import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
	if is_torch_available():
		import torch
	if is_tf_available():
		import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


lowercase__    :       Dict						=			logging.get_logger(__name__)

lowercase__    :       Union[str, Any]						=			{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

lowercase__    :       int						=			{
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

lowercase__    :       List[Any]						=			{
    '''Salesforce/codegen-350M-mono''': 2_0_4_8,
}

@lru_cache()
def      SCREAMING_SNAKE_CASE_  (		)  ->			Dict:
							lowerCAmelCase				=	(
							    list(range(ord('''!'''							)     ,						ord('''~'''							) + 1							)							) + list(range(ord('''¡'''							)     ,						ord('''¬'''							) + 1							)							) + list(range(ord('''®'''							)     ,						ord('''ÿ'''							) + 1							)							)
							)
							lowerCAmelCase				=	bs[:]
							lowerCAmelCase				=	0
							for b in range(2**8							):
														if b not in bs:
																					bs.append(__a							)
																					cs.append(2**8 + n							)
																					n += 1
							lowerCAmelCase				=	[chr(__a							) for n in cs]
							return dict(zip(__a     ,						__a							)							)

def      SCREAMING_SNAKE_CASE_  (		snake_case__							)  ->			Tuple:
							lowerCAmelCase				=	set()
							lowerCAmelCase				=	word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)							)
														lowerCAmelCase				=	char
							return pairs






class      lowercase_						(						UpperCamelCase__				):



	"""simple docstring"""





	UpperCAmelCase_						:      Optional[int]     =       VOCAB_FILES_NAMES
	UpperCAmelCase_						:      List[str]     =       PRETRAINED_VOCAB_FILES_MAP
	UpperCAmelCase_						:      Dict     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCAmelCase_						:      Optional[Any]     =       ["""input_ids""", """attention_mask"""]


	def __init__(							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE="replace"	,    __SCREAMING_SNAKE_CASE="<|endoftext|>"	,    __SCREAMING_SNAKE_CASE="<|endoftext|>"	,    __SCREAMING_SNAKE_CASE="<|endoftext|>"	,    __SCREAMING_SNAKE_CASE=None	,    __SCREAMING_SNAKE_CASE=False	,    __SCREAMING_SNAKE_CASE=False	,    **__SCREAMING_SNAKE_CASE	,    )		->Optional[Any]:
								lowerCAmelCase				=	AddedToken(snake_case_	,    lstrip=snake_case_	,    rstrip=snake_case_       ) if isinstance(snake_case_	,    snake_case_       ) else bos_token
								lowerCAmelCase				=	AddedToken(snake_case_	,    lstrip=snake_case_	,    rstrip=snake_case_       ) if isinstance(snake_case_	,    snake_case_       ) else eos_token
								lowerCAmelCase				=	AddedToken(snake_case_	,    lstrip=snake_case_	,    rstrip=snake_case_       ) if isinstance(snake_case_	,    snake_case_       ) else unk_token
								lowerCAmelCase				=	AddedToken(snake_case_	,    lstrip=snake_case_	,    rstrip=snake_case_       ) if isinstance(snake_case_	,    snake_case_       ) else pad_token
								super().__init__(
								    errors=snake_case_	,    unk_token=snake_case_	,    bos_token=snake_case_	,    eos_token=snake_case_	,    pad_token=snake_case_	,    add_prefix_space=snake_case_	,    add_bos_token=snake_case_	,    **snake_case_	,    )
								lowerCAmelCase				=	add_bos_token

								with open(snake_case_	,    encoding='''utf-8'''       ) as vocab_handle:
															lowerCAmelCase				=	json.load(snake_case_       )
								lowerCAmelCase				=	{v: k for k, v in self.encoder.items()}
								lowerCAmelCase				=	errors  # how to handle errors in decoding
								lowerCAmelCase				=	bytes_to_unicode()
								lowerCAmelCase				=	{v: k for k, v in self.byte_encoder.items()}
								with open(snake_case_	,    encoding='''utf-8'''       ) as merges_handle:
															lowerCAmelCase				=	merges_handle.read().split('''\n'''       )[1:-1]
								lowerCAmelCase				=	[tuple(merge.split()       ) for merge in bpe_merges]
								lowerCAmelCase				=	dict(zip(snake_case_	,    range(len(snake_case_       )       )       )       )
								lowerCAmelCase				=	{}
								lowerCAmelCase				=	add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								lowerCAmelCase				=	re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''       )


	@property
	def     SCREAMING_SNAKE_CASE_       (							self       )		->Any:
								return len(self.encoder       )


	def     SCREAMING_SNAKE_CASE_       (							self       )		->Tuple:
								return dict(self.encoder	,    **self.added_tokens_encoder       )


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Optional[int]:
								if token in self.cache:
															return self.cache[token]
								lowerCAmelCase				=	tuple(snake_case_       )
								lowerCAmelCase				=	get_pairs(snake_case_       )

								if not pairs:
															return token

								while True:
															lowerCAmelCase				=	min(snake_case_	,    key=lambda __SCREAMING_SNAKE_CASE       : self.bpe_ranks.get(snake_case_	,    float('''inf'''       )       )       )
															if bigram not in self.bpe_ranks:
																						break
															lowerCAmelCase				=	bigram
															lowerCAmelCase				=	[]
															lowerCAmelCase				=	0
															while i < len(snake_case_       ):
																						try:
																													lowerCAmelCase				=	word.index(snake_case_	,    snake_case_       )
																						except ValueError:
																													new_word.extend(word[i:]       )
																													break
																						else:
																													new_word.extend(word[i:j]       )
																													lowerCAmelCase				=	j

																						if word[i] == first and i < len(snake_case_       ) - 1 and word[i + 1] == second:
																													new_word.append(first + second       )
																													i += 2
																						else:
																													new_word.append(word[i]       )
																													i += 1
															lowerCAmelCase				=	tuple(snake_case_       )
															lowerCAmelCase				=	new_word
															if len(snake_case_       ) == 1:
																						break
															else:
																						lowerCAmelCase				=	get_pairs(snake_case_       )
								lowerCAmelCase				=	''' '''.join(snake_case_       )
								lowerCAmelCase				=	word
								return word


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE=None       )		->Any:
								if self.add_bos_token:
															lowerCAmelCase				=	[self.bos_token_id]
								else:
															lowerCAmelCase				=	[]

								lowerCAmelCase				=	bos_token_ids + token_ids_a

								if token_ids_a is None:
															return output

								return output + bos_token_ids + token_ids_a


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->List[Any]:
								lowerCAmelCase				=	[]
								for token in re.findall(self.pat	,    snake_case_       ):
															lowerCAmelCase				=	''''''.join(
															    self.byte_encoder[b] for b in token.encode('''utf-8'''       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case_       ).split(''' '''       )       )
								return bpe_tokens


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Dict:
								return self.encoder.get(snake_case_	,    self.encoder.get(self.unk_token       )       )


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Any:
								return self.decoder.get(snake_case_       )


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Any:
								lowerCAmelCase				=	''''''.join(snake_case_       )
								lowerCAmelCase				=	bytearray([self.byte_decoder[c] for c in text]       ).decode('''utf-8'''	,    errors=self.errors       )
								return text


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE = None       )		->Tuple[str]:
								if not os.path.isdir(snake_case_       ):
															logger.error(F"Vocabulary path ({save_directory}) should be a directory"       )
															return
								lowerCAmelCase				=	os.path.join(
								    snake_case_	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
								lowerCAmelCase				=	os.path.join(
								    snake_case_	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )

								with open(snake_case_	,    '''w'''	,    encoding='''utf-8'''       ) as f:
															f.write(json.dumps(self.encoder	,    indent=2	,    sort_keys=snake_case_	,    ensure_ascii=snake_case_       ) + '''\n'''       )

								lowerCAmelCase				=	0
								with open(snake_case_	,    '''w'''	,    encoding='''utf-8'''       ) as writer:
															writer.write('''#version: 0.2\n'''       )
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,    key=lambda __SCREAMING_SNAKE_CASE       : kv[1]       ):
																						if index != token_index:
																													logger.warning(
																													    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																													    ''' Please check that the tokenizer is not corrupted!'''       )
																													lowerCAmelCase				=	token_index
																						writer.write(''' '''.join(snake_case_       ) + '''\n'''       )
																						index += 1

								return vocab_file, merge_file


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE=False	,    **__SCREAMING_SNAKE_CASE       )		->Dict:
								lowerCAmelCase				=	kwargs.pop('''add_prefix_space'''	,    self.add_prefix_space       )
								if is_split_into_words or add_prefix_space:
															lowerCAmelCase				=	''' ''' + text
								return (text, kwargs)


	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE = False	,    __SCREAMING_SNAKE_CASE = None	,    __SCREAMING_SNAKE_CASE = None	,    **__SCREAMING_SNAKE_CASE	,    )		->str:
								lowerCAmelCase				=	super()._decode(
								    token_ids=snake_case_	,    skip_special_tokens=snake_case_	,    clean_up_tokenization_spaces=snake_case_	,    **snake_case_	,    )

								if truncate_before_pattern is not None and len(snake_case_       ) > 0:
															lowerCAmelCase				=	self.truncate(snake_case_	,    snake_case_       )

								return decoded_text




	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )		->Union[str, Any]:
								def find_re(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       ):
															lowerCAmelCase				=	pattern.search(snake_case_	,    snake_case_       )
															return m.start() if m else -1

								lowerCAmelCase				=	[re.compile(snake_case_	,    re.MULTILINE       ) for pattern in truncate_before_pattern]

								lowerCAmelCase				=	list(re.finditer('''^print'''	,    snake_case_	,    re.MULTILINE       )       )

								if len(snake_case_       ) > 1:
															lowerCAmelCase				=	completion[: prints[1].start()]

								lowerCAmelCase				=	list(re.finditer('''^def'''	,    snake_case_	,    re.MULTILINE       )       )

								if len(snake_case_       ) > 1:
															lowerCAmelCase				=	completion[: defs[1].start()]

								lowerCAmelCase				=	0

								lowerCAmelCase				=	[
								    pos for pos in [find_re(snake_case_	,    snake_case_	,    snake_case_       ) for terminal in terminals] if pos != -1
								]

								if len(snake_case_       ) > 0:
															return completion[: min(snake_case_       )]
								else:
															return completion




from typing import Callable, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__    :       Optional[Any]						=			logging.get_logger(__name__)

lowercase__    :       Union[str, Any]						=			{
    '''microsoft/prophetnet-large-uncased''': (
        '''https://huggingface.co/microsoft/prophetnet-large-uncased/resolve/main/config.json'''
    ),
}




class      lowercase_						(						UpperCamelCase_		):



	"""simple docstring"""





	UpperCAmelCase_						:      Optional[int]     =       """prophetnet"""
	UpperCAmelCase_						:      Optional[int]     =       ["""past_key_values"""]
	UpperCAmelCase_						:      Any     =       {
	    """num_attention_heads""": """num_encoder_attention_heads""",
	}


	def __init__(							self	,    __SCREAMING_SNAKE_CASE = 0.1	,    __SCREAMING_SNAKE_CASE = "gelu"	,    __SCREAMING_SNAKE_CASE = 30522	,    __SCREAMING_SNAKE_CASE = 1024	,    __SCREAMING_SNAKE_CASE = 4096	,    __SCREAMING_SNAKE_CASE = 12	,    __SCREAMING_SNAKE_CASE = 16	,    __SCREAMING_SNAKE_CASE = 4096	,    __SCREAMING_SNAKE_CASE = 12	,    __SCREAMING_SNAKE_CASE = 16	,    __SCREAMING_SNAKE_CASE = 0.1	,    __SCREAMING_SNAKE_CASE = 0.1	,    __SCREAMING_SNAKE_CASE = 512	,    __SCREAMING_SNAKE_CASE = 0.0_2	,    __SCREAMING_SNAKE_CASE = True	,    __SCREAMING_SNAKE_CASE = True	,    __SCREAMING_SNAKE_CASE = 0	,    __SCREAMING_SNAKE_CASE = 2	,    __SCREAMING_SNAKE_CASE = 32	,    __SCREAMING_SNAKE_CASE = 128	,    __SCREAMING_SNAKE_CASE = False	,    __SCREAMING_SNAKE_CASE = 0.0	,    __SCREAMING_SNAKE_CASE = True	,    __SCREAMING_SNAKE_CASE = 0	,    __SCREAMING_SNAKE_CASE = 1	,    __SCREAMING_SNAKE_CASE = 2	,    **__SCREAMING_SNAKE_CASE	,    )		->int:
								lowerCAmelCase				=	vocab_size
								lowerCAmelCase				=	hidden_size
								lowerCAmelCase				=	encoder_ffn_dim
								lowerCAmelCase				=	num_encoder_layers
								lowerCAmelCase				=	num_encoder_attention_heads
								lowerCAmelCase				=	decoder_ffn_dim
								lowerCAmelCase				=	num_decoder_layers
								lowerCAmelCase				=	num_decoder_attention_heads
								lowerCAmelCase				=	max_position_embeddings
								lowerCAmelCase				=	init_std  # Normal(0, this parameter)
								lowerCAmelCase				=	activation_function

								# parameters for prophetnet
								lowerCAmelCase				=	ngram
								lowerCAmelCase				=	num_buckets
								lowerCAmelCase				=	relative_max_distance
								lowerCAmelCase				=	disable_ngram_loss
								lowerCAmelCase				=	eps

								# 3 Types of Dropout
								lowerCAmelCase				=	attention_dropout
								lowerCAmelCase				=	activation_dropout
								lowerCAmelCase				=	dropout

								lowerCAmelCase				=	use_cache

								super().__init__(
								    pad_token_id=__SCREAMING_SNAKE_CASE	,    bos_token_id=__SCREAMING_SNAKE_CASE	,    eos_token_id=__SCREAMING_SNAKE_CASE	,    is_encoder_decoder=__SCREAMING_SNAKE_CASE	,    add_cross_attention=__SCREAMING_SNAKE_CASE	,    decoder_start_token_id=__SCREAMING_SNAKE_CASE	,    **__SCREAMING_SNAKE_CASE	,    )


	@property
	def     SCREAMING_SNAKE_CASE_       (							self       )		->int:
								return self.num_encoder_layers + self.num_decoder_layers




	@num_hidden_layers.setter
	def     SCREAMING_SNAKE_CASE_       (							self	,    __SCREAMING_SNAKE_CASE       )		->Union[str, Any]:
								raise NotImplementedError(
								    '''This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and'''
								    ''' `num_decoder_layers`.'''       )



