



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
				if is_torch_available():
								import torch
				if is_tf_available():
								import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__lowerCamelCase  				=					logging.get_logger(__name__)

__lowerCamelCase  				=					{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

__lowerCamelCase  				=					{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

__lowerCamelCase  				=					{
    """Salesforce/codegen-350M-mono""": 2_0_4_8,
}




@lru_cache()
def     UpperCAmelCase__ (							)   -> Dict:
		_A           =    (
		    list(range(ord('''!'''							)						,				ord('''~'''							) + 1							)							) + list(range(ord('''¡'''							)						,				ord('''¬'''							) + 1							)							) + list(range(ord('''®'''							)						,				ord('''ÿ'''							) + 1							)							)
		)
		_A           =    bs[:]
		_A           =    0
		for b in range(2**8							):
				if b not in bs:
						bs.append(__snake_case							)
						cs.append(2**8 + n							)
						n += 1
		_A           =    [chr(__snake_case							) for n in cs]
		return dict(zip(__snake_case						,				__snake_case							)							)




def     UpperCAmelCase__ (							__snake_case							)   -> List[str]:
		_A           =    set()
		_A           =    word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)							)
				_A           =    char
		return pairs



class 		_snake_case      (			lowerCamelCase				):





	"""simple docstring"""



	lowerCamelCase_          =			VOCAB_FILES_NAMES
	lowerCamelCase_          =			PRETRAINED_VOCAB_FILES_MAP
	lowerCamelCase_          =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCamelCase_          =			['''input_ids''', '''attention_mask''']
	def __init__(						self	,    a	,    a	,    a="replace"	,    a="<|endoftext|>"	,    a="<|endoftext|>"	,    a="<|endoftext|>"	,    a=None	,    a=False	,    a=False	,    **a	,    )	->							Tuple:




			"""simple docstring"""







			_A           =    AddedToken(a	,    lstrip=a	,    rstrip=a					) if isinstance(a	,    a					) else bos_token
			_A           =    AddedToken(a	,    lstrip=a	,    rstrip=a					) if isinstance(a	,    a					) else eos_token
			_A           =    AddedToken(a	,    lstrip=a	,    rstrip=a					) if isinstance(a	,    a					) else unk_token
			_A           =    AddedToken(a	,    lstrip=a	,    rstrip=a					) if isinstance(a	,    a					) else pad_token
			super().__init__(
			    errors=a	,    unk_token=a	,    bos_token=a	,    eos_token=a	,    pad_token=a	,    add_prefix_space=a	,    add_bos_token=a	,    **a	,    )
			_A           =    add_bos_token

			with open(a	,    encoding='''utf-8'''					) as vocab_handle:
					_A           =    json.load(a					)
			_A           =    {v: k for k, v in self.encoder.items()}
			_A           =    errors  # how to handle errors in decoding
			_A           =    bytes_to_unicode()
			_A           =    {v: k for k, v in self.byte_encoder.items()}
			with open(a	,    encoding='''utf-8'''					) as merges_handle:
					_A           =    merges_handle.read().split('''\n'''					)[1:-1]
			_A           =    [tuple(merge.split()					) for merge in bpe_merges]
			_A           =    dict(zip(a	,    range(len(a					)					)					)					)
			_A           =    {}
			_A           =    add_prefix_space

			# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
			_A           =    re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''					)
	@property
	def lowercase_		(						self					)	->							Optional[int]:




			"""simple docstring"""







			return len(self.encoder					)
	def lowercase_		(						self					)	->							Optional[int]:




			"""simple docstring"""







			return dict(self.encoder	,    **self.added_tokens_encoder					)
	def lowercase_		(						self	,    a					)	->							int:




			"""simple docstring"""







			if token in self.cache:
					return self.cache[token]
			_A           =    tuple(a					)
			_A           =    get_pairs(a					)

			if not pairs:
					return token

			while True:
					_A           =    min(a	,    key=lambda a					: self.bpe_ranks.get(a	,    float('''inf'''					)					)					)
					if bigram not in self.bpe_ranks:
							break
					_A    ,   _A           =    bigram
					_A           =    []
					_A           =    0
					while i < len(a					):
							try:
									_A           =    word.index(a	,    a					)
							except ValueError:
									new_word.extend(word[i:]					)
									break
							else:
									new_word.extend(word[i:j]					)
									_A           =    j

							if word[i] == first and i < len(a					) - 1 and word[i + 1] == second:
									new_word.append(first + second					)
									i += 2
							else:
									new_word.append(word[i]					)
									i += 1
					_A           =    tuple(a					)
					_A           =    new_word
					if len(a					) == 1:
							break
					else:
							_A           =    get_pairs(a					)
			_A           =    ''' '''.join(a					)
			_A           =    word
			return word
	def lowercase_		(						self	,    a	,    a=None					)	->							List[Any]:




			"""simple docstring"""







			if self.add_bos_token:
					_A           =    [self.bos_token_id]
			else:
					_A           =    []

			_A           =    bos_token_ids + token_ids_a

			if token_ids_a is None:
					return output

			return output + bos_token_ids + token_ids_a
	def lowercase_		(						self	,    a					)	->							List[Any]:




			"""simple docstring"""







			_A           =    []
			for token in re.findall(self.pat	,    a					):
					_A           =    ''''''.join(
					    self.byte_encoder[b] for b in token.encode('''utf-8'''					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
					bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a					).split(''' '''					)					)
			return bpe_tokens
	def lowercase_		(						self	,    a					)	->							Union[str, Any]:




			"""simple docstring"""







			return self.encoder.get(a	,    self.encoder.get(self.unk_token					)					)
	def lowercase_		(						self	,    a					)	->							Any:




			"""simple docstring"""







			return self.decoder.get(a					)
	def lowercase_		(						self	,    a					)	->							Optional[int]:




			"""simple docstring"""







			_A           =    ''''''.join(a					)
			_A           =    bytearray([self.byte_decoder[c] for c in text]					).decode('''utf-8'''	,    errors=self.errors					)
			return text
	def lowercase_		(						self	,    a	,    a = None					)	->							Tuple[str]:




			"""simple docstring"""







			if not os.path.isdir(a					):
					logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
					return
			_A           =    os.path.join(
			    a	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
			_A           =    os.path.join(
			    a	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']					)

			with open(a	,    '''w'''	,    encoding='''utf-8'''					) as f:
					f.write(json.dumps(self.encoder	,    indent=2	,    sort_keys=a	,    ensure_ascii=a					) + '''\n'''					)

			_A           =    0
			with open(a	,    '''w'''	,    encoding='''utf-8'''					) as writer:
					writer.write('''#version: 0.2\n'''					)
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,    key=lambda a					: kv[1]					):
							if index != token_index:
									logger.warning(
									    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
									    ''' Please check that the tokenizer is not corrupted!'''					)
									_A           =    token_index
							writer.write(''' '''.join(a					) + '''\n'''					)
							index += 1

			return vocab_file, merge_file
	def lowercase_		(						self	,    a	,    a=False	,    **a					)	->							Tuple:




			"""simple docstring"""







			_A           =    kwargs.pop('''add_prefix_space'''	,    self.add_prefix_space					)
			if is_split_into_words or add_prefix_space:
					_A           =    ''' ''' + text
			return (text, kwargs)
	def lowercase_		(						self	,    a	,    a = False	,    a = None	,    a = None	,    **a	,    )	->							str:




			"""simple docstring"""







			_A           =    super()._decode(
			    token_ids=a	,    skip_special_tokens=a	,    clean_up_tokenization_spaces=a	,    **a	,    )

			if truncate_before_pattern is not None and len(a					) > 0:
					_A           =    self.truncate(a	,    a					)

			return decoded_text



	def lowercase_		(						self	,    a	,    a					)	->							Optional[Any]:




			"""simple docstring"""







			def find_re(a	,    a	,    a					):
					_A           =    pattern.search(a	,    a					)
					return m.start() if m else -1

			_A           =    [re.compile(a	,    re.MULTILINE					) for pattern in truncate_before_pattern]

			_A           =    list(re.finditer('''^print'''	,    a	,    re.MULTILINE					)					)

			if len(a					) > 1:
					_A           =    completion[: prints[1].start()]

			_A           =    list(re.finditer('''^def'''	,    a	,    re.MULTILINE					)					)

			if len(a					) > 1:
					_A           =    completion[: defs[1].start()]

			_A           =    0

			_A           =    [
			    pos for pos in [find_re(a	,    a	,    a					) for terminal in terminals] if pos != -1
			]

			if len(a					) > 0:
					return completion[: min(a					)]
			else:
					return completion




import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase  				=					logging.get_logger(__name__)

__lowerCamelCase  				=					{}



class 		_snake_case      (			lowerCamelCase				):





 """simple docstring"""



 lowerCamelCase_          =			'''fsmt_decoder'''
 def __init__(						self	,    a=0	,    a=0					)	->							str:




   """simple docstring"""







   super().__init__()
   _A           =    vocab_size
   _A           =    bos_token_id







class 		_snake_case      (			lowerCamelCase				):





 """simple docstring"""



 lowerCamelCase_          =			'''fsmt'''
 lowerCamelCase_          =			{'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}
 def __init__(						self	,    a=["en", "de"]	,    a=4_2_0_2_4	,    a=4_2_0_2_4	,    a="relu"	,    a=1_0_2_4	,    a=2_0_0	,    a=1_0_2_4	,    a=4_0_9_6	,    a=1_2	,    a=1_6	,    a=0.0	,    a=4_0_9_6	,    a=1_2	,    a=1_6	,    a=0.0	,    a=0.0	,    a=0.1	,    a=0.0	,    a=0.02	,    a=2	,    a=True	,    a=True	,    a=False	,    a=5	,    a=1.0	,    a=False	,    a=True	,    a=1	,    a=0	,    a=2	,    a=2	,    **a	,    )	->							Union[str, Any]:




   """simple docstring"""







   _A           =    langs
   _A           =    src_vocab_size
   _A           =    tgt_vocab_size
   _A           =    d_model  # encoder_embed_dim and decoder_embed_dim

   _A           =    encoder_ffn_dim
   _A           =    _A           =    encoder_layers
   _A           =    encoder_attention_heads
   _A           =    encoder_layerdrop
   _A           =    decoder_layerdrop
   _A           =    decoder_ffn_dim
   _A           =    decoder_layers
   _A           =    decoder_attention_heads
   _A           =    max_position_embeddings
   _A           =    init_std  # Normal(0, this parameter)
   _A           =    activation_function

   _A           =    DecoderConfig(vocab_size=a	,    bos_token_id=a					)
   if "decoder" in common_kwargs:
     del common_kwargs["decoder"]

   _A           =    scale_embedding  # scale factor will be sqrt(d_model) if True

   # 3 Types of Dropout
   _A           =    attention_dropout
   _A           =    activation_dropout
   _A           =    dropout

   _A           =    use_cache
   super().__init__(
       pad_token_id=a	,    bos_token_id=a	,    eos_token_id=a	,    decoder_start_token_id=a	,    is_encoder_decoder=a	,    tie_word_embeddings=a	,    forced_eos_token_id=a	,    max_length=a	,    num_beams=a	,    length_penalty=a	,    early_stopping=a	,    **a	,    )



 def lowercase_		(						self					)	->							Any:




   """simple docstring"""







   _A           =    copy.deepcopy(self.__dict__					)
   _A           =    self.decoder.to_dict()
   _A           =    self.__class__.model_type
   return output