"""simple docstring"""



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
				if is_torch_available():
								import torch
				if is_tf_available():
								import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_UpperCamelCase         =		logging.get_logger(__name__)

_UpperCamelCase         =		{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_UpperCamelCase         =		{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
}

_UpperCamelCase         =		{
    "Salesforce/codegen-350M-mono": 20_48,
}

@lru_cache()
def    _A(		):
							A__					:    Dict														= (
							    list(range(ord("""!"""       ) ,     ord("""~"""       ) + 1       )       ) + list(range(ord("""¡"""       ) ,     ord("""¬"""       ) + 1       )       ) + list(range(ord("""®"""       ) ,     ord("""ÿ"""       ) + 1       )       )
							)
							A__					:    str														= bs[:]
							A__					:    List[Any]														= 0
							for b in range(2**8       ):
														if b not in bs:
																					bs.append(lowerCAmelCase       )
																					cs.append(2**8 + n       )
																					n += 1
							A__					:    List[Any]														= [chr(lowerCAmelCase       ) for n in cs]
							return dict(zip(lowerCAmelCase ,     lowerCAmelCase       )       )

def    _A(		lowerCAmelCase       ):
							A__					:    Tuple														= set()
							A__					:    Optional[Any]														= word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)       )
														A__					:    Dict														= char
							return pairs



class 	__UpperCAmelCase						(__A    ):







			'''simple docstring'''
			_UpperCamelCase     :	int				    =							VOCAB_FILES_NAMES
			_UpperCamelCase     :	Optional[int]				    =							PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase     :	Optional[int]				    =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCamelCase     :	Optional[int]				    =							['input_ids', 'attention_mask']





			def __init__( self      ,			snake_case_      ,			snake_case_      ,			snake_case_="replace"      ,			snake_case_="<|endoftext|>"      ,			snake_case_="<|endoftext|>"      ,			snake_case_="<|endoftext|>"      ,			snake_case_=None      ,			snake_case_=False      ,			snake_case_=False      ,			**snake_case_      ,			):







										'''simple docstring'''


										A__					:    int														= AddedToken(snake_case_      ,			lstrip=snake_case_      ,			rstrip=snake_case_       ) if isinstance(snake_case_      ,			snake_case_       ) else bos_token
										A__					:    Optional[Any]														= AddedToken(snake_case_      ,			lstrip=snake_case_      ,			rstrip=snake_case_       ) if isinstance(snake_case_      ,			snake_case_       ) else eos_token
										A__					:    Dict														= AddedToken(snake_case_      ,			lstrip=snake_case_      ,			rstrip=snake_case_       ) if isinstance(snake_case_      ,			snake_case_       ) else unk_token
										A__					:    str														= AddedToken(snake_case_      ,			lstrip=snake_case_      ,			rstrip=snake_case_       ) if isinstance(snake_case_      ,			snake_case_       ) else pad_token
										super().__init__(
										    errors=snake_case_      ,			unk_token=snake_case_      ,			bos_token=snake_case_      ,			eos_token=snake_case_      ,			pad_token=snake_case_      ,			add_prefix_space=snake_case_      ,			add_bos_token=snake_case_      ,			**snake_case_      ,			)
										A__					:    Any														= add_bos_token

										with open(snake_case_      ,			encoding="""utf-8"""       ) as vocab_handle:
																	A__					:    Optional[Any]														= json.load(snake_case_       )
										A__					:    Dict														= {v: k for k, v in self.encoder.items()}
										A__					:    Optional[Any]														= errors  # how to handle errors in decoding
										A__					:    int														= bytes_to_unicode()
										A__					:    Dict														= {v: k for k, v in self.byte_encoder.items()}
										with open(snake_case_      ,			encoding="""utf-8"""       ) as merges_handle:
																	A__					:    List[str]														= merges_handle.read().split("""\n"""       )[1:-1]
										A__					:    List[Any]														= [tuple(merge.split()       ) for merge in bpe_merges]
										A__					:    List[Any]														= dict(zip(snake_case_      ,			range(len(snake_case_       )       )       )       )
										A__					:    List[str]														= {}
										A__					:    int														= add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										A__					:    int														= re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""       )





			@property
			def 		lowerCamelCase				( self       ):







										'''simple docstring'''


										return len(self.encoder       )





			def 		lowerCamelCase				( self       ):







										'''simple docstring'''


										return dict(self.encoder      ,			**self.added_tokens_encoder       )





			def 		lowerCamelCase				( self      ,			snake_case_       ):







										'''simple docstring'''


										if token in self.cache:
																	return self.cache[token]
										A__					:    Tuple														= tuple(snake_case_       )
										A__					:    Any														= get_pairs(snake_case_       )

										if not pairs:
																	return token

										while True:
																	A__					:    Dict														= min(snake_case_      ,			key=lambda snake_case_       : self.bpe_ranks.get(snake_case_      ,			float("""inf"""       )       )       )
																	if bigram not in self.bpe_ranks:
																								break
																	A__							,       A__					:    Optional[Any]														= bigram
																	A__					:    int														= []
																	A__					:    List[Any]														= 0
																	while i < len(snake_case_       ):
																								try:
																															A__					:    str														= word.index(snake_case_      ,			snake_case_       )
																								except ValueError:
																															new_word.extend(word[i:]       )
																															break
																								else:
																															new_word.extend(word[i:j]       )
																															A__					:    List[Any]														= j

																								if word[i] == first and i < len(snake_case_       ) - 1 and word[i + 1] == second:
																															new_word.append(first + second       )
																															i += 2
																								else:
																															new_word.append(word[i]       )
																															i += 1
																	A__					:    List[str]														= tuple(snake_case_       )
																	A__					:    Optional[int]														= new_word
																	if len(snake_case_       ) == 1:
																								break
																	else:
																								A__					:    List[Any]														= get_pairs(snake_case_       )
										A__					:    Tuple														= """ """.join(snake_case_       )
										A__					:    Optional[Any]														= word
										return word





			def 		lowerCamelCase				( self      ,			snake_case_      ,			snake_case_=None       ):







										'''simple docstring'''


										if self.add_bos_token:
																	A__					:    Any														= [self.bos_token_id]
										else:
																	A__					:    Optional[Any]														= []

										A__					:    Optional[Any]														= bos_token_ids + token_ids_a

										if token_ids_a is None:
																	return output

										return output + bos_token_ids + token_ids_a





			def 		lowerCamelCase				( self      ,			snake_case_       ):







										'''simple docstring'''


										A__					:    Any														= []
										for token in re.findall(self.pat      ,			snake_case_       ):
																	A__					:    str														= """""".join(
																	    self.byte_encoder[b] for b in token.encode("""utf-8"""       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																	bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case_       ).split(""" """       )       )
										return bpe_tokens





			def 		lowerCamelCase				( self      ,			snake_case_       ):







										'''simple docstring'''


										return self.encoder.get(snake_case_      ,			self.encoder.get(self.unk_token       )       )





			def 		lowerCamelCase				( self      ,			snake_case_       ):







										'''simple docstring'''


										return self.decoder.get(snake_case_       )





			def 		lowerCamelCase				( self      ,			snake_case_       ):







										'''simple docstring'''


										A__					:    Optional[int]														= """""".join(snake_case_       )
										A__					:    str														= bytearray([self.byte_decoder[c] for c in text]       ).decode("""utf-8"""      ,			errors=self.errors       )
										return text





			def 		lowerCamelCase				( self      ,			snake_case_      ,			snake_case_ = None       ):







										'''simple docstring'''


										if not os.path.isdir(snake_case_       ):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
																	return
										A__					:    Dict														= os.path.join(
										    snake_case_      ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
										A__					:    Optional[Any]														= os.path.join(
										    snake_case_      ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

										with open(snake_case_      ,			"""w"""      ,			encoding="""utf-8"""       ) as f:
																	f.write(json.dumps(self.encoder      ,			indent=2      ,			sort_keys=snake_case_      ,			ensure_ascii=snake_case_       ) + """\n"""       )

										A__					:    List[str]														= 0
										with open(snake_case_      ,			"""w"""      ,			encoding="""utf-8"""       ) as writer:
																	writer.write("""#version: 0.2\n"""       )
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,			key=lambda snake_case_       : kv[1]       ):
																								if index != token_index:
																															logger.warning(
																															    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																															    """ Please check that the tokenizer is not corrupted!"""       )
																															A__					:    Any														= token_index
																								writer.write(""" """.join(snake_case_       ) + """\n"""       )
																								index += 1

										return vocab_file, merge_file





			def 		lowerCamelCase				( self      ,			snake_case_      ,			snake_case_=False      ,			**snake_case_       ):







										'''simple docstring'''


										A__					:    Optional[Any]														= kwargs.pop("""add_prefix_space"""      ,			self.add_prefix_space       )
										if is_split_into_words or add_prefix_space:
																	A__					:    Union[str, Any]														= """ """ + text
										return (text, kwargs)





			def 		lowerCamelCase				( self      ,			snake_case_      ,			snake_case_ = False      ,			snake_case_ = None      ,			snake_case_ = None      ,			**snake_case_      ,			):







										'''simple docstring'''


										A__					:    Tuple														= super()._decode(
										    token_ids=snake_case_      ,			skip_special_tokens=snake_case_      ,			clean_up_tokenization_spaces=snake_case_      ,			**snake_case_      ,			)

										if truncate_before_pattern is not None and len(snake_case_       ) > 0:
																	A__					:    List[Any]														= self.truncate(snake_case_      ,			snake_case_       )

										return decoded_text





			def 		lowerCamelCase				( self      ,			snake_case_      ,			snake_case_       ):







										'''simple docstring'''


										def find_re(snake_case_      ,			snake_case_      ,			snake_case_       ):
																	A__					:    str														= pattern.search(snake_case_      ,			snake_case_       )
																	return m.start() if m else -1

										A__					:    Union[str, Any]														= [re.compile(snake_case_      ,			re.MULTILINE       ) for pattern in truncate_before_pattern]

										A__					:    List[str]														= list(re.finditer("""^print"""      ,			snake_case_      ,			re.MULTILINE       )       )

										if len(snake_case_       ) > 1:
																	A__					:    int														= completion[: prints[1].start()]

										A__					:    Dict														= list(re.finditer("""^def"""      ,			snake_case_      ,			re.MULTILINE       )       )

										if len(snake_case_       ) > 1:
																	A__					:    List[str]														= completion[: defs[1].start()]

										A__					:    Union[str, Any]														= 0

										A__					:    Any														= [
										    pos for pos in [find_re(snake_case_      ,			snake_case_      ,			snake_case_       ) for terminal in terminals] if pos != -1
										]

										if len(snake_case_       ) > 0:
																	return completion[: min(snake_case_       )]
										else:
																	return completion






"""simple docstring"""



def    _A(		lowerCAmelCase       ):
							if not isinstance(lowerCAmelCase ,     lowerCAmelCase       ):
														A__					:    int														= F'''Input value of [number={number}] must be an integer'''
														raise TypeError(lowerCAmelCase       )
							if number < 0 or number % 2 == 1:
														return False
							A__					:    List[Any]														= int(number**0.5       )
							return number == number_sqrt * (number_sqrt + 1)


if __name__ == "__main__":
				import doctest

				doctest.testmod()





