






"""simple docstring"""


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__SCREAMING_SNAKE_CASE      =      logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE      =      {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__SCREAMING_SNAKE_CASE      =      {
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

__SCREAMING_SNAKE_CASE      =      {
    'openai-gpt': 5_12,
}


def 					UpperCAmelCase			(				a__   ):





						'''simple docstring'''






						lowerCAmelCase      :Union[str, Any]				  = text.strip()
						if not text:
												return []
						lowerCAmelCase      :List[Any]				  = text.split()
						return tokens




class 	__UpperCamelCase    (    UpperCamelCase	):
					def __init__(       self       :  Union[str, Any]    ,   UpperCAmelCase       :  Dict=True    ,   UpperCAmelCase       :  List[Any]=None    ,   UpperCAmelCase       :  int=True    ,   UpperCAmelCase       :  Any=None    ,   UpperCAmelCase       :  Dict=True    ,   )  ->					Any:
											if never_split is None:
																	lowerCAmelCase      :int				  = []
											lowerCAmelCase      :int				  = do_lower_case
											lowerCAmelCase      :Optional[int]				  = set(UpperCAmelCase      )
											lowerCAmelCase      :int				  = tokenize_chinese_chars
											lowerCAmelCase      :List[Any]				  = strip_accents
											lowerCAmelCase      :Union[str, Any]				  = do_split_on_punc
					def 					UpperCAmelCase__   (       self       :  List[str]    ,   UpperCAmelCase       :  int    ,   UpperCAmelCase       :  Optional[Any]=None      )  ->					Dict:
											lowerCAmelCase      :Any				  = self.never_split.union(set(UpperCAmelCase      )      ) if never_split else self.never_split
											lowerCAmelCase      :str				  = self._clean_text(UpperCAmelCase      )

											# This was added on November 1st, 2018 for the multilingual and Chinese
											# models. This is also applied to the English models now, but it doesn't
											# matter since the English models were not trained on any Chinese data
											# and generally don't have any Chinese data in them (there are Chinese
											# characters in the vocabulary because Wikipedia does have some Chinese
											# words in the English Wikipedia.).
											if self.tokenize_chinese_chars:
																	lowerCAmelCase      :Union[str, Any]				  = self._tokenize_chinese_chars(UpperCAmelCase      )
											# prevents treating the same character with different unicode codepoints as different characters
											lowerCAmelCase      :int				  = unicodedata.normalize('NFC'    ,   UpperCAmelCase      )
											lowerCAmelCase      :List[Any]				  = whitespace_tokenize(UpperCAmelCase      )
											lowerCAmelCase      :Union[str, Any]				  = []
											for token in orig_tokens:
																	if token not in never_split:
																							if self.do_lower_case:
																													lowerCAmelCase      :Dict				  = token.lower()
																													if self.strip_accents is not False:
																																			lowerCAmelCase      :str				  = self._run_strip_accents(UpperCAmelCase      )
																							elif self.strip_accents:
																													lowerCAmelCase      :List[str]				  = self._run_strip_accents(UpperCAmelCase      )
																	split_tokens.extend(self._run_split_on_punc(UpperCAmelCase    ,   UpperCAmelCase      )      )

											lowerCAmelCase      :List[Any]				  = whitespace_tokenize(' '.join(UpperCAmelCase      )      )
											return output_tokens
					def 					UpperCAmelCase__   (       self       :  int    ,   UpperCAmelCase       :  Union[str, Any]      )  ->					int:
											lowerCAmelCase      :List[Any]				  = unicodedata.normalize('NFD'    ,   UpperCAmelCase      )
											lowerCAmelCase      :Dict				  = []
											for char in text:
																	lowerCAmelCase      :Any				  = unicodedata.category(UpperCAmelCase      )
																	if cat == "Mn":
																							continue
																	output.append(UpperCAmelCase      )
											return "".join(UpperCAmelCase      )
					def 					UpperCAmelCase__   (       self       :  Dict    ,   UpperCAmelCase       :  Any    ,   UpperCAmelCase       :  List[Any]=None      )  ->					List[str]:
											if not self.do_split_on_punc or (never_split is not None and text in never_split):
																	return [text]
											lowerCAmelCase      :str				  = list(UpperCAmelCase      )
											lowerCAmelCase      :List[Any]				  = 0
											lowerCAmelCase      :Optional[Any]				  = True
											lowerCAmelCase      :List[Any]				  = []
											while i < len(UpperCAmelCase      ):
																	lowerCAmelCase      :Tuple				  = chars[i]
																	if _is_punctuation(UpperCAmelCase      ):
																							output.append([char]      )
																							lowerCAmelCase      :Optional[Any]				  = True
																	else:
																							if start_new_word:
																													output.append([]      )
																							lowerCAmelCase      :List[Any]				  = False
																							output[-1].append(UpperCAmelCase      )
																	i += 1

											return ["".join(UpperCAmelCase      ) for x in output]
					def 					UpperCAmelCase__   (       self       :  Dict    ,   UpperCAmelCase       :  Any      )  ->					str:
											lowerCAmelCase      :Optional[Any]				  = []
											for char in text:
																	lowerCAmelCase      :Tuple				  = ord(UpperCAmelCase      )
																	if self._is_chinese_char(UpperCAmelCase      ):
																							output.append(' '      )
																							output.append(UpperCAmelCase      )
																							output.append(' '      )
																	else:
																							output.append(UpperCAmelCase      )
											return "".join(UpperCAmelCase      )
					def 					UpperCAmelCase__   (       self       :  Optional[int]    ,   UpperCAmelCase       :  Optional[Any]      )  ->					Optional[Any]:
											# This defines a "chinese character" as anything in the CJK Unicode block:
											#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
											#
											# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
											# despite its name. The modern Korean Hangul alphabet is a different block,
											# as is Japanese Hiragana and Katakana. Those alphabets are used to write
											# space-separated words, so they are not treated specially and handled
											# like the all of the other languages.
											if (
											    (cp >= 0X4_E00 and cp <= 0X9_FFF)
											    or (cp >= 0X3_400 and cp <= 0X4_DBF)  #
											    or (cp >= 0X20_000 and cp <= 0X2A_6DF)  #
											    or (cp >= 0X2A_700 and cp <= 0X2B_73F)  #
											    or (cp >= 0X2B_740 and cp <= 0X2B_81F)  #
											    or (cp >= 0X2B_820 and cp <= 0X2C_EAF)  #
											    or (cp >= 0XF_900 and cp <= 0XF_AFF)
											    or (cp >= 0X2F_800 and cp <= 0X2F_A1F)  #
											):  #
																	return True

											return False
					def 					UpperCAmelCase__   (       self       :  List[Any]    ,   UpperCAmelCase       :  List[str]      )  ->					int:
											lowerCAmelCase      :Optional[int]				  = []
											for char in text:
																	lowerCAmelCase      :Optional[Any]				  = ord(UpperCAmelCase      )
																	if cp == 0 or cp == 0XF_FFD or _is_control(UpperCAmelCase      ):
																							continue
																	if _is_whitespace(UpperCAmelCase      ):
																							output.append(' '      )
																	else:
																							output.append(UpperCAmelCase      )
											return "".join(UpperCAmelCase      )


def 					UpperCAmelCase			(				a__   ):





						'''simple docstring'''






						lowerCAmelCase      :Tuple				  = set()
						lowerCAmelCase      :Union[str, Any]				  = word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)   )
												lowerCAmelCase      :str				  = char
						return pairs


def 					UpperCAmelCase			(				a__   ):





						'''simple docstring'''






						lowerCAmelCase      :Any				  = text.replace('—'    ,			'-'   )
						lowerCAmelCase      :int				  = text.replace('–'    ,			'-'   )
						lowerCAmelCase      :Tuple				  = text.replace('―'    ,			'-'   )
						lowerCAmelCase      :List[Any]				  = text.replace('…'    ,			'...'   )
						lowerCAmelCase      :Any				  = text.replace('´'    ,			'\''   )
						lowerCAmelCase      :Dict				  = re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'    ,			R' \1 '    ,			a__   )
						lowerCAmelCase      :Any				  = re.sub(R'\s*\n\s*'    ,			' \n '    ,			a__   )
						lowerCAmelCase      :int				  = re.sub(R'[^\S\n]+'    ,			' '    ,			a__   )
						return text.strip()







class 	__UpperCamelCase    (    UpperCamelCase	):
					lowercase_						:							Any			=     VOCAB_FILES_NAMES
					lowercase_						:							Any			=     PRETRAINED_VOCAB_FILES_MAP
					lowercase_						:							int			=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase_						:							Tuple			=     ["""input_ids""", """attention_mask"""]
					def __init__(       self       :  Union[str, Any]    ,   UpperCAmelCase       :  List[str]    ,   UpperCAmelCase       :  Dict    ,   UpperCAmelCase       :  List[str]="<unk>"    ,   **UpperCAmelCase       :  str      )  ->					Tuple:
											super().__init__(unk_token=UpperCAmelCase    ,   **UpperCAmelCase      )

											try:
																	import ftfy
																	from spacy.lang.en import English

																	lowerCAmelCase      :List[Any]				  = English()
																	lowerCAmelCase      :Any				  = _nlp.tokenizer
																	lowerCAmelCase      :Dict				  = ftfy.fix_text
											except ImportError:
																	logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'      )
																	lowerCAmelCase      :List[Any]				  = BasicTokenizer(do_lower_case=UpperCAmelCase      )
																	lowerCAmelCase      :Tuple				  = None

											with open(UpperCAmelCase    ,   encoding='utf-8'      ) as vocab_handle:
																	lowerCAmelCase      :Dict				  = json.load(UpperCAmelCase      )
											lowerCAmelCase      :str				  = {v: k for k, v in self.encoder.items()}
											with open(UpperCAmelCase    ,   encoding='utf-8'      ) as merges_handle:
																	lowerCAmelCase      :Tuple				  = merges_handle.read().split('\n'      )[1:-1]
											lowerCAmelCase      :Optional[Any]				  = [tuple(merge.split()      ) for merge in merges]
											lowerCAmelCase      :Optional[Any]				  = dict(zip(UpperCAmelCase    ,   range(len(UpperCAmelCase      )      )      )      )
											lowerCAmelCase      :Union[str, Any]				  = {}
					@property
					def 					UpperCAmelCase__   (       self       :  List[str]      )  ->					Dict:
											return True
					@property
					def 					UpperCAmelCase__   (       self       :  Dict      )  ->					str:
											return len(self.encoder      )
					def 					UpperCAmelCase__   (       self       :  Union[str, Any]      )  ->					Dict:
											return dict(self.encoder    ,   **self.added_tokens_encoder      )
					def 					UpperCAmelCase__   (       self       :  List[Any]    ,   UpperCAmelCase       :  str      )  ->					Dict:
											lowerCAmelCase      :Optional[Any]				  = tuple(token[:-1]      ) + (token[-1] + '</w>',)
											if token in self.cache:
																	return self.cache[token]
											lowerCAmelCase      :Union[str, Any]				  = get_pairs(UpperCAmelCase      )

											if not pairs:
																	return token + "</w>"

											while True:
																	lowerCAmelCase      :Optional[Any]				  = min(UpperCAmelCase    ,   key=lambda UpperCAmelCase      : self.bpe_ranks.get(UpperCAmelCase    ,   float('inf'      )      )      )
																	if bigram not in self.bpe_ranks:
																							break
																	lowerCAmelCase  ,   lowerCAmelCase      :List[Any]				  = bigram
																	lowerCAmelCase      :Optional[int]				  = []
																	lowerCAmelCase      :Optional[int]				  = 0
																	while i < len(UpperCAmelCase      ):
																							try:
																													lowerCAmelCase      :Optional[int]				  = word.index(UpperCAmelCase    ,   UpperCAmelCase      )
																							except ValueError:
																													new_word.extend(word[i:]      )
																													break
																							else:
																													new_word.extend(word[i:j]      )
																													lowerCAmelCase      :str				  = j

																							if word[i] == first and i < len(UpperCAmelCase      ) - 1 and word[i + 1] == second:
																													new_word.append(first + second      )
																													i += 2
																							else:
																													new_word.append(word[i]      )
																													i += 1
																	lowerCAmelCase      :Dict				  = tuple(UpperCAmelCase      )
																	lowerCAmelCase      :Tuple				  = new_word
																	if len(UpperCAmelCase      ) == 1:
																							break
																	else:
																							lowerCAmelCase      :List[Any]				  = get_pairs(UpperCAmelCase      )
											lowerCAmelCase      :int				  = ' '.join(UpperCAmelCase      )
											if word == "\n  </w>":
																	lowerCAmelCase      :List[Any]				  = '\n</w>'
											lowerCAmelCase      :List[Any]				  = word
											return word
					def 					UpperCAmelCase__   (       self       :  Any    ,   UpperCAmelCase       :  Optional[Any]      )  ->					Optional[Any]:
											lowerCAmelCase      :List[str]				  = []
											if self.fix_text is None:
																	# Using BERT's BasicTokenizer
																	lowerCAmelCase      :Dict				  = self.nlp.tokenize(UpperCAmelCase      )
																	for token in text:
																							split_tokens.extend(list(self.bpe(UpperCAmelCase      ).split(' '      )      )      )
											else:
																	# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
																	lowerCAmelCase      :List[str]				  = self.nlp(text_standardize(self.fix_text(UpperCAmelCase      )      )      )
																	for token in text:
																							split_tokens.extend(list(self.bpe(token.text.lower()      ).split(' '      )      )      )
											return split_tokens
					def 					UpperCAmelCase__   (       self       :  List[str]    ,   UpperCAmelCase       :  List[str]      )  ->					List[Any]:
											return self.encoder.get(UpperCAmelCase    ,   self.encoder.get(self.unk_token      )      )
					def 					UpperCAmelCase__   (       self       :  str    ,   UpperCAmelCase       :  List[str]      )  ->					Optional[int]:
											return self.decoder.get(UpperCAmelCase    ,   self.unk_token      )
					def 					UpperCAmelCase__   (       self       :  List[Any]    ,   UpperCAmelCase       :  Dict      )  ->					Tuple:
											lowerCAmelCase      :Any				  = ''.join(UpperCAmelCase      ).replace('</w>'    ,   ' '      ).strip()
											return out_string
					def 					UpperCAmelCase__   (       self       :  Optional[int]    ,   UpperCAmelCase       :  str    ,   UpperCAmelCase       :  Optional[str] = None      )  ->					Tuple[str]:
											if not os.path.isdir(UpperCAmelCase      ):
																	logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""      )
																	return
											lowerCAmelCase      :Union[str, Any]				  = os.path.join(
											    UpperCAmelCase    ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
											lowerCAmelCase      :Union[str, Any]				  = os.path.join(
											    UpperCAmelCase    ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

											with open(UpperCAmelCase    ,   'w'    ,   encoding='utf-8'      ) as f:
																	f.write(json.dumps(self.encoder    ,   indent=2    ,   sort_keys=UpperCAmelCase    ,   ensure_ascii=UpperCAmelCase      ) + '\n'      )

											lowerCAmelCase      :str				  = 0
											with open(UpperCAmelCase    ,   'w'    ,   encoding='utf-8'      ) as writer:
																	writer.write('#version: 0.2\n'      )
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,   key=lambda UpperCAmelCase      : kv[1]      ):
																							if index != token_index:
																													logger.warning(
																													    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																													    ' Please check that the tokenizer is not corrupted!'      )
																													lowerCAmelCase      :List[Any]				  = token_index
																							writer.write(' '.join(UpperCAmelCase      ) + '\n'      )
																							index += 1

											return vocab_file, merge_file







"""simple docstring"""


from .number_theory.prime_numbers import next_prime




class 	__UpperCamelCase    :
					def __init__(       self       :  List[str]    ,   UpperCAmelCase       :  int    ,   UpperCAmelCase       :  int | None = None    ,   UpperCAmelCase       :  float | None = None    ,   )  ->					None:
											lowerCAmelCase      :Tuple				  = size_table
											lowerCAmelCase      :List[Any]				  = [None] * self.size_table
											lowerCAmelCase      :Tuple				  = 0.7_5 if lim_charge is None else lim_charge
											lowerCAmelCase      :str				  = 1 if charge_factor is None else charge_factor
											lowerCAmelCase      :list				  = []
											lowerCAmelCase      :dict				  = {}
					def 					UpperCAmelCase__   (       self       :  Optional[int]      )  ->					Union[str, Any]:
											return self._keys
					def 					UpperCAmelCase__   (       self       :  Dict      )  ->					Optional[Any]:
											return sum(1 for slot in self.values if slot is not None      ) / (
											    self.size_table * self.charge_factor
											)
					def 					UpperCAmelCase__   (       self       :  Any    ,   UpperCAmelCase       :  List[str]      )  ->					int:
											return key % self.size_table
					def 					UpperCAmelCase__   (       self       :  Dict    ,   UpperCAmelCase       :  Union[str, Any]      )  ->					int:
											print(f"""step {step_ord}"""      )
											print(list(range(len(self.values      )      )      )      )
											print(self.values      )
					def 					UpperCAmelCase__   (       self       :  Any    ,   UpperCAmelCase       :  Dict      )  ->					Any:
											lowerCAmelCase      :List[str]				  = 1
											lowerCAmelCase      :Union[str, Any]				  = values
											for value in values:
																	self.insert_data(UpperCAmelCase      )
																	self._step_by_step(UpperCAmelCase      )
																	i += 1
					def 					UpperCAmelCase__   (       self       :  str    ,   UpperCAmelCase       :  Dict    ,   UpperCAmelCase       :  Union[str, Any]      )  ->					Optional[Any]:
											lowerCAmelCase      :List[Any]				  = data
											lowerCAmelCase      :str				  = data
					def 					UpperCAmelCase__   (       self       :  Optional[int]    ,   UpperCAmelCase       :  str    ,   UpperCAmelCase       :  str=None      )  ->					Tuple:
											lowerCAmelCase      :Optional[Any]				  = self.hash_function(key + 1      )

											while self.values[new_key] is not None and self.values[new_key] != key:
																	if self.values.count(UpperCAmelCase      ) > 0:
																							lowerCAmelCase      :Tuple				  = self.hash_function(new_key + 1      )
																	else:
																							lowerCAmelCase      :Union[str, Any]				  = None
																							break

											return new_key
					def 					UpperCAmelCase__   (       self       :  Any      )  ->					Any:
											lowerCAmelCase      :Optional[Any]				  = [value for value in self.values if value is not None]
											lowerCAmelCase      :List[str]				  = next_prime(self.size_table    ,   factor=2      )
											self._keys.clear()
											lowerCAmelCase      :int				  = [None] * self.size_table  # hell's pointers D: don't DRY ;/
											for value in survivor_values:
																	self.insert_data(UpperCAmelCase      )
					def 					UpperCAmelCase__   (       self       :  Optional[int]    ,   UpperCAmelCase       :  Tuple      )  ->					int:
											lowerCAmelCase      :Union[str, Any]				  = self.hash_function(UpperCAmelCase      )

											if self.values[key] is None:
																	self._set_value(UpperCAmelCase    ,   UpperCAmelCase      )

											elif self.values[key] == data:
																	pass

											else:
																	lowerCAmelCase      :Tuple				  = self._collision_resolution(UpperCAmelCase    ,   UpperCAmelCase      )
																	if collision_resolution is not None:
																							self._set_value(UpperCAmelCase    ,   UpperCAmelCase      )
																	else:
																							self.rehashing()
																							self.insert_data(UpperCAmelCase      )