

import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


UpperCAmelCase_       :				List[Any]		   =      logging.get_logger(__name__)

UpperCAmelCase_       :				Union[str, Any]		   =      {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

UpperCAmelCase_       :				Union[str, Any]		   =      {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

UpperCAmelCase_       :				Tuple		   =      {
    '''openai-gpt''': 512,
}
def        __SCREAMING_SNAKE_CASE    (      a__						:      Dict					)				->							Any:
	__A   : int								= text.strip()
	if not text:
		return []
	__A   : int								= text.split()
	return tokens






class     lowerCamelCase_			(  _lowercase  ):




	def __init__(    self	:    List[str]				,			__A	:    Optional[int]=True				,			__A	:    int=None				,			__A	:    int=True				,			__A	:    int=None				,			__A	:    str=True				,			):
		if never_split is None:
			__A   : Tuple								= []
		__A   : Any								= do_lower_case
		__A   : Any								= set(__A       )
		__A   : Optional[int]								= tokenize_chinese_chars
		__A   : Union[str, Any]								= strip_accents
		__A   : str								= do_split_on_punc




	def 					lowerCAmelCase_					(    self	:    List[str]				,			__A	:    Optional[Any]				,			__A	:    Optional[int]=None       ):
		__A   : Optional[Any]								= self.never_split.union(set(__A       )       ) if never_split else self.never_split
		__A   : List[Any]								= self._clean_text(__A       )

		# This was added on November 1st, 2018 for the multilingual and Chinese
		# models. This is also applied to the English models now, but it doesn't
		# matter since the English models were not trained on any Chinese data
		# and generally don't have any Chinese data in them (there are Chinese
		# characters in the vocabulary because Wikipedia does have some Chinese
		# words in the English Wikipedia.).
		if self.tokenize_chinese_chars:
			__A   : List[str]								= self._tokenize_chinese_chars(__A       )
		# prevents treating the same character with different unicode codepoints as different characters
		__A   : List[Any]								= unicodedata.normalize("""NFC"""				,			__A       )
		__A   : Tuple								= whitespace_tokenize(__A       )
		__A   : Any								= []
		for token in orig_tokens:
			if token not in never_split:
				if self.do_lower_case:
					__A   : Optional[int]								= token.lower()
					if self.strip_accents is not False:
						__A   : Dict								= self._run_strip_accents(__A       )
				elif self.strip_accents:
					__A   : List[Any]								= self._run_strip_accents(__A       )
			split_tokens.extend(self._run_split_on_punc(__A				,			__A       )       )

		__A   : str								= whitespace_tokenize(""" """.join(__A       )       )
		return output_tokens




	def 					lowerCAmelCase_					(    self	:    List[str]				,			__A	:    Dict       ):
		__A   : Any								= unicodedata.normalize("""NFD"""				,			__A       )
		__A   : Dict								= []
		for char in text:
			__A   : Union[str, Any]								= unicodedata.category(__A       )
			if cat == "Mn":
				continue
			output.append(__A       )
		return "".join(__A       )




	def 					lowerCAmelCase_					(    self	:    int				,			__A	:    int				,			__A	:    List[str]=None       ):
		if not self.do_split_on_punc or (never_split is not None and text in never_split):
			return [text]
		__A   : Union[str, Any]								= list(__A       )
		__A   : str								= 0
		__A   : int								= True
		__A   : Dict								= []
		while i < len(__A       ):
			__A   : Tuple								= chars[i]
			if _is_punctuation(__A       ):
				output.append([char]       )
				__A   : Tuple								= True
			else:
				if start_new_word:
					output.append([]       )
				__A   : str								= False
				output[-1].append(__A       )
			i += 1

		return ["".join(__A       ) for x in output]




	def 					lowerCAmelCase_					(    self	:    Tuple				,			__A	:    Any       ):
		__A   : Optional[Any]								= []
		for char in text:
			__A   : Optional[int]								= ord(__A       )
			if self._is_chinese_char(__A       ):
				output.append(""" """       )
				output.append(__A       )
				output.append(""" """       )
			else:
				output.append(__A       )
		return "".join(__A       )




	def 					lowerCAmelCase_					(    self	:    Dict				,			__A	:    Optional[int]       ):
		# This defines a "chinese character" as anything in the CJK Unicode block:
		#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
		#
		# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
		# despite its name. The modern Korean Hangul alphabet is a different block,
		# as is Japanese Hiragana and Katakana. Those alphabets are used to write
		# space-separated words, so they are not treated specially and handled
		# like the all of the other languages.
		if (
		    (cp >= 0X4E00 and cp <= 0X9FFF)
		    or (cp >= 0X3400 and cp <= 0X4DBF)  #
		    or (cp >= 0X20000 and cp <= 0X2A6DF)  #
		    or (cp >= 0X2A700 and cp <= 0X2B73F)  #
		    or (cp >= 0X2B740 and cp <= 0X2B81F)  #
		    or (cp >= 0X2B820 and cp <= 0X2CEAF)  #
		    or (cp >= 0XF900 and cp <= 0XFAFF)
		    or (cp >= 0X2F800 and cp <= 0X2FA1F)  #
		):  #
			return True

		return False




	def 					lowerCAmelCase_					(    self	:    Union[str, Any]				,			__A	:    Any       ):
		__A   : Tuple								= []
		for char in text:
			__A   : int								= ord(__A       )
			if cp == 0 or cp == 0XFFFD or _is_control(__A       ):
				continue
			if _is_whitespace(__A       ):
				output.append(""" """       )
			else:
				output.append(__A       )
		return "".join(__A       )






def        __SCREAMING_SNAKE_CASE    (      a__						:      Tuple					)				->							Any:
	__A   : Dict								= set()
	__A   : Dict								= word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)					)
		__A   : Optional[int]								= char
	return pairs
def        __SCREAMING_SNAKE_CASE    (      a__						:      Any					)				->							Dict:
	__A   : List[Any]								= text.replace("""—""" ,"""-"""					)
	__A   : Union[str, Any]								= text.replace("""–""" ,"""-"""					)
	__A   : Optional[Any]								= text.replace("""―""" ,"""-"""					)
	__A   : str								= text.replace("""…""" ,"""..."""					)
	__A   : Any								= text.replace("""´""" ,"""'"""					)
	__A   : List[str]								= re.sub(r"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)""" ,r""" \1 """ ,a__					)
	__A   : str								= re.sub(r"""\s*\n\s*""" ,""" \n """ ,a__					)
	__A   : Optional[Any]								= re.sub(r"""[^\S\n]+""" ,""" """ ,a__					)
	return text.strip()







class     lowerCamelCase_			(  _lowercase  ):
	_lowercase		:      str    				=							VOCAB_FILES_NAMES
	_lowercase		:      Dict    				=							PRETRAINED_VOCAB_FILES_MAP
	_lowercase		:      Any    				=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowercase		:      Union[str, Any]    				=							['''input_ids''', '''attention_mask''']




	def __init__(    self	:    Union[str, Any]				,			__A	:    Optional[Any]				,			__A	:    Dict				,			__A	:    str="<unk>"				,			**__A	:    Dict       ):
		super().__init__(unk_token=__A				,			**__A       )

		try:
			import ftfy
			from spacy.lang.en import English

			__A   : int								= English()
			__A   : Union[str, Any]								= _nlp.tokenizer
			__A   : Tuple								= ftfy.fix_text
		except ImportError:
			logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""       )
			__A   : int								= BasicTokenizer(do_lower_case=__A       )
			__A   : Union[str, Any]								= None

		with open(__A				,			encoding="""utf-8"""       ) as vocab_handle:
			__A   : Union[str, Any]								= json.load(__A       )
		__A   : Any								= {v: k for k, v in self.encoder.items()}
		with open(__A				,			encoding="""utf-8"""       ) as merges_handle:
			__A   : Union[str, Any]								= merges_handle.read().split("""\n"""       )[1:-1]
		__A   : List[Any]								= [tuple(merge.split()       ) for merge in merges]
		__A   : Union[str, Any]								= dict(zip(__A				,			range(len(__A       )       )       )       )
		__A   : Union[str, Any]								= {}




	@property
	def 					lowerCAmelCase_					(    self	:    Optional[int]       ):
		return True




	@property
	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
		return len(self.encoder       )




	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
		return dict(self.encoder				,			**self.added_tokens_encoder       )




	def 					lowerCAmelCase_					(    self	:    Dict				,			__A	:    Any       ):
		__A   : Any								= tuple(token[:-1]       ) + (token[-1] + """</w>""",)
		if token in self.cache:
			return self.cache[token]
		__A   : Dict								= get_pairs(__A       )

		if not pairs:
			return token + "</w>"

		while True:
			__A   : Dict								= min(__A				,			key=lambda __A       : self.bpe_ranks.get(__A				,			float("""inf"""       )       )       )
			if bigram not in self.bpe_ranks:
				break
			__A						,							__A   : List[str]								= bigram
			__A   : Union[str, Any]								= []
			__A   : List[str]								= 0
			while i < len(__A       ):
				try:
					__A   : Optional[int]								= word.index(__A				,			__A       )
				except ValueError:
					new_word.extend(word[i:]       )
					break
				else:
					new_word.extend(word[i:j]       )
					__A   : List[str]								= j

				if word[i] == first and i < len(__A       ) - 1 and word[i + 1] == second:
					new_word.append(first + second       )
					i += 2
				else:
					new_word.append(word[i]       )
					i += 1
			__A   : List[str]								= tuple(__A       )
			__A   : Dict								= new_word
			if len(__A       ) == 1:
				break
			else:
				__A   : List[Any]								= get_pairs(__A       )
		__A   : Any								= """ """.join(__A       )
		if word == "\n  </w>":
			__A   : Optional[Any]								= """\n</w>"""
		__A   : Dict								= word
		return word




	def 					lowerCAmelCase_					(    self	:    List[str]				,			__A	:    Tuple       ):
		__A   : Union[str, Any]								= []
		if self.fix_text is None:
			# Using BERT's BasicTokenizer
			__A   : Union[str, Any]								= self.nlp.tokenize(__A       )
			for token in text:
				split_tokens.extend(list(self.bpe(__A       ).split(""" """       )       )       )
		else:
			# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
			__A   : Union[str, Any]								= self.nlp(text_standardize(self.fix_text(__A       )       )       )
			for token in text:
				split_tokens.extend(list(self.bpe(token.text.lower()       ).split(""" """       )       )       )
		return split_tokens




	def 					lowerCAmelCase_					(    self	:    List[str]				,			__A	:    Union[str, Any]       ):
		return self.encoder.get(__A				,			self.encoder.get(self.unk_token       )       )




	def 					lowerCAmelCase_					(    self	:    int				,			__A	:    Union[str, Any]       ):
		return self.decoder.get(__A				,			self.unk_token       )




	def 					lowerCAmelCase_					(    self	:    int				,			__A	:    Optional[int]       ):
		__A   : Union[str, Any]								= """""".join(__A       ).replace("""</w>"""				,			""" """       ).strip()
		return out_string




	def 					lowerCAmelCase_					(    self	:    Optional[Any]				,			__A	:    str				,			__A	:    Optional[str] = None       ):
		if not os.path.isdir(__A       ):
			logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
			return
		__A   : List[str]								= os.path.join(
		    __A				,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
		__A   : Dict								= os.path.join(
		    __A				,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

		with open(__A				,			"""w"""				,			encoding="""utf-8"""       ) as f:
			f.write(json.dumps(self.encoder				,			indent=2				,			sort_keys=__A				,			ensure_ascii=__A       ) + """\n"""       )

		__A   : Dict								= 0
		with open(__A				,			"""w"""				,			encoding="""utf-8"""       ) as writer:
			writer.write("""#version: 0.2\n"""       )
			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,			key=lambda __A       : kv[1]       ):
				if index != token_index:
					logger.warning(
					    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
					    """ Please check that the tokenizer is not corrupted!"""       )
					__A   : Any								= token_index
				writer.write(""" """.join(__A       ) + """\n"""       )
				index += 1

		return vocab_file, merge_file








import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase






@require_tokenizers
class     lowerCamelCase_			(  unittest.TestCase  ):
	_lowercase		:      List[str]    				=							EsmTokenizer




	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
		super().setUp()
		__A   : Dict								= tempfile.mkdtemp()
		# fmt: off
		__A   : List[str]								= ["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
		# fmt: on
		__A   : int								= os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES["""vocab_file"""]       )
		with open(self.vocab_file				,			"""w"""				,			encoding="""utf-8"""       ) as vocab_writer:
			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )




	def 					lowerCAmelCase_					(    self	:    Dict				,			**__A	:    int       ):
		return [self.get_tokenizer(**__A       )]




	def 					lowerCAmelCase_					(    self	:    Optional[int]				,			**__A	:    List[Any]       ):
		return self.tokenizer_class.from_pretrained(self.tmpdirname				,			**__A       )




	def 					lowerCAmelCase_					(    self	:    List[str]       ):
		__A   : Union[str, Any]								= self.tokenizer_class(self.vocab_file       )

		__A   : int								= tokenizer.tokenize("""LAGVS"""       )
		self.assertListEqual(__A				,			["""L""", """A""", """G""", """V""", """S"""]       )
		self.assertListEqual(tokenizer.convert_tokens_to_ids(__A       )				,			[4, 5, 6, 7, 8]       )




	def 					lowerCAmelCase_					(    self	:    str       ):
		__A   : Union[str, Any]								= self.tokenizer_class(self.vocab_file       )

		__A   : int								= """LAGVS"""
		self.assertListEqual(tokenizer.encode(__A       )				,			[0, 4, 5, 6, 7, 8, 2]       )




	def 					lowerCAmelCase_					(    self	:    Dict       ):
		__A   : str								= self.tokenizer_class(self.vocab_file       )

		__A   : Tuple								= ["""LAGVS""", """WCB"""]
		__A   : str								= tokenizer(__A				,			padding=__A       )["""input_ids"""]

		self.assertListEqual(__A				,			[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]       )




	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
		__A   : Dict								= self.tokenizer_class(self.vocab_file       )

		__A   : Any								= ["""LAGVS""", """WCB"""]
		__A   : str								= tokenizer(__A				,			padding=__A       )["""input_ids"""]

		self.assertListEqual(__A				,			[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]       )




	def 					lowerCAmelCase_					(    self	:    List[str]       ):
		__A   : Optional[int]								= self.get_tokenizers(fast=__A       )
		for tokenizer in tokenizers:
			with self.subTest(F"""{tokenizer.__class__.__name__}"""       ):
				__A   : Any								= """<unk>"""
				__A   : str								= """<mask>"""

				__A   : Optional[Any]								= tokenizer.tokenize(__A       )
				__A   : str								= tokenizer.tokenize(__A       )

				self.assertEqual(len(__A       )				,			1       )
				self.assertEqual(len(__A       )				,			1       )
				self.assertEqual(token_a[0]				,			__A       )
				self.assertEqual(token_a[0]				,			__A       )





