



import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


UpperCAmelCase_          =				logging.get_logger(__name__)

UpperCAmelCase_          =				{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

UpperCAmelCase_          =				{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

UpperCAmelCase_          =				{
    'openai-gpt': 5_1_2,
}
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						List[Any]					)->str:
      _lowerCAmelCase			      =	text.strip()
      if not text:
            return []
      _lowerCAmelCase			      =	text.split()
      return tokens





class 		UpperCAmelCase    (    __UpperCamelCase					):





   def __init__(						self							,  _lowerCAmelCase=True							,  _lowerCAmelCase=None							,  _lowerCAmelCase=True							,  _lowerCAmelCase=None							,  _lowerCAmelCase=True							,  ):






         if never_split is None:
               _lowerCAmelCase			      =	[]
         _lowerCAmelCase			      =	do_lower_case
         _lowerCAmelCase			      =	set(__UpperCamelCase      )
         _lowerCAmelCase			      =	tokenize_chinese_chars
         _lowerCAmelCase			      =	strip_accents
         _lowerCAmelCase			      =	do_split_on_punc





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase							,  _lowerCAmelCase=None      ):
         _lowerCAmelCase			      =	self.never_split.union(set(__UpperCamelCase      )      ) if never_split else self.never_split
         _lowerCAmelCase			      =	self._clean_text(__UpperCamelCase      )

         # This was added on November 1st, 2018 for the multilingual and Chinese
         # models. This is also applied to the English models now, but it doesn't
         # matter since the English models were not trained on any Chinese data
         # and generally don't have any Chinese data in them (there are Chinese
         # characters in the vocabulary because Wikipedia does have some Chinese
         # words in the English Wikipedia.).
         if self.tokenize_chinese_chars:
               _lowerCAmelCase			      =	self._tokenize_chinese_chars(__UpperCamelCase      )
         # prevents treating the same character with different unicode codepoints as different characters
         _lowerCAmelCase			      =	unicodedata.normalize('''NFC'''							,  __UpperCamelCase      )
         _lowerCAmelCase			      =	whitespace_tokenize(__UpperCamelCase      )
         _lowerCAmelCase			      =	[]
         for token in orig_tokens:
               if token not in never_split:
                     if self.do_lower_case:
                           _lowerCAmelCase			      =	token.lower()
                           if self.strip_accents is not False:
                                 _lowerCAmelCase			      =	self._run_strip_accents(__UpperCamelCase      )
                     elif self.strip_accents:
                           _lowerCAmelCase			      =	self._run_strip_accents(__UpperCamelCase      )
               split_tokens.extend(self._run_split_on_punc(__UpperCamelCase							,  __UpperCamelCase      )      )

         _lowerCAmelCase			      =	whitespace_tokenize(''' '''.join(__UpperCamelCase      )      )
         return output_tokens





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase      ):
         _lowerCAmelCase			      =	unicodedata.normalize('''NFD'''							,  __UpperCamelCase      )
         _lowerCAmelCase			      =	[]
         for char in text:
               _lowerCAmelCase			      =	unicodedata.category(__UpperCamelCase      )
               if cat == "Mn":
                     continue
               output.append(__UpperCamelCase      )
         return "".join(__UpperCamelCase      )





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase							,  _lowerCAmelCase=None      ):






         if not self.do_split_on_punc or (never_split is not None and text in never_split):
               return [text]
         _lowerCAmelCase			      =	list(__UpperCamelCase      )
         _lowerCAmelCase			      =	0
         _lowerCAmelCase			      =	True
         _lowerCAmelCase			      =	[]
         while i < len(__UpperCamelCase      ):
               _lowerCAmelCase			      =	chars[i]
               if _is_punctuation(__UpperCamelCase      ):
                     output.append([char]      )
                     _lowerCAmelCase			      =	True
               else:
                     if start_new_word:
                           output.append([]      )
                     _lowerCAmelCase			      =	False
                     output[-1].append(__UpperCamelCase      )
               i += 1

         return ["".join(__UpperCamelCase      ) for x in output]





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase      ):
         _lowerCAmelCase			      =	[]
         for char in text:
               _lowerCAmelCase			      =	ord(__UpperCamelCase      )
               if self._is_chinese_char(__UpperCamelCase      ):
                     output.append(''' '''      )
                     output.append(__UpperCamelCase      )
                     output.append(''' '''      )
               else:
                     output.append(__UpperCamelCase      )
         return "".join(__UpperCamelCase      )





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase      ):






         if (
             (cp >= 0x4e_00 and cp <= 0x9f_ff)
             or (cp >= 0x34_00 and cp <= 0x4d_bf)  #
             or (cp >= 0x2_00_00 and cp <= 0x2_a6_df)  #
             or (cp >= 0x2_a7_00 and cp <= 0x2_b7_3f)  #
             or (cp >= 0x2_b7_40 and cp <= 0x2_b8_1f)  #
             or (cp >= 0x2_b8_20 and cp <= 0x2_ce_af)  #
             or (cp >= 0xf9_00 and cp <= 0xfa_ff)
             or (cp >= 0x2_f8_00 and cp <= 0x2_fa_1f)  #
         ):  #
               return True

         return False





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase      ):
         _lowerCAmelCase			      =	[]
         for char in text:
               _lowerCAmelCase			      =	ord(__UpperCamelCase      )
               if cp == 0 or cp == 0xff_fd or _is_control(__UpperCamelCase      ):
                     continue
               if _is_whitespace(__UpperCamelCase      ):
                     output.append(''' '''      )
               else:
                     output.append(__UpperCamelCase      )
         return "".join(__UpperCamelCase      )






def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						List[Any]					)->List[str]:
      _lowerCAmelCase			      =	set()
      _lowerCAmelCase			      =	word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)					)
            _lowerCAmelCase			      =	char
      return pairs
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						List[str]					)->Optional[Any]:
      _lowerCAmelCase			      =	text.replace('''—'''  ,					'''-'''					)
      _lowerCAmelCase			      =	text.replace('''–'''  ,					'''-'''					)
      _lowerCAmelCase			      =	text.replace('''―'''  ,					'''-'''					)
      _lowerCAmelCase			      =	text.replace('''…'''  ,					'''...'''					)
      _lowerCAmelCase			      =	text.replace('''´'''  ,					'''\''''					)
      _lowerCAmelCase			      =	re.sub(r'''(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''  ,					r''' \1 '''  ,					_SCREAMING_SNAKE_CASE					)
      _lowerCAmelCase			      =	re.sub(r'''\s*\n\s*'''  ,					''' \n '''  ,					_SCREAMING_SNAKE_CASE					)
      _lowerCAmelCase			      =	re.sub(r'''[^\S\n]+'''  ,					''' '''  ,					_SCREAMING_SNAKE_CASE					)
      return text.strip()





class 		UpperCAmelCase    (    __UpperCamelCase					):
   SCREAMING_SNAKE_CASE__					      =  VOCAB_FILES_NAMES
   SCREAMING_SNAKE_CASE__					      =  PRETRAINED_VOCAB_FILES_MAP
   SCREAMING_SNAKE_CASE__					      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   SCREAMING_SNAKE_CASE__					      =  ['''input_ids''', '''attention_mask''']





   def __init__(						self							,  _lowerCAmelCase							,  _lowerCAmelCase							,  _lowerCAmelCase="<unk>"							,  **_lowerCAmelCase      ):
         super().__init__(unk_token=__UpperCamelCase							,  **__UpperCamelCase      )

         try:
               import ftfy
               from spacy.lang.en import English

               _lowerCAmelCase			      =	English()
               _lowerCAmelCase			      =	_nlp.tokenizer
               _lowerCAmelCase			      =	ftfy.fix_text
         except ImportError:
               logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''      )
               _lowerCAmelCase			      =	BasicTokenizer(do_lower_case=__UpperCamelCase      )
               _lowerCAmelCase			      =	None

         with open(__UpperCamelCase							,  encoding='''utf-8'''      ) as vocab_handle:
               _lowerCAmelCase			      =	json.load(__UpperCamelCase      )
         _lowerCAmelCase			      =	{v: k for k, v in self.encoder.items()}
         with open(__UpperCamelCase							,  encoding='''utf-8'''      ) as merges_handle:
               _lowerCAmelCase			      =	merges_handle.read().split('''\n'''      )[1:-1]
         _lowerCAmelCase			      =	[tuple(merge.split()      ) for merge in merges]
         _lowerCAmelCase			      =	dict(zip(__UpperCamelCase							,  range(len(__UpperCamelCase      )      )      )      )
         _lowerCAmelCase			      =	{}





   @property
   def 						__lowerCAmelCase				(						self      ):
         return True





   @property
   def 						__lowerCAmelCase				(						self      ):
         return len(self.encoder      )





   def 						__lowerCAmelCase				(						self      ):
         return dict(self.encoder							,  **self.added_tokens_encoder      )





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase      ):
         _lowerCAmelCase			      =	tuple(token[:-1]      ) + (token[-1] + """</w>""",)
         if token in self.cache:
               return self.cache[token]
         _lowerCAmelCase			      =	get_pairs(__UpperCamelCase      )

         if not pairs:
               return token + "</w>"

         while True:
               _lowerCAmelCase			      =	min(__UpperCamelCase							,  key=lambda _lowerCAmelCase      : self.bpe_ranks.get(__UpperCamelCase							,  float('''inf'''      )      )      )
               if bigram not in self.bpe_ranks:
                     break
               _lowerCAmelCase			      =	bigram
               _lowerCAmelCase			      =	[]
               _lowerCAmelCase			      =	0
               while i < len(__UpperCamelCase      ):
                     try:
                           _lowerCAmelCase			      =	word.index(__UpperCamelCase							,  __UpperCamelCase      )
                     except ValueError:
                           new_word.extend(word[i:]      )
                           break
                     else:
                           new_word.extend(word[i:j]      )
                           _lowerCAmelCase			      =	j

                     if word[i] == first and i < len(__UpperCamelCase      ) - 1 and word[i + 1] == second:
                           new_word.append(first + second      )
                           i += 2
                     else:
                           new_word.append(word[i]      )
                           i += 1
               _lowerCAmelCase			      =	tuple(__UpperCamelCase      )
               _lowerCAmelCase			      =	new_word
               if len(__UpperCamelCase      ) == 1:
                     break
               else:
                     _lowerCAmelCase			      =	get_pairs(__UpperCamelCase      )
         _lowerCAmelCase			      =	""" """.join(__UpperCamelCase      )
         if word == "\n  </w>":
               _lowerCAmelCase			      =	"""\n</w>"""
         _lowerCAmelCase			      =	word
         return word





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase      ):
         _lowerCAmelCase			      =	[]
         if self.fix_text is None:
               # Using BERT's BasicTokenizer
               _lowerCAmelCase			      =	self.nlp.tokenize(__UpperCamelCase      )
               for token in text:
                     split_tokens.extend(list(self.bpe(__UpperCamelCase      ).split(''' '''      )      )      )
         else:
               # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
               _lowerCAmelCase			      =	self.nlp(text_standardize(self.fix_text(__UpperCamelCase      )      )      )
               for token in text:
                     split_tokens.extend(list(self.bpe(token.text.lower()      ).split(''' '''      )      )      )
         return split_tokens





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase      ):
         return self.encoder.get(__UpperCamelCase							,  self.encoder.get(self.unk_token      )      )





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase      ):
         return self.decoder.get(__UpperCamelCase							,  self.unk_token      )





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase      ):
         _lowerCAmelCase			      =	"""""".join(__UpperCamelCase      ).replace('''</w>'''							,  ''' '''      ).strip()
         return out_string





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase							,  _lowerCAmelCase = None      ):






         if not os.path.isdir(__UpperCamelCase      ):
               logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
               return
         _lowerCAmelCase			      =	os.path.join(
             __UpperCamelCase							,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )
         _lowerCAmelCase			      =	os.path.join(
             __UpperCamelCase							,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']      )

         with open(__UpperCamelCase							,  '''w'''							,  encoding='''utf-8'''      ) as f:
               f.write(json.dumps(self.encoder							,  indent=2							,  sort_keys=__UpperCamelCase							,  ensure_ascii=__UpperCamelCase      ) + '''\n'''      )

         _lowerCAmelCase			      =	0
         with open(__UpperCamelCase							,  '''w'''							,  encoding='''utf-8'''      ) as writer:
               writer.write('''#version: 0.2\n'''      )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,  key=lambda _lowerCAmelCase      : kv[1]      ):
                     if index != token_index:
                           logger.warning(
                               F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                               ''' Please check that the tokenizer is not corrupted!'''      )
                           _lowerCAmelCase			      =	token_index
                     writer.write(''' '''.join(__UpperCamelCase      ) + '''\n'''      )
                     index += 1

         return vocab_file, merge_file

'''simple docstring'''







import unittest

from cer import CER


__SCREAMING_SNAKE_CASE		:  List[str]        =      CER()
class  SCREAMING_SNAKE_CASE__       (  unittest.TestCase   ):
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Optional[Any]								=	["""White House"""]
					__a       :     Tuple								=	["""white house"""]
					# S = 2, D = 0, I = 0, N = 11, CER = 2 / 11
					__a       :     Any								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.1_8_1_8_1_8_1_8_1_8			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Optional[int]								=	["""were wolf"""]
					__a       :     Tuple								=	["""werewolf"""]
					# S = 0, D = 0, I = 1, N = 9, CER = 1 / 9
					__a       :     Any								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.1_1_1_1_1_1_1			) < 1E-6			)

					__a       :     str								=	["""werewolf"""]
					__a       :     List[Any]								=	["""weae     wolf"""]
					# S = 1, D = 1, I = 0, N = 8, CER = 0.25
					__a       :     Tuple								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.2_5			) < 1E-6			)

					# consecutive whitespaces case 1
					__a       :     int								=	["""were wolf"""]
					__a       :     Dict								=	["""were               wolf"""]
					# S = 0, D = 0, I = 0, N = 9, CER = 0
					__a       :     str								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.0			) < 1E-6			)

					# consecutive whitespaces case 2
					__a       :     List[str]								=	["""were   wolf"""]
					__a       :     Optional[int]								=	["""were               wolf"""]
					# S = 0, D = 0, I = 0, N = 9, CER = 0
					__a       :     Optional[Any]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.0			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Dict								=	["""werewolf"""]
					__a       :     str								=	["""weaewolf"""]
					# S = 1, D = 0, I = 0, N = 8, CER = 0.125
					__a       :     Optional[int]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.1_2_5			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[str]								=	["""werewolf"""]
					__a       :     Tuple								=	["""wereawolf"""]
					# S = 0, D = 1, I = 0, N = 8, CER = 0.125
					__a       :     Optional[Any]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.1_2_5			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[str]								=	["""werewolf"""]
					__a       :     Optional[Any]								=	["""wereolf"""]
					# S = 0, D = 0, I = 1, N = 8, CER = 0.125
					__a       :     int								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.1_2_5			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Dict								=	["""werewolf"""]
					__a       :     Tuple								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertEqual(__UpperCamelCase  ,    0.0			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[str]								=	["""werewolf""", """I am your father"""]
					__a       :     int								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertEqual(__UpperCamelCase  ,    0.0			)

					__a       :     Dict								=	["""werewolf""", """I am your father""", """doge"""]
					__a       :     Optional[Any]								=	["""werxwolf""", """I       am your father""", """doge"""]
					# S = 1, D = 0, I = 0, N = 28, CER = 1 / 28
					__a       :     Union[str, Any]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.0_3_5_7_1_4_2_8			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[str]								=	["""My hovercraft""", """is full of eels"""]
					__a       :     Union[str, Any]								=	["""My hovercraft is full""", """ of eels"""]
					# S = 0, D = 0, I = 2, N = 28, CER = 2 / 28
					# whitespace at the front of " of eels" will be strip during preporcessing
					# so need to insert 2 whitespaces
					__a       :     str								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase  ,    concatenate_texts=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.0_7_1_4_2_8			) < 1E-6			)
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     int								=	["""我能吞下玻璃而不伤身体"""]
					__a       :     Optional[Any]								=	[""" 能吞虾玻璃而 不霜身体啦"""]
					# S = 3, D = 2, I = 0, N = 11, CER = 5 / 11
					__a       :     List[Any]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.4_5_4_5_4_5_4_5_4_5			) < 1E-6			)

					__a       :     Union[str, Any]								=	["""我能吞下玻璃""", """而不伤身体"""]
					__a       :     Optional[Any]								=	["""我    能 吞 下 玻 璃""", """而不伤身体"""]
					# S = 0, D = 5, I = 0, N = 11, CER = 5 / 11
					__a       :     List[str]								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertTrue(abs(char_error_rate - 0.4_5_4_5_4_5_4_5_4_5_4_5			) < 1E-6			)

					__a       :     Union[str, Any]								=	["""我能吞下玻璃而不伤身体"""]
					__a       :     Dict								=	cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)
					self.assertFalse(__UpperCamelCase  ,    0.0			)

	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     List[Any]								=	[""""""]
					__a       :     Dict								=	["""Hypothesis"""]
					with self.assertRaises(__UpperCamelCase			):
									cer.compute(predictions=__UpperCamelCase  ,    references=__UpperCamelCase			)


if __name__ == "__main__":
					unittest.main()