


import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_	:     List[Any]							     =	logging.get_logger(__name__)

UpperCAmelCase_	:     int							     =	{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

UpperCAmelCase_	:     Dict							     =	{
    'vocab_file': {
        'microsoft/biogpt': 'https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json',
    },
    'merges_file': {'microsoft/biogpt': 'https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt'},
}

UpperCAmelCase_	:     List[Any]							     =	{
    'microsoft/biogpt': 1024,
}







def     snake_case_     (						SCREAMING_SNAKE_CASE__  ):







				"""simple docstring"""





				_SCREAMING_SNAKE_CASE				: str  	=   set()
				_SCREAMING_SNAKE_CASE				: List[Any]  	=   word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)  )
								_SCREAMING_SNAKE_CASE				: Dict  	=   char
				return pairs







class 					lowercase__						(		_snake_case      ):





							'''simple docstring'''


							A_							:   Union[str, Any]   			=						VOCAB_FILES_NAMES
							A_							:   Union[str, Any]   			=						PRETRAINED_VOCAB_FILES_MAP
							A_							:   Union[str, Any]   			=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							A_							:   str   			=						["""input_ids""", """attention_mask"""]






							def __init__(					self					,      __snake_case					,      __snake_case					,      __snake_case="<unk>"					,      __snake_case="<s>"					,      __snake_case="</s>"					,      __snake_case="</s>"					,      __snake_case="<pad>"					,      **__snake_case					,      ):
											super().__init__(
											    bos_token=__snake_case					,      eos_token=__snake_case					,      sep_token=__snake_case					,      unk_token=__snake_case					,      pad_token=__snake_case					,      **__snake_case					,      )

											try:
															import sacremoses
											except ImportError:
															raise ImportError(
															    """You need to install sacremoses to use BioGptTokenizer. """
															    """See https://pypi.org/project/sacremoses/ for installation."""						)

											_SCREAMING_SNAKE_CASE				: str  	=   '''en'''
											_SCREAMING_SNAKE_CASE				: str  	=   sacremoses
											# cache of sm.MosesTokenizer instance
											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   {}
											_SCREAMING_SNAKE_CASE				: Tuple  	=   {}
											with open(__snake_case					,      encoding="""utf-8"""						) as vocab_handle:
															_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   json.load(__snake_case						)
											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   {v: k for k, v in self.encoder.items()}
											with open(__snake_case					,      encoding="""utf-8"""						) as merges_handle:
															_SCREAMING_SNAKE_CASE				: Any  	=   merges_handle.read().split("""\n"""						)[:-1]
											_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   [tuple(merge.split()[:2]						) for merge in merges]
											_SCREAMING_SNAKE_CASE				: Dict  	=   dict(zip(__snake_case					,      range(len(__snake_case						)						)						)						)
											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   {}






							@property
							def UpperCAmelCase_			(					self						):
											return len(self.encoder						)






							def UpperCAmelCase_			(					self						):
											return dict(self.encoder					,      **self.added_tokens_encoder						)






							def UpperCAmelCase_			(					self					,      __snake_case					,      __snake_case						):
											if lang not in self.cache_moses_tokenizer:
															_SCREAMING_SNAKE_CASE				: Dict  	=   self.sm.MosesTokenizer(lang=__snake_case						)
															_SCREAMING_SNAKE_CASE				: Dict  	=   moses_tokenizer
											return self.cache_moses_tokenizer[lang].tokenize(
											    __snake_case					,      aggressive_dash_splits=__snake_case					,      return_str=__snake_case					,      escape=__snake_case						)






							def UpperCAmelCase_			(					self					,      __snake_case					,      __snake_case						):
											if lang not in self.cache_moses_detokenizer:
															_SCREAMING_SNAKE_CASE				: List[Any]  	=   self.sm.MosesDetokenizer(lang=__snake_case						)
															_SCREAMING_SNAKE_CASE				: str  	=   moses_detokenizer
											return self.cache_moses_detokenizer[lang].detokenize(__snake_case						)






							def UpperCAmelCase_			(					self					,      __snake_case						):
											_SCREAMING_SNAKE_CASE				: int  	=   tuple(token[:-1]						) + (token[-1] + '''</w>''',)
											if token in self.cache:
															return self.cache[token]
											_SCREAMING_SNAKE_CASE				: Dict  	=   get_pairs(__snake_case						)

											if not pairs:
															return token + "</w>"

											while True:
															_SCREAMING_SNAKE_CASE				: Dict  	=   min(__snake_case					,      key=lambda __snake_case						: self.bpe_ranks.get(__snake_case					,      float("""inf"""						)						)						)
															if bigram not in self.bpe_ranks:
																			break
															_SCREAMING_SNAKE_CASE				: str  	=   bigram
															_SCREAMING_SNAKE_CASE				: str  	=   []
															_SCREAMING_SNAKE_CASE				: int  	=   0
															while i < len(__snake_case						):
																			try:
																							_SCREAMING_SNAKE_CASE				: str  	=   word.index(__snake_case					,      __snake_case						)
																			except ValueError:
																							new_word.extend(word[i:]						)
																							break
																			else:
																							new_word.extend(word[i:j]						)
																							_SCREAMING_SNAKE_CASE				: Any  	=   j

																			if word[i] == first and i < len(__snake_case						) - 1 and word[i + 1] == second:
																							new_word.append(first + second						)
																							i += 2
																			else:
																							new_word.append(word[i]						)
																							i += 1
															_SCREAMING_SNAKE_CASE				: Tuple  	=   tuple(__snake_case						)
															_SCREAMING_SNAKE_CASE				: Any  	=   new_word
															if len(__snake_case						) == 1:
																			break
															else:
																			_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   get_pairs(__snake_case						)
											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   ''' '''.join(__snake_case						)
											if word == "\n  </w>":
															_SCREAMING_SNAKE_CASE				: Tuple  	=   '''\n</w>'''
											_SCREAMING_SNAKE_CASE				: Dict  	=   word
											return word






							def UpperCAmelCase_			(					self					,      __snake_case					,      __snake_case=False						):
											if bypass_tokenizer:
															_SCREAMING_SNAKE_CASE				: str  	=   text.split()
											else:
															_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   self.moses_tokenize(__snake_case					,      self.lang						)

											_SCREAMING_SNAKE_CASE				: str  	=   []
											for token in text:
															if token:
																			split_tokens.extend(list(self.bpe(__snake_case						).split(""" """						)						)						)

											return split_tokens






							def UpperCAmelCase_			(					self					,      __snake_case						):
											return self.encoder.get(__snake_case					,      self.encoder.get(self.unk_token						)						)






							def UpperCAmelCase_			(					self					,      __snake_case						):
											return self.decoder.get(__snake_case					,      self.unk_token						)






							def UpperCAmelCase_			(					self					,      __snake_case						):
											_SCREAMING_SNAKE_CASE				: Any  	=   [t.replace(""" """					,      """"""						).replace("""</w>"""					,      """ """						) for t in tokens]
											_SCREAMING_SNAKE_CASE				: Any  	=   ''''''.join(__snake_case						).split()
											# detokenize
											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   self.moses_detokenize(__snake_case					,      self.lang						)
											return text






							def UpperCAmelCase_			(					self					,      __snake_case					,      __snake_case = None						):
											if token_ids_a is None:
															return [self.sep_token_id] + token_ids_a
											_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   [self.sep_token_id]
											return sep + token_ids_a + sep + token_ids_a






							def UpperCAmelCase_			(					self					,      __snake_case					,      __snake_case = None					,      __snake_case = False						):
											if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=__snake_case					,      token_ids_a=__snake_case					,      already_has_special_tokens=__snake_case						)
											# no bos used in fairseq
											if token_ids_a is not None:
															return [1] + ([0] * len(__snake_case						)) + [1] + ([0] * len(__snake_case						))
											return [1] + ([0] * len(__snake_case						))






							def UpperCAmelCase_			(					self					,      __snake_case					,      __snake_case = None						):
											_SCREAMING_SNAKE_CASE				: int  	=   [self.sep_token_id]

											# no bos used in fairseq
											if token_ids_a is None:
															return len(token_ids_a + sep						) * [0]
											return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






							def UpperCAmelCase_			(					self					,      __snake_case					,      __snake_case = None						):
											if not os.path.isdir(__snake_case						):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
															return
											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   os.path.join(
											    __snake_case					,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
											_SCREAMING_SNAKE_CASE				: List[str]  	=   os.path.join(
											    __snake_case					,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]						)

											with open(__snake_case					,      """w"""					,      encoding="""utf-8"""						) as f:
															f.write(json.dumps(self.encoder					,      indent=2					,      sort_keys=__snake_case					,      ensure_ascii=__snake_case						) + """\n"""						)

											_SCREAMING_SNAKE_CASE				: Tuple  	=   0
											with open(__snake_case					,      """w"""					,      encoding="""utf-8"""						) as writer:
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,      key=lambda __snake_case						: kv[1]						):
																			if index != token_index:
																							logger.warning(
																							    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																							    """ Please check that the tokenizer is not corrupted!"""						)
																							_SCREAMING_SNAKE_CASE				: Any  	=   token_index
																			writer.write(""" """.join(__snake_case						) + """\n"""						)
																			index += 1

											return vocab_file, merge_file






							def __getstate__(					self						):
											_SCREAMING_SNAKE_CASE				: Optional[Any]  	=   self.__dict__.copy()
											_SCREAMING_SNAKE_CASE				: List[str]  	=   None
											return state






							def __setstate__(					self					,      __snake_case						):
											_SCREAMING_SNAKE_CASE				: Union[str, Any]  	=   d

											try:
															import sacremoses
											except ImportError:
															raise ImportError(
															    """You need to install sacremoses to use XLMTokenizer. """
															    """See https://pypi.org/project/sacremoses/ for installation."""						)

											_SCREAMING_SNAKE_CASE				: Optional[int]  	=   sacremoses








'''simple docstring'''


from ..utils import DummyObject, requires_backends





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   str   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Union[str, Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Tuple   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Tuple   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   str   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Dict   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   str   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   str   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Tuple   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   int   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Tuple   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Dict   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[int]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Union[str, Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Dict   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Tuple   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[str]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Any   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Dict   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Union[str, Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   Optional[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)





class 					lowercase__						(		metaclass=_snake_case      ):





							'''simple docstring'''


							A_							:   List[Any]   			=						["""tokenizers"""]






							def __init__(					self					,      *__snake_case					,      **__snake_case						):
											requires_backends(self					,      ["""tokenizers"""]						)




