
'''simple docstring'''
import itertools
import os
import subprocess
from os.path import dirname

from parameterized import parameterized

from tests.trainer.test_trainer import TrainerIntegrationCommon  # noqa
from transformers import is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_tests_dir,
    require_deepspeed,
    require_torch_gpu,
    slow,
)
from transformers.trainer_utils import set_seed


if is_torch_available():
     from tests.trainer.test_trainer import (  # noqa
         RegressionModelConfig,
         RegressionPreTrainedModel,
         get_regression_trainer,
     )


set_seed(42)

A__ :		Dict					=get_tests_dir('fixtures')
A__ :		Optional[Any]					=os.path.join(dirname(get_tests_dir()))
A__ :		List[str]					=dirname(os.path.abspath(__file__))

# default torch.distributed port
A__ :		Tuple					='''10999'''

A__ :		str					='''t5-small'''

# *** Working Models ***
A__ :		Any					='''hf-internal-testing/tiny-albert'''
A__ :		Union[str, Any]					='''sshleifer/bart-tiny-random'''
A__ :		Union[str, Any]					='''hf-internal-testing/tiny-bert'''
A__ :		int					='''hf-internal-testing/tiny-random-bigbird_pegasus'''
A__ :		Union[str, Any]					='''hf-internal-testing/tiny-random-big_bird'''
A__ :		List[str]					='''hf-internal-testing/tiny-random-blenderbot'''
A__ :		List[Any]					='''bigscience/bigscience-small-testing'''
A__ :		int					='''hf-internal-testing/tiny-random-deberta'''
A__ :		Optional[Any]					='''hf-internal-testing/tiny-random-deberta-v2'''
A__ :		Dict					='''sshleifer/tiny-distilbert-base-cased'''
A__ :		Dict					='''hf-internal-testing/tiny-electra'''
A__ :		Union[str, Any]					='''hf-internal-testing/tiny-random-flaubert'''
A__ :		int					='''stas/tiny-wmt19-en-de'''
A__ :		Union[str, Any]					='''hf-internal-testing/tiny-random-funnel'''
A__ :		Optional[Any]					='''sshleifer/tiny-gpt2'''
A__ :		Tuple					='''hf-internal-testing/tiny-random-gptj'''
A__ :		Optional[int]					='''hf-internal-testing/tiny-random-gpt_neo'''
A__ :		int					='''hf-internal-testing/tiny-layoutlm'''
A__ :		List[Any]					='''hf-internal-testing/tiny-random-led'''
A__ :		Tuple					='''hf-internal-testing/tiny-random-longformer'''
A__ :		List[str]					='''stas/tiny-m2m_100'''  # hf tiny model is unsuitable
A__ :		List[Any]					='''sshleifer/tiny-marian-en-de'''
A__ :		int					='''sshleifer/tiny-mbart'''
A__ :		Union[str, Any]					='''hf-internal-testing/tiny-random-mobilebert'''
A__ :		Union[str, Any]					='''hf-internal-testing/tiny-random-mpnet'''
A__ :		List[Any]					='''stas/pegasus-cnn_dailymail-tiny-random'''
A__ :		str					='''hf-internal-testing/tiny-random-prophetnet'''
A__ :		Union[str, Any]					='''sshleifer/tiny-distilroberta-base'''
A__ :		Optional[int]					='''hf-internal-testing/tiny-random-squeezebert'''
A__ :		Tuple					='''patrickvonplaten/t5-tiny-random'''
A__ :		Tuple					='''hf-internal-testing/tiny-random-t5-v1.1'''
A__ :		Any					='''hf-internal-testing/tiny-random-vit'''
A__ :		Tuple					='''hf-internal-testing/tiny-xlm-roberta'''
A__ :		Union[str, Any]					='''sshleifer/tiny-xlnet-base-cased'''


# *** To Fix ***


# *** tiny model issues ***
# missing model files:
A__ :		int					='''hf-internal-testing/tiny-random-mt5'''
A__ :		Optional[Any]					='''hf-internal-testing/tiny-random-camembert'''
A__ :		int					='''hf-internal-testing/tiny-random-openai-gpt'''

# missing tokenizer files
A__ :		Union[str, Any]					='''hf-internal-testing/tiny-random-convbert'''
A__ :		Dict					='''hf-internal-testing/tiny-random-layoutlmv2'''
A__ :		List[str]					='''hf-internal-testing/tiny-random-hubert'''

# issues with tokenizer
A__ :		Tuple					='''hf-internal-testing/tiny-random-ctrl'''
A__ :		Optional[int]					='''hf-internal-testing/tiny-random-transfo-xl'''  # same as ctrl

# other issues with tiny models
A__ :		List[Any]					='''hf-internal-testing/tiny-random-ibert'''  # multiple issues with either mlm/qa/clas
A__ :		int					='''hf-internal-testing/tiny-random-reformer'''  # multiple issues with either mlm/qa/clas

# *** Lacking official examples to test with ***
# or not working with examples
A__ :		Tuple					='''hf-internal-testing/tiny-random-dpr'''
# - "dpr"  examples/research_projects/rag-end2end-retriever/
A__ :		Any					='''hf-internal-testing/tiny-random-rag'''
# - "rag" research_projects
A__ :		int					=''''''
# - "luke" Entities classes - no plan to make such example
A__ :		List[str]					='''hf-internal-testing/tiny-random-lxmert'''
# - "lxmert" doesn't work with run_qa.py
A__ :		List[str]					='''hf-internal-testing/tiny-random-clip'''
# - "clip" nothing under pytorch examples - XXX: Suraj is working on adding some - check by end of Sep
A__ :		str					='''hf-internal-testing/tiny-random-speech_to_text'''
# - "speech_to_text", nothing under pytorch examples


# *** Reactive mode ***
# models with low usage, unstable API, things about to change - do nothing about the following until someone runs into a problem
A__ :		Optional[Any]					='''hf-internal-testing/tiny-random-tapas'''







def 							A_  (    __SCREAMING_SNAKE_CASE    :	int=False			)					->							int:





  """simple docstring"""

  __A		:  List[Any]       				=		min(2      ,  get_gpu_count()			) if distributed else 1
  __A		:  int       				=		os.environ.get("""DS_TEST_PORT"""      ,  __snake_case			)
  return F"deepspeed --num_nodes 1 --num_gpus {num_gpus} --master_port {master_port}".split()







def 							A_  (    )					->							Union[str, Any]:





  """simple docstring"""

  __A		:  Tuple       				=		F"{FIXTURE_DIRECTORY}/tests_samples"
  __A		:  Any       				=		F"{data_dir_samples}/wmt_en_ro"
  __A		:  Dict       				=		F"{data_dir_samples}/xsum"
  __A		:  Optional[int]       				=		"""\n        --do_train\n        --max_train_samples 4\n        --per_device_train_batch_size 2\n        --num_train_epochs 1\n        --fp16\n        --report_to none\n        --overwrite_output_dir\n        """.split()

  # try to cover as many models as possible once (it's enough to run on one task per model)
  # but need a tiny model for each
  #
  # should have "{model_type.upper()}_TINY" corresponding vars defined, e.g., T5_TINY, etc.
  __A		:  Tuple       				=		{
      """trans""": [
          """bart""",
          """fsmt""",
          """m2m_100""",
          """marian""",
          """mbart""",
          """t5""",
          """t5_v1""",
          # "mt5", missing model files
      ],
      """sum""": [
          """pegasus""",
      ],
      """clm""": [
          """big_bird""",
          """bigbird_pegasus""",
          """blenderbot""",
          """bloom""",
          """gpt2""",
          """gpt_neo""",
          """gptj""",
          """xlm-roberta""",
          """prophetnet""",
          # "camembert", missing model files
      ],
      """mlm""": [
          """albert""",
          """deberta""",
          """deberta-v2""",
          """distilbert""",
          """electra""",
          """flaubert""",
          """funnel""",
          """layoutlm""",
          # "reformer", # multiple issues with either mlm/qa/clas
      ],
      """qa""": [
          """led""",
          """longformer""",
          """mobilebert""",
          """mpnet""",
          """roberta""",
          """squeezebert""",
          # "convbert", # missing tokenizer files
          # "layoutlmv2", missing model files
      ],
      """clas""": [
          """bert""",
          """xlnet""",
          # "hubert", # missing tokenizer files
          # "ibert", # multiple issues with either mlm/qa/clas
          # "transfo-xl", # tokenizer issues as ctrl
          # "ctrl", # tokenizer issues
          # "openai-gpt", missing model files
          # "tapas", multiple issues
      ],
      """img_clas""": [
          """vit""",
      ],
  }

  __A		:  Optional[int]       				=		F"{ROOT_DIRECTORY}/examples/pytorch"

  __A		:  str       				=		{
      """trans""": F"\n        {scripts_dir}/translation/run_translation.py\n        --train_file {data_dir_wmt}/train.json\n        --source_lang en\n        --target_lang ro\n        ",
      """sum""": F"\n        {scripts_dir}/summarization/run_summarization.py\n        --train_file {data_dir_xsum}/sample.json\n        --max_source_length 12\n        --max_target_length 12\n        --lang en\n        ",
      """clm""": F"\n        {scripts_dir}/language-modeling/run_clm.py\n        --train_file {FIXTURE_DIRECTORY}/sample_text.txt\n        --block_size 8\n        ",
      """mlm""": F"\n        {scripts_dir}/language-modeling/run_mlm.py\n        --train_file {FIXTURE_DIRECTORY}/sample_text.txt\n        ",
      """qa""": F"\n        {scripts_dir}/question-answering/run_qa.py\n        --train_file {data_dir_samples}/SQUAD/sample.json\n        ",
      """clas""": F"\n        {scripts_dir}/text-classification/run_glue.py\n        --train_file {data_dir_samples}/MRPC/train.csv\n        --max_seq_length 12\n        --task_name MRPC\n        ",
      """img_clas""": F"\n        {scripts_dir}/image-classification/run_image_classification.py\n            --dataset_name hf-internal-testing/cats_vs_dogs_sample\n            --remove_unused_columns False\n            --max_steps 10\n            --image_processor_name {DS_TESTS_DIRECTORY}/vit_feature_extractor.json\n        ",
  }

  __A		:  Tuple       				=		get_launcher(distributed=__snake_case			)

  __A		:  List[Any]       				=		{}
  for task, args in tasks.items():
    __A		:  str       				=		args.split()
    for model in tasksamodels[task]:
      __A		:  str       				=		globals()[F"{model.upper().replace('-'      ,  '_'			)}_TINY"]
      __A		:  int       				=		F"--model_name_or_path {model_name}".split()
      __A		:  Optional[int]       				=		launcher + args + args_model + args_main

      # # generation special case
      # if task == "gen":
      #     launcher = f"deepspeed --num_nodes 1 --num_gpus 1".split()
      #     args_model += f"--model_type {model}".split()
      #     cmds[f"{task}_{model}"] = launcher + args + args_model
      # else:

  return cmds


A__ :		Dict					=make_task_cmds()

A__ :		str					='''zero2'''
A__ :		Tuple					='''zero3'''

A__ :		Optional[int]					=[ZEROa, ZEROa]







def 							A_  (    __SCREAMING_SNAKE_CASE    :	Union[str, Any]      ,  __SCREAMING_SNAKE_CASE    :	List[Any]      ,  __SCREAMING_SNAKE_CASE    :	Dict			)					->							Dict:





  """simple docstring"""

  __A		:  Dict       				=		parameterized.to_safe_name("""_""".join(str(__snake_case			) for x in param.args			)			)
  return F"{func.__name__}_{param_based_name}"


# Cartesian-product of zero stages with models to test
A__ :		int					=list(itertools.product(stages, task_cmds.keys()))







@slow
@require_deepspeed
@require_torch_gpu
class 			__A					(			_UpperCAmelCase      ):
 def 		lowercase_( self      :     List[str]					, lowerCamelCase      :     Dict					, lowerCamelCase      :     List[str]							):



   """simple docstring"""





   if task not in task_cmds:
     raise ValueError(f"don\'t know of task {task}, have {task_cmds.keys()}"							)

   __A		:  Tuple       				=		task_cmds[task]
   __A		:  Optional[Any]       				=		f"--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json".split()

   __A		:  List[Any]       				=		self.get_auto_remove_tmp_dir()
   __A		:  int       				=		f"--output_dir {output_dir}".split()

   cmd += args_ds + args_out

   return cmd, output_dir






 @parameterized.expand(_lowerCAmelCase					, name_func=_lowerCAmelCase							)
 def 		lowercase_( self      :     Tuple					, lowerCamelCase      :     List[Any]					, lowerCamelCase      :     Tuple							):



   """simple docstring"""





   __A				,       __A		:  Optional[Any]       				=		self.get_task_cmd(_lowerCAmelCase					, _lowerCAmelCase							)

   # 1. generate the checkpoint
   cmd += "--save_steps 1".split()
   # keep for quick debug
   # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] + cmd)); die
   execute_subprocess_async(_lowerCAmelCase					, env=self.get_env()							)

   # 2. test that the fp32 weights get reconsolidated
   __A		:  Tuple       				=		f"{output_dir}/checkpoint-1"
   __A		:  Tuple       				=		f"{chkpt_dir}/out.bin"
   __A		:  Dict       				=		f"{chkpt_dir}/zero_to_fp32.py {chkpt_dir} {recovered_model_path}"
   # keep for quick debug
   # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
   subprocess.check_call(_lowerCAmelCase					, shell=_lowerCAmelCase							)
   assert os.path.exists(_lowerCAmelCase							), f"{recovered_model_path} was not found"

   # possibly could also test that the resulting saved model is usable but given that we use
   # random models we won't know if it's any good







import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class      __snake_case				(   _UpperCAmelCase	,      unittest.TestCase):
					SCREAMING_SNAKE_CASE_		:       str								=			ElectraTokenizer
					SCREAMING_SNAKE_CASE_		:       Any								=			ElectraTokenizerFast
					SCREAMING_SNAKE_CASE_		:       Tuple								=			True
					SCREAMING_SNAKE_CASE_		:       str								=			True
					SCREAMING_SNAKE_CASE_		:       str								=			filter_non_english
					def 	a			(						self       :						int  )							->					Optional[int]:



											'''simple docstring'''





											super().setUp()

											__magic_name__					=      [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											__magic_name__					=      os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES['vocab_file']  )
											with open(self.vocab_file	,      'w'	,      encoding='utf-8'  ) as vocab_writer:
																	vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
					def 	a			(						self       :						Tuple	,      _lowerCAmelCase       :						Dict  )							->					List[Any]:



											'''simple docstring'''





											__magic_name__					=      'UNwant\u00E9d,running'
											__magic_name__					=      'unwanted, running'
											return input_text, output_text
					def 	a			(						self       :						Union[str, Any]  )							->					Tuple:



											'''simple docstring'''





											__magic_name__					=      self.tokenizer_class(self.vocab_file  )

											__magic_name__					=      tokenizer.tokenize('UNwant\u00E9d,running'  )
											self.assertListEqual(_lowerCAmelCase	,      ['un', '##want', '##ed', ',', 'runn', '##ing']  )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase  )	,      [9, 6, 7, 1_2, 1_0, 1_1]  )
					def 	a			(						self       :						Optional[int]  )							->					Dict:



											'''simple docstring'''





											if not self.test_rust_tokenizer:
																	return

											__magic_name__					=      self.get_tokenizer()
											__magic_name__					=      self.get_rust_tokenizer()

											__magic_name__					=      'UNwant\u00E9d,running'

											__magic_name__					=      tokenizer.tokenize(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.tokenize(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      self.get_rust_tokenizer()
											__magic_name__					=      tokenizer.encode(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											# With lower casing
											__magic_name__					=      self.get_tokenizer(do_lower_case=_lowerCAmelCase  )
											__magic_name__					=      self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase  )

											__magic_name__					=      'UNwant\u00E9d,running'

											__magic_name__					=      tokenizer.tokenize(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.tokenize(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      self.get_rust_tokenizer()
											__magic_name__					=      tokenizer.encode(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )
					def 	a			(						self       :						str  )							->					Optional[Any]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )	,      ['ah', '\u535A', '\u63A8', 'zz']  )
					def 	a			(						self       :						Optional[int]  )							->					int:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )	,      ['hello', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['hello']  )
					def 	a			(						self       :						int  )							->					Union[str, Any]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['hällo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['h\u00E9llo']  )
					def 	a			(						self       :						Any  )							->					Tuple:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['hello']  )
					def 	a			(						self       :						List[str]  )							->					str:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['hello']  )
					def 	a			(						self       :						Union[str, Any]  )							->					Optional[Any]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )	,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
					def 	a			(						self       :						int  )							->					int:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
					def 	a			(						self       :						Dict  )							->					int:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
					def 	a			(						self       :						Tuple  )							->					List[str]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      never_split=['[UNK]']  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )	,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
					def 	a			(						self       :						Dict  )							->					List[str]:



											'''simple docstring'''





											__magic_name__					=      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											__magic_name__					=      {}
											for i, token in enumerate(_lowerCAmelCase  ):
																	__magic_name__					=      i
											__magic_name__					=      WordpieceTokenizer(vocab=_lowerCAmelCase	,      unk_token='[UNK]'  )

											self.assertListEqual(tokenizer.tokenize(''  )	,      []  )

											self.assertListEqual(tokenizer.tokenize('unwanted running'  )	,      ['un', '##want', '##ed', 'runn', '##ing']  )

											self.assertListEqual(tokenizer.tokenize('unwantedX running'  )	,      ['[UNK]', 'runn', '##ing']  )
					def 	a			(						self       :						List[str]  )							->					Union[str, Any]:



											'''simple docstring'''





											self.assertTrue(_is_whitespace(' '  )  )
											self.assertTrue(_is_whitespace('\t'  )  )
											self.assertTrue(_is_whitespace('\r'  )  )
											self.assertTrue(_is_whitespace('\n'  )  )
											self.assertTrue(_is_whitespace('\u00A0'  )  )

											self.assertFalse(_is_whitespace('A'  )  )
											self.assertFalse(_is_whitespace('-'  )  )
					def 	a			(						self       :						Optional[int]  )							->					Dict:



											'''simple docstring'''





											self.assertTrue(_is_control('\u0005'  )  )

											self.assertFalse(_is_control('A'  )  )
											self.assertFalse(_is_control(' '  )  )
											self.assertFalse(_is_control('\t'  )  )
											self.assertFalse(_is_control('\r'  )  )
					def 	a			(						self       :						List[Any]  )							->					List[str]:



											'''simple docstring'''





											self.assertTrue(_is_punctuation('-'  )  )
											self.assertTrue(_is_punctuation('$'  )  )
											self.assertTrue(_is_punctuation('`'  )  )
											self.assertTrue(_is_punctuation('.'  )  )

											self.assertFalse(_is_punctuation('A'  )  )
											self.assertFalse(_is_punctuation(' '  )  )
					def 	a			(						self       :						Optional[Any]  )							->					int:



											'''simple docstring'''





											__magic_name__					=      self.get_tokenizer()
											__magic_name__					=      self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase  ) for t in ['Test', '\xad', 'test']]	,      [['[UNK]'], [], ['[UNK]']]  )

											self.assertListEqual(
											    [rust_tokenizer.tokenize(_lowerCAmelCase  ) for t in ['Test', '\xad', 'test']]	,      [['[UNK]'], [], ['[UNK]']]  )
					@slow
					def 	a			(						self       :						str  )							->					List[str]:



											'''simple docstring'''





											__magic_name__					=      self.tokenizer_class.from_pretrained('google/electra-base-discriminator'  )

											__magic_name__					=      tokenizer.encode('sequence builders'	,      add_special_tokens=_lowerCAmelCase  )
											__magic_name__					=      tokenizer.encode('multi-sequence build'	,      add_special_tokens=_lowerCAmelCase  )

											__magic_name__					=      tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase  )
											__magic_name__					=      tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase	,      _lowerCAmelCase  )

											assert encoded_sentence == [1_0_1] + text + [1_0_2]
											assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
					def 	a			(						self       :						int  )							->					Union[str, Any]:



											'''simple docstring'''





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																							__magic_name__					=      self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )

																							__magic_name__					=      F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																							__magic_name__					=      tokenizer_r.encode_plus(
																							    _lowerCAmelCase	,      return_attention_mask=_lowerCAmelCase	,      return_token_type_ids=_lowerCAmelCase	,      return_offsets_mapping=_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase	,      )

																							__magic_name__					=      tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase	,      'do_lower_case'  ) else False
																							__magic_name__					=      (
																							    [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'A'),
																							        ((1, 2), ','),
																							        ((3, 5), 'na'),
																							        ((5, 6), '##ï'),
																							        ((6, 8), '##ve'),
																							        ((9, 1_5), tokenizer_r.mask_token),
																							        ((1_6, 2_1), 'Allen'),
																							        ((2_1, 2_3), '##NL'),
																							        ((2_3, 2_4), '##P'),
																							        ((2_5, 3_3), 'sentence'),
																							        ((3_3, 3_4), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							    if not do_lower_case
																							    else [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'a'),
																							        ((1, 2), ','),
																							        ((3, 8), 'naive'),
																							        ((9, 1_5), tokenizer_r.mask_token),
																							        ((1_6, 2_1), 'allen'),
																							        ((2_1, 2_3), '##nl'),
																							        ((2_3, 2_4), '##p'),
																							        ((2_5, 3_3), 'sentence'),
																							        ((3_3, 3_4), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							)

																							self.assertEqual(
																							    [e[1] for e in expected_results]	,      tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
																							self.assertEqual([e[0] for e in expected_results]	,      tokens['offset_mapping']  )







					def 	a			(						self       :						Any  )							->					int:



											'''simple docstring'''





											__magic_name__					=      ['的', '人', '有']
											__magic_name__					=      ''.join(_lowerCAmelCase  )
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																							__magic_name__					=      True
																							__magic_name__					=      self.tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )
																							__magic_name__					=      self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_p.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_r.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase  )

																							# it is expected that each Chinese character is not preceded by "##"
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

																							__magic_name__					=      False
																							__magic_name__					=      self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )
																							__magic_name__					=      self.tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_r.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_p.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase  )

																							# it is expected that only the first Chinese character is not preceded by "##".
																							__magic_name__					=      [
																							    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase  )
																							]
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

