



# This test is meant to be run in on an instance with TPUs like this:
#
#   python examples/pytorch/xla_spawn.py --num_cores=8 tests/test_trainer_tpu.py
#
# Replace 8 with the number of TPU cores you have.
#

import sys
from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.utils import logging


UpperCAmelCase_          =				logging.get_logger(__name__)


if is_torch_available():
    import torch
    from torch import nn
    from torch.utils.data import Dataset

    from transformers import Trainer





    class 		UpperCAmelCase    (    snake_case_					):





       def __init__(						self							,  _lowerCAmelCase = 101      ):
             _lowerCAmelCase			      =	length





       def __len__(						self      ):
             return self.length





       def __getitem__(						self							,  _lowerCAmelCase      ):
             return i






    class 		UpperCAmelCase    :





       def __call__(						self							,  _lowerCAmelCase      ):
             return {"input_ids": torch.tensor(_lowerCAmelCase      ), "labels": torch.tensor(_lowerCAmelCase      )}






    class 		UpperCAmelCase    (    nn.Module					):





       def __init__(						self      ):
             super().__init__()
             # Add some (unused) params otherwise DDP will complain.
             _lowerCAmelCase			      =	nn.Linear(120							,  80      )





       def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase							,  _lowerCAmelCase=None      ):
             if labels is not None:
                   return torch.tensor(0.0							,  device=input_ids.device      ), input_ids
             else:
                   return input_ids
def 				UpperCAmelCase__		(					)->str:
      _lowerCAmelCase			      =	HfArgumentParser((TrainingArguments,)					)
      sys.argv += ["--output_dir", "./examples"]
      _lowerCAmelCase			      =	parser.parse_args_into_dataclasses()[0]

      logger.warning(
          f'''Process rank: {training_args.local_rank}, device: {training_args.device}, '''
          f'''tpu_num_cores: {training_args.tpu_num_cores}'''  ,					)

      # Essentially, what we want to verify in the distributed case is
      # that we get all samples back, in the right order.
      # (this is crucial for prediction for instance)
      for dataset_length in [1_0_0_1, 2_5_6, 1_5]:
            _lowerCAmelCase			      =	DummyDataset(_SCREAMING_SNAKE_CASE					)

            def compute_metrics(_SCREAMING_SNAKE_CASE :						EvalPrediction					) -> Dict:
                  _lowerCAmelCase			      =	list(range(len(_SCREAMING_SNAKE_CASE					)					)					)
                  _lowerCAmelCase			      =	p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
                  return {"success": success}

            _lowerCAmelCase			      =	Trainer(
                model=DummyModel()  ,					args=_SCREAMING_SNAKE_CASE  ,					data_collator=DummyDataCollator()  ,					eval_dataset=_SCREAMING_SNAKE_CASE  ,					compute_metrics=_SCREAMING_SNAKE_CASE  ,					)
            _lowerCAmelCase			      =	trainer.evaluate()
            logger.info(_SCREAMING_SNAKE_CASE					)
            if metrics["eval_success"] is not True:
                  logger.error(_SCREAMING_SNAKE_CASE					)
                  exit(1					)

            _lowerCAmelCase			      =	trainer.predict(_SCREAMING_SNAKE_CASE					)
            logger.info(p.metrics					)
            if p.metrics["test_success"] is not True:
                  logger.error(p.metrics					)
                  exit(1					)

            _lowerCAmelCase			      =	2

            _lowerCAmelCase			      =	trainer.evaluate()
            logger.info(_SCREAMING_SNAKE_CASE					)
            if metrics["eval_success"] is not True:
                  logger.error(_SCREAMING_SNAKE_CASE					)
                  exit(1					)

            _lowerCAmelCase			      =	trainer.predict(_SCREAMING_SNAKE_CASE					)
            logger.info(p.metrics					)
            if p.metrics["test_success"] is not True:
                  logger.error(p.metrics					)
                  exit(1					)

            _lowerCAmelCase			      =	None

      logger.info('''🔥 All distributed tests successful'''					)
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Any					)->str:
      # For xla_spawn (TPUs)
      main()


if __name__ == "__main__":
    main()




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 		UpperCAmelCase    (    snake_case_							,unittest.TestCase					):
   SCREAMING_SNAKE_CASE__					      =  PreTrainedTokenizerFast
   SCREAMING_SNAKE_CASE__					      =  False
   SCREAMING_SNAKE_CASE__					      =  True
   SCREAMING_SNAKE_CASE__					      =  '''tokenizer_file'''





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	False  # because we don't have pretrained_vocab_files_map
         super().setUp()
         _lowerCAmelCase			      =	True

         _lowerCAmelCase			      =	['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
         _lowerCAmelCase			      =	'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

         # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
         _lowerCAmelCase			      =	[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

         _lowerCAmelCase			      =	PreTrainedTokenizerFast.from_pretrained(model_paths[0]      )
         tokenizer.save_pretrained(self.tmpdirname      )





   def 						__lowerCAmelCase				(						self      ):
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass





   def 						__lowerCAmelCase				(						self      ):
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass





   def 						__lowerCAmelCase				(						self      ):
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass





   def 						__lowerCAmelCase				(						self      ):
         # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
         pass





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
                     try:
                           _lowerCAmelCase			      =	tempfile.mkdtemp()
                           _lowerCAmelCase			      =	self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase							,  **_lowerCAmelCase      )

                           tokenizer.save_pretrained(self.tmpdirname      )
                           super().test_training_new_tokenizer()
                     finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname      )
                           _lowerCAmelCase			      =	tmpdirname_orig





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
                     try:
                           _lowerCAmelCase			      =	tempfile.mkdtemp()
                           _lowerCAmelCase			      =	self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase							,  **_lowerCAmelCase      )

                           tokenizer.save_pretrained(self.tmpdirname      )
                           super().test_training_new_tokenizer_with_special_tokens_change()
                     finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname      )
                           _lowerCAmelCase			      =	tmpdirname_orig





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name      )

         _lowerCAmelCase			      =	('''a''' for _ in range(1_000      ))
         _lowerCAmelCase			      =	tokenizer.train_new_from_iterator(text_iterator=_lowerCAmelCase							,  length=1_000							,  vocab_size=50      )

         _lowerCAmelCase			      =	new_tokenizer.encode('''a🤗'''      )
         self.assertEqual(_lowerCAmelCase							,  [64, 172, 253, 97, 245]      )






@require_tokenizers
class 		UpperCAmelCase    (    unittest.TestCase					):





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	AutoTokenizer.from_pretrained('''bert-base-cased'''      )
         _lowerCAmelCase			      =	json.loads(tokenizer._tokenizer.to_str()      )
         _lowerCAmelCase			      =	len(_lowerCAmelCase      )

         with tempfile.TemporaryDirectory() as tmp_dir:
               # Hack to save this in the tokenizer_config.json
               _lowerCAmelCase			      =	['''tokenizer.4.0.0.json''']
               tokenizer.save_pretrained(_lowerCAmelCase      )
               json.dump(_lowerCAmelCase							,  open(os.path.join(_lowerCAmelCase							,  '''tokenizer.4.0.0.json'''      )							,  '''w'''      )      )

               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               _lowerCAmelCase			      =	AutoTokenizer.from_pretrained(_lowerCAmelCase      )
               self.assertEqual(len(_lowerCAmelCase      )							,  len(_lowerCAmelCase      ) + 1      )
               _lowerCAmelCase			      =	json.loads(new_tokenizer._tokenizer.to_str()      )
               self.assertIn('''huggingface'''							,  json_tokenizer['''model''']['''vocab''']      )

               # Will need to be adjusted if we reach v42 and this test is still here.
               # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
               shutil.move(os.path.join(_lowerCAmelCase							,  '''tokenizer.4.0.0.json'''      )							,  os.path.join(_lowerCAmelCase							,  '''tokenizer.42.0.0.json'''      )      )
               _lowerCAmelCase			      =	['''tokenizer.42.0.0.json''']
               tokenizer.save_pretrained(_lowerCAmelCase      )
               _lowerCAmelCase			      =	AutoTokenizer.from_pretrained(_lowerCAmelCase      )
               self.assertEqual(len(_lowerCAmelCase      )							,  len(_lowerCAmelCase      )      )
               _lowerCAmelCase			      =	json.loads(new_tokenizer._tokenizer.to_str()      )
               self.assertNotIn('''huggingface'''							,  json_tokenizer['''model''']['''vocab''']      )





   def 						__lowerCAmelCase				(						self      ):
         # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
         _lowerCAmelCase			      =	'''hf-internal-testing/test-two-tokenizers'''

         # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
         _lowerCAmelCase			      =	AutoTokenizer.from_pretrained(_lowerCAmelCase      )
         self.assertEqual(len(_lowerCAmelCase      )							,  28_997      )
         _lowerCAmelCase			      =	json.loads(tokenizer._tokenizer.to_str()      )
         self.assertIn('''huggingface'''							,  json_tokenizer['''model''']['''vocab''']      )

         # Testing an older version by monkey-patching the version in the module it's used.
         import transformers as old_transformers

         _lowerCAmelCase			      =	'''3.0.0'''
         _lowerCAmelCase			      =	old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCAmelCase      )
         self.assertEqual(len(_lowerCAmelCase      )							,  28_996      )
         _lowerCAmelCase			      =	json.loads(old_tokenizer._tokenizer.to_str()      )
         self.assertNotIn('''huggingface'''							,  json_tokenizer['''model''']['''vocab''']      )






@require_tokenizers
class 		UpperCAmelCase    (    unittest.TestCase					):





   def 						__lowerCAmelCase				(						self      ):
         # See https://github.com/huggingface/transformers/pull/12550
         # and https://github.com/huggingface/tokenizers/issues/537
         _lowerCAmelCase			      =	PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''      )
         _lowerCAmelCase			      =	'''The Matrix is a 1999 science fiction action film.'''

         with concurrent.futures.ThreadPoolExecutor() as executor:
               _lowerCAmelCase			      =	[executor.submit(self.fetch							,  _lowerCAmelCase							,  _lowerCAmelCase      ) for i in range(10      )]
               _lowerCAmelCase			      =	[future.result() for future in futures]
               self.assertEqual(_lowerCAmelCase							,  [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10      )]      )





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase							,  _lowerCAmelCase      ):
         return tokenizer.encode(_lowerCAmelCase							,  truncation='''longest_first'''							,  padding='''longest'''      )