




"""simple docstring"""
import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class  _SCREAMING_SNAKE_CASE       (     SCREAMING_SNAKE_CASE__			,				unittest.TestCase				):



 '''simple docstring'''





 __UpperCAmelCase		   =							PreTrainedTokenizerFast
 __UpperCAmelCase		   =							False
 __UpperCAmelCase		   =							True
 __UpperCAmelCase		   =							"""tokenizer_file"""






 def lowercase_      (self   ):

  '''simple docstring'''







  _UpperCamelCase      :	Optional[int]					      =  False  # because we don't have pretrained_vocab_files_map
  super().setUp()
  _UpperCamelCase      :	Any					      =  True

  _UpperCamelCase      :	Union[str, Any]					      =  ["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
  _UpperCamelCase      :	Dict					      =  "SaulLu/dummy-tokenizer-bytelevel-bpe"

  # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
  _UpperCamelCase      :	List[str]					      =  [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

  _UpperCamelCase      :	int					      =  PreTrainedTokenizerFast.from_pretrained(model_paths[0]   )
  tokenizer.save_pretrained(self.tmpdirname   )






 def lowercase_      (self   ):

  '''simple docstring'''







  pass






 def lowercase_      (self   ):

  '''simple docstring'''







  pass






 def lowercase_      (self   ):

  '''simple docstring'''







  pass






 def lowercase_      (self   ):

  '''simple docstring'''







  pass






 def lowercase_      (self   ):

  '''simple docstring'''







  _UpperCamelCase      :	Dict					      =  self.tmpdirname
  # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
   with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
    try:
     _UpperCamelCase      :	Union[str, Any]					      =  tempfile.mkdtemp()
     _UpperCamelCase      :	Dict					      =  self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__					,     **lowerCAmelCase__   )

     tokenizer.save_pretrained(self.tmpdirname   )
     super().test_training_new_tokenizer()
    finally:
     # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
     # is restored
     shutil.rmtree(self.tmpdirname   )
     _UpperCamelCase      :	int					      =  tmpdirname_orig






 def lowercase_      (self   ):

  '''simple docstring'''







  _UpperCamelCase      :	Any					      =  self.tmpdirname
  # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
   with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
    try:
     _UpperCamelCase      :	int					      =  tempfile.mkdtemp()
     _UpperCamelCase      :	Dict					      =  self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__					,     **lowerCAmelCase__   )

     tokenizer.save_pretrained(self.tmpdirname   )
     super().test_training_new_tokenizer_with_special_tokens_change()
    finally:
     # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
     # is restored
     shutil.rmtree(self.tmpdirname   )
     _UpperCamelCase      :	Dict					      =  tmpdirname_orig






 def lowercase_      (self   ):

  '''simple docstring'''







  _UpperCamelCase      :	Any					      =  self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name   )

  _UpperCamelCase      :	List[Any]					      =  ("a" for _ in range(10_00   ))
  _UpperCamelCase      :	Any					      =  tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase__					,     length=10_00					,     vocab_size=50   )

  _UpperCamelCase      :	str					      =  new_tokenizer.encode("a🤗"   )
  self.assertEqual(lowerCAmelCase__					,     [64, 1_72, 2_53, 97, 2_45]   )







@require_tokenizers
class  _SCREAMING_SNAKE_CASE       (     unittest.TestCase				):



 '''simple docstring'''





 def lowercase_      (self   ):

  '''simple docstring'''







  _UpperCamelCase      :	str					      =  AutoTokenizer.from_pretrained("bert-base-cased"   )
  _UpperCamelCase      :	List[str]					      =  json.loads(tokenizer._tokenizer.to_str()   )
  _UpperCamelCase      :	Any					      =  len(lowerCAmelCase__   )

  with tempfile.TemporaryDirectory() as tmp_dir:
   # Hack to save this in the tokenizer_config.json
   _UpperCamelCase      :	Optional[Any]					      =  ["tokenizer.4.0.0.json"]
   tokenizer.save_pretrained(lowerCAmelCase__   )
   json.dump(lowerCAmelCase__					,     open(os.path.join(lowerCAmelCase__					,     "tokenizer.4.0.0.json"   )					,     "w"   )   )

   # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
   _UpperCamelCase      :	str					      =  AutoTokenizer.from_pretrained(lowerCAmelCase__   )
   self.assertEqual(len(lowerCAmelCase__   )					,     len(lowerCAmelCase__   ) + 1   )
   _UpperCamelCase      :	Union[str, Any]					      =  json.loads(new_tokenizer._tokenizer.to_str()   )
   self.assertIn("huggingface"					,     json_tokenizer["model"]["vocab"]   )

   # Will need to be adjusted if we reach v42 and this test is still here.
   # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
   shutil.move(os.path.join(lowerCAmelCase__					,     "tokenizer.4.0.0.json"   )					,     os.path.join(lowerCAmelCase__					,     "tokenizer.42.0.0.json"   )   )
   _UpperCamelCase      :	Optional[Any]					      =  ["tokenizer.42.0.0.json"]
   tokenizer.save_pretrained(lowerCAmelCase__   )
   _UpperCamelCase      :	str					      =  AutoTokenizer.from_pretrained(lowerCAmelCase__   )
   self.assertEqual(len(lowerCAmelCase__   )					,     len(lowerCAmelCase__   )   )
   _UpperCamelCase      :	str					      =  json.loads(new_tokenizer._tokenizer.to_str()   )
   self.assertNotIn("huggingface"					,     json_tokenizer["model"]["vocab"]   )






 def lowercase_      (self   ):

  '''simple docstring'''







  _UpperCamelCase      :	Any					      =  "hf-internal-testing/test-two-tokenizers"

  # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
  _UpperCamelCase      :	Optional[int]					      =  AutoTokenizer.from_pretrained(lowerCAmelCase__   )
  self.assertEqual(len(lowerCAmelCase__   )					,     2_89_97   )
  _UpperCamelCase      :	Tuple					      =  json.loads(tokenizer._tokenizer.to_str()   )
  self.assertIn("huggingface"					,     json_tokenizer["model"]["vocab"]   )

  # Testing an older version by monkey-patching the version in the module it's used.
  import transformers as old_transformers

  _UpperCamelCase      :	Optional[int]					      =  "3.0.0"
  _UpperCamelCase      :	Dict					      =  old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase__   )
  self.assertEqual(len(lowerCAmelCase__   )					,     2_89_96   )
  _UpperCamelCase      :	Optional[Any]					      =  json.loads(old_tokenizer._tokenizer.to_str()   )
  self.assertNotIn("huggingface"					,     json_tokenizer["model"]["vocab"]   )







@require_tokenizers
class  _SCREAMING_SNAKE_CASE       (     unittest.TestCase				):



 '''simple docstring'''





 def lowercase_      (self   ):

  '''simple docstring'''







  _UpperCamelCase      :	List[str]					      =  PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"   )
  _UpperCamelCase      :	Any					      =  "The Matrix is a 1999 science fiction action film."

  with concurrent.futures.ThreadPoolExecutor() as executor:
   _UpperCamelCase      :	str					      =  [executor.submit(self.fetch					,     lowerCAmelCase__					,     lowerCAmelCase__   ) for i in range(10   )]
   _UpperCamelCase      :	Dict					      =  [future.result() for future in futures]
   self.assertEqual(lowerCAmelCase__					,     [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10   )]   )






 def lowercase_      (self					,     lowerCAmelCase__					,     lowerCAmelCase__   ):

  '''simple docstring'''







  return tokenizer.encode(lowerCAmelCase__					,     truncation="longest_first"					,     padding="longest"   )









"""simple docstring"""
import numpy as np
from numpy import floataa
from numpy.typing import NDArray






def 				__lowerCAmelCase					(					__lowerCAmelCase			:		NDArray[floataa]  ,	__lowerCAmelCase			:		NDArray[floataa]						)    ->       NDArray[floataa]:
 _UpperCamelCase      ,	_UpperCamelCase      :	Optional[Any]					      =  np.shape(__lowerCAmelCase						)

 _UpperCamelCase      :	NDArray[floataa]					      =  np.zeros((rows, 1)  ,	dtype=__lowerCAmelCase						)
 for row in reversed(range(__lowerCAmelCase						)						):
  _UpperCamelCase      :	List[str]					      =  0
  for col in range(row + 1  ,	__lowerCAmelCase						):
   total += coefficients[row, col] * x[col]

  _UpperCamelCase      :	Dict					      =  (vector[row] - total) / coefficients[row, row]

 return x






def 				__lowerCAmelCase					(					__lowerCAmelCase			:		NDArray[floataa]  ,	__lowerCAmelCase			:		NDArray[floataa]						)    ->       NDArray[floataa]:
 _UpperCamelCase      ,	_UpperCamelCase      :	Optional[Any]					      =  np.shape(__lowerCAmelCase						)
 if rows != columns:
  return np.array(()  ,	dtype=__lowerCAmelCase						)

 # augmented matrix
 _UpperCamelCase      :	NDArray[floataa]					      =  np.concatenate((coefficients, vector)  ,	axis=1						)
 _UpperCamelCase      :	List[str]					      =  augmented_mat.astype("float64"						)

 # scale the matrix leaving it triangular
 for row in range(rows - 1						):
  _UpperCamelCase      :	Optional[Any]					      =  augmented_mat[row, row]
  for col in range(row + 1  ,	__lowerCAmelCase						):
   _UpperCamelCase      :	Union[str, Any]					      =  augmented_mat[col, row] / pivot
   augmented_mat[col, :] -= factor * augmented_mat[row, :]

 _UpperCamelCase      :	Any					      =  retroactive_resolution(
     augmented_mat[:, 0:columns]  ,	augmented_mat[:, columns : columns + 1]						)

 return x


if __name__ == "__main__":
    import doctest

    doctest.testmod()



