



'''simple docstring'''






import logging

# standard libraries
import sys

import numpy as np
import pytest  # type: ignore

# Custom/local libraries
from matrix import matrix_operation as matop

A_				:					Any							=      [[1_2, 1_0], [3, 9]]
A_				:					Tuple							=      [[3, 4], [7, 4]]
A_				:					List[str]							=      [[3, 0, 2], [2, 0, -2], [0, 1, 1]]
A_				:					Union[str, Any]							=      [[3, 0, -2], [2, 0, 2], [0, 1, 1]]
A_				:					Optional[int]							=      [[3, 0, 2], [2, 0, -2], [0, 1, 1], [2, 0, -2]]
A_				:					Tuple							=      [1]
A_				:					List[Any]							=      [2]

A_				:					Dict							=      logging.getLogger()
A_				:					Optional[int]							=      logging.DEBUG
A_				:					List[Any]							=      logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)




@pytest.mark.mat_ops
@pytest.mark.parametrize(
    ("""mat1""", """mat2""")			,  [(mat_a, mat_b), (mat_c, mat_d), (mat_d, mat_e), (mat_f, mat_h)]						)
def        snake_case_   (	lowerCAmelCase_			,  lowerCAmelCase_						)->						str:





       '''simple docstring'''


       if (np.array(lowerCAmelCase_						)).shape < (2, 2) or (np.array(lowerCAmelCase_						)).shape < (2, 2):
              with pytest.raises(lowerCAmelCase_						):
                     logger.info(F'''\n\t{test_addition.__name__} returned integer'''						)
                     matop.add(lowerCAmelCase_			,  lowerCAmelCase_						)
       elif (np.array(lowerCAmelCase_						)).shape == (np.array(lowerCAmelCase_						)).shape:
              logger.info(F'''\n\t{test_addition.__name__} with same matrix dims'''						)
              _UpperCAmelCase    :		int     	=					(np.array(lowerCAmelCase_						) + np.array(lowerCAmelCase_						)).tolist()
              _UpperCAmelCase    :		int     	=					matop.add(lowerCAmelCase_			,  lowerCAmelCase_						)
              assert theo == act
       else:
              with pytest.raises(lowerCAmelCase_						):
                     logger.info(F'''\n\t{test_addition.__name__} with different matrix dims'''						)
                     matop.add(lowerCAmelCase_			,  lowerCAmelCase_						)




@pytest.mark.mat_ops
@pytest.mark.parametrize(
    ("""mat1""", """mat2""")			,  [(mat_a, mat_b), (mat_c, mat_d), (mat_d, mat_e), (mat_f, mat_h)]						)
def        snake_case_   (	lowerCAmelCase_			,  lowerCAmelCase_						)->						Optional[Any]:





       '''simple docstring'''


       if (np.array(lowerCAmelCase_						)).shape < (2, 2) or (np.array(lowerCAmelCase_						)).shape < (2, 2):
              with pytest.raises(lowerCAmelCase_						):
                     logger.info(F'''\n\t{test_subtraction.__name__} returned integer'''						)
                     matop.subtract(lowerCAmelCase_			,  lowerCAmelCase_						)
       elif (np.array(lowerCAmelCase_						)).shape == (np.array(lowerCAmelCase_						)).shape:
              logger.info(F'''\n\t{test_subtraction.__name__} with same matrix dims'''						)
              _UpperCAmelCase    :		Tuple     	=					(np.array(lowerCAmelCase_						) - np.array(lowerCAmelCase_						)).tolist()
              _UpperCAmelCase    :		int     	=					matop.subtract(lowerCAmelCase_			,  lowerCAmelCase_						)
              assert theo == act
       else:
              with pytest.raises(lowerCAmelCase_						):
                     logger.info(F'''\n\t{test_subtraction.__name__} with different matrix dims'''						)
                     assert matop.subtract(lowerCAmelCase_			,  lowerCAmelCase_						)




@pytest.mark.mat_ops
@pytest.mark.parametrize(
    ("""mat1""", """mat2""")			,  [(mat_a, mat_b), (mat_c, mat_d), (mat_d, mat_e), (mat_f, mat_h)]						)
def        snake_case_   (	lowerCAmelCase_			,  lowerCAmelCase_						)->						List[Any]:





       '''simple docstring'''


       if (np.array(lowerCAmelCase_						)).shape < (2, 2) or (np.array(lowerCAmelCase_						)).shape < (2, 2):
              logger.info(F'''\n\t{test_multiplication.__name__} returned integer'''						)
              with pytest.raises(lowerCAmelCase_						):
                     matop.add(lowerCAmelCase_			,  lowerCAmelCase_						)
       elif (np.array(lowerCAmelCase_						)).shape == (np.array(lowerCAmelCase_						)).shape:
              logger.info(F'''\n\t{test_multiplication.__name__} meets dim requirements'''						)
              _UpperCAmelCase    :		str     	=					(np.matmul(lowerCAmelCase_			,  lowerCAmelCase_						)).tolist()
              _UpperCAmelCase    :		Union[str, Any]     	=					matop.multiply(lowerCAmelCase_			,  lowerCAmelCase_						)
              assert theo == act
       else:
              with pytest.raises(lowerCAmelCase_						):
                     logger.info(
                         F'''\n\t{test_multiplication.__name__} does not meet dim requirements'''						)
                     assert matop.subtract(lowerCAmelCase_			,  lowerCAmelCase_						)




@pytest.mark.mat_ops
def        snake_case_   (	)->						Optional[Any]:





       '''simple docstring'''


       _UpperCAmelCase    :		Optional[Any]     	=					(3.5 * np.array(lowerCAmelCase_						)).tolist()
       _UpperCAmelCase    :		List[Any]     	=					matop.scalar_multiply(lowerCAmelCase_			,  3.5						)
       assert theo == act




@pytest.mark.mat_ops
def        snake_case_   (	)->						Optional[Any]:





       '''simple docstring'''


       _UpperCAmelCase    :		Dict     	=					(np.identity(5						)).tolist()
       _UpperCAmelCase    :		Union[str, Any]     	=					matop.identity(5						)
       assert theo == act




@pytest.mark.mat_ops
@pytest.mark.parametrize("""mat"""			,  [mat_a, mat_b, mat_c, mat_d, mat_e, mat_f]						)
def        snake_case_   (	lowerCAmelCase_						)->						List[Any]:





       '''simple docstring'''


       if (np.array(lowerCAmelCase_						)).shape < (2, 2):
              with pytest.raises(lowerCAmelCase_						):
                     logger.info(F'''\n\t{test_transpose.__name__} returned integer'''						)
                     matop.transpose(lowerCAmelCase_						)
       else:
              _UpperCAmelCase    :		str     	=					(np.transpose(lowerCAmelCase_						)).tolist()
              _UpperCAmelCase    :		List[str]     	=					matop.transpose(lowerCAmelCase_			,  return_map=lowerCAmelCase_						)
              assert theo == act











import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase		,       unittest.TestCase):
						_SCREAMING_SNAKE_CASE						:							Tuple				=       PreTrainedTokenizerFast
						_SCREAMING_SNAKE_CASE						:							Any				=       False
						_SCREAMING_SNAKE_CASE						:							Optional[int]				=       True
						_SCREAMING_SNAKE_CASE						:							int				=       '''tokenizer_file'''






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    False  # because we don't have pretrained_vocab_files_map
												super().setUp()
												lowerCAmelCase__         =    True

												lowerCAmelCase__         =    ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
												lowerCAmelCase__         =    'SaulLu/dummy-tokenizer-bytelevel-bpe'

												# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
												lowerCAmelCase__         =    [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

												lowerCAmelCase__         =    PreTrainedTokenizerFast.from_pretrained(model_paths[0] )
												tokenizer.save_pretrained(self.tmpdirname )






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
												# model
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
												pass






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tmpdirname
												# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
																								try:
																														lowerCAmelCase__         =    tempfile.mkdtemp()
																														lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(_UpperCamelCase  ,    **_UpperCamelCase )

																														tokenizer.save_pretrained(self.tmpdirname )
																														super().test_training_new_tokenizer()
																								finally:
																														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																														# is restored
																														shutil.rmtree(self.tmpdirname )
																														lowerCAmelCase__         =    tmpdirname_orig






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tmpdirname
												# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
																								try:
																														lowerCAmelCase__         =    tempfile.mkdtemp()
																														lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(_UpperCamelCase  ,    **_UpperCamelCase )

																														tokenizer.save_pretrained(self.tmpdirname )
																														super().test_training_new_tokenizer_with_special_tokens_change()
																								finally:
																														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																														# is restored
																														shutil.rmtree(self.tmpdirname )
																														lowerCAmelCase__         =    tmpdirname_orig







						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name )

												lowerCAmelCase__         =    ('a' for _ in range(10_00 ))
												lowerCAmelCase__         =    tokenizer.train_new_from_iterator(text_iterator=_UpperCamelCase  ,    length=10_00  ,    vocab_size=50 )

												lowerCAmelCase__         =    new_tokenizer.encode('a🤗' )
												self.assertEqual(_UpperCamelCase  ,    [64, 1_72, 2_53, 97, 2_45] )




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( unittest.TestCase):






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    AutoTokenizer.from_pretrained('bert-base-cased' )
												lowerCAmelCase__         =    json.loads(tokenizer._tokenizer.to_str() )
												lowerCAmelCase__         =    len(_UpperCamelCase )

												with tempfile.TemporaryDirectory() as tmp_dir:
																		# Hack to save this in the tokenizer_config.json
																		lowerCAmelCase__         =    ['tokenizer.4.0.0.json']
																		tokenizer.save_pretrained(_UpperCamelCase )
																		json.dump(_UpperCamelCase  ,    open(os.path.join(_UpperCamelCase  ,    'tokenizer.4.0.0.json' )  ,    'w' ) )

																		# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
																		lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
																		self.assertEqual(len(_UpperCamelCase )  ,    len(_UpperCamelCase ) + 1 )
																		lowerCAmelCase__         =    json.loads(new_tokenizer._tokenizer.to_str() )
																		self.assertIn('huggingface'  ,    json_tokenizer['model']['vocab'] )

																		# Will need to be adjusted if we reach v42 and this test is still here.
																		# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
																		shutil.move(os.path.join(_UpperCamelCase  ,    'tokenizer.4.0.0.json' )  ,    os.path.join(_UpperCamelCase  ,    'tokenizer.42.0.0.json' ) )
																		lowerCAmelCase__         =    ['tokenizer.42.0.0.json']
																		tokenizer.save_pretrained(_UpperCamelCase )
																		lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
																		self.assertEqual(len(_UpperCamelCase )  ,    len(_UpperCamelCase ) )
																		lowerCAmelCase__         =    json.loads(new_tokenizer._tokenizer.to_str() )
																		self.assertNotIn('huggingface'  ,    json_tokenizer['model']['vocab'] )







						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
												lowerCAmelCase__         =    'hf-internal-testing/test-two-tokenizers'

												# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
												lowerCAmelCase__         =    AutoTokenizer.from_pretrained(_UpperCamelCase )
												self.assertEqual(len(_UpperCamelCase )  ,    2_89_97 )
												lowerCAmelCase__         =    json.loads(tokenizer._tokenizer.to_str() )
												self.assertIn('huggingface'  ,    json_tokenizer['model']['vocab'] )

												# Testing an older version by monkey-patching the version in the module it's used.
												import transformers as old_transformers

												lowerCAmelCase__         =    '3.0.0'
												lowerCAmelCase__         =    old_transformers.models.auto.AutoTokenizer.from_pretrained(_UpperCamelCase )
												self.assertEqual(len(_UpperCamelCase )  ,    2_89_96 )
												lowerCAmelCase__         =    json.loads(old_tokenizer._tokenizer.to_str() )
												self.assertNotIn('huggingface'  ,    json_tokenizer['model']['vocab'] )




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( unittest.TestCase):






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												# See https://github.com/huggingface/transformers/pull/12550
												# and https://github.com/huggingface/tokenizers/issues/537
												lowerCAmelCase__         =    PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel' )
												lowerCAmelCase__         =    'The Matrix is a 1999 science fiction action film.'

												with concurrent.futures.ThreadPoolExecutor() as executor:
																		lowerCAmelCase__         =    [executor.submit(self.fetch  ,    _UpperCamelCase  ,    _UpperCamelCase ) for i in range(10 )]
																		lowerCAmelCase__         =    [future.result() for future in futures]
																		self.assertEqual(_UpperCamelCase  ,    [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10 )] )







						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase ):






												"""simple docstring"""
												return tokenizer.encode(_UpperCamelCase  ,    truncation='longest_first'  ,    padding='longest' )





