


import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a				:							Any					=			get_tests_dir('fixtures/spiece.model')

@require_sentencepiece
@require_tokenizers
class      lowerCamelCase	(  SCREAMING_SNAKE_CASE,			unittest.TestCase     ):
       _UpperCamelCase    		=      ErnieMTokenizer
       _UpperCamelCase    		=      False
       _UpperCamelCase    		=      True
       _UpperCamelCase    		=      False
       _UpperCamelCase    		=      False

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        super().setUp()

        # We have a SentencePiece fixture for testing
        snake_case   : Union[str, Any]	  =      ErnieMTokenizer(a__	,						unk_token='''<unk>'''	,						pad_token='''<pad>'''					)
        tokenizer.save_pretrained(self.tmpdirname					)

       def 				UpperCAmelCase				(			self	,						a__					):






        '''simple docstring'''






        snake_case   : int	  =      '''this is a test'''
        snake_case   : Any	  =      '''this is a test'''
        return input_text, output_text

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        snake_case   : List[Any]	  =      '''<pad>'''
        snake_case   : Any	  =      0

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__					)	,						a__					)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__					)	,						a__					)

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        snake_case   : Optional[int]	  =      list(self.get_tokenizer().get_vocab().keys()					)

        self.assertEqual(vocab_keys[0]	,						'''<pad>'''					)
        self.assertEqual(vocab_keys[1]	,						'''<unk>'''					)
        self.assertEqual(vocab_keys[-1]	,						'''▁eloquent'''					)
        self.assertEqual(len(a__					)	,						3_00_00					)

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        self.assertEqual(self.get_tokenizer().vocab_size	,						3_00_00					)

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        if not self.test_rust_tokenizer:
         return

        snake_case   : List[Any]	  =      self.get_tokenizer()
        snake_case   : Union[str, Any]	  =      self.get_rust_tokenizer()

        snake_case   : Optional[Any]	  =      '''I was born in 92000, and this is falsé.'''

        snake_case   : str	  =      tokenizer.tokenize(a__					)
        snake_case   : Dict	  =      rust_tokenizer.tokenize(a__					)
        self.assertListEqual(a__	,						a__					)

        snake_case   : int	  =      tokenizer.encode(a__	,						add_special_tokens=a__					)
        snake_case   : Any	  =      rust_tokenizer.encode(a__	,						add_special_tokens=a__					)
        self.assertListEqual(a__	,						a__					)

        snake_case   : Dict	  =      self.get_rust_tokenizer()
        snake_case   : Any	  =      tokenizer.encode(a__					)
        snake_case   : Optional[int]	  =      rust_tokenizer.encode(a__					)
        self.assertListEqual(a__	,						a__					)

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        snake_case   : str	  =      ErnieMTokenizer(a__	,						do_lower_case=a__	,						unk_token='''<unk>'''	,						pad_token='''<pad>'''					)

        snake_case   : int	  =      tokenizer.tokenize('''This is a test'''					)
        self.assertListEqual(a__	,						['''▁this''', '''▁is''', '''▁a''', '''▁test''']					)

        self.assertListEqual(tokenizer.convert_tokens_to_ids(a__					)	,						[48, 25, 21, 12_89]					)

        snake_case   : Union[str, Any]	  =      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''					)
        # ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
        self.assertListEqual(
            a__	,						['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''']					)
        snake_case   : Union[str, Any]	  =      tokenizer.convert_tokens_to_ids(a__					)
        self.assertListEqual(a__	,						[31, 23, 3_86, 19, 5_18, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]					)

        snake_case   : str	  =      tokenizer.convert_ids_to_tokens(a__					)
        self.assertListEqual(
            a__	,						['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''']	,						)

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        snake_case   : Tuple	  =      ErnieMTokenizer(a__	,						unk_token='''<unk>'''	,						pad_token='''<pad>'''					)

        snake_case   : str	  =      tokenizer.encode('''sequence builders'''					)
        snake_case   : Any	  =      tokenizer.encode('''multi-sequence build'''					)

        snake_case   : Union[str, Any]	  =      tokenizer.build_inputs_with_special_tokens(a__					)
        snake_case   : Dict	  =      tokenizer.build_inputs_with_special_tokens(a__	,						a__					)

        assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
        assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
            tokenizer.sep_token_id
        ] + text_a + [tokenizer.sep_token_id]



       @slow
       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        snake_case   : Tuple	  =      {'''input_ids''': [[0, 1_10_62, 8_27_72, 7, 15, 8_27_72, 5_38, 5_15_29, 2_37, 1_71_98, 12_90, 2_06, 9, 21_51_75, 13_14, 1_36, 1_71_98, 12_90, 2_06, 9, 5_63_59, 42, 12_20_09, 9, 1_64_66, 16, 8_73_44, 45_37, 9, 47_17, 7_83_81, 6, 15_99_58, 7, 15, 2_44_80, 6_18, 4, 5_27, 2_26_93, 9, 3_04, 4, 27_77, 2_44_80, 98_74, 4, 4_35_23, 5_94, 4, 8_03, 1_83_92, 3_31_89, 18, 4, 4_35_23, 2_44_47, 5, 5, 5, 16, 1_00, 2_49_55, 8_36_58, 96_26, 14_40_57, 15, 8_39, 2_23_35, 16, 1_36, 2_49_55, 8_36_58, 8_34_79, 15, 3_91_02, 7_24, 16, 6_78, 6_45, 64_60, 13_28, 45_89, 42, 12_20_09, 11_57_74, 23, 35_59, 13_28, 4_68_76, 7, 1_36, 5_38_94, 19_40, 4_22_27, 4_11_59, 1_77_21, 8_23, 4_25, 4, 2_75_12, 9_87_22, 2_06, 1_36, 55_31, 49_70, 9_19, 1_73_36, 5, 2], [0, 2_00_80, 6_18, 83, 8_27_75, 47, 4_79, 9, 15_17, 73, 5_38_94, 3_33, 8_05_81, 11_01_17, 1_88_11, 52_56, 12_95, 51, 15_25_26, 2_97, 79_86, 3_90, 12_44_16, 5_38, 3_54_31, 2_14, 98, 1_50_44, 2_57_37, 1_36, 71_08, 4_37_01, 23, 7_56, 13_53_55, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_81, 6_37_73, 11_94_55, 6, 14_77_97, 8_82_03, 7, 6_45, 70, 21, 32_85, 1_02_69, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=a__	,						model_name='''susnato/ernie-m-base_pytorch'''	,						sequences=[
                '''Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '''
                '''general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural '''
                '''Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained '''
                '''models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.''',
                '''BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '''
                '''conditioning on both left and right context in all layers.''',
                '''The quick brown fox jumps over the lazy dog.''',
            ]	,						)










import os
import sys

from . import rsa_key_generator as rkg

a				:							str					=			128
a				:							str					=			256






def    _UpperCAmelCase	(   __lowerCamelCase ,     __lowerCamelCase = DEFAULT_BLOCK_SIZE       )		->list[int]:


 '''simple docstring'''
 snake_case   : List[str]	  =      message.encode('''ascii'''       )

 snake_case   : List[Any]	  =      []
 for block_start in range(0 ,     len(__lowerCamelCase       ) ,     __lowerCamelCase       ):
  snake_case   : int	  =      0
  for i in range(__lowerCamelCase ,     min(block_start + block_size ,     len(__lowerCamelCase       )       )       ):
   block_int += message_bytes[i] * (BYTE_SIZE ** (i % block_size))
  block_ints.append(__lowerCamelCase       )
 return block_ints






def    _UpperCAmelCase	(   __lowerCamelCase ,     __lowerCamelCase ,     __lowerCamelCase = DEFAULT_BLOCK_SIZE       )		->str:


 '''simple docstring'''
 snake_case   : list[str]	  =      []
 for block_int in block_ints:
  snake_case   : list[str]	  =      []
  for i in range(block_size - 1 ,     -1 ,     -1       ):
   if len(__lowerCamelCase       ) + i < message_length:
    snake_case   : Any	  =      block_int // (BYTE_SIZE**i)
    snake_case   : Optional[Any]	  =      block_int % (BYTE_SIZE**i)
    block_message.insert(0 ,     chr(__lowerCamelCase       )       )
  message.extend(__lowerCamelCase       )
 return "".join(__lowerCamelCase       )






def    _UpperCAmelCase	(   __lowerCamelCase ,     __lowerCamelCase ,     __lowerCamelCase = DEFAULT_BLOCK_SIZE       )		->list[int]:


 '''simple docstring'''
 snake_case   : str	  =      []
 snake_case  ,					snake_case   : Optional[Any]	  =      key

 for block in get_blocks_from_text(__lowerCamelCase ,     __lowerCamelCase       ):
  encrypted_blocks.append(pow(__lowerCamelCase ,     __lowerCamelCase ,     __lowerCamelCase       )       )
 return encrypted_blocks






def    _UpperCAmelCase	(   __lowerCamelCase ,     __lowerCamelCase ,     __lowerCamelCase ,     __lowerCamelCase = DEFAULT_BLOCK_SIZE ,     )		->str:


 '''simple docstring'''
 snake_case   : List[Any]	  =      []
 snake_case  ,					snake_case   : List[str]	  =      key
 for block in encrypted_blocks:
  decrypted_blocks.append(pow(__lowerCamelCase ,     __lowerCamelCase ,     __lowerCamelCase       )       )
 return get_text_from_blocks(__lowerCamelCase ,     __lowerCamelCase ,     __lowerCamelCase       )






def    _UpperCAmelCase	(   __lowerCamelCase       )		->tuple[int, int, int]:


 '''simple docstring'''
 with open(__lowerCamelCase       ) as fo:
  snake_case   : List[Any]	  =      fo.read()
 snake_case  ,					snake_case  ,					snake_case   : Dict	  =      content.split(''','''       )
 return (int(__lowerCamelCase       ), int(__lowerCamelCase       ), int(__lowerCamelCase       ))






def    _UpperCAmelCase	(   __lowerCamelCase ,     __lowerCamelCase ,     __lowerCamelCase ,     __lowerCamelCase = DEFAULT_BLOCK_SIZE ,     )		->str:


 '''simple docstring'''
 snake_case  ,					snake_case  ,					snake_case   : List[Any]	  =      read_key_file(__lowerCamelCase       )
 if key_size < block_size * 8:
  sys.exit(
      '''ERROR: Block size is {} bits and key size is {} bits. The RSA cipher '''
      '''requires the block size to be equal to or greater than the key size. '''
      '''Either decrease the block size or use different keys.'''.format(
          block_size * 8 ,     __lowerCamelCase       )       )

 snake_case   : Any	  =      [str(__lowerCamelCase       ) for i in encrypt_message(__lowerCamelCase ,     (n, e) ,     __lowerCamelCase       )]

 snake_case   : List[str]	  =      ''','''.join(__lowerCamelCase       )
 snake_case   : str	  =      f'''{len(__lowerCamelCase       )}_{block_size}_{encrypted_content}'''
 with open(__lowerCamelCase ,     '''w'''       ) as fo:
  fo.write(__lowerCamelCase       )
 return encrypted_content






def    _UpperCAmelCase	(   __lowerCamelCase ,     __lowerCamelCase       )		->str:


 '''simple docstring'''
 snake_case  ,					snake_case  ,					snake_case   : str	  =      read_key_file(__lowerCamelCase       )
 with open(__lowerCamelCase       ) as fo:
  snake_case   : Dict	  =      fo.read()
 snake_case  ,					snake_case  ,					snake_case   : Union[str, Any]	  =      content.split('''_'''       )
 snake_case   : str	  =      int(__lowerCamelCase       )
 snake_case   : int	  =      int(__lowerCamelCase       )

 if key_size < block_size * 8:
  sys.exit(
      '''ERROR: Block size is {} bits and key size is {} bits. The RSA cipher '''
      '''requires the block size to be equal to or greater than the key size. '''
      '''Did you specify the correct key file and encrypted file?'''.format(
          block_size * 8 ,     __lowerCamelCase       )       )

 snake_case   : str	  =      []
 for block in encrypted_message.split(''','''       ):
  encrypted_blocks.append(int(__lowerCamelCase       )       )

 return decrypt_message(__lowerCamelCase ,     __lowerCamelCase ,     (n, d) ,     __lowerCamelCase       )






def    _UpperCAmelCase	(   )		->None:


 '''simple docstring'''
 snake_case   : List[Any]	  =      '''encrypted_file.txt'''
 snake_case   : Optional[int]	  =      input(R'''Encrypt\Decrypt [e\d]: '''       )

 if response.lower().startswith('''e'''       ):
  snake_case   : Dict	  =      '''encrypt'''
 elif response.lower().startswith('''d'''       ):
  snake_case   : Dict	  =      '''decrypt'''

 if mode == "encrypt":
  if not os.path.exists('''rsa_pubkey.txt'''       ):
   rkg.make_key_files('''rsa''' ,     1024       )

  snake_case   : List[Any]	  =      input('''\nEnter message: '''       )
  snake_case   : Union[str, Any]	  =      '''rsa_pubkey.txt'''
  print(f'''Encrypting and writing to {filename}...'''       )
  snake_case   : List[str]	  =      encrypt_and_write_to_file(__lowerCamelCase ,     __lowerCamelCase ,     __lowerCamelCase       )

  print('''\nEncrypted text:'''       )
  print(__lowerCamelCase       )

 elif mode == "decrypt":
  snake_case   : str	  =      '''rsa_privkey.txt'''
  print(f'''Reading from {filename} and decrypting...'''       )
  snake_case   : int	  =      read_from_file_and_decrypt(__lowerCamelCase ,     __lowerCamelCase       )
  print('''writing decryption to rsa_decryption.txt...'''       )
  with open('''rsa_decryption.txt''' ,     '''w'''       ) as dec:
   dec.write(__lowerCamelCase       )

  print('''\nDecryption:'''       )
  print(__lowerCamelCase       )


if __name__ == "__main__":
  main()






