
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


UpperCamelCase_	:    Optional[Any]  					=       {
    """auto_factory""": ["""get_values"""],
    """configuration_auto""": ["""ALL_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CONFIG_MAPPING""", """MODEL_NAMES_MAPPING""", """AutoConfig"""],
    """feature_extraction_auto""": ["""FEATURE_EXTRACTOR_MAPPING""", """AutoFeatureExtractor"""],
    """image_processing_auto""": ["""IMAGE_PROCESSOR_MAPPING""", """AutoImageProcessor"""],
    """processing_auto""": ["""PROCESSOR_MAPPING""", """AutoProcessor"""],
    """tokenization_auto""": ["""TOKENIZER_MAPPING""", """AutoTokenizer"""],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCamelCase_	:    Optional[Any]  					=       [
           """MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_AUDIO_XVECTOR_MAPPING""",
           """MODEL_FOR_BACKBONE_MAPPING""",
           """MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING""",
           """MODEL_FOR_CAUSAL_LM_MAPPING""",
           """MODEL_FOR_CTC_MAPPING""",
           """MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING""",
           """MODEL_FOR_DEPTH_ESTIMATION_MAPPING""",
           """MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_IMAGE_SEGMENTATION_MAPPING""",
           """MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING""",
           """MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING""",
           """MODEL_FOR_MASKED_LM_MAPPING""",
           """MODEL_FOR_MASK_GENERATION_MAPPING""",
           """MODEL_FOR_MULTIPLE_CHOICE_MAPPING""",
           """MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING""",
           """MODEL_FOR_OBJECT_DETECTION_MAPPING""",
           """MODEL_FOR_PRETRAINING_MAPPING""",
           """MODEL_FOR_QUESTION_ANSWERING_MAPPING""",
           """MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING""",
           """MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING""",
           """MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING""",
           """MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING""",
           """MODEL_FOR_TEXT_ENCODING_MAPPING""",
           """MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING""",
           """MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_VISION_2_SEQ_MAPPING""",
           """MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING""",
           """MODEL_MAPPING""",
           """MODEL_WITH_LM_HEAD_MAPPING""",
           """MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING""",
           """MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING""",
           """AutoModel""",
           """AutoBackbone""",
           """AutoModelForAudioClassification""",
           """AutoModelForAudioFrameClassification""",
           """AutoModelForAudioXVector""",
           """AutoModelForCausalLM""",
           """AutoModelForCTC""",
           """AutoModelForDepthEstimation""",
           """AutoModelForImageClassification""",
           """AutoModelForImageSegmentation""",
           """AutoModelForInstanceSegmentation""",
           """AutoModelForMaskGeneration""",
           """AutoModelForTextEncoding""",
           """AutoModelForMaskedImageModeling""",
           """AutoModelForMaskedLM""",
           """AutoModelForMultipleChoice""",
           """AutoModelForNextSentencePrediction""",
           """AutoModelForObjectDetection""",
           """AutoModelForPreTraining""",
           """AutoModelForQuestionAnswering""",
           """AutoModelForSemanticSegmentation""",
           """AutoModelForSeq2SeqLM""",
           """AutoModelForSequenceClassification""",
           """AutoModelForSpeechSeq2Seq""",
           """AutoModelForTableQuestionAnswering""",
           """AutoModelForTokenClassification""",
           """AutoModelForUniversalSegmentation""",
           """AutoModelForVideoClassification""",
           """AutoModelForVision2Seq""",
           """AutoModelForVisualQuestionAnswering""",
           """AutoModelForDocumentQuestionAnswering""",
           """AutoModelWithLMHead""",
           """AutoModelForZeroShotImageClassification""",
           """AutoModelForZeroShotObjectDetection""",
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCamelCase_	:    Tuple  					=       [
           """TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING""",
           """TF_MODEL_FOR_CAUSAL_LM_MAPPING""",
           """TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING""",
           """TF_MODEL_FOR_MASK_GENERATION_MAPPING""",
           """TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING""",
           """TF_MODEL_FOR_MASKED_LM_MAPPING""",
           """TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING""",
           """TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING""",
           """TF_MODEL_FOR_PRETRAINING_MAPPING""",
           """TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING""",
           """TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING""",
           """TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING""",
           """TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING""",
           """TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING""",
           """TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING""",
           """TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING""",
           """TF_MODEL_FOR_TEXT_ENCODING_MAPPING""",
           """TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING""",
           """TF_MODEL_FOR_VISION_2_SEQ_MAPPING""",
           """TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING""",
           """TF_MODEL_MAPPING""",
           """TF_MODEL_WITH_LM_HEAD_MAPPING""",
           """TFAutoModel""",
           """TFAutoModelForAudioClassification""",
           """TFAutoModelForCausalLM""",
           """TFAutoModelForImageClassification""",
           """TFAutoModelForMaskedImageModeling""",
           """TFAutoModelForMaskedLM""",
           """TFAutoModelForMaskGeneration""",
           """TFAutoModelForMultipleChoice""",
           """TFAutoModelForNextSentencePrediction""",
           """TFAutoModelForPreTraining""",
           """TFAutoModelForDocumentQuestionAnswering""",
           """TFAutoModelForQuestionAnswering""",
           """TFAutoModelForSemanticSegmentation""",
           """TFAutoModelForSeq2SeqLM""",
           """TFAutoModelForSequenceClassification""",
           """TFAutoModelForSpeechSeq2Seq""",
           """TFAutoModelForTableQuestionAnswering""",
           """TFAutoModelForTextEncoding""",
           """TFAutoModelForTokenClassification""",
           """TFAutoModelForVision2Seq""",
           """TFAutoModelForZeroShotImageClassification""",
           """TFAutoModelWithLMHead""",
       ]

try:
       if not is_flax_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCamelCase_	:    Optional[int]  					=       [
           """FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING""",
           """FLAX_MODEL_FOR_CAUSAL_LM_MAPPING""",
           """FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING""",
           """FLAX_MODEL_FOR_MASKED_LM_MAPPING""",
           """FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING""",
           """FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING""",
           """FLAX_MODEL_FOR_PRETRAINING_MAPPING""",
           """FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING""",
           """FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING""",
           """FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING""",
           """FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING""",
           """FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING""",
           """FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING""",
           """FLAX_MODEL_MAPPING""",
           """FlaxAutoModel""",
           """FlaxAutoModelForCausalLM""",
           """FlaxAutoModelForImageClassification""",
           """FlaxAutoModelForMaskedLM""",
           """FlaxAutoModelForMultipleChoice""",
           """FlaxAutoModelForNextSentencePrediction""",
           """FlaxAutoModelForPreTraining""",
           """FlaxAutoModelForQuestionAnswering""",
           """FlaxAutoModelForSeq2SeqLM""",
           """FlaxAutoModelForSequenceClassification""",
           """FlaxAutoModelForSpeechSeq2Seq""",
           """FlaxAutoModelForTokenClassification""",
           """FlaxAutoModelForVision2Seq""",
       ]


if TYPE_CHECKING:
       from .auto_factory import get_values
       from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, CONFIG_MAPPING, MODEL_NAMES_MAPPING, AutoConfig
       from .feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING, AutoFeatureExtractor
       from .image_processing_auto import IMAGE_PROCESSOR_MAPPING, AutoImageProcessor
       from .processing_auto import PROCESSOR_MAPPING, AutoProcessor
       from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_auto import (
                  MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
                  MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING,
                  MODEL_FOR_AUDIO_XVECTOR_MAPPING,
                  MODEL_FOR_BACKBONE_MAPPING,
                  MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING,
                  MODEL_FOR_CAUSAL_LM_MAPPING,
                  MODEL_FOR_CTC_MAPPING,
                  MODEL_FOR_DEPTH_ESTIMATION_MAPPING,
                  MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
                  MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
                  MODEL_FOR_IMAGE_SEGMENTATION_MAPPING,
                  MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING,
                  MODEL_FOR_MASK_GENERATION_MAPPING,
                  MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
                  MODEL_FOR_MASKED_LM_MAPPING,
                  MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
                  MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
                  MODEL_FOR_OBJECT_DETECTION_MAPPING,
                  MODEL_FOR_PRETRAINING_MAPPING,
                  MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                  MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
                  MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
                  MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                  MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
                  MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
                  MODEL_FOR_TEXT_ENCODING_MAPPING,
                  MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                  MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING,
                  MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
                  MODEL_FOR_VISION_2_SEQ_MAPPING,
                  MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING,
                  MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
                  MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING,
                  MODEL_MAPPING,
                  MODEL_WITH_LM_HEAD_MAPPING,
                  AutoBackbone,
                  AutoModel,
                  AutoModelForAudioClassification,
                  AutoModelForAudioFrameClassification,
                  AutoModelForAudioXVector,
                  AutoModelForCausalLM,
                  AutoModelForCTC,
                  AutoModelForDepthEstimation,
                  AutoModelForDocumentQuestionAnswering,
                  AutoModelForImageClassification,
                  AutoModelForImageSegmentation,
                  AutoModelForInstanceSegmentation,
                  AutoModelForMaskedImageModeling,
                  AutoModelForMaskedLM,
                  AutoModelForMaskGeneration,
                  AutoModelForMultipleChoice,
                  AutoModelForNextSentencePrediction,
                  AutoModelForObjectDetection,
                  AutoModelForPreTraining,
                  AutoModelForQuestionAnswering,
                  AutoModelForSemanticSegmentation,
                  AutoModelForSeqaSeqLM,
                  AutoModelForSequenceClassification,
                  AutoModelForSpeechSeqaSeq,
                  AutoModelForTableQuestionAnswering,
                  AutoModelForTextEncoding,
                  AutoModelForTokenClassification,
                  AutoModelForUniversalSegmentation,
                  AutoModelForVideoClassification,
                  AutoModelForVisionaSeq,
                  AutoModelForVisualQuestionAnswering,
                  AutoModelForZeroShotImageClassification,
                  AutoModelForZeroShotObjectDetection,
                  AutoModelWithLMHead,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_auto import (
                  TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
                  TF_MODEL_FOR_CAUSAL_LM_MAPPING,
                  TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
                  TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
                  TF_MODEL_FOR_MASK_GENERATION_MAPPING,
                  TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
                  TF_MODEL_FOR_MASKED_LM_MAPPING,
                  TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
                  TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
                  TF_MODEL_FOR_PRETRAINING_MAPPING,
                  TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                  TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
                  TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
                  TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                  TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
                  TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
                  TF_MODEL_FOR_TEXT_ENCODING_MAPPING,
                  TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                  TF_MODEL_FOR_VISION_2_SEQ_MAPPING,
                  TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
                  TF_MODEL_MAPPING,
                  TF_MODEL_WITH_LM_HEAD_MAPPING,
                  TFAutoModel,
                  TFAutoModelForAudioClassification,
                  TFAutoModelForCausalLM,
                  TFAutoModelForDocumentQuestionAnswering,
                  TFAutoModelForImageClassification,
                  TFAutoModelForMaskedImageModeling,
                  TFAutoModelForMaskedLM,
                  TFAutoModelForMaskGeneration,
                  TFAutoModelForMultipleChoice,
                  TFAutoModelForNextSentencePrediction,
                  TFAutoModelForPreTraining,
                  TFAutoModelForQuestionAnswering,
                  TFAutoModelForSemanticSegmentation,
                  TFAutoModelForSeqaSeqLM,
                  TFAutoModelForSequenceClassification,
                  TFAutoModelForSpeechSeqaSeq,
                  TFAutoModelForTableQuestionAnswering,
                  TFAutoModelForTextEncoding,
                  TFAutoModelForTokenClassification,
                  TFAutoModelForVisionaSeq,
                  TFAutoModelForZeroShotImageClassification,
                  TFAutoModelWithLMHead,
              )

       try:
              if not is_flax_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_flax_auto import (
                  FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
                  FLAX_MODEL_FOR_CAUSAL_LM_MAPPING,
                  FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
                  FLAX_MODEL_FOR_MASKED_LM_MAPPING,
                  FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
                  FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
                  FLAX_MODEL_FOR_PRETRAINING_MAPPING,
                  FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                  FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
                  FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                  FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
                  FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                  FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING,
                  FLAX_MODEL_MAPPING,
                  FlaxAutoModel,
                  FlaxAutoModelForCausalLM,
                  FlaxAutoModelForImageClassification,
                  FlaxAutoModelForMaskedLM,
                  FlaxAutoModelForMultipleChoice,
                  FlaxAutoModelForNextSentencePrediction,
                  FlaxAutoModelForPreTraining,
                  FlaxAutoModelForQuestionAnswering,
                  FlaxAutoModelForSeqaSeqLM,
                  FlaxAutoModelForSequenceClassification,
                  FlaxAutoModelForSpeechSeqaSeq,
                  FlaxAutoModelForTokenClassification,
                  FlaxAutoModelForVisionaSeq,
              )

else:
       import sys

       UpperCamelCase_	:    Optional[int]  					=       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






import os
import unittest

from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer

from ...test_tokenization_common import TokenizerTesterMixin




class      __lowercase	(	__snake_case      ,   unittest.TestCase					):
	_A   			=      BertweetTokenizer
	_A   			=      False






	def 			_a(self							:   Tuple						)   -> Any:
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							_lowercase					:  Tuple    			=      ["I", "m", "V@@", "R@@", "r", "e@@"]
							_lowercase					:  Tuple    			=      dict(zip(snake_case			,  range(len(snake_case						)						)						)						)
							_lowercase					:  Tuple    			=      ["#version: 0.2", "a m</w>"]
							_lowercase					:  Optional[Any]    			=      {"unk_token": "<unk>"}

							_lowercase					:  int    			=      os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["vocab_file"]						)
							_lowercase					:  Optional[int]    			=      os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["merges_file"]						)
							with open(self.vocab_file			,  "w"			,  encoding="utf-8"						) as fp:
													for token in vocab_tokens:
																			fp.write(F"""{token} {vocab_tokens[token]}\n"""						)
							with open(self.merges_file			,  "w"			,  encoding="utf-8"						) as fp:
													fp.write("\n".join(snake_case						)						)






	def 			_a(self							:   List[Any]			,  **snake_case							:   Tuple						)   -> Union[str, Any]:
							kwargs.update(self.special_tokens_map						)
							return BertweetTokenizer.from_pretrained(self.tmpdirname			,  **snake_case						)






	def 			_a(self							:   Any			,  snake_case							:   Any						)   -> List[Any]:
							_lowercase					:  Tuple    			=      "I am VinAI Research"
							_lowercase					:  Optional[int]    			=      "I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"
							return input_text, output_text






	def 			_a(self							:   str						)   -> Optional[int]:
							_lowercase					:  str    			=      BertweetTokenizer(self.vocab_file			,  self.merges_file			,  **self.special_tokens_map						)
							_lowercase					:  Optional[Any]    			=      "I am VinAI Research"
							_lowercase					:  str    			=      "I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h".split()
							_lowercase					:  Tuple    			=      tokenizer.tokenize(snake_case						)
							self.assertListEqual(snake_case			,  snake_case						)

							_lowercase					:  str    			=      tokens + [tokenizer.unk_token]

							_lowercase					:  str    			=      [4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case						)			,  snake_case						)




