





'''simple docstring'''


import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MODEL_FOR_PRETRAINING_MAPPING,
				    FunnelBaseModel,
				    FunnelForMaskedLM,
				    FunnelForMultipleChoice,
				    FunnelForPreTraining,
				    FunnelForQuestionAnswering,
				    FunnelForSequenceClassification,
				    FunnelForTokenClassification,
				    FunnelModel,
				)



class 							__UpperCAmelCase  :







		'''simple docstring'''







		def __init__(self						:      Tuple      ,      _lowerCAmelCase						:      Optional[int]      ,      _lowerCAmelCase						:      List[str]=13      ,      _lowerCAmelCase						:      int=7      ,      _lowerCAmelCase						:      Union[str, Any]=True      ,      _lowerCAmelCase						:      Union[str, Any]=True      ,      _lowerCAmelCase						:      Dict=True      ,      _lowerCAmelCase						:      Optional[Any]=True      ,      _lowerCAmelCase						:      List[str]=99      ,      _lowerCAmelCase						:      Dict=[1, 1, 2]      ,      _lowerCAmelCase						:      Optional[int]=1      ,      _lowerCAmelCase						:      List[str]=32      ,      _lowerCAmelCase						:      Optional[int]=4      ,      _lowerCAmelCase						:      Dict=8      ,      _lowerCAmelCase						:      Tuple=37      ,      _lowerCAmelCase						:      Union[str, Any]="gelu_new"      ,      _lowerCAmelCase						:      int=0.1      ,      _lowerCAmelCase						:      List[str]=0.1      ,      _lowerCAmelCase						:      Dict=0.0      ,      _lowerCAmelCase						:      str=512      ,      _lowerCAmelCase						:      int=3      ,      _lowerCAmelCase						:      Optional[int]=0.02      ,      _lowerCAmelCase						:      Dict=3      ,      _lowerCAmelCase						:      Union[str, Any]=4      ,      _lowerCAmelCase						:      Optional[int]=None      ,      _lowerCAmelCase						:      List[str]=False      ,      ):
			A							  =      parent
			A							  =      batch_size
			A							  =      seq_length
			A							  =      is_training
			A							  =      use_input_mask
			A							  =      use_token_type_ids
			A							  =      use_labels
			A							  =      vocab_size
			A							  =      block_sizes
			A							  =      num_decoder_layers
			A							  =      d_model
			A							  =      n_head
			A							  =      d_head
			A							  =      d_inner
			A							  =      hidden_act
			A							  =      hidden_dropout
			A							  =      attention_dropout
			A							  =      activation_dropout
			A							  =      max_position_embeddings
			A							  =      type_vocab_size
			A							  =      2
			A							  =      num_labels
			A							  =      num_choices
			A							  =      scope
			A							  =      initializer_std

			# Used in the tests to check the size of the first attention layer
			A							  =      n_head
			# Used in the tests to check the size of the first hidden state
			A							  =      self.d_model
			# Used in the tests to check the number of output hidden states/attentions
			A							  =      sum(self.block_sizes      ) + (0 if base else self.num_decoder_layers)
			# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
			# the last hidden state of the first block (which is the first hidden state of the decoder).
			if not base:
				A							  =      self.num_hidden_layers + 2







		def        A     (self						:      List[str]      ):
			A							  =      ids_tensor([self.batch_size, self.seq_length]      ,      self.vocab_size      )

			A							  =      None
			if self.use_input_mask:
				A							  =      ids_tensor([self.batch_size, self.seq_length]      ,      vocab_size=2      )

			A							  =      None
			if self.use_token_type_ids:
				A							  =      ids_tensor([self.batch_size, self.seq_length]      ,      self.type_vocab_size      )

			A							  =      None
			A							  =      None
			A							  =      None
			if self.use_labels:
				A							  =      ids_tensor([self.batch_size]      ,      self.type_sequence_label_size      )
				A							  =      ids_tensor([self.batch_size, self.seq_length]      ,      self.num_labels      )
				A							  =      ids_tensor([self.batch_size]      ,      self.num_choices      )
				A							  =      ids_tensor([self.batch_size, self.seq_length]      ,      1      )

			A							  =      self.get_config()

			return (
			    config,
			    input_ids,
			    token_type_ids,
			    input_mask,
			    sequence_labels,
			    token_labels,
			    choice_labels,
			    fake_token_labels,
			)







		def        A     (self						:      List[Any]      ):
			return FunnelConfig(
			    vocab_size=self.vocab_size      ,      block_sizes=self.block_sizes      ,      num_decoder_layers=self.num_decoder_layers      ,      d_model=self.d_model      ,      n_head=self.n_head      ,      d_head=self.d_head      ,      d_inner=self.d_inner      ,      hidden_act=self.hidden_act      ,      hidden_dropout=self.hidden_dropout      ,      attention_dropout=self.attention_dropout      ,      activation_dropout=self.activation_dropout      ,      max_position_embeddings=self.max_position_embeddings      ,      type_vocab_size=self.type_vocab_size      ,      initializer_std=self.initializer_std      ,      )







		def        A     (self						:      Dict      ,      _lowerCAmelCase						:      Optional[Any]      ,      _lowerCAmelCase						:      Optional[Any]      ,      _lowerCAmelCase						:      Union[str, Any]      ,      _lowerCAmelCase						:      List[Any]      ,      _lowerCAmelCase						:      Union[str, Any]      ,      _lowerCAmelCase						:      List[Any]      ,      _lowerCAmelCase						:      Tuple      ,      _lowerCAmelCase						:      str      ,      ):
			A							  =      FunnelModel(config=_lowerCAmelCase      )
			model.to(_lowerCAmelCase      )
			model.eval()
			A							  =      model(_lowerCAmelCase      ,      attention_mask=_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      )
			A							  =      model(_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      )
			A							  =      model(_lowerCAmelCase      )
			self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, self.seq_length, self.d_model)      )

			A							  =      False
			A							  =      model(_lowerCAmelCase      )
			self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, self.seq_length, self.d_model)      )

			A							  =      False
			A							  =      model(_lowerCAmelCase      )
			self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, self.seq_length, self.d_model)      )







		def        A     (self						:      Optional[Any]      ,      _lowerCAmelCase						:      List[Any]      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      Union[str, Any]      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      Any      ,      _lowerCAmelCase						:      Tuple      ,      _lowerCAmelCase						:      Optional[int]      ,      ):
			A							  =      FunnelBaseModel(config=_lowerCAmelCase      )
			model.to(_lowerCAmelCase      )
			model.eval()
			A							  =      model(_lowerCAmelCase      ,      attention_mask=_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      )
			A							  =      model(_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      )
			A							  =      model(_lowerCAmelCase      )
			self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, 2, self.d_model)      )

			A							  =      False
			A							  =      model(_lowerCAmelCase      )
			self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, 3, self.d_model)      )

			A							  =      False
			A							  =      model(_lowerCAmelCase      )
			self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, 2, self.d_model)      )







		def        A     (self						:      Optional[Any]      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      Union[str, Any]      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      Dict      ,      _lowerCAmelCase						:      Dict      ,      _lowerCAmelCase						:      Union[str, Any]      ,      _lowerCAmelCase						:      int      ,      ):
			A							  =      self.num_labels
			A							  =      FunnelForPreTraining(config=_lowerCAmelCase      )
			model.to(_lowerCAmelCase      )
			model.eval()
			A							  =      model(_lowerCAmelCase      ,      attention_mask=_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      ,      labels=_lowerCAmelCase      )
			self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length)      )







		def        A     (self						:      Tuple      ,      _lowerCAmelCase						:      Any      ,      _lowerCAmelCase						:      Optional[Any]      ,      _lowerCAmelCase						:      Union[str, Any]      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      Optional[int]      ,      _lowerCAmelCase						:      List[Any]      ,      _lowerCAmelCase						:      Any      ,      _lowerCAmelCase						:      Any      ,      ):
			A							  =      FunnelForMaskedLM(config=_lowerCAmelCase      )
			model.to(_lowerCAmelCase      )
			model.eval()
			A							  =      model(_lowerCAmelCase      ,      attention_mask=_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      ,      labels=_lowerCAmelCase      )
			self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length, self.vocab_size)      )







		def        A     (self						:      str      ,      _lowerCAmelCase						:      Any      ,      _lowerCAmelCase						:      Dict      ,      _lowerCAmelCase						:      Union[str, Any]      ,      _lowerCAmelCase						:      Tuple      ,      _lowerCAmelCase						:      List[str]      ,      _lowerCAmelCase						:      List[str]      ,      _lowerCAmelCase						:      List[Any]      ,      _lowerCAmelCase						:      List[Any]      ,      ):
			A							  =      self.num_labels
			A							  =      FunnelForSequenceClassification(_lowerCAmelCase      )
			model.to(_lowerCAmelCase      )
			model.eval()
			A							  =      model(_lowerCAmelCase      ,      attention_mask=_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      ,      labels=_lowerCAmelCase      )
			self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.num_labels)      )







		def        A     (self						:      Dict      ,      _lowerCAmelCase						:      Union[str, Any]      ,      _lowerCAmelCase						:      Any      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      List[Any]      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      Optional[int]      ,      _lowerCAmelCase						:      Any      ,      _lowerCAmelCase						:      List[str]      ,      ):
			A							  =      self.num_choices
			A							  =      FunnelForMultipleChoice(config=_lowerCAmelCase      )
			model.to(_lowerCAmelCase      )
			model.eval()
			A							  =      input_ids.unsqueeze(1      ).expand(-1      ,      self.num_choices      ,      -1      ).contiguous()
			A							  =      token_type_ids.unsqueeze(1      ).expand(-1      ,      self.num_choices      ,      -1      ).contiguous()
			A							  =      input_mask.unsqueeze(1      ).expand(-1      ,      self.num_choices      ,      -1      ).contiguous()
			A							  =      model(
			    _lowerCAmelCase      ,      attention_mask=_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      ,      labels=_lowerCAmelCase      ,      )
			self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.num_choices)      )







		def        A     (self						:      Any      ,      _lowerCAmelCase						:      Optional[int]      ,      _lowerCAmelCase						:      Union[str, Any]      ,      _lowerCAmelCase						:      Union[str, Any]      ,      _lowerCAmelCase						:      Union[str, Any]      ,      _lowerCAmelCase						:      Optional[int]      ,      _lowerCAmelCase						:      Tuple      ,      _lowerCAmelCase						:      List[str]      ,      _lowerCAmelCase						:      Union[str, Any]      ,      ):
			A							  =      self.num_labels
			A							  =      FunnelForTokenClassification(config=_lowerCAmelCase      )
			model.to(_lowerCAmelCase      )
			model.eval()
			A							  =      model(_lowerCAmelCase      ,      attention_mask=_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      ,      labels=_lowerCAmelCase      )
			self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length, self.num_labels)      )







		def        A     (self						:      str      ,      _lowerCAmelCase						:      Optional[int]      ,      _lowerCAmelCase						:      List[str]      ,      _lowerCAmelCase						:      Any      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      Optional[Any]      ,      _lowerCAmelCase						:      List[Any]      ,      _lowerCAmelCase						:      List[Any]      ,      _lowerCAmelCase						:      Dict      ,      ):
			A							  =      FunnelForQuestionAnswering(config=_lowerCAmelCase      )
			model.to(_lowerCAmelCase      )
			model.eval()
			A							  =      model(
			    _lowerCAmelCase      ,      attention_mask=_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      ,      start_positions=_lowerCAmelCase      ,      end_positions=_lowerCAmelCase      ,      )
			self.parent.assertEqual(result.start_logits.shape      ,      (self.batch_size, self.seq_length)      )
			self.parent.assertEqual(result.end_logits.shape      ,      (self.batch_size, self.seq_length)      )







		def        A     (self						:      Dict      ):
			A							  =      self.prepare_config_and_inputs()
			(
			    (
			    A
			)  ,    (
			    A
			)  ,    (
			    A
			)  ,    (
			    A
			)  ,    (
			    A
			)  ,    (
			    A
			)  ,    (
			    A
			)  ,    (
			    A
			)  ,    
			)							  =      config_and_inputs
			A							  =      {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
			return config, inputs_dict






@require_torch
class 							__UpperCAmelCase  (       A__		,					A__		,					unittest.TestCase						):







		'''simple docstring'''







		__lowerCAmelCase 				=    False
		__lowerCAmelCase 				=    False
		__lowerCAmelCase 				=    (
		    (
		        FunnelModel,
		        FunnelForMaskedLM,
		        FunnelForPreTraining,
		        FunnelForQuestionAnswering,
		        FunnelForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		__lowerCAmelCase 				=    (
		    {
		        '''feature-extraction''': (FunnelBaseModel, FunnelModel),
		        '''fill-mask''': FunnelForMaskedLM,
		        '''question-answering''': FunnelForQuestionAnswering,
		        '''text-classification''': FunnelForSequenceClassification,
		        '''token-classification''': FunnelForTokenClassification,
		        '''zero-shot''': FunnelForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)







		def        A     (self						:      Tuple      ,      _lowerCAmelCase						:      Optional[int]      ,      _lowerCAmelCase						:      List[Any]      ,      _lowerCAmelCase						:      str=False      ):
			A							  =      super()._prepare_for_class(_lowerCAmelCase      ,      _lowerCAmelCase      ,      return_labels=_lowerCAmelCase      )

			if return_labels:
				if model_class in get_values(_lowerCAmelCase      ):
					A							  =      torch.zeros(
					    (self.model_tester.batch_size, self.model_tester.seq_length)      ,      dtype=torch.long      ,      device=_lowerCAmelCase      )
			return inputs_dict







		def        A     (self						:      int      ):
			A							  =      FunnelModelTester(self      )
			A							  =      ConfigTester(self      ,      config_class=_lowerCAmelCase      )







		def        A     (self						:      Tuple      ):
			self.config_tester.run_common_tests()







		def        A     (self						:      Optional[Any]      ):
			A							  =      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*_lowerCAmelCase      )







		def        A     (self						:      Tuple      ):
			A							  =      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_pretraining(*_lowerCAmelCase      )







		def        A     (self						:      Optional[Any]      ):
			A							  =      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_masked_lm(*_lowerCAmelCase      )







		def        A     (self						:      Union[str, Any]      ):
			A							  =      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_token_classification(*_lowerCAmelCase      )







		def        A     (self						:      List[str]      ):
			A							  =      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_question_answering(*_lowerCAmelCase      )







		def        A     (self						:      Dict      ,      _lowerCAmelCase						:      List[str]      ):
			if hasattr(_lowerCAmelCase      ,      """weight"""      ) and module.weight is not None:
				module.weight.data.fill_(3      )
			if hasattr(_lowerCAmelCase      ,      """bias"""      ) and module.bias is not None:
				module.bias.data.fill_(3      )

			for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
				if hasattr(_lowerCAmelCase      ,      _lowerCAmelCase      ) and getattr(_lowerCAmelCase      ,      _lowerCAmelCase      ) is not None:
					A							  =      getattr(_lowerCAmelCase      ,      _lowerCAmelCase      )
					weight.data.fill_(3      )






@require_torch
class 							__UpperCAmelCase  (       A__		,					unittest.TestCase						):







		'''simple docstring'''







		__lowerCAmelCase 				=    False
		__lowerCAmelCase 				=    False
		__lowerCAmelCase 				=    (
		    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
		)







		def        A     (self						:      str      ):
			A							  =      FunnelModelTester(self      ,      base=_lowerCAmelCase      )
			A							  =      ConfigTester(self      ,      config_class=_lowerCAmelCase      )







		def        A     (self						:      List[str]      ):
			self.config_tester.run_common_tests()







		def        A     (self						:      Tuple      ):
			A							  =      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_base_model(*_lowerCAmelCase      )







		def        A     (self						:      List[str]      ):
			A							  =      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_sequence_classification(*_lowerCAmelCase      )







		def        A     (self						:      Union[str, Any]      ):
			A							  =      self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_multiple_choice(*_lowerCAmelCase      )







		def        A     (self						:      int      ):
			A  ,    A							  =      self.model_tester.prepare_config_and_inputs_for_common()
			A							  =      True

			for model_class in self.all_model_classes:
				if model_class.__name__ == "FunnelBaseModel":
					continue
				A							  =      model_class(_lowerCAmelCase      )
				model.to(_lowerCAmelCase      )
				model.train()
				A							  =      self._prepare_for_class(_lowerCAmelCase      ,      _lowerCAmelCase      ,      return_labels=_lowerCAmelCase      )
				A							  =      model(**_lowerCAmelCase      ).loss
				loss.backward()







		def        A     (self						:      Optional[int]      ,      _lowerCAmelCase						:      Union[str, Any]      ):
			if hasattr(_lowerCAmelCase      ,      """weight"""      ) and module.weight is not None:
				module.weight.data.fill_(3      )
			if hasattr(_lowerCAmelCase      ,      """bias"""      ) and module.bias is not None:
				module.bias.data.fill_(3      )

			for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
				if hasattr(_lowerCAmelCase      ,      _lowerCAmelCase      ) and getattr(_lowerCAmelCase      ,      _lowerCAmelCase      ) is not None:
					A							  =      getattr(_lowerCAmelCase      ,      _lowerCAmelCase      )
					weight.data.fill_(3      )






@require_torch
@require_sentencepiece
@require_tokenizers
class 							__UpperCAmelCase  (       unittest.TestCase						):







		'''simple docstring'''







		def        A     (self						:      Optional[Any]      ):
			A							  =      13
			A							  =      7
			A							  =      torch.arange(0      ,      batch_size * sequence_length      ).long().reshape(_lowerCAmelCase      ,      _lowerCAmelCase      )
			A							  =      [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
			A							  =      torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]      )

			A							  =      FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""      )
			A							  =      model(_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      )[0].abs()

			A							  =      torch.tensor(2_344.8_352      )
			A							  =      torch.tensor(0.8_052      )
			self.assertTrue(torch.allclose(output.sum()      ,      _lowerCAmelCase      ,      atol=1e-4      )      )
			self.assertTrue(torch.allclose(output.mean()      ,      _lowerCAmelCase      ,      atol=1e-4      )      )

			A							  =      torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]      )
			A							  =      model(_lowerCAmelCase      ,      attention_mask=_lowerCAmelCase      ,      token_type_ids=_lowerCAmelCase      )[0].abs()

			A							  =      torch.tensor(2_343.8_425      )
			A							  =      torch.tensor(0.8_049      )
			self.assertTrue(torch.allclose(output.sum()      ,      _lowerCAmelCase      ,      atol=1e-4      )      )
			self.assertTrue(torch.allclose(output.mean()      ,      _lowerCAmelCase      ,      atol=1e-4      )      )







		@slow
		def        A     (self						:      Optional[int]      ):
			A							  =      FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""      )
			A							  =      FunnelModel.from_pretrained("""huggingface/funnel-small"""      )
			A							  =      tokenizer("""Hello! I am the Funnel Transformer model."""      ,      return_tensors="""pt"""      )
			A							  =      model(**_lowerCAmelCase      )[0]

			A							  =      torch.tensor(235.7_246      )
			A							  =      torch.tensor(0.0_256      )
			self.assertTrue(torch.allclose(output.sum()      ,      _lowerCAmelCase      ,      atol=1e-4      )      )
			self.assertTrue(torch.allclose(output.mean()      ,      _lowerCAmelCase      ,      atol=1e-4      )      )








'''simple docstring'''


from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase				:	int				    =       logging.get_logger(__name__)



class 							__UpperCAmelCase  (       A__						):







		'''simple docstring'''







		__lowerCAmelCase 				=    ['''input_ids''', '''attention_mask''']







		def __init__(self						:      Tuple      ,      _lowerCAmelCase						:      str="[PAD]"      ,      _lowerCAmelCase						:      Dict="[BOS]"      ,      _lowerCAmelCase						:      Any="[EOS]"      ,      _lowerCAmelCase						:      List[Any]="[MASK]"      ,      _lowerCAmelCase						:      int="[CLS]"      ,      _lowerCAmelCase						:      Optional[Any]="[SEP]"      ,      _lowerCAmelCase						:      List[str]=2048      ,      **_lowerCAmelCase						:      Dict      ,      ):
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else pad_token
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else bos_token
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else eos_token
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else mask_token
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else cls_token
			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else sep_token

			super().__init__(
			    pad_token=_lowerCAmelCase      ,      bos_token=_lowerCAmelCase      ,      eos_token=_lowerCAmelCase      ,      mask_token=_lowerCAmelCase      ,      cls_token=_lowerCAmelCase      ,      sep_token=_lowerCAmelCase      ,      model_max_length=_lowerCAmelCase      ,      **_lowerCAmelCase      ,      )

			A							  =      2**8  # utf is 8 bits

			# define special tokens dict
			A							  =      {
			    self.pad_token: 0,
			    self.bos_token: 1,
			    self.eos_token: 2,
			    self.mask_token: 3,
			    self.cls_token: 4,
			    self.sep_token: 5,
			}
			A							  =      len(self.special_tokens_encoder      )
			A							  =      {v: k for k, v in self.special_tokens_encoder.items()}







		def        A     (self						:      Tuple      ):
			A							  =      self.special_tokens_encoder.copy()
			vocab.update(self.added_tokens_encoder      )
			for i in range(self._utf_vocab_size      ):
				A							  =      chr(_lowerCAmelCase      )
				A							  =      i + len(self.special_tokens_encoder      )
			return vocab







		@property
		def        A     (self						:      Tuple      ):
			return self._utf_vocab_size + self._num_special_tokens







		def        A     (self						:      Union[str, Any]      ,      _lowerCAmelCase						:      List[int]      ,      _lowerCAmelCase						:      Optional[List[int]] = None      ,      _lowerCAmelCase						:      bool = False      ):
			if already_has_special_tokens:
				return super().get_special_tokens_mask(
				    token_ids_a=_lowerCAmelCase      ,      token_ids_a=_lowerCAmelCase      ,      already_has_special_tokens=_lowerCAmelCase      )

			# normal case: some special tokens
			if token_ids_a is None:
				return [1] + [0] * len(_lowerCAmelCase      ) + [1]
			return [1] + ([0] * len(_lowerCAmelCase      )) + [1] + ([0] * len(_lowerCAmelCase      )) + [1]







		def        A     (self						:      List[Any]      ,      _lowerCAmelCase						:      List[int]      ,      _lowerCAmelCase						:      Optional[List[int]] = None      ):
			if token_ids_a is None:
				return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
			else:
				return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]







		def        A     (self						:      str      ,      _lowerCAmelCase						:      str      ):
			A							  =      [chr(_lowerCAmelCase      ) for i in text.encode("""utf-8"""      )]
			return tokens







		def        A     (self						:      int      ,      _lowerCAmelCase						:      List[str]      ):
			if token in self.special_tokens_encoder:
				A							  =      self.special_tokens_encoder[token]
			elif token in self.added_tokens_encoder:
				A							  =      self.added_tokens_encoder[token]
			elif len(_lowerCAmelCase      ) != 1:
				A							  =      self.unk_token_id
			else:
				A							  =      ord(_lowerCAmelCase      ) + self._num_special_tokens
			return token_id







		def        A     (self						:      Optional[int]      ,      _lowerCAmelCase						:      Any      ):
			if index in self.special_tokens_decoder:
				A							  =      self.special_tokens_decoder[index]
			elif index in self.added_tokens_decoder:
				A							  =      self.added_tokens_decoder[index]
			else:
				A							  =      chr(index - self._num_special_tokens      )
			return token







		def        A     (self						:      Dict      ,      _lowerCAmelCase						:      List[str]      ):
			A							  =      b""""""
			for token in tokens:
				if token in self.special_tokens_decoder:
					A							  =      self.special_tokens_decoder[token].encode("""utf-8"""      )
				elif token in self.added_tokens_decoder:
					A							  =      self.special_tokens_decoder[token].encode("""utf-8"""      )
				elif token in self.special_tokens_encoder:
					A							  =      token.encode("""utf-8"""      )
				elif token in self.added_tokens_encoder:
					A							  =      token.encode("""utf-8"""      )
				else:
					A							  =      bytes([ord(_lowerCAmelCase      )]      )
				bstring += tok_string
			A							  =      bstring.decode("""utf-8"""      ,      errors="""replace"""      )
			return string







		def        A     (self						:      Tuple      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      Optional[str] = None      ):
			return ()

