

import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__        =			logging.get_logger(__name__)

A__        =			{}







class    UpperCamelCase_			(  _snake_case			):


  """simple docstring"""
  SCREAMING_SNAKE_CASE__				: str										=      """fsmt_decoder"""



  def __init__(	self    :				Union[str, Any]     ,  A_    :				Optional[int]=0     ,  A_    :				Dict=0	)					-> int:
      super().__init__()
      SCREAMING_SNAKE_CASE__					     =						vocab_size
      SCREAMING_SNAKE_CASE__					     =						bos_token_id







class    UpperCamelCase_			(  _snake_case			):


  """simple docstring"""
  SCREAMING_SNAKE_CASE__				: str										=      """fsmt"""
  SCREAMING_SNAKE_CASE__				: Optional[Any]										=      {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}



  def __init__(	self    :				Dict     ,  A_    :				List[str]=["en", "de"]     ,  A_    :				Optional[Any]=4_2024     ,  A_    :				Any=4_2024     ,  A_    :				Tuple="relu"     ,  A_    :				Tuple=1024     ,  A_    :				str=200     ,  A_    :				str=1024     ,  A_    :				Dict=4096     ,  A_    :				Optional[int]=12     ,  A_    :				Any=16     ,  A_    :				Optional[Any]=0.0     ,  A_    :				List[Any]=4096     ,  A_    :				Union[str, Any]=12     ,  A_    :				int=16     ,  A_    :				Any=0.0     ,  A_    :				Dict=0.0     ,  A_    :				Union[str, Any]=0.1     ,  A_    :				List[Any]=0.0     ,  A_    :				Any=0.02     ,  A_    :				Optional[int]=2     ,  A_    :				Union[str, Any]=True     ,  A_    :				str=True     ,  A_    :				Dict=False     ,  A_    :				Any=5     ,  A_    :				Dict=1.0     ,  A_    :				str=False     ,  A_    :				List[Any]=True     ,  A_    :				int=1     ,  A_    :				List[str]=0     ,  A_    :				List[Any]=2     ,  A_    :				Tuple=2     ,  **A_    :				Tuple     ,  )					-> str:
      SCREAMING_SNAKE_CASE__					     =						langs
      SCREAMING_SNAKE_CASE__					     =						src_vocab_size
      SCREAMING_SNAKE_CASE__					     =						tgt_vocab_size
      SCREAMING_SNAKE_CASE__					     =						d_model  # encoder_embed_dim and decoder_embed_dim

      SCREAMING_SNAKE_CASE__					     =						encoder_ffn_dim
      SCREAMING_SNAKE_CASE__					     =						SCREAMING_SNAKE_CASE__					     =						encoder_layers
      SCREAMING_SNAKE_CASE__					     =						encoder_attention_heads
      SCREAMING_SNAKE_CASE__					     =						encoder_layerdrop
      SCREAMING_SNAKE_CASE__					     =						decoder_layerdrop
      SCREAMING_SNAKE_CASE__					     =						decoder_ffn_dim
      SCREAMING_SNAKE_CASE__					     =						decoder_layers
      SCREAMING_SNAKE_CASE__					     =						decoder_attention_heads
      SCREAMING_SNAKE_CASE__					     =						max_position_embeddings
      SCREAMING_SNAKE_CASE__					     =						init_std  # Normal(0, this parameter)
      SCREAMING_SNAKE_CASE__					     =						activation_function

      SCREAMING_SNAKE_CASE__					     =						DecoderConfig(vocab_size=A_     ,  bos_token_id=A_	)
      if "decoder" in common_kwargs:
          del common_kwargs["decoder"]

      SCREAMING_SNAKE_CASE__					     =						scale_embedding  # scale factor will be sqrt(d_model) if True

      # 3 Types of Dropout
      SCREAMING_SNAKE_CASE__					     =						attention_dropout
      SCREAMING_SNAKE_CASE__					     =						activation_dropout
      SCREAMING_SNAKE_CASE__					     =						dropout

      SCREAMING_SNAKE_CASE__					     =						use_cache
      super().__init__(
          pad_token_id=A_     ,  bos_token_id=A_     ,  eos_token_id=A_     ,  decoder_start_token_id=A_     ,  is_encoder_decoder=A_     ,  tie_word_embeddings=A_     ,  forced_eos_token_id=A_     ,  max_length=A_     ,  num_beams=A_     ,  length_penalty=A_     ,  early_stopping=A_     ,  **A_     ,  )







  def      _UpperCamelCase    (	self    :				Optional[int]	)					-> str:
      SCREAMING_SNAKE_CASE__					     =						copy.deepcopy(self.__dict__	)
      SCREAMING_SNAKE_CASE__					     =						self.decoder.to_dict()
      SCREAMING_SNAKE_CASE__					     =						self.__class__.model_type
      return output





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


A__        =			logging.get_logger(__name__)

A__        =			{"vocab_file": "vocab.txt"}

A__        =			{
    "vocab_file": {
        "tau/splinter-base": "https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt",
        "tau/splinter-base-qass": "https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt",
        "tau/splinter-large": "https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt",
        "tau/splinter-large-qass": "https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt",
    }
}

A__        =			{
    "tau/splinter-base": 512,
    "tau/splinter-base-qass": 512,
    "tau/splinter-large": 512,
    "tau/splinter-large-qass": 512,
}

A__        =			{
    "tau/splinter-base": {"do_lower_case": False},
    "tau/splinter-base-qass": {"do_lower_case": False},
    "tau/splinter-large": {"do_lower_case": False},
    "tau/splinter-large-qass": {"do_lower_case": False},
}







class    UpperCamelCase_			(  _snake_case			):


  """simple docstring"""
  SCREAMING_SNAKE_CASE__				: Dict										=      VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE__				: Optional[int]										=      PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE__				: List[Any]										=      PRETRAINED_INIT_CONFIGURATION
  SCREAMING_SNAKE_CASE__				: str										=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE__				: Optional[Any]										=      SplinterTokenizer



  def __init__(	self    :				Tuple     ,  A_    :				Optional[Any]=None     ,  A_    :				Optional[int]=None     ,  A_    :				Optional[Any]=True     ,  A_    :				str="[UNK]"     ,  A_    :				Optional[Any]="[SEP]"     ,  A_    :				str="[PAD]"     ,  A_    :				List[Any]="[CLS]"     ,  A_    :				Union[str, Any]="[MASK]"     ,  A_    :				Any="[QUESTION]"     ,  A_    :				List[Any]=True     ,  A_    :				Union[str, Any]=None     ,  **A_    :				Optional[Any]     ,  )					-> Optional[Any]:
      super().__init__(
          A_     ,  tokenizer_file=A_     ,  do_lower_case=A_     ,  unk_token=A_     ,  sep_token=A_     ,  pad_token=A_     ,  cls_token=A_     ,  mask_token=A_     ,  tokenize_chinese_chars=A_     ,  strip_accents=A_     ,  additional_special_tokens=(question_token,)     ,  **A_     ,  )

      SCREAMING_SNAKE_CASE__					     =						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
      if (
          pre_tok_state.get("lowercase"     ,  A_	) != do_lower_case
          or pre_tok_state.get("strip_accents"     ,  A_	) != strip_accents
      ):
          SCREAMING_SNAKE_CASE__					     =						getattr(A_     ,  pre_tok_state.pop("type"	)	)
          SCREAMING_SNAKE_CASE__					     =						do_lower_case
          SCREAMING_SNAKE_CASE__					     =						strip_accents
          SCREAMING_SNAKE_CASE__					     =						pre_tok_class(**A_	)

      SCREAMING_SNAKE_CASE__					     =						do_lower_case



  @property
  def      _UpperCamelCase    (	self    :				List[Any]	)					-> Optional[Any]:
      return self.convert_tokens_to_ids(self.question_token	)



  def      _UpperCamelCase    (	self    :				List[str]     ,  A_    :				List[int]     ,  A_    :				Optional[List[int]] = None	)					-> List[int]:
      if token_ids_a is None:
          return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      SCREAMING_SNAKE_CASE__					     =						[self.cls_token_id]
      SCREAMING_SNAKE_CASE__					     =						[self.sep_token_id]
      SCREAMING_SNAKE_CASE__					     =						[self.question_token_id] + [self.convert_tokens_to_ids("."	)]
      if self.padding_side == "right":
          # Input is question-then-context
          return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
      else:
          # Input is context-then-question
          return cls + token_ids_a + sep + token_ids_a + question_suffix + sep



  def      _UpperCamelCase    (	self    :				Optional[int]     ,  A_    :				List[int]     ,  A_    :				Optional[List[int]] = None	)					-> List[int]:
      SCREAMING_SNAKE_CASE__					     =						[self.sep_token_id]
      SCREAMING_SNAKE_CASE__					     =						[self.cls_token_id]
      SCREAMING_SNAKE_CASE__					     =						[self.question_token_id] + [self.convert_tokens_to_ids("."	)]
      if token_ids_a is None:
          return len(cls + token_ids_a + sep	) * [0]

      if self.padding_side == "right":
          # Input is question-then-context
          return len(cls + token_ids_a + question_suffix + sep	) * [0] + len(token_ids_a + sep	) * [1]
      else:
          # Input is context-then-question
          return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + question_suffix + sep	) * [1]







  def      _UpperCamelCase    (	self    :				Optional[int]     ,  A_    :				str     ,  A_    :				Optional[str] = None	)					-> Tuple[str]:
      SCREAMING_SNAKE_CASE__					     =						self._tokenizer.model.save(A_     ,  name=A_	)
      return tuple(A_	)


