

"""simple docstring"""




from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


SCREAMING_SNAKE_CASE_  :      List[str]							=      {
    '''configuration_utils''': ['''GenerationConfig'''],
    '''streamers''': ['''TextIteratorStreamer''', '''TextStreamer'''],
}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     SCREAMING_SNAKE_CASE_  :      Union[str, Any]							=      [
         '''Constraint''',
         '''ConstraintListState''',
         '''DisjunctiveConstraint''',
         '''PhrasalConstraint''',
     ]
     SCREAMING_SNAKE_CASE_  :      Union[str, Any]							=      [
         '''BeamHypotheses''',
         '''BeamScorer''',
         '''BeamSearchScorer''',
         '''ConstrainedBeamSearchScorer''',
     ]
     SCREAMING_SNAKE_CASE_  :      List[str]							=      [
         '''EpsilonLogitsWarper''',
         '''EtaLogitsWarper''',
         '''ForcedBOSTokenLogitsProcessor''',
         '''ForcedEOSTokenLogitsProcessor''',
         '''HammingDiversityLogitsProcessor''',
         '''InfNanRemoveLogitsProcessor''',
         '''LogitsProcessor''',
         '''LogitsProcessorList''',
         '''LogitsWarper''',
         '''MinLengthLogitsProcessor''',
         '''MinNewTokensLengthLogitsProcessor''',
         '''NoBadWordsLogitsProcessor''',
         '''NoRepeatNGramLogitsProcessor''',
         '''PrefixConstrainedLogitsProcessor''',
         '''RepetitionPenaltyLogitsProcessor''',
         '''SequenceBiasLogitsProcessor''',
         '''EncoderRepetitionPenaltyLogitsProcessor''',
         '''TemperatureLogitsWarper''',
         '''TopKLogitsWarper''',
         '''TopPLogitsWarper''',
         '''TypicalLogitsWarper''',
         '''EncoderNoRepeatNGramLogitsProcessor''',
         '''ExponentialDecayLengthPenalty''',
         '''LogitNormalization''',
     ]
     SCREAMING_SNAKE_CASE_  :      Any							=      [
         '''MaxNewTokensCriteria''',
         '''MaxLengthCriteria''',
         '''MaxTimeCriteria''',
         '''StoppingCriteria''',
         '''StoppingCriteriaList''',
         '''validate_stopping_criteria''',
     ]
     SCREAMING_SNAKE_CASE_  :      List[str]							=      [
         '''GenerationMixin''',
         '''top_k_top_p_filtering''',
         '''GreedySearchEncoderDecoderOutput''',
         '''GreedySearchDecoderOnlyOutput''',
         '''SampleEncoderDecoderOutput''',
         '''SampleDecoderOnlyOutput''',
         '''BeamSearchEncoderDecoderOutput''',
         '''BeamSearchDecoderOnlyOutput''',
         '''BeamSampleEncoderDecoderOutput''',
         '''BeamSampleDecoderOnlyOutput''',
         '''ContrastiveSearchEncoderDecoderOutput''',
         '''ContrastiveSearchDecoderOnlyOutput''',
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     SCREAMING_SNAKE_CASE_  :      Optional[Any]							=      [
         '''TFForcedBOSTokenLogitsProcessor''',
         '''TFForcedEOSTokenLogitsProcessor''',
         '''TFLogitsProcessor''',
         '''TFLogitsProcessorList''',
         '''TFLogitsWarper''',
         '''TFMinLengthLogitsProcessor''',
         '''TFNoBadWordsLogitsProcessor''',
         '''TFNoRepeatNGramLogitsProcessor''',
         '''TFRepetitionPenaltyLogitsProcessor''',
         '''TFTemperatureLogitsWarper''',
         '''TFTopKLogitsWarper''',
         '''TFTopPLogitsWarper''',
         '''TFForceTokensLogitsProcessor''',
         '''TFSuppressTokensAtBeginLogitsProcessor''',
         '''TFSuppressTokensLogitsProcessor''',
     ]
     SCREAMING_SNAKE_CASE_  :      List[Any]							=      [
         '''TFGenerationMixin''',
         '''tf_top_k_top_p_filtering''',
         '''TFGreedySearchDecoderOnlyOutput''',
         '''TFGreedySearchEncoderDecoderOutput''',
         '''TFSampleEncoderDecoderOutput''',
         '''TFSampleDecoderOnlyOutput''',
         '''TFBeamSearchEncoderDecoderOutput''',
         '''TFBeamSearchDecoderOnlyOutput''',
         '''TFBeamSampleEncoderDecoderOutput''',
         '''TFBeamSampleDecoderOnlyOutput''',
         '''TFContrastiveSearchEncoderDecoderOutput''',
         '''TFContrastiveSearchDecoderOnlyOutput''',
     ]

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     SCREAMING_SNAKE_CASE_  :      Optional[int]							=      [
         '''FlaxForcedBOSTokenLogitsProcessor''',
         '''FlaxForcedEOSTokenLogitsProcessor''',
         '''FlaxLogitsProcessor''',
         '''FlaxLogitsProcessorList''',
         '''FlaxLogitsWarper''',
         '''FlaxMinLengthLogitsProcessor''',
         '''FlaxTemperatureLogitsWarper''',
         '''FlaxTopKLogitsWarper''',
         '''FlaxTopPLogitsWarper''',
     ]
     SCREAMING_SNAKE_CASE_  :      Optional[Any]							=      [
         '''FlaxGenerationMixin''',
         '''FlaxGreedySearchOutput''',
         '''FlaxSampleOutput''',
         '''FlaxBeamSearchOutput''',
     ]

if TYPE_CHECKING:
     from .configuration_utils import GenerationConfig
     from .streamers import TextIteratorStreamer, TextStreamer

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
          from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
          from .logits_process import (
              EncoderNoRepeatNGramLogitsProcessor,
              EncoderRepetitionPenaltyLogitsProcessor,
              EpsilonLogitsWarper,
              EtaLogitsWarper,
              ExponentialDecayLengthPenalty,
              ForcedBOSTokenLogitsProcessor,
              ForcedEOSTokenLogitsProcessor,
              HammingDiversityLogitsProcessor,
              InfNanRemoveLogitsProcessor,
              LogitNormalization,
              LogitsProcessor,
              LogitsProcessorList,
              LogitsWarper,
              MinLengthLogitsProcessor,
              MinNewTokensLengthLogitsProcessor,
              NoBadWordsLogitsProcessor,
              NoRepeatNGramLogitsProcessor,
              PrefixConstrainedLogitsProcessor,
              RepetitionPenaltyLogitsProcessor,
              SequenceBiasLogitsProcessor,
              TemperatureLogitsWarper,
              TopKLogitsWarper,
              TopPLogitsWarper,
              TypicalLogitsWarper,
          )
          from .stopping_criteria import (
              MaxLengthCriteria,
              MaxNewTokensCriteria,
              MaxTimeCriteria,
              StoppingCriteria,
              StoppingCriteriaList,
              validate_stopping_criteria,
          )
          from .utils import (
              BeamSampleDecoderOnlyOutput,
              BeamSampleEncoderDecoderOutput,
              BeamSearchDecoderOnlyOutput,
              BeamSearchEncoderDecoderOutput,
              ContrastiveSearchDecoderOnlyOutput,
              ContrastiveSearchEncoderDecoderOutput,
              GenerationMixin,
              GreedySearchDecoderOnlyOutput,
              GreedySearchEncoderDecoderOutput,
              SampleDecoderOnlyOutput,
              SampleEncoderDecoderOutput,
              top_k_top_p_filtering,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tf_logits_process import (
              TFForcedBOSTokenLogitsProcessor,
              TFForcedEOSTokenLogitsProcessor,
              TFForceTokensLogitsProcessor,
              TFLogitsProcessor,
              TFLogitsProcessorList,
              TFLogitsWarper,
              TFMinLengthLogitsProcessor,
              TFNoBadWordsLogitsProcessor,
              TFNoRepeatNGramLogitsProcessor,
              TFRepetitionPenaltyLogitsProcessor,
              TFSuppressTokensAtBeginLogitsProcessor,
              TFSuppressTokensLogitsProcessor,
              TFTemperatureLogitsWarper,
              TFTopKLogitsWarper,
              TFTopPLogitsWarper,
          )
          from .tf_utils import (
              TFBeamSampleDecoderOnlyOutput,
              TFBeamSampleEncoderDecoderOutput,
              TFBeamSearchDecoderOnlyOutput,
              TFBeamSearchEncoderDecoderOutput,
              TFContrastiveSearchDecoderOnlyOutput,
              TFContrastiveSearchEncoderDecoderOutput,
              TFGenerationMixin,
              TFGreedySearchDecoderOnlyOutput,
              TFGreedySearchEncoderDecoderOutput,
              TFSampleDecoderOnlyOutput,
              TFSampleEncoderDecoderOutput,
              tf_top_k_top_p_filtering,
          )

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .flax_logits_process import (
              FlaxForcedBOSTokenLogitsProcessor,
              FlaxForcedEOSTokenLogitsProcessor,
              FlaxLogitsProcessor,
              FlaxLogitsProcessorList,
              FlaxLogitsWarper,
              FlaxMinLengthLogitsProcessor,
              FlaxTemperatureLogitsWarper,
              FlaxTopKLogitsWarper,
              FlaxTopPLogitsWarper,
          )
          from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
     import sys

     SCREAMING_SNAKE_CASE_  :      str							=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)







"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


SCREAMING_SNAKE_CASE_  :      str							=      logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_  :      List[Any]							=      {'''vocab_file''': '''vocab.txt'''}

SCREAMING_SNAKE_CASE_  :      int							=      {
    '''vocab_file''': {
        '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
        '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
        '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
        '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
    }
}

SCREAMING_SNAKE_CASE_  :      Union[str, Any]							=      {
    '''tau/splinter-base''': 512,
    '''tau/splinter-base-qass''': 512,
    '''tau/splinter-large''': 512,
    '''tau/splinter-large-qass''': 512,
}

SCREAMING_SNAKE_CASE_  :      List[Any]							=      {
    '''tau/splinter-base''': {'''do_lower_case''': False},
    '''tau/splinter-base-qass''': {'''do_lower_case''': False},
    '''tau/splinter-large''': {'''do_lower_case''': False},
    '''tau/splinter-large-qass''': {'''do_lower_case''': False},
}


class 		_A							(      __a       ):
      __a             =						VOCAB_FILES_NAMES
      __a             =						PRETRAINED_VOCAB_FILES_MAP
      __a             =						PRETRAINED_INIT_CONFIGURATION
      __a             =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __a             =						SplinterTokenizer





      def __init__(	self     ,    SCREAMING_SNAKE_CASE__=None     ,    SCREAMING_SNAKE_CASE__=None     ,    SCREAMING_SNAKE_CASE__=True     ,    SCREAMING_SNAKE_CASE__="[UNK]"     ,    SCREAMING_SNAKE_CASE__="[SEP]"     ,    SCREAMING_SNAKE_CASE__="[PAD]"     ,    SCREAMING_SNAKE_CASE__="[CLS]"     ,    SCREAMING_SNAKE_CASE__="[MASK]"     ,    SCREAMING_SNAKE_CASE__="[QUESTION]"     ,    SCREAMING_SNAKE_CASE__=True     ,    SCREAMING_SNAKE_CASE__=None     ,    **SCREAMING_SNAKE_CASE__     ,    ) ->       Optional[Any]:
         super().__init__(
             SCREAMING_SNAKE_CASE__     ,    tokenizer_file=SCREAMING_SNAKE_CASE__     ,    do_lower_case=SCREAMING_SNAKE_CASE__     ,    unk_token=SCREAMING_SNAKE_CASE__     ,    sep_token=SCREAMING_SNAKE_CASE__     ,    pad_token=SCREAMING_SNAKE_CASE__     ,    cls_token=SCREAMING_SNAKE_CASE__     ,    mask_token=SCREAMING_SNAKE_CASE__     ,    tokenize_chinese_chars=SCREAMING_SNAKE_CASE__     ,    strip_accents=SCREAMING_SNAKE_CASE__     ,    additional_special_tokens=(question_token,)     ,    **SCREAMING_SNAKE_CASE__     ,    )

         lowerCamelCase__						       =      json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
         if (
             pre_tok_state.get("lowercase"     ,    SCREAMING_SNAKE_CASE__		) != do_lower_case
             or pre_tok_state.get("strip_accents"     ,    SCREAMING_SNAKE_CASE__		) != strip_accents
         ):
            lowerCamelCase__						       =      getattr(SCREAMING_SNAKE_CASE__     ,    pre_tok_state.pop("type"		)		)
            lowerCamelCase__						       =      do_lower_case
            lowerCamelCase__						       =      strip_accents
            lowerCamelCase__						       =      pre_tok_class(**SCREAMING_SNAKE_CASE__		)

         lowerCamelCase__						       =      do_lower_case





      @property
      def        _lowerCamelCase	(	self		) ->       Optional[Any]:
         return self.convert_tokens_to_ids(self.question_token		)





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__ = None		) ->       List[int]:
         if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         lowerCamelCase__						       =      [self.cls_token_id]
         lowerCamelCase__						       =      [self.sep_token_id]
         lowerCamelCase__						       =      [self.question_token_id] + [self.convert_tokens_to_ids("."		)]
         if self.padding_side == "right":
            # Input is question-then-context
            return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
         else:
            # Input is context-then-question
            return cls + token_ids_a + sep + token_ids_a + question_suffix + sep





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__ = None		) ->       List[int]:
         lowerCamelCase__						       =      [self.sep_token_id]
         lowerCamelCase__						       =      [self.cls_token_id]
         lowerCamelCase__						       =      [self.question_token_id] + [self.convert_tokens_to_ids("."		)]
         if token_ids_a is None:
            return len(cls + token_ids_a + sep		) * [0]

         if self.padding_side == "right":
            # Input is question-then-context
            return len(cls + token_ids_a + question_suffix + sep		) * [0] + len(token_ids_a + sep		) * [1]
         else:
            # Input is context-then-question
            return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + question_suffix + sep		) * [1]





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__ = None		) ->       Tuple[str]:
         lowerCamelCase__						       =      self._tokenizer.model.save(SCREAMING_SNAKE_CASE__     ,    name=SCREAMING_SNAKE_CASE__		)
         return tuple(SCREAMING_SNAKE_CASE__		)




