




'''simple docstring'''





import importlib.metadata
import sys

from transformers.testing_utils import TestCasePlus
from transformers.utils.versions import require_version, require_version_core


lowercase__			=	importlib.metadata.version('''numpy''')
lowercase__			=	'''.'''.join([str(x) for x in sys.version_info[:3]])






class 						lowercase     (   UpperCAmelCase_):






      def    A__(self					)       ->   Optional[int]:
         # lt + different version strings
         require_version_core('''numpy<1000.4.5'''					)
         require_version_core('''numpy<1000.4'''					)
         require_version_core('''numpy<1000'''					)

         # le
         require_version_core('''numpy<=1000.4.5'''					)
         require_version_core(f"""numpy<={numpy_ver}"""					)

         # eq
         require_version_core(f"""numpy=={numpy_ver}"""					)

         # ne
         require_version_core('''numpy!=1000.4.5'''					)

         # ge
         require_version_core('''numpy>=1.0'''					)
         require_version_core('''numpy>=1.0.0'''					)
         require_version_core(f"""numpy>={numpy_ver}"""					)

         # gt
         require_version_core('''numpy>1.0.0'''					)

         # mix
         require_version_core('''numpy>1.0.0,<1000'''					)

         # requirement w/o version
         require_version_core('''numpy'''					)

         # unmet requirements due to version conflict
         for req in ["numpy==1.0.0", "numpy>=1000.0.0", f"""numpy<{numpy_ver}"""]:
            try:
               require_version_core(a__					)
            except ImportError as e:
               self.assertIn(f"""{req} is required""" ,  str(a__					)					)
               self.assertIn('''but found''' ,  str(a__					)					)

        # unmet requirements due to missing module
         for req in ["numpipypie>1", "numpipypie2"]:
            try:
               require_version_core(a__					)
            except importlib.metadata.PackageNotFoundError as e:
               self.assertIn(f"""The '{req}' distribution was not found and is required by this application""" ,  str(a__					)					)
               self.assertIn('''Try: pip install transformers -U''' ,  str(a__					)					)

        # bogus requirements formats:
        # 1. whole thing
         for req in ["numpy??1.0.0", "numpy1.0.0"]:
            try:
               require_version_core(a__					)
            except ValueError as e:
               self.assertIn('''requirement needs to be in the pip package format''' ,  str(a__					)					)
        # 2. only operators
         for req in ["numpy=1.0.0", "numpy == 1.00", "numpy<>1.0.0", "numpy><1.00", "numpy>>1.0.0"]:
            try:
               require_version_core(a__					)
            except ValueError as e:
               self.assertIn('''need one of ''' ,  str(a__					)					)






      def    A__(self					)       ->   Any:
         # matching requirement
         require_version('''python>=3.6.0'''					)

         # not matching requirements
         for req in ["python>9.9.9", "python<3.0.0"]:
            try:
               require_version_core(a__					)
            except ImportError as e:
               self.assertIn(f"""{req} is required""" ,  str(a__					)					)
               self.assertIn(f"""but found python=={python_ver}""" ,  str(a__					)					)







'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowercase__			=	logging.get_logger(__name__)

lowercase__			=	{'''vocab_file''': '''vocab.txt'''}

lowercase__			=	{
    '''vocab_file''': {
        '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
        '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
        '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
        '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
    }
}

lowercase__			=	{
    '''tau/splinter-base''': 5_1_2,
    '''tau/splinter-base-qass''': 5_1_2,
    '''tau/splinter-large''': 5_1_2,
    '''tau/splinter-large-qass''': 5_1_2,
}

lowercase__			=	{
    '''tau/splinter-base''': {'''do_lower_case''': False},
    '''tau/splinter-base-qass''': {'''do_lower_case''': False},
    '''tau/splinter-large''': {'''do_lower_case''': False},
    '''tau/splinter-large-qass''': {'''do_lower_case''': False},
}






class 						lowercase     (   UpperCAmelCase_):
      __lowercase 		=  VOCAB_FILES_NAMES
      __lowercase 		=  PRETRAINED_VOCAB_FILES_MAP
      __lowercase 		=  PRETRAINED_INIT_CONFIGURATION
      __lowercase 		=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __lowercase 		=  SplinterTokenizer






      def __init__(self ,  a__=None ,  a__=None ,  a__=True ,  a__="[UNK]" ,  a__="[SEP]" ,  a__="[PAD]" ,  a__="[CLS]" ,  a__="[MASK]" ,  a__="[QUESTION]" ,  a__=True ,  a__=None ,  **a__ ,  )       ->   Any:
         super().__init__(
             a__ ,  tokenizer_file=a__ ,  do_lower_case=a__ ,  unk_token=a__ ,  sep_token=a__ ,  pad_token=a__ ,  cls_token=a__ ,  mask_token=a__ ,  tokenize_chinese_chars=a__ ,  strip_accents=a__ ,  additional_special_tokens=(question_token,) ,  **a__ ,  )

         SCREAMING_SNAKE_CASE__			:				Tuple       						= json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
         if (
             pre_tok_state.get('''lowercase''' ,  a__					) != do_lower_case
             or pre_tok_state.get('''strip_accents''' ,  a__					) != strip_accents
         ):
            SCREAMING_SNAKE_CASE__			:				List[Any]       						= getattr(a__ ,  pre_tok_state.pop('''type'''					)					)
            SCREAMING_SNAKE_CASE__			:				Optional[int]       						= do_lower_case
            SCREAMING_SNAKE_CASE__			:				Optional[int]       						= strip_accents
            SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= pre_tok_class(**a__					)

         SCREAMING_SNAKE_CASE__			:				Optional[int]       						= do_lower_case






      @property
      def    A__(self					)       ->   List[str]:
         return self.convert_tokens_to_ids(self.question_token					)






      def    A__(self ,  a__ ,  a__ = None					)       ->   List[int]:
         if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         SCREAMING_SNAKE_CASE__			:				Dict       						= [self.cls_token_id]
         SCREAMING_SNAKE_CASE__			:				Tuple       						= [self.sep_token_id]
         SCREAMING_SNAKE_CASE__			:				List[Any]       						= [self.question_token_id] + [self.convert_tokens_to_ids('''.'''					)]
         if self.padding_side == "right":
            # Input is question-then-context
            return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
         else:
            # Input is context-then-question
            return cls + token_ids_a + sep + token_ids_a + question_suffix + sep






      def    A__(self ,  a__ ,  a__ = None					)       ->   List[int]:
         SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= [self.sep_token_id]
         SCREAMING_SNAKE_CASE__			:				Any       						= [self.cls_token_id]
         SCREAMING_SNAKE_CASE__			:				Optional[Any]       						= [self.question_token_id] + [self.convert_tokens_to_ids('''.'''					)]
         if token_ids_a is None:
            return len(cls + token_ids_a + sep					) * [0]

         if self.padding_side == "right":
            # Input is question-then-context
            return len(cls + token_ids_a + question_suffix + sep					) * [0] + len(token_ids_a + sep					) * [1]
         else:
            # Input is context-then-question
            return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + question_suffix + sep					) * [1]






      def    A__(self ,  a__ ,  a__ = None					)       ->   Tuple[str]:
         SCREAMING_SNAKE_CASE__			:				int       						= self._tokenizer.model.save(a__ ,  name=a__					)
         return tuple(a__					)

