

'''simple docstring'''





import os
import shutil
import unittest
from unittest.mock import patch

from transformers.testing_utils import CaptureStd, is_pt_tf_cross_test, require_torch







class    UpperCamelCase_			(  unittest.TestCase			):


  """simple docstring"""



  @patch("sys.argv"     ,  ["fakeprogrampath", "env"]	)
  def      _UpperCamelCase    (	self    :				List[str]	)					-> Optional[int]:
      # test transformers-cli env
      import transformers.commands.transformers_cli

      with CaptureStd() as cs:
          transformers.commands.transformers_cli.main()
      self.assertIn("Python version"     ,  cs.out	)
      self.assertIn("Platform"     ,  cs.out	)
      self.assertIn("Using distributed or parallel set-up in script?"     ,  cs.out	)



  @is_pt_tf_cross_test
  @patch(
      "sys.argv"     ,  ["fakeprogrampath", "pt-to-tf", "--model-name", "hf-internal-testing/tiny-random-gptj", "--no-pr"]	)
  def      _UpperCamelCase    (	self    :				List[str]	)					-> Optional[Any]:
      import transformers.commands.transformers_cli

      shutil.rmtree("/tmp/hf-internal-testing/tiny-random-gptj"     ,  ignore_errors=A_	)  # cleans potential past runs
      transformers.commands.transformers_cli.main()

      # The original repo has no TF weights -- if they exist, they were created by the CLI
      self.assertTrue(os.path.exists("/tmp/hf-internal-testing/tiny-random-gptj/tf_model.h5"	)	)



  @require_torch
  @patch("sys.argv"     ,  ["fakeprogrampath", "download", "hf-internal-testing/tiny-random-gptj", "--cache-dir", "/tmp"]	)
  def      _UpperCamelCase    (	self    :				Any	)					-> Any:
      import transformers.commands.transformers_cli

      # # remove any previously downloaded model to start clean
      shutil.rmtree("/tmp/models--hf-internal-testing--tiny-random-gptj"     ,  ignore_errors=A_	)

      # run the command
      transformers.commands.transformers_cli.main()

      # check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--tiny-random-gptj
      self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/blobs"	)	)
      self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/refs"	)	)
      self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/snapshots"	)	)







  @require_torch
  @patch(
      "sys.argv"     ,  [
          "fakeprogrampath",
          "download",
          "hf-internal-testing/test_dynamic_model_with_tokenizer",
          "--trust-remote-code",
          "--cache-dir",
          "/tmp",
      ]     ,  )
  def      _UpperCamelCase    (	self    :				str	)					-> Optional[Any]:
      import transformers.commands.transformers_cli

      # # remove any previously downloaded model to start clean
      shutil.rmtree("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer"     ,  ignore_errors=A_	)

      # run the command
      transformers.commands.transformers_cli.main()

      # check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer
      self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/blobs"	)	)
      self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/refs"	)	)
      self.assertTrue(
          os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/snapshots"	)	)





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


A__        =			logging.get_logger(__name__)

A__        =			{"vocab_file": "vocab.txt"}

A__        =			{
    "vocab_file": {
        "tau/splinter-base": "https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt",
        "tau/splinter-base-qass": "https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt",
        "tau/splinter-large": "https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt",
        "tau/splinter-large-qass": "https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt",
    }
}

A__        =			{
    "tau/splinter-base": 512,
    "tau/splinter-base-qass": 512,
    "tau/splinter-large": 512,
    "tau/splinter-large-qass": 512,
}

A__        =			{
    "tau/splinter-base": {"do_lower_case": False},
    "tau/splinter-base-qass": {"do_lower_case": False},
    "tau/splinter-large": {"do_lower_case": False},
    "tau/splinter-large-qass": {"do_lower_case": False},
}







class    UpperCamelCase_			(  _snake_case			):


  """simple docstring"""
  SCREAMING_SNAKE_CASE__				: Dict										=      VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE__				: Optional[int]										=      PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE__				: List[Any]										=      PRETRAINED_INIT_CONFIGURATION
  SCREAMING_SNAKE_CASE__				: str										=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE__				: Optional[Any]										=      SplinterTokenizer



  def __init__(	self    :				Tuple     ,  A_    :				Optional[Any]=None     ,  A_    :				Optional[int]=None     ,  A_    :				Optional[Any]=True     ,  A_    :				str="[UNK]"     ,  A_    :				Optional[Any]="[SEP]"     ,  A_    :				str="[PAD]"     ,  A_    :				List[Any]="[CLS]"     ,  A_    :				Union[str, Any]="[MASK]"     ,  A_    :				Any="[QUESTION]"     ,  A_    :				List[Any]=True     ,  A_    :				Union[str, Any]=None     ,  **A_    :				Optional[Any]     ,  )					-> Optional[Any]:
      super().__init__(
          A_     ,  tokenizer_file=A_     ,  do_lower_case=A_     ,  unk_token=A_     ,  sep_token=A_     ,  pad_token=A_     ,  cls_token=A_     ,  mask_token=A_     ,  tokenize_chinese_chars=A_     ,  strip_accents=A_     ,  additional_special_tokens=(question_token,)     ,  **A_     ,  )

      SCREAMING_SNAKE_CASE__					     =						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
      if (
          pre_tok_state.get("lowercase"     ,  A_	) != do_lower_case
          or pre_tok_state.get("strip_accents"     ,  A_	) != strip_accents
      ):
          SCREAMING_SNAKE_CASE__					     =						getattr(A_     ,  pre_tok_state.pop("type"	)	)
          SCREAMING_SNAKE_CASE__					     =						do_lower_case
          SCREAMING_SNAKE_CASE__					     =						strip_accents
          SCREAMING_SNAKE_CASE__					     =						pre_tok_class(**A_	)

      SCREAMING_SNAKE_CASE__					     =						do_lower_case



  @property
  def      _UpperCamelCase    (	self    :				List[Any]	)					-> Optional[Any]:
      return self.convert_tokens_to_ids(self.question_token	)



  def      _UpperCamelCase    (	self    :				List[str]     ,  A_    :				List[int]     ,  A_    :				Optional[List[int]] = None	)					-> List[int]:
      if token_ids_a is None:
          return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      SCREAMING_SNAKE_CASE__					     =						[self.cls_token_id]
      SCREAMING_SNAKE_CASE__					     =						[self.sep_token_id]
      SCREAMING_SNAKE_CASE__					     =						[self.question_token_id] + [self.convert_tokens_to_ids("."	)]
      if self.padding_side == "right":
          # Input is question-then-context
          return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
      else:
          # Input is context-then-question
          return cls + token_ids_a + sep + token_ids_a + question_suffix + sep



  def      _UpperCamelCase    (	self    :				Optional[int]     ,  A_    :				List[int]     ,  A_    :				Optional[List[int]] = None	)					-> List[int]:
      SCREAMING_SNAKE_CASE__					     =						[self.sep_token_id]
      SCREAMING_SNAKE_CASE__					     =						[self.cls_token_id]
      SCREAMING_SNAKE_CASE__					     =						[self.question_token_id] + [self.convert_tokens_to_ids("."	)]
      if token_ids_a is None:
          return len(cls + token_ids_a + sep	) * [0]

      if self.padding_side == "right":
          # Input is question-then-context
          return len(cls + token_ids_a + question_suffix + sep	) * [0] + len(token_ids_a + sep	) * [1]
      else:
          # Input is context-then-question
          return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + question_suffix + sep	) * [1]







  def      _UpperCamelCase    (	self    :				Optional[int]     ,  A_    :				str     ,  A_    :				Optional[str] = None	)					-> Tuple[str]:
      SCREAMING_SNAKE_CASE__					     =						self._tokenizer.model.save(A_     ,  name=A_	)
      return tuple(A_	)


