




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__A						:		Any          =						logging.get_logger(__name__)

__A						:		Optional[Any]          =						{"vocab_file": "vocab.txt"}

__A						:		str          =						{
    "vocab_file": {
        "tau/splinter-base": "https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt",
        "tau/splinter-base-qass": "https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt",
        "tau/splinter-large": "https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt",
        "tau/splinter-large-qass": "https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt",
    }
}

__A						:		Tuple          =						{
    "tau/splinter-base": 512,
    "tau/splinter-base-qass": 512,
    "tau/splinter-large": 512,
    "tau/splinter-large-qass": 512,
}

__A						:		Dict          =						{
    "tau/splinter-base": {"do_lower_case": False},
    "tau/splinter-base-qass": {"do_lower_case": False},
    "tau/splinter-large": {"do_lower_case": False},
    "tau/splinter-large-qass": {"do_lower_case": False},
}

class 		lowercase_       (	lowerCAmelCase__				):
					__UpperCamelCase     	= VOCAB_FILES_NAMES
					__UpperCamelCase     	= PRETRAINED_VOCAB_FILES_MAP
					__UpperCamelCase     	= PRETRAINED_INIT_CONFIGURATION
					__UpperCamelCase     	= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__UpperCamelCase     	= SplinterTokenizer




					def __init__(   self:		List[Any],     _lowercase:		List[str]=None,     _lowercase:		int=None,     _lowercase:		List[str]=True,     _lowercase:		Dict="[UNK]",     _lowercase:		Tuple="[SEP]",     _lowercase:		Union[str, Any]="[PAD]",     _lowercase:		Optional[Any]="[CLS]",     _lowercase:		Optional[int]="[MASK]",     _lowercase:		str="[QUESTION]",     _lowercase:		int=True,     _lowercase:		str=None,     **_lowercase:		List[Any],     ):


									'''simple docstring'''


									super().__init__(
									    _lowercase,     tokenizer_file=_lowercase,     do_lower_case=_lowercase,     unk_token=_lowercase,     sep_token=_lowercase,     pad_token=_lowercase,     cls_token=_lowercase,     mask_token=_lowercase,     tokenize_chinese_chars=_lowercase,     strip_accents=_lowercase,     additional_special_tokens=(question_token,),     **_lowercase,     )

									__lowerCAmelCase					=  json.loads(self.backend_tokenizer.normalizer.__getstate__())
									if (
									    pre_tok_state.get("""lowercase""",     _lowercase) != do_lower_case
									    or pre_tok_state.get("""strip_accents""",     _lowercase) != strip_accents
									):
													__lowerCAmelCase					=  getattr(_lowercase,     pre_tok_state.pop("""type"""))
													__lowerCAmelCase					=  do_lower_case
													__lowerCAmelCase					=  strip_accents
													__lowerCAmelCase					=  pre_tok_class(**_lowercase)

									__lowerCAmelCase					=  do_lower_case




					@property
					def     _lowercase						(   self:		List[str]):


									'''simple docstring'''


									return self.convert_tokens_to_ids(self.question_token)




					def     _lowercase						(   self:		Optional[int],     _lowercase:		List[int],     _lowercase:		Optional[List[int]] = None):


									'''simple docstring'''


									if token_ids_a is None:
													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									__lowerCAmelCase					=  [self.cls_token_id]
									__lowerCAmelCase					=  [self.sep_token_id]
									__lowerCAmelCase					=  [self.question_token_id] + [self.convert_tokens_to_ids(""".""")]
									if self.padding_side == "right":
													# Input is question-then-context
													return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
									else:
													# Input is context-then-question
													return cls + token_ids_a + sep + token_ids_a + question_suffix + sep




					def     _lowercase						(   self:		str,     _lowercase:		List[int],     _lowercase:		Optional[List[int]] = None):


									'''simple docstring'''


									__lowerCAmelCase					=  [self.sep_token_id]
									__lowerCAmelCase					=  [self.cls_token_id]
									__lowerCAmelCase					=  [self.question_token_id] + [self.convert_tokens_to_ids(""".""")]
									if token_ids_a is None:
													return len(cls + token_ids_a + sep) * [0]

									if self.padding_side == "right":
													# Input is question-then-context
													return len(cls + token_ids_a + question_suffix + sep) * [0] + len(token_ids_a + sep) * [1]
									else:
													# Input is context-then-question
													return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + question_suffix + sep) * [1]







					def     _lowercase						(   self:		Optional[int],     _lowercase:		str,     _lowercase:		Optional[str] = None):


									'''simple docstring'''


									__lowerCAmelCase					=  self._tokenizer.model.save(_lowercase,     name=_lowercase)
									return tuple(_lowercase)








import os

from apache_beam.io.filesystems import FileSystems
from apache_beam.pipeline import Pipeline

from .logging import get_logger


lowerCAmelCase__             =		2 << 2_0  # 2mb
lowerCAmelCase__             =		get_logger(__name__)







class 						lowercase     (					_lowercase				):


		"""simple docstring"""




		def 		A__       (					self):
			_UpperCamelCase :		Optional[Any]								=    self._options.get_all_options().get('runner')
			return runner in [None, "DirectRunner", "PortableRunner"]






def        lowerCamelCase_  (				UpperCAmelCase_				:  Any				,							UpperCAmelCase_				:  Optional[Any]				,							UpperCAmelCase_				:  str=False						)    ->					str:




	'''simple docstring'''




	_UpperCamelCase :		Optional[int]								=    FileSystems
	if fs.exists(UpperCAmelCase_						):
		if force_upload:
			logger.info(F'''Remote path already exist: {remote_file_path}. Overwriting it as force_upload=True.'''						)
		else:
			logger.info(F'''Remote path already exist: {remote_file_path}. Skipping it as force_upload=False.'''						)
			return
	with fs.create(UpperCAmelCase_						) as remote_file:
		with open(UpperCAmelCase_				,							'rb'						) as local_file:
			_UpperCamelCase :		List[str]								=    local_file.read(UpperCAmelCase_						)
			while chunk:
				remote_file.write(UpperCAmelCase_						)
				_UpperCamelCase :		List[str]								=    local_file.read(UpperCAmelCase_						)







def        lowerCamelCase_  (				UpperCAmelCase_				:  Any				,							UpperCAmelCase_				:  List[Any]				,							UpperCAmelCase_				:  Optional[int]=False						)    ->					Any:




	'''simple docstring'''




	_UpperCamelCase :		List[str]								=    FileSystems
	if os.path.exists(UpperCAmelCase_						):
		if force_download:
			logger.info(F'''Local path already exist: {remote_file_path}. Overwriting it as force_upload=True.'''						)
		else:
			logger.info(F'''Local path already exist: {remote_file_path}. Skipping it as force_upload=False.'''						)
			return
	with fs.open(UpperCAmelCase_						) as remote_file:
		with open(UpperCAmelCase_				,							'wb'						) as local_file:
			_UpperCamelCase :		Union[str, Any]								=    remote_file.read(UpperCAmelCase_						)
			while chunk:
				local_file.write(UpperCAmelCase_						)
				_UpperCamelCase :		List[str]								=    remote_file.read(UpperCAmelCase_						)




