
'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__lowerCAmelCase  							= logging.get_logger(__name__)

__lowerCAmelCase  							= {"""vocab_file""": """vocab.txt"""}

__lowerCAmelCase  							= {
    """vocab_file""": {
        """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt""",
        """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt""",
        """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt""",
        """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt""",
    }
}

__lowerCAmelCase  							= {
    """tau/splinter-base""": 512,
    """tau/splinter-base-qass""": 512,
    """tau/splinter-large""": 512,
    """tau/splinter-large-qass""": 512,
}

__lowerCAmelCase  							= {
    """tau/splinter-base""": {"""do_lower_case""": False},
    """tau/splinter-base-qass""": {"""do_lower_case""": False},
    """tau/splinter-large""": {"""do_lower_case""": False},
    """tau/splinter-large-qass""": {"""do_lower_case""": False},
}


class     _lowerCAmelCase					(					a_   ):







							'''simple docstring'''

							lowerCAmelCase_      		=		VOCAB_FILES_NAMES
							lowerCAmelCase_      		=		PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase_      		=		PRETRAINED_INIT_CONFIGURATION
							lowerCAmelCase_      		=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase_      		=		SplinterTokenizer

							def __init__(self						,						UpperCAmelCase=None						,						UpperCAmelCase=None						,						UpperCAmelCase=True						,						UpperCAmelCase="[UNK]"						,						UpperCAmelCase="[SEP]"						,						UpperCAmelCase="[PAD]"						,						UpperCAmelCase="[CLS]"						,						UpperCAmelCase="[MASK]"						,						UpperCAmelCase="[QUESTION]"						,						UpperCAmelCase=True						,						UpperCAmelCase=None						,						**UpperCAmelCase						,						)  ->     int:
														super().__init__(
														    _snake_case						,						tokenizer_file=_snake_case						,						do_lower_case=_snake_case						,						unk_token=_snake_case						,						sep_token=_snake_case						,						pad_token=_snake_case						,						cls_token=_snake_case						,						mask_token=_snake_case						,						tokenize_chinese_chars=_snake_case						,						strip_accents=_snake_case						,						additional_special_tokens=(question_token,)						,						**_snake_case						,						)

														_snake_case     =						json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
														if (
														    pre_tok_state.get("""lowercase"""						,						_snake_case						) != do_lower_case
														    or pre_tok_state.get("""strip_accents"""						,						_snake_case						) != strip_accents
														):
																					_snake_case     =						getattr(_snake_case						,						pre_tok_state.pop("""type"""						)						)
																					_snake_case     =						do_lower_case
																					_snake_case     =						strip_accents
																					_snake_case     =						pre_tok_class(**_snake_case						)

														_snake_case     =						do_lower_case

							@property
							def  lowercase							(self						)  ->     str:
														return self.convert_tokens_to_ids(self.question_token						)

							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase = None						)  ->     List[int]:
														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

														_snake_case     =						[self.cls_token_id]
														_snake_case     =						[self.sep_token_id]
														_snake_case     =						[self.question_token_id] + [self.convert_tokens_to_ids("""."""						)]
														if self.padding_side == "right":
																					# Input is question-then-context
																					return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
														else:
																					# Input is context-then-question
																					return cls + token_ids_a + sep + token_ids_a + question_suffix + sep

							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase = None						)  ->     List[int]:
														_snake_case     =						[self.sep_token_id]
														_snake_case     =						[self.cls_token_id]
														_snake_case     =						[self.question_token_id] + [self.convert_tokens_to_ids("""."""						)]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep						) * [0]

														if self.padding_side == "right":
																					# Input is question-then-context
																					return len(cls + token_ids_a + question_suffix + sep						) * [0] + len(token_ids_a + sep						) * [1]
														else:
																					# Input is context-then-question
																					return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + question_suffix + sep						) * [1]





							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase = None						)  ->     Tuple[str]:
														_snake_case     =						self._tokenizer.model.save(_snake_case						,						name=_snake_case						)
														return tuple(_snake_case						)

'''simple docstring'''
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_xglm import XGLMTokenizer
else:
				__lowerCAmelCase  							= None


__lowerCAmelCase  							= logging.get_logger(__name__)

__lowerCAmelCase  							= {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase  							= {
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json',
    },
}

__lowerCAmelCase  							= {
    'facebook/xglm-564M': 2_048,
}


class     _lowerCAmelCase					(					__snake_case   ):







							'''simple docstring'''

							lowerCAmelCase_      		=		VOCAB_FILES_NAMES
							lowerCAmelCase_      		=		PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase_      		=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase_      		=		["input_ids", "attention_mask"]
							lowerCAmelCase_      		=		XGLMTokenizer

							def __init__(self						,						UpperCAmelCase=None						,						UpperCAmelCase=None						,						UpperCAmelCase="<s>"						,						UpperCAmelCase="</s>"						,						UpperCAmelCase="</s>"						,						UpperCAmelCase="<s>"						,						UpperCAmelCase="<unk>"						,						UpperCAmelCase="<pad>"						,						**UpperCAmelCase						,						)  ->     List[Any]:
														# Compatibility with the original tokenizer
														_snake_case     =						7
														_snake_case     =						[f"""<madeupword{i}>""" for i in range(self.num_madeup_words						)]

														_snake_case     =						kwargs.get("""additional_special_tokens"""						,						[]						)
														kwargs["additional_special_tokens"] += [
														    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
														]

														super().__init__(
														    UpperCAmelCase						,						tokenizer_file=UpperCAmelCase						,						bos_token=UpperCAmelCase						,						eos_token=UpperCAmelCase						,						sep_token=UpperCAmelCase						,						cls_token=UpperCAmelCase						,						unk_token=UpperCAmelCase						,						pad_token=UpperCAmelCase						,						**UpperCAmelCase						,						)

														_snake_case     =						vocab_file
														_snake_case     =						False if not self.vocab_file else True

							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase = None						)  ->     List[int]:

														if token_ids_a is None:
																					return [self.sep_token_id] + token_ids_a
														_snake_case     =						[self.sep_token_id]
														return sep + token_ids_a + sep + sep + token_ids_a

							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase = None						)  ->     List[int]:
														_snake_case     =						[self.sep_token_id]

														if token_ids_a is None:
																					return len(sep + token_ids_a						) * [0]
														return len(sep + token_ids_a + sep + sep + token_ids_a						) * [0]





							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase = None						)  ->     Tuple[str]:
														if not self.can_save_slow_tokenizer:
																					raise ValueError(
																					    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
																					    """tokenizer."""						)

														if not os.path.isdir(UpperCAmelCase						):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""						)
																					return
														_snake_case     =						os.path.join(
														    UpperCAmelCase						,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

														if os.path.abspath(self.vocab_file						) != os.path.abspath(UpperCAmelCase						):
																					copyfile(self.vocab_file						,						UpperCAmelCase						)

														return (out_vocab_file,)