
'''simple docstring'''
import os

from ...utils import logging
from .utils import DataProcessor, InputExample


__lowerCAmelCase  							= logging.get_logger(__name__)


class     _lowerCAmelCase					(					__snake_case   ):







							'''simple docstring'''

							def __init__(self						,						UpperCAmelCase						,						UpperCAmelCase=None						)  ->     Optional[int]:
														_snake_case     =						language
														_snake_case     =						train_language

							def  lowercase							(self						,						UpperCAmelCase						)  ->     int:
														_snake_case     =						self.language if self.train_language is None else self.train_language
														_snake_case     =						self._read_tsv(os.path.join(UpperCAmelCase						,						f"""XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv"""						)						)
														_snake_case     =						[]
														for i, line in enumerate(UpperCAmelCase						):
																					if i == 0:
																												continue
																					_snake_case     =						f"""train-{i}"""
																					_snake_case     =						line[0]
																					_snake_case     =						line[1]
																					_snake_case     =						"""contradiction""" if line[2] == """contradictory""" else line[2]
																					if not isinstance(UpperCAmelCase						,						UpperCAmelCase						):
																												raise ValueError(f"""Training input {text_a} is not a string"""						)
																					if not isinstance(UpperCAmelCase						,						UpperCAmelCase						):
																												raise ValueError(f"""Training input {text_b} is not a string"""						)
																					if not isinstance(UpperCAmelCase						,						UpperCAmelCase						):
																												raise ValueError(f"""Training label {label} is not a string"""						)
																					examples.append(InputExample(guid=UpperCAmelCase						,						text_a=UpperCAmelCase						,						text_b=UpperCAmelCase						,						label=UpperCAmelCase						)						)
														return examples

							def  lowercase							(self						,						UpperCAmelCase						)  ->     Optional[int]:
														_snake_case     =						self._read_tsv(os.path.join(UpperCAmelCase						,						"""XNLI-1.0/xnli.test.tsv"""						)						)
														_snake_case     =						[]
														for i, line in enumerate(UpperCAmelCase						):
																					if i == 0:
																												continue
																					_snake_case     =						line[0]
																					if language != self.language:
																												continue
																					_snake_case     =						f"""test-{i}"""
																					_snake_case     =						line[6]
																					_snake_case     =						line[7]
																					_snake_case     =						line[1]
																					if not isinstance(UpperCAmelCase						,						UpperCAmelCase						):
																												raise ValueError(f"""Training input {text_a} is not a string"""						)
																					if not isinstance(UpperCAmelCase						,						UpperCAmelCase						):
																												raise ValueError(f"""Training input {text_b} is not a string"""						)
																					if not isinstance(UpperCAmelCase						,						UpperCAmelCase						):
																												raise ValueError(f"""Training label {label} is not a string"""						)
																					examples.append(InputExample(guid=UpperCAmelCase						,						text_a=UpperCAmelCase						,						text_b=UpperCAmelCase						,						label=UpperCAmelCase						)						)
														return examples





							def  lowercase							(self						)  ->     str:
														return ["contradiction", "entailment", "neutral"]
__lowerCAmelCase  							= {
    'xnli': XnliProcessor,
}

__lowerCAmelCase  							= {
    'xnli': 'classification',
}

__lowerCAmelCase  							= {
    'xnli': 3,
}

'''simple docstring'''
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_xglm import XGLMTokenizer
else:
				__lowerCAmelCase  							= None


__lowerCAmelCase  							= logging.get_logger(__name__)

__lowerCAmelCase  							= {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase  							= {
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json',
    },
}

__lowerCAmelCase  							= {
    'facebook/xglm-564M': 2_048,
}


class     _lowerCAmelCase					(					__snake_case   ):







							'''simple docstring'''

							lowerCAmelCase_      		=		VOCAB_FILES_NAMES
							lowerCAmelCase_      		=		PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase_      		=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase_      		=		["input_ids", "attention_mask"]
							lowerCAmelCase_      		=		XGLMTokenizer

							def __init__(self						,						UpperCAmelCase=None						,						UpperCAmelCase=None						,						UpperCAmelCase="<s>"						,						UpperCAmelCase="</s>"						,						UpperCAmelCase="</s>"						,						UpperCAmelCase="<s>"						,						UpperCAmelCase="<unk>"						,						UpperCAmelCase="<pad>"						,						**UpperCAmelCase						,						)  ->     List[Any]:
														# Compatibility with the original tokenizer
														_snake_case     =						7
														_snake_case     =						[f"""<madeupword{i}>""" for i in range(self.num_madeup_words						)]

														_snake_case     =						kwargs.get("""additional_special_tokens"""						,						[]						)
														kwargs["additional_special_tokens"] += [
														    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
														]

														super().__init__(
														    UpperCAmelCase						,						tokenizer_file=UpperCAmelCase						,						bos_token=UpperCAmelCase						,						eos_token=UpperCAmelCase						,						sep_token=UpperCAmelCase						,						cls_token=UpperCAmelCase						,						unk_token=UpperCAmelCase						,						pad_token=UpperCAmelCase						,						**UpperCAmelCase						,						)

														_snake_case     =						vocab_file
														_snake_case     =						False if not self.vocab_file else True

							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase = None						)  ->     List[int]:

														if token_ids_a is None:
																					return [self.sep_token_id] + token_ids_a
														_snake_case     =						[self.sep_token_id]
														return sep + token_ids_a + sep + sep + token_ids_a

							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase = None						)  ->     List[int]:
														_snake_case     =						[self.sep_token_id]

														if token_ids_a is None:
																					return len(sep + token_ids_a						) * [0]
														return len(sep + token_ids_a + sep + sep + token_ids_a						) * [0]





							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase = None						)  ->     Tuple[str]:
														if not self.can_save_slow_tokenizer:
																					raise ValueError(
																					    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
																					    """tokenizer."""						)

														if not os.path.isdir(UpperCAmelCase						):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""						)
																					return
														_snake_case     =						os.path.join(
														    UpperCAmelCase						,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

														if os.path.abspath(self.vocab_file						) != os.path.abspath(UpperCAmelCase						):
																					copyfile(self.vocab_file						,						UpperCAmelCase						)

														return (out_vocab_file,)