



'''simple docstring'''
import os
import unittest

from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer

from ...test_tokenization_common import TokenizerTesterMixin






class 							snake_case__			(							lowercase		,      unittest.TestCase     ):
	lowercase__						       =    BertweetTokenizer
	lowercase__						       =    False





	def 			UpperCamelCase_						(				self      :       Union[str, Any]				):







							'''simple docstring'''
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							_UpperCamelCase      :	Union[str, Any]				       =     ['I', 'm', 'V@@', 'R@@', 'r', 'e@@']
							_UpperCamelCase      :	Any				       =     dict(zip(lowerCamelCase__				,range(len(lowerCamelCase__				)				)				)				)
							_UpperCamelCase      :	str				       =     ['#version: 0.2', 'a m</w>']
							_UpperCamelCase      :	Any				       =     {'unk_token': '<unk>'}

							_UpperCamelCase      :	Union[str, Any]				       =     os.path.join(self.tmpdirname				,VOCAB_FILES_NAMES['vocab_file']				)
							_UpperCamelCase      :	Optional[Any]				       =     os.path.join(self.tmpdirname				,VOCAB_FILES_NAMES['merges_file']				)
							with open(self.vocab_file				,'w'				,encoding='utf-8'				) as fp:
													for token in vocab_tokens:
																			fp.write(F'{token} {vocab_tokens[token]}\n'				)
							with open(self.merges_file				,'w'				,encoding='utf-8'				) as fp:
													fp.write('\n'.join(lowerCamelCase__				)				)





	def 			UpperCamelCase_						(				self      :       str				,**lowerCamelCase__      :       List[Any]				):







							'''simple docstring'''
							kwargs.update(self.special_tokens_map				)
							return BertweetTokenizer.from_pretrained(self.tmpdirname				,**lowerCamelCase__				)





	def 			UpperCamelCase_						(				self      :       Tuple				,lowerCamelCase__      :       List[Any]				):







							'''simple docstring'''
							_UpperCamelCase      :	str				       =     'I am VinAI Research'
							_UpperCamelCase      :	Optional[Any]				       =     'I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'
							return input_text, output_text





	def 			UpperCamelCase_						(				self      :       Dict				):







							'''simple docstring'''
							_UpperCamelCase      :	List[str]				       =     BertweetTokenizer(self.vocab_file				,self.merges_file				,**self.special_tokens_map				)
							_UpperCamelCase      :	int				       =     'I am VinAI Research'
							_UpperCamelCase      :	int				       =     'I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'.split()
							_UpperCamelCase      :	List[Any]				       =     tokenizer.tokenize(lowerCamelCase__				)
							self.assertListEqual(lowerCamelCase__				,lowerCamelCase__				)

							_UpperCamelCase      :	Dict				       =     tokens + [tokenizer.unk_token]

							_UpperCamelCase      :	Dict				       =     [4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__				)				,lowerCamelCase__				)







'''simple docstring'''
import os

from ...utils import logging
from .utils import DataProcessor, InputExample


snake_case_				:			Any						 =						logging.get_logger(__name__)






class 							lowercase__			(							lowercase     ):





	def __init__(				self      :       Union[str, Any]				,lowerCamelCase__      :       Tuple				,lowerCamelCase__      :       Optional[int]=None				):







							'''simple docstring'''
							_UpperCamelCase      :	Any				       =     language
							_UpperCamelCase      :	int				       =     train_language





	def 			UpperCamelCase_						(				self      :       Optional[int]				,lowerCamelCase__      :       Dict				):







							'''simple docstring'''
							_UpperCamelCase      :	Tuple				       =     self.language if self.train_language is None else self.train_language
							_UpperCamelCase      :	List[Any]				       =     self._read_tsv(os.path.join(lowerCamelCase__				,F'XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv'				)				)
							_UpperCamelCase      :	Union[str, Any]				       =     []
							for i, line in enumerate(lowerCamelCase__				):
													if i == 0:
																			continue
													_UpperCamelCase      :	Optional[Any]				       =     F'train-{i}'
													_UpperCamelCase      :	List[str]				       =     line[0]
													_UpperCamelCase      :	Union[str, Any]				       =     line[1]
													_UpperCamelCase      :	Tuple				       =     'contradiction' if line[2] == 'contradictory' else line[2]
													if not isinstance(lowerCamelCase__				,lowerCamelCase__				):
																			raise ValueError(F'Training input {text_a} is not a string'				)
													if not isinstance(lowerCamelCase__				,lowerCamelCase__				):
																			raise ValueError(F'Training input {text_b} is not a string'				)
													if not isinstance(lowerCamelCase__				,lowerCamelCase__				):
																			raise ValueError(F'Training label {label} is not a string'				)
													examples.append(InputExample(guid=lowerCamelCase__				,text_a=lowerCamelCase__				,text_b=lowerCamelCase__				,label=lowerCamelCase__				)				)
							return examples





	def 			UpperCamelCase_						(				self      :       Optional[Any]				,lowerCamelCase__      :       List[str]				):







							'''simple docstring'''
							_UpperCamelCase      :	Optional[int]				       =     self._read_tsv(os.path.join(lowerCamelCase__				,'XNLI-1.0/xnli.test.tsv'				)				)
							_UpperCamelCase      :	List[str]				       =     []
							for i, line in enumerate(lowerCamelCase__				):
													if i == 0:
																			continue
													_UpperCamelCase      :	Optional[int]				       =     line[0]
													if language != self.language:
																			continue
													_UpperCamelCase      :	Optional[int]				       =     F'test-{i}'
													_UpperCamelCase      :	Optional[int]				       =     line[6]
													_UpperCamelCase      :	List[str]				       =     line[7]
													_UpperCamelCase      :	str				       =     line[1]
													if not isinstance(lowerCamelCase__				,lowerCamelCase__				):
																			raise ValueError(F'Training input {text_a} is not a string'				)
													if not isinstance(lowerCamelCase__				,lowerCamelCase__				):
																			raise ValueError(F'Training input {text_b} is not a string'				)
													if not isinstance(lowerCamelCase__				,lowerCamelCase__				):
																			raise ValueError(F'Training label {label} is not a string'				)
													examples.append(InputExample(guid=lowerCamelCase__				,text_a=lowerCamelCase__				,text_b=lowerCamelCase__				,label=lowerCamelCase__				)				)
							return examples





	def 			UpperCamelCase_						(				self      :       str				):







							'''simple docstring'''
							return ["contradiction", "entailment", "neutral"]
snake_case_				:			int						 =						{
    'xnli': XnliProcessor,
}

snake_case_				:			Tuple						 =						{
    'xnli': 'classification',
}

snake_case_				:			Any						 =						{
    'xnli': 3,
}


