




import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 						SCREAMING_SNAKE_CASE__		(_a						,				unittest.TestCase ):
     lowercase_   :			Dict   =   ElectraTokenizer
     lowercase_   :			str   =   ElectraTokenizerFast
     lowercase_   :			Any   =   True
     lowercase_   :			Any   =   True
     lowercase_   :			Optional[Any]   =   filter_non_english
     def 					A__			(	self  :		int	):



       """simple docstring"""
       super().setUp()

       lowerCAmelCase__								=			[
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       lowerCAmelCase__								=			os.path.join(self.tmpdirname					,	VOCAB_FILES_NAMES['''vocab_file''']	)
       with open(self.vocab_file					,	'''w'''					,	encoding='''utf-8'''	) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)
     def 					A__			(	self  :		Tuple					,	__lowerCamelCase  :		List[Any]	):



       """simple docstring"""
       lowerCAmelCase__								=			'''UNwant\u00E9d,running'''
       lowerCAmelCase__								=			'''unwanted, running'''
       return input_text, output_text
     def 					A__			(	self  :		Dict	):



       """simple docstring"""
       lowerCAmelCase__								=			self.tokenizer_class(self.vocab_file	)

       lowerCAmelCase__								=			tokenizer.tokenize('''UNwant\u00E9d,running'''	)
       self.assertListEqual(__lowerCamelCase					,	['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']	)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase	)					,	[9, 6, 7, 12, 10, 11]	)
     def 					A__			(	self  :		Optional[Any]	):



       """simple docstring"""
       if not self.test_rust_tokenizer:
         return

       lowerCAmelCase__								=			self.get_tokenizer()
       lowerCAmelCase__								=			self.get_rust_tokenizer()

       lowerCAmelCase__								=			'''UNwant\u00E9d,running'''

       lowerCAmelCase__								=			tokenizer.tokenize(__lowerCamelCase	)
       lowerCAmelCase__								=			rust_tokenizer.tokenize(__lowerCamelCase	)
       self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

       lowerCAmelCase__								=			tokenizer.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
       lowerCAmelCase__								=			rust_tokenizer.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
       self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

       lowerCAmelCase__								=			self.get_rust_tokenizer()
       lowerCAmelCase__								=			tokenizer.encode(__lowerCamelCase	)
       lowerCAmelCase__								=			rust_tokenizer.encode(__lowerCamelCase	)
       self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

       # With lower casing
       lowerCAmelCase__								=			self.get_tokenizer(do_lower_case=__lowerCamelCase	)
       lowerCAmelCase__								=			self.get_rust_tokenizer(do_lower_case=__lowerCamelCase	)

       lowerCAmelCase__								=			'''UNwant\u00E9d,running'''

       lowerCAmelCase__								=			tokenizer.tokenize(__lowerCamelCase	)
       lowerCAmelCase__								=			rust_tokenizer.tokenize(__lowerCamelCase	)
       self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

       lowerCAmelCase__								=			tokenizer.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
       lowerCAmelCase__								=			rust_tokenizer.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
       self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

       lowerCAmelCase__								=			self.get_rust_tokenizer()
       lowerCAmelCase__								=			tokenizer.encode(__lowerCamelCase	)
       lowerCAmelCase__								=			rust_tokenizer.encode(__lowerCamelCase	)
       self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)
     def 					A__			(	self  :		Any	):



       """simple docstring"""
       lowerCAmelCase__								=			BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''	)					,	['''ah''', '''\u535A''', '''\u63A8''', '''zz''']	)
     def 					A__			(	self  :		List[Any]	):



       """simple docstring"""
       lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)					,	['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)					,	['''hello''']	)
     def 					A__			(	self  :		Union[str, Any]	):



       """simple docstring"""
       lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase					,	strip_accents=__lowerCamelCase	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)					,	['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)					,	['''h\u00E9llo''']	)
     def 					A__			(	self  :		int	):



       """simple docstring"""
       lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase					,	strip_accents=__lowerCamelCase	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)					,	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)					,	['''hello''']	)
     def 					A__			(	self  :		int	):



       """simple docstring"""
       lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)					,	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)					,	['''hello''']	)
     def 					A__			(	self  :		Dict	):



       """simple docstring"""
       lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)					,	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)
     def 					A__			(	self  :		Optional[int]	):



       """simple docstring"""
       lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase					,	strip_accents=__lowerCamelCase	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)					,	['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)
     def 					A__			(	self  :		Union[str, Any]	):



       """simple docstring"""
       lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase					,	strip_accents=__lowerCamelCase	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)					,	['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)
     def 					A__			(	self  :		List[str]	):



       """simple docstring"""
       lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase					,	never_split=['''[UNK]''']	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''	)					,	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']	)
     def 					A__			(	self  :		List[str]	):



       """simple docstring"""
       lowerCAmelCase__								=			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

       lowerCAmelCase__								=			{}
       for i, token in enumerate(__lowerCamelCase	):
         lowerCAmelCase__								=			i
       lowerCAmelCase__								=			WordpieceTokenizer(vocab=__lowerCamelCase					,	unk_token='''[UNK]'''	)

       self.assertListEqual(tokenizer.tokenize(''''''	)					,	[]	)

       self.assertListEqual(tokenizer.tokenize('''unwanted running'''	)					,	['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']	)

       self.assertListEqual(tokenizer.tokenize('''unwantedX running'''	)					,	['''[UNK]''', '''runn''', '''##ing''']	)
     def 					A__			(	self  :		Union[str, Any]	):



       """simple docstring"""
       self.assertTrue(_is_whitespace(''' '''	)	)
       self.assertTrue(_is_whitespace('''\t'''	)	)
       self.assertTrue(_is_whitespace('''\r'''	)	)
       self.assertTrue(_is_whitespace('''\n'''	)	)
       self.assertTrue(_is_whitespace('''\u00A0'''	)	)

       self.assertFalse(_is_whitespace('''A'''	)	)
       self.assertFalse(_is_whitespace('''-'''	)	)
     def 					A__			(	self  :		Dict	):



       """simple docstring"""
       self.assertTrue(_is_control('''\u0005'''	)	)

       self.assertFalse(_is_control('''A'''	)	)
       self.assertFalse(_is_control(''' '''	)	)
       self.assertFalse(_is_control('''\t'''	)	)
       self.assertFalse(_is_control('''\r'''	)	)
     def 					A__			(	self  :		Dict	):



       """simple docstring"""
       self.assertTrue(_is_punctuation('''-'''	)	)
       self.assertTrue(_is_punctuation('''$'''	)	)
       self.assertTrue(_is_punctuation('''`'''	)	)
       self.assertTrue(_is_punctuation('''.'''	)	)

       self.assertFalse(_is_punctuation('''A'''	)	)
       self.assertFalse(_is_punctuation(''' '''	)	)
     def 					A__			(	self  :		Optional[Any]	):



       """simple docstring"""
       lowerCAmelCase__								=			self.get_tokenizer()
       lowerCAmelCase__								=			self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(__lowerCamelCase	) for t in ['''Test''', '''\xad''', '''test''']]					,	[['''[UNK]'''], [], ['''[UNK]''']]	)

       self.assertListEqual(
           [rust_tokenizer.tokenize(__lowerCamelCase	) for t in ['''Test''', '''\xad''', '''test''']]					,	[['''[UNK]'''], [], ['''[UNK]''']]	)
     @slow
     def 					A__			(	self  :		Dict	):



       """simple docstring"""
       lowerCAmelCase__								=			self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''	)

       lowerCAmelCase__								=			tokenizer.encode('''sequence builders'''					,	add_special_tokens=__lowerCamelCase	)
       lowerCAmelCase__								=			tokenizer.encode('''multi-sequence build'''					,	add_special_tokens=__lowerCamelCase	)

       lowerCAmelCase__								=			tokenizer.build_inputs_with_special_tokens(__lowerCamelCase	)
       lowerCAmelCase__								=			tokenizer.build_inputs_with_special_tokens(__lowerCamelCase					,	__lowerCamelCase	)

       assert encoded_sentence == [1_01] + text + [1_02]
       assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
     def 					A__			(	self  :		str	):



       """simple docstring"""
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
           lowerCAmelCase__								=			self.rust_tokenizer_class.from_pretrained(__lowerCamelCase					,	**__lowerCamelCase	)

           lowerCAmelCase__								=			F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
           lowerCAmelCase__								=			tokenizer_r.encode_plus(
               __lowerCamelCase					,	return_attention_mask=__lowerCamelCase					,	return_token_type_ids=__lowerCamelCase					,	return_offsets_mapping=__lowerCamelCase					,	add_special_tokens=__lowerCamelCase					,	)

           lowerCAmelCase__								=			tokenizer_r.do_lower_case if hasattr(__lowerCamelCase					,	'''do_lower_case'''	) else False
           lowerCAmelCase__								=			(
               [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''A'''),
                   ((1, 2), ''','''),
                   ((3, 5), '''na'''),
                   ((5, 6), '''##ï'''),
                   ((6, 8), '''##ve'''),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), '''Allen'''),
                   ((21, 23), '''##NL'''),
                   ((23, 24), '''##P'''),
                   ((25, 33), '''sentence'''),
                   ((33, 34), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
               if not do_lower_case
               else [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''a'''),
                   ((1, 2), ''','''),
                   ((3, 8), '''naive'''),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), '''allen'''),
                   ((21, 23), '''##nl'''),
                   ((23, 24), '''##p'''),
                   ((25, 33), '''sentence'''),
                   ((33, 34), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
           )

           self.assertEqual(
               [e[1] for e in expected_results]					,	tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']	)	)
           self.assertEqual([e[0] for e in expected_results]					,	tokens['''offset_mapping''']	)



     def 					A__			(	self  :		Union[str, Any]	):



       """simple docstring"""
       lowerCAmelCase__								=			['''的''', '''人''', '''有''']
       lowerCAmelCase__								=			''''''.join(__lowerCamelCase	)
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
           lowerCAmelCase__								=			True
           lowerCAmelCase__								=			self.tokenizer_class.from_pretrained(__lowerCamelCase					,	**__lowerCamelCase	)
           lowerCAmelCase__								=			self.rust_tokenizer_class.from_pretrained(__lowerCamelCase					,	**__lowerCamelCase	)

           lowerCAmelCase__								=			tokenizer_p.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
           lowerCAmelCase__								=			tokenizer_r.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)

           lowerCAmelCase__								=			tokenizer_r.convert_ids_to_tokens(__lowerCamelCase	)
           lowerCAmelCase__								=			tokenizer_p.convert_ids_to_tokens(__lowerCamelCase	)

           # it is expected that each Chinese character is not preceded by "##"
           self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)
           self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

           lowerCAmelCase__								=			False
           lowerCAmelCase__								=			self.rust_tokenizer_class.from_pretrained(__lowerCamelCase					,	**__lowerCamelCase	)
           lowerCAmelCase__								=			self.tokenizer_class.from_pretrained(__lowerCamelCase					,	**__lowerCamelCase	)

           lowerCAmelCase__								=			tokenizer_r.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
           lowerCAmelCase__								=			tokenizer_p.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)

           lowerCAmelCase__								=			tokenizer_r.convert_ids_to_tokens(__lowerCamelCase	)
           lowerCAmelCase__								=			tokenizer_p.convert_ids_to_tokens(__lowerCamelCase	)

           # it is expected that only the first Chinese character is not preceded by "##".
           lowerCAmelCase__								=			[
               F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__lowerCamelCase	)
           ]
           self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)
           self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)







import os

from ...utils import logging
from .utils import DataProcessor, InputExample


__magic_name__     :     str  							=					logging.get_logger(__name__)





class 						SCREAMING_SNAKE_CASE__		(_a ):
     def __init__(	self  :		Any					,	__lowerCamelCase  :		Any					,	__lowerCamelCase  :		Optional[int]=None	):



       """simple docstring"""
       lowerCAmelCase__								=			language
       lowerCAmelCase__								=			train_language
     def 					A__			(	self  :		str					,	__lowerCamelCase  :		List[str]	):



       """simple docstring"""
       lowerCAmelCase__								=			self.language if self.train_language is None else self.train_language
       lowerCAmelCase__								=			self._read_tsv(os.path.join(__lowerCamelCase					,	F"""XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv"""	)	)
       lowerCAmelCase__								=			[]
       for i, line in enumerate(__lowerCamelCase	):
         if i == 0:
           continue
         lowerCAmelCase__								=			F"""train-{i}"""
         lowerCAmelCase__								=			line[0]
         lowerCAmelCase__								=			line[1]
         lowerCAmelCase__								=			'''contradiction''' if line[2] == '''contradictory''' else line[2]
         if not isinstance(__lowerCamelCase					,	__lowerCamelCase	):
           raise ValueError(F"""Training input {text_a} is not a string"""	)
         if not isinstance(__lowerCamelCase					,	__lowerCamelCase	):
           raise ValueError(F"""Training input {text_b} is not a string"""	)
         if not isinstance(__lowerCamelCase					,	__lowerCamelCase	):
           raise ValueError(F"""Training label {label} is not a string"""	)
         examples.append(InputExample(guid=__lowerCamelCase					,	text_a=__lowerCamelCase					,	text_b=__lowerCamelCase					,	label=__lowerCamelCase	)	)
       return examples
     def 					A__			(	self  :		Dict					,	__lowerCamelCase  :		Union[str, Any]	):



       """simple docstring"""
       lowerCAmelCase__								=			self._read_tsv(os.path.join(__lowerCamelCase					,	'''XNLI-1.0/xnli.test.tsv'''	)	)
       lowerCAmelCase__								=			[]
       for i, line in enumerate(__lowerCamelCase	):
         if i == 0:
           continue
         lowerCAmelCase__								=			line[0]
         if language != self.language:
           continue
         lowerCAmelCase__								=			F"""test-{i}"""
         lowerCAmelCase__								=			line[6]
         lowerCAmelCase__								=			line[7]
         lowerCAmelCase__								=			line[1]
         if not isinstance(__lowerCamelCase					,	__lowerCamelCase	):
           raise ValueError(F"""Training input {text_a} is not a string"""	)
         if not isinstance(__lowerCamelCase					,	__lowerCamelCase	):
           raise ValueError(F"""Training input {text_b} is not a string"""	)
         if not isinstance(__lowerCamelCase					,	__lowerCamelCase	):
           raise ValueError(F"""Training label {label} is not a string"""	)
         examples.append(InputExample(guid=__lowerCamelCase					,	text_a=__lowerCamelCase					,	text_b=__lowerCamelCase					,	label=__lowerCamelCase	)	)
       return examples



     def 					A__			(	self  :		List[str]	):



       """simple docstring"""
       return ["contradiction", "entailment", "neutral"]



__magic_name__     :     Tuple  							=					{
    """xnli""": XnliProcessor,
}

__magic_name__     :     List[Any]  							=					{
    """xnli""": """classification""",
}

__magic_name__     :     str  							=					{
    """xnli""": 3,
}

