






import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class      UpperCamelCase_							(				snake_case__ ,       unittest.TestCase       ):
						lowercase									=  RealmTokenizer
						lowercase									=  RealmTokenizerFast
						lowercase									=  True
						lowercase									=  True
						lowercase									=  filter_non_english




						def      snake_case__(    self       )			->						Any:
													super().setUp()

													_a  :    Tuple			=						[
													    '''[UNK]''',
													    '''[CLS]''',
													    '''[SEP]''',
													    '''[PAD]''',
													    '''[MASK]''',
													    '''want''',
													    '''##want''',
													    '''##ed''',
													    '''wa''',
													    '''un''',
													    '''runn''',
													    '''##ing''',
													    ''',''',
													    '''low''',
													    '''lowest''',
													]
													_a  :    Optional[int]			=						os.path.join(self.tmpdirname		,	VOCAB_FILES_NAMES['''vocab_file''']       )
													with open(self.vocab_file		,	'''w'''		,	encoding='''utf-8'''       ) as vocab_writer:
																				vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )




						def      snake_case__(    self		,	lowercase       )			->						Optional[Any]:
													_a  :    Tuple			=						'''UNwant\u00E9d,running'''
													_a  :    List[str]			=						'''unwanted, running'''
													return input_text, output_text




						def      snake_case__(    self       )			->						Tuple:
													_a  :    Tuple			=						self.tokenizer_class(self.vocab_file       )

													_a  :    str			=						tokenizer.tokenize('''UNwant\u00E9d,running'''       )
													self.assertListEqual(lowerCAmelCase_		,	['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']       )
													self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_       )		,	[9, 6, 7, 12, 10, 11]       )




						def      snake_case__(    self       )			->						Optional[int]:
													if not self.test_rust_tokenizer:
																				return

													_a  :    Optional[int]			=						self.get_tokenizer()
													_a  :    List[str]			=						self.get_rust_tokenizer()

													_a  :    Optional[int]			=						'''UNwant\u00E9d,running'''

													_a  :    Dict			=						tokenizer.tokenize(lowerCAmelCase_       )
													_a  :    List[str]			=						rust_tokenizer.tokenize(lowerCAmelCase_       )
													self.assertListEqual(lowerCAmelCase_		,	lowerCAmelCase_       )

													_a  :    Optional[int]			=						tokenizer.encode(lowerCAmelCase_		,	add_special_tokens=lowerCAmelCase_       )
													_a  :    Optional[int]			=						rust_tokenizer.encode(lowerCAmelCase_		,	add_special_tokens=lowerCAmelCase_       )
													self.assertListEqual(lowerCAmelCase_		,	lowerCAmelCase_       )

													_a  :    List[Any]			=						self.get_rust_tokenizer()
													_a  :    int			=						tokenizer.encode(lowerCAmelCase_       )
													_a  :    List[Any]			=						rust_tokenizer.encode(lowerCAmelCase_       )
													self.assertListEqual(lowerCAmelCase_		,	lowerCAmelCase_       )

													# With lower casing
													_a  :    List[Any]			=						self.get_tokenizer(do_lower_case=lowerCAmelCase_       )
													_a  :    Any			=						self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_       )

													_a  :    Dict			=						'''UNwant\u00E9d,running'''

													_a  :    Optional[Any]			=						tokenizer.tokenize(lowerCAmelCase_       )
													_a  :    Dict			=						rust_tokenizer.tokenize(lowerCAmelCase_       )
													self.assertListEqual(lowerCAmelCase_		,	lowerCAmelCase_       )

													_a  :    int			=						tokenizer.encode(lowerCAmelCase_		,	add_special_tokens=lowerCAmelCase_       )
													_a  :    Tuple			=						rust_tokenizer.encode(lowerCAmelCase_		,	add_special_tokens=lowerCAmelCase_       )
													self.assertListEqual(lowerCAmelCase_		,	lowerCAmelCase_       )

													_a  :    Optional[int]			=						self.get_rust_tokenizer()
													_a  :    List[str]			=						tokenizer.encode(lowerCAmelCase_       )
													_a  :    int			=						rust_tokenizer.encode(lowerCAmelCase_       )
													self.assertListEqual(lowerCAmelCase_		,	lowerCAmelCase_       )




						def      snake_case__(    self       )			->						Tuple:
													_a  :    Any			=						BasicTokenizer()

													self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''       )		,	['''ah''', '''\u535A''', '''\u63A8''', '''zz''']       )




						def      snake_case__(    self       )			->						Optional[Any]:
													_a  :    List[str]			=						BasicTokenizer(do_lower_case=lowerCAmelCase_       )

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )		,	['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
													self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )		,	['''hello''']       )




						def      snake_case__(    self       )			->						Union[str, Any]:
													_a  :    List[Any]			=						BasicTokenizer(do_lower_case=lowerCAmelCase_		,	strip_accents=lowerCAmelCase_       )

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )		,	['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
													self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )		,	['''h\u00E9llo''']       )




						def      snake_case__(    self       )			->						List[str]:
													_a  :    str			=						BasicTokenizer(do_lower_case=lowerCAmelCase_		,	strip_accents=lowerCAmelCase_       )

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )		,	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
													self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )		,	['''hello''']       )




						def      snake_case__(    self       )			->						Dict:
													_a  :    Any			=						BasicTokenizer(do_lower_case=lowerCAmelCase_       )

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )		,	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
													self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )		,	['''hello''']       )




						def      snake_case__(    self       )			->						str:
													_a  :    List[str]			=						BasicTokenizer(do_lower_case=lowerCAmelCase_       )

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )		,	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )




						def      snake_case__(    self       )			->						Union[str, Any]:
													_a  :    Tuple			=						BasicTokenizer(do_lower_case=lowerCAmelCase_		,	strip_accents=lowerCAmelCase_       )

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )		,	['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )




						def      snake_case__(    self       )			->						int:
													_a  :    List[Any]			=						BasicTokenizer(do_lower_case=lowerCAmelCase_		,	strip_accents=lowerCAmelCase_       )

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )		,	['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )




						def      snake_case__(    self       )			->						Dict:
													_a  :    Union[str, Any]			=						BasicTokenizer(do_lower_case=lowerCAmelCase_		,	never_split=['''[UNK]''']       )

													self.assertListEqual(
													    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''       )		,	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']       )




						def      snake_case__(    self       )			->						Tuple:
													_a  :    str			=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

													_a  :    Union[str, Any]			=						{}
													for i, token in enumerate(lowerCAmelCase_       ):
																				_a  :    List[Any]			=						i
													_a  :    Dict			=						WordpieceTokenizer(vocab=lowerCAmelCase_		,	unk_token='''[UNK]'''       )

													self.assertListEqual(tokenizer.tokenize(''''''       )		,	[]       )

													self.assertListEqual(tokenizer.tokenize('''unwanted running'''       )		,	['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']       )

													self.assertListEqual(tokenizer.tokenize('''unwantedX running'''       )		,	['''[UNK]''', '''runn''', '''##ing''']       )




						def      snake_case__(    self       )			->						Optional[Any]:
													self.assertTrue(_is_whitespace(''' '''       )       )
													self.assertTrue(_is_whitespace('''\t'''       )       )
													self.assertTrue(_is_whitespace('''\r'''       )       )
													self.assertTrue(_is_whitespace('''\n'''       )       )
													self.assertTrue(_is_whitespace('''\u00A0'''       )       )

													self.assertFalse(_is_whitespace('''A'''       )       )
													self.assertFalse(_is_whitespace('''-'''       )       )




						def      snake_case__(    self       )			->						Optional[int]:
													self.assertTrue(_is_control('''\u0005'''       )       )

													self.assertFalse(_is_control('''A'''       )       )
													self.assertFalse(_is_control(''' '''       )       )
													self.assertFalse(_is_control('''\t'''       )       )
													self.assertFalse(_is_control('''\r'''       )       )




						def      snake_case__(    self       )			->						List[str]:
													self.assertTrue(_is_punctuation('''-'''       )       )
													self.assertTrue(_is_punctuation('''$'''       )       )
													self.assertTrue(_is_punctuation('''`'''       )       )
													self.assertTrue(_is_punctuation('''.'''       )       )

													self.assertFalse(_is_punctuation('''A'''       )       )
													self.assertFalse(_is_punctuation(''' '''       )       )




						def      snake_case__(    self       )			->						str:
													_a  :    Dict			=						self.get_tokenizer()

													# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
													self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_       ) for t in ['''Test''', '''\xad''', '''test''']]		,	[['''[UNK]'''], [], ['''[UNK]''']]       )

													if self.test_rust_tokenizer:
																				_a  :    Tuple			=						self.get_rust_tokenizer()
																				self.assertListEqual(
																				    [rust_tokenizer.tokenize(lowerCAmelCase_       ) for t in ['''Test''', '''\xad''', '''test''']]		,	[['''[UNK]'''], [], ['''[UNK]''']]       )




						@slow
						def      snake_case__(    self       )			->						List[Any]:
													_a  :    Optional[int]			=						self.tokenizer_class.from_pretrained('''bert-base-uncased'''       )

													_a  :    Dict			=						tokenizer.encode('''sequence builders'''		,	add_special_tokens=lowerCAmelCase_       )
													_a  :    Tuple			=						tokenizer.encode('''multi-sequence build'''		,	add_special_tokens=lowerCAmelCase_       )

													_a  :    Dict			=						tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_       )
													_a  :    Union[str, Any]			=						tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_		,	lowerCAmelCase_       )

													assert encoded_sentence == [101] + text + [102]
													assert encoded_pair == [101] + text + [102] + text_a + [102]




						def      snake_case__(    self       )			->						Tuple:
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
																											_a  :    List[Any]			=						self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_		,	**lowerCAmelCase_       )

																											_a  :    Dict			=						F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																											_a  :    Any			=						tokenizer_r.encode_plus(
																											    lowerCAmelCase_		,	return_attention_mask=lowerCAmelCase_		,	return_token_type_ids=lowerCAmelCase_		,	return_offsets_mapping=lowerCAmelCase_		,	add_special_tokens=lowerCAmelCase_		,	)

																											_a  :    List[Any]			=						tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_		,	'''do_lower_case'''       ) else False
																											_a  :    int			=						(
																											    [
																											        ((0, 0), tokenizer_r.cls_token),
																											        ((0, 1), '''A'''),
																											        ((1, 2), ''','''),
																											        ((3, 5), '''na'''),
																											        ((5, 6), '''##ï'''),
																											        ((6, 8), '''##ve'''),
																											        ((9, 15), tokenizer_r.mask_token),
																											        ((16, 21), '''Allen'''),
																											        ((21, 23), '''##NL'''),
																											        ((23, 24), '''##P'''),
																											        ((25, 33), '''sentence'''),
																											        ((33, 34), '''.'''),
																											        ((0, 0), tokenizer_r.sep_token),
																											    ]
																											    if not do_lower_case
																											    else [
																											        ((0, 0), tokenizer_r.cls_token),
																											        ((0, 1), '''a'''),
																											        ((1, 2), ''','''),
																											        ((3, 8), '''naive'''),
																											        ((9, 15), tokenizer_r.mask_token),
																											        ((16, 21), '''allen'''),
																											        ((21, 23), '''##nl'''),
																											        ((23, 24), '''##p'''),
																											        ((25, 33), '''sentence'''),
																											        ((33, 34), '''.'''),
																											        ((0, 0), tokenizer_r.sep_token),
																											    ]
																											)

																											self.assertEqual(
																											    [e[1] for e in expected_results]		,	tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']       )       )
																											self.assertEqual([e[0] for e in expected_results]		,	tokens['''offset_mapping''']       )







						@slow
						def      snake_case__(    self       )			->						Any:
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																				with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
																											_a  :    Union[str, Any]			=						self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_		,	**lowerCAmelCase_       )
																											_a  :    List[Any]			=						self.tokenizer_class.from_pretrained(lowerCAmelCase_		,	**lowerCAmelCase_       )
																											_a  :    Union[str, Any]			=						[['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

																											_a  :    List[Any]			=						tokenizer_r.batch_encode_candidates(lowerCAmelCase_		,	max_length=10		,	return_tensors='''np'''       )
																											_a  :    Tuple			=						tokenizer_p.batch_encode_candidates(lowerCAmelCase_		,	max_length=10		,	return_tensors='''np'''       )

																											_a  :    Dict			=						(2, 2, 10)

																											self.assertEqual(encoded_sentence_r['''input_ids'''].shape		,	lowerCAmelCase_       )
																											self.assertEqual(encoded_sentence_r['''attention_mask'''].shape		,	lowerCAmelCase_       )
																											self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape		,	lowerCAmelCase_       )

																											self.assertEqual(encoded_sentence_p['''input_ids'''].shape		,	lowerCAmelCase_       )
																											self.assertEqual(encoded_sentence_p['''attention_mask'''].shape		,	lowerCAmelCase_       )
																											self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape		,	lowerCAmelCase_       )
import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata







def 						UpperCAmelCase							(		lowerCAmelCase_							)						->						str:
     a	:     Optional[Any]						      =    min(re.search('''^ +'''    ,   lowerCAmelCase_							).end() if t.startswith(''' '''							) else 0 for t in string.splitlines()							)
     return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(lowerCAmelCase_							)]							)


UpperCAmelCase				 =   """\
---
language:
- zh
- en
task_ids:
- sentiment-classification
---
# Begin of markdown

Some cool dataset card
"""

UpperCAmelCase				 =   """\
---
---
# Begin of markdown

Some cool dataset card
"""


UpperCAmelCase				 =   """\
# Begin of markdown

Some cool dataset card
"""






class 				_SCREAMING_SNAKE_CASE( unittest.TestCase						):
   def _lowercase(     self     )						->List[str]:
        with tempfile.TemporaryDirectory() as tmp_dir:
             a	:     List[str]						      =    Path(lowerCAmelCase_     ) / '''README.md'''
             with open(lowerCAmelCase_				,  '''w+'''     ) as readme_file:
                  readme_file.write(lowerCAmelCase_     )
             a	:     Dict						      =    DatasetMetadata.from_readme(lowerCAmelCase_     )
             self.assertDictEqual(lowerCAmelCase_				,  {'''language''': ['''zh''', '''en'''], '''task_ids''': ['''sentiment-classification''']}     )

             with open(lowerCAmelCase_				,  '''w+'''     ) as readme_file:
                  readme_file.write(lowerCAmelCase_     )
             a	:     List[str]						      =    DatasetMetadata.from_readme(lowerCAmelCase_     )
             self.assertDictEqual(lowerCAmelCase_				,  {}     )

             with open(lowerCAmelCase_				,  '''w+'''     ) as readme_file:
                  readme_file.write(lowerCAmelCase_     )
             a	:     List[Any]						      =    DatasetMetadata.from_readme(lowerCAmelCase_     )
             self.assertEqual(lowerCAmelCase_				,  {}     )


   def _lowercase(     self     )						->Optional[Any]:
        a	:     Tuple						      =    _dedent(
            '''\
            annotations_creators:
            - found
            language_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - question-answering
            task_ids:
            - open-domain-qa
            '''     )
        assert DatasetMetadata.from_yaml_string(lowerCAmelCase_     )

        a	:     Dict						      =    _dedent(
            '''\
            annotations_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - question-answering
            task_ids:
            - open-domain-qa
            task_ids:
            - open-domain-qa
            '''     )
        with self.assertRaises(lowerCAmelCase_     ):
             DatasetMetadata.from_yaml_string(lowerCAmelCase_     )

        a	:     Optional[int]						      =    _dedent(
            '''\
            annotations_creators:
            - found
            language_creators:
            - found
            language:
            - en
            license:
            - unknown
            multilinguality:
            - monolingual
            pretty_name: Test Dataset
            size_categories:
            - 10K<n<100K
            source_datasets:
            - extended|other-yahoo-webscope-l6
            task_categories:
            - text-classification
            task_ids:
            - multi-class-classification
            paperswithcode_id:
            - squad
            configs:
            - en
            train-eval-index:
            - config: en
              task: text-classification
              task_id: multi_class_classification
              splits:
                train_split: train
                eval_split: test
              col_mapping:
                text: text
                label: target
              metrics:
                - type: accuracy
                  name: Accuracy
            extra_gated_prompt: |
              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:
              [RESEARCHER_FULLNAME] (the "Researcher") has requested permission to use the ImageNet database (the "Database") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:
              1. Researcher shall use the Database only for non-commercial research and educational purposes.
            extra_gated_fields:
              Company: text
              Country: text
              I agree to use this model for non-commerical use ONLY: checkbox
            '''     )
        assert DatasetMetadata.from_yaml_string(lowerCAmelCase_     )