
'''simple docstring'''
import csv
import json
import os

import datasets


# TODO: Add BibTeX citation
# Find for instance the citation on arxiv or on the dataset repo/website
__lowerCAmelCase  							= '\\n@InProceedings{huggingface:dataset,\ntitle = {A great new dataset},\nauthor={huggingface, Inc.\n},\nyear={2020}\n}\n'

# TODO: Add description of the dataset here
# You can copy an official description
__lowerCAmelCase  							= '\\nThis new dataset is designed to solve this great NLP task and is crafted with a lot of care.\n'

# TODO: Add a link to an official homepage for the dataset here
__lowerCAmelCase  							= ''

# TODO: Add the licence for the dataset here if you can find it
__lowerCAmelCase  							= ''

# TODO: Add link to the official dataset URLs here
# The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
__lowerCAmelCase  							= {
    'first_domain': 'https://huggingface.co/great-new-dataset-first_domain.zip',
    'second_domain': 'https://huggingface.co/great-new-dataset-second_domain.zip',
}


class      _lowerCAmelCase					(					datasets.GeneratorBasedBuilder   ):







							'''simple docstring'''

							lowerCAmelCase_      		=		datasets.Version("1.1.0"   )

							# This is an example of a dataset with multiple configurations.
							# If you don't want/need to define several sub-sets in your dataset,
							# just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.

							# If you need to make complex sub-parts in the datasets with configurable options
							# You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
							# BUILDER_CONFIG_CLASS = MyBuilderConfig

							# You will be able to load one or the other configurations in the following list with
							# data = datasets.load_dataset('my_dataset', 'first_domain')
							# data = datasets.load_dataset('my_dataset', 'second_domain')
							lowerCAmelCase_      		=		[
							    datasets.BuilderConfig(name="first_domain"  ,							version=__snake_case  ,							description="This part of my dataset covers a first domain"   ),
							    datasets.BuilderConfig(name="second_domain"  ,							version=__snake_case  ,							description="This part of my dataset covers a second domain"   ),
							]

							lowerCAmelCase_      		=		"first_domain"  # It's not mandatory to have a default configuration. Just use one if it make sense.

							def  lowercase							(self						)  ->     int:
														# TODO: This method specifies the datasets.DatasetInfo object which contains informations and typings for the dataset
														if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
																					_snake_case     =						datasets.Features(
																					    {
																					        """sentence""": datasets.Value("""string"""						),
																					        """option1""": datasets.Value("""string"""						),
																					        """answer""": datasets.Value("""string"""						)
																					        # These are the features of your dataset like images, labels ...
																					    }						)
														else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
																					_snake_case     =						datasets.Features(
																					    {
																					        """sentence""": datasets.Value("""string"""						),
																					        """option2""": datasets.Value("""string"""						),
																					        """second_domain_answer""": datasets.Value("""string"""						)
																					        # These are the features of your dataset like images, labels ...
																					    }						)
														return datasets.DatasetInfo(
														    # This is the description that will appear on the datasets page.
														    description=_DESCRIPTION						,						features=UpperCAmelCase						,						homepage=_HOMEPAGE						,						license=_LICENSE						,						citation=_CITATION						,						)

							def  lowercase							(self						,						UpperCAmelCase						)  ->     List[str]:
														# TODO: This method is tasked with downloading/extracting the data and defining the splits depending on the configuration
														# If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name

														# dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS
														# It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.
														# By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive
														_snake_case     =						_URLS[self.config.name]
														_snake_case     =						dl_manager.download_and_extract(UpperCAmelCase						)
														return [
														    datasets.SplitGenerator(
														        name=datasets.Split.TRAIN						,						gen_kwargs={
														            """filepath""": os.path.join(UpperCAmelCase						,						"""train.jsonl"""						),
														            """split""": """train""",
														        }						,						),
														    datasets.SplitGenerator(
														        name=datasets.Split.VALIDATION						,						gen_kwargs={
														            """filepath""": os.path.join(UpperCAmelCase						,						"""dev.jsonl"""						),
														            """split""": """dev""",
														        }						,						),
														    datasets.SplitGenerator(
														        name=datasets.Split.TEST						,						gen_kwargs={
														            """filepath""": os.path.join(UpperCAmelCase						,						"""test.jsonl"""						),
														            """split""": """test"""
														        }						,						),
														]





							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase						)  ->     Any:
														# TODO: This method handles input defined in _split_generators to yield (key, example) tuples from the dataset.
														# The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example.
														with open(UpperCAmelCase						,						encoding="""utf-8"""						) as f:
																					for key, row in enumerate(UpperCAmelCase						):
																												_snake_case     =						json.loads(UpperCAmelCase						)
																												if self.config.name == "first_domain":
																																			# Yields examples as (key, example) tuples
																																			yield key, {
																																			    "sentence": data["sentence"],
																																			    "option1": data["option1"],
																																			    "answer": "" if split == "test" else data["answer"],
																																			}
																												else:
																																			yield key, {
																																			    "sentence": data["sentence"],
																																			    "option2": data["option2"],
																																			    "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
																																			}

'''simple docstring'''
import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
				from .tokenization_deberta_va import DebertaVaTokenizer
else:
				__lowerCAmelCase  							= None

__lowerCAmelCase  							= logging.get_logger(__name__)

__lowerCAmelCase  							= {'vocab_file': 'spm.model', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase  							= {
    'vocab_file': {
        'microsoft/deberta-v2-xlarge': 'https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model',
        'microsoft/deberta-v2-xxlarge': 'https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model',
        'microsoft/deberta-v2-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model'
        ),
        'microsoft/deberta-v2-xxlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model'
        ),
    }
}

__lowerCAmelCase  							= {
    'microsoft/deberta-v2-xlarge': 512,
    'microsoft/deberta-v2-xxlarge': 512,
    'microsoft/deberta-v2-xlarge-mnli': 512,
    'microsoft/deberta-v2-xxlarge-mnli': 512,
}

__lowerCAmelCase  							= {
    'microsoft/deberta-v2-xlarge': {'do_lower_case': False},
    'microsoft/deberta-v2-xxlarge': {'do_lower_case': False},
    'microsoft/deberta-v2-xlarge-mnli': {'do_lower_case': False},
    'microsoft/deberta-v2-xxlarge-mnli': {'do_lower_case': False},
}


class     _lowerCAmelCase					(					__snake_case   ):







							'''simple docstring'''

							lowerCAmelCase_      		=		VOCAB_FILES_NAMES
							lowerCAmelCase_      		=		PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase_      		=		PRETRAINED_INIT_CONFIGURATION
							lowerCAmelCase_      		=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase_      		=		DebertaVaTokenizer

							def __init__(self						,						UpperCAmelCase=None						,						UpperCAmelCase=None						,						UpperCAmelCase=False						,						UpperCAmelCase=False						,						UpperCAmelCase="[CLS]"						,						UpperCAmelCase="[SEP]"						,						UpperCAmelCase="[UNK]"						,						UpperCAmelCase="[SEP]"						,						UpperCAmelCase="[PAD]"						,						UpperCAmelCase="[CLS]"						,						UpperCAmelCase="[MASK]"						,						**UpperCAmelCase						,						)  ->     None:
														super().__init__(
														    UpperCAmelCase						,						tokenizer_file=UpperCAmelCase						,						do_lower_case=UpperCAmelCase						,						bos_token=UpperCAmelCase						,						eos_token=UpperCAmelCase						,						unk_token=UpperCAmelCase						,						sep_token=UpperCAmelCase						,						pad_token=UpperCAmelCase						,						cls_token=UpperCAmelCase						,						mask_token=UpperCAmelCase						,						split_by_punct=UpperCAmelCase						,						**UpperCAmelCase						,						)

														_snake_case     =						do_lower_case
														_snake_case     =						split_by_punct
														_snake_case     =						vocab_file
														_snake_case     =						False if not self.vocab_file else True

							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase=None						)  ->     List[Any]:

														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														_snake_case     =						[self.cls_token_id]
														_snake_case     =						[self.sep_token_id]
														return cls + token_ids_a + sep + token_ids_a + sep

							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase=None						,						UpperCAmelCase=False						)  ->     Union[str, Any]:

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=UpperCAmelCase						,						token_ids_a=UpperCAmelCase						,						already_has_special_tokens=UpperCAmelCase						)

														if token_ids_a is not None:
																					return [1] + ([0] * len(UpperCAmelCase						)) + [1] + ([0] * len(UpperCAmelCase						)) + [1]
														return [1] + ([0] * len(UpperCAmelCase						)) + [1]

							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase=None						)  ->     str:
														_snake_case     =						[self.sep_token_id]
														_snake_case     =						[self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep						) * [0]
														return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]





							def  lowercase							(self						,						UpperCAmelCase						,						UpperCAmelCase = None						)  ->     Tuple[str]:
														if not self.can_save_slow_tokenizer:
																					raise ValueError(
																					    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
																					    """tokenizer."""						)

														if not os.path.isdir(UpperCAmelCase						):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
																					return
														_snake_case     =						os.path.join(
														    UpperCAmelCase						,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

														if os.path.abspath(self.vocab_file						) != os.path.abspath(UpperCAmelCase						):
																					copyfile(self.vocab_file						,						UpperCAmelCase						)

														return (out_vocab_file,)