



"""simple docstring"""






import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase				:     List[str]								=  logging.get_logger(__name__)

lowerCamelCase				:     Optional[Any]								=  {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowerCamelCase				:     str								=  {
    'vocab_file': {
        'microsoft/biogpt': 'https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json',
    },
    'merges_file': {'microsoft/biogpt': 'https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt'},
}

lowerCamelCase				:     Tuple								=  {
    'microsoft/biogpt': 1_0_2_4,
}


def 							A__ (     UpperCamelCase__    ):


  '''simple docstring'''


  _SCREAMING_SNAKE_CASE        =			set()
  _SCREAMING_SNAKE_CASE        =			word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)    )
    _SCREAMING_SNAKE_CASE        =			char
  return pairs







class    __snake_case(					__A       ):
    _A       = VOCAB_FILES_NAMES
    _A       = PRETRAINED_VOCAB_FILES_MAP
    _A       = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _A       = ['''input_ids''', '''attention_mask''']




    def __init__(   self    ,							A_    ,							A_    ,							A_="<unk>"    ,							A_="<s>"    ,							A_="</s>"    ,							A_="</s>"    ,							A_="<pad>"    ,							**A_    ,							):





      '''simple docstring'''



      super().__init__(
          bos_token=lowercase__    ,							eos_token=lowercase__    ,							sep_token=lowercase__    ,							unk_token=lowercase__    ,							pad_token=lowercase__    ,							**lowercase__    ,							)

      try:
        import sacremoses
      except ImportError:
        raise ImportError(
            '''You need to install sacremoses to use BioGptTokenizer. '''
            '''See https://pypi.org/project/sacremoses/ for installation.'''			)

      _SCREAMING_SNAKE_CASE        =			'''en'''
      _SCREAMING_SNAKE_CASE        =			sacremoses
      # cache of sm.MosesTokenizer instance
      _SCREAMING_SNAKE_CASE        =			{}
      _SCREAMING_SNAKE_CASE        =			{}
      with open(lowercase__    ,							encoding='''utf-8'''			) as vocab_handle:
        _SCREAMING_SNAKE_CASE        =			json.load(lowercase__			)
      _SCREAMING_SNAKE_CASE        =			{v: k for k, v in self.encoder.items()}
      with open(lowercase__    ,							encoding='''utf-8'''			) as merges_handle:
        _SCREAMING_SNAKE_CASE        =			merges_handle.read().split('''\n'''			)[:-1]
      _SCREAMING_SNAKE_CASE        =			[tuple(merge.split()[:2]			) for merge in merges]
      _SCREAMING_SNAKE_CASE        =			dict(zip(lowercase__    ,							range(len(lowercase__			)			)			)			)
      _SCREAMING_SNAKE_CASE        =			{}




    @property
    def   A   (   self			):





      '''simple docstring'''



      return len(self.encoder			)




    def   A   (   self			):





      '''simple docstring'''



      return dict(self.encoder    ,							**self.added_tokens_encoder			)




    def   A   (   self    ,							A_    ,							A_			):





      '''simple docstring'''



      if lang not in self.cache_moses_tokenizer:
        _SCREAMING_SNAKE_CASE        =			self.sm.MosesTokenizer(lang=lowercase__			)
        _SCREAMING_SNAKE_CASE        =			moses_tokenizer
      return self.cache_moses_tokenizer[lang].tokenize(
          lowercase__    ,							aggressive_dash_splits=lowercase__    ,							return_str=lowercase__    ,							escape=lowercase__			)




    def   A   (   self    ,							A_    ,							A_			):





      '''simple docstring'''



      if lang not in self.cache_moses_detokenizer:
        _SCREAMING_SNAKE_CASE        =			self.sm.MosesDetokenizer(lang=lowercase__			)
        _SCREAMING_SNAKE_CASE        =			moses_detokenizer
      return self.cache_moses_detokenizer[lang].detokenize(lowercase__			)




    def   A   (   self    ,							A_			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			tuple(token[:-1]			) + (token[-1] + '''</w>''',)
      if token in self.cache:
        return self.cache[token]
      _SCREAMING_SNAKE_CASE        =			get_pairs(lowercase__			)

      if not pairs:
        return token + "</w>"

      while True:
        _SCREAMING_SNAKE_CASE        =			min(lowercase__    ,							key=lambda A_			: self.bpe_ranks.get(lowercase__    ,							float('''inf'''			)			)			)
        if bigram not in self.bpe_ranks:
          break
        _SCREAMING_SNAKE_CASE    ,     _SCREAMING_SNAKE_CASE        =			bigram
        _SCREAMING_SNAKE_CASE        =			[]
        _SCREAMING_SNAKE_CASE        =			0
        while i < len(lowercase__			):
          try:
            _SCREAMING_SNAKE_CASE        =			word.index(lowercase__    ,							lowercase__			)
          except ValueError:
            new_word.extend(word[i:]			)
            break
          else:
            new_word.extend(word[i:j]			)
            _SCREAMING_SNAKE_CASE        =			j

          if word[i] == first and i < len(lowercase__			) - 1 and word[i + 1] == second:
            new_word.append(first + second			)
            i += 2
          else:
            new_word.append(word[i]			)
            i += 1
        _SCREAMING_SNAKE_CASE        =			tuple(lowercase__			)
        _SCREAMING_SNAKE_CASE        =			new_word
        if len(lowercase__			) == 1:
          break
        else:
          _SCREAMING_SNAKE_CASE        =			get_pairs(lowercase__			)
      _SCREAMING_SNAKE_CASE        =			''' '''.join(lowercase__			)
      if word == "\n  </w>":
        _SCREAMING_SNAKE_CASE        =			'''\n</w>'''
      _SCREAMING_SNAKE_CASE        =			word
      return word




    def   A   (   self    ,							A_    ,							A_=False			):





      '''simple docstring'''



      if bypass_tokenizer:
        _SCREAMING_SNAKE_CASE        =			text.split()
      else:
        _SCREAMING_SNAKE_CASE        =			self.moses_tokenize(lowercase__    ,							self.lang			)

      _SCREAMING_SNAKE_CASE        =			[]
      for token in text:
        if token:
          split_tokens.extend(list(self.bpe(lowercase__			).split(''' '''			)			)			)

      return split_tokens




    def   A   (   self    ,							A_			):





      '''simple docstring'''



      return self.encoder.get(lowercase__    ,							self.encoder.get(self.unk_token			)			)




    def   A   (   self    ,							A_			):





      '''simple docstring'''



      return self.decoder.get(lowercase__    ,							self.unk_token			)




    def   A   (   self    ,							A_			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			[t.replace(''' '''    ,							''''''			).replace('''</w>'''    ,							''' '''			) for t in tokens]
      _SCREAMING_SNAKE_CASE        =			''''''.join(lowercase__			).split()
      # detokenize
      _SCREAMING_SNAKE_CASE        =			self.moses_detokenize(lowercase__    ,							self.lang			)
      return text




    def   A   (   self    ,							A_    ,							A_ = None			):





      '''simple docstring'''



      if token_ids_a is None:
        return [self.sep_token_id] + token_ids_a
      _SCREAMING_SNAKE_CASE        =			[self.sep_token_id]
      return sep + token_ids_a + sep + token_ids_a




    def   A   (   self    ,							A_    ,							A_ = None    ,							A_ = False			):





      '''simple docstring'''



      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=lowercase__    ,							token_ids_a=lowercase__    ,							already_has_special_tokens=lowercase__			)
      # no bos used in fairseq
      if token_ids_a is not None:
        return [1] + ([0] * len(lowercase__			)) + [1] + ([0] * len(lowercase__			))
      return [1] + ([0] * len(lowercase__			))




    def   A   (   self    ,							A_    ,							A_ = None			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			[self.sep_token_id]

      # no bos used in fairseq
      if token_ids_a is None:
        return len(token_ids_a + sep			) * [0]
      return len(token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]




    def   A   (   self    ,							A_    ,							A_ = None			):





      '''simple docstring'''



      if not os.path.isdir(lowercase__			):
        logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''			)
        return
      _SCREAMING_SNAKE_CASE        =			os.path.join(
          lowercase__    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)
      _SCREAMING_SNAKE_CASE        =			os.path.join(
          lowercase__    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']			)

      with open(lowercase__    ,							'''w'''    ,							encoding='''utf-8'''			) as f:
        f.write(json.dumps(self.encoder    ,							indent=2    ,							sort_keys=lowercase__    ,							ensure_ascii=lowercase__			) + '''\n'''			)

      _SCREAMING_SNAKE_CASE        =			0
      with open(lowercase__    ,							'''w'''    ,							encoding='''utf-8'''			) as writer:
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,							key=lambda A_			: kv[1]			):
          if index != token_index:
            logger.warning(
                F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                ''' Please check that the tokenizer is not corrupted!'''			)
            _SCREAMING_SNAKE_CASE        =			token_index
          writer.write(''' '''.join(lowercase__			) + '''\n'''			)
          index += 1

      return vocab_file, merge_file




    def __getstate__(   self			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			self.__dict__.copy()
      _SCREAMING_SNAKE_CASE        =			None
      return state




    def __setstate__(   self    ,							A_			):





      '''simple docstring'''



      _SCREAMING_SNAKE_CASE        =			d

      try:
        import sacremoses
      except ImportError:
        raise ImportError(
            '''You need to install sacremoses to use XLMTokenizer. '''
            '''See https://pypi.org/project/sacremoses/ for installation.'''			)

      _SCREAMING_SNAKE_CASE        =			sacremoses








'''simple docstring'''







import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode


def     _UpperCamelCase				(  snake_case__				,      **snake_case__				):
      return RunBeamCommand(
          args.dataset				,      args.name				,      args.cache_dir				,      args.beam_pipeline_options				,      args.data_dir				,      args.all_configs				,      args.save_info or args.save_infos				,      args.ignore_verifications				,      args.force_redownload				,      **snake_case__				,      )






class        a__     (      lowerCamelCase    ):
     '''simple docstring'''







     @staticmethod
     def      _UpperCamelCase (lowercase__     :						ArgumentParser       )	->							List[Any]:
           __A		 =   parser.add_parser("run_beam"					,					help="Run a Beam dataset processing pipeline"       )
           run_beam_parser.add_argument("dataset"					,					type=lowercase__					,					help="Name of the dataset to download"       )
           run_beam_parser.add_argument("--name"					,					type=lowercase__					,					default=lowercase__					,					help="Dataset config name"       )
           run_beam_parser.add_argument(
               "--cache_dir"					,					type=lowercase__					,					default=lowercase__					,					help="Cache directory where the datasets are stored"					,					)
           run_beam_parser.add_argument(
               "--beam_pipeline_options"					,					type=lowercase__					,					default=""					,					help="Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`"					,					)
           run_beam_parser.add_argument(
               "--data_dir"					,					type=lowercase__					,					default=lowercase__					,					help="Can be used to specify a manual directory to get the files from"					,					)
           run_beam_parser.add_argument("--all_configs"					,					action="store_true"					,					help="Test all dataset configurations"       )
           run_beam_parser.add_argument("--save_info"					,					action="store_true"					,					help="Save the dataset infos file"       )
           run_beam_parser.add_argument(
               "--ignore_verifications"					,					action="store_true"					,					help="Run the test without checksums and splits checks"       )
           run_beam_parser.add_argument("--force_redownload"					,					action="store_true"					,					help="Force dataset redownload"       )
           # aliases
           run_beam_parser.add_argument("--save_infos"					,					action="store_true"					,					help="alias for save_info"       )
           run_beam_parser.set_defaults(func=lowercase__       )






     def __init__(self     :						Any					,					lowercase__     :						str					,					lowercase__     :						str					,					lowercase__     :						str					,					lowercase__     :						str					,					lowercase__     :						str					,					lowercase__     :						bool					,					lowercase__     :						bool					,					lowercase__     :						bool					,					lowercase__     :						bool					,					**lowercase__     :						Tuple					,					)	->							List[Any]:
           __A		 =   dataset
           __A		 =   name
           __A		 =   cache_dir
           __A		 =   beam_pipeline_options
           __A		 =   data_dir
           __A		 =   all_configs
           __A		 =   save_infos
           __A		 =   ignore_verifications
           __A		 =   force_redownload
           __A		 =   config_kwargs






     def      _UpperCamelCase (self     :						int       )	->							Dict:
           import apache_beam as beam

           if self._name is not None and self._all_configs:
                 print("Both parameters `name` and `all_configs` can't be used at once."       )
                 exit(1       )
           __A ,					__A		 =   self._dataset, self._name
           __A		 =   dataset_module_factory(lowercase__       )
           __A		 =   import_main_class(dataset_module.module_path       )
           __A		 =   []
           if self._beam_pipeline_options:
                 __A		 =   beam.options.pipeline_options.PipelineOptions(
                     flags=[f"--{opt.strip()}" for opt in self._beam_pipeline_options.split(","       ) if opt]       )
           else:
                 __A		 =   None
           if self._all_configs and len(builder_cls.BUILDER_CONFIGS       ) > 0:
                 for builder_config in builder_cls.BUILDER_CONFIGS:
                       builders.append(
                           builder_cls(
                               config_name=builder_config.name					,					data_dir=self._data_dir					,					hash=dataset_module.hash					,					beam_options=lowercase__					,					cache_dir=self._cache_dir					,					base_path=dataset_module.builder_kwargs.get("base_path"       )					,					)       )
           else:
                 builders.append(
                     builder_cls(
                         config_name=lowercase__					,					data_dir=self._data_dir					,					beam_options=lowercase__					,					cache_dir=self._cache_dir					,					base_path=dataset_module.builder_kwargs.get("base_path"       )					,					**self._config_kwargs					,					)       )

           for builder in builders:
                 builder.download_and_prepare(
                     download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
                     if not self._force_redownload
                     else DownloadMode.FORCE_REDOWNLOAD					,					download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH       )					,					verification_mode=VerificationMode.NO_CHECKS
                     if self._ignore_verifications
                     else VerificationMode.ALL_CHECKS					,					try_from_hf_gcs=lowercase__					,					)
                 if self._save_infos:
                       builder._save_infos()

           print("Apache beam run successful."       )

           # If save_infos=True, the dataset infos file is created next to the loaded module file.
           # Let's move it to the original directory of the dataset script, to allow the user to
           # upload them on S3 at the same time afterwards.
           if self._save_infos:
                 __A		 =   os.path.join(builder_cls.get_imported_module_dir()					,					config.DATASETDICT_INFOS_FILENAME       )

                 __A		 =   Path(lowercase__       ).name + ".py"

                 __A		 =   os.path.join(lowercase__					,					lowercase__       )
                 if os.path.isfile(lowercase__       ):
                       __A		 =   os.path.dirname(lowercase__       )
                 elif os.path.isfile(lowercase__       ):
                       __A		 =   path
                 else:  # in case of a remote dataset
                       print(f"Dataset Infos file saved at {dataset_infos_path}"       )
                       exit(1       )

                 # Move datasetinfo back to the user
                 __A		 =   os.path.join(lowercase__					,					config.DATASETDICT_INFOS_FILENAME       )
                 copyfile(lowercase__					,					lowercase__       )
                 print(f"Dataset Infos file saved at {user_dataset_infos_path}"       )


