




import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


lowercase__   :    Union[str, Any] 					=      logging.getLogger(__name__)

def       SCREAMING_SNAKE_CASE							(     )    ->  Tuple:
     a											=   argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset.")
     parser.add_argument(
         "--dataset_name"	,      type=__UpperCamelCase	,      default="wikitext"	,      help="Name of the training. Explore datasets at: hf.co/datasets."	,      )
     parser.add_argument(
         "--dataset_config"	,      type=__UpperCamelCase	,      default="wikitext-103-raw-v1"	,      help="Configuration name of the dataset.")
     parser.add_argument(
         "--batch_size"	,      type=__UpperCamelCase	,      default=10_00	,      help="Batch size during training."	,      )
     parser.add_argument(
         "--vocab_size"	,      type=__UpperCamelCase	,      default=1_00_48	,      help="Size of the desired vocabulary."	,      )
     parser.add_argument(
         "--limit"	,      default=__UpperCamelCase	,      type=__UpperCamelCase	,      help="Limit the number of shards (used for debugging)."	,      )
     parser.add_argument(
         "--export_to_hub"	,      action="store_true"	,      )

     a											=   parser.parse_args()
     return args

def       SCREAMING_SNAKE_CASE							(     __UpperCamelCase)    ->  str:
     a											=   datasets.load_dataset(args.dataset_name	,      args.dataset_config	,      split="train")

     if args.limit is not None:
          a											=   min(len(__UpperCamelCase)	,      args.limit)
          a											=   dataset.select(range(__UpperCamelCase))
          logger.info(f'''Limiting the dataset to {args.limit} entries.''')

     def batch_iterator():
          for i in range(0	,      len(__UpperCamelCase)	,      args.batch_size):
               yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
     a											=   Tokenizer(Unigram())
     a											=   normalizers.Sequence([normalizers.Replace("``"	,      "\""), normalizers.Replace("''"	,      "\"")])
     a											=   pre_tokenizers.Metaspace()

     # Prepare the trainer.
     a											=   UnigramTrainer(
         unk_token="<unk>"	,      special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"]	,      vocab_size=args.vocab_size	,      )

     logger.info("Training the tokenizer.")
     tokenizer.train_from_iterator(batch_iterator()	,      trainer=__UpperCamelCase)
     logger.info("Tokenizer training complete!")

     a											=   tokenizer.token_to_id("[CLS]")
     a											=   tokenizer.token_to_id("[SEP]")
     a											=   processors.TemplateProcessing(
         single="[CLS]:0 $A:0 [SEP]:0"	,      pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"	,      special_tokens=[
             ("[CLS]", cls_token_id),
             ("[SEP]", sep_token_id),
         ]	,      )
     a											=   decoders.Metaspace()

     if args.export_to_hub:
          logger.info("Exporting the trained tokenzier to Hub.")
          a											=   AlbertTokenizerFast(tokenizer_object=__UpperCamelCase)
          new_tokenizer.push_to_hub("unigram-tokenizer-dataset")


if __name__ == "__main__":
   lowercase__   :    Union[str, Any] 					=      parse_args()
   main(args)







import safs




class     a__ (      safs.SaFileSystem   ):
     pass

