


import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


UpperCAmelCase     =       logging.getLogger(__name__)






def 			__lowerCAmelCase       ()->    int:




      """simple docstring"""

      snake_case_			= argparse.ArgumentParser(description='''Train a unigram tokenizer on the wikitext dataset.'''   )
      parser.add_argument(
          '''--dataset_name'''   ,   type=_UpperCamelCase   ,   default='''wikitext'''   ,   help='''Name of the training. Explore datasets at: hf.co/datasets.'''   ,   )
      parser.add_argument(
          '''--dataset_config'''   ,   type=_UpperCamelCase   ,   default='''wikitext-103-raw-v1'''   ,   help='''Configuration name of the dataset.'''   )
      parser.add_argument(
          '''--batch_size'''   ,   type=_UpperCamelCase   ,   default=1000   ,   help='''Batch size during training.'''   ,   )
      parser.add_argument(
          '''--vocab_size'''   ,   type=_UpperCamelCase   ,   default=1_0048   ,   help='''Size of the desired vocabulary.'''   ,   )
      parser.add_argument(
          '''--limit'''   ,   default=_UpperCamelCase   ,   type=_UpperCamelCase   ,   help='''Limit the number of shards (used for debugging).'''   ,   )
      parser.add_argument(
          '''--export_to_hub'''   ,   action='''store_true'''   ,   )

      snake_case_			= parser.parse_args()
      return args






def 			__lowerCAmelCase       (SCREAMING_SNAKE_CASE   )->    str:




      """simple docstring"""

      snake_case_			= datasets.load_dataset(args.dataset_name   ,   args.dataset_config   ,   split='''train'''   )

      if args.limit is not None:
            snake_case_			= min(len(_UpperCamelCase   )   ,   args.limit   )
            snake_case_			= dataset.select(range(_UpperCamelCase   )   )
            logger.info(f'''Limiting the dataset to {args.limit} entries.'''   )

      def batch_iterator():
            for i in range(0   ,   len(_UpperCamelCase   )   ,   args.batch_size   ):
                  yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
      snake_case_			= Tokenizer(Unigram()   )
      snake_case_			= normalizers.Sequence([normalizers.Replace('''``'''   ,   '''"'''   ), normalizers.Replace('''\'\''''   ,   '''"'''   )]   )
      snake_case_			= pre_tokenizers.Metaspace()

      # Prepare the trainer.
      snake_case_			= UnigramTrainer(
          unk_token='''<unk>'''   ,   special_tokens=['''[CLS]''', '''[SEP]''', '''<unk>''', '''<pad>''', '''[MASK]''']   ,   vocab_size=args.vocab_size   ,   )

      logger.info('''Training the tokenizer.'''   )
      tokenizer.train_from_iterator(batch_iterator()   ,   trainer=_UpperCamelCase   )
      logger.info('''Tokenizer training complete!'''   )

      snake_case_			= tokenizer.token_to_id('''[CLS]'''   )
      snake_case_			= tokenizer.token_to_id('''[SEP]'''   )
      snake_case_			= processors.TemplateProcessing(
          single='''[CLS]:0 $A:0 [SEP]:0'''   ,   pair='''[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'''   ,   special_tokens=[
              ('''[CLS]''', cls_token_id),
              ('''[SEP]''', sep_token_id),
          ]   ,   )
      snake_case_			= decoders.Metaspace()

      if args.export_to_hub:
            logger.info('''Exporting the trained tokenzier to Hub.'''   )
            snake_case_			= AlbertTokenizerFast(tokenizer_object=_UpperCamelCase   )
            new_tokenizer.push_to_hub('''unigram-tokenizer-dataset'''   )


if __name__ == "__main__":
   UpperCAmelCase     =       parse_args()
   main(args)
'''simple docstring'''




import argparse
import logging
import sys
import time

import tensorflow as tf
from datasets import load_dataset

from transformers import AutoTokenizer, TFAutoModelForSequenceClassification


if __name__ == "__main__":
  __snake_case  : Any		       =    argparse.ArgumentParser()

  # Hyperparameters sent by the client are passed as command-line arguments to the script.
  parser.add_argument('--epochs', type=int, default=1)
  parser.add_argument('--per_device_train_batch_size', type=int, default=16)
  parser.add_argument('--per_device_eval_batch_size', type=int, default=8)
  parser.add_argument('--model_name_or_path', type=str)
  parser.add_argument('--learning_rate', type=str, default=5E-5)
  parser.add_argument('--do_train', type=bool, default=True)
  parser.add_argument('--do_eval', type=bool, default=True)
  parser.add_argument('--output_dir', type=str)

  __snake_case       ,       __snake_case  : Dict		       =    parser.parse_known_args()

  # overwrite batch size until we have tf_glue.py
  __snake_case  : Any		       =    16
  __snake_case  : Optional[Any]		       =    16

  # Set up logging
  __snake_case  : Union[str, Any]		       =    logging.getLogger(__name__)

  logging.basicConfig(
      level=logging.getLevelName('INFO'),
      handlers=[logging.StreamHandler(sys.stdout)],
      format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
  )

  # Load model and tokenizer
  __snake_case  : Optional[Any]		       =    TFAutoModelForSequenceClassification.from_pretrained(args.model_name_or_path)
  __snake_case  : Optional[Any]		       =    AutoTokenizer.from_pretrained(args.model_name_or_path)

  # Load dataset
  __snake_case       ,       __snake_case  : Optional[int]		       =    load_dataset('imdb', split=['train', 'test'])
  __snake_case  : List[str]		       =    train_dataset.shuffle().select(range(5_000))  # smaller the size for train dataset to 5k
  __snake_case  : List[str]		       =    test_dataset.shuffle().select(range(500))  # smaller the size for test dataset to 500

  # Preprocess train dataset
  __snake_case  : int		       =    train_dataset.map(
      lambda e: tokenizer(e['text'], truncation=True, padding='max_length'), batched=True
  )
  train_dataset.set_format(type='tensorflow', columns=['input_ids', 'attention_mask', 'label'])

  __snake_case  : Dict		       =    {
      x: train_dataset[x].to_tensor(default_value=0, shape=[None, tokenizer.model_max_length])
      for x in ['input_ids', 'attention_mask']
  }
  __snake_case  : str		       =    tf.data.Dataset.from_tensor_slices((train_features, train_dataset['label'])).batch(
      args.per_device_train_batch_size
  )

  # Preprocess test dataset
  __snake_case  : Tuple		       =    test_dataset.map(
      lambda e: tokenizer(e['text'], truncation=True, padding='max_length'), batched=True
  )
  test_dataset.set_format(type='tensorflow', columns=['input_ids', 'attention_mask', 'label'])

  __snake_case  : Optional[int]		       =    {
      x: test_dataset[x].to_tensor(default_value=0, shape=[None, tokenizer.model_max_length])
      for x in ['input_ids', 'attention_mask']
  }
  __snake_case  : List[str]		       =    tf.data.Dataset.from_tensor_slices((test_features, test_dataset['label'])).batch(
      args.per_device_eval_batch_size
  )

  # fine optimizer and loss
  __snake_case  : Optional[int]		       =    tf.keras.optimizers.Adam(learning_rate=args.learning_rate)
  __snake_case  : Union[str, Any]		       =    tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
  __snake_case  : int		       =    [tf.keras.metrics.SparseCategoricalAccuracy()]
  model.compile(optimizer=optimizer, loss=loss, metrics=metrics)

  __snake_case  : Any		       =    time.time()
  __snake_case  : int		       =    model.fit(tf_train_dataset, epochs=args.epochs, batch_size=args.per_device_train_batch_size)
  __snake_case  : Union[str, Any]		       =    time.time() - start_train_time

  logger.info('*** Train ***')
  logger.info(F"""train_runtime = {end_train_time}""")
  for key, value in train_results.history.items():
    logger.info(F"""  {key} = {value}""")






