






import argparse
import time

import torch
import transformers
from measures_util import end_measure, log_measures, start_measure
from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForSeqaSeqLM, AutoTokenizer

from accelerate.utils import compute_module_sizes


A_	:	List[str]				=				{
    'gpt-j-6b': {'is_causal': True, 'model': 'sgugger/sharded-gpt-j-6B', 'tokenizer': 'EleutherAI/gpt-j-6B'},
    'gpt-neox': {'is_causal': True, 'model': 'EleutherAI/gpt-neox-20b'},
    'opt': {'is_causal': True, 'model': 'facebook/opt-30b'},
    'T0pp': {'is_causal': False, 'model': 'bigscience/T0pp', 'model_revision': 'sharded'},
}

A_	:	Any				=				[
    'Hello, my name is',
    'Are unicorns real? Unicorns are',
    'For the first time in several years,',
    'My name is Julien and I am',
    'The goal of life is',
    'Whenever I\'m sad, I like to',
]

def 						snake_case      () ->       Any:
  UpperCamelCase_:      str					   =       argparse.ArgumentParser(description='Run and time generations on a big model using Accelerate.'						)
  parser.add_argument('model_name'			,					type=UpperCAmelCase__			,					default=UpperCAmelCase__			,					help='The name of the model to try.'						)
  parser.add_argument(
      '--tokenizer_name'			,					type=UpperCAmelCase__			,					default=UpperCAmelCase__			,					help='The name of the tokenizer (if different from the model.'						)
  parser.add_argument('--is_causal'			,					type=UpperCAmelCase__			,					default=UpperCAmelCase__			,					help='Whether or not the model is causal.'						)
  parser.add_argument(
      '--model_revision'			,					type=UpperCAmelCase__			,					default=UpperCAmelCase__			,					help='The revision to use for the model checkpoint.'						)
  parser.add_argument('--torch_dtype'			,					type=UpperCAmelCase__			,					default=UpperCAmelCase__			,					help='The dtype for the model.'						)
  parser.add_argument('--disk_offload'			,					action='store_true'						)

  UpperCamelCase_:      Optional[Any]					   =       parser.parse_args()

  # Sanitize args
  if args.model_name in DEFAULT_MODELS:
    UpperCamelCase_:      List[Any]					   =       DEFAULT_MODELS[args.model_name]
    UpperCamelCase_:      Union[str, Any]					   =       defaults['model']
    if args.tokenizer_name is None:
      UpperCamelCase_:      Tuple					   =       defaults.get('tokenizer'			,					args.model_name						)
    if args.is_causal is None:
      UpperCamelCase_:      Optional[Any]					   =       defaults['is_causal']
    if args.model_revision is None:
      UpperCamelCase_:      List[str]					   =       defaults.get('model_revision'			,					'main'						)

  if args.is_causal is None:
    raise ValueError('Could not infer the default for `--is_causal`, pass either True or False for it.'						)
  if args.tokenizer_name is None:
    UpperCamelCase_:      Optional[Any]					   =       args.model_name
  if args.model_revision is None:
    UpperCamelCase_:      str					   =       'main'

  return args

def 						snake_case      () ->       List[Any]:
  transformers.utils.logging.set_verbosity_error()
  UpperCamelCase_:      Tuple					   =       parse_args()

  if args.torch_dtype is None:
    UpperCamelCase_:      int					   =       AutoConfig.from_pretrained(args.model_name						)
    UpperCamelCase_:      Optional[Any]					   =       getattr(UpperCAmelCase__			,					'torch_dtype'			,					torch.floataa						)
  else:
    UpperCamelCase_:      List[Any]					   =       getattr(UpperCAmelCase__			,					args.torch_dtype						)
  UpperCamelCase_:      List[str]					   =       AutoModelForCausalLM if args.is_causal else AutoModelForSeqaSeqLM
  UpperCamelCase_:      str					   =       {
      'torch_dtype': torch_dtype,
      'revision': args.model_revision,
  }
  if args.disk_offload:
    UpperCamelCase_:      Union[str, Any]					   =       'tmp_offload'
    UpperCamelCase_:      List[Any]					   =       True

  UpperCamelCase_:      Any					   =       start_measure()
  UpperCamelCase_:      Tuple					   =       model_cls.from_pretrained(args.model_name			,					device_map='auto'			,					**UpperCAmelCase__						)
  UpperCamelCase_:      Any					   =       end_measure(UpperCAmelCase__						)
  log_measures(UpperCAmelCase__			,					'Model loading'						)

  UpperCamelCase_:      Optional[Any]					   =       compute_module_sizes(UpperCAmelCase__						)
  UpperCamelCase_:      Any					   =       {v: 0 for v in model.hf_device_map.values()}
  for module, device in model.hf_device_map.items():
    device_size[device] += module_sizes[module]
  UpperCamelCase_:      Optional[Any]					   =       '\n'.join([F'''- {device}: {size // 2**2_0}MiB''' for device, size in device_size.items()]						)
  print(F'''\nTheoretical use:\n{message}'''						)

  UpperCamelCase_:      int					   =       AutoTokenizer.from_pretrained(args.tokenizer_name						)

  UpperCamelCase_:      Dict					   =       start_measure()
  UpperCamelCase_:      List[str]					   =       []
  UpperCamelCase_:      str					   =       []
  UpperCamelCase_:      Optional[int]					   =       []
  for prompt in PROMPTS:
    UpperCamelCase_:      int					   =       tokenizer(UpperCAmelCase__			,					return_tensors='pt'						).to(0						)
    UpperCamelCase_:      Any					   =       inputs['input_ids'][0].tolist()
    UpperCamelCase_:      str					   =       time.time()
    UpperCamelCase_:      Optional[Any]					   =       model.generate(inputs['input_ids']						)
    UpperCamelCase_:      str					   =       time.time()
    UpperCamelCase_:      Dict					   =       outputs[0].tolist()
    UpperCamelCase_:      Any					   =       len(UpperCAmelCase__						) if outputs[: len(UpperCAmelCase__						)] != tokens else len(UpperCAmelCase__						) - len(UpperCAmelCase__						)
    UpperCamelCase_:      Optional[int]					   =       after_generate - before_generate

    UpperCamelCase_:      Optional[int]					   =       tokenizer.decode(UpperCAmelCase__			,					skip_special_tokens=UpperCAmelCase__						)
    texts_outs.append(UpperCAmelCase__						)
    generation_times.append(UpperCAmelCase__						)
    gen_tokens.append(UpperCAmelCase__						)
    print(F'''Prompt: {prompt}\nGeneration {text_out}\nIn {generation_time:.2f}s for {num_gen_tokens} tokens\n'''						)

  UpperCamelCase_:      Optional[int]					   =       end_measure(UpperCAmelCase__						)
  log_measures(UpperCAmelCase__			,					'Model generation'						)

  UpperCamelCase_:      str					   =       [gen / tok for gen, tok in zip(UpperCAmelCase__			,					UpperCAmelCase__						)]
  UpperCamelCase_:      List[str]					   =       sum(UpperCAmelCase__						) / len(UpperCAmelCase__						)
  print(F'''Average time of generation per token: {avg_gen:.2f}s'''						)
  print(F'''First generation (avg time per token): {generation_times_per_token[0]:.2f}s'''						)
  UpperCamelCase_:      Dict					   =       sum(generation_times_per_token[1:]						) / (len(UpperCAmelCase__						) - 1)
  print(F'''Average time of generation per token (excluding the first): {avg_gen:.2f}s'''						)


if __name__ == "__main__":
   main()







# Lint as: python3

import hashlib
from typing import Union

def 						snake_case      (UpperCAmelCase__						) ->       bytes:
  if isinstance(UpperCAmelCase__			,					UpperCAmelCase__						):
    # Data already in bytes, returns as it as
    return hash_data
  elif isinstance(UpperCAmelCase__			,					UpperCAmelCase__						):
    # We keep the data as it as for it ot be later encoded to UTF-8
    # However replace `\\` with `/` for Windows compatibility
    UpperCamelCase_:      List[str]					   =       hash_data.replace('\\'			,					'/'						)
  elif isinstance(UpperCAmelCase__			,					UpperCAmelCase__						):
    UpperCamelCase_:      Union[str, Any]					   =       str(UpperCAmelCase__						)
  else:
    # If data is not of the required type, raise error
    raise InvalidKeyError(UpperCAmelCase__						)

  return hash_data.encode('utf-8'						)




class        _lowerCAmelCase( UpperCAmelCase_      ):





       """simple docstring"""







       def __init__(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      int					   =       '\nFAILURE TO GENERATE DATASET: Invalid key type detected'
         UpperCamelCase_:      Dict					   =       f'''\nFound Key {hash_data} of type {type(_lowerCamelCase							)}'''
         UpperCamelCase_:      Any					   =       '\nKeys should be either str, int or bytes type'
         super().__init__(f'''{self.prefix}{self.err_msg}{self.suffix}'''							)







class        _lowerCAmelCase( UpperCAmelCase_      ):





       """simple docstring"""







       def __init__(      self						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase=""							):
         UpperCamelCase_:      int					   =       key
         UpperCamelCase_:      Dict					   =       duplicate_key_indices
         UpperCamelCase_:      int					   =       fix_msg
         UpperCamelCase_:      List[Any]					   =       'Found multiple examples generated with the same key'
         if len(_lowerCamelCase							) <= 2_0:
           UpperCamelCase_:      Tuple					   =       f'''\nThe examples at index {', '.join(_lowerCamelCase							)} have the key {key}'''
         else:
           UpperCamelCase_:      Optional[int]					   =       f'''\nThe examples at index {', '.join(duplicate_key_indices[:2_0]							)}... ({len(_lowerCamelCase							) - 2_0} more) have the key {key}'''
         UpperCamelCase_:      List[Any]					   =       '\n' + fix_msg if fix_msg else ''
         super().__init__(f'''{self.prefix}{self.err_msg}{self.suffix}'''							)







class        _lowerCAmelCase:





       """simple docstring"""







       def __init__(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      int					   =       hashlib.mda(_as_bytes(_lowerCamelCase							)							)





       def     _a					(      self						,    _lowerCamelCase							):
         UpperCamelCase_:      Optional[int]					   =       self._split_mda.copy()
         UpperCamelCase_:      str					   =       _as_bytes(_lowerCamelCase							)
         mda.update(_lowerCamelCase							)
         # Convert to integer with hexadecimal conversion
         return int(mda.hexdigest()						,    1_6							)