
import argparse
import logging
import math
import time

import torch

from transformers import TransfoXLCorpus, TransfoXLLMHeadModel


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
_UpperCAmelCase   :					List[str]    =			logging.getLogger(__name__)


def  A			(   )  ->    List[Any]:
		'''simple docstring'''

		UpperCamelCase            =	argparse.ArgumentParser(description='PyTorch Transformer Language Model'					)
		parser.add_argument('--model_name'       ,			type=lowercase       ,			default='transfo-xl-wt103'       ,			help='pretrained model name'					)
		parser.add_argument(
		    '--split'       ,			type=lowercase       ,			default='test'       ,			choices=['all', 'valid', 'test']       ,			help='which split to evaluate'					)
		parser.add_argument('--batch_size'       ,			type=lowercase       ,			default=10       ,			help='batch size'					)
		parser.add_argument('--tgt_len'       ,			type=lowercase       ,			default=128       ,			help='number of tokens to predict'					)
		parser.add_argument('--ext_len'       ,			type=lowercase       ,			default=0       ,			help='length of the extended context'					)
		parser.add_argument('--mem_len'       ,			type=lowercase       ,			default=1_600       ,			help='length of the retained previous heads'					)
		parser.add_argument('--clamp_len'       ,			type=lowercase       ,			default=1_000       ,			help='max positional embedding index'					)
		parser.add_argument('--no_cuda'       ,			action='store_true'       ,			help='Do not use CUDA even though CUA is available'					)
		parser.add_argument('--work_dir'       ,			type=lowercase       ,			required=lowercase       ,			help='path to the work_dir'					)
		parser.add_argument('--no_log'       ,			action='store_true'       ,			help='do not log the eval result'					)
		parser.add_argument('--same_length'       ,			action='store_true'       ,			help='set same length attention with masking'					)
		parser.add_argument('--server_ip'       ,			type=lowercase       ,			default=''       ,			help='Can be used for distant debugging.'					)
		parser.add_argument('--server_port'       ,			type=lowercase       ,			default=''       ,			help='Can be used for distant debugging.'					)
		UpperCamelCase            =	parser.parse_args()
		assert args.ext_len >= 0, "extended context length must be non-negative"

		if args.server_ip and args.server_port:
				# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
				import ptvsd

				print('Waiting for debugger attach'					)
				ptvsd.enable_attach(address=(args.server_ip, args.server_port)       ,			redirect_output=lowercase					)
				ptvsd.wait_for_attach()

		UpperCamelCase            =	torch.device('cuda' if torch.cuda.is_available() and not args.no_cuda else 'cpu'					)
		logger.info('device: {}'.format(lowercase					)					)

		# Load a pre-processed dataset
		# You can also build the corpus yourself using TransfoXLCorpus methods
		# The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
		# and tokenizing the dataset
		# The pre-processed corpus is a convertion (using the conversion script )
		UpperCamelCase            =	TransfoXLCorpus.from_pretrained(args.model_name					)

		UpperCamelCase            =	corpus.get_iterator('valid'       ,			args.batch_size       ,			args.tgt_len       ,			device=lowercase       ,			ext_len=args.ext_len					)
		UpperCamelCase            =	corpus.get_iterator('test'       ,			args.batch_size       ,			args.tgt_len       ,			device=lowercase       ,			ext_len=args.ext_len					)

		# Load a pre-trained model
		UpperCamelCase            =	TransfoXLLMHeadModel.from_pretrained(args.model_name					)
		model.to(lowercase					)

		logger.info(
		    'Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}'.format(
		        args.batch_size       ,			args.tgt_len       ,			args.ext_len       ,			args.mem_len       ,			args.clamp_len					)					)

		model.reset_memory_length(args.mem_len					)
		if args.clamp_len > 0:
				UpperCamelCase            =	args.clamp_len
		if args.same_length:
				UpperCamelCase            =	True

		###############################################################################
		# Evaluation code
		###############################################################################
		def evaluate(lowercase					):
				# Turn on evaluation mode which disables dropout.
				model.eval()
				UpperCamelCase      ,    UpperCamelCase            =	0, 0.0
				UpperCamelCase            =	time.time()
				with torch.no_grad():
						UpperCamelCase            =	None
						for idx, (data, target, seq_len) in enumerate(lowercase					):
								UpperCamelCase            =	model(lowercase       ,			lm_labels=lowercase       ,			mems=lowercase					)
								UpperCamelCase      ,    UpperCamelCase      ,    UpperCamelCase            =	ret
								UpperCamelCase            =	loss.mean()
								total_loss += seq_len * loss.item()
								total_len += seq_len
						UpperCamelCase            =	time.time() - start_time
				logger.info('Time : {:.2f}s, {:.2f}ms/segment'.format(lowercase       ,			1_000 * total_time / (idx + 1)					)					)
				return total_loss / total_len

		# Run on test data.
		if args.split == "all":
				UpperCamelCase            =	evaluate(lowercase					)
				UpperCamelCase            =	evaluate(lowercase					)
		elif args.split == "valid":
				UpperCamelCase            =	evaluate(lowercase					)
				UpperCamelCase            =	None
		elif args.split == "test":
				UpperCamelCase            =	evaluate(lowercase					)
				UpperCamelCase            =	None

		def format_log(lowercase       ,			lowercase					):
				UpperCamelCase            =	'| {0} loss {1:5.2f} | {0} ppl {2:9.3f} '.format(lowercase       ,			lowercase       ,			math.exp(lowercase					)					)
				return log_str

		UpperCamelCase            =	''
		if valid_loss is not None:
				log_str += format_log(lowercase       ,			'valid'					)
		if test_loss is not None:
				log_str += format_log(lowercase       ,			'test'					)

		logger.info('=' * 100					)
		logger.info(lowercase					)
		logger.info('=' * 100					)


if __name__ == "__main__":
					main()


import json
import os
from dataclasses import dataclass

import numpy as np
import pyarrow as pa

import datasets
from utils import get_duration


_UpperCAmelCase   :					str    =			100_000_000_000
_UpperCAmelCase   :					Optional[int]    =			10_000

_UpperCAmelCase  ,_UpperCAmelCase   :					Optional[int]    =			os.path.split(__file__)
_UpperCAmelCase   :					Optional[int]    =			os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))


def  A			(   lowercase       ,			lowercase					)  ->    datasets.Dataset:
		'''simple docstring'''

		UpperCamelCase            =	pa.Table.from_pydict({'col': [0] * chunk_size}					)
		UpperCamelCase            =	pa.concat_tables([table] * (num_examples // chunk_size)					)
		return datasets.Dataset(lowercase       ,			fingerprint='table_100B'					)


@dataclass
class 							lowercase   :
		__lowercase			:			int
		__lowercase			:			int
		__lowercase			:			int
		__lowercase			:			int



		def __UpperCamelCase					(					self			)    ->   Optional[int]:
				"""simple docstring"""
				UpperCamelCase            =	np.random.default_rng(self.seed			)
				UpperCamelCase            =	rng.integers(low=self.low							, high=self.high							, size=self.size			).tolist()



		def __iter__(					self			)    ->   List[str]:
				"""simple docstring"""
				return iter(self._sampled_values			)



		def __len__(					self			)    ->   int:
				"""simple docstring"""
				return self.size






@get_duration
def  A			(   lowercase					)  ->    List[Any]:
		'''simple docstring'''

		UpperCamelCase            =	dataset[0]


@get_duration
def  A			(   lowercase					)  ->    Any:
		'''simple docstring'''

		UpperCamelCase            =	dataset[-1]


@get_duration
def  A			(   lowercase					)  ->    List[Any]:
		'''simple docstring'''

		UpperCamelCase            =	dataset[range(len(lowercase					) // 2       ,			len(lowercase					) // 2 + 1_024					)]


@get_duration
def  A			(   lowercase					)  ->    Any:
		'''simple docstring'''

		UpperCamelCase            =	dataset[RandIter(0       ,			len(lowercase					)       ,			1_024       ,			seed=42					)]


def  A			(   )  ->    List[Any]:
		'''simple docstring'''

		UpperCamelCase            =	{'num examples': SPEED_TEST_N_EXAMPLES}
		UpperCamelCase            =	(get_first_row, get_last_row, get_batch_of_aaaa_rows, get_batch_of_aaaa_random_rows)
		print('generating dataset'					)
		UpperCamelCase            =	generate_100B_dataset(num_examples=lowercase       ,			chunk_size=lowercase					)
		print('Functions'					)
		for func in functions:
				print(func.__name__					)
				UpperCamelCase            =	func(lowercase					)

		with open(lowercase       ,			'wb'					) as f:
				f.write(json.dumps(lowercase					).encode('utf-8'					)					)


if __name__ == "__main__":  # useful to run the profiler
					benchmark_table_100B()
