'''simple docstring'''


import os
import pickle
import unittest
from collections import UserDict, namedtuple

import torch

from accelerate.test_utils.testing import require_cuda, require_torch_min_version
from accelerate.test_utils.training import RegressionModel
from accelerate.utils import (
    convert_outputs_to_fpaa,
    extract_model_from_parallel,
    find_device,
    listify,
    patch_environment,
    recursively_apply,
    send_to_device,
)


snake_case__	:     Optional[int]      =    namedtuple('''ExampleNamedTuple''', '''a b c''')






class 	__SCREAMING_SNAKE_CASE		(			unittest.TestCase			):







  '''simple docstring'''






  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = torch.randn(5 ,						2		)
       UpperCAmelCase_				:  Optional[Any]  = torch.device('cuda'		) if torch.cuda.is_available() else torch.device('cpu'		)

       UpperCAmelCase_				:  Dict  = send_to_device(snake_case_ ,						snake_case_		)
       self.assertTrue(torch.equal(resulta.cpu() ,						snake_case_		)		)

       UpperCAmelCase_				:  Tuple  = send_to_device((tensor, [tensor, tensor], 1) ,						snake_case_		)
       self.assertIsInstance(snake_case_ ,						snake_case_		)
       self.assertTrue(torch.equal(resulta[0].cpu() ,						snake_case_		)		)
       self.assertIsInstance(resulta[1] ,						snake_case_		)
       self.assertTrue(torch.equal(resulta[1][0].cpu() ,						snake_case_		)		)
       self.assertTrue(torch.equal(resulta[1][1].cpu() ,						snake_case_		)		)
       self.assertEqual(resulta[2] ,						1		)

       UpperCAmelCase_				:  List[Any]  = send_to_device({'a': tensor, 'b': [tensor, tensor], 'c': 1} ,						snake_case_		)
       self.assertIsInstance(snake_case_ ,						snake_case_		)
       self.assertTrue(torch.equal(resulta['a'].cpu() ,						snake_case_		)		)
       self.assertIsInstance(resulta['b'] ,						snake_case_		)
       self.assertTrue(torch.equal(resulta['b'][0].cpu() ,						snake_case_		)		)
       self.assertTrue(torch.equal(resulta['b'][1].cpu() ,						snake_case_		)		)
       self.assertEqual(resulta['c'] ,						1		)

       UpperCAmelCase_				:  Tuple  = send_to_device(ExampleNamedTuple(a=snake_case_ ,						b=[tensor, tensor] ,						c=1		) ,						snake_case_		)
       self.assertIsInstance(snake_case_ ,						snake_case_		)
       self.assertTrue(torch.equal(resulta.a.cpu() ,						snake_case_		)		)
       self.assertIsInstance(resulta.b ,						snake_case_		)
       self.assertTrue(torch.equal(resulta.b[0].cpu() ,						snake_case_		)		)
       self.assertTrue(torch.equal(resulta.b[1].cpu() ,						snake_case_		)		)
       self.assertEqual(resulta.c ,						1		)

       UpperCAmelCase_				:  int  = send_to_device(UserDict({'a': tensor, 'b': [tensor, tensor], 'c': 1}		) ,						snake_case_		)
       self.assertIsInstance(snake_case_ ,						snake_case_		)
       self.assertTrue(torch.equal(resulta['a'].cpu() ,						snake_case_		)		)
       self.assertIsInstance(resulta['b'] ,						snake_case_		)
       self.assertTrue(torch.equal(resulta['b'][0].cpu() ,						snake_case_		)		)
       self.assertTrue(torch.equal(resulta['b'][1].cpu() ,						snake_case_		)		)
       self.assertEqual(resulta['c'] ,						1		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       with self.assertRaises(snake_case_		) as cm:
            UpperCAmelCase_				:  Optional[Any]  = recursively_apply(torch.tensor ,						(torch.tensor(1		), 1) ,						error_on_other_type=snake_case_		)
       self.assertEqual(
           str(cm.exception		) ,						'Unsupported types (<class \'int\'>) passed to `tensor`. Only nested list/tuple/dicts of objects that are valid for `is_torch_tensor` should be passed.' ,						)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Dict  = torch.tensor([1, 2, 3, 4, 5]		)
       self.assertEqual(listify(snake_case_		) ,						[1, 2, 3, 4, 5]		)

       UpperCAmelCase_				:  List[str]  = torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]]		)
       self.assertEqual(listify(snake_case_		) ,						[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]]		)

       UpperCAmelCase_				:  int  = torch.tensor([[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]], [[1_1, 1_2, 1_3, 1_4, 1_5], [1_6, 1_7, 1_8, 1_9, 2_0]]]		)
       self.assertEqual(
           listify(snake_case_		) ,						[[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]], [[1_1, 1_2, 1_3, 1_4, 1_5], [1_6, 1_7, 1_8, 1_9, 2_0]]]		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       with patch_environment(aa=1 ,						BB=2		):
            self.assertEqual(os.environ.get('AA'		) ,						'1'		)
            self.assertEqual(os.environ.get('BB'		) ,						'2'		)

       self.assertNotIn('AA' ,						os.environ		)
       self.assertNotIn('BB' ,						os.environ		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[int]  = RegressionModel()
       UpperCAmelCase_				:  Optional[Any]  = model.forward
       UpperCAmelCase_				:  List[Any]  = convert_outputs_to_fpaa(model.forward		)
       UpperCAmelCase_				:  Any  = extract_model_from_parallel(snake_case_ ,						keep_fpaa_wrapper=snake_case_		)
       UpperCAmelCase_				:  Optional[Any]  = pickle.dumps(snake_case_		)


  @require_cuda
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Tuple  = RegressionModel()
       UpperCAmelCase_				:  Tuple  = model.forward
       UpperCAmelCase_				:  Optional[Any]  = torch.cuda.amp.autocast(dtype=torch.floataa		)(model.forward		)
       UpperCAmelCase_				:  int  = convert_outputs_to_fpaa(model.forward		)
       UpperCAmelCase_				:  Optional[Any]  = extract_model_from_parallel(snake_case_ ,						keep_fpaa_wrapper=snake_case_		)
       UpperCAmelCase_				:  List[Any]  = pickle.dumps(snake_case_		)


  @require_cuda
  @require_torch_min_version(version='2.0'		)
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Tuple  = RegressionModel()
       UpperCAmelCase_				:  str  = model.forward
       UpperCAmelCase_				:  str  = torch.cuda.amp.autocast(dtype=torch.floataa		)(model.forward		)
       UpperCAmelCase_				:  List[str]  = convert_outputs_to_fpaa(model.forward		)
       UpperCAmelCase_				:  List[Any]  = torch.compile(model.forward ,						backend='inductor'		)
       UpperCAmelCase_				:  List[Any]  = torch.randn(4 ,						1_0		).cuda()
       UpperCAmelCase_				:  List[str]  = model(snake_case_		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = RegressionModel()
       # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
       UpperCAmelCase_				:  List[Any]  = torch.nn.parallel.DataParallel(snake_case_		)
       UpperCAmelCase_				:  List[str]  = extract_model_from_parallel(snake_case_		)

       self.assertEqual(snake_case_ ,						snake_case_		)


  @require_torch_min_version(version='2.0'		)
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[Any]  = RegressionModel()
       UpperCAmelCase_				:  List[str]  = torch.compile(snake_case_		)

       # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
       UpperCAmelCase_				:  Optional[int]  = torch.nn.parallel.DataParallel(snake_case_		)
       UpperCAmelCase_				:  List[str]  = torch.compile(snake_case_		)
       UpperCAmelCase_				:  Union[str, Any]  = extract_model_from_parallel(snake_case_		)

       self.assertEqual(compiled_model._orig_mod ,						compiled_model_unwrapped._orig_mod		)






  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       self.assertEqual(find_device([1, 'a', torch.tensor([1, 2, 3]		)]		) ,						torch.device('cpu'		)		)
       self.assertEqual(find_device({'a': 1, 'b': torch.tensor([1, 2, 3]		)}		) ,						torch.device('cpu'		)		)
       self.assertIsNone(find_device([1, 'a']		)		)













"""simple docstring"""






import os
from dataclasses import replace

import jax
import wandb
from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
from datasets import load_dataset
from flax import jax_utils

from transformers import BigBirdTokenizerFast


if __name__ == "__main__":
    print('''#################### AVAILABLE DEVICES ####################''')
    print(jax.devices())
    print('''###########################################################''')

    # setup for wandb sweep
    lowerCAmelCase_      =				Args()
    lowerCAmelCase_      =				wandb.init(project='''bigbird-natural-questions''', config=args.__dict__)
    lowerCAmelCase_      =				dict(logger.config)
    del wandb_args["batch_size"]
    lowerCAmelCase_      =				replace(args, **wandb_args)
    lowerCAmelCase_      =				args.base_dir + '''-''' + wandb.run.id
    lowerCAmelCase_      =				replace(args, base_dir=base_dir)
    print(args)

    lowerCAmelCase_      =				load_dataset('''json''', data_files=args.tr_data_path)['''train''']
    lowerCAmelCase_      =				load_dataset('''json''', data_files=args.val_data_path)['''train''']

    # drop extra batch for now
    lowerCAmelCase_      =				range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
    lowerCAmelCase_      =				tr_dataset.shuffle().select(indices)
    lowerCAmelCase_      =				range(len(val_dataset) - len(val_dataset) % args.batch_size)
    lowerCAmelCase_      =				val_dataset.shuffle().select(indices)

    if os.environ.get('''TRAIN_ON_SMALL''', '''false''') == "true":
        lowerCAmelCase_      =				tr_dataset.shuffle().select(range(80_000))
        lowerCAmelCase_      =				val_dataset.shuffle().select(range(8_000))

    print(tr_dataset)
    print(val_dataset)

    lowerCAmelCase_      =				FlaxBigBirdForNaturalQuestions.from_pretrained(
        args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
    )
    lowerCAmelCase_      =				BigBirdTokenizerFast.from_pretrained(args.model_id)
    lowerCAmelCase_      =				DataCollator(pad_id=tokenizer.pad_token_id, max_length=4_096)

    lowerCAmelCase_      =				{
        '''lr''': args.lr,
        '''init_lr''': args.init_lr,
        '''warmup_steps''': args.warmup_steps,
        '''num_train_steps''': args.max_epochs * (len(tr_dataset) // args.batch_size),
        '''weight_decay''': args.weight_decay,
    }
    lowerCAmelCase_      ,					lowerCAmelCase_      =				build_tx(**tx_args)

    lowerCAmelCase_      =				Trainer(
        args=args,
        data_collator=data_collator,
        model_save_fn=model.save_pretrained,
        train_step_fn=train_step,
        val_step_fn=val_step,
        logger=logger,
        scheduler_fn=lr,
    )

    lowerCAmelCase_      =				None
    lowerCAmelCase_      =				trainer.create_state(model, tx, num_train_steps=tx_args['''num_train_steps'''], ckpt_dir=ckpt_dir)
    try:
        trainer.train(state, tr_dataset, val_dataset)
    except KeyboardInterrupt:
        print('''Oooops; TRAINING STOPPED UNFORTUNATELY''')

    print('''SAVING WEIGHTS IN `final-weights`''')
    lowerCAmelCase_      =				jax_utils.unreplicate(state.params)
    model.save_pretrained(os.path.join(args.base_dir, '''final-weights'''), params=params)






