




import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 						SCREAMING_SNAKE_CASE__		(_a						,				unittest.TestCase ):
					lowercase_   :			Dict   =   ElectraTokenizer
					lowercase_   :			str   =   ElectraTokenizerFast
					lowercase_   :			Any   =   True
					lowercase_   :			Any   =   True
					lowercase_   :			Optional[Any]   =   filter_non_english
					def 					A__			(	self  :		int	):



							"""simple docstring"""
							super().setUp()

							lowerCAmelCase__								=			[
							    '''[UNK]''',
							    '''[CLS]''',
							    '''[SEP]''',
							    '''[PAD]''',
							    '''[MASK]''',
							    '''want''',
							    '''##want''',
							    '''##ed''',
							    '''wa''',
							    '''un''',
							    '''runn''',
							    '''##ing''',
							    ''',''',
							    '''low''',
							    '''lowest''',
							]
							lowerCAmelCase__								=			os.path.join(self.tmpdirname					,	VOCAB_FILES_NAMES['''vocab_file''']	)
							with open(self.vocab_file					,	'''w'''					,	encoding='''utf-8'''	) as vocab_writer:
									vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)
					def 					A__			(	self  :		Tuple					,	__lowerCamelCase  :		List[Any]	):



							"""simple docstring"""
							lowerCAmelCase__								=			'''UNwant\u00E9d,running'''
							lowerCAmelCase__								=			'''unwanted, running'''
							return input_text, output_text
					def 					A__			(	self  :		Dict	):



							"""simple docstring"""
							lowerCAmelCase__								=			self.tokenizer_class(self.vocab_file	)

							lowerCAmelCase__								=			tokenizer.tokenize('''UNwant\u00E9d,running'''	)
							self.assertListEqual(__lowerCamelCase					,	['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']	)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase	)					,	[9, 6, 7, 12, 10, 11]	)
					def 					A__			(	self  :		Optional[Any]	):



							"""simple docstring"""
							if not self.test_rust_tokenizer:
									return

							lowerCAmelCase__								=			self.get_tokenizer()
							lowerCAmelCase__								=			self.get_rust_tokenizer()

							lowerCAmelCase__								=			'''UNwant\u00E9d,running'''

							lowerCAmelCase__								=			tokenizer.tokenize(__lowerCamelCase	)
							lowerCAmelCase__								=			rust_tokenizer.tokenize(__lowerCamelCase	)
							self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

							lowerCAmelCase__								=			tokenizer.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
							lowerCAmelCase__								=			rust_tokenizer.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
							self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

							lowerCAmelCase__								=			self.get_rust_tokenizer()
							lowerCAmelCase__								=			tokenizer.encode(__lowerCamelCase	)
							lowerCAmelCase__								=			rust_tokenizer.encode(__lowerCamelCase	)
							self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

							# With lower casing
							lowerCAmelCase__								=			self.get_tokenizer(do_lower_case=__lowerCamelCase	)
							lowerCAmelCase__								=			self.get_rust_tokenizer(do_lower_case=__lowerCamelCase	)

							lowerCAmelCase__								=			'''UNwant\u00E9d,running'''

							lowerCAmelCase__								=			tokenizer.tokenize(__lowerCamelCase	)
							lowerCAmelCase__								=			rust_tokenizer.tokenize(__lowerCamelCase	)
							self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

							lowerCAmelCase__								=			tokenizer.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
							lowerCAmelCase__								=			rust_tokenizer.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
							self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

							lowerCAmelCase__								=			self.get_rust_tokenizer()
							lowerCAmelCase__								=			tokenizer.encode(__lowerCamelCase	)
							lowerCAmelCase__								=			rust_tokenizer.encode(__lowerCamelCase	)
							self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)
					def 					A__			(	self  :		Any	):



							"""simple docstring"""
							lowerCAmelCase__								=			BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''	)					,	['''ah''', '''\u535A''', '''\u63A8''', '''zz''']	)
					def 					A__			(	self  :		List[Any]	):



							"""simple docstring"""
							lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)					,	['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)					,	['''hello''']	)
					def 					A__			(	self  :		Union[str, Any]	):



							"""simple docstring"""
							lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase					,	strip_accents=__lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)					,	['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)					,	['''h\u00E9llo''']	)
					def 					A__			(	self  :		int	):



							"""simple docstring"""
							lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase					,	strip_accents=__lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)					,	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)					,	['''hello''']	)
					def 					A__			(	self  :		int	):



							"""simple docstring"""
							lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)					,	['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)					,	['''hello''']	)
					def 					A__			(	self  :		Dict	):



							"""simple docstring"""
							lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)					,	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)
					def 					A__			(	self  :		Optional[int]	):



							"""simple docstring"""
							lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase					,	strip_accents=__lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)					,	['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)
					def 					A__			(	self  :		Union[str, Any]	):



							"""simple docstring"""
							lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase					,	strip_accents=__lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)					,	['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)
					def 					A__			(	self  :		List[str]	):



							"""simple docstring"""
							lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase					,	never_split=['''[UNK]''']	)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''	)					,	['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']	)
					def 					A__			(	self  :		List[str]	):



							"""simple docstring"""
							lowerCAmelCase__								=			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							lowerCAmelCase__								=			{}
							for i, token in enumerate(__lowerCamelCase	):
									lowerCAmelCase__								=			i
							lowerCAmelCase__								=			WordpieceTokenizer(vocab=__lowerCamelCase					,	unk_token='''[UNK]'''	)

							self.assertListEqual(tokenizer.tokenize(''''''	)					,	[]	)

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''	)					,	['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']	)

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''	)					,	['''[UNK]''', '''runn''', '''##ing''']	)
					def 					A__			(	self  :		Union[str, Any]	):



							"""simple docstring"""
							self.assertTrue(_is_whitespace(''' '''	)	)
							self.assertTrue(_is_whitespace('''\t'''	)	)
							self.assertTrue(_is_whitespace('''\r'''	)	)
							self.assertTrue(_is_whitespace('''\n'''	)	)
							self.assertTrue(_is_whitespace('''\u00A0'''	)	)

							self.assertFalse(_is_whitespace('''A'''	)	)
							self.assertFalse(_is_whitespace('''-'''	)	)
					def 					A__			(	self  :		Dict	):



							"""simple docstring"""
							self.assertTrue(_is_control('''\u0005'''	)	)

							self.assertFalse(_is_control('''A'''	)	)
							self.assertFalse(_is_control(''' '''	)	)
							self.assertFalse(_is_control('''\t'''	)	)
							self.assertFalse(_is_control('''\r'''	)	)
					def 					A__			(	self  :		Dict	):



							"""simple docstring"""
							self.assertTrue(_is_punctuation('''-'''	)	)
							self.assertTrue(_is_punctuation('''$'''	)	)
							self.assertTrue(_is_punctuation('''`'''	)	)
							self.assertTrue(_is_punctuation('''.'''	)	)

							self.assertFalse(_is_punctuation('''A'''	)	)
							self.assertFalse(_is_punctuation(''' '''	)	)
					def 					A__			(	self  :		Optional[Any]	):



							"""simple docstring"""
							lowerCAmelCase__								=			self.get_tokenizer()
							lowerCAmelCase__								=			self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(__lowerCamelCase	) for t in ['''Test''', '''\xad''', '''test''']]					,	[['''[UNK]'''], [], ['''[UNK]''']]	)

							self.assertListEqual(
							    [rust_tokenizer.tokenize(__lowerCamelCase	) for t in ['''Test''', '''\xad''', '''test''']]					,	[['''[UNK]'''], [], ['''[UNK]''']]	)
					@slow
					def 					A__			(	self  :		Dict	):



							"""simple docstring"""
							lowerCAmelCase__								=			self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''	)

							lowerCAmelCase__								=			tokenizer.encode('''sequence builders'''					,	add_special_tokens=__lowerCamelCase	)
							lowerCAmelCase__								=			tokenizer.encode('''multi-sequence build'''					,	add_special_tokens=__lowerCamelCase	)

							lowerCAmelCase__								=			tokenizer.build_inputs_with_special_tokens(__lowerCamelCase	)
							lowerCAmelCase__								=			tokenizer.build_inputs_with_special_tokens(__lowerCamelCase					,	__lowerCamelCase	)

							assert encoded_sentence == [1_01] + text + [1_02]
							assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
					def 					A__			(	self  :		str	):



							"""simple docstring"""
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
											lowerCAmelCase__								=			self.rust_tokenizer_class.from_pretrained(__lowerCamelCase					,	**__lowerCamelCase	)

											lowerCAmelCase__								=			F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
											lowerCAmelCase__								=			tokenizer_r.encode_plus(
											    __lowerCamelCase					,	return_attention_mask=__lowerCamelCase					,	return_token_type_ids=__lowerCamelCase					,	return_offsets_mapping=__lowerCamelCase					,	add_special_tokens=__lowerCamelCase					,	)

											lowerCAmelCase__								=			tokenizer_r.do_lower_case if hasattr(__lowerCamelCase					,	'''do_lower_case'''	) else False
											lowerCAmelCase__								=			(
											    [
											        ((0, 0), tokenizer_r.cls_token),
											        ((0, 1), '''A'''),
											        ((1, 2), ''','''),
											        ((3, 5), '''na'''),
											        ((5, 6), '''##ï'''),
											        ((6, 8), '''##ve'''),
											        ((9, 15), tokenizer_r.mask_token),
											        ((16, 21), '''Allen'''),
											        ((21, 23), '''##NL'''),
											        ((23, 24), '''##P'''),
											        ((25, 33), '''sentence'''),
											        ((33, 34), '''.'''),
											        ((0, 0), tokenizer_r.sep_token),
											    ]
											    if not do_lower_case
											    else [
											        ((0, 0), tokenizer_r.cls_token),
											        ((0, 1), '''a'''),
											        ((1, 2), ''','''),
											        ((3, 8), '''naive'''),
											        ((9, 15), tokenizer_r.mask_token),
											        ((16, 21), '''allen'''),
											        ((21, 23), '''##nl'''),
											        ((23, 24), '''##p'''),
											        ((25, 33), '''sentence'''),
											        ((33, 34), '''.'''),
											        ((0, 0), tokenizer_r.sep_token),
											    ]
											)

											self.assertEqual(
											    [e[1] for e in expected_results]					,	tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']	)	)
											self.assertEqual([e[0] for e in expected_results]					,	tokens['''offset_mapping''']	)



					def 					A__			(	self  :		Union[str, Any]	):



							"""simple docstring"""
							lowerCAmelCase__								=			['''的''', '''人''', '''有''']
							lowerCAmelCase__								=			''''''.join(__lowerCamelCase	)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
											lowerCAmelCase__								=			True
											lowerCAmelCase__								=			self.tokenizer_class.from_pretrained(__lowerCamelCase					,	**__lowerCamelCase	)
											lowerCAmelCase__								=			self.rust_tokenizer_class.from_pretrained(__lowerCamelCase					,	**__lowerCamelCase	)

											lowerCAmelCase__								=			tokenizer_p.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
											lowerCAmelCase__								=			tokenizer_r.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)

											lowerCAmelCase__								=			tokenizer_r.convert_ids_to_tokens(__lowerCamelCase	)
											lowerCAmelCase__								=			tokenizer_p.convert_ids_to_tokens(__lowerCamelCase	)

											# it is expected that each Chinese character is not preceded by "##"
											self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)
											self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)

											lowerCAmelCase__								=			False
											lowerCAmelCase__								=			self.rust_tokenizer_class.from_pretrained(__lowerCamelCase					,	**__lowerCamelCase	)
											lowerCAmelCase__								=			self.tokenizer_class.from_pretrained(__lowerCamelCase					,	**__lowerCamelCase	)

											lowerCAmelCase__								=			tokenizer_r.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)
											lowerCAmelCase__								=			tokenizer_p.encode(__lowerCamelCase					,	add_special_tokens=__lowerCamelCase	)

											lowerCAmelCase__								=			tokenizer_r.convert_ids_to_tokens(__lowerCamelCase	)
											lowerCAmelCase__								=			tokenizer_p.convert_ids_to_tokens(__lowerCamelCase	)

											# it is expected that only the first Chinese character is not preceded by "##".
											lowerCAmelCase__								=			[
											    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__lowerCamelCase	)
											]
											self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)
											self.assertListEqual(__lowerCamelCase					,	__lowerCamelCase	)







import os
from dataclasses import replace

import jax
import wandb
from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
from datasets import load_dataset
from flax import jax_utils

from transformers import BigBirdTokenizerFast


if __name__ == "__main__":
      print("""#################### AVAILABLE DEVICES ####################""")
      print(jax.devices())
      print("""###########################################################""")

      # setup for wandb sweep
      __magic_name__     :     Optional[Any]  							=					Args()
      __magic_name__     :     List[Any]  							=					wandb.init(project="""bigbird-natural-questions""", config=args.__dict__)
      __magic_name__     :     Any  							=					dict(logger.config)
      del wandb_args["batch_size"]
      __magic_name__     :     int  							=					replace(args, **wandb_args)
      __magic_name__     :     Tuple  							=					args.base_dir + """-""" + wandb.run.id
      __magic_name__     :     Union[str, Any]  							=					replace(args, base_dir=base_dir)
      print(args)

      __magic_name__     :     Union[str, Any]  							=					load_dataset("""json""", data_files=args.tr_data_path)["""train"""]
      __magic_name__     :     List[Any]  							=					load_dataset("""json""", data_files=args.val_data_path)["""train"""]

      # drop extra batch for now
      __magic_name__     :     List[str]  							=					range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
      __magic_name__     :     Dict  							=					tr_dataset.shuffle().select(indices)
      __magic_name__     :     Any  							=					range(len(val_dataset) - len(val_dataset) % args.batch_size)
      __magic_name__     :     Dict  							=					val_dataset.shuffle().select(indices)

      if os.environ.get("""TRAIN_ON_SMALL""", """false""") == "true":
            __magic_name__     :     Any  							=					tr_dataset.shuffle().select(range(8_00_00))
            __magic_name__     :     Optional[int]  							=					val_dataset.shuffle().select(range(80_00))

      print(tr_dataset)
      print(val_dataset)

      __magic_name__     :     Union[str, Any]  							=					FlaxBigBirdForNaturalQuestions.from_pretrained(
          args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
      )
      __magic_name__     :     Tuple  							=					BigBirdTokenizerFast.from_pretrained(args.model_id)
      __magic_name__     :     Tuple  							=					DataCollator(pad_id=tokenizer.pad_token_id, max_length=40_96)

      __magic_name__     :     int  							=					{
          """lr""": args.lr,
          """init_lr""": args.init_lr,
          """warmup_steps""": args.warmup_steps,
          """num_train_steps""": args.max_epochs * (len(tr_dataset) // args.batch_size),
          """weight_decay""": args.weight_decay,
      }
      __magic_name__							,   __magic_name__     :     Optional[int]  							=					build_tx(**tx_args)

      __magic_name__     :     List[str]  							=					Trainer(
          args=args,
          data_collator=data_collator,
          model_save_fn=model.save_pretrained,
          train_step_fn=train_step,
          val_step_fn=val_step,
          logger=logger,
          scheduler_fn=lr,
      )

      __magic_name__     :     Union[str, Any]  							=					None
      __magic_name__     :     Dict  							=					trainer.create_state(model, tx, num_train_steps=tx_args["""num_train_steps"""], ckpt_dir=ckpt_dir)
      try:
            trainer.train(state, tr_dataset, val_dataset)
      except KeyboardInterrupt:
            print("""Oooops; TRAINING STOPPED UNFORTUNATELY""")

      print("""SAVING WEIGHTS IN `final-weights`""")
      __magic_name__     :     List[Any]  							=					jax_utils.unreplicate(state.params)
      model.save_pretrained(os.path.join(args.base_dir, """final-weights"""), params=params)

