
import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


SCREAMING_SNAKE_CASE							:Dict											=       logging.getLogger(__name__)


def 		UpperCAmelCase    (			)	->			int:







				"""simple docstring"""



				__A         =				argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."		)
				parser.add_argument(
				    "--dataset_name"	,     type=a_	,     default="wikitext"	,     help="Name of the training. Explore datasets at: hf.co/datasets."	,     )
				parser.add_argument(
				    "--dataset_config"	,     type=a_	,     default="wikitext-103-raw-v1"	,     help="Configuration name of the dataset."		)
				parser.add_argument(
				    "--batch_size"	,     type=a_	,     default=1_0_0_0	,     help="Batch size during training."	,     )
				parser.add_argument(
				    "--vocab_size"	,     type=a_	,     default=1_0_0_4_8	,     help="Size of the desired vocabulary."	,     )
				parser.add_argument(
				    "--limit"	,     default=a_	,     type=a_	,     help="Limit the number of shards (used for debugging)."	,     )
				parser.add_argument(
				    "--export_to_hub"	,     action="store_true"	,     )

				__A         =				parser.parse_args()
				return args


def 		UpperCAmelCase    (			a_		)	->			Optional[Any]:







				"""simple docstring"""



				__A         =				datasets.load_dataset(args.dataset_name	,     args.dataset_config	,     split="train"		)

				if args.limit is not None:
								__A         =				min(len(a_		)	,     args.limit		)
								__A         =				dataset.select(range(a_		)		)
								logger.info(F'''Limiting the dataset to {args.limit} entries.'''		)

				def batch_iterator():
								for i in range(0	,     len(a_		)	,     args.batch_size		):
												yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
				__A         =				Tokenizer(Unigram()		)
				__A         =				normalizers.Sequence([normalizers.Replace("``"	,     "\""		), normalizers.Replace("''"	,     "\""		)]		)
				__A         =				pre_tokenizers.Metaspace()

				# Prepare the trainer.
				__A         =				UnigramTrainer(
				    unk_token="<unk>"	,     special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"]	,     vocab_size=args.vocab_size	,     )

				logger.info("Training the tokenizer."		)
				tokenizer.train_from_iterator(batch_iterator()	,     trainer=a_		)
				logger.info("Tokenizer training complete!"		)

				__A         =				tokenizer.token_to_id("[CLS]"		)
				__A         =				tokenizer.token_to_id("[SEP]"		)
				__A         =				processors.TemplateProcessing(
				    single="[CLS]:0 $A:0 [SEP]:0"	,     pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"	,     special_tokens=[
				        ("[CLS]", cls_token_id),
				        ("[SEP]", sep_token_id),
				    ]	,     )
				__A         =				decoders.Metaspace()

				if args.export_to_hub:
								logger.info("Exporting the trained tokenzier to Hub."		)
								__A         =				AlbertTokenizerFast(tokenizer_object=a_		)
								new_tokenizer.push_to_hub("unigram-tokenizer-dataset"		)


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE							:int											=       parse_args()
					main(args)


import unittest

from transformers import SpeechaTextaConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers.models.speech_to_text_a.modeling_speech_to_text_a import (
					    SpeechaTextaDecoder,
					    SpeechaTextaForCausalLM,
					)





@require_torch
class 							UpperCAmelCase   :




	'''simple docstring'''





	def __init__( self      :				Dict							,A      :				Tuple							,A      :				Union[str, Any]=99							,A      :				Dict=13							,A      :				int=16							,A      :				int=7							,A      :				Optional[int]=True							,A      :				str=True							,A      :				Optional[Any]=True							,A      :				Optional[int]=False							,A      :				Dict=True							,A      :				Any=2							,A      :				List[Any]=32							,A      :				Optional[Any]=4							,A      :				Tuple=4							,A      :				List[Any]=30							,A      :				Optional[Any]=0							,A      :				str=1							,A      :				List[Any]=2							,A      :				List[Any]=None							,):
					__A         =				parent
					__A         =				batch_size
					__A         =				decoder_seq_length
					# For common tests
					__A         =				self.decoder_seq_length
					__A         =				is_training
					__A         =				use_attention_mask
					__A         =				use_labels

					__A         =				vocab_size
					__A         =				d_model
					__A         =				d_model
					__A         =				decoder_layers
					__A         =				decoder_layers
					__A         =				decoder_ffn_dim
					__A         =				decoder_attention_heads
					__A         =				decoder_attention_heads
					__A         =				eos_token_id
					__A         =				bos_token_id
					__A         =				pad_token_id
					__A         =				decoder_start_token_id
					__A         =				use_cache
					__A         =				max_position_embeddings

					__A         =				None
					__A         =				decoder_seq_length
					__A         =				2
					__A         =				1






	def        UpperCamelCase_						( self      :				Tuple				):
					__A         =				ids_tensor([self.batch_size, self.decoder_seq_length]							,self.vocab_size				)

					__A         =				None
					if self.use_attention_mask:
									__A         =				ids_tensor([self.batch_size, self.decoder_seq_length]							,vocab_size=2				)

					__A         =				None
					if self.use_labels:
									__A         =				ids_tensor([self.batch_size, self.decoder_seq_length]							,self.vocab_size				)

					__A         =				SpeechaTextaConfig(
					    vocab_size=self.vocab_size							,d_model=self.d_model							,decoder_layers=self.decoder_layers							,decoder_ffn_dim=self.decoder_ffn_dim							,decoder_attention_heads=self.decoder_attention_heads							,eos_token_id=self.eos_token_id							,bos_token_id=self.bos_token_id							,use_cache=self.use_cache							,pad_token_id=self.pad_token_id							,decoder_start_token_id=self.decoder_start_token_id							,max_position_embeddings=self.max_position_embeddings							,)

					return (
					    config,
					    input_ids,
					    attention_mask,
					    lm_labels,
					)






	def        UpperCamelCase_						( self      :				Tuple							,A      :				List[str]							,A      :				Tuple							,A      :				Optional[int]							,A      :				List[str]							,):
					__A         =				True
					__A         =				SpeechaTextaDecoder(config=A				).to(A				).eval()
					__A         =				input_ids[:2]

					input_ids[input_ids == 0] += 1
					# first forward pass
					__A         =				model(A							,use_cache=A				)
					__A         =				model(A				)
					__A         =				model(A							,use_cache=A				)

					self.parent.assertTrue(len(A				) == len(A				)				)
					self.parent.assertTrue(len(A				) == len(A				) + 1				)

					__A         =				outputs["past_key_values"]

					# create hypothetical next token and extent to next_input_ids
					__A         =				ids_tensor((2, 1)							,config.vocab_size - 1				) + 1

					# append to next input_ids and
					__A         =				torch.cat([input_ids, next_tokens]							,dim=-1				)

					print(A				)

					__A         =				model(A				)["last_hidden_state"]
					__A         =				model(A							,past_key_values=A				)["last_hidden_state"]

					# select random slice
					__A         =				ids_tensor((1,)							,output_from_past.shape[-1]				).item()
					__A         =				output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
					__A         =				output_from_past[:, 0, random_slice_idx].detach()

					# test that outputs are equal for slice
					assert torch.allclose(A							,A							,atol=1E-3				)







	def        UpperCamelCase_						( self      :				Any				):
					__A         =				self.prepare_config_and_inputs()
					(
					    (
					    __A
					)    ,							(
					    __A
					)    ,							(
					    __A
					)    ,							(
					    __A
					)    ,							
					)         =				config_and_inputs

					__A         =				{
					    "input_ids": input_ids,
					    "attention_mask": attention_mask,
					}
					return config, inputs_dict





@require_torch
class 							UpperCAmelCase   (						__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE    ,    unittest.TestCase						):




	'''simple docstring'''





	snake_case_      					=  (SpeechaTextaDecoder, SpeechaTextaForCausalLM) if is_torch_available() else ()
	snake_case_      					=  (SpeechaTextaForCausalLM,) if is_torch_available() else ()
	snake_case_      					=  {"text-generation": SpeechaTextaForCausalLM} if is_torch_available() else {}
	snake_case_      					=  True
	snake_case_      					=  False






	def        UpperCamelCase_						( self      :				Any							,):
					__A         =				SpeechaTextaStandaloneDecoderModelTester(self							,is_training=A				)
					__A         =				ConfigTester(self							,config_class=A				)






	def        UpperCamelCase_						( self      :				Optional[Any]				):
					pass






	def        UpperCamelCase_						( self      :				int				):
					pass






	def        UpperCamelCase_						( self      :				Tuple				):
					pass






	def        UpperCamelCase_						( self      :				str				):
					self.config_tester.run_common_tests()






	def        UpperCamelCase_						( self      :				Dict				):
					__A         =				self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_decoder_model_past(*A				)







	def        UpperCamelCase_						( self      :				str				):
					return
