





'''simple docstring'''






import warnings
from dataclasses import dataclass, field
from typing import Optional, Tuple

from .training_args import TrainingArguments
from .utils import cached_property, is_tf_available, logging, requires_backends


__UpperCamelCase      :   str								=     logging.get_logger(__name__)

if is_tf_available():
	import tensorflow as tf



@dataclass
class  lowerCamelCase__				(						snake_case__						):


	"""simple docstring"""





	__magic_name__							   =		"""tf"""
	__magic_name__							   =		field(
	    default=snake_case__  ,							metadata={"""help""": """Name of TPU"""}  ,							)

	__magic_name__							   =		field(
	    default=snake_case__  ,							metadata={"""help""": """Zone of TPU"""}  ,							)

	__magic_name__							   =		field(
	    default=snake_case__  ,							metadata={"""help""": """Name of Cloud TPU-enabled project"""}  ,							)

	__magic_name__							   =		field(
	    default=1.0  ,							metadata={"""help""": """Power for the Polynomial decay LR scheduler."""}  ,							)

	__magic_name__							   =		field(default=snake_case__  ,							metadata={"""help""": """Whether to activate the XLA compilation or not"""}						)




	@cached_property
	def 				_lowerCamelCase				(					self			)       ->				Dict:
					requires_backends(self   ,     ['''tf''']			)
					logger.info('''Tensorflow: setting up strategy'''			)

					_A :				Any				=  tf.config.list_physical_devices('''GPU'''			)

					# Set to float16 at first
					if self.fpaa:
									tf.keras.mixed_precision.set_global_policy('''mixed_float16'''			)

					if self.no_cuda:
									_A :				int				=  tf.distribute.OneDeviceStrategy(device='''/cpu:0'''			)
					else:
									try:
													if self.tpu_name:
																	_A :				Optional[int]				=  tf.distribute.cluster_resolver.TPUClusterResolver(
																	    self.tpu_name   ,     zone=self.tpu_zone   ,     project=self.gcp_project			)
													else:
																	_A :				List[Any]				=  tf.distribute.cluster_resolver.TPUClusterResolver()
									except ValueError:
													if self.tpu_name:
																	raise RuntimeError(F"""Couldn't connect to TPU {self.tpu_name}!"""			)
													else:
																	_A :				Optional[int]				=  None

									if tpu:
													# Set to bfloat16 in case of TPU
													if self.fpaa:
																	tf.keras.mixed_precision.set_global_policy('''mixed_bfloat16'''			)

													tf.config.experimental_connect_to_cluster(UpperCAmelCase_			)
													tf.tpu.experimental.initialize_tpu_system(UpperCAmelCase_			)

													_A :				Tuple				=  tf.distribute.TPUStrategy(UpperCAmelCase_			)

									elif len(UpperCAmelCase_			) == 0:
													_A :				List[Any]				=  tf.distribute.OneDeviceStrategy(device='''/cpu:0'''			)
									elif len(UpperCAmelCase_			) == 1:
													_A :				Tuple				=  tf.distribute.OneDeviceStrategy(device='''/gpu:0'''			)
									elif len(UpperCAmelCase_			) > 1:
													# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
													_A :				List[Any]				=  tf.distribute.MirroredStrategy()
									else:
													raise ValueError('''Cannot find the proper strategy, please check your environment properties.'''			)

					return strategy




	@property
	def 				_lowerCamelCase				(					self			)       ->				Union[str, Any]:
					requires_backends(self   ,     ['''tf''']			)
					return self._setup_strategy




	@property
	def 				_lowerCamelCase				(					self			)       ->				List[str]:
					requires_backends(self   ,     ['''tf''']			)
					return self._setup_strategy.num_replicas_in_sync




	@property
	def 				_lowerCamelCase				(					self			)       ->				Optional[Any]:
					return False  # TF Logging is handled by Keras not the Trainer




	@property
	def 				_lowerCamelCase				(					self			)       ->				int:
					if self.per_gpu_train_batch_size:
									logger.warning(
									    '''Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future '''
									    '''version. Using `--per_device_train_batch_size` is preferred.'''			)
					_A :				Optional[int]				=  self.per_gpu_train_batch_size or self.per_device_train_batch_size
					return per_device_batch_size * self.n_replicas




	@property
	def 				_lowerCamelCase				(					self			)       ->				int:
					if self.per_gpu_eval_batch_size:
									logger.warning(
									    '''Using deprecated `--per_gpu_eval_batch_size` argument which will be removed in a future '''
									    '''version. Using `--per_device_eval_batch_size` is preferred.'''			)
					_A :				Optional[int]				=  self.per_gpu_eval_batch_size or self.per_device_eval_batch_size
					return per_device_batch_size * self.n_replicas




	@property
	def 				_lowerCamelCase				(					self			)       ->				Tuple:
					requires_backends(self   ,     ['''tf''']			)
					warnings.warn(
					    '''The n_gpu argument is deprecated and will be removed in a future version, use n_replicas instead.'''   ,     UpperCAmelCase_   ,     )
					return self._setup_strategy.num_replicas_in_sync







'''simple docstring'''

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_xglm import XGLMTokenizer
else:
			lowerCAmelCase__      :   List[str]      	=			None


lowerCAmelCase__      :   Union[str, Any]      	=			logging.get_logger(__name__)

lowerCAmelCase__      :   Tuple      	=			{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

lowerCAmelCase__      :   List[str]      	=			{
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json",
    },
}

lowerCAmelCase__      :   int      	=			{
    "facebook/xglm-564M": 20_48,
}


class      SCREAMING_SNAKE_CASE__      (   snake_case__		):







					"""simple docstring"""






					SCREAMING_SNAKE_CASE							=	VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE							=	PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE							=	['''input_ids''', '''attention_mask''']
					SCREAMING_SNAKE_CASE							=	XGLMTokenizer
					def __init__(   self       :      Optional[int]		,	UpperCAmelCase_       :      List[Any]=None		,	UpperCAmelCase_       :      str=None		,	UpperCAmelCase_       :      str="<s>"		,	UpperCAmelCase_       :      List[Any]="</s>"		,	UpperCAmelCase_       :      Dict="</s>"		,	UpperCAmelCase_       :      Optional[Any]="<s>"		,	UpperCAmelCase_       :      List[str]="<unk>"		,	UpperCAmelCase_       :      Union[str, Any]="<pad>"		,	**UpperCAmelCase_       :      List[Any]		,	):







							"""simple docstring"""
							# Compatibility with the original tokenizer
							__UpperCAmelCase	:							Union[str, Any]     =     7
							__UpperCAmelCase	:							Tuple     =     [f"<madeupword{i}>" for i in range(self.num_madeup_words		)]

							__UpperCAmelCase	:							List[Any]     =     kwargs.get("additional_special_tokens"		,	[]		)
							kwargs["additional_special_tokens"] += [
							    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    UpperCAmelCase_		,	tokenizer_file=UpperCAmelCase_		,	bos_token=UpperCAmelCase_		,	eos_token=UpperCAmelCase_		,	sep_token=UpperCAmelCase_		,	cls_token=UpperCAmelCase_		,	unk_token=UpperCAmelCase_		,	pad_token=UpperCAmelCase_		,	**UpperCAmelCase_		,	)

							__UpperCAmelCase	:							Dict     =     vocab_file
							__UpperCAmelCase	:							Optional[int]     =     False if not self.vocab_file else True
					def 		lowerCamelCase_   (   self       :      int		,	UpperCAmelCase_       :      List[int]		,	UpperCAmelCase_       :      Optional[List[int]] = None		):







							"""simple docstring"""

							if token_ids_a is None:
									return [self.sep_token_id] + token_ids_a
							__UpperCAmelCase	:							Any     =     [self.sep_token_id]
							return sep + token_ids_a + sep + sep + token_ids_a
					def 		lowerCamelCase_   (   self       :      Optional[Any]		,	UpperCAmelCase_       :      List[int]		,	UpperCAmelCase_       :      Optional[List[int]] = None		):







							"""simple docstring"""
							__UpperCAmelCase	:							List[Any]     =     [self.sep_token_id]

							if token_ids_a is None:
									return len(sep + token_ids_a		) * [0]
							return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]






					def 		lowerCamelCase_   (   self       :      List[Any]		,	UpperCAmelCase_       :      str		,	UpperCAmelCase_       :      Optional[str] = None		):







							"""simple docstring"""
							if not self.can_save_slow_tokenizer:
									raise ValueError(
									    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
									    "tokenizer."		)

							if not os.path.isdir(UpperCAmelCase_		):
									logger.error(f"Vocabulary path ({save_directory}) should be a directory."		)
									return
							__UpperCAmelCase	:							str     =     os.path.join(
							    UpperCAmelCase_		,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

							if os.path.abspath(self.vocab_file		) != os.path.abspath(UpperCAmelCase_		):
									copyfile(self.vocab_file		,	UpperCAmelCase_		)

							return (out_vocab_file,)




