




'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


lowerCAmelCase_    	=							logging.get_logger(__name__)

lowerCAmelCase_    	=							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase_    	=							{
    '''vocab_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json''',
    },
}

lowerCAmelCase_    	=							{
    '''microsoft/mpnet-base''': 5_1_2,
}

lowerCAmelCase_    	=							{
    '''microsoft/mpnet-base''': {'''do_lower_case''': True},
}

class   _snake_case( UpperCAmelCase			):
 __snake_case:       Optional[Any]									=				VOCAB_FILES_NAMES
 __snake_case:       str									=				PRETRAINED_VOCAB_FILES_MAP
 __snake_case:       List[str]									=				PRETRAINED_INIT_CONFIGURATION
 __snake_case:       str									=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __snake_case:       List[str]									=				MPNetTokenizer
 __snake_case:       Union[str, Any]									=				['''input_ids''', '''attention_mask''']






 def __init__(self					:     Union[str, Any]						,		a					:     Optional[int]=None						,		a					:     List[str]=None						,		a					:     List[Any]=True						,		a					:     List[Any]="<s>"						,		a					:     List[str]="</s>"						,		a					:     List[str]="</s>"						,		a					:     Union[str, Any]="<s>"						,		a					:     str="[UNK]"						,		a					:     str="<pad>"						,		a					:     int="<mask>"						,		a					:     Any=True						,		a					:     Any=None						,		**a					:     Union[str, Any]						,		)					->      List[Any]:






    """simple docstring"""


    super().__init__(
        a						,		tokenizer_file=a						,		do_lower_case=a						,		bos_token=a						,		eos_token=a						,		sep_token=a						,		cls_token=a						,		unk_token=a						,		pad_token=a						,		mask_token=a						,		tokenize_chinese_chars=a						,		strip_accents=a						,		**a						,		)

    A__         =    json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
    if (
        pre_tok_state.get("lowercase"						,		a						) != do_lower_case
        or pre_tok_state.get("strip_accents"						,		a						) != strip_accents
    ):
       A__         =    getattr(a						,		pre_tok_state.pop("type"						)						)
       A__         =    do_lower_case
       A__         =    strip_accents
       A__         =    pre_tok_class(**a						)

    A__         =    do_lower_case






 @property
 def 	_UpperCamelCase   (self					:     int						)					->      str:






    """simple docstring"""


    if self._mask_token is None:
       if self.verbose:
          logger.error("Using mask_token, but it is not set yet."						)
       return None
    return str(self._mask_token						)






 @mask_token.setter
 def 	_UpperCamelCase   (self					:     int						,		a					:     Optional[int]						)					->      Dict:






    """simple docstring"""


    A__         =    AddedToken(a						,		lstrip=a						,		rstrip=a						) if isinstance(a						,		a						) else value
    A__         =    value






 def 	_UpperCamelCase   (self					:     Tuple						,		a					:     Optional[Any]						,		a					:     str=None						)					->      str:






    """simple docstring"""


    A__         =    [self.bos_token_id] + token_ids_a + [self.eos_token_id]
    if token_ids_a is None:
       return output

    return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






 def 	_UpperCamelCase   (self					:     Tuple						,		a					:     List[int]						,		a					:     Optional[List[int]] = None						)					->      List[int]:






    """simple docstring"""


    A__         =    [self.sep_token_id]
    A__         =    [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep						) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]






 def 	_UpperCamelCase   (self					:     int						,		a					:     str						,		a					:     Optional[str] = None						)					->      Tuple[str]:






    """simple docstring"""


    A__         =    self._tokenizer.model.save(a						,		name=a						)
    return tuple(a						)








'''simple docstring'''







import warnings
from dataclasses import dataclass, field
from typing import Optional, Tuple

from .training_args import TrainingArguments
from .utils import cached_property, is_tf_available, logging, requires_backends


lowerCAmelCase_    	=							logging.get_logger(__name__)

if is_tf_available():
    import tensorflow as tf

@dataclass
class   _snake_case( UpperCAmelCase			):
 __snake_case:       Optional[Any]									=				'''tf'''
 __snake_case:       Optional[str]									=				field(
     default=UpperCAmelCase					,   metadata={'''help''': '''Name of TPU'''}					,   )

 __snake_case:       Optional[str]									=				field(
     default=UpperCAmelCase					,   metadata={'''help''': '''Zone of TPU'''}					,   )

 __snake_case:       Optional[str]									=				field(
     default=UpperCAmelCase					,   metadata={'''help''': '''Name of Cloud TPU-enabled project'''}					,   )

 __snake_case:       float									=				field(
     default=1.0					,   metadata={'''help''': '''Power for the Polynomial decay LR scheduler.'''}					,   )

 __snake_case:       bool									=				field(default=UpperCAmelCase					,   metadata={'''help''': '''Whether to activate the XLA compilation or not'''}			)






 @cached_property
 def 	_UpperCamelCase   (self					:     Tuple						)					->      Tuple["tf.distribute.Strategy", int]:






    """simple docstring"""


    requires_backends(self						,		['tf']						)
    logger.info('Tensorflow: setting up strategy'						)

    A__         =    tf.config.list_physical_devices('GPU'						)

    # Set to float16 at first
    if self.fpaa:
       tf.keras.mixed_precision.set_global_policy('mixed_float16'						)

    if self.no_cuda:
       A__         =    tf.distribute.OneDeviceStrategy(device='/cpu:0'						)
    else:
       try:
          if self.tpu_name:
             A__         =    tf.distribute.cluster_resolver.TPUClusterResolver(
                 self.tpu_name						,		zone=self.tpu_zone						,		project=self.gcp_project						)
          else:
             A__         =    tf.distribute.cluster_resolver.TPUClusterResolver()
       except ValueError:
          if self.tpu_name:
             raise RuntimeError(f"""Couldn't connect to TPU {self.tpu_name}!"""						)
          else:
             A__         =    None

       if tpu:
          # Set to bfloat16 in case of TPU
          if self.fpaa:
             tf.keras.mixed_precision.set_global_policy('mixed_bfloat16'						)

          tf.config.experimental_connect_to_cluster(a						)
          tf.tpu.experimental.initialize_tpu_system(a						)

          A__         =    tf.distribute.TPUStrategy(a						)

       elif len(a						) == 0:
          A__         =    tf.distribute.OneDeviceStrategy(device='/cpu:0'						)
       elif len(a						) == 1:
          A__         =    tf.distribute.OneDeviceStrategy(device='/gpu:0'						)
       elif len(a						) > 1:
          # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
          A__         =    tf.distribute.MirroredStrategy()
       else:
          raise ValueError('Cannot find the proper strategy, please check your environment properties.'						)

    return strategy






 @property
 def 	_UpperCamelCase   (self					:     List[Any]						)					->      "tf.distribute.Strategy":






    """simple docstring"""


    requires_backends(self						,		['tf']						)
    return self._setup_strategy






 @property
 def 	_UpperCamelCase   (self					:     Tuple						)					->      int:






    """simple docstring"""


    requires_backends(self						,		['tf']						)
    return self._setup_strategy.num_replicas_in_sync






 @property
 def 	_UpperCamelCase   (self					:     Optional[Any]						)					->      int:






    """simple docstring"""


    return False  # TF Logging is handled by Keras not the Trainer






 @property
 def 	_UpperCamelCase   (self					:     Union[str, Any]						)					->      int:






    """simple docstring"""


    if self.per_gpu_train_batch_size:
       logger.warning(
           'Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future '
           'version. Using `--per_device_train_batch_size` is preferred.'						)
    A__         =    self.per_gpu_train_batch_size or self.per_device_train_batch_size
    return per_device_batch_size * self.n_replicas






 @property
 def 	_UpperCamelCase   (self					:     Optional[int]						)					->      int:






    """simple docstring"""


    if self.per_gpu_eval_batch_size:
       logger.warning(
           'Using deprecated `--per_gpu_eval_batch_size` argument which will be removed in a future '
           'version. Using `--per_device_eval_batch_size` is preferred.'						)
    A__         =    self.per_gpu_eval_batch_size or self.per_device_eval_batch_size
    return per_device_batch_size * self.n_replicas






 @property
 def 	_UpperCamelCase   (self					:     Optional[Any]						)					->      int:






    """simple docstring"""


    requires_backends(self						,		['tf']						)
    warnings.warn(
        'The n_gpu argument is deprecated and will be removed in a future version, use n_replicas instead.'						,		a						,		)
    return self._setup_strategy.num_replicas_in_sync


