



"""simple docstring"""







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
		UpperCAmelCase:  Union[str, Any]			     =				None


UpperCAmelCase:  int			     =				logging.get_logger(__name__)

UpperCAmelCase:  Dict			     =				{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase:  List[Any]			     =				{
    """vocab_file""": {
        """xlm-roberta-base""": """https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large""": """https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large-finetuned-conll02-dutch""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll02-spanish""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-english""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-german""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """xlm-roberta-base""": """https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json""",
        """xlm-roberta-large""": """https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json""",
        """xlm-roberta-large-finetuned-conll02-dutch""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json"""
        ),
        """xlm-roberta-large-finetuned-conll02-spanish""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json"""
        ),
        """xlm-roberta-large-finetuned-conll03-english""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json"""
        ),
        """xlm-roberta-large-finetuned-conll03-german""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCAmelCase:  Tuple			     =				{
    """xlm-roberta-base""": 512,
    """xlm-roberta-large""": 512,
    """xlm-roberta-large-finetuned-conll02-dutch""": 512,
    """xlm-roberta-large-finetuned-conll02-spanish""": 512,
    """xlm-roberta-large-finetuned-conll03-english""": 512,
    """xlm-roberta-large-finetuned-conll03-german""": 512,
}





class   UpperCamelCase	(     snake_case       ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_							:       Optional[Any]						       =  VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE_							:       Any						       =  PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE_							:       Optional[int]						       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE_							:       Union[str, Any]						       =  ["input_ids", "attention_mask"]
	SCREAMING_SNAKE_CASE_							:       Optional[Any]						       =  XLMRobertaTokenizer






	def __init__(     self				,UpperCAmelCase_=None				,UpperCAmelCase_=None				,UpperCAmelCase_="<s>"				,UpperCAmelCase_="</s>"				,UpperCAmelCase_="</s>"				,UpperCAmelCase_="<s>"				,UpperCAmelCase_="<unk>"				,UpperCAmelCase_="<pad>"				,UpperCAmelCase_="<mask>"				,**UpperCAmelCase_				,):
			# Mask token behave like a normal word, i.e. include the space before it
			_lowercase				: Optional[int]     =   AddedToken(UpperCAmelCase_				,lstrip=UpperCAmelCase_				,rstrip=UpperCAmelCase_   ) if isinstance(UpperCAmelCase_				,UpperCAmelCase_   ) else mask_token

			super().__init__(
			    UpperCAmelCase_				,tokenizer_file=UpperCAmelCase_				,bos_token=UpperCAmelCase_				,eos_token=UpperCAmelCase_				,sep_token=UpperCAmelCase_				,cls_token=UpperCAmelCase_				,unk_token=UpperCAmelCase_				,pad_token=UpperCAmelCase_				,mask_token=UpperCAmelCase_				,**UpperCAmelCase_				,)

			_lowercase				: Tuple     =   vocab_file
			_lowercase				: List[Any]     =   False if not self.vocab_file else True






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_ = None   ):

			if token_ids_a is None:
					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
			_lowercase				: int     =   [self.cls_token_id]
			_lowercase				: Tuple     =   [self.sep_token_id]
			return cls + token_ids_a + sep + sep + token_ids_a + sep






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_ = None   ):
			_lowercase				: Dict     =   [self.sep_token_id]
			_lowercase				: Dict     =   [self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep   ) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]






	def    lowerCamelCase__					(     self				,UpperCAmelCase_				,UpperCAmelCase_ = None   ):
			if not self.can_save_slow_tokenizer:
					raise ValueError(
					    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
					    """tokenizer."""   )

			if not os.path.isdir(UpperCAmelCase_   ):
					logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""   )
					return
			_lowercase				: Union[str, Any]     =   os.path.join(
			    UpperCAmelCase_				,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )

			if os.path.abspath(self.vocab_file   ) != os.path.abspath(UpperCAmelCase_   ):
					copyfile(self.vocab_file				,UpperCAmelCase_   )

			return (out_vocab_file,)







"""simple docstring"""







import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


__a      			= [nn.LayerNorm]

__a      			= logging.get_logger(__name__)

__a      			= version.parse(version.parse(torch.__version__).base_version)

__a      			= parsed_torch_version_base >= version.parse("2.0")
__a      			= parsed_torch_version_base >= version.parse("1.12")
__a      			= parsed_torch_version_base >= version.parse("1.11")
__a      			= parsed_torch_version_base < version.parse("1.11")






def        a__			(							_UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase						,      self			):





						"""simple docstring"""

						from torch import _softmax_backward_data

						if is_torch_less_than_1_11:
												return _softmax_backward_data(_UpperCamelCase						,      _UpperCamelCase						,      parent.dim						,      self			)
						else:
												return _softmax_backward_data(_UpperCamelCase						,      _UpperCamelCase						,      parent.dim						,      self.dtype			)






def        a__			(							_UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase = 0			):





						"""simple docstring"""

						UpperCAmelCase__												=     index.to(layer.weight.device			)
						UpperCAmelCase__												=     layer.weight.index_select(_UpperCamelCase						,      _UpperCamelCase			).clone().detach()
						if layer.bias is not None:
												if dim == 1:
																		UpperCAmelCase__												=     layer.bias.clone().detach()
												else:
																		UpperCAmelCase__												=     layer.bias[index].clone().detach()
						UpperCAmelCase__												=     list(layer.weight.size()			)
						UpperCAmelCase__												=     len(_UpperCamelCase			)
						UpperCAmelCase__												=     nn.Linear(new_size[1]						,      new_size[0]						,      bias=layer.bias is not None			).to(layer.weight.device			)
						UpperCAmelCase__												=     False
						new_layer.weight.copy_(W.contiguous()			)
						UpperCAmelCase__												=     True
						if layer.bias is not None:
												UpperCAmelCase__												=     False
												new_layer.bias.copy_(b.contiguous()			)
												UpperCAmelCase__												=     True
						return new_layer
class 				lowerCamelCase							(      nn.Module			):


							"""simple docstring"""






							def __init__(    self			:  str		,			lowerCamelCase_			:  Any		,			lowerCamelCase_			:  Dict					):
													super().__init__()
													UpperCAmelCase__												=     nf
													UpperCAmelCase__												=     nn.Parameter(torch.empty(lowerCamelCase_		,			lowerCamelCase_					)					)
													UpperCAmelCase__												=     nn.Parameter(torch.zeros(lowerCamelCase_					)					)
													nn.init.normal_(self.weight		,			std=0.02					)






							def        lowerCAmelCase  (    self			:  Any		,			lowerCamelCase_			:  Tuple					):
													UpperCAmelCase__												=     x.size()[:-1] + (self.nf,)
													UpperCAmelCase__												=     torch.addmm(self.bias		,			x.view(-1		,			x.size(-1					)					)		,			self.weight					)
													UpperCAmelCase__												=     x.view(lowerCamelCase_					)
													return x






def        a__			(							_UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase = 1			):





						"""simple docstring"""

						UpperCAmelCase__												=     index.to(layer.weight.device			)
						UpperCAmelCase__												=     layer.weight.index_select(_UpperCamelCase						,      _UpperCamelCase			).clone().detach()
						if dim == 0:
												UpperCAmelCase__												=     layer.bias.clone().detach()
						else:
												UpperCAmelCase__												=     layer.bias[index].clone().detach()
						UpperCAmelCase__												=     list(layer.weight.size()			)
						UpperCAmelCase__												=     len(_UpperCamelCase			)
						UpperCAmelCase__												=     ConvaD(new_size[1]						,      new_size[0]			).to(layer.weight.device			)
						UpperCAmelCase__												=     False
						new_layer.weight.copy_(W.contiguous()			)
						UpperCAmelCase__												=     True
						UpperCAmelCase__												=     False
						new_layer.bias.copy_(b.contiguous()			)
						UpperCAmelCase__												=     True
						return new_layer






def        a__			(							_UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase = None			):





						"""simple docstring"""

						if isinstance(_UpperCamelCase						,      nn.Linear			):
												return prune_linear_layer(_UpperCamelCase						,      _UpperCamelCase						,      dim=0 if dim is None else dim			)
						elif isinstance(_UpperCamelCase						,      _UpperCamelCase			):
												return prune_convad_layer(_UpperCamelCase						,      _UpperCamelCase						,      dim=1 if dim is None else dim			)
						else:
												raise ValueError(F"""Can't prune layer of class {layer.__class__}"""			)






def        a__			(							_UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase						,      *_UpperCamelCase			):





						"""simple docstring"""

						assert len(_UpperCamelCase			) > 0, F"""{input_tensors} has to be a tuple/list of tensors"""

						# inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
						UpperCAmelCase__												=     len(inspect.signature(_UpperCamelCase			).parameters			)
						if num_args_in_forward_chunk_fn != len(_UpperCamelCase			):
												raise ValueError(
												    F"""forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(_UpperCamelCase			)} input """
												    '''tensors are given'''			)

						if chunk_size > 0:
												UpperCAmelCase__												=     input_tensors[0].shape[chunk_dim]
												for input_tensor in input_tensors:
																		if input_tensor.shape[chunk_dim] != tensor_shape:
																								raise ValueError(
																								    F"""All input tenors have to be of the same shape: {tensor_shape}, """
																								    F"""found shape {input_tensor.shape[chunk_dim]}"""			)

												if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
																		raise ValueError(
																		    F"""The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk """
																		    F"""size {chunk_size}"""			)

												UpperCAmelCase__												=     input_tensors[0].shape[chunk_dim] // chunk_size

												# chunk input tensor into tuples
												UpperCAmelCase__												=     tuple(input_tensor.chunk(_UpperCamelCase						,      dim=_UpperCamelCase			) for input_tensor in input_tensors			)
												# apply forward fn to every tuple
												UpperCAmelCase__												=     tuple(forward_fn(*_UpperCamelCase			) for input_tensors_chunk in zip(*_UpperCamelCase			)			)
												# concatenate output at same dimension
												return torch.cat(_UpperCamelCase						,      dim=_UpperCamelCase			)

						return forward_fn(*_UpperCamelCase			)






def        a__			(							_UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase			):





						"""simple docstring"""

						UpperCAmelCase__												=     torch.ones(_UpperCamelCase						,      _UpperCamelCase			)
						UpperCAmelCase__												=     set(_UpperCamelCase			) - already_pruned_heads  # Convert to set and remove already pruned heads
						for head in heads:
												# Compute how many pruned heads are before the head and move the index accordingly
												UpperCAmelCase__												=     head - sum(1 if h < head else 0 for h in already_pruned_heads			)
												UpperCAmelCase__												=     0
						UpperCAmelCase__												=     mask.view(-1			).contiguous().eq(1			)
						UpperCAmelCase__												=     torch.arange(len(_UpperCamelCase			)			)[mask].long()
						return heads, index






def        a__			(							*_UpperCamelCase						,      _UpperCamelCase = None			):





						"""simple docstring"""

						return torch.meshgrid(*_UpperCamelCase						,      indexing=_UpperCamelCase			)






def        a__			(							_UpperCamelCase			):





						"""simple docstring"""

						return tensor.device, storage_ptr(_UpperCamelCase			), storage_size(_UpperCamelCase			)