



'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


_UpperCamelCase  						=			logging.get_logger(__name__)

_UpperCamelCase  						=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCamelCase  						=			{
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """allenai/longformer-base-4096""": (
            """https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
    },
}

_UpperCamelCase  						=			{
    """allenai/longformer-base-4096""": 4096,
    """allenai/longformer-large-4096""": 4096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4096,
}

class      lowerCamelCase__     (     _A				):






	'''simple docstring'''

	A__		=	VOCAB_FILES_NAMES
	A__		=	PRETRAINED_VOCAB_FILES_MAP
	A__		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A__		=	['''input_ids''', '''attention_mask''']
	A__		=	LongformerTokenizer




	def __init__(  self		:  List[Any]   ,		__A		:  List[str]=None   ,		__A		:  List[Any]=None   ,		__A		:  Union[str, Any]=None   ,		__A		:  List[Any]="replace"   ,		__A		:  str="<s>"   ,		__A		:  Tuple="</s>"   ,		__A		:  List[str]="</s>"   ,		__A		:  Tuple="<s>"   ,		__A		:  int="<unk>"   ,		__A		:  List[Any]="<pad>"   ,		__A		:  str="<mask>"   ,		__A		:  Optional[int]=False   ,		__A		:  Tuple=True   ,		**__A		:  Union[str, Any]   ,		)     ->			int:




								'''simple docstring'''





								super().__init__(
								    __A   ,		__A   ,		tokenizer_file=__A   ,		errors=__A   ,		bos_token=__A   ,		eos_token=__A   ,		sep_token=__A   ,		cls_token=__A   ,		unk_token=__A   ,		pad_token=__A   ,		mask_token=__A   ,		add_prefix_space=__A   ,		trim_offsets=__A   ,		**__A   ,		)

								lowerCAmelCase__  =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
								if pre_tok_state.get("""add_prefix_space"""   ,		__A					) != add_prefix_space:
															lowerCAmelCase__  =   getattr(__A   ,		pre_tok_state.pop("""type"""					)					)
															lowerCAmelCase__  =   add_prefix_space
															lowerCAmelCase__  =   pre_tok_class(**__A					)

								lowerCAmelCase__  =   add_prefix_space

								lowerCAmelCase__  =   """post_processor"""
								lowerCAmelCase__  =   getattr(self.backend_tokenizer   ,		__A   ,		__A					)
								if tokenizer_component_instance:
															lowerCAmelCase__  =   json.loads(tokenizer_component_instance.__getstate__()					)

															# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
															if "sep" in state:
																						lowerCAmelCase__  =   tuple(state["""sep"""]					)
															if "cls" in state:
																						lowerCAmelCase__  =   tuple(state["""cls"""]					)

															lowerCAmelCase__  =   False

															if state.get("""add_prefix_space"""   ,		__A					) != add_prefix_space:
																						lowerCAmelCase__  =   add_prefix_space
																						lowerCAmelCase__  =   True

															if state.get("""trim_offsets"""   ,		__A					) != trim_offsets:
																						lowerCAmelCase__  =   trim_offsets
																						lowerCAmelCase__  =   True

															if changes_to_apply:
																						lowerCAmelCase__  =   getattr(__A   ,		state.pop("""type"""					)					)
																						lowerCAmelCase__  =   component_class(**__A					)
																						setattr(self.backend_tokenizer   ,		__A   ,		__A					)




	@property
	def      lowercase__							(  self		:  Any					)     ->			str:




								'''simple docstring'''





								if self._mask_token is None:
															if self.verbose:
																						logger.error("""Using mask_token, but it is not set yet."""					)
															return None
								return str(self._mask_token					)




	@mask_token.setter
	def      lowercase__							(  self		:  str   ,		__A		:  Dict					)     ->			Optional[Any]:




								'''simple docstring'''





								lowerCAmelCase__  =   AddedToken(__A   ,		lstrip=__A   ,		rstrip=__A					) if isinstance(__A   ,		__A					) else value
								lowerCAmelCase__  =   value




	def      lowercase__							(  self		:  List[Any]   ,		*__A		:  str   ,		**__A		:  Dict					)     ->			BatchEncoding:




								'''simple docstring'''





								lowerCAmelCase__  =   kwargs.get("""is_split_into_words"""   ,		__A					)
								assert self.add_prefix_space or not is_split_into_words, (
								    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*__A   ,		**__A					)




	def      lowercase__							(  self		:  Dict   ,		*__A		:  Optional[int]   ,		**__A		:  Optional[Any]					)     ->			BatchEncoding:




								'''simple docstring'''





								lowerCAmelCase__  =   kwargs.get("""is_split_into_words"""   ,		__A					)

								assert self.add_prefix_space or not is_split_into_words, (
								    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*__A   ,		**__A					)




	def      lowercase__							(  self		:  Optional[Any]   ,		__A		:  str   ,		__A		:  Optional[str] = None					)     ->			Tuple[str]:




								'''simple docstring'''





								lowerCAmelCase__  =   self._tokenizer.model.save(__A   ,		name=__A					)
								return tuple(__A					)




	def      lowercase__							(  self		:  Any   ,		__A		:  List[Any]   ,		__A		:  str=None					)     ->			Union[str, Any]:




								'''simple docstring'''





								lowerCAmelCase__  =   [self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
															return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




	def      lowercase__							(  self		:  Union[str, Any]   ,		__A		:  List[int]   ,		__A		:  Optional[List[int]] = None					)     ->			List[int]:




								'''simple docstring'''





								lowerCAmelCase__  =   [self.sep_token_id]
								lowerCAmelCase__  =   [self.cls_token_id]

								if token_ids_a is None:
															return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]










'''simple docstring'''







from ..utils import DummyObject, requires_backends

class      lowerCamelCase__     (     metaclass=_A				):






	'''simple docstring'''

	A__		=	['''torch''', '''librosa''']




	def __init__(  self		:  Optional[Any]   ,		*__A		:  Tuple   ,		**__A		:  Tuple					)     ->			Union[str, Any]:




								'''simple docstring'''





								requires_backends(self   ,		["""torch""", """librosa"""]					)




	@classmethod
	def      lowercase__							(  cls		:  Dict   ,		*__A		:  List[Any]   ,		**__A		:  List[str]					)     ->			Union[str, Any]:




								'''simple docstring'''





								requires_backends(cls   ,		["""torch""", """librosa"""]					)




	@classmethod
	def      lowercase__							(  cls		:  Optional[int]   ,		*__A		:  List[str]   ,		**__A		:  Tuple					)     ->			Any:




								'''simple docstring'''





								requires_backends(cls   ,		["""torch""", """librosa"""]					)





class      lowerCamelCase__     (     metaclass=_A				):






	'''simple docstring'''

	A__		=	['''torch''', '''librosa''']




	def __init__(  self		:  int   ,		*__A		:  Dict   ,		**__A		:  List[Any]					)     ->			List[Any]:




								'''simple docstring'''





								requires_backends(self   ,		["""torch""", """librosa"""]					)




	@classmethod
	def      lowercase__							(  cls		:  Optional[int]   ,		*__A		:  Optional[int]   ,		**__A		:  List[Any]					)     ->			Union[str, Any]:




								'''simple docstring'''





								requires_backends(cls   ,		["""torch""", """librosa"""]					)




	@classmethod
	def      lowercase__							(  cls		:  List[str]   ,		*__A		:  List[str]   ,		**__A		:  int					)     ->			List[str]:




								'''simple docstring'''





								requires_backends(cls   ,		["""torch""", """librosa"""]					)





