


import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


lowerCamelCase		=	logging.get_logger(__name__)

lowerCamelCase		=	{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase		=	{
    '''vocab_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''allenai/longformer-base-4096''': (
            '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCamelCase		=	{
    '''allenai/longformer-base-4096''': 4096,
    '''allenai/longformer-large-4096''': 4096,
    '''allenai/longformer-large-4096-finetuned-triviaqa''': 4096,
    '''allenai/longformer-base-4096-extra.pos.embd.only''': 4096,
    '''allenai/longformer-large-4096-extra.pos.embd.only''': 4096,
}





class       _a							(       _lowercase):
	_a						:						Optional[Any]   					=							VOCAB_FILES_NAMES
	_a						:						Tuple   					=							PRETRAINED_VOCAB_FILES_MAP
	_a						:						List[str]   					=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_a						:						Optional[int]   					=							['''input_ids''', '''attention_mask''']
	_a						:						Optional[int]   					=							LongformerTokenizer




	def __init__(      self		:	Union[str, Any]	,      _SCREAMING_SNAKE_CASE		:	Dict=None	,      _SCREAMING_SNAKE_CASE		:	Any=None	,      _SCREAMING_SNAKE_CASE		:	Optional[Any]=None	,      _SCREAMING_SNAKE_CASE		:	str="replace"	,      _SCREAMING_SNAKE_CASE		:	Optional[Any]="<s>"	,      _SCREAMING_SNAKE_CASE		:	Tuple="</s>"	,      _SCREAMING_SNAKE_CASE		:	Dict="</s>"	,      _SCREAMING_SNAKE_CASE		:	Dict="<s>"	,      _SCREAMING_SNAKE_CASE		:	Dict="<unk>"	,      _SCREAMING_SNAKE_CASE		:	Union[str, Any]="<pad>"	,      _SCREAMING_SNAKE_CASE		:	Optional[int]="<mask>"	,      _SCREAMING_SNAKE_CASE		:	str=False	,      _SCREAMING_SNAKE_CASE		:	List[str]=True	,      **_SCREAMING_SNAKE_CASE		:	Any	,      )-> Dict:
			super().__init__(
			    _SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	,      tokenizer_file=_SCREAMING_SNAKE_CASE	,      errors=_SCREAMING_SNAKE_CASE	,      bos_token=_SCREAMING_SNAKE_CASE	,      eos_token=_SCREAMING_SNAKE_CASE	,      sep_token=_SCREAMING_SNAKE_CASE	,      cls_token=_SCREAMING_SNAKE_CASE	,      unk_token=_SCREAMING_SNAKE_CASE	,      pad_token=_SCREAMING_SNAKE_CASE	,      mask_token=_SCREAMING_SNAKE_CASE	,      add_prefix_space=_SCREAMING_SNAKE_CASE	,      trim_offsets=_SCREAMING_SNAKE_CASE	,      **_SCREAMING_SNAKE_CASE	,      )

			lowerCAmelCase__     :    Tuple					  =      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
			if pre_tok_state.get('''add_prefix_space'''	,      _SCREAMING_SNAKE_CASE	) != add_prefix_space:
					lowerCAmelCase__     :    Any					  =      getattr(_SCREAMING_SNAKE_CASE	,      pre_tok_state.pop('''type'''	)	)
					lowerCAmelCase__     :    Any					  =      add_prefix_space
					lowerCAmelCase__     :    List[str]					  =      pre_tok_class(**_SCREAMING_SNAKE_CASE	)

			lowerCAmelCase__     :    Tuple					  =      add_prefix_space

			lowerCAmelCase__     :    Optional[int]					  =      '''post_processor'''
			lowerCAmelCase__     :    Tuple					  =      getattr(self.backend_tokenizer	,      _SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)
			if tokenizer_component_instance:
					lowerCAmelCase__     :    Optional[Any]					  =      json.loads(tokenizer_component_instance.__getstate__()	)

					# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
					if "sep" in state:
							lowerCAmelCase__     :    List[Any]					  =      tuple(state['''sep''']	)
					if "cls" in state:
							lowerCAmelCase__     :    Dict					  =      tuple(state['''cls''']	)

					lowerCAmelCase__     :    List[Any]					  =      False

					if state.get('''add_prefix_space'''	,      _SCREAMING_SNAKE_CASE	) != add_prefix_space:
							lowerCAmelCase__     :    Dict					  =      add_prefix_space
							lowerCAmelCase__     :    Tuple					  =      True

					if state.get('''trim_offsets'''	,      _SCREAMING_SNAKE_CASE	) != trim_offsets:
							lowerCAmelCase__     :    List[str]					  =      trim_offsets
							lowerCAmelCase__     :    Union[str, Any]					  =      True

					if changes_to_apply:
							lowerCAmelCase__     :    Union[str, Any]					  =      getattr(_SCREAMING_SNAKE_CASE	,      state.pop('''type'''	)	)
							lowerCAmelCase__     :    Optional[Any]					  =      component_class(**_SCREAMING_SNAKE_CASE	)
							setattr(self.backend_tokenizer	,      _SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	)




	@property
	def 							UpperCAmelCase__(      self		:	str	)-> str:
			if self._mask_token is None:
					if self.verbose:
							logger.error('''Using mask_token, but it is not set yet.'''	)
					return None
			return str(self._mask_token	)




	@mask_token.setter
	def 							UpperCAmelCase__(      self		:	Optional[Any]	,      _SCREAMING_SNAKE_CASE		:	Optional[Any]	)-> Optional[Any]:
			lowerCAmelCase__     :    Union[str, Any]					  =      AddedToken(_SCREAMING_SNAKE_CASE	,      lstrip=_SCREAMING_SNAKE_CASE	,      rstrip=_SCREAMING_SNAKE_CASE	) if isinstance(_SCREAMING_SNAKE_CASE	,      _SCREAMING_SNAKE_CASE	) else value
			lowerCAmelCase__     :    List[Any]					  =      value




	def 							UpperCAmelCase__(      self		:	Optional[int]	,      *_SCREAMING_SNAKE_CASE		:	Any	,      **_SCREAMING_SNAKE_CASE		:	List[Any]	)-> BatchEncoding:
			lowerCAmelCase__     :    Optional[Any]					  =      kwargs.get('''is_split_into_words'''	,      _SCREAMING_SNAKE_CASE	)
			assert self.add_prefix_space or not is_split_into_words, (
			    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
			    "to use it with pretokenized inputs."
			)

			return super()._batch_encode_plus(*_SCREAMING_SNAKE_CASE	,      **_SCREAMING_SNAKE_CASE	)




	def 							UpperCAmelCase__(      self		:	Union[str, Any]	,      *_SCREAMING_SNAKE_CASE		:	List[str]	,      **_SCREAMING_SNAKE_CASE		:	Optional[Any]	)-> BatchEncoding:
			lowerCAmelCase__     :    Any					  =      kwargs.get('''is_split_into_words'''	,      _SCREAMING_SNAKE_CASE	)

			assert self.add_prefix_space or not is_split_into_words, (
			    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
			    "to use it with pretokenized inputs."
			)

			return super()._encode_plus(*_SCREAMING_SNAKE_CASE	,      **_SCREAMING_SNAKE_CASE	)




	def 							UpperCAmelCase__(      self		:	List[Any]	,      _SCREAMING_SNAKE_CASE		:	str	,      _SCREAMING_SNAKE_CASE		:	Optional[str] = None	)-> Tuple[str]:
			lowerCAmelCase__     :    str					  =      self._tokenizer.model.save(_SCREAMING_SNAKE_CASE	,      name=_SCREAMING_SNAKE_CASE	)
			return tuple(_SCREAMING_SNAKE_CASE	)




	def 							UpperCAmelCase__(      self		:	int	,      _SCREAMING_SNAKE_CASE		:	int	,      _SCREAMING_SNAKE_CASE		:	int=None	)-> Optional[int]:
			lowerCAmelCase__     :    str					  =      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
			if token_ids_a is None:
					return output

			return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




	def 							UpperCAmelCase__(      self		:	Any	,      _SCREAMING_SNAKE_CASE		:	List[int]	,      _SCREAMING_SNAKE_CASE		:	Optional[List[int]] = None	)-> List[int]:
			lowerCAmelCase__     :    Optional[Any]					  =      [self.sep_token_id]
			lowerCAmelCase__     :    Optional[Any]					  =      [self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep	) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


lowerCamelCase		=	{
    '''configuration_segformer''': ['''SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''SegformerConfig''', '''SegformerOnnxConfig''']
}

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase		=	['''SegformerFeatureExtractor''']
		lowerCamelCase		=	['''SegformerImageProcessor''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase		=	[
		    '''SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''SegformerDecodeHead''',
		    '''SegformerForImageClassification''',
		    '''SegformerForSemanticSegmentation''',
		    '''SegformerLayer''',
		    '''SegformerModel''',
		    '''SegformerPreTrainedModel''',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase		=	[
		    '''TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''TFSegformerDecodeHead''',
		    '''TFSegformerForImageClassification''',
		    '''TFSegformerForSemanticSegmentation''',
		    '''TFSegformerModel''',
		    '''TFSegformerPreTrainedModel''',
		]


if TYPE_CHECKING:
		from .configuration_segformer import SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, SegformerConfig, SegformerOnnxConfig

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_segformer import SegformerFeatureExtractor
				from .image_processing_segformer import SegformerImageProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_segformer import (
				    SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    SegformerDecodeHead,
				    SegformerForImageClassification,
				    SegformerForSemanticSegmentation,
				    SegformerLayer,
				    SegformerModel,
				    SegformerPreTrainedModel,
				)
		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_segformer import (
				    TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFSegformerDecodeHead,
				    TFSegformerForImageClassification,
				    TFSegformerForSemanticSegmentation,
				    TFSegformerModel,
				    TFSegformerPreTrainedModel,
				)

else:
		import sys

		lowerCamelCase		=	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




