

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


_SCREAMING_SNAKE_CASE          = logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE          = {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_SCREAMING_SNAKE_CASE          = {
    '''vocab_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''allenai/longformer-base-4096''': (
            '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'''
        ),
    },
}

_SCREAMING_SNAKE_CASE          = {
    '''allenai/longformer-base-4096''': 4_0_9_6,
    '''allenai/longformer-large-4096''': 4_0_9_6,
    '''allenai/longformer-large-4096-finetuned-triviaqa''': 4_0_9_6,
    '''allenai/longformer-base-4096-extra.pos.embd.only''': 4_0_9_6,
    '''allenai/longformer-large-4096-extra.pos.embd.only''': 4_0_9_6,
}


class     lowerCamelCase_   ( _lowerCAmelCase):
 _lowerCAmelCase       		=  VOCAB_FILES_NAMES
 _lowerCAmelCase       		=  PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase       		=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase       		=  ['''input_ids''', '''attention_mask''']
 _lowerCAmelCase       		=  LongformerTokenizer







 def __init__( self     ,     _UpperCAmelCase=None     ,     _UpperCAmelCase=None     ,     _UpperCAmelCase=None     ,     _UpperCAmelCase="replace"     ,     _UpperCAmelCase="<s>"     ,     _UpperCAmelCase="</s>"     ,     _UpperCAmelCase="</s>"     ,     _UpperCAmelCase="<s>"     ,     _UpperCAmelCase="<unk>"     ,     _UpperCAmelCase="<pad>"     ,     _UpperCAmelCase="<mask>"     ,     _UpperCAmelCase=False     ,     _UpperCAmelCase=True     ,     **_UpperCAmelCase     ,     ):
   super().__init__(
       _UpperCAmelCase     ,     _UpperCAmelCase     ,     tokenizer_file=_UpperCAmelCase     ,     errors=_UpperCAmelCase     ,     bos_token=_UpperCAmelCase     ,     eos_token=_UpperCAmelCase     ,     sep_token=_UpperCAmelCase     ,     cls_token=_UpperCAmelCase     ,     unk_token=_UpperCAmelCase     ,     pad_token=_UpperCAmelCase     ,     mask_token=_UpperCAmelCase     ,     add_prefix_space=_UpperCAmelCase     ,     trim_offsets=_UpperCAmelCase     ,     **_UpperCAmelCase     ,     )

   __UpperCAmelCase  :     Any		=    json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()       )
   if pre_tok_state.get('add_prefix_space'     ,     _UpperCAmelCase       ) != add_prefix_space:
     __UpperCAmelCase  :     Optional[int]		=    getattr(_UpperCAmelCase     ,     pre_tok_state.pop('type'       )       )
     __UpperCAmelCase  :     Optional[Any]		=    add_prefix_space
     __UpperCAmelCase  :     List[Any]		=    pre_tok_class(**_UpperCAmelCase       )

   __UpperCAmelCase  :     int		=    add_prefix_space

   __UpperCAmelCase  :     Optional[int]		=    'post_processor'
   __UpperCAmelCase  :     Dict		=    getattr(self.backend_tokenizer     ,     _UpperCAmelCase     ,     _UpperCAmelCase       )
   if tokenizer_component_instance:
     __UpperCAmelCase  :     List[Any]		=    json.loads(tokenizer_component_instance.__getstate__()       )

     # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
     if "sep" in state:
       __UpperCAmelCase  :     Optional[int]		=    tuple(state['sep']       )
     if "cls" in state:
       __UpperCAmelCase  :     List[str]		=    tuple(state['cls']       )

     __UpperCAmelCase  :     Optional[int]		=    False

     if state.get('add_prefix_space'     ,     _UpperCAmelCase       ) != add_prefix_space:
       __UpperCAmelCase  :     str		=    add_prefix_space
       __UpperCAmelCase  :     Tuple		=    True

     if state.get('trim_offsets'     ,     _UpperCAmelCase       ) != trim_offsets:
       __UpperCAmelCase  :     List[str]		=    trim_offsets
       __UpperCAmelCase  :     Union[str, Any]		=    True

     if changes_to_apply:
       __UpperCAmelCase  :     Optional[Any]		=    getattr(_UpperCAmelCase     ,     state.pop('type'       )       )
       __UpperCAmelCase  :     List[str]		=    component_class(**_UpperCAmelCase       )
       setattr(self.backend_tokenizer     ,     _UpperCAmelCase     ,     _UpperCAmelCase       )







 @property
 def 						a					( self       ):
   if self._mask_token is None:
     if self.verbose:
       logger.error('Using mask_token, but it is not set yet.'       )
     return None
   return str(self._mask_token       )







 @mask_token.setter
 def 						a					( self     ,     _UpperCAmelCase       ):
   __UpperCAmelCase  :     Union[str, Any]		=    AddedToken(_UpperCAmelCase     ,     lstrip=_UpperCAmelCase     ,     rstrip=_UpperCAmelCase       ) if isinstance(_UpperCAmelCase     ,     _UpperCAmelCase       ) else value
   __UpperCAmelCase  :     int		=    value







 def 						a					( self     ,     *_UpperCAmelCase     ,     **_UpperCAmelCase       ):
   __UpperCAmelCase  :     str		=    kwargs.get('is_split_into_words'     ,     _UpperCAmelCase       )
   assert self.add_prefix_space or not is_split_into_words, (
       F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
       "to use it with pretokenized inputs."
   )

   return super()._batch_encode_plus(*_UpperCAmelCase     ,     **_UpperCAmelCase       )







 def 						a					( self     ,     *_UpperCAmelCase     ,     **_UpperCAmelCase       ):
   __UpperCAmelCase  :     Dict		=    kwargs.get('is_split_into_words'     ,     _UpperCAmelCase       )

   assert self.add_prefix_space or not is_split_into_words, (
       F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
       "to use it with pretokenized inputs."
   )

   return super()._encode_plus(*_UpperCAmelCase     ,     **_UpperCAmelCase       )







 def 						a					( self     ,     _UpperCAmelCase     ,     _UpperCAmelCase = None       ):
   __UpperCAmelCase  :     Any		=    self._tokenizer.model.save(_UpperCAmelCase     ,     name=_UpperCAmelCase       )
   return tuple(_UpperCAmelCase       )







 def 						a					( self     ,     _UpperCAmelCase     ,     _UpperCAmelCase=None       ):
   __UpperCAmelCase  :     Any		=    [self.bos_token_id] + token_ids_a + [self.eos_token_id]
   if token_ids_a is None:
     return output

   return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







 def 						a					( self     ,     _UpperCAmelCase     ,     _UpperCAmelCase = None       ):
   __UpperCAmelCase  :     Optional[Any]		=    [self.sep_token_id]
   __UpperCAmelCase  :     Optional[int]		=    [self.cls_token_id]

   if token_ids_a is None:
     return len(cls + token_ids_a + sep       ) * [0]
   return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]








from __future__ import annotations

from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any


class     lowerCamelCase_   :
 _lowerCAmelCase       		=  ('''_front''', '''_back''', '''_len''')


 @dataclass
 class     lowerCamelCase_   :
  _lowerCAmelCase       		=  None
  _lowerCAmelCase       		=  None
  _lowerCAmelCase       		=  None



 class     lowerCamelCase_   :
  _lowerCAmelCase       		=  ('''_cur''',)




  def __init__( self     ,     _UpperCAmelCase       ):
    __UpperCAmelCase  :     Tuple		=    cur




  def __iter__( self       ):
    return self




  def 						a					( self       ):
    if self._cur is None:
      # finished iterating
      raise StopIteration
    __UpperCAmelCase  :     str		=    self._cur.val
    __UpperCAmelCase  :     Dict		=    self._cur.next_node

    return val




 def __init__( self     ,     _UpperCAmelCase = None       ):
   __UpperCAmelCase  :     Any		=    None
   __UpperCAmelCase  :     Any		=    None
   __UpperCAmelCase  :     int		=    0

   if iterable is not None:
     # append every value to the deque
     for val in iterable:
       self.append(_UpperCAmelCase       )




 def 						a					( self     ,     _UpperCAmelCase       ):
   __UpperCAmelCase  :     Optional[Any]		=    self._Node(_UpperCAmelCase     ,     _UpperCAmelCase     ,     _UpperCAmelCase       )
   if self.is_empty():
     # front = back
     __UpperCAmelCase  :     Dict		=    node
     __UpperCAmelCase  :     List[str]		=    1
   else:
     # connect nodes
     __UpperCAmelCase  :     Any		=    node
     __UpperCAmelCase  :     Tuple		=    self._back
     __UpperCAmelCase  :     Optional[Any]		=    node  # assign new back to the new node

     self._len += 1

     # make sure there were no errors
     assert not self.is_empty(), "Error on appending value."




 def 						a					( self     ,     _UpperCAmelCase       ):
   __UpperCAmelCase  :     Optional[Any]		=    self._Node(_UpperCAmelCase     ,     _UpperCAmelCase     ,     _UpperCAmelCase       )
   if self.is_empty():
     # front = back
     __UpperCAmelCase  :     str		=    node
     __UpperCAmelCase  :     List[str]		=    1
   else:
     # connect nodes
     __UpperCAmelCase  :     List[str]		=    self._front
     __UpperCAmelCase  :     Any		=    node
     __UpperCAmelCase  :     Dict		=    node  # assign new front to the new node

     self._len += 1

     # make sure there were no errors
     assert not self.is_empty(), "Error on appending value."




 def 						a					( self     ,     _UpperCAmelCase       ):
   for val in iterable:
     self.append(_UpperCAmelCase       )




 def 						a					( self     ,     _UpperCAmelCase       ):
   for val in iterable:
     self.appendleft(_UpperCAmelCase       )




 def 						a					( self       ):
   assert not self.is_empty(), "Deque is empty."

   __UpperCAmelCase  :     List[Any]		=    self._back
   __UpperCAmelCase  :     Optional[int]		=    self._back.prev_node  # set new back
   # drop the last node - python will deallocate memory automatically
   __UpperCAmelCase  :     int		=    None

   self._len -= 1

   return topop.val




 def 						a					( self       ):
   assert not self.is_empty(), "Deque is empty."

   __UpperCAmelCase  :     List[Any]		=    self._front
   __UpperCAmelCase  :     Union[str, Any]		=    self._front.next_node  # set new front and drop the first node
   __UpperCAmelCase  :     Union[str, Any]		=    None

   self._len -= 1

   return topop.val




 def 						a					( self       ):
   return self._front is None




 def __len__( self       ):
   return self._len




 def __eq__( self     ,     _UpperCAmelCase       ):

   if not isinstance(_UpperCAmelCase     ,     _UpperCAmelCase       ):
     return NotImplemented

   __UpperCAmelCase  :     List[str]		=    self._front
   __UpperCAmelCase  :     str		=    other._front

   # if the length of the dequeues are not the same, they are not equal
   if len(self       ) != len(_UpperCAmelCase       ):
     return False

   while me is not None and oth is not None:
     # compare every value
     if me.val != oth.val:
       return False
     __UpperCAmelCase  :     Any		=    me.next_node
     __UpperCAmelCase  :     str		=    oth.next_node

   return True




 def __iter__( self       ):
   return Deque._Iterator(self._front       )




 def __repr__( self       ):
   __UpperCAmelCase  :     int		=    []
   __UpperCAmelCase  :     Dict		=    self._front
   while aux is not None:
     # append the values in a list to display
     values_list.append(aux.val       )
     __UpperCAmelCase  :     List[str]		=    aux.next_node

   return F"[{', '.join(repr(_UpperCAmelCase       ) for val in values_list       )}]"


if __name__ == "__main__":
      import doctest

      doctest.testmod()





