

"""simple docstring"""




import collections
import inspect
import unittest

from transformers import DonutSwinConfig
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import is_torch_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch
     from torch import nn

     from transformers import DonutSwinModel
     from transformers.models.donut.modeling_donut_swin import DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST


class 		_A							:





      def __init__(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__=13     ,    SCREAMING_SNAKE_CASE__=32     ,    SCREAMING_SNAKE_CASE__=2     ,    SCREAMING_SNAKE_CASE__=3     ,    SCREAMING_SNAKE_CASE__=16     ,    SCREAMING_SNAKE_CASE__=[1, 2, 1]     ,    SCREAMING_SNAKE_CASE__=[2, 2, 4]     ,    SCREAMING_SNAKE_CASE__=2     ,    SCREAMING_SNAKE_CASE__=2.0     ,    SCREAMING_SNAKE_CASE__=True     ,    SCREAMING_SNAKE_CASE__=0.0     ,    SCREAMING_SNAKE_CASE__=0.0     ,    SCREAMING_SNAKE_CASE__=0.1     ,    SCREAMING_SNAKE_CASE__="gelu"     ,    SCREAMING_SNAKE_CASE__=False     ,    SCREAMING_SNAKE_CASE__=True     ,    SCREAMING_SNAKE_CASE__=0.02     ,    SCREAMING_SNAKE_CASE__=1e-5     ,    SCREAMING_SNAKE_CASE__=True     ,    SCREAMING_SNAKE_CASE__=None     ,    SCREAMING_SNAKE_CASE__=True     ,    SCREAMING_SNAKE_CASE__=10     ,    SCREAMING_SNAKE_CASE__=8     ,    ) ->       int:
         lowerCamelCase__						       =      parent
         lowerCamelCase__						       =      batch_size
         lowerCamelCase__						       =      image_size
         lowerCamelCase__						       =      patch_size
         lowerCamelCase__						       =      num_channels
         lowerCamelCase__						       =      embed_dim
         lowerCamelCase__						       =      depths
         lowerCamelCase__						       =      num_heads
         lowerCamelCase__						       =      window_size
         lowerCamelCase__						       =      mlp_ratio
         lowerCamelCase__						       =      qkv_bias
         lowerCamelCase__						       =      hidden_dropout_prob
         lowerCamelCase__						       =      attention_probs_dropout_prob
         lowerCamelCase__						       =      drop_path_rate
         lowerCamelCase__						       =      hidden_act
         lowerCamelCase__						       =      use_absolute_embeddings
         lowerCamelCase__						       =      patch_norm
         lowerCamelCase__						       =      layer_norm_eps
         lowerCamelCase__						       =      initializer_range
         lowerCamelCase__						       =      is_training
         lowerCamelCase__						       =      scope
         lowerCamelCase__						       =      use_labels
         lowerCamelCase__						       =      type_sequence_label_size
         lowerCamelCase__						       =      encoder_stride





      def        _lowerCamelCase	(	self		) ->       str:
         lowerCamelCase__						       =      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]		)

         lowerCamelCase__						       =      None
         if self.use_labels:
            lowerCamelCase__						       =      ids_tensor([self.batch_size]     ,    self.type_sequence_label_size		)

         lowerCamelCase__						       =      self.get_config()

         return config, pixel_values, labels





      def        _lowerCamelCase	(	self		) ->       List[str]:
         return DonutSwinConfig(
             image_size=self.image_size     ,    patch_size=self.patch_size     ,    num_channels=self.num_channels     ,    embed_dim=self.embed_dim     ,    depths=self.depths     ,    num_heads=self.num_heads     ,    window_size=self.window_size     ,    mlp_ratio=self.mlp_ratio     ,    qkv_bias=self.qkv_bias     ,    hidden_dropout_prob=self.hidden_dropout_prob     ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,    drop_path_rate=self.drop_path_rate     ,    hidden_act=self.hidden_act     ,    use_absolute_embeddings=self.use_absolute_embeddings     ,    path_norm=self.patch_norm     ,    layer_norm_eps=self.layer_norm_eps     ,    initializer_range=self.initializer_range     ,    encoder_stride=self.encoder_stride     ,    )





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		) ->       str:
         lowerCamelCase__						       =      DonutSwinModel(config=SCREAMING_SNAKE_CASE__		)
         model.to(SCREAMING_SNAKE_CASE__		)
         model.eval()
         lowerCamelCase__						       =      model(SCREAMING_SNAKE_CASE__		)

         lowerCamelCase__						       =      ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths		) - 1))
         lowerCamelCase__						       =      int(config.embed_dim * 2 ** (len(config.depths		) - 1)		)

         self.parent.assertEqual(result.last_hidden_state.shape     ,    (self.batch_size, expected_seq_len, expected_dim)		)





      def        _lowerCamelCase	(	self		) ->       Optional[int]:
         lowerCamelCase__						       =      self.prepare_config_and_inputs()
         (
             (
             lowerCamelCase__
         )       ,							(
             lowerCamelCase__
         )       ,							(
             lowerCamelCase__
         )       ,							
         )						       =      config_and_inputs
         lowerCamelCase__						       =      {"pixel_values": pixel_values}
         return config, inputs_dict






@require_torch
class 		_A							(      __a					,							__a					,							unittest.TestCase       ):
      __a             =						(DonutSwinModel,) if is_torch_available() else ()
      __a             =						{'feature-extraction': DonutSwinModel} if is_torch_available() else {}
      __a             =						True

      __a             =						False
      __a             =						False
      __a             =						False





      def        _lowerCamelCase	(	self		) ->       List[Any]:
         lowerCamelCase__						       =      DonutSwinModelTester(self		)
         lowerCamelCase__						       =      ConfigTester(self     ,    config_class=SCREAMING_SNAKE_CASE__     ,    embed_dim=37		)





      def        _lowerCamelCase	(	self		) ->       List[Any]:
         self.create_and_test_config_common_properties()
         self.config_tester.create_and_test_config_to_json_string()
         self.config_tester.create_and_test_config_to_json_file()
         self.config_tester.create_and_test_config_from_and_save_pretrained()
         self.config_tester.create_and_test_config_with_num_labels()
         self.config_tester.check_config_can_be_init_without_params()
         self.config_tester.check_config_arguments_init()





      def        _lowerCamelCase	(	self		) ->       int:
         return





      def        _lowerCamelCase	(	self		) ->       Dict:
         lowerCamelCase__						       =      self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE__		)





      def        _lowerCamelCase	(	self		) ->       Optional[int]:
         # DonutSwin does not use inputs_embeds
         pass





      def        _lowerCamelCase	(	self		) ->       Optional[Any]:
         lowerCamelCase__       ,							lowerCamelCase__						       =      self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
            lowerCamelCase__						       =      model_class(SCREAMING_SNAKE_CASE__		)
            self.assertIsInstance(model.get_input_embeddings()     ,    (nn.Module)		)
            lowerCamelCase__						       =      model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(SCREAMING_SNAKE_CASE__     ,    nn.Linear		)		)





      def        _lowerCamelCase	(	self		) ->       int:
         lowerCamelCase__       ,							lowerCamelCase__						       =      self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
            lowerCamelCase__						       =      model_class(SCREAMING_SNAKE_CASE__		)
            lowerCamelCase__						       =      inspect.signature(model.forward		)
            # signature.parameters is an OrderedDict => so arg_names order is deterministic
            lowerCamelCase__						       =      [*signature.parameters.keys()]

            lowerCamelCase__						       =      ["pixel_values"]
            self.assertListEqual(arg_names[:1]     ,    SCREAMING_SNAKE_CASE__		)





      def        _lowerCamelCase	(	self		) ->       Tuple:
         lowerCamelCase__       ,							lowerCamelCase__						       =      self.model_tester.prepare_config_and_inputs_for_common()
         lowerCamelCase__						       =      True

         for model_class in self.all_model_classes:
            lowerCamelCase__						       =      True
            lowerCamelCase__						       =      False
            lowerCamelCase__						       =      True
            lowerCamelCase__						       =      model_class(SCREAMING_SNAKE_CASE__		)
            model.to(SCREAMING_SNAKE_CASE__		)
            model.eval()
            with torch.no_grad():
               lowerCamelCase__						       =      model(**self._prepare_for_class(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		)		)
            lowerCamelCase__						       =      outputs.attentions
            lowerCamelCase__						       =      len(self.model_tester.depths		)
            self.assertEqual(len(SCREAMING_SNAKE_CASE__		)     ,    SCREAMING_SNAKE_CASE__		)

            # check that output_attentions also work using config
            del inputs_dict["output_attentions"]
            lowerCamelCase__						       =      True
            lowerCamelCase__						       =      config.window_size**2
            lowerCamelCase__						       =      model_class(SCREAMING_SNAKE_CASE__		)
            model.to(SCREAMING_SNAKE_CASE__		)
            model.eval()
            with torch.no_grad():
               lowerCamelCase__						       =      model(**self._prepare_for_class(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		)		)
            lowerCamelCase__						       =      outputs.attentions
            self.assertEqual(len(SCREAMING_SNAKE_CASE__		)     ,    SCREAMING_SNAKE_CASE__		)

            self.assertListEqual(
                list(attentions[0].shape[-3:]		)     ,    [self.model_tester.num_heads[0], window_size_squared, window_size_squared]     ,    )
            lowerCamelCase__						       =      len(SCREAMING_SNAKE_CASE__		)

            # Check attention is always last and order is fine
            lowerCamelCase__						       =      True
            lowerCamelCase__						       =      True
            lowerCamelCase__						       =      model_class(SCREAMING_SNAKE_CASE__		)
            model.to(SCREAMING_SNAKE_CASE__		)
            model.eval()
            with torch.no_grad():
               lowerCamelCase__						       =      model(**self._prepare_for_class(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		)		)

            if hasattr(self.model_tester     ,    "num_hidden_states_types"		):
               lowerCamelCase__						       =      self.model_tester.num_hidden_states_types
            else:
               # also another +1 for reshaped_hidden_states
               lowerCamelCase__						       =      2
            self.assertEqual(out_len + added_hidden_states     ,    len(SCREAMING_SNAKE_CASE__		)		)

            lowerCamelCase__						       =      outputs.attentions

            self.assertEqual(len(SCREAMING_SNAKE_CASE__		)     ,    SCREAMING_SNAKE_CASE__		)

            self.assertListEqual(
                list(self_attentions[0].shape[-3:]		)     ,    [self.model_tester.num_heads[0], window_size_squared, window_size_squared]     ,    )





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		) ->       Dict:
         lowerCamelCase__						       =      model_class(SCREAMING_SNAKE_CASE__		)
         model.to(SCREAMING_SNAKE_CASE__		)
         model.eval()

         with torch.no_grad():
            lowerCamelCase__						       =      model(**self._prepare_for_class(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		)		)

         lowerCamelCase__						       =      outputs.hidden_states

         lowerCamelCase__						       =      getattr(
             self.model_tester     ,    "expected_num_hidden_layers"     ,    len(self.model_tester.depths		) + 1		)
         self.assertEqual(len(SCREAMING_SNAKE_CASE__		)     ,    SCREAMING_SNAKE_CASE__		)

         # DonutSwin has a different seq_length
         lowerCamelCase__						       =      (
             config.patch_size
             if isinstance(config.patch_size     ,    collections.abc.Iterable		)
             else (config.patch_size, config.patch_size)
         )

         lowerCamelCase__						       =      (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

         self.assertListEqual(
             list(hidden_states[0].shape[-2:]		)     ,    [num_patches, self.model_tester.embed_dim]     ,    )

         lowerCamelCase__						       =      outputs.reshaped_hidden_states
         self.assertEqual(len(SCREAMING_SNAKE_CASE__		)     ,    SCREAMING_SNAKE_CASE__		)

         lowerCamelCase__       ,							lowerCamelCase__       ,							lowerCamelCase__       ,							lowerCamelCase__						       =      reshaped_hidden_states[0].shape
         lowerCamelCase__						       =      (
             reshaped_hidden_states[0].view(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    height * width		).permute(0     ,    2     ,    1		)
         )
         self.assertListEqual(
             list(reshaped_hidden_states.shape[-2:]		)     ,    [num_patches, self.model_tester.embed_dim]     ,    )





      def        _lowerCamelCase	(	self		) ->       List[str]:
         lowerCamelCase__       ,							lowerCamelCase__						       =      self.model_tester.prepare_config_and_inputs_for_common()

         lowerCamelCase__						       =      (
             self.model_tester.image_size
             if isinstance(self.model_tester.image_size     ,    collections.abc.Iterable		)
             else (self.model_tester.image_size, self.model_tester.image_size)
         )

         for model_class in self.all_model_classes:
            lowerCamelCase__						       =      True
            self.check_hidden_states_output(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		)

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            lowerCamelCase__						       =      True

            self.check_hidden_states_output(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		)





      def        _lowerCamelCase	(	self		) ->       Union[str, Any]:
         lowerCamelCase__       ,							lowerCamelCase__						       =      self.model_tester.prepare_config_and_inputs_for_common()
         lowerCamelCase__						       =      3

         lowerCamelCase__						       =      (
             self.model_tester.image_size
             if isinstance(self.model_tester.image_size     ,    collections.abc.Iterable		)
             else (self.model_tester.image_size, self.model_tester.image_size)
         )
         lowerCamelCase__						       =      (
             config.patch_size
             if isinstance(config.patch_size     ,    collections.abc.Iterable		)
             else (config.patch_size, config.patch_size)
         )

         lowerCamelCase__						       =      image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
         lowerCamelCase__						       =      image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

         for model_class in self.all_model_classes:
            lowerCamelCase__						       =      True
            self.check_hidden_states_output(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    (padded_height, padded_width)		)

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            lowerCamelCase__						       =      True
            self.check_hidden_states_output(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    (padded_height, padded_width)		)





      @slow
      def        _lowerCamelCase	(	self		) ->       str:
         for model_name in DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            lowerCamelCase__						       =      DonutSwinModel.from_pretrained(SCREAMING_SNAKE_CASE__		)
            self.assertIsNotNone(SCREAMING_SNAKE_CASE__		)





      def        _lowerCamelCase	(	self		) ->       str:
         lowerCamelCase__       ,							lowerCamelCase__						       =      self.model_tester.prepare_config_and_inputs_for_common()

         lowerCamelCase__						       =      _config_zero_init(SCREAMING_SNAKE_CASE__		)
         for model_class in self.all_model_classes:
            lowerCamelCase__						       =      model_class(config=SCREAMING_SNAKE_CASE__		)
            for name, param in model.named_parameters():
               if "embeddings" not in name and param.requires_grad:
                  self.assertIn(
                      ((param.data.mean() * 1e9).round() / 1e9).item()     ,    [0.0, 1.0]     ,    msg=f'Parameter {name} of model {model_class} seems not properly initialized'     ,    )







"""simple docstring"""




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation

SCREAMING_SNAKE_CASE_  :      Optional[int]							=      logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_  :      Optional[int]							=      {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

SCREAMING_SNAKE_CASE_  :      Dict							=      {
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
}

SCREAMING_SNAKE_CASE_  :      Optional[Any]							=      {
    '''gpt2''': 1024,
    '''gpt2-medium''': 1024,
    '''gpt2-large''': 1024,
    '''gpt2-xl''': 1024,
    '''distilgpt2''': 1024,
}



@lru_cache()
def 						UpperCAmelCase__ (					)   ->   Optional[int]:




   """simple docstring"""
   lowerCamelCase__						       =      (
       list(range(ord("!" )						,     ord("~" ) + 1 ) ) + list(range(ord("¡" )						,     ord("¬" ) + 1 ) ) + list(range(ord("®" )						,     ord("ÿ" ) + 1 ) )
   )
   lowerCamelCase__						       =      bs[:]
   lowerCamelCase__						       =      0
   for b in range(2**8 ):
      if b not in bs:
         bs.append(A__ )
         cs.append(2**8 + n )
         n += 1
   lowerCamelCase__						       =      [chr(A__ ) for n in cs]
   return dict(zip(A__						,     A__ ) )



def 						UpperCAmelCase__ (					A__ )   ->   int:




   """simple docstring"""
   lowerCamelCase__						       =      set()
   lowerCamelCase__						       =      word[0]
   for char in word[1:]:
      pairs.add((prev_char, char) )
      lowerCamelCase__						       =      char
   return pairs


class 		_A							(      __a       ):
      __a             =						VOCAB_FILES_NAMES
      __a             =						PRETRAINED_VOCAB_FILES_MAP
      __a             =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __a             =						['input_ids', 'attention_mask']





      def __init__(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__="replace"     ,    SCREAMING_SNAKE_CASE__="<|endoftext|>"     ,    SCREAMING_SNAKE_CASE__="<|endoftext|>"     ,    SCREAMING_SNAKE_CASE__="<|endoftext|>"     ,    SCREAMING_SNAKE_CASE__=None     ,    SCREAMING_SNAKE_CASE__=False     ,    SCREAMING_SNAKE_CASE__=False     ,    **SCREAMING_SNAKE_CASE__     ,    ) ->       List[str]:
         lowerCamelCase__						       =      AddedToken(SCREAMING_SNAKE_CASE__     ,    lstrip=SCREAMING_SNAKE_CASE__     ,    rstrip=SCREAMING_SNAKE_CASE__		) if isinstance(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		) else bos_token
         lowerCamelCase__						       =      AddedToken(SCREAMING_SNAKE_CASE__     ,    lstrip=SCREAMING_SNAKE_CASE__     ,    rstrip=SCREAMING_SNAKE_CASE__		) if isinstance(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		) else eos_token
         lowerCamelCase__						       =      AddedToken(SCREAMING_SNAKE_CASE__     ,    lstrip=SCREAMING_SNAKE_CASE__     ,    rstrip=SCREAMING_SNAKE_CASE__		) if isinstance(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		) else unk_token
         lowerCamelCase__						       =      AddedToken(SCREAMING_SNAKE_CASE__     ,    lstrip=SCREAMING_SNAKE_CASE__     ,    rstrip=SCREAMING_SNAKE_CASE__		) if isinstance(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		) else pad_token
         super().__init__(
             errors=SCREAMING_SNAKE_CASE__     ,    unk_token=SCREAMING_SNAKE_CASE__     ,    bos_token=SCREAMING_SNAKE_CASE__     ,    eos_token=SCREAMING_SNAKE_CASE__     ,    pad_token=SCREAMING_SNAKE_CASE__     ,    add_prefix_space=SCREAMING_SNAKE_CASE__     ,    add_bos_token=SCREAMING_SNAKE_CASE__     ,    **SCREAMING_SNAKE_CASE__     ,    )
         lowerCamelCase__						       =      add_bos_token

         with open(SCREAMING_SNAKE_CASE__     ,    encoding="utf-8"		) as vocab_handle:
            lowerCamelCase__						       =      json.load(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						       =      {v: k for k, v in self.encoder.items()}
         lowerCamelCase__						       =      errors  # how to handle errors in decoding
         lowerCamelCase__						       =      bytes_to_unicode()
         lowerCamelCase__						       =      {v: k for k, v in self.byte_encoder.items()}
         with open(SCREAMING_SNAKE_CASE__     ,    encoding="utf-8"		) as merges_handle:
            lowerCamelCase__						       =      merges_handle.read().split("\n"		)[1:-1]
         lowerCamelCase__						       =      [tuple(merge.split()		) for merge in bpe_merges]
         lowerCamelCase__						       =      dict(zip(SCREAMING_SNAKE_CASE__     ,    range(len(SCREAMING_SNAKE_CASE__		)		)		)		)
         lowerCamelCase__						       =      {}
         lowerCamelCase__						       =      add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         lowerCamelCase__						       =      re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"		)





      @property
      def        _lowerCamelCase	(	self		) ->       Optional[int]:
         return len(self.encoder		)





      def        _lowerCamelCase	(	self		) ->       str:
         return dict(self.encoder     ,    **self.added_tokens_encoder		)





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       Any:
         if token in self.cache:
            return self.cache[token]
         lowerCamelCase__						       =      tuple(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						       =      get_pairs(SCREAMING_SNAKE_CASE__		)

         if not pairs:
            return token

         while True:
            lowerCamelCase__						       =      min(SCREAMING_SNAKE_CASE__     ,    key=lambda SCREAMING_SNAKE_CASE__		: self.bpe_ranks.get(SCREAMING_SNAKE_CASE__     ,    float("inf"		)		)		)
            if bigram not in self.bpe_ranks:
               break
            lowerCamelCase__       ,							lowerCamelCase__						       =      bigram
            lowerCamelCase__						       =      []
            lowerCamelCase__						       =      0
            while i < len(SCREAMING_SNAKE_CASE__		):
               try:
                  lowerCamelCase__						       =      word.index(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		)
               except ValueError:
                  new_word.extend(word[i:]		)
                  break
               else:
                  new_word.extend(word[i:j]		)
                  lowerCamelCase__						       =      j

               if word[i] == first and i < len(SCREAMING_SNAKE_CASE__		) - 1 and word[i + 1] == second:
                  new_word.append(first + second		)
                  i += 2
               else:
                  new_word.append(word[i]		)
                  i += 1
            lowerCamelCase__						       =      tuple(SCREAMING_SNAKE_CASE__		)
            lowerCamelCase__						       =      new_word
            if len(SCREAMING_SNAKE_CASE__		) == 1:
               break
            else:
               lowerCamelCase__						       =      get_pairs(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						       =      " ".join(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						       =      word
         return word





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__=None		) ->       Optional[int]:
         if self.add_bos_token:
            lowerCamelCase__						       =      [self.bos_token_id]
         else:
            lowerCamelCase__						       =      []

         lowerCamelCase__						       =      bos_token_ids + token_ids_a

         if token_ids_a is None:
            return output

         return output + bos_token_ids + token_ids_a





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__ = None     ,    SCREAMING_SNAKE_CASE__ = False		) ->       List[int]:
         if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=SCREAMING_SNAKE_CASE__     ,    token_ids_a=SCREAMING_SNAKE_CASE__     ,    already_has_special_tokens=SCREAMING_SNAKE_CASE__		)

         if not self.add_bos_token:
            return super().get_special_tokens_mask(
                token_ids_a=SCREAMING_SNAKE_CASE__     ,    token_ids_a=SCREAMING_SNAKE_CASE__     ,    already_has_special_tokens=SCREAMING_SNAKE_CASE__		)

         if token_ids_a is None:
            return [1] + ([0] * len(SCREAMING_SNAKE_CASE__		))
         return [1] + ([0] * len(SCREAMING_SNAKE_CASE__		)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE__		))





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       Tuple:
         lowerCamelCase__						       =      []
         for token in re.findall(self.pat     ,    SCREAMING_SNAKE_CASE__		):
            lowerCamelCase__						       =      "".join(
                self.byte_encoder[b] for b in token.encode("utf-8"		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(SCREAMING_SNAKE_CASE__		).split(" "		)		)
         return bpe_tokens





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       Tuple:
         return self.encoder.get(SCREAMING_SNAKE_CASE__     ,    self.encoder.get(self.unk_token		)		)





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       Any:
         return self.decoder.get(SCREAMING_SNAKE_CASE__		)





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       str:
         lowerCamelCase__						       =      "".join(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						       =      bytearray([self.byte_decoder[c] for c in text]		).decode("utf-8"     ,    errors=self.errors		)
         return text





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__ = None		) ->       Tuple[str]:
         if not os.path.isdir(SCREAMING_SNAKE_CASE__		):
            logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
            return
         lowerCamelCase__						       =      os.path.join(
             SCREAMING_SNAKE_CASE__     ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
         lowerCamelCase__						       =      os.path.join(
             SCREAMING_SNAKE_CASE__     ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]		)

         with open(SCREAMING_SNAKE_CASE__     ,    "w"     ,    encoding="utf-8"		) as f:
            f.write(json.dumps(self.encoder     ,    indent=2     ,    sort_keys=SCREAMING_SNAKE_CASE__     ,    ensure_ascii=SCREAMING_SNAKE_CASE__		) + "\n"		)

         lowerCamelCase__						       =      0
         with open(SCREAMING_SNAKE_CASE__     ,    "w"     ,    encoding="utf-8"		) as writer:
            writer.write("#version: 0.2\n"		)
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,    key=lambda SCREAMING_SNAKE_CASE__		: kv[1]		):
               if index != token_index:
                  logger.warning(
                      f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                      " Please check that the tokenizer is not corrupted!"		)
                  lowerCamelCase__						       =      token_index
               writer.write(" ".join(SCREAMING_SNAKE_CASE__		) + "\n"		)
               index += 1

         return vocab_file, merge_file





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__=False     ,    **SCREAMING_SNAKE_CASE__		) ->       int:
         lowerCamelCase__						       =      kwargs.pop("add_prefix_space"     ,    self.add_prefix_space		)
         if is_split_into_words or add_prefix_space:
            lowerCamelCase__						       =      " " + text
         return (text, kwargs)





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       List[int]:
         lowerCamelCase__						       =      []
         for is_user, text in conversation.iter_texts():
            input_ids.extend(self.encode(SCREAMING_SNAKE_CASE__     ,    add_special_tokens=SCREAMING_SNAKE_CASE__		) + [self.eos_token_id]		)
         if len(SCREAMING_SNAKE_CASE__		) > self.model_max_length:
            lowerCamelCase__						       =      input_ids[-self.model_max_length :]
         return input_ids




