


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation

__A  							=  logging.get_logger(__name__)

__A  							=  {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__A  							=  {
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
}

__A  							=  {
    "gpt2": 10_24,
    "gpt2-medium": 10_24,
    "gpt2-large": 10_24,
    "gpt2-xl": 10_24,
    "distilgpt2": 10_24,
}







@lru_cache()
def 			lowercase__  (				)  ->							Tuple:







       """simple docstring"""






       __UpperCAmelCase    							=(
           list(range(ord("""!"""		)     ,			ord("""~"""		) + 1		)		) + list(range(ord("""¡"""		)     ,			ord("""¬"""		) + 1		)		) + list(range(ord("""®"""		)     ,			ord("""ÿ"""		) + 1		)		)
       )
       __UpperCAmelCase    							=bs[:]
       __UpperCAmelCase    							=0
       for b in range(2**8		):
              if b not in bs:
                     bs.append(A_		)
                     cs.append(2**8 + n		)
                     n += 1
       __UpperCAmelCase    							=[chr(A_		) for n in cs]
       return dict(zip(A_     ,			A_		)		)







def 			lowercase__  (				A_:	List[str]		)  ->							Any:







       """simple docstring"""






       __UpperCAmelCase    							=set()
       __UpperCAmelCase    							=word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)		)
              __UpperCAmelCase    							=char
       return pairs





class 							_A (    UpperCamelCase							):
    """simple docstring"""

    lowerCamelCase  :       Optional[Any]   =   VOCAB_FILES_NAMES
    lowerCamelCase  :       int   =   PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase  :       int   =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase  :       str   =   ['input_ids', 'attention_mask']



    def __init__(							self			:    int   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    Any   ,    __SCREAMING_SNAKE_CASE			:    List[Any]="replace"   ,    __SCREAMING_SNAKE_CASE			:    Any="<|endoftext|>"   ,    __SCREAMING_SNAKE_CASE			:    int="<|endoftext|>"   ,    __SCREAMING_SNAKE_CASE			:    Tuple="<|endoftext|>"   ,    __SCREAMING_SNAKE_CASE			:    List[str]=None   ,    __SCREAMING_SNAKE_CASE			:    Any=False   ,    __SCREAMING_SNAKE_CASE			:    str=False   ,    **__SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    )			->       Union[str, Any]:
           __UpperCAmelCase    							=AddedToken(__SCREAMING_SNAKE_CASE   ,    lstrip=__SCREAMING_SNAKE_CASE   ,    rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE   ,    __SCREAMING_SNAKE_CASE			) else bos_token
           __UpperCAmelCase    							=AddedToken(__SCREAMING_SNAKE_CASE   ,    lstrip=__SCREAMING_SNAKE_CASE   ,    rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE   ,    __SCREAMING_SNAKE_CASE			) else eos_token
           __UpperCAmelCase    							=AddedToken(__SCREAMING_SNAKE_CASE   ,    lstrip=__SCREAMING_SNAKE_CASE   ,    rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE   ,    __SCREAMING_SNAKE_CASE			) else unk_token
           __UpperCAmelCase    							=AddedToken(__SCREAMING_SNAKE_CASE   ,    lstrip=__SCREAMING_SNAKE_CASE   ,    rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE   ,    __SCREAMING_SNAKE_CASE			) else pad_token
           super().__init__(
               errors=__SCREAMING_SNAKE_CASE   ,    unk_token=__SCREAMING_SNAKE_CASE   ,    bos_token=__SCREAMING_SNAKE_CASE   ,    eos_token=__SCREAMING_SNAKE_CASE   ,    pad_token=__SCREAMING_SNAKE_CASE   ,    add_prefix_space=__SCREAMING_SNAKE_CASE   ,    add_bos_token=__SCREAMING_SNAKE_CASE   ,    **__SCREAMING_SNAKE_CASE   ,    )
           __UpperCAmelCase    							=add_bos_token

           with open(__SCREAMING_SNAKE_CASE   ,    encoding="""utf-8"""			) as vocab_handle:
                  __UpperCAmelCase    							=json.load(__SCREAMING_SNAKE_CASE			)
           __UpperCAmelCase    							={v: k for k, v in self.encoder.items()}
           __UpperCAmelCase    							=errors  # how to handle errors in decoding
           __UpperCAmelCase    							=bytes_to_unicode()
           __UpperCAmelCase    							={v: k for k, v in self.byte_encoder.items()}
           with open(__SCREAMING_SNAKE_CASE   ,    encoding="""utf-8"""			) as merges_handle:
                  __UpperCAmelCase    							=merges_handle.read().split("""\n"""			)[1:-1]
           __UpperCAmelCase    							=[tuple(merge.split()			) for merge in bpe_merges]
           __UpperCAmelCase    							=dict(zip(__SCREAMING_SNAKE_CASE   ,    range(len(__SCREAMING_SNAKE_CASE			)			)			)			)
           __UpperCAmelCase    							={}
           __UpperCAmelCase    							=add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           __UpperCAmelCase    							=re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""			)



    @property
    def _a       (							self			:    Tuple			)			->       Dict:
           return len(self.encoder			)



    def _a       (							self			:    Union[str, Any]			)			->       Optional[int]:
           return dict(self.encoder   ,    **self.added_tokens_encoder			)



    def _a       (							self			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]			)			->       Optional[int]:
           if token in self.cache:
                  return self.cache[token]
           __UpperCAmelCase    							=tuple(__SCREAMING_SNAKE_CASE			)
           __UpperCAmelCase    							=get_pairs(__SCREAMING_SNAKE_CASE			)

           if not pairs:
                  return token

           while True:
                  __UpperCAmelCase    							=min(__SCREAMING_SNAKE_CASE   ,    key=lambda __SCREAMING_SNAKE_CASE			: self.bpe_ranks.get(__SCREAMING_SNAKE_CASE   ,    float("""inf"""			)			)			)
                  if bigram not in self.bpe_ranks:
                         break
                  __UpperCAmelCase							,   __UpperCAmelCase    							=bigram
                  __UpperCAmelCase    							=[]
                  __UpperCAmelCase    							=0
                  while i < len(__SCREAMING_SNAKE_CASE			):
                         try:
                                __UpperCAmelCase    							=word.index(__SCREAMING_SNAKE_CASE   ,    __SCREAMING_SNAKE_CASE			)
                         except ValueError:
                                new_word.extend(word[i:]			)
                                break
                         else:
                                new_word.extend(word[i:j]			)
                                __UpperCAmelCase    							=j

                         if word[i] == first and i < len(__SCREAMING_SNAKE_CASE			) - 1 and word[i + 1] == second:
                                new_word.append(first + second			)
                                i += 2
                         else:
                                new_word.append(word[i]			)
                                i += 1
                  __UpperCAmelCase    							=tuple(__SCREAMING_SNAKE_CASE			)
                  __UpperCAmelCase    							=new_word
                  if len(__SCREAMING_SNAKE_CASE			) == 1:
                         break
                  else:
                         __UpperCAmelCase    							=get_pairs(__SCREAMING_SNAKE_CASE			)
           __UpperCAmelCase    							=""" """.join(__SCREAMING_SNAKE_CASE			)
           __UpperCAmelCase    							=word
           return word



    def _a       (							self			:    str   ,    __SCREAMING_SNAKE_CASE			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]=None			)			->       Union[str, Any]:
           if self.add_bos_token:
                  __UpperCAmelCase    							=[self.bos_token_id]
           else:
                  __UpperCAmelCase    							=[]

           __UpperCAmelCase    							=bos_token_ids + token_ids_a

           if token_ids_a is None:
                  return output

           return output + bos_token_ids + token_ids_a



    def _a       (							self			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    List[int]   ,    __SCREAMING_SNAKE_CASE			:    Optional[List[int]] = None   ,    __SCREAMING_SNAKE_CASE			:    bool = False			)			->       List[int]:
           if already_has_special_tokens:
                  return super().get_special_tokens_mask(
                      token_ids_a=__SCREAMING_SNAKE_CASE   ,    token_ids_a=__SCREAMING_SNAKE_CASE   ,    already_has_special_tokens=__SCREAMING_SNAKE_CASE			)

           if not self.add_bos_token:
                  return super().get_special_tokens_mask(
                      token_ids_a=__SCREAMING_SNAKE_CASE   ,    token_ids_a=__SCREAMING_SNAKE_CASE   ,    already_has_special_tokens=__SCREAMING_SNAKE_CASE			)

           if token_ids_a is None:
                  return [1] + ([0] * len(__SCREAMING_SNAKE_CASE			))
           return [1] + ([0] * len(__SCREAMING_SNAKE_CASE			)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE			))



    def _a       (							self			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    int			)			->       int:
           __UpperCAmelCase    							=[]
           for token in re.findall(self.pat   ,    __SCREAMING_SNAKE_CASE			):
                  __UpperCAmelCase    							="""""".join(
                      self.byte_encoder[b] for b in token.encode("""utf-8"""			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                  bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__SCREAMING_SNAKE_CASE			).split(""" """			)			)
           return bpe_tokens



    def _a       (							self			:    int   ,    __SCREAMING_SNAKE_CASE			:    Dict			)			->       List[Any]:
           return self.encoder.get(__SCREAMING_SNAKE_CASE   ,    self.encoder.get(self.unk_token			)			)



    def _a       (							self			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    Tuple			)			->       Dict:
           return self.decoder.get(__SCREAMING_SNAKE_CASE			)



    def _a       (							self			:    int   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]			)			->       List[str]:
           __UpperCAmelCase    							="""""".join(__SCREAMING_SNAKE_CASE			)
           __UpperCAmelCase    							=bytearray([self.byte_decoder[c] for c in text]			).decode("""utf-8"""   ,    errors=self.errors			)
           return text



    def _a       (							self			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    str   ,    __SCREAMING_SNAKE_CASE			:    Optional[str] = None			)			->       Tuple[str]:
           if not os.path.isdir(__SCREAMING_SNAKE_CASE			):
                  logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''			)
                  return
           __UpperCAmelCase    							=os.path.join(
               __SCREAMING_SNAKE_CASE   ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)
           __UpperCAmelCase    							=os.path.join(
               __SCREAMING_SNAKE_CASE   ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]			)

           with open(__SCREAMING_SNAKE_CASE   ,    """w"""   ,    encoding="""utf-8"""			) as f:
                  f.write(json.dumps(self.encoder   ,    indent=2   ,    sort_keys=__SCREAMING_SNAKE_CASE   ,    ensure_ascii=__SCREAMING_SNAKE_CASE			) + """\n"""			)

           __UpperCAmelCase    							=0
           with open(__SCREAMING_SNAKE_CASE   ,    """w"""   ,    encoding="""utf-8"""			) as writer:
                  writer.write("""#version: 0.2\n"""			)
                  for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,    key=lambda __SCREAMING_SNAKE_CASE			: kv[1]			):
                         if index != token_index:
                                logger.warning(
                                    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                    """ Please check that the tokenizer is not corrupted!"""			)
                                __UpperCAmelCase    							=token_index
                         writer.write(""" """.join(__SCREAMING_SNAKE_CASE			) + """\n"""			)
                         index += 1

           return vocab_file, merge_file



    def _a       (							self			:    Optional[Any]   ,    __SCREAMING_SNAKE_CASE			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    int=False   ,    **__SCREAMING_SNAKE_CASE			:    List[Any]			)			->       Optional[int]:
           __UpperCAmelCase    							=kwargs.pop("""add_prefix_space"""   ,    self.add_prefix_space			)
           if is_split_into_words or add_prefix_space:
                  __UpperCAmelCase    							=""" """ + text
           return (text, kwargs)



    def _a       (							self			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    "Conversation"			)			->       List[int]:
           __UpperCAmelCase    							=[]
           for is_user, text in conversation.iter_texts():
                  input_ids.extend(self.encode(__SCREAMING_SNAKE_CASE   ,    add_special_tokens=__SCREAMING_SNAKE_CASE			) + [self.eos_token_id]			)
           if len(__SCREAMING_SNAKE_CASE			) > self.model_max_length:
                  __UpperCAmelCase    							=input_ids[-self.model_max_length :]
           return input_ids












from collections import defaultdict







def 			lowercase__  (				A_:	int = 5		)  ->							int:







       """simple docstring"""






       __UpperCAmelCase    							=defaultdict(A_		)
       __UpperCAmelCase    							=0

       while True:
              __UpperCAmelCase    							=get_digits(A_		)
              freqs[digits].append(A_		)

              if len(freqs[digits]		) == max_base:
                     __UpperCAmelCase    							=freqs[digits][0] ** 3
                     return base

              num += 1







def 			lowercase__  (				A_:	int		)  ->							str:







       """simple docstring"""






       return "".join(sorted(str(num**3		)		)		)


if __name__ == "__main__":
   print(F"""{solution() = }""")




