'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation

__UpperCAmelCase  							=logging.get_logger(__name__)

__UpperCAmelCase  							={
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__UpperCAmelCase  							={
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
}

__UpperCAmelCase  							={
    "gpt2": 1_0_2_4,
    "gpt2-medium": 1_0_2_4,
    "gpt2-large": 1_0_2_4,
    "gpt2-xl": 1_0_2_4,
    "distilgpt2": 1_0_2_4,
}






@lru_cache()
def __lowerCAmelCase				(							)			->					Any:
    __lowerCamelCase						=						(
        list(range(ord('''!'''       )       ,						ord('''~'''       ) + 1       )       ) + list(range(ord('''¡'''       )       ,						ord('''¬'''       ) + 1       )       ) + list(range(ord('''®'''       )       ,						ord('''ÿ'''       ) + 1       )       )
    )
    __lowerCamelCase						=						bs[:]
    __lowerCamelCase						=						0
    for b in range(2**8       ):
        if b not in bs:
            bs.append(UpperCamelCase__       )
            cs.append(2**8 + n       )
            n += 1
    __lowerCamelCase						=						[chr(UpperCamelCase__       ) for n in cs]
    return dict(zip(UpperCamelCase__       ,						UpperCamelCase__       )       )






def __lowerCAmelCase				(							UpperCamelCase__       )			->					List[str]:
    __lowerCamelCase						=						set()
    __lowerCamelCase						=						word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)       )
        __lowerCamelCase						=						char
    return pairs




class 					a__   (     UpperCAmelCase__				):
     lowerCamelCase						:  Optional[int]							=VOCAB_FILES_NAMES
     lowerCamelCase						:  List[Any]							=PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase						:  List[Any]							=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase						:  int							=["input_ids", "attention_mask"]






     def __init__(				self	:	List[Any]						,       a	:	Union[str, Any]						,       a	:	Dict						,       a	:	str="replace"						,       a	:	str="<|endoftext|>"						,       a	:	Tuple="<|endoftext|>"						,       a	:	Dict="<|endoftext|>"						,       a	:	Tuple=None						,       a	:	Optional[int]=False						,       a	:	int=False						,       **a	:	int						,       ):





         """simple docstring"""



         __lowerCamelCase						=						AddedToken(a						,       lstrip=a						,       rstrip=a					) if isinstance(a						,       a					) else bos_token
         __lowerCamelCase						=						AddedToken(a						,       lstrip=a						,       rstrip=a					) if isinstance(a						,       a					) else eos_token
         __lowerCamelCase						=						AddedToken(a						,       lstrip=a						,       rstrip=a					) if isinstance(a						,       a					) else unk_token
         __lowerCamelCase						=						AddedToken(a						,       lstrip=a						,       rstrip=a					) if isinstance(a						,       a					) else pad_token
         super().__init__(
             errors=a						,       unk_token=a						,       bos_token=a						,       eos_token=a						,       pad_token=a						,       add_prefix_space=a						,       add_bos_token=a						,       **a						,       )
         __lowerCamelCase						=						add_bos_token

         with open(a						,       encoding='''utf-8'''					) as vocab_handle:
             __lowerCamelCase						=						json.load(a					)
         __lowerCamelCase						=						{v: k for k, v in self.encoder.items()}
         __lowerCamelCase						=						errors  # how to handle errors in decoding
         __lowerCamelCase						=						bytes_to_unicode()
         __lowerCamelCase						=						{v: k for k, v in self.byte_encoder.items()}
         with open(a						,       encoding='''utf-8'''					) as merges_handle:
             __lowerCamelCase						=						merges_handle.read().split('''\n'''					)[1:-1]
         __lowerCamelCase						=						[tuple(merge.split()					) for merge in bpe_merges]
         __lowerCamelCase						=						dict(zip(a						,       range(len(a					)					)					)					)
         __lowerCamelCase						=						{}
         __lowerCamelCase						=						add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         __lowerCamelCase						=						re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''					)






     @property
     def      SCREAMING_SNAKE_CASE__     (				self	:	Tuple					):





         """simple docstring"""



         return len(self.encoder					)






     def      SCREAMING_SNAKE_CASE__     (				self	:	Optional[Any]					):





         """simple docstring"""



         return dict(self.encoder						,       **self.added_tokens_encoder					)






     def      SCREAMING_SNAKE_CASE__     (				self	:	List[Any]						,       a	:	Tuple					):





         """simple docstring"""



         if token in self.cache:
             return self.cache[token]
         __lowerCamelCase						=						tuple(a					)
         __lowerCamelCase						=						get_pairs(a					)

         if not pairs:
             return token

         while True:
             __lowerCamelCase						=						min(a						,       key=lambda a					: self.bpe_ranks.get(a						,       float('''inf'''					)					)					)
             if bigram not in self.bpe_ranks:
                 break
             __lowerCamelCase			,						__lowerCamelCase						=						bigram
             __lowerCamelCase						=						[]
             __lowerCamelCase						=						0
             while i < len(a					):
                 try:
                     __lowerCamelCase						=						word.index(a						,       a					)
                 except ValueError:
                     new_word.extend(word[i:]					)
                     break
                 else:
                     new_word.extend(word[i:j]					)
                     __lowerCamelCase						=						j

                 if word[i] == first and i < len(a					) - 1 and word[i + 1] == second:
                     new_word.append(first + second					)
                     i += 2
                 else:
                     new_word.append(word[i]					)
                     i += 1
             __lowerCamelCase						=						tuple(a					)
             __lowerCamelCase						=						new_word
             if len(a					) == 1:
                 break
             else:
                 __lowerCamelCase						=						get_pairs(a					)
         __lowerCamelCase						=						''' '''.join(a					)
         __lowerCamelCase						=						word
         return word






     def      SCREAMING_SNAKE_CASE__     (				self	:	Any						,       a	:	Union[str, Any]						,       a	:	Union[str, Any]=None					):





         """simple docstring"""



         if self.add_bos_token:
             __lowerCamelCase						=						[self.bos_token_id]
         else:
             __lowerCamelCase						=						[]

         __lowerCamelCase						=						bos_token_ids + token_ids_a

         if token_ids_a is None:
             return output

         return output + bos_token_ids + token_ids_a






     def      SCREAMING_SNAKE_CASE__     (				self	:	int						,       a	:	List[int]						,       a	:	Optional[List[int]] = None						,       a	:	bool = False					):





         """simple docstring"""



         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=a						,       token_ids_a=a						,       already_has_special_tokens=a					)

         if not self.add_bos_token:
             return super().get_special_tokens_mask(
                 token_ids_a=a						,       token_ids_a=a						,       already_has_special_tokens=a					)

         if token_ids_a is None:
             return [1] + ([0] * len(a					))
         return [1] + ([0] * len(a					)) + [1] + ([0] * len(a					))






     def      SCREAMING_SNAKE_CASE__     (				self	:	int						,       a	:	Optional[int]					):





         """simple docstring"""



         __lowerCamelCase						=						[]
         for token in re.findall(self.pat						,       a					):
             __lowerCamelCase						=						''''''.join(
                 self.byte_encoder[b] for b in token.encode('''utf-8'''					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
             bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a					).split(''' '''					)					)
         return bpe_tokens






     def      SCREAMING_SNAKE_CASE__     (				self	:	Optional[Any]						,       a	:	Any					):





         """simple docstring"""



         return self.encoder.get(a						,       self.encoder.get(self.unk_token					)					)






     def      SCREAMING_SNAKE_CASE__     (				self	:	Any						,       a	:	List[str]					):





         """simple docstring"""



         return self.decoder.get(a					)






     def      SCREAMING_SNAKE_CASE__     (				self	:	Optional[Any]						,       a	:	str					):





         """simple docstring"""



         __lowerCamelCase						=						''''''.join(a					)
         __lowerCamelCase						=						bytearray([self.byte_decoder[c] for c in text]					).decode('''utf-8'''						,       errors=self.errors					)
         return text






     def      SCREAMING_SNAKE_CASE__     (				self	:	List[str]						,       a	:	str						,       a	:	Optional[str] = None					):





         """simple docstring"""



         if not os.path.isdir(a					):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
             return
         __lowerCamelCase						=						os.path.join(
             a						,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
         __lowerCamelCase						=						os.path.join(
             a						,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']					)

         with open(a						,       '''w'''						,       encoding='''utf-8'''					) as f:
             f.write(json.dumps(self.encoder						,       indent=2						,       sort_keys=a						,       ensure_ascii=a					) + '''\n'''					)

         __lowerCamelCase						=						0
         with open(a						,       '''w'''						,       encoding='''utf-8'''					) as writer:
             writer.write('''#version: 0.2\n'''					)
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,       key=lambda a					: kv[1]					):
                 if index != token_index:
                     logger.warning(
                         f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                         ''' Please check that the tokenizer is not corrupted!'''					)
                     __lowerCamelCase						=						token_index
                 writer.write(''' '''.join(a					) + '''\n'''					)
                 index += 1

         return vocab_file, merge_file






     def      SCREAMING_SNAKE_CASE__     (				self	:	Union[str, Any]						,       a	:	int						,       a	:	Optional[int]=False						,       **a	:	Any					):





         """simple docstring"""



         __lowerCamelCase						=						kwargs.pop('''add_prefix_space'''						,       self.add_prefix_space					)
         if is_split_into_words or add_prefix_space:
             __lowerCamelCase						=						''' ''' + text
         return (text, kwargs)






     def      SCREAMING_SNAKE_CASE__     (				self	:	Tuple						,       a	:	"Conversation"					):





         """simple docstring"""



         __lowerCamelCase						=						[]
         for is_user, text in conversation.iter_texts():
             input_ids.extend(self.encode(a						,       add_special_tokens=a					) + [self.eos_token_id]					)
         if len(a					) > self.model_max_length:
             __lowerCamelCase						=						input_ids[-self.model_max_length :]
         return input_ids

'''simple docstring'''




from __future__ import annotations






def __lowerCAmelCase				(							UpperCamelCase__       ,						UpperCamelCase__       ,						UpperCamelCase__       )			->					tuple[float, float]:
    assert (
        c % greatest_common_divisor(UpperCamelCase__       ,						UpperCamelCase__       ) == 0
    )  # greatest_common_divisor(a,b) function implemented below
    ((__lowerCamelCase)			,						(__lowerCamelCase)			,						(__lowerCamelCase))						=						extended_gcd(UpperCamelCase__       ,						UpperCamelCase__       )  # extended_gcd(a,b) function implemented below
    __lowerCamelCase						=						c / d
    return (r * x, r * y)






def __lowerCAmelCase				(							UpperCamelCase__       ,						UpperCamelCase__       ,						UpperCamelCase__       ,						UpperCamelCase__ = 2       )			->					None:
    ((__lowerCamelCase)			,						(__lowerCamelCase))						=						diophantine(UpperCamelCase__       ,						UpperCamelCase__       ,						UpperCamelCase__       )  # Initial value
    __lowerCamelCase						=						greatest_common_divisor(UpperCamelCase__       ,						UpperCamelCase__       )
    __lowerCamelCase						=						a // d
    __lowerCamelCase						=						b // d

    for i in range(UpperCamelCase__       ):
        __lowerCamelCase						=						xa + i * q
        __lowerCamelCase						=						ya - i * p
        print(UpperCamelCase__       ,						UpperCamelCase__       )






def __lowerCAmelCase				(							UpperCamelCase__       ,						UpperCamelCase__       )			->					int:
    if a < b:
        __lowerCamelCase			,						__lowerCamelCase						=						b, a

    while a % b != 0:
        __lowerCamelCase			,						__lowerCamelCase						=						b, a % b

    return b






def __lowerCAmelCase				(							UpperCamelCase__       ,						UpperCamelCase__       )			->					tuple[int, int, int]:
    assert a >= 0 and b >= 0

    if b == 0:
        __lowerCamelCase			,						__lowerCamelCase			,						__lowerCamelCase						=						a, 1, 0
    else:
        ((__lowerCamelCase)			,						(__lowerCamelCase)			,						(__lowerCamelCase))						=						extended_gcd(UpperCamelCase__       ,						a % b       )
        __lowerCamelCase						=						q
        __lowerCamelCase						=						p - q * (a // b)

    assert a % d == 0 and b % d == 0
    assert d == a * x + b * y

    return (d, x, y)


if __name__ == "__main__":
  from doctest import testmod

  testmod(name="diophantine", verbose=True)
  testmod(name="diophantine_all_soln", verbose=True)
  testmod(name="extended_gcd", verbose=True)
  testmod(name="greatest_common_divisor", verbose=True)
