
import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_snake_case			:       List[str]      =						logging.get_logger(__name__)

_snake_case			:       int      =						{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_snake_case			:       List[Any]      =						{
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

_snake_case			:       Union[str, Any]      =						{
    "openai-gpt": 5_12,
}

def    __snake_case       (  __magic_name__	):







    '''simple docstring'''


    lowercase						    =			text.strip()
    if not text:
        return []
    lowercase						    =			text.split()
    return tokens





class      UpperCamelCase_			(			__a		):







      '''simple docstring'''



      def __init__( self       :str		,						lowerCAmelCase__       :Optional[Any]=True		,						lowerCAmelCase__       :Optional[int]=None		,						lowerCAmelCase__       :Any=True		,						lowerCAmelCase__       :Union[str, Any]=None		,						lowerCAmelCase__       :int=True		,						)     ->Union[str, Any]:
          if never_split is None:
              lowercase						    =			[]
          lowercase						    =			do_lower_case
          lowercase						    =			set(lowerCAmelCase__    )
          lowercase						    =			tokenize_chinese_chars
          lowercase						    =			strip_accents
          lowercase						    =			do_split_on_punc





      def 					SCREAMING_SNAKE_CASE( self       :str		,						lowerCAmelCase__       :Optional[Any]		,						lowerCAmelCase__       :List[str]=None    )     ->Optional[int]:
          lowercase						    =			self.never_split.union(set(lowerCAmelCase__    )    ) if never_split else self.never_split
          lowercase						    =			self._clean_text(lowerCAmelCase__    )

          # This was added on November 1st, 2018 for the multilingual and Chinese
          # models. This is also applied to the English models now, but it doesn't
          # matter since the English models were not trained on any Chinese data
          # and generally don't have any Chinese data in them (there are Chinese
          # characters in the vocabulary because Wikipedia does have some Chinese
          # words in the English Wikipedia.).
          if self.tokenize_chinese_chars:
              lowercase						    =			self._tokenize_chinese_chars(lowerCAmelCase__    )
          # prevents treating the same character with different unicode codepoints as different characters
          lowercase						    =			unicodedata.normalize("NFC"		,						lowerCAmelCase__    )
          lowercase						    =			whitespace_tokenize(lowerCAmelCase__    )
          lowercase						    =			[]
          for token in orig_tokens:
              if token not in never_split:
                  if self.do_lower_case:
                      lowercase						    =			token.lower()
                      if self.strip_accents is not False:
                          lowercase						    =			self._run_strip_accents(lowerCAmelCase__    )
                  elif self.strip_accents:
                      lowercase						    =			self._run_strip_accents(lowerCAmelCase__    )
              split_tokens.extend(self._run_split_on_punc(lowerCAmelCase__		,						lowerCAmelCase__    )    )

          lowercase						    =			whitespace_tokenize(" ".join(lowerCAmelCase__    )    )
          return output_tokens





      def 					SCREAMING_SNAKE_CASE( self       :Union[str, Any]		,						lowerCAmelCase__       :Any    )     ->Tuple:
          lowercase						    =			unicodedata.normalize("NFD"		,						lowerCAmelCase__    )
          lowercase						    =			[]
          for char in text:
              lowercase						    =			unicodedata.category(lowerCAmelCase__    )
              if cat == "Mn":
                  continue
              output.append(lowerCAmelCase__    )
          return "".join(lowerCAmelCase__    )





      def 					SCREAMING_SNAKE_CASE( self       :Tuple		,						lowerCAmelCase__       :str		,						lowerCAmelCase__       :Any=None    )     ->Tuple:
          if not self.do_split_on_punc or (never_split is not None and text in never_split):
              return [text]
          lowercase						    =			list(lowerCAmelCase__    )
          lowercase						    =			0
          lowercase						    =			True
          lowercase						    =			[]
          while i < len(lowerCAmelCase__    ):
              lowercase						    =			chars[i]
              if _is_punctuation(lowerCAmelCase__    ):
                  output.append([char]    )
                  lowercase						    =			True
              else:
                  if start_new_word:
                      output.append([]    )
                  lowercase						    =			False
                  output[-1].append(lowerCAmelCase__    )
              i += 1

          return ["".join(lowerCAmelCase__    ) for x in output]





      def 					SCREAMING_SNAKE_CASE( self       :Dict		,						lowerCAmelCase__       :List[str]    )     ->Union[str, Any]:
          lowercase						    =			[]
          for char in text:
              lowercase						    =			ord(lowerCAmelCase__    )
              if self._is_chinese_char(lowerCAmelCase__    ):
                  output.append(" "    )
                  output.append(lowerCAmelCase__    )
                  output.append(" "    )
              else:
                  output.append(lowerCAmelCase__    )
          return "".join(lowerCAmelCase__    )





      def 					SCREAMING_SNAKE_CASE( self       :str		,						lowerCAmelCase__       :Optional[int]    )     ->List[Any]:
          # This defines a "chinese character" as anything in the CJK Unicode block:
          #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
          #
          # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
          # despite its name. The modern Korean Hangul alphabet is a different block,
          # as is Japanese Hiragana and Katakana. Those alphabets are used to write
          # space-separated words, so they are not treated specially and handled
          # like the all of the other languages.
          if (
              (cp >= 0x4E00 and cp <= 0x9FFF)
              or (cp >= 0x3400 and cp <= 0x4DBF)  #
              or (cp >= 0x2_0000 and cp <= 0x2_A6DF)  #
              or (cp >= 0x2_A700 and cp <= 0x2_B73F)  #
              or (cp >= 0x2_B740 and cp <= 0x2_B81F)  #
              or (cp >= 0x2_B820 and cp <= 0x2_CEAF)  #
              or (cp >= 0xF900 and cp <= 0xFAFF)
              or (cp >= 0x2_F800 and cp <= 0x2_FA1F)  #
          ):  #
              return True

          return False





      def 					SCREAMING_SNAKE_CASE( self       :List[Any]		,						lowerCAmelCase__       :List[Any]    )     ->List[str]:
          lowercase						    =			[]
          for char in text:
              lowercase						    =			ord(lowerCAmelCase__    )
              if cp == 0 or cp == 0xFFFD or _is_control(lowerCAmelCase__    ):
                  continue
              if _is_whitespace(lowerCAmelCase__    ):
                  output.append(" "    )
              else:
                  output.append(lowerCAmelCase__    )
          return "".join(lowerCAmelCase__    )




def    __snake_case       (  __magic_name__	):







    '''simple docstring'''


    lowercase						    =			set()
    lowercase						    =			word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)	)
        lowercase						    =			char
    return pairs

def    __snake_case       (  __magic_name__	):







    '''simple docstring'''


    lowercase						    =			text.replace("—"		,						"-"	)
    lowercase						    =			text.replace("–"		,						"-"	)
    lowercase						    =			text.replace("―"		,						"-"	)
    lowercase						    =			text.replace("…"		,						"..."	)
    lowercase						    =			text.replace("´"		,						"'"	)
    lowercase						    =			re.sub(r"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"		,						r" \1 "		,						__magic_name__	)
    lowercase						    =			re.sub(r"\s*\n\s*"		,						" \n "		,						__magic_name__	)
    lowercase						    =			re.sub(r"[^\S\n]+"		,						" "		,						__magic_name__	)
    return text.strip()





class      UpperCamelCase_			(			__a		):







      '''simple docstring'''



      UpperCamelCase			:     List[Any]							      =				VOCAB_FILES_NAMES
      UpperCamelCase			:     Union[str, Any]							      =				PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase			:     List[Any]							      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase			:     int							      =				['''input_ids''', '''attention_mask''']





      def __init__( self       :Optional[Any]		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :Dict		,						lowerCAmelCase__       :List[str]="<unk>"		,						**lowerCAmelCase__       :Optional[Any]    )     ->Tuple:
          super().__init__(unk_token=lowerCAmelCase__		,						**lowerCAmelCase__    )

          try:
              import ftfy
              from spacy.lang.en import English

              lowercase						    =			English()
              lowercase						    =			_nlp.tokenizer
              lowercase						    =			ftfy.fix_text
          except ImportError:
              logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."    )
              lowercase						    =			BasicTokenizer(do_lower_case=lowerCAmelCase__    )
              lowercase						    =			None

          with open(lowerCAmelCase__		,						encoding="utf-8"    ) as vocab_handle:
              lowercase						    =			json.load(lowerCAmelCase__    )
          lowercase						    =			{v: k for k, v in self.encoder.items()}
          with open(lowerCAmelCase__		,						encoding="utf-8"    ) as merges_handle:
              lowercase						    =			merges_handle.read().split("\n"    )[1:-1]
          lowercase						    =			[tuple(merge.split()    ) for merge in merges]
          lowercase						    =			dict(zip(lowerCAmelCase__		,						range(len(lowerCAmelCase__    )    )    )    )
          lowercase						    =			{}





      @property
      def 					SCREAMING_SNAKE_CASE( self       :List[str]    )     ->Optional[Any]:
          return True





      @property
      def 					SCREAMING_SNAKE_CASE( self       :Dict    )     ->Optional[int]:
          return len(self.encoder    )





      def 					SCREAMING_SNAKE_CASE( self       :List[str]    )     ->Dict:
          return dict(self.encoder		,						**self.added_tokens_encoder    )





      def 					SCREAMING_SNAKE_CASE( self       :Optional[Any]		,						lowerCAmelCase__       :Union[str, Any]    )     ->Optional[Any]:
          lowercase						    =			tuple(token[:-1]    ) + (token[-1] + "</w>",)
          if token in self.cache:
              return self.cache[token]
          lowercase						    =			get_pairs(lowerCAmelCase__    )

          if not pairs:
              return token + "</w>"

          while True:
              lowercase						    =			min(lowerCAmelCase__		,						key=lambda lowerCAmelCase__    : self.bpe_ranks.get(lowerCAmelCase__		,						float("inf"    )    )    )
              if bigram not in self.bpe_ranks:
                  break
              lowercase    ,  lowercase						    =			bigram
              lowercase						    =			[]
              lowercase						    =			0
              while i < len(lowerCAmelCase__    ):
                  try:
                      lowercase						    =			word.index(lowerCAmelCase__		,						lowerCAmelCase__    )
                  except ValueError:
                      new_word.extend(word[i:]    )
                      break
                  else:
                      new_word.extend(word[i:j]    )
                      lowercase						    =			j

                  if word[i] == first and i < len(lowerCAmelCase__    ) - 1 and word[i + 1] == second:
                      new_word.append(first + second    )
                      i += 2
                  else:
                      new_word.append(word[i]    )
                      i += 1
              lowercase						    =			tuple(lowerCAmelCase__    )
              lowercase						    =			new_word
              if len(lowerCAmelCase__    ) == 1:
                  break
              else:
                  lowercase						    =			get_pairs(lowerCAmelCase__    )
          lowercase						    =			" ".join(lowerCAmelCase__    )
          if word == "\n  </w>":
              lowercase						    =			"\n</w>"
          lowercase						    =			word
          return word





      def 					SCREAMING_SNAKE_CASE( self       :List[Any]		,						lowerCAmelCase__       :str    )     ->List[Any]:
          lowercase						    =			[]
          if self.fix_text is None:
              # Using BERT's BasicTokenizer
              lowercase						    =			self.nlp.tokenize(lowerCAmelCase__    )
              for token in text:
                  split_tokens.extend(list(self.bpe(lowerCAmelCase__    ).split(" "    )    )    )
          else:
              # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
              lowercase						    =			self.nlp(text_standardize(self.fix_text(lowerCAmelCase__    )    )    )
              for token in text:
                  split_tokens.extend(list(self.bpe(token.text.lower()    ).split(" "    )    )    )
          return split_tokens





      def 					SCREAMING_SNAKE_CASE( self       :Tuple		,						lowerCAmelCase__       :List[Any]    )     ->Union[str, Any]:
          return self.encoder.get(lowerCAmelCase__		,						self.encoder.get(self.unk_token    )    )





      def 					SCREAMING_SNAKE_CASE( self       :Dict		,						lowerCAmelCase__       :Any    )     ->Dict:
          return self.decoder.get(lowerCAmelCase__		,						self.unk_token    )





      def 					SCREAMING_SNAKE_CASE( self       :int		,						lowerCAmelCase__       :List[str]    )     ->Optional[int]:
          lowercase						    =			"".join(lowerCAmelCase__    ).replace("</w>"		,						" "    ).strip()
          return out_string





      def 					SCREAMING_SNAKE_CASE( self       :Union[str, Any]		,						lowerCAmelCase__       :str		,						lowerCAmelCase__       :Optional[str] = None    )     ->Tuple[str]:
          if not os.path.isdir(lowerCAmelCase__    ):
              logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
              return
          lowercase						    =			os.path.join(
              lowerCAmelCase__		,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )
          lowercase						    =			os.path.join(
              lowerCAmelCase__		,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]    )

          with open(lowerCAmelCase__		,						"w"		,						encoding="utf-8"    ) as f:
              f.write(json.dumps(self.encoder		,						indent=2		,						sort_keys=lowerCAmelCase__		,						ensure_ascii=lowerCAmelCase__    ) + "\n"    )

          lowercase						    =			0
          with open(lowerCAmelCase__		,						"w"		,						encoding="utf-8"    ) as writer:
              writer.write("#version: 0.2\n"    )
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,						key=lambda lowerCAmelCase__    : kv[1]    ):
                  if index != token_index:
                      logger.warning(
                          F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                          " Please check that the tokenizer is not corrupted!"    )
                      lowercase						    =			token_index
                  writer.write(" ".join(lowerCAmelCase__    ) + "\n"    )
                  index += 1

          return vocab_file, merge_file


from functools import partial
from typing import Any, Callable, Dict, List, Type, TypeVar, Union, overload

import torch
import torch.nn as nn
import torch.types

def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):







				'''simple docstring'''


				if not inplace:
								lowercase						    =			ma + ma
				else:
								ma += ma

				return ma

def    __snake_case       (  __magic_name__		,						__magic_name__	):







				'''simple docstring'''


				lowercase						    =			-1 * len(__magic_name__	)
				lowercase						    =			list(range(len(tensor.shape[:zero_index]	)	)	)
				return tensor.permute(first_inds + [zero_index + i for i in inds]	)

def    __snake_case       (  __magic_name__		,						__magic_name__	):







				'''simple docstring'''


				return t.reshape(t.shape[:-no_dims] + (-1,)	)

def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__		,						__magic_name__ = 1e-4	):







				'''simple docstring'''


				lowercase						    =			mask.expand(*value.shape	)
				return torch.sum(mask * value		,						dim=__magic_name__	) / (eps + torch.sum(__magic_name__		,						dim=__magic_name__	))

def    __snake_case       (  __magic_name__		,						__magic_name__ = 2.3_1_2_5		,						__magic_name__ = 2_1.6_8_7_5		,						__magic_name__ = 64	):







				'''simple docstring'''


				lowercase						    =			torch.linspace(__magic_name__		,						__magic_name__		,						no_bins - 1		,						device=pts.device	)
				lowercase						    =			torch.sqrt(torch.sum((pts.unsqueeze(-2	) - pts.unsqueeze(-3	)) ** 2		,						dim=-1	)	)
				return torch.bucketize(__magic_name__		,						__magic_name__	)

def    __snake_case       (  __magic_name__		,						__magic_name__	):







				'''simple docstring'''


				lowercase						    =			dicts[0]
				lowercase						    =			{}
				for k, v in first.items():
								lowercase						    =			[d[k] for d in dicts]
								if isinstance(__magic_name__		,						__magic_name__	):
												lowercase						    =			dict_multimap(__magic_name__		,						__magic_name__	)
								else:
												lowercase						    =			fn(__magic_name__	)

				return new_dict

def    __snake_case       (  __magic_name__		,						__magic_name__	):







				'''simple docstring'''


				lowercase						    =			v_bins.view(((1,) * len(x.shape	)) + (len(__magic_name__	),)	)
				lowercase						    =			x[..., None] - reshaped_bins
				lowercase						    =			torch.argmin(torch.abs(__magic_name__	)		,						dim=-1	)
				return nn.functional.one_hot(__magic_name__		,						num_classes=len(__magic_name__	)	).float()

def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__ = 0		,						__magic_name__ = 0	):







				'''simple docstring'''


				lowercase						    =			[]
				for i, s in enumerate(data.shape[:no_batch_dims]	):
								lowercase						    =			torch.arange(__magic_name__	)
								lowercase						    =			r.view(*(*((1,) * i), -1, *((1,) * (len(inds.shape	) - i - 1)))	)
								ranges.append(__magic_name__	)

				lowercase						    =			[slice(__magic_name__	) for _ in range(len(data.shape	) - no_batch_dims	)]
				lowercase						    =			inds
				ranges.extend(__magic_name__	)
				# Matt note: Editing this to get around the behaviour of using a list as an array index changing
				# in recent Numpy versions
				return data[tuple(__magic_name__	)]


_snake_case			:       List[Any]      =						TypeVar("T")

def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):







				'''simple docstring'''


				lowercase						    =			{}
				for k, v in dic.items():
								if isinstance(__magic_name__		,						__magic_name__	):
												lowercase						    =			dict_map(__magic_name__		,						__magic_name__		,						__magic_name__	)
								else:
												lowercase						    =			tree_map(__magic_name__		,						__magic_name__		,						__magic_name__	)

				return new_dict

@overload
def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):







				'''simple docstring'''


				...

@overload
def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):







				'''simple docstring'''


				...

@overload
def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):







				'''simple docstring'''


				...

@overload
def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):







				'''simple docstring'''


				...

def    __snake_case       (  __magic_name__		,						__magic_name__		,						__magic_name__	):







				'''simple docstring'''


				if isinstance(__magic_name__		,						__magic_name__	):
								return dict_map(__magic_name__		,						__magic_name__		,						__magic_name__	)
				elif isinstance(__magic_name__		,						__magic_name__	):
								return [tree_map(__magic_name__		,						__magic_name__		,						__magic_name__	) for x in tree]
				elif isinstance(__magic_name__		,						__magic_name__	):
								return tuple(tree_map(__magic_name__		,						__magic_name__		,						__magic_name__	) for x in tree	)
				elif isinstance(__magic_name__		,						__magic_name__	):
								return fn(__magic_name__	)
				else:
								print(type(__magic_name__	)	)
								raise ValueError("Not supported"	)




_snake_case			:       Union[str, Any]      =						partial(tree_map, leaf_type=torch.Tensor)
