

import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowercase_									=    logging.get_logger(__name__)

lowercase_									=    {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

lowercase_									=    {
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

lowercase_									=    {
    "openai-gpt": 512,
}
def 		_snake_case(       SCREAMING_SNAKE_CASE__     :      Union[str, Any] )				-> Optional[int]:





    '''simple docstring'''






    A__        =  text.strip()
    if not text:
        return []
    A__        =  text.split()
    return tokens


class 					A		(				lowercase__					):







    """simple docstring"""





    def __init__( self					:			List[str],lowercase_					:			Union[str, Any]=True,lowercase_					:			Optional[Any]=None,lowercase_					:			Optional[int]=True,lowercase_					:			Optional[Any]=None,lowercase_					:			Dict=True,)->       Any:





        '''simple docstring'''







        if never_split is None:
            A__        =  []
        A__        =  do_lower_case
        A__        =  set(__UpperCamelCase						)
        A__        =  tokenize_chinese_chars
        A__        =  strip_accents
        A__        =  do_split_on_punc


    def 		snake_case__	( self					:			Any,lowercase_					:			Any,lowercase_					:			Any=None						)->       int:





        '''simple docstring'''







        A__        =  self.never_split.union(set(__UpperCamelCase						)						) if never_split else self.never_split
        A__        =  self._clean_text(__UpperCamelCase						)

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        if self.tokenize_chinese_chars:
            A__        =  self._tokenize_chinese_chars(__UpperCamelCase						)
        # prevents treating the same character with different unicode codepoints as different characters
        A__        =  unicodedata.normalize('NFC',__UpperCamelCase						)
        A__        =  whitespace_tokenize(__UpperCamelCase						)
        A__        =  []
        for token in orig_tokens:
            if token not in never_split:
                if self.do_lower_case:
                    A__        =  token.lower()
                    if self.strip_accents is not False:
                        A__        =  self._run_strip_accents(__UpperCamelCase						)
                elif self.strip_accents:
                    A__        =  self._run_strip_accents(__UpperCamelCase						)
            split_tokens.extend(self._run_split_on_punc(__UpperCamelCase,__UpperCamelCase						)						)

        A__        =  whitespace_tokenize(' '.join(__UpperCamelCase						)						)
        return output_tokens


    def 		snake_case__	( self					:			str,lowercase_					:			List[Any]						)->       Union[str, Any]:





        '''simple docstring'''







        A__        =  unicodedata.normalize('NFD',__UpperCamelCase						)
        A__        =  []
        for char in text:
            A__        =  unicodedata.category(__UpperCamelCase						)
            if cat == "Mn":
                continue
            output.append(__UpperCamelCase						)
        return "".join(__UpperCamelCase						)


    def 		snake_case__	( self					:			str,lowercase_					:			Optional[int],lowercase_					:			List[Any]=None						)->       Union[str, Any]:





        '''simple docstring'''







        if not self.do_split_on_punc or (never_split is not None and text in never_split):
            return [text]
        A__        =  list(__UpperCamelCase						)
        A__        =  0
        A__        =  True
        A__        =  []
        while i < len(__UpperCamelCase						):
            A__        =  chars[i]
            if _is_punctuation(__UpperCamelCase						):
                output.append([char]						)
                A__        =  True
            else:
                if start_new_word:
                    output.append([]						)
                A__        =  False
                output[-1].append(__UpperCamelCase						)
            i += 1

        return ["".join(__UpperCamelCase						) for x in output]


    def 		snake_case__	( self					:			Optional[Any],lowercase_					:			Tuple						)->       Optional[int]:





        '''simple docstring'''







        A__        =  []
        for char in text:
            A__        =  ord(__UpperCamelCase						)
            if self._is_chinese_char(__UpperCamelCase						):
                output.append(' '						)
                output.append(__UpperCamelCase						)
                output.append(' '						)
            else:
                output.append(__UpperCamelCase						)
        return "".join(__UpperCamelCase						)


    def 		snake_case__	( self					:			Union[str, Any],lowercase_					:			Union[str, Any]						)->       str:





        '''simple docstring'''







        if (
            (cp >= 0X4_e_0_0 and cp <= 0X9_f_f_f)
            or (cp >= 0X3_4_0_0 and cp <= 0X4_d_b_f)  #
            or (cp >= 0X2_0_0_0_0 and cp <= 0X2_a_6_d_f)  #
            or (cp >= 0X2_a_7_0_0 and cp <= 0X2_b_7_3_f)  #
            or (cp >= 0X2_b_7_4_0 and cp <= 0X2_b_8_1_f)  #
            or (cp >= 0X2_b_8_2_0 and cp <= 0X2_c_e_a_f)  #
            or (cp >= 0Xf_9_0_0 and cp <= 0Xf_a_f_f)
            or (cp >= 0X2_f_8_0_0 and cp <= 0X2_f_a_1_f)  #
        ):  #
            return True

        return False


    def 		snake_case__	( self					:			List[str],lowercase_					:			str						)->       Any:





        '''simple docstring'''







        A__        =  []
        for char in text:
            A__        =  ord(__UpperCamelCase						)
            if cp == 0 or cp == 0Xf_f_f_d or _is_control(__UpperCamelCase						):
                continue
            if _is_whitespace(__UpperCamelCase						):
                output.append(' '						)
            else:
                output.append(__UpperCamelCase						)
        return "".join(__UpperCamelCase						)




def 		_snake_case(       SCREAMING_SNAKE_CASE__     :      Optional[int] )				-> Union[str, Any]:





    '''simple docstring'''






    A__        =  set()
    A__        =  word[0]
    for char in word[1:]:
        pairs.add((prev_char, char) )
        A__        =  char
    return pairs
def 		_snake_case(       SCREAMING_SNAKE_CASE__     :      Tuple )				-> Dict:





    '''simple docstring'''






    A__        =  text.replace('—'       ,     '-' )
    A__        =  text.replace('–'       ,     '-' )
    A__        =  text.replace('―'       ,     '-' )
    A__        =  text.replace('…'       ,     '...' )
    A__        =  text.replace('´'       ,     '\'' )
    A__        =  re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'       ,     R' \1 '       ,     UpperCamelCase__ )
    A__        =  re.sub(R'\s*\n\s*'       ,     ' \n '       ,     UpperCamelCase__ )
    A__        =  re.sub(R'[^\S\n]+'       ,     ' '       ,     UpperCamelCase__ )
    return text.strip()


class 					A		(				lowercase__					):







    """simple docstring"""





    lowerCamelCase						      =				VOCAB_FILES_NAMES
    lowerCamelCase						      =				PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase						      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase						      =				['input_ids', 'attention_mask']


    def __init__( self					:			Optional[Any],lowercase_					:			Any,lowercase_					:			Any,lowercase_					:			int="<unk>",**lowercase_					:			str						)->       Optional[Any]:





        '''simple docstring'''







        super().__init__(unk_token=__UpperCamelCase,**__UpperCamelCase						)

        try:
            import ftfy
            from spacy.lang.en import English

            A__        =  English()
            A__        =  _nlp.tokenizer
            A__        =  ftfy.fix_text
        except ImportError:
            logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'						)
            A__        =  BasicTokenizer(do_lower_case=__UpperCamelCase						)
            A__        =  None

        with open(__UpperCamelCase,encoding='utf-8'						) as vocab_handle:
            A__        =  json.load(__UpperCamelCase						)
        A__        =  {v: k for k, v in self.encoder.items()}
        with open(__UpperCamelCase,encoding='utf-8'						) as merges_handle:
            A__        =  merges_handle.read().split('\n'						)[1:-1]
        A__        =  [tuple(merge.split()						) for merge in merges]
        A__        =  dict(zip(__UpperCamelCase,range(len(__UpperCamelCase						)						)						)						)
        A__        =  {}


    @property
    def 		snake_case__	( self					:			Dict						)->       Optional[int]:





        '''simple docstring'''







        return True


    @property
    def 		snake_case__	( self					:			str						)->       Any:





        '''simple docstring'''







        return len(self.encoder						)


    def 		snake_case__	( self					:			str						)->       Tuple:





        '''simple docstring'''







        return dict(self.encoder,**self.added_tokens_encoder						)


    def 		snake_case__	( self					:			Dict,lowercase_					:			Dict						)->       Union[str, Any]:





        '''simple docstring'''







        A__        =  tuple(token[:-1]						) + (token[-1] + '''</w>''',)
        if token in self.cache:
            return self.cache[token]
        A__        =  get_pairs(__UpperCamelCase						)

        if not pairs:
            return token + "</w>"

        while True:
            A__        =  min(__UpperCamelCase,key=lambda lowercase_						: self.bpe_ranks.get(__UpperCamelCase,float('inf'						)						)						)
            if bigram not in self.bpe_ranks:
                break
            A__        =  bigram
            A__        =  []
            A__        =  0
            while i < len(__UpperCamelCase						):
                try:
                    A__        =  word.index(__UpperCamelCase,__UpperCamelCase						)
                except ValueError:
                    new_word.extend(word[i:]						)
                    break
                else:
                    new_word.extend(word[i:j]						)
                    A__        =  j

                if word[i] == first and i < len(__UpperCamelCase						) - 1 and word[i + 1] == second:
                    new_word.append(first + second						)
                    i += 2
                else:
                    new_word.append(word[i]						)
                    i += 1
            A__        =  tuple(__UpperCamelCase						)
            A__        =  new_word
            if len(__UpperCamelCase						) == 1:
                break
            else:
                A__        =  get_pairs(__UpperCamelCase						)
        A__        =  ''' '''.join(__UpperCamelCase						)
        if word == "\n  </w>":
            A__        =  '''\n</w>'''
        A__        =  word
        return word


    def 		snake_case__	( self					:			Optional[Any],lowercase_					:			Optional[int]						)->       List[Any]:





        '''simple docstring'''







        A__        =  []
        if self.fix_text is None:
            # Using BERT's BasicTokenizer
            A__        =  self.nlp.tokenize(__UpperCamelCase						)
            for token in text:
                split_tokens.extend(list(self.bpe(__UpperCamelCase						).split(' '						)						)						)
        else:
            # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
            A__        =  self.nlp(text_standardize(self.fix_text(__UpperCamelCase						)						)						)
            for token in text:
                split_tokens.extend(list(self.bpe(token.text.lower()						).split(' '						)						)						)
        return split_tokens


    def 		snake_case__	( self					:			Tuple,lowercase_					:			Union[str, Any]						)->       List[Any]:





        '''simple docstring'''







        return self.encoder.get(__UpperCamelCase,self.encoder.get(self.unk_token						)						)


    def 		snake_case__	( self					:			str,lowercase_					:			Optional[Any]						)->       Optional[int]:





        '''simple docstring'''







        return self.decoder.get(__UpperCamelCase,self.unk_token						)


    def 		snake_case__	( self					:			Optional[Any],lowercase_					:			int						)->       int:





        '''simple docstring'''







        A__        =  ''''''.join(__UpperCamelCase						).replace('</w>',' '						).strip()
        return out_string


    def 		snake_case__	( self					:			Optional[int],lowercase_					:			str,lowercase_					:			Optional[str] = None						)->       Tuple[str]:





        '''simple docstring'''







        if not os.path.isdir(__UpperCamelCase						):
            logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
            return
        A__        =  os.path.join(
            __UpperCamelCase,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)
        A__        =  os.path.join(
            __UpperCamelCase,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']						)

        with open(__UpperCamelCase,'w',encoding='utf-8'						) as f:
            f.write(json.dumps(self.encoder,indent=2,sort_keys=__UpperCamelCase,ensure_ascii=__UpperCamelCase						) + '\n'						)

        A__        =  0
        with open(__UpperCamelCase,'w',encoding='utf-8'						) as writer:
            writer.write('#version: 0.2\n'						)
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),key=lambda lowercase_						: kv[1]						):
                if index != token_index:
                    logger.warning(
                        F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                        ' Please check that the tokenizer is not corrupted!'						)
                    A__        =  token_index
                writer.write(' '.join(__UpperCamelCase						) + '\n'						)
                index += 1

        return vocab_file, merge_file






'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import _LazyModule


_lowercase       :						List[str]						={"tokenization_bert_japanese": ["BertJapaneseTokenizer", "CharacterTokenizer", "MecabTokenizer"]}


if TYPE_CHECKING:
				from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer

else:
				import sys

				_lowercase       :						Dict						=_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




