'''simple docstring'''






import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
 import importlib.resources as pkg_resources
except ImportError:
 # Try backported to PY<37 `importlib_resources`.
 import importlib_resources as pkg_resources

from . import resources


_UpperCAmelCase						:    List[str]     					=					'''https://github.com/huggingface/datasets/tree/main/src/datasets/utils'''
_UpperCAmelCase						:    int     					=					F'''{BASE_REF_URL}/{__file__}'''
_UpperCAmelCase						:    Any     					=					logging.getLogger(__name__)





def    _SCREAMING_SNAKE_CASE		(			__snake_case       : str							):
     _A								=     pkg_resources.read_text(_UpperCAmelCase      ,		_UpperCAmelCase							)
     return yaml.safe_load(_UpperCAmelCase							), F'{BASE_REF_URL}/resources/{resource}'


_UpperCAmelCase					,     _UpperCAmelCase						:    Union[str, Any]     					=					load_yaml_resource('''readme_structure.yaml''')

_UpperCAmelCase						:    int     					=					[
    '''[Needs More Information]''',
    '''[More Information Needed]''',
    '''(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)''',
]

# Dictionary representation of section/readme, error_list, warning_list
_UpperCAmelCase						:    Union[str, Any]     					=					Tuple[dict, List[str], List[str]]







class lowercase_					:







   """simple docstring"""

   def __init__(			self			: Optional[Any],   UpperCamelCase__			: str,   UpperCamelCase__			: str,   UpperCamelCase__			: List[str] = None,   UpperCamelCase__			: bool = False    )     -> Optional[Any]:
        _A								=     name
        _A								=     level
        _A								=     lines
        _A								=     ''
        _A								=     True
        _A								=     {}
        _A								=     []
        _A								=     []
        if self.lines is not None:
             self.parse(suppress_parsing_errors=SCREAMING_SNAKE_CASE_    )
   def   __UpperCAmelCase      (			self			: Optional[Any],   UpperCamelCase__			: bool = False    )     -> List[Any]:
        _A								=     ''
        _A								=     []
        _A								=     False
        for line in self.lines:
             if line.strip(' \n'    ) == "":
                  continue
             elif line.strip(' \n'    )[:3] == "```":
                  _A								=     not code_start
             elif line.split()[0] == self.level + "#" and not code_start:
                  if current_sub_level != "":
                       _A								=     Section(SCREAMING_SNAKE_CASE_,   self.level + '#',   SCREAMING_SNAKE_CASE_    )
                       _A								=     []
                  else:
                       if current_lines != []:
                            self.text += "".join(SCREAMING_SNAKE_CASE_    ).strip()
                            if self.text != "" and self.text not in FILLER_TEXT:
                                 _A								=     False
                            _A								=     []

                  _A								=     ' '.join(line.split()[1:]    ).strip(' \n'    )
             else:
                  current_lines.append(SCREAMING_SNAKE_CASE_    )
        else:
             if current_sub_level != "":
                  if current_sub_level in self.content:
                       self.parsing_error_list.append(
                           f'Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.'    )
                  _A								=     Section(SCREAMING_SNAKE_CASE_,   self.level + '#',   SCREAMING_SNAKE_CASE_    )
             else:
                  if current_lines != []:
                       self.text += "".join(SCREAMING_SNAKE_CASE_    ).strip()
                       if self.text != "" and self.text not in FILLER_TEXT:
                            _A								=     False

        if self.level == "" and not suppress_parsing_errors:
             if self.parsing_error_list != [] or self.parsing_warning_list != []:
                  _A								=     _A								=     '\n'.join('-\t' + x for x in self.parsing_error_list + self.parsing_warning_list    )
                  _A								=     f'The following issues were found while parsing the README at `{self.name}`:\n' + errors
                  raise ValueError(SCREAMING_SNAKE_CASE_    )
   def   __UpperCAmelCase      (			self			: Dict,   UpperCamelCase__			: dict    )     -> Optional[int]:
        _A								=     []
        _A								=     []
        if structure["allow_empty"] is False:
             # If content is expected
             if self.is_empty_text and self.content == {}:
                  # If no content is found, mention it in the error_list
                  error_list.append(f'Expected some content in section `{self.name}` but it is empty.'    )

        if structure["allow_empty_text"] is False:
             # If some text is expected
             if self.is_empty_text:
                  # If no text is found, mention it in the error_list
                  error_list.append(
                      f'Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).'    )
        # Subsections Validation
        if structure["subsections"] is not None:
             # If subsections are expected
             if self.content == {}:
                  # If no subsections are present
                  _A								=     [subsection['name'] for subsection in structure['subsections']]
                  # Mention the expected values in the error_list
                  error_list.append(
                      f'Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values]    )}. Found \'None\'.'    )
             else:
                  # If some subsections are present
                  _A								=     [subsection['name'] for subsection in structure['subsections']]
                  _A								=     False
                  for idx, name in enumerate(SCREAMING_SNAKE_CASE_    ):
                       if name not in self.content:
                            # If the expected subsection is not present
                            error_list.append(f'Section `{self.name}` is missing subsection: `{name}`.'    )
                            _A								=     True
                       else:
                            # If the subsection is present, validate subsection, return the result
                            # and concat the errors from subsection to section error_list

                            # Skip sublevel validation if current level is `###`
                            if self.level == "###":
                                 continue
                            else:
                                 _A		,					_A		,					_A								=     self.content[name].validate(
                                     structure['subsections'][idx]    )
                            error_list += subsec_error_list
                            warning_list += subsec_warning_list

                  if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
                       for name in self.content:
                            if name not in structure_names:
                                 # If an extra subsection is present
                                 warning_list.append(
                                     f'`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.'    )
        if error_list:
             # If there are errors, do not return the dictionary as it is invalid
             return {}, error_list, warning_list
        else:
             return self.to_dict(), error_list, warning_list

   def   __UpperCAmelCase      (			self			: Optional[Any]    )     -> Tuple:
        return {
            "name": self.name,
            "text": self.text,
            "is_empty_text": self.is_empty_text,
            "subsections": [value.to_dict() for value in self.content.values()],
        }







class lowercase_					(		lowerCamelCase__ ):  # Level 0







   """simple docstring"""

   def __init__(			self			: Optional[int],   UpperCamelCase__			: str,   UpperCamelCase__			: List[str],   UpperCamelCase__			: dict = None,   UpperCamelCase__			: bool = False    )     -> Union[str, Any]:
        super().__init__(name=SCREAMING_SNAKE_CASE_,   level=''    )  # Not using lines here as we need to use a child class parse
        _A								=     structure
        _A								=     -2
        _A								=     0
        _A								=     lines
        if self.lines is not None:
             self.parse(suppress_parsing_errors=SCREAMING_SNAKE_CASE_    )
   def   __UpperCAmelCase      (			self			: Tuple    )     -> Optional[Any]:
        if self.structure is None:
             _A		,					_A		,					_A								=     self._validate(SCREAMING_SNAKE_CASE_    )
        else:
             _A		,					_A		,					_A								=     self._validate(self.structure    )
        if error_list != [] or warning_list != []:
             _A								=     '\n'.join(['-\t' + x for x in error_list + warning_list]    )
             _A								=     f'The following issues were found for the README at `{self.name}`:\n' + errors
             raise ValueError(SCREAMING_SNAKE_CASE_    )
   @classmethod
   def   __UpperCAmelCase      (			cls			: Optional[int],   UpperCamelCase__			: Path,   UpperCamelCase__			: dict = None,   UpperCamelCase__			: bool = False    )     -> int:
        with open(SCREAMING_SNAKE_CASE_,   encoding='utf-8'    ) as f:
             _A								=     f.readlines()
        return cls(SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_,   suppress_parsing_errors=SCREAMING_SNAKE_CASE_    )
   @classmethod
   def   __UpperCAmelCase      (			cls			: Any,   UpperCamelCase__			: str,   UpperCamelCase__			: dict = None,   UpperCamelCase__			: str = "root",   UpperCamelCase__			: bool = False    )     -> List[Any]:
        _A								=     string.split('\n'    )
        return cls(SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_,   suppress_parsing_errors=SCREAMING_SNAKE_CASE_    )
   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: bool = False    )     -> Dict:
        # Skip Tags
        _A								=     0

        for line in self.lines:
             self.yaml_tags_line_count += 1
             if line.strip(' \n'    ) == "---":
                  self.tag_count += 1
                  if self.tag_count == 2:
                       break
             line_count += 1
        if self.tag_count == 2:
             _A								=     self.lines[line_count + 1 :]  # Get the last + 1 th item.
        else:
             _A								=     self.lines[self.tag_count :]
        super().parse(suppress_parsing_errors=SCREAMING_SNAKE_CASE_    )
   def __str__(			self			: Optional[Any]    )     -> Optional[int]:
        return str(self.to_dict()    )

   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: Optional[Any]    )     -> Tuple:
        _A								=     []
        _A								=     []
        if self.yaml_tags_line_count == 0:
             warning_list.append('Empty YAML markers are present in the README.'    )
        elif self.tag_count == 0:
             warning_list.append('No YAML markers are present in the README.'    )
        elif self.tag_count == 1:
             warning_list.append('Only the start of YAML tags present in the README.'    )
        # Check how many first level sections are present.
        _A								=     len(self.content.keys()    )
        if num_first_level_keys > 1:
             # If more than one, add to the error list, continue
             error_list.append(
                 f'The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys()    )]    )}. Only one heading is expected. Skipping further validation for this README.'    )
        elif num_first_level_keys < 1:
             # If less than one, append error.
             error_list.append(
                 'The README has no first-level headings. One heading is expected. Skipping further validation for this README.'    )

        else:
             # If one exactly
             _A								=     list(self.content.keys()    )[0]  # Get the key
             if start_key.startswith('Dataset Card for'    ):  # Check correct start
                  # If the starting is correct, validate all the sections
                  _A		,					_A		,					_A								=     self.content[start_key].validate(
                      readme_structure['subsections'][0]    )
                  error_list += sec_error_list
                  warning_list += sec_warning_list
             else:
                  # If not found, append error
                  error_list.append(
                      'No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'    )
        if error_list:
             # If there are errors, do not return the dictionary as it is invalid
             return {}, error_list, warning_list
        else:
             return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
 from argparse import ArgumentParser

 _UpperCAmelCase						:    List[str]     					=					ArgumentParser(usage='''Validate the content (excluding YAML tags) of a README.md file.''')
 ap.add_argument('''readme_filepath''')
 _UpperCAmelCase						:    Optional[int]     					=					ap.parse_args()
 _UpperCAmelCase						:    Optional[int]     					=					Path(args.readme_filepath)
 _UpperCAmelCase						:    Optional[int]     					=					ReadMe.from_readme(readme_filepath)




'''simple docstring'''






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_UpperCAmelCase						:    Optional[int]     					=					logging.get_logger(__name__)

_UpperCAmelCase						:    Optional[Any]     					=					{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_UpperCAmelCase						:    Optional[Any]     					=					{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

_UpperCAmelCase						:    int     					=					{
    '''openai-gpt''': 5_12,
}





def    _SCREAMING_SNAKE_CASE		(			__snake_case       : Any							):
     _A								=     text.strip()
     if not text:
          return []
     _A								=     text.split()
     return tokens







class lowercase_					(		_UpperCamelCase ):







   """simple docstring"""

   def __init__(			self			: Optional[int],   UpperCamelCase__			: Optional[int]=True,   UpperCamelCase__			: Optional[Any]=None,   UpperCamelCase__			: Optional[int]=True,   UpperCamelCase__			: Dict=None,   UpperCamelCase__			: Any=True,   )     -> str:
        if never_split is None:
             _A								=     []
        _A								=     do_lower_case
        _A								=     set(UpperCamelCase__    )
        _A								=     tokenize_chinese_chars
        _A								=     strip_accents
        _A								=     do_split_on_punc
   def   __UpperCAmelCase      (			self			: Any,   UpperCamelCase__			: List[str],   UpperCamelCase__			: int=None    )     -> Optional[Any]:
        _A								=     self.never_split.union(set(UpperCamelCase__    )    ) if never_split else self.never_split
        _A								=     self._clean_text(UpperCamelCase__    )

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        if self.tokenize_chinese_chars:
             _A								=     self._tokenize_chinese_chars(UpperCamelCase__    )
        # prevents treating the same character with different unicode codepoints as different characters
        _A								=     unicodedata.normalize('NFC',   UpperCamelCase__    )
        _A								=     whitespace_tokenize(UpperCamelCase__    )
        _A								=     []
        for token in orig_tokens:
             if token not in never_split:
                  if self.do_lower_case:
                       _A								=     token.lower()
                       if self.strip_accents is not False:
                            _A								=     self._run_strip_accents(UpperCamelCase__    )
                  elif self.strip_accents:
                       _A								=     self._run_strip_accents(UpperCamelCase__    )
             split_tokens.extend(self._run_split_on_punc(UpperCamelCase__,   UpperCamelCase__    )    )

        _A								=     whitespace_tokenize(' '.join(UpperCamelCase__    )    )
        return output_tokens
   def   __UpperCAmelCase      (			self			: List[Any],   UpperCamelCase__			: Dict    )     -> Dict:
        _A								=     unicodedata.normalize('NFD',   UpperCamelCase__    )
        _A								=     []
        for char in text:
             _A								=     unicodedata.category(UpperCamelCase__    )
             if cat == "Mn":
                  continue
             output.append(UpperCamelCase__    )
        return "".join(UpperCamelCase__    )
   def   __UpperCAmelCase      (			self			: Union[str, Any],   UpperCamelCase__			: Tuple,   UpperCamelCase__			: Any=None    )     -> Optional[Any]:
        if not self.do_split_on_punc or (never_split is not None and text in never_split):
             return [text]
        _A								=     list(UpperCamelCase__    )
        _A								=     0
        _A								=     True
        _A								=     []
        while i < len(UpperCamelCase__    ):
             _A								=     chars[i]
             if _is_punctuation(UpperCamelCase__    ):
                  output.append([char]    )
                  _A								=     True
             else:
                  if start_new_word:
                       output.append([]    )
                  _A								=     False
                  output[-1].append(UpperCamelCase__    )
             i += 1

        return ["".join(UpperCamelCase__    ) for x in output]
   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: str    )     -> int:
        _A								=     []
        for char in text:
             _A								=     ord(UpperCamelCase__    )
             if self._is_chinese_char(UpperCamelCase__    ):
                  output.append(' '    )
                  output.append(UpperCamelCase__    )
                  output.append(' '    )
             else:
                  output.append(UpperCamelCase__    )
        return "".join(UpperCamelCase__    )
   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: List[Any]    )     -> Union[str, Any]:
        # This defines a "chinese character" as anything in the CJK Unicode block:
        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
        #
        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
        # despite its name. The modern Korean Hangul alphabet is a different block,
        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
        # space-separated words, so they are not treated specially and handled
        # like the all of the other languages.
        if (
            (cp >= 0x4e_00 and cp <= 0x9f_ff)
            or (cp >= 0x34_00 and cp <= 0x4d_bf)  #
            or (cp >= 0x2_00_00 and cp <= 0x2_a6_df)  #
            or (cp >= 0x2_a7_00 and cp <= 0x2_b7_3f)  #
            or (cp >= 0x2_b7_40 and cp <= 0x2_b8_1f)  #
            or (cp >= 0x2_b8_20 and cp <= 0x2_ce_af)  #
            or (cp >= 0xf9_00 and cp <= 0xfa_ff)
            or (cp >= 0x2_f8_00 and cp <= 0x2_fa_1f)  #
        ):  #
             return True

        return False

   def   __UpperCAmelCase      (			self			: Optional[int],   UpperCamelCase__			: Union[str, Any]    )     -> str:
        _A								=     []
        for char in text:
             _A								=     ord(UpperCamelCase__    )
             if cp == 0 or cp == 0xff_fd or _is_control(UpperCamelCase__    ):
                  continue
             if _is_whitespace(UpperCamelCase__    ):
                  output.append(' '    )
             else:
                  output.append(UpperCamelCase__    )
        return "".join(UpperCamelCase__    )






def    _SCREAMING_SNAKE_CASE		(			__snake_case       : Optional[int]							):
     _A								=     set()
     _A								=     word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)							)
          _A								=     char
     return pairs





def    _SCREAMING_SNAKE_CASE		(			__snake_case       : Any							):
     _A								=     text.replace('—'      ,		'-'							)
     _A								=     text.replace('–'      ,		'-'							)
     _A								=     text.replace('―'      ,		'-'							)
     _A								=     text.replace('…'      ,		'...'							)
     _A								=     text.replace('´'      ,		'\''							)
     _A								=     re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'      ,		R' \1 '      ,		__snake_case							)
     _A								=     re.sub(R'\s*\n\s*'      ,		' \n '      ,		__snake_case							)
     _A								=     re.sub(R'[^\S\n]+'      ,		' '      ,		__snake_case							)
     return text.strip()







class lowercase_					(		_UpperCamelCase ):







   """simple docstring"""

   __lowerCAmelCase    						=      VOCAB_FILES_NAMES
   __lowerCAmelCase    						=      PRETRAINED_VOCAB_FILES_MAP
   __lowerCAmelCase    						=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __lowerCAmelCase    						=      ["input_ids", "attention_mask"]
   def __init__(			self			: Tuple,   UpperCamelCase__			: Tuple,   UpperCamelCase__			: List[str],   UpperCamelCase__			: Any="<unk>",   **UpperCamelCase__			: List[str]    )     -> Dict:
        super().__init__(unk_token=UpperCamelCase__,   **UpperCamelCase__    )

        try:
             import ftfy
             from spacy.lang.en import English

             _A								=     English()
             _A								=     _nlp.tokenizer
             _A								=     ftfy.fix_text
        except ImportError:
             logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'    )
             _A								=     BasicTokenizer(do_lower_case=UpperCamelCase__    )
             _A								=     None

        with open(UpperCamelCase__,   encoding='utf-8'    ) as vocab_handle:
             _A								=     json.load(UpperCamelCase__    )
        _A								=     {v: k for k, v in self.encoder.items()}
        with open(UpperCamelCase__,   encoding='utf-8'    ) as merges_handle:
             _A								=     merges_handle.read().split('\n'    )[1:-1]
        _A								=     [tuple(merge.split()    ) for merge in merges]
        _A								=     dict(zip(UpperCamelCase__,   range(len(UpperCamelCase__    )    )    )    )
        _A								=     {}
   @property
   def   __UpperCAmelCase      (			self			: Optional[int]    )     -> Tuple:
        return True
   @property
   def   __UpperCAmelCase      (			self			: str    )     -> List[str]:
        return len(self.encoder    )
   def   __UpperCAmelCase      (			self			: Tuple    )     -> Dict:
        return dict(self.encoder,   **self.added_tokens_encoder    )
   def   __UpperCAmelCase      (			self			: List[str],   UpperCamelCase__			: Dict    )     -> List[Any]:
        _A								=     tuple(token[:-1]    ) + (token[-1] + '</w>',)
        if token in self.cache:
             return self.cache[token]
        _A								=     get_pairs(UpperCamelCase__    )

        if not pairs:
             return token + "</w>"

        while True:
             _A								=     min(UpperCamelCase__,   key=lambda UpperCamelCase__    : self.bpe_ranks.get(UpperCamelCase__,   float('inf'    )    )    )
             if bigram not in self.bpe_ranks:
                  break
             _A		,					_A								=     bigram
             _A								=     []
             _A								=     0
             while i < len(UpperCamelCase__    ):
                  try:
                       _A								=     word.index(UpperCamelCase__,   UpperCamelCase__    )
                  except ValueError:
                       new_word.extend(word[i:]    )
                       break
                  else:
                       new_word.extend(word[i:j]    )
                       _A								=     j

                  if word[i] == first and i < len(UpperCamelCase__    ) - 1 and word[i + 1] == second:
                       new_word.append(first + second    )
                       i += 2
                  else:
                       new_word.append(word[i]    )
                       i += 1
             _A								=     tuple(UpperCamelCase__    )
             _A								=     new_word
             if len(UpperCamelCase__    ) == 1:
                  break
             else:
                  _A								=     get_pairs(UpperCamelCase__    )
        _A								=     ' '.join(UpperCamelCase__    )
        if word == "\n  </w>":
             _A								=     '\n</w>'
        _A								=     word
        return word
   def   __UpperCAmelCase      (			self			: Dict,   UpperCamelCase__			: Any    )     -> Union[str, Any]:
        _A								=     []
        if self.fix_text is None:
             # Using BERT's BasicTokenizer
             _A								=     self.nlp.tokenize(UpperCamelCase__    )
             for token in text:
                  split_tokens.extend(list(self.bpe(UpperCamelCase__    ).split(' '    )    )    )
        else:
             # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
             _A								=     self.nlp(text_standardize(self.fix_text(UpperCamelCase__    )    )    )
             for token in text:
                  split_tokens.extend(list(self.bpe(token.text.lower()    ).split(' '    )    )    )
        return split_tokens
   def   __UpperCAmelCase      (			self			: Any,   UpperCamelCase__			: List[Any]    )     -> List[Any]:
        return self.encoder.get(UpperCamelCase__,   self.encoder.get(self.unk_token    )    )
   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: List[str]    )     -> Tuple:
        return self.decoder.get(UpperCamelCase__,   self.unk_token    )
   def   __UpperCAmelCase      (			self			: Any,   UpperCamelCase__			: List[str]    )     -> Any:
        _A								=     ''.join(UpperCamelCase__    ).replace('</w>',   ' '    ).strip()
        return out_string

   def   __UpperCAmelCase      (			self			: Dict,   UpperCamelCase__			: str,   UpperCamelCase__			: Optional[str] = None    )     -> Tuple[str]:
        if not os.path.isdir(UpperCamelCase__    ):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory'    )
             return
        _A								=     os.path.join(
            UpperCamelCase__,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
        _A								=     os.path.join(
            UpperCamelCase__,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']    )

        with open(UpperCamelCase__,   'w',   encoding='utf-8'    ) as f:
             f.write(json.dumps(self.encoder,   indent=2,   sort_keys=UpperCamelCase__,   ensure_ascii=UpperCamelCase__    ) + '\n'    )

        _A								=     0
        with open(UpperCamelCase__,   'w',   encoding='utf-8'    ) as writer:
             writer.write('#version: 0.2\n'    )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),   key=lambda UpperCamelCase__    : kv[1]    ):
                  if index != token_index:
                       logger.warning(
                           f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                           ' Please check that the tokenizer is not corrupted!'    )
                       _A								=     token_index
                  writer.write(' '.join(UpperCamelCase__    ) + '\n'    )
                  index += 1

        return vocab_file, merge_file



