
"""simple docstring"""






import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
				if is_torch_available():
								import torch
				if is_tf_available():
								import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__UpperCAmelCase 					=    logging.get_logger(__name__)

__UpperCAmelCase 					=    {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__UpperCAmelCase 					=    {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

__UpperCAmelCase 					=    {
    'Salesforce/codegen-350M-mono': 20_48,
}






@lru_cache()
def 		_snake_case			(      )    ->							str:


			'''simple docstring'''





			lowerCAmelCase_       :Any     	=    (
			    list(range(ord("""!"""						) ,	ord("""~"""						) + 1						)						) + list(range(ord("""¡"""						) ,	ord("""¬"""						) + 1						)						) + list(range(ord("""®"""						) ,	ord("""ÿ"""						) + 1						)						)
			)
			lowerCAmelCase_       :Optional[int]     	=    bs[:]
			lowerCAmelCase_       :int     	=    0
			for b in range(2**8						):
						if b not in bs:
									bs.append(lowercase__						)
									cs.append(2**8 + n						)
									n += 1
			lowerCAmelCase_       :Any     	=    [chr(lowercase__						) for n in cs]
			return dict(zip(lowercase__ ,	lowercase__						)						)






def 		_snake_case			(      lowercase__						:    Dict						)    ->							List[str]:


			'''simple docstring'''





			lowerCAmelCase_       :List[str]     	=    set()
			lowerCAmelCase_       :int     	=    word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)						)
						lowerCAmelCase_       :List[Any]     	=    char
			return pairs




class      _SCREAMING_SNAKE_CASE   (     A__				):
			UpperCAmelCase_    :str												=		VOCAB_FILES_NAMES
			UpperCAmelCase_    :Optional[int]												=		PRETRAINED_VOCAB_FILES_MAP
			UpperCAmelCase_    :int												=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCAmelCase_    :Optional[Any]												=		["input_ids", "attention_mask"]






			def __init__(   self			,  __A			,  __A			,  __A="replace"			,  __A="<|endoftext|>"			,  __A="<|endoftext|>"			,  __A="<|endoftext|>"			,  __A=None			,  __A=False			,  __A=False			,  **__A			,  )     ->	str:
						lowerCAmelCase_       :Any     	=    AddedToken(__A			,  lstrip=__A			,  rstrip=__A			) if isinstance(__A			,  __A			) else bos_token
						lowerCAmelCase_       :List[Any]     	=    AddedToken(__A			,  lstrip=__A			,  rstrip=__A			) if isinstance(__A			,  __A			) else eos_token
						lowerCAmelCase_       :Any     	=    AddedToken(__A			,  lstrip=__A			,  rstrip=__A			) if isinstance(__A			,  __A			) else unk_token
						lowerCAmelCase_       :str     	=    AddedToken(__A			,  lstrip=__A			,  rstrip=__A			) if isinstance(__A			,  __A			) else pad_token
						super().__init__(
						    errors=__A			,  unk_token=__A			,  bos_token=__A			,  eos_token=__A			,  pad_token=__A			,  add_prefix_space=__A			,  add_bos_token=__A			,  **__A			,  )
						lowerCAmelCase_       :Union[str, Any]     	=    add_bos_token

						with open(__A			,  encoding="""utf-8"""			) as vocab_handle:
									lowerCAmelCase_       :int     	=    json.load(__A			)
						lowerCAmelCase_       :Optional[int]     	=    {v: k for k, v in self.encoder.items()}
						lowerCAmelCase_       :Optional[Any]     	=    errors  # how to handle errors in decoding
						lowerCAmelCase_       :int     	=    bytes_to_unicode()
						lowerCAmelCase_       :str     	=    {v: k for k, v in self.byte_encoder.items()}
						with open(__A			,  encoding="""utf-8"""			) as merges_handle:
									lowerCAmelCase_       :Optional[Any]     	=    merges_handle.read().split("""\n"""			)[1:-1]
						lowerCAmelCase_       :str     	=    [tuple(merge.split()			) for merge in bpe_merges]
						lowerCAmelCase_       :int     	=    dict(zip(__A			,  range(len(__A			)			)			)			)
						lowerCAmelCase_       :Dict     	=    {}
						lowerCAmelCase_       :List[str]     	=    add_prefix_space

						# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
						lowerCAmelCase_       :Any     	=    re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""			)






			@property
			def 					__lowerCAmelCase					(   self			)     ->	Any:
						return len(self.encoder			)






			def 					__lowerCAmelCase					(   self			)     ->	Dict:
						return dict(self.encoder			,  **self.added_tokens_encoder			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Any:
						if token in self.cache:
									return self.cache[token]
						lowerCAmelCase_       :List[str]     	=    tuple(__A			)
						lowerCAmelCase_       :Dict     	=    get_pairs(__A			)

						if not pairs:
									return token

						while True:
									lowerCAmelCase_       :str     	=    min(__A			,  key=lambda __A			: self.bpe_ranks.get(__A			,  float("""inf"""			)			)			)
									if bigram not in self.bpe_ranks:
												break
									lowerCAmelCase_	,			lowerCAmelCase_       :int     	=    bigram
									lowerCAmelCase_       :Union[str, Any]     	=    []
									lowerCAmelCase_       :List[str]     	=    0
									while i < len(__A			):
												try:
															lowerCAmelCase_       :List[Any]     	=    word.index(__A			,  __A			)
												except ValueError:
															new_word.extend(word[i:]			)
															break
												else:
															new_word.extend(word[i:j]			)
															lowerCAmelCase_       :Optional[int]     	=    j

												if word[i] == first and i < len(__A			) - 1 and word[i + 1] == second:
															new_word.append(first + second			)
															i += 2
												else:
															new_word.append(word[i]			)
															i += 1
									lowerCAmelCase_       :Tuple     	=    tuple(__A			)
									lowerCAmelCase_       :Union[str, Any]     	=    new_word
									if len(__A			) == 1:
												break
									else:
												lowerCAmelCase_       :List[Any]     	=    get_pairs(__A			)
						lowerCAmelCase_       :str     	=    """ """.join(__A			)
						lowerCAmelCase_       :int     	=    word
						return word






			def 					__lowerCAmelCase					(   self			,  __A			,  __A=None			)     ->	Tuple:
						if self.add_bos_token:
									lowerCAmelCase_       :List[Any]     	=    [self.bos_token_id]
						else:
									lowerCAmelCase_       :Optional[Any]     	=    []

						lowerCAmelCase_       :List[str]     	=    bos_token_ids + token_ids_a

						if token_ids_a is None:
									return output

						return output + bos_token_ids + token_ids_a






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Tuple:
						lowerCAmelCase_       :int     	=    []
						for token in re.findall(self.pat			,  __A			):
									lowerCAmelCase_       :List[Any]     	=    """""".join(
									    self.byte_encoder[b] for b in token.encode("""utf-8"""			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
									bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__A			).split(""" """			)			)
						return bpe_tokens






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	List[str]:
						return self.encoder.get(__A			,  self.encoder.get(self.unk_token			)			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Dict:
						return self.decoder.get(__A			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Union[str, Any]:
						lowerCAmelCase_       :Optional[Any]     	=    """""".join(__A			)
						lowerCAmelCase_       :str     	=    bytearray([self.byte_decoder[c] for c in text]			).decode("""utf-8"""			,  errors=self.errors			)
						return text






			def 					__lowerCAmelCase					(   self			,  __A			,  __A = None			)     ->	Tuple[str]:
						if not os.path.isdir(__A			):
									logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""			)
									return
						lowerCAmelCase_       :str     	=    os.path.join(
						    __A			,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)
						lowerCAmelCase_       :Dict     	=    os.path.join(
						    __A			,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]			)

						with open(__A			,  """w"""			,  encoding="""utf-8"""			) as f:
									f.write(json.dumps(self.encoder			,  indent=2			,  sort_keys=__A			,  ensure_ascii=__A			) + """\n"""			)

						lowerCAmelCase_       :str     	=    0
						with open(__A			,  """w"""			,  encoding="""utf-8"""			) as writer:
									writer.write("""#version: 0.2\n"""			)
									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,  key=lambda __A			: kv[1]			):
												if index != token_index:
															logger.warning(
															    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
															    """ Please check that the tokenizer is not corrupted!"""			)
															lowerCAmelCase_       :List[str]     	=    token_index
												writer.write(""" """.join(__A			) + """\n"""			)
												index += 1

						return vocab_file, merge_file






			def 					__lowerCAmelCase					(   self			,  __A			,  __A=False			,  **__A			)     ->	Dict:
						lowerCAmelCase_       :str     	=    kwargs.pop("""add_prefix_space"""			,  self.add_prefix_space			)
						if is_split_into_words or add_prefix_space:
									lowerCAmelCase_       :Union[str, Any]     	=    """ """ + text
						return (text, kwargs)






			def 					__lowerCAmelCase					(   self			,  __A			,  __A = False			,  __A = None			,  __A = None			,  **__A			,  )     ->	str:
						lowerCAmelCase_       :Any     	=    super()._decode(
						    token_ids=__A			,  skip_special_tokens=__A			,  clean_up_tokenization_spaces=__A			,  **__A			,  )

						if truncate_before_pattern is not None and len(__A			) > 0:
									lowerCAmelCase_       :Dict     	=    self.truncate(__A			,  __A			)

						return decoded_text






			def 					__lowerCAmelCase					(   self			,  __A			,  __A			)     ->	Any:
						def find_re(__A			,  __A			,  __A			):
									lowerCAmelCase_       :Union[str, Any]     	=    pattern.search(__A			,  __A			)
									return m.start() if m else -1

						lowerCAmelCase_       :List[str]     	=    [re.compile(__A			,  re.MULTILINE			) for pattern in truncate_before_pattern]

						lowerCAmelCase_       :Tuple     	=    list(re.finditer("""^print"""			,  __A			,  re.MULTILINE			)			)

						if len(__A			) > 1:
									lowerCAmelCase_       :List[str]     	=    completion[: prints[1].start()]

						lowerCAmelCase_       :Any     	=    list(re.finditer("""^def"""			,  __A			,  re.MULTILINE			)			)

						if len(__A			) > 1:
									lowerCAmelCase_       :Optional[Any]     	=    completion[: defs[1].start()]

						lowerCAmelCase_       :Optional[Any]     	=    0

						lowerCAmelCase_       :int     	=    [
						    pos for pos in [find_re(__A			,  __A			,  __A			) for terminal in terminals] if pos != -1
						]

						if len(__A			) > 0:
									return completion[: min(__A			)]
						else:
									return completion








"""simple docstring"""






from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy




class      _SCREAMING_SNAKE_CASE   (     A__				):
			UpperCAmelCase_    :Union[str, Any]												=		"MarkupLMFeatureExtractor"
			UpperCAmelCase_    :Optional[Any]												=		("MarkupLMTokenizer", "MarkupLMTokenizerFast")
			UpperCAmelCase_    :List[Any]												=		True






			def __call__(   self			,  __A=None			,  __A=None			,  __A=None			,  __A=None			,  __A=None			,  __A = True			,  __A = False			,  __A = None			,  __A = None			,  __A = 0			,  __A = None			,  __A = None			,  __A = None			,  __A = False			,  __A = False			,  __A = False			,  __A = False			,  __A = True			,  __A = None			,  **__A			,  )     ->	BatchEncoding:
						# first, create nodes and xpaths
						if self.parse_html:
									if html_strings is None:
												raise ValueError("""Make sure to pass HTML strings in case `parse_html` is set to `True`"""			)

									if nodes is not None or xpaths is not None or node_labels is not None:
												raise ValueError(
												    """Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"""			)

									lowerCAmelCase_       :Union[str, Any]     	=    self.feature_extractor(__A			)
									lowerCAmelCase_       :Any     	=    features["""nodes"""]
									lowerCAmelCase_       :Union[str, Any]     	=    features["""xpaths"""]
						else:
									if html_strings is not None:
												raise ValueError("""You have passed HTML strings but `parse_html` is set to `False`."""			)
									if nodes is None or xpaths is None:
												raise ValueError("""Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"""			)

        # # second, apply the tokenizer
						if questions is not None and self.parse_html:
									if isinstance(__A			,  __A			):
												lowerCAmelCase_       :Dict     	=    [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

						lowerCAmelCase_       :Optional[Any]     	=    self.tokenizer(
						    text=questions if questions is not None else nodes			,  text_pair=nodes if questions is not None else None			,  xpaths=__A			,  node_labels=__A			,  add_special_tokens=__A			,  padding=__A			,  truncation=__A			,  max_length=__A			,  stride=__A			,  pad_to_multiple_of=__A			,  return_token_type_ids=__A			,  return_attention_mask=__A			,  return_overflowing_tokens=__A			,  return_special_tokens_mask=__A			,  return_offsets_mapping=__A			,  return_length=__A			,  verbose=__A			,  return_tensors=__A			,  **__A			,  )

						return encoded_inputs






			def 					__lowerCAmelCase					(   self			,  *__A			,  **__A			)     ->	Tuple:
						return self.tokenizer.batch_decode(*__A			,  **__A			)






			def 					__lowerCAmelCase					(   self			,  *__A			,  **__A			)     ->	str:
						return self.tokenizer.decode(*__A			,  **__A			)






			@property
			def 					__lowerCAmelCase					(   self			)     ->	Any:
						lowerCAmelCase_       :Any     	=    self.tokenizer.model_input_names
						return tokenizer_input_names






