






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowerCamelCase_		  =						logging.get_logger(__name__)

lowerCamelCase_		  =						{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowerCamelCase_		  =						{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

lowerCamelCase_		  =						{
    'openai-gpt': 5_1_2,
}
def   lowerCamelCase      (		a_			)			->    Union[str, Any]:
							lowerCAmelCase_       		=  text.strip()
							if not text:
														return []
							lowerCAmelCase_       		=  text.split()
							return tokens




class 							a_				(		A__	):


							'''simple docstring'''






							def __init__(	self    ,    lowercase_=True    ,    lowercase_=None    ,    lowercase_=True    ,    lowercase_=None    ,    lowercase_=True    ,    )      ->      int:
														'''simple docstring'''

														if never_split is None:
																					lowerCAmelCase_       		=  []
														lowerCAmelCase_       		=  do_lower_case
														lowerCAmelCase_       		=  set(_lowerCAmelCase     )
														lowerCAmelCase_       		=  tokenize_chinese_chars
														lowerCAmelCase_       		=  strip_accents
														lowerCAmelCase_       		=  do_split_on_punc

							def   _lowercase (	self    ,    lowercase_    ,    lowercase_=None     )      ->      Tuple:
														'''simple docstring'''

														lowerCAmelCase_       		=  self.never_split.union(set(_lowerCAmelCase     )     ) if never_split else self.never_split
														lowerCAmelCase_       		=  self._clean_text(_lowerCAmelCase     )

														# This was added on November 1st, 2018 for the multilingual and Chinese
														# models. This is also applied to the English models now, but it doesn't
														# matter since the English models were not trained on any Chinese data
														# and generally don't have any Chinese data in them (there are Chinese
														# characters in the vocabulary because Wikipedia does have some Chinese
														# words in the English Wikipedia.).
														if self.tokenize_chinese_chars:
																					lowerCAmelCase_       		=  self._tokenize_chinese_chars(_lowerCAmelCase     )
														# prevents treating the same character with different unicode codepoints as different characters
														lowerCAmelCase_       		=  unicodedata.normalize('NFC'    ,    _lowerCAmelCase     )
														lowerCAmelCase_       		=  whitespace_tokenize(_lowerCAmelCase     )
														lowerCAmelCase_       		=  []
														for token in orig_tokens:
																					if token not in never_split:
																												if self.do_lower_case:
																																			lowerCAmelCase_       		=  token.lower()
																																			if self.strip_accents is not False:
																																										lowerCAmelCase_       		=  self._run_strip_accents(_lowerCAmelCase     )
																												elif self.strip_accents:
																																			lowerCAmelCase_       		=  self._run_strip_accents(_lowerCAmelCase     )
																					split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase    ,    _lowerCAmelCase     )     )

														lowerCAmelCase_       		=  whitespace_tokenize(' '.join(_lowerCAmelCase     )     )
														return output_tokens

							def   _lowercase (	self    ,    lowercase_     )      ->      Any:
														'''simple docstring'''

														lowerCAmelCase_       		=  unicodedata.normalize('NFD'    ,    _lowerCAmelCase     )
														lowerCAmelCase_       		=  []
														for char in text:
																					lowerCAmelCase_       		=  unicodedata.category(_lowerCAmelCase     )
																					if cat == "Mn":
																												continue
																					output.append(_lowerCAmelCase     )
														return "".join(_lowerCAmelCase     )

							def   _lowercase (	self    ,    lowercase_    ,    lowercase_=None     )      ->      str:
														'''simple docstring'''

														if not self.do_split_on_punc or (never_split is not None and text in never_split):
																					return [text]
														lowerCAmelCase_       		=  list(_lowerCAmelCase     )
														lowerCAmelCase_       		=  0
														lowerCAmelCase_       		=  True
														lowerCAmelCase_       		=  []
														while i < len(_lowerCAmelCase     ):
																					lowerCAmelCase_       		=  chars[i]
																					if _is_punctuation(_lowerCAmelCase     ):
																												output.append([char]     )
																												lowerCAmelCase_       		=  True
																					else:
																												if start_new_word:
																																			output.append([]     )
																												lowerCAmelCase_       		=  False
																												output[-1].append(_lowerCAmelCase     )
																					i += 1

														return ["".join(_lowerCAmelCase     ) for x in output]

							def   _lowercase (	self    ,    lowercase_     )      ->      str:
														'''simple docstring'''

														lowerCAmelCase_       		=  []
														for char in text:
																					lowerCAmelCase_       		=  ord(_lowerCAmelCase     )
																					if self._is_chinese_char(_lowerCAmelCase     ):
																												output.append(' '     )
																												output.append(_lowerCAmelCase     )
																												output.append(' '     )
																					else:
																												output.append(_lowerCAmelCase     )
														return "".join(_lowerCAmelCase     )

							def   _lowercase (	self    ,    lowercase_     )      ->      Optional[Any]:
														'''simple docstring'''

														if (
														    (cp >= 0X4E00 and cp <= 0X9FFF)
														    or (cp >= 0X3400 and cp <= 0X4DBF)  #
														    or (cp >= 0X20000 and cp <= 0X2A6DF)  #
														    or (cp >= 0X2A700 and cp <= 0X2B73F)  #
														    or (cp >= 0X2B740 and cp <= 0X2B81F)  #
														    or (cp >= 0X2B820 and cp <= 0X2CEAF)  #
														    or (cp >= 0XF900 and cp <= 0XFAFF)
														    or (cp >= 0X2F800 and cp <= 0X2FA1F)  #
														):  #
																					return True

														return False






							def   _lowercase (	self    ,    lowercase_     )      ->      List[Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  []
														for char in text:
																					lowerCAmelCase_       		=  ord(_lowerCAmelCase     )
																					if cp == 0 or cp == 0XFFFD or _is_control(_lowerCAmelCase     ):
																												continue
																					if _is_whitespace(_lowerCAmelCase     ):
																												output.append(' '     )
																					else:
																												output.append(_lowerCAmelCase     )
														return "".join(_lowerCAmelCase     )


def   lowerCamelCase      (		a_			)			->    str:
							lowerCAmelCase_       		=  set()
							lowerCAmelCase_       		=  word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)			)
														lowerCAmelCase_       		=  char
							return pairs
def   lowerCamelCase      (		a_			)			->    List[str]:
							lowerCAmelCase_       		=  text.replace('—'				,		'-'			)
							lowerCAmelCase_       		=  text.replace('–'				,		'-'			)
							lowerCAmelCase_       		=  text.replace('―'				,		'-'			)
							lowerCAmelCase_       		=  text.replace('…'				,		'...'			)
							lowerCAmelCase_       		=  text.replace('´'				,		'\''			)
							lowerCAmelCase_       		=  re.sub(R'(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'				,		R' \1 '				,		a_			)
							lowerCAmelCase_       		=  re.sub(R'\s*\n\s*'				,		' \n '				,		a_			)
							lowerCAmelCase_       		=  re.sub(R'[^\S\n]+'				,		' '				,		a_			)
							return text.strip()




class 							a_				(		A__	):


							'''simple docstring'''






							__a:			int									=					VOCAB_FILES_NAMES
							__a:			List[str]									=					PRETRAINED_VOCAB_FILES_MAP
							__a:			List[Any]									=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a:			int									=					['''input_ids''', '''attention_mask''']

							def __init__(	self    ,    lowercase_    ,    lowercase_    ,    lowercase_="<unk>"    ,    **lowercase_     )      ->      Union[str, Any]:
														'''simple docstring'''

														super().__init__(unk_token=_lowerCAmelCase    ,    **_lowerCAmelCase     )

														try:
																					import ftfy
																					from spacy.lang.en import English

																					lowerCAmelCase_       		=  English()
																					lowerCAmelCase_       		=  _nlp.tokenizer
																					lowerCAmelCase_       		=  ftfy.fix_text
														except ImportError:
																					logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'     )
																					lowerCAmelCase_       		=  BasicTokenizer(do_lower_case=_lowerCAmelCase     )
																					lowerCAmelCase_       		=  None

														with open(_lowerCAmelCase    ,    encoding='utf-8'     ) as vocab_handle:
																					lowerCAmelCase_       		=  json.load(_lowerCAmelCase     )
														lowerCAmelCase_       		=  {v: k for k, v in self.encoder.items()}
														with open(_lowerCAmelCase    ,    encoding='utf-8'     ) as merges_handle:
																					lowerCAmelCase_       		=  merges_handle.read().split('\n'     )[1:-1]
														lowerCAmelCase_       		=  [tuple(merge.split()     ) for merge in merges]
														lowerCAmelCase_       		=  dict(zip(_lowerCAmelCase    ,    range(len(_lowerCAmelCase     )     )     )     )
														lowerCAmelCase_       		=  {}

							@property
							def   _lowercase (	self     )      ->      int:
														'''simple docstring'''

														return True

							@property
							def   _lowercase (	self     )      ->      List[str]:
														'''simple docstring'''

														return len(self.encoder     )

							def   _lowercase (	self     )      ->      List[Any]:
														'''simple docstring'''

														return dict(self.encoder    ,    **self.added_tokens_encoder     )

							def   _lowercase (	self    ,    lowercase_     )      ->      int:
														'''simple docstring'''

														lowerCAmelCase_       		=  tuple(token[:-1]     ) + (token[-1] + '</w>',)
														if token in self.cache:
																					return self.cache[token]
														lowerCAmelCase_       		=  get_pairs(_lowerCAmelCase     )

														if not pairs:
																					return token + "</w>"

														while True:
																					lowerCAmelCase_       		=  min(_lowerCAmelCase    ,    key=lambda lowercase_     : self.bpe_ranks.get(_lowerCAmelCase    ,    float('inf'     )     )     )
																					if bigram not in self.bpe_ranks:
																												break
																					lowerCAmelCase_		,     lowerCAmelCase_       		=  bigram
																					lowerCAmelCase_       		=  []
																					lowerCAmelCase_       		=  0
																					while i < len(_lowerCAmelCase     ):
																												try:
																																			lowerCAmelCase_       		=  word.index(_lowerCAmelCase    ,    _lowerCAmelCase     )
																												except ValueError:
																																			new_word.extend(word[i:]     )
																																			break
																												else:
																																			new_word.extend(word[i:j]     )
																																			lowerCAmelCase_       		=  j

																												if word[i] == first and i < len(_lowerCAmelCase     ) - 1 and word[i + 1] == second:
																																			new_word.append(first + second     )
																																			i += 2
																												else:
																																			new_word.append(word[i]     )
																																			i += 1
																					lowerCAmelCase_       		=  tuple(_lowerCAmelCase     )
																					lowerCAmelCase_       		=  new_word
																					if len(_lowerCAmelCase     ) == 1:
																												break
																					else:
																												lowerCAmelCase_       		=  get_pairs(_lowerCAmelCase     )
														lowerCAmelCase_       		=  ' '.join(_lowerCAmelCase     )
														if word == "\n  </w>":
																					lowerCAmelCase_       		=  '\n</w>'
														lowerCAmelCase_       		=  word
														return word

							def   _lowercase (	self    ,    lowercase_     )      ->      List[Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  []
														if self.fix_text is None:
																					# Using BERT's BasicTokenizer
																					lowerCAmelCase_       		=  self.nlp.tokenize(_lowerCAmelCase     )
																					for token in text:
																												split_tokens.extend(list(self.bpe(_lowerCAmelCase     ).split(' '     )     )     )
														else:
																					# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
																					lowerCAmelCase_       		=  self.nlp(text_standardize(self.fix_text(_lowerCAmelCase     )     )     )
																					for token in text:
																												split_tokens.extend(list(self.bpe(token.text.lower()     ).split(' '     )     )     )
														return split_tokens

							def   _lowercase (	self    ,    lowercase_     )      ->      List[str]:
														'''simple docstring'''

														return self.encoder.get(_lowerCAmelCase    ,    self.encoder.get(self.unk_token     )     )

							def   _lowercase (	self    ,    lowercase_     )      ->      int:
														'''simple docstring'''

														return self.decoder.get(_lowerCAmelCase    ,    self.unk_token     )

							def   _lowercase (	self    ,    lowercase_     )      ->      Union[str, Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  ''.join(_lowerCAmelCase     ).replace('</w>'    ,    ' '     ).strip()
														return out_string






							def   _lowercase (	self    ,    lowercase_    ,    lowercase_ = None     )      ->      Optional[Any]:
														'''simple docstring'''

														if not os.path.isdir(_lowerCAmelCase     ):
																					logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
																					return
														lowerCAmelCase_       		=  os.path.join(
														    _lowerCAmelCase    ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )
														lowerCAmelCase_       		=  os.path.join(
														    _lowerCAmelCase    ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']     )

														with open(_lowerCAmelCase    ,    'w'    ,    encoding='utf-8'     ) as f:
																					f.write(json.dumps(self.encoder    ,    indent=2    ,    sort_keys=_lowerCAmelCase    ,    ensure_ascii=_lowerCAmelCase     ) + '\n'     )

														lowerCAmelCase_       		=  0
														with open(_lowerCAmelCase    ,    'w'    ,    encoding='utf-8'     ) as writer:
																					writer.write('#version: 0.2\n'     )
																					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,    key=lambda lowercase_     : kv[1]     ):
																												if index != token_index:
																																			logger.warning(
																																			    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																																			    ' Please check that the tokenizer is not corrupted!'     )
																																			lowerCAmelCase_       		=  token_index
																												writer.write(' '.join(_lowerCAmelCase     ) + '\n'     )
																												index += 1

														return vocab_file, merge_file














import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class 							a_				(		a_	):


							'''simple docstring'''






							__a:			int									=					['''image_processor''', '''tokenizer''']
							__a:			int									=					'''VideoMAEImageProcessor'''
							__a:			Dict									=					('''CLIPTokenizer''', '''CLIPTokenizerFast''')

							def __init__(	self    ,    lowercase_=None    ,    lowercase_=None    ,    **lowercase_     )      ->      Optional[Any]:
														'''simple docstring'''

														lowerCAmelCase_       		=  None
														if "feature_extractor" in kwargs:
																					warnings.warn(
																					    'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
																					    ' instead.'    ,    lowercase_    ,    )
																					lowerCAmelCase_       		=  kwargs.pop('feature_extractor'     )

														lowerCAmelCase_       		=  image_processor if image_processor is not None else feature_extractor
														if image_processor is None:
																					raise ValueError('You need to specify an `image_processor`.'     )
														if tokenizer is None:
																					raise ValueError('You need to specify a `tokenizer`.'     )

														super().__init__(lowercase_    ,    lowercase_     )
														lowerCAmelCase_       		=  self.image_processor

							def __call__(	self    ,    lowercase_=None    ,    lowercase_=None    ,    lowercase_=None    ,    **lowercase_     )      ->      Optional[int]:
														'''simple docstring'''

														if text is None and videos is None:
																					raise ValueError('You have to specify either text or videos. Both cannot be none.'     )

														if text is not None:
																					lowerCAmelCase_       		=  self.tokenizer(lowercase_    ,    return_tensors=lowercase_    ,    **lowercase_     )

														if videos is not None:
																					lowerCAmelCase_       		=  self.image_processor(lowercase_    ,    return_tensors=lowercase_    ,    **lowercase_     )

														if text is not None and videos is not None:
																					lowerCAmelCase_       		=  image_features.pixel_values
																					return encoding
														elif text is not None:
																					return encoding
														else:
																					return BatchEncoding(data=dict(**lowercase_     )    ,    tensor_type=lowercase_     )

							def   _lowercase (	self    ,    *lowercase_    ,    **lowercase_     )      ->      Union[str, Any]:
														'''simple docstring'''

														return self.tokenizer.batch_decode(*lowercase_    ,    **lowercase_     )

							def   _lowercase (	self    ,    *lowercase_    ,    **lowercase_     )      ->      str:
														'''simple docstring'''

														return self.tokenizer.decode(*lowercase_    ,    **lowercase_     )

							@property
							def   _lowercase (	self     )      ->      Any:
														'''simple docstring'''

														return ["input_ids", "attention_mask", "position_ids", "pixel_values"]

							@property
							def   _lowercase (	self     )      ->      List[str]:
														'''simple docstring'''

														warnings.warn(
														    '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'    ,    lowercase_    ,    )
														return self.image_processor_class






							@property
							def   _lowercase (	self     )      ->      int:
														'''simple docstring'''

														warnings.warn(
														    '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'    ,    lowercase_    ,    )
														return self.image_processor



