
"""simple docstring"""




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__lowerCAmelCase							:  Any 				=		logging.get_logger(__name__)

__lowerCAmelCase							:  Optional[Any] 				=		{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__lowerCAmelCase							:  int 				=		{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

__lowerCAmelCase							:  Optional[int] 				=		{
    '''openai-gpt''': 512,
}

def __snake_case (      UpperCamelCase    )       ->    Union[str, Any]:


		"""simple docstring"""





		a__   		=     text.strip()
		if not text:
				return []
		a__   		=     text.split()
		return tokens
class SCREAMING_SNAKE_CASE       (      lowerCAmelCase					):






	'''simple docstring'''







	def __init__(  self :List[str]       ,    __magic_name__ :List[Any]=True       ,    __magic_name__ :Optional[Any]=None       ,    __magic_name__ :int=True       ,    __magic_name__ :str=None       ,    __magic_name__ :str=True       ,    )			->   Union[str, Any]:

			'''simple docstring'''






			if never_split is None:
					a__   		=     []
			a__   		=     do_lower_case
			a__   		=     set(__magic_name__			)
			a__   		=     tokenize_chinese_chars
			a__   		=     strip_accents
			a__   		=     do_split_on_punc


	def    _UpperCamelCase							(  self :Optional[int]       ,    __magic_name__ :Tuple       ,    __magic_name__ :Optional[Any]=None			)			->   Any:

			'''simple docstring'''






			a__   		=     self.never_split.union(set(__magic_name__			)			) if never_split else self.never_split
			a__   		=     self._clean_text(__magic_name__			)

			# This was added on November 1st, 2018 for the multilingual and Chinese
			# models. This is also applied to the English models now, but it doesn't
			# matter since the English models were not trained on any Chinese data
			# and generally don't have any Chinese data in them (there are Chinese
			# characters in the vocabulary because Wikipedia does have some Chinese
			# words in the English Wikipedia.).
			if self.tokenize_chinese_chars:
					a__   		=     self._tokenize_chinese_chars(__magic_name__			)
			# prevents treating the same character with different unicode codepoints as different characters
			a__   		=     unicodedata.normalize('''NFC'''       ,    __magic_name__			)
			a__   		=     whitespace_tokenize(__magic_name__			)
			a__   		=     []
			for token in orig_tokens:
					if token not in never_split:
							if self.do_lower_case:
									a__   		=     token.lower()
									if self.strip_accents is not False:
											a__   		=     self._run_strip_accents(__magic_name__			)
							elif self.strip_accents:
									a__   		=     self._run_strip_accents(__magic_name__			)
					split_tokens.extend(self._run_split_on_punc(__magic_name__       ,    __magic_name__			)			)

			a__   		=     whitespace_tokenize(''' '''.join(__magic_name__			)			)
			return output_tokens


	def    _UpperCamelCase							(  self :int       ,    __magic_name__ :Union[str, Any]			)			->   Tuple:

			'''simple docstring'''






			a__   		=     unicodedata.normalize('''NFD'''       ,    __magic_name__			)
			a__   		=     []
			for char in text:
					a__   		=     unicodedata.category(__magic_name__			)
					if cat == "Mn":
							continue
					output.append(__magic_name__			)
			return "".join(__magic_name__			)


	def    _UpperCamelCase							(  self :Optional[int]       ,    __magic_name__ :Dict       ,    __magic_name__ :Dict=None			)			->   Optional[int]:

			'''simple docstring'''






			if not self.do_split_on_punc or (never_split is not None and text in never_split):
					return [text]
			a__   		=     list(__magic_name__			)
			a__   		=     0
			a__   		=     True
			a__   		=     []
			while i < len(__magic_name__			):
					a__   		=     chars[i]
					if _is_punctuation(__magic_name__			):
							output.append([char]			)
							a__   		=     True
					else:
							if start_new_word:
									output.append([]			)
							a__   		=     False
							output[-1].append(__magic_name__			)
					i += 1

			return ["".join(__magic_name__			) for x in output]


	def    _UpperCamelCase							(  self :str       ,    __magic_name__ :Dict			)			->   Optional[Any]:

			'''simple docstring'''






			a__   		=     []
			for char in text:
					a__   		=     ord(__magic_name__			)
					if self._is_chinese_char(__magic_name__			):
							output.append(''' '''			)
							output.append(__magic_name__			)
							output.append(''' '''			)
					else:
							output.append(__magic_name__			)
			return "".join(__magic_name__			)


	def    _UpperCamelCase							(  self :int       ,    __magic_name__ :Optional[Any]			)			->   Optional[Any]:

			'''simple docstring'''






			if (
			    (cp >= 0x4e_00 and cp <= 0x9f_ff)
			    or (cp >= 0x34_00 and cp <= 0x4d_bf)  #
			    or (cp >= 0x2_00_00 and cp <= 0x2_a6_df)  #
			    or (cp >= 0x2_a7_00 and cp <= 0x2_b7_3f)  #
			    or (cp >= 0x2_b7_40 and cp <= 0x2_b8_1f)  #
			    or (cp >= 0x2_b8_20 and cp <= 0x2_ce_af)  #
			    or (cp >= 0xf9_00 and cp <= 0xfa_ff)
			    or (cp >= 0x2_f8_00 and cp <= 0x2_fa_1f)  #
			):  #
					return True

			return False






	def    _UpperCamelCase							(  self :Dict       ,    __magic_name__ :Tuple			)			->   Dict:

			'''simple docstring'''






			a__   		=     []
			for char in text:
					a__   		=     ord(__magic_name__			)
					if cp == 0 or cp == 0xff_fd or _is_control(__magic_name__			):
							continue
					if _is_whitespace(__magic_name__			):
							output.append(''' '''			)
					else:
							output.append(__magic_name__			)
			return "".join(__magic_name__			)

def __snake_case (      UpperCamelCase    )       ->    Optional[Any]:


		"""simple docstring"""





		a__   		=     set()
		a__   		=     word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)    )
				a__   		=     char
		return pairs

def __snake_case (      UpperCamelCase    )       ->    int:


		"""simple docstring"""





		a__   		=     text.replace('''—'''						,					'''-'''    )
		a__   		=     text.replace('''–'''						,					'''-'''    )
		a__   		=     text.replace('''―'''						,					'''-'''    )
		a__   		=     text.replace('''…'''						,					'''...'''    )
		a__   		=     text.replace('''´'''						,					'''\''''    )
		a__   		=     re.sub(r'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''						,					r''' \1 '''						,					UpperCamelCase    )
		a__   		=     re.sub(r'''\s*\n\s*'''						,					''' \n '''						,					UpperCamelCase    )
		a__   		=     re.sub(r'''[^\S\n]+'''						,					''' '''						,					UpperCamelCase    )
		return text.strip()




class SCREAMING_SNAKE_CASE       (      lowerCAmelCase					):






	'''simple docstring'''







	snake_case__							:		Union[str, Any]         =		VOCAB_FILES_NAMES
	snake_case__							:		List[Any]         =		PRETRAINED_VOCAB_FILES_MAP
	snake_case__							:		Optional[Any]         =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__							:		str         =		['input_ids', 'attention_mask']


	def __init__(  self :Optional[int]       ,    __magic_name__ :Optional[int]       ,    __magic_name__ :Optional[Any]       ,    __magic_name__ :int="<unk>"       ,    **__magic_name__ :Optional[int]			)			->   Union[str, Any]:

			'''simple docstring'''






			super().__init__(unk_token=__magic_name__       ,    **__magic_name__			)

			try:
					import ftfy
					from spacy.lang.en import English

					a__   		=     English()
					a__   		=     _nlp.tokenizer
					a__   		=     ftfy.fix_text
			except ImportError:
					logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''			)
					a__   		=     BasicTokenizer(do_lower_case=__magic_name__			)
					a__   		=     None

			with open(__magic_name__       ,    encoding='''utf-8'''			) as vocab_handle:
					a__   		=     json.load(__magic_name__			)
			a__   		=     {v: k for k, v in self.encoder.items()}
			with open(__magic_name__       ,    encoding='''utf-8'''			) as merges_handle:
					a__   		=     merges_handle.read().split('''\n'''			)[1:-1]
			a__   		=     [tuple(merge.split()			) for merge in merges]
			a__   		=     dict(zip(__magic_name__       ,    range(len(__magic_name__			)			)			)			)
			a__   		=     {}


	@property
	def    _UpperCamelCase							(  self :Union[str, Any]			)			->   Union[str, Any]:

			'''simple docstring'''






			return True


	@property
	def    _UpperCamelCase							(  self :Any			)			->   Optional[int]:

			'''simple docstring'''






			return len(self.encoder			)


	def    _UpperCamelCase							(  self :Optional[int]			)			->   Union[str, Any]:

			'''simple docstring'''






			return dict(self.encoder       ,    **self.added_tokens_encoder			)


	def    _UpperCamelCase							(  self :Union[str, Any]       ,    __magic_name__ :List[str]			)			->   str:

			'''simple docstring'''






			a__   		=     tuple(token[:-1]			) + (token[-1] + '''</w>''',)
			if token in self.cache:
					return self.cache[token]
			a__   		=     get_pairs(__magic_name__			)

			if not pairs:
					return token + "</w>"

			while True:
					a__   		=     min(__magic_name__       ,    key=lambda __magic_name__			: self.bpe_ranks.get(__magic_name__       ,    float('''inf'''			)			)			)
					if bigram not in self.bpe_ranks:
							break
					a__		,	a__   		=     bigram
					a__   		=     []
					a__   		=     0
					while i < len(__magic_name__			):
							try:
									a__   		=     word.index(__magic_name__       ,    __magic_name__			)
							except ValueError:
									new_word.extend(word[i:]			)
									break
							else:
									new_word.extend(word[i:j]			)
									a__   		=     j

							if word[i] == first and i < len(__magic_name__			) - 1 and word[i + 1] == second:
									new_word.append(first + second			)
									i += 2
							else:
									new_word.append(word[i]			)
									i += 1
					a__   		=     tuple(__magic_name__			)
					a__   		=     new_word
					if len(__magic_name__			) == 1:
							break
					else:
							a__   		=     get_pairs(__magic_name__			)
			a__   		=     ''' '''.join(__magic_name__			)
			if word == "\n  </w>":
					a__   		=     '''\n</w>'''
			a__   		=     word
			return word


	def    _UpperCamelCase							(  self :str       ,    __magic_name__ :Any			)			->   Optional[Any]:

			'''simple docstring'''






			a__   		=     []
			if self.fix_text is None:
					# Using BERT's BasicTokenizer
					a__   		=     self.nlp.tokenize(__magic_name__			)
					for token in text:
							split_tokens.extend(list(self.bpe(__magic_name__			).split(''' '''			)			)			)
			else:
					# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
					a__   		=     self.nlp(text_standardize(self.fix_text(__magic_name__			)			)			)
					for token in text:
							split_tokens.extend(list(self.bpe(token.text.lower()			).split(''' '''			)			)			)
			return split_tokens


	def    _UpperCamelCase							(  self :str       ,    __magic_name__ :List[Any]			)			->   Union[str, Any]:

			'''simple docstring'''






			return self.encoder.get(__magic_name__       ,    self.encoder.get(self.unk_token			)			)


	def    _UpperCamelCase							(  self :Tuple       ,    __magic_name__ :List[Any]			)			->   Optional[Any]:

			'''simple docstring'''






			return self.decoder.get(__magic_name__       ,    self.unk_token			)


	def    _UpperCamelCase							(  self :Optional[Any]       ,    __magic_name__ :Optional[Any]			)			->   Optional[Any]:

			'''simple docstring'''






			a__   		=     ''''''.join(__magic_name__			).replace('''</w>'''       ,    ''' '''			).strip()
			return out_string






	def    _UpperCamelCase							(  self :str       ,    __magic_name__ :str       ,    __magic_name__ :Optional[str] = None			)			->   Tuple[str]:

			'''simple docstring'''






			if not os.path.isdir(__magic_name__			):
					logger.error(F"Vocabulary path ({save_directory}) should be a directory"			)
					return
			a__   		=     os.path.join(
			    __magic_name__       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)
			a__   		=     os.path.join(
			    __magic_name__       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']			)

			with open(__magic_name__       ,    '''w'''       ,    encoding='''utf-8'''			) as f:
					f.write(json.dumps(self.encoder       ,    indent=2       ,    sort_keys=__magic_name__       ,    ensure_ascii=__magic_name__			) + '''\n'''			)

			a__   		=     0
			with open(__magic_name__       ,    '''w'''       ,    encoding='''utf-8'''			) as writer:
					writer.write('''#version: 0.2\n'''			)
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,    key=lambda __magic_name__			: kv[1]			):
							if index != token_index:
									logger.warning(
									    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
									    ''' Please check that the tokenizer is not corrupted!'''			)
									a__   		=     token_index
							writer.write(''' '''.join(__magic_name__			) + '''\n'''			)
							index += 1

			return vocab_file, merge_file






"""simple docstring"""




import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
					import torch

if is_vision_available():
					from PIL import Image

					from transformers import MobileNetVaImageProcessor
class SCREAMING_SNAKE_CASE       (      unittest.TestCase					):






	'''simple docstring'''







	def __init__(  self :Tuple       ,    __magic_name__ :str       ,    __magic_name__ :Optional[Any]=7       ,    __magic_name__ :int=3       ,    __magic_name__ :Union[str, Any]=18       ,    __magic_name__ :Union[str, Any]=30       ,    __magic_name__ :str=400       ,    __magic_name__ :str=True       ,    __magic_name__ :Any=None       ,    __magic_name__ :Tuple=True       ,    __magic_name__ :List[str]=None       ,    )			->   Optional[int]:

			'''simple docstring'''






			a__   		=     size if size is not None else {'''shortest_edge''': 20}
			a__   		=     crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}
			a__   		=     parent
			a__   		=     batch_size
			a__   		=     num_channels
			a__   		=     image_size
			a__   		=     min_resolution
			a__   		=     max_resolution
			a__   		=     do_resize
			a__   		=     size
			a__   		=     do_center_crop
			a__   		=     crop_size






	def    _UpperCamelCase							(  self :Tuple			)			->   Any:

			'''simple docstring'''






			return {
			    "do_resize": self.do_resize,
			    "size": self.size,
			    "do_center_crop": self.do_center_crop,
			    "crop_size": self.crop_size,
			}

@require_torch
@require_vision
class SCREAMING_SNAKE_CASE       (      lowerCAmelCase					,   unittest.TestCase					):






	'''simple docstring'''







	snake_case__							:		str         =		MobileNetVaImageProcessor if is_vision_available() else None


	def    _UpperCamelCase							(  self :Union[str, Any]			)			->   str:

			'''simple docstring'''






			a__   		=     MobileNetVaImageProcessingTester(self			)


	@property
	def    _UpperCamelCase							(  self :List[str]			)			->   Optional[Any]:

			'''simple docstring'''






			return self.image_processor_tester.prepare_image_processor_dict()


	def    _UpperCamelCase							(  self :Tuple			)			->   Tuple:

			'''simple docstring'''






			a__   		=     self.image_processing_class(**self.image_processor_dict			)
			self.assertTrue(hasattr(__magic_name__       ,    '''do_resize'''			)			)
			self.assertTrue(hasattr(__magic_name__       ,    '''size'''			)			)
			self.assertTrue(hasattr(__magic_name__       ,    '''do_center_crop'''			)			)
			self.assertTrue(hasattr(__magic_name__       ,    '''center_crop'''			)			)


	def    _UpperCamelCase							(  self :List[Any]			)			->   int:

			'''simple docstring'''






			a__   		=     self.image_processing_class.from_dict(self.image_processor_dict			)
			self.assertEqual(image_processor.size       ,    {'''shortest_edge''': 20}			)
			self.assertEqual(image_processor.crop_size       ,    {'''height''': 18, '''width''': 18}			)

			a__   		=     self.image_processing_class.from_dict(self.image_processor_dict       ,    size=42       ,    crop_size=84			)
			self.assertEqual(image_processor.size       ,    {'''shortest_edge''': 42}			)
			self.assertEqual(image_processor.crop_size       ,    {'''height''': 84, '''width''': 84}			)


	def    _UpperCamelCase							(  self :Optional[int]			)			->   Any:

			'''simple docstring'''






			pass


	def    _UpperCamelCase							(  self :Dict			)			->   Tuple:

			'''simple docstring'''






			a__   		=     self.image_processing_class(**self.image_processor_dict			)
			# create random PIL images
			a__   		=     prepare_image_inputs(self.image_processor_tester       ,    equal_resolution=__magic_name__			)
			for image in image_inputs:
					self.assertIsInstance(__magic_name__       ,    Image.Image			)

			# Test not batched input
			a__   		=     image_processing(image_inputs[0]       ,    return_tensors='''pt'''			).pixel_values
			self.assertEqual(
			    encoded_images.shape       ,    (
			        1,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['''height'''],
			        self.image_processor_tester.crop_size['''width'''],
			    )       ,    )

			# Test batched
			a__   		=     image_processing(__magic_name__       ,    return_tensors='''pt'''			).pixel_values
			self.assertEqual(
			    encoded_images.shape       ,    (
			        self.image_processor_tester.batch_size,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['''height'''],
			        self.image_processor_tester.crop_size['''width'''],
			    )       ,    )


	def    _UpperCamelCase							(  self :Optional[Any]			)			->   Union[str, Any]:

			'''simple docstring'''






			a__   		=     self.image_processing_class(**self.image_processor_dict			)
			# create random numpy tensors
			a__   		=     prepare_image_inputs(self.image_processor_tester       ,    equal_resolution=__magic_name__       ,    numpify=__magic_name__			)
			for image in image_inputs:
					self.assertIsInstance(__magic_name__       ,    np.ndarray			)

			# Test not batched input
			a__   		=     image_processing(image_inputs[0]       ,    return_tensors='''pt'''			).pixel_values
			self.assertEqual(
			    encoded_images.shape       ,    (
			        1,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['''height'''],
			        self.image_processor_tester.crop_size['''width'''],
			    )       ,    )

			# Test batched
			a__   		=     image_processing(__magic_name__       ,    return_tensors='''pt'''			).pixel_values
			self.assertEqual(
			    encoded_images.shape       ,    (
			        self.image_processor_tester.batch_size,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['''height'''],
			        self.image_processor_tester.crop_size['''width'''],
			    )       ,    )






	def    _UpperCamelCase							(  self :List[Any]			)			->   str:

			'''simple docstring'''






			a__   		=     self.image_processing_class(**self.image_processor_dict			)
			# create random PyTorch tensors
			a__   		=     prepare_image_inputs(self.image_processor_tester       ,    equal_resolution=__magic_name__       ,    torchify=__magic_name__			)
			for image in image_inputs:
					self.assertIsInstance(__magic_name__       ,    torch.Tensor			)

			# Test not batched input
			a__   		=     image_processing(image_inputs[0]       ,    return_tensors='''pt'''			).pixel_values
			self.assertEqual(
			    encoded_images.shape       ,    (
			        1,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['''height'''],
			        self.image_processor_tester.crop_size['''width'''],
			    )       ,    )

			# Test batched
			a__   		=     image_processing(__magic_name__       ,    return_tensors='''pt'''			).pixel_values
			self.assertEqual(
			    encoded_images.shape       ,    (
			        self.image_processor_tester.batch_size,
			        self.image_processor_tester.num_channels,
			        self.image_processor_tester.crop_size['''height'''],
			        self.image_processor_tester.crop_size['''width'''],
			    )       ,    )




