
def       _lowercase					(      __SCREAMING_SNAKE_CASE				,    __SCREAMING_SNAKE_CASE )	->     list:
			UpperCamelCase__    :		int     =				len(__SCREAMING_SNAKE_CASE )
			UpperCamelCase__    :		int     =				len(__SCREAMING_SNAKE_CASE )
			UpperCamelCase__    :		int     =				(
			    first_input_list_length
			    if first_input_list_length > second_input_list_length
			    else second_input_list_length
			)
			UpperCamelCase__    :		list     =				[]
			for char_count in range(__SCREAMING_SNAKE_CASE ):
						if char_count < first_input_list_length:
									output_result_list.append(first_input_list[char_count] )
						if char_count < second_input_list_length:
									output_result_list.append(second_input_list[char_count] )

			return output_result_list


if __name__ == "__main__":
							print(alternative_list_arrange(['''A''', '''B''', '''C'''], [1, 2, 3, 4, 5]), end=''' ''')





import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


UpperCAmelCase__				:  Optional[Any]       					=				logging.get_logger(__name__)

UpperCAmelCase__				:  List[str]       					=				{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

UpperCAmelCase__				:  Optional[int]       					=				{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

UpperCAmelCase__				:  Any       					=				{
    '''openai-gpt''': 512,
}



def       _lowercase					(      __SCREAMING_SNAKE_CASE )	->     Optional[int]:
			UpperCamelCase__    :		List[Any]     =				text.strip()
			if not text:
						return []
			UpperCamelCase__    :		List[Any]     =				text.split()
			return tokens



class 			UpperCamelCase_			(						__UpperCamelCase   ):




							'''simple docstring'''

							def __init__(				self     ,						UpperCamelCase=True     ,						UpperCamelCase=None     ,						UpperCamelCase=True     ,						UpperCamelCase=None     ,						UpperCamelCase=True     ,						) ->	Tuple:
										if never_split is None:
													UpperCamelCase__    :		str     =				[]
										UpperCamelCase__    :		Optional[Any]     =				do_lower_case
										UpperCamelCase__    :		Optional[int]     =				set(UpperCamelCase)
										UpperCamelCase__    :		List[Any]     =				tokenize_chinese_chars
										UpperCamelCase__    :		Optional[int]     =				strip_accents
										UpperCamelCase__    :		Any     =				do_split_on_punc
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase     ,						UpperCamelCase=None) ->	List[str]:
										UpperCamelCase__    :		Tuple     =				self.never_split.union(set(UpperCamelCase)) if never_split else self.never_split
										UpperCamelCase__    :		Optional[int]     =				self._clean_text(UpperCamelCase)

										# This was added on November 1st, 2018 for the multilingual and Chinese
										# models. This is also applied to the English models now, but it doesn't
										# matter since the English models were not trained on any Chinese data
										# and generally don't have any Chinese data in them (there are Chinese
										# characters in the vocabulary because Wikipedia does have some Chinese
										# words in the English Wikipedia.).
										if self.tokenize_chinese_chars:
													UpperCamelCase__    :		List[str]     =				self._tokenize_chinese_chars(UpperCamelCase)
										# prevents treating the same character with different unicode codepoints as different characters
										UpperCamelCase__    :		Dict     =				unicodedata.normalize('NFC'     ,						UpperCamelCase)
										UpperCamelCase__    :		Optional[int]     =				whitespace_tokenize(UpperCamelCase)
										UpperCamelCase__    :		List[Any]     =				[]
										for token in orig_tokens:
													if token not in never_split:
																if self.do_lower_case:
																			UpperCamelCase__    :		str     =				token.lower()
																			if self.strip_accents is not False:
																						UpperCamelCase__    :		str     =				self._run_strip_accents(UpperCamelCase)
																elif self.strip_accents:
																			UpperCamelCase__    :		Union[str, Any]     =				self._run_strip_accents(UpperCamelCase)
													split_tokens.extend(self._run_split_on_punc(UpperCamelCase     ,						UpperCamelCase))

										UpperCamelCase__    :		Dict     =				whitespace_tokenize(' '.join(UpperCamelCase))
										return output_tokens
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[str]:
										UpperCamelCase__    :		Union[str, Any]     =				unicodedata.normalize('NFD'     ,						UpperCamelCase)
										UpperCamelCase__    :		Dict     =				[]
										for char in text:
													UpperCamelCase__    :		Union[str, Any]     =				unicodedata.category(UpperCamelCase)
													if cat == "Mn":
																continue
													output.append(UpperCamelCase)
										return "".join(UpperCamelCase)
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase     ,						UpperCamelCase=None) ->	Dict:
										if not self.do_split_on_punc or (never_split is not None and text in never_split):
													return [text]
										UpperCamelCase__    :		Dict     =				list(UpperCamelCase)
										UpperCamelCase__    :		Optional[int]     =				0
										UpperCamelCase__    :		Dict     =				True
										UpperCamelCase__    :		Optional[int]     =				[]
										while i < len(UpperCamelCase):
													UpperCamelCase__    :		List[str]     =				chars[i]
													if _is_punctuation(UpperCamelCase):
																output.append([char])
																UpperCamelCase__    :		Tuple     =				True
													else:
																if start_new_word:
																			output.append([])
																UpperCamelCase__    :		Union[str, Any]     =				False
																output[-1].append(UpperCamelCase)
													i += 1

										return ["".join(UpperCamelCase) for x in output]
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	Tuple:
										UpperCamelCase__    :		List[str]     =				[]
										for char in text:
													UpperCamelCase__    :		List[str]     =				ord(UpperCamelCase)
													if self._is_chinese_char(UpperCamelCase):
																output.append(' ')
																output.append(UpperCamelCase)
																output.append(' ')
													else:
																output.append(UpperCamelCase)
										return "".join(UpperCamelCase)
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	Union[str, Any]:
										# This defines a "chinese character" as anything in the CJK Unicode block:
										#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
										#
										# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
										# despite its name. The modern Korean Hangul alphabet is a different block,
										# as is Japanese Hiragana and Katakana. Those alphabets are used to write
										# space-separated words, so they are not treated specially and handled
										# like the all of the other languages.
										if (
										    (cp >= 0X4e00 and cp <= 0X9fff)
										    or (cp >= 0X3400 and cp <= 0X4dbf)  #
										    or (cp >= 0X2_0000 and cp <= 0X2_a6df)  #
										    or (cp >= 0X2_a700 and cp <= 0X2_b73f)  #
										    or (cp >= 0X2_b740 and cp <= 0X2_b81f)  #
										    or (cp >= 0X2_b820 and cp <= 0X2_ceaf)  #
										    or (cp >= 0Xf900 and cp <= 0Xfaff)
										    or (cp >= 0X2_f800 and cp <= 0X2_fa1f)  #
										):  #
													return True

										return False






							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[str]:
										UpperCamelCase__    :		List[Any]     =				[]
										for char in text:
													UpperCamelCase__    :		Dict     =				ord(UpperCamelCase)
													if cp == 0 or cp == 0Xfffd or _is_control(UpperCamelCase):
																continue
													if _is_whitespace(UpperCamelCase):
																output.append(' ')
													else:
																output.append(UpperCamelCase)
										return "".join(UpperCamelCase)



def       _lowercase					(      __SCREAMING_SNAKE_CASE )	->     Union[str, Any]:
			UpperCamelCase__    :		Optional[int]     =				set()
			UpperCamelCase__    :		str     =				word[0]
			for char in word[1:]:
						pairs.add((prev_char, char) )
						UpperCamelCase__    :		Any     =				char
			return pairs



def       _lowercase					(      __SCREAMING_SNAKE_CASE )	->     Optional[Any]:
			UpperCamelCase__    :		Tuple     =				text.replace('—'				,    '-' )
			UpperCamelCase__    :		Any     =				text.replace('–'				,    '-' )
			UpperCamelCase__    :		Any     =				text.replace('―'				,    '-' )
			UpperCamelCase__    :		Tuple     =				text.replace('…'				,    '...' )
			UpperCamelCase__    :		Dict     =				text.replace('´'				,    '\'' )
			UpperCamelCase__    :		int     =				re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'				,    R' \1 '				,    __SCREAMING_SNAKE_CASE )
			UpperCamelCase__    :		int     =				re.sub(R'\s*\n\s*'				,    ' \n '				,    __SCREAMING_SNAKE_CASE )
			UpperCamelCase__    :		str     =				re.sub(R'[^\S\n]+'				,    ' '				,    __SCREAMING_SNAKE_CASE )
			return text.strip()



class 			UpperCamelCase_			(						__UpperCamelCase   ):




							'''simple docstring'''

							UpperCamelCase_					   =							VOCAB_FILES_NAMES
							UpperCamelCase_					   =							PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase_					   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase_					   =							["""input_ids""", """attention_mask"""]
							def __init__(				self     ,						UpperCamelCase     ,						UpperCamelCase     ,						UpperCamelCase="<unk>"     ,						**UpperCamelCase) ->	Union[str, Any]:
										super().__init__(unk_token=UpperCamelCase     ,						**UpperCamelCase)

										try:
													import ftfy
													from spacy.lang.en import English

													UpperCamelCase__    :		Optional[int]     =				English()
													UpperCamelCase__    :		Dict     =				_nlp.tokenizer
													UpperCamelCase__    :		List[Any]     =				ftfy.fix_text
										except ImportError:
													logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.')
													UpperCamelCase__    :		Dict     =				BasicTokenizer(do_lower_case=UpperCamelCase)
													UpperCamelCase__    :		List[Any]     =				None

										with open(UpperCamelCase     ,						encoding='utf-8') as vocab_handle:
													UpperCamelCase__    :		int     =				json.load(UpperCamelCase)
										UpperCamelCase__    :		Dict     =				{v: k for k, v in self.encoder.items()}
										with open(UpperCamelCase     ,						encoding='utf-8') as merges_handle:
													UpperCamelCase__    :		Optional[int]     =				merges_handle.read().split('\n')[1:-1]
										UpperCamelCase__    :		int     =				[tuple(merge.split()) for merge in merges]
										UpperCamelCase__    :		int     =				dict(zip(UpperCamelCase     ,						range(len(UpperCamelCase))))
										UpperCamelCase__    :		Optional[Any]     =				{}
							@property
							def        lowerCAmelCase__ (				self) ->	Dict:
										return True
							@property
							def        lowerCAmelCase__ (				self) ->	Union[str, Any]:
										return len(self.encoder)
							def        lowerCAmelCase__ (				self) ->	Tuple:
										return dict(self.encoder     ,						**self.added_tokens_encoder)
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[Any]:
										UpperCamelCase__    :		Optional[Any]     =				tuple(token[:-1]) + (token[-1] + '</w>',)
										if token in self.cache:
													return self.cache[token]
										UpperCamelCase__    :		int     =				get_pairs(UpperCamelCase)

										if not pairs:
													return token + "</w>"

										while True:
													UpperCamelCase__    :		str     =				min(UpperCamelCase     ,						key=lambda UpperCamelCase: self.bpe_ranks.get(UpperCamelCase     ,						float('inf')))
													if bigram not in self.bpe_ranks:
																break
													UpperCamelCase__   ,			UpperCamelCase__    :		Optional[int]     =				bigram
													UpperCamelCase__    :		str     =				[]
													UpperCamelCase__    :		List[Any]     =				0
													while i < len(UpperCamelCase):
																try:
																			UpperCamelCase__    :		Union[str, Any]     =				word.index(UpperCamelCase     ,						UpperCamelCase)
																except ValueError:
																			new_word.extend(word[i:])
																			break
																else:
																			new_word.extend(word[i:j])
																			UpperCamelCase__    :		Tuple     =				j

																if word[i] == first and i < len(UpperCamelCase) - 1 and word[i + 1] == second:
																			new_word.append(first + second)
																			i += 2
																else:
																			new_word.append(word[i])
																			i += 1
													UpperCamelCase__    :		int     =				tuple(UpperCamelCase)
													UpperCamelCase__    :		List[Any]     =				new_word
													if len(UpperCamelCase) == 1:
																break
													else:
																UpperCamelCase__    :		List[Any]     =				get_pairs(UpperCamelCase)
										UpperCamelCase__    :		Optional[int]     =				' '.join(UpperCamelCase)
										if word == "\n  </w>":
													UpperCamelCase__    :		Optional[int]     =				'\n</w>'
										UpperCamelCase__    :		Optional[Any]     =				word
										return word
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[Any]:
										UpperCamelCase__    :		int     =				[]
										if self.fix_text is None:
													# Using BERT's BasicTokenizer
													UpperCamelCase__    :		List[str]     =				self.nlp.tokenize(UpperCamelCase)
													for token in text:
																split_tokens.extend(list(self.bpe(UpperCamelCase).split(' ')))
										else:
													# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
													UpperCamelCase__    :		Dict     =				self.nlp(text_standardize(self.fix_text(UpperCamelCase)))
													for token in text:
																split_tokens.extend(list(self.bpe(token.text.lower()).split(' ')))
										return split_tokens
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	Tuple:
										return self.encoder.get(UpperCamelCase     ,						self.encoder.get(self.unk_token))
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[Any]:
										return self.decoder.get(UpperCamelCase     ,						self.unk_token)
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[str]:
										UpperCamelCase__    :		Optional[Any]     =				''.join(UpperCamelCase).replace('</w>'     ,						' ').strip()
										return out_string






							def        lowerCAmelCase__ (				self     ,						UpperCamelCase     ,						UpperCamelCase = None) ->	Tuple[str]:
										if not os.path.isdir(UpperCamelCase):
													logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
													return
										UpperCamelCase__    :		int     =				os.path.join(
										    UpperCamelCase     ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
										UpperCamelCase__    :		List[str]     =				os.path.join(
										    UpperCamelCase     ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

										with open(UpperCamelCase     ,						'w'     ,						encoding='utf-8') as f:
													f.write(json.dumps(self.encoder     ,						indent=2     ,						sort_keys=UpperCamelCase     ,						ensure_ascii=UpperCamelCase) + '\n')

										UpperCamelCase__    :		Any     =				0
										with open(UpperCamelCase     ,						'w'     ,						encoding='utf-8') as writer:
													writer.write('#version: 0.2\n')
													for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,						key=lambda UpperCamelCase: kv[1]):
																if index != token_index:
																			logger.warning(
																			    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																			    ' Please check that the tokenizer is not corrupted!')
																			UpperCamelCase__    :		Dict     =				token_index
																writer.write(' '.join(UpperCamelCase) + '\n')
																index += 1

										return vocab_file, merge_file



