



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation

UpperCAmelCase							=       logging.get_logger(__name__)

UpperCAmelCase							=       {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

UpperCAmelCase							=       {
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
}

UpperCAmelCase							=       {
    """gpt2""": 1_024,
    """gpt2-medium""": 1_024,
    """gpt2-large""": 1_024,
    """gpt2-xl""": 1_024,
    """distilgpt2""": 1_024,
}

@lru_cache()
def       A_   (						):

				"""simple docstring"""
				a__					   =		(
				    list(range(ord("""!"""		)			,		ord("""~"""		) + 1		)		) + list(range(ord("""¡"""		)			,		ord("""¬"""		) + 1		)		) + list(range(ord("""®"""		)			,		ord("""ÿ"""		) + 1		)		)
				)
				a__					   =		bs[:]
				a__					   =		0
				for b in range(2**8		):
								if b not in bs:
												bs.append(__a		)
												cs.append(2**8 + n		)
												n += 1
				a__					   =		[chr(__a		) for n in cs]
				return dict(zip(__a			,		__a		)		)

def       A_   (						__a  :					Optional[int]		):

				"""simple docstring"""
				a__					   =		set()
				a__					   =		word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)		)
								a__					   =		char
				return pairs




class 			__snake_case   ( SCREAMING_SNAKE_CASE):







							'''simple docstring'''






							UpperCamelCase__					:   Optional[Any]       =       VOCAB_FILES_NAMES
							UpperCamelCase__					:   Any       =       PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase__					:   Tuple       =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase__					:   Any       =       ["""input_ids""", """attention_mask"""]


							def __init__(				self	,	a_	,	a_	,	a_="replace"	,	a_="<|endoftext|>"	,	a_="<|endoftext|>"	,	a_="<|endoftext|>"	,	a_=None	,	a_=False	,	a_=False	,	**a_	,	):
											a__					   =		AddedToken(a_	,	lstrip=a_	,	rstrip=a_		) if isinstance(a_	,	a_		) else bos_token
											a__					   =		AddedToken(a_	,	lstrip=a_	,	rstrip=a_		) if isinstance(a_	,	a_		) else eos_token
											a__					   =		AddedToken(a_	,	lstrip=a_	,	rstrip=a_		) if isinstance(a_	,	a_		) else unk_token
											a__					   =		AddedToken(a_	,	lstrip=a_	,	rstrip=a_		) if isinstance(a_	,	a_		) else pad_token
											super().__init__(
											    errors=a_	,	unk_token=a_	,	bos_token=a_	,	eos_token=a_	,	pad_token=a_	,	add_prefix_space=a_	,	add_bos_token=a_	,	**a_	,	)
											a__					   =		add_bos_token

											with open(a_	,	encoding="""utf-8"""		) as vocab_handle:
															a__					   =		json.load(a_		)
											a__					   =		{v: k for k, v in self.encoder.items()}
											a__					   =		errors  # how to handle errors in decoding
											a__					   =		bytes_to_unicode()
											a__					   =		{v: k for k, v in self.byte_encoder.items()}
											with open(a_	,	encoding="""utf-8"""		) as merges_handle:
															a__					   =		merges_handle.read().split("""\n"""		)[1:-1]
											a__					   =		[tuple(merge.split()		) for merge in bpe_merges]
											a__					   =		dict(zip(a_	,	range(len(a_		)		)		)		)
											a__					   =		{}
											a__					   =		add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											a__					   =		re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""		)


							@property
							def _a   (				self		):
											return len(self.encoder		)


							def _a   (				self		):
											return dict(self.encoder	,	**self.added_tokens_encoder		)


							def _a   (				self	,	a_		):
											if token in self.cache:
															return self.cache[token]
											a__					   =		tuple(a_		)
											a__					   =		get_pairs(a_		)

											if not pairs:
															return token

											while True:
															a__					   =		min(a_	,	key=lambda a_		: self.bpe_ranks.get(a_	,	float("""inf"""		)		)		)
															if bigram not in self.bpe_ranks:
																			break
															a__   ,      a__					   =		bigram
															a__					   =		[]
															a__					   =		0
															while i < len(a_		):
																			try:
																							a__					   =		word.index(a_	,	a_		)
																			except ValueError:
																							new_word.extend(word[i:]		)
																							break
																			else:
																							new_word.extend(word[i:j]		)
																							a__					   =		j

																			if word[i] == first and i < len(a_		) - 1 and word[i + 1] == second:
																							new_word.append(first + second		)
																							i += 2
																			else:
																							new_word.append(word[i]		)
																							i += 1
															a__					   =		tuple(a_		)
															a__					   =		new_word
															if len(a_		) == 1:
																			break
															else:
																			a__					   =		get_pairs(a_		)
											a__					   =		""" """.join(a_		)
											a__					   =		word
											return word


							def _a   (				self	,	a_	,	a_=None		):
											if self.add_bos_token:
															a__					   =		[self.bos_token_id]
											else:
															a__					   =		[]

											a__					   =		bos_token_ids + token_ids_a

											if token_ids_a is None:
															return output

											return output + bos_token_ids + token_ids_a


							def _a   (				self	,	a_	,	a_ = None	,	a_ = False		):
											if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=a_	,	token_ids_a=a_	,	already_has_special_tokens=a_		)

											if not self.add_bos_token:
															return super().get_special_tokens_mask(
															    token_ids_a=a_	,	token_ids_a=a_	,	already_has_special_tokens=a_		)

											if token_ids_a is None:
															return [1] + ([0] * len(a_		))
											return [1] + ([0] * len(a_		)) + [1] + ([0] * len(a_		))


							def _a   (				self	,	a_		):
											a__					   =		[]
											for token in re.findall(self.pat	,	a_		):
															a__					   =		"""""".join(
															    self.byte_encoder[b] for b in token.encode("""utf-8"""		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a_		).split(""" """		)		)
											return bpe_tokens


							def _a   (				self	,	a_		):
											return self.encoder.get(a_	,	self.encoder.get(self.unk_token		)		)


							def _a   (				self	,	a_		):
											return self.decoder.get(a_		)


							def _a   (				self	,	a_		):
											a__					   =		"""""".join(a_		)
											a__					   =		bytearray([self.byte_decoder[c] for c in text]		).decode("""utf-8"""	,	errors=self.errors		)
											return text


							def _a   (				self	,	a_	,	a_ = None		):
											if not os.path.isdir(a_		):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
															return
											a__					   =		os.path.join(
											    a_	,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
											a__					   =		os.path.join(
											    a_	,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]		)

											with open(a_	,	"""w"""	,	encoding="""utf-8"""		) as f:
															f.write(json.dumps(self.encoder	,	indent=2	,	sort_keys=a_	,	ensure_ascii=a_		) + """\n"""		)

											a__					   =		0
											with open(a_	,	"""w"""	,	encoding="""utf-8"""		) as writer:
															writer.write("""#version: 0.2\n"""		)
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,	key=lambda a_		: kv[1]		):
																			if index != token_index:
																							logger.warning(
																							    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																							    """ Please check that the tokenizer is not corrupted!"""		)
																							a__					   =		token_index
																			writer.write(""" """.join(a_		) + """\n"""		)
																			index += 1

											return vocab_file, merge_file


							def _a   (				self	,	a_	,	a_=False	,	**a_		):
											a__					   =		kwargs.pop("""add_prefix_space"""	,	self.add_prefix_space		)
											if is_split_into_words or add_prefix_space:
															a__					   =		""" """ + text
											return (text, kwargs)







							def _a   (				self	,	a_		):
											a__					   =		[]
											for is_user, text in conversation.iter_texts():
															input_ids.extend(self.encode(a_	,	add_special_tokens=a_		) + [self.eos_token_id]		)
											if len(a_		) > self.model_max_length:
															a__					   =		input_ids[-self.model_max_length :]
											return input_ids









import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class __lowerCamelCase					( snake_case__	,			unittest.TestCase):





	"""simple docstring"""






	UpperCamelCase__								= GPTNeoXJapaneseTokenizer
	UpperCamelCase__								= False
	UpperCamelCase__								= {"do_clean_text": False, "add_prefix_space": False}

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							super().setUp()

							_UpperCAmelCase							   =     [
							    'こん',
							    'こんに',
							    'にちは',
							    'ばんは',
							    '世界,㔺界',
							    '、',
							    '。',
							    '<BR>',
							    '<SP>',
							    '<TAB>',
							    '<URL>',
							    '<EMAIL>',
							    '<TEL>',
							    '<DATE>',
							    '<PRICE>',
							    '<BLOCK>',
							    '<KIGOU>',
							    '<U2000U2BFF>',
							    '<|emoji1|>',
							    '<unk>',
							    '<|startoftext|>',
							    '<|endoftext|>',
							]
							_UpperCAmelCase							   =     {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
							_UpperCAmelCase							   =     {'unk_token': '<unk>'}

							_UpperCAmelCase							   =     os.path.join(self.tmpdirname			,       VOCAB_FILES_NAMES['vocab_file']       )
							_UpperCAmelCase							   =     os.path.join(self.tmpdirname			,       VOCAB_FILES_NAMES['emoji_file']       )
							with open(self.vocab_file			,       'w'			,       encoding='utf-8'       ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )
							with open(self.emoji_file			,       'w'       ) as emoji_writer:
													emoji_writer.write(json.dumps(UpperCAmelCase       )       )

	def 			UpperCamelCase       (				self			,       **UpperCAmelCase       ):
							"""simple docstring"""







							kwargs.update(self.special_tokens_map       )
							return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname			,       **UpperCAmelCase       )

	def 			UpperCamelCase       (				self			,       UpperCAmelCase       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     'こんにちは、世界。 \nこんばんは、㔺界。😀'
							_UpperCAmelCase							   =     'こんにちは、世界。 \nこんばんは、世界。😀'
							return input_text, output_text

	def 			UpperCamelCase       (				self			,       UpperCAmelCase       ):
							"""simple docstring"""







							_UpperCAmelCase   ,					_UpperCAmelCase							   =     self.get_input_output_texts(UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.encode(UpperCAmelCase			,       add_special_tokens=UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.decode(UpperCAmelCase			,       clean_up_tokenization_spaces=UpperCAmelCase       )
							return text, ids

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass  # TODO add if relevant

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass  # TODO add if relevant

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass  # TODO add if relevant

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.get_tokenizer()

							# Testing tokenization
							_UpperCAmelCase							   =     'こんにちは、世界。　こんばんは、㔺界。'
							_UpperCAmelCase							   =     ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
							_UpperCAmelCase							   =     tokenizer.tokenize(UpperCAmelCase       )
							self.assertListEqual(UpperCAmelCase			,       UpperCAmelCase       )

							# Testing conversion to ids without special tokens
							_UpperCAmelCase							   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
							_UpperCAmelCase							   =     tokenizer.convert_tokens_to_ids(UpperCAmelCase       )
							self.assertListEqual(UpperCAmelCase			,       UpperCAmelCase       )

							# Testing conversion to ids with special tokens
							_UpperCAmelCase							   =     tokens + [tokenizer.unk_token]
							_UpperCAmelCase							   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
							_UpperCAmelCase							   =     tokenizer.convert_tokens_to_ids(UpperCAmelCase       )
							self.assertListEqual(UpperCAmelCase			,       UpperCAmelCase       )

	@slow
	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.tokenizer_class.from_pretrained('abeja/gpt-neox-japanese-2.7b'       )

							_UpperCAmelCase							   =     tokenizer.encode('ありがとう。'			,       add_special_tokens=UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.encode('どういたしまして。'			,       add_special_tokens=UpperCAmelCase       )

							_UpperCAmelCase							   =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase			,       UpperCAmelCase       )

							assert encoded_sentence == ids_a
							assert encoded_pair == ids_a + ids_a

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass
