
"""simple docstring"""






import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation






def 		_snake_case			(      lowercase__						:    torch.Tensor ,	lowercase__						:    int ,	lowercase__						:    bool = False ,	lowercase__						:    float = 1 ,	lowercase__						:    float = 1 ,	lowercase__						:    int = 1_0_0_0_0 ,	)    ->							Optional[int]:


			'''simple docstring'''





			assert len(timesteps.shape						) == 1, "Timesteps should be a 1d-array"

			lowerCAmelCase_       :Any     	=    embedding_dim // 2
			lowerCAmelCase_       :Optional[Any]     	=    -math.log(lowercase__						) * torch.arange(
			    start=0 ,	end=lowercase__ ,	dtype=torch.floataa ,	device=timesteps.device						)
			lowerCAmelCase_       :str     	=    exponent / (half_dim - downscale_freq_shift)

			lowerCAmelCase_       :int     	=    torch.exp(lowercase__						)
			lowerCAmelCase_       :Optional[int]     	=    timesteps[:, None].float() * emb[None, :]

			# scale embeddings
			lowerCAmelCase_       :Any     	=    scale * emb

			# concat sine and cosine embeddings
			lowerCAmelCase_       :Union[str, Any]     	=    torch.cat([torch.sin(lowercase__						), torch.cos(lowercase__						)] ,	dim=-1						)

			# flip sine and cosine embeddings
			if flip_sin_to_cos:
						lowerCAmelCase_       :Dict     	=    torch.cat([emb[:, half_dim:], emb[:, :half_dim]] ,	dim=-1						)

			# zero pad
			if embedding_dim % 2 == 1:
						lowerCAmelCase_       :Union[str, Any]     	=    torch.nn.functional.pad(lowercase__ ,	(0, 1, 0, 0)						)
			return emb






def 		_snake_case			(      lowercase__						:    List[str] ,	lowercase__						:    Optional[int] ,	lowercase__						:    Any=False ,	lowercase__						:    Tuple=0						)    ->							str:


			'''simple docstring'''





			lowerCAmelCase_       :Any     	=    np.arange(lowercase__ ,	dtype=np.floataa						)
			lowerCAmelCase_       :int     	=    np.arange(lowercase__ ,	dtype=np.floataa						)
			lowerCAmelCase_       :Dict     	=    np.meshgrid(lowercase__ ,	lowercase__						)  # here w goes first
			lowerCAmelCase_       :List[str]     	=    np.stack(lowercase__ ,	axis=0						)

			lowerCAmelCase_       :Tuple     	=    grid.reshape([2, 1, grid_size, grid_size]						)
			lowerCAmelCase_       :List[Any]     	=    get_ad_sincos_pos_embed_from_grid(lowercase__ ,	lowercase__						)
			if cls_token and extra_tokens > 0:
						lowerCAmelCase_       :int     	=    np.concatenate([np.zeros([extra_tokens, embed_dim]						), pos_embed] ,	axis=0						)
			return pos_embed






def 		_snake_case			(      lowercase__						:    Optional[int] ,	lowercase__						:    str						)    ->							Union[str, Any]:


			'''simple docstring'''





			if embed_dim % 2 != 0:
						raise ValueError("""embed_dim must be divisible by 2"""						)

			# use half of dimensions to encode grid_h
			lowerCAmelCase_       :Dict     	=    get_ad_sincos_pos_embed_from_grid(embed_dim // 2 ,	grid[0]						)  # (H*W, D/2)
			lowerCAmelCase_       :Union[str, Any]     	=    get_ad_sincos_pos_embed_from_grid(embed_dim // 2 ,	grid[1]						)  # (H*W, D/2)

			lowerCAmelCase_       :Optional[int]     	=    np.concatenate([emb_h, emb_w] ,	axis=1						)  # (H*W, D)
			return emb






def 		_snake_case			(      lowercase__						:    Dict ,	lowercase__						:    Any						)    ->							Union[str, Any]:


			'''simple docstring'''





			if embed_dim % 2 != 0:
						raise ValueError("""embed_dim must be divisible by 2"""						)

			lowerCAmelCase_       :int     	=    np.arange(embed_dim // 2 ,	dtype=np.floataa						)
			omega /= embed_dim / 2.0
			lowerCAmelCase_       :Union[str, Any]     	=    1.0 / 1_0_0_0_0**omega  # (D/2,)

			lowerCAmelCase_       :Optional[Any]     	=    pos.reshape(-1						)  # (M,)
			lowerCAmelCase_       :int     	=    np.einsum("""m,d->md""" ,	lowercase__ ,	lowercase__						)  # (M, D/2), outer product

			lowerCAmelCase_       :Optional[Any]     	=    np.sin(lowercase__						)  # (M, D/2)
			lowerCAmelCase_       :Union[str, Any]     	=    np.cos(lowercase__						)  # (M, D/2)

			lowerCAmelCase_       :str     	=    np.concatenate([emb_sin, emb_cos] ,	axis=1						)  # (M, D)
			return emb




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A=224			,  __A=224			,  __A=16			,  __A=3			,  __A=768			,  __A=False			,  __A=True			,  __A=True			,  )     ->	str:
						super().__init__()

						lowerCAmelCase_       :Optional[Any]     	=    (height // patch_size) * (width // patch_size)
						lowerCAmelCase_       :str     	=    flatten
						lowerCAmelCase_       :Union[str, Any]     	=    layer_norm

						lowerCAmelCase_       :Union[str, Any]     	=    nn.Convad(
						    __A			,  __A			,  kernel_size=(patch_size, patch_size)			,  stride=__A			,  bias=__A			)
						if layer_norm:
									lowerCAmelCase_       :Any     	=    nn.LayerNorm(__A			,  elementwise_affine=__A			,  eps=1E-6			)
						else:
									lowerCAmelCase_       :Union[str, Any]     	=    None

						lowerCAmelCase_       :str     	=    get_ad_sincos_pos_embed(__A			,  int(num_patches**0.5			)			)
						self.register_buffer("""pos_embed"""			,  torch.from_numpy(__A			).float().unsqueeze(0			)			,  persistent=__A			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	int:
						lowerCAmelCase_       :Optional[int]     	=    self.proj(__A			)
						if self.flatten:
									lowerCAmelCase_       :Optional[Any]     	=    latent.flatten(2			).transpose(1			,  2			)  # BCHW -> BNC
						if self.layer_norm:
									lowerCAmelCase_       :List[str]     	=    self.norm(__A			)
						return latent + self.pos_embed




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A			,  __A			,  __A = "silu"			,  __A = None			,  __A = None			,  __A=None			,  )     ->	List[str]:
						super().__init__()

						lowerCAmelCase_       :int     	=    nn.Linear(__A			,  __A			)

						if cond_proj_dim is not None:
									lowerCAmelCase_       :int     	=    nn.Linear(__A			,  __A			,  bias=__A			)
						else:
									lowerCAmelCase_       :int     	=    None

						lowerCAmelCase_       :Tuple     	=    get_activation(__A			)

						if out_dim is not None:
									lowerCAmelCase_       :Dict     	=    out_dim
						else:
									lowerCAmelCase_       :List[Any]     	=    time_embed_dim
						lowerCAmelCase_       :str     	=    nn.Linear(__A			,  __A			)

						if post_act_fn is None:
									lowerCAmelCase_       :Optional[Any]     	=    None
						else:
									lowerCAmelCase_       :Optional[int]     	=    get_activation(__A			)






			def 					__lowerCAmelCase					(   self			,  __A			,  __A=None			)     ->	Optional[Any]:
						if condition is not None:
									lowerCAmelCase_       :Union[str, Any]     	=    sample + self.cond_proj(__A			)
						lowerCAmelCase_       :Union[str, Any]     	=    self.linear_a(__A			)

						if self.act is not None:
									lowerCAmelCase_       :List[str]     	=    self.act(__A			)

						lowerCAmelCase_       :Optional[Any]     	=    self.linear_a(__A			)

						if self.post_act is not None:
									lowerCAmelCase_       :Dict     	=    self.post_act(__A			)
						return sample




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A			,  __A			,  __A			)     ->	str:
						super().__init__()
						lowerCAmelCase_       :List[str]     	=    num_channels
						lowerCAmelCase_       :Dict     	=    flip_sin_to_cos
						lowerCAmelCase_       :int     	=    downscale_freq_shift






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	List[Any]:
						lowerCAmelCase_       :Any     	=    get_timestep_embedding(
						    __A			,  self.num_channels			,  flip_sin_to_cos=self.flip_sin_to_cos			,  downscale_freq_shift=self.downscale_freq_shift			,  )
						return t_emb




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A = 256			,  __A = 1.0			,  __A=True			,  __A=True			,  __A=False			)     ->	List[Any]:
						super().__init__()
						lowerCAmelCase_       :List[str]     	=    nn.Parameter(torch.randn(__A			) * scale			,  requires_grad=__A			)
						lowerCAmelCase_       :List[str]     	=    log
						lowerCAmelCase_       :Union[str, Any]     	=    flip_sin_to_cos

						if set_W_to_weight:
									# to delete later
									lowerCAmelCase_       :List[Any]     	=    nn.Parameter(torch.randn(__A			) * scale			,  requires_grad=__A			)

									lowerCAmelCase_       :Union[str, Any]     	=    self.W






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	List[Any]:
						if self.log:
									lowerCAmelCase_       :Optional[Any]     	=    torch.log(__A			)

						lowerCAmelCase_       :Tuple     	=    x[:, None] * self.weight[None, :] * 2 * np.pi

						if self.flip_sin_to_cos:
									lowerCAmelCase_       :Tuple     	=    torch.cat([torch.cos(__A			), torch.sin(__A			)]			,  dim=-1			)
						else:
									lowerCAmelCase_       :str     	=    torch.cat([torch.sin(__A			), torch.cos(__A			)]			,  dim=-1			)
						return out




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A			,  __A			,  __A			,  __A			,  )     ->	List[str]:
						super().__init__()

						lowerCAmelCase_       :int     	=    height
						lowerCAmelCase_       :Dict     	=    width
						lowerCAmelCase_       :Optional[Any]     	=    num_embed
						lowerCAmelCase_       :List[str]     	=    embed_dim

						lowerCAmelCase_       :str     	=    nn.Embedding(self.num_embed			,  __A			)
						lowerCAmelCase_       :List[Any]     	=    nn.Embedding(self.height			,  __A			)
						lowerCAmelCase_       :Tuple     	=    nn.Embedding(self.width			,  __A			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Tuple:
						lowerCAmelCase_       :List[Any]     	=    self.emb(__A			)

						lowerCAmelCase_       :List[str]     	=    self.height_emb(torch.arange(self.height			,  device=index.device			).view(1			,  self.height			)			)

						# 1 x H x D -> 1 x H x 1 x D
						lowerCAmelCase_       :List[Any]     	=    height_emb.unsqueeze(2			)

						lowerCAmelCase_       :Optional[int]     	=    self.width_emb(torch.arange(self.width			,  device=index.device			).view(1			,  self.width			)			)

						# 1 x W x D -> 1 x 1 x W x D
						lowerCAmelCase_       :str     	=    width_emb.unsqueeze(1			)

						lowerCAmelCase_       :Dict     	=    height_emb + width_emb

						# 1 x H x W x D -> 1 x L xD
						lowerCAmelCase_       :Optional[int]     	=    pos_emb.view(1			,  self.height * self.width			,  -1			)

						lowerCAmelCase_       :Tuple     	=    emb + pos_emb[:, : emb.shape[1], :]

						return emb




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A			,  __A			,  __A			)     ->	List[str]:
						super().__init__()
						lowerCAmelCase_       :Dict     	=    dropout_prob > 0
						lowerCAmelCase_       :Any     	=    nn.Embedding(num_classes + use_cfg_embedding			,  __A			)
						lowerCAmelCase_       :int     	=    num_classes
						lowerCAmelCase_       :Union[str, Any]     	=    dropout_prob






			def 					__lowerCAmelCase					(   self			,  __A			,  __A=None			)     ->	Optional[int]:
						if force_drop_ids is None:
									lowerCAmelCase_       :List[str]     	=    torch.rand(labels.shape[0]			,  device=labels.device			) < self.dropout_prob
						else:
									lowerCAmelCase_       :Union[str, Any]     	=    torch.tensor(force_drop_ids == 1			)
						lowerCAmelCase_       :int     	=    torch.where(__A			,  self.num_classes			,  __A			)
						return labels






			def 					__lowerCAmelCase					(   self			,  __A			,  __A=None			)     ->	List[str]:
						lowerCAmelCase_       :List[Any]     	=    self.dropout_prob > 0
						if (self.training and use_dropout) or (force_drop_ids is not None):
									lowerCAmelCase_       :int     	=    self.token_drop(__A			,  __A			)
						lowerCAmelCase_       :int     	=    self.embedding_table(__A			)
						return embeddings




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A = 1024			,  __A = 768			,  __A = 768			,  __A = 10			,  )     ->	Dict:
						super().__init__()

						lowerCAmelCase_       :Dict     	=    num_image_text_embeds
						lowerCAmelCase_       :Tuple     	=    nn.Linear(__A			,  self.num_image_text_embeds * cross_attention_dim			)
						lowerCAmelCase_       :List[Any]     	=    nn.Linear(__A			,  __A			)






			def 					__lowerCAmelCase					(   self			,  __A			,  __A			)     ->	List[Any]:
						lowerCAmelCase_       :Optional[int]     	=    text_embeds.shape[0]

						# image
						lowerCAmelCase_       :int     	=    self.image_embeds(__A			)
						lowerCAmelCase_       :Tuple     	=    image_text_embeds.reshape(__A			,  self.num_image_text_embeds			,  -1			)

						# text
						lowerCAmelCase_       :List[Any]     	=    self.text_proj(__A			)

						return torch.cat([image_text_embeds, text_embeds]			,  dim=1			)




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A = 768			,  __A = 768			,  __A = 32			,  )     ->	Dict:
						super().__init__()

						lowerCAmelCase_       :Union[str, Any]     	=    num_image_text_embeds
						lowerCAmelCase_       :List[Any]     	=    nn.Linear(__A			,  self.num_image_text_embeds * cross_attention_dim			)
						lowerCAmelCase_       :Any     	=    nn.LayerNorm(__A			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Optional[int]:
						lowerCAmelCase_       :List[Any]     	=    image_embeds.shape[0]

						# image
						lowerCAmelCase_       :Optional[Any]     	=    self.image_embeds(__A			)
						lowerCAmelCase_       :Tuple     	=    image_embeds.reshape(__A			,  self.num_image_text_embeds			,  -1			)
						lowerCAmelCase_       :Optional[int]     	=    self.norm(__A			)
						return image_embeds




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A			,  __A			,  __A=0.1			)     ->	Dict:
						super().__init__()

						lowerCAmelCase_       :Tuple     	=    Timesteps(num_channels=256			,  flip_sin_to_cos=__A			,  downscale_freq_shift=1			)
						lowerCAmelCase_       :Optional[Any]     	=    TimestepEmbedding(in_channels=256			,  time_embed_dim=__A			)
						lowerCAmelCase_       :str     	=    LabelEmbedding(__A			,  __A			,  __A			)






			def 					__lowerCAmelCase					(   self			,  __A			,  __A			,  __A=None			)     ->	int:
						lowerCAmelCase_       :int     	=    self.time_proj(__A			)
						lowerCAmelCase_       :Any     	=    self.timestep_embedder(timesteps_proj.to(dtype=__A			)			)  # (N, D)

						lowerCAmelCase_       :int     	=    self.class_embedder(__A			)  # (N, D)

						lowerCAmelCase_       :Optional[Any]     	=    timesteps_emb + class_labels  # (N, D)

						return conditioning




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A			,  __A			,  __A = 64			)     ->	Optional[Any]:
						super().__init__()
						lowerCAmelCase_       :List[str]     	=    nn.LayerNorm(__A			)
						lowerCAmelCase_       :int     	=    AttentionPooling(__A			,  __A			)
						lowerCAmelCase_       :Optional[int]     	=    nn.Linear(__A			,  __A			)
						lowerCAmelCase_       :Union[str, Any]     	=    nn.LayerNorm(__A			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Any:
						lowerCAmelCase_       :str     	=    self.norma(__A			)
						lowerCAmelCase_       :str     	=    self.pool(__A			)
						lowerCAmelCase_       :List[str]     	=    self.proj(__A			)
						lowerCAmelCase_       :Optional[Any]     	=    self.norma(__A			)
						return hidden_states




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A = 768			,  __A = 768			,  __A = 1536			)     ->	Optional[Any]:
						super().__init__()
						lowerCAmelCase_       :List[Any]     	=    nn.Linear(__A			,  __A			)
						lowerCAmelCase_       :Any     	=    nn.LayerNorm(__A			)
						lowerCAmelCase_       :str     	=    nn.Linear(__A			,  __A			)






			def 					__lowerCAmelCase					(   self			,  __A			,  __A			)     ->	Optional[int]:
						# text
						lowerCAmelCase_       :Any     	=    self.text_proj(__A			)
						lowerCAmelCase_       :int     	=    self.text_norm(__A			)

						# image
						lowerCAmelCase_       :Optional[int]     	=    self.image_proj(__A			)

						return time_image_embeds + time_text_embeds




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A = 768			,  __A = 1536			)     ->	str:
						super().__init__()
						lowerCAmelCase_       :Union[str, Any]     	=    nn.Linear(__A			,  __A			)
						lowerCAmelCase_       :Any     	=    nn.LayerNorm(__A			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Tuple:
						# image
						lowerCAmelCase_       :Optional[int]     	=    self.image_proj(__A			)
						lowerCAmelCase_       :Optional[int]     	=    self.image_norm(__A			)
						return time_image_embeds




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A = 768			,  __A = 1536			)     ->	str:
						super().__init__()
						lowerCAmelCase_       :Optional[Any]     	=    nn.Linear(__A			,  __A			)
						lowerCAmelCase_       :int     	=    nn.LayerNorm(__A			)
						lowerCAmelCase_       :Any     	=    nn.Sequential(
						    nn.Convad(3			,  16			,  3			,  padding=1			)			,  nn.SiLU()			,  nn.Convad(16			,  16			,  3			,  padding=1			)			,  nn.SiLU()			,  nn.Convad(16			,  32			,  3			,  padding=1			,  stride=2			)			,  nn.SiLU()			,  nn.Convad(32			,  32			,  3			,  padding=1			)			,  nn.SiLU()			,  nn.Convad(32			,  96			,  3			,  padding=1			,  stride=2			)			,  nn.SiLU()			,  nn.Convad(96			,  96			,  3			,  padding=1			)			,  nn.SiLU()			,  nn.Convad(96			,  256			,  3			,  padding=1			,  stride=2			)			,  nn.SiLU()			,  nn.Convad(256			,  4			,  3			,  padding=1			)			,  )






			def 					__lowerCAmelCase					(   self			,  __A			,  __A			)     ->	List[Any]:
						# image
						lowerCAmelCase_       :str     	=    self.image_proj(__A			)
						lowerCAmelCase_       :Any     	=    self.image_norm(__A			)
						lowerCAmelCase_       :List[Any]     	=    self.input_hint_block(__A			)
						return time_image_embeds, hint




class      _SCREAMING_SNAKE_CASE   (     nn.Module				):






			def __init__(   self			,  __A			,  __A			,  __A=None			)     ->	Tuple:
						super().__init__()
						lowerCAmelCase_       :Dict     	=    dtype
						lowerCAmelCase_       :List[Any]     	=    nn.Parameter(torch.randn(1			,  __A			) / embed_dim**0.5			)
						lowerCAmelCase_       :str     	=    nn.Linear(__A			,  __A			,  dtype=self.dtype			)
						lowerCAmelCase_       :int     	=    nn.Linear(__A			,  __A			,  dtype=self.dtype			)
						lowerCAmelCase_       :Union[str, Any]     	=    nn.Linear(__A			,  __A			,  dtype=self.dtype			)
						lowerCAmelCase_       :str     	=    num_heads
						lowerCAmelCase_       :Optional[int]     	=    embed_dim // self.num_heads






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Optional[Any]:
						lowerCAmelCase_	,			lowerCAmelCase_	,			lowerCAmelCase_       :List[str]     	=    x.size()

						def shape(__A			):
									# (bs, length, width) --> (bs, length, n_heads, dim_per_head)
									lowerCAmelCase_       :Union[str, Any]     	=    x.view(__A			,  -1			,  self.num_heads			,  self.dim_per_head			)
									# (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
									lowerCAmelCase_       :Optional[int]     	=    x.transpose(1			,  2			)
									# (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
									lowerCAmelCase_       :Union[str, Any]     	=    x.reshape(bs * self.num_heads			,  -1			,  self.dim_per_head			)
									# (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
									lowerCAmelCase_       :Tuple     	=    x.transpose(1			,  2			)
									return x

						lowerCAmelCase_       :Optional[int]     	=    x.mean(dim=1			,  keepdim=__A			) + self.positional_embedding.to(x.dtype			)
						lowerCAmelCase_       :Union[str, Any]     	=    torch.cat([class_token, x]			,  dim=1			)  # (bs, length+1, width)

						# (bs*n_heads, class_token_length, dim_per_head)
						lowerCAmelCase_       :Optional[Any]     	=    shape(self.q_proj(__A			)			)
						# (bs*n_heads, length+class_token_length, dim_per_head)
						lowerCAmelCase_       :Dict     	=    shape(self.k_proj(__A			)			)
						lowerCAmelCase_       :Any     	=    shape(self.v_proj(__A			)			)

						# (bs*n_heads, class_token_length, length+class_token_length):
						lowerCAmelCase_       :Optional[int]     	=    1 / math.sqrt(math.sqrt(self.dim_per_head			)			)
						lowerCAmelCase_       :str     	=    torch.einsum("""bct,bcs->bts"""			,  q * scale			,  k * scale			)  # More stable with f16 than dividing afterwards
						lowerCAmelCase_       :int     	=    torch.softmax(weight.float()			,  dim=-1			).type(weight.dtype			)

						# (bs*n_heads, dim_per_head, class_token_length)
						lowerCAmelCase_       :Optional[int]     	=    torch.einsum("""bts,bcs->bct"""			,  __A			,  __A			)

						# (bs, length+1, width)
						lowerCAmelCase_       :List[str]     	=    a.reshape(__A			,  -1			,  1			).transpose(1			,  2			)

						return a[:, 0, :]  # cls_token








"""simple docstring"""






import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class      _SCREAMING_SNAKE_CASE   (     A__      ,					unittest.TestCase				):
			UpperCAmelCase_    :List[str]												=		GPTNeoXJapaneseTokenizer
			UpperCAmelCase_    :List[str]												=		False
			UpperCAmelCase_    :int												=		{"do_clean_text": False, "add_prefix_space": False}






			def 					__lowerCAmelCase					(   self			)     ->	Optional[Any]:
						super().setUp()

						lowerCAmelCase_       :int     	=    [
						    """こん""",
						    """こんに""",
						    """にちは""",
						    """ばんは""",
						    """世界,㔺界""",
						    """、""",
						    """。""",
						    """<BR>""",
						    """<SP>""",
						    """<TAB>""",
						    """<URL>""",
						    """<EMAIL>""",
						    """<TEL>""",
						    """<DATE>""",
						    """<PRICE>""",
						    """<BLOCK>""",
						    """<KIGOU>""",
						    """<U2000U2BFF>""",
						    """<|emoji1|>""",
						    """<unk>""",
						    """<|startoftext|>""",
						    """<|endoftext|>""",
						]
						lowerCAmelCase_       :str     	=    {"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
						lowerCAmelCase_       :Dict     	=    {"""unk_token""": """<unk>"""}

						lowerCAmelCase_       :Union[str, Any]     	=    os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["""vocab_file"""]			)
						lowerCAmelCase_       :Union[str, Any]     	=    os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["""emoji_file"""]			)
						with open(self.vocab_file			,  """w"""			,  encoding="""utf-8"""			) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)
						with open(self.emoji_file			,  """w"""			) as emoji_writer:
									emoji_writer.write(json.dumps(__A			)			)






			def 					__lowerCAmelCase					(   self			,  **__A			)     ->	str:
						kwargs.update(self.special_tokens_map			)
						return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname			,  **__A			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	List[Any]:
						lowerCAmelCase_       :Optional[Any]     	=    """こんにちは、世界。 \nこんばんは、㔺界。😀"""
						lowerCAmelCase_       :List[Any]     	=    """こんにちは、世界。 \nこんばんは、世界。😀"""
						return input_text, output_text






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Optional[Any]:
						lowerCAmelCase_	,			lowerCAmelCase_       :Optional[int]     	=    self.get_input_output_texts(__A			)
						lowerCAmelCase_       :List[Any]     	=    tokenizer.encode(__A			,  add_special_tokens=__A			)
						lowerCAmelCase_       :Optional[Any]     	=    tokenizer.decode(__A			,  clean_up_tokenization_spaces=__A			)
						return text, ids






			def 					__lowerCAmelCase					(   self			)     ->	str:
						pass  # TODO add if relevant






			def 					__lowerCAmelCase					(   self			)     ->	Any:
						pass  # TODO add if relevant






			def 					__lowerCAmelCase					(   self			)     ->	Optional[int]:
						pass  # TODO add if relevant






			def 					__lowerCAmelCase					(   self			)     ->	Optional[Any]:
						lowerCAmelCase_       :int     	=    self.get_tokenizer()

						# Testing tokenization
						lowerCAmelCase_       :List[str]     	=    """こんにちは、世界。　こんばんは、㔺界。"""
						lowerCAmelCase_       :int     	=    ["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
						lowerCAmelCase_       :List[Any]     	=    tokenizer.tokenize(__A			)
						self.assertListEqual(__A			,  __A			)

						# Testing conversion to ids without special tokens
						lowerCAmelCase_       :List[str]     	=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
						lowerCAmelCase_       :Dict     	=    tokenizer.convert_tokens_to_ids(__A			)
						self.assertListEqual(__A			,  __A			)

						# Testing conversion to ids with special tokens
						lowerCAmelCase_       :Tuple     	=    tokens + [tokenizer.unk_token]
						lowerCAmelCase_       :str     	=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
						lowerCAmelCase_       :Any     	=    tokenizer.convert_tokens_to_ids(__A			)
						self.assertListEqual(__A			,  __A			)






			@slow
			def 					__lowerCAmelCase					(   self			)     ->	Any:
						lowerCAmelCase_       :str     	=    self.tokenizer_class.from_pretrained("""abeja/gpt-neox-japanese-2.7b"""			)

						lowerCAmelCase_       :Union[str, Any]     	=    tokenizer.encode("""ありがとう。"""			,  add_special_tokens=__A			)
						lowerCAmelCase_       :Dict     	=    tokenizer.encode("""どういたしまして。"""			,  add_special_tokens=__A			)

						lowerCAmelCase_       :Union[str, Any]     	=    tokenizer.build_inputs_with_special_tokens(__A			)
						lowerCAmelCase_       :Optional[int]     	=    tokenizer.build_inputs_with_special_tokens(__A			,  __A			)

						assert encoded_sentence == ids_a
						assert encoded_pair == ids_a + ids_a






			def 					__lowerCAmelCase					(   self			)     ->	Dict:
						# Intentionally convert some words to accommodate character fluctuations unique to Japanese
						pass






			def 					__lowerCAmelCase					(   self			)     ->	List[str]:
						# tokenizer has no padding token
						pass






