






import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
		import torch

if is_vision_available():
		from PIL import Image

		from transformers import MobileNetVaImageProcessor





class __lowerCamelCase					( unittest.TestCase):





	"""simple docstring"""






	def __init__(				self			,       UpperCAmelCase			,       UpperCAmelCase=7			,       UpperCAmelCase=3			,       UpperCAmelCase=18			,       UpperCAmelCase=30			,       UpperCAmelCase=400			,       UpperCAmelCase=True			,       UpperCAmelCase=None			,       UpperCAmelCase=True			,       UpperCAmelCase=None			,       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     size if size is not None else {'shortest_edge': 20}
							_UpperCAmelCase							   =     crop_size if crop_size is not None else {'height': 18, 'width': 18}
							_UpperCAmelCase							   =     parent
							_UpperCAmelCase							   =     batch_size
							_UpperCAmelCase							   =     num_channels
							_UpperCAmelCase							   =     image_size
							_UpperCAmelCase							   =     min_resolution
							_UpperCAmelCase							   =     max_resolution
							_UpperCAmelCase							   =     do_resize
							_UpperCAmelCase							   =     size
							_UpperCAmelCase							   =     do_center_crop
							_UpperCAmelCase							   =     crop_size

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							return {
							    "do_resize": self.do_resize,
							    "size": self.size,
							    "do_center_crop": self.do_center_crop,
							    "crop_size": self.crop_size,
							}





@require_torch
@require_vision
class __lowerCamelCase					( snake_case__	,			unittest.TestCase):





	"""simple docstring"""






	UpperCamelCase__								= MobileNetVaImageProcessor if is_vision_available() else None

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     MobileNetVaImageProcessingTester(self       )

	@property
	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							return self.image_processor_tester.prepare_image_processor_dict()

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.image_processing_class(**self.image_processor_dict       )
							self.assertTrue(hasattr(UpperCAmelCase			,       'do_resize'       )       )
							self.assertTrue(hasattr(UpperCAmelCase			,       'size'       )       )
							self.assertTrue(hasattr(UpperCAmelCase			,       'do_center_crop'       )       )
							self.assertTrue(hasattr(UpperCAmelCase			,       'center_crop'       )       )

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.image_processing_class.from_dict(self.image_processor_dict       )
							self.assertEqual(image_processor.size			,       {'shortest_edge': 20}       )
							self.assertEqual(image_processor.crop_size			,       {'height': 18, 'width': 18}       )

							_UpperCAmelCase							   =     self.image_processing_class.from_dict(self.image_processor_dict			,       size=42			,       crop_size=84       )
							self.assertEqual(image_processor.size			,       {'shortest_edge': 42}       )
							self.assertEqual(image_processor.crop_size			,       {'height': 84, 'width': 84}       )

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.image_processing_class(**self.image_processor_dict       )
							# create random PIL images
							_UpperCAmelCase							   =     prepare_image_inputs(self.image_processor_tester			,       equal_resolution=UpperCAmelCase       )
							for image in image_inputs:
													self.assertIsInstance(UpperCAmelCase			,       Image.Image       )

							# Test not batched input
							_UpperCAmelCase							   =     image_processing(image_inputs[0]			,       return_tensors='pt'       ).pixel_values
							self.assertEqual(
							    encoded_images.shape			,       (
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,       )

							# Test batched
							_UpperCAmelCase							   =     image_processing(UpperCAmelCase			,       return_tensors='pt'       ).pixel_values
							self.assertEqual(
							    encoded_images.shape			,       (
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,       )

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.image_processing_class(**self.image_processor_dict       )
							# create random numpy tensors
							_UpperCAmelCase							   =     prepare_image_inputs(self.image_processor_tester			,       equal_resolution=UpperCAmelCase			,       numpify=UpperCAmelCase       )
							for image in image_inputs:
													self.assertIsInstance(UpperCAmelCase			,       np.ndarray       )

							# Test not batched input
							_UpperCAmelCase							   =     image_processing(image_inputs[0]			,       return_tensors='pt'       ).pixel_values
							self.assertEqual(
							    encoded_images.shape			,       (
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,       )

							# Test batched
							_UpperCAmelCase							   =     image_processing(UpperCAmelCase			,       return_tensors='pt'       ).pixel_values
							self.assertEqual(
							    encoded_images.shape			,       (
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,       )

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.image_processing_class(**self.image_processor_dict       )
							# create random PyTorch tensors
							_UpperCAmelCase							   =     prepare_image_inputs(self.image_processor_tester			,       equal_resolution=UpperCAmelCase			,       torchify=UpperCAmelCase       )
							for image in image_inputs:
													self.assertIsInstance(UpperCAmelCase			,       torch.Tensor       )

							# Test not batched input
							_UpperCAmelCase							   =     image_processing(image_inputs[0]			,       return_tensors='pt'       ).pixel_values
							self.assertEqual(
							    encoded_images.shape			,       (
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,       )

							# Test batched
							_UpperCAmelCase							   =     image_processing(UpperCAmelCase			,       return_tensors='pt'       ).pixel_values
							self.assertEqual(
							    encoded_images.shape			,       (
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size['height'],
							        self.image_processor_tester.crop_size['width'],
							    )			,       )








import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class __lowerCamelCase					( snake_case__	,			unittest.TestCase):





	"""simple docstring"""






	UpperCamelCase__								= GPTNeoXJapaneseTokenizer
	UpperCamelCase__								= False
	UpperCamelCase__								= {"do_clean_text": False, "add_prefix_space": False}

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							super().setUp()

							_UpperCAmelCase							   =     [
							    'こん',
							    'こんに',
							    'にちは',
							    'ばんは',
							    '世界,㔺界',
							    '、',
							    '。',
							    '<BR>',
							    '<SP>',
							    '<TAB>',
							    '<URL>',
							    '<EMAIL>',
							    '<TEL>',
							    '<DATE>',
							    '<PRICE>',
							    '<BLOCK>',
							    '<KIGOU>',
							    '<U2000U2BFF>',
							    '<|emoji1|>',
							    '<unk>',
							    '<|startoftext|>',
							    '<|endoftext|>',
							]
							_UpperCAmelCase							   =     {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
							_UpperCAmelCase							   =     {'unk_token': '<unk>'}

							_UpperCAmelCase							   =     os.path.join(self.tmpdirname			,       VOCAB_FILES_NAMES['vocab_file']       )
							_UpperCAmelCase							   =     os.path.join(self.tmpdirname			,       VOCAB_FILES_NAMES['emoji_file']       )
							with open(self.vocab_file			,       'w'			,       encoding='utf-8'       ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )
							with open(self.emoji_file			,       'w'       ) as emoji_writer:
													emoji_writer.write(json.dumps(UpperCAmelCase       )       )

	def 			UpperCamelCase       (				self			,       **UpperCAmelCase       ):
							"""simple docstring"""







							kwargs.update(self.special_tokens_map       )
							return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname			,       **UpperCAmelCase       )

	def 			UpperCamelCase       (				self			,       UpperCAmelCase       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     'こんにちは、世界。 \nこんばんは、㔺界。😀'
							_UpperCAmelCase							   =     'こんにちは、世界。 \nこんばんは、世界。😀'
							return input_text, output_text

	def 			UpperCamelCase       (				self			,       UpperCAmelCase       ):
							"""simple docstring"""







							_UpperCAmelCase   ,					_UpperCAmelCase							   =     self.get_input_output_texts(UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.encode(UpperCAmelCase			,       add_special_tokens=UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.decode(UpperCAmelCase			,       clean_up_tokenization_spaces=UpperCAmelCase       )
							return text, ids

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass  # TODO add if relevant

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass  # TODO add if relevant

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass  # TODO add if relevant

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.get_tokenizer()

							# Testing tokenization
							_UpperCAmelCase							   =     'こんにちは、世界。　こんばんは、㔺界。'
							_UpperCAmelCase							   =     ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
							_UpperCAmelCase							   =     tokenizer.tokenize(UpperCAmelCase       )
							self.assertListEqual(UpperCAmelCase			,       UpperCAmelCase       )

							# Testing conversion to ids without special tokens
							_UpperCAmelCase							   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
							_UpperCAmelCase							   =     tokenizer.convert_tokens_to_ids(UpperCAmelCase       )
							self.assertListEqual(UpperCAmelCase			,       UpperCAmelCase       )

							# Testing conversion to ids with special tokens
							_UpperCAmelCase							   =     tokens + [tokenizer.unk_token]
							_UpperCAmelCase							   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
							_UpperCAmelCase							   =     tokenizer.convert_tokens_to_ids(UpperCAmelCase       )
							self.assertListEqual(UpperCAmelCase			,       UpperCAmelCase       )

	@slow
	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							_UpperCAmelCase							   =     self.tokenizer_class.from_pretrained('abeja/gpt-neox-japanese-2.7b'       )

							_UpperCAmelCase							   =     tokenizer.encode('ありがとう。'			,       add_special_tokens=UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.encode('どういたしまして。'			,       add_special_tokens=UpperCAmelCase       )

							_UpperCAmelCase							   =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase       )
							_UpperCAmelCase							   =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase			,       UpperCAmelCase       )

							assert encoded_sentence == ids_a
							assert encoded_pair == ids_a + ids_a

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass

	def 			UpperCamelCase       (				self       ):
							"""simple docstring"""







							pass
