
'''simple docstring'''







import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 					lowerCAmelCase__  (    lowerCamelCase_       ,		unittest.TestCase	):
					lowerCAmelCase_	  =	HerbertTokenizer
					lowerCAmelCase_	  =	HerbertTokenizerFast
					lowerCAmelCase_	  =	True






					def       _snake_case       (  self						):


											"""simple docstring"""





											super().setUp()

											# Use a simpler test file without japanese/chinese characters
											with open(F'''{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'''							,				encoding='''utf-8'''						) as f_data:
																	lowercase_      :	Optional[int]						=			f_data.read().replace('''\n\n'''							,				'''\n'''						).strip()

											lowercase_      :	str						=			[
											    '''<s>''',
											    '''</s>''',
											    '''l''',
											    '''o''',
											    '''w''',
											    '''e''',
											    '''r''',
											    '''s''',
											    '''t''',
											    '''i''',
											    '''d''',
											    '''n''',
											    '''w</w>''',
											    '''r</w>''',
											    '''t</w>''',
											    '''lo''',
											    '''low''',
											    '''er</w>''',
											    '''low</w>''',
											    '''lowest</w>''',
											    '''newer</w>''',
											    '''wider</w>''',
											    ''',</w>''',
											    '''<unk>''',
											]
											lowercase_      :	Optional[Any]						=			dict(zip(__SCREAMING_SNAKE_CASE							,				range(len(__SCREAMING_SNAKE_CASE						)						)						)						)
											lowercase_      :	Any						=			['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

											lowercase_      :	Union[str, Any]						=			os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES['''vocab_file''']						)
											lowercase_      :	str						=			os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES['''merges_file''']						)
											with open(self.vocab_file							,				'''w'''						) as fp:
																	fp.write(json.dumps(__SCREAMING_SNAKE_CASE						)						)
											with open(self.merges_file							,				'''w'''						) as fp:
																	fp.write('''\n'''.join(__SCREAMING_SNAKE_CASE						)						)






					def       _snake_case       (  self							,				__SCREAMING_SNAKE_CASE						):


											"""simple docstring"""





											lowercase_      :	Optional[int]						=			'''lower newer'''
											lowercase_      :	List[Any]						=			'''lower newer'''
											return input_text, output_text






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	Tuple						=			self.tokenizer_class(vocab_file=self.vocab_file							,				merges_file=self.merges_file						)

											lowercase_      :	Optional[Any]						=			'''lower'''
											lowercase_      :	List[Any]						=			['''low''', '''er</w>''']
											lowercase_      :	Union[str, Any]						=			tokenizer.tokenize(__SCREAMING_SNAKE_CASE						)
											self.assertListEqual(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						)

											lowercase_      :	Any						=			tokens + ['''<unk>''']
											lowercase_      :	List[str]						=			[16, 17, 23]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE						)							,				__SCREAMING_SNAKE_CASE						)






					def       _snake_case       (  self						):


											"""simple docstring"""





											if not self.test_rust_tokenizer:
																	return

											lowercase_      :	List[Any]						=			self.get_tokenizer()
											lowercase_      :	List[Any]						=			self.get_rust_tokenizer()

											lowercase_      :	Union[str, Any]						=			'''lower,newer'''

											lowercase_      :	str						=			tokenizer.tokenize(__SCREAMING_SNAKE_CASE						)
											lowercase_      :	Optional[Any]						=			rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE						)
											self.assertListEqual(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						)

											lowercase_      :	Optional[int]						=			tokenizer.encode(__SCREAMING_SNAKE_CASE							,				add_special_tokens=__SCREAMING_SNAKE_CASE						)
											lowercase_      :	Optional[int]						=			rust_tokenizer.encode(__SCREAMING_SNAKE_CASE							,				add_special_tokens=__SCREAMING_SNAKE_CASE						)
											self.assertListEqual(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						)

											lowercase_      :	str						=			self.get_rust_tokenizer()
											lowercase_      :	int						=			tokenizer.encode(__SCREAMING_SNAKE_CASE						)
											lowercase_      :	Dict						=			rust_tokenizer.encode(__SCREAMING_SNAKE_CASE						)
											self.assertListEqual(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						)






					@slow
					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	List[str]						=			self.tokenizer_class.from_pretrained('''allegro/herbert-base-cased'''						)

											lowercase_      :	Dict						=			tokenizer.encode('''konstruowanie sekwencji'''							,				add_special_tokens=__SCREAMING_SNAKE_CASE						)
											lowercase_      :	Optional[int]						=			tokenizer.encode('''konstruowanie wielu sekwencji'''							,				add_special_tokens=__SCREAMING_SNAKE_CASE						)

											lowercase_      :	Optional[int]						=			tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE						)
											lowercase_      :	Dict						=			tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE						)

											assert encoded_sentence == [0] + text + [2]
											assert encoded_pair == [0] + text + [2] + text_a + [2]





'''simple docstring'''







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
							import torch

if is_vision_available():
							from PIL import Image

							from transformers import DeiTImageProcessor






class 					lowerCAmelCase__  (    unittest.TestCase	):






					def __init__(  self							,				__SCREAMING_SNAKE_CASE							,				__SCREAMING_SNAKE_CASE=7							,				__SCREAMING_SNAKE_CASE=3							,				__SCREAMING_SNAKE_CASE=18							,				__SCREAMING_SNAKE_CASE=30							,				__SCREAMING_SNAKE_CASE=4_00							,				__SCREAMING_SNAKE_CASE=True							,				__SCREAMING_SNAKE_CASE=None							,				__SCREAMING_SNAKE_CASE=True							,				__SCREAMING_SNAKE_CASE=None							,				__SCREAMING_SNAKE_CASE=True							,				__SCREAMING_SNAKE_CASE=[0.5, 0.5, 0.5]							,				__SCREAMING_SNAKE_CASE=[0.5, 0.5, 0.5]							,				):


											"""simple docstring"""





											lowercase_      :	Union[str, Any]						=			size if size is not None else {'''height''': 20, '''width''': 20}
											lowercase_      :	List[Any]						=			crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}

											lowercase_      :	List[str]						=			parent
											lowercase_      :	Optional[Any]						=			batch_size
											lowercase_      :	int						=			num_channels
											lowercase_      :	List[str]						=			image_size
											lowercase_      :	Optional[Any]						=			min_resolution
											lowercase_      :	Tuple						=			max_resolution
											lowercase_      :	Union[str, Any]						=			do_resize
											lowercase_      :	List[str]						=			size
											lowercase_      :	Tuple						=			do_center_crop
											lowercase_      :	Dict						=			crop_size
											lowercase_      :	int						=			do_normalize
											lowercase_      :	Optional[int]						=			image_mean
											lowercase_      :	List[str]						=			image_std






					def       _snake_case       (  self						):


											"""simple docstring"""





											return {
											    "do_resize": self.do_resize,
											    "size": self.size,
											    "do_center_crop": self.do_center_crop,
											    "crop_size": self.crop_size,
											    "do_normalize": self.do_normalize,
											    "image_mean": self.image_mean,
											    "image_std": self.image_std,
											}







@require_torch
@require_vision
class 					lowerCAmelCase__  (    lowerCamelCase_       ,		unittest.TestCase	):
					lowerCAmelCase_	  =	DeiTImageProcessor if is_vision_available() else None
					lowerCAmelCase_	  =	True






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	List[str]						=			DeiTImageProcessingTester(self						)






					@property
					def       _snake_case       (  self						):


											"""simple docstring"""





											return self.image_processor_tester.prepare_image_processor_dict()






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	str						=			self.image_processing_class(**self.image_processor_dict						)
											self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE							,				'''do_resize'''						)						)
											self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE							,				'''size'''						)						)
											self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE							,				'''do_center_crop'''						)						)
											self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE							,				'''center_crop'''						)						)
											self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE							,				'''do_normalize'''						)						)
											self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE							,				'''image_mean'''						)						)
											self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE							,				'''image_std'''						)						)






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	List[str]						=			self.image_processing_class.from_dict(self.image_processor_dict						)
											self.assertEqual(image_processor.size							,				{'''height''': 20, '''width''': 20}						)
											self.assertEqual(image_processor.crop_size							,				{'''height''': 18, '''width''': 18}						)

											lowercase_      :	List[str]						=			self.image_processing_class.from_dict(self.image_processor_dict							,				size=42							,				crop_size=84						)
											self.assertEqual(image_processor.size							,				{'''height''': 42, '''width''': 42}						)
											self.assertEqual(image_processor.crop_size							,				{'''height''': 84, '''width''': 84}						)






					def       _snake_case       (  self						):


											"""simple docstring"""





											pass






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	Tuple						=			self.image_processing_class(**self.image_processor_dict						)
											# create random PIL images
											lowercase_      :	Optional[int]						=			prepare_image_inputs(self.image_processor_tester							,				equal_resolution=__SCREAMING_SNAKE_CASE						)
											for image in image_inputs:
																	self.assertIsInstance(__SCREAMING_SNAKE_CASE							,				Image.Image						)

											# Test not batched input
											lowercase_      :	List[Any]						=			image_processing(image_inputs[0]							,				return_tensors='''pt'''						).pixel_values
											self.assertEqual(
											    encoded_images.shape							,				(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['''height'''],
											        self.image_processor_tester.crop_size['''width'''],
											    )							,				)

											# Test batched
											lowercase_      :	List[str]						=			image_processing(__SCREAMING_SNAKE_CASE							,				return_tensors='''pt'''						).pixel_values
											self.assertEqual(
											    encoded_images.shape							,				(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['''height'''],
											        self.image_processor_tester.crop_size['''width'''],
											    )							,				)






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	List[Any]						=			self.image_processing_class(**self.image_processor_dict						)
											# create random numpy tensors
											lowercase_      :	Tuple						=			prepare_image_inputs(self.image_processor_tester							,				equal_resolution=__SCREAMING_SNAKE_CASE							,				numpify=__SCREAMING_SNAKE_CASE						)
											for image in image_inputs:
																	self.assertIsInstance(__SCREAMING_SNAKE_CASE							,				np.ndarray						)

											# Test not batched input
											lowercase_      :	Dict						=			image_processing(image_inputs[0]							,				return_tensors='''pt'''						).pixel_values
											self.assertEqual(
											    encoded_images.shape							,				(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['''height'''],
											        self.image_processor_tester.crop_size['''width'''],
											    )							,				)

											# Test batched
											lowercase_      :	Union[str, Any]						=			image_processing(__SCREAMING_SNAKE_CASE							,				return_tensors='''pt'''						).pixel_values
											self.assertEqual(
											    encoded_images.shape							,				(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['''height'''],
											        self.image_processor_tester.crop_size['''width'''],
											    )							,				)






					def       _snake_case       (  self						):


											"""simple docstring"""





											lowercase_      :	List[str]						=			self.image_processing_class(**self.image_processor_dict						)
											# create random PyTorch tensors
											lowercase_      :	List[Any]						=			prepare_image_inputs(self.image_processor_tester							,				equal_resolution=__SCREAMING_SNAKE_CASE							,				torchify=__SCREAMING_SNAKE_CASE						)
											for image in image_inputs:
																	self.assertIsInstance(__SCREAMING_SNAKE_CASE							,				torch.Tensor						)

											# Test not batched input
											lowercase_      :	List[str]						=			image_processing(image_inputs[0]							,				return_tensors='''pt'''						).pixel_values
											self.assertEqual(
											    encoded_images.shape							,				(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['''height'''],
											        self.image_processor_tester.crop_size['''width'''],
											    )							,				)

											# Test batched
											lowercase_      :	Optional[int]						=			image_processing(__SCREAMING_SNAKE_CASE							,				return_tensors='''pt'''						).pixel_values
											self.assertEqual(
											    encoded_images.shape							,				(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['''height'''],
											        self.image_processor_tester.crop_size['''width'''],
											    )							,				)



