






import json
import os
import re
import warnings
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a   			=						logging.get_logger(__name__)

_a   			=						{
    '''source_spm''': '''source.spm''',
    '''target_spm''': '''target.spm''',
    '''vocab''': '''vocab.json''',
    '''target_vocab_file''': '''target_vocab.json''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

_a   			=						{
    '''source_spm''': {
        '''Helsinki-NLP/opus-mt-en-de''': '''https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/source.spm'''
    },
    '''target_spm''': {
        '''Helsinki-NLP/opus-mt-en-de''': '''https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/target.spm'''
    },
    '''vocab''': {
        '''Helsinki-NLP/opus-mt-en-de''': '''https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/vocab.json'''
    },
    '''tokenizer_config_file''': {
        '''Helsinki-NLP/opus-mt-en-de''': (
            '''https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/tokenizer_config.json'''
        )
    },
}

_a   			=						{'''Helsinki-NLP/opus-mt-en-de''': 512}
_a   			=						{}





class __lowerCamelCase				(   snake_case__):





	"""simple docstring"""






	UpperCamelCase__			=   VOCAB_FILES_NAMES
	UpperCamelCase__			=   PRETRAINED_VOCAB_FILES_MAP
	UpperCamelCase__			=   PRETRAINED_INIT_CONFIGURATION
	UpperCamelCase__			=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCamelCase__			=   ["input_ids", "attention_mask"]
	UpperCamelCase__			=   re.compile(">>.+<<")  # type: re.Pattern

	def __init__(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase=None ,					UpperCAmelCase=None ,					UpperCAmelCase=None ,					UpperCAmelCase="<unk>" ,					UpperCAmelCase="</s>" ,					UpperCAmelCase="<pad>" ,					UpperCAmelCase=512 ,					UpperCAmelCase = None ,					UpperCAmelCase=False ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							_UpperCAmelCase             =    {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    # bos_token=bos_token,  unused. Start decoding with config.decoder_start_token_id
							    source_lang=UpperCAmelCase ,					target_lang=UpperCAmelCase ,					unk_token=UpperCAmelCase ,					eos_token=UpperCAmelCase ,					pad_token=UpperCAmelCase ,					model_max_length=UpperCAmelCase ,					sp_model_kwargs=self.sp_model_kwargs ,					target_vocab_file=UpperCAmelCase ,					separate_vocabs=UpperCAmelCase ,					**UpperCAmelCase ,					)
							assert Path(UpperCAmelCase					).exists(), F"""cannot find spm source {source_spm}"""

							_UpperCAmelCase             =    separate_vocabs
							_UpperCAmelCase             =    load_json(UpperCAmelCase					)
							if self.unk_token not in self.encoder:
													raise KeyError('<unk> token must be in vocab'					)
							assert self.pad_token in self.encoder

							if separate_vocabs:
													_UpperCAmelCase             =    load_json(UpperCAmelCase					)
													_UpperCAmelCase             =    {v: k for k, v in self.target_encoder.items()}
													_UpperCAmelCase             =    []
							else:
													_UpperCAmelCase             =    {v: k for k, v in self.encoder.items()}
													_UpperCAmelCase             =    [k for k in self.encoder if k.startswith('>>'					) and k.endswith('<<'					)]

							_UpperCAmelCase             =    source_lang
							_UpperCAmelCase             =    target_lang
							_UpperCAmelCase             =    [source_spm, target_spm]

							# load SentencePiece model for pre-processing
							_UpperCAmelCase             =    load_spm(UpperCAmelCase ,					self.sp_model_kwargs					)
							_UpperCAmelCase             =    load_spm(UpperCAmelCase ,					self.sp_model_kwargs					)
							_UpperCAmelCase             =    self.spm_source
							_UpperCAmelCase             =    self.encoder

							# Multilingual target side: default to using first supported language code.

							self._setup_normalizer()

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							try:
													from sacremoses import MosesPunctNormalizer

													_UpperCAmelCase             =    MosesPunctNormalizer(self.source_lang					).normalize
							except (ImportError, FileNotFoundError):
													warnings.warn('Recommended: pip install sacremoses.'					)
													_UpperCAmelCase             =    lambda UpperCAmelCase					: x

	def  UpperCamelCase					(							self ,					UpperCAmelCase					):
							"""simple docstring"""







							return self.punc_normalizer(UpperCAmelCase					) if x else ""

	def  UpperCamelCase					(							self ,					UpperCAmelCase					):
							"""simple docstring"""







							return self.current_encoder.get(UpperCAmelCase ,					self.current_encoder[self.unk_token]					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase					):
							"""simple docstring"""







							_UpperCAmelCase             =    self.language_code_re.match(UpperCAmelCase					)
							_UpperCAmelCase             =    [match.group(0					)] if match else []
							return code, self.language_code_re.sub('' ,					UpperCAmelCase					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase					):
							"""simple docstring"""







							_UpperCAmelCase      ,				_UpperCAmelCase             =    self.remove_language_code(UpperCAmelCase					)
							_UpperCAmelCase             =    self.current_spm.encode(UpperCAmelCase ,					out_type=UpperCAmelCase					)
							return code + pieces

	def  UpperCamelCase					(							self ,					UpperCAmelCase					):
							"""simple docstring"""







							return self.decoder.get(UpperCAmelCase ,					self.unk_token					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					**UpperCAmelCase					):
							"""simple docstring"""







							return super().batch_decode(UpperCAmelCase ,					**UpperCAmelCase					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					**UpperCAmelCase					):
							"""simple docstring"""







							return super().decode(UpperCAmelCase ,					**UpperCAmelCase					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase					):
							"""simple docstring"""







							_UpperCAmelCase             =    self.spm_source if self._decode_use_source_tokenizer else self.spm_target
							_UpperCAmelCase             =    []
							_UpperCAmelCase             =    ''
							for token in tokens:
													# make sure that special tokens are not decoded using sentencepiece model
													if token in self.all_special_tokens:
																			out_string += sp_model.decode_pieces(UpperCAmelCase					) + token + " "
																			_UpperCAmelCase             =    []
													else:
																			current_sub_tokens.append(UpperCAmelCase					)
							out_string += sp_model.decode_pieces(UpperCAmelCase					)
							return out_string.strip()

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase=None					):
							"""simple docstring"""







							if token_ids_a is None:
													return token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return token_ids_a + token_ids_a + [self.eos_token_id]

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							_UpperCAmelCase             =    self.spm_source
							_UpperCAmelCase             =    self.encoder

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							_UpperCAmelCase             =    self.spm_target
							if self.separate_vocabs:
													_UpperCAmelCase             =    self.target_encoder

	@property
	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							return len(self.encoder					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase = None					):
							"""simple docstring"""







							if not os.path.isdir(UpperCAmelCase					):
													logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""					)
													return
							_UpperCAmelCase             =    []

							if self.separate_vocabs:
													_UpperCAmelCase             =    os.path.join(
													    UpperCAmelCase ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab'] ,					)
													_UpperCAmelCase             =    os.path.join(
													    UpperCAmelCase ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['target_vocab_file'] ,					)
													save_json(self.encoder ,					UpperCAmelCase					)
													save_json(self.target_encoder ,					UpperCAmelCase					)
													saved_files.append(UpperCAmelCase					)
													saved_files.append(UpperCAmelCase					)
							else:
													_UpperCAmelCase             =    os.path.join(
													    UpperCAmelCase ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab']					)
													save_json(self.encoder ,					UpperCAmelCase					)
													saved_files.append(UpperCAmelCase					)

							for spm_save_filename, spm_orig_path, spm_model in zip(
							    [VOCAB_FILES_NAMES['source_spm'], VOCAB_FILES_NAMES['target_spm']] ,					self.spm_files ,					[self.spm_source, self.spm_target] ,					):
													_UpperCAmelCase             =    os.path.join(
													    UpperCAmelCase ,					(filename_prefix + '-' if filename_prefix else '') + spm_save_filename					)
													if os.path.abspath(UpperCAmelCase					) != os.path.abspath(UpperCAmelCase					) and os.path.isfile(UpperCAmelCase					):
																			copyfile(UpperCAmelCase ,					UpperCAmelCase					)
																			saved_files.append(UpperCAmelCase					)
													elif not os.path.isfile(UpperCAmelCase					):
																			with open(UpperCAmelCase ,					'wb'					) as fi:
																									_UpperCAmelCase             =    spm_model.serialized_model_proto()
																									fi.write(UpperCAmelCase					)
																			saved_files.append(UpperCAmelCase					)

							return tuple(UpperCAmelCase					)

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							return self.get_src_vocab()

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							return dict(self.encoder ,					**self.added_tokens_encoder					)

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							return dict(self.target_encoder ,					**self.added_tokens_decoder					)

	def __getstate__(							self					):
							"""simple docstring"""







							_UpperCAmelCase             =    self.__dict__.copy()
							state.update(
							    {k: None for k in ['spm_source', 'spm_target', 'current_spm', 'punc_normalizer', 'target_vocab_file']}					)
							return state

	def __setstate__(							self ,					UpperCAmelCase					):
							"""simple docstring"""







							_UpperCAmelCase             =    d

							# for backward compatibility
							if not hasattr(self ,					'sp_model_kwargs'					):
													_UpperCAmelCase             =    {}

							_UpperCAmelCase      ,				_UpperCAmelCase             =    (load_spm(UpperCAmelCase ,					self.sp_model_kwargs					) for f in self.spm_files)
							_UpperCAmelCase             =    self.spm_source
							self._setup_normalizer()

	def  UpperCamelCase					(							self ,					*UpperCAmelCase ,					**UpperCAmelCase					):
							"""simple docstring"""







							return 1

	def  UpperCamelCase					(							self ,					UpperCAmelCase					):
							"""simple docstring"""







							_UpperCAmelCase             =    set(self.all_special_ids					)  # call it once instead of inside list comp
							all_special_ids.remove(self.unk_token_id					)  # <unk> is only sometimes special
							return [1 if x in all_special_ids else 0 for x in seq]

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase = None ,					UpperCAmelCase = False					):
							"""simple docstring"""







							if already_has_special_tokens:
													return self._special_token_mask(UpperCAmelCase					)
							elif token_ids_a is None:
													return self._special_token_mask(UpperCAmelCase					) + [1]
							else:
													return self._special_token_mask(token_ids_a + token_ids_a					) + [1]



def __A					(   __lowerCAmelCase				,	__lowerCAmelCase			)->						sentencepiece.SentencePieceProcessor:





						"""simple docstring"""



						_UpperCAmelCase             =    sentencepiece.SentencePieceProcessor(**__lowerCAmelCase			)
						spm.Load(__lowerCAmelCase			)
						return spm


def __A					(   __lowerCAmelCase				,	__lowerCAmelCase			)->						None:





						"""simple docstring"""



						with open(__lowerCAmelCase				,	'w'			) as f:
												json.dump(__lowerCAmelCase				,	__lowerCAmelCase				,	indent=2			)


def __A					(   __lowerCAmelCase			)->						Union[Dict, List]:





						"""simple docstring"""



						with open(__lowerCAmelCase				,	'r'			) as f:
												return json.load(__lowerCAmelCase			)








from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


_a   			=						logging.get_logger(__name__)


if is_vision_available():
		import PIL





class __lowerCamelCase				(   snake_case__):





	"""simple docstring"""






	UpperCamelCase__			=   ["pixel_values"]

	def __init__(							self ,					UpperCAmelCase = True ,					UpperCAmelCase = None ,					UpperCAmelCase = PILImageResampling.BICUBIC ,					UpperCAmelCase = True ,					UpperCAmelCase = None ,					UpperCAmelCase = True ,					UpperCAmelCase = 1 / 255 ,					UpperCAmelCase = True ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = True ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							super().__init__(**UpperCAmelCase					)
							_UpperCAmelCase             =    size if size is not None else {'shortest_edge': 224}
							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase ,					default_to_square=UpperCAmelCase					)
							_UpperCAmelCase             =    crop_size if crop_size is not None else {'height': 224, 'width': 224}
							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase					)

							_UpperCAmelCase             =    do_resize
							_UpperCAmelCase             =    size
							_UpperCAmelCase             =    resample
							_UpperCAmelCase             =    do_center_crop
							_UpperCAmelCase             =    crop_size
							_UpperCAmelCase             =    do_rescale
							_UpperCAmelCase             =    rescale_factor
							_UpperCAmelCase             =    do_normalize
							_UpperCAmelCase             =    image_mean if image_mean is not None else OPENAI_CLIP_MEAN
							_UpperCAmelCase             =    image_std if image_std is not None else OPENAI_CLIP_STD
							_UpperCAmelCase             =    do_convert_rgb

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase = PILImageResampling.BICUBIC ,					UpperCAmelCase = None ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase ,					default_to_square=UpperCAmelCase					)
							_UpperCAmelCase             =    get_resize_output_image_size(
							    UpperCAmelCase ,					size=(size['height'], size['width']) ,					default_to_square=UpperCAmelCase					)
							return resize(UpperCAmelCase ,					size=UpperCAmelCase ,					resample=UpperCAmelCase ,					data_format=UpperCAmelCase ,					**UpperCAmelCase					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase = None ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase					)
							return center_crop(UpperCAmelCase ,					size=(size['height'], size['width']) ,					data_format=UpperCAmelCase ,					**UpperCAmelCase					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase = None ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							return rescale(UpperCAmelCase ,					scale=UpperCAmelCase ,					data_format=UpperCAmelCase ,					**UpperCAmelCase					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase = None ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							return normalize(UpperCAmelCase ,					mean=UpperCAmelCase ,					std=UpperCAmelCase ,					data_format=UpperCAmelCase ,					**UpperCAmelCase					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = ChannelDimension.FIRST ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							_UpperCAmelCase             =    do_resize if do_resize is not None else self.do_resize
							_UpperCAmelCase             =    size if size is not None else self.size
							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase ,					default_to_square=UpperCAmelCase					)
							_UpperCAmelCase             =    resample if resample is not None else self.resample
							_UpperCAmelCase             =    do_center_crop if do_center_crop is not None else self.do_center_crop
							_UpperCAmelCase             =    crop_size if crop_size is not None else self.crop_size
							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase					)
							_UpperCAmelCase             =    do_rescale if do_rescale is not None else self.do_rescale
							_UpperCAmelCase             =    rescale_factor if rescale_factor is not None else self.rescale_factor
							_UpperCAmelCase             =    do_normalize if do_normalize is not None else self.do_normalize
							_UpperCAmelCase             =    image_mean if image_mean is not None else self.image_mean
							_UpperCAmelCase             =    image_std if image_std is not None else self.image_std
							_UpperCAmelCase             =    do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

							_UpperCAmelCase             =    make_list_of_images(UpperCAmelCase					)

							if not valid_images(UpperCAmelCase					):
													raise ValueError(
													    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
													    'torch.Tensor, tf.Tensor or jax.ndarray.'					)

							if do_resize and size is None:
													raise ValueError('Size must be specified if do_resize is True.'					)

							if do_center_crop and crop_size is None:
													raise ValueError('Crop size must be specified if do_center_crop is True.'					)

							if do_rescale and rescale_factor is None:
													raise ValueError('Rescale factor must be specified if do_rescale is True.'					)

							if do_normalize and (image_mean is None or image_std is None):
													raise ValueError('Image mean and std must be specified if do_normalize is True.'					)

							# PIL RGBA images are converted to RGB
							if do_convert_rgb:
													_UpperCAmelCase             =    [convert_to_rgb(UpperCAmelCase					) for image in images]

							# All transformations expect numpy arrays.
							_UpperCAmelCase             =    [to_numpy_array(UpperCAmelCase					) for image in images]

							if do_resize:
													_UpperCAmelCase             =    [self.resize(image=UpperCAmelCase ,					size=UpperCAmelCase ,					resample=UpperCAmelCase					) for image in images]

							if do_center_crop:
													_UpperCAmelCase             =    [self.center_crop(image=UpperCAmelCase ,					size=UpperCAmelCase					) for image in images]

							if do_rescale:
													_UpperCAmelCase             =    [self.rescale(image=UpperCAmelCase ,					scale=UpperCAmelCase					) for image in images]

							if do_normalize:
													_UpperCAmelCase             =    [self.normalize(image=UpperCAmelCase ,					mean=UpperCAmelCase ,					std=UpperCAmelCase					) for image in images]

							_UpperCAmelCase             =    [to_channel_dimension_format(UpperCAmelCase ,					UpperCAmelCase					) for image in images]

							_UpperCAmelCase             =    {'pixel_values': images}
							return BatchFeature(data=UpperCAmelCase ,					tensor_type=UpperCAmelCase					)
