





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_mbartaa import MBartaaTokenizer
else:
							UpperCAmelCase__    :      Union[str, Any]       					=None


UpperCAmelCase__    :      Union[str, Any]       					=logging.get_logger(__name__)

UpperCAmelCase__    :      Optional[Any]       					={'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__    :      int       					={
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase__    :      Optional[Any]       					={
    '''facebook/mbart-large-50-one-to-many-mmt''': 10_24,
}

# fmt: off
UpperCAmelCase__    :      Optional[int]       					=['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']

class    __A  (    a     ):
				__A  =					VOCAB_FILES_NAMES
				__A  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A  =					PRETRAINED_VOCAB_FILES_MAP
				__A  =					["""input_ids""", """attention_mask"""]
				__A  =					MBartaaTokenizer

				__A  =					[]
				__A  =					[]


				def __init__(		self ,			UpperCAmelCase_=None ,			UpperCAmelCase_=None ,			UpperCAmelCase_=None ,			UpperCAmelCase_=None ,			UpperCAmelCase_="</s>" ,			UpperCAmelCase_="</s>" ,			UpperCAmelCase_="<s>" ,			UpperCAmelCase_="<unk>" ,			UpperCAmelCase_="<pad>" ,			UpperCAmelCase_="<mask>" ,			**UpperCAmelCase_ ,			):
								# Mask token behave like a normal word, i.e. include the space before it
								lowerCamelCase									=AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_				) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_				) else mask_token

								lowerCamelCase									=kwargs.get("""additional_special_tokens""" ,			[]				)
								kwargs["additional_special_tokens"] += [
								    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
								]

								super().__init__(
								    UpperCAmelCase_ ,			src_lang=UpperCAmelCase_ ,			tgt_lang=UpperCAmelCase_ ,			tokenizer_file=UpperCAmelCase_ ,			eos_token=UpperCAmelCase_ ,			sep_token=UpperCAmelCase_ ,			cls_token=UpperCAmelCase_ ,			unk_token=UpperCAmelCase_ ,			pad_token=UpperCAmelCase_ ,			mask_token=UpperCAmelCase_ ,			**UpperCAmelCase_ ,			)

								lowerCamelCase									=vocab_file
								lowerCamelCase									=False if not self.vocab_file else True

								lowerCamelCase									={
								    lang_code: self.convert_tokens_to_ids(UpperCAmelCase_				) for lang_code in FAIRSEQ_LANGUAGE_CODES
								}

								lowerCamelCase									=src_lang if src_lang is not None else """en_XX"""
								lowerCamelCase									=tgt_lang
								lowerCamelCase									=self.lang_code_to_id[self._src_lang]
								self.set_src_lang_special_tokens(self._src_lang				)


				@property
				def _snake_case							(		self				):
								return self._src_lang


				@src_lang.setter
				def _snake_case							(		self ,			UpperCAmelCase_				):
								lowerCamelCase									=new_src_lang
								self.set_src_lang_special_tokens(self._src_lang				)


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_ = None				):
								if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + self.suffix_tokens
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_ = "en_XX" ,			UpperCAmelCase_ = None ,			UpperCAmelCase_ = "ro_RO" ,			**UpperCAmelCase_ ,			):
								lowerCamelCase									=src_lang
								lowerCamelCase									=tgt_lang
								return super().prepare_seqaseq_batch(UpperCAmelCase_ ,			UpperCAmelCase_ ,			**UpperCAmelCase_				)


				def _snake_case							(		self				):
								return self.set_src_lang_special_tokens(self.src_lang				)


				def _snake_case							(		self				):
								return self.set_tgt_lang_special_tokens(self.tgt_lang				)


				def _snake_case							(		self ,			UpperCAmelCase_				):
								lowerCamelCase									=self.convert_tokens_to_ids(UpperCAmelCase_				)
								lowerCamelCase									=[self.cur_lang_code_id]
								lowerCamelCase									=[self.eos_token_id]

								lowerCamelCase									=self.convert_ids_to_tokens(self.prefix_tokens				)
								lowerCamelCase									=self.convert_ids_to_tokens(self.suffix_tokens				)

								lowerCamelCase									=processors.TemplateProcessing(
								    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str ,			pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str ,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,			self.prefix_tokens + self.suffix_tokens				)				) ,			)


				def _snake_case							(		self ,			UpperCAmelCase_				):
								lowerCamelCase									=self.convert_tokens_to_ids(UpperCAmelCase_				)
								lowerCamelCase									=[self.cur_lang_code_id]
								lowerCamelCase									=[self.eos_token_id]

								lowerCamelCase									=self.convert_ids_to_tokens(self.prefix_tokens				)
								lowerCamelCase									=self.convert_ids_to_tokens(self.suffix_tokens				)

								lowerCamelCase									=processors.TemplateProcessing(
								    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str ,			pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str ,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,			self.prefix_tokens + self.suffix_tokens				)				) ,			)


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_ ,			UpperCAmelCase_ ,			UpperCAmelCase_ ,			**UpperCAmelCase_				):
								if src_lang is None or tgt_lang is None:
												raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""				)
								lowerCamelCase									=src_lang
								lowerCamelCase									=self(UpperCAmelCase_ ,			add_special_tokens=UpperCAmelCase_ ,			return_tensors=UpperCAmelCase_ ,			**UpperCAmelCase_				)
								lowerCamelCase									=self.convert_tokens_to_ids(UpperCAmelCase_				)
								lowerCamelCase									=tgt_lang_id
								return inputs


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_ = None				):
								if not self.can_save_slow_tokenizer:
												raise ValueError(
												    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
												    """tokenizer."""				)

								if not os.path.isdir(UpperCAmelCase_				):
												logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
												return
								lowerCamelCase									=os.path.join(
								    UpperCAmelCase_ ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

								if os.path.abspath(self.vocab_file				) != os.path.abspath(UpperCAmelCase_				):
												copyfile(self.vocab_file ,			UpperCAmelCase_				)

								return (out_vocab_file,)












import os
import pickle
import unittest
from collections import UserDict, namedtuple

import torch

from accelerate.test_utils.testing import require_cuda, require_torch_min_version
from accelerate.test_utils.training import RegressionModel
from accelerate.utils import (
    convert_outputs_to_fpaa,
    extract_model_from_parallel,
    find_device,
    listify,
    patch_environment,
    recursively_apply,
    send_to_device,
)


UpperCAmelCase__    :      List[Any]       					=namedtuple('''ExampleNamedTuple''', '''a b c''')

class    __A  (    unittest.TestCase     ):


				def _snake_case							(		self				):
								lowerCamelCase									=torch.randn(5 ,			2				)
								lowerCamelCase									=torch.device("""cuda"""				) if torch.cuda.is_available() else torch.device("""cpu"""				)

								lowerCamelCase									=send_to_device(UpperCAmelCase_ ,			UpperCAmelCase_				)
								self.assertTrue(torch.equal(resulta.cpu() ,			UpperCAmelCase_				)				)

								lowerCamelCase									=send_to_device((tensor, [tensor, tensor], 1) ,			UpperCAmelCase_				)
								self.assertIsInstance(UpperCAmelCase_ ,			UpperCAmelCase_				)
								self.assertTrue(torch.equal(resulta[0].cpu() ,			UpperCAmelCase_				)				)
								self.assertIsInstance(resulta[1] ,			UpperCAmelCase_				)
								self.assertTrue(torch.equal(resulta[1][0].cpu() ,			UpperCAmelCase_				)				)
								self.assertTrue(torch.equal(resulta[1][1].cpu() ,			UpperCAmelCase_				)				)
								self.assertEqual(resulta[2] ,			1				)

								lowerCamelCase									=send_to_device({"""a""": tensor, """b""": [tensor, tensor], """c""": 1} ,			UpperCAmelCase_				)
								self.assertIsInstance(UpperCAmelCase_ ,			UpperCAmelCase_				)
								self.assertTrue(torch.equal(resulta["""a"""].cpu() ,			UpperCAmelCase_				)				)
								self.assertIsInstance(resulta["""b"""] ,			UpperCAmelCase_				)
								self.assertTrue(torch.equal(resulta["""b"""][0].cpu() ,			UpperCAmelCase_				)				)
								self.assertTrue(torch.equal(resulta["""b"""][1].cpu() ,			UpperCAmelCase_				)				)
								self.assertEqual(resulta["""c"""] ,			1				)

								lowerCamelCase									=send_to_device(ExampleNamedTuple(a=UpperCAmelCase_ ,			b=[tensor, tensor] ,			c=1				) ,			UpperCAmelCase_				)
								self.assertIsInstance(UpperCAmelCase_ ,			UpperCAmelCase_				)
								self.assertTrue(torch.equal(resulta.a.cpu() ,			UpperCAmelCase_				)				)
								self.assertIsInstance(resulta.b ,			UpperCAmelCase_				)
								self.assertTrue(torch.equal(resulta.b[0].cpu() ,			UpperCAmelCase_				)				)
								self.assertTrue(torch.equal(resulta.b[1].cpu() ,			UpperCAmelCase_				)				)
								self.assertEqual(resulta.c ,			1				)

								lowerCamelCase									=send_to_device(UserDict({"""a""": tensor, """b""": [tensor, tensor], """c""": 1}				) ,			UpperCAmelCase_				)
								self.assertIsInstance(UpperCAmelCase_ ,			UpperCAmelCase_				)
								self.assertTrue(torch.equal(resulta["""a"""].cpu() ,			UpperCAmelCase_				)				)
								self.assertIsInstance(resulta["""b"""] ,			UpperCAmelCase_				)
								self.assertTrue(torch.equal(resulta["""b"""][0].cpu() ,			UpperCAmelCase_				)				)
								self.assertTrue(torch.equal(resulta["""b"""][1].cpu() ,			UpperCAmelCase_				)				)
								self.assertEqual(resulta["""c"""] ,			1				)


				def _snake_case							(		self				):
								with self.assertRaises(UpperCAmelCase_				) as cm:
												lowerCamelCase									=recursively_apply(torch.tensor ,			(torch.tensor(1				), 1) ,			error_on_other_type=UpperCAmelCase_				)
								self.assertEqual(
								    str(cm.exception				) ,			"""Unsupported types (<class 'int'>) passed to `tensor`. Only nested list/tuple/dicts of objects that are valid for `is_torch_tensor` should be passed.""" ,			)


				def _snake_case							(		self				):
								lowerCamelCase									=torch.tensor([1, 2, 3, 4, 5]				)
								self.assertEqual(listify(UpperCAmelCase_				) ,			[1, 2, 3, 4, 5]				)

								lowerCamelCase									=torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]				)
								self.assertEqual(listify(UpperCAmelCase_				) ,			[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]				)

								lowerCamelCase									=torch.tensor([[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]], [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20]]]				)
								self.assertEqual(
								    listify(UpperCAmelCase_				) ,			[[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]], [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20]]]				)


				def _snake_case							(		self				):
								with patch_environment(aa=1 ,			BB=2				):
												self.assertEqual(os.environ.get("""AA"""				) ,			"""1"""				)
												self.assertEqual(os.environ.get("""BB"""				) ,			"""2"""				)

								self.assertNotIn("""AA""" ,			os.environ				)
								self.assertNotIn("""BB""" ,			os.environ				)


				def _snake_case							(		self				):
								lowerCamelCase									=RegressionModel()
								lowerCamelCase									=model.forward
								lowerCamelCase									=convert_outputs_to_fpaa(model.forward				)
								lowerCamelCase									=extract_model_from_parallel(UpperCAmelCase_ ,			keep_fpaa_wrapper=UpperCAmelCase_				)
								lowerCamelCase									=pickle.dumps(UpperCAmelCase_				)


				@require_cuda
				def _snake_case							(		self				):
								lowerCamelCase									=RegressionModel()
								lowerCamelCase									=model.forward
								lowerCamelCase									=torch.cuda.amp.autocast(dtype=torch.floataa				)(model.forward				)
								lowerCamelCase									=convert_outputs_to_fpaa(model.forward				)
								lowerCamelCase									=extract_model_from_parallel(UpperCAmelCase_ ,			keep_fpaa_wrapper=UpperCAmelCase_				)
								lowerCamelCase									=pickle.dumps(UpperCAmelCase_				)


				@require_cuda
				@require_torch_min_version(version="""2.0"""				)
				def _snake_case							(		self				):
								lowerCamelCase									=RegressionModel()
								lowerCamelCase									=model.forward
								lowerCamelCase									=torch.cuda.amp.autocast(dtype=torch.floataa				)(model.forward				)
								lowerCamelCase									=convert_outputs_to_fpaa(model.forward				)
								lowerCamelCase									=torch.compile(model.forward ,			backend="""inductor"""				)
								lowerCamelCase									=torch.randn(4 ,			10				).cuda()
								lowerCamelCase									=model(UpperCAmelCase_				)


				def _snake_case							(		self				):
								lowerCamelCase									=RegressionModel()
								# could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
								lowerCamelCase									=torch.nn.parallel.DataParallel(UpperCAmelCase_				)
								lowerCamelCase									=extract_model_from_parallel(UpperCAmelCase_				)

								self.assertEqual(UpperCAmelCase_ ,			UpperCAmelCase_				)


				@require_torch_min_version(version="""2.0"""				)
				def _snake_case							(		self				):
								lowerCamelCase									=RegressionModel()
								lowerCamelCase									=torch.compile(UpperCAmelCase_				)

								# could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
								lowerCamelCase									=torch.nn.parallel.DataParallel(UpperCAmelCase_				)
								lowerCamelCase									=torch.compile(UpperCAmelCase_				)
								lowerCamelCase									=extract_model_from_parallel(UpperCAmelCase_				)

								self.assertEqual(compiled_model._orig_mod ,			compiled_model_unwrapped._orig_mod				)


				def _snake_case							(		self				):
								self.assertEqual(find_device([1, """a""", torch.tensor([1, 2, 3]				)]				) ,			torch.device("""cpu"""				)				)
								self.assertEqual(find_device({"""a""": 1, """b""": torch.tensor([1, 2, 3]				)}				) ,			torch.device("""cpu"""				)				)
								self.assertIsNone(find_device([1, """a"""]				)				)





