



'''simple docstring'''
# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import List






def A__					(  UpperCAmelCase_						,						UpperCAmelCase_							):
						if name != "training_function" and name != "main":
												raise ValueError(f'Incorrect function name passed: {name}, choose either \'main\' or \'training_function\''							)
						_UpperCamelCase       ,    _UpperCamelCase      :	Optional[int]				       =     [], False
						for line in lines:
												if not found_start and f'def {name}' in line:
																		_UpperCamelCase      :	List[Any]				       =     True
																		good_lines.append(UpperCAmelCase_							)
																		continue
												if found_start:
																		if name == "training_function" and "def main" in line:
																								return good_lines
																		if name == "main" and "if __name__" in line:
																								return good_lines
																		good_lines.append(UpperCAmelCase_							)






def A__					(  UpperCAmelCase_							):
						return [line for line in lines if not line.lstrip().startswith('#'							) and line != "\n"]






def A__					(  UpperCAmelCase_						,						UpperCAmelCase_						,						UpperCAmelCase_						,						UpperCAmelCase_ = None							):
						with open(UpperCAmelCase_						,						'r'							) as f:
												_UpperCamelCase      :	Optional[int]				       =     f.readlines()
						with open(os.path.abspath(os.path.join('examples'						,						'nlp_example.py'							)							)						,						'r'							) as f:
												_UpperCamelCase      :	List[Any]				       =     f.readlines()
						with open(UpperCAmelCase_						,						'r'							) as f:
												_UpperCamelCase      :	int				       =     f.readlines()
						if secondary_filename is not None:
												with open(UpperCAmelCase_						,						'r'							) as f:
																		_UpperCamelCase      :	Tuple				       =     f.readlines()

    # This is our base, we remove all the code from here in our `full_filename` and `feature_filename` to find the new content
						if parser_only:
												_UpperCamelCase      :	Tuple				       =     clean_lines(get_function_contents_by_name(UpperCAmelCase_						,						'main'							)							)
												_UpperCamelCase      :	Union[str, Any]				       =     clean_lines(get_function_contents_by_name(UpperCAmelCase_						,						'main'							)							)
												_UpperCamelCase      :	Dict				       =     clean_lines(get_function_contents_by_name(UpperCAmelCase_						,						'main'							)							)
												if secondary_filename is not None:
																		_UpperCamelCase      :	Any				       =     clean_lines(get_function_contents_by_name(UpperCAmelCase_						,						'main'							)							)
						else:
												_UpperCamelCase      :	Optional[int]				       =     clean_lines(get_function_contents_by_name(UpperCAmelCase_						,						'training_function'							)							)
												_UpperCamelCase      :	List[str]				       =     clean_lines(get_function_contents_by_name(UpperCAmelCase_						,						'training_function'							)							)
												_UpperCamelCase      :	List[Any]				       =     clean_lines(get_function_contents_by_name(UpperCAmelCase_						,						'training_function'							)							)
												if secondary_filename is not None:
																		_UpperCamelCase      :	int				       =     clean_lines(
																		    get_function_contents_by_name(UpperCAmelCase_						,						'training_function'							)							)

						_UpperCamelCase      :	Any				       =     'train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)\n'

						# Specific code in our script that differs from the full version, aka what is new
						_UpperCamelCase      :	Optional[Any]				       =     []
						_UpperCamelCase      :	Any				       =     []  # We keep track of the idxs just in case it's a repeated statement
						_UpperCamelCase      :	Dict				       =     iter(UpperCAmelCase_							)
						for i in range(len(UpperCAmelCase_							) - 1							):
												if i not in passed_idxs:
																		_UpperCamelCase      :	List[Any]				       =     next(UpperCAmelCase_							)
																		if (line not in full_file_func) and (line.lstrip() != _dl_line):
																								if "TESTING_MOCKED_DATALOADERS" not in line:
																														new_feature_code.append(UpperCAmelCase_							)
																														passed_idxs.append(UpperCAmelCase_							)
																								else:
																														# Skip over the `config['num_epochs'] = 2` statement
																														_UpperCamelCase      :	Dict				       =     next(UpperCAmelCase_							)

    # Extract out just the new parts from the full_file_training_func
						_UpperCamelCase      :	Optional[int]				       =     []
						_UpperCamelCase      :	str				       =     []  # We keep track of the idxs just in case it's a repeated statement
						for i, line in enumerate(UpperCAmelCase_							):
												if i not in passed_idxs:
																		if (line not in full_file_func) and (line.lstrip() != _dl_line):
																								if "TESTING_MOCKED_DATALOADERS" not in line:
																														new_full_example_parts.append(UpperCAmelCase_							)
																														passed_idxs.append(UpperCAmelCase_							)

    # Finally, get the overall diff
						_UpperCamelCase      :	Any				       =     [line for line in new_feature_code if line not in new_full_example_parts]
						if secondary_filename is not None:
												_UpperCamelCase      :	str				       =     [line for line in full_file_contents if line not in secondary_file_func]
												_UpperCamelCase      :	List[Any]				       =     [line for line in diff_from_example if line not in diff_from_two]

						return diff_from_example







'''simple docstring'''
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_mbartaa import MBartaaTokenizer
else:
					snake_case_				:			List[str]						 =						None


snake_case_				:			Union[str, Any]						 =						logging.get_logger(__name__)

snake_case_				:			Dict						 =						{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

snake_case_				:			str						 =						{
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'
        ),
    },
}

snake_case_				:			List[Any]						 =						{
    'facebook/mbart-large-50-one-to-many-mmt': 1024,
}

# fmt: off
snake_case_				:			Union[str, Any]						 =						['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']






class 							lowercase__			(							lowercase     ):
	lowercase__						       =    VOCAB_FILES_NAMES
	lowercase__						       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase__						       =    PRETRAINED_VOCAB_FILES_MAP
	lowercase__						       =    ["""input_ids""", """attention_mask"""]
	lowercase__						       =    MBartaaTokenizer

	lowercase__						       =    []
	lowercase__						       =    []





	def __init__(				self      :       Optional[int]				,lowerCamelCase__      :       Union[str, Any]=None				,lowerCamelCase__      :       List[Any]=None				,lowerCamelCase__      :       Optional[Any]=None				,lowerCamelCase__      :       str=None				,lowerCamelCase__      :       List[str]="</s>"				,lowerCamelCase__      :       List[str]="</s>"				,lowerCamelCase__      :       Dict="<s>"				,lowerCamelCase__      :       int="<unk>"				,lowerCamelCase__      :       List[Any]="<pad>"				,lowerCamelCase__      :       Optional[int]="<mask>"				,**lowerCamelCase__      :       Optional[int]				,):







							'''simple docstring'''
							# Mask token behave like a normal word, i.e. include the space before it
							_UpperCamelCase      :	str				       =     AddedToken(lowerCamelCase__				,lstrip=lowerCamelCase__				,rstrip=lowerCamelCase__				) if isinstance(lowerCamelCase__				,lowerCamelCase__				) else mask_token

							_UpperCamelCase      :	Tuple				       =     kwargs.get('additional_special_tokens'				,[]				)
							kwargs["additional_special_tokens"] += [
							    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    lowerCamelCase__				,src_lang=lowerCamelCase__				,tgt_lang=lowerCamelCase__				,tokenizer_file=lowerCamelCase__				,eos_token=lowerCamelCase__				,sep_token=lowerCamelCase__				,cls_token=lowerCamelCase__				,unk_token=lowerCamelCase__				,pad_token=lowerCamelCase__				,mask_token=lowerCamelCase__				,**lowerCamelCase__				,)

							_UpperCamelCase      :	Optional[int]				       =     vocab_file
							_UpperCamelCase      :	Union[str, Any]				       =     False if not self.vocab_file else True

							_UpperCamelCase      :	Union[str, Any]				       =     {
							    lang_code: self.convert_tokens_to_ids(lowerCamelCase__				) for lang_code in FAIRSEQ_LANGUAGE_CODES
							}

							_UpperCamelCase      :	Union[str, Any]				       =     src_lang if src_lang is not None else 'en_XX'
							_UpperCamelCase      :	Any				       =     tgt_lang
							_UpperCamelCase      :	Optional[Any]				       =     self.lang_code_to_id[self._src_lang]
							self.set_src_lang_special_tokens(self._src_lang				)





	@property
	def 			UpperCamelCase_						(				self      :       Optional[Any]				):







							'''simple docstring'''
							return self._src_lang





	@src_lang.setter
	def 			UpperCamelCase_						(				self      :       Optional[Any]				,lowerCamelCase__      :       str				):







							'''simple docstring'''
							_UpperCamelCase      :	str				       =     new_src_lang
							self.set_src_lang_special_tokens(self._src_lang				)





	def 			UpperCamelCase_						(				self      :       Tuple				,lowerCamelCase__      :       List[int]				,lowerCamelCase__      :       Optional[List[int]] = None				):







							'''simple docstring'''
							if token_ids_a is None:
													return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





	def 			UpperCamelCase_						(				self      :       Optional[Any]				,lowerCamelCase__      :       List[str]				,lowerCamelCase__      :       str = "en_XX"				,lowerCamelCase__      :       Optional[List[str]] = None				,lowerCamelCase__      :       str = "ro_RO"				,**lowerCamelCase__      :       Dict				,):







							'''simple docstring'''
							_UpperCamelCase      :	List[str]				       =     src_lang
							_UpperCamelCase      :	List[str]				       =     tgt_lang
							return super().prepare_seqaseq_batch(lowerCamelCase__				,lowerCamelCase__				,**lowerCamelCase__				)





	def 			UpperCamelCase_						(				self      :       Tuple				):







							'''simple docstring'''
							return self.set_src_lang_special_tokens(self.src_lang				)





	def 			UpperCamelCase_						(				self      :       Optional[int]				):







							'''simple docstring'''
							return self.set_tgt_lang_special_tokens(self.tgt_lang				)





	def 			UpperCamelCase_						(				self      :       Dict				,lowerCamelCase__      :       str				):







							'''simple docstring'''
							_UpperCamelCase      :	Any				       =     self.convert_tokens_to_ids(lowerCamelCase__				)
							_UpperCamelCase      :	List[Any]				       =     [self.cur_lang_code_id]
							_UpperCamelCase      :	List[Any]				       =     [self.eos_token_id]

							_UpperCamelCase      :	Optional[int]				       =     self.convert_ids_to_tokens(self.prefix_tokens				)
							_UpperCamelCase      :	List[Any]				       =     self.convert_ids_to_tokens(self.suffix_tokens				)

							_UpperCamelCase      :	Union[str, Any]				       =     processors.TemplateProcessing(
							    single=prefix_tokens_str + ['$A'] + suffix_tokens_str				,pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str				,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,self.prefix_tokens + self.suffix_tokens				)				)				,)





	def 			UpperCamelCase_						(				self      :       Union[str, Any]				,lowerCamelCase__      :       str				):







							'''simple docstring'''
							_UpperCamelCase      :	List[str]				       =     self.convert_tokens_to_ids(lowerCamelCase__				)
							_UpperCamelCase      :	int				       =     [self.cur_lang_code_id]
							_UpperCamelCase      :	int				       =     [self.eos_token_id]

							_UpperCamelCase      :	Union[str, Any]				       =     self.convert_ids_to_tokens(self.prefix_tokens				)
							_UpperCamelCase      :	int				       =     self.convert_ids_to_tokens(self.suffix_tokens				)

							_UpperCamelCase      :	Dict				       =     processors.TemplateProcessing(
							    single=prefix_tokens_str + ['$A'] + suffix_tokens_str				,pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str				,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,self.prefix_tokens + self.suffix_tokens				)				)				,)





	def 			UpperCamelCase_						(				self      :       Any				,lowerCamelCase__      :       Optional[int]				,lowerCamelCase__      :       str				,lowerCamelCase__      :       Optional[str]				,lowerCamelCase__      :       Optional[str]				,**lowerCamelCase__      :       List[Any]				):







							'''simple docstring'''
							if src_lang is None or tgt_lang is None:
													raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'				)
							_UpperCamelCase      :	Optional[int]				       =     src_lang
							_UpperCamelCase      :	Optional[Any]				       =     self(lowerCamelCase__				,add_special_tokens=lowerCamelCase__				,return_tensors=lowerCamelCase__				,**lowerCamelCase__				)
							_UpperCamelCase      :	Tuple				       =     self.convert_tokens_to_ids(lowerCamelCase__				)
							_UpperCamelCase      :	Optional[int]				       =     tgt_lang_id
							return inputs





	def 			UpperCamelCase_						(				self      :       Optional[int]				,lowerCamelCase__      :       str				,lowerCamelCase__      :       Optional[str] = None				):







							'''simple docstring'''
							if not self.can_save_slow_tokenizer:
													raise ValueError(
													    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
													    'tokenizer.'				)

							if not os.path.isdir(lowerCamelCase__				):
													logger.error(F'Vocabulary path ({save_directory}) should be a directory'				)
													return
							_UpperCamelCase      :	Optional[Any]				       =     os.path.join(
							    lowerCamelCase__				,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)

							if os.path.abspath(self.vocab_file				) != os.path.abspath(lowerCamelCase__				):
													copyfile(self.vocab_file				,lowerCamelCase__				)

							return (out_vocab_file,)


