'''simple docstring'''




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_mbartaa import MBartaaTokenizer
else:
  __UpperCAmelCase  							=None


__UpperCAmelCase  							=logging.get_logger(__name__)

__UpperCAmelCase  							={"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase  							={
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"
        ),
    },
}

__UpperCAmelCase  							={
    "facebook/mbart-large-50-one-to-many-mmt": 1_0_2_4,
}

# fmt: off
__UpperCAmelCase  							=["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]




class 					a__   (     UpperCAmelCase__				):
     lowerCamelCase						:  Any							=VOCAB_FILES_NAMES
     lowerCamelCase						:  Optional[int]							=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase						:  str							=PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase						:  List[Any]							=["input_ids", "attention_mask"]
     lowerCamelCase						:  Optional[int]							=MBartaaTokenizer

     lowerCamelCase						:  List[int]							=[]
     lowerCamelCase						:  List[int]							=[]






     def __init__(				self	:	Tuple						,       a	:	List[Any]=None						,       a	:	List[Any]=None						,       a	:	Optional[int]=None						,       a	:	List[str]=None						,       a	:	Optional[int]="</s>"						,       a	:	str="</s>"						,       a	:	Tuple="<s>"						,       a	:	Tuple="<unk>"						,       a	:	Any="<pad>"						,       a	:	Optional[Any]="<mask>"						,       **a	:	Dict						,       ):





         """simple docstring"""



         __lowerCamelCase						=						AddedToken(a						,       lstrip=a						,       rstrip=a					) if isinstance(a						,       a					) else mask_token

         __lowerCamelCase						=						kwargs.get('''additional_special_tokens'''						,       []					)
         kwargs["additional_special_tokens"] += [
             code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
         ]

         super().__init__(
             a						,       src_lang=a						,       tgt_lang=a						,       tokenizer_file=a						,       eos_token=a						,       sep_token=a						,       cls_token=a						,       unk_token=a						,       pad_token=a						,       mask_token=a						,       **a						,       )

         __lowerCamelCase						=						vocab_file
         __lowerCamelCase						=						False if not self.vocab_file else True

         __lowerCamelCase						=						{
             lang_code: self.convert_tokens_to_ids(a					) for lang_code in FAIRSEQ_LANGUAGE_CODES
         }

         __lowerCamelCase						=						src_lang if src_lang is not None else '''en_XX'''
         __lowerCamelCase						=						tgt_lang
         __lowerCamelCase						=						self.lang_code_to_id[self._src_lang]
         self.set_src_lang_special_tokens(self._src_lang					)






     @property
     def      SCREAMING_SNAKE_CASE__     (				self	:	int					):





         """simple docstring"""



         return self._src_lang






     @src_lang.setter
     def      SCREAMING_SNAKE_CASE__     (				self	:	Dict						,       a	:	str					):





         """simple docstring"""



         __lowerCamelCase						=						new_src_lang
         self.set_src_lang_special_tokens(self._src_lang					)






     def      SCREAMING_SNAKE_CASE__     (				self	:	Optional[int]						,       a	:	List[int]						,       a	:	Optional[List[int]] = None					):





         """simple docstring"""



         if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






     def      SCREAMING_SNAKE_CASE__     (				self	:	int						,       a	:	List[str]						,       a	:	str = "en_XX"						,       a	:	Optional[List[str]] = None						,       a	:	str = "ro_RO"						,       **a	:	int						,       ):





         """simple docstring"""



         __lowerCamelCase						=						src_lang
         __lowerCamelCase						=						tgt_lang
         return super().prepare_seqaseq_batch(a						,       a						,       **a					)






     def      SCREAMING_SNAKE_CASE__     (				self	:	List[str]					):





         """simple docstring"""



         return self.set_src_lang_special_tokens(self.src_lang					)






     def      SCREAMING_SNAKE_CASE__     (				self	:	Tuple					):





         """simple docstring"""



         return self.set_tgt_lang_special_tokens(self.tgt_lang					)






     def      SCREAMING_SNAKE_CASE__     (				self	:	Union[str, Any]						,       a	:	str					):





         """simple docstring"""



         __lowerCamelCase						=						self.convert_tokens_to_ids(a					)
         __lowerCamelCase						=						[self.cur_lang_code_id]
         __lowerCamelCase						=						[self.eos_token_id]

         __lowerCamelCase						=						self.convert_ids_to_tokens(self.prefix_tokens					)
         __lowerCamelCase						=						self.convert_ids_to_tokens(self.suffix_tokens					)

         __lowerCamelCase						=						processors.TemplateProcessing(
             single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str						,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str						,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,       self.prefix_tokens + self.suffix_tokens					)					)						,       )






     def      SCREAMING_SNAKE_CASE__     (				self	:	int						,       a	:	str					):





         """simple docstring"""



         __lowerCamelCase						=						self.convert_tokens_to_ids(a					)
         __lowerCamelCase						=						[self.cur_lang_code_id]
         __lowerCamelCase						=						[self.eos_token_id]

         __lowerCamelCase						=						self.convert_ids_to_tokens(self.prefix_tokens					)
         __lowerCamelCase						=						self.convert_ids_to_tokens(self.suffix_tokens					)

         __lowerCamelCase						=						processors.TemplateProcessing(
             single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str						,       pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str						,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,       self.prefix_tokens + self.suffix_tokens					)					)						,       )






     def      SCREAMING_SNAKE_CASE__     (				self	:	int						,       a	:	Dict						,       a	:	str						,       a	:	Optional[str]						,       a	:	Optional[str]						,       **a	:	Optional[Any]					):





         """simple docstring"""



         if src_lang is None or tgt_lang is None:
             raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''					)
         __lowerCamelCase						=						src_lang
         __lowerCamelCase						=						self(a						,       add_special_tokens=a						,       return_tensors=a						,       **a					)
         __lowerCamelCase						=						self.convert_tokens_to_ids(a					)
         __lowerCamelCase						=						tgt_lang_id
         return inputs






     def      SCREAMING_SNAKE_CASE__     (				self	:	str						,       a	:	str						,       a	:	Optional[str] = None					):





         """simple docstring"""



         if not self.can_save_slow_tokenizer:
             raise ValueError(
                 '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                 '''tokenizer.'''					)

         if not os.path.isdir(a					):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
             return
         __lowerCamelCase						=						os.path.join(
             a						,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

         if os.path.abspath(self.vocab_file					) != os.path.abspath(a					):
             copyfile(self.vocab_file						,       a					)

         return (out_vocab_file,)

'''simple docstring'''




from __future__ import annotations






def __lowerCAmelCase				(							UpperCamelCase__       ,						UpperCamelCase__       )			->					list:
    __lowerCamelCase						=						[None] * (len(UpperCamelCase__       ) + len(UpperCamelCase__       ))

    __lowerCamelCase						=						0  # pointer to current index for left Half
    __lowerCamelCase						=						0  # pointer to current index for the right Half
    __lowerCamelCase						=						0  # pointer to current index for the sorted array Half

    while pointera < len(UpperCamelCase__       ) and pointera < len(UpperCamelCase__       ):
        if left_half[pointera] < right_half[pointera]:
            __lowerCamelCase						=						left_half[pointera]
            pointera += 1
            index += 1
        else:
            __lowerCamelCase						=						right_half[pointera]
            pointera += 1
            index += 1
    while pointera < len(UpperCamelCase__       ):
        __lowerCamelCase						=						left_half[pointera]
        pointera += 1
        index += 1

    while pointera < len(UpperCamelCase__       ):
        __lowerCamelCase						=						right_half[pointera]
        pointera += 1
        index += 1

    return sorted_array






def __lowerCAmelCase				(							UpperCamelCase__       )			->					list:
    if len(UpperCamelCase__       ) <= 1:
        return array
    # the actual formula to calculate the middle element = left + (right - left) // 2
    # this avoids integer overflow in case of large N
    __lowerCamelCase						=						0 + (len(UpperCamelCase__       ) - 0) // 2

    # Split the array into halves till the array length becomes equal to One
    # merge the arrays of single length returned by mergeSort function and
    # pass them into the merge arrays function which merges the array
    __lowerCamelCase						=						array[:middle]
    __lowerCamelCase						=						array[middle:]

    return merge(merge_sort(UpperCamelCase__       )       ,						merge_sort(UpperCamelCase__       )       )


if __name__ == "__main__":
  import doctest

  doctest.testmod()
