




"""simple docstring"""




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device


if is_torch_available():
  import torch

  from transformers import XLMProphetNetForConditionalGeneration, XLMProphetNetTokenizer
@require_torch
class 							lowerCamelCase		( unittest.TestCase				):







   @slow
   def       a_							(							self ):
       UpperCamelCase				:   Tuple          =							XLMProphetNetForConditionalGeneration.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased""" )
       model.to(SCREAMING_SNAKE_CASE_ )

       # encoder-decoder outputs
       UpperCamelCase				:   List[Any]          =							torch.tensor([[17, 9_6208, 10_3471, 2]] ).to(SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   Optional[int]          =							torch.tensor(
           [[2, 250, 9953, 34, 6_9489, 1620, 32, 11_8424, 624, 210, 105, 2913, 1032, 351]] ).to(SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   Dict          =							model(
           input_ids=SCREAMING_SNAKE_CASE_    ,      attention_mask=SCREAMING_SNAKE_CASE_    ,      encoder_outputs=SCREAMING_SNAKE_CASE_    ,      decoder_input_ids=SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   str          =							output[0]
       UpperCamelCase				:   str          =							torch.Size((1, 14, 25_0012) )
       self.assertEqual(output_predited_logis.shape    ,      SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   str          =							torch.tensor(
           [[[-6.3986, -8.2391, 12.5189], [-6.3289, -8.0864, 12.6211], [-6.2418, -8.0445, 12.7968]]] ).to(SCREAMING_SNAKE_CASE_ )
       self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]    ,      SCREAMING_SNAKE_CASE_    ,      atol=1e-4 ) )

       # encoder outputs
       UpperCamelCase				:   str          =							model.prophetnet.encoder(SCREAMING_SNAKE_CASE_ )[0]
       UpperCamelCase				:   str          =							torch.tensor(
           [[[-1.4260, -0.7628, 0.8453], [-1.4719, -0.1391, 0.7807], [-1.7678, 0.0114, 0.4646]]] ).to(SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   int          =							torch.Size((1, 4, 1024) )
       self.assertEqual(encoder_outputs.shape    ,      SCREAMING_SNAKE_CASE_ )
       self.assertTrue(torch.allclose(encoder_outputs[:, :3, :3]    ,      SCREAMING_SNAKE_CASE_    ,      atol=1e-4 ) )

       # decoder outputs
       UpperCamelCase				:   Union[str, Any]          =							model.prophetnet.decoder(
           SCREAMING_SNAKE_CASE_    ,      encoder_hidden_states=SCREAMING_SNAKE_CASE_    ,      )
       UpperCamelCase				:   str          =							decoder_outputs[1].view(1    ,      model.config.ngram    ,      14    ,      -1 )
       UpperCamelCase				:   Optional[int]          =							model.lm_head(SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   str          =							predicting_streams_logits[:, 0]
       self.assertTrue(torch.allclose(next_first_stream_logits[:, :3, :3]    ,      SCREAMING_SNAKE_CASE_    ,      atol=1e-4 ) )







   @slow
   def       a_							(							self ):
       UpperCamelCase				:   Optional[int]          =							XLMProphetNetForConditionalGeneration.from_pretrained(
           """microsoft/xprophetnet-large-wiki100-cased-xglue-ntg""" )
       model.to(SCREAMING_SNAKE_CASE_ )

       UpperCamelCase				:   Union[str, Any]          =							torch.tensor([[17, 9_6208, 10_3471, 2]] ).to(SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   str          =							torch.tensor(
           [[2, 250, 9953, 34, 6_9489, 1620, 32, 11_8424, 624, 210, 105, 2913, 1032, 351]] ).to(SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   str          =							model(
           input_ids=SCREAMING_SNAKE_CASE_    ,      attention_mask=SCREAMING_SNAKE_CASE_    ,      encoder_outputs=SCREAMING_SNAKE_CASE_    ,      decoder_input_ids=SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   List[Any]          =							output[0]
       UpperCamelCase				:   List[str]          =							torch.Size((1, 14, 25_0012) )
       self.assertEqual(output_predited_logis.shape    ,      SCREAMING_SNAKE_CASE_ )
       # compare the actual values for a slice.
       UpperCamelCase				:   Optional[int]          =							torch.tensor(
           [[[-9.2253, -9.7173, -6.3529], [-7.6701, -9.0145, -1.9382], [-8.0195, -7.0004, -0.1523]]] ).to(SCREAMING_SNAKE_CASE_ )

       self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]    ,      SCREAMING_SNAKE_CASE_    ,      atol=1e-4 ) )







   @slow
   def       a_							(							self ):
       UpperCamelCase				:   Optional[int]          =							XLMProphetNetForConditionalGeneration.from_pretrained(
           """microsoft/xprophetnet-large-wiki100-cased-xglue-ntg""" )
       model.to(SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   Optional[int]          =							512

       UpperCamelCase				:   Optional[Any]          =							XLMProphetNetTokenizer.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased-xglue-ntg""" )

       UpperCamelCase				:   Optional[int]          =							(
           """Microsoft Corporation intends to officially end free support for the Windows 7 operating system after"""
           """ January 14, 2020, according to the official portal of the organization. From that day, users of this"""
           """ system will not be able to receive security updates, which could make their computers vulnerable to"""
           """ cyber attacks."""
       )
       UpperCamelCase				:   int          =							(
           """орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7"""
           """ после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи"""
           """ этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми"""
           """ к кибератакам."""
       )
       UpperCamelCase				:   Optional[Any]          =							(
           """根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。"""
       )

       UpperCamelCase				:   Union[str, Any]          =							tokenizer(
           [EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE]    ,      padding=SCREAMING_SNAKE_CASE_    ,      max_length=255    ,      return_tensors="""pt""" ).input_ids
       UpperCamelCase				:   List[str]          =							input_ids.to(SCREAMING_SNAKE_CASE_ )

       UpperCamelCase				:   int          =							model.generate(
           SCREAMING_SNAKE_CASE_    ,      num_beams=10    ,      length_penalty=1.0    ,      no_repeat_ngram_size=3    ,      early_stopping=SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   Tuple          =							[tokenizer.decode(SCREAMING_SNAKE_CASE_    ,      skip_special_tokens=SCREAMING_SNAKE_CASE_ ) for g in summary_ids]
       UpperCamelCase				:   Tuple          =							"""Microsoft to end Windows 7 free support after January 14, 2020"""
       UpperCamelCase				:   str          =							"""Microsoft намерена прекратить бесплатную поддержку Windows 7 после 14 января 2020 года"""
       UpperCamelCase				:   Optional[Any]          =							"""微软打算终止对Windows 7操作系统的免费支持"""
       self.assertListEqual(
           [EXPECTED_TITLE_EN, EXPECTED_TITLE_RU, EXPECTED_TITLE_ZH]    ,      SCREAMING_SNAKE_CASE_    ,      )

       UpperCamelCase				:   Optional[Any]          =							model.generate(
           SCREAMING_SNAKE_CASE_    ,      num_beams=1    ,      length_penalty=1.0    ,      no_repeat_ngram_size=3    ,      early_stopping=SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   Optional[int]          =							[
           tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_    ,      skip_special_tokens=SCREAMING_SNAKE_CASE_ ) for g in summary_ids_beama
       ]
       UpperCamelCase				:   List[str]          =							"""▁Microsoft ▁to ▁end ▁free ▁support ▁for ▁Windows ▁7""".split(""" """ )
       UpperCamelCase				:   Optional[int]          =							"""▁Microsoft ▁намерен а ▁прекрати ть ▁бес плат ную ▁поддержку ▁Windows ▁7 ▁после ▁14 ▁января ▁2020 ▁года""".split(
           """ """ )
       UpperCamelCase				:   Any          =							"""微软 公司 打算 终止 对 Windows ▁7 操作 系统的 免费 支持""".split(""" """ )
       self.assertListEqual(
           [EXPECTED_TITLE_EN_BEAM1_TOK, EXPECTED_TITLE_RU_BEAM1_TOK, EXPECTED_TITLE_ZH_BEAM1_TOK]    ,      SCREAMING_SNAKE_CASE_    ,      )










"""simple docstring"""




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_mbartaa import MBartaaTokenizer
else:
  __A :		Tuple      							=    None


__A :		Optional[Any]      							=    logging.get_logger(__name__)

__A :		Optional[Any]      							=    {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

__A :		Optional[Any]      							=    {
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'''
        ),
    },
}

__A :		List[Any]      							=    {
    '''facebook/mbart-large-50-one-to-many-mmt''': 1024,
}

# fmt: off
__A :		Dict      							=    ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']
class 							lowerCamelCase		( _UpperCAmelCase				):
   lowercase       :  List[Any]      =				VOCAB_FILES_NAMES
   lowercase       :  Optional[Any]      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowercase       :  List[Any]      =				PRETRAINED_VOCAB_FILES_MAP
   lowercase       :  Tuple      =				['input_ids', 'attention_mask']
   lowercase       :  Optional[int]      =				MBartaaTokenizer

   lowercase       :  List[int]      =				[]
   lowercase       :  List[int]      =				[]







   def __init__(							self    ,      SCREAMING_SNAKE_CASE_=None    ,      SCREAMING_SNAKE_CASE_=None    ,      SCREAMING_SNAKE_CASE_=None    ,      SCREAMING_SNAKE_CASE_=None    ,      SCREAMING_SNAKE_CASE_="</s>"    ,      SCREAMING_SNAKE_CASE_="</s>"    ,      SCREAMING_SNAKE_CASE_="<s>"    ,      SCREAMING_SNAKE_CASE_="<unk>"    ,      SCREAMING_SNAKE_CASE_="<pad>"    ,      SCREAMING_SNAKE_CASE_="<mask>"    ,      **SCREAMING_SNAKE_CASE_    ,      ):
       # Mask token behave like a normal word, i.e. include the space before it
       UpperCamelCase				:   Optional[int]          =							AddedToken(SCREAMING_SNAKE_CASE_    ,      lstrip=SCREAMING_SNAKE_CASE_    ,      rstrip=SCREAMING_SNAKE_CASE_ ) if isinstance(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_ ) else mask_token

       UpperCamelCase				:   Dict          =							kwargs.get("""additional_special_tokens"""    ,      [] )
       kwargs["additional_special_tokens"] += [
           code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
       ]

       super().__init__(
           SCREAMING_SNAKE_CASE_    ,      src_lang=SCREAMING_SNAKE_CASE_    ,      tgt_lang=SCREAMING_SNAKE_CASE_    ,      tokenizer_file=SCREAMING_SNAKE_CASE_    ,      eos_token=SCREAMING_SNAKE_CASE_    ,      sep_token=SCREAMING_SNAKE_CASE_    ,      cls_token=SCREAMING_SNAKE_CASE_    ,      unk_token=SCREAMING_SNAKE_CASE_    ,      pad_token=SCREAMING_SNAKE_CASE_    ,      mask_token=SCREAMING_SNAKE_CASE_    ,      **SCREAMING_SNAKE_CASE_    ,      )

       UpperCamelCase				:   int          =							vocab_file
       UpperCamelCase				:   Optional[Any]          =							False if not self.vocab_file else True

       UpperCamelCase				:   List[Any]          =							{
           lang_code: self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ ) for lang_code in FAIRSEQ_LANGUAGE_CODES
       }

       UpperCamelCase				:   List[str]          =							src_lang if src_lang is not None else """en_XX"""
       UpperCamelCase				:   Union[str, Any]          =							tgt_lang
       UpperCamelCase				:   Union[str, Any]          =							self.lang_code_to_id[self._src_lang]
       self.set_src_lang_special_tokens(self._src_lang )







   @property
   def       a_							(							self ):
       return self._src_lang







   @src_lang.setter
   def       a_							(							self    ,      SCREAMING_SNAKE_CASE_ ):
       UpperCamelCase				:   str          =							new_src_lang
       self.set_src_lang_special_tokens(self._src_lang )







   def       a_							(							self    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_ = None ):
       if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







   def       a_							(							self    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_ = "en_XX"    ,      SCREAMING_SNAKE_CASE_ = None    ,      SCREAMING_SNAKE_CASE_ = "ro_RO"    ,      **SCREAMING_SNAKE_CASE_    ,      ):
       UpperCamelCase				:   str          =							src_lang
       UpperCamelCase				:   Optional[Any]          =							tgt_lang
       return super().prepare_seqaseq_batch(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      **SCREAMING_SNAKE_CASE_ )







   def       a_							(							self ):
       return self.set_src_lang_special_tokens(self.src_lang )







   def       a_							(							self ):
       return self.set_tgt_lang_special_tokens(self.tgt_lang )







   def       a_							(							self    ,      SCREAMING_SNAKE_CASE_ ):
       UpperCamelCase				:   Optional[Any]          =							self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   Tuple          =							[self.cur_lang_code_id]
       UpperCamelCase				:   Optional[int]          =							[self.eos_token_id]

       UpperCamelCase				:   Dict          =							self.convert_ids_to_tokens(self.prefix_tokens )
       UpperCamelCase				:   Any          =							self.convert_ids_to_tokens(self.suffix_tokens )

       UpperCamelCase				:   Any          =							processors.TemplateProcessing(
           single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str    ,      pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str    ,      special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,      self.prefix_tokens + self.suffix_tokens ) )    ,      )







   def       a_							(							self    ,      SCREAMING_SNAKE_CASE_ ):
       UpperCamelCase				:   Any          =							self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   Optional[Any]          =							[self.cur_lang_code_id]
       UpperCamelCase				:   Tuple          =							[self.eos_token_id]

       UpperCamelCase				:   str          =							self.convert_ids_to_tokens(self.prefix_tokens )
       UpperCamelCase				:   Dict          =							self.convert_ids_to_tokens(self.suffix_tokens )

       UpperCamelCase				:   Dict          =							processors.TemplateProcessing(
           single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str    ,      pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str    ,      special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,      self.prefix_tokens + self.suffix_tokens ) )    ,      )







   def       a_							(							self    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      **SCREAMING_SNAKE_CASE_ ):
       if src_lang is None or tgt_lang is None:
           raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model""" )
       UpperCamelCase				:   Optional[int]          =							src_lang
       UpperCamelCase				:   Optional[int]          =							self(SCREAMING_SNAKE_CASE_    ,      add_special_tokens=SCREAMING_SNAKE_CASE_    ,      return_tensors=SCREAMING_SNAKE_CASE_    ,      **SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   Dict          =							self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ )
       UpperCamelCase				:   str          =							tgt_lang_id
       return inputs







   def       a_							(							self    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_ = None ):
       if not self.can_save_slow_tokenizer:
           raise ValueError(
               """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
               """tokenizer.""" )

       if not os.path.isdir(SCREAMING_SNAKE_CASE_ ):
           logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
           return
       UpperCamelCase				:   Optional[int]          =							os.path.join(
           SCREAMING_SNAKE_CASE_    ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

       if os.path.abspath(self.vocab_file ) != os.path.abspath(SCREAMING_SNAKE_CASE_ ):
           copyfile(self.vocab_file    ,      SCREAMING_SNAKE_CASE_ )

       return (out_vocab_file,)




