






"""simple docstring"""





import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation

require_version("""tokenizers>=0.13.3""")

if is_sentencepiece_available():
  from .tokenization_llama import LlamaTokenizer
else:
  UpperCamelCase					=     None

UpperCamelCase					=     logging.get_logger(__name__)
UpperCamelCase					=     {"""vocab_file""": """tokenizer.model""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase,     UpperCamelCase					=     """[INST]""", """[/INST]"""
UpperCamelCase,     UpperCamelCase					=     """<<SYS>>\n""", """\n<</SYS>>\n\n"""

# fmt: off
UpperCamelCase					=     """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
correct. If you don't know the answer to a question, please don't share false information."""







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   UpperCamelCase__     	=							VOCAB_FILES_NAMES
   UpperCamelCase__     	=							LlamaTokenizer
   UpperCamelCase__     	=							"""left"""
   UpperCamelCase__     	=							["""input_ids""", """attention_mask"""]





   def __init__(       self					, UpperCAmelCase_=None					, UpperCAmelCase_=None					, UpperCAmelCase_=False					, UpperCAmelCase_="<unk>"					, UpperCAmelCase_="<s>"					, UpperCAmelCase_="</s>"					, UpperCAmelCase_=True					, UpperCAmelCase_=False					, **UpperCAmelCase_					, )			->   Optional[Any]:




          '''simple docstring'''




          super().__init__(
              vocab_file=UpperCAmelCase_					, tokenizer_file=UpperCAmelCase_					, clean_up_tokenization_spaces=UpperCAmelCase_					, unk_token=UpperCAmelCase_					, bos_token=UpperCAmelCase_					, eos_token=UpperCAmelCase_					, **UpperCAmelCase_					, )
          lowercase__:							Union[str, Any]         =     add_bos_token
          lowercase__:							List[str]         =     add_eos_token
          self.update_post_processor()

          lowercase__:							Dict         =     vocab_file
          lowercase__:							str         =     False if not self.vocab_file else True





   def __lowercase							(       self)			->   Dict:




          '''simple docstring'''




          lowercase__:							Any         =     self.bos_token
          lowercase__:							str         =     self.bos_token_id

          lowercase__:							Optional[int]         =     self.eos_token
          lowercase__:							Any         =     self.eos_token_id

          lowercase__:							Any         =     F"""{(bos+':0 ') * self.add_bos_token}$A:0{(' '+eos+':0') * self.add_eos_token}"""
          lowercase__:							Union[str, Any]         =     F"""{single}{(' '+bos+':1') * self.add_bos_token} $B:1{(' '+eos+':1') * self.add_eos_token}"""

          lowercase__:							List[str]         =     []
          if self.add_bos_token:
                 special_tokens.append((bos, bos_token_id))
          if self.add_eos_token:
                 special_tokens.append((eos, eos_token_id))
          lowercase__:							List[str]         =     processors.TemplateProcessing(
              single=UpperCAmelCase_					, pair=UpperCAmelCase_					, special_tokens=UpperCAmelCase_)





   @property
   def __lowercase							(       self)			->   Optional[Any]:




          '''simple docstring'''




          return self._add_eos_token





   @property
   def __lowercase							(       self)			->   List[Any]:




          '''simple docstring'''




          return self._add_bos_token





   @add_eos_token.setter
   def __lowercase							(       self					, UpperCAmelCase_)			->   Optional[Any]:




          '''simple docstring'''




          lowercase__:							Union[str, Any]         =     value
          self.update_post_processor()





   @add_bos_token.setter
   def __lowercase							(       self					, UpperCAmelCase_)			->   Optional[Any]:




          '''simple docstring'''




          lowercase__:							str         =     value
          self.update_post_processor()





   def __lowercase							(       self					, UpperCAmelCase_					, UpperCAmelCase_ = None)			->   Tuple[str]:




          '''simple docstring'''




          if not self.can_save_slow_tokenizer:
                 raise ValueError(
                     "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                     "tokenizer.")

          if not os.path.isdir(UpperCAmelCase_):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
                 return
          lowercase__:							str         =     os.path.join(
              UpperCAmelCase_					, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

          if os.path.abspath(self.vocab_file) != os.path.abspath(UpperCAmelCase_):
                 copyfile(self.vocab_file					, UpperCAmelCase_)

          return (out_vocab_file,)





   def __lowercase							(       self					, UpperCAmelCase_)			->   List[str]:




          '''simple docstring'''




          lowercase__:							int         =     list(conversation.iter_texts())
          if not all([is_user for is_user, msg in dialogue[::2]]) or not all(
              [not is_user for is_user, msg in dialogue[1::2]]):
                 raise ValueError(
                     "The model only supports 'user' and 'assistant' roles, starting with user and alternating (u/a/u/a/u...)")

          lowercase__:							str         =     []
          if len(conversation.past_user_inputs) > 0:
                 if not conversation.past_user_inputs[0].startswith(UpperCAmelCase_) or E_SYS not in conversation.past_user_inputs[0]:
                        lowercase__:							Optional[int]         =     (
                            B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
                        )
          elif not dialogue[0][1].startswith(UpperCAmelCase_) or E_SYS not in dialogue[0][1]:
                 lowercase__:							Optional[Any]         =     (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

          dialog_tokens += sum(
              [
                  [self.bos_token_id]
                  + self.encode(
                      F"""{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} """					, add_special_tokens=UpperCAmelCase_)
                  + [self.eos_token_id]
                  for prompt, answer in zip(dialogue[::2]					, dialogue[1::2])
              ]					, []					, )
          if not (dialogue[-1][0]):
                 raise ValueError(F"""Last message must be from user, got {dialogue[-1]['role']}""")
          dialog_tokens += [self.bos_token_id] + self.encode(
              F"""{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"""					, add_special_tokens=UpperCAmelCase_)
          return dialog_tokens







'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFXLMForMultipleChoice,
      TFXLMForQuestionAnsweringSimple,
      TFXLMForSequenceClassification,
      TFXLMForTokenClassification,
      TFXLMModel,
      TFXLMWithLMHeadModel,
      XLMConfig,
  )






class       __lowercase		:







     """simple docstring"""






     def __init__(		self  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						str		     =	parent
         __UpperCamelCase						:						Tuple		     =	1_3
         __UpperCamelCase						:						str		     =	7
         __UpperCamelCase						:						List[str]		     =	True
         __UpperCamelCase						:						Optional[Any]		     =	True
         __UpperCamelCase						:						Tuple		     =	True
         __UpperCamelCase						:						Tuple		     =	True
         __UpperCamelCase						:						int		     =	True
         __UpperCamelCase						:						int		     =	False
         __UpperCamelCase						:						Optional[Any]		     =	False
         __UpperCamelCase						:						Any		     =	False
         __UpperCamelCase						:						Any		     =	2
         __UpperCamelCase						:						str		     =	9_9
         __UpperCamelCase						:						int		     =	0
         __UpperCamelCase						:						Any		     =	3_2
         __UpperCamelCase						:						Tuple		     =	2
         __UpperCamelCase						:						Optional[int]		     =	4
         __UpperCamelCase						:						List[str]		     =	0.1
         __UpperCamelCase						:						List[Any]		     =	0.1
         __UpperCamelCase						:						List[str]		     =	5_1_2
         __UpperCamelCase						:						int		     =	1_6
         __UpperCamelCase						:						Any		     =	2
         __UpperCamelCase						:						Dict		     =	0.0_2
         __UpperCamelCase						:						Any		     =	3
         __UpperCamelCase						:						str		     =	4
         __UpperCamelCase						:						Any		     =	'last'
         __UpperCamelCase						:						int		     =	True
         __UpperCamelCase						:						List[str]		     =	None
         __UpperCamelCase						:						Union[str, Any]		     =	0







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						List[str]		     =	ids_tensor([self.batch_size, self.seq_length]  ,      self.vocab_size     )
         __UpperCamelCase						:						str		     =	random_attention_mask([self.batch_size, self.seq_length]  ,      dtype=tf.floataa     )

         __UpperCamelCase						:						Optional[Any]		     =	None
         if self.use_input_lengths:
             __UpperCamelCase						:						Optional[int]		     =	(
                 ids_tensor([self.batch_size]  ,      vocab_size=2     ) + self.seq_length - 2
             )  # small variation of seq_length

         __UpperCamelCase						:						int		     =	None
         if self.use_token_type_ids:
             __UpperCamelCase						:						Union[str, Any]		     =	ids_tensor([self.batch_size, self.seq_length]  ,      self.n_langs     )

         __UpperCamelCase						:						Optional[Any]		     =	None
         __UpperCamelCase						:						Optional[Any]		     =	None
         __UpperCamelCase						:						Tuple		     =	None
         if self.use_labels:
             __UpperCamelCase						:						Tuple		     =	ids_tensor([self.batch_size]  ,      self.type_sequence_label_size     )
             __UpperCamelCase						:						Dict		     =	ids_tensor([self.batch_size, self.seq_length]  ,      self.num_labels     )
             __UpperCamelCase						:						Optional[int]		     =	ids_tensor([self.batch_size]  ,      2  ,      dtype=tf.floataa     )
             __UpperCamelCase						:						int		     =	ids_tensor([self.batch_size]  ,      self.num_choices     )

         __UpperCamelCase						:						int		     =	XLMConfig(
             vocab_size=self.vocab_size  ,      n_special=self.n_special  ,      emb_dim=self.hidden_size  ,      n_layers=self.num_hidden_layers  ,      n_heads=self.num_attention_heads  ,      dropout=self.hidden_dropout_prob  ,      attention_dropout=self.attention_probs_dropout_prob  ,      gelu_activation=self.gelu_activation  ,      sinusoidal_embeddings=self.sinusoidal_embeddings  ,      asm=self.asm  ,      causal=self.causal  ,      n_langs=self.n_langs  ,      max_position_embeddings=self.max_position_embeddings  ,      initializer_range=self.initializer_range  ,      summary_type=self.summary_type  ,      use_proj=self.use_proj  ,      bos_token_id=self.bos_token_id  ,      )

         return (
             config,
             input_ids,
             token_type_ids,
             input_lengths,
             sequence_labels,
             token_labels,
             is_impossible_labels,
             choice_labels,
             input_mask,
         )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						str		     =	TFXLMModel(config=_lowerCamelCase     )
         __UpperCamelCase						:						Optional[Any]		     =	{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
         __UpperCamelCase						:						str		     =	model(_lowerCamelCase     )

         __UpperCamelCase						:						Dict		     =	[input_ids, input_mask]
         __UpperCamelCase						:						Tuple		     =	model(_lowerCamelCase     )
         self.parent.assertEqual(result.last_hidden_state.shape  ,      (self.batch_size, self.seq_length, self.hidden_size)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						int		     =	TFXLMWithLMHeadModel(_lowerCamelCase     )

         __UpperCamelCase						:						List[str]		     =	{'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
         __UpperCamelCase						:						List[str]		     =	model(_lowerCamelCase     )

         __UpperCamelCase						:						Tuple		     =	outputs

         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.seq_length, self.vocab_size)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						Tuple		     =	TFXLMForQuestionAnsweringSimple(_lowerCamelCase     )

         __UpperCamelCase						:						Dict		     =	{'input_ids': input_ids, 'lengths': input_lengths}

         __UpperCamelCase						:						Optional[Any]		     =	model(_lowerCamelCase     )

         self.parent.assertEqual(result.start_logits.shape  ,      (self.batch_size, self.seq_length)     )
         self.parent.assertEqual(result.end_logits.shape  ,      (self.batch_size, self.seq_length)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						Optional[Any]		     =	TFXLMForSequenceClassification(_lowerCamelCase     )

         __UpperCamelCase						:						Union[str, Any]		     =	{'input_ids': input_ids, 'lengths': input_lengths}

         __UpperCamelCase						:						Dict		     =	model(_lowerCamelCase     )

         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.type_sequence_label_size)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						int		     =	self.num_labels
         __UpperCamelCase						:						int		     =	TFXLMForTokenClassification(config=_lowerCamelCase     )
         __UpperCamelCase						:						List[str]		     =	{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
         __UpperCamelCase						:						List[str]		     =	model(_lowerCamelCase     )
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.seq_length, self.num_labels)     )







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      ):
         __UpperCamelCase						:						Any		     =	self.num_choices
         __UpperCamelCase						:						List[Any]		     =	TFXLMForMultipleChoice(config=_lowerCamelCase     )
         __UpperCamelCase						:						Tuple		     =	tf.tile(tf.expand_dims(_lowerCamelCase  ,      1     )  ,      (1, self.num_choices, 1)     )
         __UpperCamelCase						:						Optional[int]		     =	tf.tile(tf.expand_dims(_lowerCamelCase  ,      1     )  ,      (1, self.num_choices, 1)     )
         __UpperCamelCase						:						Union[str, Any]		     =	tf.tile(tf.expand_dims(_lowerCamelCase  ,      1     )  ,      (1, self.num_choices, 1)     )
         __UpperCamelCase						:						str		     =	{
             'input_ids': multiple_choice_inputs_ids,
             'attention_mask': multiple_choice_input_mask,
             'token_type_ids': multiple_choice_token_type_ids,
         }
         __UpperCamelCase						:						Dict		     =	model(_lowerCamelCase     )
         self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.num_choices)     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						List[str]		     =	self.prepare_config_and_inputs()
         (
             (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       (
             __UpperCamelCase
         )     ,       
         )						:						Any		     =	config_and_inputs
         __UpperCamelCase						:						Union[str, Any]		     =	{
             'input_ids': input_ids,
             'token_type_ids': token_type_ids,
             'langs': token_type_ids,
             'lengths': input_lengths,
         }
         return config, inputs_dict






@require_tf
class       __lowercase		(    _lowerCamelCase      ,				_lowerCamelCase      ,				unittest.TestCase    ):







     """simple docstring"""






     SCREAMING_SNAKE_CASE__		 =    (
         (
             TFXLMModel,
             TFXLMWithLMHeadModel,
             TFXLMForSequenceClassification,
             TFXLMForQuestionAnsweringSimple,
             TFXLMForTokenClassification,
             TFXLMForMultipleChoice,
         )
         if is_tf_available()
         else ()
     )
     SCREAMING_SNAKE_CASE__		 =    (
         (TFXLMWithLMHeadModel,) if is_tf_available() else ()
     )  # TODO (PVP): Check other models whether language generation is also applicable
     SCREAMING_SNAKE_CASE__		 =    (
         {
             '''feature-extraction''': TFXLMModel,
             '''fill-mask''': TFXLMWithLMHeadModel,
             '''question-answering''': TFXLMForQuestionAnsweringSimple,
             '''text-classification''': TFXLMForSequenceClassification,
             '''text-generation''': TFXLMWithLMHeadModel,
             '''token-classification''': TFXLMForTokenClassification,
             '''zero-shot''': TFXLMForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     SCREAMING_SNAKE_CASE__		 =    False
     SCREAMING_SNAKE_CASE__		 =    False







     def lowerCAmelCase  (		self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase     ):
         if (
             pipeline_test_casse_name == "QAPipelineTests"
             and tokenizer_name is not None
             and not tokenizer_name.endswith('Fast'     )
         ):
             # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
             # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
             # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
             return True

         return False







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[Any]		     =	TFXLMModelTester(self     )
         __UpperCamelCase						:						Union[str, Any]		     =	ConfigTester(self  ,      config_class=_lowerCamelCase  ,      emb_dim=3_7     )







     def lowerCAmelCase  (		self     ):
         self.config_tester.run_common_tests()







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[int]		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_model(*_lowerCamelCase     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[int]		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_lm_head(*_lowerCamelCase     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[Any]		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_qa(*_lowerCamelCase     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Optional[Any]		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_sequence_classif(*_lowerCamelCase     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Any		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_for_token_classification(*_lowerCamelCase     )







     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Dict		     =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_for_multiple_choice(*_lowerCamelCase     )







     @slow
     def lowerCAmelCase  (		self     ):
         for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             __UpperCamelCase						:						List[Any]		     =	TFXLMModel.from_pretrained(_lowerCamelCase     )
             self.assertIsNotNone(_lowerCamelCase     )






@require_tf
class       __lowercase		(    unittest.TestCase    ):







     """simple docstring"""






     @slow
     def lowerCAmelCase  (		self     ):
         __UpperCamelCase						:						Union[str, Any]		     =	TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048'     )
         __UpperCamelCase						:						Union[str, Any]		     =	tf.convert_to_tensor([[1_4, 4_4_7]]  ,      dtype=tf.intaa     )  # the president
         __UpperCamelCase						:						List[Any]		     =	[
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
         ]  # the president the president the president the president the president the president the president the president the president the president
         # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
         __UpperCamelCase						:						str		     =	model.generate(_lowerCamelCase  ,      do_sample=_lowerCamelCase     )
         self.assertListEqual(output_ids[0].numpy().tolist()  ,      _lowerCamelCase     )



