
import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
      import torch

      from ..models.auto.modeling_auto import (
          MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
          MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
      )

if is_tf_available() and is_tensorflow_probability_available():
      import tensorflow as tf
      import tensorflow_probability as tfp

      from ..models.auto.modeling_tf_auto import (
          TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
          TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
      )

class 		lowercase_							(							__SCREAMING_SNAKE_CASE					):

 def __call__(   self  ,						__UpperCamelCase=None  ,						__UpperCamelCase=None  ,						**__UpperCamelCase  ):




        """simple docstring"""


        requires_backends(self  ,						"""pandas"""  )
        import pandas as pd

        if table is None:
               raise ValueError("""Keyword argument `table` cannot be None."""  )
        elif query is None:
               if isinstance(__UpperCamelCase  ,						__UpperCamelCase  ) and table.get("""query"""  ) is not None and table.get("""table"""  ) is not None:
                      UpperCamelCase_											=							[table]
               elif isinstance(__UpperCamelCase  ,						__UpperCamelCase  ) and len(__UpperCamelCase  ) > 0:
                      if not all(isinstance(__UpperCamelCase  ,						__UpperCamelCase  ) for d in table  ):
                             raise ValueError(
                                 f'''Keyword argument `table` should be a list of dict, but is {(type(__UpperCamelCase  ) for d in table)}'''  )

                      if table[0].get("""query"""  ) is not None and table[0].get("""table"""  ) is not None:
                             UpperCamelCase_											=							table
                      else:
                             raise ValueError(
                                 """If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`"""
                                 f''' and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys.'''  )
               elif Dataset is not None and isinstance(__UpperCamelCase  ,						__UpperCamelCase  ) or isinstance(__UpperCamelCase  ,						types.GeneratorType  ):
                      return table
               else:
                      raise ValueError(
                          """Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but """
                          f'''is {type(__UpperCamelCase  )})'''  )
        else:
               UpperCamelCase_											=							[{"""table""": table, """query""": query}]

        for tqa_pipeline_input in tqa_pipeline_inputs:
               if not isinstance(tqa_pipeline_input["""table"""]  ,						pd.DataFrame  ):
                      if tqa_pipeline_input["table"] is None:
                             raise ValueError("""Table cannot be None."""  )

                      UpperCamelCase_											=							pd.DataFrame(tqa_pipeline_input["""table"""]  )

        return tqa_pipeline_inputs



@add_end_docstrings(__SCREAMING_SNAKE_CASE					)
class 		lowercase_							(							__SCREAMING_SNAKE_CASE					):
 A__    :					Optional[int]    =     'table,query'

 def __init__(   self  ,						__UpperCamelCase=TableQuestionAnsweringArgumentHandler()  ,						*__UpperCamelCase  ,						**__UpperCamelCase  ):




        """simple docstring"""


        super().__init__(*__UpperCamelCase  ,						**__UpperCamelCase  )
        UpperCamelCase_											=							args_parser

        self.check_model_type(
            dict(
                TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
                + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()  )
            if self.framework == """tf"""
            else dict(
                MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()  )  )

        UpperCamelCase_											=							bool(getattr(self.model.config  ,						"""aggregation_labels"""  ,						__UpperCamelCase  )  ) and bool(
            getattr(self.model.config  ,						"""num_aggregation_labels"""  ,						__UpperCamelCase  )  )
        UpperCamelCase_											=							"""tapas""" if hasattr(self.model.config  ,						"""aggregation_labels"""  ) else None

 def lowerCamelCase_					(   self  ,						**__UpperCamelCase  ):




        """simple docstring"""


        return self.model(**__UpperCamelCase  )

 def lowerCamelCase_					(   self  ,						**__UpperCamelCase  ):




        """simple docstring"""


        if self.framework == "pt":
               UpperCamelCase_											=							[]
               UpperCamelCase_											=							[]
               UpperCamelCase_											=							None
               UpperCamelCase_											=							inputs["""input_ids"""].shape[0]

               UpperCamelCase_											=							inputs["""input_ids"""].to(self.device  )
               UpperCamelCase_											=							inputs["""attention_mask"""].to(self.device  )
               UpperCamelCase_											=							inputs["""token_type_ids"""].to(self.device  )
               UpperCamelCase_											=							None

               for index in range(__UpperCamelCase  ):
                      # If sequences have already been processed, the token type IDs will be created according to the previous
                      # answer.
                      if prev_answers is not None:
                             UpperCamelCase_											=							token_type_ids_example[:, 3]  # shape (seq_len,)
                             UpperCamelCase_											=							np.zeros_like(prev_labels_example.cpu().numpy()  )  # shape (seq_len,)

                             UpperCamelCase_											=							token_type_ids[index]  # shape (seq_len, 7)
                             for i in range(model_labels.shape[0]  ):
                                    UpperCamelCase_											=							token_type_ids_example[:, 0].tolist()[i]
                                    UpperCamelCase_											=							token_type_ids_example[:, 1].tolist()[i] - 1
                                    UpperCamelCase_											=							token_type_ids_example[:, 2].tolist()[i] - 1

                                    if row_id >= 0 and col_id >= 0 and segment_id == 1:
                                           UpperCamelCase_											=							int(prev_answers[(col_id, row_id)]  )

                             UpperCamelCase_											=							torch.from_numpy(__UpperCamelCase  ).type(torch.long  ).to(self.device  )

                      UpperCamelCase_											=							input_ids[index]
                      UpperCamelCase_											=							attention_mask[index]  # shape (seq_len,)
                      UpperCamelCase_											=							token_type_ids[index]  # shape (seq_len, 7)
                      UpperCamelCase_											=							self.model(
                          input_ids=input_ids_example.unsqueeze(0  )  ,						attention_mask=attention_mask_example.unsqueeze(0  )  ,						token_type_ids=token_type_ids_example.unsqueeze(0  )  ,						)
                      UpperCamelCase_											=							outputs.logits

                      if self.aggregate:
                             all_aggregations.append(outputs.logits_aggregation  )

                      all_logits.append(__UpperCamelCase  )

                      UpperCamelCase_											=							torch.distributions.Bernoulli(logits=__UpperCamelCase  )
                      UpperCamelCase_											=							dist_per_token.probs * attention_mask_example.type(torch.floataa  ).to(
                          dist_per_token.probs.device  )

                      UpperCamelCase_											=							collections.defaultdict(__UpperCamelCase  )
                      for i, p in enumerate(probabilities.squeeze().tolist()  ):
                             UpperCamelCase_											=							token_type_ids_example[:, 0].tolist()[i]
                             UpperCamelCase_											=							token_type_ids_example[:, 1].tolist()[i] - 1
                             UpperCamelCase_											=							token_type_ids_example[:, 2].tolist()[i] - 1
                             if col >= 0 and row >= 0 and segment_id == 1:
                                    coords_to_probs[(col, row)].append(__UpperCamelCase  )

                      UpperCamelCase_											=							{key: np.array(coords_to_probs[key]  ).mean() > 0.5 for key in coords_to_probs}

               UpperCamelCase_											=							torch.cat(tuple(__UpperCamelCase  )  ,						0  )

               return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(__UpperCamelCase  )  ,						0  ))
        else:
               UpperCamelCase_											=							[]
               UpperCamelCase_											=							[]
               UpperCamelCase_											=							None
               UpperCamelCase_											=							inputs["""input_ids"""].shape[0]

               UpperCamelCase_											=							inputs["""input_ids"""]
               UpperCamelCase_											=							inputs["""attention_mask"""]
               UpperCamelCase_											=							inputs["""token_type_ids"""].numpy()
               UpperCamelCase_											=							None

               for index in range(__UpperCamelCase  ):
                      # If sequences have already been processed, the token type IDs will be created according to the previous
                      # answer.
                      if prev_answers is not None:
                             UpperCamelCase_											=							token_type_ids_example[:, 3]  # shape (seq_len,)
                             UpperCamelCase_											=							np.zeros_like(__UpperCamelCase  ,						dtype=np.intaa  )  # shape (seq_len,)

                             UpperCamelCase_											=							token_type_ids[index]  # shape (seq_len, 7)
                             for i in range(model_labels.shape[0]  ):
                                    UpperCamelCase_											=							token_type_ids_example[:, 0].tolist()[i]
                                    UpperCamelCase_											=							token_type_ids_example[:, 1].tolist()[i] - 1
                                    UpperCamelCase_											=							token_type_ids_example[:, 2].tolist()[i] - 1

                                    if row_id >= 0 and col_id >= 0 and segment_id == 1:
                                           UpperCamelCase_											=							int(prev_answers[(col_id, row_id)]  )

                             UpperCamelCase_											=							model_labels

                      UpperCamelCase_											=							input_ids[index]
                      UpperCamelCase_											=							attention_mask[index]  # shape (seq_len,)
                      UpperCamelCase_											=							token_type_ids[index]  # shape (seq_len, 7)
                      UpperCamelCase_											=							self.model(
                          input_ids=np.expand_dims(__UpperCamelCase  ,						axis=0  )  ,						attention_mask=np.expand_dims(__UpperCamelCase  ,						axis=0  )  ,						token_type_ids=np.expand_dims(__UpperCamelCase  ,						axis=0  )  ,						)
                      UpperCamelCase_											=							outputs.logits

                      if self.aggregate:
                             all_aggregations.append(outputs.logits_aggregation  )

                      all_logits.append(__UpperCamelCase  )

                      UpperCamelCase_											=							tfp.distributions.Bernoulli(logits=__UpperCamelCase  )
                      UpperCamelCase_											=							dist_per_token.probs_parameter() * tf.cast(__UpperCamelCase  ,						tf.floataa  )

                      UpperCamelCase_											=							collections.defaultdict(__UpperCamelCase  )
                      UpperCamelCase_											=							token_type_ids_example
                      for i, p in enumerate(tf.squeeze(__UpperCamelCase  ).numpy().tolist()  ):
                             UpperCamelCase_											=							token_type_ids_example[:, 0].tolist()[i]
                             UpperCamelCase_											=							token_type_ids_example[:, 1].tolist()[i] - 1
                             UpperCamelCase_											=							token_type_ids_example[:, 2].tolist()[i] - 1
                             if col >= 0 and row >= 0 and segment_id == 1:
                                    coords_to_probs[(col, row)].append(__UpperCamelCase  )

                      UpperCamelCase_											=							{key: np.array(coords_to_probs[key]  ).mean() > 0.5 for key in coords_to_probs}

               UpperCamelCase_											=							tf.concat(tuple(__UpperCamelCase  )  ,						0  )

               return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(__UpperCamelCase  )  ,						0  ))

 def __call__(   self  ,						*__UpperCamelCase  ,						**__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_											=							self._args_parser(*__UpperCamelCase  ,						**__UpperCamelCase  )

        UpperCamelCase_											=							super().__call__(__UpperCamelCase  ,						**__UpperCamelCase  )
        if len(__UpperCamelCase  ) == 1:
               return results[0]
        return results

 def lowerCamelCase_					(   self  ,						__UpperCamelCase=None  ,						__UpperCamelCase=None  ,						__UpperCamelCase=None  ,						**__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_											=							{}
        if padding is not None:
               UpperCamelCase_											=							padding
        if truncation is not None:
               UpperCamelCase_											=							truncation

        UpperCamelCase_											=							{}
        if sequential is not None:
               UpperCamelCase_											=							sequential
        return preprocess_params, forward_params, {}

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase=None  ,						__UpperCamelCase=True  ,						__UpperCamelCase=None  ):




        """simple docstring"""


        if truncation is None:
               if self.type == "tapas":
                      UpperCamelCase_											=							"""drop_rows_to_fit"""
               else:
                      UpperCamelCase_											=							"""do_not_truncate"""

        UpperCamelCase_							,					UpperCamelCase_											=							pipeline_input["""table"""], pipeline_input["""query"""]
        if table.empty:
               raise ValueError("""table is empty"""  )
        if query is None or query == "":
               raise ValueError("""query is empty"""  )
        UpperCamelCase_											=							self.tokenizer(__UpperCamelCase  ,						__UpperCamelCase  ,						return_tensors=self.framework  ,						truncation=__UpperCamelCase  ,						padding=__UpperCamelCase  )
        UpperCamelCase_											=							table
        return inputs

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase=False  ):




        """simple docstring"""


        UpperCamelCase_											=							model_inputs.pop("""table"""  )

        if self.type == "tapas":
               if sequential:
                      UpperCamelCase_											=							self.sequential_inference(**__UpperCamelCase  )
               else:
                      UpperCamelCase_											=							self.batch_inference(**__UpperCamelCase  )
        else:
               UpperCamelCase_											=							self.model.generate(**__UpperCamelCase  )
        UpperCamelCase_											=							{"""model_inputs""": model_inputs, """table""": table, """outputs""": outputs}
        return model_outputs

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_											=							model_outputs["""model_inputs"""]
        UpperCamelCase_											=							model_outputs["""table"""]
        UpperCamelCase_											=							model_outputs["""outputs"""]
        if self.type == "tapas":
               if self.aggregate:
                      UpperCamelCase_							,					UpperCamelCase_											=							outputs[:2]
                      UpperCamelCase_											=							self.tokenizer.convert_logits_to_predictions(__UpperCamelCase  ,						__UpperCamelCase  ,						__UpperCamelCase  )
                      UpperCamelCase_							,					UpperCamelCase_											=							predictions
                      UpperCamelCase_											=							{i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(__UpperCamelCase  )}

                      UpperCamelCase_											=							self.model.config.no_aggregation_label_index
                      UpperCamelCase_											=							{
                          i: aggregators[i] + """ > """ for i, pred in enumerate(__UpperCamelCase  ) if pred != no_agg_label_index
                      }
               else:
                      UpperCamelCase_											=							outputs[0]
                      UpperCamelCase_											=							self.tokenizer.convert_logits_to_predictions(__UpperCamelCase  ,						__UpperCamelCase  )
                      UpperCamelCase_											=							predictions[0]
                      UpperCamelCase_											=							{}
                      UpperCamelCase_											=							{}
               UpperCamelCase_											=							[]
               for index, coordinates in enumerate(__UpperCamelCase  ):
                      UpperCamelCase_											=							[table.iat[coordinate] for coordinate in coordinates]
                      UpperCamelCase_											=							aggregators.get(__UpperCamelCase  ,						""""""  )
                      UpperCamelCase_											=							aggregators_prefix.get(__UpperCamelCase  ,						""""""  )
                      UpperCamelCase_											=							{
                          """answer""": aggregator_prefix + """, """.join(__UpperCamelCase  ),
                          """coordinates""": coordinates,
                          """cells""": [table.iat[coordinate] for coordinate in coordinates],
                      }
                      if aggregator:
                             UpperCamelCase_											=							aggregator

                      answers.append(__UpperCamelCase  )
               if len(__UpperCamelCase  ) == 0:
                      raise PipelineException("""Empty answer"""  )
        else:
               UpperCamelCase_											=							[{"""answer""": answer} for answer in self.tokenizer.batch_decode(__UpperCamelCase  ,						skip_special_tokens=__UpperCamelCase  )]

        return answers if len(__UpperCamelCase  ) > 1 else answers[0]





"""simple docstring"""




import random
import shutil
import tempfile
import unittest

import numpy as np

from transformers import TaTokenizer, TaTokenizerFast
from transformers.testing_utils import require_sentencepiece, require_torch
from transformers.utils.import_utils import is_speech_available, is_torch_available


if is_torch_available():
					pass

if is_speech_available():
					from transformers import EncodecFeatureExtractor, MusicgenProcessor


__lowerCAmelCase							:  Union[str, Any] 				=		random.Random()

def __snake_case (      UpperCamelCase						,					UpperCamelCase=1.0						,					UpperCamelCase=None						,					UpperCamelCase=None    )       ->    Tuple:


		"""simple docstring"""





		if rng is None:
				a__   		=     global_rng

		a__   		=     []
		for batch_idx in range(shape[0]    ):
				values.append([]    )
				for _ in range(shape[1]    ):
						values[-1].append(rng.random() * scale    )

		return values




@require_torch
@require_sentencepiece
class SCREAMING_SNAKE_CASE       (      unittest.TestCase					):






	'''simple docstring'''







	def    _UpperCamelCase							(  self :Optional[int]			)			->   List[str]:

			'''simple docstring'''






			a__   		=     '''facebook/musicgen-small'''
			a__   		=     tempfile.mkdtemp()


	def    _UpperCamelCase							(  self :Union[str, Any]       ,    **__magic_name__ :Union[str, Any]			)			->   str:

			'''simple docstring'''






			return TaTokenizer.from_pretrained(self.checkpoint       ,    **__magic_name__			)


	def    _UpperCamelCase							(  self :Optional[int]       ,    **__magic_name__ :Any			)			->   Optional[int]:

			'''simple docstring'''






			return EncodecFeatureExtractor.from_pretrained(self.checkpoint       ,    **__magic_name__			)


	def    _UpperCamelCase							(  self :Optional[int]			)			->   Optional[int]:

			'''simple docstring'''






			shutil.rmtree(self.tmpdirname			)


	def    _UpperCamelCase							(  self :Tuple			)			->   Any:

			'''simple docstring'''






			a__   		=     self.get_tokenizer()
			a__   		=     self.get_feature_extractor()

			a__   		=     MusicgenProcessor(tokenizer=__magic_name__       ,    feature_extractor=__magic_name__			)

			processor.save_pretrained(self.tmpdirname			)
			a__   		=     MusicgenProcessor.from_pretrained(self.tmpdirname			)

			self.assertEqual(processor.tokenizer.get_vocab()       ,    tokenizer.get_vocab()			)
			self.assertIsInstance(processor.tokenizer       ,    __magic_name__			)

			self.assertEqual(processor.feature_extractor.to_json_string()       ,    feature_extractor.to_json_string()			)
			self.assertIsInstance(processor.feature_extractor       ,    __magic_name__			)


	def    _UpperCamelCase							(  self :Dict			)			->   Optional[int]:

			'''simple docstring'''






			a__   		=     MusicgenProcessor(tokenizer=self.get_tokenizer()       ,    feature_extractor=self.get_feature_extractor()			)
			processor.save_pretrained(self.tmpdirname			)

			a__   		=     self.get_tokenizer(bos_token='''(BOS)'''       ,    eos_token='''(EOS)'''			)
			a__   		=     self.get_feature_extractor(do_normalize=__magic_name__       ,    padding_value=1.0			)

			a__   		=     MusicgenProcessor.from_pretrained(
			    self.tmpdirname       ,    bos_token='''(BOS)'''       ,    eos_token='''(EOS)'''       ,    do_normalize=__magic_name__       ,    padding_value=1.0			)

			self.assertEqual(processor.tokenizer.get_vocab()       ,    tokenizer_add_kwargs.get_vocab()			)
			self.assertIsInstance(processor.tokenizer       ,    __magic_name__			)

			self.assertEqual(processor.feature_extractor.to_json_string()       ,    feature_extractor_add_kwargs.to_json_string()			)
			self.assertIsInstance(processor.feature_extractor       ,    __magic_name__			)


	def    _UpperCamelCase							(  self :Any			)			->   Tuple:

			'''simple docstring'''






			a__   		=     self.get_feature_extractor()
			a__   		=     self.get_tokenizer()

			a__   		=     MusicgenProcessor(tokenizer=__magic_name__       ,    feature_extractor=__magic_name__			)

			a__   		=     floats_list((3, 1000)			)

			a__   		=     feature_extractor(__magic_name__       ,    return_tensors='''np'''			)
			a__   		=     processor(__magic_name__       ,    return_tensors='''np'''			)

			for key in input_feat_extract.keys():
					self.assertAlmostEqual(input_feat_extract[key].sum()       ,    input_processor[key].sum()       ,    delta=1e-2			)


	def    _UpperCamelCase							(  self :Tuple			)			->   Dict:

			'''simple docstring'''






			a__   		=     self.get_feature_extractor()
			a__   		=     self.get_tokenizer()

			a__   		=     MusicgenProcessor(tokenizer=__magic_name__       ,    feature_extractor=__magic_name__			)

			a__   		=     '''This is a test string'''

			a__   		=     processor(text=__magic_name__			)

			a__   		=     tokenizer(__magic_name__			)

			for key in encoded_tok.keys():
					self.assertListEqual(encoded_tok[key]       ,    encoded_processor[key]			)


	def    _UpperCamelCase							(  self :Tuple			)			->   Optional[Any]:

			'''simple docstring'''






			a__   		=     self.get_feature_extractor()
			a__   		=     self.get_tokenizer()

			a__   		=     MusicgenProcessor(tokenizer=__magic_name__       ,    feature_extractor=__magic_name__			)

			a__   		=     [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

			a__   		=     processor.batch_decode(sequences=__magic_name__			)
			a__   		=     tokenizer.batch_decode(__magic_name__			)

			self.assertListEqual(__magic_name__       ,    __magic_name__			)


	def    _UpperCamelCase							(  self :Optional[Any]			)			->   int:

			'''simple docstring'''






			a__   		=     self.get_feature_extractor()
			a__   		=     self.get_tokenizer()

			a__   		=     MusicgenProcessor(tokenizer=__magic_name__       ,    feature_extractor=__magic_name__			)

			self.assertListEqual(
			    processor.model_input_names       ,    feature_extractor.model_input_names       ,    msg='''`processor` and `feature_extractor` model input names do not match'''       ,    )






	def    _UpperCamelCase							(  self :Any			)			->   List[str]:

			'''simple docstring'''






			a__   		=     self.get_feature_extractor(padding_side='''left'''			)
			a__   		=     self.get_tokenizer()

			a__   		=     MusicgenProcessor(tokenizer=__magic_name__       ,    feature_extractor=__magic_name__			)

			a__   		=     [floats_list((1, x)			)[0] for x in range(5       ,    20       ,    5			)]
			a__   		=     processor(__magic_name__			).padding_mask

			a__   		=     np.asarray(floats_list((3, 20)			)			)[:, None, :]
			a__   		=     processor.batch_decode(__magic_name__       ,    padding_mask=__magic_name__			)

			self.assertIsInstance(__magic_name__       ,    __magic_name__			)

			for audio in decoded_audios:
					self.assertIsInstance(__magic_name__       ,    np.ndarray			)

			self.assertTrue(decoded_audios[0].shape == (1, 10)			)
			self.assertTrue(decoded_audios[1].shape == (1, 15)			)
			self.assertTrue(decoded_audios[2].shape == (1, 20)			)




