
"""simple docstring"""


from typing import List

from packaging import version
from sklearn.metrics import fa_score

import datasets
from datasets.config import PY_VERSION


if PY_VERSION < version.parse('3.8'):
   import importlib_metadata
else:
   import importlib.metadata as importlib_metadata


# TODO(Patrick/Anton)
__A 		= '\\n'

__A 		= '\\n    XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages.\n    XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval.\n'

__A 		= '\nCompute XTREME-S evaluation metric associated to each XTREME-S dataset.\nArgs:\n    predictions: list of predictions to score.\n        Each translation should be tokenized into a list of tokens.\n    references: list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\n    bleu_kwargs: optional dict of keywords to be passed when computing \'bleu\'.\n        Keywords include Dict can be one of \'smooth_method\', \'smooth_value\', \'force\', \'lowercase\',\n        \'tokenize\', \'use_effective_order\'.\n    wer_kwargs: optional dict of keywords to be passed when computing \'wer\' and \'cer\'.\n        Keywords include \'concatenate_texts\'.\nReturns: depending on the XTREME-S task, one or several of:\n    "accuracy": Accuracy - for \'fleurs-lang_id\', \'minds14\'\n    "f1": F1 score - for \'minds14\'\n    "wer": Word error rate - for \'mls\', \'fleurs-asr\', \'voxpopuli\', \'babel\'\n    "cer": Character error rate - for \'mls\', \'fleurs-asr\', \'voxpopuli\', \'babel\'\n    "bleu": BLEU score according to the `sacrebleu` metric - for \'covost2\'\nExamples:\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'mls\')  # \'mls\', \'voxpopuli\', \'fleurs-asr\' or \'babel\'\n    >>> references = ["it is sunny here", "paper and pen are essentials"]\n    >>> predictions = ["it\'s sunny", "paper pen are essential"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'wer\': 0.56, \'cer\': 0.27}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'covost2\')\n    >>> references = ["bonjour paris", "il est necessaire de faire du sport de temps en temp"]\n    >>> predictions = ["bonjour paris", "il est important de faire du sport souvent"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'bleu\': 31.65}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'fleurs-lang_id\')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'accuracy\': 0.6}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'minds14\')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'f1\': 0.58, \'accuracy\': 0.6}\n'

__A 		= ['fleurs-asr', 'mls', 'voxpopuli', 'babel', 'covost2', 'fleurs-lang_id', 'minds14']
__A 		= ''

try:
   from jiwer import transforms as tr

   __A 		= True
except ImportError:
   __A 		= False

if _jiwer_available and version.parse(importlib_metadata.version('jiwer')) < version.parse('2.3.0'):







   class 	lowerCamelCase__      (      tr.AbstractTransform       ):





      '''simple docstring'''



      def __init__(							self  ,  __UpperCAmelCase = " "						)				->  Union[str, Any]:
       _lowerCAmelCase			=sentence_delimiter



      def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase						)				->  Any:
       return list(__UpperCAmelCase						)



      def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase						)				->  Tuple:
       _lowerCAmelCase			=[]
       for sent_idx, sentence in enumerate(__UpperCAmelCase						):
        chars.extend(self.process_string(__UpperCAmelCase						)						)
        if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(__UpperCAmelCase						) - 1:
         chars.append(self.sentence_delimiter						)
       return chars







   __A 		= tr.Compose(
       [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
   )
elif _jiwer_available:
   __A 		= tr.Compose(
       [
           tr.RemoveMultipleSpaces(),
           tr.Strip(),
           tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
           tr.ReduceToListOfListOfChars(),
       ]
   )
else:
   __A 		= None



def 					_lowerCamelCase(__UpperCamelCase					,				__UpperCamelCase )       ->				Optional[int]:
 return float((preds == labels).mean() )



def 					_lowerCamelCase(__UpperCamelCase					,				__UpperCamelCase )       ->				Dict:
 return {
     "f1": float(fa_score(y_true=_A					,				y_pred=_A					,				average="""macro""" ) ),
     "accuracy": simple_accuracy(_A					,				_A ),
 }



def 					_lowerCamelCase(__UpperCamelCase					,				__UpperCamelCase					,				__UpperCamelCase="exp"					,				__UpperCamelCase=None					,				__UpperCamelCase=False					,				__UpperCamelCase=False					,				__UpperCamelCase=None					,				__UpperCamelCase=False					,				)       ->				List[Any]:
 _lowerCAmelCase			=[[label] for label in labels]
 _lowerCAmelCase			=list(_A )
 try:
  import sacrebleu as scb
 except ImportError:
  raise ValueError(
      """sacrebleu has to be installed in order to apply the bleu metric for covost2."""
      """You can install it via `pip install sacrebleu`.""" )

 if version.parse(scb.__version__ ) < version.parse("""1.4.12""" ):
  raise ImportWarning(
      """To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n"""
      """You can install it with `pip install \"sacrebleu>=1.4.12\"`.""" )

 _lowerCAmelCase			=len(labels[0] )
 if any(len(_A ) != references_per_prediction for refs in labels ):
  raise ValueError("""Sacrebleu requires the same number of references for each prediction""" )
 _lowerCAmelCase			=[[refs[i] for refs in labels] for i in range(_A )]
 _lowerCAmelCase			=scb.corpus_bleu(
     _A					,				_A					,				smooth_method=_A					,				smooth_value=_A					,				force=_A					,				lowercase=_A					,				use_effective_order=_A					,				**({"""tokenize""": tokenize} if tokenize else {})					,				)
 return {"bleu": output.score}



def 					_lowerCamelCase(__UpperCamelCase					,				__UpperCamelCase					,				__UpperCamelCase					,				__UpperCamelCase )       ->				List[str]:


 try:
  from jiwer import compute_measures
 except ImportError:
  raise ValueError(
      F'''jiwer has to be installed in order to apply the wer metric for {config_name}.'''
      """You can install it via `pip install jiwer`.""" )

 if concatenate_texts:
  _lowerCAmelCase			=compute_measures(_A					,				_A )["""wer"""]

  _lowerCAmelCase			=compute_measures(_A					,				_A					,				truth_transform=_A					,				hypothesis_transform=_A )["""wer"""]
  return {"wer": wer, "cer": cer}
 else:

  def compute_score(__UpperCamelCase					,				__UpperCamelCase					,				__UpperCamelCase="wer" ):
   _lowerCAmelCase			=0
   _lowerCAmelCase			=0
   for prediction, reference in zip(_A					,				_A ):
    if score_type == "wer":
     _lowerCAmelCase			=compute_measures(_A					,				_A )
    elif score_type == "cer":
     _lowerCAmelCase			=compute_measures(
         _A					,				_A					,				truth_transform=_A					,				hypothesis_transform=_A )
    incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
    total += measures["substitutions"] + measures["deletions"] + measures["hits"]
   return incorrect / total

  return {"wer": compute_score(_A					,				_A					,				"""wer""" ), "cer": compute_score(_A					,				_A					,				"""cer""" )}







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,			_KWARGS_DESCRIPTION       )
class 	lowerCamelCase__      (      datasets.Metric       ):





   '''simple docstring'''



   def       _lowerCAmelCase	(							self						)				->  List[str]:






    if self.config_name not in _CONFIG_NAMES:
     raise KeyError(f'''You should supply a configuration name selected in {_CONFIG_NAMES}'''						)

    _lowerCAmelCase			="""int64""" if self.config_name in ["""fleurs-lang_id""", """minds14"""] else """string"""

    return datasets.MetricInfo(
        description=_DESCRIPTION  ,  citation=_CITATION  ,  inputs_description=_KWARGS_DESCRIPTION  ,  features=datasets.Features(
            {"""predictions""": datasets.Value(__UpperCAmelCase						), """references""": datasets.Value(__UpperCAmelCase						)}						)  ,  codebase_urls=[]  ,  reference_urls=[]  ,  format="""numpy"""  ,  )



   def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase  ,  __UpperCAmelCase  ,  __UpperCAmelCase=None  ,  __UpperCAmelCase=None						)				->  List[Any]:
    _lowerCAmelCase			=bleu_kwargs if bleu_kwargs is not None else {}
    _lowerCAmelCase			=wer_kwargs if wer_kwargs is not None else {}

    if self.config_name == "fleurs-lang_id":
     return {"accuracy": simple_accuracy(__UpperCAmelCase  ,  __UpperCAmelCase						)}
    elif self.config_name == "minds14":
     return fa_and_simple_accuracy(__UpperCAmelCase  ,  __UpperCAmelCase						)
    elif self.config_name == "covost2":
     _lowerCAmelCase			=bleu_kwargs.pop("""smooth_method"""  ,  """exp"""						)
     _lowerCAmelCase			=bleu_kwargs.pop("""smooth_value"""  ,  __UpperCAmelCase						)
     _lowerCAmelCase			=bleu_kwargs.pop("""force"""  ,  __UpperCAmelCase						)
     _lowerCAmelCase			=bleu_kwargs.pop("""lowercase"""  ,  __UpperCAmelCase						)
     _lowerCAmelCase			=bleu_kwargs.pop("""tokenize"""  ,  __UpperCAmelCase						)
     _lowerCAmelCase			=bleu_kwargs.pop("""use_effective_order"""  ,  __UpperCAmelCase						)
     return bleu(
         preds=__UpperCAmelCase  ,  labels=__UpperCAmelCase  ,  smooth_method=__UpperCAmelCase  ,  smooth_value=__UpperCAmelCase  ,  force=__UpperCAmelCase  ,  lowercase=__UpperCAmelCase  ,  tokenize=__UpperCAmelCase  ,  use_effective_order=__UpperCAmelCase  ,  )
    elif self.config_name in ["fleurs-asr", "mls", "voxpopuli", "babel"]:
     _lowerCAmelCase			=wer_kwargs.pop("""concatenate_texts"""  ,  __UpperCAmelCase						)
     return wer_and_cer(__UpperCAmelCase  ,  __UpperCAmelCase  ,  __UpperCAmelCase  ,  self.config_name						)
    else:
     raise KeyError(f'''You should supply a configuration name selected in {_CONFIG_NAMES}'''						)





"""simple docstring"""


import argparse
import json
import logging
import os
import sys
from unittest import skip
from unittest.mock import patch

import tensorflow as tf

from transformers.testing_utils import TestCasePlus, get_gpu_count, slow


__A 		= [
    os.path.join(os.path.dirname(__file__), dirname)
    for dirname in [
        'text-generation',
        'text-classification',
        'token-classification',
        'language-modeling',
        'multiple-choice',
        'question-answering',
        'summarization',
        'translation',
        'image-classification',
    ]
]
sys.path.extend(SRC_DIRS)


if SRC_DIRS is not None:
			import run_clm
			import run_image_classification
			import run_mlm
			import run_ner
			import run_qa as run_squad
			import run_summarization
			import run_swag
			import run_text_classification
			import run_translation


logging.basicConfig(level=logging.DEBUG)

__A 		= logging.getLogger()



def 					_lowerCamelCase()       ->				str:
	_lowerCAmelCase			=argparse.ArgumentParser()
	parser.add_argument("""-f""" )
	_lowerCAmelCase			=parser.parse_args()
	return args.f



def 					_lowerCamelCase(__UpperCamelCase )       ->				List[str]:
	_lowerCAmelCase			={}
	_lowerCAmelCase			=os.path.join(__UpperCamelCase					,				"""all_results.json""" )
	if os.path.exists(__UpperCamelCase ):
		with open(__UpperCamelCase					,				"""r""" ) as f:
			_lowerCAmelCase			=json.load(__UpperCamelCase )
	else:
		raise ValueError(F'''can\'t find {path}''' )
	return results



def 					_lowerCamelCase()       ->				List[Any]:
	return bool(tf.config.list_physical_devices("""GPU""" ) )


__A 		= logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)







class 	lowerCamelCase__      (      __magic_name__       ):





			'''simple docstring'''



			@skip("""Skipping until shape inference for to_tf_dataset PR is merged."""						)
			def       _lowerCAmelCase	(							self						)				->  List[str]:
				_lowerCAmelCase			=self.get_auto_remove_tmp_dir()
				_lowerCAmelCase			=f'''
            run_text_classification.py
            --model_name_or_path distilbert-base-uncased
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
            --do_train
            --do_eval
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --learning_rate=1e-4
            --max_steps=10
            --warmup_steps=2
            --seed=42
            --max_seq_length=128
            '''.split()

				if is_cuda_available():
					testargs.append("""--fp16"""						)

				with patch.object(__UpperCAmelCase  ,  """argv"""  ,  __UpperCAmelCase						):
					run_text_classification.main()
					# Reset the mixed precision policy so we don't break other tests
					tf.keras.mixed_precision.set_global_policy("""float32"""						)
					_lowerCAmelCase			=get_results(__UpperCAmelCase						)
					self.assertGreaterEqual(result["""eval_accuracy"""]  ,  0.7_5						)



			def       _lowerCAmelCase	(							self						)				->  int:
				_lowerCAmelCase			=self.get_auto_remove_tmp_dir()
				_lowerCAmelCase			=f'''
            run_clm.py
            --model_name_or_path distilgpt2
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --do_train
            --do_eval
            --block_size 128
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --num_train_epochs 2
            --output_dir {tmp_dir}
            --overwrite_output_dir
            '''.split()

				if len(tf.config.list_physical_devices("""GPU"""						)						) > 1:
					# Skipping because there are not enough batches to train the model + would need a drop_last to work.
					return

				with patch.object(__UpperCAmelCase  ,  """argv"""  ,  __UpperCAmelCase						):
					run_clm.main()
					_lowerCAmelCase			=get_results(__UpperCAmelCase						)
					self.assertLess(result["""eval_perplexity"""]  ,  1_00						)



			def       _lowerCAmelCase	(							self						)				->  str:
				_lowerCAmelCase			=self.get_auto_remove_tmp_dir()
				_lowerCAmelCase			=f'''
            run_mlm.py
            --model_name_or_path distilroberta-base
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --max_seq_length 64
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --do_train
            --do_eval
            --prediction_loss_only
            --num_train_epochs=1
            --learning_rate=1e-4
        '''.split()

				with patch.object(__UpperCAmelCase  ,  """argv"""  ,  __UpperCAmelCase						):
					run_mlm.main()
					_lowerCAmelCase			=get_results(__UpperCAmelCase						)
					self.assertLess(result["""eval_perplexity"""]  ,  42						)



			def       _lowerCAmelCase	(							self						)				->  Union[str, Any]:
				# with so little data distributed training needs more epochs to get the score on par with 0/1 gpu
				_lowerCAmelCase			=7 if get_gpu_count() > 1 else 2

				_lowerCAmelCase			=self.get_auto_remove_tmp_dir()
				_lowerCAmelCase			=f'''
            run_ner.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/conll/sample.json
            --validation_file tests/fixtures/tests_samples/conll/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --do_train
            --do_eval
            --warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=2
            --num_train_epochs={epochs}
            --seed 7
        '''.split()

				with patch.object(__UpperCAmelCase  ,  """argv"""  ,  __UpperCAmelCase						):
					run_ner.main()
					_lowerCAmelCase			=get_results(__UpperCAmelCase						)
					self.assertGreaterEqual(result["""accuracy"""]  ,  0.7_5						)



			def       _lowerCAmelCase	(							self						)				->  str:
				_lowerCAmelCase			=self.get_auto_remove_tmp_dir()
				_lowerCAmelCase			=f'''
            run_qa.py
            --model_name_or_path bert-base-uncased
            --version_2_with_negative
            --train_file tests/fixtures/tests_samples/SQUAD/sample.json
            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_steps=10
            --warmup_steps=2
            --do_train
            --do_eval
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
        '''.split()

				with patch.object(__UpperCAmelCase  ,  """argv"""  ,  __UpperCAmelCase						):
					run_squad.main()
					_lowerCAmelCase			=get_results(__UpperCAmelCase						)
					self.assertGreaterEqual(result["""f1"""]  ,  30						)
					self.assertGreaterEqual(result["""exact"""]  ,  30						)



			def       _lowerCAmelCase	(							self						)				->  int:
				_lowerCAmelCase			=self.get_auto_remove_tmp_dir()
				_lowerCAmelCase			=f'''
            run_swag.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/swag/sample.json
            --validation_file tests/fixtures/tests_samples/swag/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_steps=20
            --warmup_steps=2
            --do_train
            --do_eval
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
        '''.split()

				with patch.object(__UpperCAmelCase  ,  """argv"""  ,  __UpperCAmelCase						):
					run_swag.main()
					_lowerCAmelCase			=get_results(__UpperCAmelCase						)
					self.assertGreaterEqual(result["""val_accuracy"""]  ,  0.8						)



			@slow
			def       _lowerCAmelCase	(							self						)				->  Tuple:
				_lowerCAmelCase			=self.get_auto_remove_tmp_dir()
				_lowerCAmelCase			=f'''
            run_summarization.py
            --model_name_or_path t5-small
            --train_file tests/fixtures/tests_samples/xsum/sample.json
            --validation_file tests/fixtures/tests_samples/xsum/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_steps=50
            --warmup_steps=8
            --do_train
            --do_eval
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
        '''.split()

				with patch.object(__UpperCAmelCase  ,  """argv"""  ,  __UpperCAmelCase						):
					run_summarization.main()
					_lowerCAmelCase			=get_results(__UpperCAmelCase						)
					self.assertGreaterEqual(result["""rouge1"""]  ,  10						)
					self.assertGreaterEqual(result["""rouge2"""]  ,  2						)
					self.assertGreaterEqual(result["""rougeL"""]  ,  7						)
					self.assertGreaterEqual(result["""rougeLsum"""]  ,  7						)



			@slow
			def       _lowerCAmelCase	(							self						)				->  Any:
				_lowerCAmelCase			=self.get_auto_remove_tmp_dir()
				_lowerCAmelCase			=f'''
            run_translation.py
            --model_name_or_path Rocketknight1/student_marian_en_ro_6_1
            --source_lang en
            --target_lang ro
            --train_file tests/fixtures/tests_samples/wmt16/sample.json
            --validation_file tests/fixtures/tests_samples/wmt16/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --warmup_steps=8
            --do_train
            --do_eval
            --learning_rate=3e-3
            --num_train_epochs 12
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --source_lang en_XX
            --target_lang ro_RO
        '''.split()

				with patch.object(__UpperCAmelCase  ,  """argv"""  ,  __UpperCAmelCase						):
					run_translation.main()
					_lowerCAmelCase			=get_results(__UpperCAmelCase						)
					self.assertGreaterEqual(result["""bleu"""]  ,  30						)



			def       _lowerCAmelCase	(							self						)				->  Optional[int]:
				_lowerCAmelCase			=self.get_auto_remove_tmp_dir()
				_lowerCAmelCase			=f'''
            run_image_classification.py
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --model_name_or_path microsoft/resnet-18
            --do_train
            --do_eval
            --learning_rate 1e-4
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --dataloader_num_workers 16
            --num_train_epochs 2
            --train_val_split 0.1
            --seed 42
            --ignore_mismatched_sizes True
            '''.split()

				with patch.object(__UpperCAmelCase  ,  """argv"""  ,  __UpperCAmelCase						):
					run_image_classification.main()
					_lowerCAmelCase			=get_results(__UpperCAmelCase						)
					self.assertGreaterEqual(result["""accuracy"""]  ,  0.7						)



