


import argparse
import logging
import math
import time

import torch

from transformers import TransfoXLCorpus, TransfoXLLMHeadModel


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
SCREAMING_SNAKE_CASE	:List[Any]  		=     logging.getLogger(__name__)



def 			_lowerCAmelCase						(							)->Optional[Any]:
   '''simple docstring'''






   snake_case_					    =					argparse.ArgumentParser(description="PyTorch Transformer Language Model"							)
   parser.add_argument("--model_name"		,						type=lowerCAmelCase_		,						default="transfo-xl-wt103"		,						help="pretrained model name"							)
   parser.add_argument(
       "--split"		,						type=lowerCAmelCase_		,						default="test"		,						choices=["all", "valid", "test"]		,						help="which split to evaluate"							)
   parser.add_argument("--batch_size"		,						type=lowerCAmelCase_		,						default=10		,						help="batch size"							)
   parser.add_argument("--tgt_len"		,						type=lowerCAmelCase_		,						default=128		,						help="number of tokens to predict"							)
   parser.add_argument("--ext_len"		,						type=lowerCAmelCase_		,						default=0		,						help="length of the extended context"							)
   parser.add_argument("--mem_len"		,						type=lowerCAmelCase_		,						default=1_600		,						help="length of the retained previous heads"							)
   parser.add_argument("--clamp_len"		,						type=lowerCAmelCase_		,						default=1_000		,						help="max positional embedding index"							)
   parser.add_argument("--no_cuda"		,						action="store_true"		,						help="Do not use CUDA even though CUA is available"							)
   parser.add_argument("--work_dir"		,						type=lowerCAmelCase_		,						required=lowerCAmelCase_		,						help="path to the work_dir"							)
   parser.add_argument("--no_log"		,						action="store_true"		,						help="do not log the eval result"							)
   parser.add_argument("--same_length"		,						action="store_true"		,						help="set same length attention with masking"							)
   parser.add_argument("--server_ip"		,						type=lowerCAmelCase_		,						default=""		,						help="Can be used for distant debugging."							)
   parser.add_argument("--server_port"		,						type=lowerCAmelCase_		,						default=""		,						help="Can be used for distant debugging."							)
   snake_case_					    =					parser.parse_args()
   assert args.ext_len >= 0, "extended context length must be non-negative"

   if args.server_ip and args.server_port:
      # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
      import ptvsd

      print("Waiting for debugger attach"							)
      ptvsd.enable_attach(address=(args.server_ip, args.server_port)		,						redirect_output=lowerCAmelCase_							)
      ptvsd.wait_for_attach()

   snake_case_					    =					torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu"							)
   logger.info("device: {}".format(lowerCAmelCase_							)							)

   # Load a pre-processed dataset
   # You can also build the corpus yourself using TransfoXLCorpus methods
   # The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
   # and tokenizing the dataset
   # The pre-processed corpus is a convertion (using the conversion script )
   snake_case_					    =					TransfoXLCorpus.from_pretrained(args.model_name							)

   snake_case_					    =					corpus.get_iterator("valid"		,						args.batch_size		,						args.tgt_len		,						device=lowerCAmelCase_		,						ext_len=args.ext_len							)
   snake_case_					    =					corpus.get_iterator("test"		,						args.batch_size		,						args.tgt_len		,						device=lowerCAmelCase_		,						ext_len=args.ext_len							)

   # Load a pre-trained model
   snake_case_					    =					TransfoXLLMHeadModel.from_pretrained(args.model_name							)
   model.to(lowerCAmelCase_							)

   logger.info(
       "Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}".format(
           args.batch_size		,						args.tgt_len		,						args.ext_len		,						args.mem_len		,						args.clamp_len							)							)

   model.reset_memory_length(args.mem_len							)
   if args.clamp_len > 0:
      snake_case_					    =					args.clamp_len
   if args.same_length:
      snake_case_					    =					True

   ###############################################################################
   # Evaluation code
   ###############################################################################
   def evaluate(lowerCAmelCase_							:Any							):
      # Turn on evaluation mode which disables dropout.
      model.eval()
      snake_case_    ,    snake_case_					    =					0, 0.0
      snake_case_					    =					time.time()
      with torch.no_grad():
         snake_case_					    =					None
         for idx, (data, target, seq_len) in enumerate(lowerCAmelCase_							):
            snake_case_					    =					model(lowerCAmelCase_		,						lm_labels=lowerCAmelCase_		,						mems=lowerCAmelCase_							)
            snake_case_    ,    snake_case_    ,    snake_case_					    =					ret
            snake_case_					    =					loss.mean()
            total_loss += seq_len * loss.item()
            total_len += seq_len
         snake_case_					    =					time.time() - start_time
      logger.info("Time : {:.2f}s, {:.2f}ms/segment".format(lowerCAmelCase_		,						1_000 * total_time / (idx + 1)							)							)
      return total_loss / total_len

   # Run on test data.
   if args.split == "all":
      snake_case_					    =					evaluate(lowerCAmelCase_							)
      snake_case_					    =					evaluate(lowerCAmelCase_							)
   elif args.split == "valid":
      snake_case_					    =					evaluate(lowerCAmelCase_							)
      snake_case_					    =					None
   elif args.split == "test":
      snake_case_					    =					evaluate(lowerCAmelCase_							)
      snake_case_					    =					None

   def format_log(lowerCAmelCase_							:Dict		,						lowerCAmelCase_							:str							):
      snake_case_					    =					"| {0} loss {1:5.2f} | {0} ppl {2:9.3f} ".format(lowerCAmelCase_		,						lowerCAmelCase_		,						math.exp(lowerCAmelCase_							)							)
      return log_str

   snake_case_					    =					""
   if valid_loss is not None:
      log_str += format_log(lowerCAmelCase_		,						"valid"							)
   if test_loss is not None:
      log_str += format_log(lowerCAmelCase_		,						"test"							)

   logger.info("=" * 100							)
   logger.info(lowerCAmelCase_							)
   logger.info("=" * 100							)


if __name__ == "__main__":
   main()








import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    SummarizationPipeline,
    TFPreTrainedModel,
    pipeline,
)
from transformers.testing_utils import get_gpu_count, is_pipeline_test, require_tf, require_torch, slow, torch_device
from transformers.tokenization_utils import TruncationStrategy

from .test_pipelines_common import ANY


SCREAMING_SNAKE_CASE	:Tuple  		=     -1 if torch_device == '''cpu''' else 0


@is_pipeline_test
class     __lowerCAmelCase     ( unittest.TestCase					):






  """simple docstring"""





  _SCREAMING_SNAKE_CASE		    =       MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
  _SCREAMING_SNAKE_CASE		    =       TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING



  def 					lowerCAmelCase__  (     self       :       List[Any]    ,				_lowerCAmelCase       :       Tuple    ,				_lowerCAmelCase       :       List[str]    ,				_lowerCAmelCase       :       List[str]  )   -> Dict:




     """simple docstring"""
     snake_case_					    =					SummarizationPipeline(model=_lowerCAmelCase    ,				tokenizer=_lowerCAmelCase  )
     return summarizer, ["(CNN)The Palestinian Authority officially became", "Some other text"]



  def 					lowerCAmelCase__  (     self       :       Union[str, Any]    ,				_lowerCAmelCase       :       int    ,				_lowerCAmelCase       :       int  )   -> Union[str, Any]:




     """simple docstring"""
     snake_case_					    =					summarizer.model

     snake_case_					    =					summarizer("(CNN)The Palestinian Authority officially became"  )
     self.assertEqual(_lowerCAmelCase    ,				[{"summary_text": ANY(_lowerCAmelCase  )}]  )

     snake_case_					    =					summarizer(
         "(CNN)The Palestinian Authority officially became "    ,				num_beams=2    ,				min_length=2    ,				max_length=5    ,				)
     self.assertEqual(_lowerCAmelCase    ,				[{"summary_text": ANY(_lowerCAmelCase  )}]  )

     # Some models (Switch Transformers, LED, T5, LongT5, etc) can handle long sequences.
     snake_case_					    =					[
         "SwitchTransformersConfig",
         "T5Config",
         "LongT5Config",
         "LEDConfig",
         "PegasusXConfig",
         "FSMTConfig",
         "M2M100Config",
         "ProphetNetConfig",  # positional embeddings up to a fixed maximum size (otherwise clamping the values)
     ]
     if model.config.__class__.__name__ not in model_can_handle_longer_seq:
        # Too long and exception is expected.
        # For TF models, if the weights are initialized in GPU context, we won't get expected index error from
        # the embedding layer.
        if not (
            isinstance(_lowerCAmelCase    ,				_lowerCAmelCase  )
            and get_gpu_count() > 0
            and len(summarizer.model.trainable_weights  ) > 0
        ):
           with self.assertRaises(_lowerCAmelCase  ):
              snake_case_					    =					summarizer("This " * 1_0_0_0  )
     snake_case_					    =					summarizer("This " * 1_0_0_0    ,				truncation=TruncationStrategy.ONLY_FIRST  )



  @require_torch
  def 					lowerCAmelCase__  (     self       :       List[Any]  )   -> str:




     """simple docstring"""
     snake_case_					    =					pipeline(task="summarization"    ,				model="sshleifer/tiny-mbart"    ,				framework="pt"  )
     snake_case_					    =					summarizer("This is a small test"  )
     self.assertEqual(
         _lowerCAmelCase    ,				[
             {
                 "summary_text": "เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป"
             }
         ]    ,				)



  @require_tf
  def 					lowerCAmelCase__  (     self       :       Dict  )   -> Optional[int]:




     """simple docstring"""
     snake_case_					    =					pipeline(task="summarization"    ,				model="sshleifer/tiny-mbart"    ,				framework="tf"  )
     snake_case_					    =					summarizer("This is a small test"  )
     self.assertEqual(
         _lowerCAmelCase    ,				[
             {
                 "summary_text": "เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป"
             }
         ]    ,				)



  @require_torch
  @slow
  def 					lowerCAmelCase__  (     self       :       List[Any]  )   -> Union[str, Any]:




     """simple docstring"""
     snake_case_					    =					pipeline(task="summarization"    ,				device=_lowerCAmelCase  )
     snake_case_					    =					(
         " (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on"
         " Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The"
         " formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based."
         " The Palestinians signed the ICC's founding Rome Statute in January, when they also accepted its"
         " jurisdiction over alleged crimes committed \"in the occupied Palestinian territory, including East"
         " Jerusalem, since June 13, 2014.\" Later that month, the ICC opened a preliminary examination into the"
         " situation in Palestinian territories, paving the way for possible war crimes investigations against"
         " Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and"
         " the United States, neither of which is an ICC member, opposed the Palestinians' efforts to join the"
         " body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday's ceremony, said it was a"
         " move toward greater justice. \"As Palestine formally becomes a State Party to the Rome Statute today, the"
         " world is also a step closer to ending a long era of impunity and injustice,\" he said, according to an"
         " ICC news release. \"Indeed, today brings us closer to our shared goals of justice and peace.\" Judge"
         " Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the"
         " Palestinians. \"As the Rome Statute today enters into force for the State of Palestine, Palestine"
         " acquires all the rights as well as responsibilities that come with being a State Party to the Statute."
         " These are substantive commitments, which cannot be taken lightly,\" she said. Rights group Human Rights"
         " Watch welcomed the development. \"Governments seeking to penalize Palestine for joining the ICC should"
         " immediately end their pressure, and countries that support universal acceptance of the court's treaty"
         " should speak out to welcome its membership,\" said Balkees Jarrah, international justice counsel for the"
         " group. \"What's objectionable is the attempts to undermine international justice, not Palestine's"
         " decision to join a treaty to which over 100 countries around the world are members.\" In January, when"
         " the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an"
         " outrage, saying the court was overstepping its boundaries. The United States also said it \"strongly\""
         " disagreed with the court's decision. \"As we have said repeatedly, we do not believe that Palestine is a"
         " state and therefore we do not believe that it is eligible to join the ICC,\" the State Department said in"
         " a statement. It urged the warring sides to resolve their differences through direct negotiations. \"We"
         " will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace,\""
         " it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the"
         " territories as \"Palestine.\" While a preliminary examination is not a formal investigation, it allows the"
         " court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou"
         " Bensouda said her office would \"conduct its analysis in full independence and impartiality.\" The war"
         " between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry"
         " will include alleged war crimes committed since June. The International Criminal Court was set up in"
         " 2002 to prosecute genocide, crimes against humanity and war crimes. CNN's Vasco Cotovio, Kareem Khadder"
         " and Faith Karimi contributed to this report."
     )
     snake_case_					    =					(
         " The Palestinian Authority becomes the 123rd member of the International Criminal Court . The move gives"
         " the court jurisdiction over alleged crimes in Palestinian territories . Israel and the United States"
         " opposed the Palestinians' efforts to join the court . Rights group Human Rights Watch welcomes the move,"
         " says governments seeking to penalize Palestine should end pressure ."
     )
     snake_case_					    =					summarizer(_lowerCAmelCase  )
     self.assertEqual(result[0]["summary_text"]    ,				_lowerCAmelCase  )




