





'''simple docstring'''

import inspect
import os
from argparse import ArgumentParser, Namespace
from importlib import import_module

import huggingface_hub
import numpy as np
from packaging import version

from .. import (
    FEATURE_EXTRACTOR_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    PROCESSOR_MAPPING,
    TOKENIZER_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoTokenizer,
    is_datasets_available,
    is_tf_available,
    is_torch_available,
)
from ..utils import TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging
from . import BaseTransformersCLICommand


if is_tf_available():
  import tensorflow as tf

  tf.config.experimental.enable_tensor_float_aa_execution(False)

if is_torch_available():
  import torch

if is_datasets_available():
  from datasets import load_dataset


__lowerCAmelCase							:			Tuple				=5E-5  # larger error tolerance than in our internal tests, to avoid flaky user-facing errors





def 		UpperCamelCase       (							_lowerCamelCase      :      Namespace ):
    return PTtoTFCommand(
        args.model_name		, args.local_dir		, args.max_error		, args.new_weights		, args.no_pr		, args.push		, args.extra_commit_description		, args.override_model_class		, )






class      UpperCAmelCase			( UpperCamelCase__       ):
 @staticmethod
 def   UpperCAmelCase_							(       lowercase_   :ArgumentParser							)->							int:
     A__       =    parser.add_parser(
         "pt-to-tf"		, help=(
             "CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."
             " Can also be used to validate existing weights without opening PRs, with --no-pr."
         )		, )
     train_parser.add_argument(
         "--model-name"		, type=lowercase_		, required=lowercase_		, help="The model name, including owner/organization, as seen on the hub."		, )
     train_parser.add_argument(
         "--local-dir"		, type=lowercase_		, default=""		, help="Optional local directory of the model repository. Defaults to /tmp/{model_name}"		, )
     train_parser.add_argument(
         "--max-error"		, type=lowercase_		, default=lowercase_		, help=(
             F"Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk."
         )		, )
     train_parser.add_argument(
         "--new-weights"		, action="store_true"		, help="Optional flag to create new TensorFlow weights, even if they already exist."		, )
     train_parser.add_argument(
         "--no-pr"		, action="store_true"		, help="Optional flag to NOT open a PR with converted weights."							)
     train_parser.add_argument(
         "--push"		, action="store_true"		, help="Optional flag to push the weights directly to `main` (requires permissions)"		, )
     train_parser.add_argument(
         "--extra-commit-description"		, type=lowercase_		, default=""		, help="Optional additional commit description to use when opening a PR (e.g. to tag the owner)."		, )
     train_parser.add_argument(
         "--override-model-class"		, type=lowercase_		, default=lowercase_		, help="If you think you know better than the auto-detector, you can specify the model class here. "
         "Can be either an AutoModel class or a specific model class like BertForSequenceClassification."		, )
     train_parser.set_defaults(func=lowercase_							)
 @staticmethod
 def   UpperCAmelCase_							(       lowercase_   :int		, lowercase_   :int							)->							str:
     A__       =    set(pt_outputs.keys()							)
     A__       =    set(tf_outputs.keys()							)
     if pt_out_attrs != tf_out_attrs:
         raise ValueError(
             F"The model outputs have different attributes, aborting. (Pytorch: {pt_out_attrs}, TensorFlow:"
             F" {tf_out_attrs})"							)

     # 2. For each output attribute, computes the difference
     def _find_pt_tf_differences(lowercase_   :str		, lowercase_   :Optional[int]		, lowercase_   :List[Any]		, lowercase_   :Tuple=""							):
         # If the current attribute is a tensor, it is a leaf and we make the comparison. Otherwise, we will dig in
         # recursivelly, keeping the name of the attribute.
         if isinstance(lowercase_		, torch.Tensor							):
             A__       =    np.max(np.abs(pt_out.numpy() - tf_out.numpy()							)							)
             A__       =    tensor_difference
         else:
             A__       =    attr_name
             for i, pt_item in enumerate(lowercase_							):
                 # If it is a named attribute, we keep the name. Otherwise, just its index.
                 if isinstance(lowercase_		, lowercase_							):
                     A__       =    root_name + pt_item
                     A__       =    tf_out[pt_item]
                     A__       =    pt_out[pt_item]
                 else:
                     A__       =    root_name + F"[{i}]"
                     A__       =    tf_out[i]
                 A__       =    _find_pt_tf_differences(lowercase_		, lowercase_		, lowercase_		, lowercase_							)

         return differences

     return _find_pt_tf_differences(lowercase_		, lowercase_		, {}							)
 def __init__(       self   :Any		, lowercase_   :str		, lowercase_   :str		, lowercase_   :float		, lowercase_   :bool		, lowercase_   :bool		, lowercase_   :bool		, lowercase_   :str		, lowercase_   :str		, *lowercase_   :List[Any]		, )->							List[str]:
     A__       =    logging.get_logger("transformers-cli/pt_to_tf"							)
     A__       =    model_name
     A__       =    local_dir if local_dir else os.path.join("/tmp"		, lowercase_							)
     A__       =    max_error
     A__       =    new_weights
     A__       =    no_pr
     A__       =    push
     A__       =    extra_commit_description
     A__       =    override_model_class
 def   UpperCAmelCase_							(       self   :Any		, lowercase_   :List[Any]		, lowercase_   :List[Any]		, lowercase_   :Dict							)->							List[str]:

     def _get_audio_input():
         A__       =    load_dataset("hf-internal-testing/librispeech_asr_dummy"		, "clean"		, split="validation"							)
         A__       =    ds.sort("id"							).select(range(2							)							)[:2]["audio"]
         A__       =    [x["array"] for x in speech_samples]
         return raw_samples

     A__       =    type(pt_model.config							)
     if model_config_class in PROCESSOR_MAPPING:
         A__       =    AutoProcessor.from_pretrained(self._local_dir							)
         if model_config_class in TOKENIZER_MAPPING and processor.tokenizer.pad_token is None:
             A__       =    processor.tokenizer.eos_token
     elif model_config_class in IMAGE_PROCESSOR_MAPPING:
         A__       =    AutoImageProcessor.from_pretrained(self._local_dir							)
     elif model_config_class in FEATURE_EXTRACTOR_MAPPING:
         A__       =    AutoFeatureExtractor.from_pretrained(self._local_dir							)
     elif model_config_class in TOKENIZER_MAPPING:
         A__       =    AutoTokenizer.from_pretrained(self._local_dir							)
         if processor.pad_token is None:
             A__       =    processor.eos_token
     else:
         raise ValueError(F"Unknown data processing type (model config type: {model_config_class})"							)

     A__       =    set(inspect.signature(pt_model.forward							).parameters.keys()							)
     A__       =    {}
     if "input_ids" in model_forward_signature:
         processor_inputs.update(
             {
                 "text": ["Hi there!", "I am a batch with more than one row and different input lengths."],
                 "padding": True,
                 "truncation": True,
             }							)
     if "pixel_values" in model_forward_signature:
         A__       =    load_dataset("cifar10"		, "plain_text"		, split="test"							)[:2]["img"]
         processor_inputs.update({"images": sample_images}							)
     if "input_features" in model_forward_signature:
         A__       =    inspect.signature(processor.feature_extractor							).parameters
         # Pad to the largest input length by default but take feature extractor default
         # padding value if it exists e.g. "max_length" and is not False or None
         if "padding" in feature_extractor_signature:
             A__       =    feature_extractor_signature["padding"].default
             if default_strategy is not False and default_strategy is not None:
                 A__       =    default_strategy
             else:
                 A__       =    True
         else:
             A__       =    True
         processor_inputs.update({"audio": _get_audio_input(), "padding": padding_strategy}							)
     if "input_values" in model_forward_signature:  # Wav2Vec2 audio input
         processor_inputs.update({"audio": _get_audio_input(), "padding": True}							)
     A__       =    processor(**lowercase_		, return_tensors="pt"							)
     A__       =    processor(**lowercase_		, return_tensors="tf"							)

     # Extra input requirements, in addition to the input modality
     if (
         config.is_encoder_decoder
         or (hasattr(lowercase_		, "encoder"							) and hasattr(lowercase_		, "decoder"							))
         or "decoder_input_ids" in tf_dummy_inputs
     ):
         A__       =    np.asarray([[1], [1]]		, dtype=lowercase_							) * (pt_model.config.decoder_start_token_id or 0)
         pt_input.update({"decoder_input_ids": torch.tensor(lowercase_							)}							)
         tf_input.update({"decoder_input_ids": tf.convert_to_tensor(lowercase_							)}							)

     return pt_input, tf_input



 def   UpperCAmelCase_							(       self   :Optional[int]							)->							Optional[int]:
     # hub version 0.9.0 introduced the possibility of programmatically opening PRs with normal write tokens.
     if version.parse(huggingface_hub.__version__							) < version.parse("0.9.0"							):
         raise ImportError(
             "The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub"
             " installation."							)
     else:
         from huggingface_hub import Repository, create_commit
         from huggingface_hub._commit_api import CommitOperationAdd

     # Fetch remote data
     A__       =    Repository(local_dir=self._local_dir		, clone_from=self._model_name							)

     # Load config and get the appropriate architecture -- the latter is needed to convert the head's weights
     A__       =    AutoConfig.from_pretrained(self._local_dir							)
     A__       =    config.architectures
     if self._override_model_class is not None:
         if self._override_model_class.startswith("TF"							):
             A__       =    [self._override_model_class[2:]]
         else:
             A__       =    [self._override_model_class]
         try:
             A__       =    getattr(import_module("transformers"							)		, architectures[0]							)
         except AttributeError:
             raise ValueError(F"Model class {self._override_model_class} not found in transformers."							)
         try:
             A__       =    getattr(import_module("transformers"							)		, "TF" + architectures[0]							)
         except AttributeError:
             raise ValueError(F"TF model class TF{self._override_model_class} not found in transformers."							)
     elif architectures is None:  # No architecture defined -- use auto classes
         A__       =    getattr(import_module("transformers"							)		, "AutoModel"							)
         A__       =    getattr(import_module("transformers"							)		, "TFAutoModel"							)
         self._logger.warning("No detected architecture, using AutoModel/TFAutoModel"							)
     else:  # Architecture defined -- use it
         if len(lowercase_							) > 1:
             raise ValueError(F"More than one architecture was found, aborting. (architectures = {architectures})"							)
         self._logger.warning(F"Detected architecture: {architectures[0]}"							)
         A__       =    getattr(import_module("transformers"							)		, architectures[0]							)
         try:
             A__       =    getattr(import_module("transformers"							)		, "TF" + architectures[0]							)
         except AttributeError:
             raise AttributeError(F"The TensorFlow equivalent of {architectures[0]} doesn't exist in transformers."							)

        # Check the TF dummy inputs to see what keys we need in the forward pass
     A__       =    tf_class.from_config(lowercase_							)
     A__       =    tf_from_pt_model.dummy_inputs

     del tf_from_pt_model  # Try to keep only one model in memory at a time

     # Load the model and get some basic inputs
     A__       =    pt_class.from_pretrained(self._local_dir							)
     pt_model.eval()

     A__,			A__       =    self.get_inputs(lowercase_		, lowercase_		, lowercase_							)

     with torch.no_grad():
         A__       =    pt_model(**lowercase_		, output_hidden_states=lowercase_							)
     del pt_model  # will no longer be used, and may have a large memory footprint

     A__       =    tf_class.from_pretrained(self._local_dir		, from_pt=lowercase_							)
     A__       =    tf_from_pt_model(**lowercase_		, output_hidden_states=lowercase_		, training=lowercase_							)

     # Confirms that cross loading PT weights into TF worked.
     A__       =    self.find_pt_tf_differences(lowercase_		, lowercase_							)
     A__       =    {k: v for k, v in crossload_differences.items() if "hidden" not in k}
     A__       =    {k: v for k, v in crossload_differences.items() if "hidden" in k}
     if len(lowercase_							) == 0 and architectures is not None:
         raise ValueError(
             F"Something went wrong -- the config file has architectures ({architectures}), but no model head"
             " output was found. All outputs start with 'hidden'"							)
     A__       =    max(output_differences.values()							) if output_differences else 0.0
     A__       =    max(hidden_differences.values()							)
     if max_crossload_output_diff > self._max_error or max_crossload_hidden_diff > self._max_error:
         raise ValueError(
             "The cross-loaded TensorFlow model has different outputs, something went wrong!\n"
             + F"\nList of maximum output differences above the threshold ({self._max_error}):\n"
             + "\n".join([F"{k}: {v:.3e}" for k, v in output_differences.items() if v > self._max_error]							)
             + F"\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n"
             + "\n".join([F"{k}: {v:.3e}" for k, v in hidden_differences.items() if v > self._max_error]							)							)

     # Save the weights in a TF format (if needed) and confirms that the results are still good
     A__       =    os.path.join(self._local_dir		, lowercase_							)
     A__       =    os.path.join(self._local_dir		, lowercase_							)
     if (not os.path.exists(lowercase_							) and not os.path.exists(lowercase_							)) or self._new_weights:
         tf_from_pt_model.save_pretrained(self._local_dir							)
     del tf_from_pt_model  # will no longer be used, and may have a large memory footprint

     A__       =    tf_class.from_pretrained(self._local_dir							)
     A__       =    tf_model(**lowercase_		, output_hidden_states=lowercase_							)

     A__       =    self.find_pt_tf_differences(lowercase_		, lowercase_							)
     A__       =    {k: v for k, v in conversion_differences.items() if "hidden" not in k}
     A__       =    {k: v for k, v in conversion_differences.items() if "hidden" in k}
     if len(lowercase_							) == 0 and architectures is not None:
         raise ValueError(
             F"Something went wrong -- the config file has architectures ({architectures}), but no model head"
             " output was found. All outputs start with 'hidden'"							)
     A__       =    max(output_differences.values()							) if output_differences else 0.0
     A__       =    max(hidden_differences.values()							)
     if max_conversion_output_diff > self._max_error or max_conversion_hidden_diff > self._max_error:
         raise ValueError(
             "The converted TensorFlow model has different outputs, something went wrong!\n"
             + F"\nList of maximum output differences above the threshold ({self._max_error}):\n"
             + "\n".join([F"{k}: {v:.3e}" for k, v in output_differences.items() if v > self._max_error]							)
             + F"\n\nList of maximum hidden layer differences above the threshold ({self._max_error}):\n"
             + "\n".join([F"{k}: {v:.3e}" for k, v in hidden_differences.items() if v > self._max_error]							)							)

     A__       =    "Update TF weights" if self._new_weights else "Add TF weights"
     if self._push:
         repo.git_add(auto_lfs_track=lowercase_							)
         repo.git_commit(lowercase_							)
         repo.git_push(blocking=lowercase_							)  # this prints a progress bar with the upload
         self._logger.warning(F"TF weights pushed into {self._model_name}"							)
     elif not self._no_pr:
         self._logger.warning("Uploading the weights into a new PR..."							)
         A__       =    (
             "Model converted by the [`transformers`' `pt_to_tf`"
             " CLI](https://github.com/huggingface/transformers/blob/main/src/transformers/commands/pt_to_tf.py). "
             "All converted model outputs and hidden layers were validated against its PyTorch counterpart.\n\n"
             F"Maximum crossload output difference={max_crossload_output_diff:.3e}; "
             F"Maximum crossload hidden layer difference={max_crossload_hidden_diff:.3e};\n"
             F"Maximum conversion output difference={max_conversion_output_diff:.3e}; "
             F"Maximum conversion hidden layer difference={max_conversion_hidden_diff:.3e};\n"
         )
         if self._max_error > MAX_ERROR:
             commit_descrition += (
                 F"\n\nCAUTION: The maximum admissible error was manually increased to {self._max_error}!"
             )
         if self._extra_commit_description:
             commit_descrition += "\n\n" + self._extra_commit_description

         # sharded model -> adds all related files (index and .h5 shards)
         if os.path.exists(lowercase_							):
             A__       =    [
                 CommitOperationAdd(path_in_repo=lowercase_		, path_or_fileobj=lowercase_							)
             ]
             for shard_path in tf.io.gfile.glob(self._local_dir + "/tf_model-*.h5"							):
                 operations += [
                     CommitOperationAdd(path_in_repo=os.path.basename(lowercase_							)		, path_or_fileobj=lowercase_							)
                 ]
         else:
             A__       =    [CommitOperationAdd(path_in_repo=lowercase_		, path_or_fileobj=lowercase_							)]

         A__       =    create_commit(
             repo_id=self._model_name		, operations=lowercase_		, commit_message=lowercase_		, commit_description=lowercase_		, repo_type="model"		, create_pr=lowercase_		, ).pr_url
         self._logger.warning(F"PR open in {hub_pr_url}"							)










'''simple docstring'''

import torch

from transformers import AutoTokenizer






class      UpperCAmelCase			( UpperCamelCase__       ):
 def __init__(       self   :Any		, lowercase_   :Dict							)->							Any:
     A__       =    AutoTokenizer.from_pretrained(lowercase_							)
 def   UpperCAmelCase_							(       self   :Tuple		, lowercase_   :int							)->							Optional[int]:

     if isinstance(lowercase_		, lowercase_							) and all(isinstance(_x		, lowercase_							) for _x in x							):
         A__       =    None
         for l in x:
             A__       =    self.tokenizer(
                 lowercase_		, padding="max_length"		, max_length=3_84		, truncation=lowercase_		, return_tensors="pt"		, )
             A__       =    torch.tensor([len(lowercase_							)]							)
             if d is not None:
                 for k in d.keys():
                     A__       =    torch.cat((d[k], t[k])		, 0							)
             else:
                 A__       =    t

         A__       =    torch.tensor(self.tokenizer.convert_tokens_to_ids("[E]"							)							)
         A__       =    torch.tensor(self.tokenizer.convert_tokens_to_ids("[/E]"							)							)

     elif isinstance(lowercase_		, lowercase_							) and all(isinstance(_x		, lowercase_							) for _x in x							):
         A__       =    self.tokenizer(
             lowercase_		, padding="max_length"		, max_length=3_84		, truncation=lowercase_		, return_tensors="pt"		, )

     else:
         raise Exception(
             "Type of parameter x was not recognized! Only `list of strings` for query or `list of lists of"
             " strings` for supports are supported."							)

     return d



 def   UpperCAmelCase_							(       self   :Union[str, Any]		, lowercase_   :List[str]		, lowercase_   :str		, lowercase_   :str		, lowercase_   :List[Any]		, lowercase_   :Dict=0.7_0							)->							int:
     A__       =    []
     for idx in range(len(W_query["input_ids"]							)							):
         A__       =    A__       =    range(p_start.shape[1]							)

         A__       =    []
         for start_id in start_indexes:
             for end_id in end_indexes:
                 if start_id < end_id:
                     output.append(
                         (
                             start_id,
                             end_id,
                             p_start[idx][start_id].item(),
                             p_end[idx][end_id].item(),
                         )							)

         output.sort(key=lambda lowercase_							: (tup[2] * tup[3])		, reverse=lowercase_							)
         A__       =    []
         for k in range(len(lowercase_							)							):
             if output[k][2] * output[k][3] >= thresh:
                 A__,			A__       =    output[k][0], output[k][1]
                 A__       =    self.tokenizer.decode(W_query["input_ids"][idx][c_start_pos:c_end_pos]							)
                 temp.append((decoded, output[k][2] * output[k][3])							)

         final_outputs.append(lowercase_							)

     return final_outputs



