





import subprocess
import sys
import warnings
from argparse import ArgumentParser
from pathlib import Path

from packaging import version

from .. import AutoFeatureExtractor, AutoImageProcessor, AutoProcessor, AutoTokenizer
from ..utils import logging
from ..utils.import_utils import is_optimum_available
from .convert import export, validate_model_outputs
from .features import FeaturesManager
from .utils import get_preprocessor


_UpperCamelCase       :				List[str]           =			"""1.5.0"""

_UpperCamelCase       :				Optional[Any]           =			["""vision-encoder-decoder"""]
def  __UpperCamelCase    (						snake_case      )    ->		Tuple:


   '''simple docstring'''







   if is_optimum_available():
      from optimum.version import __version__ as optimum_version

      __A    =  version.parse(snake_case      )
      if parsed_optimum_version < version.parse(snake_case      ):
         raise RuntimeError(
             F"transformers.onnx requires optimum >= {MIN_OPTIMUM_VERSION} but {optimum_version} is installed. You "
             '''can upgrade optimum by running: pip install -U optimum[exporters]'''      )
   else:
      raise RuntimeError(
          '''transformers.onnx requires optimum to run, you can install the library by running: pip install '''
          '''optimum[exporters]'''      )
   __A    =  [
       sys.executable,
       '''-m''',
       '''optimum.exporters.onnx''',
       F"--model {args.model}",
       F"--task {args.feature}",
       F"--framework {args.framework}" if args.framework is not None else '''''',
       F"{args.output}",
   ]
   __A    =  subprocess.Popen(''' '''.join(snake_case      )							,			stdout=subprocess.PIPE							,			shell=snake_case      )
   proc.wait()

   logger.info(
       '''The export was done by optimum.exporters.onnx. We recommend using to use this package directly in future, as '''
       '''transformers.onnx is deprecated, and will be removed in v5. You can find more information here: '''
       '''https://huggingface.co/docs/optimum/exporters/onnx/usage_guides/export_a_model.'''      )
def  __UpperCamelCase    (						snake_case      )    ->		List[Any]:


   '''simple docstring'''







   __A    =  args.output if args.output.is_file() else args.output.joinpath('''model.onnx'''      )
   if not args.output.parent.exists():
      args.output.parent.mkdir(parents=snake_case      )

   # Allocate the model
   __A    =  FeaturesManager.get_model_from_feature(
       args.feature							,			args.model							,			framework=args.framework							,			cache_dir=args.cache_dir      )

   __A   ,		__A    =  FeaturesManager.check_supported_model_or_raise(snake_case							,			feature=args.feature      )
   __A    =  model_onnx_config(model.config      )

   if model_kind in ENCODER_DECODER_MODELS:
      __A    =  model.get_encoder()
      __A    =  model.get_decoder()

      __A    =  onnx_config.get_encoder_config(encoder_model.config      )
      __A    =  onnx_config.get_decoder_config(
          encoder_model.config							,			decoder_model.config							,			feature=args.feature      )

      if args.opset is None:
         __A    =  max(encoder_onnx_config.default_onnx_opset							,			decoder_onnx_config.default_onnx_opset      )

      if args.opset < min(encoder_onnx_config.default_onnx_opset							,			decoder_onnx_config.default_onnx_opset      ):
         raise ValueError(
             F"Opset {args.opset} is not sufficient to export {model_kind}. At least "
             F" {min(encoder_onnx_config.default_onnx_opset							,			decoder_onnx_config.default_onnx_opset      )} is required."      )

      __A    =  AutoFeatureExtractor.from_pretrained(args.model      )

      __A   ,		__A    =  export(
          snake_case							,			snake_case							,			snake_case							,			args.opset							,			args.output.parent.joinpath('''encoder_model.onnx'''      )							,			)

      validate_model_outputs(
          snake_case							,			snake_case							,			snake_case							,			args.output.parent.joinpath('''encoder_model.onnx'''      )							,			snake_case							,			args.atol if args.atol else encoder_onnx_config.atol_for_validation							,			)

      __A    =  AutoTokenizer.from_pretrained(args.model      )

      __A   ,		__A    =  export(
          snake_case							,			snake_case							,			snake_case							,			args.opset							,			args.output.parent.joinpath('''decoder_model.onnx'''      )							,			)

      validate_model_outputs(
          snake_case							,			snake_case							,			snake_case							,			args.output.parent.joinpath('''decoder_model.onnx'''      )							,			snake_case							,			args.atol if args.atol else decoder_onnx_config.atol_for_validation							,			)
      logger.info(
          F"All good, model saved at: {args.output.parent.joinpath('encoder_model.onnx'      ).as_posix()},"
          F" {args.output.parent.joinpath('decoder_model.onnx'      ).as_posix()}"      )

   else:
      # Instantiate the appropriate preprocessor
      if args.preprocessor == "auto":
         __A    =  get_preprocessor(args.model      )
      elif args.preprocessor == "tokenizer":
         __A    =  AutoTokenizer.from_pretrained(args.model      )
      elif args.preprocessor == "image_processor":
         __A    =  AutoImageProcessor.from_pretrained(args.model      )
      elif args.preprocessor == "feature_extractor":
         __A    =  AutoFeatureExtractor.from_pretrained(args.model      )
      elif args.preprocessor == "processor":
         __A    =  AutoProcessor.from_pretrained(args.model      )
      else:
         raise ValueError(F"Unknown preprocessor type '{args.preprocessor}'"      )

      # Ensure the requested opset is sufficient
      if args.opset is None:
         __A    =  onnx_config.default_onnx_opset

      if args.opset < onnx_config.default_onnx_opset:
         raise ValueError(
             F"Opset {args.opset} is not sufficient to export {model_kind}. "
             F"At least  {onnx_config.default_onnx_opset} is required."      )

      __A   ,		__A    =  export(
          snake_case							,			snake_case							,			snake_case							,			args.opset							,			args.output							,			)

      if args.atol is None:
         __A    =  onnx_config.atol_for_validation

      validate_model_outputs(snake_case							,			snake_case							,			snake_case							,			args.output							,			snake_case							,			args.atol      )
      logger.info(F"All good, model saved at: {args.output.as_posix()}"      )
      warnings.warn(
          '''The export was done by transformers.onnx which is deprecated and will be removed in v5. We recommend'''
          ''' using optimum.exporters.onnx in future. You can find more information here:'''
          ''' https://huggingface.co/docs/optimum/exporters/onnx/usage_guides/export_a_model.'''							,			snake_case							,			)
def  __UpperCamelCase    (						)    ->		str:


   '''simple docstring'''







   __A    =  ArgumentParser('''Hugging Face Transformers ONNX exporter'''      )
   parser.add_argument(
       '''-m'''							,			'''--model'''							,			type=snake_case							,			required=snake_case							,			help='''Model ID on huggingface.co or path on disk to load model from.'''      )
   parser.add_argument(
       '''--feature'''							,			default='''default'''							,			help='''The type of features to export the model with.'''							,			)
   parser.add_argument('''--opset'''							,			type=snake_case							,			default=snake_case							,			help='''ONNX opset version to export the model with.'''      )
   parser.add_argument(
       '''--atol'''							,			type=snake_case							,			default=snake_case							,			help='''Absolute difference tolerance when validating the model.'''      )
   parser.add_argument(
       '''--framework'''							,			type=snake_case							,			choices=['''pt''', '''tf''']							,			default=snake_case							,			help=(
           '''The framework to use for the ONNX export.'''
           ''' If not provided, will attempt to use the local checkpoint\'s original framework'''
           ''' or what is available in the environment.'''
       )							,			)
   parser.add_argument('''output'''							,			type=snake_case							,			help='''Path indicating where to store generated ONNX model.'''      )
   parser.add_argument('''--cache_dir'''							,			type=snake_case							,			default=snake_case							,			help='''Path indicating where to store cache.'''      )
   parser.add_argument(
       '''--preprocessor'''							,			type=snake_case							,			choices=['''auto''', '''tokenizer''', '''feature_extractor''', '''image_processor''', '''processor''']							,			default='''auto'''							,			help='''Which type of preprocessor to use. \'auto\' tries to automatically detect it.'''							,			)
   parser.add_argument(
       '''--export_with_transformers'''							,			action='''store_true'''							,			help=(
           '''Whether to use transformers.onnx instead of optimum.exporters.onnx to perform the ONNX export. It can be '''
           '''useful when exporting a model supported in transformers but not in optimum, otherwise it is not '''
           '''recommended.'''
       )							,			)

   __A    =  parser.parse_args()
   if args.export_with_transformers or not is_optimum_available():
      export_with_transformers(snake_case      )
   else:
      export_with_optimum(snake_case      )


if __name__ == "__main__":
    _UpperCamelCase       :				List[Any]           =			logging.get_logger("""transformers.onnx""")  # pylint: disable=invalid-name
    logger.setLevel(logging.INFO)
    main()








import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase       :				Optional[Any]           =			logging.get_logger(__name__)


_UpperCamelCase       :				Optional[int]           =			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
_UpperCamelCase       :				int           =			{
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
}

_UpperCamelCase       :				str           =			{
    """RUCAIBox/mvp""": 1_0_2_4,
}
@lru_cache()
def  __UpperCamelCase    (						)    ->		Tuple:


   '''simple docstring'''







   __A    =  (
       list(range(ord('''!'''      )							,			ord('''~'''      ) + 1      )      ) + list(range(ord('''¡'''      )							,			ord('''¬'''      ) + 1      )      ) + list(range(ord('''®'''      )							,			ord('''ÿ'''      ) + 1      )      )
   )
   __A    =  bs[:]
   __A    =  0
   for b in range(2**8      ):
      if b not in bs:
         bs.append(snake_case      )
         cs.append(2**8 + n      )
         n += 1
   __A    =  [chr(snake_case      ) for n in cs]
   return dict(zip(snake_case							,			snake_case      )      )
def  __UpperCamelCase    (						snake_case      )    ->		int:


   '''simple docstring'''







   __A    =  set()
   __A    =  word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)      )
      __A    =  char
   return pairs







class 		_lowerCAmelCase(		_a):

  """simple docstring"""







  lowerCamelCase__     	=       VOCAB_FILES_NAMES
  lowerCamelCase__     	=       PRETRAINED_VOCAB_FILES_MAP
  lowerCamelCase__     	=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCamelCase__     	=       ['''input_ids''', '''attention_mask''']

  def __init__(   self    ,							UpperCAmelCase    ,							UpperCAmelCase    ,							UpperCAmelCase="replace"    ,							UpperCAmelCase="<s>"    ,							UpperCAmelCase="</s>"    ,							UpperCAmelCase="</s>"    ,							UpperCAmelCase="<s>"    ,							UpperCAmelCase="<unk>"    ,							UpperCAmelCase="<pad>"    ,							UpperCAmelCase="<mask>"    ,							UpperCAmelCase=False    ,							**UpperCAmelCase    ,							)->   Optional[int]:
     __A    =  AddedToken(UpperCAmelCase    ,							lstrip=UpperCAmelCase    ,							rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,							UpperCAmelCase      ) else bos_token
     __A    =  AddedToken(UpperCAmelCase    ,							lstrip=UpperCAmelCase    ,							rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,							UpperCAmelCase      ) else eos_token
     __A    =  AddedToken(UpperCAmelCase    ,							lstrip=UpperCAmelCase    ,							rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,							UpperCAmelCase      ) else sep_token
     __A    =  AddedToken(UpperCAmelCase    ,							lstrip=UpperCAmelCase    ,							rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,							UpperCAmelCase      ) else cls_token
     __A    =  AddedToken(UpperCAmelCase    ,							lstrip=UpperCAmelCase    ,							rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,							UpperCAmelCase      ) else unk_token
     __A    =  AddedToken(UpperCAmelCase    ,							lstrip=UpperCAmelCase    ,							rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,							UpperCAmelCase      ) else pad_token

     # Mask token behave like a normal word, i.e. include the space before it
     __A    =  AddedToken(UpperCAmelCase    ,							lstrip=UpperCAmelCase    ,							rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,							UpperCAmelCase      ) else mask_token

     super().__init__(
         errors=UpperCAmelCase    ,							bos_token=UpperCAmelCase    ,							eos_token=UpperCAmelCase    ,							unk_token=UpperCAmelCase    ,							sep_token=UpperCAmelCase    ,							cls_token=UpperCAmelCase    ,							pad_token=UpperCAmelCase    ,							mask_token=UpperCAmelCase    ,							add_prefix_space=UpperCAmelCase    ,							**UpperCAmelCase    ,							)
     with open(UpperCAmelCase    ,							encoding='''utf-8'''      ) as vocab_handle:
        __A    =  json.load(UpperCAmelCase      )
     __A    =  {v: k for k, v in self.encoder.items()}
     __A    =  errors  # how to handle errors in decoding
     __A    =  bytes_to_unicode()
     __A    =  {v: k for k, v in self.byte_encoder.items()}
     with open(UpperCAmelCase    ,							encoding='''utf-8'''      ) as merges_handle:
        __A    =  merges_handle.read().split('''\n'''      )[1:-1]
     __A    =  [tuple(merge.split()      ) for merge in bpe_merges]
     __A    =  dict(zip(UpperCAmelCase    ,							range(len(UpperCAmelCase      )      )      )      )
     __A    =  {}
     __A    =  add_prefix_space

     # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
     __A    =  re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''      )

  @property
  def 						SCREAMING_SNAKE_CASE__			(   self      )->   Tuple:
     return len(self.encoder      )

  def 						SCREAMING_SNAKE_CASE__			(   self      )->   str:
     return dict(self.encoder    ,							**self.added_tokens_encoder      )

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase      )->   List[str]:
     if token in self.cache:
        return self.cache[token]
     __A    =  tuple(UpperCAmelCase      )
     __A    =  get_pairs(UpperCAmelCase      )

     if not pairs:
        return token

     while True:
        __A    =  min(UpperCAmelCase    ,							key=lambda UpperCAmelCase      : self.bpe_ranks.get(UpperCAmelCase    ,							float('''inf'''      )      )      )
        if bigram not in self.bpe_ranks:
           break
        __A   ,		__A    =  bigram
        __A    =  []
        __A    =  0
        while i < len(UpperCAmelCase      ):
           try:
              __A    =  word.index(UpperCAmelCase    ,							UpperCAmelCase      )
           except ValueError:
              new_word.extend(word[i:]      )
              break
           else:
              new_word.extend(word[i:j]      )
              __A    =  j

           if word[i] == first and i < len(UpperCAmelCase      ) - 1 and word[i + 1] == second:
              new_word.append(first + second      )
              i += 2
           else:
              new_word.append(word[i]      )
              i += 1
        __A    =  tuple(UpperCAmelCase      )
        __A    =  new_word
        if len(UpperCAmelCase      ) == 1:
           break
        else:
           __A    =  get_pairs(UpperCAmelCase      )
     __A    =  ''' '''.join(UpperCAmelCase      )
     __A    =  word
     return word

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase      )->   List[Any]:
     __A    =  []
     for token in re.findall(self.pat    ,							UpperCAmelCase      ):
        __A    =  ''''''.join(
            self.byte_encoder[b] for b in token.encode('''utf-8'''      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
        bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase      ).split(''' '''      )      )
     return bpe_tokens

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase      )->   Any:
     return self.encoder.get(UpperCAmelCase    ,							self.encoder.get(self.unk_token      )      )

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase      )->   List[Any]:
     return self.decoder.get(UpperCAmelCase      )

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase      )->   Tuple:
     __A    =  ''''''.join(UpperCAmelCase      )
     __A    =  bytearray([self.byte_decoder[c] for c in text]      ).decode('''utf-8'''    ,							errors=self.errors      )
     return text

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase = None      )->   Tuple[str]:
     if not os.path.isdir(UpperCAmelCase      ):
        logger.error(f"Vocabulary path ({save_directory}) should be a directory"      )
        return
     __A    =  os.path.join(
         UpperCAmelCase    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )
     __A    =  os.path.join(
         UpperCAmelCase    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']      )

     with open(UpperCAmelCase    ,							'''w'''    ,							encoding='''utf-8'''      ) as f:
        f.write(json.dumps(self.encoder    ,							indent=2    ,							sort_keys=UpperCAmelCase    ,							ensure_ascii=UpperCAmelCase      ) + '''\n'''      )

     __A    =  0
     with open(UpperCAmelCase    ,							'''w'''    ,							encoding='''utf-8'''      ) as writer:
        writer.write('''#version: 0.2\n'''      )
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,							key=lambda UpperCAmelCase      : kv[1]      ):
           if index != token_index:
              logger.warning(
                  f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                  ''' Please check that the tokenizer is not corrupted!'''      )
              __A    =  token_index
           writer.write(''' '''.join(UpperCAmelCase      ) + '''\n'''      )
           index += 1

     return vocab_file, merge_file

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase = None      )->   List[int]:
     if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
     __A    =  [self.cls_token_id]
     __A    =  [self.sep_token_id]
     return cls + token_ids_a + sep + sep + token_ids_a + sep

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase = None    ,							UpperCAmelCase = False      )->   List[int]:
     if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=UpperCAmelCase    ,							token_ids_a=UpperCAmelCase    ,							already_has_special_tokens=UpperCAmelCase      )

     if token_ids_a is None:
        return [1] + ([0] * len(UpperCAmelCase      )) + [1]
     return [1] + ([0] * len(UpperCAmelCase      )) + [1, 1] + ([0] * len(UpperCAmelCase      )) + [1]

  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase = None      )->   List[int]:
     __A    =  [self.sep_token_id]
     __A    =  [self.cls_token_id]

     if token_ids_a is None:
        return len(cls + token_ids_a + sep      ) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]





  def 						SCREAMING_SNAKE_CASE__			(   self    ,							UpperCAmelCase    ,							UpperCAmelCase=False    ,							**UpperCAmelCase      )->   Union[str, Any]:
     __A    =  kwargs.pop('''add_prefix_space'''    ,							self.add_prefix_space      )
     if (is_split_into_words or add_prefix_space) and (len(UpperCAmelCase      ) > 0 and not text[0].isspace()):
        __A    =  ''' ''' + text
     return (text, kwargs)

