
'''simple docstring'''





import argparse
import json
import os

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ConvNextImageProcessor, ConvNextVaConfig, ConvNextVaForImageClassification
from transformers.image_utils import PILImageResampling
from transformers.utils import logging


logging.set_verbosity_info()
a__		: List[Any]  							=							logging.get_logger(__name__)





def 							snake_case				(		UpperCAmelCase						)->      List[str]:







      """simple docstring"""
      __A													=      ConvNextVaConfig()

      if "atto" in checkpoint_url:
            __A													=      [2, 2, 6, 2]
            __A													=      [4_0, 8_0, 1_6_0, 3_2_0]
      if "femto" in checkpoint_url:
            __A													=      [2, 2, 6, 2]
            __A													=      [4_8, 9_6, 1_9_2, 3_8_4]
      if "pico" in checkpoint_url:
            __A													=      [2, 2, 6, 2]
            __A													=      [6_4, 1_2_8, 2_5_6, 5_1_2]
      if "nano" in checkpoint_url:
            __A													=      [2, 2, 8, 2]
            __A													=      [8_0, 1_6_0, 3_2_0, 6_4_0]
      if "tiny" in checkpoint_url:
            __A													=      [3, 3, 9, 3]
            __A													=      [9_6, 1_9_2, 3_8_4, 7_6_8]
      if "base" in checkpoint_url:
            __A													=      [3, 3, 2_7, 3]
            __A													=      [1_2_8, 2_5_6, 5_1_2, 1_0_2_4]
      if "large" in checkpoint_url:
            __A													=      [3, 3, 2_7, 3]
            __A													=      [1_9_2, 3_8_4, 7_6_8, 1_5_3_6]
      if "huge" in checkpoint_url:
            __A													=      [3, 3, 2_7, 3]
            __A													=      [3_5_2, 7_0_4, 1_4_0_8, 2_8_1_6]

      __A													=      1_0_0_0
      __A													=      'imagenet-1k-id2label.json'
      __A													=      (1, 1_0_0_0)

      __A													=      'huggingface/label-files'
      __A													=      num_labels
      __A													=      json.load(open(hf_hub_download(UpperCAmelCase   ,	UpperCAmelCase   ,	repo_type='dataset'						)   ,	'r'						)						)
      __A													=      {int(UpperCAmelCase						): v for k, v in idalabel.items()}

      __A													=      idalabel
      __A													=      {v: k for k, v in idalabel.items()}
      __A													=      hidden_sizes
      __A													=      depths

      return config, expected_shape





def 							snake_case				(		UpperCAmelCase						)->      List[str]:







      """simple docstring"""
      if "downsample_layers.0.0" in name:
            __A													=      name.replace('downsample_layers.0.0'   ,	'embeddings.patch_embeddings'						)
      if "downsample_layers.0.1" in name:
            __A													=      name.replace('downsample_layers.0.1'   ,	'embeddings.norm'						)  # we rename to layernorm later on
      if "downsample_layers.1.0" in name:
            __A													=      name.replace('downsample_layers.1.0'   ,	'stages.1.downsampling_layer.0'						)
      if "downsample_layers.1.1" in name:
            __A													=      name.replace('downsample_layers.1.1'   ,	'stages.1.downsampling_layer.1'						)
      if "downsample_layers.2.0" in name:
            __A													=      name.replace('downsample_layers.2.0'   ,	'stages.2.downsampling_layer.0'						)
      if "downsample_layers.2.1" in name:
            __A													=      name.replace('downsample_layers.2.1'   ,	'stages.2.downsampling_layer.1'						)
      if "downsample_layers.3.0" in name:
            __A													=      name.replace('downsample_layers.3.0'   ,	'stages.3.downsampling_layer.0'						)
      if "downsample_layers.3.1" in name:
            __A													=      name.replace('downsample_layers.3.1'   ,	'stages.3.downsampling_layer.1'						)
      if "stages" in name and "downsampling_layer" not in name:
            # stages.0.0. for instance should be renamed to stages.0.layers.0.
            __A													=      name[: len('stages.0'						)] + '.layers' + name[len('stages.0'						) :]
      if "gamma" in name:
            __A													=      name.replace('gamma'   ,	'weight'						)
      if "beta" in name:
            __A													=      name.replace('beta'   ,	'bias'						)
      if "stages" in name:
            __A													=      name.replace('stages'   ,	'encoder.stages'						)
      if "norm" in name:
            __A													=      name.replace('norm'   ,	'layernorm'						)
      if "head" in name:
            __A													=      name.replace('head'   ,	'classifier'						)

      return name





def 							snake_case				(		)->      Union[str, Any]:







      """simple docstring"""
      __A													=      'http://images.cocodataset.org/val2017/000000039769.jpg'
      __A													=      Image.open(requests.get(UpperCAmelCase   ,	stream=UpperCAmelCase						).raw						)
      return im





def 							snake_case				(		UpperCAmelCase						)->      Tuple:







      """simple docstring"""
      if "224" in checkpoint_url:
            __A													=      2_2_4
            __A													=      2_2_4 / 2_5_6
      elif "384" in checkpoint_url:
            __A													=      3_8_4
            __A													=      None
      else:
            __A													=      5_1_2
            __A													=      None

      return ConvNextImageProcessor(
          size=UpperCAmelCase   ,	crop_pct=UpperCAmelCase   ,	image_mean=[0.485, 0.456, 0.406]   ,	image_std=[0.229, 0.224, 0.225]   ,	resample=PILImageResampling.BICUBIC   ,	)





@torch.no_grad()
def 							snake_case				(		UpperCAmelCase   ,	UpperCAmelCase   ,	UpperCAmelCase   ,	UpperCAmelCase						)->      Any:







      """simple docstring"""
      print('Downloading original model from checkpoint...'						)
      # define ConvNeXTV2 configuration based on URL
      __A						,   __A													=      get_convnextva_config(UpperCAmelCase						)
      # load original state_dict from URL
      __A													=      torch.hub.load_state_dict_from_url(UpperCAmelCase						)['model']

      print('Converting model parameters...'						)
      # rename keys
      for key in state_dict.copy().keys():
            __A													=      state_dict.pop(UpperCAmelCase						)
            __A													=      val
      # add prefix to all keys expect classifier head
      for key in state_dict.copy().keys():
            __A													=      state_dict.pop(UpperCAmelCase						)
            if not key.startswith('classifier'						):
                  __A													=      'convnextv2.' + key
            __A													=      val

      # load HuggingFace model
      __A													=      ConvNextVaForImageClassification(UpperCAmelCase						)
      model.load_state_dict(UpperCAmelCase						)
      model.eval()

      # Check outputs on an image, prepared by ConvNextImageProcessor
      __A													=      convert_preprocessor(UpperCAmelCase						)
      __A													=      preprocessor(images=prepare_img()   ,	return_tensors='pt'						)
      __A													=      model(**UpperCAmelCase						).logits

      # note: the logits below were obtained without center cropping
      if checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_atto_1k_224_ema.pt":
            __A													=      torch.tensor([-0.3930, 0.1747, -0.5246, 0.4177, 0.4295]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_femto_1k_224_ema.pt":
            __A													=      torch.tensor([-0.1727, -0.5341, -0.7818, -0.4745, -0.6566]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_pico_1k_224_ema.pt":
            __A													=      torch.tensor([-0.0333, 0.1563, -0.9137, 0.1054, 0.0381]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_nano_1k_224_ema.pt":
            __A													=      torch.tensor([-0.1744, -0.1555, -0.0713, 0.0950, -0.1431]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_tiny_1k_224_ema.pt":
            __A													=      torch.tensor([0.9996, 0.1966, -0.4386, -0.3472, 0.6661]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_base_1k_224_ema.pt":
            __A													=      torch.tensor([-0.2553, -0.6708, -0.1359, 0.2518, -0.2488]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_large_1k_224_ema.pt":
            __A													=      torch.tensor([-0.0673, -0.5627, -0.3753, -0.2722, 0.0178]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_huge_1k_224_ema.pt":
            __A													=      torch.tensor([-0.6377, -0.7458, -0.2150, 0.1184, -0.0597]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_nano_22k_224_ema.pt":
            __A													=      torch.tensor([1.0799, 0.2322, -0.8860, 1.0219, 0.6231]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_nano_22k_384_ema.pt":
            __A													=      torch.tensor([0.3766, 0.4917, -1.1426, 0.9942, 0.6024]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_tiny_22k_224_ema.pt":
            __A													=      torch.tensor([0.4220, -0.6919, -0.4317, -0.2881, -0.6609]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_tiny_22k_384_ema.pt":
            __A													=      torch.tensor([0.1082, -0.8286, -0.5095, 0.4681, -0.8085]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_base_22k_224_ema.pt":
            __A													=      torch.tensor([-0.2419, -0.6221, 0.2176, -0.0980, -0.7527]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_base_22k_384_ema.pt":
            __A													=      torch.tensor([0.0391, -0.4371, 0.3786, 0.1251, -0.2784]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_large_22k_224_ema.pt":
            __A													=      torch.tensor([-0.0504, 0.5636, -0.1729, -0.6507, -0.3949]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_large_22k_384_ema.pt":
            __A													=      torch.tensor([0.3560, 0.9486, 0.3149, -0.2667, -0.5138]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_huge_22k_384_ema.pt":
            __A													=      torch.tensor([-0.2469, -0.4550, -0.5853, -0.0810, 0.0309]						)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_huge_22k_512_ema.pt":
            __A													=      torch.tensor([-0.3090, 0.0802, -0.0682, -0.1979, -0.2826]						)
      else:
            raise ValueError(f'Unknown URL: {checkpoint_url}'						)

      assert torch.allclose(logits[0, :5]   ,	UpperCAmelCase   ,	atol=1e-3						)
      assert logits.shape == expected_shape
      print('Model outputs match the original results!'						)

      if save_model:
            print('Saving model to local...'						)
            # Create folder to save model
            if not os.path.isdir(UpperCAmelCase						):
                  os.mkdir(UpperCAmelCase						)

            model.save_pretrained(UpperCAmelCase						)
            preprocessor.save_pretrained(UpperCAmelCase						)

      __A													=      'convnextv2'
      if "atto" in checkpoint_url:
            model_name += "-atto"
      if "femto" in checkpoint_url:
            model_name += "-femto"
      if "pico" in checkpoint_url:
            model_name += "-pico"
      if "nano" in checkpoint_url:
            model_name += "-nano"
      elif "tiny" in checkpoint_url:
            model_name += "-tiny"
      elif "base" in checkpoint_url:
            model_name += "-base"
      elif "large" in checkpoint_url:
            model_name += "-large"
      elif "huge" in checkpoint_url:
            model_name += "-huge"
      if "22k" in checkpoint_url and "1k" not in checkpoint_url:
            model_name += "-22k"
      elif "22k" in checkpoint_url and "1k" in checkpoint_url:
            model_name += "-22k-1k"
      elif "1k" in checkpoint_url:
            model_name += "-1k"
      if "224" in checkpoint_url:
            model_name += "-224"
      elif "384" in checkpoint_url:
            model_name += "-384"
      elif "512" in checkpoint_url:
            model_name += "-512"

      if push_to_hub:
            print(f'Pushing {model_name} to the hub...'						)
            model.push_to_hub(UpperCAmelCase						)
            preprocessor.push_to_hub(UpperCAmelCase						)


if __name__ == "__main__":
 a__		: int  							=							argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     "--checkpoint_url",
     default="https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_atto_1k_224_ema.pt",
     type=str,
     help="URL of the original ConvNeXTV2 checkpoint you'd like to convert.",
 )
 parser.add_argument(
     "--pytorch_dump_folder_path",
     default="model",
     type=str,
     help="Path to the output PyTorch model directory.",
 )
 parser.add_argument("--save_model", action="store_true", help="Save model to local")
 parser.add_argument("--push_to_hub", action="store_true", help="Push model and image preprocessor to the hub")

 a__		: Any  							=							parser.parse_args()
 convert_convnextva_checkpoint(
     args.checkpoint_url, args.pytorch_dump_folder_path, args.save_model, args.push_to_hub
 )














"""simple docstring"""





import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)







class 						_a  :





   '''simple docstring'''






   def __init__(       self					, UpperCAmelCase_)			->   List[Any]:




          '''simple docstring'''




          lowercase__:							List[str]         =     foo





   def __call__(       self)			->   Union[str, Any]:




          '''simple docstring'''




          return self.foo







class 						_a  (		datasets.Dataset       ):





   '''simple docstring'''






   @fingerprint_transform(inplace=UpperCAmelCase_)
   def __lowercase							(       self					, UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_)			->   int:




          '''simple docstring'''




          return DatasetChild(self.data					, fingerprint=UpperCAmelCase_)





   @fingerprint_transform(inplace=UpperCAmelCase_)
   def __lowercase							(       self					, UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_)			->   int:




          '''simple docstring'''




          return DatasetChild(self.data					, fingerprint=UpperCAmelCase_)







class 						_a  :





   '''simple docstring'''






   def __init__(       self					, UpperCAmelCase_)			->   Optional[int]:




          '''simple docstring'''




          lowercase__:							Tuple         =     callable





   def __call__(       self					, *UpperCAmelCase_					, **UpperCAmelCase_)			->   Optional[int]:




          '''simple docstring'''




          if self.callable is not None:
                 return self.callable(*UpperCAmelCase_					, **UpperCAmelCase_)





   def __getstate__(       self)			->   int:




          '''simple docstring'''




          raise pickle.PicklingError()







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   @require_transformers
   @pytest.mark.integration
   def __lowercase							(       self)			->   List[Any]:




          '''simple docstring'''




          from transformers import AutoTokenizer

          def encode(UpperCAmelCase_):
                 return tokenizer(UpperCAmelCase_)

          # TODO: add hash consistency tests across sessions
          lowercase__:							Tuple         =     AutoTokenizer.from_pretrained("bert-base-uncased")
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase_: tokenizer(UpperCAmelCase_))).hexdigest()
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Tuple         =     AutoTokenizer.from_pretrained("bert-base-cased")
          lowercase__:							int         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase_: tokenizer(UpperCAmelCase_))).hexdigest()
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Optional[int]         =     AutoTokenizer.from_pretrained("bert-base-uncased")
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							str         =     mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase_: tokenizer(UpperCAmelCase_))).hexdigest()
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_transformers
   @pytest.mark.integration
   def __lowercase							(       self)			->   Optional[int]:




          '''simple docstring'''




          from transformers import AutoTokenizer

          lowercase__:							Optional[int]         =     AutoTokenizer.from_pretrained("gpt2")
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          tokenizer("Hello world !")  # call once to change the tokenizer's cache
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_regex
   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          import regex

          lowercase__:							List[Any]         =     regex.Regex("foo")
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							List[Any]         =     regex.Regex("bar")
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Union[str, Any]         =     regex.Regex("foo")
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          def func():
                 return foo

          lowercase__:							int         =     [0]
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Dict         =     [1]
          lowercase__:							int         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Union[str, Any]         =     [0]
          lowercase__:							List[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Dict:




          '''simple docstring'''




          def func():
                 pass

          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()

          def func():
                 pass

          lowercase__:							List[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(Foo([1]))).hexdigest()
          lowercase__:							int         =     mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Union[str, Any]:




          '''simple docstring'''




          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
          lowercase__:							List[Any]         =     mda(datasets.utils.py_utils.dumps(Foo([1]).__call__)).hexdigest()
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Union[str, Any]:




          '''simple docstring'''




          lowercase__:							List[Any]         =     (
              "co_argcount",
              "co_kwonlyargcount",
              "co_nlocals",
              "co_stacksize",
              "co_flags",
              "co_code",
              "co_consts",
              "co_names",
              "co_varnames",
              "co_filename",
              "co_name",
              "co_firstlineno",
              "co_lnotab",
              "co_freevars",
              "co_cellvars",
          )

          def _create_code_pyaa(*UpperCAmelCase_):
                 if hasattr(args[-3]					, "encode"):
                        lowercase__:							Tuple         =     list(UpperCAmelCase_)
                        lowercase__:							str         =     args[-3].encode()  # co_lnotab
                        lowercase__:							Any         =     args[-10].encode()  # co_code
                 if hasattr(UpperCAmelCase_					, "co_posonlyargcount"):
                        if len(UpperCAmelCase_) == 16:
                               return CodeType(*UpperCAmelCase_)
                        elif len(UpperCAmelCase_) == 15:
                               return CodeType(args[0]					, 0					, *args[1:])
                        return CodeType(args[0]					, 0					, 0					, *args[1:])
                 elif hasattr(UpperCAmelCase_					, "co_kwonlyargcount"):
                        if len(UpperCAmelCase_) == 16:
                               return CodeType(args[0]					, *args[2:])
                        elif len(UpperCAmelCase_) == 15:
                               return CodeType(*UpperCAmelCase_)
                        return CodeType(args[0]					, 0					, *args[1:])
                 if len(UpperCAmelCase_) == 16:
                        return CodeType(args[0]					, *args[3:])
                 elif len(UpperCAmelCase_) == 15:
                        return CodeType(args[0]					, *args[2:])
                 return CodeType(*UpperCAmelCase_)

          def create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_):
                 def func():
                        return returned_obj

                 lowercase__:							Dict         =     func.__code__
                 # Use _create_code from dill in order to make it work for different python versions
                 if datasets.config.PY_VERSION < version.parse("3.8"):
                        lowercase__:							Union[str, Any]         =     _create_code_pyaa(
                            *[getattr(UpperCAmelCase_					, UpperCAmelCase_) if k != "co_filename" else co_filename for k in code_args_pyaa])
                 else:
                        lowercase__:							Tuple         =     code.replace(co_filename=UpperCAmelCase_)
                 return FunctionType(UpperCAmelCase_					, func.__globals__					, func.__name__					, func.__defaults__					, func.__closure__)

          lowercase__    ,      lowercase__:							Any         =     "<ipython-input-2-e0383a102aae>", [0]
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Optional[Any]         =     "<ipython-input-2-e0383a102aae>", [1]
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Tuple         =     "<ipython-input-5-713f6613acf3>", [0]
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)

          lowercase__    ,      lowercase__:							Dict         =     os.path.join(gettempdir()					, "ipykernel_12345"					, "321456789.py"), [0]
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Dict         =     os.path.join(gettempdir()					, "ipykernel_12345"					, "321456789.py"), [1]
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Any         =     os.path.join(gettempdir()					, "ipykernel_12345"					, "654123987.py"), [0]
          lowercase__:							Dict         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Dict:




          '''simple docstring'''




          lowercase__    ,      lowercase__:							str         =     [0], [1]

          def func():
                 return foo, bar

          lowercase__:							List[str]         =     "__main__"

          def globalvars_mocka_side_effect(UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_):
                 return {"foo": foo, "bar": bar}

          def globalvars_mocka_side_effect(UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_):
                 return {"bar": bar, "foo": foo}

          with patch("dill.detect.globalvars"					, side_effect=UpperCAmelCase_) as globalvars_mocka:
                 lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
                 self.assertGreater(globalvars_mocka.call_count					, 0)
          with patch("dill.detect.globalvars"					, side_effect=UpperCAmelCase_) as globalvars_mocka:
                 lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
                 self.assertGreater(globalvars_mocka.call_count					, 0)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          lowercase__:							List[Any]         =     Hasher.hash("hello")
          lowercase__:							Optional[int]         =     Hasher.hash("hello")
          lowercase__:							int         =     Hasher.hash("there")
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Any:




          '''simple docstring'''




          lowercase__:							str         =     Hasher.hash(Foo("hello"))
          lowercase__:							Tuple         =     Hasher.hash(Foo("hello"))
          lowercase__:							Union[str, Any]         =     Hasher.hash(Foo("there"))
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          lowercase__:							str         =     Hasher()
          for x in ["hello", Foo("hello")]:
                 hasher.update(UpperCAmelCase_)
          lowercase__:							Dict         =     hasher.hexdigest()
          lowercase__:							Any         =     Hasher()
          for x in ["hello", Foo("hello")]:
                 hasher.update(UpperCAmelCase_)
          lowercase__:							List[Any]         =     hasher.hexdigest()
          lowercase__:							Optional[int]         =     Hasher()
          for x in ["there", Foo("there")]:
                 hasher.update(UpperCAmelCase_)
          lowercase__:							Tuple         =     hasher.hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   List[str]:




          '''simple docstring'''




          with self.assertRaises(pickle.PicklingError):
                 Hasher.hash(UnpicklableCallable(Foo("hello")))





   def __lowercase							(       self)			->   Optional[Any]:




          '''simple docstring'''




          lowercase__:							Optional[Any]         =     "abc"
          lowercase__:							Any         =     [string, string]  # two strings have the same ids
          lowercase__:							str         =     [string, string]
          lowercase__:							Optional[int]         =     json.loads(F"""[\"{string}\", \"{string}\"]""")  # two strings have different ids
          self.assertIs(obja[0]					, UpperCAmelCase_)
          self.assertIs(obja[0]					, obja[1])
          self.assertIs(obja[0]					, UpperCAmelCase_)
          self.assertIs(obja[0]					, obja[1])
          self.assertIsNot(obja[0]					, UpperCAmelCase_)
          self.assertIsNot(obja[0]					, obja[1])
          lowercase__:							List[Any]         =     Hasher.hash(UpperCAmelCase_)
          lowercase__:							str         =     Hasher.hash(UpperCAmelCase_)
          lowercase__:							Tuple         =     Hasher.hash(UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_tiktoken
   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          import tiktoken

          lowercase__:							Union[str, Any]         =     tiktoken.get_encoding("gpt2")
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Dict         =     tiktoken.get_encoding("r50k_base")
          lowercase__:							Dict         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							List[Any]         =     tiktoken.get_encoding("gpt2")
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_torch
   def __lowercase							(       self)			->   List[Any]:




          '''simple docstring'''




          import torch

          lowercase__:							Tuple         =     torch.tensor([1.0])
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Dict         =     torch.tensor([2.0])
          lowercase__:							Union[str, Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Tuple         =     torch.tensor([1.0])
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_spacy
   @require_spacy_model("en_core_web_sm")
   @require_spacy_model("fr_core_news_sm")
   @pytest.mark.integration
   def __lowercase							(       self)			->   Any:




          '''simple docstring'''




          import spacy

          lowercase__:							Tuple         =     spacy.load("en_core_web_sm")
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Any         =     spacy.load("fr_core_news_sm")
          lowercase__:							str         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							int         =     spacy.load("en_core_web_sm")
          lowercase__:							str         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)







@pytest.mark.integration
def 							A(    snake_case_						):
       """simple docstring"""





       lowercase__:							Optional[Any]         =     tmp_path / "dir1"
       lowercase__:							Tuple         =     tmp_path / "dir2"
       dira.mkdir()
       dira.mkdir()
       lowercase__:							Union[str, Any]         =     "script.py"
       lowercase__:							Any         =     dedent(
           "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "						)
       lowercase__:							List[str]         =     dira / script_filename
       lowercase__:							int         =     dira / script_filename
       with script_patha.open("w"						) as f:
              f.write(snake_case_						)
       with script_patha.open("w"						) as f:
              f.write(snake_case_						)
       lowercase__:							Any         =     subprocess.check_output(["python", str(snake_case_						)]						)
       lowercase__:							Optional[Any]         =     subprocess.check_output(["python", str(snake_case_						)]						)
       assert fingerprinta == fingerprinta






def 							A(    ):
       """simple docstring"""





       lowercase__:							Optional[int]         =     {"a": [0, 1, 2]}
       lowercase__:							Dict         =     DatasetChild(InMemoryTable.from_pydict(snake_case_						)						)
       lowercase__:							List[Any]         =     dataset.funca()._fingerprint
       assert expected_fingerprint == dataset.funca()._fingerprint
       assert expected_fingerprint != dataset.funca()._fingerprint

       with Pool(2						) as p:
              assert expected_fingerprint == p.apply_async(dataset.funca						).get()._fingerprint
              assert expected_fingerprint != p.apply_async(dataset.funca						).get()._fingerprint






def 							A(    ):
       """simple docstring"""





       lowercase__:							List[str]         =     {"a": [0, 1, 2]}







       class 						_a  (		datasets.Dataset       ):





          '''simple docstring'''






          @fingerprint_transform(inplace=UpperCAmelCase_)
          def __lowercase							(       self					, UpperCAmelCase_)			->   List[str]:




                 '''simple docstring'''




                 return DummyDatasetChild(self.data					, fingerprint=UpperCAmelCase_)







       lowercase__:							Tuple         =     DummyDatasetChild(InMemoryTable.from_pydict(snake_case_						)						).func()







       class 						_a  (		datasets.Dataset       ):





          '''simple docstring'''






          @fingerprint_transform(inplace=UpperCAmelCase_					, version="1.0.0")
          def __lowercase							(       self					, UpperCAmelCase_)			->   Optional[int]:




                 '''simple docstring'''




                 return DummyDatasetChild(self.data					, fingerprint=UpperCAmelCase_)







       lowercase__:							Optional[int]         =     DummyDatasetChild(InMemoryTable.from_pydict(snake_case_						)						).func()







       class 						_a  (		datasets.Dataset       ):





          '''simple docstring'''






          @fingerprint_transform(inplace=UpperCAmelCase_					, version="2.0.0")
          def __lowercase							(       self					, UpperCAmelCase_)			->   Any:




                 '''simple docstring'''




                 return DummyDatasetChild(self.data					, fingerprint=UpperCAmelCase_)







       lowercase__:							Optional[int]         =     DummyDatasetChild(InMemoryTable.from_pydict(snake_case_						)						).func()

       assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}						) == 3







def 							A(    ):
       """simple docstring"""





       lowercase__:							int         =     Hasher()
       hasher.update(lambda snake_case_						: x						)






