



'''simple docstring'''







import copy
import fnmatch
import json
import os
import pickle as pkl
import shutil
import sys
import tarfile
import tempfile
from collections import OrderedDict
from contextlib import contextmanager
from functools import partial
from hashlib import shaaaa
from io import BytesIO
from pathlib import Path
from urllib.parse import urlparse
from zipfile import ZipFile, is_zipfile

import cva
import numpy as np
import requests
import wget
from filelock import FileLock
from PIL import Image
from tqdm.auto import tqdm
from yaml import Loader, dump, load


try:
		import torch

		_UpperCamelCase  						=			True
except ImportError:
		_UpperCamelCase  						=			False


try:
		from torch.hub import _get_torch_home

		_UpperCamelCase  						=			_get_torch_home()
except ImportError:
		_UpperCamelCase  						=			os.path.expanduser(
		    os.getenv("""TORCH_HOME""", os.path.join(os.getenv("""XDG_CACHE_HOME""", """~/.cache"""), """torch"""))
		)

_UpperCamelCase  						=			os.path.join(torch_cache_home, """transformers""")

_UpperCamelCase  						=			"""https://cdn.huggingface.co"""
_UpperCamelCase  						=			"""https://s3.amazonaws.com/models.huggingface.co/bert"""
_UpperCamelCase  						=			"""/""".join(str(Path(__file__).resolve()).split("""/""")[:-1])
_UpperCamelCase  						=			os.path.join(PATH, """config.yaml""")
_UpperCamelCase  						=			os.path.join(PATH, """attributes.txt""")
_UpperCamelCase  						=			os.path.join(PATH, """objects.txt""")
_UpperCamelCase  						=			os.getenv("""PYTORCH_PRETRAINED_BERT_CACHE""", default_cache_path)
_UpperCamelCase  						=			os.getenv("""PYTORCH_TRANSFORMERS_CACHE""", PYTORCH_PRETRAINED_BERT_CACHE)
_UpperCamelCase  						=			os.getenv("""TRANSFORMERS_CACHE""", PYTORCH_TRANSFORMERS_CACHE)
_UpperCamelCase  						=			"""pytorch_model.bin"""
_UpperCamelCase  						=			"""config.yaml"""
def 			_lowerCAmelCase(      UpperCAmelCase_      :	str=OBJECTS					, UpperCAmelCase_      :	Tuple=ATTRIBUTES						) ->							List[Any]:
							lowerCAmelCase__  =   []
							with open(snake_case_						) as f:
														for object in f.readlines():
																					vg_classes.append(object.split(""","""						)[0].lower().strip()						)

							lowerCAmelCase__  =   []
							with open(snake_case_						) as f:
														for object in f.readlines():
																					vg_attrs.append(object.split(""","""						)[0].lower().strip()						)
							return vg_classes, vg_attrs
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Any						) ->							Union[str, Any]:
							lowerCAmelCase__  =   OrderedDict()
							with open(snake_case_					, """rb"""						) as f:
														lowerCAmelCase__  =   pkl.load(snake_case_						)["model"]
							for k in copy.deepcopy(list(ckp.keys()						)						):
														lowerCAmelCase__  =   ckp.pop(snake_case_						)
														if isinstance(snake_case_					, np.ndarray						):
																					lowerCAmelCase__  =   torch.tensor(snake_case_						)
														else:
																					assert isinstance(snake_case_					, torch.tensor						), type(snake_case_						)
														lowerCAmelCase__  =   v
							return r

class      lowerCamelCase__     :






	'''simple docstring'''

	A__		=	{}




	def __init__(  self		:  int   ,		__A		:  str   ,		__A		:  Optional[int] = "root"   ,		__A		:  Tuple=0					)     ->			str:




								'''simple docstring'''





								lowerCAmelCase__  =   name
								lowerCAmelCase__  =   level
								lowerCAmelCase__  =   {}
								for k, v in dictionary.items():
															if v is None:
																						raise ValueError()
															lowerCAmelCase__  =   copy.deepcopy(UpperCAmelCase_					)
															lowerCAmelCase__  =   copy.deepcopy(UpperCAmelCase_					)
															if isinstance(UpperCAmelCase_   ,		UpperCAmelCase_					):
																						lowerCAmelCase__  =   Config(UpperCAmelCase_   ,		name=UpperCAmelCase_   ,		level=level + 1					)
															lowerCAmelCase__  =   v
															setattr(self   ,		UpperCAmelCase_   ,		UpperCAmelCase_					)

								lowerCAmelCase__  =   d




	def __repr__(  self		:  List[str]					)     ->			Tuple:




								'''simple docstring'''





								return str(list((self._pointer.keys())					)					)




	def __setattr__(  self		:  str   ,		__A		:  List[Any]   ,		__A		:  Dict					)     ->			Tuple:




								'''simple docstring'''





								lowerCAmelCase__  =   val
								lowerCAmelCase__  =   val
								lowerCAmelCase__  =   key.split("""."""					)
								lowerCAmelCase__  =   len(UpperCAmelCase_					) - 1
								lowerCAmelCase__  =   self._pointer
								if len(UpperCAmelCase_					) > 1:
															for i, l in enumerate(UpperCAmelCase_					):
																						if hasattr(self   ,		UpperCAmelCase_					) and isinstance(getattr(self   ,		UpperCAmelCase_					)   ,		UpperCAmelCase_					):
																													setattr(getattr(self   ,		UpperCAmelCase_					)   ,		""".""".join(levels[i:]					)   ,		UpperCAmelCase_					)
																						if l == last_level:
																													lowerCAmelCase__  =   val
																						else:
																													lowerCAmelCase__  =   pointer[l]




	def      lowercase__							(  self		:  Any					)     ->			Dict:




								'''simple docstring'''





								return self._pointer




	def      lowercase__							(  self		:  List[Any]   ,		__A		:  Dict   ,		__A		:  Union[str, Any]					)     ->			Dict:




								'''simple docstring'''





								with open(f'''{file_name}'''   ,		"""w"""					) as stream:
															dump(UpperCAmelCase_   ,		UpperCAmelCase_					)




	def      lowercase__							(  self		:  List[str]   ,		__A		:  List[Any]   ,		__A		:  str					)     ->			Dict:




								'''simple docstring'''





								with open(f'''{file_name}'''   ,		"""w"""					) as stream:
															json.dump(UpperCAmelCase_   ,		UpperCAmelCase_					)




	@staticmethod
	def      lowercase__							(  __A		:  Tuple					)     ->			Optional[int]:




								'''simple docstring'''





								with open(UpperCAmelCase_					) as stream:
															lowerCAmelCase__  =   load(UpperCAmelCase_   ,		Loader=UpperCAmelCase_					)
								return data




	def __str__(  self		:  str					)     ->			Any:




								'''simple docstring'''





								lowerCAmelCase__  =   "    "
								if self._name != "root":
															lowerCAmelCase__  =   f'''{t * (self._level-1)}{self._name}:\n'''
								else:
															lowerCAmelCase__  =   ""
								lowerCAmelCase__  =   self._level
								for i, (k, v) in enumerate(self._pointer.items()					):
															if isinstance(UpperCAmelCase_   ,		UpperCAmelCase_					):
																						r += f'''{t * (self._level)}{v}\n'''
																						self._level += 1
															else:
																						r += f'''{t * (self._level)}{k}: {v} ({type(UpperCAmelCase_					).__name__})\n'''
															lowerCAmelCase__  =   level
								return r[:-1]




	@classmethod
	def      lowercase__							(  cls		:  str   ,		__A		:  str   ,		**__A		:  List[Any]					)     ->			int:




								'''simple docstring'''





								lowerCAmelCase__  =   cls.get_config_dict(UpperCAmelCase_   ,		**UpperCAmelCase_					)
								return cls(UpperCAmelCase_					)




	@classmethod
	def      lowercase__							(  cls		:  List[str]   ,		__A		:  Optional[int]   ,		**__A		:  str					)     ->			Any:




								'''simple docstring'''





								lowerCAmelCase__  =   kwargs.pop("""cache_dir"""   ,		UpperCAmelCase_					)
								lowerCAmelCase__  =   kwargs.pop("""force_download"""   ,		UpperCAmelCase_					)
								lowerCAmelCase__  =   kwargs.pop("""resume_download"""   ,		UpperCAmelCase_					)
								lowerCAmelCase__  =   kwargs.pop("""proxies"""   ,		UpperCAmelCase_					)
								lowerCAmelCase__  =   kwargs.pop("""local_files_only"""   ,		UpperCAmelCase_					)

								if os.path.isdir(UpperCAmelCase_					):
															lowerCAmelCase__  =   os.path.join(UpperCAmelCase_   ,		UpperCAmelCase_					)
								elif os.path.isfile(UpperCAmelCase_					) or is_remote_url(UpperCAmelCase_					):
															lowerCAmelCase__  =   pretrained_model_name_or_path
								else:
															lowerCAmelCase__  =   hf_bucket_url(UpperCAmelCase_   ,		filename=UpperCAmelCase_   ,		use_cdn=UpperCAmelCase_					)

								try:
															# Load from URL or cache if already cached
															lowerCAmelCase__  =   cached_path(
															    UpperCAmelCase_   ,		cache_dir=UpperCAmelCase_   ,		force_download=UpperCAmelCase_   ,		proxies=UpperCAmelCase_   ,		resume_download=UpperCAmelCase_   ,		local_files_only=UpperCAmelCase_   ,		)
															# Load config dict
															if resolved_config_file is None:
																						raise EnvironmentError

															lowerCAmelCase__  =   Config.load_yaml(UpperCAmelCase_					)

								except EnvironmentError:
															lowerCAmelCase__  =   "Can't load config for"
															raise EnvironmentError(UpperCAmelCase_					)

								if resolved_config_file == config_file:
															print("""loading configuration file from path"""					)
								else:
															print("""loading configuration file cache"""					)

								return Config.load_yaml(UpperCAmelCase_					), kwargs





def 			_lowerCAmelCase(      UpperCAmelCase_      :	Optional[int]						) ->							List[str]:
							lowerCAmelCase__  =   torch.load("""dump.pt"""					, map_location=in_tensor.device						)
							lowerCAmelCase__  =   in_tensor.numpy()
							lowerCAmelCase__  =   out_tensor.numpy()[0]
							print(na.shape					, na[0, 0, :5]						)
							print(na.shape					, na[0, 0, :5]						)
							assert np.allclose(snake_case_					, snake_case_					, rtol=0.01					, atol=0.1						), (
							    F'''{sum([1 for x in np.isclose(snake_case_					, snake_case_					, rtol=0.01					, atol=0.1						).flatten() if x is False]						)/len(na.flatten()						)*100:.4f} %'''
							    " element-wise mismatch"
							)
							raise Exception("""tensors are all good"""						)

							# Hugging face functions below
def 			_lowerCAmelCase(      UpperCAmelCase_      :	List[Any]						) ->							Union[str, Any]:
							lowerCAmelCase__  =   urlparse(snake_case_						)
							return parsed.scheme in ("http", "https")
def 			_lowerCAmelCase(      UpperCAmelCase_      :	str					, UpperCAmelCase_      :	Union[str, Any]					, UpperCAmelCase_      :	Dict=True						) ->							int:
							lowerCAmelCase__  =   CLOUDFRONT_DISTRIB_PREFIX if use_cdn else S3_BUCKET_PREFIX
							lowerCAmelCase__  =   "/" not in model_id
							if legacy_format:
														return F'''{endpoint}/{model_id}-{filename}'''
							else:
														return F'''{endpoint}/{model_id}/{filename}'''
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Tuple					, UpperCAmelCase_      :	Optional[int]					, UpperCAmelCase_      :	str=None					, UpperCAmelCase_      :	int=0					, UpperCAmelCase_      :	int=None					, ) ->							Optional[Any]:
							lowerCAmelCase__  =   "python/{}".format(sys.version.split()[0]						)
							if _torch_available:
														ua += "; torch/{}".format(torch.__version__						)
							if isinstance(snake_case_					, snake_case_						):
														ua += "; " + "; ".join("""{}/{}""".format(snake_case_					, snake_case_						) for k, v in user_agent.items()						)
							elif isinstance(snake_case_					, snake_case_						):
														ua += "; " + user_agent
							lowerCAmelCase__  =   {"user-agent": ua}
							if resume_size > 0:
														lowerCAmelCase__  =   "bytes=%d-" % (resume_size,)
							lowerCAmelCase__  =   requests.get(snake_case_					, stream=snake_case_					, proxies=snake_case_					, headers=snake_case_						)
							if response.status_code == 416:  # Range not satisfiable
														return
							lowerCAmelCase__  =   response.headers.get("""Content-Length"""						)
							lowerCAmelCase__  =   resume_size + int(snake_case_						) if content_length is not None else None
							lowerCAmelCase__  =   tqdm(
							    unit="""B"""					, unit_scale=snake_case_					, total=snake_case_					, initial=snake_case_					, desc="""Downloading"""					, )
							for chunk in response.iter_content(chunk_size=1024						):
														if chunk:  # filter out keep-alive new chunks
																					progress.update(len(snake_case_						)						)
																					temp_file.write(snake_case_						)
							progress.close()
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Optional[Any]					, UpperCAmelCase_      :	str=None					, UpperCAmelCase_      :	Optional[Any]=False					, UpperCAmelCase_      :	Tuple=None					, UpperCAmelCase_      :	Dict=10					, UpperCAmelCase_      :	int=False					, UpperCAmelCase_      :	List[Any]=None					, UpperCAmelCase_      :	List[str]=False					, ) ->							str:





							if cache_dir is None:
														lowerCAmelCase__  =   TRANSFORMERS_CACHE
							if isinstance(snake_case_					, snake_case_						):
														lowerCAmelCase__  =   str(snake_case_						)

							os.makedirs(snake_case_					, exist_ok=snake_case_						)

							lowerCAmelCase__  =   None
							if not local_files_only:
														try:
																					lowerCAmelCase__  =   requests.head(snake_case_					, allow_redirects=snake_case_					, proxies=snake_case_					, timeout=snake_case_						)
																					if response.status_code == 200:
																												lowerCAmelCase__  =   response.headers.get("""ETag"""						)
														except (EnvironmentError, requests.exceptions.Timeout):
																					# etag is already None
																					pass

							lowerCAmelCase__  =   url_to_filename(snake_case_					, snake_case_						)

							# get cache path to put the file
							lowerCAmelCase__  =   os.path.join(snake_case_					, snake_case_						)

							# etag is None = we don't have a connection, or url doesn't exist, or is otherwise inaccessible.
							# try to get the last downloaded one
							if etag is None:
														if os.path.exists(snake_case_						):
																					return cache_path
														else:
																					lowerCAmelCase__  =   [
																					    file
																					    for file in fnmatch.filter(os.listdir(snake_case_						)					, filename + """.*"""						)
																					    if not file.endswith(""".json"""						) and not file.endswith(""".lock"""						)
																					]
																					if len(snake_case_						) > 0:
																												return os.path.join(snake_case_					, matching_files[-1]						)
																					else:
																												# If files cannot be found and local_files_only=True,
																												# the models might've been found if local_files_only=False
																												# Notify the user about that
																												if local_files_only:
																																			raise ValueError(
																																			    """Cannot find the requested files in the cached path and outgoing traffic has been"""
																																			    """ disabled. To enable model look-ups and downloads online, set 'local_files_only'"""
																																			    """ to False."""						)
																												return None

    # From now on, etag is not None.
							if os.path.exists(snake_case_						) and not force_download:
														return cache_path

							# Prevent parallel downloads of the same file with a lock.
							lowerCAmelCase__  =   cache_path + ".lock"
							with FileLock(snake_case_						):
														# If the download just completed while the lock was activated.
														if os.path.exists(snake_case_						) and not force_download:
																					# Even if returning early like here, the lock will be released.
																					return cache_path

														if resume_download:
																					lowerCAmelCase__  =   cache_path + ".incomplete"

																					@contextmanager
																					def _resumable_file_manager():
																												with open(snake_case_					, """a+b"""						) as f:
																																			yield f

																					lowerCAmelCase__  =   _resumable_file_manager
																					if os.path.exists(snake_case_						):
																												lowerCAmelCase__  =   os.stat(snake_case_						).st_size
																					else:
																												lowerCAmelCase__  =   0
														else:
																					lowerCAmelCase__  =   partial(tempfile.NamedTemporaryFile					, dir=snake_case_					, delete=snake_case_						)
																					lowerCAmelCase__  =   0

														# Download to temporary file, then copy to cache dir once finished.
														# Otherwise you get corrupt cache entries if the download gets interrupted.
														with temp_file_manager() as temp_file:
																					print(
																					    """%s not found in cache or force_download set to True, downloading to %s"""					, snake_case_					, temp_file.name					, )

																					http_get(
																					    snake_case_					, snake_case_					, proxies=snake_case_					, resume_size=snake_case_					, user_agent=snake_case_					, )

														os.replace(temp_file.name					, snake_case_						)

														lowerCAmelCase__  =   {"url": url, "etag": etag}
														lowerCAmelCase__  =   cache_path + ".json"
														with open(snake_case_					, """w"""						) as meta_file:
																					json.dump(snake_case_					, snake_case_						)

							return cache_path
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Optional[Any]					, UpperCAmelCase_      :	Tuple=None						) ->							Any:
							lowerCAmelCase__  =   url.encode("""utf-8"""						)
							lowerCAmelCase__  =   shaaaa(snake_case_						)
							lowerCAmelCase__  =   url_hash.hexdigest()

							if etag:
														lowerCAmelCase__  =   etag.encode("""utf-8"""						)
														lowerCAmelCase__  =   shaaaa(snake_case_						)
														filename += "." + etag_hash.hexdigest()

							if url.endswith(""".h5"""						):
														filename += ".h5"

							return filename
def 			_lowerCAmelCase(      UpperCAmelCase_      :	str					, UpperCAmelCase_      :	Any=None					, UpperCAmelCase_      :	Any=False					, UpperCAmelCase_      :	Optional[int]=None					, UpperCAmelCase_      :	Union[str, Any]=False					, UpperCAmelCase_      :	Tuple=None					, UpperCAmelCase_      :	Tuple=False					, UpperCAmelCase_      :	Any=False					, UpperCAmelCase_      :	int=False					, ) ->							List[str]:





							if cache_dir is None:
														lowerCAmelCase__  =   TRANSFORMERS_CACHE
							if isinstance(snake_case_					, snake_case_						):
														lowerCAmelCase__  =   str(snake_case_						)
							if isinstance(snake_case_					, snake_case_						):
														lowerCAmelCase__  =   str(snake_case_						)

							if is_remote_url(snake_case_						):
														# URL, so get it from the cache (downloading if necessary)
														lowerCAmelCase__  =   get_from_cache(
														    snake_case_					, cache_dir=snake_case_					, force_download=snake_case_					, proxies=snake_case_					, resume_download=snake_case_					, user_agent=snake_case_					, local_files_only=snake_case_					, )
							elif os.path.exists(snake_case_						):
														# File, and it exists.
														lowerCAmelCase__  =   url_or_filename
							elif urlparse(snake_case_						).scheme == "":
														# File, but it doesn't exist.
														raise EnvironmentError("""file {} not found""".format(snake_case_						)						)
							else:
														# Something unknown
														raise ValueError("""unable to parse {} as a URL or as a local path""".format(snake_case_						)						)

							if extract_compressed_file:
														if not is_zipfile(snake_case_						) and not tarfile.is_tarfile(snake_case_						):
																					return output_path

														# Path where we extract compressed archives
														# We avoid '.' in dir name and add "-extracted" at the end: "./model.zip" => "./model-zip-extracted/"
														lowerCAmelCase__  =   os.path.split(snake_case_						)
														lowerCAmelCase__  =   output_file.replace("""."""					, """-"""						) + "-extracted"
														lowerCAmelCase__  =   os.path.join(snake_case_					, snake_case_						)

														if os.path.isdir(snake_case_						) and os.listdir(snake_case_						) and not force_extract:
																					return output_path_extracted

														# Prevent parallel extractions
														lowerCAmelCase__  =   output_path + ".lock"
														with FileLock(snake_case_						):
																					shutil.rmtree(snake_case_					, ignore_errors=snake_case_						)
																					os.makedirs(snake_case_						)
																					if is_zipfile(snake_case_						):
																												with ZipFile(snake_case_					, """r"""						) as zip_file:
																																			zip_file.extractall(snake_case_						)
																																			zip_file.close()
																					elif tarfile.is_tarfile(snake_case_						):
																												lowerCAmelCase__  =   tarfile.open(snake_case_						)
																												tar_file.extractall(snake_case_						)
																												tar_file.close()
																					else:
																												raise EnvironmentError("""Archive format of {} could not be identified""".format(snake_case_						)						)

														return output_path_extracted

							return output_path
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Optional[Any]					, UpperCAmelCase_      :	str=","						) ->							List[str]:
							assert isinstance(snake_case_					, snake_case_						)
							if os.path.isfile(snake_case_						):
														with open(snake_case_						) as f:
																					lowerCAmelCase__  =   eval(f.read()						)
							else:
														lowerCAmelCase__  =   requests.get(snake_case_						)
														try:
																					lowerCAmelCase__  =   requests.json()
														except Exception:
																					lowerCAmelCase__  =   req.content.decode()
																					assert data is not None, "could not connect"
																					try:
																												lowerCAmelCase__  =   eval(snake_case_						)
																					except Exception:
																												lowerCAmelCase__  =   data.split("""\n"""						)
														req.close()
							return data
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Dict						) ->							List[Any]:
							lowerCAmelCase__  =   requests.get(snake_case_						)
							lowerCAmelCase__  =   np.array(Image.open(BytesIO(response.content						)						)						)
							return img
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Union[str, Any]						) ->							Dict:
							lowerCAmelCase__  =   url.split("""/"""						)[-1]
							if fn not in os.listdir(os.getcwd()						):
														wget.download(snake_case_						)
							with open(snake_case_					, """rb"""						) as stream:
														lowerCAmelCase__  =   pkl.load(snake_case_						)
							lowerCAmelCase__  =   weights.pop("""model"""						)
							lowerCAmelCase__  =   {}
							for k, v in model.items():
														lowerCAmelCase__  =   torch.from_numpy(snake_case_						)
														if "running_var" in k:
																					lowerCAmelCase__  =   torch.tensor([0]						)
																					lowerCAmelCase__  =   k.replace("""running_var"""					, """num_batches_tracked"""						)
																					lowerCAmelCase__  =   zero
							return new
def 			_lowerCAmelCase(      ) ->							Any:
							print(F'''{os.path.abspath(os.path.join(snake_case_					, os.pardir						)						)}/demo.ipynb'''						)
def 			_lowerCAmelCase(      UpperCAmelCase_      :	Tuple					, UpperCAmelCase_      :	List[str]="RGB"						) ->							Any:
							assert isinstance(snake_case_					, snake_case_						)
							if os.path.isfile(snake_case_						):
														lowerCAmelCase__  =   cva.imread(snake_case_						)
							else:
														lowerCAmelCase__  =   get_image_from_url(snake_case_						)
														assert img is not None, F'''could not connect to: {im}'''
							lowerCAmelCase__  =   cva.cvtColor(snake_case_					, cva.COLOR_BGR2RGB						)
							if input_format == "RGB":
														lowerCAmelCase__  =   img[:, :, ::-1]
							return img






def 			_lowerCAmelCase(      UpperCAmelCase_      :	Union[str, Any]					, UpperCAmelCase_      :	str=1						) ->							List[str]:
							return (images[i : i + batch] for i in range(0					, len(snake_case_						)					, snake_case_						))













"""simple docstring"""





import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)







class 						_a  :





   '''simple docstring'''






   def __init__(       self					, UpperCAmelCase_)			->   List[Any]:




          '''simple docstring'''




          lowercase__:							List[str]         =     foo





   def __call__(       self)			->   Union[str, Any]:




          '''simple docstring'''




          return self.foo







class 						_a  (		datasets.Dataset       ):





   '''simple docstring'''






   @fingerprint_transform(inplace=UpperCAmelCase_)
   def __lowercase							(       self					, UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_)			->   int:




          '''simple docstring'''




          return DatasetChild(self.data					, fingerprint=UpperCAmelCase_)





   @fingerprint_transform(inplace=UpperCAmelCase_)
   def __lowercase							(       self					, UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_)			->   int:




          '''simple docstring'''




          return DatasetChild(self.data					, fingerprint=UpperCAmelCase_)







class 						_a  :





   '''simple docstring'''






   def __init__(       self					, UpperCAmelCase_)			->   Optional[int]:




          '''simple docstring'''




          lowercase__:							Tuple         =     callable





   def __call__(       self					, *UpperCAmelCase_					, **UpperCAmelCase_)			->   Optional[int]:




          '''simple docstring'''




          if self.callable is not None:
                 return self.callable(*UpperCAmelCase_					, **UpperCAmelCase_)





   def __getstate__(       self)			->   int:




          '''simple docstring'''




          raise pickle.PicklingError()







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   @require_transformers
   @pytest.mark.integration
   def __lowercase							(       self)			->   List[Any]:




          '''simple docstring'''




          from transformers import AutoTokenizer

          def encode(UpperCAmelCase_):
                 return tokenizer(UpperCAmelCase_)

          # TODO: add hash consistency tests across sessions
          lowercase__:							Tuple         =     AutoTokenizer.from_pretrained("bert-base-uncased")
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase_: tokenizer(UpperCAmelCase_))).hexdigest()
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Tuple         =     AutoTokenizer.from_pretrained("bert-base-cased")
          lowercase__:							int         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase_: tokenizer(UpperCAmelCase_))).hexdigest()
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Optional[int]         =     AutoTokenizer.from_pretrained("bert-base-uncased")
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							str         =     mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase_: tokenizer(UpperCAmelCase_))).hexdigest()
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_transformers
   @pytest.mark.integration
   def __lowercase							(       self)			->   Optional[int]:




          '''simple docstring'''




          from transformers import AutoTokenizer

          lowercase__:							Optional[int]         =     AutoTokenizer.from_pretrained("gpt2")
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          tokenizer("Hello world !")  # call once to change the tokenizer's cache
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_regex
   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          import regex

          lowercase__:							List[Any]         =     regex.Regex("foo")
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							List[Any]         =     regex.Regex("bar")
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Union[str, Any]         =     regex.Regex("foo")
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          def func():
                 return foo

          lowercase__:							int         =     [0]
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Dict         =     [1]
          lowercase__:							int         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Union[str, Any]         =     [0]
          lowercase__:							List[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Dict:




          '''simple docstring'''




          def func():
                 pass

          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()

          def func():
                 pass

          lowercase__:							List[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(Foo([1]))).hexdigest()
          lowercase__:							int         =     mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Union[str, Any]:




          '''simple docstring'''




          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
          lowercase__:							List[Any]         =     mda(datasets.utils.py_utils.dumps(Foo([1]).__call__)).hexdigest()
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Union[str, Any]:




          '''simple docstring'''




          lowercase__:							List[Any]         =     (
              "co_argcount",
              "co_kwonlyargcount",
              "co_nlocals",
              "co_stacksize",
              "co_flags",
              "co_code",
              "co_consts",
              "co_names",
              "co_varnames",
              "co_filename",
              "co_name",
              "co_firstlineno",
              "co_lnotab",
              "co_freevars",
              "co_cellvars",
          )

          def _create_code_pyaa(*UpperCAmelCase_):
                 if hasattr(args[-3]					, "encode"):
                        lowercase__:							Tuple         =     list(UpperCAmelCase_)
                        lowercase__:							str         =     args[-3].encode()  # co_lnotab
                        lowercase__:							Any         =     args[-10].encode()  # co_code
                 if hasattr(UpperCAmelCase_					, "co_posonlyargcount"):
                        if len(UpperCAmelCase_) == 16:
                               return CodeType(*UpperCAmelCase_)
                        elif len(UpperCAmelCase_) == 15:
                               return CodeType(args[0]					, 0					, *args[1:])
                        return CodeType(args[0]					, 0					, 0					, *args[1:])
                 elif hasattr(UpperCAmelCase_					, "co_kwonlyargcount"):
                        if len(UpperCAmelCase_) == 16:
                               return CodeType(args[0]					, *args[2:])
                        elif len(UpperCAmelCase_) == 15:
                               return CodeType(*UpperCAmelCase_)
                        return CodeType(args[0]					, 0					, *args[1:])
                 if len(UpperCAmelCase_) == 16:
                        return CodeType(args[0]					, *args[3:])
                 elif len(UpperCAmelCase_) == 15:
                        return CodeType(args[0]					, *args[2:])
                 return CodeType(*UpperCAmelCase_)

          def create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_):
                 def func():
                        return returned_obj

                 lowercase__:							Dict         =     func.__code__
                 # Use _create_code from dill in order to make it work for different python versions
                 if datasets.config.PY_VERSION < version.parse("3.8"):
                        lowercase__:							Union[str, Any]         =     _create_code_pyaa(
                            *[getattr(UpperCAmelCase_					, UpperCAmelCase_) if k != "co_filename" else co_filename for k in code_args_pyaa])
                 else:
                        lowercase__:							Tuple         =     code.replace(co_filename=UpperCAmelCase_)
                 return FunctionType(UpperCAmelCase_					, func.__globals__					, func.__name__					, func.__defaults__					, func.__closure__)

          lowercase__    ,      lowercase__:							Any         =     "<ipython-input-2-e0383a102aae>", [0]
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Optional[Any]         =     "<ipython-input-2-e0383a102aae>", [1]
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Tuple         =     "<ipython-input-5-713f6613acf3>", [0]
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)

          lowercase__    ,      lowercase__:							Dict         =     os.path.join(gettempdir()					, "ipykernel_12345"					, "321456789.py"), [0]
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Dict         =     os.path.join(gettempdir()					, "ipykernel_12345"					, "321456789.py"), [1]
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Any         =     os.path.join(gettempdir()					, "ipykernel_12345"					, "654123987.py"), [0]
          lowercase__:							Dict         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Dict:




          '''simple docstring'''




          lowercase__    ,      lowercase__:							str         =     [0], [1]

          def func():
                 return foo, bar

          lowercase__:							List[str]         =     "__main__"

          def globalvars_mocka_side_effect(UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_):
                 return {"foo": foo, "bar": bar}

          def globalvars_mocka_side_effect(UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_):
                 return {"bar": bar, "foo": foo}

          with patch("dill.detect.globalvars"					, side_effect=UpperCAmelCase_) as globalvars_mocka:
                 lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
                 self.assertGreater(globalvars_mocka.call_count					, 0)
          with patch("dill.detect.globalvars"					, side_effect=UpperCAmelCase_) as globalvars_mocka:
                 lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
                 self.assertGreater(globalvars_mocka.call_count					, 0)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          lowercase__:							List[Any]         =     Hasher.hash("hello")
          lowercase__:							Optional[int]         =     Hasher.hash("hello")
          lowercase__:							int         =     Hasher.hash("there")
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Any:




          '''simple docstring'''




          lowercase__:							str         =     Hasher.hash(Foo("hello"))
          lowercase__:							Tuple         =     Hasher.hash(Foo("hello"))
          lowercase__:							Union[str, Any]         =     Hasher.hash(Foo("there"))
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          lowercase__:							str         =     Hasher()
          for x in ["hello", Foo("hello")]:
                 hasher.update(UpperCAmelCase_)
          lowercase__:							Dict         =     hasher.hexdigest()
          lowercase__:							Any         =     Hasher()
          for x in ["hello", Foo("hello")]:
                 hasher.update(UpperCAmelCase_)
          lowercase__:							List[Any]         =     hasher.hexdigest()
          lowercase__:							Optional[int]         =     Hasher()
          for x in ["there", Foo("there")]:
                 hasher.update(UpperCAmelCase_)
          lowercase__:							Tuple         =     hasher.hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   List[str]:




          '''simple docstring'''




          with self.assertRaises(pickle.PicklingError):
                 Hasher.hash(UnpicklableCallable(Foo("hello")))





   def __lowercase							(       self)			->   Optional[Any]:




          '''simple docstring'''




          lowercase__:							Optional[Any]         =     "abc"
          lowercase__:							Any         =     [string, string]  # two strings have the same ids
          lowercase__:							str         =     [string, string]
          lowercase__:							Optional[int]         =     json.loads(F"""[\"{string}\", \"{string}\"]""")  # two strings have different ids
          self.assertIs(obja[0]					, UpperCAmelCase_)
          self.assertIs(obja[0]					, obja[1])
          self.assertIs(obja[0]					, UpperCAmelCase_)
          self.assertIs(obja[0]					, obja[1])
          self.assertIsNot(obja[0]					, UpperCAmelCase_)
          self.assertIsNot(obja[0]					, obja[1])
          lowercase__:							List[Any]         =     Hasher.hash(UpperCAmelCase_)
          lowercase__:							str         =     Hasher.hash(UpperCAmelCase_)
          lowercase__:							Tuple         =     Hasher.hash(UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_tiktoken
   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          import tiktoken

          lowercase__:							Union[str, Any]         =     tiktoken.get_encoding("gpt2")
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Dict         =     tiktoken.get_encoding("r50k_base")
          lowercase__:							Dict         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							List[Any]         =     tiktoken.get_encoding("gpt2")
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_torch
   def __lowercase							(       self)			->   List[Any]:




          '''simple docstring'''




          import torch

          lowercase__:							Tuple         =     torch.tensor([1.0])
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Dict         =     torch.tensor([2.0])
          lowercase__:							Union[str, Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Tuple         =     torch.tensor([1.0])
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_spacy
   @require_spacy_model("en_core_web_sm")
   @require_spacy_model("fr_core_news_sm")
   @pytest.mark.integration
   def __lowercase							(       self)			->   Any:




          '''simple docstring'''




          import spacy

          lowercase__:							Tuple         =     spacy.load("en_core_web_sm")
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Any         =     spacy.load("fr_core_news_sm")
          lowercase__:							str         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							int         =     spacy.load("en_core_web_sm")
          lowercase__:							str         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)







@pytest.mark.integration
def 							A(    snake_case_						):
       """simple docstring"""





       lowercase__:							Optional[Any]         =     tmp_path / "dir1"
       lowercase__:							Tuple         =     tmp_path / "dir2"
       dira.mkdir()
       dira.mkdir()
       lowercase__:							Union[str, Any]         =     "script.py"
       lowercase__:							Any         =     dedent(
           "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "						)
       lowercase__:							List[str]         =     dira / script_filename
       lowercase__:							int         =     dira / script_filename
       with script_patha.open("w"						) as f:
              f.write(snake_case_						)
       with script_patha.open("w"						) as f:
              f.write(snake_case_						)
       lowercase__:							Any         =     subprocess.check_output(["python", str(snake_case_						)]						)
       lowercase__:							Optional[Any]         =     subprocess.check_output(["python", str(snake_case_						)]						)
       assert fingerprinta == fingerprinta






def 							A(    ):
       """simple docstring"""





       lowercase__:							Optional[int]         =     {"a": [0, 1, 2]}
       lowercase__:							Dict         =     DatasetChild(InMemoryTable.from_pydict(snake_case_						)						)
       lowercase__:							List[Any]         =     dataset.funca()._fingerprint
       assert expected_fingerprint == dataset.funca()._fingerprint
       assert expected_fingerprint != dataset.funca()._fingerprint

       with Pool(2						) as p:
              assert expected_fingerprint == p.apply_async(dataset.funca						).get()._fingerprint
              assert expected_fingerprint != p.apply_async(dataset.funca						).get()._fingerprint






def 							A(    ):
       """simple docstring"""





       lowercase__:							List[str]         =     {"a": [0, 1, 2]}







       class 						_a  (		datasets.Dataset       ):





          '''simple docstring'''






          @fingerprint_transform(inplace=UpperCAmelCase_)
          def __lowercase							(       self					, UpperCAmelCase_)			->   List[str]:




                 '''simple docstring'''




                 return DummyDatasetChild(self.data					, fingerprint=UpperCAmelCase_)







       lowercase__:							Tuple         =     DummyDatasetChild(InMemoryTable.from_pydict(snake_case_						)						).func()







       class 						_a  (		datasets.Dataset       ):





          '''simple docstring'''






          @fingerprint_transform(inplace=UpperCAmelCase_					, version="1.0.0")
          def __lowercase							(       self					, UpperCAmelCase_)			->   Optional[int]:




                 '''simple docstring'''




                 return DummyDatasetChild(self.data					, fingerprint=UpperCAmelCase_)







       lowercase__:							Optional[int]         =     DummyDatasetChild(InMemoryTable.from_pydict(snake_case_						)						).func()







       class 						_a  (		datasets.Dataset       ):





          '''simple docstring'''






          @fingerprint_transform(inplace=UpperCAmelCase_					, version="2.0.0")
          def __lowercase							(       self					, UpperCAmelCase_)			->   Any:




                 '''simple docstring'''




                 return DummyDatasetChild(self.data					, fingerprint=UpperCAmelCase_)







       lowercase__:							Optional[int]         =     DummyDatasetChild(InMemoryTable.from_pydict(snake_case_						)						).func()

       assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}						) == 3







def 							A(    ):
       """simple docstring"""





       lowercase__:							int         =     Hasher()
       hasher.update(lambda snake_case_						: x						)






