import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)




class 					_snake_case						:
					def __init__(  self:				List[str]	,  __lowerCamelCase:				int     )    ->				Any:
												__UpperCAmelCase						:			Union[str, Any]         =   foo





					def __call__(  self:				Any     )    ->				str:
												return self.foo




class 					_snake_case						(						datasets.Dataset				):
					@fingerprint_transform(inplace=__lowerCamelCase     )
					def    _lowerCamelCase (  self:				Dict	,  __lowerCamelCase:				Any	,  *__lowerCamelCase:				List[Any]	,  **__lowerCamelCase:				Dict     )    ->				List[str]:
												return DatasetChild(self.data	,  fingerprint=__lowerCamelCase     )





					@fingerprint_transform(inplace=__lowerCamelCase     )
					def    _lowerCamelCase (  self:				List[Any]	,  __lowerCamelCase:				Dict	,  *__lowerCamelCase:				List[Any]	,  **__lowerCamelCase:				List[str]     )    ->				Optional[Any]:
												return DatasetChild(self.data	,  fingerprint=__lowerCamelCase     )




class 					_snake_case						:
					def __init__(  self:				Dict	,  __lowerCamelCase:				Dict     )    ->				Optional[Any]:
												__UpperCAmelCase						:			List[str]         =   callable
					def __call__(  self:				List[str]	,  *__lowerCamelCase:				Any	,  **__lowerCamelCase:				List[Any]     )    ->				str:
												if self.callable is not None:
																			return self.callable(*__lowerCamelCase	,  **__lowerCamelCase     )





					def __getstate__(  self:				int     )    ->				Union[str, Any]:
												raise pickle.PicklingError()




class 					_snake_case						(						_lowercase				):
					@require_transformers
					@pytest.mark.integration
					def    _lowerCamelCase (  self:				List[Any]     )    ->				List[str]:
												from transformers import AutoTokenizer

												def encode(__lowerCamelCase:				Tuple     ):
																			return tokenizer(__lowerCamelCase     )

												# TODO: add hash consistency tests across sessions
												__UpperCAmelCase						:			Optional[int]         =   AutoTokenizer.from_pretrained("bert-base-uncased"     )
												__UpperCAmelCase						:			Any         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			Optional[Any]         =   mda(datasets.utils.py_utils.dumps(lambda __lowerCamelCase     : tokenizer(__lowerCamelCase     )     )     ).hexdigest()
												__UpperCAmelCase						:			Any         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			Tuple         =   AutoTokenizer.from_pretrained("bert-base-cased"     )
												__UpperCAmelCase						:			List[str]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			Tuple         =   mda(datasets.utils.py_utils.dumps(lambda __lowerCamelCase     : tokenizer(__lowerCamelCase     )     )     ).hexdigest()
												__UpperCAmelCase						:			Optional[int]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			Optional[int]         =   AutoTokenizer.from_pretrained("bert-base-uncased"     )
												__UpperCAmelCase						:			Any         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			Any         =   mda(datasets.utils.py_utils.dumps(lambda __lowerCamelCase     : tokenizer(__lowerCamelCase     )     )     ).hexdigest()
												__UpperCAmelCase						:			List[str]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )
					@require_transformers
					@pytest.mark.integration
					def    _lowerCamelCase (  self:				Tuple     )    ->				List[str]:
												from transformers import AutoTokenizer

												__UpperCAmelCase						:			List[str]         =   AutoTokenizer.from_pretrained("gpt2"     )
												__UpperCAmelCase						:			Optional[int]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												tokenizer("Hello world !"     )  # call once to change the tokenizer's cache
												__UpperCAmelCase						:			Optional[int]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )





					@require_regex
					def    _lowerCamelCase (  self:				int     )    ->				List[str]:
												import regex

												__UpperCAmelCase						:			Dict         =   regex.Regex("foo"     )
												__UpperCAmelCase						:			List[Any]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			Any         =   regex.Regex("bar"     )
												__UpperCAmelCase						:			Tuple         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			List[str]         =   regex.Regex("foo"     )
												__UpperCAmelCase						:			Any         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )




class 					_snake_case						(						_lowercase				):
					def    _lowerCamelCase (  self:				Optional[int]     )    ->				Optional[Any]:
												def func():
																			return foo

												__UpperCAmelCase						:			int         =   [0]
												__UpperCAmelCase						:			Tuple         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			Dict         =   [1]
												__UpperCAmelCase						:			Union[str, Any]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			int         =   [0]
												__UpperCAmelCase						:			str         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )
					def    _lowerCamelCase (  self:				Optional[Any]     )    ->				int:
												def func():
																			pass

												__UpperCAmelCase						:			List[str]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()

												def func():
																			pass

												__UpperCAmelCase						:			Tuple         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
					def    _lowerCamelCase (  self:				int     )    ->				Tuple:
												__UpperCAmelCase						:			str         =   mda(datasets.utils.py_utils.dumps(Foo([0]     )     )     ).hexdigest()
												__UpperCAmelCase						:			Any         =   mda(datasets.utils.py_utils.dumps(Foo([1]     )     )     ).hexdigest()
												__UpperCAmelCase						:			int         =   mda(datasets.utils.py_utils.dumps(Foo([0]     )     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )
					def    _lowerCamelCase (  self:				Tuple     )    ->				str:
												__UpperCAmelCase						:			List[str]         =   mda(datasets.utils.py_utils.dumps(Foo([0]     ).__call__     )     ).hexdigest()
												__UpperCAmelCase						:			Optional[Any]         =   mda(datasets.utils.py_utils.dumps(Foo([1]     ).__call__     )     ).hexdigest()
												__UpperCAmelCase						:			Any         =   mda(datasets.utils.py_utils.dumps(Foo([0]     ).__call__     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )
					def    _lowerCamelCase (  self:				Dict     )    ->				Union[str, Any]:
												__UpperCAmelCase						:			str         =   (
												    "co_argcount",
												    "co_kwonlyargcount",
												    "co_nlocals",
												    "co_stacksize",
												    "co_flags",
												    "co_code",
												    "co_consts",
												    "co_names",
												    "co_varnames",
												    "co_filename",
												    "co_name",
												    "co_firstlineno",
												    "co_lnotab",
												    "co_freevars",
												    "co_cellvars",
												)

												def _create_code_pyaa(*__lowerCamelCase:				Any     ):
																			if hasattr(args[-3]	,  "encode"     ):
																										__UpperCAmelCase						:			List[Any]         =   list(__lowerCamelCase     )
																										__UpperCAmelCase						:			str         =   args[-3].encode()  # co_lnotab
																										__UpperCAmelCase						:			str         =   args[-10].encode()  # co_code
																			if hasattr(__lowerCamelCase	,  "co_posonlyargcount"     ):
																										if len(__lowerCamelCase     ) == 16:
																																	return CodeType(*__lowerCamelCase     )
																										elif len(__lowerCamelCase     ) == 15:
																																	return CodeType(args[0]	,  0	,  *args[1:]     )
																										return CodeType(args[0]	,  0	,  0	,  *args[1:]     )
																			elif hasattr(__lowerCamelCase	,  "co_kwonlyargcount"     ):
																										if len(__lowerCamelCase     ) == 16:
																																	return CodeType(args[0]	,  *args[2:]     )
																										elif len(__lowerCamelCase     ) == 15:
																																	return CodeType(*__lowerCamelCase     )
																										return CodeType(args[0]	,  0	,  *args[1:]     )
																			if len(__lowerCamelCase     ) == 16:
																										return CodeType(args[0]	,  *args[3:]     )
																			elif len(__lowerCamelCase     ) == 15:
																										return CodeType(args[0]	,  *args[2:]     )
																			return CodeType(*__lowerCamelCase     )

												def create_ipython_func(__lowerCamelCase:				Dict	,  __lowerCamelCase:				str     ):
																			def func():
																										return returned_obj

																			__UpperCAmelCase						:			Optional[int]         =   func.__code__
																			# Use _create_code from dill in order to make it work for different python versions
																			if datasets.config.PY_VERSION < version.parse("3.8"     ):
																										__UpperCAmelCase						:			str         =   _create_code_pyaa(
																										    *[getattr(__lowerCamelCase	,  __lowerCamelCase     ) if k != "co_filename" else co_filename for k in code_args_pyaa]     )
																			else:
																										__UpperCAmelCase						:			List[str]         =   code.replace(co_filename=__lowerCamelCase     )
																			return FunctionType(__lowerCamelCase	,  func.__globals__	,  func.__name__	,  func.__defaults__	,  func.__closure__     )

												__UpperCAmelCase    ,	__UpperCAmelCase						:			Optional[Any]         =   "<ipython-input-2-e0383a102aae>", [0]
												__UpperCAmelCase						:			Dict         =   mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase	,  __lowerCamelCase     )     )     ).hexdigest()
												__UpperCAmelCase    ,	__UpperCAmelCase						:			Union[str, Any]         =   "<ipython-input-2-e0383a102aae>", [1]
												__UpperCAmelCase						:			List[str]         =   mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase	,  __lowerCamelCase     )     )     ).hexdigest()
												__UpperCAmelCase    ,	__UpperCAmelCase						:			Union[str, Any]         =   "<ipython-input-5-713f6613acf3>", [0]
												__UpperCAmelCase						:			Union[str, Any]         =   mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase	,  __lowerCamelCase     )     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )

												__UpperCAmelCase    ,	__UpperCAmelCase						:			Union[str, Any]         =   os.path.join(gettempdir()	,  "ipykernel_12345"	,  "321456789.py"     ), [0]
												__UpperCAmelCase						:			int         =   mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase	,  __lowerCamelCase     )     )     ).hexdigest()
												__UpperCAmelCase    ,	__UpperCAmelCase						:			Union[str, Any]         =   os.path.join(gettempdir()	,  "ipykernel_12345"	,  "321456789.py"     ), [1]
												__UpperCAmelCase						:			List[str]         =   mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase	,  __lowerCamelCase     )     )     ).hexdigest()
												__UpperCAmelCase    ,	__UpperCAmelCase						:			List[str]         =   os.path.join(gettempdir()	,  "ipykernel_12345"	,  "654123987.py"     ), [0]
												__UpperCAmelCase						:			int         =   mda(datasets.utils.py_utils.dumps(create_ipython_func(__lowerCamelCase	,  __lowerCamelCase     )     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )





					def    _lowerCamelCase (  self:				str     )    ->				List[Any]:
												__UpperCAmelCase    ,	__UpperCAmelCase						:			Optional[int]         =   [0], [1]

												def func():
																			return foo, bar

												__UpperCAmelCase						:			Tuple         =   "__main__"

												def globalvars_mocka_side_effect(__lowerCamelCase:				str	,  *__lowerCamelCase:				Tuple	,  **__lowerCamelCase:				List[str]     ):
																			return {"foo": foo, "bar": bar}

												def globalvars_mocka_side_effect(__lowerCamelCase:				Any	,  *__lowerCamelCase:				List[str]	,  **__lowerCamelCase:				Tuple     ):
																			return {"bar": bar, "foo": foo}

												with patch("dill.detect.globalvars"	,  side_effect=__lowerCamelCase     ) as globalvars_mocka:
																			__UpperCAmelCase						:			Union[str, Any]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
																			self.assertGreater(globalvars_mocka.call_count	,  0     )
												with patch("dill.detect.globalvars"	,  side_effect=__lowerCamelCase     ) as globalvars_mocka:
																			__UpperCAmelCase						:			Any         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
																			self.assertGreater(globalvars_mocka.call_count	,  0     )
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )




class 					_snake_case						(						_lowercase				):
					def    _lowerCamelCase (  self:				str     )    ->				List[str]:
												__UpperCAmelCase						:			str         =   Hasher.hash("hello"     )
												__UpperCAmelCase						:			Union[str, Any]         =   Hasher.hash("hello"     )
												__UpperCAmelCase						:			Optional[int]         =   Hasher.hash("there"     )
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )
					def    _lowerCamelCase (  self:				List[str]     )    ->				Optional[int]:
												__UpperCAmelCase						:			List[str]         =   Hasher.hash(Foo("hello"     )     )
												__UpperCAmelCase						:			Optional[Any]         =   Hasher.hash(Foo("hello"     )     )
												__UpperCAmelCase						:			Any         =   Hasher.hash(Foo("there"     )     )
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )
					def    _lowerCamelCase (  self:				Any     )    ->				str:
												__UpperCAmelCase						:			List[Any]         =   Hasher()
												for x in ["hello", Foo("hello"     )]:
																			hasher.update(__lowerCamelCase     )
												__UpperCAmelCase						:			Dict         =   hasher.hexdigest()
												__UpperCAmelCase						:			Optional[int]         =   Hasher()
												for x in ["hello", Foo("hello"     )]:
																			hasher.update(__lowerCamelCase     )
												__UpperCAmelCase						:			Optional[int]         =   hasher.hexdigest()
												__UpperCAmelCase						:			Union[str, Any]         =   Hasher()
												for x in ["there", Foo("there"     )]:
																			hasher.update(__lowerCamelCase     )
												__UpperCAmelCase						:			Optional[int]         =   hasher.hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )
					def    _lowerCamelCase (  self:				Union[str, Any]     )    ->				List[Any]:
												with self.assertRaises(pickle.PicklingError     ):
																			Hasher.hash(UnpicklableCallable(Foo("hello"     )     )     )
					def    _lowerCamelCase (  self:				Dict     )    ->				List[str]:
												__UpperCAmelCase						:			int         =   "abc"
												__UpperCAmelCase						:			Optional[int]         =   [string, string]  # two strings have the same ids
												__UpperCAmelCase						:			Union[str, Any]         =   [string, string]
												__UpperCAmelCase						:			Any         =   json.loads(f'''["{string}", "{string}"]'''     )  # two strings have different ids
												self.assertIs(obja[0]	,  __lowerCamelCase     )
												self.assertIs(obja[0]	,  obja[1]     )
												self.assertIs(obja[0]	,  __lowerCamelCase     )
												self.assertIs(obja[0]	,  obja[1]     )
												self.assertIsNot(obja[0]	,  __lowerCamelCase     )
												self.assertIsNot(obja[0]	,  obja[1]     )
												__UpperCAmelCase						:			Tuple         =   Hasher.hash(__lowerCamelCase     )
												__UpperCAmelCase						:			Optional[Any]         =   Hasher.hash(__lowerCamelCase     )
												__UpperCAmelCase						:			Optional[int]         =   Hasher.hash(__lowerCamelCase     )
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
					@require_tiktoken
					def    _lowerCamelCase (  self:				Tuple     )    ->				int:
												import tiktoken

												__UpperCAmelCase						:			Tuple         =   tiktoken.get_encoding("gpt2"     )
												__UpperCAmelCase						:			Optional[Any]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			Optional[int]         =   tiktoken.get_encoding("r50k_base"     )
												__UpperCAmelCase						:			Optional[int]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			List[str]         =   tiktoken.get_encoding("gpt2"     )
												__UpperCAmelCase						:			Dict         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )
					@require_torch
					def    _lowerCamelCase (  self:				Tuple     )    ->				Tuple:
												import torch

												__UpperCAmelCase						:			List[str]         =   torch.tensor([1.0]     )
												__UpperCAmelCase						:			List[str]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			int         =   torch.tensor([2.0]     )
												__UpperCAmelCase						:			Any         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			int         =   torch.tensor([1.0]     )
												__UpperCAmelCase						:			Dict         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )





					@require_spacy
					@require_spacy_model("en_core_web_sm"     )
					@require_spacy_model("fr_core_news_sm"     )
					@pytest.mark.integration
					def    _lowerCamelCase (  self:				Union[str, Any]     )    ->				List[str]:
												import spacy

												__UpperCAmelCase						:			Union[str, Any]         =   spacy.load("en_core_web_sm"     )
												__UpperCAmelCase						:			int         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			Any         =   spacy.load("fr_core_news_sm"     )
												__UpperCAmelCase						:			Optional[int]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												__UpperCAmelCase						:			Any         =   spacy.load("en_core_web_sm"     )
												__UpperCAmelCase						:			Union[str, Any]         =   mda(datasets.utils.py_utils.dumps(__lowerCamelCase     )     ).hexdigest()
												self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
												self.assertNotEqual(__lowerCamelCase	,  __lowerCamelCase     )




@pytest.mark.integration
def   _UpperCamelCase   (			snake_case__							)						->   Dict:
							__UpperCAmelCase						:			str         =   tmp_path / "dir1"
							__UpperCAmelCase						:			int         =   tmp_path / "dir2"
							dira.mkdir()
							dira.mkdir()
							__UpperCAmelCase						:			List[str]         =   "script.py"
							__UpperCAmelCase						:			int         =   dedent(
							    "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "							)
							__UpperCAmelCase						:			str         =   dira / script_filename
							__UpperCAmelCase						:			Any         =   dira / script_filename
							with script_patha.open("w"							) as f:
														f.write(snake_case__							)
							with script_patha.open("w"							) as f:
														f.write(snake_case__							)
							__UpperCAmelCase						:			int         =   subprocess.check_output(["python", str(snake_case__							)]							)
							__UpperCAmelCase						:			List[Any]         =   subprocess.check_output(["python", str(snake_case__							)]							)
							assert fingerprinta == fingerprinta




def   _UpperCamelCase   (			)						->   int:
							__UpperCAmelCase						:			List[str]         =   {"a": [0, 1, 2]}
							__UpperCAmelCase						:			Any         =   DatasetChild(InMemoryTable.from_pydict(snake_case__							)							)
							__UpperCAmelCase						:			Tuple         =   dataset.funca()._fingerprint
							assert expected_fingerprint == dataset.funca()._fingerprint
							assert expected_fingerprint != dataset.funca()._fingerprint

							with Pool(2							) as p:
														assert expected_fingerprint == p.apply_async(dataset.funca							).get()._fingerprint
														assert expected_fingerprint != p.apply_async(dataset.funca							).get()._fingerprint




def   _UpperCamelCase   (			)						->   Any:
							__UpperCAmelCase						:			Tuple         =   {"a": [0, 1, 2]}




							class 					_snake_case						(						datasets.Dataset				):
												@fingerprint_transform(inplace=__lowerCamelCase     )
												def    _lowerCamelCase (  self:				Dict	,  __lowerCamelCase:				Optional[int]     )    ->				List[str]:
																			return DummyDatasetChild(self.data	,  fingerprint=__lowerCamelCase     )




							__UpperCAmelCase						:			Optional[int]         =   DummyDatasetChild(InMemoryTable.from_pydict(snake_case__							)							).func()




							class 					_snake_case						(						datasets.Dataset				):
												@fingerprint_transform(inplace=__lowerCamelCase	,  version="1.0.0"     )
												def    _lowerCamelCase (  self:				Tuple	,  __lowerCamelCase:				Union[str, Any]     )    ->				str:
																			return DummyDatasetChild(self.data	,  fingerprint=__lowerCamelCase     )




							__UpperCAmelCase						:			int         =   DummyDatasetChild(InMemoryTable.from_pydict(snake_case__							)							).func()




							class 					_snake_case						(						datasets.Dataset				):
												@fingerprint_transform(inplace=__lowerCamelCase	,  version="2.0.0"     )
												def    _lowerCamelCase (  self:				int	,  __lowerCamelCase:				List[Any]     )    ->				List[str]:
																			return DummyDatasetChild(self.data	,  fingerprint=__lowerCamelCase     )




							__UpperCAmelCase						:			List[str]         =   DummyDatasetChild(InMemoryTable.from_pydict(snake_case__							)							).func()

							assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}							) == 3







def   _UpperCamelCase   (			)						->   List[Any]:
							# AttributeError: module 'dill._dill' has no attribute 'stack'
							__UpperCAmelCase						:			Tuple         =   Hasher()
							hasher.update(lambda snake_case__							: x							)





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
	from tokenizers import Tokenizer
	from tokenizers.models import WordPiece




class 					_snake_case						(						unittest.TestCase				):
					def    _lowerCamelCase (  self:				str	,  __lowerCamelCase:				int     )    ->				List[Any]:
												__UpperCAmelCase						:			List[str]         =   list(tokenizer_class.max_model_input_sizes.keys()     )
												for model_name in sa_models[:1]:
																			__UpperCAmelCase						:			Any         =   tokenizer_class.from_pretrained(__lowerCamelCase     )
																			self.assertIsNotNone(__lowerCamelCase     )
																			self.assertIsInstance(__lowerCamelCase	,  __lowerCamelCase     )
																			self.assertIsInstance(__lowerCamelCase	,  __lowerCamelCase     )

																			for special_tok in tokenizer.all_special_tokens:
																										self.assertIsInstance(__lowerCamelCase	,  __lowerCamelCase     )
																										__UpperCAmelCase						:			str         =   tokenizer.convert_tokens_to_ids(__lowerCamelCase     )
																										self.assertIsInstance(__lowerCamelCase	,  __lowerCamelCase     )
					def    _lowerCamelCase (  self:				List[Any]	,  __lowerCamelCase:				BatchEncoding	,  __lowerCamelCase:				Optional[Callable] = None     )    ->				Dict:
												__UpperCAmelCase						:			int         =   pickle.dumps(__lowerCamelCase     )
												self.assertIsNotNone(__lowerCamelCase     )

												__UpperCAmelCase						:			Dict         =   pickle.loads(__lowerCamelCase     )

												# Ensure is_fast is correctly restored
												self.assertEqual(be_restored.is_fast	,  be_original.is_fast     )

												# Ensure encodings are potentially correctly restored
												if be_original.is_fast:
																			self.assertIsNotNone(be_restored.encodings     )
												else:
																			self.assertIsNone(be_restored.encodings     )

												# Ensure the keys are the same
												for original_v, restored_v in zip(be_original.values()	,  be_restored.values()     ):
																			if equal_op:
																										self.assertTrue(equal_op(__lowerCamelCase	,  __lowerCamelCase     )     )
																			else:
																										self.assertEqual(__lowerCamelCase	,  __lowerCamelCase     )
					@slow
					def    _lowerCamelCase (  self:				str     )    ->				Optional[Any]:
												self.check_tokenizer_from_pretrained(__lowerCamelCase     )
					def    _lowerCamelCase (  self:				str     )    ->				Dict:
												self.assertEqual(TensorType("tf"     )	,  TensorType.TENSORFLOW     )
												self.assertEqual(TensorType("pt"     )	,  TensorType.PYTORCH     )
												self.assertEqual(TensorType("np"     )	,  TensorType.NUMPY     )
					@require_tokenizers
					def    _lowerCamelCase (  self:				Union[str, Any]     )    ->				int:
												import numpy as np

												__UpperCAmelCase						:			str         =   BertTokenizer.from_pretrained("bert-base-cased"     )
												__UpperCAmelCase						:			Any         =   BertTokenizerFast.from_pretrained("bert-base-cased"     )

												# Python no tensor
												with self.subTest("BatchEncoding (Python, return_tensors=None)"     ):
																			self.assert_dump_and_restore(tokenizer_p("Small example to encode"     )     )

												with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_p("Small example to encode"	,  return_tensors=TensorType.NUMPY     )	,  np.array_equal     )

												with self.subTest("BatchEncoding (Rust, return_tensors=None)"     ):
																			self.assert_dump_and_restore(tokenizer_r("Small example to encode"     )     )

												with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_r("Small example to encode"	,  return_tensors=TensorType.NUMPY     )	,  np.array_equal     )
					@require_tf
					@require_tokenizers
					def    _lowerCamelCase (  self:				Tuple     )    ->				str:
												import tensorflow as tf

												def tf_array_equals(__lowerCamelCase:				Dict	,  __lowerCamelCase:				Tuple     ):
																			return tf.reduce_all(tf.equal(__lowerCamelCase	,  __lowerCamelCase     )     )

												__UpperCAmelCase						:			List[str]         =   BertTokenizer.from_pretrained("bert-base-cased"     )
												__UpperCAmelCase						:			Optional[int]         =   BertTokenizerFast.from_pretrained("bert-base-cased"     )

												with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_p("Small example to encode"	,  return_tensors=TensorType.TENSORFLOW     )	,  __lowerCamelCase     )

												with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_r("Small example to encode"	,  return_tensors=TensorType.TENSORFLOW     )	,  __lowerCamelCase     )
					@require_torch
					@require_tokenizers
					def    _lowerCamelCase (  self:				Tuple     )    ->				Union[str, Any]:
												import torch

												__UpperCAmelCase						:			Optional[int]         =   BertTokenizer.from_pretrained("bert-base-cased"     )
												__UpperCAmelCase						:			Optional[Any]         =   BertTokenizerFast.from_pretrained("bert-base-cased"     )

												with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_p("Small example to encode"	,  return_tensors=TensorType.PYTORCH     )	,  torch.equal     )

												with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"     ):
																			self.assert_dump_and_restore(
																			    tokenizer_r("Small example to encode"	,  return_tensors=TensorType.PYTORCH     )	,  torch.equal     )
					@require_tokenizers
					def    _lowerCamelCase (  self:				int     )    ->				Any:
												__UpperCAmelCase						:			Tuple         =   BertTokenizer.from_pretrained("bert-base-cased"     )
												__UpperCAmelCase						:			Tuple         =   BertTokenizerFast.from_pretrained("bert-base-cased"     )

												with self.subTest("Python Tokenizer"     ):
																			self.assertFalse(tokenizer_p("Small example to_encode"     ).is_fast     )

												with self.subTest("Rust Tokenizer"     ):
																			self.assertTrue(tokenizer_r("Small example to_encode"     ).is_fast     )
					@require_tokenizers
					def    _lowerCamelCase (  self:				Union[str, Any]     )    ->				List[str]:
												__UpperCAmelCase						:			Tuple         =   BertTokenizerFast.from_pretrained("bert-base-cased"     )
												__UpperCAmelCase						:			Tuple         =   tokenizer_r(["Test", "\xad", "test"]	,  is_split_into_words=__lowerCamelCase     )

												self.assertEqual(encoded.word_to_tokens(0     )	,  TokenSpan(start=1	,  end=2     )     )
												self.assertEqual(encoded.word_to_tokens(1     )	,  __lowerCamelCase     )
												self.assertEqual(encoded.word_to_tokens(2     )	,  TokenSpan(start=2	,  end=3     )     )
					def    _lowerCamelCase (  self:				List[str]     )    ->				List[Any]:
												__UpperCAmelCase						:			str         =   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
												__UpperCAmelCase						:			str         =   batch.convert_to_tensors(tensor_type="np"     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (2, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (2,)     )
												# test converting the converted
												with CaptureStderr() as cs:
																			__UpperCAmelCase						:			Tuple         =   batch.convert_to_tensors(tensor_type="np"     )
												self.assertFalse(len(cs.err     )	,  msg=f'''should have no warning, but got {cs.err}'''     )

												__UpperCAmelCase						:			Optional[Any]         =   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
												__UpperCAmelCase						:			Any         =   batch.convert_to_tensors(tensor_type="np"	,  prepend_batch_axis=__lowerCamelCase     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (1, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (1,)     )
					@require_torch
					def    _lowerCamelCase (  self:				Dict     )    ->				int:
												__UpperCAmelCase						:			str         =   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
												__UpperCAmelCase						:			Union[str, Any]         =   batch.convert_to_tensors(tensor_type="pt"     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (2, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (2,)     )
												# test converting the converted
												with CaptureStderr() as cs:
																			__UpperCAmelCase						:			Optional[int]         =   batch.convert_to_tensors(tensor_type="pt"     )
												self.assertFalse(len(cs.err     )	,  msg=f'''should have no warning, but got {cs.err}'''     )

												__UpperCAmelCase						:			Optional[Any]         =   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
												__UpperCAmelCase						:			Union[str, Any]         =   batch.convert_to_tensors(tensor_type="pt"	,  prepend_batch_axis=__lowerCamelCase     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (1, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (1,)     )
					@require_tf
					def    _lowerCamelCase (  self:				Any     )    ->				Dict:
												__UpperCAmelCase						:			Any         =   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
												__UpperCAmelCase						:			Union[str, Any]         =   batch.convert_to_tensors(tensor_type="tf"     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (2, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (2,)     )
												# test converting the converted
												with CaptureStderr() as cs:
																			__UpperCAmelCase						:			Tuple         =   batch.convert_to_tensors(tensor_type="tf"     )
												self.assertFalse(len(cs.err     )	,  msg=f'''should have no warning, but got {cs.err}'''     )

												__UpperCAmelCase						:			Tuple         =   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
												__UpperCAmelCase						:			int         =   batch.convert_to_tensors(tensor_type="tf"	,  prepend_batch_axis=__lowerCamelCase     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (1, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (1,)     )
					@require_flax
					def    _lowerCamelCase (  self:				Any     )    ->				Tuple:
												__UpperCAmelCase						:			Optional[Any]         =   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}     )
												__UpperCAmelCase						:			List[Any]         =   batch.convert_to_tensors(tensor_type="jax"     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (2, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (2,)     )
												# test converting the converted
												with CaptureStderr() as cs:
																			__UpperCAmelCase						:			Tuple         =   batch.convert_to_tensors(tensor_type="jax"     )
												self.assertFalse(len(cs.err     )	,  msg=f'''should have no warning, but got {cs.err}'''     )

												__UpperCAmelCase						:			str         =   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}     )
												__UpperCAmelCase						:			str         =   batch.convert_to_tensors(tensor_type="jax"	,  prepend_batch_axis=__lowerCamelCase     )
												self.assertEqual(tensor_batch["inputs"].shape	,  (1, 3)     )
												self.assertEqual(tensor_batch["labels"].shape	,  (1,)     )
					def    _lowerCamelCase (  self:				Optional[int]     )    ->				Optional[int]:
												__UpperCAmelCase						:			List[Any]         =   [{"input_ids": np.array([0, 1, 2]     )}, {"input_ids": np.array([0, 1, 2, 3]     )}]
												__UpperCAmelCase						:			Any         =   BertTokenizer.from_pretrained("bert-base-cased"     )

												__UpperCAmelCase						:			Union[str, Any]         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase     )
												self.assertTrue(isinstance(batch["input_ids"]	,  np.ndarray     )     )
												self.assertEqual(batch["input_ids"].tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
												__UpperCAmelCase						:			Union[str, Any]         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase	,  return_tensors="np"     )
												self.assertTrue(isinstance(batch["input_ids"]	,  np.ndarray     )     )
												self.assertEqual(batch["input_ids"].tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
					@require_torch
					def    _lowerCamelCase (  self:				List[Any]     )    ->				Optional[Any]:
												import torch

												__UpperCAmelCase						:			Any         =   [{"input_ids": torch.tensor([0, 1, 2]     )}, {"input_ids": torch.tensor([0, 1, 2, 3]     )}]
												__UpperCAmelCase						:			Any         =   BertTokenizer.from_pretrained("bert-base-cased"     )

												__UpperCAmelCase						:			str         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase     )
												self.assertTrue(isinstance(batch["input_ids"]	,  torch.Tensor     )     )
												self.assertEqual(batch["input_ids"].tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
												__UpperCAmelCase						:			Union[str, Any]         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase	,  return_tensors="pt"     )
												self.assertTrue(isinstance(batch["input_ids"]	,  torch.Tensor     )     )
												self.assertEqual(batch["input_ids"].tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
					@require_tf
					def    _lowerCamelCase (  self:				int     )    ->				Union[str, Any]:
												import tensorflow as tf

												__UpperCAmelCase						:			Any         =   [{"input_ids": tf.constant([0, 1, 2]     )}, {"input_ids": tf.constant([0, 1, 2, 3]     )}]
												__UpperCAmelCase						:			List[str]         =   BertTokenizer.from_pretrained("bert-base-cased"     )

												__UpperCAmelCase						:			str         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase     )
												self.assertTrue(isinstance(batch["input_ids"]	,  tf.Tensor     )     )
												self.assertEqual(batch["input_ids"].numpy().tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
												__UpperCAmelCase						:			Dict         =   tokenizer.pad(__lowerCamelCase	,  padding=__lowerCamelCase	,  return_tensors="tf"     )
												self.assertTrue(isinstance(batch["input_ids"]	,  tf.Tensor     )     )
												self.assertEqual(batch["input_ids"].numpy().tolist()	,  [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
					@require_tokenizers
					def    _lowerCamelCase (  self:				Dict     )    ->				int:
												__UpperCAmelCase						:			Optional[int]         =   Tokenizer(WordPiece(unk_token="[UNK]"     )     )
												PreTrainedTokenizerFast(tokenizer_object=__lowerCamelCase     )





					@require_tokenizers
					def    _lowerCamelCase (  self:				str     )    ->				Dict:
												__UpperCAmelCase						:			Tuple         =   Tokenizer(WordPiece(unk_token="[UNK]"     )     )
												with tempfile.TemporaryDirectory() as tmpdirname:
																			bert_tokenizer.save(os.path.join(__lowerCamelCase	,  "tokenizer.json"     )     )
																			PreTrainedTokenizerFast(tokenizer_file=os.path.join(__lowerCamelCase	,  "tokenizer.json"     )     )




