




import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)


class        UpperCAmelCase__				:



     '''simple docstring'''
     def __init__(				self ,    _lowerCAmelCase	):
            a			   =foo






     def __call__(				self	):
            return self.foo



class        UpperCAmelCase__				(							datasets.Dataset				):



     '''simple docstring'''
     @fingerprint_transform(inplace=_lowerCAmelCase	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    *_lowerCAmelCase ,    **_lowerCAmelCase	):
            return DatasetChild(self.data ,    fingerprint=_lowerCAmelCase	)






     @fingerprint_transform(inplace=_lowerCAmelCase	)
     def        lowerCAmelCase__  (				self ,    _lowerCAmelCase ,    *_lowerCAmelCase ,    **_lowerCAmelCase	):
            return DatasetChild(self.data ,    fingerprint=_lowerCAmelCase	)



class        UpperCAmelCase__				:



     '''simple docstring'''
     def __init__(				self ,    _lowerCAmelCase	):
            a			   =callable
     def __call__(				self ,    *_lowerCAmelCase ,    **_lowerCAmelCase	):
            if self.callable is not None:
                   return self.callable(*_lowerCAmelCase ,    **_lowerCAmelCase	)






     def __getstate__(				self	):
            raise pickle.PicklingError()



class        UpperCAmelCase__				(							UpperCAmelCase__				):



     '''simple docstring'''
     @require_transformers
     @pytest.mark.integration
     def        lowerCAmelCase__  (				self	):
            from transformers import AutoTokenizer

            def encode(_lowerCAmelCase	):
                   return tokenizer(_lowerCAmelCase	)

            # TODO: add hash consistency tests across sessions
            a			   =AutoTokenizer.from_pretrained("""bert-base-uncased"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =mda(datasets.utils.py_utils.dumps(lambda _lowerCAmelCase	: tokenizer(_lowerCAmelCase	)	)	).hexdigest()
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =AutoTokenizer.from_pretrained("""bert-base-cased"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =mda(datasets.utils.py_utils.dumps(lambda _lowerCAmelCase	: tokenizer(_lowerCAmelCase	)	)	).hexdigest()
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =AutoTokenizer.from_pretrained("""bert-base-uncased"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =mda(datasets.utils.py_utils.dumps(lambda _lowerCAmelCase	: tokenizer(_lowerCAmelCase	)	)	).hexdigest()
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     @require_transformers
     @pytest.mark.integration
     def        lowerCAmelCase__  (				self	):
            from transformers import AutoTokenizer

            a			   =AutoTokenizer.from_pretrained("""gpt2"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            tokenizer("""Hello world !"""	)  # call once to change the tokenizer's cache
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)






     @require_regex
     def        lowerCAmelCase__  (				self	):
            import regex

            a			   =regex.Regex("""foo"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =regex.Regex("""bar"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =regex.Regex("""foo"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)



class        UpperCAmelCase__				(							UpperCAmelCase__				):



     '''simple docstring'''
     def        lowerCAmelCase__  (				self	):
            def func():
                   return foo

            a			   =[0]
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =[1]
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =[0]
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     def        lowerCAmelCase__  (				self	):
            def func():
                   pass

            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()

            def func():
                   pass

            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =mda(datasets.utils.py_utils.dumps(Foo([0]	)	)	).hexdigest()
            a			   =mda(datasets.utils.py_utils.dumps(Foo([1]	)	)	).hexdigest()
            a			   =mda(datasets.utils.py_utils.dumps(Foo([0]	)	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =mda(datasets.utils.py_utils.dumps(Foo([0]	).__call__	)	).hexdigest()
            a			   =mda(datasets.utils.py_utils.dumps(Foo([1]	).__call__	)	).hexdigest()
            a			   =mda(datasets.utils.py_utils.dumps(Foo([0]	).__call__	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =(
                """co_argcount""",
                """co_kwonlyargcount""",
                """co_nlocals""",
                """co_stacksize""",
                """co_flags""",
                """co_code""",
                """co_consts""",
                """co_names""",
                """co_varnames""",
                """co_filename""",
                """co_name""",
                """co_firstlineno""",
                """co_lnotab""",
                """co_freevars""",
                """co_cellvars""",
            )

            def _create_code_pyaa(*_lowerCAmelCase	):
                   if hasattr(args[-3] ,    """encode"""	):
                          a			   =list(_lowerCAmelCase	)
                          a			   =args[-3].encode()  # co_lnotab
                          a			   =args[-10].encode()  # co_code
                   if hasattr(_lowerCAmelCase ,    """co_posonlyargcount"""	):
                          if len(_lowerCAmelCase	) == 16:
                                 return CodeType(*_lowerCAmelCase	)
                          elif len(_lowerCAmelCase	) == 15:
                                 return CodeType(args[0] ,    0 ,    *args[1:]	)
                          return CodeType(args[0] ,    0 ,    0 ,    *args[1:]	)
                   elif hasattr(_lowerCAmelCase ,    """co_kwonlyargcount"""	):
                          if len(_lowerCAmelCase	) == 16:
                                 return CodeType(args[0] ,    *args[2:]	)
                          elif len(_lowerCAmelCase	) == 15:
                                 return CodeType(*_lowerCAmelCase	)
                          return CodeType(args[0] ,    0 ,    *args[1:]	)
                   if len(_lowerCAmelCase	) == 16:
                          return CodeType(args[0] ,    *args[3:]	)
                   elif len(_lowerCAmelCase	) == 15:
                          return CodeType(args[0] ,    *args[2:]	)
                   return CodeType(*_lowerCAmelCase	)

            def create_ipython_func(_lowerCAmelCase ,    _lowerCAmelCase	):
                   def func():
                          return returned_obj

                   a			   =func.__code__
                   # Use _create_code from dill in order to make it work for different python versions
                   if datasets.config.PY_VERSION < version.parse("""3.8"""	):
                          a			   =_create_code_pyaa(
                              *[getattr(_lowerCAmelCase ,    _lowerCAmelCase	) if k != """co_filename""" else co_filename for k in code_args_pyaa]	)
                   else:
                          a			   =code.replace(co_filename=_lowerCAmelCase	)
                   return FunctionType(_lowerCAmelCase ,    func.__globals__ ,    func.__name__ ,    func.__defaults__ ,    func.__closure__	)

            a   ,      a			   ="""<ipython-input-2-e0383a102aae>""", [0]
            a			   =mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase ,    _lowerCAmelCase	)	)	).hexdigest()
            a   ,      a			   ="""<ipython-input-2-e0383a102aae>""", [1]
            a			   =mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase ,    _lowerCAmelCase	)	)	).hexdigest()
            a   ,      a			   ="""<ipython-input-5-713f6613acf3>""", [0]
            a			   =mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase ,    _lowerCAmelCase	)	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)

            a   ,      a			   =os.path.join(gettempdir() ,    """ipykernel_12345""" ,    """321456789.py"""	), [0]
            a			   =mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase ,    _lowerCAmelCase	)	)	).hexdigest()
            a   ,      a			   =os.path.join(gettempdir() ,    """ipykernel_12345""" ,    """321456789.py"""	), [1]
            a			   =mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase ,    _lowerCAmelCase	)	)	).hexdigest()
            a   ,      a			   =os.path.join(gettempdir() ,    """ipykernel_12345""" ,    """654123987.py"""	), [0]
            a			   =mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase ,    _lowerCAmelCase	)	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)






     def        lowerCAmelCase__  (				self	):
            a   ,      a			   =[0], [1]

            def func():
                   return foo, bar

            a			   ="""__main__"""

            def globalvars_mocka_side_effect(_lowerCAmelCase ,    *_lowerCAmelCase ,    **_lowerCAmelCase	):
                   return {"foo": foo, "bar": bar}

            def globalvars_mocka_side_effect(_lowerCAmelCase ,    *_lowerCAmelCase ,    **_lowerCAmelCase	):
                   return {"bar": bar, "foo": foo}

            with patch("""dill.detect.globalvars""" ,    side_effect=_lowerCAmelCase	) as globalvars_mocka:
                   a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
                   self.assertGreater(globalvars_mocka.call_count ,    0	)
            with patch("""dill.detect.globalvars""" ,    side_effect=_lowerCAmelCase	) as globalvars_mocka:
                   a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
                   self.assertGreater(globalvars_mocka.call_count ,    0	)
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)



class        UpperCAmelCase__				(							UpperCAmelCase__				):



     '''simple docstring'''
     def        lowerCAmelCase__  (				self	):
            a			   =Hasher.hash("""hello"""	)
            a			   =Hasher.hash("""hello"""	)
            a			   =Hasher.hash("""there"""	)
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =Hasher.hash(Foo("""hello"""	)	)
            a			   =Hasher.hash(Foo("""hello"""	)	)
            a			   =Hasher.hash(Foo("""there"""	)	)
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     def        lowerCAmelCase__  (				self	):
            a			   =Hasher()
            for x in ["hello", Foo("""hello"""	)]:
                   hasher.update(_lowerCAmelCase	)
            a			   =hasher.hexdigest()
            a			   =Hasher()
            for x in ["hello", Foo("""hello"""	)]:
                   hasher.update(_lowerCAmelCase	)
            a			   =hasher.hexdigest()
            a			   =Hasher()
            for x in ["there", Foo("""there"""	)]:
                   hasher.update(_lowerCAmelCase	)
            a			   =hasher.hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     def        lowerCAmelCase__  (				self	):
            with self.assertRaises(pickle.PicklingError	):
                   Hasher.hash(UnpicklableCallable(Foo("""hello"""	)	)	)
     def        lowerCAmelCase__  (				self	):
            a			   ="""abc"""
            a			   =[string, string]  # two strings have the same ids
            a			   =[string, string]
            a			   =json.loads(F'''["{string}", "{string}"]'''	)  # two strings have different ids
            self.assertIs(obja[0] ,    _lowerCAmelCase	)
            self.assertIs(obja[0] ,    obja[1]	)
            self.assertIs(obja[0] ,    _lowerCAmelCase	)
            self.assertIs(obja[0] ,    obja[1]	)
            self.assertIsNot(obja[0] ,    _lowerCAmelCase	)
            self.assertIsNot(obja[0] ,    obja[1]	)
            a			   =Hasher.hash(_lowerCAmelCase	)
            a			   =Hasher.hash(_lowerCAmelCase	)
            a			   =Hasher.hash(_lowerCAmelCase	)
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     @require_tiktoken
     def        lowerCAmelCase__  (				self	):
            import tiktoken

            a			   =tiktoken.get_encoding("""gpt2"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =tiktoken.get_encoding("""r50k_base"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =tiktoken.get_encoding("""gpt2"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
     @require_torch
     def        lowerCAmelCase__  (				self	):
            import torch

            a			   =torch.tensor([1.0]	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =torch.tensor([2.0]	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =torch.tensor([1.0]	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)






     @require_spacy
     @require_spacy_model("""en_core_web_sm"""	)
     @require_spacy_model("""fr_core_news_sm"""	)
     @pytest.mark.integration
     def        lowerCAmelCase__  (				self	):
            import spacy

            a			   =spacy.load("""en_core_web_sm"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =spacy.load("""fr_core_news_sm"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            a			   =spacy.load("""en_core_web_sm"""	)
            a			   =mda(datasets.utils.py_utils.dumps(_lowerCAmelCase	)	).hexdigest()
            self.assertEqual(_lowerCAmelCase ,    _lowerCAmelCase	)
            self.assertNotEqual(_lowerCAmelCase ,    _lowerCAmelCase	)





@pytest.mark.integration
def 							lowerCamelCase    (							UpperCAmelCase_	:      Path      )->			Optional[int]:

       """simple docstring"""




       a			   =tmp_path / """dir1"""
       a			   =tmp_path / """dir2"""
       dira.mkdir()
       dira.mkdir()
       a			   ="""script.py"""
       a			   =dedent(
           """
    from datasets.fingerprint import Hasher
    def foo():
        pass
    print(Hasher.hash(foo))
    """      )
       a			   =dira / script_filename
       a			   =dira / script_filename
       with script_patha.open("""w"""      ) as f:
              f.write(UpperCAmelCase_      )
       with script_patha.open("""w"""      ) as f:
              f.write(UpperCAmelCase_      )
       a			   =subprocess.check_output(["""python""", str(UpperCAmelCase_      )]      )
       a			   =subprocess.check_output(["""python""", str(UpperCAmelCase_      )]      )
       assert fingerprinta == fingerprinta





def 							lowerCamelCase    (							)->			List[Any]:

       """simple docstring"""




       a			   ={"""a""": [0, 1, 2]}
       a			   =DatasetChild(InMemoryTable.from_pydict(UpperCAmelCase_      )      )
       a			   =dataset.funca()._fingerprint
       assert expected_fingerprint == dataset.funca()._fingerprint
       assert expected_fingerprint != dataset.funca()._fingerprint

       with Pool(2      ) as p:
              assert expected_fingerprint == p.apply_async(dataset.funca      ).get()._fingerprint
              assert expected_fingerprint != p.apply_async(dataset.funca      ).get()._fingerprint





def 							lowerCamelCase    (							)->			str:

       """simple docstring"""




       a			   ={"""a""": [0, 1, 2]}


       class        UpperCAmelCase__				(							datasets.Dataset				):



            '''simple docstring'''
            @fingerprint_transform(inplace=_lowerCAmelCase	)
            def        lowerCAmelCase__  (				self ,    _lowerCAmelCase	):
                   return DummyDatasetChild(self.data ,    fingerprint=_lowerCAmelCase	)



       a			   =DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase_      )      ).func()


       class        UpperCAmelCase__				(							datasets.Dataset				):



            '''simple docstring'''
            @fingerprint_transform(inplace=_lowerCAmelCase ,    version="""1.0.0"""	)
            def        lowerCAmelCase__  (				self ,    _lowerCAmelCase	):
                   return DummyDatasetChild(self.data ,    fingerprint=_lowerCAmelCase	)



       a			   =DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase_      )      ).func()


       class        UpperCAmelCase__				(							datasets.Dataset				):



            '''simple docstring'''
            @fingerprint_transform(inplace=_lowerCAmelCase ,    version="""2.0.0"""	)
            def        lowerCAmelCase__  (				self ,    _lowerCAmelCase	):
                   return DummyDatasetChild(self.data ,    fingerprint=_lowerCAmelCase	)



       a			   =DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase_      )      ).func()

       assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}      ) == 3






def 							lowerCamelCase    (							)->			Optional[Any]:

       """simple docstring"""




       a			   =Hasher()
       hasher.update(lambda UpperCAmelCase_      : x      )






import math
from timeit import timeit





def 							lowerCamelCase    (							UpperCAmelCase_	:      int      )->			int:

       """simple docstring"""




       a			   =0
       a			   =abs(UpperCAmelCase_      )
       while True:
              a			   =n // 10
              digits += 1
              if n == 0:
                     break
       return digits





def 							lowerCamelCase    (							UpperCAmelCase_	:      int      )->			int:

       """simple docstring"""




       return 1 if n == 0 else math.floor(math.log(abs(UpperCAmelCase_      )   ,       10      ) + 1      )





def 							lowerCamelCase    (							UpperCAmelCase_	:      int      )->			int:

       """simple docstring"""




       return len(str(abs(UpperCAmelCase_      )      )      )





def 							lowerCamelCase    (							)->			None:

       """simple docstring"""




       from collections.abc import Callable

       def benchmark_a_function(UpperCAmelCase_	:      Callable   ,       UpperCAmelCase_	:      int      ) -> None:
              a			   =F'''{func.__name__}({value})'''
              a			   =timeit(F'''__main__.{call}'''   ,       setup="""import __main__"""      )
              print(F'''{call}: {func(UpperCAmelCase_      )} -- {timing} seconds'''      )

       for value in (26_2144, 1125_8999_0684_2624, 126_7650_6002_2822_9401_4967_0320_5376):
              for func in (num_digits, num_digits_fast, num_digits_faster):
                     benchmark_a_function(UpperCAmelCase_   ,       UpperCAmelCase_      )
              print()


if __name__ == "__main__":
    import doctest

    doctest.testmod()
    benchmark()
