


import glob
import os
from typing import Dict, List, Union

import torch

from diffusers.utils import is_safetensors_available


if is_safetensors_available():
    import safetensors.torch

from huggingface_hub import snapshot_download

from diffusers import DiffusionPipeline, __version__
from diffusers.schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME
from diffusers.utils import CONFIG_NAME, DIFFUSERS_CACHE, ONNX_WEIGHTS_NAME, WEIGHTS_NAME







class   lowerCAmelCase__ (       a):




     '''simple docstring'''


     def __init__(		self)    ->       Optional[Any]:
            self.register_to_config()
            super().__init__()


     def 				_lowerCamelCase				(		self				,		__lowerCamelCase				,		__lowerCamelCase)    ->       Any:
            if dicta == dicta:
                   return True
            else:
                   _A					,    _A   : List[Any]       					=				self._remove_meta_keys(__lowerCamelCase)
                   _A					,    _A   : Tuple       					=				self._remove_meta_keys(__lowerCamelCase)
                   if configa == configa:
                          print(F"Warning !: Mismatch in keys {meta_keysa} and {meta_keysa}.")
                          return True
            return False


     def 				_lowerCamelCase				(		self				,		__lowerCamelCase)    ->       Tuple:
            _A   : Any       					=				[]
            _A   : List[Any]       					=				config_dict.copy()
            for key in config_dict.keys():
                   if key.startswith("_"):
                          temp_dict.pop(__lowerCamelCase)
                          meta_keys.append(__lowerCamelCase)
            return (temp_dict, meta_keys)


     @torch.no_grad()
     def 				_lowerCamelCase				(		self				,		__lowerCamelCase				,		**__lowerCamelCase)    ->       List[Any]:
            _A   : Any       					=				kwargs.pop("cache_dir"				,		__lowerCamelCase)
            _A   : List[Any]       					=				kwargs.pop("resume_download"				,		__lowerCamelCase)
            _A   : int       					=				kwargs.pop("force_download"				,		__lowerCamelCase)
            _A   : List[Any]       					=				kwargs.pop("proxies"				,		__lowerCamelCase)
            _A   : Optional[int]       					=				kwargs.pop("local_files_only"				,		__lowerCamelCase)
            _A   : Union[str, Any]       					=				kwargs.pop("use_auth_token"				,		__lowerCamelCase)
            _A   : int       					=				kwargs.pop("revision"				,		__lowerCamelCase)
            _A   : Tuple       					=				kwargs.pop("torch_dtype"				,		__lowerCamelCase)
            _A   : List[Any]       					=				kwargs.pop("device_map"				,		__lowerCamelCase)

            _A   : List[Any]       					=				kwargs.pop("alpha"				,		0.5)
            _A   : Any       					=				kwargs.pop("interp"				,		__lowerCamelCase)

            print("Received list"				,		__lowerCamelCase)
            print(F"Combining with alpha={alpha}, interpolation mode={interp}")

            _A   : Tuple       					=				len(__lowerCamelCase)
            # Ignore result from model_index_json comparision of the two checkpoints
            _A   : Tuple       					=				kwargs.pop("force"				,		__lowerCamelCase)

            # If less than 2 checkpoints, nothing to merge. If more than 3, not supported for now.
            if checkpoint_count > 3 or checkpoint_count < 2:
                   raise ValueError(
                       "Received incorrect number of checkpoints to merge. Ensure that either 2 or 3 checkpoints are being"
                       " passed.")

            print("Received the right number of checkpoints")
            # chkpt0, chkpt1 = pretrained_model_name_or_path_list[0:2]
            # chkpt2 = pretrained_model_name_or_path_list[2] if checkpoint_count == 3 else None

            # Validate that the checkpoints can be merged
            # Step 1: Load the model config and compare the checkpoints. We'll compare the model_index.json first while ignoring the keys starting with '_'
            _A   : Union[str, Any]       					=				[]
            for pretrained_model_name_or_path in pretrained_model_name_or_path_list:
                   _A   : Dict       					=				DiffusionPipeline.load_config(
                       __lowerCamelCase				,		cache_dir=__lowerCamelCase				,		resume_download=__lowerCamelCase				,		force_download=__lowerCamelCase				,		proxies=__lowerCamelCase				,		local_files_only=__lowerCamelCase				,		use_auth_token=__lowerCamelCase				,		revision=__lowerCamelCase				,		)
                   config_dicts.append(__lowerCamelCase)

            _A   : Any       					=				True
            for idx in range(1				,		len(__lowerCamelCase)):
                   comparison_result &= self._compare_model_configs(config_dicts[idx - 1]				,		config_dicts[idx])
                   if not force and comparison_result is False:
                          raise ValueError("Incompatible checkpoints. Please check model_index.json for the models.")
                          print(config_dicts[0]				,		config_dicts[1])
            print("Compatible model_index.json files found")
            # Step 2: Basic Validation has succeeded. Let's download the models and save them into our local files.
            _A   : Optional[Any]       					=				[]
            for pretrained_model_name_or_path, config_dict in zip(__lowerCamelCase				,		__lowerCamelCase):
                   _A   : int       					=				[k for k in config_dict.keys() if not k.startswith("_")]
                   _A   : Any       					=				[os.path.join(__lowerCamelCase				,		"*") for k in folder_names]
                   allow_patterns += [
                       WEIGHTS_NAME,
                       SCHEDULER_CONFIG_NAME,
                       CONFIG_NAME,
                       ONNX_WEIGHTS_NAME,
                       DiffusionPipeline.config_name,
                   ]
                   _A   : int       					=				config_dict.get("_class_name")
                   _A   : int       					=				{"diffusers": __version__, "pipeline_class": requested_pipeline_class}

                   _A   : Tuple       					=				(
                       pretrained_model_name_or_path
                       if os.path.isdir(__lowerCamelCase)
                       else snapshot_download(
                           __lowerCamelCase				,		cache_dir=__lowerCamelCase				,		resume_download=__lowerCamelCase				,		proxies=__lowerCamelCase				,		local_files_only=__lowerCamelCase				,		use_auth_token=__lowerCamelCase				,		revision=__lowerCamelCase				,		allow_patterns=__lowerCamelCase				,		user_agent=__lowerCamelCase				,		)
                   )
                   print("Cached Folder"				,		__lowerCamelCase)
                   cached_folders.append(__lowerCamelCase)

            # Step 3:-
            # Load the first checkpoint as a diffusion pipeline and modify its module state_dict in place
            _A   : str       					=				DiffusionPipeline.from_pretrained(
                cached_folders[0]				,		torch_dtype=__lowerCamelCase				,		device_map=__lowerCamelCase)
            final_pipe.to(self.device)

            _A   : Any       					=				None
            if len(__lowerCamelCase) > 2:
                   _A   : Optional[Any]       					=				os.path.join(cached_folders[2])

            if interp == "sigmoid":
                   _A   : List[str]       					=				CheckpointMergerPipeline.sigmoid
            elif interp == "inv_sigmoid":
                   _A   : Dict       					=				CheckpointMergerPipeline.inv_sigmoid
            elif interp == "add_diff":
                   _A   : Dict       					=				CheckpointMergerPipeline.add_difference
            else:
                   _A   : List[str]       					=				CheckpointMergerPipeline.weighted_sum

            # Find each module's state dict.
            for attr in final_pipe.config.keys():
                   if not attr.startswith("_"):
                          _A   : Optional[int]       					=				os.path.join(cached_folders[1]				,		__lowerCamelCase)
                          if os.path.exists(__lowerCamelCase):
                                 _A   : Dict       					=				[
                                     *glob.glob(os.path.join(__lowerCamelCase				,		"*.safetensors")),
                                     *glob.glob(os.path.join(__lowerCamelCase				,		"*.bin")),
                                 ]
                                 _A   : Tuple       					=				files[0] if len(__lowerCamelCase) > 0 else None
                          if len(__lowerCamelCase) < 3:
                                 _A   : int       					=				None
                          else:
                                 _A   : int       					=				os.path.join(cached_folders[2]				,		__lowerCamelCase)
                                 if os.path.exists(__lowerCamelCase):
                                        _A   : Any       					=				[
                                            *glob.glob(os.path.join(__lowerCamelCase				,		"*.safetensors")),
                                            *glob.glob(os.path.join(__lowerCamelCase				,		"*.bin")),
                                        ]
                                        _A   : Dict       					=				files[0] if len(__lowerCamelCase) > 0 else None
                # For an attr if both checkpoint_path_1 and 2 are None, ignore.
                # If atleast one is present, deal with it according to interp method, of course only if the state_dict keys match.
                          if checkpoint_path_a is None and checkpoint_path_a is None:
                                 print(F"Skipping {attr}: not present in 2nd or 3d model")
                                 continue
                          try:
                                 _A   : Any       					=				getattr(__lowerCamelCase				,		__lowerCamelCase)
                                 if isinstance(__lowerCamelCase				,		__lowerCamelCase):  # ignore requires_safety_checker boolean
                                        continue
                                 _A   : Dict       					=				getattr(__lowerCamelCase				,		"state_dict")
                                 _A   : List[str]       					=				theta_a()

                                 _A   : List[str]       					=				getattr(__lowerCamelCase				,		"load_state_dict")
                                 _A   : List[Any]       					=				(
                                     safetensors.torch.load_file(__lowerCamelCase)
                                     if (is_safetensors_available() and checkpoint_path_a.endswith(".safetensors"))
                                     else torch.load(__lowerCamelCase				,		map_location="cpu")
                                 )
                                 _A   : List[str]       					=				None
                                 if checkpoint_path_a:
                                        _A   : Union[str, Any]       					=				(
                                            safetensors.torch.load_file(__lowerCamelCase)
                                            if (is_safetensors_available() and checkpoint_path_a.endswith(".safetensors"))
                                            else torch.load(__lowerCamelCase				,		map_location="cpu")
                                        )

                                 if not theta_a.keys() == theta_a.keys():
                                        print(F"Skipping {attr}: key mismatch")
                                        continue
                                 if theta_a and not theta_a.keys() == theta_a.keys():
                                        print(F"Skipping {attr}:y mismatch")
                          except Exception as e:
                                 print(F"Skipping {attr} do to an unexpected error: {str(__lowerCamelCase)}")
                                 continue
                          print(F"MERGING {attr}")

                          for key in theta_a.keys():
                                 if theta_a:
                                        _A   : List[str]       					=				theta_func(theta_a[key]				,		theta_a[key]				,		theta_a[key]				,		__lowerCamelCase)
                                 else:
                                        _A   : Optional[int]       					=				theta_func(theta_a[key]				,		theta_a[key]				,		__lowerCamelCase				,		__lowerCamelCase)

                          del theta_a
                          del theta_a
                          update_theta_a(__lowerCamelCase)

                          del theta_a
            return final_pipe


     @staticmethod
     def 				_lowerCamelCase				(		__lowerCamelCase				,		__lowerCamelCase				,		__lowerCamelCase				,		__lowerCamelCase)    ->       str:
            return ((1 - alpha) * thetaa) + (alpha * thetaa)


     @staticmethod
     def 				_lowerCamelCase				(		__lowerCamelCase				,		__lowerCamelCase				,		__lowerCamelCase				,		__lowerCamelCase)    ->       List[Any]:
            _A   : Tuple       					=				alpha * alpha * (3 - (2 * alpha))
            return thetaa + ((thetaa - thetaa) * alpha)


     @staticmethod
     def 				_lowerCamelCase				(		__lowerCamelCase				,		__lowerCamelCase				,		__lowerCamelCase				,		__lowerCamelCase)    ->       Union[str, Any]:
            import math

            _A   : Optional[int]       					=				0.5 - math.sin(math.asin(1.0 - 2.0 * alpha) / 3.0)
            return thetaa + ((thetaa - thetaa) * alpha)


     @staticmethod
     def 				_lowerCamelCase				(		__lowerCamelCase				,		__lowerCamelCase				,		__lowerCamelCase				,		__lowerCamelCase)    ->       Optional[Any]:
            return thetaa + (thetaa - thetaa) * (1.0 - alpha)




import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


lowerCAmelCase__   	=				get_logger(__name__)


if TYPE_CHECKING:
    from .builder import DatasetBuilder
def 							_UpperCAmelCase       (UpperCamelCase__  :						Optional[Any]       ,							UpperCamelCase__  :						Optional[Union[str, bool]] = None		):
       _A   : Dict       					=				importlib.import_module(UpperCamelCase__		)

       # TODO(QL): always update the module to add subsequent new authentication
       if hasattr(UpperCamelCase__       ,							"_patched_for_streaming"		) and module._patched_for_streaming:
              return

       def wrap_auth(UpperCamelCase__  :						Optional[int]		):
              @wraps(UpperCamelCase__		)
              def wrapper(*UpperCamelCase__  :						str       ,							**UpperCamelCase__  :						Optional[int]		):
                     return function(*UpperCamelCase__       ,							use_auth_token=UpperCamelCase__       ,							**UpperCamelCase__		)

              _A   : List[Any]       					=				"wrap_auth"
              return wrapper

       # open files in a streaming fashion
       patch_submodule(UpperCamelCase__       ,							"open"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"os.listdir"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"os.walk"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"glob.glob"       ,							wrap_auth(UpperCamelCase__		)		).start()
       # allow to navigate in remote zip files
       patch_submodule(UpperCamelCase__       ,							"os.path.join"       ,							UpperCamelCase__		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.dirname"       ,							UpperCamelCase__		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.basename"       ,							UpperCamelCase__		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.relpath"       ,							UpperCamelCase__		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.split"       ,							UpperCamelCase__		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.splitext"       ,							UpperCamelCase__		).start()
       # allow checks on paths
       patch_submodule(UpperCamelCase__       ,							"os.path.exists"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.isdir"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.isfile"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"os.path.getsize"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"pathlib.Path"       ,							UpperCamelCase__		).start()
       # file readers
       patch_submodule(UpperCamelCase__       ,							"gzip.open"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"numpy.load"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"pandas.read_csv"       ,							wrap_auth(UpperCamelCase__		)       ,							attrs=["__version__"]		).start()
       patch_submodule(UpperCamelCase__       ,							"pandas.read_excel"       ,							wrap_auth(UpperCamelCase__		)       ,							attrs=["__version__"]		).start()
       patch_submodule(UpperCamelCase__       ,							"scipy.io.loadmat"       ,							wrap_auth(UpperCamelCase__		)       ,							attrs=["__version__"]		).start()
       patch_submodule(UpperCamelCase__       ,							"xml.etree.ElementTree.parse"       ,							wrap_auth(UpperCamelCase__		)		).start()
       patch_submodule(UpperCamelCase__       ,							"xml.dom.minidom.parse"       ,							wrap_auth(UpperCamelCase__		)		).start()
       _A   : Any       					=				True






def 							_UpperCAmelCase       (UpperCamelCase__  :						"DatasetBuilder"		):
       extend_module_for_streaming(builder.__module__       ,							use_auth_token=builder.use_auth_token		)
       # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
       if not builder.__module__.startswith("datasets."		):  # check that it's not a packaged builder like csv
              for imports in get_imports(inspect.getfile(builder.__class__		)		):
                     if imports[0] == "internal":
                            _A   : Optional[Any]       					=				imports[1]
                            _A   : int       					=				".".join(builder.__module__.split("."		)[:-1] + [internal_import_name]		)
                            extend_module_for_streaming(UpperCamelCase__       ,							use_auth_token=builder.use_auth_token		)

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
       from .builder import DatasetBuilder

       _A   : Tuple       					=				[
           cls.__module__
           for cls in type(UpperCamelCase__		).__mro__[1:]  # make sure it's not the same module we've already patched
           if issubclass(cls       ,							UpperCamelCase__		) and cls.__module__ != DatasetBuilder.__module__
       ]  # check it's not a standard builder from datasets.builder
       for module in parent_builder_modules:
              extend_module_for_streaming(UpperCamelCase__       ,							use_auth_token=builder.use_auth_token		)
