
import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
				if is_torch_available():
								import torch
				if is_tf_available():
								import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


snake_case__  							=							logging.get_logger(__name__)

snake_case__  							=							{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

snake_case__  							=							{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

snake_case__  							=							{
    """Salesforce/codegen-350M-mono""": 20_48,
}

@lru_cache()
def        lowerCamelCase_      (			):
						lowercase       :    List[Any]											=  (
						    list(range(ord('''!'''       )							,			ord('''~'''       ) + 1       )       ) + list(range(ord('''¡'''       )							,			ord('''¬'''       ) + 1       )       ) + list(range(ord('''®'''       )							,			ord('''ÿ'''       ) + 1       )       )
						)
						lowercase       :    List[str]											=  bs[:]
						lowercase       :    List[str]											=  0
						for b in range(2**8       ):
												if b not in bs:
																		bs.append(UpperCAmelCase_       )
																		cs.append(2**8 + n       )
																		n += 1
						lowercase       :    List[str]											=  [chr(UpperCAmelCase_       ) for n in cs]
						return dict(zip(UpperCAmelCase_							,			UpperCAmelCase_       )       )

def        lowerCamelCase_      (			UpperCAmelCase_							:				Union[str, Any]       ):
						lowercase       :    List[str]											=  set()
						lowercase       :    int											=  word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)       )
												lowercase       :    Optional[Any]											=  char
						return pairs







class 							UpperCAmelCase	(   __lowerCamelCase					):
		a__: Tuple			  =				VOCAB_FILES_NAMES
		a__: Union[str, Any]			  =				PRETRAINED_VOCAB_FILES_MAP
		a__: List[str]			  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a__: Any			  =				["""input_ids""", """attention_mask"""]

		def __init__(			self      :					List[str] ,      lowerCAmelCase      :					Tuple ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					str="replace" ,      lowerCAmelCase      :					int="<|endoftext|>" ,      lowerCAmelCase      :					Optional[Any]="<|endoftext|>" ,      lowerCAmelCase      :					int="<|endoftext|>" ,      lowerCAmelCase      :					List[str]=None ,      lowerCAmelCase      :					List[str]=False ,      lowerCAmelCase      :					int=False ,      **lowerCAmelCase      :					Any ,      ):
								lowercase       :    str											=  AddedToken(lowerCAmelCase ,      lstrip=lowerCAmelCase ,      rstrip=lowerCAmelCase					) if isinstance(lowerCAmelCase ,      lowerCAmelCase					) else bos_token
								lowercase       :    Any											=  AddedToken(lowerCAmelCase ,      lstrip=lowerCAmelCase ,      rstrip=lowerCAmelCase					) if isinstance(lowerCAmelCase ,      lowerCAmelCase					) else eos_token
								lowercase       :    Optional[Any]											=  AddedToken(lowerCAmelCase ,      lstrip=lowerCAmelCase ,      rstrip=lowerCAmelCase					) if isinstance(lowerCAmelCase ,      lowerCAmelCase					) else unk_token
								lowercase       :    Dict											=  AddedToken(lowerCAmelCase ,      lstrip=lowerCAmelCase ,      rstrip=lowerCAmelCase					) if isinstance(lowerCAmelCase ,      lowerCAmelCase					) else pad_token
								super().__init__(
								    errors=lowerCAmelCase ,      unk_token=lowerCAmelCase ,      bos_token=lowerCAmelCase ,      eos_token=lowerCAmelCase ,      pad_token=lowerCAmelCase ,      add_prefix_space=lowerCAmelCase ,      add_bos_token=lowerCAmelCase ,      **lowerCAmelCase ,      )
								lowercase       :    Optional[int]											=  add_bos_token

								with open(lowerCAmelCase ,      encoding='''utf-8'''					) as vocab_handle:
														lowercase       :    Any											=  json.load(lowerCAmelCase					)
								lowercase       :    Union[str, Any]											=  {v: k for k, v in self.encoder.items()}
								lowercase       :    Any											=  errors  # how to handle errors in decoding
								lowercase       :    str											=  bytes_to_unicode()
								lowercase       :    int											=  {v: k for k, v in self.byte_encoder.items()}
								with open(lowerCAmelCase ,      encoding='''utf-8'''					) as merges_handle:
														lowercase       :    str											=  merges_handle.read().split('''\n'''					)[1:-1]
								lowercase       :    Any											=  [tuple(merge.split()					) for merge in bpe_merges]
								lowercase       :    List[str]											=  dict(zip(lowerCAmelCase ,      range(len(lowerCAmelCase					)					)					)					)
								lowercase       :    str											=  {}
								lowercase       :    int											=  add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								lowercase       :    Union[str, Any]											=  re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''					)

		@property
		def        _lowerCAmelCase	(			self      :					str					):
								return len(self.encoder					)

		def        _lowerCAmelCase	(			self      :					List[Any]					):
								return dict(self.encoder ,      **self.added_tokens_encoder					)

		def        _lowerCAmelCase	(			self      :					str ,      lowerCAmelCase      :					Optional[int]					):
								if token in self.cache:
														return self.cache[token]
								lowercase       :    Union[str, Any]											=  tuple(lowerCAmelCase					)
								lowercase       :    Dict											=  get_pairs(lowerCAmelCase					)

								if not pairs:
														return token

								while True:
														lowercase       :    int											=  min(lowerCAmelCase ,      key=lambda lowerCAmelCase					: self.bpe_ranks.get(lowerCAmelCase ,      float('''inf'''					)					)					)
														if bigram not in self.bpe_ranks:
																				break
														lowercase       :    Optional[Any]											=  bigram
														lowercase       :    List[Any]											=  []
														lowercase       :    Optional[Any]											=  0
														while i < len(lowerCAmelCase					):
																				try:
																										lowercase       :    List[str]											=  word.index(lowerCAmelCase ,      lowerCAmelCase					)
																				except ValueError:
																										new_word.extend(word[i:]					)
																										break
																				else:
																										new_word.extend(word[i:j]					)
																										lowercase       :    str											=  j

																				if word[i] == first and i < len(lowerCAmelCase					) - 1 and word[i + 1] == second:
																										new_word.append(first + second					)
																										i += 2
																				else:
																										new_word.append(word[i]					)
																										i += 1
														lowercase       :    int											=  tuple(lowerCAmelCase					)
														lowercase       :    List[Any]											=  new_word
														if len(lowerCAmelCase					) == 1:
																				break
														else:
																				lowercase       :    int											=  get_pairs(lowerCAmelCase					)
								lowercase       :    Tuple											=  ''' '''.join(lowerCAmelCase					)
								lowercase       :    Optional[int]											=  word
								return word

		def        _lowerCAmelCase	(			self      :					Any ,      lowerCAmelCase      :					Optional[int] ,      lowerCAmelCase      :					Optional[Any]=None					):
								if self.add_bos_token:
														lowercase       :    Union[str, Any]											=  [self.bos_token_id]
								else:
														lowercase       :    Union[str, Any]											=  []

								lowercase       :    List[Any]											=  bos_token_ids + token_ids_a

								if token_ids_a is None:
														return output

								return output + bos_token_ids + token_ids_a

		def        _lowerCAmelCase	(			self      :					Dict ,      lowerCAmelCase      :					Dict					):
								lowercase       :    Optional[int]											=  []
								for token in re.findall(self.pat ,      lowerCAmelCase					):
														lowercase       :    List[str]											=  ''''''.join(
														    self.byte_encoder[b] for b in token.encode('''utf-8'''					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
														bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase					).split(''' '''					)					)
								return bpe_tokens

		def        _lowerCAmelCase	(			self      :					str ,      lowerCAmelCase      :					Any					):
								return self.encoder.get(lowerCAmelCase ,      self.encoder.get(self.unk_token					)					)

		def        _lowerCAmelCase	(			self      :					int ,      lowerCAmelCase      :					int					):
								return self.decoder.get(lowerCAmelCase					)

		def        _lowerCAmelCase	(			self      :					List[Any] ,      lowerCAmelCase      :					Tuple					):
								lowercase       :    List[str]											=  ''''''.join(lowerCAmelCase					)
								lowercase       :    Optional[int]											=  bytearray([self.byte_decoder[c] for c in text]					).decode('''utf-8''' ,      errors=self.errors					)
								return text

		def        _lowerCAmelCase	(			self      :					Optional[int] ,      lowerCAmelCase      :					str ,      lowerCAmelCase      :					Optional[str] = None					):
								if not os.path.isdir(lowerCAmelCase					):
														logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
														return
								lowercase       :    Optional[Any]											=  os.path.join(
								    lowerCAmelCase ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
								lowercase       :    Dict											=  os.path.join(
								    lowerCAmelCase ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']					)

								with open(lowerCAmelCase ,      '''w''' ,      encoding='''utf-8'''					) as f:
														f.write(json.dumps(self.encoder ,      indent=2 ,      sort_keys=lowerCAmelCase ,      ensure_ascii=lowerCAmelCase					) + '''\n'''					)

								lowercase       :    List[Any]											=  0
								with open(lowerCAmelCase ,      '''w''' ,      encoding='''utf-8'''					) as writer:
														writer.write('''#version: 0.2\n'''					)
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,      key=lambda lowerCAmelCase					: kv[1]					):
																				if index != token_index:
																										logger.warning(
																										    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																										    ''' Please check that the tokenizer is not corrupted!'''					)
																										lowercase       :    Dict											=  token_index
																				writer.write(''' '''.join(lowerCAmelCase					) + '''\n'''					)
																				index += 1

								return vocab_file, merge_file

		def        _lowerCAmelCase	(			self      :					str ,      lowerCAmelCase      :					int ,      lowerCAmelCase      :					Union[str, Any]=False ,      **lowerCAmelCase      :					List[Any]					):
								lowercase       :    Union[str, Any]											=  kwargs.pop('''add_prefix_space''' ,      self.add_prefix_space					)
								if is_split_into_words or add_prefix_space:
														lowercase       :    Tuple											=  ''' ''' + text
								return (text, kwargs)

		def        _lowerCAmelCase	(			self      :					Any ,      lowerCAmelCase      :					Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"] ,      lowerCAmelCase      :					bool = False ,      lowerCAmelCase      :					bool = None ,      lowerCAmelCase      :					Optional[List[str]] = None ,      **lowerCAmelCase      :					int ,      ):
								lowercase       :    int											=  super()._decode(
								    token_ids=lowerCAmelCase ,      skip_special_tokens=lowerCAmelCase ,      clean_up_tokenization_spaces=lowerCAmelCase ,      **lowerCAmelCase ,      )

								if truncate_before_pattern is not None and len(lowerCAmelCase					) > 0:
														lowercase       :    Any											=  self.truncate(lowerCAmelCase ,      lowerCAmelCase					)

								return decoded_text




		def        _lowerCAmelCase	(			self      :					str ,      lowerCAmelCase      :					Any ,      lowerCAmelCase      :					str					):
								def find_re(lowerCAmelCase      :					Dict ,      lowerCAmelCase      :					List[str] ,      lowerCAmelCase      :					int					):
														lowercase       :    Dict											=  pattern.search(lowerCAmelCase ,      lowerCAmelCase					)
														return m.start() if m else -1

								lowercase       :    Tuple											=  [re.compile(lowerCAmelCase ,      re.MULTILINE					) for pattern in truncate_before_pattern]

								lowercase       :    List[Any]											=  list(re.finditer('''^print''' ,      lowerCAmelCase ,      re.MULTILINE					)					)

								if len(lowerCAmelCase					) > 1:
														lowercase       :    Dict											=  completion[: prints[1].start()]

								lowercase       :    Dict											=  list(re.finditer('''^def''' ,      lowerCAmelCase ,      re.MULTILINE					)					)

								if len(lowerCAmelCase					) > 1:
														lowercase       :    Optional[Any]											=  completion[: defs[1].start()]

								lowercase       :    Any											=  0

								lowercase       :    str											=  [
								    pos for pos in [find_re(lowerCAmelCase ,      lowerCAmelCase ,      lowerCAmelCase					) for terminal in terminals] if pos != -1
								]

								if len(lowerCAmelCase					) > 0:
														return completion[: min(lowerCAmelCase					)]
								else:
														return completion




from ..utils import (
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_librosa_available,
    is_note_seq_available,
    is_onnx_available,
    is_torch_available,
    is_transformers_available,
)


try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				from ..utils.dummy_pt_objects import *  # noqa F403
else:
				from .consistency_models import ConsistencyModelPipeline
				from .dance_diffusion import DanceDiffusionPipeline
				from .ddim import DDIMPipeline
				from .ddpm import DDPMPipeline
				from .dit import DiTPipeline
				from .latent_diffusion import LDMSuperResolutionPipeline
				from .latent_diffusion_uncond import LDMPipeline
				from .pipeline_utils import AudioPipelineOutput, DiffusionPipeline, ImagePipelineOutput
				from .pndm import PNDMPipeline
				from .repaint import RePaintPipeline
				from .score_sde_ve import ScoreSdeVePipeline
				from .stochastic_karras_ve import KarrasVePipeline

try:
				if not (is_torch_available() and is_librosa_available()):
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				from ..utils.dummy_torch_and_librosa_objects import *  # noqa F403
else:
				from .audio_diffusion import AudioDiffusionPipeline, Mel

try:
				if not (is_torch_available() and is_transformers_available()):
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				from ..utils.dummy_torch_and_transformers_objects import *  # noqa F403
else:
				from .alt_diffusion import AltDiffusionImgaImgPipeline, AltDiffusionPipeline
				from .audioldm import AudioLDMPipeline
				from .controlnet import (
				    StableDiffusionControlNetImgaImgPipeline,
				    StableDiffusionControlNetInpaintPipeline,
				    StableDiffusionControlNetPipeline,
				)
				from .deepfloyd_if import (
				    IFImgaImgPipeline,
				    IFImgaImgSuperResolutionPipeline,
				    IFInpaintingPipeline,
				    IFInpaintingSuperResolutionPipeline,
				    IFPipeline,
				    IFSuperResolutionPipeline,
				)
				from .kandinsky import (
				    KandinskyImgaImgPipeline,
				    KandinskyInpaintPipeline,
				    KandinskyPipeline,
				    KandinskyPriorPipeline,
				)
				from .kandinskya_a import (
				    KandinskyVaaControlnetImgaImgPipeline,
				    KandinskyVaaControlnetPipeline,
				    KandinskyVaaImgaImgPipeline,
				    KandinskyVaaInpaintPipeline,
				    KandinskyVaaPipeline,
				    KandinskyVaaPriorEmbaEmbPipeline,
				    KandinskyVaaPriorPipeline,
				)
				from .latent_diffusion import LDMTextToImagePipeline
				from .paint_by_example import PaintByExamplePipeline
				from .semantic_stable_diffusion import SemanticStableDiffusionPipeline
				from .shap_e import ShapEImgaImgPipeline, ShapEPipeline
				from .stable_diffusion import (
				    CycleDiffusionPipeline,
				    StableDiffusionAttendAndExcitePipeline,
				    StableDiffusionDepthaImgPipeline,
				    StableDiffusionDiffEditPipeline,
				    StableDiffusionImageVariationPipeline,
				    StableDiffusionImgaImgPipeline,
				    StableDiffusionInpaintPipeline,
				    StableDiffusionInpaintPipelineLegacy,
				    StableDiffusionInstructPixaPixPipeline,
				    StableDiffusionLatentUpscalePipeline,
				    StableDiffusionLDMaDPipeline,
				    StableDiffusionModelEditingPipeline,
				    StableDiffusionPanoramaPipeline,
				    StableDiffusionParadigmsPipeline,
				    StableDiffusionPipeline,
				    StableDiffusionPixaPixZeroPipeline,
				    StableDiffusionSAGPipeline,
				    StableDiffusionUpscalePipeline,
				    StableUnCLIPImgaImgPipeline,
				    StableUnCLIPPipeline,
				)
				from .stable_diffusion_safe import StableDiffusionPipelineSafe
				from .text_to_video_synthesis import TextToVideoSDPipeline, TextToVideoZeroPipeline, VideoToVideoSDPipeline
				from .unclip import UnCLIPImageVariationPipeline, UnCLIPPipeline
				from .unidiffuser import ImageTextPipelineOutput, UniDiffuserModel, UniDiffuserPipeline, UniDiffuserTextDecoder
				from .versatile_diffusion import (
				    VersatileDiffusionDualGuidedPipeline,
				    VersatileDiffusionImageVariationPipeline,
				    VersatileDiffusionPipeline,
				    VersatileDiffusionTextToImagePipeline,
				)
				from .vq_diffusion import VQDiffusionPipeline


try:
				if not (is_torch_available() and is_transformers_available() and is_invisible_watermark_available()):
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				from ..utils.dummy_torch_and_transformers_and_invisible_watermark_objects import *  # noqa F403
else:
				from .stable_diffusion_xl import StableDiffusionXLImgaImgPipeline, StableDiffusionXLPipeline

try:
				if not is_onnx_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				from ..utils.dummy_onnx_objects import *  # noqa F403
else:
				from .onnx_utils import OnnxRuntimeModel

try:
				if not (is_torch_available() and is_transformers_available() and is_onnx_available()):
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				from ..utils.dummy_torch_and_transformers_and_onnx_objects import *  # noqa F403
else:
				from .stable_diffusion import (
				    OnnxStableDiffusionImgaImgPipeline,
				    OnnxStableDiffusionInpaintPipeline,
				    OnnxStableDiffusionInpaintPipelineLegacy,
				    OnnxStableDiffusionPipeline,
				    OnnxStableDiffusionUpscalePipeline,
				    StableDiffusionOnnxPipeline,
				)

try:
				if not (is_torch_available() and is_transformers_available() and is_k_diffusion_available()):
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				from ..utils.dummy_torch_and_transformers_and_k_diffusion_objects import *  # noqa F403
else:
				from .stable_diffusion import StableDiffusionKDiffusionPipeline

try:
				if not is_flax_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				from ..utils.dummy_flax_objects import *  # noqa F403
else:
				from .pipeline_flax_utils import FlaxDiffusionPipeline


try:
				if not (is_flax_available() and is_transformers_available()):
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				from ..utils.dummy_flax_and_transformers_objects import *  # noqa F403
else:
				from .controlnet import FlaxStableDiffusionControlNetPipeline
				from .stable_diffusion import (
				    FlaxStableDiffusionImgaImgPipeline,
				    FlaxStableDiffusionInpaintPipeline,
				    FlaxStableDiffusionPipeline,
				)
try:
				if not (is_transformers_available() and is_torch_available() and is_note_seq_available()):
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				from ..utils.dummy_transformers_and_torch_and_note_seq_objects import *  # noqa F403
else:
				from .spectrogram_diffusion import MidiProcessor, SpectrogramDiffusionPipeline


