



#
# Sort large text files in a minimum amount of memory
#
import argparse
import os



class        lowerCAmelCase_	:
       UpperCAmelCase__       :      Tuple       =						"block_{0}.dat"
       def __init__(    self,    SCREAMING_SNAKE_CASE_ )  ->	int:
          UpperCamelCase					:						int					=			filename
          UpperCamelCase					:						Optional[Any]					=			[]
       def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )  ->	int:
          UpperCamelCase					:						Dict					=			self.BLOCK_FILENAME_FORMAT.format(SCREAMING_SNAKE_CASE_ )
          with open(SCREAMING_SNAKE_CASE_,    'w' ) as file:
             file.write(SCREAMING_SNAKE_CASE_ )
          self.block_filenames.append(SCREAMING_SNAKE_CASE_ )
       def       snake_case_ (    self )  ->	str:
          return self.block_filenames
       def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_=None )  ->	int:
          UpperCamelCase					:						Any					=			0
          with open(self.filename ) as file:
             while True:
                UpperCamelCase					:						Union[str, Any]					=			file.readlines(SCREAMING_SNAKE_CASE_ )

                if lines == []:
                   break

                if sort_key is None:
                   lines.sort()
                else:
                   lines.sort(key=SCREAMING_SNAKE_CASE_ )

                self.write_block(''.join(SCREAMING_SNAKE_CASE_ ),    SCREAMING_SNAKE_CASE_ )
                i += 1






       def       snake_case_ (    self )  ->	List[str]:
          map(os.remove,    self.block_filenames )






class        lowerCAmelCase_	:
       def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_ )  ->	Optional[int]:
          UpperCamelCase					:						Tuple					=			-1
          UpperCamelCase					:						int					=			None

          for i in range(len(SCREAMING_SNAKE_CASE_ ) ):
             if min_str is None or choices[i] < min_str:
                UpperCamelCase					:						Tuple					=			i

          return min_index






class        lowerCAmelCase_	:
       def __init__(    self,    SCREAMING_SNAKE_CASE_ )  ->	Tuple:
          UpperCamelCase					:						Union[str, Any]					=			files
          UpperCamelCase					:						Optional[int]					=			set()
          UpperCamelCase					:						Optional[Any]					=			len(SCREAMING_SNAKE_CASE_ )
          UpperCamelCase					:						str					=			{i: None for i in range(self.num_buffers )}
       def       snake_case_ (    self )  ->	Optional[int]:
          return {
              i: self.buffers[i] for i in range(self.num_buffers ) if i not in self.empty
          }
       def       snake_case_ (    self )  ->	Union[str, Any]:
          for i in range(self.num_buffers ):
             if self.buffers[i] is None and i not in self.empty:
                UpperCamelCase					:						List[str]					=			self.files[i].readline()

                if self.buffers[i] == "":
                   self.empty.add(SCREAMING_SNAKE_CASE_ )
                   self.files[i].close()

          if len(self.empty ) == self.num_buffers:
             return False

          return True






       def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_ )  ->	List[str]:
          UpperCamelCase					:						str					=			self.buffers[index]
          UpperCamelCase					:						Optional[Any]					=			None

          return value






class        lowerCAmelCase_	:
       def __init__(    self,    SCREAMING_SNAKE_CASE_ )  ->	Optional[Any]:
          UpperCamelCase					:						List[str]					=			merge_strategy
       def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )  ->	Any:
          UpperCamelCase					:						Any					=			FilesArray(self.get_file_handles(SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ ) )
          with open(SCREAMING_SNAKE_CASE_,    'w',    SCREAMING_SNAKE_CASE_ ) as outfile:
             while buffers.refresh():
                UpperCamelCase					:						Tuple					=			self.merge_strategy.select(buffers.get_dict() )
                outfile.write(buffers.unshift(SCREAMING_SNAKE_CASE_ ) )






       def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )  ->	Any:
          UpperCamelCase					:						List[Any]					=			{}

          for i in range(len(SCREAMING_SNAKE_CASE_ ) ):
             UpperCamelCase					:						List[str]					=			open(filenames[i],    'r',    SCREAMING_SNAKE_CASE_ )  # noqa: UP015

          return files






class        lowerCAmelCase_	:
       def __init__(    self,    SCREAMING_SNAKE_CASE_ )  ->	Dict:
          UpperCamelCase					:						Any					=			block_size
       def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_=None )  ->	int:
          UpperCamelCase					:						Optional[Any]					=			self.get_number_blocks(SCREAMING_SNAKE_CASE_,    self.block_size )
          UpperCamelCase					:						Any					=			FileSplitter(SCREAMING_SNAKE_CASE_ )
          splitter.split(self.block_size,    SCREAMING_SNAKE_CASE_ )

          UpperCamelCase					:						List[Any]					=			FileMerger(NWayMerge() )
          UpperCamelCase					:						Optional[int]					=			self.block_size / (num_blocks + 1)
          merger.merge(splitter.get_block_filenames(),    filename + '.out',    SCREAMING_SNAKE_CASE_ )

          splitter.cleanup()






       def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )  ->	List[Any]:
          return (os.stat(SCREAMING_SNAKE_CASE_ ).st_size / block_size) + 1






def 	UpperCamelCase			(     snake_case__ :					List[str]			)			->  str:
   if string[-1].lower() == "k":
      return int(string[:-1]			) * 1024
   elif string[-1].lower() == "m":
      return int(string[:-1]			) * 1024 * 1024
   elif string[-1].lower() == "g":
      return int(string[:-1]			) * 1024 * 1024 * 1024
   else:
      return int(snake_case__			)
def 	UpperCamelCase			(     )			->  str:
   UpperCamelCase					:						str					=			argparse.ArgumentParser()
   parser.add_argument(
       '-m' ,  '--mem' ,  help='amount of memory to use for sorting' ,  default='100M'			)
   parser.add_argument(
       'filename' ,  metavar='<filename>' ,  nargs=1 ,  help='name of file to sort'			)
   UpperCamelCase					:						int					=			parser.parse_args()

   UpperCamelCase					:						Optional[Any]					=			ExternalSort(parse_memory(args.mem			)			)
   sorter.sort(args.filename[0]			)


if __name__ == "__main__":
 main()








import numpy as np
import torch
import torch.nn as nn
from transformers import CLIPConfig, CLIPVisionModel, PreTrainedModel

from ...utils import logging


__UpperCAmelCase      = logging.get_logger(__name__)
def 	UpperCamelCase			(     snake_case__ :					List[str] ,  snake_case__ :					str			)			->  Dict:
   UpperCamelCase					:						int					=			nn.functional.normalize(snake_case__			)
   UpperCamelCase					:						Union[str, Any]					=			nn.functional.normalize(snake_case__			)
   return torch.mm(snake_case__ ,  normalized_text_embeds.t()			)



class        lowerCAmelCase_	(						a__							):
       UpperCAmelCase__       :      int       =						CLIPConfig

       UpperCAmelCase__       :      Union[str, Any]       =						["CLIPEncoderLayer"]
       def __init__(    self,    SCREAMING_SNAKE_CASE_ )  ->	List[str]:
          super().__init__(SCREAMING_SNAKE_CASE_ )

          UpperCamelCase					:						int					=			CLIPVisionModel(config.vision_config )
          UpperCamelCase					:						int					=			nn.Linear(config.vision_config.hidden_size,    config.projection_dim,    bias=SCREAMING_SNAKE_CASE_ )

          UpperCamelCase					:						List[Any]					=			nn.Parameter(torch.ones(17,    config.projection_dim ),    requires_grad=SCREAMING_SNAKE_CASE_ )
          UpperCamelCase					:						str					=			nn.Parameter(torch.ones(3,    config.projection_dim ),    requires_grad=SCREAMING_SNAKE_CASE_ )

          UpperCamelCase					:						Dict					=			nn.Parameter(torch.ones(17 ),    requires_grad=SCREAMING_SNAKE_CASE_ )
          UpperCamelCase					:						Optional[Any]					=			nn.Parameter(torch.ones(3 ),    requires_grad=SCREAMING_SNAKE_CASE_ )
       @torch.no_grad()
       def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )  ->	Union[str, Any]:
          UpperCamelCase					:						Optional[int]					=			self.vision_model(SCREAMING_SNAKE_CASE_ )[1]  # pooled_output
          UpperCamelCase					:						int					=			self.visual_projection(SCREAMING_SNAKE_CASE_ )

          # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
          UpperCamelCase					:						Optional[Any]					=			cosine_distance(SCREAMING_SNAKE_CASE_,    self.special_care_embeds ).cpu().float().numpy()
          UpperCamelCase					:						Any					=			cosine_distance(SCREAMING_SNAKE_CASE_,    self.concept_embeds ).cpu().float().numpy()

          UpperCamelCase					:						str					=			[]
          UpperCamelCase					:						int					=			image_embeds.shape[0]
          for i in range(SCREAMING_SNAKE_CASE_ ):
             UpperCamelCase					:						Optional[int]					=			{'special_scores': {}, 'special_care': [], 'concept_scores': {}, 'bad_concepts': []}

             # increase this value to create a stronger `nfsw` filter
             # at the cost of increasing the possibility of filtering benign images
             UpperCamelCase					:						Optional[Any]					=			0.0

             for concept_idx in range(len(special_cos_dist[0] ) ):
                UpperCamelCase					:						Union[str, Any]					=			special_cos_dist[i][concept_idx]
                UpperCamelCase					:						Optional[Any]					=			self.special_care_embeds_weights[concept_idx].item()
                UpperCamelCase					:						str					=			round(concept_cos - concept_threshold + adjustment,    3 )
                if result_img["special_scores"][concept_idx] > 0:
                   result_img["special_care"].append({concept_idx, result_img['special_scores'][concept_idx]} )
                   UpperCamelCase					:						Dict					=			0.01

             for concept_idx in range(len(cos_dist[0] ) ):
                UpperCamelCase					:						List[Any]					=			cos_dist[i][concept_idx]
                UpperCamelCase					:						Optional[int]					=			self.concept_embeds_weights[concept_idx].item()
                UpperCamelCase					:						Union[str, Any]					=			round(concept_cos - concept_threshold + adjustment,    3 )
                if result_img["concept_scores"][concept_idx] > 0:
                   result_img["bad_concepts"].append(SCREAMING_SNAKE_CASE_ )

             result.append(SCREAMING_SNAKE_CASE_ )

          UpperCamelCase					:						int					=			[len(res['bad_concepts'] ) > 0 for res in result]

          for idx, has_nsfw_concept in enumerate(SCREAMING_SNAKE_CASE_ ):
             if has_nsfw_concept:
                if torch.is_tensor(SCREAMING_SNAKE_CASE_ ) or torch.is_tensor(images[0] ):
                   UpperCamelCase					:						List[str]					=			torch.zeros_like(images[idx] )  # black image
                else:
                   UpperCamelCase					:						int					=			np.zeros(images[idx].shape )  # black image

          if any(SCREAMING_SNAKE_CASE_ ):
             logger.warning(
                 'Potential NSFW content was detected in one or more images. A black image will be returned instead.'
                 ' Try again with a different prompt and/or seed.' )

          return images, has_nsfw_concepts






       @torch.no_grad()
       def       snake_case_ (    self,    SCREAMING_SNAKE_CASE_,    SCREAMING_SNAKE_CASE_ )  ->	Optional[int]:
          UpperCamelCase					:						Union[str, Any]					=			self.vision_model(SCREAMING_SNAKE_CASE_ )[1]  # pooled_output
          UpperCamelCase					:						str					=			self.visual_projection(SCREAMING_SNAKE_CASE_ )

          UpperCamelCase					:						str					=			cosine_distance(SCREAMING_SNAKE_CASE_,    self.special_care_embeds )
          UpperCamelCase					:						Any					=			cosine_distance(SCREAMING_SNAKE_CASE_,    self.concept_embeds )

          # increase this value to create a stronger `nsfw` filter
          # at the cost of increasing the possibility of filtering benign images
          UpperCamelCase					:						Dict					=			0.0

          UpperCamelCase					:						str					=			special_cos_dist - self.special_care_embeds_weights + adjustment
          # special_scores = special_scores.round(decimals=3)
          UpperCamelCase					:						List[str]					=			torch.any(special_scores > 0,    dim=1 )
          UpperCamelCase					:						List[Any]					=			special_care * 0.01
          UpperCamelCase					:						List[Any]					=			special_adjustment.unsqueeze(1 ).expand(-1,    cos_dist.shape[1] )

          UpperCamelCase					:						List[str]					=			(cos_dist - self.concept_embeds_weights) + special_adjustment
          # concept_scores = concept_scores.round(decimals=3)
          UpperCamelCase					:						Tuple					=			torch.any(concept_scores > 0,    dim=1 )

          UpperCamelCase					:						Any					=			0.0  # black image

          return images, has_nsfw_concepts



