






'''simple docstring'''





import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__snake_case							      =						16
__snake_case							      =						32


def 						_A     (       SCREAMING_SNAKE_CASE__						:			Optional[Any] ):
    return int(x / 2**20 )







class     UpperCAmelCase_		:



      """simple docstring"""
      def __enter__(       self				)       ->   Optional[Any]:
          gc.collect()
          torch.cuda.empty_cache()
          torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
          UpperCamelCase :Union[str, Any]										=     torch.cuda.memory_allocated()
          UpperCamelCase :int										=     psutil.Process()

          UpperCamelCase :Union[str, Any]										=     self.cpu_mem_used()
          UpperCamelCase :Tuple										=     True
          UpperCamelCase :Optional[int]										=     threading.Thread(target=self.peak_monitor_func				)
          UpperCamelCase :Union[str, Any]										=     True
          peak_monitor_thread.start()
          return self
      def 					UpperCAmelCase		(       self				)       ->   Optional[Any]:
          return self.process.memory_info().rss
      def 					UpperCAmelCase		(       self				)       ->   Any:
          UpperCamelCase :Dict										=     -1

          while True:
              UpperCamelCase :Union[str, Any]										=     max(self.cpu_mem_used()					,       self.cpu_peak				)

              # can't sleep or will not catch the peak right (this comment is here on purpose)
              # time.sleep(0.001) # 1msec

              if not self.peak_monitoring:
                  break




      def __exit__(       self					,       *SCREAMING_SNAKE_CASE_				)       ->   List[str]:
          UpperCamelCase :List[str]										=     False

          gc.collect()
          torch.cuda.empty_cache()
          UpperCamelCase :List[Any]										=     torch.cuda.memory_allocated()
          UpperCamelCase :Dict										=     torch.cuda.max_memory_allocated()
          UpperCamelCase :Union[str, Any]										=     bamb(self.end - self.begin				)
          UpperCamelCase :Any										=     bamb(self.peak - self.begin				)

          UpperCamelCase :List[Any]										=     self.cpu_mem_used()
          UpperCamelCase :str										=     bamb(self.cpu_end - self.cpu_begin				)
          UpperCamelCase :Optional[int]										=     bamb(self.cpu_peak - self.cpu_begin				)
          # print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
 from accelerate.test_utils.training import mocked_dataloaders

 __snake_case							      =						mocked_dataloaders  # noqa: F811


def 						_A     (       SCREAMING_SNAKE_CASE__						:			List[Any]  ,						SCREAMING_SNAKE_CASE__						:			Dict ):
    # For testing only
    if os.environ.get('''TESTING_MOCKED_DATALOADERS'''  ,						SCREAMING_SNAKE_CASE__ ) == "1":
        UpperCamelCase :Any										=     2

    # New Code #
    # Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
    UpperCamelCase :Dict										=     FullyShardedDataParallelPlugin(
        state_dict_config=FullStateDictConfig(offload_to_cpu=SCREAMING_SNAKE_CASE__  ,						ranka_only=SCREAMING_SNAKE_CASE__ )  ,						optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=SCREAMING_SNAKE_CASE__  ,						ranka_only=SCREAMING_SNAKE_CASE__ )  ,						)

    # Initialize accelerator
    if args.with_tracking:
        UpperCamelCase :Dict										=     Accelerator(
            cpu=args.cpu  ,						mixed_precision=args.mixed_precision  ,						log_with='''wandb'''  ,						project_dir=args.logging_dir  ,						fsdp_plugin=SCREAMING_SNAKE_CASE__  ,						)
    else:
        UpperCamelCase :int										=     Accelerator(fsdp_plugin=SCREAMING_SNAKE_CASE__ )
    accelerator.print(accelerator.distributed_type )

    if hasattr(args.checkpointing_steps  ,						'''isdigit''' ):
        if args.checkpointing_steps == "epoch":
            UpperCamelCase :Tuple										=     args.checkpointing_steps
        elif args.checkpointing_steps.isdigit():
            UpperCamelCase :Optional[Any]										=     int(args.checkpointing_steps )
        else:
            raise ValueError(
                F'''Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.''' )
    else:
        UpperCamelCase :int										=     None
    # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
    UpperCamelCase :Any										=     config['''lr''']
    UpperCamelCase :Tuple										=     int(config['''num_epochs'''] )
    UpperCamelCase :Union[str, Any]										=     int(config['''seed'''] )
    UpperCamelCase :Union[str, Any]										=     int(config['''batch_size'''] )

    # We need to initialize the trackers we use, and also store our configuration
    if args.with_tracking:
        UpperCamelCase :Tuple										=     vars(SCREAMING_SNAKE_CASE__ )
        accelerator.init_trackers('''fsdp_glue_no_trainer'''  ,						SCREAMING_SNAKE_CASE__ )

    UpperCamelCase :Optional[Any]										=     AutoTokenizer.from_pretrained(args.model_name_or_path )
    UpperCamelCase :List[str]										=     load_dataset('''glue'''  ,						'''mrpc''' )
    UpperCamelCase :List[str]										=     evaluate.load('''glue'''  ,						'''mrpc''' )

    def tokenize_function(SCREAMING_SNAKE_CASE__						:			List[Any] ):
        # max_length=None => use the model max length (it's actually the default)
        UpperCamelCase :int										=     tokenizer(examples['''sentence1''']  ,						examples['''sentence2''']  ,						truncation=SCREAMING_SNAKE_CASE__  ,						max_length=SCREAMING_SNAKE_CASE__ )
        return outputs

    # Apply the method we just defined to all the examples in all the splits of the dataset
    # starting with the main process first:
    with accelerator.main_process_first():
        UpperCamelCase :int										=     datasets.map(
            SCREAMING_SNAKE_CASE__  ,						batched=SCREAMING_SNAKE_CASE__  ,						remove_columns=['''idx''', '''sentence1''', '''sentence2''']  ,						)

    # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
    # transformers library
    UpperCamelCase :str										=     tokenized_datasets.rename_column('''label'''  ,						'''labels''' )

    # If the batch size is too big we use gradient accumulation
    UpperCamelCase :Union[str, Any]										=     1
    if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
        UpperCamelCase :Union[str, Any]										=     batch_size // MAX_GPU_BATCH_SIZE
        UpperCamelCase :Dict										=     MAX_GPU_BATCH_SIZE

    def collate_fn(SCREAMING_SNAKE_CASE__						:			List[Any] ):
        # On TPU it's best to pad everything to the same length or training will be very slow.
        UpperCamelCase :Union[str, Any]										=     128 if accelerator.distributed_type == DistributedType.TPU else None
        # When using mixed precision we want round multiples of 8/16
        if accelerator.mixed_precision == "fp8":
            UpperCamelCase :List[Any]										=     16
        elif accelerator.mixed_precision != "no":
            UpperCamelCase :int										=     8
        else:
            UpperCamelCase :Dict										=     None

        return tokenizer.pad(
            SCREAMING_SNAKE_CASE__  ,						padding='''longest'''  ,						max_length=SCREAMING_SNAKE_CASE__  ,						pad_to_multiple_of=SCREAMING_SNAKE_CASE__  ,						return_tensors='''pt'''  ,						)

    # Instantiate dataloaders.
    UpperCamelCase :Optional[Any]										=     DataLoader(
        tokenized_datasets['''train''']  ,						shuffle=SCREAMING_SNAKE_CASE__  ,						collate_fn=SCREAMING_SNAKE_CASE__  ,						batch_size=SCREAMING_SNAKE_CASE__ )
    UpperCamelCase :int										=     DataLoader(
        tokenized_datasets['''validation''']  ,						shuffle=SCREAMING_SNAKE_CASE__  ,						collate_fn=SCREAMING_SNAKE_CASE__  ,						batch_size=SCREAMING_SNAKE_CASE__ )

    set_seed(SCREAMING_SNAKE_CASE__ )

    # Instantiate the model (we build the model here so that the seed also control new weights initialization)
    UpperCamelCase :List[Any]										=     AutoModelForSequenceClassification.from_pretrained(
        args.model_name_or_path  ,						return_dict=SCREAMING_SNAKE_CASE__  ,						low_cpu_mem_usage=SCREAMING_SNAKE_CASE__ )

    # New Code #
    # For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
    UpperCamelCase :int										=     accelerator.prepare(SCREAMING_SNAKE_CASE__ )
    accelerator.print(SCREAMING_SNAKE_CASE__ )

    # Instantiate optimizer
    # New Code #
    # For FSDP feature, at present it doesn't support multiple parameter groups,
    # so we need to create a single parameter group for the whole model
    UpperCamelCase :str										=     torch.optim.AdamW(params=model.parameters()  ,						lr=SCREAMING_SNAKE_CASE__  ,						weight_decay=2e-4 )

    # Instantiate scheduler
    UpperCamelCase :Optional[Any]										=     get_linear_schedule_with_warmup(
        optimizer=SCREAMING_SNAKE_CASE__  ,						num_warmup_steps=10  ,						num_training_steps=(len(SCREAMING_SNAKE_CASE__ ) * num_epochs) // gradient_accumulation_steps  ,						)

    # New Code #
    # For FSDP feature, prepare everything except the model as we have already prepared the model
    # before creating the optimizer
    # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
    # prepare method.
    UpperCamelCase :List[Any]										=     accelerator.prepare(
        SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ )

    UpperCamelCase :List[Any]										=     0

    # Potentially load in the weights and states from a previous save
    if args.resume_from_checkpoint:
        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
            accelerator.print(F'''Resumed from checkpoint: {args.resume_from_checkpoint}''' )
            accelerator.load_state(args.resume_from_checkpoint )
            UpperCamelCase :List[str]										=     os.path.basename(args.resume_from_checkpoint )
        else:
            # Get the most recent checkpoint
            UpperCamelCase :Optional[int]										=     [f.name for f in os.scandir(os.getcwd() ) if f.is_dir()]
            dirs.sort(key=os.path.getctime )
            UpperCamelCase :Dict										=     dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
        # Extract `epoch_{i}` or `step_{i}`
        UpperCamelCase :Dict										=     os.path.splitext(SCREAMING_SNAKE_CASE__ )[0]

        if "epoch" in training_difference:
            num_epochs -= int(training_difference.replace('''epoch_'''  ,						'''''' ) )
            UpperCamelCase :Dict										=     None
        else:
            UpperCamelCase :str										=     int(training_difference.replace('''step_'''  ,						'''''' ) )
            num_epochs -= resume_step // len(SCREAMING_SNAKE_CASE__ )
            # If resuming by step, we also need to know exactly how far into the DataLoader we went
            UpperCamelCase :List[str]										=     (num_epochs * len(SCREAMING_SNAKE_CASE__ )) - resume_step

    # Now we train the model
    for epoch in range(SCREAMING_SNAKE_CASE__ ):
        # New Code #
        # context manager to track the peak memory usage during the training epoch
        with TorchTracemalloc() as tracemalloc:
            model.train()
            if args.with_tracking:
                UpperCamelCase :List[str]										=     0
            for step, batch in enumerate(SCREAMING_SNAKE_CASE__ ):
                # We need to skip steps until we reach the resumed step
                if args.resume_from_checkpoint and epoch == 0:
                    if resume_step is not None and step < resume_step:
                        pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
                batch.to(accelerator.device )
                UpperCamelCase :Optional[Any]										=     model(**SCREAMING_SNAKE_CASE__ )
                UpperCamelCase :Dict										=     outputs.loss
                UpperCamelCase :Tuple										=     loss / gradient_accumulation_steps
                # We keep track of the loss at each epoch
                if args.with_tracking:
                    total_loss += loss.detach().float()
                accelerator.backward(SCREAMING_SNAKE_CASE__ )
                if step % gradient_accumulation_steps == 0:
                    optimizer.step()
                    lr_scheduler.step()
                    optimizer.zero_grad()
                    # accelerator.print(lr_scheduler.get_lr())

                overall_step += 1

                if isinstance(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ ):
                    UpperCamelCase :Any										=     F'''step_{overall_step}'''
                    if overall_step % checkpointing_steps == 0:
                        if args.output_dir is not None:
                            UpperCamelCase :Optional[Any]										=     os.path.join(args.output_dir  ,						SCREAMING_SNAKE_CASE__ )
                        accelerator.save_state(SCREAMING_SNAKE_CASE__ )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
        accelerator.print('''Memory before entering the train : {}'''.format(bamb(tracemalloc.begin ) ) )
        accelerator.print('''Memory consumed at the end of the train (end-begin): {}'''.format(tracemalloc.used ) )
        accelerator.print('''Peak Memory consumed during the train (max-begin): {}'''.format(tracemalloc.peaked ) )
        accelerator.print(
            '''Total Peak Memory consumed during the train (max): {}'''.format(
                tracemalloc.peaked + bamb(tracemalloc.begin ) ) )
        # Logging the peak memory usage of the GPU to the tracker
        if args.with_tracking:
            accelerator.log(
                {
                    '''train_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin ),
                }  ,						step=SCREAMING_SNAKE_CASE__  ,						)

        # New Code #
        # context manager to track the peak memory usage during the evaluation
        with TorchTracemalloc() as tracemalloc:
            model.eval()
            for step, batch in enumerate(SCREAMING_SNAKE_CASE__ ):
                # We could avoid this line since we set the accelerator with `device_placement=True`.
                batch.to(accelerator.device )
                with torch.no_grad():
                    UpperCamelCase :Union[str, Any]										=     model(**SCREAMING_SNAKE_CASE__ )
                UpperCamelCase :List[str]										=     outputs.logits.argmax(dim=-1 )
                UpperCamelCase :List[str]										=     accelerator.gather_for_metrics((predictions, batch['''labels''']) )
                metric.add_batch(
                    predictions=SCREAMING_SNAKE_CASE__  ,						references=SCREAMING_SNAKE_CASE__  ,						)

            UpperCamelCase :Union[str, Any]										=     metric.compute()
            # Use accelerator.print to print only on the main process.
            accelerator.print(F'''epoch {epoch}:'''  ,						SCREAMING_SNAKE_CASE__ )
            if args.with_tracking:
                accelerator.log(
                    {
                        '''accuracy''': eval_metric['''accuracy'''],
                        '''f1''': eval_metric['''f1'''],
                        '''train_loss''': total_loss.item() / len(SCREAMING_SNAKE_CASE__ ),
                    }  ,						step=SCREAMING_SNAKE_CASE__  ,						)

            if checkpointing_steps == "epoch":
                UpperCamelCase :Optional[int]										=     F'''epoch_{epoch}'''
                if args.output_dir is not None:
                    UpperCamelCase :Any										=     os.path.join(args.output_dir  ,						SCREAMING_SNAKE_CASE__ )
                accelerator.save_state(SCREAMING_SNAKE_CASE__ )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
        accelerator.print('''Memory before entering the eval : {}'''.format(bamb(tracemalloc.begin ) ) )
        accelerator.print('''Memory consumed at the end of the eval (end-begin): {}'''.format(tracemalloc.used ) )
        accelerator.print('''Peak Memory consumed during the eval (max-begin): {}'''.format(tracemalloc.peaked ) )
        accelerator.print(
            '''Total Peak Memory consumed during the eval (max): {}'''.format(tracemalloc.peaked + bamb(tracemalloc.begin ) ) )
        # Logging the peak memory usage of the GPU to the tracker
        if args.with_tracking:
            accelerator.log(
                {
                    '''eval_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin ),
                }  ,						step=SCREAMING_SNAKE_CASE__  ,						)

    if args.with_tracking:
        accelerator.end_training()


def 						_A     (       ):
    UpperCamelCase :str										=     argparse.ArgumentParser(description='''Simple example of training script.''' )
    parser.add_argument(
        '''--mixed_precision'''  ,						type=SCREAMING_SNAKE_CASE__  ,						default=SCREAMING_SNAKE_CASE__  ,						choices=['''no''', '''fp16''', '''bf16''', '''fp8''']  ,						help='''Whether to use mixed precision. Choose'''
        '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
        '''and an Nvidia Ampere GPU.'''  ,						)
    parser.add_argument('''--cpu'''  ,						action='''store_true'''  ,						help='''If passed, will train on the CPU.''' )
    parser.add_argument(
        '''--checkpointing_steps'''  ,						type=SCREAMING_SNAKE_CASE__  ,						default=SCREAMING_SNAKE_CASE__  ,						help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''  ,						)
    parser.add_argument(
        '''--resume_from_checkpoint'''  ,						type=SCREAMING_SNAKE_CASE__  ,						default=SCREAMING_SNAKE_CASE__  ,						help='''If the training should continue from a checkpoint folder.'''  ,						)
    parser.add_argument(
        '''--with_tracking'''  ,						action='''store_true'''  ,						help='''Whether to load in all available experiment trackers from the environment and use them for logging.'''  ,						)
    parser.add_argument(
        '''--output_dir'''  ,						type=SCREAMING_SNAKE_CASE__  ,						default='''.'''  ,						help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''  ,						)
    parser.add_argument(
        '''--logging_dir'''  ,						type=SCREAMING_SNAKE_CASE__  ,						default='''logs'''  ,						help='''Location on where to store experiment tracking logs`'''  ,						)
    parser.add_argument(
        '''--model_name_or_path'''  ,						type=SCREAMING_SNAKE_CASE__  ,						help='''Path to pretrained model or model identifier from huggingface.co/models.'''  ,						required=SCREAMING_SNAKE_CASE__  ,						)
    UpperCamelCase :str										=     parser.parse_args()
    UpperCamelCase :List[Any]										=     {'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
    training_function(SCREAMING_SNAKE_CASE__  ,						SCREAMING_SNAKE_CASE__ )


if __name__ == "__main__":
 main()













from __future__ import annotations

import inspect
import unittest

import numpy as np

from transformers import SwinConfig
from transformers.testing_utils import require_tf, require_vision, slow, to_atuple
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers.models.swin.modeling_tf_swin import (
     TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
     TFSwinForImageClassification,
     TFSwinForMaskedImageModeling,
     TFSwinModel,
 )


if is_vision_available():
 from PIL import Image

 from transformers import AutoImageProcessor







class     UpperCAmelCase_		:



      """simple docstring"""
      def __init__(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_=13					,       SCREAMING_SNAKE_CASE_=32					,       SCREAMING_SNAKE_CASE_=2					,       SCREAMING_SNAKE_CASE_=3					,       SCREAMING_SNAKE_CASE_=16					,       SCREAMING_SNAKE_CASE_=[1, 2, 1]					,       SCREAMING_SNAKE_CASE_=[2, 2, 4]					,       SCREAMING_SNAKE_CASE_=2					,       SCREAMING_SNAKE_CASE_=2.0					,       SCREAMING_SNAKE_CASE_=True					,       SCREAMING_SNAKE_CASE_=0.0					,       SCREAMING_SNAKE_CASE_=0.0					,       SCREAMING_SNAKE_CASE_=0.1					,       SCREAMING_SNAKE_CASE_="gelu"					,       SCREAMING_SNAKE_CASE_=False					,       SCREAMING_SNAKE_CASE_=True					,       SCREAMING_SNAKE_CASE_=0.02					,       SCREAMING_SNAKE_CASE_=1e-5					,       SCREAMING_SNAKE_CASE_=True					,       SCREAMING_SNAKE_CASE_=None					,       SCREAMING_SNAKE_CASE_=True					,       SCREAMING_SNAKE_CASE_=10					,       SCREAMING_SNAKE_CASE_=8					,       )       ->   None:
          UpperCamelCase :Optional[Any]										=     parent
          UpperCamelCase :Optional[Any]										=     batch_size
          UpperCamelCase :List[Any]										=     image_size
          UpperCamelCase :List[Any]										=     patch_size
          UpperCamelCase :List[Any]										=     num_channels
          UpperCamelCase :Optional[Any]										=     embed_dim
          UpperCamelCase :List[Any]										=     depths
          UpperCamelCase :List[Any]										=     num_heads
          UpperCamelCase :Tuple										=     window_size
          UpperCamelCase :Optional[int]										=     mlp_ratio
          UpperCamelCase :int										=     qkv_bias
          UpperCamelCase :str										=     hidden_dropout_prob
          UpperCamelCase :Dict										=     attention_probs_dropout_prob
          UpperCamelCase :Dict										=     drop_path_rate
          UpperCamelCase :Optional[Any]										=     hidden_act
          UpperCamelCase :int										=     use_absolute_embeddings
          UpperCamelCase :Any										=     patch_norm
          UpperCamelCase :str										=     layer_norm_eps
          UpperCamelCase :List[Any]										=     initializer_range
          UpperCamelCase :str										=     is_training
          UpperCamelCase :Union[str, Any]										=     scope
          UpperCamelCase :List[str]										=     use_labels
          UpperCamelCase :Union[str, Any]										=     type_sequence_label_size
          UpperCamelCase :int										=     encoder_stride
      def 					UpperCAmelCase		(       self				)       ->   Optional[int]:
          UpperCamelCase :Dict										=     floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)

          UpperCamelCase :List[Any]										=     None
          if self.use_labels:
              UpperCamelCase :Tuple										=     ids_tensor([self.batch_size]					,       self.type_sequence_label_size				)

          UpperCamelCase :List[str]										=     self.get_config()

          return config, pixel_values, labels
      def 					UpperCAmelCase		(       self				)       ->   int:
          return SwinConfig(
              image_size=self.image_size					,       patch_size=self.patch_size					,       num_channels=self.num_channels					,       embed_dim=self.embed_dim					,       depths=self.depths					,       num_heads=self.num_heads					,       window_size=self.window_size					,       mlp_ratio=self.mlp_ratio					,       qkv_bias=self.qkv_bias					,       hidden_dropout_prob=self.hidden_dropout_prob					,       attention_probs_dropout_prob=self.attention_probs_dropout_prob					,       drop_path_rate=self.drop_path_rate					,       hidden_act=self.hidden_act					,       use_absolute_embeddings=self.use_absolute_embeddings					,       path_norm=self.patch_norm					,       layer_norm_eps=self.layer_norm_eps					,       initializer_range=self.initializer_range					,       encoder_stride=self.encoder_stride					,       )
      def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   List[Any]:
          UpperCamelCase :int										=     TFSwinModel(config=SCREAMING_SNAKE_CASE_				)
          UpperCamelCase :Tuple										=     model(SCREAMING_SNAKE_CASE_				)

          UpperCamelCase :str										=     ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths				) - 1))
          UpperCamelCase :Union[str, Any]										=     int(config.embed_dim * 2 ** (len(config.depths				) - 1)				)

          self.parent.assertEqual(result.last_hidden_state.shape					,       (self.batch_size, expected_seq_len, expected_dim)				)
      def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   Any:
          UpperCamelCase :int										=     TFSwinForMaskedImageModeling(config=SCREAMING_SNAKE_CASE_				)
          UpperCamelCase :List[str]										=     model(SCREAMING_SNAKE_CASE_				)
          self.parent.assertEqual(
              result.logits.shape					,       (self.batch_size, self.num_channels, self.image_size, self.image_size)				)

          # test greyscale images
          UpperCamelCase :Optional[int]										=     1
          UpperCamelCase :int										=     TFSwinForMaskedImageModeling(SCREAMING_SNAKE_CASE_				)

          UpperCamelCase :str										=     floats_tensor([self.batch_size, 1, self.image_size, self.image_size]				)
          UpperCamelCase :str										=     model(SCREAMING_SNAKE_CASE_				)
          self.parent.assertEqual(result.logits.shape					,       (self.batch_size, 1, self.image_size, self.image_size)				)
      def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   List[str]:
          UpperCamelCase :Optional[Any]										=     self.type_sequence_label_size
          UpperCamelCase :Optional[int]										=     TFSwinForImageClassification(SCREAMING_SNAKE_CASE_				)
          UpperCamelCase :Tuple										=     model(SCREAMING_SNAKE_CASE_					,       labels=SCREAMING_SNAKE_CASE_				)
          self.parent.assertEqual(result.logits.shape					,       (self.batch_size, self.type_sequence_label_size)				)

          # test greyscale images
          UpperCamelCase :Union[str, Any]										=     1
          UpperCamelCase :Tuple										=     TFSwinForImageClassification(SCREAMING_SNAKE_CASE_				)
          UpperCamelCase :Any										=     floats_tensor([self.batch_size, 1, self.image_size, self.image_size]				)
          UpperCamelCase :Tuple										=     model(SCREAMING_SNAKE_CASE_				)
          self.parent.assertEqual(result.logits.shape					,       (self.batch_size, self.type_sequence_label_size)				)




      def 					UpperCAmelCase		(       self				)       ->   int:
          UpperCamelCase :List[str]										=     self.prepare_config_and_inputs()
          UpperCamelCase							,     UpperCamelCase							,     UpperCamelCase :Optional[Any]										=     config_and_inputs
          UpperCamelCase :Any										=     {'''pixel_values''': pixel_values}
          return config, inputs_dict







@require_tf
class     UpperCAmelCase_		(      lowercase,			lowercase,			unittest.TestCase				):



      """simple docstring"""
      UpperCamelCase_       :      int     =(
          (
              TFSwinModel,
              TFSwinForImageClassification,
              TFSwinForMaskedImageModeling,
          )
          if is_tf_available()
          else ()
      )
      UpperCamelCase_       :      Any     =(
          {'feature-extraction': TFSwinModel, 'image-classification': TFSwinForImageClassification}
          if is_tf_available()
          else {}
      )

      UpperCamelCase_       :      Optional[int]     =False
      UpperCamelCase_       :      int     =False
      UpperCamelCase_       :      Optional[int]     =False
      UpperCamelCase_       :      Any     =False
      def 					UpperCAmelCase		(       self				)       ->   Tuple:
          UpperCamelCase :str										=     TFSwinModelTester(self				)
          UpperCamelCase :str										=     ConfigTester(self					,       config_class=SCREAMING_SNAKE_CASE_					,       embed_dim=37				)
      def 					UpperCAmelCase		(       self				)       ->   Dict:
          self.create_and_test_config_common_properties()
          self.config_tester.create_and_test_config_to_json_string()
          self.config_tester.create_and_test_config_to_json_file()
          self.config_tester.create_and_test_config_from_and_save_pretrained()
          self.config_tester.create_and_test_config_with_num_labels()
          self.config_tester.check_config_can_be_init_without_params()
          self.config_tester.check_config_arguments_init()
      def 					UpperCAmelCase		(       self				)       ->   Union[str, Any]:
          return
      def 					UpperCAmelCase		(       self				)       ->   Optional[Any]:
          UpperCamelCase :Any										=     self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_				)
      def 					UpperCAmelCase		(       self				)       ->   Optional[int]:
          UpperCamelCase :Optional[int]										=     self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_image_modeling(*SCREAMING_SNAKE_CASE_				)
      def 					UpperCAmelCase		(       self				)       ->   Optional[Any]:
          UpperCamelCase :Any										=     self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_image_classification(*SCREAMING_SNAKE_CASE_				)
      @unittest.skip(reason='''Swin does not use inputs_embeds'''				)
      def 					UpperCAmelCase		(       self				)       ->   Union[str, Any]:
          pass
      def 					UpperCAmelCase		(       self				)       ->   Optional[int]:
          UpperCamelCase							,     UpperCamelCase :int										=     self.model_tester.prepare_config_and_inputs_for_common()

          for model_class in self.all_model_classes:
              UpperCamelCase :Optional[int]										=     model_class(SCREAMING_SNAKE_CASE_				)
              self.assertIsInstance(model.get_input_embeddings()					,       tf.keras.layers.Layer				)
              UpperCamelCase :Union[str, Any]										=     model.get_output_embeddings()
              self.assertTrue(x is None or isinstance(SCREAMING_SNAKE_CASE_					,       tf.keras.layers.Dense				)				)
      def 					UpperCAmelCase		(       self				)       ->   str:
          UpperCamelCase							,     UpperCamelCase :str										=     self.model_tester.prepare_config_and_inputs_for_common()

          for model_class in self.all_model_classes:
              UpperCamelCase :Union[str, Any]										=     model_class(SCREAMING_SNAKE_CASE_				)
              UpperCamelCase :Union[str, Any]										=     inspect.signature(model.call				)
              # signature.parameters is an OrderedDict => so arg_names order is deterministic
              UpperCamelCase :Dict										=     [*signature.parameters.keys()]

              UpperCamelCase :Tuple										=     ['''pixel_values''']
              self.assertListEqual(arg_names[:1]					,       SCREAMING_SNAKE_CASE_				)
      def 					UpperCAmelCase		(       self				)       ->   List[Any]:
          UpperCamelCase							,     UpperCamelCase :Optional[int]										=     self.model_tester.prepare_config_and_inputs_for_common()
          UpperCamelCase :str										=     True

          for model_class in self.all_model_classes:
              UpperCamelCase :List[Any]										=     True
              UpperCamelCase :Optional[Any]										=     False
              UpperCamelCase :List[str]										=     True
              UpperCamelCase :Optional[int]										=     model_class(SCREAMING_SNAKE_CASE_				)
              UpperCamelCase :Union[str, Any]										=     model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)				)
              UpperCamelCase :List[Any]										=     outputs.attentions
              UpperCamelCase :int										=     len(self.model_tester.depths				)
              self.assertEqual(len(SCREAMING_SNAKE_CASE_				)					,       SCREAMING_SNAKE_CASE_				)

              # check that output_attentions also work using config
              del inputs_dict["output_attentions"]
              UpperCamelCase :Optional[Any]										=     True
              UpperCamelCase :List[Any]										=     config.window_size**2
              UpperCamelCase :Dict										=     model_class(SCREAMING_SNAKE_CASE_				)
              UpperCamelCase :Dict										=     model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)				)
              UpperCamelCase :List[Any]										=     outputs.attentions
              self.assertEqual(len(SCREAMING_SNAKE_CASE_				)					,       SCREAMING_SNAKE_CASE_				)

              self.assertListEqual(
                  list(attentions[0].shape[-3:]				)					,       [self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,       )
              UpperCamelCase :List[Any]										=     len(SCREAMING_SNAKE_CASE_				)

              # Check attention is always last and order is fine
              UpperCamelCase :int										=     True
              UpperCamelCase :List[str]										=     True
              UpperCamelCase :List[str]										=     model_class(SCREAMING_SNAKE_CASE_				)
              UpperCamelCase :Tuple										=     model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)				)

              if hasattr(self.model_tester					,       '''num_hidden_states_types'''				):
                  UpperCamelCase :Union[str, Any]										=     self.model_tester.num_hidden_states_types
              else:
                  # also another +1 for reshaped_hidden_states
                  UpperCamelCase :Optional[int]										=     2
              self.assertEqual(out_len + added_hidden_states					,       len(SCREAMING_SNAKE_CASE_				)				)

              UpperCamelCase :Optional[int]										=     outputs.attentions

              self.assertEqual(len(SCREAMING_SNAKE_CASE_				)					,       SCREAMING_SNAKE_CASE_				)

              self.assertListEqual(
                  list(self_attentions[0].shape[-3:]				)					,       [self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,       )
      def 					UpperCAmelCase		(       self					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)       ->   Dict:
          UpperCamelCase :Optional[int]										=     model_class(SCREAMING_SNAKE_CASE_				)
          UpperCamelCase :Any										=     model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)				)
          UpperCamelCase :Union[str, Any]										=     outputs.hidden_states

          UpperCamelCase :Any										=     getattr(
              self.model_tester					,       '''expected_num_hidden_layers'''					,       len(self.model_tester.depths				) + 1				)
          self.assertEqual(len(SCREAMING_SNAKE_CASE_				)					,       SCREAMING_SNAKE_CASE_				)

          # Swin has a different seq_length
          UpperCamelCase :str										=     to_atuple(config.patch_size				)

          UpperCamelCase :Optional[int]										=     (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

          self.assertListEqual(
              list(hidden_states[0].shape[-2:]				)					,       [num_patches, self.model_tester.embed_dim]					,       )

          UpperCamelCase :str										=     outputs.reshaped_hidden_states
          self.assertEqual(len(SCREAMING_SNAKE_CASE_				)					,       SCREAMING_SNAKE_CASE_				)

          UpperCamelCase							,     UpperCamelCase							,     UpperCamelCase							,     UpperCamelCase :Optional[int]										=     reshaped_hidden_states[0].shape

          UpperCamelCase :int										=     tf.reshape(reshaped_hidden_states[0]					,       (batch_size, num_channels, height * width)				)
          UpperCamelCase :Dict										=     tf.transpose(SCREAMING_SNAKE_CASE_					,       (0, 2, 1)				)

          self.assertListEqual(
              list(reshaped_hidden_states.shape[-2:]				)					,       [num_patches, self.model_tester.embed_dim]					,       )
      def 					UpperCAmelCase		(       self				)       ->   str:
          UpperCamelCase							,     UpperCamelCase :int										=     self.model_tester.prepare_config_and_inputs_for_common()

          UpperCamelCase :List[Any]										=     to_atuple(self.model_tester.image_size				)

          for model_class in self.all_model_classes:
              UpperCamelCase :List[Any]										=     True
              self.check_hidden_states_output(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)

              # check that output_hidden_states also work using config
              del inputs_dict["output_hidden_states"]
              UpperCamelCase :List[str]										=     True

              self.check_hidden_states_output(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_				)
      def 					UpperCAmelCase		(       self				)       ->   Optional[Any]:
          UpperCamelCase							,     UpperCamelCase :List[Any]										=     self.model_tester.prepare_config_and_inputs_for_common()
          UpperCamelCase :List[Any]										=     3

          UpperCamelCase :Any										=     to_atuple(self.model_tester.image_size				)
          UpperCamelCase :List[Any]										=     to_atuple(config.patch_size				)

          UpperCamelCase :Dict										=     image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
          UpperCamelCase :List[str]										=     image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

          for model_class in self.all_model_classes:
              UpperCamelCase :List[Any]										=     True
              self.check_hidden_states_output(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       (padded_height, padded_width)				)

              # check that output_hidden_states also work using config
              del inputs_dict["output_hidden_states"]
              UpperCamelCase :str										=     True
              self.check_hidden_states_output(SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       SCREAMING_SNAKE_CASE_					,       (padded_height, padded_width)				)




      @slow
      def 					UpperCAmelCase		(       self				)       ->   List[Any]:
          for model_name in TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              UpperCamelCase :Dict										=     TFSwinModel.from_pretrained(SCREAMING_SNAKE_CASE_				)
              self.assertIsNotNone(SCREAMING_SNAKE_CASE_				)







@require_vision
@require_tf
class     UpperCAmelCase_		(      unittest.TestCase				):



      """simple docstring"""
      @cached_property
      def 					UpperCAmelCase		(       self				)       ->   Union[str, Any]:
          return (
              AutoImageProcessor.from_pretrained('''microsoft/swin-tiny-patch4-window7-224'''				)
              if is_vision_available()
              else None
          )




      @slow
      def 					UpperCAmelCase		(       self				)       ->   Optional[int]:
          UpperCamelCase :Tuple										=     TFSwinForImageClassification.from_pretrained('''microsoft/swin-tiny-patch4-window7-224'''				)
          UpperCamelCase :int										=     self.default_image_processor

          UpperCamelCase :Optional[int]										=     Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''				)
          UpperCamelCase :List[str]										=     image_processor(images=SCREAMING_SNAKE_CASE_					,       return_tensors='''tf'''				)

          # forward pass
          UpperCamelCase :Optional[Any]										=     model(SCREAMING_SNAKE_CASE_				)

          # verify the logits
          UpperCamelCase :Tuple										=     tf.TensorShape((1, 1000)				)
          self.assertEqual(outputs.logits.shape					,       SCREAMING_SNAKE_CASE_				)
          UpperCamelCase :List[Any]										=     tf.constant([-0.0948, -0.6454, -0.0921]				)
          self.assertTrue(np.allclose(outputs.logits[0, :3]					,       SCREAMING_SNAKE_CASE_					,       atol=1e-4				)				)





