




'''simple docstring'''






import argparse
import logging
import os
import sys
import time

import tensorflow as tf
from datasets import load_dataset
from tqdm import tqdm

from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
from transformers.utils import is_sagemaker_dp_enabled


if os.environ.get('''SDP_ENABLED''') or is_sagemaker_dp_enabled():
      lowercase	:      Optional[Any]				=       True
      lowercase	:      Any				=       '''p3dn.24xlarge'''
      import smdistributed.dataparallel.tensorflow as sdp
else:
      lowercase	:      Tuple				=       False




def 		a_( lowerCamelCase   ,				lowerCamelCase   ,				lowerCamelCase   ,				lowerCamelCase   ,				lowerCamelCase   ,				lowerCamelCase   ,				lowerCamelCase=None      )				->Tuple:






       '''simple docstring'''




       lowercase  :		Optional[Any]						=      tqdm(lowerCamelCase      )
       for i, batch in enumerate(lowerCamelCase      ):
              with tf.GradientTape() as tape:
                     lowercase,     lowercase  :		Dict						=      batch
                     lowercase  :		Tuple						=      model(lowerCamelCase      )
                     lowercase  :		Optional[int]						=      loss(lowerCamelCase   ,				outputs.logits      )

              if SDP_ENABLED:
                     lowercase  :		Any						=      sdp.DistributedGradientTape(lowerCamelCase   ,				sparse_as_dense=lowerCamelCase      )

              lowercase  :		int						=      tape.gradient(lowerCamelCase   ,				model.trainable_variables      )
              opt.apply_gradients(zip(lowerCamelCase   ,				model.trainable_variables      )      )

              pbar.set_description(f"""Loss: {loss_value:.4f}"""      )

              if SDP_ENABLED and i == 0:
                     sdp.broadcast_variables(model.variables   ,				root_rank=0      )
                     sdp.broadcast_variables(opt.variables()   ,				root_rank=0      )

              if max_steps and i >= max_steps:
                     break

       lowercase  :		List[Any]						=      {'''loss''': loss_value.numpy()}
       return train_results




def 		a_( lowerCamelCase   ,				lowerCamelCase   ,				lowerCamelCase      )				->int:






       '''simple docstring'''




       lowercase,     lowercase  :		str						=      load_dataset('''imdb'''   ,				split=['''train''', '''test''']      )

       # Preprocess train dataset
       lowercase  :		int						=      train_dataset.map(
           lambda lowerCamelCase      : tokenizer(e['''text''']   ,				truncation=lowerCamelCase   ,				padding='''max_length'''      )   ,				batched=lowerCamelCase      )
       train_dataset.set_format(type='''tensorflow'''   ,				columns=['''input_ids''', '''attention_mask''', '''label''']      )

       lowercase  :		Any						=      {
           x: train_dataset[x].to_tensor(default_value=0   ,				shape=[None, tokenizer.model_max_length]      )
           for x in ['''input_ids''', '''attention_mask''']
       }
       lowercase  :		Any						=      tf.data.Dataset.from_tensor_slices((train_features, train_dataset['''label'''])      )

       # Preprocess test dataset
       lowercase  :		str						=      test_dataset.map(
           lambda lowerCamelCase      : tokenizer(e['''text''']   ,				truncation=lowerCamelCase   ,				padding='''max_length'''      )   ,				batched=lowerCamelCase      )
       test_dataset.set_format(type='''tensorflow'''   ,				columns=['''input_ids''', '''attention_mask''', '''label''']      )

       lowercase  :		Dict						=      {
           x: test_dataset[x].to_tensor(default_value=0   ,				shape=[None, tokenizer.model_max_length]      )
           for x in ['''input_ids''', '''attention_mask''']
       }
       lowercase  :		Tuple						=      tf.data.Dataset.from_tensor_slices((test_features, test_dataset['''label'''])      )

       if SDP_ENABLED:
              lowercase  :		Union[str, Any]						=      tf_train_dataset.shard(sdp.size()   ,				sdp.rank()      )
              lowercase  :		int						=      tf_test_dataset.shard(sdp.size()   ,				sdp.rank()      )
       lowercase  :		List[str]						=      tf_train_dataset.batch(lowerCamelCase   ,				drop_remainder=lowerCamelCase      )
       lowercase  :		Any						=      tf_test_dataset.batch(lowerCamelCase   ,				drop_remainder=lowerCamelCase      )

       return tf_train_dataset, tf_test_dataset


if __name__ == "__main__":
      lowercase	:      Union[str, Any]				=       argparse.ArgumentParser()

      # Hyperparameters sent by the client are passed as command-line arguments to the script.
      parser.add_argument('''--epochs''', type=int, default=3)
      parser.add_argument('''--per_device_train_batch_size''', type=int, default=16)
      parser.add_argument('''--per_device_eval_batch_size''', type=int, default=8)
      parser.add_argument('''--model_name_or_path''', type=str)
      parser.add_argument('''--learning_rate''', type=str, default=5e-5)
      parser.add_argument('''--do_train''', type=bool, default=True)
      parser.add_argument('''--do_eval''', type=bool, default=True)
      parser.add_argument('''--output_dir''', type=str)
      parser.add_argument('''--max_steps''', type=int, default=None)

      # Data, model, and output directories
      parser.add_argument('''--output_data_dir''', type=str, default=os.environ['''SM_OUTPUT_DATA_DIR'''])
      parser.add_argument('''--model_dir''', type=str, default=os.environ['''SM_MODEL_DIR'''])
      parser.add_argument('''--n_gpus''', type=str, default=os.environ['''SM_NUM_GPUS'''])

      lowercase	,	lowercase	:      Union[str, Any]				=       parser.parse_known_args()

      # Set up logging
      lowercase	:      List[str]				=       logging.getLogger(__name__)

      logging.basicConfig(
          level=logging.getLevelName('''INFO'''),
          handlers=[logging.StreamHandler(sys.stdout)],
          format='''%(asctime)s - %(name)s - %(levelname)s - %(message)s''',
      )

      if SDP_ENABLED:
            sdp.init()

            lowercase	:      Dict				=       tf.config.experimental.list_physical_devices('''GPU''')
            for gpu in gpus:
                  tf.config.experimental.set_memory_growth(gpu, True)
            if gpus:
                  tf.config.experimental.set_visible_devices(gpus[sdp.local_rank()], '''GPU''')

    # Load model and tokenizer
      lowercase	:      Union[str, Any]				=       TFAutoModelForSequenceClassification.from_pretrained(args.model_name_or_path)
      lowercase	:      int				=       AutoTokenizer.from_pretrained(args.model_name_or_path)

      # get datasets
      lowercase	,	lowercase	:      int				=       get_datasets(
          tokenizer=tokenizer,
          train_batch_size=args.per_device_train_batch_size,
          eval_batch_size=args.per_device_eval_batch_size,
      )

      # fine optimizer and loss
      lowercase	:      str				=       tf.keras.optimizers.Adam(learning_rate=args.learning_rate)
      lowercase	:      Union[str, Any]				=       tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
      lowercase	:      int				=       [tf.keras.metrics.SparseCategoricalAccuracy()]
      model.compile(optimizer=optimizer, loss=loss, metrics=metrics)

      # Training
      if args.do_train:
            # train_results = model.fit(tf_train_dataset, epochs=args.epochs, batch_size=args.train_batch_size)
            lowercase	:      List[Any]				=       time.time()
            lowercase	:      Optional[Any]				=       fit(
                model,
                loss,
                optimizer,
                tf_train_dataset,
                args.epochs,
                args.per_device_train_batch_size,
                max_steps=args.max_steps,
            )
            lowercase	:      List[str]				=       time.time() - start_train_time
            logger.info('''*** Train ***''')
            logger.info(f'train_runtime = {end_train_time}')

            lowercase	:      Optional[int]				=       os.path.join(args.output_dir, '''train_results.txt''')

            if not SDP_ENABLED or sdp.rank() == 0:
                  with open(output_eval_file, '''w''') as writer:
                        logger.info('''***** Train results *****''')
                        logger.info(train_results)
                        for key, value in train_results.items():
                              logger.info(f'  {key} = {value}')
                              writer.write(f'{key} = {value}\n')

    # Evaluation
      if args.do_eval and (not SDP_ENABLED or sdp.rank() == 0):
            lowercase	:      Optional[Any]				=       model.evaluate(tf_test_dataset, batch_size=args.per_device_eval_batch_size, return_dict=True)
            logger.info('''*** Evaluate ***''')

            lowercase	:      List[str]				=       os.path.join(args.output_dir, '''eval_results.txt''')

            with open(output_eval_file, '''w''') as writer:
                  logger.info('''***** Eval results *****''')
                  logger.info(result)
                  for key, value in result.items():
                        logger.info(f'  {key} = {value}')
                        writer.write(f'{key} = {value}\n')

    # Save result
      if SDP_ENABLED:
            if sdp.rank() == 0:
                  model.save_pretrained(args.output_dir)
                  tokenizer.save_pretrained(args.output_dir)
      else:
            model.save_pretrained(args.output_dir)
            tokenizer.save_pretrained(args.output_dir)










'''simple docstring'''






import inspect
import unittest

from transformers import DataaVecVisionConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch
      from torch import nn

      from transformers import (
          MODEL_MAPPING,
          DataaVecVisionForImageClassification,
          DataaVecVisionForSemanticSegmentation,
          DataaVecVisionModel,
      )
      from transformers.models.dataavec.modeling_dataavec_vision import DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
      from PIL import Image

      from transformers import BeitImageProcessor

class   SCREAMING_SNAKE_CASE_		:



 def __init__(  self   ,		A   ,		A=100   ,		A=13   ,		A=30   ,		A=2   ,		A=3   ,		A=True   ,		A=True   ,		A=32   ,		A=4   ,		A=4   ,		A=37   ,		A="gelu"   ,		A=0.1   ,		A=0.1   ,		A=10   ,		A=0.0_2   ,		A=3   ,		A=None   ,		A=[0, 1, 2, 3]   ,		):






        '''simple docstring'''




        lowercase  :		Optional[Any]						=      parent
        lowercase  :		str						=      100
        lowercase  :		Dict						=      batch_size
        lowercase  :		Tuple						=      image_size
        lowercase  :		int						=      patch_size
        lowercase  :		Optional[int]						=      num_channels
        lowercase  :		Any						=      is_training
        lowercase  :		Dict						=      use_labels
        lowercase  :		Any						=      hidden_size
        lowercase  :		List[Any]						=      num_hidden_layers
        lowercase  :		int						=      num_attention_heads
        lowercase  :		Dict						=      intermediate_size
        lowercase  :		Optional[int]						=      hidden_act
        lowercase  :		Union[str, Any]						=      hidden_dropout_prob
        lowercase  :		str						=      attention_probs_dropout_prob
        lowercase  :		List[str]						=      type_sequence_label_size
        lowercase  :		Dict						=      initializer_range
        lowercase  :		str						=      scope
        lowercase  :		Tuple						=      out_indices
        lowercase  :		Union[str, Any]						=      num_labels

        # in BeiT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
        lowercase  :		Union[str, Any]						=      (image_size // patch_size) ** 2
        lowercase  :		List[Any]						=      num_patches + 1



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		List[Any]						=      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size] )

        lowercase  :		Union[str, Any]						=      None
        lowercase  :		int						=      None
        if self.use_labels:
               lowercase  :		Dict						=      ids_tensor([self.batch_size]   ,		self.type_sequence_label_size )
               lowercase  :		Tuple						=      ids_tensor([self.batch_size, self.image_size, self.image_size]   ,		self.num_labels )

        lowercase  :		int						=      self.get_config()

        return config, pixel_values, labels, pixel_labels



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        return DataaVecVisionConfig(
            vocab_size=self.vocab_size   ,		image_size=self.image_size   ,		patch_size=self.patch_size   ,		num_channels=self.num_channels   ,		hidden_size=self.hidden_size   ,		num_hidden_layers=self.num_hidden_layers   ,		num_attention_heads=self.num_attention_heads   ,		intermediate_size=self.intermediate_size   ,		hidden_act=self.hidden_act   ,		hidden_dropout_prob=self.hidden_dropout_prob   ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,		is_decoder=A   ,		initializer_range=self.initializer_range   ,		out_indices=self.out_indices   ,		)



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		List[str]						=      DataaVecVisionModel(config=A )
        model.to(A )
        model.eval()
        lowercase  :		Tuple						=      model(A )
        # expected sequence length = num_patches + 1 (we add 1 for the [CLS] token)
        lowercase  :		Union[str, Any]						=      (self.image_size // self.patch_size) ** 2
        self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, num_patches + 1, self.hidden_size) )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		Any						=      self.type_sequence_label_size
        lowercase  :		Any						=      DataaVecVisionForImageClassification(A )
        model.to(A )
        model.eval()
        lowercase  :		Dict						=      model(A   ,		labels=A )
        self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.type_sequence_label_size) )



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A ):






        '''simple docstring'''




        lowercase  :		List[str]						=      self.num_labels
        lowercase  :		List[Any]						=      DataaVecVisionForSemanticSegmentation(A )
        model.to(A )
        model.eval()
        lowercase  :		int						=      model(A )
        self.parent.assertEqual(
            result.logits.shape   ,		(self.batch_size, self.num_labels, self.image_size * 2, self.image_size * 2) )
        lowercase  :		List[str]						=      model(A   ,		labels=A )
        self.parent.assertEqual(
            result.logits.shape   ,		(self.batch_size, self.num_labels, self.image_size * 2, self.image_size * 2) )





 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		List[str]						=      self.prepare_config_and_inputs()
        lowercase,     lowercase,     lowercase,     lowercase  :		Any						=      config_and_inputs
        lowercase  :		Union[str, Any]						=      {'''pixel_values''': pixel_values}
        return config, inputs_dict






@require_torch
class   SCREAMING_SNAKE_CASE_		(       lowerCamelCase_		,     lowerCamelCase_		,     unittest.TestCase				):
 __snake_case	:     Tuple			   =      (
     (DataaVecVisionModel, DataaVecVisionForImageClassification, DataaVecVisionForSemanticSegmentation)
     if is_torch_available()
     else ()
 )
 __snake_case	:     Optional[Any]			   =      (
     {
         "feature-extraction": DataaVecVisionModel,
         "image-classification": DataaVecVisionForImageClassification,
         "image-segmentation": DataaVecVisionForSemanticSegmentation,
     }
     if is_torch_available()
     else {}
 )

 __snake_case	:     str			   =      False
 __snake_case	:     str			   =      False
 __snake_case	:     int			   =      False



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Dict						=      DataaVecVisionModelTester(self )
        lowercase  :		List[str]						=      ConfigTester(
            self   ,		config_class=A   ,		has_text_modality=A   ,		hidden_size=37 )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        self.config_tester.run_common_tests()



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        pass



 @require_torch_multi_gpu
 @unittest.skip(
     reason='''Data2VecVision has some layers using `add_module` which doesn\'t work well with `nn.DataParallel`''' )
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        pass



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		Tuple						=      self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
               lowercase  :		int						=      model_class(A )
               self.assertIsInstance(model.get_input_embeddings()   ,		(nn.Module) )
               lowercase  :		Optional[int]						=      model.get_output_embeddings()
               self.assertTrue(x is None or isinstance(A   ,		nn.Linear ) )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		Dict						=      self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
               lowercase  :		Tuple						=      model_class(A )
               lowercase  :		List[str]						=      inspect.signature(model.forward )
               # signature.parameters is an OrderedDict => so arg_names order is deterministic
               lowercase  :		Tuple						=      [*signature.parameters.keys()]

               lowercase  :		List[Any]						=      ['''pixel_values''']
               self.assertListEqual(arg_names[:1]   ,		A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[Any]						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Optional[int]						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_segmentation(*A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        if not self.model_tester.is_training:
               return

        lowercase,     lowercase  :		Tuple						=      self.model_tester.prepare_config_and_inputs_for_common()
        lowercase  :		Union[str, Any]						=      True

        for model_class in self.all_model_classes:
               if model_class in [*get_values(A )]:
                      continue

               lowercase  :		Optional[int]						=      model_class(A )
               model.to(A )
               model.train()
               lowercase  :		int						=      self._prepare_for_class(A   ,		A   ,		return_labels=A )
               lowercase  :		List[str]						=      model(**A ).loss
               loss.backward()



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		Tuple						=      self.model_tester.prepare_config_and_inputs_for_common()
        if not self.model_tester.is_training:
               return

        lowercase  :		List[str]						=      False
        lowercase  :		str						=      True

        for model_class in self.all_model_classes:
               if model_class in [*get_values(A )] or not model_class.supports_gradient_checkpointing:
                      continue
               # TODO: remove the following 3 lines once we have a MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING
               # this can then be incorporated into _prepare_for_class in test_modeling_common.py
               elif model_class.__name__ == "Data2VecVisionForSemanticSegmentation":
                      lowercase,     lowercase,     lowercase,     lowercase  :		Tuple						=      inputs_dict['''pixel_values'''].shape
                      lowercase  :		int						=      torch.zeros(
                          [self.model_tester.batch_size, height, width]   ,		device=A ).long()
               lowercase  :		Optional[Any]						=      model_class(A )
               model.gradient_checkpointing_enable()
               model.to(A )
               model.train()
               lowercase  :		Any						=      self._prepare_for_class(A   ,		A   ,		return_labels=A )
               lowercase  :		Optional[int]						=      model(**A ).loss
               loss.backward()



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase,     lowercase  :		List[Any]						=      self.model_tester.prepare_config_and_inputs_for_common()

        lowercase  :		Union[str, Any]						=      _config_zero_init(A )
        for model_class in self.all_model_classes:
               lowercase  :		Dict						=      model_class(config=A )
               for name, param in model.named_parameters():
                      # we skip lambda parameters as these require special initial values
                      # determined by config.layer_scale_init_value
                      if "lambda" in name:
                             continue
                      if param.requires_grad:
                             self.assertIn(
                                 ((param.data.mean() * 1E9).round() / 1E9).item()   ,		[0.0, 1.0]   ,		msg=f"""Parameter {name} of model {model_class} seems not properly initialized"""   ,		)



 def    SCREAMING_SNAKE_CASE(  self   ,		A   ,		A   ,		A   ,		A=2E-4   ,		A="outputs"   ,		A=None ):






        '''simple docstring'''




        super().check_pt_tf_outputs(A   ,		A   ,		A   ,		A   ,		A   ,		A )



 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Any						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*A )





 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        for model_name in DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               lowercase  :		List[Any]						=      DataaVecVisionModel.from_pretrained(A )
               self.assertIsNotNone(A )






def 		a_( )				->Any:






       '''simple docstring'''




       lowercase  :		Optional[Any]						=      Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''      )
       return image





@require_torch
@require_vision
class   SCREAMING_SNAKE_CASE_		(       unittest.TestCase				):



 @cached_property
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        return (
            BeitImageProcessor.from_pretrained('''facebook/data2vec-vision-base-ft1k''' ) if is_vision_available() else None
        )





 @slow
 def    SCREAMING_SNAKE_CASE(  self ):






        '''simple docstring'''




        lowercase  :		Union[str, Any]						=      DataaVecVisionForImageClassification.from_pretrained('''facebook/data2vec-vision-base-ft1k''' ).to(
            A )

        lowercase  :		List[str]						=      self.default_image_processor
        lowercase  :		List[Any]						=      prepare_img()
        lowercase  :		List[Any]						=      image_processor(images=A   ,		return_tensors='''pt''' ).to(A )

        # forward pass
        with torch.no_grad():
               lowercase  :		List[Any]						=      model(**A )
        lowercase  :		Any						=      outputs.logits

        # verify the logits
        lowercase  :		List[Any]						=      torch.Size((1, 1000) )
        self.assertEqual(logits.shape   ,		A )

        lowercase  :		str						=      torch.tensor([0.3_2_7_7, -0.1_3_9_5, 0.0_9_1_1] ).to(A )

        self.assertTrue(torch.allclose(logits[0, :3]   ,		A   ,		atol=1E-4 ) )

        lowercase  :		Dict						=      [model.config.labelaid[i] for i in ['''remote control, remote''', '''tabby, tabby cat''']]
        self.assertEqual(logits[0].topk(2 ).indices.cpu().tolist()   ,		A )




