import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


lowerCAmelCase__      		=					logging.getLogger(__name__)
class 							a_					(		SCREAMING_SNAKE_CASE		):






 '''simple docstring'''






 def 						__snake_case		(     self   :    Tuple     ,       lowercase__   :    Dict     ,       lowercase__   :    List[str]     ,       lowercase__   :    Dict=None     ,       lowercase__   :    List[Any]=None):







        '''simple docstring'''



        if current_layer == 0:
               lowerCAmelCase__	      =   self.embedding_hidden_mapping_in(_lowerCamelCase)
        else:
               lowerCAmelCase__	      =   hidden_states[0]

        lowerCAmelCase__	      =   int(self.config.num_hidden_layers / self.config.num_hidden_groups)

        # Index of the hidden group
        lowerCAmelCase__	      =   int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups))

        lowerCAmelCase__	      =   self.albert_layer_groups[group_idx](
            _lowerCamelCase     ,       _lowerCamelCase     ,       head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]     ,       )
        lowerCAmelCase__	      =   layer_group_output[0]

        return (hidden_states,)






@add_start_docstrings(
    'The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.'			,	SCREAMING_SNAKE_CASE			,	)
class 							a_					(		SCREAMING_SNAKE_CASE		):






 '''simple docstring'''






 def __init__(     self   :    List[str]     ,       lowercase__   :    Optional[int]):







        '''simple docstring'''



        super().__init__(_lowerCamelCase)

        lowerCAmelCase__	      =   AlbertTransformerWithPabee(_lowerCamelCase)

        self.init_weights()
        lowerCAmelCase__	      =   0
        lowerCAmelCase__	      =   0
        lowerCAmelCase__	      =   0

        lowerCAmelCase__	      =   0




 def 						__snake_case		(     self   :    List[Any]     ,       lowercase__   :    Optional[Any]):







        '''simple docstring'''



        lowerCAmelCase__	      =   threshold




 def 						__snake_case		(     self   :    Optional[int]     ,       lowercase__   :    Optional[Any]):







        '''simple docstring'''



        lowerCAmelCase__	      =   patience




 def 						__snake_case		(     self   :    Union[str, Any]):







        '''simple docstring'''



        lowerCAmelCase__	      =   0
        lowerCAmelCase__	      =   0




 def 						__snake_case		(     self   :    Optional[int]):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.inference_layers_num / self.inference_instances_num
        lowerCAmelCase__	      =   (
            F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
            F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
        )
        print(_lowerCamelCase)




 @add_start_docstrings_to_model_forward(_lowerCamelCase)
 def 						__snake_case		(     self   :    Dict     ,       lowercase__   :    List[str]=None     ,       lowercase__   :    Optional[Any]=None     ,       lowercase__   :    List[Any]=None     ,       lowercase__   :    Union[str, Any]=None     ,       lowercase__   :    int=None     ,       lowercase__   :    Dict=None     ,       lowercase__   :    Optional[Any]=None     ,       lowercase__   :    Optional[int]=None     ,       lowercase__   :    Union[str, Any]=False     ,       ):







        '''simple docstring'''



        if input_ids is not None and inputs_embeds is not None:
               raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time')
        elif input_ids is not None:
               lowerCAmelCase__	      =   input_ids.size()
        elif inputs_embeds is not None:
               lowerCAmelCase__	      =   inputs_embeds.size()[:-1]
        else:
               raise ValueError('You have to specify either input_ids or inputs_embeds')

        lowerCAmelCase__	      =   input_ids.device if input_ids is not None else inputs_embeds.device

        if attention_mask is None:
               lowerCAmelCase__	      =   torch.ones(_lowerCamelCase     ,       device=_lowerCamelCase)
        if token_type_ids is None:
               lowerCAmelCase__	      =   torch.zeros(_lowerCamelCase     ,       dtype=torch.long     ,       device=_lowerCamelCase)

        lowerCAmelCase__	      =   attention_mask.unsqueeze(1).unsqueeze(2)
        lowerCAmelCase__	      =   extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
        lowerCAmelCase__	      =   (1.0 - extended_attention_mask) * -10_000.0
        lowerCAmelCase__	      =   self.get_head_mask(_lowerCamelCase     ,       self.config.num_hidden_layers)

        lowerCAmelCase__	      =   self.embeddings(
            _lowerCamelCase     ,       position_ids=_lowerCamelCase     ,       token_type_ids=_lowerCamelCase     ,       inputs_embeds=_lowerCamelCase)
        lowerCAmelCase__	      =   embedding_output

        if self.training:
               lowerCAmelCase__	      =   []
               for i in range(self.config.num_hidden_layers):
                      lowerCAmelCase__	      =   self.encoder.adaptive_forward(
                          _lowerCamelCase     ,       current_layer=_lowerCamelCase     ,       attention_mask=_lowerCamelCase     ,       head_mask=_lowerCamelCase     ,       )

                      lowerCAmelCase__	      =   self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]))
                      lowerCAmelCase__	      =   output_layers[i](output_dropout(_lowerCamelCase))
                      res.append(_lowerCamelCase)
        elif self.patience == 0:  # Use all layers for inference
               lowerCAmelCase__	      =   self.encoder(_lowerCamelCase     ,       _lowerCamelCase     ,       head_mask=_lowerCamelCase)
               lowerCAmelCase__	      =   self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]))
               lowerCAmelCase__	      =   [output_layers[self.config.num_hidden_layers - 1](_lowerCamelCase)]
        else:
               lowerCAmelCase__	      =   0
               lowerCAmelCase__	      =   None
               lowerCAmelCase__	      =   0
               for i in range(self.config.num_hidden_layers):
                      calculated_layer_num += 1
                      lowerCAmelCase__	      =   self.encoder.adaptive_forward(
                          _lowerCamelCase     ,       current_layer=_lowerCamelCase     ,       attention_mask=_lowerCamelCase     ,       head_mask=_lowerCamelCase     ,       )

                      lowerCAmelCase__	      =   self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]))
                      lowerCAmelCase__	      =   output_layers[i](_lowerCamelCase)
                      if regression:
                             lowerCAmelCase__	      =   logits.detach()
                             if patient_result is not None:
                                    lowerCAmelCase__	      =   patient_result.detach()
                             if (patient_result is not None) and torch.abs(patient_result - labels) < self.regression_threshold:
                                    patient_counter += 1
                             else:
                                    lowerCAmelCase__	      =   0
                      else:
                             lowerCAmelCase__	      =   logits.detach().argmax(dim=1)
                             if patient_result is not None:
                                    lowerCAmelCase__	      =   patient_result.detach().argmax(dim=1)
                             if (patient_result is not None) and torch.all(labels.eq(_lowerCamelCase)):
                                    patient_counter += 1
                             else:
                                    lowerCAmelCase__	      =   0

                      lowerCAmelCase__	      =   logits
                      if patient_counter == self.patience:
                             break
               lowerCAmelCase__	      =   [patient_result]
               self.inference_layers_num += calculated_layer_num
               self.inference_instances_num += 1

        return res






@add_start_docstrings(
    'Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '			,	SCREAMING_SNAKE_CASE			,	)
class 							a_					(		SCREAMING_SNAKE_CASE		):






 '''simple docstring'''






 def __init__(     self   :    Dict     ,       lowercase__   :    str):







        '''simple docstring'''



        super().__init__(_lowerCamelCase)
        lowerCAmelCase__	      =   config.num_labels

        lowerCAmelCase__	      =   AlbertModelWithPabee(_lowerCamelCase)
        lowerCAmelCase__	      =   nn.Dropout(config.classifier_dropout_prob)
        lowerCAmelCase__	      =   nn.ModuleList(
            [nn.Linear(config.hidden_size     ,       self.config.num_labels) for _ in range(config.num_hidden_layers)])

        self.init_weights()




 @add_start_docstrings_to_model_forward(_lowerCamelCase)
 def 						__snake_case		(     self   :    Optional[Any]     ,       lowercase__   :    Any=None     ,       lowercase__   :    Optional[int]=None     ,       lowercase__   :    Optional[int]=None     ,       lowercase__   :    Union[str, Any]=None     ,       lowercase__   :    List[str]=None     ,       lowercase__   :    Optional[Any]=None     ,       lowercase__   :    List[str]=None     ,       ):







        '''simple docstring'''



        lowerCAmelCase__	      =   self.albert(
            input_ids=_lowerCamelCase     ,       attention_mask=_lowerCamelCase     ,       token_type_ids=_lowerCamelCase     ,       position_ids=_lowerCamelCase     ,       head_mask=_lowerCamelCase     ,       inputs_embeds=_lowerCamelCase     ,       output_dropout=self.dropout     ,       output_layers=self.classifiers     ,       regression=self.num_labels == 1     ,       )

        lowerCAmelCase__	      =   (logits[-1],)

        if labels is not None:
               lowerCAmelCase__	      =   None
               lowerCAmelCase__	      =   0
               for ix, logits_item in enumerate(_lowerCamelCase):
                      if self.num_labels == 1:
                             #  We are doing regression
                             lowerCAmelCase__	      =   MSELoss()
                             lowerCAmelCase__	      =   loss_fct(logits_item.view(-1)     ,       labels.view(-1))
                      else:
                             lowerCAmelCase__	      =   CrossEntropyLoss()
                             lowerCAmelCase__	      =   loss_fct(logits_item.view(-1     ,       self.num_labels)     ,       labels.view(-1))
                      if total_loss is None:
                             lowerCAmelCase__	      =   loss
                      else:
                             total_loss += loss * (ix + 1)
                      total_weights += ix + 1
               lowerCAmelCase__	      =   (total_loss / total_weights,) + outputs

        return outputs


import math




def 		lowerCAmelCase_     (     _SCREAMING_SNAKE_CASE  )   ->      list[int]:

							"""simple docstring"""






							if not isinstance(_SCREAMING_SNAKE_CASE      ,	_SCREAMING_SNAKE_CASE  ):
														_UpperCAmelCase  :			Optional[Any]	       =		F"""Input value of [number={number}] must be an integer"""
														raise TypeError(_SCREAMING_SNAKE_CASE  )
							if number < 1:
														_UpperCAmelCase  :			Dict	       =		F"""Input value of [number={number}] must be a positive integer"""
														raise ValueError(_SCREAMING_SNAKE_CASE  )
							_UpperCAmelCase  :			List[Any]	       =		[number]
							while number != 1:
														if number % 2 == 0:
																					_UpperCAmelCase  :			List[str]	       =		math.floor(math.sqrt(_SCREAMING_SNAKE_CASE  )  )
														else:
																					_UpperCAmelCase  :			Dict	       =		math.floor(
																					    math.sqrt(_SCREAMING_SNAKE_CASE  ) * math.sqrt(_SCREAMING_SNAKE_CASE  ) * math.sqrt(_SCREAMING_SNAKE_CASE  )  )
														sequence.append(_SCREAMING_SNAKE_CASE  )
							return sequence


if __name__ == "__main__":
				import doctest

				doctest.testmod()




