

import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


_lowerCamelCase       :     List[str]           =							logging.getLogger(__name__)





class       __UpperCAmelCase  ( lowerCamelCase__		):



   def  __magic_name__					(							self							:  Optional[int],	__A							:  str,	__A							:  Tuple,	__A							:  List[str]=None,	__A							:  Optional[int]=None		):
     if current_layer == 0:
       UpperCAmelCase :							Dict								=				self.embedding_hidden_mapping_in(__A		)
     else:
       UpperCAmelCase :							Union[str, Any]								=				hidden_states[0]

     UpperCAmelCase :							Union[str, Any]								=				int(self.config.num_hidden_layers / self.config.num_hidden_groups		)

     # Index of the hidden group
     UpperCAmelCase :							Union[str, Any]								=				int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)		)

     UpperCAmelCase :							Tuple								=				self.albert_layer_groups[group_idx](
         __A,	__A,	head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group],	)
     UpperCAmelCase :							Union[str, Any]								=				layer_group_output[0]

     return (hidden_states,)






@add_start_docstrings(
    """The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top."""    ,						lowerCamelCase__    ,						)
class       __UpperCAmelCase  ( lowerCamelCase__		):



   def __init__(							self							:  Dict,	__A							:  Union[str, Any]		):
     super().__init__(__A		)

     UpperCAmelCase :							List[str]								=				AlbertTransformerWithPabee(__A		)

     self.init_weights()
     UpperCAmelCase :							Dict								=				0
     UpperCAmelCase :							List[str]								=				0
     UpperCAmelCase :							str								=				0

     UpperCAmelCase :							Dict								=				0



   def  __magic_name__					(							self							:  Dict,	__A							:  str		):
     UpperCAmelCase :							Tuple								=				threshold



   def  __magic_name__					(							self							:  int,	__A							:  Union[str, Any]		):
     UpperCAmelCase :							str								=				patience



   def  __magic_name__					(							self							:  List[str]		):
     UpperCAmelCase :							List[Any]								=				0
     UpperCAmelCase :							Optional[Any]								=				0



   def  __magic_name__					(							self							:  Optional[int]		):
     UpperCAmelCase :							Optional[int]								=				self.inference_layers_num / self.inference_instances_num
     UpperCAmelCase :							Tuple								=				(
         F'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
         F''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
     )
     print(__A		)





   @add_start_docstrings_to_model_forward(__A		)
   def  __magic_name__					(							self							:  Dict,	__A							:  Optional[Any]=None,	__A							:  Tuple=None,	__A							:  str=None,	__A							:  Any=None,	__A							:  Any=None,	__A							:  str=None,	__A							:  str=None,	__A							:  Optional[int]=None,	__A							:  str=False,	):

     if input_ids is not None and inputs_embeds is not None:
       raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''		)
     elif input_ids is not None:
       UpperCAmelCase :							Optional[int]								=				input_ids.size()
     elif inputs_embeds is not None:
       UpperCAmelCase :							List[Any]								=				inputs_embeds.size()[:-1]
     else:
       raise ValueError('''You have to specify either input_ids or inputs_embeds'''		)

     UpperCAmelCase :							List[Any]								=				input_ids.device if input_ids is not None else inputs_embeds.device

     if attention_mask is None:
       UpperCAmelCase :							int								=				torch.ones(__A,	device=__A		)
     if token_type_ids is None:
       UpperCAmelCase :							Dict								=				torch.zeros(__A,	dtype=torch.long,	device=__A		)

     UpperCAmelCase :							Dict								=				attention_mask.unsqueeze(1		).unsqueeze(2		)
     UpperCAmelCase :							Optional[Any]								=				extended_attention_mask.to(dtype=self.dtype		)  # fp16 compatibility
     UpperCAmelCase :							List[str]								=				(1.0 - extended_attention_mask) * -1_0_0_0_0.0
     UpperCAmelCase :							List[Any]								=				self.get_head_mask(__A,	self.config.num_hidden_layers		)

     UpperCAmelCase :							Union[str, Any]								=				self.embeddings(
         __A,	position_ids=__A,	token_type_ids=__A,	inputs_embeds=__A		)
     UpperCAmelCase :							List[Any]								=				embedding_output

     if self.training:
       UpperCAmelCase :							Optional[Any]								=				[]
       for i in range(self.config.num_hidden_layers		):
         UpperCAmelCase :							List[str]								=				self.encoder.adaptive_forward(
             __A,	current_layer=__A,	attention_mask=__A,	head_mask=__A,	)

         UpperCAmelCase :							Union[str, Any]								=				self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]		)		)
         UpperCAmelCase :							Union[str, Any]								=				output_layers[i](output_dropout(__A		)		)
         res.append(__A		)
     elif self.patience == 0:  # Use all layers for inference
       UpperCAmelCase :							Dict								=				self.encoder(__A,	__A,	head_mask=__A		)
       UpperCAmelCase :							Optional[int]								=				self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]		)		)
       UpperCAmelCase :							Optional[Any]								=				[output_layers[self.config.num_hidden_layers - 1](__A		)]
     else:
       UpperCAmelCase :							List[str]								=				0
       UpperCAmelCase :							List[Any]								=				None
       UpperCAmelCase :							Tuple								=				0
       for i in range(self.config.num_hidden_layers		):
         calculated_layer_num += 1
         UpperCAmelCase :							int								=				self.encoder.adaptive_forward(
             __A,	current_layer=__A,	attention_mask=__A,	head_mask=__A,	)

         UpperCAmelCase :							str								=				self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]		)		)
         UpperCAmelCase :							Optional[Any]								=				output_layers[i](__A		)
         if regression:
           UpperCAmelCase :							Union[str, Any]								=				logits.detach()
           if patient_result is not None:
             UpperCAmelCase :							Optional[int]								=				patient_result.detach()
           if (patient_result is not None) and torch.abs(patient_result - labels		) < self.regression_threshold:
             patient_counter += 1
           else:
             UpperCAmelCase :							Any								=				0
         else:
           UpperCAmelCase :							Union[str, Any]								=				logits.detach().argmax(dim=1		)
           if patient_result is not None:
             UpperCAmelCase :							Optional[int]								=				patient_result.detach().argmax(dim=1		)
           if (patient_result is not None) and torch.all(labels.eq(__A		)		):
             patient_counter += 1
           else:
             UpperCAmelCase :							str								=				0

         UpperCAmelCase :							str								=				logits
         if patient_counter == self.patience:
           break
       UpperCAmelCase :							str								=				[patient_result]
       self.inference_layers_num += calculated_layer_num
       self.inference_instances_num += 1

     return res






@add_start_docstrings(
    """Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. """    ,						lowerCamelCase__    ,						)
class       __UpperCAmelCase  ( lowerCamelCase__		):



   def __init__(							self							:  Tuple,	__A							:  Tuple		):
     super().__init__(__A		)
     UpperCAmelCase :							Tuple								=				config.num_labels

     UpperCAmelCase :							Optional[int]								=				AlbertModelWithPabee(__A		)
     UpperCAmelCase :							Dict								=				nn.Dropout(config.classifier_dropout_prob		)
     UpperCAmelCase :							Tuple								=				nn.ModuleList(
         [nn.Linear(config.hidden_size,	self.config.num_labels		) for _ in range(config.num_hidden_layers		)]		)

     self.init_weights()





   @add_start_docstrings_to_model_forward(__A		)
   def  __magic_name__					(							self							:  Tuple,	__A							:  Union[str, Any]=None,	__A							:  str=None,	__A							:  str=None,	__A							:  str=None,	__A							:  Tuple=None,	__A							:  Optional[int]=None,	__A							:  Any=None,	):
     UpperCAmelCase :							List[Any]								=				self.albert(
         input_ids=__A,	attention_mask=__A,	token_type_ids=__A,	position_ids=__A,	head_mask=__A,	inputs_embeds=__A,	output_dropout=self.dropout,	output_layers=self.classifiers,	regression=self.num_labels == 1,	)

     UpperCAmelCase :							str								=				(logits[-1],)

     if labels is not None:
       UpperCAmelCase :							Tuple								=				None
       UpperCAmelCase :							Union[str, Any]								=				0
       for ix, logits_item in enumerate(__A		):
         if self.num_labels == 1:
           #  We are doing regression
           UpperCAmelCase :							Dict								=				MSELoss()
           UpperCAmelCase :							Any								=				loss_fct(logits_item.view(-1		),	labels.view(-1		)		)
         else:
           UpperCAmelCase :							Optional[int]								=				CrossEntropyLoss()
           UpperCAmelCase :							List[Any]								=				loss_fct(logits_item.view(-1,	self.num_labels		),	labels.view(-1		)		)
         if total_loss is None:
           UpperCAmelCase :							Union[str, Any]								=				loss
         else:
           total_loss += loss * (ix + 1)
         total_weights += ix + 1
       UpperCAmelCase :							str								=				(total_loss / total_weights,) + outputs

     return outputs








import decimal

def     a__  (	)						->		int:
  UpperCAmelCase :							List[Any]								=				0
  UpperCAmelCase :							Dict								=				decimal.Context(prec=105    )
  for i in range(2       ,				100    ):
    UpperCAmelCase :							List[str]								=				decimal.Decimal(UpperCAmelCase    )
    UpperCAmelCase :							List[str]								=				number.sqrt(UpperCAmelCase    )
    if len(str(UpperCAmelCase    )    ) > 1:
      answer += int(str(UpperCAmelCase    )[0]    )
      UpperCAmelCase :							str								=				str(UpperCAmelCase    )[2:101]
      answer += sum(int(UpperCAmelCase    ) for x in sqrt_number_str    )
  return answer


if __name__ == "__main__":
     import doctest

     doctest.testmod()
     print(f"""{solution() = }""")





