




'''simple docstring'''

import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class __magic_name__(						__lowerCAmelCase    ,		unittest.TestCase      ):
						UpperCAmelCase_		:					Union[str, Any]    =  ElectraTokenizer
						UpperCAmelCase_		:					str    =  ElectraTokenizerFast
						UpperCAmelCase_		:					List[Any]    =  True
						UpperCAmelCase_		:					str    =  True
						UpperCAmelCase_		:					Dict    =  filter_non_english






						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									super().setUp()

									snake_case__          =						[
									    """[UNK]""",
									    """[CLS]""",
									    """[SEP]""",
									    """[PAD]""",
									    """[MASK]""",
									    """want""",
									    """##want""",
									    """##ed""",
									    """wa""",
									    """un""",
									    """runn""",
									    """##ing""",
									    """,""",
									    """low""",
									    """lowest""",
									]
									snake_case__          =						os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["""vocab_file"""]       )
									with open(self.vocab_file					,   """w"""					,   encoding="""utf-8"""       ) as vocab_writer:
												vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]					,   __UpperCamelCase						:						Union[str, Any]       ):



									'''simple docstring'''






									snake_case__          =						"""UNwant\u00E9d,running"""
									snake_case__          =						"""unwanted, running"""
									return input_text, output_text






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									snake_case__          =						self.tokenizer_class(self.vocab_file       )

									snake_case__          =						tokenizer.tokenize("""UNwant\u00E9d,running"""       )
									self.assertListEqual(__UpperCamelCase					,   ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]       )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase       )					,   [9, 6, 7, 1_2, 1_0, 1_1]       )






						def       __lowerCAmelCase(   self						:						int       ):



									'''simple docstring'''






									if not self.test_rust_tokenizer:
												return

									snake_case__          =						self.get_tokenizer()
									snake_case__          =						self.get_rust_tokenizer()

									snake_case__          =						"""UNwant\u00E9d,running"""

									snake_case__          =						tokenizer.tokenize(__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.tokenize(__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )

									snake_case__          =						tokenizer.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )

									snake_case__          =						self.get_rust_tokenizer()
									snake_case__          =						tokenizer.encode(__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.encode(__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )

									# With lower casing
									snake_case__          =						self.get_tokenizer(do_lower_case=__UpperCamelCase       )
									snake_case__          =						self.get_rust_tokenizer(do_lower_case=__UpperCamelCase       )

									snake_case__          =						"""UNwant\u00E9d,running"""

									snake_case__          =						tokenizer.tokenize(__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.tokenize(__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )

									snake_case__          =						tokenizer.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )

									snake_case__          =						self.get_rust_tokenizer()
									snake_case__          =						tokenizer.encode(__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.encode(__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Optional[Any]       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""       )					,   ["""ah""", """\u535A""", """\u63A8""", """zz"""]       )






						def       __lowerCAmelCase(   self						:						Dict       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       )					,   ["""hello""", """!""", """how""", """are""", """you""", """?"""]       )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )					,   ["""hello"""]       )






						def       __lowerCAmelCase(   self						:						Optional[Any]       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase					,   strip_accents=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )					,   ["""hällo""", """!""", """how""", """are""", """you""", """?"""]       )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )					,   ["""h\u00E9llo"""]       )






						def       __lowerCAmelCase(   self						:						Dict       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase					,   strip_accents=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )					,   ["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )					,   ["""hello"""]       )






						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )					,   ["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )					,   ["""hello"""]       )






						def       __lowerCAmelCase(   self						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       )					,   ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase					,   strip_accents=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )					,   ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )






						def       __lowerCAmelCase(   self						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase					,   strip_accents=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )					,   ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase					,   never_split=["""[UNK]"""]       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""       )					,   ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]       ):



									'''simple docstring'''






									snake_case__          =						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									snake_case__          =						{}
									for i, token in enumerate(__UpperCamelCase       ):
												snake_case__          =						i
									snake_case__          =						WordpieceTokenizer(vocab=__UpperCamelCase					,   unk_token="""[UNK]"""       )

									self.assertListEqual(tokenizer.tokenize(""""""       )					,   []       )

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""       )					,   ["""un""", """##want""", """##ed""", """runn""", """##ing"""]       )

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""       )					,   ["""[UNK]""", """runn""", """##ing"""]       )






						def       __lowerCAmelCase(   self						:						Dict       ):



									'''simple docstring'''






									self.assertTrue(_is_whitespace(""" """       )       )
									self.assertTrue(_is_whitespace("""\t"""       )       )
									self.assertTrue(_is_whitespace("""\r"""       )       )
									self.assertTrue(_is_whitespace("""\n"""       )       )
									self.assertTrue(_is_whitespace("""\u00A0"""       )       )

									self.assertFalse(_is_whitespace("""A"""       )       )
									self.assertFalse(_is_whitespace("""-"""       )       )






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									self.assertTrue(_is_control("""\u0005"""       )       )

									self.assertFalse(_is_control("""A"""       )       )
									self.assertFalse(_is_control(""" """       )       )
									self.assertFalse(_is_control("""\t"""       )       )
									self.assertFalse(_is_control("""\r"""       )       )






						def       __lowerCAmelCase(   self						:						int       ):



									'''simple docstring'''






									self.assertTrue(_is_punctuation("""-"""       )       )
									self.assertTrue(_is_punctuation("""$"""       )       )
									self.assertTrue(_is_punctuation("""`"""       )       )
									self.assertTrue(_is_punctuation("""."""       )       )

									self.assertFalse(_is_punctuation("""A"""       )       )
									self.assertFalse(_is_punctuation(""" """       )       )






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									snake_case__          =						self.get_tokenizer()
									snake_case__          =						self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(__UpperCamelCase       ) for t in ["""Test""", """\xad""", """test"""]]					,   [["""[UNK]"""], [], ["""[UNK]"""]]       )

									self.assertListEqual(
									    [rust_tokenizer.tokenize(__UpperCamelCase       ) for t in ["""Test""", """\xad""", """test"""]]					,   [["""[UNK]"""], [], ["""[UNK]"""]]       )






						@slow
						def       __lowerCAmelCase(   self						:						Tuple       ):



									'''simple docstring'''






									snake_case__          =						self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""       )

									snake_case__          =						tokenizer.encode("""sequence builders"""					,   add_special_tokens=__UpperCamelCase       )
									snake_case__          =						tokenizer.encode("""multi-sequence build"""					,   add_special_tokens=__UpperCamelCase       )

									snake_case__          =						tokenizer.build_inputs_with_special_tokens(__UpperCamelCase       )
									snake_case__          =						tokenizer.build_inputs_with_special_tokens(__UpperCamelCase					,   __UpperCamelCase       )

									assert encoded_sentence == [1_0_1] + text + [1_0_2]
									assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]






						def       __lowerCAmelCase(   self						:						int       ):



									'''simple docstring'''






									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
															snake_case__          =						self.rust_tokenizer_class.from_pretrained(__UpperCamelCase					,   **__UpperCamelCase       )

															snake_case__          =						f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
															snake_case__          =						tokenizer_r.encode_plus(
															    __UpperCamelCase					,   return_attention_mask=__UpperCamelCase					,   return_token_type_ids=__UpperCamelCase					,   return_offsets_mapping=__UpperCamelCase					,   add_special_tokens=__UpperCamelCase					,   )

															snake_case__          =						tokenizer_r.do_lower_case if hasattr(__UpperCamelCase					,   """do_lower_case"""       ) else False
															snake_case__          =						(
															    [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), """A"""),
															        ((1, 2), ""","""),
															        ((3, 5), """na"""),
															        ((5, 6), """##ï"""),
															        ((6, 8), """##ve"""),
															        ((9, 1_5), tokenizer_r.mask_token),
															        ((1_6, 2_1), """Allen"""),
															        ((2_1, 2_3), """##NL"""),
															        ((2_3, 2_4), """##P"""),
															        ((2_5, 3_3), """sentence"""),
															        ((3_3, 3_4), """."""),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															    if not do_lower_case
															    else [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), """a"""),
															        ((1, 2), ""","""),
															        ((3, 8), """naive"""),
															        ((9, 1_5), tokenizer_r.mask_token),
															        ((1_6, 2_1), """allen"""),
															        ((2_1, 2_3), """##nl"""),
															        ((2_3, 2_4), """##p"""),
															        ((2_5, 3_3), """sentence"""),
															        ((3_3, 3_4), """."""),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															)

															self.assertEqual(
															    [e[1] for e in expected_results]					,   tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]       )       )
															self.assertEqual([e[0] for e in expected_results]					,   tokens["""offset_mapping"""]       )







						def       __lowerCAmelCase(   self						:						Tuple       ):



									'''simple docstring'''






									snake_case__          =						["""的""", """人""", """有"""]
									snake_case__          =						"""""".join(__UpperCamelCase       )
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
															snake_case__          =						True
															snake_case__          =						self.tokenizer_class.from_pretrained(__UpperCamelCase					,   **__UpperCamelCase       )
															snake_case__          =						self.rust_tokenizer_class.from_pretrained(__UpperCamelCase					,   **__UpperCamelCase       )

															snake_case__          =						tokenizer_p.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )
															snake_case__          =						tokenizer_r.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )

															snake_case__          =						tokenizer_r.convert_ids_to_tokens(__UpperCamelCase       )
															snake_case__          =						tokenizer_p.convert_ids_to_tokens(__UpperCamelCase       )

															# it is expected that each Chinese character is not preceded by "##"
															self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )
															self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )

															snake_case__          =						False
															snake_case__          =						self.rust_tokenizer_class.from_pretrained(__UpperCamelCase					,   **__UpperCamelCase       )
															snake_case__          =						self.tokenizer_class.from_pretrained(__UpperCamelCase					,   **__UpperCamelCase       )

															snake_case__          =						tokenizer_r.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )
															snake_case__          =						tokenizer_p.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )

															snake_case__          =						tokenizer_r.convert_ids_to_tokens(__UpperCamelCase       )
															snake_case__          =						tokenizer_p.convert_ids_to_tokens(__UpperCamelCase       )

															# it is expected that only the first Chinese character is not preceded by "##".
															snake_case__          =						[
															    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__UpperCamelCase       )
															]
															self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )
															self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )







"""simple docstring"""




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


UpperCamelCase											=     logging.getLogger(__name__)




class  lowercase_     (_UpperCAmelCase  ):






      def       lowerCamelCase__     (     self  ,  a_  ,  a_  ,  a_=None  ,  a_=None						)						->Optional[int]:







        '''simple docstring'''


        if current_layer == 0:
          _a					      =						self.embedding_hidden_mapping_in(a_						)
        else:
          _a					      =						hidden_states[0]

        _a					      =						int(self.config.num_hidden_layers / self.config.num_hidden_groups						)

        # Index of the hidden group
        _a					      =						int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)						)

        _a					      =						self.albert_layer_groups[group_idx](
            a_  ,  a_  ,  head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]  ,  )
        _a					      =						layer_group_output[0]

        return (hidden_states,)







@add_start_docstrings(
    '''The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.''',     _UpperCAmelCase,     )
class  lowercase_     (_UpperCAmelCase  ):






      def __init__(     self  ,  a_						)						->Dict:







        '''simple docstring'''


        super().__init__(a_						)

        _a					      =						AlbertTransformerWithPabee(a_						)

        self.init_weights()
        _a					      =						0
        _a					      =						0
        _a					      =						0

        _a					      =						0






      def       lowerCamelCase__     (     self  ,  a_						)						->Optional[int]:







        '''simple docstring'''


        _a					      =						threshold






      def       lowerCamelCase__     (     self  ,  a_						)						->Union[str, Any]:







        '''simple docstring'''


        _a					      =						patience






      def       lowerCamelCase__     (     self						)						->int:







        '''simple docstring'''


        _a					      =						0
        _a					      =						0






      def       lowerCamelCase__     (     self						)						->Tuple:







        '''simple docstring'''


        _a					      =						self.inference_layers_num / self.inference_instances_num
        _a					      =						(
            f'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
            f''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
        )
        print(a_						)






      @add_start_docstrings_to_model_forward(a_						)
      def       lowerCamelCase__     (     self  ,  a_=None  ,  a_=None  ,  a_=None  ,  a_=None  ,  a_=None  ,  a_=None  ,  a_=None  ,  a_=None  ,  a_=False  ,  )						->int:







        '''simple docstring'''


        if input_ids is not None and inputs_embeds is not None:
          raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"						)
        elif input_ids is not None:
          _a					      =						input_ids.size()
        elif inputs_embeds is not None:
          _a					      =						inputs_embeds.size()[:-1]
        else:
          raise ValueError("You have to specify either input_ids or inputs_embeds"						)

        _a					      =						input_ids.device if input_ids is not None else inputs_embeds.device

        if attention_mask is None:
          _a					      =						torch.ones(a_  ,  device=a_						)
        if token_type_ids is None:
          _a					      =						torch.zeros(a_  ,  dtype=torch.long  ,  device=a_						)

        _a					      =						attention_mask.unsqueeze(1						).unsqueeze(2						)
        _a					      =						extended_attention_mask.to(dtype=self.dtype						)  # fp16 compatibility
        _a					      =						(1.0 - extended_attention_mask) * -10_000.0
        _a					      =						self.get_head_mask(a_  ,  self.config.num_hidden_layers						)

        _a					      =						self.embeddings(
            a_  ,  position_ids=a_  ,  token_type_ids=a_  ,  inputs_embeds=a_						)
        _a					      =						embedding_output

        if self.training:
          _a					      =						[]
          for i in range(self.config.num_hidden_layers						):
            _a					      =						self.encoder.adaptive_forward(
                a_  ,  current_layer=a_  ,  attention_mask=a_  ,  head_mask=a_  ,  )

            _a					      =						self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
            _a					      =						output_layers[i](output_dropout(a_						)						)
            res.append(a_						)
        elif self.patience == 0:  # Use all layers for inference
          _a					      =						self.encoder(a_  ,  a_  ,  head_mask=a_						)
          _a					      =						self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
          _a					      =						[output_layers[self.config.num_hidden_layers - 1](a_						)]
        else:
          _a					      =						0
          _a					      =						None
          _a					      =						0
          for i in range(self.config.num_hidden_layers						):
            calculated_layer_num += 1
            _a					      =						self.encoder.adaptive_forward(
                a_  ,  current_layer=a_  ,  attention_mask=a_  ,  head_mask=a_  ,  )

            _a					      =						self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]						)						)
            _a					      =						output_layers[i](a_						)
            if regression:
              _a					      =						logits.detach()
              if patient_result is not None:
                _a					      =						patient_result.detach()
              if (patient_result is not None) and torch.abs(patient_result - labels						) < self.regression_threshold:
                patient_counter += 1
              else:
                _a					      =						0
            else:
              _a					      =						logits.detach().argmax(dim=1						)
              if patient_result is not None:
                _a					      =						patient_result.detach().argmax(dim=1						)
              if (patient_result is not None) and torch.all(labels.eq(a_						)						):
                patient_counter += 1
              else:
                _a					      =						0

            _a					      =						logits
            if patient_counter == self.patience:
              break
          _a					      =						[patient_result]
          self.inference_layers_num += calculated_layer_num
          self.inference_instances_num += 1

        return res







@add_start_docstrings(
    '''Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. ''',     _UpperCAmelCase,     )
class  lowercase_     (_UpperCAmelCase  ):






      def __init__(     self  ,  a_						)						->Dict:







        '''simple docstring'''


        super().__init__(a_						)
        _a					      =						config.num_labels

        _a					      =						AlbertModelWithPabee(a_						)
        _a					      =						nn.Dropout(config.classifier_dropout_prob						)
        _a					      =						nn.ModuleList(
            [nn.Linear(config.hidden_size  ,  self.config.num_labels						) for _ in range(config.num_hidden_layers						)]						)

        self.init_weights()






      @add_start_docstrings_to_model_forward(a_						)
      def       lowerCamelCase__     (     self  ,  a_=None  ,  a_=None  ,  a_=None  ,  a_=None  ,  a_=None  ,  a_=None  ,  a_=None  ,  )						->List[Any]:







        '''simple docstring'''


        _a					      =						self.albert(
            input_ids=a_  ,  attention_mask=a_  ,  token_type_ids=a_  ,  position_ids=a_  ,  head_mask=a_  ,  inputs_embeds=a_  ,  output_dropout=self.dropout  ,  output_layers=self.classifiers  ,  regression=self.num_labels == 1  ,  )

        _a					      =						(logits[-1],)

        if labels is not None:
          _a					      =						None
          _a					      =						0
          for ix, logits_item in enumerate(a_						):
            if self.num_labels == 1:
              #  We are doing regression
              _a					      =						MSELoss()
              _a					      =						loss_fct(logits_item.view(-1						)  ,  labels.view(-1						)						)
            else:
              _a					      =						CrossEntropyLoss()
              _a					      =						loss_fct(logits_item.view(-1  ,  self.num_labels						)  ,  labels.view(-1						)						)
            if total_loss is None:
              _a					      =						loss
            else:
              total_loss += loss * (ix + 1)
            total_weights += ix + 1
          _a					      =						(total_loss / total_weights,) + outputs

        return outputs


