



'''simple docstring'''




import argparse
import gc
import os
import sys
from pathlib import Path
from typing import List  # noqa: F401

import pytorch_lightning as pl
import torch
from finetune import SummarizationModule, TranslationModule
from finetune import main as ft_main
from make_student import create_student_by_copying_alternating_layers, get_layers_to_supervise
from torch import nn

from transformers import AutoModelForSeqaSeqLM, MBartTokenizer, TaForConditionalGeneration
from transformers.models.bart.modeling_bart import shift_tokens_right
from utils import calculate_bleu, check_output_dir, freeze_params, label_smoothed_nll_loss, use_task_specific_params


# need the parent dir module
sys.path.insert(2, str(Path(__file__).resolve().parents[1]))
from lightning_base import generic_train  # noqa

class      UpperCamelCase_					(				__magic_name__     ):
	lowercase		=					['loss', 'ce_loss', 'mlm_loss', 'hid_loss_enc', 'hid_loss_dec']





	def __init__(    self						,						A       )     ->     Any:
								assert Path(hparams.data_dir       ).exists()
								UpperCAmelCase	:						Dict					=						Path(hparams.output_dir       )
								self.output_dir.mkdir(exist_ok=A       )

								UpperCAmelCase	:						int					=						self.output_dir.joinpath("""student"""       )

								UpperCAmelCase	:						List[Any]					=						str(A       )  # Tell lightning we are training the student
								UpperCAmelCase	:						int					=						AutoModelForSeqaSeqLM.from_pretrained(hparams.teacher       ).eval()
								use_task_specific_params(A						,						hparams.task       )  # We copy good generation parameters to student by default
								if hparams.student is not None:
															UpperCAmelCase	:						Optional[Any]					=						AutoModelForSeqaSeqLM.from_pretrained(hparams.student       )
															use_task_specific_params(A						,						hparams.task       )
															UpperCAmelCase	,	UpperCAmelCase	:						List[str]					=						None, None
								else:
															UpperCAmelCase	,	UpperCAmelCase	,	UpperCAmelCase	:						int					=						create_student_by_copying_alternating_layers(
															    A						,						e=hparams.student_encoder_layers						,						d=hparams.student_decoder_layers						,						save_path=A       )

								if hparams.length_penalty != -1:
															UpperCAmelCase	:						int					=						hparams.length_penalty
								UpperCAmelCase	:						str					=						hparams.teacher  # Use teacher's tokenizer
								super().__init__(A						,						model=A						,						config=student.config       )
								assert student.config.model_type == teacher.config.model_type, (
								    f'''teacher, student model types should be the same, got {student.config.model_type} !='''
								    f''' {teacher.config.model_type}'''
								)

								if student.config.model_type == "t5":
															UpperCAmelCase	:						Optional[int]					=						len(student.get_encoder().block       )
															UpperCAmelCase	:						Any					=						len(student.get_decoder().block       )
															UpperCAmelCase	:						Union[str, Any]					=						len(teacher.get_encoder().block       )
															UpperCAmelCase	:						Optional[int]					=						len(teacher.get_decoder().block       )
								else:
															UpperCAmelCase	:						Dict					=						student.config.encoder_layers
															UpperCAmelCase	:						int					=						student.config.decoder_layers
															UpperCAmelCase	:						Any					=						teacher.config.encoder_layers
															UpperCAmelCase	:						List[Any]					=						teacher.config.decoder_layers

								UpperCAmelCase	:						Dict					=						not (hparams.student is None or hparams.teacher == hparams.student)
								UpperCAmelCase	:						Optional[int]					=						(not self.different_base_models) and hparams.alpha_hid > 0
								UpperCAmelCase	:						Union[str, Any]					=						self.different_base_models or (student_encoder_layers != teacher_encoder_layers)
								# self.different_encoder determines whether we need to run the teacher encoder
								UpperCAmelCase	:						Dict					=						teacher
								freeze_params(self.teacher       )

								if not self.different_encoder:  # To save RAM, delete teacher encoder and freeze student encoder.
															try:
																						del self.teacher.model.encoder
															except AttributeError:  # T5
																						del self.teacher.encoder

								if e_layer_ids is None:
															UpperCAmelCase	:						Tuple					=						list(range(A       )       )
								if d_layer_ids is None:
															UpperCAmelCase	:						List[Any]					=						list(range(A       )       )

								UpperCAmelCase	,	UpperCAmelCase	:						str					=						e_layer_ids, d_layer_ids  # type: List[int], List[int]

								if self.do_calc_hidden_loss:  # Intermediate supervision: Decide which layers to supervise
															if hparams.supervise_forward:
																						UpperCAmelCase	:						List[str]					=						get_layers_to_supervise(
																						    n_student=len(self.e_layer_ids       )						,						n_teacher=A       )
																						UpperCAmelCase	:						int					=						get_layers_to_supervise(
																						    n_student=len(self.d_layer_ids       )						,						n_teacher=A       )
															else:  # student layer should emulate hidden states of the teacher layer it was copied from
																						UpperCAmelCase	:						Dict					=						self.e_layer_ids
																						UpperCAmelCase	:						List[Any]					=						self.d_layer_ids
								else:
															UpperCAmelCase	:						Tuple					=						None
															UpperCAmelCase	:						Union[str, Any]					=						None

								UpperCAmelCase	:						Any					=						nn.KLDivLoss(reduction="""batchmean"""       )
								UpperCAmelCase	:						Tuple					=						2.0
								UpperCAmelCase	:						Tuple					=						hparams.alpha_mlm
								UpperCAmelCase	:						List[str]					=						hparams.alpha_ce
								UpperCAmelCase	:						Dict					=						hparams.alpha_hid
								gc.collect()
								torch.cuda.empty_cache()





	def 					_lowercase(    self						,						A						,						A						,						A       )     ->     Any:
								UpperCAmelCase	:						Union[str, Any]					=						mask[:, :, None].expand_as(A       )
								UpperCAmelCase	:						str					=						s_logits.size(-1       )
								UpperCAmelCase	:						Optional[Any]					=						torch.masked_select(A						,						A       )  # (bs * seq_length * voc_size) modulo the 1s in mask
								UpperCAmelCase	:						List[str]					=						torch.masked_select(A						,						A       )  # (bs * seq_length * voc_size) modulo the 1s in mask
								UpperCAmelCase	:						List[str]					=						s_logits_slct.view(-1						,						A       )  # (bs * seq_length, voc_size) modulo the 1s in mask
								UpperCAmelCase	:						Optional[Any]					=						t_logits_slct.view(-1						,						A       )  # (bs * seq_length, voc_size) modulo the 1s in mask
								assert t_logits_slct.size() == s_logits_slct.size()
								UpperCAmelCase	:						int					=						(
								    self.ce_loss_fct(
								        nn.functional.log_softmax(s_logits_slct / self.temperature						,						dim=-1       )						,						nn.functional.softmax(t_logits_slct / self.temperature						,						dim=-1       )						,						)
								    * (self.temperature) ** 2
								)
								return loss_ce





	@staticmethod
	def 					_lowercase(    A						,						A       )     ->     Optional[int]:
								SummarizationModule.add_model_specific_args(A						,						A       )
								add_distill_args(A       )
								return parser





	def 					_lowercase(    self						,						A       )     ->     tuple:
								UpperCAmelCase	:						Union[str, Any]					=						self.tokenizer.pad_token_id
								UpperCAmelCase	,	UpperCAmelCase	,	UpperCAmelCase	:						Optional[int]					=						batch["""input_ids"""], batch["""attention_mask"""], batch["""labels"""]
								if isinstance(self.model						,						A       ):
															UpperCAmelCase	:						List[str]					=						self.model._shift_right(A       )
								else:
															UpperCAmelCase	:						str					=						shift_tokens_right(A						,						A       )

								# noinspection PyCallingNonCallable
								UpperCAmelCase	:						Union[str, Any]					=						self(
								    A						,						attention_mask=A						,						decoder_input_ids=A						,						output_hidden_states=self.do_calc_hidden_loss						,						output_attentions=A						,						use_cache=A						,						)
								UpperCAmelCase	:						Dict					=						student_outputs["""logits"""]

								# Same cross entropy vs. label smoothing logic as finetune.py
								assert lm_logits.shape[-1] == self.model.config.vocab_size
								if self.hparams.label_smoothing == 0:
															# Same behavior as modeling_bart.py, besides ignoring pad_token_id
															UpperCAmelCase	:						Optional[Any]					=						nn.CrossEntropyLoss(ignore_index=A       )
															UpperCAmelCase	:						Tuple					=						loss_fct(lm_logits.view(-1						,						lm_logits.shape[-1]       )						,						labels.view(-1       )       )
								else:
															UpperCAmelCase	:						int					=						nn.functional.log_softmax(A						,						dim=-1       )
															UpperCAmelCase	,	UpperCAmelCase	:						List[Any]					=						label_smoothed_nll_loss(
															    A						,						A						,						self.hparams.label_smoothing						,						ignore_index=A       )

								def zero_tensor():
															return torch.tensor(0.0       ).type_as(A       )

								UpperCAmelCase	:						Union[str, Any]					=						student_outputs[
								    """encoder_last_hidden_state"""
								]  # use this unless self.different_base_models
								UpperCAmelCase	,	UpperCAmelCase	:						Optional[Any]					=						zero_tensor(), zero_tensor()
								if self.different_encoder:  # compute encoder hidden state loss
															UpperCAmelCase	:						int					=						self.teacher.get_encoder()(
															    A						,						attention_mask=A						,						output_hidden_states=self.do_calc_hidden_loss						,						)
															if self.different_base_models:
																						UpperCAmelCase	:						Any					=						all_teacher_encoder_outputs["""last_hidden_state"""]
															elif self.do_calc_hidden_loss:
																						UpperCAmelCase	:						List[str]					=						self.calc_hidden_loss(
																						    A						,						student_outputs["""encoder_hidden_states"""]						,						all_teacher_encoder_outputs["""hidden_states"""]						,						self.e_matches						,						normalize_hidden=self.hparams.normalize_hidden						,						)

								UpperCAmelCase	:						str					=						self.teacher(
								    A						,						attention_mask=A						,						encoder_outputs=(teacher_enc_outputs,)						,						decoder_input_ids=A						,						output_hidden_states=self.do_calc_hidden_loss						,						use_cache=A						,						)
								UpperCAmelCase	:						Union[str, Any]					=						decoder_input_ids.ne(A       )
								UpperCAmelCase	:						Union[str, Any]					=						self.calc_ce_loss(A						,						A						,						teacher_outputs["""logits"""]       )
								if self.do_calc_hidden_loss:  # Intermediate supervision of decoder hidden states
															UpperCAmelCase	:						List[Any]					=						self.calc_hidden_loss(
															    A						,						student_outputs["""decoder_hidden_states"""]						,						teacher_outputs["""decoder_hidden_states"""]						,						self.d_matches						,						normalize_hidden=self.hparams.normalize_hidden						,						)

								UpperCAmelCase	:						Dict					=						(
								    self.alpha_ce * loss_ce
								    + self.alpha_mlm * student_lm_loss
								    + self.hparams.alpha_hid * (hid_loss_enc + hid_loss_dec)
								)
								return blended_loss, loss_ce, student_lm_loss, hid_loss_enc, hid_loss_dec






	@staticmethod
	def 					_lowercase(    A						,						A						,						A						,						A						,						A       )     ->     Optional[Any]:
								UpperCAmelCase	:						Optional[int]					=						"""expected list or tuple for hidden_states, got tensor of shape: """
								assert not isinstance(A						,						torch.Tensor       ), f'''{msg}{hidden_states.shape}'''
								assert not isinstance(A						,						torch.Tensor       ), f'''{msg}{hidden_states_T.shape}'''
								UpperCAmelCase	:						Optional[Any]					=						attention_mask.to(hidden_states[0]       )
								UpperCAmelCase	:						str					=						mask.sum() * hidden_states[0].size(-1       )
								UpperCAmelCase	:						Dict					=						torch.stack([hidden_states[i] for i in range(len(A       )       )]       )
								UpperCAmelCase	:						Any					=						torch.stack([hidden_states_T[j] for j in matches]       )
								assert student_states.shape == teacher_states.shape, f'''{student_states.shape} != {teacher_states.shape}'''
								if normalize_hidden:
															UpperCAmelCase	:						Union[str, Any]					=						nn.functional.layer_norm(A						,						student_states.shape[1:]       )
															UpperCAmelCase	:						str					=						nn.functional.layer_norm(A						,						teacher_states.shape[1:]       )
								UpperCAmelCase	:						List[str]					=						nn.functional.mse_loss(A						,						A						,						reduction="""none"""       )
								UpperCAmelCase	:						List[Any]					=						(mse * mask.unsqueeze(0       ).unsqueeze(-1       )).sum() / valid_count
								return masked_mse




def  __lowerCamelCase    (   _lowercase						) ->     Dict:
							# NOTE: if --student argument was specified and the teacher and student base models
							# are different, the models still have to have the same tokenizer, specified by
							# --tokenizer_name. So, for example, you can distill from t5_large to t5_small but not
							# from bart to t5. This s because if the tokenizers are different, the output space
							# for the two models is also different and their logits are not comparable.
							parser.add_argument("""--teacher"""      ,							type=_lowercase						)
							parser.add_argument("""--alpha_ce"""      ,							default=0.8      ,							type=_lowercase						)
							parser.add_argument("""--alpha_mlm"""      ,							default=0.2      ,							type=_lowercase						)
							parser.add_argument("""--alpha_hid"""      ,							default=0.0      ,							type=_lowercase      ,							required=_lowercase						)
							parser.add_argument("""--student"""      ,							type=_lowercase      ,							required=_lowercase						)
							parser.add_argument("""--student_decoder_layers"""      ,							default=1_2      ,							type=_lowercase      ,							required=_lowercase						)
							parser.add_argument("""--student_encoder_layers"""      ,							default=1_2      ,							type=_lowercase      ,							required=_lowercase						)
							parser.add_argument("""--no_teacher"""      ,							action="""store_true"""      ,							default=_lowercase						)
							parser.add_argument("""--length_penalty"""      ,							type=_lowercase      ,							default=-1						)
							parser.add_argument("""--supervise_forward"""      ,							action="""store_true"""      ,							default=_lowercase						)
							parser.add_argument("""--normalize_hidden"""      ,							action="""store_true"""      ,							default=_lowercase						)

class      UpperCamelCase_					(				__magic_name__     ):
	lowercase		=					'translation'
	lowercase		=					['bleu']
	lowercase		=					'bleu'





	def __init__(    self						,						A						,						**A       )     ->     str:
								super().__init__(A						,						**A       )
								assert hparams.src_lang is not None
								assert hparams.tgt_lang is not None
								UpperCAmelCase	:						Any					=						hparams.src_lang
								UpperCAmelCase	:						Optional[Any]					=						hparams.tgt_lang
								if self.model.config.decoder_start_token_id is None and isinstance(self.tokenizer						,						A       ):
															UpperCAmelCase	:						Union[str, Any]					=						self.tokenizer.lang_code_to_id[hparams.tgt_lang]





	def 					_lowercase(    self						,						A						,						A       )     ->     dict:
								return calculate_bleu(A						,						A       )






	@staticmethod
	def 					_lowercase(    A						,						A       )     ->     Dict:
								TranslationModule.add_model_specific_args(A						,						A       )
								add_distill_args(A       )
								return parser




def  __lowerCamelCase    (   _lowercase						) ->     Tuple:
							if args.no_teacher:
														UpperCAmelCase	:						Optional[int]					=						TranslationModule if """translation""" in args.task else SummarizationModule
							else:  # DISTILL WITH TEACHER
														UpperCAmelCase	:						List[Any]					=						TranslationDistiller if """translation""" in args.task else SummarizationDistiller
							UpperCAmelCase	:						str					=						module_cls.__name__
							print(F'''using module {args.setup_cls}'''						)
							UpperCAmelCase	:						Optional[Any]					=						module_cls(_lowercase						)
							return model



def  __lowerCamelCase    (   _lowercase						) ->     Optional[int]:
							Path(args.output_dir						).mkdir(exist_ok=_lowercase						)
							check_output_dir(_lowercase      ,							expected_items=3						)

							UpperCAmelCase	:						int					=						create_module(_lowercase						)
							return ft_main(_lowercase      ,							model=_lowercase						)


if __name__ == "__main__":
							a  :     Any       =       argparse.ArgumentParser()
							a  :     Tuple       =       pl.Trainer.add_argparse_args(parser)
							a  :     Optional[Any]       =       SummarizationDistiller.add_model_specific_args(parser, os.getcwd())
							a  :     Any       =       parser.parse_args()

							distill_main(args)






'''simple docstring'''




import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a  :     Union[str, Any]       =       logging.get_logger(__name__)

a  :     int       =       {
    """facebook/hubert-base-ls960""": """https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json""",
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}

class      UpperCamelCase_					(				__magic_name__     ):
	lowercase		=					'hubert'





	def __init__(    self						,						A=32						,						A=768						,						A=12						,						A=12						,						A=3072						,						A="gelu"						,						A=0.1						,						A=0.1						,						A=0.1						,						A=True						,						A=0.0						,						A=0.1						,						A=0.1						,						A=0.0_2						,						A=1e-5						,						A="group"						,						A="gelu"						,						A=(512, 512, 512, 512, 512, 512, 512)						,						A=(5, 2, 2, 2, 2, 2, 2)						,						A=(10, 3, 3, 3, 3, 2, 2)						,						A=False						,						A=128						,						A=16						,						A=False						,						A=True						,						A=0.0_5						,						A=10						,						A=2						,						A=0.0						,						A=10						,						A=0						,						A="sum"						,						A=False						,						A=False						,						A=256						,						A=0						,						A=1						,						A=2						,						**A						,						)     ->     List[str]:
								super().__init__(**A						,						pad_token_id=A						,						bos_token_id=A						,						eos_token_id=A       )
								UpperCAmelCase	:						List[Any]					=						hidden_size
								UpperCAmelCase	:						List[Any]					=						feat_extract_norm
								UpperCAmelCase	:						Optional[Any]					=						feat_extract_activation
								UpperCAmelCase	:						Optional[int]					=						list(A       )
								UpperCAmelCase	:						int					=						list(A       )
								UpperCAmelCase	:						Any					=						list(A       )
								UpperCAmelCase	:						Any					=						conv_bias
								UpperCAmelCase	:						Dict					=						num_conv_pos_embeddings
								UpperCAmelCase	:						Tuple					=						num_conv_pos_embedding_groups
								UpperCAmelCase	:						str					=						len(self.conv_dim       )
								UpperCAmelCase	:						Optional[Any]					=						num_hidden_layers
								UpperCAmelCase	:						int					=						intermediate_size
								UpperCAmelCase	:						Any					=						hidden_act
								UpperCAmelCase	:						Tuple					=						num_attention_heads
								UpperCAmelCase	:						int					=						hidden_dropout
								UpperCAmelCase	:						Any					=						attention_dropout
								UpperCAmelCase	:						Any					=						activation_dropout
								UpperCAmelCase	:						str					=						feat_proj_layer_norm
								UpperCAmelCase	:						int					=						feat_proj_dropout
								UpperCAmelCase	:						List[Any]					=						final_dropout
								UpperCAmelCase	:						int					=						layerdrop
								UpperCAmelCase	:						Dict					=						layer_norm_eps
								UpperCAmelCase	:						Union[str, Any]					=						initializer_range
								UpperCAmelCase	:						Any					=						vocab_size
								UpperCAmelCase	:						Optional[int]					=						do_stable_layer_norm
								UpperCAmelCase	:						Optional[int]					=						use_weighted_layer_sum
								UpperCAmelCase	:						Dict					=						classifier_proj_size

								if (
								    (len(self.conv_stride       ) != self.num_feat_extract_layers)
								    or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
								    or (len(self.conv_dim       ) != self.num_feat_extract_layers)
								):
															raise ValueError(
															    """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
															    """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
															    f''' {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,'''
															    f''' `len(config.conv_kernel) = {len(self.conv_kernel       )}`.'''       )

								# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
								UpperCAmelCase	:						Union[str, Any]					=						apply_spec_augment
								UpperCAmelCase	:						Tuple					=						mask_time_prob
								UpperCAmelCase	:						Optional[Any]					=						mask_time_length
								UpperCAmelCase	:						List[Any]					=						mask_time_min_masks
								UpperCAmelCase	:						Any					=						mask_feature_prob
								UpperCAmelCase	:						Any					=						mask_feature_length
								UpperCAmelCase	:						Tuple					=						mask_feature_min_masks

								# ctc loss
								UpperCAmelCase	:						int					=						ctc_loss_reduction
								UpperCAmelCase	:						Any					=						ctc_zero_infinity






	@property
	def 					_lowercase(    self       )     ->     Union[str, Any]:
								return functools.reduce(operator.mul						,						self.conv_stride						,						1       )

