


import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


_A							     =       logging.getLogger(__name__)

class 			lowerCamelCase_					(    _SCREAMING_SNAKE_CASE    ):





							def   __magic_name__      (							self			,       _SCREAMING_SNAKE_CASE			,       _SCREAMING_SNAKE_CASE			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None    ):
								if current_layer == 0:
									a_     			=   self.embedding_hidden_mapping_in(_SCREAMING_SNAKE_CASE    )
								else:
									a_     			=   hidden_states[0]

								a_     			=   int(self.config.num_hidden_layers / self.config.num_hidden_groups    )

								# Index of the hidden group
								a_     			=   int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)    )

								a_     			=   self.albert_layer_groups[group_idx](
								    _SCREAMING_SNAKE_CASE			,       _SCREAMING_SNAKE_CASE			,       head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]			,       )
								a_     			=   layer_group_output[0]

								return (hidden_states,)







@add_start_docstrings(
    """The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top."""   ,		_SCREAMING_SNAKE_CASE   ,		)
class 			lowerCamelCase_					(    _SCREAMING_SNAKE_CASE    ):





							def __init__(							self			,       _SCREAMING_SNAKE_CASE    ):
								super().__init__(_SCREAMING_SNAKE_CASE    )

								a_     			=   AlbertTransformerWithPabee(_SCREAMING_SNAKE_CASE    )

								self.init_weights()
								a_     			=   0
								a_     			=   0
								a_     			=   0

								a_     			=   0





							def   __magic_name__      (							self			,       _SCREAMING_SNAKE_CASE    ):
								a_     			=   threshold





							def   __magic_name__      (							self			,       _SCREAMING_SNAKE_CASE    ):
								a_     			=   patience





							def   __magic_name__      (							self    ):
								a_     			=   0
								a_     			=   0





							def   __magic_name__      (							self    ):
								a_     			=   self.inference_layers_num / self.inference_instances_num
								a_     			=   (
								    f"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
								    f""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
								)
								print(_SCREAMING_SNAKE_CASE    )





							@add_start_docstrings_to_model_forward(_SCREAMING_SNAKE_CASE    )
							def   __magic_name__      (							self			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=False			,       ):

								if input_ids is not None and inputs_embeds is not None:
									raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""    )
								elif input_ids is not None:
									a_     			=   input_ids.size()
								elif inputs_embeds is not None:
									a_     			=   inputs_embeds.size()[:-1]
								else:
									raise ValueError("""You have to specify either input_ids or inputs_embeds"""    )

								a_     			=   input_ids.device if input_ids is not None else inputs_embeds.device

								if attention_mask is None:
									a_     			=   torch.ones(_SCREAMING_SNAKE_CASE			,       device=_SCREAMING_SNAKE_CASE    )
								if token_type_ids is None:
									a_     			=   torch.zeros(_SCREAMING_SNAKE_CASE			,       dtype=torch.long			,       device=_SCREAMING_SNAKE_CASE    )

								a_     			=   attention_mask.unsqueeze(1    ).unsqueeze(2    )
								a_     			=   extended_attention_mask.to(dtype=self.dtype    )  # fp16 compatibility
								a_     			=   (1.0 - extended_attention_mask) * -1_0_0_0_0.0
								a_     			=   self.get_head_mask(_SCREAMING_SNAKE_CASE			,       self.config.num_hidden_layers    )

								a_     			=   self.embeddings(
								    _SCREAMING_SNAKE_CASE			,       position_ids=_SCREAMING_SNAKE_CASE			,       token_type_ids=_SCREAMING_SNAKE_CASE			,       inputs_embeds=_SCREAMING_SNAKE_CASE    )
								a_     			=   embedding_output

								if self.training:
									a_     			=   []
									for i in range(self.config.num_hidden_layers    ):
										a_     			=   self.encoder.adaptive_forward(
										    _SCREAMING_SNAKE_CASE			,       current_layer=_SCREAMING_SNAKE_CASE			,       attention_mask=_SCREAMING_SNAKE_CASE			,       head_mask=_SCREAMING_SNAKE_CASE			,       )

										a_     			=   self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]    )    )
										a_     			=   output_layers[i](output_dropout(_SCREAMING_SNAKE_CASE    )    )
										res.append(_SCREAMING_SNAKE_CASE    )
								elif self.patience == 0:  # Use all layers for inference
									a_     			=   self.encoder(_SCREAMING_SNAKE_CASE			,       _SCREAMING_SNAKE_CASE			,       head_mask=_SCREAMING_SNAKE_CASE    )
									a_     			=   self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]    )    )
									a_     			=   [output_layers[self.config.num_hidden_layers - 1](_SCREAMING_SNAKE_CASE    )]
								else:
									a_     			=   0
									a_     			=   None
									a_     			=   0
									for i in range(self.config.num_hidden_layers    ):
										calculated_layer_num += 1
										a_     			=   self.encoder.adaptive_forward(
										    _SCREAMING_SNAKE_CASE			,       current_layer=_SCREAMING_SNAKE_CASE			,       attention_mask=_SCREAMING_SNAKE_CASE			,       head_mask=_SCREAMING_SNAKE_CASE			,       )

										a_     			=   self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]    )    )
										a_     			=   output_layers[i](_SCREAMING_SNAKE_CASE    )
										if regression:
											a_     			=   logits.detach()
											if patient_result is not None:
												a_     			=   patient_result.detach()
											if (patient_result is not None) and torch.abs(patient_result - labels    ) < self.regression_threshold:
												patient_counter += 1
											else:
												a_     			=   0
										else:
											a_     			=   logits.detach().argmax(dim=1    )
											if patient_result is not None:
												a_     			=   patient_result.detach().argmax(dim=1    )
											if (patient_result is not None) and torch.all(labels.eq(_SCREAMING_SNAKE_CASE    )    ):
												patient_counter += 1
											else:
												a_     			=   0

										a_     			=   logits
										if patient_counter == self.patience:
											break
									a_     			=   [patient_result]
									self.inference_layers_num += calculated_layer_num
									self.inference_instances_num += 1

								return res







@add_start_docstrings(
    """Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. """   ,		_SCREAMING_SNAKE_CASE   ,		)
class 			lowerCamelCase_					(    _SCREAMING_SNAKE_CASE    ):





							def __init__(							self			,       _SCREAMING_SNAKE_CASE    ):
								super().__init__(_SCREAMING_SNAKE_CASE    )
								a_     			=   config.num_labels

								a_     			=   AlbertModelWithPabee(_SCREAMING_SNAKE_CASE    )
								a_     			=   nn.Dropout(config.classifier_dropout_prob    )
								a_     			=   nn.ModuleList(
								    [nn.Linear(config.hidden_size			,       self.config.num_labels    ) for _ in range(config.num_hidden_layers    )]    )

								self.init_weights()





							@add_start_docstrings_to_model_forward(_SCREAMING_SNAKE_CASE    )
							def   __magic_name__      (							self			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       _SCREAMING_SNAKE_CASE=None			,       ):
								a_     			=   self.albert(
								    input_ids=_SCREAMING_SNAKE_CASE			,       attention_mask=_SCREAMING_SNAKE_CASE			,       token_type_ids=_SCREAMING_SNAKE_CASE			,       position_ids=_SCREAMING_SNAKE_CASE			,       head_mask=_SCREAMING_SNAKE_CASE			,       inputs_embeds=_SCREAMING_SNAKE_CASE			,       output_dropout=self.dropout			,       output_layers=self.classifiers			,       regression=self.num_labels == 1			,       )

								a_     			=   (logits[-1],)

								if labels is not None:
									a_     			=   None
									a_     			=   0
									for ix, logits_item in enumerate(_SCREAMING_SNAKE_CASE    ):
										if self.num_labels == 1:
											#  We are doing regression
											a_     			=   MSELoss()
											a_     			=   loss_fct(logits_item.view(-1    )			,       labels.view(-1    )    )
										else:
											a_     			=   CrossEntropyLoss()
											a_     			=   loss_fct(logits_item.view(-1			,       self.num_labels    )			,       labels.view(-1    )    )
										if total_loss is None:
											a_     			=   loss
										else:
											total_loss += loss * (ix + 1)
										total_weights += ix + 1
									a_     			=   (total_loss / total_weights,) + outputs

								return outputs



from collections import deque

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      Dict	)     ->	Tuple:



	"""simple docstring"""






	return input(UpperCamelCase	).strip().split(""" """	)

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      int     ,					UpperCamelCase		:      int	)     ->	dict[int, list[int]]:



	"""simple docstring"""






	a_     			=   {}
	for i in range(UpperCamelCase	):
		a_     			=   []

	for e in range(UpperCamelCase	):
		a_			, a_     			=   (int(UpperCamelCase	) for i in _input(F"""Edge {e + 1}: <node1> <node2> """	))
		graph[x].append(UpperCamelCase	)
	return graph

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      int     ,					UpperCamelCase		:      int	)     ->	dict[int, list[int]]:



	"""simple docstring"""






	a_     			=   {}
	for i in range(UpperCamelCase	):
		a_     			=   []

	for e in range(UpperCamelCase	):
		a_			, a_     			=   (int(UpperCamelCase	) for i in _input(F"""Edge {e + 1}: <node1> <node2> """	))
		graph[x].append(UpperCamelCase	)
		graph[y].append(UpperCamelCase	)
	return graph

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      int     ,					UpperCamelCase		:      int	)     ->	dict[int, list[tuple[int, int]]]:



	"""simple docstring"""






	a_     			=   {}
	for i in range(UpperCamelCase	):
		a_     			=   []

	for e in range(UpperCamelCase	):
		a_			, a_			, a_     			=   (int(UpperCamelCase	) for i in _input(F"""Edge {e + 1}: <node1> <node2> <weight> """	))
		graph[x].append((y, w)	)
		graph[y].append((x, w)	)
	return graph


if __name__ == "__main__":
			_A     ,				_A							     =       (int(i) for i in _input('Number of nodes and edges: '))

			_A							     =       int(
			    _input(
			        'Press 1 or 2 or 3 \n'
			        '1. Unweighted directed \n'
			        '2. Unweighted undirected \n'
			        '3. Weighted undirected \n'
			    )[0]
			)

			_A							     =       {
			    1: initialize_unweighted_directed_graph,
			    2: initialize_unweighted_undirected_graph,
			    3: initialize_weighted_undirected_graph,
			}[graph_choice](n, m)

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      Tuple     ,					UpperCamelCase		:      Tuple	)     ->	List[Any]:



	"""simple docstring"""






	a_			, a_     			=   {s}, [s]
	print(UpperCamelCase	)
	while _s:
		a_     			=   0
		for i in g[_s[-1]]:
			if i not in vis:
				_s.append(UpperCamelCase	)
				vis.add(UpperCamelCase	)
				a_     			=   1
				print(UpperCamelCase	)
				break
		if not flag:
			_s.pop()

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      Tuple     ,					UpperCamelCase		:      int	)     ->	int:



	"""simple docstring"""






	a_			, a_     			=   {s}, deque([s]	)
	print(UpperCamelCase	)
	while q:
		a_     			=   q.popleft()
		for v in g[u]:
			if v not in vis:
				vis.add(UpperCamelCase	)
				q.append(UpperCamelCase	)
				print(UpperCamelCase	)

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      int     ,					UpperCamelCase		:      Any	)     ->	Union[str, Any]:



	"""simple docstring"""






	a_			, a_			, a_     			=   {s: 0}, set(), {s: 0}
	while True:
		if len(UpperCamelCase	) == len(UpperCamelCase	) - 1:
			break
		a_     			=   10_0000
		for i in dist:
			if i not in known and dist[i] < mini:
				a_     			=   dist[i]
				a_     			=   i
		known.add(UpperCamelCase	)
		for v in g[u]:
			if v[0] not in known and dist[u] + v[1] < dist.get(v[0]     ,					10_0000	):
				a_     			=   dist[u] + v[1]
				a_     			=   u
	for i in dist:
		if i != s:
			print(dist[i]	)

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      Dict     ,					UpperCamelCase		:      int=None     ,					UpperCamelCase		:      int=None	)     ->	Union[str, Any]:



	"""simple docstring"""






	if q is None:
		a_     			=   [1]
	if ind is None:
		a_     			=   [0] * (len(UpperCamelCase	) + 1)  # SInce oth Index is ignored
		for u in g:
			for v in g[u]:
				ind[v] += 1
		a_     			=   deque()
		for i in g:
			if ind[i] == 0:
				q.append(UpperCamelCase	)
	if len(UpperCamelCase	) == 0:
		return
	a_     			=   q.popleft()
	print(UpperCamelCase	)
	for w in g[v]:
		ind[w] -= 1
		if ind[w] == 0:
			q.append(UpperCamelCase	)
	topo(UpperCamelCase     ,					UpperCamelCase     ,					UpperCamelCase	)

def 			__SCREAMING_SNAKE_CASE		(		)     ->	List[Any]:



	"""simple docstring"""






	a_     			=   input().strip()
	a_     			=   []
	for _ in range(UpperCamelCase	):
		a.append(map(UpperCamelCase     ,					input().strip().split()	)	)
	return a, n

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      List[Any]	)     ->	List[str]:



	"""simple docstring"""






	((a_)			, (a_))     			=   a_and_n
	a_     			=   list(UpperCamelCase	)
	a_     			=   [[0] * n for i in range(UpperCamelCase	)]
	for k in range(UpperCamelCase	):
		for i in range(UpperCamelCase	):
			for j in range(UpperCamelCase	):
				if dist[i][j] > dist[i][k] + dist[k][j]:
					a_     			=   dist[i][k] + dist[k][j]
					a_     			=   k
	print(UpperCamelCase	)

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      Dict     ,					UpperCamelCase		:      List[Any]	)     ->	str:



	"""simple docstring"""






	a_			, a_			, a_     			=   {s: 0}, set(), {s: 0}
	while True:
		if len(UpperCamelCase	) == len(UpperCamelCase	) - 1:
			break
		a_     			=   10_0000
		for i in dist:
			if i not in known and dist[i] < mini:
				a_     			=   dist[i]
				a_     			=   i
		known.add(UpperCamelCase	)
		for v in g[u]:
			if v[0] not in known and v[1] < dist.get(v[0]     ,					10_0000	):
				a_     			=   v[1]
				a_     			=   u
	return dist

def 			__SCREAMING_SNAKE_CASE		(		)     ->	int:



	"""simple docstring"""






	a_			, a_     			=   map(UpperCamelCase     ,					input().split(""" """	)	)
	a_     			=   []
	for _ in range(UpperCamelCase	):
		edges.append(map(UpperCamelCase     ,					input().split(""" """	)	)	)
	return edges, n

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      List[str]	)     ->	List[str]:



	"""simple docstring"""






	((a_)			, (a_))     			=   e_and_n
	e.sort(reverse=UpperCamelCase     ,					key=lambda UpperCamelCase	: x[2]	)
	a_     			=   [{i} for i in range(1     ,					n + 1	)]
	while True:
		if len(UpperCamelCase	) == 1:
			break
		print(UpperCamelCase	)
		a_     			=   e.pop()
		for i in range(len(UpperCamelCase	)	):
			if x[0] in s[i]:
				break
		for j in range(len(UpperCamelCase	)	):
			if x[1] in s[j]:
				if i == j:
					break
				s[j].update(s[i]	)
				s.pop(UpperCamelCase	)
				break

def 			__SCREAMING_SNAKE_CASE		(		UpperCamelCase		:      str	)     ->	Optional[Any]:



	"""simple docstring"""






	a_     			=   []
	for node in graph:
		if not graph[node]:
			isolated.append(UpperCamelCase	)
	return isolated