


'''simple docstring'''
import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig




def      __snake_case       (       lowerCamelCase_ :				List[Any]	,  lowerCamelCase_ :				List[Any]	,  lowerCamelCase_ :				int=0						):
				'''simple docstring'''




				if name is None:
								__magic_name__	  = None
				else:
								__magic_name__	  = "." * max(0	,  spaces - 2						) + "# {:" + str(50 - spaces						) + "s}"
								__magic_name__	  = fmt.format(lowerCamelCase_						)

				# Print and recurse (if needed).
				if isinstance(lowerCamelCase_	,  lowerCamelCase_						):
								if msg is not None:
												print(lowerCamelCase_						)
								for k in val.keys():
												recursive_print(lowerCamelCase_	,  val[k]	,  spaces + 2						)
				elif isinstance(lowerCamelCase_	,  torch.Tensor						):
								print(lowerCamelCase_	,  ":"	,  val.size()						)
				else:
								print(lowerCamelCase_	,  ":"	,  lowerCamelCase_						)




def      __snake_case       (       lowerCamelCase_ :				Optional[int]	,  lowerCamelCase_ :				Dict	,  lowerCamelCase_ :				int	,  lowerCamelCase_ :				Optional[Any]						):
				'''simple docstring'''




				__magic_name__	  = param.size()
				# other versions store [num_heads * num_splits * hidden_size, :]
				__magic_name__	  = (num_heads, num_splits, hidden_size) + input_shape[1:]
				__magic_name__	  = param.view(*lowerCamelCase_						)
				__magic_name__	  = param.transpose(0	,  1						).contiguous()
				__magic_name__	  = param.view(*lowerCamelCase_						)
				return param




def      __snake_case       (       lowerCamelCase_ :				Optional[Any]	,  lowerCamelCase_ :				List[str]						):
				'''simple docstring'''




				__magic_name__	  = config.n_positions
				__magic_name__	  = config.n_layer
				__magic_name__	  = config.vocab_size
				__magic_name__	  = config.n_head
				__magic_name__	  = config.n_embd // config.n_head

				__magic_name__	  = sd_megatron["model.language_model.embedding.word_embeddings.weight"][:vocab_size, :]
				__magic_name__	  = {
				    "transformer.wte.weight": word_embeddings,
				    "transformer.wpe.weight": sd_megatron["model.language_model.embedding.position_embeddings.weight"],
				    "transformer.ln_f.weight": sd_megatron["model.language_model.encoder.final_layernorm.weight"],
				    "transformer.ln_f.bias": sd_megatron["model.language_model.encoder.final_layernorm.bias"],
				}

				__magic_name__	  = "model.language_model.encoder.layers."
				for i in range(lowerCamelCase_						):
								__magic_name__	  = torch.tril(torch.ones((n_positions, n_positions)	,  dtype=torch.bool						)						)
								__magic_name__	  = causal_mask.view(1	,  1	,  lowerCamelCase_	,  lowerCamelCase_						)
								__magic_name__	  = causal_mask
								__magic_name__	  = torch.tensor(-1e4	,  dtype=torch.bfloataa						)

								__magic_name__	  = sd_megatron[F'{pf}{i}.input_layernorm.weight']
								__magic_name__	  = sd_megatron[F'{pf}{i}.input_layernorm.bias']

								__magic_name__	  = sd_megatron[F'{pf}{i}.self_attention.query_key_value.weight']
								__magic_name__	  = fix_query_key_value_ordering(lowerCamelCase_	,  3	,  lowerCamelCase_	,  lowerCamelCase_						)
								__magic_name__	  = vala.transpose(0	,  1						).contiguous()

								__magic_name__	  = sd_megatron[F'{pf}{i}.self_attention.query_key_value.bias']
								__magic_name__	  = fix_query_key_value_ordering(lowerCamelCase_	,  3	,  lowerCamelCase_	,  lowerCamelCase_						)
								__magic_name__	  = vala

								__magic_name__	  = sd_megatron[F'{pf}{i}.self_attention.dense.weight'].transpose(
								    0	,  1						)
								__magic_name__	  = sd_megatron[F'{pf}{i}.self_attention.dense.bias']
								__magic_name__	  = sd_megatron[F'{pf}{i}.post_attention_layernorm.weight']
								__magic_name__	  = sd_megatron[F'{pf}{i}.post_attention_layernorm.bias']
								__magic_name__	  = sd_megatron[F'{pf}{i}.mlp.dense_h_to_4h.weight'].transpose(0	,  1						)
								__magic_name__	  = sd_megatron[F'{pf}{i}.mlp.dense_h_to_4h.bias']
								__magic_name__	  = sd_megatron[F'{pf}{i}.mlp.dense_4h_to_h.weight'].transpose(
								    0	,  1						)
								__magic_name__	  = sd_megatron[F'{pf}{i}.mlp.dense_4h_to_h.bias']

				# For LM head, transformers' wants the matrix to weight embeddings.
				__magic_name__	  = word_embeddings

				return sd_hf




def      __snake_case       (       lowerCamelCase_ :				List[str]	,  lowerCamelCase_ :				List[str]						):
				'''simple docstring'''




				__magic_name__	  = 6_4000
				__magic_name__	  = config_megatron["encoder_seq_length"]
				__magic_name__	  = config_megatron["hidden_size"]
				__magic_name__	  = config_megatron["num_layers"]
				__magic_name__	  = config_megatron["num_attention_heads"]
				__magic_name__	  = config_megatron["ffn_hidden_size"]
				__magic_name__	  = "gelu"
				__magic_name__	  = 0.1
				__magic_name__	  = 0.1
				__magic_name__	  = 0.1
				__magic_name__	  = config_megatron["layernorm_epsilon"]  # 1e-5
				__magic_name__	  = config_megatron["init_method_std"]  # 0.02
				__magic_name__	  = config_megatron["apply_query_key_layer_scaling"]  # True
				__magic_name__	  = True
				__magic_name__	  = True

				# This identifies the 6.7B (7B) model which uses a different tokenizer
				if config_megatron["hidden_size"] == 4096:
								__magic_name__	  = 1  # <|endoftext|>
								__magic_name__	  = 1  # <|endoftext|>
								__magic_name__	  = 0  # <unk>
				else:
								__magic_name__	  = 2  # <s>
								__magic_name__	  = 3  # <|endoftext|>
								__magic_name__	  = 0  # <pad>

				return config_hf




def      __snake_case       (       lowerCamelCase_ :				Tuple						):
				'''simple docstring'''




				print(lowerCamelCase_						)

				__magic_name__	  = args.checkpoint_path
				__magic_name__	  = args.save_path
				if isfile(lowerCamelCase_						):
								raise FileNotFoundError(F'ERROR! could not find file {checkpoint_path}'						)

				# Load the model.
				__magic_name__	  = torch.load(lowerCamelCase_	,  map_location="cpu"						)

				# Load the config.
				__magic_name__	  = checkpoint["hyper_parameters"]["cfg"]
				__magic_name__	  = GPTaConfig()
				__magic_name__	  = copy_config(config_hf=lowerCamelCase_	,  config_megatron=lowerCamelCase_						)
				__magic_name__	  = ["GPT2LMHeadModel"]

				__magic_name__	  = checkpoint["state_dict"]

				# Convert.
				print("Converting"						)
				__magic_name__	  = convert_megatron_checkpoint(lowerCamelCase_	,  lowerCamelCase_						)

				# Print the structure of converted state dict.
				if args.print_checkpoint_structure:
								recursive_print(lowerCamelCase_	,  lowerCamelCase_						)

				__magic_name__	  = "GPTSw3Tokenizer"

				# Store the config to file.
				print("Saving config"						)
				config_hf.save_pretrained(lowerCamelCase_						)

				# Store the state_dict to file.
				__magic_name__	  = os.path.join(lowerCamelCase_	,  "pytorch_model.bin"						)
				print(F'Saving checkpoint to "{output_checkpoint_file}"'						)
				torch.save(lowerCamelCase_	,  lowerCamelCase_						)


if __name__ == "__main__":
						__magic_name__   :   Optional[int]  							=argparse.ArgumentParser()
						parser.add_argument(
						    '--checkpoint_path',
						    type=str,
						    required=True,
						    help='e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000',
						)
						parser.add_argument('--save_path', type=str, required=True, help='e.g. /home/user/gpt-sw3/hf')
						parser.add_argument('--print-checkpoint-structure', action='store_true')
						__magic_name__   :   Optional[Any]  							=parser.parse_args()
						main(_args)







'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__magic_name__   :   List[Any]  							={'configuration_yoso': ['YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP', 'YosoConfig']}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__magic_name__   :   Union[str, Any]  							=[
						    'YOSO_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'YosoForMaskedLM',
						    'YosoForMultipleChoice',
						    'YosoForQuestionAnswering',
						    'YosoForSequenceClassification',
						    'YosoForTokenClassification',
						    'YosoLayer',
						    'YosoModel',
						    'YosoPreTrainedModel',
						]


if TYPE_CHECKING:
						from .configuration_yoso import YOSO_PRETRAINED_CONFIG_ARCHIVE_MAP, YosoConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_yoso import (
												    YOSO_PRETRAINED_MODEL_ARCHIVE_LIST,
												    YosoForMaskedLM,
												    YosoForMultipleChoice,
												    YosoForQuestionAnswering,
												    YosoForSequenceClassification,
												    YosoForTokenClassification,
												    YosoLayer,
												    YosoModel,
												    YosoPreTrainedModel,
												)


else:
						import sys

						__magic_name__   :   Optional[int]  							=_LazyModule(__name__, globals()['__file__'], _import_structure)



