

import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig


def       __a       (   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE=0      )      ->     Any:






    '''simple docstring'''
    # Format the message.
    if name is None:
        __UpperCAmelCase									=			None
    else:
        __UpperCAmelCase									=			'''.''' * max(0       ,   spaces - 2      ) + '''# {:''' + str(5_0 - spaces      ) + '''s}'''
        __UpperCAmelCase									=			fmt.format(SCREAMING_SNAKE_CASE      )

    # Print and recurse (if needed).
    if isinstance(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      ):
        if msg is not None:
            print(SCREAMING_SNAKE_CASE      )
        for k in val.keys():
            recursive_print(SCREAMING_SNAKE_CASE       ,   val[k]       ,   spaces + 2      )
    elif isinstance(SCREAMING_SNAKE_CASE       ,   torch.Tensor      ):
        print(SCREAMING_SNAKE_CASE       ,   ''':'''       ,   val.size()      )
    else:
        print(SCREAMING_SNAKE_CASE       ,   ''':'''       ,   SCREAMING_SNAKE_CASE      )


def       __a       (   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )      ->     Optional[Any]:






    '''simple docstring'''
    # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
    # for compatibility with later versions of NVIDIA Megatron-LM.
    # The inverse operation is performed inside Megatron-LM to read checkpoints:
    # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
    # If param is the weight tensor of the self-attention block, the returned tensor
    # will have to be transposed one more time to be read by HuggingFace GPT2.
    __UpperCAmelCase									=			param.size()
    # other versions store [num_heads * num_splits * hidden_size, :]
    __UpperCAmelCase									=			(num_heads, num_splits, hidden_size) + input_shape[1:]
    __UpperCAmelCase									=			param.view(*SCREAMING_SNAKE_CASE      )
    __UpperCAmelCase									=			param.transpose(0       ,   1      ).contiguous()
    __UpperCAmelCase									=			param.view(*SCREAMING_SNAKE_CASE      )
    return param


def       __a       (   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )      ->     Optional[Any]:






    '''simple docstring'''
    __UpperCAmelCase									=			config.n_positions
    __UpperCAmelCase									=			config.n_layer
    __UpperCAmelCase									=			config.vocab_size
    __UpperCAmelCase									=			config.n_head
    __UpperCAmelCase									=			config.n_embd // config.n_head

    __UpperCAmelCase									=			sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
    __UpperCAmelCase									=			{
        '''transformer.wte.weight''': word_embeddings,
        '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
        '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
        '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
    }

    __UpperCAmelCase									=			'''model.language_model.encoder.layers.'''
    for i in range(SCREAMING_SNAKE_CASE      ):
        __UpperCAmelCase									=			torch.tril(torch.ones((n_positions, n_positions)       ,   dtype=torch.bool      )      )
        __UpperCAmelCase									=			causal_mask.view(1       ,   1       ,   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )
        __UpperCAmelCase									=			causal_mask
        __UpperCAmelCase									=			torch.tensor(-1e4       ,   dtype=torch.bfloataa      )

        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.input_layernorm.weight''']
        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.input_layernorm.bias''']

        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.self_attention.query_key_value.weight''']
        __UpperCAmelCase									=			fix_query_key_value_ordering(SCREAMING_SNAKE_CASE       ,   3       ,   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )
        __UpperCAmelCase									=			vala.transpose(0       ,   1      ).contiguous()

        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.self_attention.query_key_value.bias''']
        __UpperCAmelCase									=			fix_query_key_value_ordering(SCREAMING_SNAKE_CASE       ,   3       ,   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )
        __UpperCAmelCase									=			vala

        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.self_attention.dense.weight'''].transpose(
            0       ,   1      )
        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.self_attention.dense.bias''']
        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.post_attention_layernorm.weight''']
        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.post_attention_layernorm.bias''']
        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0       ,   1      )
        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.mlp.dense_h_to_4h.bias''']
        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
            0       ,   1      )
        __UpperCAmelCase									=			sd_megatron[f'''{pf}{i}.mlp.dense_4h_to_h.bias''']

    # For LM head, transformers' wants the matrix to weight embeddings.
    __UpperCAmelCase									=			word_embeddings

    return sd_hf


def       __a       (   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )      ->     List[Any]:






    '''simple docstring'''
    __UpperCAmelCase									=			6_4_0_0_0
    __UpperCAmelCase									=			config_megatron['''encoder_seq_length''']
    __UpperCAmelCase									=			config_megatron['''hidden_size''']
    __UpperCAmelCase									=			config_megatron['''num_layers''']
    __UpperCAmelCase									=			config_megatron['''num_attention_heads''']
    __UpperCAmelCase									=			config_megatron['''ffn_hidden_size''']
    __UpperCAmelCase									=			'''gelu'''
    __UpperCAmelCase									=			0.1
    __UpperCAmelCase									=			0.1
    __UpperCAmelCase									=			0.1
    __UpperCAmelCase									=			config_megatron['''layernorm_epsilon''']  # 1e-5
    __UpperCAmelCase									=			config_megatron['''init_method_std''']  # 0.02
    __UpperCAmelCase									=			config_megatron['''apply_query_key_layer_scaling''']  # True
    __UpperCAmelCase									=			True
    __UpperCAmelCase									=			True

    # This identifies the 6.7B (7B) model which uses a different tokenizer
    if config_megatron["hidden_size"] == 4_0_9_6:
        __UpperCAmelCase									=			1  # <|endoftext|>
        __UpperCAmelCase									=			1  # <|endoftext|>
        __UpperCAmelCase									=			0  # <unk>
    else:
        __UpperCAmelCase									=			2  # <s>
        __UpperCAmelCase									=			3  # <|endoftext|>
        __UpperCAmelCase									=			0  # <pad>

    return config_hf


def       __a       (   SCREAMING_SNAKE_CASE      )      ->     Union[str, Any]:






    '''simple docstring'''
    print(SCREAMING_SNAKE_CASE      )

    __UpperCAmelCase									=			args.checkpoint_path
    __UpperCAmelCase									=			args.save_path
    if isfile(SCREAMING_SNAKE_CASE      ):
        raise FileNotFoundError(f'''ERROR! could not find file {checkpoint_path}'''      )

    # Load the model.
    __UpperCAmelCase									=			torch.load(SCREAMING_SNAKE_CASE       ,   map_location='''cpu'''      )

    # Load the config.
    __UpperCAmelCase									=			checkpoint['''hyper_parameters''']['''cfg''']
    __UpperCAmelCase									=			GPTaConfig()
    __UpperCAmelCase									=			copy_config(config_hf=SCREAMING_SNAKE_CASE       ,   config_megatron=SCREAMING_SNAKE_CASE      )
    __UpperCAmelCase									=			['''GPT2LMHeadModel''']

    __UpperCAmelCase									=			checkpoint['''state_dict''']

    # Convert.
    print('''Converting'''      )
    __UpperCAmelCase									=			convert_megatron_checkpoint(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )

    # Print the structure of converted state dict.
    if args.print_checkpoint_structure:
        recursive_print(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )

    __UpperCAmelCase									=			'''GPTSw3Tokenizer'''

    # Store the config to file.
    print('''Saving config'''      )
    config_hf.save_pretrained(SCREAMING_SNAKE_CASE      )

    # Store the state_dict to file.
    __UpperCAmelCase									=			os.path.join(SCREAMING_SNAKE_CASE       ,   '''pytorch_model.bin'''      )
    print(f'''Saving checkpoint to "{output_checkpoint_file}"'''      )
    torch.save(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )


if __name__ == "__main__":
       A_		:			str										=   argparse.ArgumentParser()
       parser.add_argument(
           '--checkpoint_path',
           type=str,
           required=True,
           help='e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000',
       )
       parser.add_argument('--save_path', type=str, required=True, help='e.g. /home/user/gpt-sw3/hf')
       parser.add_argument('--print-checkpoint-structure', action='store_true')
       A_		:			List[str]										=   parser.parse_args()
       main(_args)







from __future__ import annotations

import unittest

from transformers import MPNetConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers.models.mpnet.modeling_tf_mpnet import (
           TFMPNetForMaskedLM,
           TFMPNetForMultipleChoice,
           TFMPNetForQuestionAnswering,
           TFMPNetForSequenceClassification,
           TFMPNetForTokenClassification,
           TFMPNetModel,
       )
class 					A_   :


    '''simple docstring'''






    def __init__(self     ,      lowercase__     ,      lowercase__=13     ,      lowercase__=7     ,      lowercase__=True     ,      lowercase__=True     ,      lowercase__=False     ,      lowercase__=True     ,      lowercase__=99     ,      lowercase__=64     ,      lowercase__=2     ,      lowercase__=4     ,      lowercase__=64     ,      lowercase__="gelu"     ,      lowercase__=0.1     ,      lowercase__=0.1     ,      lowercase__=512     ,      lowercase__=16     ,      lowercase__=2     ,      lowercase__=0.02     ,      lowercase__=3     ,      lowercase__=4     ,      lowercase__=None     ,      )     ->      Tuple:
        __UpperCAmelCase									=			parent
        __UpperCAmelCase									=			batch_size
        __UpperCAmelCase									=			seq_length
        __UpperCAmelCase									=			is_training
        __UpperCAmelCase									=			use_input_mask
        __UpperCAmelCase									=			use_token_type_ids
        __UpperCAmelCase									=			use_labels
        __UpperCAmelCase									=			vocab_size
        __UpperCAmelCase									=			hidden_size
        __UpperCAmelCase									=			num_hidden_layers
        __UpperCAmelCase									=			num_attention_heads
        __UpperCAmelCase									=			intermediate_size
        __UpperCAmelCase									=			hidden_act
        __UpperCAmelCase									=			hidden_dropout_prob
        __UpperCAmelCase									=			attention_probs_dropout_prob
        __UpperCAmelCase									=			max_position_embeddings
        __UpperCAmelCase									=			type_vocab_size
        __UpperCAmelCase									=			type_sequence_label_size
        __UpperCAmelCase									=			initializer_range
        __UpperCAmelCase									=			num_labels
        __UpperCAmelCase									=			num_choices
        __UpperCAmelCase									=			scope


    def      lowerCAmelCase_						(self  )     ->      str:
        __UpperCAmelCase									=			ids_tensor([self.batch_size, self.seq_length]     ,      self.vocab_size  )

        __UpperCAmelCase									=			None
        if self.use_input_mask:
            __UpperCAmelCase									=			random_attention_mask([self.batch_size, self.seq_length]  )

        __UpperCAmelCase									=			None
        __UpperCAmelCase									=			None
        __UpperCAmelCase									=			None
        if self.use_labels:
            __UpperCAmelCase									=			ids_tensor([self.batch_size]     ,      self.type_sequence_label_size  )
            __UpperCAmelCase									=			ids_tensor([self.batch_size, self.seq_length]     ,      self.num_labels  )
            __UpperCAmelCase									=			ids_tensor([self.batch_size]     ,      self.num_choices  )

        __UpperCAmelCase									=			MPNetConfig(
            vocab_size=self.vocab_size     ,      hidden_size=self.hidden_size     ,      num_hidden_layers=self.num_hidden_layers     ,      num_attention_heads=self.num_attention_heads     ,      intermediate_size=self.intermediate_size     ,      hidden_act=self.hidden_act     ,      hidden_dropout_prob=self.hidden_dropout_prob     ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,      max_position_embeddings=self.max_position_embeddings     ,      initializer_range=self.initializer_range     ,      )
        return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__  )     ->      List[Any]:
        __UpperCAmelCase									=			TFMPNetModel(config=lowercase__  )
        __UpperCAmelCase									=			{'''input_ids''': input_ids, '''attention_mask''': input_mask}
        __UpperCAmelCase									=			model(lowercase__  )
        __UpperCAmelCase									=			[input_ids, input_mask]
        __UpperCAmelCase									=			model(lowercase__  )
        self.parent.assertEqual(result.last_hidden_state.shape     ,      (self.batch_size, self.seq_length, self.hidden_size)  )


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__  )     ->      Dict:
        __UpperCAmelCase									=			TFMPNetForMaskedLM(config=lowercase__  )
        __UpperCAmelCase									=			{'''input_ids''': input_ids, '''attention_mask''': input_mask}
        __UpperCAmelCase									=			model(lowercase__  )
        self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.vocab_size)  )


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__  )     ->      List[Any]:
        __UpperCAmelCase									=			TFMPNetForQuestionAnswering(config=lowercase__  )
        __UpperCAmelCase									=			{
            '''input_ids''': input_ids,
            '''attention_mask''': input_mask,
        }
        __UpperCAmelCase									=			model(lowercase__  )
        self.parent.assertEqual(result.start_logits.shape     ,      (self.batch_size, self.seq_length)  )
        self.parent.assertEqual(result.end_logits.shape     ,      (self.batch_size, self.seq_length)  )


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__  )     ->      str:
        __UpperCAmelCase									=			self.num_labels
        __UpperCAmelCase									=			TFMPNetForSequenceClassification(lowercase__  )
        __UpperCAmelCase									=			{'''input_ids''': input_ids, '''attention_mask''': input_mask}
        __UpperCAmelCase									=			model(lowercase__  )
        self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.num_labels)  )


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__  )     ->      str:
        __UpperCAmelCase									=			self.num_choices
        __UpperCAmelCase									=			TFMPNetForMultipleChoice(lowercase__  )
        __UpperCAmelCase									=			tf.tile(tf.expand_dims(lowercase__     ,      1  )     ,      (1, self.num_choices, 1)  )
        __UpperCAmelCase									=			tf.tile(tf.expand_dims(lowercase__     ,      1  )     ,      (1, self.num_choices, 1)  )
        __UpperCAmelCase									=			{
            '''input_ids''': multiple_choice_inputs_ids,
            '''attention_mask''': multiple_choice_input_mask,
        }
        __UpperCAmelCase									=			model(lowercase__  )
        self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.num_choices)  )


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__  )     ->      List[str]:
        __UpperCAmelCase									=			self.num_labels
        __UpperCAmelCase									=			TFMPNetForTokenClassification(lowercase__  )
        __UpperCAmelCase									=			{'''input_ids''': input_ids, '''attention_mask''': input_mask}
        __UpperCAmelCase									=			model(lowercase__  )
        self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.num_labels)  )






    def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
        __UpperCAmelCase									=			self.prepare_config_and_inputs()
        ((__UpperCAmelCase)					,     (__UpperCAmelCase)					,     (__UpperCAmelCase)					,     (__UpperCAmelCase)					,     (__UpperCAmelCase)					,     (__UpperCAmelCase))									=			config_and_inputs
        __UpperCAmelCase									=			{'''input_ids''': input_ids, '''attention_mask''': input_mask}
        return config, inputs_dict






@require_tf
class 					A_   (      _a ,     _a ,     unittest.TestCase						):


    '''simple docstring'''






    a__     							=       (
        (
            TFMPNetForMaskedLM,
            TFMPNetForMultipleChoice,
            TFMPNetForQuestionAnswering,
            TFMPNetForSequenceClassification,
            TFMPNetForTokenClassification,
            TFMPNetModel,
        )
        if is_tf_available()
        else ()
    )
    a__     							=       (
        {
            "feature-extraction": TFMPNetModel,
            "fill-mask": TFMPNetForMaskedLM,
            "question-answering": TFMPNetForQuestionAnswering,
            "text-classification": TFMPNetForSequenceClassification,
            "token-classification": TFMPNetForTokenClassification,
            "zero-shot": TFMPNetForSequenceClassification,
        }
        if is_tf_available()
        else {}
    )
    a__     							=       False
    a__     							=       False


    def      lowerCAmelCase_						(self  )     ->      int:
        __UpperCAmelCase									=			TFMPNetModelTester(self  )
        __UpperCAmelCase									=			ConfigTester(self     ,      config_class=lowercase__     ,      hidden_size=37  )


    def      lowerCAmelCase_						(self  )     ->      Optional[int]:
        self.config_tester.run_common_tests()


    def      lowerCAmelCase_						(self  )     ->      Dict:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mpnet_model(*lowercase__  )


    def      lowerCAmelCase_						(self  )     ->      Any:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mpnet_for_masked_lm(*lowercase__  )


    def      lowerCAmelCase_						(self  )     ->      int:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mpnet_for_question_answering(*lowercase__  )


    def      lowerCAmelCase_						(self  )     ->      Optional[int]:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mpnet_for_sequence_classification(*lowercase__  )


    def      lowerCAmelCase_						(self  )     ->      str:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mpnet_for_multiple_choice(*lowercase__  )


    def      lowerCAmelCase_						(self  )     ->      str:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_mpnet_for_token_classification(*lowercase__  )






    @slow
    def      lowerCAmelCase_						(self  )     ->      List[str]:
        for model_name in ["microsoft/mpnet-base"]:
            __UpperCAmelCase									=			TFMPNetModel.from_pretrained(lowercase__  )
            self.assertIsNotNone(lowercase__  )






@require_tf
class 					A_   (      unittest.TestCase						):


    '''simple docstring'''






    @slow
    def      lowerCAmelCase_						(self  )     ->      Any:
        __UpperCAmelCase									=			TFMPNetModel.from_pretrained('''microsoft/mpnet-base'''  )
        __UpperCAmelCase									=			tf.constant([[0, 1, 2, 3, 4, 5]]  )
        __UpperCAmelCase									=			model(lowercase__  )[0]

        __UpperCAmelCase									=			[1, 6, 768]
        self.assertEqual(output.shape     ,      lowercase__  )

        __UpperCAmelCase									=			tf.constant(
            [
                [
                    [-0.1067172, 0.08216473, 0.0024543],
                    [-0.03465879, 0.8354118, -0.03252288],
                    [-0.06569476, -0.12424111, -0.0494436],
                ]
            ]  )
        tf.debugging.assert_near(output[:, :3, :3]     ,      lowercase__     ,      atol=1E-4  )




