Update modeling_phi3.py

modeling_phi3.py

@@ -262,8 +262,8 @@ class Phi3MLP(nn.Module):
         self.config = config
 
         self.gate = nn.Linear(config.hidden_size, self.config.num_experts, bias=False)
-        self.gate_up_proj = nn.ModuleList([nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False) for i in range(2)])
-        self.down_proj = nn.ModuleList([nn.Linear(config.intermediate_size, config.hidden_size, bias=False) for i in range(2)])
+        self.gate_up_proj = nn.ModuleList([nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False) for i in range(self.config.num_experts)])
+        self.down_proj = nn.ModuleList([nn.Linear(config.intermediate_size, config.hidden_size, bias=False) for i in range(self.config.num_experts)])
         self.activation_fn = ACT2FN[config.hidden_act]
 
     def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: