GazOmatic commited on
Commit
127f214
·
verified ·
1 Parent(s): cc74479
Files changed (1) hide show
  1. config.json +30 -0
config.json CHANGED
@@ -12,6 +12,36 @@
12
  "hidden_size": 1024,
13
  "initializer_range": 0.02,
14
  "intermediate_size": 3072,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
  "max_position_embeddings": 40960,
16
  "max_window_layers": 28,
17
  "model_type": "qwen3",
 
12
  "hidden_size": 1024,
13
  "initializer_range": 0.02,
14
  "intermediate_size": 3072,
15
+ "layer_types": [
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention",
41
+ "full_attention",
42
+ "full_attention",
43
+ "full_attention"
44
+ ],
45
  "max_position_embeddings": 40960,
46
  "max_window_layers": 28,
47
  "model_type": "qwen3",