Hugging Face's logo

Tensoic
/
DeepSeek-Mini

Text Generation
Transformers
Safetensors
deepseek_v3
conversational
custom_code
fp8
Model card Files Community

Model Architecture:

Total Trainable Params: 2189403864 

DeepseekV3ForCausalLM(
  (model): DeepseekV3Model(
    (embed_tokens): Embedding(129280, 2048)
    (layers): ModuleList(
      (0-2): 3 x DeepseekV3DecoderLayer(
        (self_attn): DeepseekV3Attention(
          (q_a_proj): Linear(in_features=2048, out_features=512, bias=False)
          (q_a_layernorm): DeepseekV3RMSNorm()
          (q_b_proj): Linear(in_features=512, out_features=3072, bias=False)
          (kv_a_proj_with_mqa): Linear(in_features=2048, out_features=192, bias=False)
          (kv_a_layernorm): DeepseekV3RMSNorm()
          (kv_b_proj): Linear(in_features=128, out_features=4096, bias=False)
          (o_proj): Linear(in_features=2048, out_features=2048, bias=False)
          (rotary_emb): DeepseekV3YarnRotaryEmbedding()
        )
        (mlp): DeepseekV3MLP(
          (gate_proj): Linear(in_features=2048, out_features=8192, bias=False)
          (up_proj): Linear(in_features=2048, out_features=8192, bias=False)
          (down_proj): Linear(in_features=8192, out_features=2048, bias=False)
          (act_fn): SiLU()
        )
        (input_layernorm): DeepseekV3RMSNorm()
        (post_attention_layernorm): DeepseekV3RMSNorm()
      )
      (3-11): 9 x DeepseekV3DecoderLayer(
        (self_attn): DeepseekV3Attention(
          (q_a_proj): Linear(in_features=2048, out_features=512, bias=False)
          (q_a_layernorm): DeepseekV3RMSNorm()
          (q_b_proj): Linear(in_features=512, out_features=3072, bias=False)
          (kv_a_proj_with_mqa): Linear(in_features=2048, out_features=192, bias=False)
          (kv_a_layernorm): DeepseekV3RMSNorm()
          (kv_b_proj): Linear(in_features=128, out_features=4096, bias=False)
          (o_proj): Linear(in_features=2048, out_features=2048, bias=False)
          (rotary_emb): DeepseekV3YarnRotaryEmbedding()
        )
        (mlp): DeepseekV3MoE(
          (experts): ModuleList(
            (0-23): 24 x DeepseekV3MLP(
              (gate_proj): Linear(in_features=2048, out_features=1024, bias=False)
              (up_proj): Linear(in_features=2048, out_features=1024, bias=False)
              (down_proj): Linear(in_features=1024, out_features=2048, bias=False)
              (act_fn): SiLU()
            )
          )
          (gate): MoEGate()
          (shared_experts): DeepseekV3MLP(
            (gate_proj): Linear(in_features=2048, out_features=1024, bias=False)
            (up_proj): Linear(in_features=2048, out_features=1024, bias=False)
            (down_proj): Linear(in_features=1024, out_features=2048, bias=False)
            (act_fn): SiLU()
          )
        )
        (input_layernorm): DeepseekV3RMSNorm()
        (post_attention_layernorm): DeepseekV3RMSNorm()
      )
    )
    (norm): DeepseekV3RMSNorm()
  )
  (lm_head): Linear(in_features=2048, out_features=129280, bias=False)
)

+-----------------------------------------------------+------------+
|                       Modules                       | Parameters |
+-----------------------------------------------------+------------+
|              model.embed_tokens.weight              | 264765440  |
|       model.layers.0.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.0.self_attn.q_a_layernorm.weight    |    512     |
|       model.layers.0.self_attn.q_b_proj.weight      |  1572864   |
|  model.layers.0.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|    model.layers.0.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.0.self_attn.kv_b_proj.weight      |   524288   |
|        model.layers.0.self_attn.o_proj.weight       |  4194304   |
|         model.layers.0.mlp.gate_proj.weight         |  16777216  |
|          model.layers.0.mlp.up_proj.weight          |  16777216  |
|         model.layers.0.mlp.down_proj.weight         |  16777216  |
|        model.layers.0.input_layernorm.weight        |    2048    |
|    model.layers.0.post_attention_layernorm.weight   |    2048    |
|       model.layers.1.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.1.self_attn.q_a_layernorm.weight    |    512     |
|       model.layers.1.self_attn.q_b_proj.weight      |  1572864   |
|  model.layers.1.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|    model.layers.1.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.1.self_attn.kv_b_proj.weight      |   524288   |
|        model.layers.1.self_attn.o_proj.weight       |  4194304   |
|         model.layers.1.mlp.gate_proj.weight         |  16777216  |
|          model.layers.1.mlp.up_proj.weight          |  16777216  |
|         model.layers.1.mlp.down_proj.weight         |  16777216  |
|        model.layers.1.input_layernorm.weight        |    2048    |
|    model.layers.1.post_attention_layernorm.weight   |    2048    |
|       model.layers.2.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.2.self_attn.q_a_layernorm.weight    |    512     |
|       model.layers.2.self_attn.q_b_proj.weight      |  1572864   |
|  model.layers.2.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|    model.layers.2.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.2.self_attn.kv_b_proj.weight      |   524288   |
|        model.layers.2.self_attn.o_proj.weight       |  4194304   |
|         model.layers.2.mlp.gate_proj.weight         |  16777216  |
|          model.layers.2.mlp.up_proj.weight          |  16777216  |
|         model.layers.2.mlp.down_proj.weight         |  16777216  |
|        model.layers.2.input_layernorm.weight        |    2048    |
|    model.layers.2.post_attention_layernorm.weight   |    2048    |
|       model.layers.3.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.3.self_attn.q_a_layernorm.weight    |    512     |
|       model.layers.3.self_attn.q_b_proj.weight      |  1572864   |
|  model.layers.3.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|    model.layers.3.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.3.self_attn.kv_b_proj.weight      |   524288   |
|        model.layers.3.self_attn.o_proj.weight       |  4194304   |
|    model.layers.3.mlp.experts.0.gate_proj.weight    |  2097152   |
|     model.layers.3.mlp.experts.0.up_proj.weight     |  2097152   |
|    model.layers.3.mlp.experts.0.down_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.1.gate_proj.weight    |  2097152   |
|     model.layers.3.mlp.experts.1.up_proj.weight     |  2097152   |
|    model.layers.3.mlp.experts.1.down_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.2.gate_proj.weight    |  2097152   |
|     model.layers.3.mlp.experts.2.up_proj.weight     |  2097152   |
|    model.layers.3.mlp.experts.2.down_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.3.gate_proj.weight    |  2097152   |
|     model.layers.3.mlp.experts.3.up_proj.weight     |  2097152   |
|    model.layers.3.mlp.experts.3.down_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.4.gate_proj.weight    |  2097152   |
|     model.layers.3.mlp.experts.4.up_proj.weight     |  2097152   |
|    model.layers.3.mlp.experts.4.down_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.5.gate_proj.weight    |  2097152   |
|     model.layers.3.mlp.experts.5.up_proj.weight     |  2097152   |
|    model.layers.3.mlp.experts.5.down_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.6.gate_proj.weight    |  2097152   |
|     model.layers.3.mlp.experts.6.up_proj.weight     |  2097152   |
|    model.layers.3.mlp.experts.6.down_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.7.gate_proj.weight    |  2097152   |
|     model.layers.3.mlp.experts.7.up_proj.weight     |  2097152   |
|    model.layers.3.mlp.experts.7.down_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.8.gate_proj.weight    |  2097152   |
|     model.layers.3.mlp.experts.8.up_proj.weight     |  2097152   |
|    model.layers.3.mlp.experts.8.down_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.9.gate_proj.weight    |  2097152   |
|     model.layers.3.mlp.experts.9.up_proj.weight     |  2097152   |
|    model.layers.3.mlp.experts.9.down_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.10.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.10.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.10.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.11.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.11.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.11.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.12.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.12.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.12.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.13.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.13.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.13.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.14.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.14.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.14.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.15.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.15.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.15.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.16.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.16.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.16.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.17.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.17.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.17.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.18.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.18.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.18.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.19.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.19.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.19.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.20.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.20.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.20.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.21.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.21.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.21.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.22.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.22.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.22.down_proj.weight   |  2097152   |
|    model.layers.3.mlp.experts.23.gate_proj.weight   |  2097152   |
|     model.layers.3.mlp.experts.23.up_proj.weight    |  2097152   |
|    model.layers.3.mlp.experts.23.down_proj.weight   |  2097152   |
|            model.layers.3.mlp.gate.weight           |   49152    |
|   model.layers.3.mlp.gate.e_score_correction_bias   |     24     |
|  model.layers.3.mlp.shared_experts.gate_proj.weight |  2097152   |
|   model.layers.3.mlp.shared_experts.up_proj.weight  |  2097152   |
|  model.layers.3.mlp.shared_experts.down_proj.weight |  2097152   |
|        model.layers.3.input_layernorm.weight        |    2048    |
|    model.layers.3.post_attention_layernorm.weight   |    2048    |
|       model.layers.4.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.4.self_attn.q_a_layernorm.weight    |    512     |
|       model.layers.4.self_attn.q_b_proj.weight      |  1572864   |
|  model.layers.4.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|    model.layers.4.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.4.self_attn.kv_b_proj.weight      |   524288   |
|        model.layers.4.self_attn.o_proj.weight       |  4194304   |
|    model.layers.4.mlp.experts.0.gate_proj.weight    |  2097152   |
|     model.layers.4.mlp.experts.0.up_proj.weight     |  2097152   |
|    model.layers.4.mlp.experts.0.down_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.1.gate_proj.weight    |  2097152   |
|     model.layers.4.mlp.experts.1.up_proj.weight     |  2097152   |
|    model.layers.4.mlp.experts.1.down_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.2.gate_proj.weight    |  2097152   |
|     model.layers.4.mlp.experts.2.up_proj.weight     |  2097152   |
|    model.layers.4.mlp.experts.2.down_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.3.gate_proj.weight    |  2097152   |
|     model.layers.4.mlp.experts.3.up_proj.weight     |  2097152   |
|    model.layers.4.mlp.experts.3.down_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.4.gate_proj.weight    |  2097152   |
|     model.layers.4.mlp.experts.4.up_proj.weight     |  2097152   |
|    model.layers.4.mlp.experts.4.down_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.5.gate_proj.weight    |  2097152   |
|     model.layers.4.mlp.experts.5.up_proj.weight     |  2097152   |
|    model.layers.4.mlp.experts.5.down_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.6.gate_proj.weight    |  2097152   |
|     model.layers.4.mlp.experts.6.up_proj.weight     |  2097152   |
|    model.layers.4.mlp.experts.6.down_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.7.gate_proj.weight    |  2097152   |
|     model.layers.4.mlp.experts.7.up_proj.weight     |  2097152   |
|    model.layers.4.mlp.experts.7.down_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.8.gate_proj.weight    |  2097152   |
|     model.layers.4.mlp.experts.8.up_proj.weight     |  2097152   |
|    model.layers.4.mlp.experts.8.down_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.9.gate_proj.weight    |  2097152   |
|     model.layers.4.mlp.experts.9.up_proj.weight     |  2097152   |
|    model.layers.4.mlp.experts.9.down_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.10.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.10.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.10.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.11.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.11.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.11.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.12.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.12.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.12.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.13.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.13.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.13.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.14.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.14.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.14.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.15.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.15.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.15.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.16.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.16.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.16.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.17.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.17.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.17.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.18.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.18.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.18.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.19.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.19.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.19.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.20.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.20.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.20.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.21.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.21.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.21.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.22.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.22.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.22.down_proj.weight   |  2097152   |
|    model.layers.4.mlp.experts.23.gate_proj.weight   |  2097152   |
|     model.layers.4.mlp.experts.23.up_proj.weight    |  2097152   |
|    model.layers.4.mlp.experts.23.down_proj.weight   |  2097152   |
|            model.layers.4.mlp.gate.weight           |   49152    |
|   model.layers.4.mlp.gate.e_score_correction_bias   |     24     |
|  model.layers.4.mlp.shared_experts.gate_proj.weight |  2097152   |
|   model.layers.4.mlp.shared_experts.up_proj.weight  |  2097152   |
|  model.layers.4.mlp.shared_experts.down_proj.weight |  2097152   |
|        model.layers.4.input_layernorm.weight        |    2048    |
|    model.layers.4.post_attention_layernorm.weight   |    2048    |
|       model.layers.5.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.5.self_attn.q_a_layernorm.weight    |    512     |
|       model.layers.5.self_attn.q_b_proj.weight      |  1572864   |
|  model.layers.5.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|    model.layers.5.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.5.self_attn.kv_b_proj.weight      |   524288   |
|        model.layers.5.self_attn.o_proj.weight       |  4194304   |
|    model.layers.5.mlp.experts.0.gate_proj.weight    |  2097152   |
|     model.layers.5.mlp.experts.0.up_proj.weight     |  2097152   |
|    model.layers.5.mlp.experts.0.down_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.1.gate_proj.weight    |  2097152   |
|     model.layers.5.mlp.experts.1.up_proj.weight     |  2097152   |
|    model.layers.5.mlp.experts.1.down_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.2.gate_proj.weight    |  2097152   |
|     model.layers.5.mlp.experts.2.up_proj.weight     |  2097152   |
|    model.layers.5.mlp.experts.2.down_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.3.gate_proj.weight    |  2097152   |
|     model.layers.5.mlp.experts.3.up_proj.weight     |  2097152   |
|    model.layers.5.mlp.experts.3.down_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.4.gate_proj.weight    |  2097152   |
|     model.layers.5.mlp.experts.4.up_proj.weight     |  2097152   |
|    model.layers.5.mlp.experts.4.down_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.5.gate_proj.weight    |  2097152   |
|     model.layers.5.mlp.experts.5.up_proj.weight     |  2097152   |
|    model.layers.5.mlp.experts.5.down_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.6.gate_proj.weight    |  2097152   |
|     model.layers.5.mlp.experts.6.up_proj.weight     |  2097152   |
|    model.layers.5.mlp.experts.6.down_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.7.gate_proj.weight    |  2097152   |
|     model.layers.5.mlp.experts.7.up_proj.weight     |  2097152   |
|    model.layers.5.mlp.experts.7.down_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.8.gate_proj.weight    |  2097152   |
|     model.layers.5.mlp.experts.8.up_proj.weight     |  2097152   |
|    model.layers.5.mlp.experts.8.down_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.9.gate_proj.weight    |  2097152   |
|     model.layers.5.mlp.experts.9.up_proj.weight     |  2097152   |
|    model.layers.5.mlp.experts.9.down_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.10.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.10.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.10.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.11.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.11.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.11.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.12.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.12.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.12.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.13.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.13.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.13.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.14.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.14.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.14.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.15.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.15.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.15.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.16.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.16.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.16.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.17.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.17.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.17.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.18.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.18.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.18.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.19.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.19.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.19.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.20.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.20.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.20.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.21.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.21.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.21.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.22.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.22.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.22.down_proj.weight   |  2097152   |
|    model.layers.5.mlp.experts.23.gate_proj.weight   |  2097152   |
|     model.layers.5.mlp.experts.23.up_proj.weight    |  2097152   |
|    model.layers.5.mlp.experts.23.down_proj.weight   |  2097152   |
|            model.layers.5.mlp.gate.weight           |   49152    |
|   model.layers.5.mlp.gate.e_score_correction_bias   |     24     |
|  model.layers.5.mlp.shared_experts.gate_proj.weight |  2097152   |
|   model.layers.5.mlp.shared_experts.up_proj.weight  |  2097152   |
|  model.layers.5.mlp.shared_experts.down_proj.weight |  2097152   |
|        model.layers.5.input_layernorm.weight        |    2048    |
|    model.layers.5.post_attention_layernorm.weight   |    2048    |
|       model.layers.6.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.6.self_attn.q_a_layernorm.weight    |    512     |
|       model.layers.6.self_attn.q_b_proj.weight      |  1572864   |
|  model.layers.6.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|    model.layers.6.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.6.self_attn.kv_b_proj.weight      |   524288   |
|        model.layers.6.self_attn.o_proj.weight       |  4194304   |
|    model.layers.6.mlp.experts.0.gate_proj.weight    |  2097152   |
|     model.layers.6.mlp.experts.0.up_proj.weight     |  2097152   |
|    model.layers.6.mlp.experts.0.down_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.1.gate_proj.weight    |  2097152   |
|     model.layers.6.mlp.experts.1.up_proj.weight     |  2097152   |
|    model.layers.6.mlp.experts.1.down_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.2.gate_proj.weight    |  2097152   |
|     model.layers.6.mlp.experts.2.up_proj.weight     |  2097152   |
|    model.layers.6.mlp.experts.2.down_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.3.gate_proj.weight    |  2097152   |
|     model.layers.6.mlp.experts.3.up_proj.weight     |  2097152   |
|    model.layers.6.mlp.experts.3.down_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.4.gate_proj.weight    |  2097152   |
|     model.layers.6.mlp.experts.4.up_proj.weight     |  2097152   |
|    model.layers.6.mlp.experts.4.down_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.5.gate_proj.weight    |  2097152   |
|     model.layers.6.mlp.experts.5.up_proj.weight     |  2097152   |
|    model.layers.6.mlp.experts.5.down_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.6.gate_proj.weight    |  2097152   |
|     model.layers.6.mlp.experts.6.up_proj.weight     |  2097152   |
|    model.layers.6.mlp.experts.6.down_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.7.gate_proj.weight    |  2097152   |
|     model.layers.6.mlp.experts.7.up_proj.weight     |  2097152   |
|    model.layers.6.mlp.experts.7.down_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.8.gate_proj.weight    |  2097152   |
|     model.layers.6.mlp.experts.8.up_proj.weight     |  2097152   |
|    model.layers.6.mlp.experts.8.down_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.9.gate_proj.weight    |  2097152   |
|     model.layers.6.mlp.experts.9.up_proj.weight     |  2097152   |
|    model.layers.6.mlp.experts.9.down_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.10.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.10.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.10.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.11.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.11.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.11.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.12.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.12.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.12.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.13.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.13.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.13.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.14.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.14.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.14.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.15.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.15.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.15.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.16.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.16.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.16.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.17.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.17.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.17.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.18.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.18.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.18.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.19.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.19.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.19.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.20.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.20.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.20.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.21.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.21.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.21.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.22.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.22.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.22.down_proj.weight   |  2097152   |
|    model.layers.6.mlp.experts.23.gate_proj.weight   |  2097152   |
|     model.layers.6.mlp.experts.23.up_proj.weight    |  2097152   |
|    model.layers.6.mlp.experts.23.down_proj.weight   |  2097152   |
|            model.layers.6.mlp.gate.weight           |   49152    |
|   model.layers.6.mlp.gate.e_score_correction_bias   |     24     |
|  model.layers.6.mlp.shared_experts.gate_proj.weight |  2097152   |
|   model.layers.6.mlp.shared_experts.up_proj.weight  |  2097152   |
|  model.layers.6.mlp.shared_experts.down_proj.weight |  2097152   |
|        model.layers.6.input_layernorm.weight        |    2048    |
|    model.layers.6.post_attention_layernorm.weight   |    2048    |
|       model.layers.7.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.7.self_attn.q_a_layernorm.weight    |    512     |
|       model.layers.7.self_attn.q_b_proj.weight      |  1572864   |
|  model.layers.7.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|    model.layers.7.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.7.self_attn.kv_b_proj.weight      |   524288   |
|        model.layers.7.self_attn.o_proj.weight       |  4194304   |
|    model.layers.7.mlp.experts.0.gate_proj.weight    |  2097152   |
|     model.layers.7.mlp.experts.0.up_proj.weight     |  2097152   |
|    model.layers.7.mlp.experts.0.down_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.1.gate_proj.weight    |  2097152   |
|     model.layers.7.mlp.experts.1.up_proj.weight     |  2097152   |
|    model.layers.7.mlp.experts.1.down_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.2.gate_proj.weight    |  2097152   |
|     model.layers.7.mlp.experts.2.up_proj.weight     |  2097152   |
|    model.layers.7.mlp.experts.2.down_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.3.gate_proj.weight    |  2097152   |
|     model.layers.7.mlp.experts.3.up_proj.weight     |  2097152   |
|    model.layers.7.mlp.experts.3.down_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.4.gate_proj.weight    |  2097152   |
|     model.layers.7.mlp.experts.4.up_proj.weight     |  2097152   |
|    model.layers.7.mlp.experts.4.down_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.5.gate_proj.weight    |  2097152   |
|     model.layers.7.mlp.experts.5.up_proj.weight     |  2097152   |
|    model.layers.7.mlp.experts.5.down_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.6.gate_proj.weight    |  2097152   |
|     model.layers.7.mlp.experts.6.up_proj.weight     |  2097152   |
|    model.layers.7.mlp.experts.6.down_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.7.gate_proj.weight    |  2097152   |
|     model.layers.7.mlp.experts.7.up_proj.weight     |  2097152   |
|    model.layers.7.mlp.experts.7.down_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.8.gate_proj.weight    |  2097152   |
|     model.layers.7.mlp.experts.8.up_proj.weight     |  2097152   |
|    model.layers.7.mlp.experts.8.down_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.9.gate_proj.weight    |  2097152   |
|     model.layers.7.mlp.experts.9.up_proj.weight     |  2097152   |
|    model.layers.7.mlp.experts.9.down_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.10.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.10.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.10.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.11.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.11.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.11.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.12.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.12.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.12.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.13.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.13.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.13.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.14.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.14.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.14.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.15.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.15.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.15.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.16.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.16.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.16.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.17.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.17.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.17.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.18.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.18.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.18.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.19.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.19.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.19.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.20.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.20.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.20.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.21.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.21.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.21.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.22.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.22.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.22.down_proj.weight   |  2097152   |
|    model.layers.7.mlp.experts.23.gate_proj.weight   |  2097152   |
|     model.layers.7.mlp.experts.23.up_proj.weight    |  2097152   |
|    model.layers.7.mlp.experts.23.down_proj.weight   |  2097152   |
|            model.layers.7.mlp.gate.weight           |   49152    |
|   model.layers.7.mlp.gate.e_score_correction_bias   |     24     |
|  model.layers.7.mlp.shared_experts.gate_proj.weight |  2097152   |
|   model.layers.7.mlp.shared_experts.up_proj.weight  |  2097152   |
|  model.layers.7.mlp.shared_experts.down_proj.weight |  2097152   |
|        model.layers.7.input_layernorm.weight        |    2048    |
|    model.layers.7.post_attention_layernorm.weight   |    2048    |
|       model.layers.8.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.8.self_attn.q_a_layernorm.weight    |    512     |
|       model.layers.8.self_attn.q_b_proj.weight      |  1572864   |
|  model.layers.8.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|    model.layers.8.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.8.self_attn.kv_b_proj.weight      |   524288   |
|        model.layers.8.self_attn.o_proj.weight       |  4194304   |
|    model.layers.8.mlp.experts.0.gate_proj.weight    |  2097152   |
|     model.layers.8.mlp.experts.0.up_proj.weight     |  2097152   |
|    model.layers.8.mlp.experts.0.down_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.1.gate_proj.weight    |  2097152   |
|     model.layers.8.mlp.experts.1.up_proj.weight     |  2097152   |
|    model.layers.8.mlp.experts.1.down_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.2.gate_proj.weight    |  2097152   |
|     model.layers.8.mlp.experts.2.up_proj.weight     |  2097152   |
|    model.layers.8.mlp.experts.2.down_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.3.gate_proj.weight    |  2097152   |
|     model.layers.8.mlp.experts.3.up_proj.weight     |  2097152   |
|    model.layers.8.mlp.experts.3.down_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.4.gate_proj.weight    |  2097152   |
|     model.layers.8.mlp.experts.4.up_proj.weight     |  2097152   |
|    model.layers.8.mlp.experts.4.down_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.5.gate_proj.weight    |  2097152   |
|     model.layers.8.mlp.experts.5.up_proj.weight     |  2097152   |
|    model.layers.8.mlp.experts.5.down_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.6.gate_proj.weight    |  2097152   |
|     model.layers.8.mlp.experts.6.up_proj.weight     |  2097152   |
|    model.layers.8.mlp.experts.6.down_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.7.gate_proj.weight    |  2097152   |
|     model.layers.8.mlp.experts.7.up_proj.weight     |  2097152   |
|    model.layers.8.mlp.experts.7.down_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.8.gate_proj.weight    |  2097152   |
|     model.layers.8.mlp.experts.8.up_proj.weight     |  2097152   |
|    model.layers.8.mlp.experts.8.down_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.9.gate_proj.weight    |  2097152   |
|     model.layers.8.mlp.experts.9.up_proj.weight     |  2097152   |
|    model.layers.8.mlp.experts.9.down_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.10.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.10.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.10.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.11.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.11.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.11.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.12.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.12.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.12.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.13.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.13.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.13.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.14.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.14.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.14.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.15.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.15.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.15.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.16.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.16.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.16.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.17.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.17.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.17.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.18.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.18.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.18.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.19.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.19.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.19.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.20.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.20.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.20.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.21.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.21.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.21.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.22.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.22.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.22.down_proj.weight   |  2097152   |
|    model.layers.8.mlp.experts.23.gate_proj.weight   |  2097152   |
|     model.layers.8.mlp.experts.23.up_proj.weight    |  2097152   |
|    model.layers.8.mlp.experts.23.down_proj.weight   |  2097152   |
|            model.layers.8.mlp.gate.weight           |   49152    |
|   model.layers.8.mlp.gate.e_score_correction_bias   |     24     |
|  model.layers.8.mlp.shared_experts.gate_proj.weight |  2097152   |
|   model.layers.8.mlp.shared_experts.up_proj.weight  |  2097152   |
|  model.layers.8.mlp.shared_experts.down_proj.weight |  2097152   |
|        model.layers.8.input_layernorm.weight        |    2048    |
|    model.layers.8.post_attention_layernorm.weight   |    2048    |
|       model.layers.9.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.9.self_attn.q_a_layernorm.weight    |    512     |
|       model.layers.9.self_attn.q_b_proj.weight      |  1572864   |
|  model.layers.9.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|    model.layers.9.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.9.self_attn.kv_b_proj.weight      |   524288   |
|        model.layers.9.self_attn.o_proj.weight       |  4194304   |
|    model.layers.9.mlp.experts.0.gate_proj.weight    |  2097152   |
|     model.layers.9.mlp.experts.0.up_proj.weight     |  2097152   |
|    model.layers.9.mlp.experts.0.down_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.1.gate_proj.weight    |  2097152   |
|     model.layers.9.mlp.experts.1.up_proj.weight     |  2097152   |
|    model.layers.9.mlp.experts.1.down_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.2.gate_proj.weight    |  2097152   |
|     model.layers.9.mlp.experts.2.up_proj.weight     |  2097152   |
|    model.layers.9.mlp.experts.2.down_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.3.gate_proj.weight    |  2097152   |
|     model.layers.9.mlp.experts.3.up_proj.weight     |  2097152   |
|    model.layers.9.mlp.experts.3.down_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.4.gate_proj.weight    |  2097152   |
|     model.layers.9.mlp.experts.4.up_proj.weight     |  2097152   |
|    model.layers.9.mlp.experts.4.down_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.5.gate_proj.weight    |  2097152   |
|     model.layers.9.mlp.experts.5.up_proj.weight     |  2097152   |
|    model.layers.9.mlp.experts.5.down_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.6.gate_proj.weight    |  2097152   |
|     model.layers.9.mlp.experts.6.up_proj.weight     |  2097152   |
|    model.layers.9.mlp.experts.6.down_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.7.gate_proj.weight    |  2097152   |
|     model.layers.9.mlp.experts.7.up_proj.weight     |  2097152   |
|    model.layers.9.mlp.experts.7.down_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.8.gate_proj.weight    |  2097152   |
|     model.layers.9.mlp.experts.8.up_proj.weight     |  2097152   |
|    model.layers.9.mlp.experts.8.down_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.9.gate_proj.weight    |  2097152   |
|     model.layers.9.mlp.experts.9.up_proj.weight     |  2097152   |
|    model.layers.9.mlp.experts.9.down_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.10.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.10.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.10.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.11.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.11.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.11.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.12.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.12.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.12.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.13.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.13.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.13.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.14.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.14.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.14.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.15.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.15.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.15.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.16.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.16.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.16.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.17.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.17.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.17.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.18.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.18.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.18.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.19.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.19.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.19.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.20.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.20.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.20.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.21.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.21.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.21.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.22.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.22.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.22.down_proj.weight   |  2097152   |
|    model.layers.9.mlp.experts.23.gate_proj.weight   |  2097152   |
|     model.layers.9.mlp.experts.23.up_proj.weight    |  2097152   |
|    model.layers.9.mlp.experts.23.down_proj.weight   |  2097152   |
|            model.layers.9.mlp.gate.weight           |   49152    |
|   model.layers.9.mlp.gate.e_score_correction_bias   |     24     |
|  model.layers.9.mlp.shared_experts.gate_proj.weight |  2097152   |
|   model.layers.9.mlp.shared_experts.up_proj.weight  |  2097152   |
|  model.layers.9.mlp.shared_experts.down_proj.weight |  2097152   |
|        model.layers.9.input_layernorm.weight        |    2048    |
|    model.layers.9.post_attention_layernorm.weight   |    2048    |
|      model.layers.10.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.10.self_attn.q_a_layernorm.weight   |    512     |
|      model.layers.10.self_attn.q_b_proj.weight      |  1572864   |
| model.layers.10.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|   model.layers.10.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.10.self_attn.kv_b_proj.weight     |   524288   |
|       model.layers.10.self_attn.o_proj.weight       |  4194304   |
|    model.layers.10.mlp.experts.0.gate_proj.weight   |  2097152   |
|     model.layers.10.mlp.experts.0.up_proj.weight    |  2097152   |
|    model.layers.10.mlp.experts.0.down_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.1.gate_proj.weight   |  2097152   |
|     model.layers.10.mlp.experts.1.up_proj.weight    |  2097152   |
|    model.layers.10.mlp.experts.1.down_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.2.gate_proj.weight   |  2097152   |
|     model.layers.10.mlp.experts.2.up_proj.weight    |  2097152   |
|    model.layers.10.mlp.experts.2.down_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.3.gate_proj.weight   |  2097152   |
|     model.layers.10.mlp.experts.3.up_proj.weight    |  2097152   |
|    model.layers.10.mlp.experts.3.down_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.4.gate_proj.weight   |  2097152   |
|     model.layers.10.mlp.experts.4.up_proj.weight    |  2097152   |
|    model.layers.10.mlp.experts.4.down_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.5.gate_proj.weight   |  2097152   |
|     model.layers.10.mlp.experts.5.up_proj.weight    |  2097152   |
|    model.layers.10.mlp.experts.5.down_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.6.gate_proj.weight   |  2097152   |
|     model.layers.10.mlp.experts.6.up_proj.weight    |  2097152   |
|    model.layers.10.mlp.experts.6.down_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.7.gate_proj.weight   |  2097152   |
|     model.layers.10.mlp.experts.7.up_proj.weight    |  2097152   |
|    model.layers.10.mlp.experts.7.down_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.8.gate_proj.weight   |  2097152   |
|     model.layers.10.mlp.experts.8.up_proj.weight    |  2097152   |
|    model.layers.10.mlp.experts.8.down_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.9.gate_proj.weight   |  2097152   |
|     model.layers.10.mlp.experts.9.up_proj.weight    |  2097152   |
|    model.layers.10.mlp.experts.9.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.10.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.10.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.10.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.11.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.11.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.11.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.12.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.12.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.12.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.13.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.13.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.13.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.14.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.14.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.14.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.15.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.15.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.15.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.16.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.16.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.16.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.17.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.17.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.17.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.18.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.18.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.18.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.19.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.19.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.19.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.20.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.20.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.20.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.21.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.21.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.21.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.22.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.22.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.22.down_proj.weight   |  2097152   |
|   model.layers.10.mlp.experts.23.gate_proj.weight   |  2097152   |
|    model.layers.10.mlp.experts.23.up_proj.weight    |  2097152   |
|   model.layers.10.mlp.experts.23.down_proj.weight   |  2097152   |
|           model.layers.10.mlp.gate.weight           |   49152    |
|   model.layers.10.mlp.gate.e_score_correction_bias  |     24     |
| model.layers.10.mlp.shared_experts.gate_proj.weight |  2097152   |
|  model.layers.10.mlp.shared_experts.up_proj.weight  |  2097152   |
| model.layers.10.mlp.shared_experts.down_proj.weight |  2097152   |
|        model.layers.10.input_layernorm.weight       |    2048    |
|   model.layers.10.post_attention_layernorm.weight   |    2048    |
|      model.layers.11.self_attn.q_a_proj.weight      |  1048576   |
|    model.layers.11.self_attn.q_a_layernorm.weight   |    512     |
|      model.layers.11.self_attn.q_b_proj.weight      |  1572864   |
| model.layers.11.self_attn.kv_a_proj_with_mqa.weight |   393216   |
|   model.layers.11.self_attn.kv_a_layernorm.weight   |    128     |
|      model.layers.11.self_attn.kv_b_proj.weight     |   524288   |
|       model.layers.11.self_attn.o_proj.weight       |  4194304   |
|    model.layers.11.mlp.experts.0.gate_proj.weight   |  2097152   |
|     model.layers.11.mlp.experts.0.up_proj.weight    |  2097152   |
|    model.layers.11.mlp.experts.0.down_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.1.gate_proj.weight   |  2097152   |
|     model.layers.11.mlp.experts.1.up_proj.weight    |  2097152   |
|    model.layers.11.mlp.experts.1.down_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.2.gate_proj.weight   |  2097152   |
|     model.layers.11.mlp.experts.2.up_proj.weight    |  2097152   |
|    model.layers.11.mlp.experts.2.down_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.3.gate_proj.weight   |  2097152   |
|     model.layers.11.mlp.experts.3.up_proj.weight    |  2097152   |
|    model.layers.11.mlp.experts.3.down_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.4.gate_proj.weight   |  2097152   |
|     model.layers.11.mlp.experts.4.up_proj.weight    |  2097152   |
|    model.layers.11.mlp.experts.4.down_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.5.gate_proj.weight   |  2097152   |
|     model.layers.11.mlp.experts.5.up_proj.weight    |  2097152   |
|    model.layers.11.mlp.experts.5.down_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.6.gate_proj.weight   |  2097152   |
|     model.layers.11.mlp.experts.6.up_proj.weight    |  2097152   |
|    model.layers.11.mlp.experts.6.down_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.7.gate_proj.weight   |  2097152   |
|     model.layers.11.mlp.experts.7.up_proj.weight    |  2097152   |
|    model.layers.11.mlp.experts.7.down_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.8.gate_proj.weight   |  2097152   |
|     model.layers.11.mlp.experts.8.up_proj.weight    |  2097152   |
|    model.layers.11.mlp.experts.8.down_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.9.gate_proj.weight   |  2097152   |
|     model.layers.11.mlp.experts.9.up_proj.weight    |  2097152   |
|    model.layers.11.mlp.experts.9.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.10.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.10.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.10.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.11.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.11.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.11.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.12.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.12.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.12.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.13.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.13.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.13.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.14.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.14.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.14.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.15.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.15.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.15.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.16.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.16.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.16.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.17.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.17.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.17.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.18.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.18.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.18.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.19.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.19.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.19.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.20.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.20.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.20.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.21.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.21.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.21.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.22.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.22.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.22.down_proj.weight   |  2097152   |
|   model.layers.11.mlp.experts.23.gate_proj.weight   |  2097152   |
|    model.layers.11.mlp.experts.23.up_proj.weight    |  2097152   |
|   model.layers.11.mlp.experts.23.down_proj.weight   |  2097152   |
|           model.layers.11.mlp.gate.weight           |   49152    |
|   model.layers.11.mlp.gate.e_score_correction_bias  |     24     |
| model.layers.11.mlp.shared_experts.gate_proj.weight |  2097152   |
|  model.layers.11.mlp.shared_experts.up_proj.weight  |  2097152   |
| model.layers.11.mlp.shared_experts.down_proj.weight |  2097152   |
|        model.layers.11.input_layernorm.weight       |    2048    |
|   model.layers.11.post_attention_layernorm.weight   |    2048    |
|                  model.norm.weight                  |    2048    |
|                    lm_head.weight                   | 264765440  |
+-----------------------------------------------------+------------+
Downloads last month
6
Safetensors
Model size
2.19B params
Tensor type
F32
·
Inference Providers NEW
Text Generation
This model is not currently available via any of the supported third-party Inference Providers, and HF Inference API does not yet support model repos that contain custom code.