{
    "hidden_size": 4096,
    "intermediate_size": 14336,
    "hidden_act": "silu",
    "n_layer": 32,
    "mla_layers": [
        0,
        2,
        4,
        6,
        8,
        10,
        12,
	    14,
	    16,
	    18,
	    20,
	    22,
	    24,
	    26,
	    28,
        30
    ],
    "rms_norm_eps": 1e-05,
    "num_attention_heads": 32,
    "num_key_value_heads": 8,
    "kv_lora_rank": 160,
    "q_lora_rank": 2048,
    "use_lora_layer_norm": false,
    "use_full_kv_head": false,
    "qk_rope_head_dim": 64,
    "v_head_dim": 128,
    "qk_nope_head_dim": 64,
    "qkv_rank_divisor": 8,
    "max_position_embeddings": 131072,
    "rope_theta": 500000.0,
    "rope_scaling": {
        "factor": 8.0,
        "original_max_position_embeddings": 8192,
        "rope_type": "yarn"
    },
    "attention_bias": false,
    "attention_dropout": 0.0,
    "d_model": 4096,
    "ssm_cfg": {
        "expand": 1,
        "ngroups": 32,
        "d_state": 128,
        "repeat_kv_before_conv": false
    },
    "d_inner": 4096,
    "d_xb": 1024
}