{ "version": "0.1.0", "model_type": "decomp_llama", "quantization": "q0f16", "model_config": { "hidden_size": 3584, "intermediate_size": 14336, "num_attention_heads": 28, "num_hidden_layers": 32, "rms_norm_eps": 1e-05, "vocab_size": 77209, "tie_word_embeddings": false, "position_embedding_base": 500000.0, "rope_scaling": { "factor": 8.0, "high_freq_factor": 4.0, "low_freq_factor": 1.0, "original_max_position_embeddings": 8192, "rope_type": "llama3" }, "context_window_size": 768, "prefill_chunk_size": 768, "num_key_value_heads": 7, "head_dim": 128, "tensor_parallel_shards": 1, "truncation_ranks": { "model.layers.0.mlp.gate_up_proj": 2508, "model.layers.0.self_attn.o_proj": 764, "model.layers.0.self_attn.qkv_proj": 929, "model.layers.1.mlp.gate_up_proj": 2508, "model.layers.1.self_attn.o_proj": 477, "model.layers.1.self_attn.qkv_proj": 581, "model.layers.10.mlp.down_proj": 302, "model.layers.10.mlp.gate_up_proj": 869, "model.layers.10.self_attn.o_proj": 477, "model.layers.10.self_attn.qkv_proj": 581, "model.layers.11.mlp.down_proj": 302, "model.layers.11.mlp.gate_up_proj": 348, "model.layers.11.self_attn.o_proj": 477, "model.layers.11.self_attn.qkv_proj": 581, "model.layers.12.mlp.down_proj": 1207, "model.layers.12.mlp.gate_up_proj": 1391, "model.layers.12.self_attn.o_proj": 477, "model.layers.12.self_attn.qkv_proj": 581, "model.layers.13.self_attn.o_proj": 191, "model.layers.13.self_attn.qkv_proj": 581, "model.layers.14.self_attn.o_proj": 764, "model.layers.14.self_attn.qkv_proj": 929, "model.layers.15.mlp.down_proj": 1659, "model.layers.15.mlp.gate_up_proj": 869, "model.layers.15.self_attn.o_proj": 764, "model.layers.15.self_attn.qkv_proj": 1278, "model.layers.16.mlp.gate_up_proj": 1912, "model.layers.16.self_attn.o_proj": 477, "model.layers.16.self_attn.qkv_proj": 929, "model.layers.17.mlp.gate_up_proj": 1912, "model.layers.17.self_attn.o_proj": 477, "model.layers.17.self_attn.qkv_proj": 929, "model.layers.18.self_attn.o_proj": 477, "model.layers.18.self_attn.qkv_proj": 929, "model.layers.19.mlp.down_proj": 1659, "model.layers.19.mlp.gate_up_proj": 1912, "model.layers.19.self_attn.o_proj": 477, "model.layers.19.self_attn.qkv_proj": 581, "model.layers.2.self_attn.o_proj": 477, "model.layers.2.self_attn.qkv_proj": 1278, "model.layers.20.mlp.down_proj": 1207, "model.layers.20.mlp.gate_up_proj": 1391, "model.layers.20.self_attn.o_proj": 191, "model.layers.20.self_attn.qkv_proj": 232, "model.layers.21.mlp.down_proj": 2565, "model.layers.21.mlp.gate_up_proj": 2433, "model.layers.21.self_attn.o_proj": 477, "model.layers.21.self_attn.qkv_proj": 581, "model.layers.22.mlp.down_proj": 2112, "model.layers.22.mlp.gate_up_proj": 1912, "model.layers.22.self_attn.o_proj": 477, "model.layers.22.self_attn.qkv_proj": 581, "model.layers.23.mlp.down_proj": 1207, "model.layers.23.mlp.gate_up_proj": 1391, "model.layers.23.self_attn.o_proj": 477, "model.layers.23.self_attn.qkv_proj": 581, "model.layers.24.mlp.down_proj": 1207, "model.layers.24.mlp.gate_up_proj": 1391, "model.layers.24.self_attn.o_proj": 191, "model.layers.24.self_attn.qkv_proj": 581, "model.layers.25.mlp.down_proj": 1207, "model.layers.25.mlp.gate_up_proj": 1391, "model.layers.25.self_attn.o_proj": 477, "model.layers.25.self_attn.qkv_proj": 581, "model.layers.26.mlp.down_proj": 1207, "model.layers.26.mlp.gate_up_proj": 1391, "model.layers.26.self_attn.o_proj": 191, "model.layers.26.self_attn.qkv_proj": 581, "model.layers.27.mlp.down_proj": 1207, "model.layers.27.mlp.gate_up_proj": 1391, "model.layers.27.self_attn.o_proj": 191, "model.layers.27.self_attn.qkv_proj": 581, "model.layers.28.mlp.down_proj": 2707, "model.layers.28.mlp.gate_up_proj": 1971, "model.layers.28.self_attn.o_proj": 191, "model.layers.28.self_attn.qkv_proj": 581, "model.layers.29.self_attn.o_proj": 191, "model.layers.29.self_attn.qkv_proj": 581, "model.layers.3.self_attn.o_proj": 764, "model.layers.3.self_attn.qkv_proj": 1278, "model.layers.30.self_attn.o_proj": 191, "model.layers.30.self_attn.qkv_proj": 581, "model.layers.31.mlp.down_proj": 2229, "model.layers.31.self_attn.o_proj": 477, "model.layers.31.self_attn.qkv_proj": 581, "model.layers.4.mlp.down_proj": 2112, "model.layers.4.mlp.gate_up_proj": 2433, "model.layers.4.self_attn.o_proj": 477, "model.layers.4.self_attn.qkv_proj": 929, "model.layers.5.mlp.down_proj": 1659, "model.layers.5.mlp.gate_up_proj": 2433, "model.layers.5.self_attn.o_proj": 477, "model.layers.5.self_attn.qkv_proj": 929, "model.layers.6.self_attn.o_proj": 477, "model.layers.6.self_attn.qkv_proj": 581, "model.layers.7.mlp.down_proj": 1207, "model.layers.7.mlp.gate_up_proj": 1912, "model.layers.7.self_attn.o_proj": 477, "model.layers.7.self_attn.qkv_proj": 581, "model.layers.8.mlp.down_proj": 754, "model.layers.8.mlp.gate_up_proj": 1391, "model.layers.8.self_attn.o_proj": 477, "model.layers.8.self_attn.qkv_proj": 581, "model.layers.9.mlp.down_proj": 302, "model.layers.9.mlp.gate_up_proj": 869, "model.layers.9.self_attn.o_proj": 477, "model.layers.9.self_attn.qkv_proj": 581 }, "pruned_layers": [ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 ], "pruned_intermediate_size": 11468, "ori_hidden_size": 4096, "pipeline_parallel_stages": 1, "max_batch_size": 1 }, "vocab_size": 77209, "context_window_size": 768, "sliding_window_size": -1, "prefill_chunk_size": 768, "attention_sink_size": -1, "tensor_parallel_shards": 1, "pipeline_parallel_stages": 1, "temperature": 0.6, "presence_penalty": 0.0, "frequency_penalty": 0.0, "repetition_penalty": 1.0, "top_p": 0.9, "tokenizer_files": [ "tokenizer.json", "tokenizer_config.json" ], "tokenizer_info": { "token_postproc_method": "byte_level", "prepend_space_in_encode": false, "strip_space_in_decode": false }, "conv_template": { "name": "llama-3_1", "system_template": "<|start_header_id|>system<|end_header_id|>\n\n{system_message}<|eot_id|>", "system_message": "You are a helpful, respectful and honest assistant.", "system_prefix_token_ids": [ 76953 ], "add_role_after_system_message": true, "roles": { "user": "<|start_header_id|>user", "assistant": "<|start_header_id|>assistant", "tool": "<|start_header_id|>ipython" }, "role_templates": { "user": "{user_message}", "assistant": "{assistant_message}", "tool": "{tool_message}" }, "messages": [], "seps": [ "<|eot_id|>" ], "role_content_sep": "<|end_header_id|>\n\n", "role_empty_sep": "<|end_header_id|>\n\n", "stop_str": [], "stop_token_ids": [ 76954, 76961, 76962 ], "function_string": "", "use_function_calling": false }, "pad_token_id": 76957, "bos_token_id": 76953, "eos_token_id": [ 76954, 76961, 76962 ] }