diff --git "a/.cache/MatMulNBits_2_0_meta.json" "b/.cache/MatMulNBits_2_0_meta.json" new file mode 100644--- /dev/null +++ "b/.cache/MatMulNBits_2_0_meta.json" @@ -0,0 +1,31667 @@ +{ + "dd_meta_major_version": 1, + "dd_meta_minor_version": 4, + "state_table_updates": [ + { + "state_table_idx": 0, + "update_func": 1, + "update_arg": 1 + } + ], + "op_list": [ + { + "name": "MatMulNBits_2_0", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.0/input_layernorm/output_0.out5_4_0" + ], + "const_args": [ + "model.layers.0.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.0.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.0/attn/qk_proj/Add/output_0.out5_4_0" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.0.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.0/input_layernorm/output_0.out5_4_0" + ], + "const_args": [ + "model.layers.0.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.v_proj.Add.bias.preformat", + "model.layers.0.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.0.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "3", + "1" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.0/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.0/attn/qk_proj/Add/output_0.out5_4_0", + "past_key_values.0.key", + "past_key_values.0.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.0/attn/GroupQueryAttention/output_0.out2_0", + "present.0.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "0", + "0", + "2", + "0", + "1", + "1", + "6", + "0", + "2", + "0" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.0.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.0/attn/GroupQueryAttention/output_0.out2_0" + ], + "const_args": [ + "model.layers.0.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.0/attn/o_proj/MatMulNBits/output_0.out5_4_1" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_0", + "type": "FlatRMSAdd", + "in_args": [ + "/model/embed_tokens/Gather/output_0.out4_0", + "/model/layers.0/attn/o_proj/MatMulNBits/output_0.out5_4_1" + ], + "const_args": [ + "model.layers.0.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.0/post_attention_layernorm/output_3.out4_0", + "/model/layers.0/post_attention_layernorm/output_0.out4_0" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_0", + "type": "FlatMLP", + "in_args": [ + "/model/layers.0/post_attention_layernorm/output_0.out4_0" + ], + "const_args": [ + "model.layers.0.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.0.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.0.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.0.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.0.mlp.up_proj.MatMulNBits.qweight", + "model.layers.0.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.0.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.0.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.0/mlp/Mul/output_0.out3_0" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.0.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.0/mlp/Mul/output_0.out3_0" + ], + "const_args": [ + "model.layers.0.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.0/mlp/down_proj/MatMulNBits/output_0.out5_4_2" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_1", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.0/post_attention_layernorm/output_3.out4_0", + "/model/layers.0/mlp/down_proj/MatMulNBits/output_0.out5_4_2" + ], + "const_args": [ + "model.layers.1.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.1/input_layernorm/output_3.out4_1", + "/model/layers.1/input_layernorm/output_0.out4_1" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_1", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.1/input_layernorm/output_0.out4_1" + ], + "const_args": [ + "model.layers.1.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.1.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.1/attn/qk_proj/Add/output_0.out5_4_3" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.1.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.1/input_layernorm/output_0.out4_1" + ], + "const_args": [ + "model.layers.1.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.v_proj.Add.bias.preformat", + "model.layers.1.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.1.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "7", + "3" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.1/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.1/attn/qk_proj/Add/output_0.out5_4_3", + "past_key_values.1.key", + "past_key_values.1.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.1/attn/GroupQueryAttention/output_0.out2_1", + "present.1.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "4", + "2", + "2", + "0", + "5", + "3", + "6", + "0", + "6", + "2" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.1.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.1/attn/GroupQueryAttention/output_0.out2_1" + ], + "const_args": [ + "model.layers.1.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.1/attn/o_proj/MatMulNBits/output_0.out5_4_4" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_2", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.1/input_layernorm/output_3.out4_1", + "/model/layers.1/attn/o_proj/MatMulNBits/output_0.out5_4_4" + ], + "const_args": [ + "model.layers.1.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.1/post_attention_layernorm/output_3.out4_2", + "/model/layers.1/post_attention_layernorm/output_0.out4_2" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_1", + "type": "FlatMLP", + "in_args": [ + "/model/layers.1/post_attention_layernorm/output_0.out4_2" + ], + "const_args": [ + "model.layers.1.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.1.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.1.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.1.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.1.mlp.up_proj.MatMulNBits.qweight", + "model.layers.1.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.1.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.1.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.1/mlp/Mul/output_0.out3_1" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.1.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.1/mlp/Mul/output_0.out3_1" + ], + "const_args": [ + "model.layers.1.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.1/mlp/down_proj/MatMulNBits/output_0.out5_4_5" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_3", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.1/post_attention_layernorm/output_3.out4_2", + "/model/layers.1/mlp/down_proj/MatMulNBits/output_0.out5_4_5" + ], + "const_args": [ + "model.layers.2.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.2/input_layernorm/output_3.out4_3", + "/model/layers.2/input_layernorm/output_0.out4_3" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_2", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.2/input_layernorm/output_0.out4_3" + ], + "const_args": [ + "model.layers.2.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.2.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.2/attn/qk_proj/Add/output_0.out5_4_6" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.2.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.2/input_layernorm/output_0.out4_3" + ], + "const_args": [ + "model.layers.2.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.v_proj.Add.bias.preformat", + "model.layers.2.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.2.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "11", + "5" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.2/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.2/attn/qk_proj/Add/output_0.out5_4_6", + "past_key_values.2.key", + "past_key_values.2.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.2/attn/GroupQueryAttention/output_0.out2_2", + "present.2.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "8", + "4", + "2", + "0", + "9", + "5", + "6", + "0", + "10", + "4" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.2.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.2/attn/GroupQueryAttention/output_0.out2_2" + ], + "const_args": [ + "model.layers.2.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.2/attn/o_proj/MatMulNBits/output_0.out5_4_7" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_4", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.2/input_layernorm/output_3.out4_3", + "/model/layers.2/attn/o_proj/MatMulNBits/output_0.out5_4_7" + ], + "const_args": [ + "model.layers.2.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.2/post_attention_layernorm/output_3.out4_4", + "/model/layers.2/post_attention_layernorm/output_0.out4_4" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_2", + "type": "FlatMLP", + "in_args": [ + "/model/layers.2/post_attention_layernorm/output_0.out4_4" + ], + "const_args": [ + "model.layers.2.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.2.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.2.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.2.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.2.mlp.up_proj.MatMulNBits.qweight", + "model.layers.2.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.2.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.2.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.2/mlp/Mul/output_0.out3_2" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.2.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.2/mlp/Mul/output_0.out3_2" + ], + "const_args": [ + "model.layers.2.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.2/mlp/down_proj/MatMulNBits/output_0.out5_4_8" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_5", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.2/post_attention_layernorm/output_3.out4_4", + "/model/layers.2/mlp/down_proj/MatMulNBits/output_0.out5_4_8" + ], + "const_args": [ + "model.layers.3.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.3/input_layernorm/output_3.out4_5", + "/model/layers.3/input_layernorm/output_0.out4_5" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_3", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.3/input_layernorm/output_0.out4_5" + ], + "const_args": [ + "model.layers.3.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.3.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.3/attn/qk_proj/Add/output_0.out5_4_9" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.3.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.3/input_layernorm/output_0.out4_5" + ], + "const_args": [ + "model.layers.3.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.v_proj.Add.bias.preformat", + "model.layers.3.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.3.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "15", + "7" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.3/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.3/attn/qk_proj/Add/output_0.out5_4_9", + "past_key_values.3.key", + "past_key_values.3.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.3/attn/GroupQueryAttention/output_0.out2_3", + "present.3.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "12", + "6", + "2", + "0", + "13", + "7", + "6", + "0", + "14", + "6" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.3.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.3/attn/GroupQueryAttention/output_0.out2_3" + ], + "const_args": [ + "model.layers.3.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.3/attn/o_proj/MatMulNBits/output_0.out5_4_10" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_6", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.3/input_layernorm/output_3.out4_5", + "/model/layers.3/attn/o_proj/MatMulNBits/output_0.out5_4_10" + ], + "const_args": [ + "model.layers.3.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.3/post_attention_layernorm/output_3.out4_6", + "/model/layers.3/post_attention_layernorm/output_0.out4_6" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_3", + "type": "FlatMLP", + "in_args": [ + "/model/layers.3/post_attention_layernorm/output_0.out4_6" + ], + "const_args": [ + "model.layers.3.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.3.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.3.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.3.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.3.mlp.up_proj.MatMulNBits.qweight", + "model.layers.3.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.3.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.3.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.3/mlp/Mul/output_0.out3_3" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.3.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.3/mlp/Mul/output_0.out3_3" + ], + "const_args": [ + "model.layers.3.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.3/mlp/down_proj/MatMulNBits/output_0.out5_4_11" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_7", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.3/post_attention_layernorm/output_3.out4_6", + "/model/layers.3/mlp/down_proj/MatMulNBits/output_0.out5_4_11" + ], + "const_args": [ + "model.layers.4.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.4/input_layernorm/output_3.out4_7", + "/model/layers.4/input_layernorm/output_0.out4_7" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_4", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.4/input_layernorm/output_0.out4_7" + ], + "const_args": [ + "model.layers.4.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.4.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.4/attn/qk_proj/Add/output_0.out5_4_12" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.4.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.4/input_layernorm/output_0.out4_7" + ], + "const_args": [ + "model.layers.4.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.v_proj.Add.bias.preformat", + "model.layers.4.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.4.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "19", + "9" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.4/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.4/attn/qk_proj/Add/output_0.out5_4_12", + "past_key_values.4.key", + "past_key_values.4.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.4/attn/GroupQueryAttention/output_0.out2_4", + "present.4.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "16", + "8", + "2", + "0", + "17", + "9", + "6", + "0", + "18", + "8" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.4.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.4/attn/GroupQueryAttention/output_0.out2_4" + ], + "const_args": [ + "model.layers.4.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.4/attn/o_proj/MatMulNBits/output_0.out5_4_13" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_8", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.4/input_layernorm/output_3.out4_7", + "/model/layers.4/attn/o_proj/MatMulNBits/output_0.out5_4_13" + ], + "const_args": [ + "model.layers.4.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.4/post_attention_layernorm/output_3.out4_8", + "/model/layers.4/post_attention_layernorm/output_0.out4_8" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_4", + "type": "FlatMLP", + "in_args": [ + "/model/layers.4/post_attention_layernorm/output_0.out4_8" + ], + "const_args": [ + "model.layers.4.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.4.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.4.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.4.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.4.mlp.up_proj.MatMulNBits.qweight", + "model.layers.4.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.4.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.4.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.4/mlp/Mul/output_0.out3_4" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.4.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.4/mlp/Mul/output_0.out3_4" + ], + "const_args": [ + "model.layers.4.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.4/mlp/down_proj/MatMulNBits/output_0.out5_4_14" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_9", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.4/post_attention_layernorm/output_3.out4_8", + "/model/layers.4/mlp/down_proj/MatMulNBits/output_0.out5_4_14" + ], + "const_args": [ + "model.layers.5.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.5/input_layernorm/output_3.out4_9", + "/model/layers.5/input_layernorm/output_0.out4_9" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_5", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.5/input_layernorm/output_0.out4_9" + ], + "const_args": [ + "model.layers.5.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.5.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.5/attn/qk_proj/Add/output_0.out5_4_15" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.5.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.5/input_layernorm/output_0.out4_9" + ], + "const_args": [ + "model.layers.5.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.v_proj.Add.bias.preformat", + "model.layers.5.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.5.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "23", + "11" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.5/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.5/attn/qk_proj/Add/output_0.out5_4_15", + "past_key_values.5.key", + "past_key_values.5.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.5/attn/GroupQueryAttention/output_0.out2_5", + "present.5.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "20", + "10", + "2", + "0", + "21", + "11", + "6", + "0", + "22", + "10" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.5.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.5/attn/GroupQueryAttention/output_0.out2_5" + ], + "const_args": [ + "model.layers.5.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.5/attn/o_proj/MatMulNBits/output_0.out5_4_16" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_10", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.5/input_layernorm/output_3.out4_9", + "/model/layers.5/attn/o_proj/MatMulNBits/output_0.out5_4_16" + ], + "const_args": [ + "model.layers.5.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.5/post_attention_layernorm/output_3.out4_10", + "/model/layers.5/post_attention_layernorm/output_0.out4_10" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_5", + "type": "FlatMLP", + "in_args": [ + "/model/layers.5/post_attention_layernorm/output_0.out4_10" + ], + "const_args": [ + "model.layers.5.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.5.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.5.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.5.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.5.mlp.up_proj.MatMulNBits.qweight", + "model.layers.5.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.5.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.5.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.5/mlp/Mul/output_0.out3_5" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.5.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.5/mlp/Mul/output_0.out3_5" + ], + "const_args": [ + "model.layers.5.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.5/mlp/down_proj/MatMulNBits/output_0.out5_4_17" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_11", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.5/post_attention_layernorm/output_3.out4_10", + "/model/layers.5/mlp/down_proj/MatMulNBits/output_0.out5_4_17" + ], + "const_args": [ + "model.layers.6.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.6/input_layernorm/output_3.out4_11", + "/model/layers.6/input_layernorm/output_0.out4_11" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_6", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.6/input_layernorm/output_0.out4_11" + ], + "const_args": [ + "model.layers.6.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.6.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.6/attn/qk_proj/Add/output_0.out5_4_18" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.6.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.6/input_layernorm/output_0.out4_11" + ], + "const_args": [ + "model.layers.6.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.v_proj.Add.bias.preformat", + "model.layers.6.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.6.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "27", + "13" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.6/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.6/attn/qk_proj/Add/output_0.out5_4_18", + "past_key_values.6.key", + "past_key_values.6.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.6/attn/GroupQueryAttention/output_0.out2_6", + "present.6.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "24", + "12", + "2", + "0", + "25", + "13", + "6", + "0", + "26", + "12" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.6.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.6/attn/GroupQueryAttention/output_0.out2_6" + ], + "const_args": [ + "model.layers.6.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.6/attn/o_proj/MatMulNBits/output_0.out5_4_19" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_12", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.6/input_layernorm/output_3.out4_11", + "/model/layers.6/attn/o_proj/MatMulNBits/output_0.out5_4_19" + ], + "const_args": [ + "model.layers.6.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.6/post_attention_layernorm/output_3.out4_12", + "/model/layers.6/post_attention_layernorm/output_0.out4_12" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_6", + "type": "FlatMLP", + "in_args": [ + "/model/layers.6/post_attention_layernorm/output_0.out4_12" + ], + "const_args": [ + "model.layers.6.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.6.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.6.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.6.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.6.mlp.up_proj.MatMulNBits.qweight", + "model.layers.6.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.6.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.6.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.6/mlp/Mul/output_0.out3_6" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.6.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.6/mlp/Mul/output_0.out3_6" + ], + "const_args": [ + "model.layers.6.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.6/mlp/down_proj/MatMulNBits/output_0.out5_4_20" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_13", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.6/post_attention_layernorm/output_3.out4_12", + "/model/layers.6/mlp/down_proj/MatMulNBits/output_0.out5_4_20" + ], + "const_args": [ + "model.layers.7.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.7/input_layernorm/output_3.out4_13", + "/model/layers.7/input_layernorm/output_0.out4_13" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_7", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.7/input_layernorm/output_0.out4_13" + ], + "const_args": [ + "model.layers.7.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.7.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.7/attn/qk_proj/Add/output_0.out5_4_21" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.7.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.7/input_layernorm/output_0.out4_13" + ], + "const_args": [ + "model.layers.7.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.v_proj.Add.bias.preformat", + "model.layers.7.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.7.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "31", + "15" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.7/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.7/attn/qk_proj/Add/output_0.out5_4_21", + "past_key_values.7.key", + "past_key_values.7.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.7/attn/GroupQueryAttention/output_0.out2_7", + "present.7.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "28", + "14", + "2", + "0", + "29", + "15", + "6", + "0", + "30", + "14" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.7.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.7/attn/GroupQueryAttention/output_0.out2_7" + ], + "const_args": [ + "model.layers.7.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.7/attn/o_proj/MatMulNBits/output_0.out5_4_22" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_14", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.7/input_layernorm/output_3.out4_13", + "/model/layers.7/attn/o_proj/MatMulNBits/output_0.out5_4_22" + ], + "const_args": [ + "model.layers.7.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.7/post_attention_layernorm/output_3.out4_14", + "/model/layers.7/post_attention_layernorm/output_0.out4_14" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_7", + "type": "FlatMLP", + "in_args": [ + "/model/layers.7/post_attention_layernorm/output_0.out4_14" + ], + "const_args": [ + "model.layers.7.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.7.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.7.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.7.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.7.mlp.up_proj.MatMulNBits.qweight", + "model.layers.7.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.7.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.7.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.7/mlp/Mul/output_0.out3_7" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.7.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.7/mlp/Mul/output_0.out3_7" + ], + "const_args": [ + "model.layers.7.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.7/mlp/down_proj/MatMulNBits/output_0.out5_4_23" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_15", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.7/post_attention_layernorm/output_3.out4_14", + "/model/layers.7/mlp/down_proj/MatMulNBits/output_0.out5_4_23" + ], + "const_args": [ + "model.layers.8.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.8/input_layernorm/output_3.out4_15", + "/model/layers.8/input_layernorm/output_0.out4_15" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_8", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.8/input_layernorm/output_0.out4_15" + ], + "const_args": [ + "model.layers.8.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.8.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.8/attn/qk_proj/Add/output_0.out5_4_24" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.8.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.8/input_layernorm/output_0.out4_15" + ], + "const_args": [ + "model.layers.8.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.v_proj.Add.bias.preformat", + "model.layers.8.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.8.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "35", + "17" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.8/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.8/attn/qk_proj/Add/output_0.out5_4_24", + "past_key_values.8.key", + "past_key_values.8.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.8/attn/GroupQueryAttention/output_0.out2_8", + "present.8.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "32", + "16", + "2", + "0", + "33", + "17", + "6", + "0", + "34", + "16" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.8.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.8/attn/GroupQueryAttention/output_0.out2_8" + ], + "const_args": [ + "model.layers.8.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.8/attn/o_proj/MatMulNBits/output_0.out5_4_25" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_16", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.8/input_layernorm/output_3.out4_15", + "/model/layers.8/attn/o_proj/MatMulNBits/output_0.out5_4_25" + ], + "const_args": [ + "model.layers.8.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.8/post_attention_layernorm/output_3.out4_16", + "/model/layers.8/post_attention_layernorm/output_0.out4_16" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_8", + "type": "FlatMLP", + "in_args": [ + "/model/layers.8/post_attention_layernorm/output_0.out4_16" + ], + "const_args": [ + "model.layers.8.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.8.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.8.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.8.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.8.mlp.up_proj.MatMulNBits.qweight", + "model.layers.8.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.8.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.8.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.8/mlp/Mul/output_0.out3_8" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.8.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.8/mlp/Mul/output_0.out3_8" + ], + "const_args": [ + "model.layers.8.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.8/mlp/down_proj/MatMulNBits/output_0.out5_4_26" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_17", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.8/post_attention_layernorm/output_3.out4_16", + "/model/layers.8/mlp/down_proj/MatMulNBits/output_0.out5_4_26" + ], + "const_args": [ + "model.layers.9.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.9/input_layernorm/output_3.out4_17", + "/model/layers.9/input_layernorm/output_0.out4_17" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_9", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.9/input_layernorm/output_0.out4_17" + ], + "const_args": [ + "model.layers.9.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.9.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.9/attn/qk_proj/Add/output_0.out5_4_27" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.9.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.9/input_layernorm/output_0.out4_17" + ], + "const_args": [ + "model.layers.9.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.v_proj.Add.bias.preformat", + "model.layers.9.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.9.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "39", + "19" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.9/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.9/attn/qk_proj/Add/output_0.out5_4_27", + "past_key_values.9.key", + "past_key_values.9.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.9/attn/GroupQueryAttention/output_0.out2_9", + "present.9.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "36", + "18", + "2", + "0", + "37", + "19", + "6", + "0", + "38", + "18" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.9.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.9/attn/GroupQueryAttention/output_0.out2_9" + ], + "const_args": [ + "model.layers.9.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.9/attn/o_proj/MatMulNBits/output_0.out5_4_28" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_18", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.9/input_layernorm/output_3.out4_17", + "/model/layers.9/attn/o_proj/MatMulNBits/output_0.out5_4_28" + ], + "const_args": [ + "model.layers.9.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.9/post_attention_layernorm/output_3.out4_18", + "/model/layers.9/post_attention_layernorm/output_0.out4_18" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_9", + "type": "FlatMLP", + "in_args": [ + "/model/layers.9/post_attention_layernorm/output_0.out4_18" + ], + "const_args": [ + "model.layers.9.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.9.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.9.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.9.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.9.mlp.up_proj.MatMulNBits.qweight", + "model.layers.9.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.9.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.9.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.9/mlp/Mul/output_0.out3_9" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.9.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.9/mlp/Mul/output_0.out3_9" + ], + "const_args": [ + "model.layers.9.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.9/mlp/down_proj/MatMulNBits/output_0.out5_4_29" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_19", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.9/post_attention_layernorm/output_3.out4_18", + "/model/layers.9/mlp/down_proj/MatMulNBits/output_0.out5_4_29" + ], + "const_args": [ + "model.layers.10.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.10/input_layernorm/output_3.out4_19", + "/model/layers.10/input_layernorm/output_0.out4_19" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_10", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.10/input_layernorm/output_0.out4_19" + ], + "const_args": [ + "model.layers.10.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.10.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.10/attn/qk_proj/Add/output_0.out5_4_30" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.10.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.10/input_layernorm/output_0.out4_19" + ], + "const_args": [ + "model.layers.10.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.v_proj.Add.bias.preformat", + "model.layers.10.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.10.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "43", + "21" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.10/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.10/attn/qk_proj/Add/output_0.out5_4_30", + "past_key_values.10.key", + "past_key_values.10.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.10/attn/GroupQueryAttention/output_0.out2_10", + "present.10.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "40", + "20", + "2", + "0", + "41", + "21", + "6", + "0", + "42", + "20" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.10.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.10/attn/GroupQueryAttention/output_0.out2_10" + ], + "const_args": [ + "model.layers.10.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.10/attn/o_proj/MatMulNBits/output_0.out5_4_31" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_20", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.10/input_layernorm/output_3.out4_19", + "/model/layers.10/attn/o_proj/MatMulNBits/output_0.out5_4_31" + ], + "const_args": [ + "model.layers.10.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.10/post_attention_layernorm/output_3.out4_20", + "/model/layers.10/post_attention_layernorm/output_0.out4_20" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_10", + "type": "FlatMLP", + "in_args": [ + "/model/layers.10/post_attention_layernorm/output_0.out4_20" + ], + "const_args": [ + "model.layers.10.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.10.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.10.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.10.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.10.mlp.up_proj.MatMulNBits.qweight", + "model.layers.10.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.10.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.10.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.10/mlp/Mul/output_0.out3_10" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.10.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.10/mlp/Mul/output_0.out3_10" + ], + "const_args": [ + "model.layers.10.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.10/mlp/down_proj/MatMulNBits/output_0.out5_4_32" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_21", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.10/post_attention_layernorm/output_3.out4_20", + "/model/layers.10/mlp/down_proj/MatMulNBits/output_0.out5_4_32" + ], + "const_args": [ + "model.layers.11.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.11/input_layernorm/output_3.out4_21", + "/model/layers.11/input_layernorm/output_0.out4_21" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_11", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.11/input_layernorm/output_0.out4_21" + ], + "const_args": [ + "model.layers.11.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.11.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.11/attn/qk_proj/Add/output_0.out5_4_33" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.11.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.11/input_layernorm/output_0.out4_21" + ], + "const_args": [ + "model.layers.11.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.v_proj.Add.bias.preformat", + "model.layers.11.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.11.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "47", + "23" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.11/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.11/attn/qk_proj/Add/output_0.out5_4_33", + "past_key_values.11.key", + "past_key_values.11.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.11/attn/GroupQueryAttention/output_0.out2_11", + "present.11.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "44", + "22", + "2", + "0", + "45", + "23", + "6", + "0", + "46", + "22" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.11.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.11/attn/GroupQueryAttention/output_0.out2_11" + ], + "const_args": [ + "model.layers.11.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.11/attn/o_proj/MatMulNBits/output_0.out5_4_34" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_22", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.11/input_layernorm/output_3.out4_21", + "/model/layers.11/attn/o_proj/MatMulNBits/output_0.out5_4_34" + ], + "const_args": [ + "model.layers.11.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.11/post_attention_layernorm/output_3.out4_22", + "/model/layers.11/post_attention_layernorm/output_0.out4_22" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_11", + "type": "FlatMLP", + "in_args": [ + "/model/layers.11/post_attention_layernorm/output_0.out4_22" + ], + "const_args": [ + "model.layers.11.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.11.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.11.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.11.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.11.mlp.up_proj.MatMulNBits.qweight", + "model.layers.11.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.11.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.11.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.11/mlp/Mul/output_0.out3_11" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.11.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.11/mlp/Mul/output_0.out3_11" + ], + "const_args": [ + "model.layers.11.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.11/mlp/down_proj/MatMulNBits/output_0.out5_4_35" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_23", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.11/post_attention_layernorm/output_3.out4_22", + "/model/layers.11/mlp/down_proj/MatMulNBits/output_0.out5_4_35" + ], + "const_args": [ + "model.layers.12.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.12/input_layernorm/output_3.out4_23", + "/model/layers.12/input_layernorm/output_0.out4_23" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_12", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.12/input_layernorm/output_0.out4_23" + ], + "const_args": [ + "model.layers.12.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.12.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.12/attn/qk_proj/Add/output_0.out5_4_36" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.12.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.12/input_layernorm/output_0.out4_23" + ], + "const_args": [ + "model.layers.12.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.v_proj.Add.bias.preformat", + "model.layers.12.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.12.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "51", + "25" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.12/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.12/attn/qk_proj/Add/output_0.out5_4_36", + "past_key_values.12.key", + "past_key_values.12.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.12/attn/GroupQueryAttention/output_0.out2_12", + "present.12.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "48", + "24", + "2", + "0", + "49", + "25", + "6", + "0", + "50", + "24" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.12.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.12/attn/GroupQueryAttention/output_0.out2_12" + ], + "const_args": [ + "model.layers.12.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.12/attn/o_proj/MatMulNBits/output_0.out5_4_37" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_24", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.12/input_layernorm/output_3.out4_23", + "/model/layers.12/attn/o_proj/MatMulNBits/output_0.out5_4_37" + ], + "const_args": [ + "model.layers.12.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.12/post_attention_layernorm/output_3.out4_24", + "/model/layers.12/post_attention_layernorm/output_0.out4_24" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_12", + "type": "FlatMLP", + "in_args": [ + "/model/layers.12/post_attention_layernorm/output_0.out4_24" + ], + "const_args": [ + "model.layers.12.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.12.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.12.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.12.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.12.mlp.up_proj.MatMulNBits.qweight", + "model.layers.12.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.12.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.12.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.12/mlp/Mul/output_0.out3_12" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.12.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.12/mlp/Mul/output_0.out3_12" + ], + "const_args": [ + "model.layers.12.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.12/mlp/down_proj/MatMulNBits/output_0.out5_4_38" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_25", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.12/post_attention_layernorm/output_3.out4_24", + "/model/layers.12/mlp/down_proj/MatMulNBits/output_0.out5_4_38" + ], + "const_args": [ + "model.layers.13.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.13/input_layernorm/output_3.out4_25", + "/model/layers.13/input_layernorm/output_0.out4_25" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_13", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.13/input_layernorm/output_0.out4_25" + ], + "const_args": [ + "model.layers.13.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.13.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.13/attn/qk_proj/Add/output_0.out5_4_39" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.13.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.13/input_layernorm/output_0.out4_25" + ], + "const_args": [ + "model.layers.13.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.v_proj.Add.bias.preformat", + "model.layers.13.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.13.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "55", + "27" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.13/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.13/attn/qk_proj/Add/output_0.out5_4_39", + "past_key_values.13.key", + "past_key_values.13.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.13/attn/GroupQueryAttention/output_0.out2_13", + "present.13.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "52", + "26", + "2", + "0", + "53", + "27", + "6", + "0", + "54", + "26" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.13.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.13/attn/GroupQueryAttention/output_0.out2_13" + ], + "const_args": [ + "model.layers.13.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.13/attn/o_proj/MatMulNBits/output_0.out5_4_40" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_26", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.13/input_layernorm/output_3.out4_25", + "/model/layers.13/attn/o_proj/MatMulNBits/output_0.out5_4_40" + ], + "const_args": [ + "model.layers.13.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.13/post_attention_layernorm/output_3.out4_26", + "/model/layers.13/post_attention_layernorm/output_0.out4_26" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_13", + "type": "FlatMLP", + "in_args": [ + "/model/layers.13/post_attention_layernorm/output_0.out4_26" + ], + "const_args": [ + "model.layers.13.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.13.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.13.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.13.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.13.mlp.up_proj.MatMulNBits.qweight", + "model.layers.13.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.13.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.13.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.13/mlp/Mul/output_0.out3_13" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.13.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.13/mlp/Mul/output_0.out3_13" + ], + "const_args": [ + "model.layers.13.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.13/mlp/down_proj/MatMulNBits/output_0.out5_4_41" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_27", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.13/post_attention_layernorm/output_3.out4_26", + "/model/layers.13/mlp/down_proj/MatMulNBits/output_0.out5_4_41" + ], + "const_args": [ + "model.layers.14.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.14/input_layernorm/output_3.out4_27", + "/model/layers.14/input_layernorm/output_0.out4_27" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_14", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.14/input_layernorm/output_0.out4_27" + ], + "const_args": [ + "model.layers.14.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.14.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.14/attn/qk_proj/Add/output_0.out5_4_42" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.14.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.14/input_layernorm/output_0.out4_27" + ], + "const_args": [ + "model.layers.14.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.v_proj.Add.bias.preformat", + "model.layers.14.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.14.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "59", + "29" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.14/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.14/attn/qk_proj/Add/output_0.out5_4_42", + "past_key_values.14.key", + "past_key_values.14.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.14/attn/GroupQueryAttention/output_0.out2_14", + "present.14.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "56", + "28", + "2", + "0", + "57", + "29", + "6", + "0", + "58", + "28" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.14.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.14/attn/GroupQueryAttention/output_0.out2_14" + ], + "const_args": [ + "model.layers.14.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.14/attn/o_proj/MatMulNBits/output_0.out5_4_43" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_28", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.14/input_layernorm/output_3.out4_27", + "/model/layers.14/attn/o_proj/MatMulNBits/output_0.out5_4_43" + ], + "const_args": [ + "model.layers.14.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.14/post_attention_layernorm/output_3.out4_28", + "/model/layers.14/post_attention_layernorm/output_0.out4_28" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_14", + "type": "FlatMLP", + "in_args": [ + "/model/layers.14/post_attention_layernorm/output_0.out4_28" + ], + "const_args": [ + "model.layers.14.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.14.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.14.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.14.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.14.mlp.up_proj.MatMulNBits.qweight", + "model.layers.14.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.14.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.14.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.14/mlp/Mul/output_0.out3_14" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.14.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.14/mlp/Mul/output_0.out3_14" + ], + "const_args": [ + "model.layers.14.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.14/mlp/down_proj/MatMulNBits/output_0.out5_4_44" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_29", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.14/post_attention_layernorm/output_3.out4_28", + "/model/layers.14/mlp/down_proj/MatMulNBits/output_0.out5_4_44" + ], + "const_args": [ + "model.layers.15.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.15/input_layernorm/output_3.out4_29", + "/model/layers.15/input_layernorm/output_0.out4_29" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_15", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.15/input_layernorm/output_0.out4_29" + ], + "const_args": [ + "model.layers.15.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.15.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.15/attn/qk_proj/Add/output_0.out5_4_45" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.15.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.15/input_layernorm/output_0.out4_29" + ], + "const_args": [ + "model.layers.15.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.v_proj.Add.bias.preformat", + "model.layers.15.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.15.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "63", + "31" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.15/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.15/attn/qk_proj/Add/output_0.out5_4_45", + "past_key_values.15.key", + "past_key_values.15.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.15/attn/GroupQueryAttention/output_0.out2_15", + "present.15.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "60", + "30", + "2", + "0", + "61", + "31", + "6", + "0", + "62", + "30" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.15.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.15/attn/GroupQueryAttention/output_0.out2_15" + ], + "const_args": [ + "model.layers.15.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.15/attn/o_proj/MatMulNBits/output_0.out5_4_46" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_30", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.15/input_layernorm/output_3.out4_29", + "/model/layers.15/attn/o_proj/MatMulNBits/output_0.out5_4_46" + ], + "const_args": [ + "model.layers.15.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.15/post_attention_layernorm/output_3.out4_30", + "/model/layers.15/post_attention_layernorm/output_0.out4_30" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_15", + "type": "FlatMLP", + "in_args": [ + "/model/layers.15/post_attention_layernorm/output_0.out4_30" + ], + "const_args": [ + "model.layers.15.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.15.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.15.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.15.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.15.mlp.up_proj.MatMulNBits.qweight", + "model.layers.15.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.15.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.15.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.15/mlp/Mul/output_0.out3_15" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.15.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.15/mlp/Mul/output_0.out3_15" + ], + "const_args": [ + "model.layers.15.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.15/mlp/down_proj/MatMulNBits/output_0.out5_4_47" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_31", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.15/post_attention_layernorm/output_3.out4_30", + "/model/layers.15/mlp/down_proj/MatMulNBits/output_0.out5_4_47" + ], + "const_args": [ + "model.layers.16.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.16/input_layernorm/output_3.out4_31", + "/model/layers.16/input_layernorm/output_0.out4_31" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_16", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.16/input_layernorm/output_0.out4_31" + ], + "const_args": [ + "model.layers.16.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.16.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.16/attn/qk_proj/Add/output_0.out5_4_48" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.16.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.16/input_layernorm/output_0.out4_31" + ], + "const_args": [ + "model.layers.16.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.v_proj.Add.bias.preformat", + "model.layers.16.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.16.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "67", + "33" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.16/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.16/attn/qk_proj/Add/output_0.out5_4_48", + "past_key_values.16.key", + "past_key_values.16.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.16/attn/GroupQueryAttention/output_0.out2_16", + "present.16.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "64", + "32", + "2", + "0", + "65", + "33", + "6", + "0", + "66", + "32" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.16.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.16/attn/GroupQueryAttention/output_0.out2_16" + ], + "const_args": [ + "model.layers.16.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.16/attn/o_proj/MatMulNBits/output_0.out5_4_49" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_32", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.16/input_layernorm/output_3.out4_31", + "/model/layers.16/attn/o_proj/MatMulNBits/output_0.out5_4_49" + ], + "const_args": [ + "model.layers.16.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.16/post_attention_layernorm/output_3.out4_32", + "/model/layers.16/post_attention_layernorm/output_0.out4_32" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_16", + "type": "FlatMLP", + "in_args": [ + "/model/layers.16/post_attention_layernorm/output_0.out4_32" + ], + "const_args": [ + "model.layers.16.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.16.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.16.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.16.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.16.mlp.up_proj.MatMulNBits.qweight", + "model.layers.16.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.16.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.16.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.16/mlp/Mul/output_0.out3_16" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.16.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.16/mlp/Mul/output_0.out3_16" + ], + "const_args": [ + "model.layers.16.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.16/mlp/down_proj/MatMulNBits/output_0.out5_4_50" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_33", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.16/post_attention_layernorm/output_3.out4_32", + "/model/layers.16/mlp/down_proj/MatMulNBits/output_0.out5_4_50" + ], + "const_args": [ + "model.layers.17.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.17/input_layernorm/output_3.out4_33", + "/model/layers.17/input_layernorm/output_0.out4_33" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_17", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.17/input_layernorm/output_0.out4_33" + ], + "const_args": [ + "model.layers.17.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.17.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.17/attn/qk_proj/Add/output_0.out5_4_51" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.17.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.17/input_layernorm/output_0.out4_33" + ], + "const_args": [ + "model.layers.17.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.v_proj.Add.bias.preformat", + "model.layers.17.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.17.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "71", + "35" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.17/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.17/attn/qk_proj/Add/output_0.out5_4_51", + "past_key_values.17.key", + "past_key_values.17.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.17/attn/GroupQueryAttention/output_0.out2_17", + "present.17.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "68", + "34", + "2", + "0", + "69", + "35", + "6", + "0", + "70", + "34" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.17.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.17/attn/GroupQueryAttention/output_0.out2_17" + ], + "const_args": [ + "model.layers.17.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.17/attn/o_proj/MatMulNBits/output_0.out5_4_52" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_34", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.17/input_layernorm/output_3.out4_33", + "/model/layers.17/attn/o_proj/MatMulNBits/output_0.out5_4_52" + ], + "const_args": [ + "model.layers.17.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.17/post_attention_layernorm/output_3.out4_34", + "/model/layers.17/post_attention_layernorm/output_0.out4_34" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_17", + "type": "FlatMLP", + "in_args": [ + "/model/layers.17/post_attention_layernorm/output_0.out4_34" + ], + "const_args": [ + "model.layers.17.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.17.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.17.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.17.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.17.mlp.up_proj.MatMulNBits.qweight", + "model.layers.17.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.17.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.17.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.17/mlp/Mul/output_0.out3_17" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.17.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.17/mlp/Mul/output_0.out3_17" + ], + "const_args": [ + "model.layers.17.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.17/mlp/down_proj/MatMulNBits/output_0.out5_4_53" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_35", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.17/post_attention_layernorm/output_3.out4_34", + "/model/layers.17/mlp/down_proj/MatMulNBits/output_0.out5_4_53" + ], + "const_args": [ + "model.layers.18.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.18/input_layernorm/output_3.out4_35", + "/model/layers.18/input_layernorm/output_0.out4_35" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_18", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.18/input_layernorm/output_0.out4_35" + ], + "const_args": [ + "model.layers.18.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.18.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.18/attn/qk_proj/Add/output_0.out5_4_54" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.18.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.18/input_layernorm/output_0.out4_35" + ], + "const_args": [ + "model.layers.18.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.v_proj.Add.bias.preformat", + "model.layers.18.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.18.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "75", + "37" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.18/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.18/attn/qk_proj/Add/output_0.out5_4_54", + "past_key_values.18.key", + "past_key_values.18.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.18/attn/GroupQueryAttention/output_0.out2_18", + "present.18.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "72", + "36", + "2", + "0", + "73", + "37", + "6", + "0", + "74", + "36" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.18.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.18/attn/GroupQueryAttention/output_0.out2_18" + ], + "const_args": [ + "model.layers.18.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.18/attn/o_proj/MatMulNBits/output_0.out5_4_55" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_36", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.18/input_layernorm/output_3.out4_35", + "/model/layers.18/attn/o_proj/MatMulNBits/output_0.out5_4_55" + ], + "const_args": [ + "model.layers.18.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.18/post_attention_layernorm/output_3.out4_36", + "/model/layers.18/post_attention_layernorm/output_0.out4_36" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_18", + "type": "FlatMLP", + "in_args": [ + "/model/layers.18/post_attention_layernorm/output_0.out4_36" + ], + "const_args": [ + "model.layers.18.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.18.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.18.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.18.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.18.mlp.up_proj.MatMulNBits.qweight", + "model.layers.18.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.18.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.18.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.18/mlp/Mul/output_0.out3_18" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.18.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.18/mlp/Mul/output_0.out3_18" + ], + "const_args": [ + "model.layers.18.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.18/mlp/down_proj/MatMulNBits/output_0.out5_4_56" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_37", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.18/post_attention_layernorm/output_3.out4_36", + "/model/layers.18/mlp/down_proj/MatMulNBits/output_0.out5_4_56" + ], + "const_args": [ + "model.layers.19.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.19/input_layernorm/output_3.out4_37", + "/model/layers.19/input_layernorm/output_0.out4_37" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_19", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.19/input_layernorm/output_0.out4_37" + ], + "const_args": [ + "model.layers.19.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.19.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.19/attn/qk_proj/Add/output_0.out5_4_57" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.19.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.19/input_layernorm/output_0.out4_37" + ], + "const_args": [ + "model.layers.19.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.v_proj.Add.bias.preformat", + "model.layers.19.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.19.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "79", + "39" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.19/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.19/attn/qk_proj/Add/output_0.out5_4_57", + "past_key_values.19.key", + "past_key_values.19.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.19/attn/GroupQueryAttention/output_0.out2_19", + "present.19.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "76", + "38", + "2", + "0", + "77", + "39", + "6", + "0", + "78", + "38" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.19.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.19/attn/GroupQueryAttention/output_0.out2_19" + ], + "const_args": [ + "model.layers.19.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.19/attn/o_proj/MatMulNBits/output_0.out5_4_58" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_38", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.19/input_layernorm/output_3.out4_37", + "/model/layers.19/attn/o_proj/MatMulNBits/output_0.out5_4_58" + ], + "const_args": [ + "model.layers.19.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.19/post_attention_layernorm/output_3.out4_38", + "/model/layers.19/post_attention_layernorm/output_0.out4_38" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_19", + "type": "FlatMLP", + "in_args": [ + "/model/layers.19/post_attention_layernorm/output_0.out4_38" + ], + "const_args": [ + "model.layers.19.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.19.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.19.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.19.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.19.mlp.up_proj.MatMulNBits.qweight", + "model.layers.19.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.19.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.19.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.19/mlp/Mul/output_0.out3_19" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.19.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.19/mlp/Mul/output_0.out3_19" + ], + "const_args": [ + "model.layers.19.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.19/mlp/down_proj/MatMulNBits/output_0.out5_4_59" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_39", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.19/post_attention_layernorm/output_3.out4_38", + "/model/layers.19/mlp/down_proj/MatMulNBits/output_0.out5_4_59" + ], + "const_args": [ + "model.layers.20.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.20/input_layernorm/output_3.out4_39", + "/model/layers.20/input_layernorm/output_0.out4_39" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_20", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.20/input_layernorm/output_0.out4_39" + ], + "const_args": [ + "model.layers.20.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.20.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.20/attn/qk_proj/Add/output_0.out5_4_60" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.20.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.20/input_layernorm/output_0.out4_39" + ], + "const_args": [ + "model.layers.20.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.v_proj.Add.bias.preformat", + "model.layers.20.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.20.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "83", + "41" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.20/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.20/attn/qk_proj/Add/output_0.out5_4_60", + "past_key_values.20.key", + "past_key_values.20.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.20/attn/GroupQueryAttention/output_0.out2_20", + "present.20.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "80", + "40", + "2", + "0", + "81", + "41", + "6", + "0", + "82", + "40" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.20.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.20/attn/GroupQueryAttention/output_0.out2_20" + ], + "const_args": [ + "model.layers.20.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.20/attn/o_proj/MatMulNBits/output_0.out5_4_61" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_40", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.20/input_layernorm/output_3.out4_39", + "/model/layers.20/attn/o_proj/MatMulNBits/output_0.out5_4_61" + ], + "const_args": [ + "model.layers.20.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.20/post_attention_layernorm/output_3.out4_40", + "/model/layers.20/post_attention_layernorm/output_0.out4_40" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_20", + "type": "FlatMLP", + "in_args": [ + "/model/layers.20/post_attention_layernorm/output_0.out4_40" + ], + "const_args": [ + "model.layers.20.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.20.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.20.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.20.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.20.mlp.up_proj.MatMulNBits.qweight", + "model.layers.20.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.20.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.20.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.20/mlp/Mul/output_0.out3_20" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.20.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.20/mlp/Mul/output_0.out3_20" + ], + "const_args": [ + "model.layers.20.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.20/mlp/down_proj/MatMulNBits/output_0.out5_4_62" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_41", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.20/post_attention_layernorm/output_3.out4_40", + "/model/layers.20/mlp/down_proj/MatMulNBits/output_0.out5_4_62" + ], + "const_args": [ + "model.layers.21.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.21/input_layernorm/output_3.out4_41", + "/model/layers.21/input_layernorm/output_0.out4_41" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_21", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.21/input_layernorm/output_0.out4_41" + ], + "const_args": [ + "model.layers.21.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.21.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.21/attn/qk_proj/Add/output_0.out5_4_63" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.21.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.21/input_layernorm/output_0.out4_41" + ], + "const_args": [ + "model.layers.21.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.v_proj.Add.bias.preformat", + "model.layers.21.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.21.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "87", + "43" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.21/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.21/attn/qk_proj/Add/output_0.out5_4_63", + "past_key_values.21.key", + "past_key_values.21.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.21/attn/GroupQueryAttention/output_0.out2_21", + "present.21.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "84", + "42", + "2", + "0", + "85", + "43", + "6", + "0", + "86", + "42" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.21.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.21/attn/GroupQueryAttention/output_0.out2_21" + ], + "const_args": [ + "model.layers.21.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.21/attn/o_proj/MatMulNBits/output_0.out5_4_64" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_42", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.21/input_layernorm/output_3.out4_41", + "/model/layers.21/attn/o_proj/MatMulNBits/output_0.out5_4_64" + ], + "const_args": [ + "model.layers.21.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.21/post_attention_layernorm/output_3.out4_42", + "/model/layers.21/post_attention_layernorm/output_0.out4_42" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_21", + "type": "FlatMLP", + "in_args": [ + "/model/layers.21/post_attention_layernorm/output_0.out4_42" + ], + "const_args": [ + "model.layers.21.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.21.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.21.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.21.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.21.mlp.up_proj.MatMulNBits.qweight", + "model.layers.21.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.21.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.21.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.21/mlp/Mul/output_0.out3_21" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.21.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.21/mlp/Mul/output_0.out3_21" + ], + "const_args": [ + "model.layers.21.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.21/mlp/down_proj/MatMulNBits/output_0.out5_4_65" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_43", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.21/post_attention_layernorm/output_3.out4_42", + "/model/layers.21/mlp/down_proj/MatMulNBits/output_0.out5_4_65" + ], + "const_args": [ + "model.layers.22.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.22/input_layernorm/output_3.out4_43", + "/model/layers.22/input_layernorm/output_0.out4_43" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_22", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.22/input_layernorm/output_0.out4_43" + ], + "const_args": [ + "model.layers.22.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.22.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.22/attn/qk_proj/Add/output_0.out5_4_66" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.22.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.22/input_layernorm/output_0.out4_43" + ], + "const_args": [ + "model.layers.22.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.v_proj.Add.bias.preformat", + "model.layers.22.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.22.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "91", + "45" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.22/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.22/attn/qk_proj/Add/output_0.out5_4_66", + "past_key_values.22.key", + "past_key_values.22.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.22/attn/GroupQueryAttention/output_0.out2_22", + "present.22.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "88", + "44", + "2", + "0", + "89", + "45", + "6", + "0", + "90", + "44" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.22.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.22/attn/GroupQueryAttention/output_0.out2_22" + ], + "const_args": [ + "model.layers.22.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.22/attn/o_proj/MatMulNBits/output_0.out5_4_67" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_44", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.22/input_layernorm/output_3.out4_43", + "/model/layers.22/attn/o_proj/MatMulNBits/output_0.out5_4_67" + ], + "const_args": [ + "model.layers.22.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.22/post_attention_layernorm/output_3.out4_44", + "/model/layers.22/post_attention_layernorm/output_0.out4_44" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_22", + "type": "FlatMLP", + "in_args": [ + "/model/layers.22/post_attention_layernorm/output_0.out4_44" + ], + "const_args": [ + "model.layers.22.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.22.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.22.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.22.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.22.mlp.up_proj.MatMulNBits.qweight", + "model.layers.22.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.22.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.22.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.22/mlp/Mul/output_0.out3_22" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.22.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.22/mlp/Mul/output_0.out3_22" + ], + "const_args": [ + "model.layers.22.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.22/mlp/down_proj/MatMulNBits/output_0.out5_4_68" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_45", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.22/post_attention_layernorm/output_3.out4_44", + "/model/layers.22/mlp/down_proj/MatMulNBits/output_0.out5_4_68" + ], + "const_args": [ + "model.layers.23.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.23/input_layernorm/output_3.out4_45", + "/model/layers.23/input_layernorm/output_0.out4_45" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_23", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.23/input_layernorm/output_0.out4_45" + ], + "const_args": [ + "model.layers.23.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.23.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.23/attn/qk_proj/Add/output_0.out5_4_69" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.23.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.23/input_layernorm/output_0.out4_45" + ], + "const_args": [ + "model.layers.23.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.v_proj.Add.bias.preformat", + "model.layers.23.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.23.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "95", + "47" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.23/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.23/attn/qk_proj/Add/output_0.out5_4_69", + "past_key_values.23.key", + "past_key_values.23.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.23/attn/GroupQueryAttention/output_0.out2_23", + "present.23.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "92", + "46", + "2", + "0", + "93", + "47", + "6", + "0", + "94", + "46" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.23.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.23/attn/GroupQueryAttention/output_0.out2_23" + ], + "const_args": [ + "model.layers.23.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.23/attn/o_proj/MatMulNBits/output_0.out5_4_70" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_46", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.23/input_layernorm/output_3.out4_45", + "/model/layers.23/attn/o_proj/MatMulNBits/output_0.out5_4_70" + ], + "const_args": [ + "model.layers.23.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.23/post_attention_layernorm/output_3.out4_46", + "/model/layers.23/post_attention_layernorm/output_0.out4_46" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_23", + "type": "FlatMLP", + "in_args": [ + "/model/layers.23/post_attention_layernorm/output_0.out4_46" + ], + "const_args": [ + "model.layers.23.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.23.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.23.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.23.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.23.mlp.up_proj.MatMulNBits.qweight", + "model.layers.23.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.23.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.23.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.23/mlp/Mul/output_0.out3_23" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.23.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.23/mlp/Mul/output_0.out3_23" + ], + "const_args": [ + "model.layers.23.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.23/mlp/down_proj/MatMulNBits/output_0.out5_4_71" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_47", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.23/post_attention_layernorm/output_3.out4_46", + "/model/layers.23/mlp/down_proj/MatMulNBits/output_0.out5_4_71" + ], + "const_args": [ + "model.layers.24.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.24/input_layernorm/output_3.out4_47", + "/model/layers.24/input_layernorm/output_0.out4_47" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_24", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.24/input_layernorm/output_0.out4_47" + ], + "const_args": [ + "model.layers.24.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.24.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.24.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.24.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.24/attn/qk_proj/Add/output_0.out5_4_72" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.24.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.24/input_layernorm/output_0.out4_47" + ], + "const_args": [ + "model.layers.24.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.24.attn.v_proj.Add.bias.preformat", + "model.layers.24.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.24.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.24.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "99", + "49" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.24/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.24/attn/qk_proj/Add/output_0.out5_4_72", + "past_key_values.24.key", + "past_key_values.24.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.24/attn/GroupQueryAttention/output_0.out2_24", + "present.24.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "96", + "48", + "2", + "0", + "97", + "49", + "6", + "0", + "98", + "48" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.24.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.24/attn/GroupQueryAttention/output_0.out2_24" + ], + "const_args": [ + "model.layers.24.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.24.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.24.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.24.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.24/attn/o_proj/MatMulNBits/output_0.out5_4_73" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_48", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.24/input_layernorm/output_3.out4_47", + "/model/layers.24/attn/o_proj/MatMulNBits/output_0.out5_4_73" + ], + "const_args": [ + "model.layers.24.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.24/post_attention_layernorm/output_3.out4_48", + "/model/layers.24/post_attention_layernorm/output_0.out4_48" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_24", + "type": "FlatMLP", + "in_args": [ + "/model/layers.24/post_attention_layernorm/output_0.out4_48" + ], + "const_args": [ + "model.layers.24.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.24.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.24.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.24.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.24.mlp.up_proj.MatMulNBits.qweight", + "model.layers.24.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.24.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.24.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.24/mlp/Mul/output_0.out3_24" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.24.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.24/mlp/Mul/output_0.out3_24" + ], + "const_args": [ + "model.layers.24.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.24.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.24.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.24.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.24/mlp/down_proj/MatMulNBits/output_0.out5_4_74" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_49", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.24/post_attention_layernorm/output_3.out4_48", + "/model/layers.24/mlp/down_proj/MatMulNBits/output_0.out5_4_74" + ], + "const_args": [ + "model.layers.25.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.25/input_layernorm/output_3.out4_49", + "/model/layers.25/input_layernorm/output_0.out4_49" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_25", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.25/input_layernorm/output_0.out4_49" + ], + "const_args": [ + "model.layers.25.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.25.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.25.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.25.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.25/attn/qk_proj/Add/output_0.out5_4_75" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.25.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.25/input_layernorm/output_0.out4_49" + ], + "const_args": [ + "model.layers.25.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.25.attn.v_proj.Add.bias.preformat", + "model.layers.25.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.25.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.25.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "103", + "51" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.25/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.25/attn/qk_proj/Add/output_0.out5_4_75", + "past_key_values.25.key", + "past_key_values.25.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.25/attn/GroupQueryAttention/output_0.out2_25", + "present.25.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "100", + "50", + "2", + "0", + "101", + "51", + "6", + "0", + "102", + "50" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.25.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.25/attn/GroupQueryAttention/output_0.out2_25" + ], + "const_args": [ + "model.layers.25.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.25.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.25.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.25.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.25/attn/o_proj/MatMulNBits/output_0.out5_4_76" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_50", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.25/input_layernorm/output_3.out4_49", + "/model/layers.25/attn/o_proj/MatMulNBits/output_0.out5_4_76" + ], + "const_args": [ + "model.layers.25.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.25/post_attention_layernorm/output_3.out4_50", + "/model/layers.25/post_attention_layernorm/output_0.out4_50" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_25", + "type": "FlatMLP", + "in_args": [ + "/model/layers.25/post_attention_layernorm/output_0.out4_50" + ], + "const_args": [ + "model.layers.25.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.25.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.25.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.25.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.25.mlp.up_proj.MatMulNBits.qweight", + "model.layers.25.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.25.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.25.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.25/mlp/Mul/output_0.out3_25" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.25.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.25/mlp/Mul/output_0.out3_25" + ], + "const_args": [ + "model.layers.25.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.25.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.25.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.25.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.25/mlp/down_proj/MatMulNBits/output_0.out5_4_77" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_51", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.25/post_attention_layernorm/output_3.out4_50", + "/model/layers.25/mlp/down_proj/MatMulNBits/output_0.out5_4_77" + ], + "const_args": [ + "model.layers.26.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.26/input_layernorm/output_3.out4_51", + "/model/layers.26/input_layernorm/output_0.out4_51" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_26", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.26/input_layernorm/output_0.out4_51" + ], + "const_args": [ + "model.layers.26.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.26.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.26.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.26.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.26/attn/qk_proj/Add/output_0.out5_4_78" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.26.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.26/input_layernorm/output_0.out4_51" + ], + "const_args": [ + "model.layers.26.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.26.attn.v_proj.Add.bias.preformat", + "model.layers.26.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.26.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.26.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "107", + "53" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.26/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.26/attn/qk_proj/Add/output_0.out5_4_78", + "past_key_values.26.key", + "past_key_values.26.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.26/attn/GroupQueryAttention/output_0.out2_26", + "present.26.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "104", + "52", + "2", + "0", + "105", + "53", + "6", + "0", + "106", + "52" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.26.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.26/attn/GroupQueryAttention/output_0.out2_26" + ], + "const_args": [ + "model.layers.26.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.26.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.26.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.26.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.26/attn/o_proj/MatMulNBits/output_0.out5_4_79" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_52", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.26/input_layernorm/output_3.out4_51", + "/model/layers.26/attn/o_proj/MatMulNBits/output_0.out5_4_79" + ], + "const_args": [ + "model.layers.26.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.26/post_attention_layernorm/output_3.out4_52", + "/model/layers.26/post_attention_layernorm/output_0.out4_52" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_26", + "type": "FlatMLP", + "in_args": [ + "/model/layers.26/post_attention_layernorm/output_0.out4_52" + ], + "const_args": [ + "model.layers.26.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.26.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.26.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.26.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.26.mlp.up_proj.MatMulNBits.qweight", + "model.layers.26.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.26.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.26.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.26/mlp/Mul/output_0.out3_26" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.26.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.26/mlp/Mul/output_0.out3_26" + ], + "const_args": [ + "model.layers.26.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.26.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.26.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.26.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.26/mlp/down_proj/MatMulNBits/output_0.out5_4_80" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_53", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.26/post_attention_layernorm/output_3.out4_52", + "/model/layers.26/mlp/down_proj/MatMulNBits/output_0.out5_4_80" + ], + "const_args": [ + "model.layers.27.input_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.27/input_layernorm/output_3.out4_53", + "/model/layers.27/input_layernorm/output_0.out4_53" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "MatMulNBits_2_27", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.27/input_layernorm/output_0.out4_53" + ], + "const_args": [ + "model.layers.27.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.27.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.27.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.27.attn.qk_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.27/attn/qk_proj/Add/output_0.out5_4_81" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "4096" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "layers.27.attn.v_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.27/input_layernorm/output_0.out4_53" + ], + "const_args": [ + "model.layers.27.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.27.attn.v_proj.Add.bias.preformat", + "model.layers.27.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.27.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.27.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "512" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "111", + "55" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "5", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "/model/layers.27/attn/GroupQueryAttention", + "type": "FLATMHA", + "in_args": [ + "/model/layers.27/attn/qk_proj/Add/output_0.out5_4_81", + "past_key_values.27.key", + "past_key_values.27.value", + "attention_mask_const_uint", + "sin_cos_cache_token" + ], + "const_args": [], + "out_args": [ + "/model/layers.27/attn/GroupQueryAttention/output_0.out2_27", + "present.27.key" + ], + "attrs": { + "num_heads": { + "type": "int", + "value": [ + "28" + ] + }, + "kv_num_heads": { + "type": "int", + "value": [ + "4" + ] + }, + "scale": { + "type": "float", + "value": [ + "0.0883883461356163" + ] + }, + "softcap": { + "type": "float", + "value": [ + "0.0" + ] + }, + "do_rotary": { + "type": "int", + "value": [ + "0" + ] + }, + "rotary_interleaved": { + "type": "int", + "value": [ + "0" + ] + }, + "input_shape": { + "type": "int", + "value": [ + "4", + "28", + "1", + "4096", + "128" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "4", + "1", + "0", + "0", + "1", + "0", + "108", + "54", + "2", + "0", + "109", + "55", + "6", + "0", + "110", + "54" + ] + }, + "update_tensor_offsets": { + "type": "int", + "value": [ + "4", + "0", + "0", + "256", + "6", + "0", + "0", + "256" + ] + } + } + }, + { + "name": "layers.27.attn.o_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.27/attn/GroupQueryAttention/output_0.out2_27" + ], + "const_args": [ + "model.layers.27.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.27.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.27.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.27.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.27/attn/o_proj/MatMulNBits/output_0.out5_4_82" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_54", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.27/input_layernorm/output_3.out4_53", + "/model/layers.27/attn/o_proj/MatMulNBits/output_0.out5_4_82" + ], + "const_args": [ + "model.layers.27.post_attention_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.27/post_attention_layernorm/output_3.out4_54", + "/model/layers.27/post_attention_layernorm/output_0.out4_54" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "FlatMLP_3_27", + "type": "FlatMLP", + "in_args": [ + "/model/layers.27/post_attention_layernorm/output_0.out4_54" + ], + "const_args": [ + "model.layers.27.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.27.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.27.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.27.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.27.mlp.up_proj.MatMulNBits.qweight", + "model.layers.27.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.27.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.27.mlp.up_proj.MatMulNBits.bias.f" + ], + "out_args": [ + "/model/layers.27/mlp/Mul/output_0.out3_27" + ], + "attrs": { + "input_shape": { + "type": "int", + "value": [ + "1", + "3584", + "18944" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float", + "uint8", + "float" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16" + ] + } + } + }, + { + "name": "layers.27.mlp.down_proj", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.27/mlp/Mul/output_0.out3_27" + ], + "const_args": [ + "model.layers.27.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.27.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.27.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.27.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.27/mlp/down_proj/MatMulNBits/output_0.out5_4_83" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "18944" + ] + }, + "N": { + "type": "int", + "value": [ + "3584" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "FlatRMSAdd_4_55", + "type": "FlatRMSAdd", + "in_args": [ + "/model/layers.27/post_attention_layernorm/output_3.out4_54", + "/model/layers.27/mlp/down_proj/MatMulNBits/output_0.out5_4_83" + ], + "const_args": [ + "model.layers.28.final_norm_layernorm.weight.bf" + ], + "out_args": [ + "/model/layers.28/final_norm_layernorm/output_0.dummy", + "/model/layers.28/final_norm_layernorm/output_0.out4_55" + ], + "attrs": { + "a_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "in_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "out_dtypes": { + "type": "str", + "value": [ + "bfloat16", + "bfloat16" + ] + }, + "c_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "b_shape": { + "type": "int", + "value": [ + "1", + "1", + "3584" + ] + }, + "is_gamma_ifm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/lm_head/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.28/final_norm_layernorm/output_0.out4_55" + ], + "const_args": [ + "lm_head.MatMulNBits.qweight.preformat", + "lm_head.MatMulNBits.bias.preformat", + "lm_head.MatMulNBits.scales.preformat", + "lm_head.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "logits.out5_4_84" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "3584" + ] + }, + "N": { + "type": "int", + "value": [ + "152064" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "flat" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + } + ], + "fused_tensors": { + "in": { + "buffer_size": 21616, + "xrt_arg_id": 0, + "packed_tensors": [ + "/model/layers.0/input_layernorm/output_0.out5_4_0", + "attention_mask_const_uint", + "/model/embed_tokens/Gather/output_0.out4_0" + ] + }, + "out": { + "buffer_size": 311296, + "xrt_arg_id": 1, + "packed_tensors": [ + "/model/layers.28/final_norm_layernorm/output_0.dummy", + "logits.out5_4_84" + ] + }, + "scratch": { + "buffer_size": 2688000, + "xrt_arg_id": 2, + "packed_tensors": [ + "/model/layers.0/attn/qk_proj/Add/output_0.out5_4_0", + "/model/layers.0/attn/GroupQueryAttention/output_0.out2_0", + "/model/layers.0/attn/o_proj/MatMulNBits/output_0.out5_4_1", + "/model/layers.0/post_attention_layernorm/output_3.out4_0", + "/model/layers.0/post_attention_layernorm/output_0.out4_0", + "/model/layers.0/mlp/Mul/output_0.out3_0", + "/model/layers.0/mlp/down_proj/MatMulNBits/output_0.out5_4_2", + "/model/layers.1/input_layernorm/output_3.out4_1", + "/model/layers.1/input_layernorm/output_0.out4_1", + "/model/layers.1/attn/qk_proj/Add/output_0.out5_4_3", + "/model/layers.1/attn/GroupQueryAttention/output_0.out2_1", + "/model/layers.1/attn/o_proj/MatMulNBits/output_0.out5_4_4", + "/model/layers.1/post_attention_layernorm/output_3.out4_2", + "/model/layers.1/post_attention_layernorm/output_0.out4_2", + "/model/layers.1/mlp/Mul/output_0.out3_1", + "/model/layers.1/mlp/down_proj/MatMulNBits/output_0.out5_4_5", + "/model/layers.2/input_layernorm/output_3.out4_3", + "/model/layers.2/input_layernorm/output_0.out4_3", + "/model/layers.2/attn/qk_proj/Add/output_0.out5_4_6", + "/model/layers.2/attn/GroupQueryAttention/output_0.out2_2", + "/model/layers.2/attn/o_proj/MatMulNBits/output_0.out5_4_7", + "/model/layers.2/post_attention_layernorm/output_3.out4_4", + "/model/layers.2/post_attention_layernorm/output_0.out4_4", + "/model/layers.2/mlp/Mul/output_0.out3_2", + "/model/layers.2/mlp/down_proj/MatMulNBits/output_0.out5_4_8", + "/model/layers.3/input_layernorm/output_3.out4_5", + "/model/layers.3/input_layernorm/output_0.out4_5", + "/model/layers.3/attn/qk_proj/Add/output_0.out5_4_9", + "/model/layers.3/attn/GroupQueryAttention/output_0.out2_3", + "/model/layers.3/attn/o_proj/MatMulNBits/output_0.out5_4_10", + "/model/layers.3/post_attention_layernorm/output_3.out4_6", + "/model/layers.3/post_attention_layernorm/output_0.out4_6", + "/model/layers.3/mlp/Mul/output_0.out3_3", + "/model/layers.3/mlp/down_proj/MatMulNBits/output_0.out5_4_11", + "/model/layers.4/input_layernorm/output_3.out4_7", + "/model/layers.4/input_layernorm/output_0.out4_7", + "/model/layers.4/attn/qk_proj/Add/output_0.out5_4_12", + "/model/layers.4/attn/GroupQueryAttention/output_0.out2_4", + "/model/layers.4/attn/o_proj/MatMulNBits/output_0.out5_4_13", + "/model/layers.4/post_attention_layernorm/output_3.out4_8", + "/model/layers.4/post_attention_layernorm/output_0.out4_8", + "/model/layers.4/mlp/Mul/output_0.out3_4", + "/model/layers.4/mlp/down_proj/MatMulNBits/output_0.out5_4_14", + "/model/layers.5/input_layernorm/output_3.out4_9", + "/model/layers.5/input_layernorm/output_0.out4_9", + "/model/layers.5/attn/qk_proj/Add/output_0.out5_4_15", + "/model/layers.5/attn/GroupQueryAttention/output_0.out2_5", + "/model/layers.5/attn/o_proj/MatMulNBits/output_0.out5_4_16", + "/model/layers.5/post_attention_layernorm/output_3.out4_10", + "/model/layers.5/post_attention_layernorm/output_0.out4_10", + "/model/layers.5/mlp/Mul/output_0.out3_5", + "/model/layers.5/mlp/down_proj/MatMulNBits/output_0.out5_4_17", + "/model/layers.6/input_layernorm/output_3.out4_11", + "/model/layers.6/input_layernorm/output_0.out4_11", + "/model/layers.6/attn/qk_proj/Add/output_0.out5_4_18", + "/model/layers.6/attn/GroupQueryAttention/output_0.out2_6", + "/model/layers.6/attn/o_proj/MatMulNBits/output_0.out5_4_19", + "/model/layers.6/post_attention_layernorm/output_3.out4_12", + "/model/layers.6/post_attention_layernorm/output_0.out4_12", + "/model/layers.6/mlp/Mul/output_0.out3_6", + "/model/layers.6/mlp/down_proj/MatMulNBits/output_0.out5_4_20", + "/model/layers.7/input_layernorm/output_3.out4_13", + "/model/layers.7/input_layernorm/output_0.out4_13", + "/model/layers.7/attn/qk_proj/Add/output_0.out5_4_21", + "/model/layers.7/attn/GroupQueryAttention/output_0.out2_7", + "/model/layers.7/attn/o_proj/MatMulNBits/output_0.out5_4_22", + "/model/layers.7/post_attention_layernorm/output_3.out4_14", + "/model/layers.7/post_attention_layernorm/output_0.out4_14", + "/model/layers.7/mlp/Mul/output_0.out3_7", + "/model/layers.7/mlp/down_proj/MatMulNBits/output_0.out5_4_23", + "/model/layers.8/input_layernorm/output_3.out4_15", + "/model/layers.8/input_layernorm/output_0.out4_15", + "/model/layers.8/attn/qk_proj/Add/output_0.out5_4_24", + "/model/layers.8/attn/GroupQueryAttention/output_0.out2_8", + "/model/layers.8/attn/o_proj/MatMulNBits/output_0.out5_4_25", + "/model/layers.8/post_attention_layernorm/output_3.out4_16", + "/model/layers.8/post_attention_layernorm/output_0.out4_16", + "/model/layers.8/mlp/Mul/output_0.out3_8", + "/model/layers.8/mlp/down_proj/MatMulNBits/output_0.out5_4_26", + "/model/layers.9/input_layernorm/output_3.out4_17", + "/model/layers.9/input_layernorm/output_0.out4_17", + "/model/layers.9/attn/qk_proj/Add/output_0.out5_4_27", + "/model/layers.9/attn/GroupQueryAttention/output_0.out2_9", + "/model/layers.9/attn/o_proj/MatMulNBits/output_0.out5_4_28", + "/model/layers.9/post_attention_layernorm/output_3.out4_18", + "/model/layers.9/post_attention_layernorm/output_0.out4_18", + "/model/layers.9/mlp/Mul/output_0.out3_9", + "/model/layers.9/mlp/down_proj/MatMulNBits/output_0.out5_4_29", + "/model/layers.10/input_layernorm/output_3.out4_19", + "/model/layers.10/input_layernorm/output_0.out4_19", + "/model/layers.10/attn/qk_proj/Add/output_0.out5_4_30", + "/model/layers.10/attn/GroupQueryAttention/output_0.out2_10", + "/model/layers.10/attn/o_proj/MatMulNBits/output_0.out5_4_31", + "/model/layers.10/post_attention_layernorm/output_3.out4_20", + "/model/layers.10/post_attention_layernorm/output_0.out4_20", + "/model/layers.10/mlp/Mul/output_0.out3_10", + "/model/layers.10/mlp/down_proj/MatMulNBits/output_0.out5_4_32", + "/model/layers.11/input_layernorm/output_3.out4_21", + "/model/layers.11/input_layernorm/output_0.out4_21", + "/model/layers.11/attn/qk_proj/Add/output_0.out5_4_33", + "/model/layers.11/attn/GroupQueryAttention/output_0.out2_11", + "/model/layers.11/attn/o_proj/MatMulNBits/output_0.out5_4_34", + "/model/layers.11/post_attention_layernorm/output_3.out4_22", + "/model/layers.11/post_attention_layernorm/output_0.out4_22", + "/model/layers.11/mlp/Mul/output_0.out3_11", + "/model/layers.11/mlp/down_proj/MatMulNBits/output_0.out5_4_35", + "/model/layers.12/input_layernorm/output_3.out4_23", + "/model/layers.12/input_layernorm/output_0.out4_23", + "/model/layers.12/attn/qk_proj/Add/output_0.out5_4_36", + "/model/layers.12/attn/GroupQueryAttention/output_0.out2_12", + "/model/layers.12/attn/o_proj/MatMulNBits/output_0.out5_4_37", + "/model/layers.12/post_attention_layernorm/output_3.out4_24", + "/model/layers.12/post_attention_layernorm/output_0.out4_24", + "/model/layers.12/mlp/Mul/output_0.out3_12", + "/model/layers.12/mlp/down_proj/MatMulNBits/output_0.out5_4_38", + "/model/layers.13/input_layernorm/output_3.out4_25", + "/model/layers.13/input_layernorm/output_0.out4_25", + "/model/layers.13/attn/qk_proj/Add/output_0.out5_4_39", + "/model/layers.13/attn/GroupQueryAttention/output_0.out2_13", + "/model/layers.13/attn/o_proj/MatMulNBits/output_0.out5_4_40", + "/model/layers.13/post_attention_layernorm/output_3.out4_26", + "/model/layers.13/post_attention_layernorm/output_0.out4_26", + "/model/layers.13/mlp/Mul/output_0.out3_13", + "/model/layers.13/mlp/down_proj/MatMulNBits/output_0.out5_4_41", + "/model/layers.14/input_layernorm/output_3.out4_27", + "/model/layers.14/input_layernorm/output_0.out4_27", + "/model/layers.14/attn/qk_proj/Add/output_0.out5_4_42", + "/model/layers.14/attn/GroupQueryAttention/output_0.out2_14", + "/model/layers.14/attn/o_proj/MatMulNBits/output_0.out5_4_43", + "/model/layers.14/post_attention_layernorm/output_3.out4_28", + "/model/layers.14/post_attention_layernorm/output_0.out4_28", + "/model/layers.14/mlp/Mul/output_0.out3_14", + "/model/layers.14/mlp/down_proj/MatMulNBits/output_0.out5_4_44", + "/model/layers.15/input_layernorm/output_3.out4_29", + "/model/layers.15/input_layernorm/output_0.out4_29", + "/model/layers.15/attn/qk_proj/Add/output_0.out5_4_45", + "/model/layers.15/attn/GroupQueryAttention/output_0.out2_15", + "/model/layers.15/attn/o_proj/MatMulNBits/output_0.out5_4_46", + "/model/layers.15/post_attention_layernorm/output_3.out4_30", + "/model/layers.15/post_attention_layernorm/output_0.out4_30", + "/model/layers.15/mlp/Mul/output_0.out3_15", + "/model/layers.15/mlp/down_proj/MatMulNBits/output_0.out5_4_47", + "/model/layers.16/input_layernorm/output_3.out4_31", + "/model/layers.16/input_layernorm/output_0.out4_31", + "/model/layers.16/attn/qk_proj/Add/output_0.out5_4_48", + "/model/layers.16/attn/GroupQueryAttention/output_0.out2_16", + "/model/layers.16/attn/o_proj/MatMulNBits/output_0.out5_4_49", + "/model/layers.16/post_attention_layernorm/output_3.out4_32", + "/model/layers.16/post_attention_layernorm/output_0.out4_32", + "/model/layers.16/mlp/Mul/output_0.out3_16", + "/model/layers.16/mlp/down_proj/MatMulNBits/output_0.out5_4_50", + "/model/layers.17/input_layernorm/output_3.out4_33", + "/model/layers.17/input_layernorm/output_0.out4_33", + "/model/layers.17/attn/qk_proj/Add/output_0.out5_4_51", + "/model/layers.17/attn/GroupQueryAttention/output_0.out2_17", + "/model/layers.17/attn/o_proj/MatMulNBits/output_0.out5_4_52", + "/model/layers.17/post_attention_layernorm/output_3.out4_34", + "/model/layers.17/post_attention_layernorm/output_0.out4_34", + "/model/layers.17/mlp/Mul/output_0.out3_17", + "/model/layers.17/mlp/down_proj/MatMulNBits/output_0.out5_4_53", + "/model/layers.18/input_layernorm/output_3.out4_35", + "/model/layers.18/input_layernorm/output_0.out4_35", + "/model/layers.18/attn/qk_proj/Add/output_0.out5_4_54", + "/model/layers.18/attn/GroupQueryAttention/output_0.out2_18", + "/model/layers.18/attn/o_proj/MatMulNBits/output_0.out5_4_55", + "/model/layers.18/post_attention_layernorm/output_3.out4_36", + "/model/layers.18/post_attention_layernorm/output_0.out4_36", + "/model/layers.18/mlp/Mul/output_0.out3_18", + "/model/layers.18/mlp/down_proj/MatMulNBits/output_0.out5_4_56", + "/model/layers.19/input_layernorm/output_3.out4_37", + "/model/layers.19/input_layernorm/output_0.out4_37", + "/model/layers.19/attn/qk_proj/Add/output_0.out5_4_57", + "/model/layers.19/attn/GroupQueryAttention/output_0.out2_19", + "/model/layers.19/attn/o_proj/MatMulNBits/output_0.out5_4_58", + "/model/layers.19/post_attention_layernorm/output_3.out4_38", + "/model/layers.19/post_attention_layernorm/output_0.out4_38", + "/model/layers.19/mlp/Mul/output_0.out3_19", + "/model/layers.19/mlp/down_proj/MatMulNBits/output_0.out5_4_59", + "/model/layers.20/input_layernorm/output_3.out4_39", + "/model/layers.20/input_layernorm/output_0.out4_39", + "/model/layers.20/attn/qk_proj/Add/output_0.out5_4_60", + "/model/layers.20/attn/GroupQueryAttention/output_0.out2_20", + "/model/layers.20/attn/o_proj/MatMulNBits/output_0.out5_4_61", + "/model/layers.20/post_attention_layernorm/output_3.out4_40", + "/model/layers.20/post_attention_layernorm/output_0.out4_40", + "/model/layers.20/mlp/Mul/output_0.out3_20", + "/model/layers.20/mlp/down_proj/MatMulNBits/output_0.out5_4_62", + "/model/layers.21/input_layernorm/output_3.out4_41", + "/model/layers.21/input_layernorm/output_0.out4_41", + "/model/layers.21/attn/qk_proj/Add/output_0.out5_4_63", + "/model/layers.21/attn/GroupQueryAttention/output_0.out2_21", + "/model/layers.21/attn/o_proj/MatMulNBits/output_0.out5_4_64", + "/model/layers.21/post_attention_layernorm/output_3.out4_42", + "/model/layers.21/post_attention_layernorm/output_0.out4_42", + "/model/layers.21/mlp/Mul/output_0.out3_21", + "/model/layers.21/mlp/down_proj/MatMulNBits/output_0.out5_4_65", + "/model/layers.22/input_layernorm/output_3.out4_43", + "/model/layers.22/input_layernorm/output_0.out4_43", + "/model/layers.22/attn/qk_proj/Add/output_0.out5_4_66", + "/model/layers.22/attn/GroupQueryAttention/output_0.out2_22", + "/model/layers.22/attn/o_proj/MatMulNBits/output_0.out5_4_67", + "/model/layers.22/post_attention_layernorm/output_3.out4_44", + "/model/layers.22/post_attention_layernorm/output_0.out4_44", + "/model/layers.22/mlp/Mul/output_0.out3_22", + "/model/layers.22/mlp/down_proj/MatMulNBits/output_0.out5_4_68", + "/model/layers.23/input_layernorm/output_3.out4_45", + "/model/layers.23/input_layernorm/output_0.out4_45", + "/model/layers.23/attn/qk_proj/Add/output_0.out5_4_69", + "/model/layers.23/attn/GroupQueryAttention/output_0.out2_23", + "/model/layers.23/attn/o_proj/MatMulNBits/output_0.out5_4_70", + "/model/layers.23/post_attention_layernorm/output_3.out4_46", + "/model/layers.23/post_attention_layernorm/output_0.out4_46", + "/model/layers.23/mlp/Mul/output_0.out3_23", + "/model/layers.23/mlp/down_proj/MatMulNBits/output_0.out5_4_71", + "/model/layers.24/input_layernorm/output_3.out4_47", + "/model/layers.24/input_layernorm/output_0.out4_47", + "/model/layers.24/attn/qk_proj/Add/output_0.out5_4_72", + "/model/layers.24/attn/GroupQueryAttention/output_0.out2_24", + "/model/layers.24/attn/o_proj/MatMulNBits/output_0.out5_4_73", + "/model/layers.24/post_attention_layernorm/output_3.out4_48", + "/model/layers.24/post_attention_layernorm/output_0.out4_48", + "/model/layers.24/mlp/Mul/output_0.out3_24", + "/model/layers.24/mlp/down_proj/MatMulNBits/output_0.out5_4_74", + "/model/layers.25/input_layernorm/output_3.out4_49", + "/model/layers.25/input_layernorm/output_0.out4_49", + "/model/layers.25/attn/qk_proj/Add/output_0.out5_4_75", + "/model/layers.25/attn/GroupQueryAttention/output_0.out2_25", + "/model/layers.25/attn/o_proj/MatMulNBits/output_0.out5_4_76", + "/model/layers.25/post_attention_layernorm/output_3.out4_50", + "/model/layers.25/post_attention_layernorm/output_0.out4_50", + "/model/layers.25/mlp/Mul/output_0.out3_25", + "/model/layers.25/mlp/down_proj/MatMulNBits/output_0.out5_4_77", + "/model/layers.26/input_layernorm/output_3.out4_51", + "/model/layers.26/input_layernorm/output_0.out4_51", + "/model/layers.26/attn/qk_proj/Add/output_0.out5_4_78", + "/model/layers.26/attn/GroupQueryAttention/output_0.out2_26", + "/model/layers.26/attn/o_proj/MatMulNBits/output_0.out5_4_79", + "/model/layers.26/post_attention_layernorm/output_3.out4_52", + "/model/layers.26/post_attention_layernorm/output_0.out4_52", + "/model/layers.26/mlp/Mul/output_0.out3_26", + "/model/layers.26/mlp/down_proj/MatMulNBits/output_0.out5_4_80", + "/model/layers.27/input_layernorm/output_3.out4_53", + "/model/layers.27/input_layernorm/output_0.out4_53", + "/model/layers.27/attn/qk_proj/Add/output_0.out5_4_81", + "/model/layers.27/attn/GroupQueryAttention/output_0.out2_27", + "/model/layers.27/attn/o_proj/MatMulNBits/output_0.out5_4_82", + "/model/layers.27/post_attention_layernorm/output_3.out4_54", + "/model/layers.27/post_attention_layernorm/output_0.out4_54", + "/model/layers.27/mlp/Mul/output_0.out3_27", + "/model/layers.27/mlp/down_proj/MatMulNBits/output_0.out5_4_83", + "/model/layers.28/final_norm_layernorm/output_0.out4_55" + ] + }, + "const": { + "buffer_size": 5437120512, + "xrt_arg_id": 3, + "packed_tensors": [ + "model.layers.0.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.0.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.0.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.v_proj.Add.bias.preformat", + "model.layers.0.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.0.post_attention_layernorm.weight.bf", + "model.layers.0.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.0.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.0.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.0.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.0.mlp.up_proj.MatMulNBits.qweight", + "model.layers.0.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.0.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.0.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.0.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.1.input_layernorm.weight.bf", + "model.layers.1.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.1.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.1.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.v_proj.Add.bias.preformat", + "model.layers.1.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.1.post_attention_layernorm.weight.bf", + "model.layers.1.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.1.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.1.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.1.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.1.mlp.up_proj.MatMulNBits.qweight", + "model.layers.1.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.1.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.1.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.1.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.2.input_layernorm.weight.bf", + "model.layers.2.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.2.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.2.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.v_proj.Add.bias.preformat", + "model.layers.2.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.2.post_attention_layernorm.weight.bf", + "model.layers.2.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.2.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.2.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.2.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.2.mlp.up_proj.MatMulNBits.qweight", + "model.layers.2.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.2.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.2.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.2.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.3.input_layernorm.weight.bf", + "model.layers.3.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.3.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.3.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.v_proj.Add.bias.preformat", + "model.layers.3.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.3.post_attention_layernorm.weight.bf", + "model.layers.3.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.3.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.3.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.3.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.3.mlp.up_proj.MatMulNBits.qweight", + "model.layers.3.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.3.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.3.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.3.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.4.input_layernorm.weight.bf", + "model.layers.4.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.4.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.4.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.v_proj.Add.bias.preformat", + "model.layers.4.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.4.post_attention_layernorm.weight.bf", + "model.layers.4.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.4.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.4.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.4.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.4.mlp.up_proj.MatMulNBits.qweight", + "model.layers.4.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.4.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.4.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.4.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.5.input_layernorm.weight.bf", + "model.layers.5.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.5.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.5.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.v_proj.Add.bias.preformat", + "model.layers.5.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.5.post_attention_layernorm.weight.bf", + "model.layers.5.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.5.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.5.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.5.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.5.mlp.up_proj.MatMulNBits.qweight", + "model.layers.5.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.5.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.5.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.5.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.6.input_layernorm.weight.bf", + "model.layers.6.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.6.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.6.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.v_proj.Add.bias.preformat", + "model.layers.6.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.6.post_attention_layernorm.weight.bf", + "model.layers.6.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.6.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.6.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.6.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.6.mlp.up_proj.MatMulNBits.qweight", + "model.layers.6.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.6.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.6.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.6.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.7.input_layernorm.weight.bf", + "model.layers.7.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.7.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.7.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.v_proj.Add.bias.preformat", + "model.layers.7.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.7.post_attention_layernorm.weight.bf", + "model.layers.7.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.7.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.7.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.7.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.7.mlp.up_proj.MatMulNBits.qweight", + "model.layers.7.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.7.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.7.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.7.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.8.input_layernorm.weight.bf", + "model.layers.8.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.8.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.8.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.v_proj.Add.bias.preformat", + "model.layers.8.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.8.post_attention_layernorm.weight.bf", + "model.layers.8.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.8.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.8.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.8.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.8.mlp.up_proj.MatMulNBits.qweight", + "model.layers.8.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.8.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.8.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.8.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.9.input_layernorm.weight.bf", + "model.layers.9.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.9.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.9.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.v_proj.Add.bias.preformat", + "model.layers.9.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.9.post_attention_layernorm.weight.bf", + "model.layers.9.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.9.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.9.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.9.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.9.mlp.up_proj.MatMulNBits.qweight", + "model.layers.9.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.9.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.9.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.9.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.10.input_layernorm.weight.bf", + "model.layers.10.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.10.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.10.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.v_proj.Add.bias.preformat", + "model.layers.10.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.10.post_attention_layernorm.weight.bf", + "model.layers.10.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.10.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.10.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.10.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.10.mlp.up_proj.MatMulNBits.qweight", + "model.layers.10.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.10.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.10.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.10.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.11.input_layernorm.weight.bf", + "model.layers.11.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.11.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.11.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.v_proj.Add.bias.preformat", + "model.layers.11.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.11.post_attention_layernorm.weight.bf", + "model.layers.11.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.11.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.11.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.11.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.11.mlp.up_proj.MatMulNBits.qweight", + "model.layers.11.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.11.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.11.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.11.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.12.input_layernorm.weight.bf", + "model.layers.12.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.12.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.12.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.v_proj.Add.bias.preformat", + "model.layers.12.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.12.post_attention_layernorm.weight.bf", + "model.layers.12.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.12.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.12.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.12.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.12.mlp.up_proj.MatMulNBits.qweight", + "model.layers.12.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.12.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.12.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.12.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.13.input_layernorm.weight.bf", + "model.layers.13.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.13.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.13.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.v_proj.Add.bias.preformat", + "model.layers.13.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.13.post_attention_layernorm.weight.bf", + "model.layers.13.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.13.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.13.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.13.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.13.mlp.up_proj.MatMulNBits.qweight", + "model.layers.13.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.13.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.13.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.13.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.14.input_layernorm.weight.bf", + "model.layers.14.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.14.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.14.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.v_proj.Add.bias.preformat", + "model.layers.14.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.14.post_attention_layernorm.weight.bf", + "model.layers.14.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.14.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.14.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.14.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.14.mlp.up_proj.MatMulNBits.qweight", + "model.layers.14.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.14.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.14.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.14.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.15.input_layernorm.weight.bf", + "model.layers.15.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.15.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.15.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.v_proj.Add.bias.preformat", + "model.layers.15.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.15.post_attention_layernorm.weight.bf", + "model.layers.15.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.15.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.15.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.15.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.15.mlp.up_proj.MatMulNBits.qweight", + "model.layers.15.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.15.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.15.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.15.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.16.input_layernorm.weight.bf", + "model.layers.16.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.16.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.16.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.v_proj.Add.bias.preformat", + "model.layers.16.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.16.post_attention_layernorm.weight.bf", + "model.layers.16.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.16.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.16.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.16.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.16.mlp.up_proj.MatMulNBits.qweight", + "model.layers.16.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.16.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.16.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.16.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.17.input_layernorm.weight.bf", + "model.layers.17.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.17.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.17.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.v_proj.Add.bias.preformat", + "model.layers.17.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.17.post_attention_layernorm.weight.bf", + "model.layers.17.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.17.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.17.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.17.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.17.mlp.up_proj.MatMulNBits.qweight", + "model.layers.17.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.17.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.17.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.17.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.18.input_layernorm.weight.bf", + "model.layers.18.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.18.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.18.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.v_proj.Add.bias.preformat", + "model.layers.18.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.18.post_attention_layernorm.weight.bf", + "model.layers.18.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.18.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.18.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.18.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.18.mlp.up_proj.MatMulNBits.qweight", + "model.layers.18.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.18.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.18.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.18.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.19.input_layernorm.weight.bf", + "model.layers.19.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.19.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.19.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.v_proj.Add.bias.preformat", + "model.layers.19.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.19.post_attention_layernorm.weight.bf", + "model.layers.19.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.19.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.19.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.19.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.19.mlp.up_proj.MatMulNBits.qweight", + "model.layers.19.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.19.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.19.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.19.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.20.input_layernorm.weight.bf", + "model.layers.20.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.20.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.20.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.v_proj.Add.bias.preformat", + "model.layers.20.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.20.post_attention_layernorm.weight.bf", + "model.layers.20.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.20.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.20.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.20.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.20.mlp.up_proj.MatMulNBits.qweight", + "model.layers.20.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.20.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.20.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.20.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.21.input_layernorm.weight.bf", + "model.layers.21.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.21.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.21.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.v_proj.Add.bias.preformat", + "model.layers.21.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.21.post_attention_layernorm.weight.bf", + "model.layers.21.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.21.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.21.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.21.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.21.mlp.up_proj.MatMulNBits.qweight", + "model.layers.21.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.21.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.21.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.21.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.22.input_layernorm.weight.bf", + "model.layers.22.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.22.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.22.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.v_proj.Add.bias.preformat", + "model.layers.22.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.22.post_attention_layernorm.weight.bf", + "model.layers.22.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.22.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.22.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.22.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.22.mlp.up_proj.MatMulNBits.qweight", + "model.layers.22.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.22.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.22.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.22.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.23.input_layernorm.weight.bf", + "model.layers.23.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.23.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.23.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.v_proj.Add.bias.preformat", + "model.layers.23.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.23.post_attention_layernorm.weight.bf", + "model.layers.23.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.23.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.23.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.23.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.23.mlp.up_proj.MatMulNBits.qweight", + "model.layers.23.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.23.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.23.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.23.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.24.input_layernorm.weight.bf", + "model.layers.24.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.24.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.24.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.24.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.24.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.24.attn.v_proj.Add.bias.preformat", + "model.layers.24.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.24.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.24.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.24.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.24.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.24.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.24.post_attention_layernorm.weight.bf", + "model.layers.24.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.24.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.24.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.24.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.24.mlp.up_proj.MatMulNBits.qweight", + "model.layers.24.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.24.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.24.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.24.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.24.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.24.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.24.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.25.input_layernorm.weight.bf", + "model.layers.25.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.25.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.25.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.25.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.25.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.25.attn.v_proj.Add.bias.preformat", + "model.layers.25.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.25.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.25.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.25.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.25.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.25.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.25.post_attention_layernorm.weight.bf", + "model.layers.25.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.25.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.25.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.25.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.25.mlp.up_proj.MatMulNBits.qweight", + "model.layers.25.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.25.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.25.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.25.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.25.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.25.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.25.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.26.input_layernorm.weight.bf", + "model.layers.26.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.26.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.26.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.26.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.26.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.26.attn.v_proj.Add.bias.preformat", + "model.layers.26.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.26.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.26.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.26.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.26.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.26.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.26.post_attention_layernorm.weight.bf", + "model.layers.26.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.26.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.26.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.26.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.26.mlp.up_proj.MatMulNBits.qweight", + "model.layers.26.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.26.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.26.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.26.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.26.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.26.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.26.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.27.input_layernorm.weight.bf", + "model.layers.27.attn.qk_proj.MatMulNBits.qweight.preformat", + "model.layers.27.attn.qk_proj.MatMulNBits.bias.preformat", + "model.layers.27.attn.qk_proj.MatMulNBits.scales.preformat", + "model.layers.27.attn.qk_proj.MatMulNBits.qzeros.preformat", + "model.layers.27.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.27.attn.v_proj.Add.bias.preformat", + "model.layers.27.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.27.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.27.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.27.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.27.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.27.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.27.post_attention_layernorm.weight.bf", + "model.layers.27.mlp.gate_proj.MatMulNBits.qweight", + "model.layers.27.mlp.gate_proj.MatMulNBits.scales.f", + "model.layers.27.mlp.gate_proj.MatMulNBits.qzeros", + "model.layers.27.mlp.gate_proj.MatMulNBits.bias.f", + "model.layers.27.mlp.up_proj.MatMulNBits.qweight", + "model.layers.27.mlp.up_proj.MatMulNBits.scales.f", + "model.layers.27.mlp.up_proj.MatMulNBits.qzeros", + "model.layers.27.mlp.up_proj.MatMulNBits.bias.f", + "model.layers.27.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.27.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.27.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.27.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.28.final_norm_layernorm.weight.bf", + "lm_head.MatMulNBits.qweight.preformat", + "lm_head.MatMulNBits.bias.preformat", + "lm_head.MatMulNBits.scales.preformat", + "lm_head.MatMulNBits.qzeros.preformat" + ] + }, + "super_instr": { + "buffer_size": 0, + "xrt_arg_id": 4, + "packed_tensors": [] + }, + "ext_buf_0": { + "buffer_size": 234881024, + "xrt_arg_id": 5, + "packed_tensors": [ + "past_key_values.0.key", + "past_key_values.0.value", + "present.0.key", + "present.0.value", + "past_key_values.1.key", + "past_key_values.1.value", + "present.1.key", + "present.1.value", + "past_key_values.2.key", + "past_key_values.2.value", + "present.2.key", + "present.2.value", + "past_key_values.3.key", + "past_key_values.3.value", + "present.3.key", + "present.3.value", + "past_key_values.4.key", + "past_key_values.4.value", + "present.4.key", + "present.4.value", + "past_key_values.5.key", + "past_key_values.5.value", + "present.5.key", + "present.5.value", + "past_key_values.6.key", + "past_key_values.6.value", + "present.6.key", + "present.6.value", + "past_key_values.7.key", + "past_key_values.7.value", + "present.7.key", + "present.7.value", + "past_key_values.8.key", + "past_key_values.8.value", + "present.8.key", + "present.8.value", + "past_key_values.9.key", + "past_key_values.9.value", + "present.9.key", + "present.9.value", + "past_key_values.10.key", + "past_key_values.10.value", + "present.10.key", + "present.10.value", + "past_key_values.11.key", + "past_key_values.11.value", + "present.11.key", + "present.11.value", + "past_key_values.12.key", + "past_key_values.12.value", + "present.12.key", + "present.12.value", + "past_key_values.13.key", + "past_key_values.13.value", + "present.13.key", + "present.13.value", + "past_key_values.14.key", + "past_key_values.14.value", + "present.14.key", + "present.14.value", + "past_key_values.15.key", + "past_key_values.15.value", + "present.15.key", + "present.15.value", + "past_key_values.16.key", + "past_key_values.16.value", + "present.16.key", + "present.16.value", + "past_key_values.17.key", + "past_key_values.17.value", + "present.17.key", + "present.17.value", + "past_key_values.18.key", + "past_key_values.18.value", + "present.18.key", + "present.18.value", + "past_key_values.19.key", + "past_key_values.19.value", + "present.19.key", + "present.19.value", + "past_key_values.20.key", + "past_key_values.20.value", + "present.20.key", + "present.20.value", + "past_key_values.21.key", + "past_key_values.21.value", + "present.21.key", + "present.21.value", + "past_key_values.22.key", + "past_key_values.22.value", + "present.22.key", + "present.22.value", + "past_key_values.23.key", + "past_key_values.23.value", + "present.23.key", + "present.23.value", + "past_key_values.24.key", + "past_key_values.24.value", + "present.24.key", + "present.24.value", + "past_key_values.25.key", + "past_key_values.25.value", + "present.25.key", + "present.25.value", + "past_key_values.26.key", + "past_key_values.26.value", + "present.26.key", + "present.26.value", + "past_key_values.27.key", + "past_key_values.27.value", + "present.27.key", + "present.27.value" + ] + }, + "ext_buf_1": { + "buffer_size": 33554432, + "xrt_arg_id": 6, + "packed_tensors": [ + "sin_cos_cache_token" + ] + } + }, + "tensor_map": { + "/model/layers.0/input_layernorm/output_0.out5_4_0": { + "packed_buffer_label": "in", + "xrt_arg_id": 0, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 7168 + }, + "attention_mask_const_uint": { + "packed_buffer_label": "in", + "xrt_arg_id": 0, + "dtype": "uint32", + "shape": [ + 1 + ], + "size_in_bytes": 4, + "op_tensor_size": 4, + "offset": 21612 + }, + "/model/embed_tokens/Gather/output_0.out4_0": { + "packed_buffer_label": "in", + "xrt_arg_id": 0, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 14340 + }, + "/model/layers.28/final_norm_layernorm/output_0.dummy": { + "packed_buffer_label": "out", + "xrt_arg_id": 1, + "dtype": "float16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 0 + }, + "logits.out5_4_84": { + "packed_buffer_label": "out", + "xrt_arg_id": 1, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 152064 + ], + "size_in_bytes": 304128, + "op_tensor_size": 304128, + "offset": 7168 + }, + "/model/layers.0/attn/qk_proj/Add/output_0.out5_4_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 0 + }, + "/model/layers.0/attn/GroupQueryAttention/output_0.out2_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 8192 + }, + "/model/layers.0/attn/o_proj/MatMulNBits/output_0.out5_4_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 15360 + }, + "/model/layers.0/post_attention_layernorm/output_3.out4_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 22528 + }, + "/model/layers.0/post_attention_layernorm/output_0.out4_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 29696 + }, + "/model/layers.0/mlp/Mul/output_0.out3_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 36864 + }, + "/model/layers.0/mlp/down_proj/MatMulNBits/output_0.out5_4_2": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 74752 + }, + "/model/layers.1/input_layernorm/output_3.out4_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 81920 + }, + "/model/layers.1/input_layernorm/output_0.out4_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 89088 + }, + "/model/layers.1/attn/qk_proj/Add/output_0.out5_4_3": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 96256 + }, + "/model/layers.1/attn/GroupQueryAttention/output_0.out2_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 104448 + }, + "/model/layers.1/attn/o_proj/MatMulNBits/output_0.out5_4_4": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 111616 + }, + "/model/layers.1/post_attention_layernorm/output_3.out4_2": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 118784 + }, + "/model/layers.1/post_attention_layernorm/output_0.out4_2": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 125952 + }, + "/model/layers.1/mlp/Mul/output_0.out3_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 133120 + }, + "/model/layers.1/mlp/down_proj/MatMulNBits/output_0.out5_4_5": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 171008 + }, + "/model/layers.2/input_layernorm/output_3.out4_3": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 178176 + }, + "/model/layers.2/input_layernorm/output_0.out4_3": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 185344 + }, + "/model/layers.2/attn/qk_proj/Add/output_0.out5_4_6": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 192512 + }, + "/model/layers.2/attn/GroupQueryAttention/output_0.out2_2": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 200704 + }, + "/model/layers.2/attn/o_proj/MatMulNBits/output_0.out5_4_7": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 207872 + }, + "/model/layers.2/post_attention_layernorm/output_3.out4_4": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 215040 + }, + "/model/layers.2/post_attention_layernorm/output_0.out4_4": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 222208 + }, + "/model/layers.2/mlp/Mul/output_0.out3_2": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 229376 + }, + "/model/layers.2/mlp/down_proj/MatMulNBits/output_0.out5_4_8": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 267264 + }, + "/model/layers.3/input_layernorm/output_3.out4_5": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 274432 + }, + "/model/layers.3/input_layernorm/output_0.out4_5": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 281600 + }, + "/model/layers.3/attn/qk_proj/Add/output_0.out5_4_9": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 288768 + }, + "/model/layers.3/attn/GroupQueryAttention/output_0.out2_3": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 296960 + }, + "/model/layers.3/attn/o_proj/MatMulNBits/output_0.out5_4_10": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 304128 + }, + "/model/layers.3/post_attention_layernorm/output_3.out4_6": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 311296 + }, + "/model/layers.3/post_attention_layernorm/output_0.out4_6": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 318464 + }, + "/model/layers.3/mlp/Mul/output_0.out3_3": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 325632 + }, + "/model/layers.3/mlp/down_proj/MatMulNBits/output_0.out5_4_11": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 363520 + }, + "/model/layers.4/input_layernorm/output_3.out4_7": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 370688 + }, + "/model/layers.4/input_layernorm/output_0.out4_7": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 377856 + }, + "/model/layers.4/attn/qk_proj/Add/output_0.out5_4_12": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 385024 + }, + "/model/layers.4/attn/GroupQueryAttention/output_0.out2_4": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 393216 + }, + "/model/layers.4/attn/o_proj/MatMulNBits/output_0.out5_4_13": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 400384 + }, + "/model/layers.4/post_attention_layernorm/output_3.out4_8": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 407552 + }, + "/model/layers.4/post_attention_layernorm/output_0.out4_8": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 414720 + }, + "/model/layers.4/mlp/Mul/output_0.out3_4": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 421888 + }, + "/model/layers.4/mlp/down_proj/MatMulNBits/output_0.out5_4_14": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 459776 + }, + "/model/layers.5/input_layernorm/output_3.out4_9": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 466944 + }, + "/model/layers.5/input_layernorm/output_0.out4_9": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 474112 + }, + "/model/layers.5/attn/qk_proj/Add/output_0.out5_4_15": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 481280 + }, + "/model/layers.5/attn/GroupQueryAttention/output_0.out2_5": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 489472 + }, + "/model/layers.5/attn/o_proj/MatMulNBits/output_0.out5_4_16": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 496640 + }, + "/model/layers.5/post_attention_layernorm/output_3.out4_10": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 503808 + }, + "/model/layers.5/post_attention_layernorm/output_0.out4_10": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 510976 + }, + "/model/layers.5/mlp/Mul/output_0.out3_5": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 518144 + }, + "/model/layers.5/mlp/down_proj/MatMulNBits/output_0.out5_4_17": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 556032 + }, + "/model/layers.6/input_layernorm/output_3.out4_11": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 563200 + }, + "/model/layers.6/input_layernorm/output_0.out4_11": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 570368 + }, + "/model/layers.6/attn/qk_proj/Add/output_0.out5_4_18": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 577536 + }, + "/model/layers.6/attn/GroupQueryAttention/output_0.out2_6": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 585728 + }, + "/model/layers.6/attn/o_proj/MatMulNBits/output_0.out5_4_19": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 592896 + }, + "/model/layers.6/post_attention_layernorm/output_3.out4_12": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 600064 + }, + "/model/layers.6/post_attention_layernorm/output_0.out4_12": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 607232 + }, + "/model/layers.6/mlp/Mul/output_0.out3_6": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 614400 + }, + "/model/layers.6/mlp/down_proj/MatMulNBits/output_0.out5_4_20": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 652288 + }, + "/model/layers.7/input_layernorm/output_3.out4_13": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 659456 + }, + "/model/layers.7/input_layernorm/output_0.out4_13": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 666624 + }, + "/model/layers.7/attn/qk_proj/Add/output_0.out5_4_21": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 673792 + }, + "/model/layers.7/attn/GroupQueryAttention/output_0.out2_7": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 681984 + }, + "/model/layers.7/attn/o_proj/MatMulNBits/output_0.out5_4_22": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 689152 + }, + "/model/layers.7/post_attention_layernorm/output_3.out4_14": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 696320 + }, + "/model/layers.7/post_attention_layernorm/output_0.out4_14": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 703488 + }, + "/model/layers.7/mlp/Mul/output_0.out3_7": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 710656 + }, + "/model/layers.7/mlp/down_proj/MatMulNBits/output_0.out5_4_23": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 748544 + }, + "/model/layers.8/input_layernorm/output_3.out4_15": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 755712 + }, + "/model/layers.8/input_layernorm/output_0.out4_15": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 762880 + }, + "/model/layers.8/attn/qk_proj/Add/output_0.out5_4_24": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 770048 + }, + "/model/layers.8/attn/GroupQueryAttention/output_0.out2_8": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 778240 + }, + "/model/layers.8/attn/o_proj/MatMulNBits/output_0.out5_4_25": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 785408 + }, + "/model/layers.8/post_attention_layernorm/output_3.out4_16": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 792576 + }, + "/model/layers.8/post_attention_layernorm/output_0.out4_16": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 799744 + }, + "/model/layers.8/mlp/Mul/output_0.out3_8": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 806912 + }, + "/model/layers.8/mlp/down_proj/MatMulNBits/output_0.out5_4_26": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 844800 + }, + "/model/layers.9/input_layernorm/output_3.out4_17": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 851968 + }, + "/model/layers.9/input_layernorm/output_0.out4_17": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 859136 + }, + "/model/layers.9/attn/qk_proj/Add/output_0.out5_4_27": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 866304 + }, + "/model/layers.9/attn/GroupQueryAttention/output_0.out2_9": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 874496 + }, + "/model/layers.9/attn/o_proj/MatMulNBits/output_0.out5_4_28": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 881664 + }, + "/model/layers.9/post_attention_layernorm/output_3.out4_18": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 888832 + }, + "/model/layers.9/post_attention_layernorm/output_0.out4_18": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 896000 + }, + "/model/layers.9/mlp/Mul/output_0.out3_9": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 903168 + }, + "/model/layers.9/mlp/down_proj/MatMulNBits/output_0.out5_4_29": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 941056 + }, + "/model/layers.10/input_layernorm/output_3.out4_19": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 948224 + }, + "/model/layers.10/input_layernorm/output_0.out4_19": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 955392 + }, + "/model/layers.10/attn/qk_proj/Add/output_0.out5_4_30": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 962560 + }, + "/model/layers.10/attn/GroupQueryAttention/output_0.out2_10": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 970752 + }, + "/model/layers.10/attn/o_proj/MatMulNBits/output_0.out5_4_31": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 977920 + }, + "/model/layers.10/post_attention_layernorm/output_3.out4_20": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 985088 + }, + "/model/layers.10/post_attention_layernorm/output_0.out4_20": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 992256 + }, + "/model/layers.10/mlp/Mul/output_0.out3_10": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 999424 + }, + "/model/layers.10/mlp/down_proj/MatMulNBits/output_0.out5_4_32": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1037312 + }, + "/model/layers.11/input_layernorm/output_3.out4_21": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1044480 + }, + "/model/layers.11/input_layernorm/output_0.out4_21": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1051648 + }, + "/model/layers.11/attn/qk_proj/Add/output_0.out5_4_33": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 1058816 + }, + "/model/layers.11/attn/GroupQueryAttention/output_0.out2_11": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1067008 + }, + "/model/layers.11/attn/o_proj/MatMulNBits/output_0.out5_4_34": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1074176 + }, + "/model/layers.11/post_attention_layernorm/output_3.out4_22": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1081344 + }, + "/model/layers.11/post_attention_layernorm/output_0.out4_22": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1088512 + }, + "/model/layers.11/mlp/Mul/output_0.out3_11": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 1095680 + }, + "/model/layers.11/mlp/down_proj/MatMulNBits/output_0.out5_4_35": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1133568 + }, + "/model/layers.12/input_layernorm/output_3.out4_23": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1140736 + }, + "/model/layers.12/input_layernorm/output_0.out4_23": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1147904 + }, + "/model/layers.12/attn/qk_proj/Add/output_0.out5_4_36": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 1155072 + }, + "/model/layers.12/attn/GroupQueryAttention/output_0.out2_12": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1163264 + }, + "/model/layers.12/attn/o_proj/MatMulNBits/output_0.out5_4_37": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1170432 + }, + "/model/layers.12/post_attention_layernorm/output_3.out4_24": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1177600 + }, + "/model/layers.12/post_attention_layernorm/output_0.out4_24": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1184768 + }, + "/model/layers.12/mlp/Mul/output_0.out3_12": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 1191936 + }, + "/model/layers.12/mlp/down_proj/MatMulNBits/output_0.out5_4_38": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1229824 + }, + "/model/layers.13/input_layernorm/output_3.out4_25": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1236992 + }, + "/model/layers.13/input_layernorm/output_0.out4_25": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1244160 + }, + "/model/layers.13/attn/qk_proj/Add/output_0.out5_4_39": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 1251328 + }, + "/model/layers.13/attn/GroupQueryAttention/output_0.out2_13": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1259520 + }, + "/model/layers.13/attn/o_proj/MatMulNBits/output_0.out5_4_40": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1266688 + }, + "/model/layers.13/post_attention_layernorm/output_3.out4_26": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1273856 + }, + "/model/layers.13/post_attention_layernorm/output_0.out4_26": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1281024 + }, + "/model/layers.13/mlp/Mul/output_0.out3_13": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 1288192 + }, + "/model/layers.13/mlp/down_proj/MatMulNBits/output_0.out5_4_41": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1326080 + }, + "/model/layers.14/input_layernorm/output_3.out4_27": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1333248 + }, + "/model/layers.14/input_layernorm/output_0.out4_27": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1340416 + }, + "/model/layers.14/attn/qk_proj/Add/output_0.out5_4_42": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 1347584 + }, + "/model/layers.14/attn/GroupQueryAttention/output_0.out2_14": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1355776 + }, + "/model/layers.14/attn/o_proj/MatMulNBits/output_0.out5_4_43": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1362944 + }, + "/model/layers.14/post_attention_layernorm/output_3.out4_28": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1370112 + }, + "/model/layers.14/post_attention_layernorm/output_0.out4_28": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1377280 + }, + "/model/layers.14/mlp/Mul/output_0.out3_14": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 1384448 + }, + "/model/layers.14/mlp/down_proj/MatMulNBits/output_0.out5_4_44": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1422336 + }, + "/model/layers.15/input_layernorm/output_3.out4_29": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1429504 + }, + "/model/layers.15/input_layernorm/output_0.out4_29": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1436672 + }, + "/model/layers.15/attn/qk_proj/Add/output_0.out5_4_45": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 1443840 + }, + "/model/layers.15/attn/GroupQueryAttention/output_0.out2_15": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1452032 + }, + "/model/layers.15/attn/o_proj/MatMulNBits/output_0.out5_4_46": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1459200 + }, + "/model/layers.15/post_attention_layernorm/output_3.out4_30": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1466368 + }, + "/model/layers.15/post_attention_layernorm/output_0.out4_30": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1473536 + }, + "/model/layers.15/mlp/Mul/output_0.out3_15": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 1480704 + }, + "/model/layers.15/mlp/down_proj/MatMulNBits/output_0.out5_4_47": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1518592 + }, + "/model/layers.16/input_layernorm/output_3.out4_31": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1525760 + }, + "/model/layers.16/input_layernorm/output_0.out4_31": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1532928 + }, + "/model/layers.16/attn/qk_proj/Add/output_0.out5_4_48": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 1540096 + }, + "/model/layers.16/attn/GroupQueryAttention/output_0.out2_16": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1548288 + }, + "/model/layers.16/attn/o_proj/MatMulNBits/output_0.out5_4_49": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1555456 + }, + "/model/layers.16/post_attention_layernorm/output_3.out4_32": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1562624 + }, + "/model/layers.16/post_attention_layernorm/output_0.out4_32": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1569792 + }, + "/model/layers.16/mlp/Mul/output_0.out3_16": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 1576960 + }, + "/model/layers.16/mlp/down_proj/MatMulNBits/output_0.out5_4_50": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1614848 + }, + "/model/layers.17/input_layernorm/output_3.out4_33": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1622016 + }, + "/model/layers.17/input_layernorm/output_0.out4_33": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1629184 + }, + "/model/layers.17/attn/qk_proj/Add/output_0.out5_4_51": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 1636352 + }, + "/model/layers.17/attn/GroupQueryAttention/output_0.out2_17": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1644544 + }, + "/model/layers.17/attn/o_proj/MatMulNBits/output_0.out5_4_52": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1651712 + }, + "/model/layers.17/post_attention_layernorm/output_3.out4_34": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1658880 + }, + "/model/layers.17/post_attention_layernorm/output_0.out4_34": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1666048 + }, + "/model/layers.17/mlp/Mul/output_0.out3_17": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 1673216 + }, + "/model/layers.17/mlp/down_proj/MatMulNBits/output_0.out5_4_53": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1711104 + }, + "/model/layers.18/input_layernorm/output_3.out4_35": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1718272 + }, + "/model/layers.18/input_layernorm/output_0.out4_35": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1725440 + }, + "/model/layers.18/attn/qk_proj/Add/output_0.out5_4_54": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 1732608 + }, + "/model/layers.18/attn/GroupQueryAttention/output_0.out2_18": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1740800 + }, + "/model/layers.18/attn/o_proj/MatMulNBits/output_0.out5_4_55": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1747968 + }, + "/model/layers.18/post_attention_layernorm/output_3.out4_36": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1755136 + }, + "/model/layers.18/post_attention_layernorm/output_0.out4_36": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1762304 + }, + "/model/layers.18/mlp/Mul/output_0.out3_18": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 1769472 + }, + "/model/layers.18/mlp/down_proj/MatMulNBits/output_0.out5_4_56": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1807360 + }, + "/model/layers.19/input_layernorm/output_3.out4_37": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1814528 + }, + "/model/layers.19/input_layernorm/output_0.out4_37": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1821696 + }, + "/model/layers.19/attn/qk_proj/Add/output_0.out5_4_57": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 1828864 + }, + "/model/layers.19/attn/GroupQueryAttention/output_0.out2_19": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1837056 + }, + "/model/layers.19/attn/o_proj/MatMulNBits/output_0.out5_4_58": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1844224 + }, + "/model/layers.19/post_attention_layernorm/output_3.out4_38": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1851392 + }, + "/model/layers.19/post_attention_layernorm/output_0.out4_38": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1858560 + }, + "/model/layers.19/mlp/Mul/output_0.out3_19": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 1865728 + }, + "/model/layers.19/mlp/down_proj/MatMulNBits/output_0.out5_4_59": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1903616 + }, + "/model/layers.20/input_layernorm/output_3.out4_39": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1910784 + }, + "/model/layers.20/input_layernorm/output_0.out4_39": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1917952 + }, + "/model/layers.20/attn/qk_proj/Add/output_0.out5_4_60": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 1925120 + }, + "/model/layers.20/attn/GroupQueryAttention/output_0.out2_20": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1933312 + }, + "/model/layers.20/attn/o_proj/MatMulNBits/output_0.out5_4_61": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1940480 + }, + "/model/layers.20/post_attention_layernorm/output_3.out4_40": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1947648 + }, + "/model/layers.20/post_attention_layernorm/output_0.out4_40": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1954816 + }, + "/model/layers.20/mlp/Mul/output_0.out3_20": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 1961984 + }, + "/model/layers.20/mlp/down_proj/MatMulNBits/output_0.out5_4_62": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1999872 + }, + "/model/layers.21/input_layernorm/output_3.out4_41": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2007040 + }, + "/model/layers.21/input_layernorm/output_0.out4_41": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2014208 + }, + "/model/layers.21/attn/qk_proj/Add/output_0.out5_4_63": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 2021376 + }, + "/model/layers.21/attn/GroupQueryAttention/output_0.out2_21": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2029568 + }, + "/model/layers.21/attn/o_proj/MatMulNBits/output_0.out5_4_64": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2036736 + }, + "/model/layers.21/post_attention_layernorm/output_3.out4_42": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2043904 + }, + "/model/layers.21/post_attention_layernorm/output_0.out4_42": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2051072 + }, + "/model/layers.21/mlp/Mul/output_0.out3_21": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 2058240 + }, + "/model/layers.21/mlp/down_proj/MatMulNBits/output_0.out5_4_65": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2096128 + }, + "/model/layers.22/input_layernorm/output_3.out4_43": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2103296 + }, + "/model/layers.22/input_layernorm/output_0.out4_43": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2110464 + }, + "/model/layers.22/attn/qk_proj/Add/output_0.out5_4_66": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 2117632 + }, + "/model/layers.22/attn/GroupQueryAttention/output_0.out2_22": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2125824 + }, + "/model/layers.22/attn/o_proj/MatMulNBits/output_0.out5_4_67": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2132992 + }, + "/model/layers.22/post_attention_layernorm/output_3.out4_44": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2140160 + }, + "/model/layers.22/post_attention_layernorm/output_0.out4_44": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2147328 + }, + "/model/layers.22/mlp/Mul/output_0.out3_22": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 2154496 + }, + "/model/layers.22/mlp/down_proj/MatMulNBits/output_0.out5_4_68": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2192384 + }, + "/model/layers.23/input_layernorm/output_3.out4_45": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2199552 + }, + "/model/layers.23/input_layernorm/output_0.out4_45": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2206720 + }, + "/model/layers.23/attn/qk_proj/Add/output_0.out5_4_69": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 2213888 + }, + "/model/layers.23/attn/GroupQueryAttention/output_0.out2_23": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2222080 + }, + "/model/layers.23/attn/o_proj/MatMulNBits/output_0.out5_4_70": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2229248 + }, + "/model/layers.23/post_attention_layernorm/output_3.out4_46": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2236416 + }, + "/model/layers.23/post_attention_layernorm/output_0.out4_46": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2243584 + }, + "/model/layers.23/mlp/Mul/output_0.out3_23": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 2250752 + }, + "/model/layers.23/mlp/down_proj/MatMulNBits/output_0.out5_4_71": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2288640 + }, + "/model/layers.24/input_layernorm/output_3.out4_47": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2295808 + }, + "/model/layers.24/input_layernorm/output_0.out4_47": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2302976 + }, + "/model/layers.24/attn/qk_proj/Add/output_0.out5_4_72": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 2310144 + }, + "/model/layers.24/attn/GroupQueryAttention/output_0.out2_24": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2318336 + }, + "/model/layers.24/attn/o_proj/MatMulNBits/output_0.out5_4_73": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2325504 + }, + "/model/layers.24/post_attention_layernorm/output_3.out4_48": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2332672 + }, + "/model/layers.24/post_attention_layernorm/output_0.out4_48": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2339840 + }, + "/model/layers.24/mlp/Mul/output_0.out3_24": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 2347008 + }, + "/model/layers.24/mlp/down_proj/MatMulNBits/output_0.out5_4_74": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2384896 + }, + "/model/layers.25/input_layernorm/output_3.out4_49": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2392064 + }, + "/model/layers.25/input_layernorm/output_0.out4_49": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2399232 + }, + "/model/layers.25/attn/qk_proj/Add/output_0.out5_4_75": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 2406400 + }, + "/model/layers.25/attn/GroupQueryAttention/output_0.out2_25": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2414592 + }, + "/model/layers.25/attn/o_proj/MatMulNBits/output_0.out5_4_76": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2421760 + }, + "/model/layers.25/post_attention_layernorm/output_3.out4_50": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2428928 + }, + "/model/layers.25/post_attention_layernorm/output_0.out4_50": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2436096 + }, + "/model/layers.25/mlp/Mul/output_0.out3_25": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 2443264 + }, + "/model/layers.25/mlp/down_proj/MatMulNBits/output_0.out5_4_77": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2481152 + }, + "/model/layers.26/input_layernorm/output_3.out4_51": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2488320 + }, + "/model/layers.26/input_layernorm/output_0.out4_51": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2495488 + }, + "/model/layers.26/attn/qk_proj/Add/output_0.out5_4_78": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 2502656 + }, + "/model/layers.26/attn/GroupQueryAttention/output_0.out2_26": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2510848 + }, + "/model/layers.26/attn/o_proj/MatMulNBits/output_0.out5_4_79": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2518016 + }, + "/model/layers.26/post_attention_layernorm/output_3.out4_52": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2525184 + }, + "/model/layers.26/post_attention_layernorm/output_0.out4_52": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2532352 + }, + "/model/layers.26/mlp/Mul/output_0.out3_26": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 2539520 + }, + "/model/layers.26/mlp/down_proj/MatMulNBits/output_0.out5_4_80": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2577408 + }, + "/model/layers.27/input_layernorm/output_3.out4_53": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2584576 + }, + "/model/layers.27/input_layernorm/output_0.out4_53": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2591744 + }, + "/model/layers.27/attn/qk_proj/Add/output_0.out5_4_81": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4096 + ], + "size_in_bytes": 8192, + "op_tensor_size": 8192, + "offset": 2598912 + }, + "/model/layers.27/attn/GroupQueryAttention/output_0.out2_27": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2607104 + }, + "/model/layers.27/attn/o_proj/MatMulNBits/output_0.out5_4_82": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2614272 + }, + "/model/layers.27/post_attention_layernorm/output_3.out4_54": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2621440 + }, + "/model/layers.27/post_attention_layernorm/output_0.out4_54": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2628608 + }, + "/model/layers.27/mlp/Mul/output_0.out3_27": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 18944 + ], + "size_in_bytes": 37888, + "op_tensor_size": 37888, + "offset": 2635776 + }, + "/model/layers.27/mlp/down_proj/MatMulNBits/output_0.out5_4_83": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2673664 + }, + "/model/layers.28/final_norm_layernorm/output_0.out4_55": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2680832 + }, + "model.layers.0.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 0, + "file_name": ".cache\\MatMulNBits_2_0_0.const", + "file_size": 14680064 + }, + "model.layers.0.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 14680064, + "file_name": ".cache\\MatMulNBits_2_0_1.const", + "file_size": 16384 + }, + "model.layers.0.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 14696448, + "file_name": ".cache\\MatMulNBits_2_0_2.const", + "file_size": 458752 + }, + "model.layers.0.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 15155200, + "file_name": ".cache\\MatMulNBits_2_0_3.const", + "file_size": 114688 + }, + "model.layers.0.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 15269888, + "file_name": ".cache\\MatMulNBits_2_0_4.const", + "file_size": 1835008 + }, + "model.layers.0.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 17104896, + "file_name": ".cache\\MatMulNBits_2_0_5.const", + "file_size": 2048 + }, + "model.layers.0.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 17106944, + "file_name": ".cache\\MatMulNBits_2_0_6.const", + "file_size": 57344 + }, + "model.layers.0.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 17164288, + "file_name": ".cache\\MatMulNBits_2_0_7.const", + "file_size": 14336 + }, + "model.layers.0.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 17178624, + "file_name": ".cache\\MatMulNBits_2_0_8.const", + "file_size": 12845056 + }, + "model.layers.0.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 30023680, + "file_name": ".cache\\MatMulNBits_2_0_9.const", + "file_size": 14336 + }, + "model.layers.0.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 30038016, + "file_name": ".cache\\MatMulNBits_2_0_10.const", + "file_size": 401408 + }, + "model.layers.0.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 30439424, + "file_name": ".cache\\MatMulNBits_2_0_11.const", + "file_size": 100352 + }, + "model.layers.0.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 30539776, + "file_name": ".cache\\MatMulNBits_2_0_12.const", + "file_size": 7168 + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 30546944, + "file_name": ".cache\\MatMulNBits_2_0_13.const", + "file_size": 33947648 + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 64494592, + "file_name": ".cache\\MatMulNBits_2_0_14.const", + "file_size": 2121728 + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 66616320, + "file_name": ".cache\\MatMulNBits_2_0_15.const", + "file_size": 265216 + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 66881536, + "file_name": ".cache\\MatMulNBits_2_0_16.const", + "file_size": 75776 + }, + "model.layers.0.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 66957312, + "file_name": ".cache\\MatMulNBits_2_0_17.const", + "file_size": 33947648 + }, + "model.layers.0.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 100904960, + "file_name": ".cache\\MatMulNBits_2_0_18.const", + "file_size": 2121728 + }, + "model.layers.0.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 103026688, + "file_name": ".cache\\MatMulNBits_2_0_19.const", + "file_size": 265216 + }, + "model.layers.0.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 103291904, + "file_name": ".cache\\MatMulNBits_2_0_20.const", + "file_size": 75776 + }, + "model.layers.0.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 103367680, + "file_name": ".cache\\MatMulNBits_2_0_21.const", + "file_size": 67895296 + }, + "model.layers.0.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 171262976, + "file_name": ".cache\\MatMulNBits_2_0_22.const", + "file_size": 14336 + }, + "model.layers.0.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 171277312, + "file_name": ".cache\\MatMulNBits_2_0_23.const", + "file_size": 2121728 + }, + "model.layers.0.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 173399040, + "file_name": ".cache\\MatMulNBits_2_0_24.const", + "file_size": 530432 + }, + "model.layers.1.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 173929472, + "file_name": ".cache\\MatMulNBits_2_0_25.const", + "file_size": 7168 + }, + "model.layers.1.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 173936640, + "file_name": ".cache\\MatMulNBits_2_0_26.const", + "file_size": 14680064 + }, + "model.layers.1.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 188616704, + "file_name": ".cache\\MatMulNBits_2_0_27.const", + "file_size": 16384 + }, + "model.layers.1.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 188633088, + "file_name": ".cache\\MatMulNBits_2_0_28.const", + "file_size": 458752 + }, + "model.layers.1.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 189091840, + "file_name": ".cache\\MatMulNBits_2_0_29.const", + "file_size": 114688 + }, + "model.layers.1.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 189206528, + "file_name": ".cache\\MatMulNBits_2_0_30.const", + "file_size": 1835008 + }, + "model.layers.1.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 191041536, + "file_name": ".cache\\MatMulNBits_2_0_31.const", + "file_size": 2048 + }, + "model.layers.1.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 191043584, + "file_name": ".cache\\MatMulNBits_2_0_32.const", + "file_size": 57344 + }, + "model.layers.1.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 191100928, + "file_name": ".cache\\MatMulNBits_2_0_33.const", + "file_size": 14336 + }, + "model.layers.1.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 191115264, + "file_name": ".cache\\MatMulNBits_2_0_34.const", + "file_size": 12845056 + }, + "model.layers.1.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 203960320, + "file_name": ".cache\\MatMulNBits_2_0_35.const", + "file_size": 14336 + }, + "model.layers.1.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 203974656, + "file_name": ".cache\\MatMulNBits_2_0_36.const", + "file_size": 401408 + }, + "model.layers.1.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 204376064, + "file_name": ".cache\\MatMulNBits_2_0_37.const", + "file_size": 100352 + }, + "model.layers.1.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 204476416, + "file_name": ".cache\\MatMulNBits_2_0_38.const", + "file_size": 7168 + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 204483584, + "file_name": ".cache\\MatMulNBits_2_0_39.const", + "file_size": 33947648 + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 238431232, + "file_name": ".cache\\MatMulNBits_2_0_40.const", + "file_size": 2121728 + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 240552960, + "file_name": ".cache\\MatMulNBits_2_0_41.const", + "file_size": 265216 + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 240818176, + "file_name": ".cache\\MatMulNBits_2_0_42.const", + "file_size": 75776 + }, + "model.layers.1.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 240893952, + "file_name": ".cache\\MatMulNBits_2_0_43.const", + "file_size": 33947648 + }, + "model.layers.1.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 274841600, + "file_name": ".cache\\MatMulNBits_2_0_44.const", + "file_size": 2121728 + }, + "model.layers.1.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 276963328, + "file_name": ".cache\\MatMulNBits_2_0_45.const", + "file_size": 265216 + }, + "model.layers.1.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 277228544, + "file_name": ".cache\\MatMulNBits_2_0_46.const", + "file_size": 75776 + }, + "model.layers.1.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 277304320, + "file_name": ".cache\\MatMulNBits_2_0_47.const", + "file_size": 67895296 + }, + "model.layers.1.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 345199616, + "file_name": ".cache\\MatMulNBits_2_0_48.const", + "file_size": 14336 + }, + "model.layers.1.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 345213952, + "file_name": ".cache\\MatMulNBits_2_0_49.const", + "file_size": 2121728 + }, + "model.layers.1.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 347335680, + "file_name": ".cache\\MatMulNBits_2_0_50.const", + "file_size": 530432 + }, + "model.layers.2.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 347866112, + "file_name": ".cache\\MatMulNBits_2_0_51.const", + "file_size": 7168 + }, + "model.layers.2.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 347873280, + "file_name": ".cache\\MatMulNBits_2_0_52.const", + "file_size": 14680064 + }, + "model.layers.2.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 362553344, + "file_name": ".cache\\MatMulNBits_2_0_53.const", + "file_size": 16384 + }, + "model.layers.2.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 362569728, + "file_name": ".cache\\MatMulNBits_2_0_54.const", + "file_size": 458752 + }, + "model.layers.2.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 363028480, + "file_name": ".cache\\MatMulNBits_2_0_55.const", + "file_size": 114688 + }, + "model.layers.2.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 363143168, + "file_name": ".cache\\MatMulNBits_2_0_56.const", + "file_size": 1835008 + }, + "model.layers.2.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 364978176, + "file_name": ".cache\\MatMulNBits_2_0_57.const", + "file_size": 2048 + }, + "model.layers.2.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 364980224, + "file_name": ".cache\\MatMulNBits_2_0_58.const", + "file_size": 57344 + }, + "model.layers.2.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 365037568, + "file_name": ".cache\\MatMulNBits_2_0_59.const", + "file_size": 14336 + }, + "model.layers.2.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 365051904, + "file_name": ".cache\\MatMulNBits_2_0_60.const", + "file_size": 12845056 + }, + "model.layers.2.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 377896960, + "file_name": ".cache\\MatMulNBits_2_0_61.const", + "file_size": 14336 + }, + "model.layers.2.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 377911296, + "file_name": ".cache\\MatMulNBits_2_0_62.const", + "file_size": 401408 + }, + "model.layers.2.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 378312704, + "file_name": ".cache\\MatMulNBits_2_0_63.const", + "file_size": 100352 + }, + "model.layers.2.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 378413056, + "file_name": ".cache\\MatMulNBits_2_0_64.const", + "file_size": 7168 + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 378420224, + "file_name": ".cache\\MatMulNBits_2_0_65.const", + "file_size": 33947648 + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 412367872, + "file_name": ".cache\\MatMulNBits_2_0_66.const", + "file_size": 2121728 + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 414489600, + "file_name": ".cache\\MatMulNBits_2_0_67.const", + "file_size": 265216 + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 414754816, + "file_name": ".cache\\MatMulNBits_2_0_68.const", + "file_size": 75776 + }, + "model.layers.2.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 414830592, + "file_name": ".cache\\MatMulNBits_2_0_69.const", + "file_size": 33947648 + }, + "model.layers.2.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 448778240, + "file_name": ".cache\\MatMulNBits_2_0_70.const", + "file_size": 2121728 + }, + "model.layers.2.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 450899968, + "file_name": ".cache\\MatMulNBits_2_0_71.const", + "file_size": 265216 + }, + "model.layers.2.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 451165184, + "file_name": ".cache\\MatMulNBits_2_0_72.const", + "file_size": 75776 + }, + "model.layers.2.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 451240960, + "file_name": ".cache\\MatMulNBits_2_0_73.const", + "file_size": 67895296 + }, + "model.layers.2.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 519136256, + "file_name": ".cache\\MatMulNBits_2_0_74.const", + "file_size": 14336 + }, + "model.layers.2.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 519150592, + "file_name": ".cache\\MatMulNBits_2_0_75.const", + "file_size": 2121728 + }, + "model.layers.2.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 521272320, + "file_name": ".cache\\MatMulNBits_2_0_76.const", + "file_size": 530432 + }, + "model.layers.3.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 521802752, + "file_name": ".cache\\MatMulNBits_2_0_77.const", + "file_size": 7168 + }, + "model.layers.3.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 521809920, + "file_name": ".cache\\MatMulNBits_2_0_78.const", + "file_size": 14680064 + }, + "model.layers.3.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 536489984, + "file_name": ".cache\\MatMulNBits_2_0_79.const", + "file_size": 16384 + }, + "model.layers.3.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 536506368, + "file_name": ".cache\\MatMulNBits_2_0_80.const", + "file_size": 458752 + }, + "model.layers.3.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 536965120, + "file_name": ".cache\\MatMulNBits_2_0_81.const", + "file_size": 114688 + }, + "model.layers.3.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 537079808, + "file_name": ".cache\\MatMulNBits_2_0_82.const", + "file_size": 1835008 + }, + "model.layers.3.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 538914816, + "file_name": ".cache\\MatMulNBits_2_0_83.const", + "file_size": 2048 + }, + "model.layers.3.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 538916864, + "file_name": ".cache\\MatMulNBits_2_0_84.const", + "file_size": 57344 + }, + "model.layers.3.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 538974208, + "file_name": ".cache\\MatMulNBits_2_0_85.const", + "file_size": 14336 + }, + "model.layers.3.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 538988544, + "file_name": ".cache\\MatMulNBits_2_0_86.const", + "file_size": 12845056 + }, + "model.layers.3.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 551833600, + "file_name": ".cache\\MatMulNBits_2_0_87.const", + "file_size": 14336 + }, + "model.layers.3.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 551847936, + "file_name": ".cache\\MatMulNBits_2_0_88.const", + "file_size": 401408 + }, + "model.layers.3.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 552249344, + "file_name": ".cache\\MatMulNBits_2_0_89.const", + "file_size": 100352 + }, + "model.layers.3.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 552349696, + "file_name": ".cache\\MatMulNBits_2_0_90.const", + "file_size": 7168 + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 552356864, + "file_name": ".cache\\MatMulNBits_2_0_91.const", + "file_size": 33947648 + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 586304512, + "file_name": ".cache\\MatMulNBits_2_0_92.const", + "file_size": 2121728 + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 588426240, + "file_name": ".cache\\MatMulNBits_2_0_93.const", + "file_size": 265216 + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 588691456, + "file_name": ".cache\\MatMulNBits_2_0_94.const", + "file_size": 75776 + }, + "model.layers.3.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 588767232, + "file_name": ".cache\\MatMulNBits_2_0_95.const", + "file_size": 33947648 + }, + "model.layers.3.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 622714880, + "file_name": ".cache\\MatMulNBits_2_0_96.const", + "file_size": 2121728 + }, + "model.layers.3.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 624836608, + "file_name": ".cache\\MatMulNBits_2_0_97.const", + "file_size": 265216 + }, + "model.layers.3.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 625101824, + "file_name": ".cache\\MatMulNBits_2_0_98.const", + "file_size": 75776 + }, + "model.layers.3.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 625177600, + "file_name": ".cache\\MatMulNBits_2_0_99.const", + "file_size": 67895296 + }, + "model.layers.3.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 693072896, + "file_name": ".cache\\MatMulNBits_2_0_100.const", + "file_size": 14336 + }, + "model.layers.3.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 693087232, + "file_name": ".cache\\MatMulNBits_2_0_101.const", + "file_size": 2121728 + }, + "model.layers.3.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 695208960, + "file_name": ".cache\\MatMulNBits_2_0_102.const", + "file_size": 530432 + }, + "model.layers.4.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 695739392, + "file_name": ".cache\\MatMulNBits_2_0_103.const", + "file_size": 7168 + }, + "model.layers.4.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 695746560, + "file_name": ".cache\\MatMulNBits_2_0_104.const", + "file_size": 14680064 + }, + "model.layers.4.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 710426624, + "file_name": ".cache\\MatMulNBits_2_0_105.const", + "file_size": 16384 + }, + "model.layers.4.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 710443008, + "file_name": ".cache\\MatMulNBits_2_0_106.const", + "file_size": 458752 + }, + "model.layers.4.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 710901760, + "file_name": ".cache\\MatMulNBits_2_0_107.const", + "file_size": 114688 + }, + "model.layers.4.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 711016448, + "file_name": ".cache\\MatMulNBits_2_0_108.const", + "file_size": 1835008 + }, + "model.layers.4.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 712851456, + "file_name": ".cache\\MatMulNBits_2_0_109.const", + "file_size": 2048 + }, + "model.layers.4.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 712853504, + "file_name": ".cache\\MatMulNBits_2_0_110.const", + "file_size": 57344 + }, + "model.layers.4.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 712910848, + "file_name": ".cache\\MatMulNBits_2_0_111.const", + "file_size": 14336 + }, + "model.layers.4.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 712925184, + "file_name": ".cache\\MatMulNBits_2_0_112.const", + "file_size": 12845056 + }, + "model.layers.4.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 725770240, + "file_name": ".cache\\MatMulNBits_2_0_113.const", + "file_size": 14336 + }, + "model.layers.4.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 725784576, + "file_name": ".cache\\MatMulNBits_2_0_114.const", + "file_size": 401408 + }, + "model.layers.4.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 726185984, + "file_name": ".cache\\MatMulNBits_2_0_115.const", + "file_size": 100352 + }, + "model.layers.4.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 726286336, + "file_name": ".cache\\MatMulNBits_2_0_116.const", + "file_size": 7168 + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 726293504, + "file_name": ".cache\\MatMulNBits_2_0_117.const", + "file_size": 33947648 + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 760241152, + "file_name": ".cache\\MatMulNBits_2_0_118.const", + "file_size": 2121728 + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 762362880, + "file_name": ".cache\\MatMulNBits_2_0_119.const", + "file_size": 265216 + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 762628096, + "file_name": ".cache\\MatMulNBits_2_0_120.const", + "file_size": 75776 + }, + "model.layers.4.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 762703872, + "file_name": ".cache\\MatMulNBits_2_0_121.const", + "file_size": 33947648 + }, + "model.layers.4.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 796651520, + "file_name": ".cache\\MatMulNBits_2_0_122.const", + "file_size": 2121728 + }, + "model.layers.4.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 798773248, + "file_name": ".cache\\MatMulNBits_2_0_123.const", + "file_size": 265216 + }, + "model.layers.4.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 799038464, + "file_name": ".cache\\MatMulNBits_2_0_124.const", + "file_size": 75776 + }, + "model.layers.4.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 799114240, + "file_name": ".cache\\MatMulNBits_2_0_125.const", + "file_size": 67895296 + }, + "model.layers.4.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 867009536, + "file_name": ".cache\\MatMulNBits_2_0_126.const", + "file_size": 14336 + }, + "model.layers.4.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 867023872, + "file_name": ".cache\\MatMulNBits_2_0_127.const", + "file_size": 2121728 + }, + "model.layers.4.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 869145600, + "file_name": ".cache\\MatMulNBits_2_0_128.const", + "file_size": 530432 + }, + "model.layers.5.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 869676032, + "file_name": ".cache\\MatMulNBits_2_0_129.const", + "file_size": 7168 + }, + "model.layers.5.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 869683200, + "file_name": ".cache\\MatMulNBits_2_0_130.const", + "file_size": 14680064 + }, + "model.layers.5.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 884363264, + "file_name": ".cache\\MatMulNBits_2_0_131.const", + "file_size": 16384 + }, + "model.layers.5.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 884379648, + "file_name": ".cache\\MatMulNBits_2_0_132.const", + "file_size": 458752 + }, + "model.layers.5.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 884838400, + "file_name": ".cache\\MatMulNBits_2_0_133.const", + "file_size": 114688 + }, + "model.layers.5.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 884953088, + "file_name": ".cache\\MatMulNBits_2_0_134.const", + "file_size": 1835008 + }, + "model.layers.5.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 886788096, + "file_name": ".cache\\MatMulNBits_2_0_135.const", + "file_size": 2048 + }, + "model.layers.5.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 886790144, + "file_name": ".cache\\MatMulNBits_2_0_136.const", + "file_size": 57344 + }, + "model.layers.5.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 886847488, + "file_name": ".cache\\MatMulNBits_2_0_137.const", + "file_size": 14336 + }, + "model.layers.5.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 886861824, + "file_name": ".cache\\MatMulNBits_2_0_138.const", + "file_size": 12845056 + }, + "model.layers.5.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 899706880, + "file_name": ".cache\\MatMulNBits_2_0_139.const", + "file_size": 14336 + }, + "model.layers.5.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 899721216, + "file_name": ".cache\\MatMulNBits_2_0_140.const", + "file_size": 401408 + }, + "model.layers.5.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 900122624, + "file_name": ".cache\\MatMulNBits_2_0_141.const", + "file_size": 100352 + }, + "model.layers.5.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 900222976, + "file_name": ".cache\\MatMulNBits_2_0_142.const", + "file_size": 7168 + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 900230144, + "file_name": ".cache\\MatMulNBits_2_0_143.const", + "file_size": 33947648 + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 934177792, + "file_name": ".cache\\MatMulNBits_2_0_144.const", + "file_size": 2121728 + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 936299520, + "file_name": ".cache\\MatMulNBits_2_0_145.const", + "file_size": 265216 + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 936564736, + "file_name": ".cache\\MatMulNBits_2_0_146.const", + "file_size": 75776 + }, + "model.layers.5.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 936640512, + "file_name": ".cache\\MatMulNBits_2_0_147.const", + "file_size": 33947648 + }, + "model.layers.5.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 970588160, + "file_name": ".cache\\MatMulNBits_2_0_148.const", + "file_size": 2121728 + }, + "model.layers.5.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 972709888, + "file_name": ".cache\\MatMulNBits_2_0_149.const", + "file_size": 265216 + }, + "model.layers.5.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 972975104, + "file_name": ".cache\\MatMulNBits_2_0_150.const", + "file_size": 75776 + }, + "model.layers.5.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 973050880, + "file_name": ".cache\\MatMulNBits_2_0_151.const", + "file_size": 67895296 + }, + "model.layers.5.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1040946176, + "file_name": ".cache\\MatMulNBits_2_0_152.const", + "file_size": 14336 + }, + "model.layers.5.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1040960512, + "file_name": ".cache\\MatMulNBits_2_0_153.const", + "file_size": 2121728 + }, + "model.layers.5.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 1043082240, + "file_name": ".cache\\MatMulNBits_2_0_154.const", + "file_size": 530432 + }, + "model.layers.6.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1043612672, + "file_name": ".cache\\MatMulNBits_2_0_155.const", + "file_size": 7168 + }, + "model.layers.6.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 1043619840, + "file_name": ".cache\\MatMulNBits_2_0_156.const", + "file_size": 14680064 + }, + "model.layers.6.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 1058299904, + "file_name": ".cache\\MatMulNBits_2_0_157.const", + "file_size": 16384 + }, + "model.layers.6.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 1058316288, + "file_name": ".cache\\MatMulNBits_2_0_158.const", + "file_size": 458752 + }, + "model.layers.6.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 1058775040, + "file_name": ".cache\\MatMulNBits_2_0_159.const", + "file_size": 114688 + }, + "model.layers.6.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 1058889728, + "file_name": ".cache\\MatMulNBits_2_0_160.const", + "file_size": 1835008 + }, + "model.layers.6.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 1060724736, + "file_name": ".cache\\MatMulNBits_2_0_161.const", + "file_size": 2048 + }, + "model.layers.6.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 1060726784, + "file_name": ".cache\\MatMulNBits_2_0_162.const", + "file_size": 57344 + }, + "model.layers.6.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1060784128, + "file_name": ".cache\\MatMulNBits_2_0_163.const", + "file_size": 14336 + }, + "model.layers.6.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 1060798464, + "file_name": ".cache\\MatMulNBits_2_0_164.const", + "file_size": 12845056 + }, + "model.layers.6.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1073643520, + "file_name": ".cache\\MatMulNBits_2_0_165.const", + "file_size": 14336 + }, + "model.layers.6.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 1073657856, + "file_name": ".cache\\MatMulNBits_2_0_166.const", + "file_size": 401408 + }, + "model.layers.6.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 1074059264, + "file_name": ".cache\\MatMulNBits_2_0_167.const", + "file_size": 100352 + }, + "model.layers.6.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1074159616, + "file_name": ".cache\\MatMulNBits_2_0_168.const", + "file_size": 7168 + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1074166784, + "file_name": ".cache\\MatMulNBits_2_0_169.const", + "file_size": 33947648 + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1108114432, + "file_name": ".cache\\MatMulNBits_2_0_170.const", + "file_size": 2121728 + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 1110236160, + "file_name": ".cache\\MatMulNBits_2_0_171.const", + "file_size": 265216 + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 1110501376, + "file_name": ".cache\\MatMulNBits_2_0_172.const", + "file_size": 75776 + }, + "model.layers.6.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1110577152, + "file_name": ".cache\\MatMulNBits_2_0_173.const", + "file_size": 33947648 + }, + "model.layers.6.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1144524800, + "file_name": ".cache\\MatMulNBits_2_0_174.const", + "file_size": 2121728 + }, + "model.layers.6.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 1146646528, + "file_name": ".cache\\MatMulNBits_2_0_175.const", + "file_size": 265216 + }, + "model.layers.6.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 1146911744, + "file_name": ".cache\\MatMulNBits_2_0_176.const", + "file_size": 75776 + }, + "model.layers.6.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 1146987520, + "file_name": ".cache\\MatMulNBits_2_0_177.const", + "file_size": 67895296 + }, + "model.layers.6.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1214882816, + "file_name": ".cache\\MatMulNBits_2_0_178.const", + "file_size": 14336 + }, + "model.layers.6.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1214897152, + "file_name": ".cache\\MatMulNBits_2_0_179.const", + "file_size": 2121728 + }, + "model.layers.6.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 1217018880, + "file_name": ".cache\\MatMulNBits_2_0_180.const", + "file_size": 530432 + }, + "model.layers.7.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1217549312, + "file_name": ".cache\\MatMulNBits_2_0_181.const", + "file_size": 7168 + }, + "model.layers.7.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 1217556480, + "file_name": ".cache\\MatMulNBits_2_0_182.const", + "file_size": 14680064 + }, + "model.layers.7.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 1232236544, + "file_name": ".cache\\MatMulNBits_2_0_183.const", + "file_size": 16384 + }, + "model.layers.7.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 1232252928, + "file_name": ".cache\\MatMulNBits_2_0_184.const", + "file_size": 458752 + }, + "model.layers.7.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 1232711680, + "file_name": ".cache\\MatMulNBits_2_0_185.const", + "file_size": 114688 + }, + "model.layers.7.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 1232826368, + "file_name": ".cache\\MatMulNBits_2_0_186.const", + "file_size": 1835008 + }, + "model.layers.7.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 1234661376, + "file_name": ".cache\\MatMulNBits_2_0_187.const", + "file_size": 2048 + }, + "model.layers.7.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 1234663424, + "file_name": ".cache\\MatMulNBits_2_0_188.const", + "file_size": 57344 + }, + "model.layers.7.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1234720768, + "file_name": ".cache\\MatMulNBits_2_0_189.const", + "file_size": 14336 + }, + "model.layers.7.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 1234735104, + "file_name": ".cache\\MatMulNBits_2_0_190.const", + "file_size": 12845056 + }, + "model.layers.7.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1247580160, + "file_name": ".cache\\MatMulNBits_2_0_191.const", + "file_size": 14336 + }, + "model.layers.7.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 1247594496, + "file_name": ".cache\\MatMulNBits_2_0_192.const", + "file_size": 401408 + }, + "model.layers.7.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 1247995904, + "file_name": ".cache\\MatMulNBits_2_0_193.const", + "file_size": 100352 + }, + "model.layers.7.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1248096256, + "file_name": ".cache\\MatMulNBits_2_0_194.const", + "file_size": 7168 + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1248103424, + "file_name": ".cache\\MatMulNBits_2_0_195.const", + "file_size": 33947648 + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1282051072, + "file_name": ".cache\\MatMulNBits_2_0_196.const", + "file_size": 2121728 + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 1284172800, + "file_name": ".cache\\MatMulNBits_2_0_197.const", + "file_size": 265216 + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 1284438016, + "file_name": ".cache\\MatMulNBits_2_0_198.const", + "file_size": 75776 + }, + "model.layers.7.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1284513792, + "file_name": ".cache\\MatMulNBits_2_0_199.const", + "file_size": 33947648 + }, + "model.layers.7.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1318461440, + "file_name": ".cache\\MatMulNBits_2_0_200.const", + "file_size": 2121728 + }, + "model.layers.7.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 1320583168, + "file_name": ".cache\\MatMulNBits_2_0_201.const", + "file_size": 265216 + }, + "model.layers.7.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 1320848384, + "file_name": ".cache\\MatMulNBits_2_0_202.const", + "file_size": 75776 + }, + "model.layers.7.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 1320924160, + "file_name": ".cache\\MatMulNBits_2_0_203.const", + "file_size": 67895296 + }, + "model.layers.7.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1388819456, + "file_name": ".cache\\MatMulNBits_2_0_204.const", + "file_size": 14336 + }, + "model.layers.7.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1388833792, + "file_name": ".cache\\MatMulNBits_2_0_205.const", + "file_size": 2121728 + }, + "model.layers.7.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 1390955520, + "file_name": ".cache\\MatMulNBits_2_0_206.const", + "file_size": 530432 + }, + "model.layers.8.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1391485952, + "file_name": ".cache\\MatMulNBits_2_0_207.const", + "file_size": 7168 + }, + "model.layers.8.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 1391493120, + "file_name": ".cache\\MatMulNBits_2_0_208.const", + "file_size": 14680064 + }, + "model.layers.8.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 1406173184, + "file_name": ".cache\\MatMulNBits_2_0_209.const", + "file_size": 16384 + }, + "model.layers.8.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 1406189568, + "file_name": ".cache\\MatMulNBits_2_0_210.const", + "file_size": 458752 + }, + "model.layers.8.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 1406648320, + "file_name": ".cache\\MatMulNBits_2_0_211.const", + "file_size": 114688 + }, + "model.layers.8.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 1406763008, + "file_name": ".cache\\MatMulNBits_2_0_212.const", + "file_size": 1835008 + }, + "model.layers.8.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 1408598016, + "file_name": ".cache\\MatMulNBits_2_0_213.const", + "file_size": 2048 + }, + "model.layers.8.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 1408600064, + "file_name": ".cache\\MatMulNBits_2_0_214.const", + "file_size": 57344 + }, + "model.layers.8.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1408657408, + "file_name": ".cache\\MatMulNBits_2_0_215.const", + "file_size": 14336 + }, + "model.layers.8.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 1408671744, + "file_name": ".cache\\MatMulNBits_2_0_216.const", + "file_size": 12845056 + }, + "model.layers.8.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1421516800, + "file_name": ".cache\\MatMulNBits_2_0_217.const", + "file_size": 14336 + }, + "model.layers.8.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 1421531136, + "file_name": ".cache\\MatMulNBits_2_0_218.const", + "file_size": 401408 + }, + "model.layers.8.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 1421932544, + "file_name": ".cache\\MatMulNBits_2_0_219.const", + "file_size": 100352 + }, + "model.layers.8.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1422032896, + "file_name": ".cache\\MatMulNBits_2_0_220.const", + "file_size": 7168 + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1422040064, + "file_name": ".cache\\MatMulNBits_2_0_221.const", + "file_size": 33947648 + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1455987712, + "file_name": ".cache\\MatMulNBits_2_0_222.const", + "file_size": 2121728 + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 1458109440, + "file_name": ".cache\\MatMulNBits_2_0_223.const", + "file_size": 265216 + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 1458374656, + "file_name": ".cache\\MatMulNBits_2_0_224.const", + "file_size": 75776 + }, + "model.layers.8.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1458450432, + "file_name": ".cache\\MatMulNBits_2_0_225.const", + "file_size": 33947648 + }, + "model.layers.8.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1492398080, + "file_name": ".cache\\MatMulNBits_2_0_226.const", + "file_size": 2121728 + }, + "model.layers.8.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 1494519808, + "file_name": ".cache\\MatMulNBits_2_0_227.const", + "file_size": 265216 + }, + "model.layers.8.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 1494785024, + "file_name": ".cache\\MatMulNBits_2_0_228.const", + "file_size": 75776 + }, + "model.layers.8.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 1494860800, + "file_name": ".cache\\MatMulNBits_2_0_229.const", + "file_size": 67895296 + }, + "model.layers.8.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1562756096, + "file_name": ".cache\\MatMulNBits_2_0_230.const", + "file_size": 14336 + }, + "model.layers.8.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1562770432, + "file_name": ".cache\\MatMulNBits_2_0_231.const", + "file_size": 2121728 + }, + "model.layers.8.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 1564892160, + "file_name": ".cache\\MatMulNBits_2_0_232.const", + "file_size": 530432 + }, + "model.layers.9.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1565422592, + "file_name": ".cache\\MatMulNBits_2_0_233.const", + "file_size": 7168 + }, + "model.layers.9.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 1565429760, + "file_name": ".cache\\MatMulNBits_2_0_234.const", + "file_size": 14680064 + }, + "model.layers.9.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 1580109824, + "file_name": ".cache\\MatMulNBits_2_0_235.const", + "file_size": 16384 + }, + "model.layers.9.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 1580126208, + "file_name": ".cache\\MatMulNBits_2_0_236.const", + "file_size": 458752 + }, + "model.layers.9.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 1580584960, + "file_name": ".cache\\MatMulNBits_2_0_237.const", + "file_size": 114688 + }, + "model.layers.9.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 1580699648, + "file_name": ".cache\\MatMulNBits_2_0_238.const", + "file_size": 1835008 + }, + "model.layers.9.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 1582534656, + "file_name": ".cache\\MatMulNBits_2_0_239.const", + "file_size": 2048 + }, + "model.layers.9.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 1582536704, + "file_name": ".cache\\MatMulNBits_2_0_240.const", + "file_size": 57344 + }, + "model.layers.9.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1582594048, + "file_name": ".cache\\MatMulNBits_2_0_241.const", + "file_size": 14336 + }, + "model.layers.9.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 1582608384, + "file_name": ".cache\\MatMulNBits_2_0_242.const", + "file_size": 12845056 + }, + "model.layers.9.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1595453440, + "file_name": ".cache\\MatMulNBits_2_0_243.const", + "file_size": 14336 + }, + "model.layers.9.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 1595467776, + "file_name": ".cache\\MatMulNBits_2_0_244.const", + "file_size": 401408 + }, + "model.layers.9.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 1595869184, + "file_name": ".cache\\MatMulNBits_2_0_245.const", + "file_size": 100352 + }, + "model.layers.9.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1595969536, + "file_name": ".cache\\MatMulNBits_2_0_246.const", + "file_size": 7168 + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1595976704, + "file_name": ".cache\\MatMulNBits_2_0_247.const", + "file_size": 33947648 + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1629924352, + "file_name": ".cache\\MatMulNBits_2_0_248.const", + "file_size": 2121728 + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 1632046080, + "file_name": ".cache\\MatMulNBits_2_0_249.const", + "file_size": 265216 + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 1632311296, + "file_name": ".cache\\MatMulNBits_2_0_250.const", + "file_size": 75776 + }, + "model.layers.9.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1632387072, + "file_name": ".cache\\MatMulNBits_2_0_251.const", + "file_size": 33947648 + }, + "model.layers.9.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1666334720, + "file_name": ".cache\\MatMulNBits_2_0_252.const", + "file_size": 2121728 + }, + "model.layers.9.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 1668456448, + "file_name": ".cache\\MatMulNBits_2_0_253.const", + "file_size": 265216 + }, + "model.layers.9.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 1668721664, + "file_name": ".cache\\MatMulNBits_2_0_254.const", + "file_size": 75776 + }, + "model.layers.9.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 1668797440, + "file_name": ".cache\\MatMulNBits_2_0_255.const", + "file_size": 67895296 + }, + "model.layers.9.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1736692736, + "file_name": ".cache\\MatMulNBits_2_0_256.const", + "file_size": 14336 + }, + "model.layers.9.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1736707072, + "file_name": ".cache\\MatMulNBits_2_0_257.const", + "file_size": 2121728 + }, + "model.layers.9.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 1738828800, + "file_name": ".cache\\MatMulNBits_2_0_258.const", + "file_size": 530432 + }, + "model.layers.10.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1739359232, + "file_name": ".cache\\MatMulNBits_2_0_259.const", + "file_size": 7168 + }, + "model.layers.10.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 1739366400, + "file_name": ".cache\\MatMulNBits_2_0_260.const", + "file_size": 14680064 + }, + "model.layers.10.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 1754046464, + "file_name": ".cache\\MatMulNBits_2_0_261.const", + "file_size": 16384 + }, + "model.layers.10.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 1754062848, + "file_name": ".cache\\MatMulNBits_2_0_262.const", + "file_size": 458752 + }, + "model.layers.10.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 1754521600, + "file_name": ".cache\\MatMulNBits_2_0_263.const", + "file_size": 114688 + }, + "model.layers.10.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 1754636288, + "file_name": ".cache\\MatMulNBits_2_0_264.const", + "file_size": 1835008 + }, + "model.layers.10.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 1756471296, + "file_name": ".cache\\MatMulNBits_2_0_265.const", + "file_size": 2048 + }, + "model.layers.10.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 1756473344, + "file_name": ".cache\\MatMulNBits_2_0_266.const", + "file_size": 57344 + }, + "model.layers.10.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1756530688, + "file_name": ".cache\\MatMulNBits_2_0_267.const", + "file_size": 14336 + }, + "model.layers.10.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 1756545024, + "file_name": ".cache\\MatMulNBits_2_0_268.const", + "file_size": 12845056 + }, + "model.layers.10.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1769390080, + "file_name": ".cache\\MatMulNBits_2_0_269.const", + "file_size": 14336 + }, + "model.layers.10.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 1769404416, + "file_name": ".cache\\MatMulNBits_2_0_270.const", + "file_size": 401408 + }, + "model.layers.10.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 1769805824, + "file_name": ".cache\\MatMulNBits_2_0_271.const", + "file_size": 100352 + }, + "model.layers.10.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1769906176, + "file_name": ".cache\\MatMulNBits_2_0_272.const", + "file_size": 7168 + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1769913344, + "file_name": ".cache\\MatMulNBits_2_0_273.const", + "file_size": 33947648 + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1803860992, + "file_name": ".cache\\MatMulNBits_2_0_274.const", + "file_size": 2121728 + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 1805982720, + "file_name": ".cache\\MatMulNBits_2_0_275.const", + "file_size": 265216 + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 1806247936, + "file_name": ".cache\\MatMulNBits_2_0_276.const", + "file_size": 75776 + }, + "model.layers.10.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1806323712, + "file_name": ".cache\\MatMulNBits_2_0_277.const", + "file_size": 33947648 + }, + "model.layers.10.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1840271360, + "file_name": ".cache\\MatMulNBits_2_0_278.const", + "file_size": 2121728 + }, + "model.layers.10.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 1842393088, + "file_name": ".cache\\MatMulNBits_2_0_279.const", + "file_size": 265216 + }, + "model.layers.10.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 1842658304, + "file_name": ".cache\\MatMulNBits_2_0_280.const", + "file_size": 75776 + }, + "model.layers.10.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 1842734080, + "file_name": ".cache\\MatMulNBits_2_0_281.const", + "file_size": 67895296 + }, + "model.layers.10.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1910629376, + "file_name": ".cache\\MatMulNBits_2_0_282.const", + "file_size": 14336 + }, + "model.layers.10.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1910643712, + "file_name": ".cache\\MatMulNBits_2_0_283.const", + "file_size": 2121728 + }, + "model.layers.10.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 1912765440, + "file_name": ".cache\\MatMulNBits_2_0_284.const", + "file_size": 530432 + }, + "model.layers.11.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1913295872, + "file_name": ".cache\\MatMulNBits_2_0_285.const", + "file_size": 7168 + }, + "model.layers.11.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 1913303040, + "file_name": ".cache\\MatMulNBits_2_0_286.const", + "file_size": 14680064 + }, + "model.layers.11.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 1927983104, + "file_name": ".cache\\MatMulNBits_2_0_287.const", + "file_size": 16384 + }, + "model.layers.11.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 1927999488, + "file_name": ".cache\\MatMulNBits_2_0_288.const", + "file_size": 458752 + }, + "model.layers.11.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 1928458240, + "file_name": ".cache\\MatMulNBits_2_0_289.const", + "file_size": 114688 + }, + "model.layers.11.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 1928572928, + "file_name": ".cache\\MatMulNBits_2_0_290.const", + "file_size": 1835008 + }, + "model.layers.11.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 1930407936, + "file_name": ".cache\\MatMulNBits_2_0_291.const", + "file_size": 2048 + }, + "model.layers.11.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 1930409984, + "file_name": ".cache\\MatMulNBits_2_0_292.const", + "file_size": 57344 + }, + "model.layers.11.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1930467328, + "file_name": ".cache\\MatMulNBits_2_0_293.const", + "file_size": 14336 + }, + "model.layers.11.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 1930481664, + "file_name": ".cache\\MatMulNBits_2_0_294.const", + "file_size": 12845056 + }, + "model.layers.11.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 1943326720, + "file_name": ".cache\\MatMulNBits_2_0_295.const", + "file_size": 14336 + }, + "model.layers.11.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 1943341056, + "file_name": ".cache\\MatMulNBits_2_0_296.const", + "file_size": 401408 + }, + "model.layers.11.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 1943742464, + "file_name": ".cache\\MatMulNBits_2_0_297.const", + "file_size": 100352 + }, + "model.layers.11.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 1943842816, + "file_name": ".cache\\MatMulNBits_2_0_298.const", + "file_size": 7168 + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1943849984, + "file_name": ".cache\\MatMulNBits_2_0_299.const", + "file_size": 33947648 + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 1977797632, + "file_name": ".cache\\MatMulNBits_2_0_300.const", + "file_size": 2121728 + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 1979919360, + "file_name": ".cache\\MatMulNBits_2_0_301.const", + "file_size": 265216 + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 1980184576, + "file_name": ".cache\\MatMulNBits_2_0_302.const", + "file_size": 75776 + }, + "model.layers.11.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 1980260352, + "file_name": ".cache\\MatMulNBits_2_0_303.const", + "file_size": 33947648 + }, + "model.layers.11.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2014208000, + "file_name": ".cache\\MatMulNBits_2_0_304.const", + "file_size": 2121728 + }, + "model.layers.11.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 2016329728, + "file_name": ".cache\\MatMulNBits_2_0_305.const", + "file_size": 265216 + }, + "model.layers.11.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 2016594944, + "file_name": ".cache\\MatMulNBits_2_0_306.const", + "file_size": 75776 + }, + "model.layers.11.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 2016670720, + "file_name": ".cache\\MatMulNBits_2_0_307.const", + "file_size": 67895296 + }, + "model.layers.11.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2084566016, + "file_name": ".cache\\MatMulNBits_2_0_308.const", + "file_size": 14336 + }, + "model.layers.11.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2084580352, + "file_name": ".cache\\MatMulNBits_2_0_309.const", + "file_size": 2121728 + }, + "model.layers.11.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 2086702080, + "file_name": ".cache\\MatMulNBits_2_0_310.const", + "file_size": 530432 + }, + "model.layers.12.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2087232512, + "file_name": ".cache\\MatMulNBits_2_0_311.const", + "file_size": 7168 + }, + "model.layers.12.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 2087239680, + "file_name": ".cache\\MatMulNBits_2_0_312.const", + "file_size": 14680064 + }, + "model.layers.12.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 2101919744, + "file_name": ".cache\\MatMulNBits_2_0_313.const", + "file_size": 16384 + }, + "model.layers.12.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 2101936128, + "file_name": ".cache\\MatMulNBits_2_0_314.const", + "file_size": 458752 + }, + "model.layers.12.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 2102394880, + "file_name": ".cache\\MatMulNBits_2_0_315.const", + "file_size": 114688 + }, + "model.layers.12.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 2102509568, + "file_name": ".cache\\MatMulNBits_2_0_316.const", + "file_size": 1835008 + }, + "model.layers.12.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 2104344576, + "file_name": ".cache\\MatMulNBits_2_0_317.const", + "file_size": 2048 + }, + "model.layers.12.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 2104346624, + "file_name": ".cache\\MatMulNBits_2_0_318.const", + "file_size": 57344 + }, + "model.layers.12.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2104403968, + "file_name": ".cache\\MatMulNBits_2_0_319.const", + "file_size": 14336 + }, + "model.layers.12.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 2104418304, + "file_name": ".cache\\MatMulNBits_2_0_320.const", + "file_size": 12845056 + }, + "model.layers.12.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2117263360, + "file_name": ".cache\\MatMulNBits_2_0_321.const", + "file_size": 14336 + }, + "model.layers.12.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 2117277696, + "file_name": ".cache\\MatMulNBits_2_0_322.const", + "file_size": 401408 + }, + "model.layers.12.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 2117679104, + "file_name": ".cache\\MatMulNBits_2_0_323.const", + "file_size": 100352 + }, + "model.layers.12.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2117779456, + "file_name": ".cache\\MatMulNBits_2_0_324.const", + "file_size": 7168 + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 2117786624, + "file_name": ".cache\\MatMulNBits_2_0_325.const", + "file_size": 33947648 + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2151734272, + "file_name": ".cache\\MatMulNBits_2_0_326.const", + "file_size": 2121728 + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 2153856000, + "file_name": ".cache\\MatMulNBits_2_0_327.const", + "file_size": 265216 + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 2154121216, + "file_name": ".cache\\MatMulNBits_2_0_328.const", + "file_size": 75776 + }, + "model.layers.12.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 2154196992, + "file_name": ".cache\\MatMulNBits_2_0_329.const", + "file_size": 33947648 + }, + "model.layers.12.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2188144640, + "file_name": ".cache\\MatMulNBits_2_0_330.const", + "file_size": 2121728 + }, + "model.layers.12.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 2190266368, + "file_name": ".cache\\MatMulNBits_2_0_331.const", + "file_size": 265216 + }, + "model.layers.12.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 2190531584, + "file_name": ".cache\\MatMulNBits_2_0_332.const", + "file_size": 75776 + }, + "model.layers.12.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 2190607360, + "file_name": ".cache\\MatMulNBits_2_0_333.const", + "file_size": 67895296 + }, + "model.layers.12.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2258502656, + "file_name": ".cache\\MatMulNBits_2_0_334.const", + "file_size": 14336 + }, + "model.layers.12.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2258516992, + "file_name": ".cache\\MatMulNBits_2_0_335.const", + "file_size": 2121728 + }, + "model.layers.12.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 2260638720, + "file_name": ".cache\\MatMulNBits_2_0_336.const", + "file_size": 530432 + }, + "model.layers.13.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2261169152, + "file_name": ".cache\\MatMulNBits_2_0_337.const", + "file_size": 7168 + }, + "model.layers.13.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 2261176320, + "file_name": ".cache\\MatMulNBits_2_0_338.const", + "file_size": 14680064 + }, + "model.layers.13.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 2275856384, + "file_name": ".cache\\MatMulNBits_2_0_339.const", + "file_size": 16384 + }, + "model.layers.13.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 2275872768, + "file_name": ".cache\\MatMulNBits_2_0_340.const", + "file_size": 458752 + }, + "model.layers.13.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 2276331520, + "file_name": ".cache\\MatMulNBits_2_0_341.const", + "file_size": 114688 + }, + "model.layers.13.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 2276446208, + "file_name": ".cache\\MatMulNBits_2_0_342.const", + "file_size": 1835008 + }, + "model.layers.13.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 2278281216, + "file_name": ".cache\\MatMulNBits_2_0_343.const", + "file_size": 2048 + }, + "model.layers.13.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 2278283264, + "file_name": ".cache\\MatMulNBits_2_0_344.const", + "file_size": 57344 + }, + "model.layers.13.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2278340608, + "file_name": ".cache\\MatMulNBits_2_0_345.const", + "file_size": 14336 + }, + "model.layers.13.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 2278354944, + "file_name": ".cache\\MatMulNBits_2_0_346.const", + "file_size": 12845056 + }, + "model.layers.13.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2291200000, + "file_name": ".cache\\MatMulNBits_2_0_347.const", + "file_size": 14336 + }, + "model.layers.13.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 2291214336, + "file_name": ".cache\\MatMulNBits_2_0_348.const", + "file_size": 401408 + }, + "model.layers.13.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 2291615744, + "file_name": ".cache\\MatMulNBits_2_0_349.const", + "file_size": 100352 + }, + "model.layers.13.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2291716096, + "file_name": ".cache\\MatMulNBits_2_0_350.const", + "file_size": 7168 + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 2291723264, + "file_name": ".cache\\MatMulNBits_2_0_351.const", + "file_size": 33947648 + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2325670912, + "file_name": ".cache\\MatMulNBits_2_0_352.const", + "file_size": 2121728 + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 2327792640, + "file_name": ".cache\\MatMulNBits_2_0_353.const", + "file_size": 265216 + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 2328057856, + "file_name": ".cache\\MatMulNBits_2_0_354.const", + "file_size": 75776 + }, + "model.layers.13.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 2328133632, + "file_name": ".cache\\MatMulNBits_2_0_355.const", + "file_size": 33947648 + }, + "model.layers.13.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2362081280, + "file_name": ".cache\\MatMulNBits_2_0_356.const", + "file_size": 2121728 + }, + "model.layers.13.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 2364203008, + "file_name": ".cache\\MatMulNBits_2_0_357.const", + "file_size": 265216 + }, + "model.layers.13.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 2364468224, + "file_name": ".cache\\MatMulNBits_2_0_358.const", + "file_size": 75776 + }, + "model.layers.13.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 2364544000, + "file_name": ".cache\\MatMulNBits_2_0_359.const", + "file_size": 67895296 + }, + "model.layers.13.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2432439296, + "file_name": ".cache\\MatMulNBits_2_0_360.const", + "file_size": 14336 + }, + "model.layers.13.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2432453632, + "file_name": ".cache\\MatMulNBits_2_0_361.const", + "file_size": 2121728 + }, + "model.layers.13.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 2434575360, + "file_name": ".cache\\MatMulNBits_2_0_362.const", + "file_size": 530432 + }, + "model.layers.14.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2435105792, + "file_name": ".cache\\MatMulNBits_2_0_363.const", + "file_size": 7168 + }, + "model.layers.14.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 2435112960, + "file_name": ".cache\\MatMulNBits_2_0_364.const", + "file_size": 14680064 + }, + "model.layers.14.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 2449793024, + "file_name": ".cache\\MatMulNBits_2_0_365.const", + "file_size": 16384 + }, + "model.layers.14.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 2449809408, + "file_name": ".cache\\MatMulNBits_2_0_366.const", + "file_size": 458752 + }, + "model.layers.14.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 2450268160, + "file_name": ".cache\\MatMulNBits_2_0_367.const", + "file_size": 114688 + }, + "model.layers.14.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 2450382848, + "file_name": ".cache\\MatMulNBits_2_0_368.const", + "file_size": 1835008 + }, + "model.layers.14.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 2452217856, + "file_name": ".cache\\MatMulNBits_2_0_369.const", + "file_size": 2048 + }, + "model.layers.14.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 2452219904, + "file_name": ".cache\\MatMulNBits_2_0_370.const", + "file_size": 57344 + }, + "model.layers.14.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2452277248, + "file_name": ".cache\\MatMulNBits_2_0_371.const", + "file_size": 14336 + }, + "model.layers.14.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 2452291584, + "file_name": ".cache\\MatMulNBits_2_0_372.const", + "file_size": 12845056 + }, + "model.layers.14.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2465136640, + "file_name": ".cache\\MatMulNBits_2_0_373.const", + "file_size": 14336 + }, + "model.layers.14.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 2465150976, + "file_name": ".cache\\MatMulNBits_2_0_374.const", + "file_size": 401408 + }, + "model.layers.14.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 2465552384, + "file_name": ".cache\\MatMulNBits_2_0_375.const", + "file_size": 100352 + }, + "model.layers.14.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2465652736, + "file_name": ".cache\\MatMulNBits_2_0_376.const", + "file_size": 7168 + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 2465659904, + "file_name": ".cache\\MatMulNBits_2_0_377.const", + "file_size": 33947648 + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2499607552, + "file_name": ".cache\\MatMulNBits_2_0_378.const", + "file_size": 2121728 + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 2501729280, + "file_name": ".cache\\MatMulNBits_2_0_379.const", + "file_size": 265216 + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 2501994496, + "file_name": ".cache\\MatMulNBits_2_0_380.const", + "file_size": 75776 + }, + "model.layers.14.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 2502070272, + "file_name": ".cache\\MatMulNBits_2_0_381.const", + "file_size": 33947648 + }, + "model.layers.14.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2536017920, + "file_name": ".cache\\MatMulNBits_2_0_382.const", + "file_size": 2121728 + }, + "model.layers.14.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 2538139648, + "file_name": ".cache\\MatMulNBits_2_0_383.const", + "file_size": 265216 + }, + "model.layers.14.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 2538404864, + "file_name": ".cache\\MatMulNBits_2_0_384.const", + "file_size": 75776 + }, + "model.layers.14.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 2538480640, + "file_name": ".cache\\MatMulNBits_2_0_385.const", + "file_size": 67895296 + }, + "model.layers.14.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2606375936, + "file_name": ".cache\\MatMulNBits_2_0_386.const", + "file_size": 14336 + }, + "model.layers.14.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2606390272, + "file_name": ".cache\\MatMulNBits_2_0_387.const", + "file_size": 2121728 + }, + "model.layers.14.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 2608512000, + "file_name": ".cache\\MatMulNBits_2_0_388.const", + "file_size": 530432 + }, + "model.layers.15.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2609042432, + "file_name": ".cache\\MatMulNBits_2_0_389.const", + "file_size": 7168 + }, + "model.layers.15.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 2609049600, + "file_name": ".cache\\MatMulNBits_2_0_390.const", + "file_size": 14680064 + }, + "model.layers.15.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 2623729664, + "file_name": ".cache\\MatMulNBits_2_0_391.const", + "file_size": 16384 + }, + "model.layers.15.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 2623746048, + "file_name": ".cache\\MatMulNBits_2_0_392.const", + "file_size": 458752 + }, + "model.layers.15.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 2624204800, + "file_name": ".cache\\MatMulNBits_2_0_393.const", + "file_size": 114688 + }, + "model.layers.15.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 2624319488, + "file_name": ".cache\\MatMulNBits_2_0_394.const", + "file_size": 1835008 + }, + "model.layers.15.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 2626154496, + "file_name": ".cache\\MatMulNBits_2_0_395.const", + "file_size": 2048 + }, + "model.layers.15.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 2626156544, + "file_name": ".cache\\MatMulNBits_2_0_396.const", + "file_size": 57344 + }, + "model.layers.15.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2626213888, + "file_name": ".cache\\MatMulNBits_2_0_397.const", + "file_size": 14336 + }, + "model.layers.15.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 2626228224, + "file_name": ".cache\\MatMulNBits_2_0_398.const", + "file_size": 12845056 + }, + "model.layers.15.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2639073280, + "file_name": ".cache\\MatMulNBits_2_0_399.const", + "file_size": 14336 + }, + "model.layers.15.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 2639087616, + "file_name": ".cache\\MatMulNBits_2_0_400.const", + "file_size": 401408 + }, + "model.layers.15.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 2639489024, + "file_name": ".cache\\MatMulNBits_2_0_401.const", + "file_size": 100352 + }, + "model.layers.15.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2639589376, + "file_name": ".cache\\MatMulNBits_2_0_402.const", + "file_size": 7168 + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 2639596544, + "file_name": ".cache\\MatMulNBits_2_0_403.const", + "file_size": 33947648 + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2673544192, + "file_name": ".cache\\MatMulNBits_2_0_404.const", + "file_size": 2121728 + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 2675665920, + "file_name": ".cache\\MatMulNBits_2_0_405.const", + "file_size": 265216 + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 2675931136, + "file_name": ".cache\\MatMulNBits_2_0_406.const", + "file_size": 75776 + }, + "model.layers.15.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 2676006912, + "file_name": ".cache\\MatMulNBits_2_0_407.const", + "file_size": 33947648 + }, + "model.layers.15.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2709954560, + "file_name": ".cache\\MatMulNBits_2_0_408.const", + "file_size": 2121728 + }, + "model.layers.15.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 2712076288, + "file_name": ".cache\\MatMulNBits_2_0_409.const", + "file_size": 265216 + }, + "model.layers.15.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 2712341504, + "file_name": ".cache\\MatMulNBits_2_0_410.const", + "file_size": 75776 + }, + "model.layers.15.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 2712417280, + "file_name": ".cache\\MatMulNBits_2_0_411.const", + "file_size": 67895296 + }, + "model.layers.15.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2780312576, + "file_name": ".cache\\MatMulNBits_2_0_412.const", + "file_size": 14336 + }, + "model.layers.15.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2780326912, + "file_name": ".cache\\MatMulNBits_2_0_413.const", + "file_size": 2121728 + }, + "model.layers.15.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 2782448640, + "file_name": ".cache\\MatMulNBits_2_0_414.const", + "file_size": 530432 + }, + "model.layers.16.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2782979072, + "file_name": ".cache\\MatMulNBits_2_0_415.const", + "file_size": 7168 + }, + "model.layers.16.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 2782986240, + "file_name": ".cache\\MatMulNBits_2_0_416.const", + "file_size": 14680064 + }, + "model.layers.16.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 2797666304, + "file_name": ".cache\\MatMulNBits_2_0_417.const", + "file_size": 16384 + }, + "model.layers.16.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 2797682688, + "file_name": ".cache\\MatMulNBits_2_0_418.const", + "file_size": 458752 + }, + "model.layers.16.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 2798141440, + "file_name": ".cache\\MatMulNBits_2_0_419.const", + "file_size": 114688 + }, + "model.layers.16.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 2798256128, + "file_name": ".cache\\MatMulNBits_2_0_420.const", + "file_size": 1835008 + }, + "model.layers.16.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 2800091136, + "file_name": ".cache\\MatMulNBits_2_0_421.const", + "file_size": 2048 + }, + "model.layers.16.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 2800093184, + "file_name": ".cache\\MatMulNBits_2_0_422.const", + "file_size": 57344 + }, + "model.layers.16.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2800150528, + "file_name": ".cache\\MatMulNBits_2_0_423.const", + "file_size": 14336 + }, + "model.layers.16.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 2800164864, + "file_name": ".cache\\MatMulNBits_2_0_424.const", + "file_size": 12845056 + }, + "model.layers.16.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2813009920, + "file_name": ".cache\\MatMulNBits_2_0_425.const", + "file_size": 14336 + }, + "model.layers.16.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 2813024256, + "file_name": ".cache\\MatMulNBits_2_0_426.const", + "file_size": 401408 + }, + "model.layers.16.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 2813425664, + "file_name": ".cache\\MatMulNBits_2_0_427.const", + "file_size": 100352 + }, + "model.layers.16.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2813526016, + "file_name": ".cache\\MatMulNBits_2_0_428.const", + "file_size": 7168 + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 2813533184, + "file_name": ".cache\\MatMulNBits_2_0_429.const", + "file_size": 33947648 + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2847480832, + "file_name": ".cache\\MatMulNBits_2_0_430.const", + "file_size": 2121728 + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 2849602560, + "file_name": ".cache\\MatMulNBits_2_0_431.const", + "file_size": 265216 + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 2849867776, + "file_name": ".cache\\MatMulNBits_2_0_432.const", + "file_size": 75776 + }, + "model.layers.16.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 2849943552, + "file_name": ".cache\\MatMulNBits_2_0_433.const", + "file_size": 33947648 + }, + "model.layers.16.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2883891200, + "file_name": ".cache\\MatMulNBits_2_0_434.const", + "file_size": 2121728 + }, + "model.layers.16.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 2886012928, + "file_name": ".cache\\MatMulNBits_2_0_435.const", + "file_size": 265216 + }, + "model.layers.16.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 2886278144, + "file_name": ".cache\\MatMulNBits_2_0_436.const", + "file_size": 75776 + }, + "model.layers.16.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 2886353920, + "file_name": ".cache\\MatMulNBits_2_0_437.const", + "file_size": 67895296 + }, + "model.layers.16.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2954249216, + "file_name": ".cache\\MatMulNBits_2_0_438.const", + "file_size": 14336 + }, + "model.layers.16.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 2954263552, + "file_name": ".cache\\MatMulNBits_2_0_439.const", + "file_size": 2121728 + }, + "model.layers.16.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 2956385280, + "file_name": ".cache\\MatMulNBits_2_0_440.const", + "file_size": 530432 + }, + "model.layers.17.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2956915712, + "file_name": ".cache\\MatMulNBits_2_0_441.const", + "file_size": 7168 + }, + "model.layers.17.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 2956922880, + "file_name": ".cache\\MatMulNBits_2_0_442.const", + "file_size": 14680064 + }, + "model.layers.17.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 2971602944, + "file_name": ".cache\\MatMulNBits_2_0_443.const", + "file_size": 16384 + }, + "model.layers.17.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 2971619328, + "file_name": ".cache\\MatMulNBits_2_0_444.const", + "file_size": 458752 + }, + "model.layers.17.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 2972078080, + "file_name": ".cache\\MatMulNBits_2_0_445.const", + "file_size": 114688 + }, + "model.layers.17.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 2972192768, + "file_name": ".cache\\MatMulNBits_2_0_446.const", + "file_size": 1835008 + }, + "model.layers.17.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 2974027776, + "file_name": ".cache\\MatMulNBits_2_0_447.const", + "file_size": 2048 + }, + "model.layers.17.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 2974029824, + "file_name": ".cache\\MatMulNBits_2_0_448.const", + "file_size": 57344 + }, + "model.layers.17.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2974087168, + "file_name": ".cache\\MatMulNBits_2_0_449.const", + "file_size": 14336 + }, + "model.layers.17.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 2974101504, + "file_name": ".cache\\MatMulNBits_2_0_450.const", + "file_size": 12845056 + }, + "model.layers.17.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 2986946560, + "file_name": ".cache\\MatMulNBits_2_0_451.const", + "file_size": 14336 + }, + "model.layers.17.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 2986960896, + "file_name": ".cache\\MatMulNBits_2_0_452.const", + "file_size": 401408 + }, + "model.layers.17.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 2987362304, + "file_name": ".cache\\MatMulNBits_2_0_453.const", + "file_size": 100352 + }, + "model.layers.17.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 2987462656, + "file_name": ".cache\\MatMulNBits_2_0_454.const", + "file_size": 7168 + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 2987469824, + "file_name": ".cache\\MatMulNBits_2_0_455.const", + "file_size": 33947648 + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3021417472, + "file_name": ".cache\\MatMulNBits_2_0_456.const", + "file_size": 2121728 + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3023539200, + "file_name": ".cache\\MatMulNBits_2_0_457.const", + "file_size": 265216 + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3023804416, + "file_name": ".cache\\MatMulNBits_2_0_458.const", + "file_size": 75776 + }, + "model.layers.17.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 3023880192, + "file_name": ".cache\\MatMulNBits_2_0_459.const", + "file_size": 33947648 + }, + "model.layers.17.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3057827840, + "file_name": ".cache\\MatMulNBits_2_0_460.const", + "file_size": 2121728 + }, + "model.layers.17.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3059949568, + "file_name": ".cache\\MatMulNBits_2_0_461.const", + "file_size": 265216 + }, + "model.layers.17.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3060214784, + "file_name": ".cache\\MatMulNBits_2_0_462.const", + "file_size": 75776 + }, + "model.layers.17.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 3060290560, + "file_name": ".cache\\MatMulNBits_2_0_463.const", + "file_size": 67895296 + }, + "model.layers.17.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3128185856, + "file_name": ".cache\\MatMulNBits_2_0_464.const", + "file_size": 14336 + }, + "model.layers.17.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3128200192, + "file_name": ".cache\\MatMulNBits_2_0_465.const", + "file_size": 2121728 + }, + "model.layers.17.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 3130321920, + "file_name": ".cache\\MatMulNBits_2_0_466.const", + "file_size": 530432 + }, + "model.layers.18.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 3130852352, + "file_name": ".cache\\MatMulNBits_2_0_467.const", + "file_size": 7168 + }, + "model.layers.18.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 3130859520, + "file_name": ".cache\\MatMulNBits_2_0_468.const", + "file_size": 14680064 + }, + "model.layers.18.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 3145539584, + "file_name": ".cache\\MatMulNBits_2_0_469.const", + "file_size": 16384 + }, + "model.layers.18.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 3145555968, + "file_name": ".cache\\MatMulNBits_2_0_470.const", + "file_size": 458752 + }, + "model.layers.18.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 3146014720, + "file_name": ".cache\\MatMulNBits_2_0_471.const", + "file_size": 114688 + }, + "model.layers.18.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 3146129408, + "file_name": ".cache\\MatMulNBits_2_0_472.const", + "file_size": 1835008 + }, + "model.layers.18.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 3147964416, + "file_name": ".cache\\MatMulNBits_2_0_473.const", + "file_size": 2048 + }, + "model.layers.18.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 3147966464, + "file_name": ".cache\\MatMulNBits_2_0_474.const", + "file_size": 57344 + }, + "model.layers.18.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3148023808, + "file_name": ".cache\\MatMulNBits_2_0_475.const", + "file_size": 14336 + }, + "model.layers.18.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 3148038144, + "file_name": ".cache\\MatMulNBits_2_0_476.const", + "file_size": 12845056 + }, + "model.layers.18.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3160883200, + "file_name": ".cache\\MatMulNBits_2_0_477.const", + "file_size": 14336 + }, + "model.layers.18.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 3160897536, + "file_name": ".cache\\MatMulNBits_2_0_478.const", + "file_size": 401408 + }, + "model.layers.18.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 3161298944, + "file_name": ".cache\\MatMulNBits_2_0_479.const", + "file_size": 100352 + }, + "model.layers.18.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 3161399296, + "file_name": ".cache\\MatMulNBits_2_0_480.const", + "file_size": 7168 + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 3161406464, + "file_name": ".cache\\MatMulNBits_2_0_481.const", + "file_size": 33947648 + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3195354112, + "file_name": ".cache\\MatMulNBits_2_0_482.const", + "file_size": 2121728 + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3197475840, + "file_name": ".cache\\MatMulNBits_2_0_483.const", + "file_size": 265216 + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3197741056, + "file_name": ".cache\\MatMulNBits_2_0_484.const", + "file_size": 75776 + }, + "model.layers.18.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 3197816832, + "file_name": ".cache\\MatMulNBits_2_0_485.const", + "file_size": 33947648 + }, + "model.layers.18.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3231764480, + "file_name": ".cache\\MatMulNBits_2_0_486.const", + "file_size": 2121728 + }, + "model.layers.18.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3233886208, + "file_name": ".cache\\MatMulNBits_2_0_487.const", + "file_size": 265216 + }, + "model.layers.18.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3234151424, + "file_name": ".cache\\MatMulNBits_2_0_488.const", + "file_size": 75776 + }, + "model.layers.18.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 3234227200, + "file_name": ".cache\\MatMulNBits_2_0_489.const", + "file_size": 67895296 + }, + "model.layers.18.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3302122496, + "file_name": ".cache\\MatMulNBits_2_0_490.const", + "file_size": 14336 + }, + "model.layers.18.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3302136832, + "file_name": ".cache\\MatMulNBits_2_0_491.const", + "file_size": 2121728 + }, + "model.layers.18.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 3304258560, + "file_name": ".cache\\MatMulNBits_2_0_492.const", + "file_size": 530432 + }, + "model.layers.19.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 3304788992, + "file_name": ".cache\\MatMulNBits_2_0_493.const", + "file_size": 7168 + }, + "model.layers.19.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 3304796160, + "file_name": ".cache\\MatMulNBits_2_0_494.const", + "file_size": 14680064 + }, + "model.layers.19.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 3319476224, + "file_name": ".cache\\MatMulNBits_2_0_495.const", + "file_size": 16384 + }, + "model.layers.19.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 3319492608, + "file_name": ".cache\\MatMulNBits_2_0_496.const", + "file_size": 458752 + }, + "model.layers.19.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 3319951360, + "file_name": ".cache\\MatMulNBits_2_0_497.const", + "file_size": 114688 + }, + "model.layers.19.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 3320066048, + "file_name": ".cache\\MatMulNBits_2_0_498.const", + "file_size": 1835008 + }, + "model.layers.19.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 3321901056, + "file_name": ".cache\\MatMulNBits_2_0_499.const", + "file_size": 2048 + }, + "model.layers.19.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 3321903104, + "file_name": ".cache\\MatMulNBits_2_0_500.const", + "file_size": 57344 + }, + "model.layers.19.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3321960448, + "file_name": ".cache\\MatMulNBits_2_0_501.const", + "file_size": 14336 + }, + "model.layers.19.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 3321974784, + "file_name": ".cache\\MatMulNBits_2_0_502.const", + "file_size": 12845056 + }, + "model.layers.19.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3334819840, + "file_name": ".cache\\MatMulNBits_2_0_503.const", + "file_size": 14336 + }, + "model.layers.19.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 3334834176, + "file_name": ".cache\\MatMulNBits_2_0_504.const", + "file_size": 401408 + }, + "model.layers.19.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 3335235584, + "file_name": ".cache\\MatMulNBits_2_0_505.const", + "file_size": 100352 + }, + "model.layers.19.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 3335335936, + "file_name": ".cache\\MatMulNBits_2_0_506.const", + "file_size": 7168 + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 3335343104, + "file_name": ".cache\\MatMulNBits_2_0_507.const", + "file_size": 33947648 + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3369290752, + "file_name": ".cache\\MatMulNBits_2_0_508.const", + "file_size": 2121728 + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3371412480, + "file_name": ".cache\\MatMulNBits_2_0_509.const", + "file_size": 265216 + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3371677696, + "file_name": ".cache\\MatMulNBits_2_0_510.const", + "file_size": 75776 + }, + "model.layers.19.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 3371753472, + "file_name": ".cache\\MatMulNBits_2_0_511.const", + "file_size": 33947648 + }, + "model.layers.19.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3405701120, + "file_name": ".cache\\MatMulNBits_2_0_512.const", + "file_size": 2121728 + }, + "model.layers.19.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3407822848, + "file_name": ".cache\\MatMulNBits_2_0_513.const", + "file_size": 265216 + }, + "model.layers.19.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3408088064, + "file_name": ".cache\\MatMulNBits_2_0_514.const", + "file_size": 75776 + }, + "model.layers.19.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 3408163840, + "file_name": ".cache\\MatMulNBits_2_0_515.const", + "file_size": 67895296 + }, + "model.layers.19.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3476059136, + "file_name": ".cache\\MatMulNBits_2_0_516.const", + "file_size": 14336 + }, + "model.layers.19.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3476073472, + "file_name": ".cache\\MatMulNBits_2_0_517.const", + "file_size": 2121728 + }, + "model.layers.19.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 3478195200, + "file_name": ".cache\\MatMulNBits_2_0_518.const", + "file_size": 530432 + }, + "model.layers.20.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 3478725632, + "file_name": ".cache\\MatMulNBits_2_0_519.const", + "file_size": 7168 + }, + "model.layers.20.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 3478732800, + "file_name": ".cache\\MatMulNBits_2_0_520.const", + "file_size": 14680064 + }, + "model.layers.20.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 3493412864, + "file_name": ".cache\\MatMulNBits_2_0_521.const", + "file_size": 16384 + }, + "model.layers.20.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 3493429248, + "file_name": ".cache\\MatMulNBits_2_0_522.const", + "file_size": 458752 + }, + "model.layers.20.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 3493888000, + "file_name": ".cache\\MatMulNBits_2_0_523.const", + "file_size": 114688 + }, + "model.layers.20.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 3494002688, + "file_name": ".cache\\MatMulNBits_2_0_524.const", + "file_size": 1835008 + }, + "model.layers.20.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 3495837696, + "file_name": ".cache\\MatMulNBits_2_0_525.const", + "file_size": 2048 + }, + "model.layers.20.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 3495839744, + "file_name": ".cache\\MatMulNBits_2_0_526.const", + "file_size": 57344 + }, + "model.layers.20.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3495897088, + "file_name": ".cache\\MatMulNBits_2_0_527.const", + "file_size": 14336 + }, + "model.layers.20.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 3495911424, + "file_name": ".cache\\MatMulNBits_2_0_528.const", + "file_size": 12845056 + }, + "model.layers.20.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3508756480, + "file_name": ".cache\\MatMulNBits_2_0_529.const", + "file_size": 14336 + }, + "model.layers.20.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 3508770816, + "file_name": ".cache\\MatMulNBits_2_0_530.const", + "file_size": 401408 + }, + "model.layers.20.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 3509172224, + "file_name": ".cache\\MatMulNBits_2_0_531.const", + "file_size": 100352 + }, + "model.layers.20.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 3509272576, + "file_name": ".cache\\MatMulNBits_2_0_532.const", + "file_size": 7168 + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 3509279744, + "file_name": ".cache\\MatMulNBits_2_0_533.const", + "file_size": 33947648 + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3543227392, + "file_name": ".cache\\MatMulNBits_2_0_534.const", + "file_size": 2121728 + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3545349120, + "file_name": ".cache\\MatMulNBits_2_0_535.const", + "file_size": 265216 + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3545614336, + "file_name": ".cache\\MatMulNBits_2_0_536.const", + "file_size": 75776 + }, + "model.layers.20.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 3545690112, + "file_name": ".cache\\MatMulNBits_2_0_537.const", + "file_size": 33947648 + }, + "model.layers.20.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3579637760, + "file_name": ".cache\\MatMulNBits_2_0_538.const", + "file_size": 2121728 + }, + "model.layers.20.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3581759488, + "file_name": ".cache\\MatMulNBits_2_0_539.const", + "file_size": 265216 + }, + "model.layers.20.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3582024704, + "file_name": ".cache\\MatMulNBits_2_0_540.const", + "file_size": 75776 + }, + "model.layers.20.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 3582100480, + "file_name": ".cache\\MatMulNBits_2_0_541.const", + "file_size": 67895296 + }, + "model.layers.20.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3649995776, + "file_name": ".cache\\MatMulNBits_2_0_542.const", + "file_size": 14336 + }, + "model.layers.20.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3650010112, + "file_name": ".cache\\MatMulNBits_2_0_543.const", + "file_size": 2121728 + }, + "model.layers.20.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 3652131840, + "file_name": ".cache\\MatMulNBits_2_0_544.const", + "file_size": 530432 + }, + "model.layers.21.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 3652662272, + "file_name": ".cache\\MatMulNBits_2_0_545.const", + "file_size": 7168 + }, + "model.layers.21.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 3652669440, + "file_name": ".cache\\MatMulNBits_2_0_546.const", + "file_size": 14680064 + }, + "model.layers.21.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 3667349504, + "file_name": ".cache\\MatMulNBits_2_0_547.const", + "file_size": 16384 + }, + "model.layers.21.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 3667365888, + "file_name": ".cache\\MatMulNBits_2_0_548.const", + "file_size": 458752 + }, + "model.layers.21.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 3667824640, + "file_name": ".cache\\MatMulNBits_2_0_549.const", + "file_size": 114688 + }, + "model.layers.21.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 3667939328, + "file_name": ".cache\\MatMulNBits_2_0_550.const", + "file_size": 1835008 + }, + "model.layers.21.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 3669774336, + "file_name": ".cache\\MatMulNBits_2_0_551.const", + "file_size": 2048 + }, + "model.layers.21.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 3669776384, + "file_name": ".cache\\MatMulNBits_2_0_552.const", + "file_size": 57344 + }, + "model.layers.21.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3669833728, + "file_name": ".cache\\MatMulNBits_2_0_553.const", + "file_size": 14336 + }, + "model.layers.21.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 3669848064, + "file_name": ".cache\\MatMulNBits_2_0_554.const", + "file_size": 12845056 + }, + "model.layers.21.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3682693120, + "file_name": ".cache\\MatMulNBits_2_0_555.const", + "file_size": 14336 + }, + "model.layers.21.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 3682707456, + "file_name": ".cache\\MatMulNBits_2_0_556.const", + "file_size": 401408 + }, + "model.layers.21.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 3683108864, + "file_name": ".cache\\MatMulNBits_2_0_557.const", + "file_size": 100352 + }, + "model.layers.21.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 3683209216, + "file_name": ".cache\\MatMulNBits_2_0_558.const", + "file_size": 7168 + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 3683216384, + "file_name": ".cache\\MatMulNBits_2_0_559.const", + "file_size": 33947648 + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3717164032, + "file_name": ".cache\\MatMulNBits_2_0_560.const", + "file_size": 2121728 + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3719285760, + "file_name": ".cache\\MatMulNBits_2_0_561.const", + "file_size": 265216 + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3719550976, + "file_name": ".cache\\MatMulNBits_2_0_562.const", + "file_size": 75776 + }, + "model.layers.21.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 3719626752, + "file_name": ".cache\\MatMulNBits_2_0_563.const", + "file_size": 33947648 + }, + "model.layers.21.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3753574400, + "file_name": ".cache\\MatMulNBits_2_0_564.const", + "file_size": 2121728 + }, + "model.layers.21.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3755696128, + "file_name": ".cache\\MatMulNBits_2_0_565.const", + "file_size": 265216 + }, + "model.layers.21.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3755961344, + "file_name": ".cache\\MatMulNBits_2_0_566.const", + "file_size": 75776 + }, + "model.layers.21.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 3756037120, + "file_name": ".cache\\MatMulNBits_2_0_567.const", + "file_size": 67895296 + }, + "model.layers.21.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3823932416, + "file_name": ".cache\\MatMulNBits_2_0_568.const", + "file_size": 14336 + }, + "model.layers.21.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3823946752, + "file_name": ".cache\\MatMulNBits_2_0_569.const", + "file_size": 2121728 + }, + "model.layers.21.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 3826068480, + "file_name": ".cache\\MatMulNBits_2_0_570.const", + "file_size": 530432 + }, + "model.layers.22.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 3826598912, + "file_name": ".cache\\MatMulNBits_2_0_571.const", + "file_size": 7168 + }, + "model.layers.22.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 3826606080, + "file_name": ".cache\\MatMulNBits_2_0_572.const", + "file_size": 14680064 + }, + "model.layers.22.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 3841286144, + "file_name": ".cache\\MatMulNBits_2_0_573.const", + "file_size": 16384 + }, + "model.layers.22.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 3841302528, + "file_name": ".cache\\MatMulNBits_2_0_574.const", + "file_size": 458752 + }, + "model.layers.22.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 3841761280, + "file_name": ".cache\\MatMulNBits_2_0_575.const", + "file_size": 114688 + }, + "model.layers.22.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 3841875968, + "file_name": ".cache\\MatMulNBits_2_0_576.const", + "file_size": 1835008 + }, + "model.layers.22.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 3843710976, + "file_name": ".cache\\MatMulNBits_2_0_577.const", + "file_size": 2048 + }, + "model.layers.22.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 3843713024, + "file_name": ".cache\\MatMulNBits_2_0_578.const", + "file_size": 57344 + }, + "model.layers.22.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3843770368, + "file_name": ".cache\\MatMulNBits_2_0_579.const", + "file_size": 14336 + }, + "model.layers.22.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 3843784704, + "file_name": ".cache\\MatMulNBits_2_0_580.const", + "file_size": 12845056 + }, + "model.layers.22.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3856629760, + "file_name": ".cache\\MatMulNBits_2_0_581.const", + "file_size": 14336 + }, + "model.layers.22.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 3856644096, + "file_name": ".cache\\MatMulNBits_2_0_582.const", + "file_size": 401408 + }, + "model.layers.22.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 3857045504, + "file_name": ".cache\\MatMulNBits_2_0_583.const", + "file_size": 100352 + }, + "model.layers.22.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 3857145856, + "file_name": ".cache\\MatMulNBits_2_0_584.const", + "file_size": 7168 + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 3857153024, + "file_name": ".cache\\MatMulNBits_2_0_585.const", + "file_size": 33947648 + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3891100672, + "file_name": ".cache\\MatMulNBits_2_0_586.const", + "file_size": 2121728 + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3893222400, + "file_name": ".cache\\MatMulNBits_2_0_587.const", + "file_size": 265216 + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3893487616, + "file_name": ".cache\\MatMulNBits_2_0_588.const", + "file_size": 75776 + }, + "model.layers.22.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 3893563392, + "file_name": ".cache\\MatMulNBits_2_0_589.const", + "file_size": 33947648 + }, + "model.layers.22.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3927511040, + "file_name": ".cache\\MatMulNBits_2_0_590.const", + "file_size": 2121728 + }, + "model.layers.22.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 3929632768, + "file_name": ".cache\\MatMulNBits_2_0_591.const", + "file_size": 265216 + }, + "model.layers.22.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 3929897984, + "file_name": ".cache\\MatMulNBits_2_0_592.const", + "file_size": 75776 + }, + "model.layers.22.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 3929973760, + "file_name": ".cache\\MatMulNBits_2_0_593.const", + "file_size": 67895296 + }, + "model.layers.22.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 3997869056, + "file_name": ".cache\\MatMulNBits_2_0_594.const", + "file_size": 14336 + }, + "model.layers.22.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 3997883392, + "file_name": ".cache\\MatMulNBits_2_0_595.const", + "file_size": 2121728 + }, + "model.layers.22.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 4000005120, + "file_name": ".cache\\MatMulNBits_2_0_596.const", + "file_size": 530432 + }, + "model.layers.23.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 4000535552, + "file_name": ".cache\\MatMulNBits_2_0_597.const", + "file_size": 7168 + }, + "model.layers.23.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 4000542720, + "file_name": ".cache\\MatMulNBits_2_0_598.const", + "file_size": 14680064 + }, + "model.layers.23.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 4015222784, + "file_name": ".cache\\MatMulNBits_2_0_599.const", + "file_size": 16384 + }, + "model.layers.23.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 4015239168, + "file_name": ".cache\\MatMulNBits_2_0_600.const", + "file_size": 458752 + }, + "model.layers.23.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 4015697920, + "file_name": ".cache\\MatMulNBits_2_0_601.const", + "file_size": 114688 + }, + "model.layers.23.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 4015812608, + "file_name": ".cache\\MatMulNBits_2_0_602.const", + "file_size": 1835008 + }, + "model.layers.23.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 4017647616, + "file_name": ".cache\\MatMulNBits_2_0_603.const", + "file_size": 2048 + }, + "model.layers.23.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 4017649664, + "file_name": ".cache\\MatMulNBits_2_0_604.const", + "file_size": 57344 + }, + "model.layers.23.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4017707008, + "file_name": ".cache\\MatMulNBits_2_0_605.const", + "file_size": 14336 + }, + "model.layers.23.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 4017721344, + "file_name": ".cache\\MatMulNBits_2_0_606.const", + "file_size": 12845056 + }, + "model.layers.23.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4030566400, + "file_name": ".cache\\MatMulNBits_2_0_607.const", + "file_size": 14336 + }, + "model.layers.23.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 4030580736, + "file_name": ".cache\\MatMulNBits_2_0_608.const", + "file_size": 401408 + }, + "model.layers.23.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 4030982144, + "file_name": ".cache\\MatMulNBits_2_0_609.const", + "file_size": 100352 + }, + "model.layers.23.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 4031082496, + "file_name": ".cache\\MatMulNBits_2_0_610.const", + "file_size": 7168 + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 4031089664, + "file_name": ".cache\\MatMulNBits_2_0_611.const", + "file_size": 33947648 + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4065037312, + "file_name": ".cache\\MatMulNBits_2_0_612.const", + "file_size": 2121728 + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 4067159040, + "file_name": ".cache\\MatMulNBits_2_0_613.const", + "file_size": 265216 + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 4067424256, + "file_name": ".cache\\MatMulNBits_2_0_614.const", + "file_size": 75776 + }, + "model.layers.23.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 4067500032, + "file_name": ".cache\\MatMulNBits_2_0_615.const", + "file_size": 33947648 + }, + "model.layers.23.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4101447680, + "file_name": ".cache\\MatMulNBits_2_0_616.const", + "file_size": 2121728 + }, + "model.layers.23.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 4103569408, + "file_name": ".cache\\MatMulNBits_2_0_617.const", + "file_size": 265216 + }, + "model.layers.23.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 4103834624, + "file_name": ".cache\\MatMulNBits_2_0_618.const", + "file_size": 75776 + }, + "model.layers.23.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 4103910400, + "file_name": ".cache\\MatMulNBits_2_0_619.const", + "file_size": 67895296 + }, + "model.layers.23.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4171805696, + "file_name": ".cache\\MatMulNBits_2_0_620.const", + "file_size": 14336 + }, + "model.layers.23.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4171820032, + "file_name": ".cache\\MatMulNBits_2_0_621.const", + "file_size": 2121728 + }, + "model.layers.23.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 4173941760, + "file_name": ".cache\\MatMulNBits_2_0_622.const", + "file_size": 530432 + }, + "model.layers.24.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 4174472192, + "file_name": ".cache\\MatMulNBits_2_0_623.const", + "file_size": 7168 + }, + "model.layers.24.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 4174479360, + "file_name": ".cache\\MatMulNBits_2_0_624.const", + "file_size": 14680064 + }, + "model.layers.24.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 4189159424, + "file_name": ".cache\\MatMulNBits_2_0_625.const", + "file_size": 16384 + }, + "model.layers.24.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 4189175808, + "file_name": ".cache\\MatMulNBits_2_0_626.const", + "file_size": 458752 + }, + "model.layers.24.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 4189634560, + "file_name": ".cache\\MatMulNBits_2_0_627.const", + "file_size": 114688 + }, + "model.layers.24.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 4189749248, + "file_name": ".cache\\MatMulNBits_2_0_628.const", + "file_size": 1835008 + }, + "model.layers.24.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 4191584256, + "file_name": ".cache\\MatMulNBits_2_0_629.const", + "file_size": 2048 + }, + "model.layers.24.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 4191586304, + "file_name": ".cache\\MatMulNBits_2_0_630.const", + "file_size": 57344 + }, + "model.layers.24.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4191643648, + "file_name": ".cache\\MatMulNBits_2_0_631.const", + "file_size": 14336 + }, + "model.layers.24.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 4191657984, + "file_name": ".cache\\MatMulNBits_2_0_632.const", + "file_size": 12845056 + }, + "model.layers.24.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4204503040, + "file_name": ".cache\\MatMulNBits_2_0_633.const", + "file_size": 14336 + }, + "model.layers.24.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 4204517376, + "file_name": ".cache\\MatMulNBits_2_0_634.const", + "file_size": 401408 + }, + "model.layers.24.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 4204918784, + "file_name": ".cache\\MatMulNBits_2_0_635.const", + "file_size": 100352 + }, + "model.layers.24.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 4205019136, + "file_name": ".cache\\MatMulNBits_2_0_636.const", + "file_size": 7168 + }, + "model.layers.24.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 4205026304, + "file_name": ".cache\\MatMulNBits_2_0_637.const", + "file_size": 33947648 + }, + "model.layers.24.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4238973952, + "file_name": ".cache\\MatMulNBits_2_0_638.const", + "file_size": 2121728 + }, + "model.layers.24.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 4241095680, + "file_name": ".cache\\MatMulNBits_2_0_639.const", + "file_size": 265216 + }, + "model.layers.24.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 4241360896, + "file_name": ".cache\\MatMulNBits_2_0_640.const", + "file_size": 75776 + }, + "model.layers.24.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 4241436672, + "file_name": ".cache\\MatMulNBits_2_0_641.const", + "file_size": 33947648 + }, + "model.layers.24.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4275384320, + "file_name": ".cache\\MatMulNBits_2_0_642.const", + "file_size": 2121728 + }, + "model.layers.24.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 4277506048, + "file_name": ".cache\\MatMulNBits_2_0_643.const", + "file_size": 265216 + }, + "model.layers.24.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 4277771264, + "file_name": ".cache\\MatMulNBits_2_0_644.const", + "file_size": 75776 + }, + "model.layers.24.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 4277847040, + "file_name": ".cache\\MatMulNBits_2_0_645.const", + "file_size": 67895296 + }, + "model.layers.24.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4345742336, + "file_name": ".cache\\MatMulNBits_2_0_646.const", + "file_size": 14336 + }, + "model.layers.24.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4345756672, + "file_name": ".cache\\MatMulNBits_2_0_647.const", + "file_size": 2121728 + }, + "model.layers.24.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 4347878400, + "file_name": ".cache\\MatMulNBits_2_0_648.const", + "file_size": 530432 + }, + "model.layers.25.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 4348408832, + "file_name": ".cache\\MatMulNBits_2_0_649.const", + "file_size": 7168 + }, + "model.layers.25.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 4348416000, + "file_name": ".cache\\MatMulNBits_2_0_650.const", + "file_size": 14680064 + }, + "model.layers.25.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 4363096064, + "file_name": ".cache\\MatMulNBits_2_0_651.const", + "file_size": 16384 + }, + "model.layers.25.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 4363112448, + "file_name": ".cache\\MatMulNBits_2_0_652.const", + "file_size": 458752 + }, + "model.layers.25.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 4363571200, + "file_name": ".cache\\MatMulNBits_2_0_653.const", + "file_size": 114688 + }, + "model.layers.25.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 4363685888, + "file_name": ".cache\\MatMulNBits_2_0_654.const", + "file_size": 1835008 + }, + "model.layers.25.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 4365520896, + "file_name": ".cache\\MatMulNBits_2_0_655.const", + "file_size": 2048 + }, + "model.layers.25.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 4365522944, + "file_name": ".cache\\MatMulNBits_2_0_656.const", + "file_size": 57344 + }, + "model.layers.25.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4365580288, + "file_name": ".cache\\MatMulNBits_2_0_657.const", + "file_size": 14336 + }, + "model.layers.25.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 4365594624, + "file_name": ".cache\\MatMulNBits_2_0_658.const", + "file_size": 12845056 + }, + "model.layers.25.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4378439680, + "file_name": ".cache\\MatMulNBits_2_0_659.const", + "file_size": 14336 + }, + "model.layers.25.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 4378454016, + "file_name": ".cache\\MatMulNBits_2_0_660.const", + "file_size": 401408 + }, + "model.layers.25.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 4378855424, + "file_name": ".cache\\MatMulNBits_2_0_661.const", + "file_size": 100352 + }, + "model.layers.25.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 4378955776, + "file_name": ".cache\\MatMulNBits_2_0_662.const", + "file_size": 7168 + }, + "model.layers.25.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 4378962944, + "file_name": ".cache\\MatMulNBits_2_0_663.const", + "file_size": 33947648 + }, + "model.layers.25.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4412910592, + "file_name": ".cache\\MatMulNBits_2_0_664.const", + "file_size": 2121728 + }, + "model.layers.25.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 4415032320, + "file_name": ".cache\\MatMulNBits_2_0_665.const", + "file_size": 265216 + }, + "model.layers.25.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 4415297536, + "file_name": ".cache\\MatMulNBits_2_0_666.const", + "file_size": 75776 + }, + "model.layers.25.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 4415373312, + "file_name": ".cache\\MatMulNBits_2_0_667.const", + "file_size": 33947648 + }, + "model.layers.25.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4449320960, + "file_name": ".cache\\MatMulNBits_2_0_668.const", + "file_size": 2121728 + }, + "model.layers.25.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 4451442688, + "file_name": ".cache\\MatMulNBits_2_0_669.const", + "file_size": 265216 + }, + "model.layers.25.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 4451707904, + "file_name": ".cache\\MatMulNBits_2_0_670.const", + "file_size": 75776 + }, + "model.layers.25.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 4451783680, + "file_name": ".cache\\MatMulNBits_2_0_671.const", + "file_size": 67895296 + }, + "model.layers.25.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4519678976, + "file_name": ".cache\\MatMulNBits_2_0_672.const", + "file_size": 14336 + }, + "model.layers.25.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4519693312, + "file_name": ".cache\\MatMulNBits_2_0_673.const", + "file_size": 2121728 + }, + "model.layers.25.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 4521815040, + "file_name": ".cache\\MatMulNBits_2_0_674.const", + "file_size": 530432 + }, + "model.layers.26.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 4522345472, + "file_name": ".cache\\MatMulNBits_2_0_675.const", + "file_size": 7168 + }, + "model.layers.26.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 4522352640, + "file_name": ".cache\\MatMulNBits_2_0_676.const", + "file_size": 14680064 + }, + "model.layers.26.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 4537032704, + "file_name": ".cache\\MatMulNBits_2_0_677.const", + "file_size": 16384 + }, + "model.layers.26.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 4537049088, + "file_name": ".cache\\MatMulNBits_2_0_678.const", + "file_size": 458752 + }, + "model.layers.26.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 4537507840, + "file_name": ".cache\\MatMulNBits_2_0_679.const", + "file_size": 114688 + }, + "model.layers.26.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 4537622528, + "file_name": ".cache\\MatMulNBits_2_0_680.const", + "file_size": 1835008 + }, + "model.layers.26.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 4539457536, + "file_name": ".cache\\MatMulNBits_2_0_681.const", + "file_size": 2048 + }, + "model.layers.26.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 4539459584, + "file_name": ".cache\\MatMulNBits_2_0_682.const", + "file_size": 57344 + }, + "model.layers.26.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4539516928, + "file_name": ".cache\\MatMulNBits_2_0_683.const", + "file_size": 14336 + }, + "model.layers.26.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 4539531264, + "file_name": ".cache\\MatMulNBits_2_0_684.const", + "file_size": 12845056 + }, + "model.layers.26.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4552376320, + "file_name": ".cache\\MatMulNBits_2_0_685.const", + "file_size": 14336 + }, + "model.layers.26.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 4552390656, + "file_name": ".cache\\MatMulNBits_2_0_686.const", + "file_size": 401408 + }, + "model.layers.26.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 4552792064, + "file_name": ".cache\\MatMulNBits_2_0_687.const", + "file_size": 100352 + }, + "model.layers.26.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 4552892416, + "file_name": ".cache\\MatMulNBits_2_0_688.const", + "file_size": 7168 + }, + "model.layers.26.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 4552899584, + "file_name": ".cache\\MatMulNBits_2_0_689.const", + "file_size": 33947648 + }, + "model.layers.26.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4586847232, + "file_name": ".cache\\MatMulNBits_2_0_690.const", + "file_size": 2121728 + }, + "model.layers.26.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 4588968960, + "file_name": ".cache\\MatMulNBits_2_0_691.const", + "file_size": 265216 + }, + "model.layers.26.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 4589234176, + "file_name": ".cache\\MatMulNBits_2_0_692.const", + "file_size": 75776 + }, + "model.layers.26.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 4589309952, + "file_name": ".cache\\MatMulNBits_2_0_693.const", + "file_size": 33947648 + }, + "model.layers.26.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4623257600, + "file_name": ".cache\\MatMulNBits_2_0_694.const", + "file_size": 2121728 + }, + "model.layers.26.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 4625379328, + "file_name": ".cache\\MatMulNBits_2_0_695.const", + "file_size": 265216 + }, + "model.layers.26.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 4625644544, + "file_name": ".cache\\MatMulNBits_2_0_696.const", + "file_size": 75776 + }, + "model.layers.26.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 4625720320, + "file_name": ".cache\\MatMulNBits_2_0_697.const", + "file_size": 67895296 + }, + "model.layers.26.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4693615616, + "file_name": ".cache\\MatMulNBits_2_0_698.const", + "file_size": 14336 + }, + "model.layers.26.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4693629952, + "file_name": ".cache\\MatMulNBits_2_0_699.const", + "file_size": 2121728 + }, + "model.layers.26.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 4695751680, + "file_name": ".cache\\MatMulNBits_2_0_700.const", + "file_size": 530432 + }, + "model.layers.27.input_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 4696282112, + "file_name": ".cache\\MatMulNBits_2_0_701.const", + "file_size": 7168 + }, + "model.layers.27.attn.qk_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 4096 + ], + "size_in_bytes": 14680064, + "op_tensor_size": 14680064, + "offset": 4696289280, + "file_name": ".cache\\MatMulNBits_2_0_702.const", + "file_size": 14680064 + }, + "model.layers.27.attn.qk_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4096 + ], + "size_in_bytes": 16384, + "op_tensor_size": 16384, + "offset": 4710969344, + "file_name": ".cache\\MatMulNBits_2_0_703.const", + "file_size": 16384 + }, + "model.layers.27.attn.qk_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 114688 + ], + "size_in_bytes": 458752, + "op_tensor_size": 458752, + "offset": 4710985728, + "file_name": ".cache\\MatMulNBits_2_0_704.const", + "file_size": 458752 + }, + "model.layers.27.attn.qk_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 114688 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 4711444480, + "file_name": ".cache\\MatMulNBits_2_0_705.const", + "file_size": 114688 + }, + "model.layers.27.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 512 + ], + "size_in_bytes": 1835008, + "op_tensor_size": 1835008, + "offset": 4711559168, + "file_name": ".cache\\MatMulNBits_2_0_706.const", + "file_size": 1835008 + }, + "model.layers.27.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 512 + ], + "size_in_bytes": 2048, + "op_tensor_size": 2048, + "offset": 4713394176, + "file_name": ".cache\\MatMulNBits_2_0_707.const", + "file_size": 2048 + }, + "model.layers.27.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 14336 + ], + "size_in_bytes": 57344, + "op_tensor_size": 57344, + "offset": 4713396224, + "file_name": ".cache\\MatMulNBits_2_0_708.const", + "file_size": 57344 + }, + "model.layers.27.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 14336 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4713453568, + "file_name": ".cache\\MatMulNBits_2_0_709.const", + "file_size": 14336 + }, + "model.layers.27.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 3584 + ], + "size_in_bytes": 12845056, + "op_tensor_size": 12845056, + "offset": 4713467904, + "file_name": ".cache\\MatMulNBits_2_0_710.const", + "file_size": 12845056 + }, + "model.layers.27.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4726312960, + "file_name": ".cache\\MatMulNBits_2_0_711.const", + "file_size": 14336 + }, + "model.layers.27.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 100352 + ], + "size_in_bytes": 401408, + "op_tensor_size": 401408, + "offset": 4726327296, + "file_name": ".cache\\MatMulNBits_2_0_712.const", + "file_size": 401408 + }, + "model.layers.27.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 100352 + ], + "size_in_bytes": 100352, + "op_tensor_size": 100352, + "offset": 4726728704, + "file_name": ".cache\\MatMulNBits_2_0_713.const", + "file_size": 100352 + }, + "model.layers.27.post_attention_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 4726829056, + "file_name": ".cache\\MatMulNBits_2_0_714.const", + "file_size": 7168 + }, + "model.layers.27.mlp.gate_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 4726836224, + "file_name": ".cache\\MatMulNBits_2_0_715.const", + "file_size": 33947648 + }, + "model.layers.27.mlp.gate_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4760783872, + "file_name": ".cache\\MatMulNBits_2_0_716.const", + "file_size": 2121728 + }, + "model.layers.27.mlp.gate_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 4762905600, + "file_name": ".cache\\MatMulNBits_2_0_717.const", + "file_size": 265216 + }, + "model.layers.27.mlp.gate_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 4763170816, + "file_name": ".cache\\MatMulNBits_2_0_718.const", + "file_size": 75776 + }, + "model.layers.27.mlp.up_proj.MatMulNBits.qweight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 18944, + 28, + 64 + ], + "size_in_bytes": 33947648, + "op_tensor_size": 33947648, + "offset": 4763246592, + "file_name": ".cache\\MatMulNBits_2_0_719.const", + "file_size": 33947648 + }, + "model.layers.27.mlp.up_proj.MatMulNBits.scales.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4797194240, + "file_name": ".cache\\MatMulNBits_2_0_720.const", + "file_size": 2121728 + }, + "model.layers.27.mlp.up_proj.MatMulNBits.qzeros": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "uint8", + "shape": [ + 265216 + ], + "size_in_bytes": 265216, + "op_tensor_size": 265216, + "offset": 4799315968, + "file_name": ".cache\\MatMulNBits_2_0_721.const", + "file_size": 265216 + }, + "model.layers.27.mlp.up_proj.MatMulNBits.bias.f": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 18944 + ], + "size_in_bytes": 75776, + "op_tensor_size": 75776, + "offset": 4799581184, + "file_name": ".cache\\MatMulNBits_2_0_722.const", + "file_size": 75776 + }, + "model.layers.27.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 18944, + 3584 + ], + "size_in_bytes": 67895296, + "op_tensor_size": 67895296, + "offset": 4799656960, + "file_name": ".cache\\MatMulNBits_2_0_723.const", + "file_size": 67895296 + }, + "model.layers.27.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 3584 + ], + "size_in_bytes": 14336, + "op_tensor_size": 14336, + "offset": 4867552256, + "file_name": ".cache\\MatMulNBits_2_0_724.const", + "file_size": 14336 + }, + "model.layers.27.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 530432 + ], + "size_in_bytes": 2121728, + "op_tensor_size": 2121728, + "offset": 4867566592, + "file_name": ".cache\\MatMulNBits_2_0_725.const", + "file_size": 2121728 + }, + "model.layers.27.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 530432 + ], + "size_in_bytes": 530432, + "op_tensor_size": 530432, + "offset": 4869688320, + "file_name": ".cache\\MatMulNBits_2_0_726.const", + "file_size": 530432 + }, + "model.layers.28.final_norm_layernorm.weight.bf": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 3584 + ], + "size_in_bytes": 7168, + "op_tensor_size": 7168, + "offset": 4870218752, + "file_name": ".cache\\MatMulNBits_2_0_727.const", + "file_size": 7168 + }, + "lm_head.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 3584, + 152064 + ], + "size_in_bytes": 544997376, + "op_tensor_size": 544997376, + "offset": 4870225920, + "file_name": ".cache\\MatMulNBits_2_0_728.const", + "file_size": 544997376 + }, + "lm_head.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 152064 + ], + "size_in_bytes": 608256, + "op_tensor_size": 608256, + "offset": 5415223296, + "file_name": ".cache\\MatMulNBits_2_0_729.const", + "file_size": 608256 + }, + "lm_head.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4257792 + ], + "size_in_bytes": 17031168, + "op_tensor_size": 17031168, + "offset": 5415831552, + "file_name": ".cache\\MatMulNBits_2_0_730.const", + "file_size": 17031168 + }, + "lm_head.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4257792 + ], + "size_in_bytes": 4257792, + "op_tensor_size": 4257792, + "offset": 5432862720, + "file_name": ".cache\\MatMulNBits_2_0_731.const", + "file_size": 4257792 + }, + "past_key_values.0.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 0 + }, + "past_key_values.0.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 4194304 + }, + "present.0.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 0 + }, + "present.0.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 4194304 + }, + "past_key_values.1.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 8388608 + }, + "past_key_values.1.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 12582912 + }, + "present.1.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 8388608 + }, + "present.1.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 12582912 + }, + "past_key_values.2.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 16777216 + }, + "past_key_values.2.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 20971520 + }, + "present.2.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 16777216 + }, + "present.2.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 20971520 + }, + "past_key_values.3.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 25165824 + }, + "past_key_values.3.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 29360128 + }, + "present.3.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 25165824 + }, + "present.3.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 29360128 + }, + "past_key_values.4.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 33554432 + }, + "past_key_values.4.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 37748736 + }, + "present.4.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 33554432 + }, + "present.4.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 37748736 + }, + "past_key_values.5.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 41943040 + }, + "past_key_values.5.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 46137344 + }, + "present.5.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 41943040 + }, + "present.5.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 46137344 + }, + "past_key_values.6.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 50331648 + }, + "past_key_values.6.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 54525952 + }, + "present.6.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 50331648 + }, + "present.6.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 54525952 + }, + "past_key_values.7.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 58720256 + }, + "past_key_values.7.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 62914560 + }, + "present.7.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 58720256 + }, + "present.7.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 62914560 + }, + "past_key_values.8.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 67108864 + }, + "past_key_values.8.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 71303168 + }, + "present.8.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 67108864 + }, + "present.8.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 71303168 + }, + "past_key_values.9.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 75497472 + }, + "past_key_values.9.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 79691776 + }, + "present.9.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 75497472 + }, + "present.9.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 79691776 + }, + "past_key_values.10.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 83886080 + }, + "past_key_values.10.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 88080384 + }, + "present.10.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 83886080 + }, + "present.10.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 88080384 + }, + "past_key_values.11.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 92274688 + }, + "past_key_values.11.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 96468992 + }, + "present.11.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 92274688 + }, + "present.11.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 96468992 + }, + "past_key_values.12.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 100663296 + }, + "past_key_values.12.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 104857600 + }, + "present.12.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 100663296 + }, + "present.12.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 104857600 + }, + "past_key_values.13.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 109051904 + }, + "past_key_values.13.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 113246208 + }, + "present.13.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 109051904 + }, + "present.13.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 113246208 + }, + "past_key_values.14.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 117440512 + }, + "past_key_values.14.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 121634816 + }, + "present.14.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 117440512 + }, + "present.14.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 121634816 + }, + "past_key_values.15.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 125829120 + }, + "past_key_values.15.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 130023424 + }, + "present.15.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 125829120 + }, + "present.15.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 130023424 + }, + "past_key_values.16.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 134217728 + }, + "past_key_values.16.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 138412032 + }, + "present.16.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 134217728 + }, + "present.16.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 138412032 + }, + "past_key_values.17.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 142606336 + }, + "past_key_values.17.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 146800640 + }, + "present.17.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 142606336 + }, + "present.17.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 146800640 + }, + "past_key_values.18.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 150994944 + }, + "past_key_values.18.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 155189248 + }, + "present.18.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 150994944 + }, + "present.18.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 155189248 + }, + "past_key_values.19.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 159383552 + }, + "past_key_values.19.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 163577856 + }, + "present.19.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 159383552 + }, + "present.19.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 163577856 + }, + "past_key_values.20.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 167772160 + }, + "past_key_values.20.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 171966464 + }, + "present.20.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 167772160 + }, + "present.20.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 171966464 + }, + "past_key_values.21.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 176160768 + }, + "past_key_values.21.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 180355072 + }, + "present.21.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 176160768 + }, + "present.21.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 180355072 + }, + "past_key_values.22.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 184549376 + }, + "past_key_values.22.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 188743680 + }, + "present.22.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 184549376 + }, + "present.22.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 188743680 + }, + "past_key_values.23.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 192937984 + }, + "past_key_values.23.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 197132288 + }, + "present.23.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 192937984 + }, + "present.23.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 197132288 + }, + "past_key_values.24.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 201326592 + }, + "past_key_values.24.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 205520896 + }, + "present.24.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 201326592 + }, + "present.24.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 205520896 + }, + "past_key_values.25.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 209715200 + }, + "past_key_values.25.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 213909504 + }, + "present.25.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 209715200 + }, + "present.25.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 213909504 + }, + "past_key_values.26.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 218103808 + }, + "past_key_values.26.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 222298112 + }, + "present.26.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 218103808 + }, + "present.26.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 222298112 + }, + "past_key_values.27.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 226492416 + }, + "past_key_values.27.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 230686720 + }, + "present.27.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 226492416 + }, + "present.27.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 4, + 4096, + 128 + ], + "size_in_bytes": 4194304, + "op_tensor_size": 4194304, + "offset": 230686720 + }, + "sin_cos_cache_token": { + "packed_buffer_label": "ext_buf_1", + "xrt_arg_id": 6, + "dtype": "bfloat16", + "shape": [ + 131072, + 128 + ], + "size_in_bytes": 33554432, + "op_tensor_size": 33554432, + "offset": 0 + } + }, + "aux_info": { + "is_llm": true + } +} \ No newline at end of file