Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes.  
							See raw diff
- config.json +70 -0
- configuration_deepseek.py +210 -0
- model-00001-of-000163.safetensors +3 -0
- model-00002-of-000163.safetensors +3 -0
- model-00003-of-000163.safetensors +3 -0
- model-00004-of-000163.safetensors +3 -0
- model-00005-of-000163.safetensors +3 -0
- model-00006-of-000163.safetensors +3 -0
- model-00007-of-000163.safetensors +3 -0
- model-00008-of-000163.safetensors +3 -0
- model-00009-of-000163.safetensors +3 -0
- model-00010-of-000163.safetensors +3 -0
- model-00011-of-000163.safetensors +3 -0
- model-00012-of-000163.safetensors +3 -0
- model-00013-of-000163.safetensors +3 -0
- model-00014-of-000163.safetensors +3 -0
- model-00015-of-000163.safetensors +3 -0
- model-00016-of-000163.safetensors +3 -0
- model-00017-of-000163.safetensors +3 -0
- model-00018-of-000163.safetensors +3 -0
- model-00019-of-000163.safetensors +3 -0
- model-00020-of-000163.safetensors +3 -0
- model-00021-of-000163.safetensors +3 -0
- model-00022-of-000163.safetensors +3 -0
- model-00023-of-000163.safetensors +3 -0
- model-00024-of-000163.safetensors +3 -0
- model-00025-of-000163.safetensors +3 -0
- model-00026-of-000163.safetensors +3 -0
- model-00027-of-000163.safetensors +3 -0
- model-00028-of-000163.safetensors +3 -0
- model-00029-of-000163.safetensors +3 -0
- model-00030-of-000163.safetensors +3 -0
- model-00031-of-000163.safetensors +3 -0
- model-00032-of-000163.safetensors +3 -0
- model-00033-of-000163.safetensors +3 -0
- model-00034-of-000163.safetensors +3 -0
- model-00035-of-000163.safetensors +3 -0
- model-00036-of-000163.safetensors +3 -0
- model-00037-of-000163.safetensors +3 -0
- model-00038-of-000163.safetensors +3 -0
- model-00039-of-000163.safetensors +3 -0
- model-00040-of-000163.safetensors +3 -0
- model-00041-of-000163.safetensors +3 -0
- model-00042-of-000163.safetensors +3 -0
- model-00043-of-000163.safetensors +3 -0
- model-00044-of-000163.safetensors +3 -0
- model-00045-of-000163.safetensors +3 -0
- model-00046-of-000163.safetensors +3 -0
- model-00047-of-000163.safetensors +3 -0
- model-00048-of-000163.safetensors +3 -0
    	
        config.json
    ADDED
    
    | @@ -0,0 +1,70 @@ | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            {
         | 
| 2 | 
            +
              "architectures": [
         | 
| 3 | 
            +
                "DeepseekV3ForCausalLM"
         | 
| 4 | 
            +
              ],
         | 
| 5 | 
            +
              "attention_bias": false,
         | 
| 6 | 
            +
              "attention_dropout": 0.0,
         | 
| 7 | 
            +
              "auto_map": {
         | 
| 8 | 
            +
                "AutoConfig": "configuration_deepseek.DeepseekV3Config",
         | 
| 9 | 
            +
                "AutoModel": "modeling_deepseek.DeepseekV3Model",
         | 
| 10 | 
            +
                "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"
         | 
| 11 | 
            +
              },
         | 
| 12 | 
            +
              "aux_loss_alpha": 0.001,
         | 
| 13 | 
            +
              "bos_token_id": 0,
         | 
| 14 | 
            +
              "eos_token_id": 1,
         | 
| 15 | 
            +
              "ep_size": 1,
         | 
| 16 | 
            +
              "first_k_dense_replace": 3,
         | 
| 17 | 
            +
              "hidden_act": "silu",
         | 
| 18 | 
            +
              "hidden_size": 7168,
         | 
| 19 | 
            +
              "initializer_range": 0.02,
         | 
| 20 | 
            +
              "intermediate_size": 18432,
         | 
| 21 | 
            +
              "kv_lora_rank": 512,
         | 
| 22 | 
            +
              "max_position_embeddings": 163840,
         | 
| 23 | 
            +
              "model_type": "deepseek_v3",
         | 
| 24 | 
            +
              "moe_intermediate_size": 2048,
         | 
| 25 | 
            +
              "moe_layer_freq": 1,
         | 
| 26 | 
            +
              "n_group": 8,
         | 
| 27 | 
            +
              "n_routed_experts": 256,
         | 
| 28 | 
            +
              "n_shared_experts": 1,
         | 
| 29 | 
            +
              "norm_topk_prob": true,
         | 
| 30 | 
            +
              "num_attention_heads": 128,
         | 
| 31 | 
            +
              "num_experts_per_tok": 8,
         | 
| 32 | 
            +
              "num_hidden_layers": 61,
         | 
| 33 | 
            +
              "num_key_value_heads": 128,
         | 
| 34 | 
            +
              "num_nextn_predict_layers": 1,
         | 
| 35 | 
            +
              "pretraining_tp": 1,
         | 
| 36 | 
            +
              "q_lora_rank": 1536,
         | 
| 37 | 
            +
              "qk_nope_head_dim": 128,
         | 
| 38 | 
            +
              "qk_rope_head_dim": 64,
         | 
| 39 | 
            +
              "quantization_config": {
         | 
| 40 | 
            +
                "activation_scheme": "dynamic",
         | 
| 41 | 
            +
                "fmt": "e4m3",
         | 
| 42 | 
            +
                "quant_method": "fp8",
         | 
| 43 | 
            +
                "weight_block_size": [
         | 
| 44 | 
            +
                  128,
         | 
| 45 | 
            +
                  128
         | 
| 46 | 
            +
                ]
         | 
| 47 | 
            +
              },
         | 
| 48 | 
            +
              "rms_norm_eps": 1e-06,
         | 
| 49 | 
            +
              "rope_scaling": {
         | 
| 50 | 
            +
                "beta_fast": 32,
         | 
| 51 | 
            +
                "beta_slow": 1,
         | 
| 52 | 
            +
                "factor": 40,
         | 
| 53 | 
            +
                "mscale": 1.0,
         | 
| 54 | 
            +
                "mscale_all_dim": 1.0,
         | 
| 55 | 
            +
                "original_max_position_embeddings": 4096,
         | 
| 56 | 
            +
                "type": "yarn"
         | 
| 57 | 
            +
              },
         | 
| 58 | 
            +
              "rope_theta": 10000,
         | 
| 59 | 
            +
              "routed_scaling_factor": 2.5,
         | 
| 60 | 
            +
              "scoring_func": "sigmoid",
         | 
| 61 | 
            +
              "seq_aux": true,
         | 
| 62 | 
            +
              "tie_word_embeddings": false,
         | 
| 63 | 
            +
              "topk_group": 4,
         | 
| 64 | 
            +
              "topk_method": "noaux_tc",
         | 
| 65 | 
            +
              "torch_dtype": "bfloat16",
         | 
| 66 | 
            +
              "transformers_version": "4.46.3",
         | 
| 67 | 
            +
              "use_cache": true,
         | 
| 68 | 
            +
              "v_head_dim": 128,
         | 
| 69 | 
            +
              "vocab_size": 129280
         | 
| 70 | 
            +
            }
         | 
    	
        configuration_deepseek.py
    ADDED
    
    | @@ -0,0 +1,210 @@ | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            from transformers.configuration_utils import PretrainedConfig
         | 
| 2 | 
            +
            from transformers.utils import logging
         | 
| 3 | 
            +
             | 
| 4 | 
            +
            logger = logging.get_logger(__name__)
         | 
| 5 | 
            +
             | 
| 6 | 
            +
            DEEPSEEK_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
         | 
| 7 | 
            +
            class DeepseekV3Config(PretrainedConfig):
         | 
| 8 | 
            +
                r"""
         | 
| 9 | 
            +
                This is the configuration class to store the configuration of a [`DeepseekV3Model`]. It is used to instantiate an DeepSeek
         | 
| 10 | 
            +
                model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
         | 
| 11 | 
            +
                defaults will yield a similar configuration to that of the DeepSeek-V3.
         | 
| 12 | 
            +
             | 
| 13 | 
            +
                Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
         | 
| 14 | 
            +
                documentation from [`PretrainedConfig`] for more information.
         | 
| 15 | 
            +
             | 
| 16 | 
            +
             | 
| 17 | 
            +
                Args:
         | 
| 18 | 
            +
                    vocab_size (`int`, *optional*, defaults to 129280):
         | 
| 19 | 
            +
                        Vocabulary size of the Deep model. Defines the number of different tokens that can be represented by the
         | 
| 20 | 
            +
                        `inputs_ids` passed when calling [`DeepseekV3Model`]
         | 
| 21 | 
            +
                    hidden_size (`int`, *optional*, defaults to 4096):
         | 
| 22 | 
            +
                        Dimension of the hidden representations.
         | 
| 23 | 
            +
                    intermediate_size (`int`, *optional*, defaults to 11008):
         | 
| 24 | 
            +
                        Dimension of the MLP representations.
         | 
| 25 | 
            +
                    moe_intermediate_size (`int`, *optional*, defaults to 1407):
         | 
| 26 | 
            +
                        Dimension of the MoE representations.
         | 
| 27 | 
            +
                    num_hidden_layers (`int`, *optional*, defaults to 32):
         | 
| 28 | 
            +
                        Number of hidden layers in the Transformer decoder.
         | 
| 29 | 
            +
                    num_nextn_predict_layers (`int`, *optional*, defaults to 1):
         | 
| 30 | 
            +
                        Number of nextn predict layers in the DeepSeekV3 Model.
         | 
| 31 | 
            +
                    num_attention_heads (`int`, *optional*, defaults to 32):
         | 
| 32 | 
            +
                        Number of attention heads for each attention layer in the Transformer decoder.
         | 
| 33 | 
            +
                    n_shared_experts (`int`, *optional*, defaults to None):
         | 
| 34 | 
            +
                        Number of shared experts, None means dense model.
         | 
| 35 | 
            +
                    n_routed_experts (`int`, *optional*, defaults to None):
         | 
| 36 | 
            +
                        Number of routed experts, None means dense model.
         | 
| 37 | 
            +
                    routed_scaling_factor (`float`, *optional*, defaults to 1.0):
         | 
| 38 | 
            +
                        Scaling factor or routed experts.
         | 
| 39 | 
            +
                    topk_method (`str`, *optional*, defaults to `gready`):
         | 
| 40 | 
            +
                        Topk method used in routed gate.
         | 
| 41 | 
            +
                    n_group (`int`, *optional*, defaults to None):
         | 
| 42 | 
            +
                        Number of groups for routed experts.
         | 
| 43 | 
            +
                    topk_group (`int`, *optional*, defaults to None):
         | 
| 44 | 
            +
                        Number of selected groups for each token(for each token, ensuring the selected experts is only within `topk_group` groups).
         | 
| 45 | 
            +
                    num_experts_per_tok (`int`, *optional*, defaults to None):
         | 
| 46 | 
            +
                        Number of selected experts, None means dense model.
         | 
| 47 | 
            +
                    moe_layer_freq (`int`, *optional*, defaults to 1):
         | 
| 48 | 
            +
                        The frequency of the MoE layer: one expert layer for every `moe_layer_freq - 1` dense layers.
         | 
| 49 | 
            +
                    first_k_dense_replace (`int`, *optional*, defaults to 0):
         | 
| 50 | 
            +
                        Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
         | 
| 51 | 
            +
                                                                        \--k dense layers--/
         | 
| 52 | 
            +
                    norm_topk_prob (`bool`, *optional*, defaults to False):
         | 
| 53 | 
            +
                        Whether to normalize the weights of the routed experts.
         | 
| 54 | 
            +
                    scoring_func (`str`, *optional*, defaults to 'softmax'):
         | 
| 55 | 
            +
                        Method of computing expert weights.
         | 
| 56 | 
            +
                    aux_loss_alpha (`float`, *optional*, defaults to 0.001):
         | 
| 57 | 
            +
                        Auxiliary loss weight coefficient.
         | 
| 58 | 
            +
                    seq_aux = (`bool`, *optional*, defaults to True):
         | 
| 59 | 
            +
                        Whether to compute the auxiliary loss for each individual sample.
         | 
| 60 | 
            +
                    num_key_value_heads (`int`, *optional*):
         | 
| 61 | 
            +
                        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
         | 
| 62 | 
            +
                        `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
         | 
| 63 | 
            +
                        `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
         | 
| 64 | 
            +
                        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
         | 
| 65 | 
            +
                        by meanpooling all the original heads within that group. For more details checkout [this
         | 
| 66 | 
            +
                        paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
         | 
| 67 | 
            +
                        `num_attention_heads`.
         | 
| 68 | 
            +
                    hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
         | 
| 69 | 
            +
                        The non-linear activation function (function or string) in the decoder.
         | 
| 70 | 
            +
                    max_position_embeddings (`int`, *optional*, defaults to 2048):
         | 
| 71 | 
            +
                        The maximum sequence length that this model might ever be used with.
         | 
| 72 | 
            +
                    initializer_range (`float`, *optional*, defaults to 0.02):
         | 
| 73 | 
            +
                        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
         | 
| 74 | 
            +
                    rms_norm_eps (`float`, *optional*, defaults to 1e-06):
         | 
| 75 | 
            +
                        The epsilon used by the rms normalization layers.
         | 
| 76 | 
            +
                    use_cache (`bool`, *optional*, defaults to `True`):
         | 
| 77 | 
            +
                        Whether or not the model should return the last key/values attentions (not used by all models). Only
         | 
| 78 | 
            +
                        relevant if `config.is_decoder=True`.
         | 
| 79 | 
            +
                    pad_token_id (`int`, *optional*):
         | 
| 80 | 
            +
                        Padding token id.
         | 
| 81 | 
            +
                    bos_token_id (`int`, *optional*, defaults to 1):
         | 
| 82 | 
            +
                        Beginning of stream token id.
         | 
| 83 | 
            +
                    eos_token_id (`int`, *optional*, defaults to 2):
         | 
| 84 | 
            +
                        End of stream token id.
         | 
| 85 | 
            +
                    pretraining_tp (`int`, *optional*, defaults to 1):
         | 
| 86 | 
            +
                        Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
         | 
| 87 | 
            +
                        document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
         | 
| 88 | 
            +
                        necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
         | 
| 89 | 
            +
                        issue](https://github.com/pytorch/pytorch/issues/76232).
         | 
| 90 | 
            +
                    tie_word_embeddings (`bool`, *optional*, defaults to `False`):
         | 
| 91 | 
            +
                        Whether to tie weight embeddings
         | 
| 92 | 
            +
                    rope_theta (`float`, *optional*, defaults to 10000.0):
         | 
| 93 | 
            +
                        The base period of the RoPE embeddings.
         | 
| 94 | 
            +
                    rope_scaling (`Dict`, *optional*):
         | 
| 95 | 
            +
                        Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
         | 
| 96 | 
            +
                        strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
         | 
| 97 | 
            +
                        `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
         | 
| 98 | 
            +
                        `max_position_embeddings` to the expected new maximum.
         | 
| 99 | 
            +
                    attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
         | 
| 100 | 
            +
                        Whether to use a bias in the query, key, value and output projection layers during self-attention.
         | 
| 101 | 
            +
                    attention_dropout (`float`, *optional*, defaults to 0.0):
         | 
| 102 | 
            +
                        The dropout ratio for the attention probabilities.
         | 
| 103 | 
            +
             | 
| 104 | 
            +
                ```python
         | 
| 105 | 
            +
                >>> from transformers import DeepseekV3Model, DeepseekV3Config
         | 
| 106 | 
            +
             | 
| 107 | 
            +
                >>> # Initializing a Deepseek-V3 style configuration
         | 
| 108 | 
            +
                >>> configuration = DeepseekV3Config()
         | 
| 109 | 
            +
             | 
| 110 | 
            +
                >>> # Accessing the model configuration
         | 
| 111 | 
            +
                >>> configuration = model.config
         | 
| 112 | 
            +
                ```"""
         | 
| 113 | 
            +
             | 
| 114 | 
            +
                model_type = "deepseek_v3"
         | 
| 115 | 
            +
                keys_to_ignore_at_inference = ["past_key_values"]
         | 
| 116 | 
            +
             | 
| 117 | 
            +
                def __init__(
         | 
| 118 | 
            +
                    self,
         | 
| 119 | 
            +
                    vocab_size=129280,
         | 
| 120 | 
            +
                    hidden_size=7168,
         | 
| 121 | 
            +
                    intermediate_size=18432,
         | 
| 122 | 
            +
                    moe_intermediate_size = 2048,
         | 
| 123 | 
            +
                    num_hidden_layers=61,
         | 
| 124 | 
            +
                    num_nextn_predict_layers=1,
         | 
| 125 | 
            +
                    num_attention_heads=128,
         | 
| 126 | 
            +
                    num_key_value_heads=128,
         | 
| 127 | 
            +
                    n_shared_experts = 1,
         | 
| 128 | 
            +
                    n_routed_experts = 256,
         | 
| 129 | 
            +
                    ep_size = 1,
         | 
| 130 | 
            +
                    routed_scaling_factor = 2.5,
         | 
| 131 | 
            +
                    kv_lora_rank = 512,
         | 
| 132 | 
            +
                    q_lora_rank = 1536,
         | 
| 133 | 
            +
                    qk_rope_head_dim = 64,
         | 
| 134 | 
            +
                    v_head_dim = 128,
         | 
| 135 | 
            +
                    qk_nope_head_dim = 128,
         | 
| 136 | 
            +
                    topk_method = 'noaux_tc',
         | 
| 137 | 
            +
                    n_group = 8,
         | 
| 138 | 
            +
                    topk_group = 4,
         | 
| 139 | 
            +
                    num_experts_per_tok = 8,
         | 
| 140 | 
            +
                    moe_layer_freq = 1,
         | 
| 141 | 
            +
                    first_k_dense_replace = 3,
         | 
| 142 | 
            +
                    norm_topk_prob = True,
         | 
| 143 | 
            +
                    scoring_func = 'sigmoid',
         | 
| 144 | 
            +
                    aux_loss_alpha = 0.001,
         | 
| 145 | 
            +
                    seq_aux = True,
         | 
| 146 | 
            +
                    hidden_act="silu",
         | 
| 147 | 
            +
                    max_position_embeddings=4096,
         | 
| 148 | 
            +
                    initializer_range=0.02,
         | 
| 149 | 
            +
                    rms_norm_eps=1e-6,
         | 
| 150 | 
            +
                    use_cache=True,
         | 
| 151 | 
            +
                    pad_token_id=None,
         | 
| 152 | 
            +
                    bos_token_id=0,
         | 
| 153 | 
            +
                    eos_token_id=1,
         | 
| 154 | 
            +
                    pretraining_tp=1,
         | 
| 155 | 
            +
                    tie_word_embeddings=False,
         | 
| 156 | 
            +
                    rope_theta=10000.0,
         | 
| 157 | 
            +
                    rope_scaling=None,
         | 
| 158 | 
            +
                    attention_bias=False,
         | 
| 159 | 
            +
                    attention_dropout=0.0,
         | 
| 160 | 
            +
                    **kwargs,
         | 
| 161 | 
            +
                ):
         | 
| 162 | 
            +
                    self.vocab_size = vocab_size
         | 
| 163 | 
            +
                    self.max_position_embeddings = max_position_embeddings
         | 
| 164 | 
            +
                    self.hidden_size = hidden_size
         | 
| 165 | 
            +
                    self.intermediate_size = intermediate_size
         | 
| 166 | 
            +
                    self.moe_intermediate_size = moe_intermediate_size
         | 
| 167 | 
            +
                    self.num_hidden_layers = num_hidden_layers
         | 
| 168 | 
            +
                    self.num_nextn_predict_layers = num_nextn_predict_layers
         | 
| 169 | 
            +
                    self.num_attention_heads = num_attention_heads
         | 
| 170 | 
            +
                    self.n_shared_experts = n_shared_experts
         | 
| 171 | 
            +
                    self.n_routed_experts = n_routed_experts
         | 
| 172 | 
            +
                    self.ep_size = ep_size
         | 
| 173 | 
            +
                    self.routed_scaling_factor = routed_scaling_factor
         | 
| 174 | 
            +
                    self.kv_lora_rank = kv_lora_rank
         | 
| 175 | 
            +
                    self.q_lora_rank = q_lora_rank
         | 
| 176 | 
            +
                    self.qk_rope_head_dim = qk_rope_head_dim
         | 
| 177 | 
            +
                    self.v_head_dim = v_head_dim
         | 
| 178 | 
            +
                    self.qk_nope_head_dim = qk_nope_head_dim
         | 
| 179 | 
            +
                    self.topk_method = topk_method
         | 
| 180 | 
            +
                    self.n_group = n_group
         | 
| 181 | 
            +
                    self.topk_group = topk_group
         | 
| 182 | 
            +
                    self.num_experts_per_tok = num_experts_per_tok
         | 
| 183 | 
            +
                    self.moe_layer_freq = moe_layer_freq
         | 
| 184 | 
            +
                    self.first_k_dense_replace = first_k_dense_replace
         | 
| 185 | 
            +
                    self.norm_topk_prob = norm_topk_prob
         | 
| 186 | 
            +
                    self.scoring_func = scoring_func
         | 
| 187 | 
            +
                    self.aux_loss_alpha = aux_loss_alpha
         | 
| 188 | 
            +
                    self.seq_aux = seq_aux
         | 
| 189 | 
            +
                    # for backward compatibility
         | 
| 190 | 
            +
                    if num_key_value_heads is None:
         | 
| 191 | 
            +
                        num_key_value_heads = num_attention_heads
         | 
| 192 | 
            +
             | 
| 193 | 
            +
                    self.num_key_value_heads = num_key_value_heads
         | 
| 194 | 
            +
                    self.hidden_act = hidden_act
         | 
| 195 | 
            +
                    self.initializer_range = initializer_range
         | 
| 196 | 
            +
                    self.rms_norm_eps = rms_norm_eps
         | 
| 197 | 
            +
                    self.pretraining_tp = pretraining_tp
         | 
| 198 | 
            +
                    self.use_cache = use_cache
         | 
| 199 | 
            +
                    self.rope_theta = rope_theta
         | 
| 200 | 
            +
                    self.rope_scaling = rope_scaling
         | 
| 201 | 
            +
                    self.attention_bias = attention_bias
         | 
| 202 | 
            +
                    self.attention_dropout = attention_dropout
         | 
| 203 | 
            +
             | 
| 204 | 
            +
                    super().__init__(
         | 
| 205 | 
            +
                        pad_token_id=pad_token_id,
         | 
| 206 | 
            +
                        bos_token_id=bos_token_id,
         | 
| 207 | 
            +
                        eos_token_id=eos_token_id,
         | 
| 208 | 
            +
                        tie_word_embeddings=tie_word_embeddings,
         | 
| 209 | 
            +
                        **kwargs,
         | 
| 210 | 
            +
                    )
         | 
    	
        model-00001-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:134f51f4642dde3f3d63aacc8e4605e6355d27aceb4ee1a1f914e30464623f10
         | 
| 3 | 
            +
            size 5234139343
         | 
    	
        model-00002-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:400b76ca459923cc0c710ca51e05c8aa6167f595af208c15107800394a66a504
         | 
| 3 | 
            +
            size 4302383966
         | 
    	
        model-00003-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:1e0192d491a2cad015aed550f8c072c69dbea143e9f55074b4f5b513c0633026
         | 
| 3 | 
            +
            size 4302384375
         | 
    	
        model-00004-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:6f8ca069011e31d606dee00db53b3dc5160834327f17102353f28cad3b387587
         | 
| 3 | 
            +
            size 4302349996
         | 
    	
        model-00005-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:1973c53f492006ff0eb70d677ebfa42bc40d2cafc1e15e64eb600765bfe7b7c1
         | 
| 3 | 
            +
            size 4302384154
         | 
    	
        model-00006-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:2250eb7be1d22707f2effe99562de640b55980c2913e0640fa24d571cdfed642
         | 
| 3 | 
            +
            size 4372073602
         | 
    	
        model-00007-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:8632d0847cd3082d9b66edefcc72d80c717b3fe4535b3d68b808da3c8806bceb
         | 
| 3 | 
            +
            size 4306080097
         | 
    	
        model-00008-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:85daa7fd14a621a084a16c2669a8b95bad9173353fa9ec2f0fd6be6726866a32
         | 
| 3 | 
            +
            size 4302384356
         | 
    	
        model-00009-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:b55f6542adae951b242e8e8444567fca59f6c2c380322a51abd49341871863be
         | 
| 3 | 
            +
            size 4302350190
         | 
    	
        model-00010-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:0317ba8c85441b4f3d0134d2d43a1535e073250ba223283ceffc46bdef5dd77b
         | 
| 3 | 
            +
            size 4302383960
         | 
    	
        model-00011-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:c5291900ff83cdf2786565e133d6b792313526df8e5cc17c120e0386426e66d7
         | 
| 3 | 
            +
            size 4302384375
         | 
    	
        model-00012-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:1596e384e28712a7da32481b414ad2f9d5205b7d914066aaa77861f52218e3e3
         | 
| 3 | 
            +
            size 1321583941
         | 
    	
        model-00013-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:0b5b8ac70defdc5393f02525d88ba2eb145f51ce9b2a11637de11b522d7c0da1
         | 
| 3 | 
            +
            size 4302317244
         | 
    	
        model-00014-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:df727377a86d6bc8b3ae4ac3f4fb0937c5476fb8b19edd28c0924d016e7199f4
         | 
| 3 | 
            +
            size 4302384328
         | 
    	
        model-00015-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:ae96924ba8d4f9e56221316ac185b1b5655ad08fd6abb425288d70f8e867d3a2
         | 
| 3 | 
            +
            size 4302350218
         | 
    	
        model-00016-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:506313cbaf4ed810cf5d104229fdbc4e19cde2b12cea9e55e9550131b171409e
         | 
| 3 | 
            +
            size 4302383932
         | 
    	
        model-00017-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:d396bde693edd1001be8440865e2c61b121d0cad562596e19ca01bf29dabcca4
         | 
| 3 | 
            +
            size 4302384377
         | 
    	
        model-00018-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:4ebc50b2e0e220583a2597770d94f1a6c3e87be325470d0f4a15984bab424354
         | 
| 3 | 
            +
            size 4302350026
         | 
    	
        model-00019-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:0a9d59229d198ee0b4966834d17d61811e8afe998e7634c87e643f51bc866d75
         | 
| 3 | 
            +
            size 4302384124
         | 
    	
        model-00020-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:ac27c6cbc27131a06e0562c14292ff26d17c62b156a068e4a41c9f56fead89f7
         | 
| 3 | 
            +
            size 4302384377
         | 
    	
        model-00021-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:8afd4121eeb804b11838f51f4b49598d0e66bafe40d70a2def88698901a595d6
         | 
| 3 | 
            +
            size 4302350413
         | 
    	
        model-00022-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:d6e5d4ea01bc15edb85b06f7d0ea9a41acf2efe18dd7c6bf1d85ddf2c4074fc0
         | 
| 3 | 
            +
            size 4302384900
         | 
    	
        model-00023-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:9406624825565d25194fdf9462b397b5acba0bcab3bf1bdf9601952280945269
         | 
| 3 | 
            +
            size 4302350808
         | 
    	
        model-00024-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:488969a9e8a378b243594f86144827dfb4c8549b3c6d719e21572a2306e742b4
         | 
| 3 | 
            +
            size 4302384504
         | 
    	
        model-00025-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:450cc3f29910ee8d38ac5a5ec39901f83f63920bbc4d15175e5f5e58119dff29
         | 
| 3 | 
            +
            size 4302384961
         | 
    	
        model-00026-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:bdfe115c31c700af118ca1f2f3cfb121c1d60e40404a688d63b686e8b54eacb7
         | 
| 3 | 
            +
            size 4302350620
         | 
    	
        model-00027-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:13e981a9bdc7527d1d496ebe47456c7edd81d843c8125597d5ee640f18d65413
         | 
| 3 | 
            +
            size 4302384692
         | 
    	
        model-00028-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:d767507070d19fa2736474f701a58317a9584e32df6d9bd010611f939c86049d
         | 
| 3 | 
            +
            size 4302384963
         | 
    	
        model-00029-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:11b1ef947996f5426c462156be33547dbf1b08d8a12eca063d27889553e77470
         | 
| 3 | 
            +
            size 4302350448
         | 
    	
        model-00030-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:6ff7740ee8707ad1e058634afd4d8c525cc2dbd13d81be8bd3ebbe2d999b9461
         | 
| 3 | 
            +
            size 4302384884
         | 
    	
        model-00031-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:07afa8b68a432adc61b74f5649265bb656efae1ffcd6da6f23549f6b53a2093a
         | 
| 3 | 
            +
            size 4302350824
         | 
    	
        model-00032-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:361f93b0bcbf04a2d5c99f2265d74c16d2ac6606fea1322633be0d4c578f93ac
         | 
| 3 | 
            +
            size 4302384488
         | 
    	
        model-00033-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:d5cd3a3bd61d43a159b186554e0d652d90ef80355a4d64cdfab2f09c59586ea1
         | 
| 3 | 
            +
            size 4302384963
         | 
    	
        model-00034-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:df4c5f0efee23e96b0420e25a63e266acffba55a72c4dbb23efcaecaee279461
         | 
| 3 | 
            +
            size 1747417474
         | 
    	
        model-00035-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:59907496ec35f620146f7e2a771acbae6315b73beb7c9b4484791a5769e92720
         | 
| 3 | 
            +
            size 4302317817
         | 
    	
        model-00036-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:816a8da951743290c6cc77ee584023fd9063447256abef3937e9d73aed442e40
         | 
| 3 | 
            +
            size 4302384914
         | 
    	
        model-00037-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:03da5c0e525f134c3fb902409a9e737608c38a30710ed489a8e6627b511a25e1
         | 
| 3 | 
            +
            size 4302350794
         | 
    	
        model-00038-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:c0e6e082c73bdba9c830cfe08ec9f6424cd83296cae517dfab2338270a848334
         | 
| 3 | 
            +
            size 4302384518
         | 
    	
        model-00039-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:a8dba03ca01e579f57e83906aed8aa9f4cd8205f6d32b9888dee455a64896b34
         | 
| 3 | 
            +
            size 4302384963
         | 
    	
        model-00040-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:2ebb24adff0b0955375de7ffa071a27381df5dfcef8273b634de47f43199b0e1
         | 
| 3 | 
            +
            size 4302350602
         | 
    	
        model-00041-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:5a50db6c7cb71fe98b0509453599af97096831686894e862dd0e780f10386882
         | 
| 3 | 
            +
            size 4302384710
         | 
    	
        model-00042-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:9666e834327b2fdc9cb3be9954eda58754c7415dd63ccce449ec1e5e492a5924
         | 
| 3 | 
            +
            size 4302384963
         | 
    	
        model-00043-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:dda5691da3edae3b462c90722d82a345dbb5713ea7d0739c865b09aea593f601
         | 
| 3 | 
            +
            size 4302350432
         | 
    	
        model-00044-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:743f7e5979b3bd6512f3eccedb3665ae2ba0911b427041d7400f898c031b13cb
         | 
| 3 | 
            +
            size 4302384900
         | 
    	
        model-00045-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:929fa255d58727dae32f88ff5323cb45a9a898e864926341cecac859570114b8
         | 
| 3 | 
            +
            size 4302350808
         | 
    	
        model-00046-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:3ae202c6641bad95cc6b5c9e75b4410470ed6ca63941572342318045972062fa
         | 
| 3 | 
            +
            size 4302384504
         | 
    	
        model-00047-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:e3c9d4d06ab071d1052790d8a401c56eef5571cb2b690778dab2e66e81242812
         | 
| 3 | 
            +
            size 4302384961
         | 
    	
        model-00048-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:b5dbbd1df4a075642f11f88308c5648e90bafbaf74bece18ae5b77f17f36d905
         | 
| 3 | 
            +
            size 4302350620
         | 
