Upload 8-bit GPTQ quantized Orpheus model

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,56 @@
+---
+license: apache-2.0
+base_model: svara-tts-v1-gptqmodel
+tags:
+- quantized
+- gptq
+- text-to-speech
+- tts
+- orpheus
+- 8bit
+library_name: transformers
+pipeline_tag: text-generation
+---
+
+# Orpheus 3B 8-bit GPTQ
+
+This is an 8-bit GPTQ quantized version of [baseten/orpheus-3b-0.1-ft](https://huggingface.co/baseten/orpheus-3b-0.1-ft).
+
+## Model Details
+
+- **Base Model**: svara-tts-v1-gptqmodel
+- **Quantization**: 8-bit GPTQ
+- **Group Size**: 128
+- **Calibration Dataset**: canopylabs/zac-sample-dataset (TTS-specific)
+- **Library**: auto-gptq
+
+## Usage
+
+```python
+from auto_gptq import AutoGPTQForCausalLM
+from transformers import AutoTokenizer
+
+# Load the quantized model
+model = AutoGPTQForCausalLM.from_quantized(
+    "Hariprasath28/svara-tts-v1-gptq",
+    device="cuda:0",  # or "cpu"
+    use_triton=False,
+    trust_remote_code=True
+)
+
+tokenizer = AutoTokenizer.from_pretrained("Hariprasath28/svara-tts-v1-gptq", trust_remote_code=True)
+
+# Generate TTS tokens
+text = "tara: Hello, this is a test of the quantized Orpheus model."
+inputs = tokenizer(text, return_tensors="pt").to("cuda:0")
+
+with torch.no_grad():
+    outputs = model.generate(
+        **inputs,
+        max_new_tokens=100,
+        temperature=0.7,
+        do_sample=True
+    )
+
+generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
+print(generated)

chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,61 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128009,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": 128263,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "bits": 4,
+    "checkpoint_format": "gptq",
+    "desc_act": false,
+    "group_size": 128,
+    "lm_head": false,
+    "meta": {
+      "act_group_aware": false,
+      "damp_auto_increment": 0.01,
+      "damp_percent": 0.01,
+      "mse": 0.0,
+      "quantizer": [
+        "gptqmodel:4.2.5"
+      ],
+      "static_groups": false,
+      "true_sequential": true,
+      "uri": "https://github.com/modelcloud/gptqmodel",
+      "v2": false,
+      "v2_alpha": 0.25
+    },
+    "pack_dtype": "int32",
+    "quant_method": "gptq",
+    "sym": true
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.1",
+  "unsloth_version": "2025.10.4",
+  "use_cache": true,
+  "vocab_size": 156940
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": 128009,
+  "pad_token_id": 128263,
+  "transformers_version": "4.57.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0772c922098a8fdb8941899724fb3c8e96238884eee66c666fbba1443365f05
+size 2432001824

quant_log.csv

@@ -0,0 +1,197 @@
+layer,module,loss,samples,damp,time
+0,self_attn.k_proj,0.0007165673,0.01000,1.079
+0,self_attn.v_proj,0.0000596307,0.01000,1.078
+0,self_attn.q_proj,0.0013268112,0.01000,1.075
+0,self_attn.o_proj,0.0000097104,0.01000,1.126
+0,mlp.gate_proj,0.0014320708,0.01000,1.095
+0,mlp.up_proj,0.0013744390,0.01000,1.092
+0,mlp.down_proj,0.0032274312,0.01000,3.047
+1,self_attn.k_proj,0.0036077098,0.01000,1.082
+1,self_attn.v_proj,0.0008333813,0.01000,1.075
+1,self_attn.q_proj,0.0073467204,0.01000,1.086
+1,self_attn.o_proj,0.0000822506,0.01000,1.119
+1,mlp.gate_proj,0.0033585912,0.01000,1.097
+1,mlp.up_proj,0.0031722694,0.01000,1.080
+1,mlp.down_proj,3.9980311538,0.01000,3.069
+2,self_attn.k_proj,0.0043991325,0.01000,1.084
+2,self_attn.v_proj,0.0012270709,0.01000,1.087
+2,self_attn.q_proj,0.0091787712,0.01000,1.140
+2,self_attn.o_proj,0.0001391864,0.01000,1.093
+2,mlp.gate_proj,0.0041719070,0.01000,1.108
+2,mlp.up_proj,0.0037999180,0.01000,1.104
+2,mlp.down_proj,0.0002487536,0.01000,3.029
+3,self_attn.k_proj,0.0020995598,0.01000,1.099
+3,self_attn.v_proj,0.0007532771,0.01000,1.132
+3,self_attn.q_proj,0.0045231522,0.01000,1.081
+3,self_attn.o_proj,0.0001802197,0.01000,1.108
+3,mlp.gate_proj,0.0055153234,0.01000,1.107
+3,mlp.up_proj,0.0048551054,0.01000,1.117
+3,mlp.down_proj,0.0003586192,0.01000,3.002
+4,self_attn.k_proj,0.0022233009,0.01000,1.147
+4,self_attn.v_proj,0.0008471522,0.01000,1.105
+4,self_attn.q_proj,0.0048226416,0.01000,1.096
+4,self_attn.o_proj,0.0002814944,0.01000,1.096
+4,mlp.gate_proj,0.0058206076,0.01000,1.112
+4,mlp.up_proj,0.0048891712,0.01000,1.157
+4,mlp.down_proj,0.0004396601,0.01000,3.006
+5,self_attn.k_proj,0.0035945541,0.01000,1.136
+5,self_attn.v_proj,0.0010127633,0.01000,1.087
+5,self_attn.q_proj,0.0069930129,0.01000,1.107
+5,self_attn.o_proj,0.0002054535,0.01000,1.112
+5,mlp.gate_proj,0.0072308896,0.01000,1.146
+5,mlp.up_proj,0.0062122815,0.01000,1.105
+5,mlp.down_proj,0.0004650744,0.01000,2.999
+6,self_attn.k_proj,0.0027636019,0.01000,1.087
+6,self_attn.v_proj,0.0008667380,0.01000,1.093
+6,self_attn.q_proj,0.0054596979,0.01000,1.100
+6,self_attn.o_proj,0.0004086048,0.01000,1.147
+6,mlp.gate_proj,0.0080465792,0.01000,1.119
+6,mlp.up_proj,0.0069164634,0.01000,1.120
+6,mlp.down_proj,0.0003812077,0.01000,3.064
+7,self_attn.k_proj,0.0027822383,0.01000,1.095
+7,self_attn.v_proj,0.0008612974,0.01000,1.094
+7,self_attn.q_proj,0.0053715444,0.01000,1.136
+7,self_attn.o_proj,0.0002576188,0.01000,1.110
+7,mlp.gate_proj,0.0085338670,0.01000,1.108
+7,mlp.up_proj,0.0077248947,0.01000,1.114
+7,mlp.down_proj,0.0005805181,0.01000,3.090
+8,self_attn.k_proj,0.0029828666,0.01000,1.105
+8,self_attn.v_proj,0.0009772369,0.01000,1.090
+8,self_attn.q_proj,0.0059334094,0.01000,1.137
+8,self_attn.o_proj,0.0003355184,0.01000,1.095
+8,mlp.gate_proj,0.0087882949,0.01000,1.107
+8,mlp.up_proj,0.0079226494,0.01000,1.120
+8,mlp.down_proj,0.0006587114,0.01000,3.024
+9,self_attn.k_proj,0.0024128832,0.01000,1.096
+9,self_attn.v_proj,0.0009061856,0.01000,1.119
+9,self_attn.q_proj,0.0055490041,0.01000,1.082
+9,self_attn.o_proj,0.0003017767,0.01000,1.097
+9,mlp.gate_proj,0.0082731644,0.01000,1.104
+9,mlp.up_proj,0.0075000601,0.01000,1.125
+9,mlp.down_proj,0.0007202323,0.01000,3.002
+10,self_attn.k_proj,0.0027228307,0.01000,1.134
+10,self_attn.v_proj,0.0009034530,0.01000,1.091
+10,self_attn.q_proj,0.0053220509,0.01000,1.113
+10,self_attn.o_proj,0.0001987794,0.01000,1.097
+10,mlp.gate_proj,0.0086941755,0.01000,1.109
+10,mlp.up_proj,0.0082439293,0.01000,1.150
+10,mlp.down_proj,0.0019673965,0.01000,2.997
+11,self_attn.k_proj,0.0023969931,0.01000,1.086
+11,self_attn.v_proj,0.0010096786,0.01000,1.078
+11,self_attn.q_proj,0.0049915458,0.01000,1.085
+11,self_attn.o_proj,0.0005560061,0.01000,1.099
+11,mlp.gate_proj,0.0107198213,0.01000,1.133
+11,mlp.up_proj,0.0102804375,0.01000,1.102
+11,mlp.down_proj,0.0011271423,0.01000,3.014
+12,self_attn.k_proj,0.0029296636,0.01000,1.076
+12,self_attn.v_proj,0.0012349977,0.01000,1.095
+12,self_attn.q_proj,0.0063043941,0.01000,1.087
+12,self_attn.o_proj,0.0007200323,0.01000,1.151
+12,mlp.gate_proj,0.0116617102,0.01000,1.107
+12,mlp.up_proj,0.0111292980,0.01000,1.111
+12,mlp.down_proj,0.0014334196,0.01000,3.021
+13,self_attn.k_proj,0.0035808150,0.01000,1.085
+13,self_attn.v_proj,0.0015270074,0.01000,1.086
+13,self_attn.q_proj,0.0074143148,0.01000,1.129
+13,self_attn.o_proj,0.0009024844,0.01000,1.096
+13,mlp.gate_proj,0.0145458823,0.01000,1.102
+13,mlp.up_proj,0.0133621042,0.01000,1.111
+13,mlp.down_proj,0.0023988670,0.01000,3.040
+14,self_attn.k_proj,0.0031979346,0.01000,1.101
+14,self_attn.v_proj,0.0019720971,0.01000,1.089
+14,self_attn.q_proj,0.0079760299,0.01000,1.134
+14,self_attn.o_proj,0.0015582386,0.01000,1.103
+14,mlp.gate_proj,0.0180456819,0.01000,1.109
+14,mlp.up_proj,0.0168532025,0.01000,1.135
+14,mlp.down_proj,0.0041922475,0.01000,3.065
+15,self_attn.k_proj,0.0036405652,0.01000,1.107
+15,self_attn.v_proj,0.0018848111,0.01000,1.147
+15,self_attn.q_proj,0.0086390972,0.01000,1.087
+15,self_attn.o_proj,0.0015424666,0.01000,1.088
+15,mlp.gate_proj,0.0214356906,0.01000,1.101
+15,mlp.up_proj,0.0195761467,0.01000,1.138
+15,mlp.down_proj,0.0055152711,0.01000,3.005
+16,self_attn.k_proj,0.0045111026,0.01000,1.142
+16,self_attn.v_proj,0.0023171834,0.01000,1.084
+16,self_attn.q_proj,0.0101644966,0.01000,1.099
+16,self_attn.o_proj,0.0011662681,0.01000,1.091
+16,mlp.gate_proj,0.0213560588,0.01000,1.135
+16,mlp.up_proj,0.0195173964,0.01000,1.118
+16,mlp.down_proj,0.0054034785,0.01000,3.009
+17,self_attn.k_proj,0.0044137718,0.01000,1.101
+17,self_attn.v_proj,0.0023232712,0.01000,1.096
+17,self_attn.q_proj,0.0101262539,0.01000,1.098
+17,self_attn.o_proj,0.0008717865,0.01000,1.113
+17,mlp.gate_proj,0.0211382581,0.01000,1.144
+17,mlp.up_proj,0.0194139481,0.01000,1.109
+17,mlp.down_proj,0.0061121312,0.01000,3.061
+18,self_attn.k_proj,0.0048886443,0.01000,1.094
+18,self_attn.v_proj,0.0027045659,0.01000,1.089
+18,self_attn.q_proj,0.0109312589,0.01000,1.094
+18,self_attn.o_proj,0.0008918150,0.01000,1.112
+18,mlp.gate_proj,0.0234650211,0.01000,1.108
+18,mlp.up_proj,0.0218645099,0.01000,1.110
+18,mlp.down_proj,0.0086717100,0.01000,3.046
+19,self_attn.k_proj,0.0044884081,0.01000,1.100
+19,self_attn.v_proj,0.0030410163,0.01000,1.093
+19,self_attn.q_proj,0.0110051451,0.01000,1.139
+19,self_attn.o_proj,0.0020651088,0.01000,1.100
+19,mlp.gate_proj,0.0245785045,0.01000,1.109
+19,mlp.up_proj,0.0234034874,0.01000,1.119
+19,mlp.down_proj,0.0092809318,0.01000,3.034
+20,self_attn.k_proj,0.0050302518,0.01000,1.081
+20,self_attn.v_proj,0.0034087377,0.01000,1.153
+20,self_attn.q_proj,0.0111182124,0.01000,1.085
+20,self_attn.o_proj,0.0028834537,0.01000,1.091
+20,mlp.gate_proj,0.0262545438,0.01000,1.120
+20,mlp.up_proj,0.0257513053,0.01000,1.103
+20,mlp.down_proj,0.0104322885,0.01000,3.031
+21,self_attn.k_proj,0.0051919348,0.01000,1.126
+21,self_attn.v_proj,0.0042079091,0.01000,1.121
+21,self_attn.q_proj,0.0125673522,0.01000,1.102
+21,self_attn.o_proj,0.0020777454,0.01000,1.096
+21,mlp.gate_proj,0.0278684226,0.01000,1.119
+21,mlp.up_proj,0.0270644968,0.01000,1.142
+21,mlp.down_proj,0.0108675939,0.01000,3.004
+22,self_attn.k_proj,0.0049076396,0.01000,1.137
+22,self_attn.v_proj,0.0043899354,0.01000,1.098
+22,self_attn.q_proj,0.0115766209,0.01000,1.093
+22,self_attn.o_proj,0.0016416336,0.01000,1.090
+22,mlp.gate_proj,0.0266865456,0.01000,1.148
+22,mlp.up_proj,0.0263432138,0.01000,1.125
+22,mlp.down_proj,0.0106157281,0.01000,3.008
+23,self_attn.k_proj,0.0050228717,0.01000,1.102
+23,self_attn.v_proj,0.0052046320,0.01000,1.092
+23,self_attn.q_proj,0.0136195882,0.01000,1.095
+23,self_attn.o_proj,0.0022992036,0.01000,1.139
+23,mlp.gate_proj,0.0328759497,0.01000,1.114
+23,mlp.up_proj,0.0312380646,0.01000,1.104
+23,mlp.down_proj,0.0120926987,0.01000,3.006
+24,self_attn.k_proj,0.0053366215,0.01000,1.104
+24,self_attn.v_proj,0.0058682263,0.01000,1.092
+24,self_attn.q_proj,0.0130573710,0.01000,1.089
+24,self_attn.o_proj,0.0041216169,0.01000,1.144
+24,mlp.gate_proj,0.0386738813,0.01000,1.134
+24,mlp.up_proj,0.0368005651,0.01000,1.131
+24,mlp.down_proj,0.0130093956,0.01000,3.077
+25,self_attn.k_proj,0.0049963995,0.01000,1.103
+25,self_attn.v_proj,0.0055508415,0.01000,1.104
+25,self_attn.q_proj,0.0166119861,0.01000,1.177
+25,self_attn.o_proj,0.0073872493,0.01000,1.132
+25,mlp.gate_proj,0.0591742414,0.01000,1.121
+25,mlp.up_proj,0.0547565439,0.01000,1.119
+25,mlp.down_proj,0.0167801850,0.01000,3.042
+26,self_attn.k_proj,0.0047088243,0.01000,1.104
+26,self_attn.v_proj,0.0059495622,0.01000,1.148
+26,self_attn.q_proj,0.0157712409,0.01000,1.094
+26,self_attn.o_proj,0.0089129083,0.01000,1.101
+26,mlp.gate_proj,0.0934302301,0.01000,1.115
+26,mlp.up_proj,0.0831171238,0.01000,1.161
+26,mlp.down_proj,0.0334263968,0.01000,3.024
+27,self_attn.k_proj,0.0035992840,0.01000,1.138
+27,self_attn.v_proj,0.0043383457,0.01000,1.100
+27,self_attn.q_proj,0.0133404605,0.01000,1.104
+27,self_attn.o_proj,0.0147618290,0.01000,1.102
+27,mlp.gate_proj,0.1286905029,0.01000,1.138
+27,mlp.up_proj,0.1280683315,0.01000,1.127
+27,mlp.down_proj,0.7720242125,0.01000,3.055

quantize_config.json

@@ -0,0 +1,24 @@
+{
+  "bits": 4,
+  "group_size": 128,
+  "desc_act": false,
+  "sym": true,
+  "lm_head": false,
+  "quant_method": "gptq",
+  "checkpoint_format": "gptq",
+  "pack_dtype": "int32",
+  "meta": {
+    "quantizer": [
+      "gptqmodel:4.2.5"
+    ],
+    "uri": "https://github.com/modelcloud/gptqmodel",
+    "damp_percent": 0.01,
+    "damp_auto_increment": 0.01,
+    "static_groups": false,
+    "true_sequential": true,
+    "mse": 0.0,
+    "v2": false,
+    "v2_alpha": 0.25,
+    "act_group_aware": false
+  }
+}

special_tokens_map.json

@@ -0,0 +1,26 @@
+{
+  "additional_special_tokens": [
+    "<|audio|>"
+  ],
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<custom_token_7>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:044e2a10201774018db120391980464472baabf223bd353cea49b17da0b66abc
+size 22849546