Update README.md to include GPTQModel usage.
Browse files
README.md
CHANGED
|
@@ -127,17 +127,3 @@ model = GPTQModel.load(model_id)
|
|
| 127 |
result = model.generate("Uncovering deep insights")[0] # tokens
|
| 128 |
print(model.tokenizer.decode(result)) # string output
|
| 129 |
```
|
| 130 |
-
|
| 131 |
-
### Run the model with GPTQModel
|
| 132 |
-
|
| 133 |
-
GPTQModel package: https://github.com/ModelCloud/GPTQModel
|
| 134 |
-
|
| 135 |
-
```
|
| 136 |
-
pip install -v gptqmodel=="1.8.0" --no-build-isolation
|
| 137 |
-
from gptqmodel import GPTQModel
|
| 138 |
-
|
| 139 |
-
model_id = 'iproskurina/bloom-3b-GPTQ-4bit-g128'
|
| 140 |
-
model = GPTQModel.load(model_id)
|
| 141 |
-
result = model.generate("Uncovering deep insights")[0] # tokens
|
| 142 |
-
print(model.tokenizer.decode(result)) # string output
|
| 143 |
-
```
|
|
|
|
| 127 |
result = model.generate("Uncovering deep insights")[0] # tokens
|
| 128 |
print(model.tokenizer.decode(result)) # string output
|
| 129 |
```
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|