tt

app.py

@@ -1,35 +1,45 @@
+import gradio as gr
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
 
-
-MODEL_NAME = "microsoft/Phi-4-mini-instruct"
-
-tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
-model = AutoModelForCausalLM.from_pretrained(MODEL_NAME)
-
-def generate_text(prompt, model, tokenizer, max_length=512, temperature=1, top_k=50, top_p=0.95):
-    inputs = tokenizer.encode(prompt, return_tensors="pt")
-
-    outputs = model.generate(
-        inputs,
-        max_length=max_length,
+# Load model and tokenizer
+model_id = "PowerInfer/SmallThinker-21BA3B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    device_map="cpu",  # Force CPU
+    torch_dtype=torch.float32
+)
+
+# Create generation pipeline
+generator = pipeline(
+    "text-generation",
+    model=model,
+    tokenizer=tokenizer,
+    device=-1  # CPU
+)
+
+# Inference function
+def chat(prompt, max_new_tokens=256, temperature=0.7):
+    output = generator(
+        prompt,
+        max_new_tokens=max_new_tokens,
         temperature=temperature,
-        top_k=top_k,
-        top_p=top_p,
-        do_sample=True
+        do_sample=True,
+        pad_token_id=tokenizer.eos_token_id
     )
-
-
-    generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
-    return generated_text
-
-def main():
-    # Define your prompt
-    prompt = "According to all known laws of aviation, there is no way a bee should be able to fly."
-
-    generated_text = generate_text(prompt, model, tokenizer)
-
-    print(generated_text)
-
-if __name__ == "__main__":
-    main()
+    return output[0]["generated_text"]
+
+# Launch Gradio app
+gr.Interface(
+    fn=chat,
+    inputs=[
+        gr.Textbox(label="Prompt", lines=4, placeholder="Ask anything..."),
+        gr.Slider(32, 512, value=256, step=16, label="Max New Tokens"),
+        gr.Slider(0.1, 1.5, value=0.7, step=0.1, label="Temperature")
+    ],
+    outputs=gr.Textbox(label="Response"),
+    title="💬 SmallThinker-21BA3B-Instruct",
+    description="Run PowerInfer/SmallThinker-21BA3B-Instruct locally on CPU using Hugging Face + Gradio"
+).launch()

requirements.txt

@@ -1,7 +1,7 @@
 gradio>=4.44.0
-transformers>=4.40.0
 torch>=2.1.0
+transformers==4.53.3
 accelerate>=0.27.0
 sentencepiece  # required for some tokenizers
 safetensors    # faster & safer model loading
-hf_xet
+hf_xet