initialize as cpu

app.py

@@ -118,7 +118,7 @@ def load_model():
     ankerbot_model = AutoModelForCausalLM.from_pretrained(
         model_name,
         quantization_config=bnb_config,
-        device_map="cuda:0",
+        device_map="cpu",
         torch_dtype=torch.float16,
         use_cache=True,
         offload_folder="../offload"
@@ -128,10 +128,10 @@ def load_model():
         torch_dtype=torch.float16,
         truncation=True,
         padding=True, )
+    ankerbot_model.to("cuda") 
     prompt_format = "<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
     generator = pipeline(task="text-generation", model=ankerbot_model, tokenizer=ankerbot_tokenizer, torch_dtype=torch.float16, trust_remote_code=True) # True for flash-attn2 else False
     generator_mini = pipeline(task="text-generation", model=ankerbot_model, tokenizer=ankerbot_tokenizer, torch_dtype=torch.float16, trust_remote_code=True) # True for flash-attn2 else False
-
 load_model()
 @spaces.GPU
 def generate_response(query, context, prompts, max_tokens, temperature, top_p):