env variable flash attn 0

app.py

@@ -8,7 +8,8 @@ from huggingface_hub import login
 import torch
 import json
 import bs4
-
+import os
+os.environ["USE_FLASH_ATTENTION"] = "0"
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
 product_strings = []
@@ -105,7 +106,6 @@ def load_model():
         truncation=True,
         padding=True, )
     ankerbot_model.to("cuda") 
-    prompt_format = "<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
     generator = pipeline(task="text-generation", model=ankerbot_model, tokenizer=ankerbot_tokenizer, torch_dtype=torch.float16, attn_implementation="eager", use_flash_attention_2=False) # True for flash-attn2 else False
     generator_mini = pipeline(task="text-generation", model=ankerbot_model, tokenizer=ankerbot_tokenizer, torch_dtype=torch.float16, attn_implementation="eager", use_flash_attention_2=False) # True for flash-attn2 else False
 load_model()