NOW. We say hello to the future.

app.py

@@ -6,7 +6,7 @@ import torch
 
 from conversation import get_default_conv_template
 import gradio as gr
-from llama_cpp import Llama
+from llama_cpp import Llama, CompletionChunk
 import json
 
 """
@@ -37,12 +37,15 @@ def m3b_talk(text):
     resp = ""
     formattedQuery = "<s> [|User|]" + text + "</s> [|Assistant|]"
 
-    for token in lcpp_model(formattedQuery, stop=["[|User|]", "\n"], echo=True):
-        resp += token
+    # for token in lcpp_model(formattedQuery, stop=["[|User|]", "\n"], echo=True):
+    #     resp += lccp_model.detokenize(token)
+    r = lcpp_model(formattedQuery, stop=["[|User|]", "\n"], echo=True)
+    for c in r["choices"]:
+        resp += c["text"]
     print(resp)
-    jsn = json.loads(resp)
-    answer = jsn["choices"][0]["text"].replace(formattedQuery, "")
-    return answer
+    # jsn = json.loads(resp)
+    # answer = jsn["choices"][0]["text"].replace(formattedQuery, "")
+    return resp.replace(formattedQuery, "")
 
 def main():
     logging.basicConfig(level=logging.INFO)
@@ -59,7 +62,7 @@ def main():
 
         m3b_talk_btn.click(m3b_talk, inputs=m3b_talk_input, outputs=m3b_talk_output, api_name="talk_m3b")
 
-    demo.queue(concurrency_count=1).launch()
+    demo.queue().launch()
 
 
 if __name__ == "__main__":