first files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/* filter=lfs diff=lfs merge=lfs -text
+*.docx filter=lfs diff=lfs merge=lfs -text

.gitattributes copy

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+metaverse_executive_summary.docx filter=lfs diff=lfs merge=lfs -text
+data/payment/payment_executive_summary.docx filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+appold.py

app.py

@@ -0,0 +1,186 @@
+import spaces
+import os
+import gradio as gr
+from models import download_models
+from rag_backend import Backend
+from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
+from llama_cpp_agent.providers import LlamaCppPythonProvider
+from llama_cpp_agent.chat_history import BasicChatHistory
+from llama_cpp_agent.chat_history.messages import Roles
+import cv2
+
+# get the models
+huggingface_token = os.environ.get('HF_TOKEN')
+download_models(huggingface_token)
+
+documents_paths = {
+    'blockchain': 'data/blockchain',
+    'metaverse': 'data/metaverse',
+    'payment': 'data/payment'
+}
+
+
+# initialize backend (not ideal as global variable...)
+backend = Backend()
+
+cv2.setNumThreads(1)
+
+@spaces.GPU(duration=20)
+def respond(
+    message,
+    history: list[tuple[str, str]],
+    model,
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+    top_k,
+    repeat_penalty,
+):
+    chat_template = MessagesFormatterType.GEMMA_2
+
+    print("HISTORY SO FAR ", history)
+
+    matched_path = None
+    words = message.lower() 
+
+    for key, path in documents_paths.items():
+        if len(history) == 1 and key in words: # check if the user mentions a path word only during second interaction (i.e history has only one entry)
+            matched_path = path
+            break     
+    print("matched_path", matched_path) 
+
+    if matched_path:  # this case would only be true in second interaction
+        original_message = history[0][0]
+        print("** matched path!!")
+        query_engine = backend.create_index_for_query_engine(matched_path)
+        message = backend.generate_prompt(query_engine, original_message)
+        
+        gr.Info("Relevant context indexed from docs...")
+
+    elif (not matched_path) and (len(history) > 1):
+        print("Using context from storage db")         
+        query_engine = backend.load_index_for_query_engine()
+        message = backend.generate_prompt(query_engine, message)
+
+        gr.Info("Relevant context extracted from db...")
+
+    # Load model only if it's not already loaded or if a new model is selected
+    if backend.llm is None or backend.llm_model != model:
+        try:
+            backend.load_model(model)
+        except Exception as e:
+            return f"Error loading model: {str(e)}"
+
+    provider = LlamaCppPythonProvider(backend.llm)
+
+    agent = LlamaCppAgent(
+        provider,
+        system_prompt=f"{system_message}",
+        predefined_messages_formatter_type=chat_template,
+        debug_output=True
+    )
+
+    settings = provider.get_provider_default_settings()
+    settings.temperature = temperature
+    settings.top_k = top_k
+    settings.top_p = top_p
+    settings.max_tokens = max_tokens
+    settings.repeat_penalty = repeat_penalty
+    settings.stream = True
+
+    messages = BasicChatHistory()
+
+    # add user and assistant messages to the history
+    for msn in history:
+        user = {'role': Roles.user, 'content': msn[0]}
+        assistant = {'role': Roles.assistant, 'content': msn[1]}
+        messages.add_message(user)
+        messages.add_message(assistant)
+
+    
+    try:
+        stream = agent.get_chat_response(
+            message, 
+            llm_sampling_settings=settings,
+            chat_history=messages,
+            returns_streaming_generator=True,
+            print_output=False
+        )
+
+        outputs = ""
+        for output in stream:
+            outputs += output
+            yield outputs
+    except Exception as e:
+        yield f"Error during response generation: {str(e)}"
+        
+
+
+demo = gr.ChatInterface(
+    fn=respond,
+    css="""
+    .gradio-container {
+        background-color: #B9D9EB;
+        color: #003366;
+    }""",
+    additional_inputs=[
+        gr.Dropdown([
+                'Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf',
+                'Mistral-Nemo-Instruct-2407-Q5_K_M.gguf',
+                'gemma-2-2b-it-Q6_K_L.gguf',
+                'openchat-3.6-8b-20240522-Q6_K.gguf',
+                'Llama-3-Groq-8B-Tool-Use-Q6_K.gguf',
+                'MiniCPM-V-2_6-Q6_K.gguf',
+                'llama-3.1-storm-8b-q5_k_m.gguf',
+                'orca-2-7b-patent-instruct-llama-2-q5_k_m.gguf'
+            ],
+            value="gemma-2-2b-it-Q6_K_L.gguf",
+            label="Model"
+        ),
+        gr.Textbox(value="""Solamente all'inizio, presentati come Odi, un assistente ricercatore italiano creato dagli Osservatori del Politecnico di Milano e specializzato nel fornire risposte precise e pertinenti solo ad argomenti di innovazione digitale. 
+        Solo nella tua prima risposta, chiedi all'utente di indicare a quale di queste tre sezioni degli Osservatori si riferisce la sua domanda: 'Blockchain', 'Payment' o 'Metaverse'.
+Per le risposte successive, utilizza la cronologia della chat o il contesto fornito per aiutare l'utente a ottenere una risposta accurata. 
+Non rispondere mai a domande che non sono pertinenti a questi argomenti.""", label="System message"),
+        gr.Slider(minimum=1, maximum=4096, value=3048, step=1, label="Max tokens"),
+        gr.Slider(minimum=0.1, maximum=4.0, value=1.2, step=0.1, label="Temperature"),
+        gr.Slider(
+            minimum=0.1,
+            maximum=1.0,
+            value=0.95,
+            step=0.05,
+            label="Top-p",
+        ),
+        gr.Slider(
+            minimum=0,
+            maximum=100,
+            value=30,
+            step=1,
+            label="Top-k",
+        ),
+        gr.Slider(
+            minimum=0.0,
+            maximum=2.0,
+            value=1.1,
+            step=0.1,
+            label="Repetition penalty",
+            
+        ),
+
+    ],
+    retry_btn="Riprova",
+    undo_btn="Annulla",
+    clear_btn="Pulisci",
+    submit_btn="Invia",
+    title="Odi, l'assistente ricercatore degli Osservatori",
+    chatbot=gr.Chatbot(
+        scale=1,
+        likeable=False,
+        show_copy_button=True
+    ),
+    examples=[["Ciao, in cosa puoi aiutarmi?"],["Quanto vale il mercato italiano?"], ["Per favore dammi informazioni sugli ambiti applicativi"], ["Svelami una buona ricetta milanese"] ],
+    cache_examples=False,
+)
+
+if __name__ == "__main__":
+    demo.launch()

data/metaverse/Glossario_metaverse.docx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:076b383bc64c0c0231fa3549545683d498fb73370fd54fdf3b8ffe9471e4dbb6
+size 31966

data/metaverse/metaverse_executive_summary.docx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:854abbec61fc27fc87669fbcaa6f4a5aafaac93ea715492e775d53a59d091a29
+size 8377251

data/payment/Glossario_payment.docx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9c26b362e192eb74e8a466bb62529d5830da926f675a91605d10dce0145f311
+size 22331

data/payment/payment_executive_summary.docx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7cd5de9017d27fb118612bb5c1d624f7697fb3716272e3a382d9e126216cc1a
+size 4110078

models.py

@@ -0,0 +1,21 @@
+from huggingface_hub import hf_hub_download
+
+def download_models(huggingface_token):
+    models = [
+        ("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF", "Meta-Llama-3.1-8B-Instruct-Q5_K_M.gguf"),
+        ("bartowski/Mistral-Nemo-Instruct-2407-GGUF", "Mistral-Nemo-Instruct-2407-Q5_K_M.gguf"),
+        ("bartowski/gemma-2-2b-it-GGUF", "gemma-2-2b-it-Q6_K_L.gguf"),
+        ("bartowski/openchat-3.6-8b-20240522-GGUF", "openchat-3.6-8b-20240522-Q6_K.gguf"),
+        ("bartowski/Llama-3-Groq-8B-Tool-Use-GGUF", "Llama-3-Groq-8B-Tool-Use-Q6_K.gguf"),
+        ("bartowski/MiniCPM-V-2_6-GGUF", "MiniCPM-V-2_6-Q6_K.gguf"),
+        ("CaioXapelaum/Llama-3.1-Storm-8B-Q5_K_M-GGUF", "llama-3.1-storm-8b-q5_k_m.gguf"),
+        ("CaioXapelaum/Orca-2-7b-Patent-Instruct-Llama-2-Q5_K_M-GGUF", "orca-2-7b-patent-instruct-llama-2-q5_k_m.gguf"),
+    ]
+    
+    for repo_id, filename in models:
+        hf_hub_download(
+            repo_id=repo_id,
+            filename=filename,
+            local_dir="./models",
+            token=huggingface_token
+        )

rag_backend.py

@@ -0,0 +1,63 @@
+import os
+from llama_cpp import Llama
+from llama_index.core import VectorStoreIndex, Settings, SimpleDirectoryReader, load_index_from_storage, StorageContext
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+
+
+Settings.llm = None
+
+class Backend:
+    def __init__(self):
+        self.llm = None
+        self.llm_model = None
+        self.embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5")
+        self.PERSIST_DIR = "./db"
+        os.makedirs(self.PERSIST_DIR, exist_ok=True)
+        
+    def load_model(self, model_path):
+        self.llm = Llama(
+            model_path=f"models/{model_path}",
+            flash_attn=True,
+            n_gpu_layers=81,
+            n_batch=1024,
+            n_ctx=8192,
+        )
+        self.llm_model = model_path
+        
+        
+    def create_index_for_query_engine(self, matched_path):
+    
+        documents = SimpleDirectoryReader(input_dir=matched_path).load_data()
+        storage_context = StorageContext.from_defaults()
+        nodes = SentenceSplitter(chunk_size=256, chunk_overlap=64, paragraph_separator="\n\n").get_nodes_from_documents(documents)
+        index = VectorStoreIndex(nodes, embed_model=self.embed_model)
+        query_engine = index.as_query_engine(
+                        similarity_top_k=4, response_mode="tree_summarize"
+                        )
+        index.storage_context.persist(persist_dir=self.PERSIST_DIR)
+        
+        return query_engine
+
+
+    # here we're leveraging an already constructed and stored FAISS index
+    def load_index_for_query_engine(self):
+        storage_context = StorageContext.from_defaults(persist_dir=self.PERSIST_DIR)
+        index = load_index_from_storage(storage_context, embed_model=self.embed_model)
+        
+        query_engine = index.as_query_engine(
+                        similarity_top_k=4, response_mode="tree_summarize"
+                        )
+        return query_engine
+
+
+    def generate_prompt(self, query_engine, message):
+        relevant_chunks = query_engine.retrieve(message)
+        print(f"Found: {len(relevant_chunks)} relevant chunks")
+        
+        prompt = "Considera questo come tua base di conoscenza personale:\n==========Conoscenza===========\n"
+        for idx, chunk in enumerate(relevant_chunks):
+            print(f"{idx + 1}) {chunk.text[:64]}...")
+            prompt += chunk.text + "\n\n"   
+        prompt += "\n======================\nDomanda: " + message
+        return prompt

requirements.txt

@@ -0,0 +1,10 @@
+spaces
+huggingface_hub
+scikit-build-core
+https://github.com/abetlen/llama-cpp-python/releases/download/v0.2.90-cu124/llama_cpp_python-0.2.90-cp310-cp310-linux_x86_64.whl
+git+https://github.com/Maximilian-Winter/llama-cpp-agent
+opencv-python
+llama-index
+llama-index-embeddings-huggingface
+llama-index-embeddings-instructor
+docx2txt