latest code from main

configs/auto_merging.yaml

@@ -2,7 +2,7 @@ source_doc: "Master_Thesis.pdf"
 rag_mode: "auto-merging retrieval"
 llm_openai_model: "gpt-4o-mini"
 embed_model: "BAAI/bge-small-en-v1.5"
-chunk_sizes: [2048, 512, 128]
-similarity_top_k: 6
+chunk_sizes: [2048, 512]
+similarity_top_k: 8
 rerank_model: "cross-encoder/ms-marco-MiniLM-L-2-v2"
 rerank_top_n: 2

configs/basic.yaml

@@ -2,6 +2,6 @@ source_doc: "Master_Thesis.pdf"
 rag_mode: "classic retrieval"
 llm_openai_model: "gpt-4o-mini"
 embed_model: "BAAI/bge-small-en-v1.5"
-similarity_top_k: 6
+similarity_top_k: 10
 rerank_model: "cross-encoder/ms-marco-MiniLM-L-2-v2"
 rerank_top_n: 2

configs/sentence_window.yaml

@@ -2,7 +2,7 @@ source_doc: "Master_Thesis.pdf"
 rag_mode: "sentence window retrieval"
 llm_openai_model: "gpt-4o-mini"
 embed_model: "BAAI/bge-small-en-v1.5"
-sentence_window_size: 3
+sentence_window_size: 4
 similarity_top_k: 6
 rerank_model: "cross-encoder/ms-marco-MiniLM-L-2-v2"
 rerank_top_n: 2

scripts/app.py

@@ -112,7 +112,7 @@ with open(welcome_message_path, encoding="utf-8") as f:
 gradio_app = gr.Interface(
     fn=chat_bot,
     inputs=[
-        gr.Textbox(placeholder=default_message, label="Query"),
+        gr.Textbox(placeholder=default_message, label="Query", lines=2),
         gr.Dropdown(
             choices=SupportedRags.__args__,
             label="RAG mode",

spaces/welcome_message.md

@@ -11,7 +11,7 @@ Here you get to choose between three RAG techniques:
 - **auto-merging retrieval**
 
 Feel free to experiment with different modes! Note that a little extra delay is to be expected when switching to another mode.
-Also, note that all your queries (as well as system responses) are automatically logged on a remote PostgreSQL database for continuous monitoring of the deployed systems.
+Also, note that all your queries (as well as system responses, and evaluation of these responses) are automatically logged on a remote PostgreSQL database for continuous monitoring of the deployed systems.
 
 Each of these systems has been optimized for performance by doing a grid search on the
 relevant parameters. Performance is quantified with five metrics:

src/mythesis_chatbot/evaluation.py

@@ -1,9 +1,11 @@
+import os
 from pathlib import Path
+from typing import Literal
 
 import numpy as np
 from tqdm import tqdm
 from trulens.apps.llamaindex import TruLlama
-from trulens.core import Feedback
+from trulens.core import Feedback, TruSession
 from trulens.providers.openai import OpenAI
 
 from src.mythesis_chatbot.utils import get_config_hash
@@ -23,7 +25,7 @@ def run_evals(eval_questions_path: Path, tru_recorder, query_engine):
 
 
 # Feedback function
-def f_answer_relevance(provider=OpenAI(), name="Answer Relevance"):
+def f_answer_relevance(provider=OpenAI(), name="Answer Relevance") -> Feedback:
     return Feedback(provider.relevance_with_cot_reasons, name=name).on_input_output()
 
 
@@ -32,7 +34,7 @@ def f_context_relevance(
     provider=OpenAI(),
     context=TruLlama.select_source_nodes().node.text,
     name="Context Relevance",
-):
+) -> Feedback:
     return (
         Feedback(provider.relevance, name=name)
         .on_input()
@@ -46,7 +48,7 @@ def f_groundedness(
     provider=OpenAI(),
     context=TruLlama.select_source_nodes().node.text,
     name="Groundedness",
-):
+) -> Feedback:
     return (
         Feedback(
             provider.groundedness_measure_with_cot_reasons,
@@ -59,7 +61,7 @@ def f_groundedness(
 
 def get_prebuilt_trulens_recorder(
     query_engine, query_engine_config: dict[str, str | int]
-):
+) -> TruLlama:
     app_name = query_engine_config["rag_mode"]
     app_version = get_config_hash(query_engine_config)
 
@@ -71,3 +73,54 @@ def get_prebuilt_trulens_recorder(
         feedbacks=[f_answer_relevance(), f_context_relevance(), f_groundedness()],
     )
     return tru_recorder
+
+
+def get_tru_session(database: Literal["prod", "dev"]) -> TruSession:
+
+    print(f"Connecting to {database.lower()} database...")
+
+    match database.lower():
+        case "prod":
+            database_url = os.getenv("SUPABASE_PROD_CONNECTION_STRING_IPV4")
+            if database_url is None:
+                raise RuntimeError(
+                    "IPv4 connection string to production database is not available as"
+                    " an environment variable."
+                )
+            else:
+                print("Using IPv4 connection string...")
+                tru = TruSession(database_url=database_url)
+                return tru
+
+        case "dev":
+            database_url = os.getenv("SUPABASE_DEV_CONNECTION_STRING_IPV6")
+            if database_url:
+                try:
+                    print("Using IPv6 connection string...")
+                    tru = TruSession(database_url=database_url)
+                    return tru
+                except Exception as e:
+                    print(
+                        "An error occurred while connecting to remote dev database with"
+                        f" IPv6 connection string: {e}"
+                    )
+                    print("Reverting to IPv4")
+            else:
+                print(
+                    "IPv6 connection string to dev database is not available as an"
+                    " environment variable. Reverting to IPv4."
+                )
+
+            database_url = os.getenv("SUPABASE_DEV_CONNECTION_STRING_IPV4")
+            if database_url is None:
+                raise RuntimeError(
+                    "IPv4 connection string to dev database is not available"
+                    " as an environment variable."
+                )
+            else:
+                tru = TruSession(database_url=database_url)
+                return tru
+        case _:
+            raise ValueError(
+                f"Invalid database: {database}. Choose betwen 'prod' and 'dev'"
+            )