Increased number of documents to 50000

app.py

@@ -1,6 +1,6 @@
 import streamlit as st
 from config import CONFIG
-from model.main import process_query
+from model.main import process_query, prepare_retriever
 
 st.title("RAG Question Answering System")
 
@@ -50,10 +50,18 @@ if st.button("Generate Answear"):
                     st.write(f"- {doc}")
                 
                 st.subheader("Generated Answer")
-                st.text_area("Generated Answer", value=answer, height=CONFIG['TEXTAREA_HEIGHT'], disabled=True)
+                st.text_area("Generated Answer", value=answer, height=CONFIG['TEXTAREA_HEIGHT'])
             except Exception as e:
                 st.error(f"An error occurred: {e}")
 
+# if st.button("Prepare Retriever"):
+#     with st.spinner("Preparing retriever..."):
+#         try:
+#             prepare_retriever()
+#             st.success("Retriever prepared successfully!")
+#         except Exception as e:
+#             st.error(f"Failed to prepare retriever: {e}")
+
 st.markdown(
     """
     <style>

config.py

@@ -1,6 +1,6 @@
 CONFIG = {
     "DATASET": "aalksii/ml-arxiv-papers",
-    "MAX_NUM_OF_RECORDS": 1000,
+    "MAX_NUM_OF_RECORDS": 50000,
     "TEXTAREA_HEIGHT": 200,
     "CHUNK_SIZE": 200,
     "OPENAI_ENGINE": "gpt-4o-mini",

model/main.py

@@ -1,14 +1,24 @@
 import streamlit as st
+import pickle
 from model.questionAnsweringBot import QuestionAnsweringBot
 from model.retriever import Retriever
 
 def process_query(llm_key, query, retrieval_method):
+    # if "retriever" not in st.session_state:
+    #     st.session_state.retriever = Retriever()
+    #     print("Loading and preparing dataset...")
+    #     st.session_state.retriever.load_and_prepare_dataset()
+    #     st.session_state.retriever.prepare_bm25()
+    #     st.session_state.retriever.compute_embeddings()
     if "retriever" not in st.session_state:
-        st.session_state.retriever = Retriever()
-        print("Loading and preparing dataset...")
-        st.session_state.retriever.load_and_prepare_dataset()
-        st.session_state.retriever.prepare_bm25()
-        st.session_state.retriever.compute_embeddings()
+        with st.spinner("Loading precomputed retriever..."):
+            try:
+                import pickle
+                with open("retriever.pkl", "rb") as f:
+                    st.session_state.retriever = pickle.load(f)
+                st.success("Preloaded retriever successfully!")
+            except Exception as e:
+                st.error(f"Failed to load precomputed retriever: {e}")
 
     retriever = st.session_state.retriever
 
@@ -42,3 +52,12 @@ def getPrompt(retrieved_docs, query):
     prompt += f"\nQuery: {query}\n"
 
     return prompt
+
+def prepare_retriever():
+    retriever = Retriever()
+    retriever.load_and_prepare_dataset()
+    retriever.prepare_bm25()
+    retriever.compute_embeddings()
+
+    with open("retriever.pkl", "wb") as f:
+        pickle.dump(retriever, f)

retriever.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82ba6dd3aacd7ce192db5c240791ce7bea2f0f7d4ff4a90eba4ae697d370939c
+size 316691228