src error

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 src/faiss_index/index.faiss filter=lfs diff=lfs merge=lfs -text
+*.faiss filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,105 @@
+# Python cache/bytecode
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+dist/
+downloads/
+eggs/
+.eggs/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Unit test / coverage / tools
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.pytest_cache/
+pytestdebug.log
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pyre/
+.mypy_cache/
+.pytype/
+.pyright/
+.ruff_cache/
+
+# Jupyter
+.ipynb_checkpoints/
+**/.ipynb_checkpoints/
+
+# Environments
+.env
+.env.*
+!.env.example
+.venv/
+venv/
+env/
+ENV/
+.python-version
+
+# Logs
+*.log
+logs/
+
+# IDE / Editor
+.vscode/
+.vscode-test/
+.idea/
+*.sublime-workspace
+*.sublime-project
+
+# OS files
+.DS_Store
+Thumbs.db
+ehthumbs.db
+Desktop.ini
+
+# Streamlit
+.streamlit/secrets.toml
+
+# Hugging Face / model caches
+.huggingface/
+**/.cache/
+transformers_cache/
+hf_cache/
+sentence_transformers_cache/
+torch_cache/
+
+# Data and artifacts
+data/
+.data/
+datasets/
+outputs/
+artifacts/
+checkpoints/
+runs/
+wandb/
+mlruns/
+
+# Vector store / indexes
+vectorstore/
+.vectorstore/
+# faiss_index/
+# indexes/
+
+# Docker (local overrides)
+docker-compose.override.yml

Dockerfile

@@ -1,20 +1,28 @@
-FROM python:3.13.5-slim
+FROM python:3.11-slim
 
 WORKDIR /app
 
-RUN apt-get update && apt-get install -y \
-    build-essential \
+# Install only what you need for build/runtime
+RUN apt-get update && apt-get install -y --no-install-recommends \
     curl \
     git \
-    && rm -rf /var/lib/apt/lists/*
+ && rm -rf /var/lib/apt/lists/*
 
-COPY requirements.txt ./
-COPY src/ ./src/
+# Copy all sources at repo root into image (no src/ subpackage anymore)
+COPY requirements.txt ./requirements.txt
+COPY . .
 
-RUN pip3 install -r requirements.txt
+# Put project root on PYTHONPATH
+ENV PYTHONPATH=/app
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Spaces sets $PORT; default locally to 8501
+ENV PORT=8501
 
 EXPOSE 8501
 
-HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health
+HEALTHCHECK CMD curl --fail http://localhost:${PORT}/_stcore/health || exit 1
 
-ENTRYPOINT ["streamlit", "run", "src/streamlit_app.py", "--server.port=8501", "--server.address=0.0.0.0"]
+# Use sh so $PORT expands at runtime
+ENTRYPOINT ["/bin/sh", "-c", "streamlit run streamlit_app.py --server.port ${PORT} --server.address 0.0.0.0"]

app.py

@@ -1,3 +1,3 @@
 # Entrypoint renamed for Hugging Face Spaces Streamlit detection.
 # Currently imports the original app content.
-from src.streamlit_app import *  # noqa
+from streamlit_app import *  # noqa

{src/faiss_index → faiss_index}/index.faiss

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:286e3c5e6d3b5a8b8a642fb64a5363ec608dde23b197374bb73dc912fae06013
-size 123574317
+oid sha256:839024862e5b0a77cb65312c01ec88994ea1104a457fb58ca535976d5c79d934
+size 15392301

{src/faiss_index → faiss_index}/index.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8afc6f5341df0262d4275907748004ea3cc27897c241e21b674486d58ca0bd69
-size 59597386
+oid sha256:3edc9118bedd5886f40903221b3286af4c23bc05fcfd47fac55c2dbdfd852d8a
+size 6473293

requirements.txt

@@ -2,6 +2,7 @@ streamlit==1.38.0
 langchain==0.2.14
 langchain-community==0.2.12
 langchain-core==0.2.33
+langchain-text-splitters==0.2.2
 langchain-groq
 sentence-transformers==3.0.1
 faiss-cpu>=1.7.4
@@ -10,4 +11,5 @@ huggingface-hub>=0.23.0
 python-dotenv==1.0.1
 requests==2.32.3
 numpy<2.0.0
+dotenv
 # Removed tiktoken (unused) to avoid rust build on HF base image.

src/__init__.py

@@ -1,6 +0,0 @@
-"""
-Local package for the Research RAG Chatbot app.
-
-This file makes the `src` directory a Python package so absolute imports
-like `from src.vector_store import ...` work both locally and in Docker.
-"""

src/api.py

@@ -0,0 +1,21 @@
+from fastapi import FastAPI
+from .vector_store import build_or_load_vectorstore
+from .retriever import get_retriever
+from .rag_pipeline import build_rag_chain
+from .ingestion import df_to_documents, preprocess_dataframe, load_data_subset
+from .config import DATA_PATH
+import os
+
+app = FastAPI()
+
+# Load documents and vectorstore at startup
+df = load_data_subset(os.path.join(DATA_PATH, "arxiv-metadata-oai-snapshot.json"))
+df = preprocess_dataframe(df)
+docs = df_to_documents(df)
+vectorstore = build_or_load_vectorstore(docs)
+retriever = get_retriever(vectorstore)
+rag_chain = build_rag_chain(retriever)
+
+@app.get("/query")
+def query_rag(q: str):
+    return {"answer": rag_chain.invoke(q).content}

src/config.py

@@ -1,17 +1,15 @@
 import os
+from dotenv import load_dotenv
+load_dotenv()
 
-# Base paths
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-DATA_PATH = os.path.join(BASE_DIR, "data")
+DATA_PATH = "data"
+FAISS_INDEX_PATH = "faiss_index"
 
-# FAISS index location (folder);
-# streamlit_app expects a file check but we'll place the file path here.
-FAISS_DIR = os.path.join(os.path.dirname(__file__), "faiss_index")
-FAISS_INDEX_PATH = os.path.join(FAISS_DIR, "index.faiss")
+EMBEDDING_MODEL = "sentence-transformers/multi-qa-MiniLM-L6-cos-v1"
 
-# API Keys / Env
-GROQ_API_KEY = os.getenv("GROQ_API_KEY", "")
+DEVICE = "cuda" if os.environ.get("CUDA_AVAILABLE", "0") == "1" else "cpu"
 
-# Ensure directories exist at runtime
-os.makedirs(FAISS_DIR, exist_ok=True)
-os.makedirs(DATA_PATH, exist_ok=True)
+GROQ_API_KEY = os.environ.get("GROQ_API_KEY", "")
+
+# Cross-encoder model for reranking
+CROSS_ENCODER_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2"

src/embeddings.py

@@ -0,0 +1,8 @@
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from .config import EMBEDDING_MODEL, DEVICE
+
+def get_embedding_model():
+    return HuggingFaceEmbeddings(
+        model_name=EMBEDDING_MODEL,
+        model_kwargs={"device": DEVICE}
+    )

src/ingestion.py

@@ -1,73 +1,46 @@
-from __future__ import annotations
+"""Data loading, cleaning and preprocessing for ArXiv dataset."""
 
-import json
 import os
-from typing import List, Dict, Any, Optional
-
+import json
+import pandas as pd
 from langchain_core.documents import Document
+from .config import DATA_PATH
+from .text_processing import clean_text
 
-
-def load_data_subset(json_path: str, num_records: int = 50000) -> List[Dict[str, Any]]:
-    """
-    Load a subset of records from an arXiv metadata JSON lines file.
-    Returns a list of dicts (not a pandas DataFrame) to keep dependencies minimal.
-    """
-    if not os.path.exists(json_path):
-        raise FileNotFoundError(f"Data file not found: {json_path}")
-
-    rows: List[Dict[str, Any]] = []
-    with open(json_path, "r", encoding="utf-8") as f:
+def load_data_subset(file_path, num_records=50000):
+    records = []
+    with open(file_path, 'r') as f:
         for i, line in enumerate(f):
             if i >= num_records:
                 break
-            try:
-                rows.append(json.loads(line))
-            except json.JSONDecodeError:
-                continue
-    return rows
-
-
-def preprocess_dataframe(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    """
-    Light preprocessing: ensure keys exist, fill defaults, and derive a 'year' field.
-    Works on list-of-dicts to avoid pandas dependency.
-    """
-    def extract_year(versions: Any) -> Optional[int]:
-        if isinstance(versions, list) and versions:
-            created = versions[0].get("created")
-            if isinstance(created, str) and len(created) >= 4 and created[:4].isdigit():
-                return int(created[:4])
-        return None
-
-    norm: List[Dict[str, Any]] = []
-    for r in rows:
-        title = str(r.get("title", "") or "")
-        abstract = str(r.get("abstract", "") or "")
-        categories = str(r.get("categories", "") or "")
-        versions = r.get("versions")
-        year = r.get("year")
-        if not isinstance(year, int):
-            year = extract_year(versions)
-        norm.append({
-            **r,
-            "title": title,
-            "abstract": abstract,
-            "categories": categories,
-            "year": year,
-        })
-    return norm
-
-
-def df_to_documents(rows: List[Dict[str, Any]]) -> List[Document]:
-    """
-    Convert rows (list-of-dicts) to LangChain Documents with metadata.
-    """
-    docs: List[Document] = []
-    for r in rows:
-        content = f"Title: {r.get('title','')}\n\n{r.get('abstract','')}"
-        meta = {
-            "categories": r.get("categories", ""),
-            "year": r.get("year", None),
+            records.append(json.loads(line))
+    return pd.DataFrame(records)
+
+def preprocess_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    df['update_date'] = pd.to_datetime(df['update_date'])
+    df['year'] = df['update_date'].dt.year
+    df = df.dropna(subset=['abstract'])
+    df = df[df['abstract'].str.strip() != '']
+    return df
+
+def df_to_documents(
+    df: pd.DataFrame,
+    lowercase: bool = False,
+    remove_stopwords: bool = False
+):
+    documents = []
+    for _, row in df.iterrows():
+        title_clean = clean_text(str(row['title']), lowercase=lowercase, remove_stopwords=remove_stopwords)
+        abstract_clean = clean_text(str(row['abstract']), lowercase=lowercase, remove_stopwords=remove_stopwords)
+        page_content = f"Title: {title_clean}\n\nAbstract: {abstract_clean}"
+        categories_raw = row.get('categories', 'N/A') or 'N/A'
+        primary_category = categories_raw.split()[0] if isinstance(categories_raw, str) else 'N/A'
+        metadata = {
+            "id": row.get('id', 'N/A'),
+            "authors": row.get('authors', 'N/A'),
+            "year": int(row.get('year')) if not pd.isna(row.get('year')) else None,
+            "categories": categories_raw,
+            "primary_category": primary_category
         }
-        docs.append(Document(page_content=content, metadata=meta))
-    return docs
+        documents.append(Document(page_content=page_content, metadata=metadata))
+    return documents

src/main.py

@@ -0,0 +1,69 @@
+import os
+import kagglehub
+from .ingestion import load_data_subset, preprocess_dataframe, df_to_documents
+from .vector_store import build_or_load_vectorstore
+from .retriever import build_advanced_retriever
+from .rag_pipeline import build_rag_chain
+from .config import DATA_PATH
+import shutil 
+
+def download_dataset():
+    """Download the ArXiv dataset via KaggleHub if not already present."""
+    os.makedirs(DATA_PATH, exist_ok=True)
+    dataset_file = os.path.join(DATA_PATH, "arxiv-metadata-oai-snapshot.json")
+
+    if not os.path.exists(dataset_file):
+        print("Downloading ArXiv dataset via KaggleHub...")
+        path = kagglehub.dataset_download("Cornell-University/arxiv")
+        extracted_file = os.path.join(path, "arxiv-metadata-oai-snapshot.json")
+        shutil.copy(extracted_file, dataset_file)  # ✅ copy works across drives
+        print(f"Dataset copied to {dataset_file}")
+    else:
+        print(f"Dataset already exists at {dataset_file}")
+
+    return dataset_file
+
+
+def run_sample_queries(rag_chain):
+    """Run a few sample queries through the RAG pipeline."""
+    sample_questions = [
+        "What are the recent advancements in graph neural networks?",
+        "Explain the applications of transformers in natural language processing.",
+        "How is reinforcement learning applied in robotics?",
+    ]
+
+    for q in sample_questions:
+        print("\n---")
+        print(f"Question: {q}")
+        answer = rag_chain.invoke(q).content
+        print(f"Answer: {answer}")
+
+
+def main():
+    dataset_file = download_dataset()
+    df = load_data_subset(dataset_file, num_records=50000)
+    df = preprocess_dataframe(df)
+    documents = df_to_documents(df, lowercase=False, remove_stopwords=False)
+    vectorstore = build_or_load_vectorstore(
+        documents,
+        force_rebuild=False,
+        chunk_method="semantic",  # fallback to recursive if semantic splitter unavailable
+        chunk_size=800,
+        chunk_overlap=120
+    )
+    retriever = build_advanced_retriever(
+        vectorstore,
+        base_k=16,
+        rerank_k=6,
+        primary_category=None,
+        year_min=None,
+        year_max=None,
+        dynamic=True,
+        use_rerank=True,
+    )
+    rag_chain = build_rag_chain(retriever)
+    run_sample_queries(rag_chain)
+
+
+if __name__ == "__main__":
+    main()

src/rag_pipeline.py

@@ -0,0 +1,30 @@
+from langchain.schema.runnable import RunnableLambda, RunnablePassthrough
+from langchain.prompts import PromptTemplate
+from langchain_groq import ChatGroq
+from .config import GROQ_API_KEY
+from .retriever import RerankRetriever
+def build_rag_chain(retriever: RerankRetriever):
+    retriever_runnable = RunnableLambda(lambda question: retriever.get_relevant_documents(question))
+    format_docs_runnable = RunnableLambda(lambda docs: "\n\n".join([d.page_content for d in docs]))
+
+    prompt_template = """Answer the following question based on the provided context.
+
+Context:
+{context}
+
+Question: {question}
+
+Answer: """
+    prompt = PromptTemplate(template=prompt_template, input_variables=["context", "question"])
+
+    llm = ChatGroq(
+        model="meta-llama/llama-4-maverick-17b-128e-instruct",
+        temperature=0.7,
+        max_tokens=512,
+        groq_api_key=GROQ_API_KEY
+    )
+
+    return {
+        "context": retriever_runnable | format_docs_runnable,
+        "question": RunnablePassthrough()
+    } | prompt | llm

src/retriever.py

@@ -0,0 +1,103 @@
+from typing import Optional, List
+from dataclasses import dataclass
+from .config import CROSS_ENCODER_MODEL
+
+try:
+    from sentence_transformers import CrossEncoder
+    _HAS_CE = True
+except ImportError:
+    _HAS_CE = False
+
+
+def get_retriever(vectorstore, k=3):
+    return vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": k})
+
+
+@dataclass
+class RetrievalParams:
+    base_k: int = 8              # initial fetch size for reranking
+    rerank_k: int = 4            # final number after rerank
+    max_k: int = 20              # max docs to fetch for long/ambiguous queries
+    min_k: int = 3               # minimum docs
+    dynamic: bool = True         # enable dynamic k logic
+    year_min: Optional[int] = None
+    year_max: Optional[int] = None
+    primary_category: Optional[str] = None
+    use_rerank: bool = True
+
+
+class RerankRetriever:
+    def __init__(self, vectorstore, params: RetrievalParams):
+        self.vs = vectorstore
+        self.params = params
+        self.cross_encoder = None
+        if params.use_rerank and _HAS_CE:
+            self.cross_encoder = CrossEncoder(CROSS_ENCODER_MODEL)
+        elif params.use_rerank:
+            print("CrossEncoder not available; install sentence-transformers to enable reranking.")
+
+    def _compute_dynamic_k(self, query: str) -> int:
+        if not self.params.dynamic:
+            return self.params.base_k
+        length = len(query.split())
+        if length <= 4:  # very short, broaden
+            return min(self.params.base_k + 6, self.params.max_k)
+        if length <= 12:
+            return self.params.base_k
+        return min(self.params.base_k + 4, self.params.max_k)
+
+    def _metadata_filter(self, docs):
+        p = self.params
+        filtered = []
+        for d in docs:
+            y = d.metadata.get("year")
+            if p.year_min is not None and (y is None or y < p.year_min):
+                continue
+            if p.year_max is not None and (y is None or y > p.year_max):
+                continue
+            if p.primary_category and d.metadata.get("primary_category") != p.primary_category:
+                continue
+            filtered.append(d)
+        return filtered
+
+    def get_relevant_documents(self, query: str):
+        fetch_k = self._compute_dynamic_k(query)
+        base_retriever = self.vs.as_retriever(search_type="similarity", search_kwargs={"k": fetch_k})
+        docs = base_retriever.get_relevant_documents(query)
+        docs = self._metadata_filter(docs)
+        if self.cross_encoder and docs:
+            pairs = [(query, d.page_content) for d in docs]
+            scores = self.cross_encoder.predict(pairs)
+            ranked = sorted(zip(docs, scores), key=lambda x: x[1], reverse=True)
+            final_k = min(self.params.rerank_k, len(ranked))
+            docs = [d for d, _ in ranked[:final_k]]
+        else:
+            # fallback: truncate
+            docs = docs[: self.params.rerank_k]
+        return docs
+
+    # For LangChain compatibility
+    def invoke(self, query: str):
+        return self.get_relevant_documents(query)
+
+
+def build_advanced_retriever(
+    vectorstore,
+    base_k: int = 12,
+    rerank_k: int = 5,
+    primary_category: Optional[str] = None,
+    year_min: Optional[int] = None,
+    year_max: Optional[int] = None,
+    dynamic: bool = True,
+    use_rerank: bool = True,
+):
+    params = RetrievalParams(
+        base_k=base_k,
+        rerank_k=rerank_k,
+        primary_category=primary_category,
+        year_min=year_min,
+        year_max=year_max,
+        dynamic=dynamic,
+        use_rerank=use_rerank,
+    )
+    return RerankRetriever(vectorstore, params)

src/text_processing.py

@@ -0,0 +1,45 @@
+import re
+from typing import Iterable
+
+# Basic English stopwords (small set to avoid extra dependency); extend if needed
+BASIC_STOPWORDS = {
+    'the','and','a','an','of','in','to','is','are','for','on','with','that','this','by','from','at','as','it','be','or','we','can','our','their','these','those','using','used'
+}
+
+LATEX_EQ_RE = re.compile(r'\$\$.*?\$\$|\$[^$]*\$', re.DOTALL)
+URL_RE = re.compile(r'https?://\S+|www\.\S+')
+MULTI_WS_RE = re.compile(r'\s+')
+INLINE_LATEX_CMD_RE = re.compile(r'\\(?:cite|ref|label|eqref|begin|end|textbf|emph|mathrm|mathbb)\{[^}]*\}')
+
+
+def remove_latex(text: str) -> str:
+    text = LATEX_EQ_RE.sub(' ', text)
+    text = INLINE_LATEX_CMD_RE.sub(' ', text)
+    return text
+
+
+def remove_urls(text: str) -> str:
+    return URL_RE.sub(' ', text)
+
+
+def normalize_whitespace(text: str) -> str:
+    return MULTI_WS_RE.sub(' ', text).strip()
+
+
+def strip_stopwords(tokens: Iterable[str]) -> str:
+    return ' '.join(t for t in tokens if t not in BASIC_STOPWORDS)
+
+
+def clean_text(text: str, lowercase: bool = False, remove_stopwords: bool = False) -> str:
+    if not text:
+        return ''
+    t = remove_urls(text)
+    t = remove_latex(t)
+    if lowercase:
+        t = t.lower()
+    # Tokenize very simply on whitespace after basic cleanup
+    t = normalize_whitespace(t)
+    if remove_stopwords:
+        tokens = t.split()
+        t = strip_stopwords(tokens)
+    return t

src/vector_store.py

@@ -0,0 +1,95 @@
+import os
+from typing import List, Optional
+from langchain_community.vectorstores import FAISS
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+try:
+    from semantic_text_splitter import TextSplitter as SemanticTextSplitter  # type: ignore
+    _HAS_SEMANTIC = True
+except ImportError:  # graceful fallback if package missing
+    _HAS_SEMANTIC = False
+from langchain_core.documents import Document
+from .embeddings import get_embedding_model
+from .config import FAISS_INDEX_PATH
+
+def _chunk_documents(
+    documents: List[Document],
+    method: str = "recursive",
+    chunk_size: int = 1000,
+    chunk_overlap: int = 120
+):
+    if method == "semantic" and _HAS_SEMANTIC:
+        try:
+            # Newer versions expose factory; fallback to direct init
+            if hasattr(SemanticTextSplitter, "from_tiktoken_encoder"):
+                splitter = SemanticTextSplitter.from_tiktoken_encoder(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                )
+            else:  # try simple init signature
+                splitter = SemanticTextSplitter(
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap,
+                )
+            semantic_chunks: List[Document] = []
+            for d in documents:
+                try:
+                    parts = splitter.chunks(d.page_content)
+                except AttributeError:
+                    # Fallback: naive sentence-ish split
+                    parts = d.page_content.split('. ')
+                for part in parts:
+                    cleaned = part.strip()
+                    if not cleaned:
+                        continue
+                    semantic_chunks.append(
+                        Document(page_content=cleaned, metadata=d.metadata)
+                    )
+            return semantic_chunks
+        except Exception as e:
+            print(f"[semantic chunking fallback] {e}; reverting to recursive splitter.")
+    # fallback / default
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size, chunk_overlap=chunk_overlap, add_start_index=True
+    )
+    return splitter.split_documents(documents)
+
+def build_or_load_vectorstore(
+    documents: List[Document],
+    force_rebuild: bool = False,
+    chunk_method: str = "recursive",  # or "semantic"
+    chunk_size: int = 1000,
+    chunk_overlap: int = 120
+):
+    if os.path.exists(FAISS_INDEX_PATH) and not force_rebuild:
+        print(f"Loading existing FAISS index from {FAISS_INDEX_PATH}...")
+        vectorstore = FAISS.load_local(
+            FAISS_INDEX_PATH,
+            get_embedding_model(),
+            allow_dangerous_deserialization=True
+        )
+        print("Vector store loaded successfully.")
+        return vectorstore
+
+    print("Building FAISS index (force_rebuild=%s, method=%s)..." % (force_rebuild, chunk_method))
+    splits = _chunk_documents(
+        documents,
+        method=chunk_method,
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap
+    )
+    print(f"Split {len(documents)} docs into {len(splits)} chunks (method={chunk_method}).")
+    vectorstore = FAISS.from_documents(splits, get_embedding_model())
+    vectorstore.save_local(FAISS_INDEX_PATH)
+    print(f"Vector store created and saved to {FAISS_INDEX_PATH}")
+    return vectorstore
+
+def build_filtered_retriever(vectorstore, primary_category: Optional[str] = None, k: int = 3):
+    base = vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": k})
+    if not primary_category:
+        return base
+    # Simple wrapper applying post-filtering by metadata; could be replaced by a VectorStore-specific filter if supported
+    def _get_relevant_documents(query):
+        docs = base.get_relevant_documents(query)
+        return [d for d in docs if d.metadata.get("primary_category") == primary_category]
+    base.get_relevant_documents = _get_relevant_documents  # monkey patch
+    return base

src/streamlit_app.py → streamlit_app.py

@@ -1,21 +1,10 @@
 import os
-import torch
 import streamlit as st
 from dotenv import load_dotenv, find_dotenv
-
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-from langchain_community.embeddings import HuggingFaceEmbeddings
-from langchain_community.vectorstores import FAISS
 from langchain.prompts import PromptTemplate
 from langchain.schema.runnable import RunnablePassthrough
 from langchain_core.runnables import RunnableLambda
 from langchain_groq import ChatGroq
-import os
-import streamlit as st
-from dotenv import load_dotenv, find_dotenv
-from langchain.prompts import PromptTemplate
-from langchain.schema.runnable import RunnablePassthrough
-from langchain_groq import ChatGroq
 
 from src.vector_store import build_or_load_vectorstore
 from src.ingestion import load_data_subset, preprocess_dataframe, df_to_documents