Refactor

Dockerfile

@@ -1,55 +1,3 @@
-# # Stage 1: Build stage
-# FROM python:3.12-slim as builder
-
-# # Set environment variables
-# ENV PYTHONDONTWRITEBYTECODE=1
-# ENV PYTHONUNBUFFERED=1
-
-# # Create a non-root user
-# RUN useradd -m -u 1000 user
-
-# # Set the working directory
-# WORKDIR /app
-
-# # Copy only the requirements file first to leverage Docker cache
-# COPY --chown=user ./requirements.txt /app/requirements.txt
-
-# # Install dependencies in a virtual environment
-# RUN python -m venv /opt/venv
-# ENV PATH="/opt/venv/bin:$PATH"
-# RUN pip install --no-cache-dir --upgrade pip && \
-#     pip install --no-cache-dir -r requirements.txt
-
-# # Copy the rest of the application code
-# COPY --chown=user . /app
-
-# # Stage 2: Runtime stage
-# FROM python:3.12-slim
-
-# # Create a non-root user
-# RUN useradd -m -u 1000 user
-# USER user
-
-# # Copy the virtual environment from the builder stage
-# COPY --from=builder /opt/venv /opt/venv
-# ENV PATH="/opt/venv/bin:$PATH"
-
-# # Set the working directory
-# WORKDIR /app
-
-# # Copy only the necessary files from the builder stage
-# COPY --from=builder --chown=user /app /app
-
-# # Expose the port the app runs on
-# EXPOSE 7860
-
-# # Health check to ensure the application is running
-# HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
-#     CMD curl -f http://localhost:7860/health || exit 1
-
-# # Command to run the application with hot reloading
-# CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "7860", "--reload"]
-
 # Stage 1: Build stage
 FROM python:3.12-slim as builder
 

docker-compose.yml

@@ -16,6 +16,8 @@ services:
       interval: 30s
       timeout: 10s
       retries: 3
+
+# Could be useful later on
 #     depends_on:
 #       - db  # If you have a database service, add it here
 

src/api/models/embedding_models.py

@@ -26,9 +26,6 @@ class ReadEmbeddingRequest(BaseModel):
 #     max_concurrent_requests: int = 10
 #     dataset_name: str = "re-mind/product_type_embedding"
 
-from pydantic import BaseModel
-from typing import Dict, List
-
 
 class UpdateEmbeddingRequest(BaseModel):
     dataset_name: str = "re-mind/product_type_embedding"

src/api/services/huggingface_service.py

@@ -47,31 +47,6 @@ class HuggingFaceService:
             logger.error(f"Failed to read dataset: {e}")
             raise DatasetNotFoundError(f"Dataset not found: {e}")
 
-    # async def update_dataset(
-    #     self, dataset_name: str, updates: Dict[str, List]
-    # ) -> Optional[pd.DataFrame]:
-    #     """Update a dataset on Hugging Face Hub."""
-
-    #     embedding_service = get_embedding_service()
-
-    #     try:
-    #         df_src = await self.read_dataset(dataset_name)
-    #         df_src = Dataset.from_dict(df_src)
-    #         df_update = Dataset.from_dict(updates)
-
-    #         df = concatenate_datasets(df_src, df_update)
-
-    #         # for column, values in updates.items():
-    #         #     if column in df.columns:
-    #         #         df[column] = values
-    #         #     else:
-    #         #         logger.warning(f"Column '{column}' not found in dataset.")
-    #         # await self.push_to_hub(df, dataset_name)
-    #         # return df
-    #     except Exception as e:
-    #         logger.error(f"Failed to update dataset: {e}")
-    #         raise DatasetPushError(f"Failed to update dataset: {e}")
-
     async def update_dataset(
         self,
         dataset_name: str,

src/main.py

@@ -1,191 +1,3 @@
-# import os
-# from fastapi import FastAPI, Depends, HTTPException
-# from fastapi.responses import JSONResponse, RedirectResponse
-# from fastapi.middleware.gzip import GZipMiddleware
-# from pydantic import BaseModel
-# from typing import List, Dict
-# from src.api.models.embedding_models import (
-#     CreateEmbeddingRequest,
-#     ReadEmbeddingRequest,
-#     UpdateEmbeddingRequest,
-#     DeleteEmbeddingRequest,
-# )
-# from src.api.database import get_db, Database, QueryExecutionError, HealthCheckError
-# from src.api.services.embedding_service import EmbeddingService
-# from src.api.services.huggingface_service import HuggingFaceService
-# from src.api.exceptions import DatasetNotFoundError, DatasetPushError, OpenAIError
-# import pandas as pd
-# import logging
-# from dotenv import load_dotenv
-
-# # Load environment variables
-# load_dotenv()
-
-# # Set up structured logging
-# logging.basicConfig(
-#     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-# )
-# logger = logging.getLogger(__name__)
-
-# description = """A FastAPI application for similarity search with PostgreSQL and OpenAI embeddings.
-
-# Direct/API URL:
-# https://re-mind-similarity-search.hf.space
-# """
-
-# # Initialize FastAPI app
-# app = FastAPI(
-#     title="Similarity Search API",
-#     description=description,
-#     version="1.0.0",
-# )
-
-# app.add_middleware(GZipMiddleware, minimum_size=1000)
-
-
-# # Root endpoint redirects to /docs
-# @app.get("/")
-# async def root():
-#     return RedirectResponse(url="/docs")
-
-
-# # Health check endpoint
-# @app.get("/health")
-# async def health_check(db: Database = Depends(get_db)):
-#     try:
-#         is_healthy = await db.health_check()
-#         if not is_healthy:
-#             raise HTTPException(status_code=500, detail="Database is unhealthy")
-#         return {"status": "healthy"}
-#     except HealthCheckError as e:
-#         raise HTTPException(status_code=500, detail=str(e))
-
-
-# # Dependency to get EmbeddingService
-# def get_embedding_service() -> EmbeddingService:
-#     return EmbeddingService(openai_api_key=os.getenv("OPENAI_API_KEY"))
-
-
-# # Dependency to get HuggingFaceService
-# def get_huggingface_service() -> HuggingFaceService:
-#     return HuggingFaceService()
-
-
-# # Endpoint to create embeddings
-# @app.post("/create_embedding")
-# async def create_embedding(
-#     request: CreateEmbeddingRequest,
-#     db: Database = Depends(get_db),
-#     embedding_service: EmbeddingService = Depends(get_embedding_service),
-#     huggingface_service: HuggingFaceService = Depends(get_huggingface_service),
-# ):
-#     """
-#     Create embeddings for the target column in the dataset.
-#     """
-#     try:
-#         # Step 1: Query the database
-#         logger.info("Fetching data from the database...")
-#         result = await db.fetch(request.query)
-#         df = pd.DataFrame(result)
-
-#         # Step 2: Generate embeddings
-#         df = await embedding_service.create_embeddings(
-#             df, request.target_column, request.output_column
-#         )
-
-#         # Step 3: Push to Hugging Face Hub
-#         await huggingface_service.push_to_hub(df, request.dataset_name)
-
-#         return JSONResponse(
-#             content={
-#                 "message": "Embeddings created and pushed to Hugging Face Hub.",
-#                 "dataset_name": request.dataset_name,
-#                 "num_rows": len(df),
-#             }
-#         )
-#     except QueryExecutionError as e:
-#         logger.error(f"Database query failed: {e}")
-#         raise HTTPException(status_code=500, detail=f"Database query failed: {e}")
-#     except OpenAIError as e:
-#         logger.error(f"OpenAI API error: {e}")
-#         raise HTTPException(status_code=500, detail=f"OpenAI API error: {e}")
-#     except DatasetPushError as e:
-#         logger.error(f"Failed to push dataset: {e}")
-#         raise HTTPException(status_code=500, detail=f"Failed to push dataset: {e}")
-#     except Exception as e:
-#         logger.error(f"An error occurred: {e}")
-#         raise HTTPException(status_code=500, detail=f"An error occurred: {e}")
-
-
-# # Endpoint to read embeddings
-# # @app.get("/read_embeddings/{dataset_name}")
-# @app.post("/read_embeddings")
-# async def read_embeddings(
-#     request: ReadEmbeddingRequest,
-#     huggingface_service: HuggingFaceService = Depends(get_huggingface_service),
-# ):
-#     """
-#     Read embeddings from a Hugging Face dataset.
-#     """
-#     try:
-#         df = await huggingface_service.read_dataset(request.dataset_name)
-#         return df
-#     except DatasetNotFoundError as e:
-#         logger.error(f"Dataset not found: {e}")
-#         raise HTTPException(status_code=404, detail=f"Dataset not found: {e}")
-#     except Exception as e:
-#         logger.error(f"An error occurred: {e}")
-#         raise HTTPException(status_code=500, detail=f"An error occurred: {e}")
-
-
-# # Endpoint to update embeddings
-# @app.post("/update_embeddings")
-# async def update_embeddings(
-#     request: UpdateEmbeddingRequest,
-#     huggingface_service: HuggingFaceService = Depends(get_huggingface_service),
-# ):
-#     """
-#     Update embeddings in a Hugging Face dataset.
-#     """
-#     try:
-#         df = await huggingface_service.update_dataset(
-#             request.dataset_name, request.updates
-#         )
-#         return {
-#             "message": "Embeddings updated successfully.",
-#             "dataset_name": request.dataset_name,
-#         }
-#     except DatasetPushError as e:
-#         logger.error(f"Failed to update dataset: {e}")
-#         raise HTTPException(status_code=500, detail=f"Failed to update dataset: {e}")
-#     except Exception as e:
-#         logger.error(f"An error occurred: {e}")
-#         raise HTTPException(status_code=500, detail=f"An error occurred: {e}")
-
-
-# # Endpoint to delete embeddings
-# @app.post("/delete_embeddings")
-# async def delete_embeddings(
-#     request: DeleteEmbeddingRequest,
-#     huggingface_service: HuggingFaceService = Depends(get_huggingface_service),
-# ):
-#     """
-#     Delete embeddings from a Hugging Face dataset.
-#     """
-#     try:
-#         await huggingface_service.delete_dataset(request.dataset_name)
-#         return {
-#             "message": "Embeddings deleted successfully.",
-#             "dataset_name": request.dataset_name,
-#         }
-#     except DatasetPushError as e:
-#         logger.error(f"Failed to delete columns: {e}")
-#         raise HTTPException(status_code=500, detail=f"Failed to delete columns: {e}")
-#     except Exception as e:
-#         logger.error(f"An error occurred: {e}")
-#         raise HTTPException(status_code=500, detail=f"An error occurred: {e}")
-
-
 import os
 from fastapi import FastAPI, Depends, HTTPException
 from fastapi.responses import JSONResponse, RedirectResponse