working on image analysis (idefics)

src/analysis/analyzeImage.mts

@@ -0,0 +1,43 @@
+
+import { client } from "@gradio/client"
+
+
+// we don't use replicas yet, because it ain't easy to get their hostname
+const instances: string[] = [
+  `${process.env.VC_ANALYSIS_SPACE_API_URL || ""}`,
+  // `${process.env.VC_UPSCALING_SPACE_API_URL_2 || ""}`,
+  // `${process.env.VC_UPSCALING_SPACE_API_URL_3 || ""}`,
+].filter(instance => instance?.length > 0)
+
+
+export async function analyzeImage(src: string, prompt: string): Promise<string> {
+
+  const instance = instances.shift()
+  instances.push(instance)
+
+  const api = await client(instance, {
+    hf_token: `${process.env.VC_HF_API_TOKEN}` as any
+  })
+  
+  const result = await api.predict(6, [		
+    "HuggingFaceM4/idefics-80b-instruct", // string (Option from: ['HuggingFaceM4/idefics-80b-instruct']) in 'Model' Dropdown component		
+    prompt, // string  in 'Text input' Textbox component		
+    "null", // any (any valid json) in 'IDEFICS' Chatbot component
+    src, 	// blob in 'Image input' Image component
+
+    // the following values come from the source code at:
+    // https://huggingface.co/spaces/HuggingFaceM4/idefics_playground/blob/main/app_dialogue.py#L416-L472
+
+    "Greedy", // string  in 'Decoding strategy' Radio component		
+    0.4, // number (numeric value between 0.0 and 5.0) in 'Sampling temperature' Slider component		
+    512, // number (numeric value between 8 and 1024) in 'Maximum number of new tokens to generate' Slider component		
+    1, // number (numeric value between 0.0 and 5.0) in 'Repetition penalty' Slider component		
+    0.8, // number (numeric value between 0.01 and 0.99) in 'Top P' Slider component
+  ])
+
+  const rawResponse = result as any 
+
+  console.log("rawResponse:", rawResponse)
+  
+  return rawResponse?.data?.[0] as string
+}

src/index.mts

@@ -4,7 +4,7 @@ import path from "node:path"
 import { validate as uuidValidate } from "uuid"
 import express from "express"
 
-import { Video, VideoStatus, VideoAPIRequest, RenderRequest, RenderedScene } from "./types.mts"
+import { Video, VideoStatus, VideoAPIRequest, RenderRequest, RenderedScene, ImageAnalysisRequest, ImageAnalysisResponse } from "./types.mts"
 
 import { parseVideoRequest } from "./utils/parseVideoRequest.mts"
 import { savePendingVideo } from "./scheduler/savePendingVideo.mts"
@@ -23,6 +23,7 @@ import { sortVideosByYoungestFirst } from "./utils/sortVideosByYoungestFirst.mts
 import { getRenderedScene, renderScene } from "./production/renderScene.mts"
 import { parseRenderRequest } from "./utils/parseRenderRequest.mts"
 import { loadRenderedSceneFromCache } from "./utils/loadRenderedSceneFromCache.mts"
+import { analyzeImage } from "./analysis/analyzeImage.mts"
 
 initFolders()
 // to disable all processing (eg. to debug)
@@ -36,6 +37,56 @@ app.use(express.json())
 
 let isRendering = false
 
+// an image analyzing pipeline
+app.post("/analyze", async (req, res) => {
+
+  console.log(req.body)
+
+  const request = req.body as ImageAnalysisRequest
+
+  if (!request.prompt) {
+    console.log("Invalid prompt")
+    res.status(400)
+    res.write(JSON.stringify({ result: "", error: "invalid prompt" }))
+    res.end()
+    return
+  }
+
+  if (!request.image) {
+    console.log("Invalid image")
+    res.status(400)
+    res.write(JSON.stringify({ result: "", error: "invalid image" }))
+    res.end()
+    return
+  }
+
+  const response: ImageAnalysisResponse = {
+    result: "",
+    error: ""
+  }
+
+  try {
+    response.result = await analyzeImage(request.image, request.prompt)
+  } catch (err) {
+    // console.log("failed to render scene!")
+    response.error = `failed to render scene: ${err}`
+  }
+
+  if (response.error.length > 0) {
+    // console.log("server error")
+    res.status(500)
+    res.write(JSON.stringify(response))
+    res.end()
+    return
+  } else {
+    // console.log("all good")
+    res.status(200)
+    res.write(JSON.stringify(response))
+    res.end()
+    return
+  }
+})
+
 // a "fast track" pipeline
 app.post("/render", async (req, res) => {
 
@@ -227,7 +278,6 @@ app.post("/:ownerId", async (req, res) => {
   }
 })
 
-
 app.get("/:ownerId/:videoId\.mp4", async (req, res) => {
     
   /*

src/types.mts

@@ -310,6 +310,17 @@ export interface RenderRequest {
   wait: boolean // wait until the job is completed
 }
 
+export interface ImageAnalysisRequest {
+  image: string // in base64
+  prompt: string
+}
+
+export interface ImageAnalysisResponse {
+  result: string
+  error?: string
+}
+
+
 export interface ImageSegmentationRequest {
   image: string // in base64
   keywords: string[]