added support for parallelism

src/main.mts

@@ -7,9 +7,10 @@ export const main = async () => {
 
   const videos = await getPendingVideos()
   if (!videos.length) {
+    console.log(`no job to process.. going to try in 200 ms`)
     setTimeout(() => {
       main()
-    }, 500)
+    }, 200)
     return
   }
 
@@ -17,12 +18,24 @@ export const main = async () => {
 
   sortPendingVideosByLeastCompletedFirst(videos)
 
-  for (const video of videos) {
-    await processVideo(video)
+  let somethingFailed = ""
+  await Promise.all(videos.map(async video => {
+    try {
+      const result = await processVideo(video)
+      return result
+    } catch (err) {
+      somethingFailed = `${err}`
+      // a video failed.. no big deal
+      return Promise.resolve(somethingFailed)
+    }
+  }))
+
+  if (somethingFailed) {
+    console.error(`one of the jobs failed: ${somethingFailed}, let's wait 3 seconds`)
+    setTimeout(() => { main() }, 3000)
+  } else {
+    console.log(`successfully worked on the jobs, let's immediately loop`)
+    setTimeout(() => { main() }, 50)
   }
-  console.log(`processed ${videos.length} videos`)
 
-  setTimeout(() => {
-    main()
-  }, 1000)
 }

src/production/generateAudio.mts

@@ -1,62 +1,76 @@
-import path from "node:path"
-
 import { v4 as uuidv4 } from "uuid"
-import tmpDir from "temp-dir"
 import puppeteer from "puppeteer"
 
 import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
 import { moveFileFromTmpToPending } from "../utils/moveFileFromTmpToPending.mts"
 
+export const state = {
+  load: 0,
+}
+
 const instances: string[] = [
   process.env.VC_AUDIO_GENERATION_SPACE_API_URL
 ]
 
 // TODO we should use an inference endpoint instead
 export async function generateAudio(prompt: string, audioFileName: string) {
-  const instance = instances.shift()
-  instances.push(instance)
 
-  const browser = await puppeteer.launch({
-    headless: true,
-    protocolTimeout: 120000,
-  })
+  if (state.load === instances.length) {
+    throw new Error(`all audio generation servers are busy, try again later..`)
+  }
+
+  state.load += 1
 
   try {
-    const page = await browser.newPage()
+    const instance = instances.shift()
+    instances.push(instance)
 
-    await page.goto(instance, {
-      waitUntil: "networkidle2",
+    const browser = await puppeteer.launch({
+      headless: true,
+      protocolTimeout: 120000,
     })
 
-    await new Promise(r => setTimeout(r, 3000))
+    try {
+      const page = await browser.newPage()
 
-    const firstTextboxInput = await page.$('input[data-testid="textbox"]')
+      await page.goto(instance, {
+        waitUntil: "networkidle2",
+      })
 
-    await firstTextboxInput.type(prompt)
+      await new Promise(r => setTimeout(r, 3000))
 
-    // console.log("looking for the button to submit")
-    const submitButton = await page.$("button.lg")
+      const firstTextboxInput = await page.$('input[data-testid="textbox"]')
 
-    // console.log("clicking on the button")
-    await submitButton.click()
+      await firstTextboxInput.type(prompt)
 
-    await page.waitForSelector("a[download]", {
-      timeout: 120000, // no need to wait for too long, generation is quick
-    })
+      // console.log("looking for the button to submit")
+      const submitButton = await page.$("button.lg")
+
+      // console.log("clicking on the button")
+      await submitButton.click()
 
-    const audioRemoteUrl = await page.$$eval("a[download]", el => el.map(x => x.getAttribute("href"))[0])
+      await page.waitForSelector("a[download]", {
+        timeout: 120000, // no need to wait for too long, generation is quick
+      })
 
+      const audioRemoteUrl = await page.$$eval("a[download]", el => el.map(x => x.getAttribute("href"))[0])
 
-    // it is always a good idea to download to a tmp dir before saving to the pending dir
-    // because there is always a risk that the download will fail
-    
-    const tmpFileName = `${uuidv4()}.mp4`
 
-    await downloadFileToTmp(audioRemoteUrl, tmpFileName)
-    await moveFileFromTmpToPending(tmpFileName, audioFileName)
+      // it is always a good idea to download to a tmp dir before saving to the pending dir
+      // because there is always a risk that the download will fail
+      
+      const tmpFileName = `${uuidv4()}.mp4`
+
+      await downloadFileToTmp(audioRemoteUrl, tmpFileName)
+      await moveFileFromTmpToPending(tmpFileName, audioFileName)
+    } catch (err) {
+      throw err
+    } finally {
+      await browser.close()
+    }
   } catch (err) {
     throw err
   } finally {
-    await browser.close()
+    state.load -= 1
   }
-}
+}

src/production/generateAudioLegacy.mts

@@ -2,6 +2,10 @@ import { client } from '@gradio/client'
 
 import { generateSeed } from "../utils/generateSeed.mts"
 
+export const state = {
+  load: 0
+}
+
 const instances: string[] = [
   process.env.VC_AUDIO_GENERATION_SPACE_API_URL
 ]
@@ -11,25 +15,38 @@ export const generateAudio = async (prompt: string, options?: {
   nbFrames: number;
   nbSteps: number;
 }) => {
-  const seed = options?.seed || generateSeed()
-  const nbFrames = options?.nbFrames || 24 // we can go up to 48 frames, but then upscaling quill require too much memory!
-  const nbSteps = options?.nbSteps || 35
-
-  const instance = instances.shift()
-  instances.push(instance)
-
-  const api = await client(instance, {
-        hf_token: `${process.env.VC_HF_API_TOKEN}` as any
-  })
-
-  const rawResponse = await api.predict('/run', [		
-    prompt, // string  in 'Prompt' Textbox component		
-    seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component		
-    nbFrames, // 24 // it is the nb of frames per seconds I think?
-    nbSteps, // 10, (numeric value between 10 and 50) in 'Number of inference steps' Slider component
-  ]) as any
-  
-  const { name } = rawResponse?.data?.[0]?.[0] as { name: string, orig_name: string }
-
-  return `${instance}/file=${name}`
-}
+
+  if (state.load === instances.length) {
+    throw new Error(`all audio generation servers are busy, try again later..`)
+  }
+
+  state.load += 1
+
+  try {
+    const seed = options?.seed || generateSeed()
+    const nbFrames = options?.nbFrames || 24 // we can go up to 48 frames, but then upscaling quill require too much memory!
+    const nbSteps = options?.nbSteps || 35
+
+    const instance = instances.shift()
+    instances.push(instance)
+
+    const api = await client(instance, {
+          hf_token: `${process.env.VC_HF_API_TOKEN}` as any
+    })
+
+    const rawResponse = await api.predict('/run', [		
+      prompt, // string  in 'Prompt' Textbox component		
+      seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component		
+      nbFrames, // 24 // it is the nb of frames per seconds I think?
+      nbSteps, // 10, (numeric value between 10 and 50) in 'Number of inference steps' Slider component
+    ]) as any
+    
+    const { name } = rawResponse?.data?.[0]?.[0] as { name: string, orig_name: string }
+
+    return `${instance}/file=${name}`
+  } catch (err) {
+    throw err
+  } finally {
+    state.load -= 1
+  }
+}

src/production/generateVideo.mts

@@ -2,6 +2,10 @@ import { client } from "@gradio/client"
 
 import { generateSeed } from "../utils/generateSeed.mts"
 
+export const state = {
+  load: 0,
+}
+
 // we don't use replicas yet, because it ain't easy to get their hostname
 const instances: string[] = [
   `${process.env.VC_ZEROSCOPE_SPACE_API_URL_1 || ""}`,
@@ -14,27 +18,40 @@ export const generateVideo = async (prompt: string, options?: {
   nbFrames: number;
   nbSteps: number;
 }) => {
-  const seed = options?.seed || generateSeed()
-  const nbFrames = options?.nbFrames || 24 // we can go up to 48 frames, but then upscaling quill require too much memory!
-  const nbSteps = options?.nbSteps || 35
-
-  const instance = instances.shift()
-  instances.push(instance)
-
-  const api = await client(instance, {
-    hf_token: `${process.env.VC_HF_API_TOKEN}` as any
-  })
-
-  const rawResponse = await api.predict('/run', [		
-    prompt, // string  in 'Prompt' Textbox component		
-    seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component		
-    nbFrames, // 24 // it is the nb of frames per seconds I think?
-    nbSteps, // 10, (numeric value between 10 and 50) in 'Number of inference steps' Slider component
-  ]) as any
-  
-  // console.log("rawResponse:", rawResponse)
-
-  const { name } = rawResponse?.data?.[0]?.[0] as { name: string, orig_name: string }
 
-  return `${instance}/file=${name}`
+  if (state.load === instances.length) {
+    throw new Error(`all video generation servers are busy, try again later..`)
+  }
+
+  state.load += 1
+
+  try {
+    const seed = options?.seed || generateSeed()
+    const nbFrames = options?.nbFrames || 24 // we can go up to 48 frames, but then upscaling quill require too much memory!
+    const nbSteps = options?.nbSteps || 35
+
+    const instance = instances.shift()
+    instances.push(instance)
+
+    const api = await client(instance, {
+      hf_token: `${process.env.VC_HF_API_TOKEN}` as any
+    })
+
+    const rawResponse = await api.predict('/run', [		
+      prompt, // string  in 'Prompt' Textbox component		
+      seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component		
+      nbFrames, // 24 // it is the nb of frames per seconds I think?
+      nbSteps, // 10, (numeric value between 10 and 50) in 'Number of inference steps' Slider component
+    ]) as any
+    
+    // console.log("rawResponse:", rawResponse)
+
+    const { name } = rawResponse?.data?.[0]?.[0] as { name: string, orig_name: string }
+
+    return `${instance}/file=${name}`
+  } catch (err) {
+    throw err
+  } finally {
+    state.load -= 1
+  }
 }

src/production/generateVoice.mts

@@ -2,61 +2,77 @@ import puppeteer from "puppeteer"
 
 import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
 
+export const state = {
+  load: 0
+}
+
 const instances: string[] = [
   process.env.VC_VOICE_GENERATION_SPACE_API_URL
 ]
 
 // TODO we should use an inference endpoint instead
 export async function generateVoice(prompt: string, voiceFileName: string) {
-  const instance = instances.shift()
-  instances.push(instance)
+  if (state.load === instances.length) {
+    throw new Error(`all voice generation servers are busy, try again later..`)
+  }
 
-  console.log("instance:", instance)
-  
-  const browser = await puppeteer.launch({
-    headless: true,
-    protocolTimeout: 800000,
-  })
+  state.load += 1
 
   try {
-    const page = await browser.newPage()
-
-    await page.goto(instance, {
-      waitUntil: "networkidle2",
+    const instance = instances.shift()
+    instances.push(instance)
+
+    console.log("instance:", instance)
+    
+    const browser = await puppeteer.launch({
+      headless: true,
+      protocolTimeout: 800000,
     })
 
-    await new Promise(r => setTimeout(r, 3000))
+    try {
+      const page = await browser.newPage()
 
-    const firstTextarea = await page.$('textarea[data-testid="textbox"]')
+      await page.goto(instance, {
+        waitUntil: "networkidle2",
+      })
 
-    await firstTextarea.type(prompt)
+      await new Promise(r => setTimeout(r, 3000))
 
-    // console.log("looking for the button to submit")
-    const submitButton = await page.$("button.lg")
+      const firstTextarea = await page.$('textarea[data-testid="textbox"]')
 
-    // console.log("clicking on the button")
-    await submitButton.click()
+      await firstTextarea.type(prompt)
 
-    await page.waitForSelector("audio", {
-      timeout: 800000, // need to be large enough in case someone else attemps to use our space
-    })
+      // console.log("looking for the button to submit")
+      const submitButton = await page.$("button.lg")
 
-    const voiceRemoteUrl = await page.$$eval("audio", el => el.map(x => x.getAttribute("src"))[0])
+      // console.log("clicking on the button")
+      await submitButton.click()
 
+      await page.waitForSelector("audio", {
+        timeout: 800000, // need to be large enough in case someone else attemps to use our space
+      })
 
-    console.log({
-      voiceRemoteUrl,
-    })
+      const voiceRemoteUrl = await page.$$eval("audio", el => el.map(x => x.getAttribute("src"))[0])
+
+
+      console.log({
+        voiceRemoteUrl,
+      })
 
 
-    console.log(`- downloading ${voiceFileName} from ${voiceRemoteUrl}`)
+      console.log(`- downloading ${voiceFileName} from ${voiceRemoteUrl}`)
 
-    await downloadFileToTmp(voiceRemoteUrl, voiceFileName)
+      await downloadFileToTmp(voiceRemoteUrl, voiceFileName)
 
-    return voiceFileName
+      return voiceFileName
+    } catch (err) {
+      throw err
+    } finally {
+      await browser.close()
+    }
   } catch (err) {
     throw err
   } finally {
-    await browser.close()
+    state.load -= 1
   }
 }

src/production/interpolateVideo.mts

@@ -1,66 +1,81 @@
 import path from "node:path"
 
 import { v4 as uuidv4 } from "uuid"
-import tmpDir from "temp-dir"
 import puppeteer from "puppeteer"
 
 import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
 import { pendingFilesDirFilePath } from "../config.mts"
 import { moveFileFromTmpToPending } from "../utils/moveFileFromTmpToPending.mts"
 
+export const state = {
+  load: 0
+}
+
 const instances: string[] = [
   process.env.VC_VIDEO_INTERPOLATION_SPACE_API_URL
 ]
 
 // TODO we should use an inference endpoint instead
 export async function interpolateVideo(fileName: string, steps: number, fps: number) {
-  const inputFilePath = path.join(pendingFilesDirFilePath, fileName)
+  if (state.load === instances.length) {
+    throw new Error(`all video interpolation servers are busy, try again later..`)
+  }
 
-  console.log(`interpolating ${fileName}`)
-  console.log(`warning: interpolateVideo parameter "${steps}" is ignored!`)
-  console.log(`warning: interpolateVideo parameter "${fps}" is ignored!`)
+  state.load += 1
 
-  const instance = instances.shift()
-  instances.push(instance)
+  try {
+    const inputFilePath = path.join(pendingFilesDirFilePath, fileName)
 
-  const browser = await puppeteer.launch({
-    headless: true,
-    protocolTimeout: 400000,
-  })
+    console.log(`interpolating ${fileName}`)
+    console.log(`warning: interpolateVideo parameter "${steps}" is ignored!`)
+    console.log(`warning: interpolateVideo parameter "${fps}" is ignored!`)
 
-  try {
-    const page = await browser.newPage()
-    await page.goto(instance, { waitUntil: 'networkidle2' })
-    
-    await new Promise(r => setTimeout(r, 3000))
+    const instance = instances.shift()
+    instances.push(instance)
 
-    const fileField = await page.$('input[type=file]')
+    const browser = await puppeteer.launch({
+      headless: true,
+      protocolTimeout: 400000,
+    })
 
-    // console.log(`uploading file..`)
-    await fileField.uploadFile(inputFilePath)
+    try {
+      const page = await browser.newPage()
+      await page.goto(instance, { waitUntil: 'networkidle2' })
+      
+      await new Promise(r => setTimeout(r, 3000))
 
-    // console.log('looking for the button to submit')
-    const submitButton = await page.$('button.lg')
+      const fileField = await page.$('input[type=file]')
 
-    // console.log('clicking on the button')
-    await submitButton.click()
+      // console.log(`uploading file..`)
+      await fileField.uploadFile(inputFilePath)
 
-    await page.waitForSelector('a[download="interpolated_result.mp4"]', {
-      timeout: 400000, // need to be large enough in case someone else attemps to use our space
-    })
+      // console.log('looking for the button to submit')
+      const submitButton = await page.$('button.lg')
+
+      // console.log('clicking on the button')
+      await submitButton.click()
+
+      await page.waitForSelector('a[download="interpolated_result.mp4"]', {
+        timeout: 400000, // need to be large enough in case someone else attemps to use our space
+      })
 
-    const interpolatedFileUrl = await page.$$eval('a[download="interpolated_result.mp4"]', el => el.map(x => x.getAttribute("href"))[0])
+      const interpolatedFileUrl = await page.$$eval('a[download="interpolated_result.mp4"]', el => el.map(x => x.getAttribute("href"))[0])
 
-    // it is always a good idea to download to a tmp dir before saving to the pending dir
-    // because there is always a risk that the download will fail
+      // it is always a good idea to download to a tmp dir before saving to the pending dir
+      // because there is always a risk that the download will fail
 
-    const tmpFileName = `${uuidv4()}.mp4`
+      const tmpFileName = `${uuidv4()}.mp4`
 
-    await downloadFileToTmp(interpolatedFileUrl, tmpFileName)
-    await moveFileFromTmpToPending(tmpFileName, fileName)
+      await downloadFileToTmp(interpolatedFileUrl, tmpFileName)
+      await moveFileFromTmpToPending(tmpFileName, fileName)
+    } catch (err) {
+      throw err
+    } finally {
+      await browser.close()
+    }
   } catch (err) {
     throw err
   } finally {
-    await browser.close()
+    state.load -= 1
   }
 }

src/production/interpolateVideoLegacy.mts

@@ -7,35 +7,50 @@ import tmpDir from "temp-dir"
 
 import { downloadFileToTmp } from '../utils/downloadFileToTmp.mts'
 
+export const state = {
+  load: 0
+}
+
 const instances: string[] = [
   process.env.VC_VIDEO_INTERPOLATION_SPACE_API_URL
 ]
 
 export const interpolateVideo = async (fileName: string, steps: number, fps: number) => {
-
-  const inputFilePath = path.join(tmpDir, fileName)
-
-  const instance = instances.shift()
-  instances.push(instance)
-
-  const api = await client(instance, {
-    hf_token: `${process.env.VC_HF_API_TOKEN}` as any
-  })
-
-  const video = await fs.readFile(inputFilePath)
-
-  const blob = new Blob([video], { type: 'video/mp4' })
-  // const blob = blobFrom(filePath)
-  const result = await api.predict(1, [
-    blob, 	// blob in 'parameter_5' Video component		
-    steps, // number (numeric value between 1 and 4) in 'Interpolation Steps' Slider component		
-    fps, // string (FALSE! it's a number)  in 'FPS output' Radio component
-  ])
-
-  const data = (result as any).data[0]
-  console.log('raw data:', data)
-  const { orig_name, data: remoteFilePath } = data
-  const remoteUrl = `${instance}/file=${remoteFilePath}`
-  console.log("remoteUrl:", remoteUrl)
-  await downloadFileToTmp(remoteUrl, fileName)
+  if (state.load === instances.length) {
+    throw new Error(`all video interpolation servers are busy, try again later..`)
+  }
+
+  state.load += 1
+
+  try {
+    const inputFilePath = path.join(tmpDir, fileName)
+
+    const instance = instances.shift()
+    instances.push(instance)
+
+    const api = await client(instance, {
+      hf_token: `${process.env.VC_HF_API_TOKEN}` as any
+    })
+
+    const video = await fs.readFile(inputFilePath)
+
+    const blob = new Blob([video], { type: 'video/mp4' })
+    // const blob = blobFrom(filePath)
+    const result = await api.predict(1, [
+      blob, 	// blob in 'parameter_5' Video component		
+      steps, // number (numeric value between 1 and 4) in 'Interpolation Steps' Slider component		
+      fps, // string (FALSE! it's a number)  in 'FPS output' Radio component
+    ])
+
+    const data = (result as any).data[0]
+    console.log('raw data:', data)
+    const { orig_name, data: remoteFilePath } = data
+    const remoteUrl = `${instance}/file=${remoteFilePath}`
+    console.log("remoteUrl:", remoteUrl)
+    await downloadFileToTmp(remoteUrl, fileName)
+  } catch (err) {
+    throw err
+  } finally {
+    state.load -= 1
+  }
 }

src/production/renderStaticScene.mts

@@ -3,11 +3,14 @@ import path from "node:path"
 import { v4 as uuidv4 } from "uuid"
 import tmpDir from "temp-dir"
 
-import { ImageSegment, RenderedScene, RenderRequest } from "../types.mts"
+import { ImageSegment, RenderedScene, RenderingJob, RenderRequest } from "../types.mts"
 import { segmentImage } from "../utils/segmentImage.mts"
 import { generateImageSDXLAsBase64 } from "../utils/generateImageSDXL.mts"
 import { writeBase64ToFile } from "../utils/writeBase64ToFile.mts"
 
+
+const pendingJobs: RenderingJob[] = []
+
 export async function renderStaticScene(scene: RenderRequest): Promise<RenderedScene> {
 
   let imageBase64 = ""

src/types.mts

@@ -312,4 +312,17 @@ export interface RenderedScene {
   error: string
   maskBase64: string
   segments: ImageSegment[]
+}
+
+// note: for video generation we are always going to have slow jobs,
+// because we need multiple seconds, minutes, hours.. of video + audio
+// but for rendering we aim at shorter delays, less than 45 seconds
+// so the goal of rendering "jobs" is mostly to give the illusion that
+// things go faster, by already providing some things like the background image,
+// before we send 
+export interface RenderingJob {
+  scene: RenderRequest
+  result: RenderedScene
+
+  status: 'pending' | 'completed' | 'error'
 }