various improvements

.env

@@ -1,3 +1,3 @@
 NEXT_PUBLIC_BASE_URL=https://jbilcke-hf-fishtank.hf.space
 # NEXT_PUBLIC_RENDERING_ENGINE_API=https://hysts-zeroscope-v2.hf.space
-RENDERING_ENGINE_API=https://jbilcke-hf-videochain-api.hf.space
+RENDERING_ENGINE_API=https://jbilcke-hf-videochain-api.hf.space

src/app/agents/ant.ts

@@ -18,23 +18,33 @@ const positions = [
   "on the ground"
 ]
 
-const times = [
+const lights = [
   "during the day",
 ]
 
+const actionnables = [
+  "ant",
+  "ground",
+  "leaf",
+  "rock",
+  "wood",
+  "water",
+  "beetle"
+]
+
 export const agent: Agent = {
   title: "Ant",
   type: "ant",
   simulate: (): Scene => {
     const action = pick(actions)
     const position = pick(positions)
-    const time = pick(times)
+    const light = pick(lights)
 
     const prompt = [
       `close-up shot of a couple of ants`,
       action,
       position,
-      time,
+      light,
       `high res`,
       `documentary`,
     ].join(", ")
@@ -42,7 +52,8 @@ export const agent: Agent = {
     return {
       action,
       position,
-      time,
+      light,
+      actionnables,
       prompt
     }
   }

src/app/agents/city.ts

@@ -20,24 +20,34 @@ const positions = [
   "theme park with one big rollercoaster"
 ]
 
-const times = [
+const lights = [
   "during the day",
   // "during the night",
 ]
 
+const actionnables = [
+  "building",
+  "road",
+  "car",
+  "tower",
+  "tree",
+  "river",
+  "sea"
+]
+
 export const agent: Agent = {
   title: "City",
   type: "city",
   simulate: (): Scene => {
     const action = pick(actions)
     const position = pick(positions)
-    const time = pick(times)
+    const light = pick(lights)
 
     const prompt = [
       `static isometrical view of 3D rendered city`,
       action,
       position,
-      time,
+      light,
       `isometric`,
       `game`,
       `high res`,
@@ -46,7 +56,8 @@ export const agent: Agent = {
     return {
       action,
       position,
-      time,
+      light,
+      actionnables,
       prompt
     }
   }

src/app/agents/dungeon.ts

@@ -22,25 +22,35 @@ const positions = [
   "a damp medieval jail cell with stone walls and wooden floor"
 ]
 
-const times = [
+const lights = [
   "lit through windows",
   "lit through wall-mounted torchs"
   // "poorly lit"
 ]
 
+const actionnables = [
+  "floor",
+  "fireplace",
+  "door",
+  "window",
+  "chair",
+  "table",
+  "torch"
+]
+
 export const agent: Agent = {
   title: "Dungeon",
   type: "dungeon",
   simulate: (): Scene => {
     const action = pick(actions)
     const position = pick(positions)
-    const time = pick(times)
+    const light = pick(lights)
 
     const prompt = [
       `first-person footage`,
       action,
       position,
-      time,
+      light,
       `medieval`,
       `photography`,
       `documentary`,
@@ -50,7 +60,8 @@ export const agent: Agent = {
     return {
       action,
       position,
-      time,
+      light,
+      actionnables,
       prompt
     }
   }

src/app/agents/fish.ts

@@ -21,23 +21,29 @@ const positions = [
   "hiding in the coral"
 ]
 
-const times = [
+const lights = [
   "during the day",
 ]
 
+const actionnables = [
+  "fish",
+  "coral",
+  "sand"
+]
+
 export const agent: Agent = {
   title: "Fish",
   type: "fish",
   simulate: (): Scene => {
     const action = pick(actions)
     const position = pick(positions)
-    const time = pick(times)
+    const light = pick(lights)
 
     const prompt = [
       `medium shot of a clownfish`,
       action,
       position,
-      time,
+      light,
       `in front of yellow coral`,
       `high res`,
       `underwater footage`,
@@ -46,7 +52,8 @@ export const agent: Agent = {
     return {
       action,
       position,
-      time,
+      light,
+      actionnables,
       prompt
     }
   }

src/app/agents/fox.ts

@@ -18,23 +18,32 @@ const positions = [
   "in front of a bush"
 ]
 
-const times = [
+const lights = [
   "during the day",
 ]
 
+const actionnables = [
+  "fox",
+  "ground",
+  "grass",
+  "water",
+  "tree",
+  "sky",
+]
+
 export const agent: Agent = {
   title: "Fox",
   type: "fox",
   simulate: (): Scene => {
     const action = pick(actions)
     const position = pick(positions)
-    const time = pick(times)
+    const light = pick(lights)
 
     const prompt = [
       `medium shot of a fox`,
       action,
       position,
-      time,
+      light,
       `high res`,
       `documentary`,
     ].join(", ")
@@ -42,7 +51,8 @@ export const agent: Agent = {
     return {
       action,
       position,
-      time,
+      light,
+      actionnables,
       prompt
     }
   }

src/app/agents/smith.ts

@@ -17,25 +17,33 @@ const positions = [
   "on the sidewalk of a street"
 ]
 
-const times = [
+const lights = [
   "during the day",
   "during the night",
 ]
 
+const actionnables = [
+  "face",
+  "costume",
+  "street",
+  "book",
+  "building",
+]
+
 export const agent: Agent = {
   title: "Smith",
   type: "smith",
   simulate: (): Scene => {
     const action = pick(actions)
     const position = pick(positions)
-    const time = pick(times)
+    const light = pick(lights)
 
     const prompt = [
       `static medium shot of Agent Smith from the Matrix`,
       `wearing a black costume with black tie and black sunglasses`,
       action,
       position,
-      time,
+      light,
       `high res`,
       `Matrix movie`,
     ].join(", ")
@@ -43,7 +51,8 @@ export const agent: Agent = {
     return {
       action,
       position,
-      time,
+      light,
+      actionnables,
       prompt
     }
   }

src/app/agents/types.ts

@@ -3,7 +3,8 @@ export type AgentType = 'ant' | 'fish' | 'fox' | 'smith' | 'city' | 'dungeon'
 export interface Scene {
   action: string
   position: string
-  time: string
+  light: string
+  actionnables: string[]
   prompt: string
 }
 

src/app/main.tsx

@@ -15,11 +15,13 @@ import {
 import { render } from "./render"
 import { Agent, AgentType, Scene } from "./agents/types"
 import { agents, defaultAgent, getAgent } from "./agents"
+import { ImageSegment } from "./types"
 
 export default function Main() {
-  const [url, setUrl] = useState<string>()
+  const [url, setUrl] = useState<string>("")
   const [isPending, startTransition] = useTransition()
   const [scene, setScene] = useState<Scene>()
+  const [segments, setSegments] = useState<ImageSegment[]>([])
   const ref = useRef<AgentType>(defaultAgent)
    
   useEffect(() => {
@@ -37,7 +39,7 @@ export default function Main() {
         const scene = agent.simulate()
 
         // console.log(`rendering scene..`)
-        const newUrl = await render(scene.prompt)
+        const rendered = await render(scene.prompt, scene.actionnables)
 
         if (type !== ref?.current) {
           console.log("agent type changed! reloading scene")
@@ -45,14 +47,16 @@ export default function Main() {
           return
         } 
 
-        if (newUrl) {
+
+        if (rendered.videoUrl) {
           // console.log(`got a new url: ${newUrl}`)
-          setUrl(newUrl)
+          setUrl(rendered.videoUrl)
           setScene(scene)
+          setSegments(rendered.segments)
           setTimeout(() => { updateView()}, 1000)
         } else {
           // console.log(`going to wait a bit more: ${newUrl}`)
-          setTimeout(() => { updateView()}, 3000)
+          setTimeout(() => { updateView()}, rendered.error ? 6000 : 3000)
         }
       })
     }
@@ -70,7 +74,7 @@ export default function Main() {
             defaultValue={defaultAgent}
             onValueChange={(value) => {
               ref.current = value as AgentType
-              // setUrl("")
+              setUrl("")
             }}>
             <SelectTrigger className="w-[180px]">
               <SelectValue placeholder="Type" />
@@ -82,10 +86,19 @@ export default function Main() {
             </SelectContent>
           </Select>
         </div>
+        <p>Note: changing the model might take up to 1 minute</p>
+          
         {(scene) ? <div>
           <p>Action: {scene.action}</p>
           <p>Position: {scene.position}</p>
+          <p>Light: {scene.light}</p>
         </div> : null}
+        <div className="flex flex-col">
+        {segments.map((segment, i) => 
+          <div key={i}>
+            {segment.label} ({segment.score})
+          </div>)}
+        </div>
       </div>
       <VideoPlayer url={url} />
     </div>

src/app/render.ts

@@ -1,10 +1,19 @@
 "use server"
 
+import { RenderAPIResponse } from "./types"
+
 // note: there is no / at the end in the variable
 // so we have to add it ourselves if needed
 const apiUrl = process.env.RENDERING_ENGINE_API
 
-export async function render(prompt: string) {
+export async function render(prompt: string, actionnables: string[] = []) {
+  let defaulResult: RenderAPIResponse = {
+    videoUrl: "",
+    maskBase64: "",
+    error: "",
+    segments: []
+  }
+
   try {
     console.log(`calling ${apiUrl}/render with prompt: ${prompt}`)
     const res = await fetch(`${apiUrl}/render`, {
@@ -14,12 +23,20 @@ export async function render(prompt: string) {
         "Content-Type": "application/json",
         // Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`,
       },
-      body: JSON.stringify({ prompt }),
+      body: JSON.stringify({
+        prompt,
+        // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation
+        nbFrames: 8,
+        nbSteps: 20,
+        segmentation: "firstframe",
+        actionnables,
+      }),
        cache: 'no-store',
      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
      // next: { revalidate: 1 }
     })
 
+    // console.log("res:", res)
     // The return value is *not* serialized
     // You can return Date, Map, Set, etc.
     
@@ -29,11 +46,11 @@ export async function render(prompt: string) {
       throw new Error('Failed to fetch data')
     }
     
-    const { url } = await res.json()
-
-    return url
+    const response = (await res.json()) as RenderAPIResponse
+    // console.log("response:", response)
+    return response
   } catch (err) {
     console.error(err)
-    return ""
+    return defaulResult
   }
 }

src/app/types.ts

@@ -31,6 +31,17 @@ export interface ImageSegment {
   score: number 
 }
 
-export interface RenderResponse {
-  
+
+export interface ImageSegment {
+  id: number
+  box: number[]
+  label: string
+  score: number 
+}
+
+export interface RenderAPIResponse {
+  videoUrl: string 
+  error: string
+  maskBase64: string
+  segments: ImageSegment[]
 }

src/components/business/video-player.tsx

@@ -4,7 +4,7 @@ export const VideoPlayer = ({ url }: { url?: string }) => {
 
   if (!url) {
     return <div className="flex w-full h-screen items-center justify-center text-center">
-      <div>Generating first frames.. (might take around 30s)</div>
+      <div>Rendering first scene (might take ~1 min)</div>
     </div>
   }