preparing the code for interactivityg

src/app/main.tsx

@@ -2,7 +2,7 @@
 
 import { useEffect, useRef, useState, useTransition } from "react"
 
-import { VideoPlayer } from "@/components/business/video-player"
+import { VideoRenderer } from "@/components/business/video-renderer"
 
 import {
   Select,
@@ -44,7 +44,7 @@ export default function Main() {
 
           []
           // note: using actionnables will add +30sec to the query
-          // scene.actionnables
+          // scene.actionnables.slice(0, 5) // too many can slow us down it seems
         )
 
         if (type !== ref?.current) {
@@ -54,9 +54,9 @@ export default function Main() {
         } 
 
 
-        if (rendered.videoUrl) {
+        if (rendered.assetUrl) {
           // console.log(`got a new url: ${newUrl}`)
-          setUrl(rendered.videoUrl)
+          setUrl(rendered.assetUrl)
           setScene(scene)
           setSegments(rendered.segments)
           setTimeout(() => { updateView()}, 1000)
@@ -106,7 +106,7 @@ export default function Main() {
           </div>)}
         </div>
       </div>
-      <VideoPlayer url={url} />
+      <VideoRenderer url={url} />
     </div>
   )
 }

src/app/render.ts

@@ -1,14 +1,14 @@
 "use server"
 
-import { RenderAPIResponse } from "./types"
+import { RenderedScene } from "./types"
 
 // note: there is no / at the end in the variable
 // so we have to add it ourselves if needed
 const apiUrl = process.env.RENDERING_ENGINE_API
 
 export async function render(prompt: string, actionnables: string[] = []) {
-  let defaulResult: RenderAPIResponse = {
-    videoUrl: "",
+  let defaulResult: RenderedScene = {
+    assetUrl: "",
     maskBase64: "",
     error: "",
     segments: []
@@ -28,8 +28,8 @@ export async function render(prompt: string, actionnables: string[] = []) {
         // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation
         nbFrames: 8,
         nbSteps: 20,
-        segmentation: "firstframe",
         actionnables,
+        segmentation: "firstframe", // one day we will remove this param, to make it automatic
       }),
        cache: 'no-store',
      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
@@ -46,7 +46,7 @@ export async function render(prompt: string, actionnables: string[] = []) {
       throw new Error('Failed to fetch data')
     }
     
-    const response = (await res.json()) as RenderAPIResponse
+    const response = (await res.json()) as RenderedScene
     // console.log("response:", response)
     return response
   } catch (err) {

src/app/types.ts

@@ -27,20 +27,13 @@ export interface RenderRequest {
 export interface ImageSegment {
   id: number
   box: number[]
+  color: number[]
   label: string
   score: number 
 }
 
-
-export interface ImageSegment {
-  id: number
-  box: number[]
-  label: string
-  score: number 
-}
-
-export interface RenderAPIResponse {
-  videoUrl: string 
+export interface RenderedScene {
+  assetUrl: string 
   error: string
   maskBase64: string
   segments: ImageSegment[]

src/components/business/image-renderer.tsx

@@ -0,0 +1,88 @@
+import { useRef } from "react"
+
+import { ImageSegment, RenderedScene } from "@/app/types"
+
+export const ImageRenderer = ({
+  assetUrl = "",
+  maskBase64 = "",
+  segments = []
+}: RenderedScene) => {
+  const imgRef = useRef<HTMLImageElement | null>(null)
+  const canvas = document.createElement('canvas')
+  const context = canvas.getContext('2d')
+  
+  const getPixelColor = (x: number, y: number) => {
+    console.log("getting pixel color")
+    if (!context) {
+      throw new Error("Unable to get context from canvas")
+    }
+
+    const imgData = context.getImageData(x, y, 1, 1).data
+
+    return `[${imgData[0]},${imgData[1]},${imgData[2]},${imgData[3]/255}]`
+  }
+
+  const handleMouseDown = (event: React.MouseEvent) => {
+    const boundingRect = imgRef.current!.getBoundingClientRect()
+  
+    const x = event.clientX - boundingRect.left
+    const y = event.clientY - boundingRect.top
+  
+    if (maskBase64) {
+      const image = new Image()
+
+      image.onload = function () {
+        if (context) {
+          context.drawImage(image, 0, 0)
+
+          const clickedColor = getPixelColor(x, y) as any
+          
+          let closestSegment: ImageSegment | null = null
+
+          let minDistance = Infinity
+          
+          segments.forEach(segment => {
+            const segmentColor = segment.color.slice(0,3) // get the RGB part only
+            
+            const distance = Math.sqrt(
+              Math.pow(clickedColor[0] - segmentColor[0], 2) +
+              Math.pow(clickedColor[1] - segmentColor[1], 2) +
+              Math.pow(clickedColor[2] - segmentColor[2], 2)
+            )
+            
+            if(distance < minDistance) {
+              minDistance = distance;
+              closestSegment = segment;
+            }
+          })
+  
+          console.log("closestSegment:", closestSegment)
+          console.log(closestSegment) // Here is the closest matching segment
+        }
+      }
+
+      image.src = maskBase64
+      // image.src = "data:image/png;base64," + maskBase64;
+    } else {
+      console.log("No mask available, aborting..")
+    }
+  }
+
+
+  if (!assetUrl) {
+    return <div className="flex w-full h-screen items-center justify-center text-center">
+      <div>Rendering first frame.. (might take around 30s)</div>
+    </div>
+  }
+
+  return (
+    <div className="w-full py-8 px-2">
+      <img
+        src={assetUrl}
+        ref={imgRef}
+        className="w-full rounded-md overflow-hidden"
+        onMouseDown={handleMouseDown}
+      />
+    </div>
+  )
+}

src/components/business/{video-player.tsx → video-renderer.tsx}

@@ -1,10 +1,10 @@
 "use client"
 
-export const VideoPlayer = ({ url }: { url?: string }) => {
+export const VideoRenderer = ({ url }: { url?: string }) => {
 
   if (!url) {
     return <div className="flex w-full h-screen items-center justify-center text-center">
-      <div>Rendering first scene (might take ~1 min)</div>
+      <div>Rendering first frames.. (might take around 30s)</div>
     </div>
   }