embed Broadway into Clapper

.gitattributes

@@ -2,3 +2,4 @@
 *.xcf filter=lfs diff=lfs merge=lfs -text
 *.jpeg filter=lfs diff=lfs merge=lfs -text
 *.jpg filter=lfs diff=lfs merge=lfs -text
+public/datasets/baby-names-us-year-of-birth-full.csv filter=lfs diff=lfs merge=lfs -text

package.json

@@ -10,20 +10,21 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@aitube/clap": "0.0.27",
-    "@aitube/engine": "0.0.24",
-    "@aitube/timeline": "0.0.29",
-    "@fal-ai/serverless-client": "^0.10.3",
+    "@aitube/broadway": "0.0.19",
+    "@aitube/clap": "0.0.29",
+    "@aitube/engine": "0.0.25",
+    "@aitube/timeline": "0.0.32",
+    "@fal-ai/serverless-client": "^0.11.0",
     "@gradio/client": "^1.1.1",
     "@huggingface/hub": "^0.15.1",
     "@huggingface/inference": "^2.7.0",
-    "@langchain/anthropic": "^0.2.0",
-    "@langchain/cohere": "^0.0.11",
-    "@langchain/core": "^0.2.6",
-    "@langchain/google-vertexai": "^0.0.18",
-    "@langchain/groq": "^0.0.12",
+    "@langchain/anthropic": "^0.2.2",
+    "@langchain/cohere": "^0.1.0",
+    "@langchain/core": "^0.2.9",
+    "@langchain/google-vertexai": "^0.0.19",
+    "@langchain/groq": "^0.0.13",
     "@langchain/mistralai": "^0.0.24",
-    "@langchain/openai": "^0.1.1",
+    "@langchain/openai": "^0.2.0",
     "@monaco-editor/react": "^4.6.0",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-avatar": "^1.0.4",
@@ -52,26 +53,19 @@
     "@react-three/uikit": "^0.3.4",
     "@react-three/uikit-lucide": "^0.3.4",
     "@tailwindcss/container-queries": "^0.1.1",
-    "@types/node": "20.12.7",
-    "@types/react": "18.3.0",
-    "@types/react-dom": "18.3.0",
-    "@types/uuid": "^9.0.8",
     "@upstash/ratelimit": "^1.1.3",
     "@upstash/redis": "^1.31.1",
-    "autoprefixer": "10.4.17",
+    "autoprefixer": "10.4.19",
     "class-variance-authority": "^0.7.0",
-    "clsx": "^2.1.0",
+    "clsx": "^2.1.1",
     "cmdk": "^0.2.1",
-    "eslint": "8.57.0",
-    "eslint-config-next": "14.1.0",
     "fluent-ffmpeg": "^2.1.3",
     "fs-extra": "^11.2.0",
-    "lucide-react": "^0.334.0",
+    "lucide-react": "^0.396.0",
     "mlt-xml": "^2.0.2",
-    "monaco-editor": "^0.49.0",
-    "next": "^14.2.3",
-    "next-themes": "^0.2.1",
-    "postcss": "8.4.38",
+    "monaco-editor": "^0.50.0",
+    "next": "^14.2.4",
+    "next-themes": "^0.3.0",
     "qs": "^6.12.1",
     "query-string": "^9.0.0",
     "react": "^18.3.1",
@@ -87,22 +81,29 @@
     "react-speakup": "^1.0.0",
     "replicate": "^0.30.2",
     "sharp": "^0.33.4",
-    "sonner": "^1.4.41",
+    "sonner": "^1.5.0",
     "tailwind-merge": "^2.3.0",
-    "tailwindcss": "^3.4.3",
     "tailwindcss-animate": "^1.0.7",
     "three": "^0.164.1",
     "ts-node": "^10.9.2",
-    "typescript": "5.4.5",
     "use-file-picker": "^2.1.2",
     "usehooks-ts": "^2.14.0",
     "uuid": "^9.0.1",
     "web-audio-beat-detector": "^8.2.10",
-    "yaml": "^2.4.2",
+    "yaml": "^2.4.5",
     "zustand": "^4.5.2",
-    "zx": "^8.1.2"
+    "zx": "^8.1.3"
   },
   "devDependencies": {
-    "@types/fluent-ffmpeg": "^2.1.24"
+    "@types/fluent-ffmpeg": "^2.1.24",
+    "@types/node": "^20",
+    "@types/react": "^18",
+    "@types/react-dom": "^18",
+    "@types/uuid": "^9.0.8",
+    "eslint": "^8",
+    "eslint-config-next": "14.2.4",
+    "postcss": "^8",
+    "tailwindcss": "^3.4.3",
+    "typescript": "^5"
   }
 }

public/datasets/baby-names-us-year-of-birth-full.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:897ebf724f4c0fd8d408d13e5babdddeee838a409203701dea89287ec52a612a
+size 31419088

src/app/api/assistant/askAnyAssistant.ts

@@ -2,7 +2,7 @@
 
 import { ClapSegmentCategory } from "@aitube/clap"
 import { RunnableLike } from "@langchain/core/runnables"
-import { ChatPromptValueInterface } from "@langchain/core/dist/prompt_values"
+import { ChatPromptValueInterface } from "@langchain/core/prompt_values"
 import { AIMessage, AIMessageChunk, HumanMessage } from "@langchain/core/messages"
 import { ChatPromptTemplate, MessagesPlaceholder } from "@langchain/core/prompts"
 import { StructuredOutputParser } from "@langchain/core/output_parsers"
@@ -60,37 +60,37 @@ export async function askAnyAssistant({
     provider === ComputeProvider.GROQ
     ? new ChatGroq({
       apiKey: settings.groqApiKey,
-      modelName: settings.groqModelForAssistant,
+      modelName: settings.assistantModel,
       // temperature: 0.7,
     })
     : provider === ComputeProvider.OPENAI
     ? new ChatOpenAI({
         openAIApiKey: settings.openaiApiKey,
-        modelName: settings.openaiModelForAssistant,
+        modelName: settings.assistantModel,
        // temperature: 0.7,
       })
     : provider === ComputeProvider.ANTHROPIC
     ? new ChatAnthropic({
       anthropicApiKey: settings.anthropicApiKey,
-      modelName: settings.anthropicModelForAssistant,
+      modelName: settings.assistantModel,
       // temperature: 0.7,
     })
     : provider === ComputeProvider.COHERE
     ? new ChatCohere({
       apiKey: settings.cohereApiKey,
-      model: settings.cohereModelForAssistant,
+      model: settings.assistantModel,
       // temperature: 0.7,
     })
     : provider === ComputeProvider.MISTRALAI
     ? new ChatMistralAI({
       apiKey: settings.mistralAiApiKey,
-      modelName: settings.mistralAiModelForAssistant,
+      modelName: settings.assistantModel,
       // temperature: 0.7,
     })
     : provider === ComputeProvider.GOOGLE
     ? new ChatVertexAI({
       apiKey: settings.googleApiKey,
-      modelName: settings.googleModelForAssistant,
+      modelName: settings.assistantModel,
       // temperature: 0.7,
     })
     : undefined

src/app/api/resolve/providers/falai/index.ts

@@ -36,18 +36,18 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
 
     let result: FalAiImageResponse | undefined = undefined
 
-    if (request.settings.falAiModelForImage === "fal-ai/pulid") {
+    if (request.settings.imageGenerationModel === "fal-ai/pulid") {
       if (!request.prompts.image.identity) {
         // throw new Error(`you selected model ${request.settings.falAiModelForImage}, but no character was found, so skipping`)
         // console.log(`warning: user selected model ${request.settings.falAiModelForImage}, but no character was found. Falling back to fal-ai/fast-sdxl`)
         
         // dirty fix to fallback to a non-face model
-        request.settings.falAiModelForImage = "fal-ai/fast-sdxl"
+        request.settings.imageGenerationModel = "fal-ai/fast-sdxl"
       }
     }
 
-    if (request.settings.falAiModelForImage === "fal-ai/pulid") {
-      result = await fal.run(request.settings.falAiModelForImage, {
+    if (request.settings.imageGenerationModel === "fal-ai/pulid") {
+      result = await fal.run(request.settings.imageGenerationModel, {
         input: {
           reference_images: [{
             image_url: request.prompts.image.identity
@@ -60,13 +60,13 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
       }) as FalAiImageResponse
   
     } else {
-      result = await fal.run(request.settings.falAiModelForImage, {
+      result = await fal.run(request.settings.imageGenerationModel, {
         input: {
           prompt: request.prompts.image.positive,
           image_size: imageSize,
           sync_mode: true,
           num_inference_steps:
-            request.settings.falAiModelForImage === "fal-ai/stable-diffusion-v3-medium"
+            request.settings.imageGenerationModel === "fal-ai/stable-diffusion-v3-medium"
               ? 40
               : 25,
           num_images: 1,
@@ -86,7 +86,7 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
   } else if (request.segment.category === ClapSegmentCategory.VIDEO) {
 
     // console.log(`request.settings.falAiModelForVideo = `, request.settings.falAiModelForVideo)
-    if (request.settings.falAiModelForVideo !== "fal-ai/stable-video") {
+    if (request.settings.videoGenerationModel !== "fal-ai/stable-video") {
       throw new Error(`only "fal-ai/stable-video" is supported by Clapper for the moment`)
     }
 
@@ -94,7 +94,7 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
     if (!storyboard) {
       throw new Error(`cannot generate a video without a storyboard (the concept of Clapper is to use storyboards)`)
     }
-    const result = await fal.run(request.settings.falAiModelForVideo, {
+    const result = await fal.run(request.settings.videoGenerationModel, {
       input: {
         image_url: storyboard.assetUrl,
    
@@ -124,7 +124,7 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
     ||
     request.segment.category === ClapSegmentCategory.MUSIC
   ) {
-    const result = await fal.run(request.settings.falAiModelForSound, {
+    const result = await fal.run(request.settings.soundGenerationModel, {
       input: {
         // note how we use the *segment* prompt for music or sound
         prompt: request.segment.prompt,
@@ -138,7 +138,7 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
   } else if (
     request.segment.category === ClapSegmentCategory.DIALOGUE
   ) {
-    const result = await fal.run(request.settings.falAiModelForVoice, {
+    const result = await fal.run(request.settings.voiceGenerationModel, {
       input: {
         text: request.segment.prompt,
 

src/app/api/resolve/providers/huggingface/generateImage.ts

@@ -5,8 +5,8 @@ import { ResolveRequest } from "@/types"
 
 export async function generateImage(request: ResolveRequest): Promise<string> {
   
-  if (!request.settings.huggingFaceModelForImage) {
-    throw new Error(`HuggingFace.generateImage: cannot generate without a valid huggingFaceModelForImage`)
+  if (!request.settings.imageGenerationModel) {
+    throw new Error(`HuggingFace.generateImage: cannot generate without a valid imageGenerationModel`)
   }
 
   if (!request.prompts.image.positive) {
@@ -20,7 +20,7 @@ export async function generateImage(request: ResolveRequest): Promise<string> {
   const hf: HfInferenceEndpoint = new HfInference(request.settings.huggingFaceApiKey)
   
   const blob: Blob = await hf.textToImage({
-    model: request.settings.huggingFaceModelForImage,
+    model: request.settings.imageGenerationModel,
     inputs: request.prompts.image.positive,
     parameters: {
       height: request.meta.height,

src/app/api/resolve/providers/huggingface/generateVideo.ts

@@ -3,8 +3,8 @@ import { callGradioApi } from "@/lib/hf/callGradioApi"
 
 export async function generateVideo(request: ResolveRequest): Promise<string> {
   
-  if (!request.settings.huggingFaceModelForVideo) {
-    throw new Error(`HuggingFace.generateVideo: cannot generate without a valid huggingFaceModelForVideo`)
+  if (!request.settings.videoGenerationModel) {
+    throw new Error(`HuggingFace.generateVideo: cannot generate without a valid videoGenerationModel`)
   }
 
   if (!request.prompts.video.image) {
@@ -16,13 +16,11 @@ export async function generateVideo(request: ResolveRequest): Promise<string> {
   }
 
   // TODO pass a type to the template function
-  const output = await callGradioApi({
-    url: request.settings.huggingFaceModelForVideo,
+  const assetUrl = await callGradioApi<string>({
+    url: request.settings.videoGenerationModel,
     inputs: request.prompts.video,
     apiKey: request.settings.huggingFaceApiKey
   })
 
-  console.log(`output from the Gradio API:`, output)
-
-  throw new Error(`please finish me`)
+  return assetUrl
 }

src/app/api/resolve/providers/huggingface/generateVoice.ts

@@ -4,8 +4,8 @@ import { ResolveRequest } from "@/types"
 
 export async function generateVoice(request: ResolveRequest): Promise<string> {
   
-  if (!request.settings.huggingFaceModelForVoice) {
-    throw new Error(`HuggingFace.generateVoice: cannot generate without a valid huggingFaceModelForVoice`)
+  if (!request.settings.voiceGenerationModel) {
+    throw new Error(`HuggingFace.generateVoice: cannot generate without a valid voiceGenerationModel`)
   }
 
   if (!request.prompts.voice.positive) {
@@ -19,7 +19,7 @@ export async function generateVoice(request: ResolveRequest): Promise<string> {
   const hf: HfInferenceEndpoint = new HfInference(request.settings.huggingFaceApiKey)
   
   const blob: Blob = await hf.textToSpeech({
-    model: request.settings.huggingFaceModelForVoice,
+    model: request.settings.voiceGenerationModel,
     inputs: request.prompts.voice.positive,
   })
 

src/app/api/resolve/providers/huggingface/index.ts

@@ -24,7 +24,7 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
   }  if (request.segment.category === ClapSegmentCategory.VIDEO) {
     segment.assetUrl = await generateVideo(request)
   } else {
-    throw new Error(`Clapper doesn't support ${request.segment.category} generation for provider "Hugging Face" with model (or space) "${request.settings.huggingFaceModelForVideo}". Please open a pull request with (working code) to solve this!`)
+    throw new Error(`Clapper doesn't support ${request.segment.category} generation for provider "Hugging Face" with model (or space) "${request.settings.videoGenerationModel}". Please open a pull request with (working code) to solve this!`)
   }
   return segment
 }

src/app/api/resolve/providers/index.ts

@@ -0,0 +1,7 @@
+export { resolveSegment as resolveSegmentUsingHuggingFace } from "./huggingface"
+export { resolveSegment as resolveSegmentUsingComfyReplicate } from "./comfy-replicate"
+export { resolveSegment as resolveSegmentUsingReplicate } from "./replicate"
+export { resolveSegment as resolveSegmentUsingComfyComfyIcu } from "./comfy-comfyicu"
+export { resolveSegment as resolveSegmentUsingFalAi } from "./falai"
+export { resolveSegment as resolveSegmentUsingModelsLab } from "./modelslab"
+export { resolveSegment as resolveSegmentUsingStabilityAi } from "./stabilityai"

src/app/api/resolve/providers/replicate/index.ts

@@ -20,12 +20,12 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
   // like we are doing for Hugging Face (match the fields etc)
   if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
     let params: object = {}
-    if (request.settings.replicateModelForImage === "fofr/pulid-lightning") {
+    if (request.settings.imageGenerationModel === "fofr/pulid-lightning") {
       params = {
         prompt: request.prompts.image.positive,
         face_image: request.prompts.image.identity,
       }
-    } else if (request.settings.replicateModelForImage === "zsxkib/pulid") {
+    } else if (request.settings.imageGenerationModel === "zsxkib/pulid") {
       params = {
         prompt: request.prompts.image.positive,
         main_face_image: request.prompts.image.identity,
@@ -36,13 +36,13 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
       }
     }
     const response = await replicate.run(
-      request.settings.replicateModelForImage as any,
+      request.settings.imageGenerationModel as any,
       { input: params }
     ) as any
     segment.assetUrl = `${response.output || ""}`
   } else if (request.segment.category === ClapSegmentCategory.DIALOGUE) {
     const response = await replicate.run(
-      request.settings.replicateModelForVoice as any, {
+      request.settings.voiceGenerationModel as any, {
       input: {
         text: request.prompts.voice.positive,
         audio: request.prompts.voice.identity,
@@ -51,7 +51,7 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
     segment.assetUrl = `${response.output || ""}`
   } else if (request.segment.category === ClapSegmentCategory.VIDEO) {
     const response = await replicate.run(
-      request.settings.replicateModelForVideo as any, {
+      request.settings.videoGenerationModel as any, {
       input: {
         image: request.prompts.video.image,
       }

src/app/api/resolve/providers/stabilityai/generateImage.ts

@@ -8,7 +8,7 @@ export async function generateImage(request: ResolveRequest): Promise<string> {
     throw new Error(`StabilityAI.generateImage: cannot generate without a valid stabilityAiApiKey`)
   }
 
-  if (!request.settings.stabilityAiModelForImage) {
+  if (!request.settings.imageGenerationModel) {
     throw new Error(`StabilityAI.generateImage: cannot generate without a valid stabilityAiModelForImage`)
   }
 
@@ -39,7 +39,7 @@ export async function generateImage(request: ResolveRequest): Promise<string> {
   body.set("negative_prompt", `${request.prompts.image.negative || ""}`)
   body.set("aspect_ratio", `${aspectRatio || ""}`)
 
-  const response = await fetch(`https://api.stability.ai/v2beta/${request.settings.stabilityAiModelForImage}`, {
+  const response = await fetch(`https://api.stability.ai/v2beta/${request.settings.imageGenerationModel}`, {
     method: "POST",
     headers: {
       Authorization: `Bearer ${request.settings.stabilityAiApiKey}`,

src/app/api/resolve/providers/stabilityai/generateVideo.ts

@@ -8,8 +8,8 @@ export async function generateVideo(request: ResolveRequest): Promise<string> {
     throw new Error(`StabilityAI.generateVideo: cannot generate without a valid stabilityAiApiKey`)
   }
 
-  if (!request.settings.stabilityAiModelForVideo) {
-    throw new Error(`StabilityAI.generateVideo: cannot generate without a valid stabilityAiModelForVideo`)
+  if (!request.settings.videoGenerationModel) {
+    throw new Error(`StabilityAI.generateVideo: cannot generate without a valid videoGenerationModel`)
   }
 
 

src/app/api/resolve/route.ts

@@ -1,13 +1,15 @@
 import { NextResponse, NextRequest } from "next/server"
 import { ClapOutputType, ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap"
 
-import { resolveSegment as resolveSegmentUsingHuggingFace } from "./providers/huggingface"
-import { resolveSegment as resolveSegmentUsingComfyReplicate } from "./providers/comfy-replicate"
-import { resolveSegment as resolveSegmentUsingReplicate } from "./providers/replicate"
-import { resolveSegment as resolveSegmentUsingComfyComfyIcu } from "./providers/comfy-comfyicu"
-import { resolveSegment as resolveSegmentUsingFalAi } from "./providers/falai"
-import { resolveSegment as resolveSegmentUsingModelsLab } from "./providers/modelslab"
-import { resolveSegment as resolveSegmentUsingStabilityAi } from "./providers/stabilityai"
+import {
+  resolveSegmentUsingHuggingFace,
+  resolveSegmentUsingComfyReplicate,
+  resolveSegmentUsingReplicate,
+  resolveSegmentUsingComfyComfyIcu,
+  resolveSegmentUsingFalAi,
+  resolveSegmentUsingModelsLab,
+  resolveSegmentUsingStabilityAi
+} from "./providers"
 
 import { ComputeProvider, ResolveRequest } from "@/types"
 import { decodeOutput } from "@/lib/utils/decodeOutput"
@@ -21,7 +23,7 @@ export async function POST(req: NextRequest) {
   // console.log(`TODO Julian: secure the endpoint`)
   // await throwIfInvalidToken(req.headers.get("Authorization"))
   const request = (await req.json()) as ResolveRequest
-  
+
   const provider =
     request.segment.category === ClapSegmentCategory.STORYBOARD
     ? request.settings.imageProvider

src/app/main.tsx

@@ -22,6 +22,7 @@ import { TopBar } from "@/components/toolbars/top-bar"
 import { Timeline } from "@/components/core/timeline"
 import { useIO } from "@/controllers/io/useIO"
 import { ChatView } from "@/components/assistant/ChatView"
+import { ScriptEditor } from "@/components/editor/ScriptEditor"
 import { useSearchParams } from "next/navigation"
 
 type DroppableThing = { files: File[] }
@@ -86,7 +87,23 @@ function MainContent() {
                 <ReflexElement
                   minSize={showTimeline ? 100 : 1}
                   >
-                  <Monitor />
+                  <ReflexContainer orientation="vertical">
+
+       
+                    <ReflexElement
+                      size={showTimeline ? 400 : 1}
+                      minSize={showTimeline ? 100 : 1}
+                      maxSize={showTimeline ? 1600 : 1}
+                      >
+                      <ScriptEditor />
+                    </ReflexElement>
+                    <ReflexSplitter />
+                    <ReflexElement
+                      minSize={showTimeline ? 100 : 1}
+                      >
+                      <Monitor />
+                    </ReflexElement>
+                  </ReflexContainer>
                 </ReflexElement>
                 <ReflexSplitter />
                 <ReflexElement

src/components/{core/ScriptEditor/index.FOR_LATER → editor/ScriptEditor/index.tsx}

@@ -1,6 +1,10 @@
-import { useEffect, useState } from "react"
+import React, { useEffect, useState } from "react"
 import MonacoEditor from "monaco-editor"
 import Editor from "@monaco-editor/react"
+import { DEFAULT_DURATION_IN_MS_PER_STEP, TimelineStore, useTimeline } from "@aitube/timeline"
+
+import { useEditor } from "@/controllers/editor/useEditor"
+import { useRenderer } from "@/controllers/renderer"
 
 const beforeMount = ({ editor }: { editor: typeof MonacoEditor.editor }) => {
   // Define a custom theme with the provided color palette
@@ -11,15 +15,15 @@ const beforeMount = ({ editor }: { editor: typeof MonacoEditor.editor }) => {
      // You can define token-specific styles here if needed
    ],
    colors: {
-     'editor.background': '#111827', // Editor background color (given)
-     'editorCursor.foreground': '#e5e7eb', // Cursor color
-     'editor.lineHighlightBackground': '#374151', // Highlighted line color
-     'editorLineNumber.foreground': '#6b7280', // Line Numbers color
-     'editor.selectionBackground': '#2c333c', // Selection color
-     'editor.foreground': '#d1d5db', // Main text color
-     'editorIndentGuide.background': '#4b5563', // Indent guides color
-     'editorIndentGuide.activeBackground': '#6b7280', // Active indent guides color
-     'editorWhitespace.foreground': '#3b4049', // Whitespace symbols color
+     'editor.background': '#292524', // Editor background color (given)
+     'editorCursor.foreground': '#f5f5f4', // Cursor color
+     'editor.lineHighlightBackground': '#44403c', // Highlighted line color
+     'editorLineNumber.foreground': '#78716c', // Line Numbers color
+     'editor.selectionBackground': '#44403c', // Selection color
+     'editor.foreground': '#d6d3d1', // Main text color
+     'editorIndentGuide.background': '#78716c', // Indent guides color
+     'editorIndentGuide.activeBackground': '#a8a29e', // Active indent guides color
+     'editorWhitespace.foreground': '#a8a29e', // Whitespace symbols color
      // Add more color overrides if needed here
    },
  })
@@ -29,14 +33,57 @@ const beforeMount = ({ editor }: { editor: typeof MonacoEditor.editor }) => {
 }
 
 export function ScriptEditor() {
-  const [editor, setEditor] = useState<MonacoEditor.editor.IStandaloneCodeEditor>()
-  const script = useApp(state => state.script)
-  const rewriteCurrentScript = useApp(state => state.rewriteCurrentScript)
+  const editor = useEditor(s => s.editor)
+  const setEditor = useEditor(s => s.setEditor)
+  const draft = useEditor(s => s.draft)
+  const setDraft = useEditor(s => s.setDraft)
+  const loadDraftFromClap = useEditor(s => s.loadDraftFromClap)
+  const onDidScrollChange = useEditor(s => s.onDidScrollChange)
+  const jumpCursorOnLineClick = useEditor(s => s.jumpCursorOnLineClick)
+  
+  // this is an expensive function, we should only call it on blur or on click on a "save button maybe"
+  const publishDraftToTimeline = useEditor(s => s.publishDraftToTimeline)
+
+  const clap = useTimeline((s: TimelineStore) => s.clap)
+  const cursorTimestampAtInMs = useTimeline(s => s.cursorTimestampAtInMs)
+  const totalDurationInMs = useTimeline(s => s.totalDurationInMs)
+  const scrollX = useTimeline(s => s.scrollX)
+  const contentWidth = useTimeline(s => s.contentWidth)
+  
+  useEffect(() => { loadDraftFromClap(clap) }, [clap])
+
+  const scrollTop = useEditor(s => s.scrollTop)
+  const scrollLeft = useEditor(s => s.scrollLeft)
+  const scrollWidth = useEditor(s => s.scrollWidth)
+  const scrollHeight = useEditor(s => s.scrollHeight)
+
+  /*
+  const script = useTimeline(state => state.script)
+
   const isPlaying = useApp(state => state.isPlaying)
   const setCursorAt = useApp((state) => state.setCursorAt)
   const [scriptContent, setScriptContent] = useState("")
+  */
+
+  const currentSegment = useRenderer(s => s.currentSegment)
   
-  const activeSegments = useApp(state => state.activeSegments)
+  const activeStartTimeInLines = currentSegment?.startTimeInLines
+
+  useEffect(() => {
+    console.log("activeStartTimeInLines:", activeStartTimeInLines)
+
+  }, [activeStartTimeInLines])
+
+  useEffect(() => {
+    console.log("scrollX:", scrollX)
+
+  }, [scrollX])
+
+  /*
+  const activeSceneLineNumber = (activeScene?.startAtLine || 0)
+  */
+
+/*
   const stepsToPreviews = useApp(state => state.stepsToPreviews)
 
   const screenplayScroll = useInterface(state => state.screenplayScroll)
@@ -48,6 +95,8 @@ export function ScriptEditor() {
 
   // console.log("linesToPreview:", linesToPreviews)
 
+  */
+ /*
   useEffect(() => {
     if (editor && leftmostVisibleScene) {
       // console.log("ScriptEditor: timelineScrollLeftInStep changed to scene " + leftmostVisibleScene.line)
@@ -58,29 +107,10 @@ export function ScriptEditor() {
       editor.revealLineInCenter(lineNumber)
     }
   }, [editor, leftmostVisibleScene])
+  */
 
 
-  const activeScene = activeSegments
-    .find(s => s.category === "video")?.scene
-
-  const activeSceneLineNumber = (activeScene?.startAtLine || 0)
-
-  useEffect(() => {
-    const fn = async () => {
-    // support both text and Blob
-      let content = ""
-      if (typeof script.content !== "string") {
-        content = await script.content.text()
-      } else {
-        content = script.content
-      }
-      editor?.setValue(content)
-      setScriptContent(content)
-    }
-    fn()
-
-  }, [editor, script.content])
-
+  /*
   useEffect(() => {
     if (editor && activeSceneLineNumber) {
       // console.log("useEffect:", activeSceneLineNumber)
@@ -107,7 +137,7 @@ export function ScriptEditor() {
       }
     }
   }, [editor, activeSceneLineNumber])
-
+  */
   const onMount = (editor: MonacoEditor.editor.IStandaloneCodeEditor) => {
     const model = editor.getModel()
     if (!model) { return }
@@ -115,39 +145,14 @@ export function ScriptEditor() {
     setEditor(editor)
 
     editor.onMouseDown((e) => {
-      const currentPosition = editor.getPosition()
-
-      const line = currentPosition?.lineNumber
-      if (typeof line !== "number") { return }
-
-      // so... due to how monaco callbacks work, we cannot use the hook context
-      // to get the linesToPreview.. but that's okay!
-      const linesToPreviews = useApp.getState().linesToPreviews
-    
-      const startTimeInSteps = linesToPreviews[line]?.startTimeInSteps
-      if (typeof startTimeInSteps !== "number") { return }
-
-      setCursorAt(startTimeInSteps * DEFAULT_DURATION_IN_MS_PER_STEP)
+      jumpCursorOnLineClick(editor.getPosition()?.lineNumber)
     })
 
     editor.onDidScrollChange(({ scrollTop, scrollLeft, scrollWidth, scrollHeight }: MonacoEditor.IScrollEvent) => {
-      /*if (scrollHeight !== screenplayScroll.scrollHeight &&
-        scrollLeft !== screenplayScroll.scrollLeft &&
-        scrollTop !== screenplayScroll.scrollTop &&
-        scrollWidth !== screenplayScroll.scrollWidth) {
-          */
-        // console.log(`ScriptEditor:onDidScrollChange(${JSON.stringify({ scrollTop, scrollLeft, scrollWidth, scrollHeight }, null, 2)})`) 
-        setScreenplayScroll({
-          shouldRerender: false,
-          scrollTop,
-          scrollLeft,
-          scrollWidth,
-          scrollHeight
-        })
-      //}
-
-      // TODO we need to grab the leftmost segment
-      // now the problem is that this might be a bit costly to do
+      onDidScrollChange(
+        { scrollTop, scrollLeft, scrollWidth, scrollHeight },
+        true // <- set to true to ignore the change and avoid an infinite loop
+      )
     })
 
     // as an optimization we can use this later, for surgical edits,
@@ -162,14 +167,7 @@ export function ScriptEditor() {
   }
 
   const onChange = (plainText?: string) => {
-    if (!plainText) { return }
-
-    if (plainText === scriptContent) { return }
-
-    console.log("generic onChange:")
-    // this function is currently *very* expensive
-    // the only optimization right now is that we debounce it
-    rewriteCurrentScript(plainText)
+   // setDraft(plainText || "")
   }
 
   return (
@@ -177,13 +175,13 @@ export function ScriptEditor() {
       <Editor
         height="100%"
         defaultLanguage="plaintext"
-        defaultValue={scriptContent}
+        defaultValue={draft}
         beforeMount={beforeMount}
         onMount={onMount}
         onChange={onChange}
         theme="customTheme"
         options={{
-          fontSize: 14
+          fontSize: 12
         }}
       />
     </div>

src/components/monitor/DynamicPlayer/StoryboardBuffer.tsx

@@ -16,11 +16,16 @@ export function StoryboardBuffer({
         `absolute`,
         `h-full rounded-md overflow-hidden`,
 
-        // iseally we could only use the ease-out and duration-150
+        // ideally we could only use the ease-out and duration-150
         // to avoid a weird fade to grey,
         // but the ease out also depends on which video is on top of each other,
         // in term of z-index, so we should also intervert this
         `transition-all duration-100 ease-out`,
+
+        // yeah well, I couldn't find a better name 
+        // (except maybe aspect-128/135, but is that really more familiar?)
+        `object-cover aspect-1024/576`,
+
         className
       )}
       src={src}

src/components/monitor/DynamicPlayer/index.tsx

@@ -75,7 +75,9 @@ export const DynamicPlayer = ({
   ])
 
   return (
-    <div className={cn(`@container flex flex-col items-center flex-grow w-full`, className)}>
+    <div className={cn(`
+    @container flex flex-col items-center flex-grow w-full
+    `, className)}>
       <DynamicBuffer
         segment={dataUriBuffer1}
         isPlaying={isPlaying}

src/components/settings/constants.ts

@@ -270,7 +270,7 @@ export const availableModelsForVideoGeneration: Partial<Record<ComputeProvider,
   ],
   [ComputeProvider.HUGGINGFACE]: [
     "spaces/hpcai-tech/open-sora",
-    "spaces/multimodalart/hallo" // supports audio input
+    "spaces/jbilcke-hf/hallo-api" // supports audio input
   ],
   [ComputeProvider.REPLICATE]: [
     // note: we need a model that accepts cinematic ratios

src/components/settings/image.tsx

@@ -12,6 +12,8 @@ export function SettingsSectionImage() {
   const imageProvider = useSettings(s => s.imageProvider)
   const setImageProvider = useSettings(s => s.setImageProvider)
 
+  /*
+  to deprecate I think - or replace by defaultHuggingFaceModelForXXXX
   const huggingFaceModelForImage = useSettings(s => s.huggingFaceModelForImage)
   const setHuggingFaceModelForImage = useSettings(s => s.setHuggingFaceModelForImage)
 
@@ -23,6 +25,7 @@ export function SettingsSectionImage() {
 
   const modelsLabModelForImage = useSettings(s => s.modelsLabModelForImage)
   const setModelsLabModelForImage = useSettings(s => s.setModelsLabModelForImage)
+  */
 
   const imagePromptPrefix = useSettings(s => s.imagePromptPrefix)
   const setImagePromptPrefix = useSettings(s => s.setImagePromptPrefix)
@@ -100,6 +103,7 @@ export function SettingsSectionImage() {
 
          </>
         : // "proprietary" parameters
+          null/*
           <>
             {imageProvider === ComputeProvider.HUGGINGFACE && <FormInput
               label="HF Model ID (must be compatible with the Inference API)"
@@ -125,7 +129,9 @@ export function SettingsSectionImage() {
               defaultValue={defaultSettings.modelsLabModelForImage}
               onChange={setModelsLabModelForImage}
             />}
-          </>}
+          </>
+          */
+          }
 
       </FormSection>
     </div>

src/components/settings/music.tsx

@@ -12,6 +12,7 @@ export function SettingsSectionMusic() {
   const musicProvider = useSettings(s => s.musicProvider)
   const setMusicProvider = useSettings(s => s.setMusicProvider)
 
+  /*
   const huggingFaceModelForMusic = useSettings(s => s.huggingFaceModelForMusic)
   const setHuggingFaceModelForMusic = useSettings(s => s.setHuggingFaceModelForMusic)
 
@@ -23,6 +24,7 @@ export function SettingsSectionMusic() {
 
   const modelsLabModelForMusic = useSettings(s => s.modelsLabModelForMusic)
   const setModelsLabModelForMusic = useSettings(s => s.setModelsLabModelForMusic)
+  */
 
   const comfyWorkflowForMusic = useSettings(s => s.comfyWorkflowForMusic)
   const setComfyWorkflowForMusic = useSettings(s => s.setComfyWorkflowForMusic)
@@ -55,6 +57,7 @@ export function SettingsSectionMusic() {
           onChange={setComfyWorkflowForMusic}
         />
         : // "proprietary" parameters
+        null /*
           <>
             {musicProvider === ComputeProvider.HUGGINGFACE && <FormInput
               label="HF Model ID (must be compatible with the Inference API)"
@@ -80,7 +83,8 @@ export function SettingsSectionMusic() {
               defaultValue={defaultSettings.modelsLabModelForMusic}
               onChange={setModelsLabModelForMusic}
             />}
-          </>}
+          </>
+          */}
       </FormSection>
     </div>
   )

src/components/settings/sound.tsx

@@ -12,6 +12,7 @@ export function SettingsSectionSound() {
   const soundProvider = useSettings(s => s.soundProvider)
   const setSoundProvider = useSettings(s => s.setSoundProvider)
 
+  /*
   const huggingFaceModelForSound = useSettings(s => s.huggingFaceModelForSound)
   const setHuggingFaceModelForSound = useSettings(s => s.setHuggingFaceModelForSound)
 
@@ -23,7 +24,7 @@ export function SettingsSectionSound() {
 
   const modelsLabModelForSound = useSettings(s => s.modelsLabModelForSound)
   const setModelsLabModelForSound = useSettings(s => s.setModelsLabModelForSound)
-
+  */
   const comfyWorkflowForSound = useSettings(s => s.comfyWorkflowForSound)
   const setComfyWorkflowForSound = useSettings(s => s.setComfyWorkflowForSound)
 
@@ -55,6 +56,7 @@ export function SettingsSectionSound() {
           onChange={setComfyWorkflowForSound}
         />
         : // "proprietary" parameters
+        null /*
           <>
             {soundProvider === ComputeProvider.HUGGINGFACE && <FormInput
               label="HF Model ID (must be compatible with the Inference API)"
@@ -80,7 +82,8 @@ export function SettingsSectionSound() {
               defaultValue={defaultSettings.modelsLabModelForSound}
               onChange={setModelsLabModelForSound}
             />}
-          </>}
+          </>
+          */}
       </FormSection>
     </div>
   )

src/components/settings/video.tsx

@@ -9,6 +9,7 @@ import { availableComputeProvidersForVideos, computeProviderShortNames } from ".
 export function SettingsSectionVideo() {
   const defaultSettings = getDefaultSettingsState()
 
+  /*
   const huggingFaceModelForVideo = useSettings(s => s.huggingFaceModelForVideo)
   const setHuggingFaceModelForVideo = useSettings(s => s.setHuggingFaceModelForVideo)
 
@@ -20,7 +21,7 @@ export function SettingsSectionVideo() {
 
   const modelsLabModelForVideo = useSettings(s => s.modelsLabModelForVideo)
   const setModelsLabModelForVideo = useSettings(s => s.setModelsLabModelForVideo)
-
+  */
 
   const videoPromptPrefix = useSettings(s => s.videoPromptPrefix)
   const setVideoPromptPrefix = useSettings(s => s.setVideoPromptPrefix)
@@ -118,7 +119,7 @@ export function SettingsSectionVideo() {
           onChange={setComfyWorkflowForVideo}
         />
         : // "proprietary" parameters
-          <>
+          null /* <>
             {videoProvider === ComputeProvider.HUGGINGFACE && <FormInput
               label="HF Model ID (must be compatible with the Inference API)"
               value={huggingFaceModelForVideo}
@@ -143,7 +144,8 @@ export function SettingsSectionVideo() {
               defaultValue={defaultSettings.modelsLabModelForVideo}
               onChange={setModelsLabModelForVideo}
             />}
-          </>}
+          </>
+          */}
 
       </FormSection>
     </div>

src/components/settings/voice.tsx

@@ -12,6 +12,7 @@ export function SettingsSectionVoice() {
   const voiceProvider = useSettings(s => s.voiceProvider)
   const setVoiceProvider = useSettings(s => s.setVoiceProvider)
 
+  /*
   const huggingFaceModelForVoice = useSettings(s => s.huggingFaceModelForVoice)
   const setHuggingFaceModelForVoice = useSettings(s => s.setHuggingFaceModelForVoice)
 
@@ -23,6 +24,7 @@ export function SettingsSectionVoice() {
 
   const modelsLabModelForVoice = useSettings(s => s.modelsLabModelForVoice)
   const setModelsLabModelForVoice = useSettings(s => s.setModelsLabModelForVoice)
+  */
 
   const comfyWorkflowForVoice = useSettings(s => s.comfyWorkflowForVoice)
   const setComfyWorkflowForVoice = useSettings(s => s.setComfyWorkflowForVoice)
@@ -55,7 +57,7 @@ export function SettingsSectionVoice() {
           onChange={setComfyWorkflowForVoice}
         />
         : // "proprietary" parameters
-          <>
+          null /* <>
             {voiceProvider === ComputeProvider.HUGGINGFACE && <FormInput
               label="HF Model ID (must be compatible with the Inference API)"
               value={huggingFaceModelForVoice}
@@ -80,7 +82,8 @@ export function SettingsSectionVoice() {
               defaultValue={defaultSettings.modelsLabModelForVoice}
               onChange={setModelsLabModelForVoice}
             />}
-          </>}
+          </>
+          */}
       </FormSection>
     </div>
   )

src/controllers/audio/startAudioSourceNode.ts

@@ -40,11 +40,23 @@ export function startAudioSourceNode({
    * So make sure it uses fresh data when it is finally executed
    */
   onEnded: (sourceId: string) => void
-}): CurrentlyPlayingAudioSource {
+}): CurrentlyPlayingAudioSource | undefined {
   if (!segment.audioBuffer) {
-    throw new Error(`Cannot playAudioBuffer on non-audio segments`)
+    // throw new Error(`startAudioSourceNode: cannot play a non-audio segment`)
+    // console.error(`startAudioSourceNode: cannot play a non-audio segment`)
+    return
   }
 
+  // well, we can't play the segment if the playback cursor is out of its range
+  if (cursorTimestampAtInMs < segment.startTimeInMs || segment.endTimeInMs < cursorTimestampAtInMs) {
+    // onsole.error(`startAudioSourceNode: cannot play a segment which is nto crossing the current cursor`)
+    return
+  }
+
+  const startTimeInMs = Math.max(0, cursorTimestampAtInMs - segment.startTimeInMs)
+
+  // console.log(`startAudioSourceNode: ${startTimeInMs}ms`)
+
   // const audioContext = new AudioContext() // initialize AudioContext
 
   // Get an AudioBufferSourceNode.
@@ -58,10 +70,10 @@ export function startAudioSourceNode({
 
   const gainNode: GainNode = audioContext.createGain()
 
-  if (isFinite(segment.outputGain)) {
+  if (!isNaN(segment.outputGain) && isFinite(segment.outputGain)) {
     gainNode.gain.value = segment.outputGain
   } else {
-    console.log(`segment.outputGain isn't finite for some reason? (got value ${segment.outputGain})`)
+    // console.error(`segment.outputGain isn't finite for some reason? (got value ${segment.outputGain})`)
     gainNode.gain.value = 1.0
   }
 
@@ -71,11 +83,8 @@ export function startAudioSourceNode({
   // connect the gain node to the destination
   gainNode.connect(audioContext.destination)
 
-  // make sure we play the segment at a specific time
-  const startTimeInMs = cursorTimestampAtInMs - segment.startTimeInMs
-
   // convert milliseconds to seconds by dividing by 1000
-  source.start(audioContext.currentTime, startTimeInMs >= 1000 ? (startTimeInMs / 1000) : 0)
+  source.start(audioContext.currentTime, startTimeInMs / 1000)
 
   const currentlyPlaying: CurrentlyPlayingAudioSource = {
     sourceId: UUID(),

src/controllers/audio/useAudio.ts

@@ -3,9 +3,10 @@
 import { create } from "zustand"
 import { TimelineStore, useTimeline, RuntimeSegment } from "@aitube/timeline"
 
-import { AudioStore } from "./types"
+import { AudioStore, CurrentlyPlayingAudioSource } from "./types"
 import { getDefaultAudioState } from "./getDefaultAudioState"
 import { startAudioSourceNode } from "./startAudioSourceNode"
+import { useRenderer } from "../renderer"
 
 export const useAudio = create<AudioStore>((set, get) => ({
   ...getDefaultAudioState(),
@@ -14,7 +15,7 @@ export const useAudio = create<AudioStore>((set, get) => ({
     // console.log("useAudio: play()")
     const { isPlaying, currentlyPlaying } = get()
     if (isPlaying) { return }
-    currentlyPlaying.forEach(p => { p.sourceNode.start() })
+    currentlyPlaying.forEach(p => p.sourceNode.start())
   },
   stop: () => {
     // console.log("useAudio: stop()")
@@ -59,7 +60,7 @@ export const useAudio = create<AudioStore>((set, get) => ({
    * @returns 
    */
   syncAudioToCurrentCursorPosition: (activeAudioSegments: RuntimeSegment[]) => {
-    // console.log("useAudio: syncAudioToCurrentCursorPosition()")
+
     const { audioContext, currentlyPlaying } = get()
 
     const timelineStore: TimelineStore = useTimeline.getState()
@@ -89,7 +90,7 @@ export const useAudio = create<AudioStore>((set, get) => ({
           })
         }
       })
-    )
+    ).filter(s => s) as CurrentlyPlayingAudioSource[]
 
     set({
       currentlyPlaying: [

src/controllers/editor/getDefaultEditorState.ts

@@ -0,0 +1,18 @@
+import { EditorState } from "./types"
+
+export function getDefaultEditorState(): EditorState {
+  const state: EditorState = {
+    editor: undefined,
+    draft: "",
+    lineNumberToMentionedSegments: {},
+
+    scrollChanges: 0,
+    scrollHeight: 0,
+    scrollLeft: 0,
+    scrollTop: 0,
+    scrollWidth: 0,
+    scrollTopInMs: 0,
+  }
+
+  return state
+}

src/controllers/editor/types.ts

@@ -0,0 +1,41 @@
+import { ClapProject, ClapSegment } from "@aitube/clap"
+import MonacoEditor from "monaco-editor"
+
+export type ScrollData = {
+  scrollHeight: number
+  scrollLeft: number
+  scrollTop: number
+  scrollWidth: number
+}
+
+export type EditorState = {
+  // reference to the React component
+  editor?: MonacoEditor.editor.IStandaloneCodeEditor
+
+  // the full-text of the screenplay
+  draft: string
+
+  // map screenplay lines to dialogue segments
+  // (note: some lines point to nothing, eg. when we have empty spaces)
+  lineNumberToMentionedSegments: Record<number, ClapSegment>
+
+  scrollChanges: number
+
+  /**
+   * the index of the first step visible in the current screenplay
+   * 
+   * (the topmost visible timeline step in the current timeline)
+   */
+  scrollTopInMs: number
+} & ScrollData
+
+export type EditorControls = {
+  setEditor: (editor?: MonacoEditor.editor.IStandaloneCodeEditor) => void
+  loadDraftFromClap: (clap: ClapProject) => void
+  setDraft: (draft: string) => void
+  publishDraftToTimeline: () => Promise<void>
+  onDidScrollChange: (scrollData: ScrollData, ignoreChange?: boolean) => void
+  jumpCursorOnLineClick: (line?: number) => void
+}
+
+export type EditorStore = EditorState & EditorControls

src/controllers/editor/useEditor.ts

@@ -0,0 +1,110 @@
+"use client"
+
+import MonacoEditor from "monaco-editor"
+import { create } from "zustand"
+
+import { EditorStore, ScrollData } from "./types"
+import { getDefaultEditorState } from "./getDefaultEditorState"
+import { ClapProject, ClapSegment, ClapSegmentCategory } from "@aitube/clap"
+import { TimelineStore, useTimeline, leftBarTrackScaleWidth } from "@aitube/timeline"
+
+export const useEditor = create<EditorStore>((set, get) => ({
+  ...getDefaultEditorState(),
+  setEditor: (editor?: MonacoEditor.editor.IStandaloneCodeEditor) => { set({ editor }) },
+  loadDraftFromClap: (clap: ClapProject) => {
+    const { setDraft } = get()
+
+    setDraft(clap.meta.screenplay)
+  },
+  setDraft: (draft: string) => {
+    const { draft: previousDraft } = get()
+    if (draft === previousDraft) { return }
+    set({ draft })
+
+    const { editor } = get()
+    if (!editor) { return }
+    editor?.setValue(draft)
+  },
+  publishDraftToTimeline: async (): Promise<void> => {
+    const { draft } = get()
+    console.log(`user asked to update the whole scene! this is expensive..`)
+    // we can do something smart, which is to only reconstruct the impacted segments
+    // and shift the rest along the time axis, without modifying it
+  },
+  onDidScrollChange: ({
+    scrollHeight,
+    scrollLeft,
+    scrollTop,
+    scrollWidth
+  }: ScrollData, ignoreChange = false) => {
+    const { 
+      scrollHeight: previousScrollHeight,
+      scrollLeft: previousScrollLeft,
+      scrollTop: previousScrollTop,
+      scrollWidth: previousScrollWidth,
+      scrollChanges
+    } = get()
+
+    // skip if nothing changed
+    if (
+      scrollHeight === previousScrollHeight &&
+      scrollLeft === previousScrollLeft &&
+      scrollTop === previousScrollTop &&
+      scrollWidth === previousScrollWidth
+    ) {
+      return
+    }
+
+    set({
+      scrollHeight,
+      scrollLeft,
+      scrollTop,
+      scrollWidth,
+
+      // optionally mark the state as stale
+      scrollChanges: scrollChanges + (ignoreChange ? 0 : 1),
+    })
+
+    const timeline: TimelineStore = useTimeline.getState()
+    if (!timeline.timelineCamera || !timeline.timelineControls) { return }
+
+    const { editor } = get()
+  
+    const scrollRatio = scrollTop / scrollHeight
+    const scrollX = leftBarTrackScaleWidth + scrollRatio * timeline.contentWidth
+    console.log({
+      scrollHeight,
+      scrollLeft,
+      scrollTop,
+      scrollWidth,
+      scrollRatio,
+      scrollX
+    })
+    useTimeline.setState({ scrollX })
+    timeline.timelineCamera.position.setX(scrollX)
+    timeline.timelineControls.target.setX(scrollX)
+  },
+  jumpCursorOnLineClick: (line?: number) => {
+    if (typeof line !== "number") { return }
+    const timeline: TimelineStore = useTimeline.getState()
+  
+    const { lineNumberToMentionedSegments } = timeline
+
+    const mentionedSegments = lineNumberToMentionedSegments[line] || []
+
+    const firstMentionedSegment = mentionedSegments.at(0)
+
+
+    if (typeof firstMentionedSegment?.startTimeInMs !== "number") { return }
+
+    const { startTimeInMs } = firstMentionedSegment
+
+    timeline.setCursorTimestampAtInMs(startTimeInMs)
+  },
+
+}))
+
+
+if (typeof window !== "undefined") {
+  (window as any).useEditor = useEditor
+}

src/controllers/io/useIO.ts

@@ -2,6 +2,7 @@
 
 import { ClapProject, ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType, newSegment, parseClap, serializeClap } from "@aitube/clap"
 import { TimelineStore, useTimeline } from "@aitube/timeline"
+import { parseScriptToClap } from "@aitube/broadway"
 import { create } from "zustand"
 import { mltToXml } from "mlt-xml"
 
@@ -98,20 +99,25 @@ export const useIO = create<IOStore>((set, get) => ({
       message: "Analyzing screenplay..",
       value: 10
     })
-    try {
-        const res = await fetch("https://jbilcke-hf-broadway-api.hf.space", {
-          method: "POST",
-          headers: { 'Content-Type': 'text/plain' },
-          body: plainText,
-        })
-      const blob = await res.blob()
-        task.setProgress({
-          message: "Loading scenes..",
-          value: 50
-        })
-      // TODO: parseClap should feature a progress callback
-      const clap = await parseClap(blob)
 
+    try {
+      // this is the old way, based on a call to a separate API hosted on HF
+      // obviously this wasn't very practical and easy to scale, so I'm dropping it
+      //
+      // const res = await fetch("https://jbilcke-hf-broadway-api.hf.space", {
+      //   method: "POST",
+      //   headers: { 'Content-Type': 'text/plain' },
+      //   body: plainText,
+      // })
+      // const blob = await res.blob()
+      // task.setProgress({
+      //   message: "Loading scenes..",
+      //   value: 50
+      // })
+      // const clap = await parseClap(blob)
+
+      // new way: we analyze the screenplay on browser side
+      const clap = await parseScriptToClap(plainText)
       clap.meta.title = `${projectName || ""}`
 
       task.setProgress({
@@ -430,7 +436,7 @@ export const useIO = create<IOStore>((set, get) => ({
   saveKdenline: async () => {
     const { saveAnyFile } = get()
     const clap: ClapProject = useTimeline.getState().clap
-    // const tracks: Tracks = useTimeline.getState().tracks
+    // const tracks: ClapTracks = useTimeline.getState().tracks
 
     throw new Error(`cannot run in a browser, unfortunately`)
 

src/controllers/resolver/useResolver.ts

@@ -507,7 +507,18 @@ export const useResolver = create<ResolverStore>((set, get) => ({
       //
       // note that video clips are also concerned: we want them to perfectly fit
       if (newSegment.category === ClapSegmentCategory.DIALOGUE) {
-        await timeline.fitSegmentToAssetDuration(newSegment)
+        // by default fitSegmentToAssetDuration() will fit the segment to the asset duration without any gap,
+        // which can be weird to hear.. so let's add a little delay
+
+        // that is assuming that our dialogue lines have been properly cut,
+        // 
+        await timeline.fitSegmentToAssetDuration(
+          newSegment,
+          typeof newSegment.assetDurationInMs === "number"
+          // this delay is arbitrary, could be another value (200, 500, 1200..)
+          ? newSegment.assetDurationInMs + 700
+          : 2000
+        )
       } else if (newSegment.category === ClapSegmentCategory.VIDEO) {
         await timeline.fitSegmentToAssetDuration(newSegment)
       }

src/controllers/settings/getDefaultSettingsState.ts

@@ -84,18 +84,31 @@ export function getDefaultSettingsState(): SettingsState {
     comfyWorkflowForSound: "{}",
     comfyWorkflowForMusic: "{}",
   
+    // those are not designed for Hugging Face specifically,
+    // but to be compatible with any Gradio API URL that the
+    // user would set manually (eg. running on localhost)
+    gradioApiUrlForAssistant: "",
+    gradioApiUrlForImage: "",
+    gradioApiUrlForVideo: "",
+    gradioApiUrlForVoice: "",
+    gradioApiUrlForSound: "",
+    gradioApiUrlForMusic: "",
+
+    /******** should we deprecated all of those? or convert to defaults? ******
+     * 
     // now how we prefix everything with "models"
     // that's because it will be possible at some point to also
     // call a space eg. spaces/openai/sora (this one is just a silly example, of course)
     // "models/HuggingFaceH4/zephyr-7b-beta"
     // "models/mistralai/Mixtral-8x7B-Instruct-v0.1",
+
     huggingFaceModelForAssistant: "models/mistralai/Mixtral-8x7B-Instruct-v0.1",
     huggingFaceModelForImage: "models/sd-community/sdxl-flash",
     huggingFaceModelForImageDepth: "",
     huggingFaceModelForImageSegmentation: "",
     huggingFaceModelForImageUpscaling: "",
 
-    // huggingFaceModelForVideo: "spaces/multimodalart/hallo",
+    // huggingFaceModelForVideo: "spaces/jbilcke-hf/hallo-api",
     huggingFaceModelForVideo: "spaces/hpcai-tech/open-sora",
     huggingFaceModelForVideoDepth: "",
     huggingFaceModelForVideoSegmentation: "",
@@ -105,16 +118,6 @@ export function getDefaultSettingsState(): SettingsState {
     huggingFaceModelForSound: "",
     huggingFaceModelForMusic: "",
   
-    // those are not designed for Hugging Face specifically,
-    // but to be compatible with any Gradio API URL that the
-    // user would set manually (eg. running on localhost)
-    gradioApiUrlForAssistant: "",
-    gradioApiUrlForImage: "",
-    gradioApiUrlForVideo: "",
-    gradioApiUrlForVoice: "",
-    gradioApiUrlForSound: "",
-    gradioApiUrlForMusic: "",
-
     replicateModelForImage: "chenxwh/sdxl-flash:001bb81139b01780380407b4106ac681df46108e002eafbeb9ccb2d8faca42e1",
     replicateModelForImageDepth: "",
     replicateModelForImageSegmentation: "",
@@ -184,6 +187,8 @@ export function getDefaultSettingsState(): SettingsState {
     kitsAiModelForVoice: "",
     cohereModelForAssistant: "command-r-plus",
     mistralAiModelForAssistant: "open-mixtral-8x22b"
+
+    */
   }
   return state
 }

src/controllers/settings/types.ts

@@ -88,6 +88,16 @@ export type SettingsState = {
 
   // ---------- MODELS FOR EACH PROVIDER --------------
 
+
+  gradioApiUrlForAssistant: string
+  gradioApiUrlForImage: string
+  gradioApiUrlForVideo: string
+  gradioApiUrlForVoice: string
+  gradioApiUrlForSound: string
+  gradioApiUrlForMusic: string
+
+  /*
+  should we deprecate this? or rename to "default<something>"?
   huggingFaceModelForAssistant: string
   huggingFaceModelForImage: string
   huggingFaceModelForImageDepth: string
@@ -101,13 +111,6 @@ export type SettingsState = {
   huggingFaceModelForSound: string
   huggingFaceModelForMusic: string
 
-  gradioApiUrlForAssistant: string
-  gradioApiUrlForImage: string
-  gradioApiUrlForVideo: string
-  gradioApiUrlForVoice: string
-  gradioApiUrlForSound: string
-  gradioApiUrlForMusic: string
-
   replicateModelForImage: string
   replicateModelForImageDepth: string
   replicateModelForImageSegmentation: string
@@ -171,6 +174,7 @@ export type SettingsState = {
   cohereModelForAssistant: string
 
   mistralAiModelForAssistant: string
+  */
 }
 
 export type SettingsControls = {
@@ -251,6 +255,15 @@ export type SettingsControls = {
   setComfyWorkflowForSound: (comfyWorkflowForSound?: string) => void
   setComfyWorkflowForMusic: (comfyWorkflowForMusic?: string) => void
 
+  setGradioApiUrlForAssistant: (gradioApiUrlForAssistant?: string) => void
+  setGradioApiUrlForImage: (gradioApiUrlForImage?: string) => void
+  setGradioApiUrlForVideo: (gradioApiUrlForVideo?: string) => void
+  setGradioApiUrlForVoice: (gradioApiUrlForVoice?: string) => void
+  setGradioApiUrlForSound: (gradioApiUrlForSound?: string) => void
+  setGradioApiUrlForMusic: (gradioApiUrlForMusic?: string) => void
+
+  /*
+  should we deprecate this? or rename to "default<something>"?
   setHuggingFaceModelForAssistant: (huggingFaceModelForAssistant?: string) => void
   setHuggingFaceModelForImage: (huggingFaceModelForImage?: string) => void
   setHuggingFaceModelForImageDepth: (huggingFaceModelForImageDepth?: string) => void
@@ -264,13 +277,6 @@ export type SettingsControls = {
   setHuggingFaceModelForSound: (huggingFaceModelForSound?: string) => void
   setHuggingFaceModelForMusic: (huggingFaceModelForMusic?: string) => void
 
-  setGradioApiUrlForAssistant: (gradioApiUrlForAssistant?: string) => void
-  setGradioApiUrlForImage: (gradioApiUrlForImage?: string) => void
-  setGradioApiUrlForVideo: (gradioApiUrlForVideo?: string) => void
-  setGradioApiUrlForVoice: (gradioApiUrlForVoice?: string) => void
-  setGradioApiUrlForSound: (gradioApiUrlForSound?: string) => void
-  setGradioApiUrlForMusic: (gradioApiUrlForMusic?: string) => void
-
   setReplicateModelForImage: (replicateModelForImage?: string) => void
   setReplicateModelForImageDepth: (replicateModelForImageDepth?: string) => void
   setReplicateModelForImageSegmentation: (replicateModelForImageSegmentation?: string) => void
@@ -333,7 +339,7 @@ export type SettingsControls = {
   setMistralAiModelForAssistant: (mistralAiModelForAssistant?: string) => void
   
   setKitsAiModelForVoice: (kitsAiModelForVoice?: string) => void
-  
+  */
   getSettings: () => SettingsState
 }
 

src/controllers/settings/useSettings.ts

@@ -281,7 +281,26 @@ export const useSettings = create<SettingsStore>()(
       setComfyWorkflowForMusic: (comfyWorkflowForMusic?: string) => {
         set({ comfyWorkflowForMusic: getValidComfyWorkflowTemplate(comfyWorkflowForMusic, getDefaultSettingsState().comfyWorkflowForMusic) })
       },
-
+      setGradioApiUrlForAssistant: (gradioApiUrlForAssistant?: string) => {
+        set({ gradioApiUrlForAssistant: getValidString(gradioApiUrlForAssistant, getDefaultSettingsState().gradioApiUrlForAssistant) })
+      },
+      setGradioApiUrlForImage: (gradioApiUrlForImage?: string) => {
+        set({ gradioApiUrlForImage: getValidString(gradioApiUrlForImage, getDefaultSettingsState().gradioApiUrlForImage) })
+      },
+      setGradioApiUrlForVideo: (gradioApiUrlForVideo?: string) => {
+        set({ gradioApiUrlForVideo: getValidString(gradioApiUrlForVideo, getDefaultSettingsState().gradioApiUrlForVideo) })
+      },
+      setGradioApiUrlForVoice: (gradioApiUrlForVoice?: string) => {
+        set({ gradioApiUrlForVoice: getValidString(gradioApiUrlForVoice, getDefaultSettingsState().gradioApiUrlForVoice) })
+      },
+      setGradioApiUrlForSound: (gradioApiUrlForSound?: string) => {
+        set({ gradioApiUrlForSound: getValidString(gradioApiUrlForSound, getDefaultSettingsState().gradioApiUrlForSound) })
+      },
+      setGradioApiUrlForMusic: (gradioApiUrlForMusic?: string) => {
+        set({ gradioApiUrlForMusic: getValidString(gradioApiUrlForMusic, getDefaultSettingsState().gradioApiUrlForMusic) })
+      },
+      /*
+      should we deprecate this? or rename to "default<something>"?
       setHuggingFaceModelForAssistant: (huggingFaceModelForAssistant?: string) => {
         set({ huggingFaceModelForAssistant: getValidString(huggingFaceModelForAssistant, getDefaultSettingsState().huggingFaceModelForAssistant) })
       },
@@ -318,24 +337,6 @@ export const useSettings = create<SettingsStore>()(
       setHuggingFaceModelForMusic: (huggingFaceModelForMusic?: string) => {
         set({ huggingFaceModelForMusic: getValidString(huggingFaceModelForMusic, getDefaultSettingsState().huggingFaceModelForMusic) })
       },
-      setGradioApiUrlForAssistant: (gradioApiUrlForAssistant?: string) => {
-        set({ gradioApiUrlForAssistant: getValidString(gradioApiUrlForAssistant, getDefaultSettingsState().gradioApiUrlForAssistant) })
-      },
-      setGradioApiUrlForImage: (gradioApiUrlForImage?: string) => {
-        set({ gradioApiUrlForImage: getValidString(gradioApiUrlForImage, getDefaultSettingsState().gradioApiUrlForImage) })
-      },
-      setGradioApiUrlForVideo: (gradioApiUrlForVideo?: string) => {
-        set({ gradioApiUrlForVideo: getValidString(gradioApiUrlForVideo, getDefaultSettingsState().gradioApiUrlForVideo) })
-      },
-      setGradioApiUrlForVoice: (gradioApiUrlForVoice?: string) => {
-        set({ gradioApiUrlForVoice: getValidString(gradioApiUrlForVoice, getDefaultSettingsState().gradioApiUrlForVoice) })
-      },
-      setGradioApiUrlForSound: (gradioApiUrlForSound?: string) => {
-        set({ gradioApiUrlForSound: getValidString(gradioApiUrlForSound, getDefaultSettingsState().gradioApiUrlForSound) })
-      },
-      setGradioApiUrlForMusic: (gradioApiUrlForMusic?: string) => {
-        set({ gradioApiUrlForMusic: getValidString(gradioApiUrlForMusic, getDefaultSettingsState().gradioApiUrlForMusic) })
-      },
       setReplicateModelForImage: (replicateModelForImage?: string) => {
         set({ replicateModelForImage: getValidString(replicateModelForImage, getDefaultSettingsState().replicateModelForImage) })
       },
@@ -489,6 +490,7 @@ export const useSettings = create<SettingsStore>()(
       setKitsAiModelForVoice: (kitsAiModelForVoice?: string) => {
         set({ kitsAiModelForVoice: getValidString(kitsAiModelForVoice, getDefaultSettingsState().kitsAiModelForVoice) })
       },
+      */
       getSettings: (): SettingsState => {
         const state = get()
         const defaultSettings = getDefaultSettingsState()
@@ -570,6 +572,15 @@ export const useSettings = create<SettingsStore>()(
           comfyWorkflowForVoice: state.comfyWorkflowForVoice || defaultSettings.comfyWorkflowForVoice,
           comfyWorkflowForSound: state.comfyWorkflowForSound || defaultSettings.comfyWorkflowForSound,
           comfyWorkflowForMusic: state.comfyWorkflowForMusic || defaultSettings.comfyWorkflowForMusic,
+          gradioApiUrlForAssistant: state.gradioApiUrlForAssistant || defaultSettings.gradioApiUrlForAssistant,
+          gradioApiUrlForImage: state.gradioApiUrlForImage || defaultSettings.gradioApiUrlForImage,
+          gradioApiUrlForVideo: state.gradioApiUrlForVideo || defaultSettings.gradioApiUrlForVideo,
+          gradioApiUrlForVoice: state.gradioApiUrlForVoice || defaultSettings.gradioApiUrlForVoice,
+          gradioApiUrlForSound: state.gradioApiUrlForSound || defaultSettings.gradioApiUrlForSound,
+          gradioApiUrlForMusic: state.gradioApiUrlForMusic || defaultSettings.gradioApiUrlForMusic,
+          
+          /*
+          should we deprecate this? or rename to "default<something>"?
           huggingFaceModelForAssistant: state.huggingFaceModelForAssistant || defaultSettings.huggingFaceModelForAssistant,
           huggingFaceModelForImage: state.huggingFaceModelForImage || defaultSettings.huggingFaceModelForImage,
           huggingFaceModelForImageDepth: state.huggingFaceModelForImageDepth || defaultSettings.huggingFaceModelForImageDepth,
@@ -582,12 +593,6 @@ export const useSettings = create<SettingsStore>()(
           huggingFaceModelForVoice: state.huggingFaceModelForVoice || defaultSettings.huggingFaceModelForVoice,
           huggingFaceModelForSound: state.huggingFaceModelForSound || defaultSettings.huggingFaceModelForSound,
           huggingFaceModelForMusic: state.huggingFaceModelForMusic || defaultSettings.huggingFaceModelForMusic,
-          gradioApiUrlForAssistant: state.gradioApiUrlForAssistant || defaultSettings.gradioApiUrlForAssistant,
-          gradioApiUrlForImage: state.gradioApiUrlForImage || defaultSettings.gradioApiUrlForImage,
-          gradioApiUrlForVideo: state.gradioApiUrlForVideo || defaultSettings.gradioApiUrlForVideo,
-          gradioApiUrlForVoice: state.gradioApiUrlForVoice || defaultSettings.gradioApiUrlForVoice,
-          gradioApiUrlForSound: state.gradioApiUrlForSound || defaultSettings.gradioApiUrlForSound,
-          gradioApiUrlForMusic: state.gradioApiUrlForMusic || defaultSettings.gradioApiUrlForMusic,
           replicateModelForImage: state.replicateModelForImage || defaultSettings.replicateModelForImage,
           replicateModelForImageDepth: state.replicateModelForImageDepth || defaultSettings.replicateModelForImageDepth,
           replicateModelForImageSegmentation: state.replicateModelForImageSegmentation || defaultSettings.replicateModelForImageSegmentation,
@@ -639,6 +644,7 @@ export const useSettings = create<SettingsStore>()(
           cohereModelForAssistant: state.cohereModelForAssistant || defaultSettings.cohereModelForAssistant,
           mistralAiModelForAssistant: state.mistralAiModelForAssistant || defaultSettings.mistralAiModelForAssistant,
           kitsAiModelForVoice: state.kitsAiModelForVoice || defaultSettings.kitsAiModelForVoice,
+          */
         }
       },
     }),
@@ -646,4 +652,8 @@ export const useSettings = create<SettingsStore>()(
       name: 'CLAPPER_REVISION_0_CONTROLLERS_USE_SETTINGS'
     },
   ),
-)
+)
+
+if (typeof window !== "undefined") {
+  (window as any).useSettings = useSettings
+}

src/lib/core/constants.ts

@@ -4,7 +4,7 @@
 export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
 
 export const APP_NAME = "Clapper.app"
-export const APP_REVISION = "r20240617-0204"
+export const APP_REVISION = "r20240623-1700"
 
 export const APP_DOMAIN = "Clapper.app"
 export const APP_LINK = "https://clapper.app"

src/lib/hf/adapter/adaptAnyInputsToGradioInputs.ts

@@ -2,6 +2,7 @@ import { GradioApiInfo, SupportedFields } from "../types"
 import { identifyField } from "./identifyField"
 import { getDefaultFields } from "./getDefaultFields"
 import { findMainGradioEndpoint } from "./findMainGradioEndpoint"
+import { base64DataUriToBlob } from "@/lib/utils/base64DataUriToBlob"
 
 /**
  * This function try to adapt arbitrary inputs to strict gradio inputs
@@ -17,7 +18,8 @@ export function adaptAnyInputsToGradioInputs({
   gradioApiInfo: GradioApiInfo
 }): {
   endpoint: string
-  inputs: Array<string | number | boolean | undefined | null>
+  inputArray: Array<string | number | boolean | Blob | undefined | null>
+  inputMap: Record<string, string | number | boolean | Blob | undefined | null>
  } {
 
   const mainGradioEndpoint = findMainGradioEndpoint({ gradioApiInfo })
@@ -34,9 +36,12 @@ export function adaptAnyInputsToGradioInputs({
     inputFields[key] = inputField
     allInputFields = {...allInputFields, ...inputField}
   }
+  console.log(`input fields passed by the parent calling function:`, inputFields)
 
   // the gradio input array
-  const gradioInputs: any[] = []
+  // apparently the new JS client also supports dictionaries, yay!
+  let inputArray: Array<string | number | boolean | Blob | undefined | null> = []
+  let inputMap: Record<string, string | number | boolean | Blob | undefined | null> = {}
 
   for (const parameter of mainGradioEndpoint.endpoint.parameters) {
     let gradioInputValue: any = undefined
@@ -55,11 +60,32 @@ export function adaptAnyInputsToGradioInputs({
     if (fields.hasInputGuidance) { gradioInputValue = allInputFields.inputGuidance }
     if (fields.hasInputSeed) { gradioInputValue = allInputFields.inputSeed }
 
-    gradioInputs.push(gradioInputValue)
+    //console.log("parameter:", parameter)
+    const valueSeemsToBeABase64Uri = typeof gradioInputValue === "string" && gradioInputValue.startsWith("data:")
+    const fieldSeemsToBeTextBased =
+      parameter.type === "string"
+      || parameter.component === "Textbox"
+      // || parameter.parameter_name.includes("base64")
+
+    // the magic doesn't end here: we need to convert base64 inputs to buffers,
+    // unless gradio expects it to be a text
+    if (valueSeemsToBeABase64Uri && !fieldSeemsToBeTextBased) {
+      gradioInputValue = base64DataUriToBlob(gradioInputValue)
+    }
+    // old, low-level way
+    inputArray.push(gradioInputValue)
+
+    // new, high-level way
+    inputMap[parameter.parameter_name] = gradioInputValue
   }
 
+  console.log(`inputArray:`, inputArray.map(x => typeof x === "string" ? x.slice(0, 255) : x))
+  
+  console.log(`await client.predict("${ mainGradioEndpoint.name}", `, inputMap);
+
   return {
     endpoint: mainGradioEndpoint.name,
-    inputs: gradioInputs
+    inputArray,
+    inputMap,
   }
 }

src/lib/hf/adapter/identifyField.ts

@@ -1,9 +1,17 @@
 import { SupportedFields } from "../types"
 
 export function identifyField(key: string, value?: any, index?: number): Partial<SupportedFields> {
-  const normalizedKey = key.toLowerCase().trim()
+  const normalizedKey =
+    key
+      .toLowerCase()
+      .replaceAll("_uri", "")
+      .replaceAll("_url", "")
+      .replaceAll("_b64", "")
+      .replaceAll("_base64", "")
+      .trim()
+  console.log(`normalizedKey: ${normalizedKey}`)
+  
   switch (normalizedKey) {
-
     case "width":
       let strWidth = ""
       let numWidth = 0

src/lib/hf/callGradioApi.ts

@@ -48,6 +48,7 @@ export async function callGradioApi<T>({
   }
   */
 
+
   const gradioApiInfo = await getGradioApiInfo({
     url: ownerAndId,
     apiKey
@@ -61,17 +62,20 @@ export async function callGradioApi<T>({
   })
   
   // console.log(`gradioEndpointInputs: `, gradioEndpointInputs)
-  
+
   const app = await Client.connect(ownerAndId, {
     hf_token: apiKey as any
   })
   // console.log(`app: `, app)
   
+  console.log(`calling Gradio API ${ownerAndId}:${gradioEndpointInputs.endpoint}`)
   const output = await app.predict(
     gradioEndpointInputs.endpoint,
-    gradioEndpointInputs.inputs
+    gradioEndpointInputs.inputMap
   )
-  console.log(`output: `, output)
-  
-  return output.data as unknown as T
-}
+  // console.log(`output: `, output)
+ 
+  const data1 = (Array.isArray( output.data) ?  output.data[0] : "") || ""
+
+  return data1 as unknown as T
+}

src/lib/hf/getSpaces.ts

@@ -44,6 +44,7 @@ export async function getSpaces({
   })) {
 
     if (sdk && space.sdk != sdk) { continue }
+
     results.push(space)
   }
 

src/lib/utils/base64DataUriToBlob.ts

@@ -0,0 +1,15 @@
+export function base64DataUriToBlob(dataURI: string) {
+  dataURI = dataURI.replace(/^data:/, '');
+
+  const match = dataURI.match(/(?:image|video|audio|text)\/[^;]+/)
+  const type = match?.[0] || ""
+  const base64 = dataURI.replace(/^[^,]+,/, '');
+  const arrayBuffer = new ArrayBuffer(base64.length);
+  const typedArray = new Uint8Array(arrayBuffer);
+
+  for (let i = 0; i < base64.length; i++) {
+    typedArray[i] = base64.charCodeAt(i);
+  }
+
+  return new Blob([arrayBuffer], { type });
+}

tailwind.config.js

@@ -17,6 +17,10 @@ module.exports = {
       },
     },
     extend: {
+      aspectRatio: {
+        '1024/576': '1024 / 576',
+      },
+    
       fontFamily: {
         salsa: ['var(--font-salsa)'],
         clock: ["var(--font-clock)"],

tsconfig.json

@@ -1,11 +1,9 @@
 {
   "compilerOptions": {
-    "target": "ES2022",
     "lib": ["dom", "dom.iterable", "esnext"],
     "allowJs": true,
     "skipLibCheck": true,
     "strict": true,
-    "forceConsistentCasingInFileNames": true,
     "noEmit": true,
     "esModuleInterop": true,
     "module": "esnext",