let's add some music

.env

@@ -1,8 +1,9 @@
 HF_API_TOKEN="<USE YOUR OWN>"
 MICROSERVICE_API_SECRET_TOKEN="<USE YOUR OWN>"
 
-AI_TUBE_URL="https://aitube.at"
-# AI_TUBE_URL="http://localhost:3000"
+# you can use this to bypass AiTube client URL
+#AITUBE_URL="https://aitube.at"
+#AITUBE_URL="http://localhost:3000"
 
 AI_TUBE_API_SECRET_JWT_KEY=""
 AI_TUBE_API_SECRET_JWT_ISSUER=""

package-lock.json

@@ -8,8 +8,8 @@
       "name": "@jbilcke-hf/ai-stories-factory",
       "version": "0.0.0",
       "dependencies": {
-        "@aitube/clap": "0.0.16",
-        "@aitube/client": "0.0.21",
+        "@aitube/clap": "0.0.17",
+        "@aitube/client": "0.0.25",
         "@radix-ui/react-accordion": "^1.1.2",
         "@radix-ui/react-avatar": "^1.0.4",
         "@radix-ui/react-checkbox": "^1.0.4",
@@ -66,9 +66,9 @@
       }
     },
     "node_modules/@aitube/clap": {
-      "version": "0.0.16",
-      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.16.tgz",
-      "integrity": "sha512-EcBu4gbc8Kx58rkMmLjh0uyCWVR4cSgWqALlQ50C1GbHRg/r81gDET6faWeos14ZGrtg4B0CJv8WUTIDE54pjg==",
+      "version": "0.0.17",
+      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.17.tgz",
+      "integrity": "sha512-g/jjePX2Hz9Eo4hk+rxd6FRwoy2Hx9sadGLgN9yWSm7dGHhr9B/DVv8eLjFabu7jgW0zvZZ1FHvlsNAby4Pr/Q==",
       "dependencies": {
         "pure-uuid": "^1.8.1",
         "yaml": "^2.4.1"
@@ -78,14 +78,14 @@
       }
     },
     "node_modules/@aitube/client": {
-      "version": "0.0.21",
-      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.21.tgz",
-      "integrity": "sha512-Nw/K4j024ffYiw1WipLDU7M29L+4cM4cQwrjvSxWW8zF4/1NLrPSKTlF6Gak5Qd/ZFWq/D8GbMiGQ5W3lQb8mA==",
+      "version": "0.0.25",
+      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.25.tgz",
+      "integrity": "sha512-gX5eJOKiigVY3xK1NcsStruUuWMQbj4o4XHTceZhUyKCgHDldC0Y15mvIWVabCtEW5FFebdmhH0EFeg+PBMCsg==",
       "dependencies": {
         "query-string": "^9.0.0"
       },
       "peerDependencies": {
-        "@aitube/clap": "0.0.16"
+        "@aitube/clap": "0.0.17"
       }
     },
     "node_modules/@alloc/quick-lru": {
@@ -3129,9 +3129,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001616",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001616.tgz",
-      "integrity": "sha512-RHVYKov7IcdNjVHJFNY/78RdG4oGVjbayxv8u5IO74Wv7Hlq4PnJE6mo/OjFijjVFNy5ijnCt6H3IIo4t+wfEw==",
+      "version": "1.0.30001617",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001617.tgz",
+      "integrity": "sha512-mLyjzNI9I+Pix8zwcrpxEbGlfqOkF9kM3ptzmKNw5tizSyYwMe+nGLTqMK9cO+0E+Bh6TsBxNAaHWEM8xwSsmA==",
       "funding": [
         {
           "type": "opencollective",
@@ -3748,9 +3748,9 @@
       "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
     },
     "node_modules/electron-to-chromium": {
-      "version": "1.4.759",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.759.tgz",
-      "integrity": "sha512-qZJc+zsuI+/5UjOSFnpkJBwwLMH1AZgyKqJ7LUNnRsB7v/cDjMu9DvXgp9kH6PTTZxjnPXGp2Uhurw+2Ll4Hjg=="
+      "version": "1.4.762",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.762.tgz",
+      "integrity": "sha512-rrFvGweLxPwwSwJOjIopy3Vr+J3cIPtZzuc74bmlvmBIgQO3VYJDvVrlj94iKZ3ukXUH64Ex31hSfRTLqvjYJQ=="
     },
     "node_modules/emoji-regex": {
       "version": "9.2.2",
@@ -4602,9 +4602,9 @@
       }
     },
     "node_modules/get-tsconfig": {
-      "version": "4.7.4",
-      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.4.tgz",
-      "integrity": "sha512-ofbkKj+0pjXjhejr007J/fLf+sW+8H7K5GCm+msC8q3IpvgjobpyPqSRFemNyIMxklC0zeJpi7VDFna19FacvQ==",
+      "version": "4.7.5",
+      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.5.tgz",
+      "integrity": "sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==",
       "dependencies": {
         "resolve-pkg-maps": "^1.0.0"
       },
@@ -5347,9 +5347,9 @@
       }
     },
     "node_modules/jose": {
-      "version": "5.2.4",
-      "resolved": "https://registry.npmjs.org/jose/-/jose-5.2.4.tgz",
-      "integrity": "sha512-6ScbIk2WWCeXkmzF6bRPmEuaqy1m8SbsRFMa/FLrSCkGIhj8OLVG/IH+XHVmNMx/KUo8cVWEE6oKR4dJ+S0Rkg==",
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/jose/-/jose-5.3.0.tgz",
+      "integrity": "sha512-IChe9AtAE79ru084ow8jzkN2lNrG3Ntfiv65Cvj9uOCE2m5LNsdHG+9EbxWxAoWRF9TgDOqLN5jm08++owDVRg==",
       "funding": {
         "url": "https://github.com/sponsors/panva"
       }
@@ -5594,9 +5594,9 @@
       }
     },
     "node_modules/minipass": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.0.tgz",
-      "integrity": "sha512-oGZRv2OT1lO2UF1zUcwdTb3wqUwI0kBGTgt/T7OdSj6M6N5m3o5uPf0AIW6lVxGGoiWUR7e2AwTE+xiwK8WQig==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.1.tgz",
+      "integrity": "sha512-UZ7eQ+h8ywIRAW1hIEl2AqdwzJucU/Kp59+8kkZeSvafXhZjul247BvIJjEVFVeON6d7lM46XX1HXCduKAS8VA==",
       "engines": {
         "node": ">=16 || 14 >=14.17"
       }
@@ -6015,9 +6015,9 @@
       "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="
     },
     "node_modules/path-scurry": {
-      "version": "1.10.2",
-      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz",
-      "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==",
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.0.tgz",
+      "integrity": "sha512-LNHTaVkzaYaLGlO+0u3rQTz7QrHTFOuKyba9JMTQutkmtNew8dw8wOD7mTU/5fCPZzCWpfW0XnQKzY61P0aTaw==",
       "dependencies": {
         "lru-cache": "^10.2.0",
         "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0"
@@ -6742,9 +6742,9 @@
       }
     },
     "node_modules/semver": {
-      "version": "7.6.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.1.tgz",
-      "integrity": "sha512-f/vbBsu+fOiYt+lmwZV0rVwJScl46HppnOA1ZvIuBWKOTlllpyJ3bfVax76/OrhCH38dyxoDIA8K7uB963IYgA==",
+      "version": "7.6.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz",
+      "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==",
       "bin": {
         "semver": "bin/semver.js"
       },

package.json

@@ -9,8 +9,8 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@aitube/clap": "0.0.16",
-    "@aitube/client": "0.0.21",
+    "@aitube/clap": "0.0.17",
+    "@aitube/client": "0.0.25",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-avatar": "^1.0.4",
     "@radix-ui/react-checkbox": "^1.0.4",

src/app/main.tsx

@@ -2,7 +2,7 @@
 
 import React, { useEffect, useRef, useTransition } from 'react'
 import { IoMdPhonePortrait } from 'react-icons/io'
-import { ClapProject, ClapMediaOrientation, ClapSegmentCategory } from '@aitube/clap'
+import { ClapProject, ClapMediaOrientation, ClapSegmentCategory, updateClap } from '@aitube/clap'
 import Image from 'next/image'
 import { useFilePicker } from 'use-file-picker'
 import { DeviceFrameset } from 'react-device-frameset'
@@ -18,6 +18,8 @@ import { createClap } from './server/aitube/createClap'
 import { editClapEntities } from './server/aitube/editClapEntities'
 import { editClapDialogues } from './server/aitube/editClapDialogues'
 import { editClapStoryboards } from './server/aitube/editClapStoryboards'
+import { editClapMusic } from './server/aitube/editClapMusic'
+import { editClapVideos } from './server/aitube/editClapVideos'
 import { exportClapToVideo } from './server/aitube/exportClapToVideo'
 
 import { useStore } from './store'
@@ -42,9 +44,11 @@ export function Main() {
   const parseGenerationStatus = useStore(s => s.parseGenerationStatus)
   const storyGenerationStatus = useStore(s => s.storyGenerationStatus)
   const assetGenerationStatus = useStore(s => s.assetGenerationStatus)
+  const musicGenerationStatus = useStore(s => s.musicGenerationStatus)
   const voiceGenerationStatus = useStore(s => s.voiceGenerationStatus)
   const imageGenerationStatus = useStore(s => s.imageGenerationStatus)
   const videoGenerationStatus = useStore(s => s.videoGenerationStatus)
+  const finalGenerationStatus = useStore(s => s.finalGenerationStatus)
   const currentClap = useStore(s => s.currentClap)
   const currentVideo = useStore(s => s.currentVideo)
   const currentVideoOrientation = useStore(s => s.currentVideoOrientation)
@@ -59,9 +63,11 @@ export function Main() {
   const setParseGenerationStatus = useStore(s => s.setParseGenerationStatus)
   const setStoryGenerationStatus = useStore(s => s.setStoryGenerationStatus)
   const setAssetGenerationStatus = useStore(s => s.setAssetGenerationStatus)
+  const setMusicGenerationStatus = useStore(s => s.setMusicGenerationStatus)
   const setVoiceGenerationStatus = useStore(s => s.setVoiceGenerationStatus)
   const setImageGenerationStatus = useStore(s => s.setImageGenerationStatus)
   const setVideoGenerationStatus = useStore(s => s.setVideoGenerationStatus)
+  const setFinalGenerationStatus = useStore(s => s.setFinalGenerationStatus)
   const setCurrentClap = useStore(s => s.setCurrentClap)
   const setCurrentVideo = useStore(s => s.setCurrentVideo)
   const progress = useStore(s => s.progress)
@@ -87,7 +93,7 @@ export function Main() {
         setStatus,
         setProgress,
         setParseGenerationStatus,
-        setVideoGenerationStatus,
+        setFinalGenerationStatus,
       } = useStore.getState()
 
       let clap: ClapProject | undefined = undefined
@@ -113,6 +119,7 @@ export function Main() {
 
       setParseGenerationStatus("finished")
 
+      /*
       try {
         setProgress(60)
         setVoiceGenerationStatus("generating")
@@ -135,8 +142,9 @@ export function Main() {
       if (!clap) {
         return
       }
+      */
 
-      setVideoGenerationStatus("generating")
+      setFinalGenerationStatus("generating")
   
       let assetUrl = ""
       try {
@@ -150,13 +158,13 @@ export function Main() {
       }
     
       if (!assetUrl) {
-        setVideoGenerationStatus("error")
+        setFinalGenerationStatus("error")
         setStatus("error")
         setProgress(0)
         return
       }
 
-      setVideoGenerationStatus("finished")
+      setFinalGenerationStatus("finished")
 
       setProgress(80)
   
@@ -169,41 +177,30 @@ export function Main() {
     fn()
   }, [fileData?.name])
   
-  const handleSubmit = async () => {
+  const generateStory = async (): Promise<ClapProject> => {
 
-    startTransition(async () => {
-      console.log(`handleSubmit(): generating a clap using prompt = "${promptDraft.current}" `)
+    let clap: ClapProject | undefined = undefined
+    try {
+      setProgress(0)
 
-      let clap: ClapProject | undefined = undefined
-      try {
-        setProgress(0)
+      setStatus("generating")
+      setStoryGenerationStatus("generating")
+      setStoryPrompt(promptDraft.current)
 
-        setStatus("generating")
-        setStoryGenerationStatus("generating")
-        setStoryPrompt(promptDraft.current)
+      clap = await createClap({
+        prompt: promptDraft.current,
+        orientation: useStore.getState().orientation,
 
-        clap = await createClap({
-          prompt: promptDraft.current,
-          orientation: useStore.getState().orientation,
-          turbo: true,
-        })
+        turbo: true,
+      })
 
-        if (!clap) { throw new Error(`failed to create the clap`) }
+      if (!clap) { throw new Error(`failed to create the clap`) }
 
-        if (clap.segments.length <= 1) { throw new Error(`failed to generate more than one segments`) }
+      if (clap.segments.length <= 1) { throw new Error(`failed to generate more than one segments`) }
 
-        console.log(`handleSubmit(): received a clap = `, clap)
-        setCurrentClap(clap)
-        setStoryGenerationStatus("finished")
-      } catch (err) {
-        setStoryGenerationStatus("error")
-        setStatus("error")
-        setError(`${err}`)
-        return
-      }
-      if (!clap) {
-        return
-      }
+      console.log(`handleSubmit(): received a clap = `, clap)
+      setCurrentClap(clap)
+      setStoryGenerationStatus("finished")
 
       console.log("-------- GENERATED STORY --------")
       console.table(clap.segments, [
@@ -213,34 +210,30 @@ export function Main() {
         'category',
         'prompt'
       ])
+      return clap
+    } catch (err) {
+      setStoryGenerationStatus("error")
+      throw err
+    }
+  }
 
- 
-      try {
-        setProgress(20)
-        setAssetGenerationStatus("generating")
-        clap = await editClapEntities({
-          clap,
+  const generateEntities = async (clap: ClapProject): Promise<ClapProject> => {
+    try {
+      setProgress(20)
+      setAssetGenerationStatus("generating")
+      clap = await editClapEntities({
+        clap,
 
-          // generating entities requires a "smart" LLM
-          turbo: false,
-          // turbo: true,
-        })
+        // generating entities requires a "smart" LLM
+        turbo: false,
+        // turbo: true,
+      })
 
-        if (!clap) { throw new Error(`failed to edit the entities`) }
+      if (!clap) { throw new Error(`failed to edit the entities`) }
 
-        console.log(`handleSubmit(): received a clap with entities = `, clap)
-        setCurrentClap(clap)
-        setAssetGenerationStatus("finished")
-      } catch (err) {
-        setAssetGenerationStatus("error")
-        setStatus("error")
-        setError(`${err}`)
-        return
-      }
-      if (!clap) {
-        return
-      }
-      
+      console.log(`handleSubmit(): received a clap with entities = `, clap)
+      setCurrentClap(clap)
+      setAssetGenerationStatus("finished")
       console.log("-------- GENERATED ENTITIES --------")
       console.table(clap.entities, [
         'category',
@@ -248,110 +241,209 @@ export function Main() {
         'imagePrompt',
         'appearance'
       ])
+      return clap
+    } catch (err) {
+      setAssetGenerationStatus("error")
+      throw err
+    }
+  }
 
-      /*
-      if (mainCharacterImage) {
-        console.log("handleSubmit(): User specified a main character image")
-        // various strategies here, for instance we can assume that the first character is the main character,
-        // or maybe a more reliable way is to count the number of occurrences.
-        // there is a risk of misgendering, so ideally we should add some kind of UI to do this,
-        // such as a list of characters.
-      }
-      */
 
-      // TODO Julian
-      console.log("handleSubmit(): TODO Julian: generate images in parallel of the dialogue using Promise.all()")
-      // this is not trivial to do btw, since we will have to merge the clap together
-      // (this could be a helper function inside @aitube/clap)
-      try {
-        setProgress(40)
-        setImageGenerationStatus("generating")
-        clap = await editClapStoryboards({
-          clap,
-          // the turbo is mandatory here,
-          // since this uses a model with character consistency,
-          // which is not the case for the non-turbo one
-          turbo: true
-        })
+  const generateMusic = async (clap: ClapProject): Promise<ClapProject> => {
+    try {
+      setProgress(30)
+      setMusicGenerationStatus("generating")
 
-        if (!clap) { throw new Error(`failed to edit the storyboards`) }
+      clap = await editClapMusic({
+        clap,
+        turbo: true
+      })
 
-        // const fusion = 
-        console.log(`handleSubmit(): received a clap with images = `, clap)
-        setCurrentClap(clap)
-        setImageGenerationStatus("finished")
-      } catch (err) {
-        setImageGenerationStatus("error")
-        setStatus("error")
-        setError(`${err}`)
-        return
-      }
-      if (!clap) {
-        return
-      }
+      if (!clap) { throw new Error(`failed to edit the music`) }
+
+      console.log(`handleSubmit(): received a clap with music = `, clap)
+      setCurrentClap(clap)
+      setMusicGenerationStatus("finished")
+      console.log("-------- GENERATED MUSIC --------")
+      console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.MUSIC), [
+        'endTimeInMs',
+        'prompt',
+        'entityId',
+      ])
+      return clap
+    } catch (err) {
+      setMusicGenerationStatus("error")
+      throw err
+    }
+  }
 
+  const generateStoryboards = async (clap: ClapProject): Promise<ClapProject> => {
+    try {
+      setProgress(40)
+      setImageGenerationStatus("generating")
+      clap = await editClapStoryboards({
+        clap,
+        // the turbo is mandatory here,
+        // since this uses a model with character consistency,
+        // which is not the case for the non-turbo one
+        turbo: true
+      })
+
+      if (!clap) { throw new Error(`failed to edit the storyboards`) }
+
+      // const fusion = 
+      console.log(`handleSubmit(): received a clap with images = `, clap)
+      setCurrentClap(clap)
+      setImageGenerationStatus("finished")
       console.log("-------- GENERATED STORYBOARDS --------")
       console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD), [
         'endTimeInMs',
         'prompt',
         'assetUrl'
       ])
+      return clap
+    } catch (err) {
+      setImageGenerationStatus("error")
+      throw err
+    }
+  }
 
-      
-      try {
-        setProgress(50)
-        setVoiceGenerationStatus("generating")
-        clap = await editClapDialogues({
-          clap,
-          turbo: true
-        })
+  const generateVideos = async (clap: ClapProject): Promise<ClapProject> => {
+    try {
+      setProgress(50)
+      setVideoGenerationStatus("generating")
 
-        if (!clap) { throw new Error(`failed to edit the dialogues`) }
+      clap = await editClapVideos({
+        clap,
+        turbo: true
+      })
 
-        console.log(`handleSubmit(): received a clap with dialogues = `, clap)
-        setCurrentClap(clap)
-        setVoiceGenerationStatus("finished")
-      } catch (err) {
-        setVoiceGenerationStatus("error")
-        setStatus("error")
-        setError(`${err}`)
-        return
-      }
-      if (!clap) {
-        console.log("aborting prematurely")
-        return
-      }
+      if (!clap) { throw new Error(`failed to edit the videos`) }
 
+      console.log(`handleSubmit(): received a clap with videos = `, clap)
+      setCurrentClap(clap)
+      setVideoGenerationStatus("finished")
+      console.log("-------- GENERATED VIDEOS --------")
+      console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.VIDEO), [
+        'endTimeInMs',
+        'prompt',
+        'entityId',
+      ])
+      return clap
+    } catch (err) {
+      setVideoGenerationStatus("error")
+      throw err
+    }
+  }
+
+  const generateDialogues = async (clap: ClapProject): Promise<ClapProject> => {
+    try {
+      setProgress(70)
+      setVoiceGenerationStatus("generating")
+      clap = await editClapDialogues({
+        clap,
+        turbo: true
+      })
+
+      if (!clap) { throw new Error(`failed to edit the dialogues`) }
+
+      console.log(`handleSubmit(): received a clap with dialogues = `, clap)
+      setCurrentClap(clap)
+      setVoiceGenerationStatus("finished")
       console.log("-------- GENERATED DIALOGUES --------")
       console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.DIALOGUE), [
         'endTimeInMs',
         'prompt',
         'entityId',
       ])
+      return clap
+    } catch (err) {
+      setVoiceGenerationStatus("error")
+      throw err
+    }
+  }
+
+  const generateFinalVideo = async (clap: ClapProject): Promise<string> => {
+
+    let assetUrl = ""
+    try {
+      setProgress(85)
+      setFinalGenerationStatus("generating")
+      assetUrl = await exportClapToVideo({
+        clap,
+        turbo: true
+      })
+
+      console.log(`handleSubmit(): received a video: ${assetUrl.slice(0, 60)}...`)
+      setFinalGenerationStatus("finished")
+      setCurrentVideo(assetUrl)
+      return assetUrl
+    } catch (err) {
+      setFinalGenerationStatus("error")
+      throw err
+    }
+  }
+  
+  const handleSubmit = async () => {
+
+    startTransition(async () => {
+      console.log(`handleSubmit(): generating a clap using prompt = "${promptDraft.current}" `)
 
-      let assetUrl = ""
       try {
-        setProgress(75)
-        setVideoGenerationStatus("generating")
-        assetUrl = await exportClapToVideo({
-          clap,
-          // turbo: true
-       })
+        let clap = await generateStory()
+        
+        const claps = await Promise.all([
+          generateMusic(clap),
+          generateVideos(clap)
+        ])
+
+        for (const newerClap of claps) {
+          console.log("newerClap:", newerClap)
+          clap = await updateClap(clap, newerClap, {
+            overwriteMeta: false,
+            inlineReplace: true,
+          })
+        }
+        console.log("finalClap: ", clap)
+
+        /*
+        clap = await claps.reduce(async (existingClap, newerClap) =>
+          updateClap(existingClap, newerClap, {
+            overwriteMeta: false,
+            inlineReplace: true,
+          })
+        , Promise.resolve(clap)
+        */
+
+       
+        // We can't have consistent characters with video (yet)
+        // clap = await generateEntities(clap)
+
+        /*
+        if (mainCharacterImage) {
+          console.log("handleSubmit(): User specified a main character image")
+          // various strategies here, for instance we can assume that the first character is the main character,
+          // or maybe a more reliable way is to count the number of occurrences.
+          // there is a risk of misgendering, so ideally we should add some kind of UI to do this,
+          // such as a list of characters.
+        }
+        */
+
+        // let's skip storyboards for now
+        // clap = await generateStoryboards(clap)
+
+        // clap = await generateVideos(clap)
+        // clap = await generateDialogues(clap)
+     
+        await generateFinalVideo(clap)
 
-        console.log(`handleSubmit(): received a video: ${assetUrl.slice(0, 60)}...`)
-        setVideoGenerationStatus("finished")
-        setCurrentVideo(assetUrl)
         setStatus("finished")
         setError("")
       } catch (err) {
-        console.error(`error: `, err)
-        setVideoGenerationStatus("error")
+        console.error(`failed to generate: `, err)
         setStatus("error")
         setError(`${err}`)
-        // setCurrentVideo("")
       }
-
-      console.log("-------- GENERATED FINAL VIDEO --------")
     })
   }
 
@@ -377,18 +469,22 @@ export function Main() {
     const progressDelayInMsPerStage: Record<GenerationStage, number> = {
       story: 2200,
       entities: 2200,
+      music: 3000,
       images: 1000,
       voices: 2000,
-      video_export: 2500,
+      videos: 2000,
+      final: 2500,
       idle: 1000
     }
 
     const maxProgressPerStage: Record<GenerationStage, number> = {
       story: 19,
-      entities: 39,
+      entities: 29,
+      music: 39,
       images: 49,
-      voices: 74,
-      video_export: 99,
+      videos: 69,
+      voices: 84,
+      final: 99,
       idle: 100
     }
 
@@ -424,20 +520,20 @@ export function Main() {
       // `bg-gradient-to-br from-amber-700 to-yellow-300`, 
 
       // warm orange, a bit flash but not bad, not bad at all
-     // `bg-gradient-to-br from-orange-700 to-yellow-400`, 
+      // `bg-gradient-to-br from-orange-700 to-yellow-400`, 
    
-     // nice "AiTube" vibe
-     `bg-gradient-to-br from-red-700 to-yellow-400`, 
+      // nice "AiTube" vibe
+      // `bg-gradient-to-br from-red-700 to-yellow-400`, 
    
-     // pretty cool lime!
-     // `bg-gradient-to-br from-lime-700 to-yellow-400`, 
+      // pretty cool lime!
+      `bg-gradient-to-br from-lime-700 to-yellow-400`, 
 
-     // new style, pretty "fresh" - maybe too bright? 
-     // use a dark logo for this one
-     // `bg-gradient-to-br from-yellow-200 to-yellow-500`, 
+      // new style, pretty "fresh" - maybe too bright? 
+      // use a dark logo for this one
+      // `bg-gradient-to-br from-yellow-200 to-yellow-500`, 
       
-     // too pastel
-     // `bg-gradient-to-br from-yellow-200 to-red-300`, 
+      // too pastel
+      // `bg-gradient-to-br from-yellow-200 to-red-300`, 
       
       // `bg-gradient-to-br from-sky-400 to-sky-300/30`, 
       `w-screen h-full overflow-y-scroll md:overflow-hidden`,
@@ -755,12 +851,14 @@ export function Main() {
                     <p className="text-2xl font-bold">{progress}%</p> 
                     <p className="text-base text-white/70">{isBusy
                       ? (
-                        storyGenerationStatus === "generating" ? "Writing the story.."
+                        storyGenerationStatus === "generating" ? "Writing story.."
                         : parseGenerationStatus === "generating" ? "Loading the project.."
                         : assetGenerationStatus === "generating" ? "Casting characters.."
+                        : musicGenerationStatus === "generating" ? "Producing music.."
                         : imageGenerationStatus === "generating" ? "Creating storyboards.."
-                        : voiceGenerationStatus === "generating" ? "Recording voices.."
-                        : videoGenerationStatus === "generating" ? "Assembling final video.."
+                        : videoGenerationStatus === "generating" ? "Filming shots.."
+                        : voiceGenerationStatus === "generating" ? "Recording dialogues.."
+                        : finalGenerationStatus === "generating" ? "Assembling final cut.."
                         : "Please wait.."
                       )
                       : status === "error"

src/app/server/aitube/editClapMusic.ts

@@ -0,0 +1,23 @@
+"use server"
+
+import { ClapProject } from "@aitube/clap"
+import { editClapMusic as apiEditClapMusic, ClapCompletionMode } from "@aitube/client"
+
+import { getToken } from "./getToken"
+
+export async function editClapMusic({
+  clap,
+  turbo = false,
+}: {
+  clap: ClapProject
+  turbo?: boolean
+}): Promise<ClapProject> {
+  const newClap: ClapProject = await apiEditClapMusic({
+    clap,
+    completionMode: ClapCompletionMode.MERGE,
+    turbo,
+    token: await getToken()
+  })
+
+  return newClap
+}

src/app/store.ts

@@ -24,9 +24,11 @@ export const useStore = create<{
   parseGenerationStatus: TaskStatus
   storyGenerationStatus: TaskStatus
   assetGenerationStatus: TaskStatus
+  musicGenerationStatus: TaskStatus
   voiceGenerationStatus: TaskStatus
   imageGenerationStatus: TaskStatus
   videoGenerationStatus: TaskStatus
+  finalGenerationStatus: TaskStatus
   isBusy: boolean
 
   currentClap?: ClapProject
@@ -47,9 +49,11 @@ export const useStore = create<{
   setParseGenerationStatus: (parseGenerationStatus: TaskStatus) => void
   setStoryGenerationStatus: (storyGenerationStatus: TaskStatus) => void
   setAssetGenerationStatus: (assetGenerationStatus: TaskStatus) => void
+  setMusicGenerationStatus: (musicGenerationStatus: TaskStatus) => void
   setVoiceGenerationStatus: (voiceGenerationStatus: TaskStatus) => void
   setImageGenerationStatus: (imageGenerationStatus: TaskStatus) => void
   setVideoGenerationStatus: (videoGenerationStatus: TaskStatus) => void
+  setFinalGenerationStatus: (finalGenerationStatus: TaskStatus) => void
   syncStatusAndStageState: () => void
   setCurrentClap: (currentClap?: ClapProject) => void
 
@@ -71,9 +75,11 @@ export const useStore = create<{
   parseGenerationStatus: "idle",
   storyGenerationStatus: "idle",
   assetGenerationStatus: "idle",
+  musicGenerationStatus: "idle",
   voiceGenerationStatus: "idle",
   imageGenerationStatus: "idle",
   videoGenerationStatus: "idle",
+  finalGenerationStatus: "idle",
   isBusy: false,
   currentClap: undefined,
   currentVideo: "",
@@ -118,6 +124,10 @@ export const useStore = create<{
     set({ assetGenerationStatus })
     get().syncStatusAndStageState()
   },
+  setMusicGenerationStatus: (musicGenerationStatus: TaskStatus) => {
+    set({ musicGenerationStatus })
+    get().syncStatusAndStageState()
+  },
   setVoiceGenerationStatus: (voiceGenerationStatus: TaskStatus) => {
     set({ voiceGenerationStatus })
     get().syncStatusAndStageState()
@@ -130,8 +140,12 @@ export const useStore = create<{
     set({ videoGenerationStatus })
     get().syncStatusAndStageState()
   },
+  setFinalGenerationStatus: (finalGenerationStatus: TaskStatus) => {
+    set({ finalGenerationStatus })
+    get().syncStatusAndStageState()
+  },
   syncStatusAndStageState: () => {
-    const { status, storyGenerationStatus, assetGenerationStatus, voiceGenerationStatus, imageGenerationStatus, videoGenerationStatus } = get()
+    const { status, storyGenerationStatus, assetGenerationStatus, musicGenerationStatus, voiceGenerationStatus, imageGenerationStatus, videoGenerationStatus, finalGenerationStatus } = get()
 
     // note: we don't really have "stages" since some things run in parallel,
     // and some parallel tasks may finish before the others
@@ -139,9 +153,11 @@ export const useStore = create<{
     let stage: GenerationStage =
       storyGenerationStatus === "generating" ? "story" :
       assetGenerationStatus === "generating" ? "entities" :
+      musicGenerationStatus === "generating" ? "music" :
       voiceGenerationStatus === "generating" ? "voices" :
       imageGenerationStatus === "generating" ? "images" :
-      videoGenerationStatus === "generating" ? "video_export" :
+      videoGenerationStatus === "generating" ? "videos" :
+      finalGenerationStatus === "generating" ? "final" :
       "idle"
 
 

src/types.ts

@@ -13,7 +13,9 @@ export type GlobalStatus =
 export type GenerationStage =
   | "story"
   | "entities"
+  | "music"
   | "voices"
   | "images"
-  | "video_export"
+  | "videos"
+  | "final"
   | "idle"