jbilcke-hf HF staff commited on
Commit
c20780a
β€’
1 Parent(s): 4fafd8c

fix bug in the .env

Browse files
.env CHANGED
@@ -70,7 +70,7 @@ RENDERING_HF_INFERENCE_API_FILE_TYPE="image/png"
70
 
71
  # An experimental RENDERING engine (sorry it is not very documented yet, so you can use one of the other engines)
72
  RENDERING_VIDEOCHAIN_API_URL="http://localhost:7860"
73
- you decide
74
  RENDERING_OPENAI_API_BASE_URL="https://api.openai.com/v1"
75
  RENDERING_OPENAI_API_MODEL="dall-e-3"
76
 
 
70
 
71
  # An experimental RENDERING engine (sorry it is not very documented yet, so you can use one of the other engines)
72
  RENDERING_VIDEOCHAIN_API_URL="http://localhost:7860"
73
+
74
  RENDERING_OPENAI_API_BASE_URL="https://api.openai.com/v1"
75
  RENDERING_OPENAI_API_MODEL="dall-e-3"
76
 
src/app/engine/render.ts CHANGED
@@ -371,6 +371,9 @@ export async function newRender({
371
  prompt,
372
  // negativePrompt, unused for now
373
 
 
 
 
374
  nbFrames,
375
 
376
  nbSteps: nbInferenceSteps, // 20 = fast, 30 = better, 50 = best
 
371
  prompt,
372
  // negativePrompt, unused for now
373
 
374
+ // for a future version of the comic factory
375
+ identityImage: "",
376
+
377
  nbFrames,
378
 
379
  nbSteps: nbInferenceSteps, // 20 = fast, 30 = better, 50 = best
src/app/interface/bottom-bar/bottom-bar.tsx CHANGED
@@ -13,28 +13,33 @@ import { SettingsDialog } from "../settings-dialog"
13
  import { useLocalStorage } from "usehooks-ts"
14
  import { localStorageKeys } from "../settings-dialog/localStorageKeys"
15
  import { defaultSettings } from "../settings-dialog/defaultSettings"
 
16
 
17
  function BottomBar() {
18
  // deprecated, as HTML-to-bitmap didn't work that well for us
19
- // const page = useStore(state => state.page)
20
- // const download = useStore(state => state.download)
21
- // const pageToImage = useStore(state => state.pageToImage)
22
 
23
- const isGeneratingStory = useStore(state => state.isGeneratingStory)
24
- const prompt = useStore(state => state.prompt)
25
- const panelGenerationStatus = useStore(state => state.panelGenerationStatus)
26
 
27
- const preset = useStore(state => state.preset)
 
 
28
 
29
  const allStatus = Object.values(panelGenerationStatus)
30
  const remainingImages = allStatus.reduce((acc, s) => (acc + (s ? 1 : 0)), 0)
31
 
32
- const upscaleQueue = useStore(state => state.upscaleQueue)
33
- const renderedScenes = useStore(state => state.renderedScenes)
34
- const removeFromUpscaleQueue = useStore(state => state.removeFromUpscaleQueue)
35
- const setRendered = useStore(state => state.setRendered)
36
  const [isUpscaling, setUpscaling] = useState(false)
37
 
 
 
38
  const [hasGeneratedAtLeastOnce, setHasGeneratedAtLeastOnce] = useLocalStorage<boolean>(
39
  localStorageKeys.hasGeneratedAtLeastOnce,
40
  defaultSettings.hasGeneratedAtLeastOnce
@@ -147,6 +152,17 @@ function BottomBar() {
147
  </Button>
148
  </div>
149
  */}
 
 
 
 
 
 
 
 
 
 
 
150
  <Button
151
  onClick={handlePrint}
152
  disabled={!prompt?.length}
 
13
  import { useLocalStorage } from "usehooks-ts"
14
  import { localStorageKeys } from "../settings-dialog/localStorageKeys"
15
  import { defaultSettings } from "../settings-dialog/defaultSettings"
16
+ import { getParam } from "@/lib/getParam"
17
 
18
  function BottomBar() {
19
  // deprecated, as HTML-to-bitmap didn't work that well for us
20
+ // const page = useStore(s => s.page)
21
+ // const download = useStore(s => s.download)
22
+ // const pageToImage = useStore(s => s.pageToImage)
23
 
24
+ const isGeneratingStory = useStore(s => s.isGeneratingStory)
25
+ const prompt = useStore(s => s.prompt)
26
+ const panelGenerationStatus = useStore(s => s.panelGenerationStatus)
27
 
28
+ const preset = useStore(s => s.preset)
29
+
30
+ const canSeeBetaFeatures = getParam<boolean>("beta", false)
31
 
32
  const allStatus = Object.values(panelGenerationStatus)
33
  const remainingImages = allStatus.reduce((acc, s) => (acc + (s ? 1 : 0)), 0)
34
 
35
+ const upscaleQueue = useStore(s => s.upscaleQueue)
36
+ const renderedScenes = useStore(s => s.renderedScenes)
37
+ const removeFromUpscaleQueue = useStore(s => s.removeFromUpscaleQueue)
38
+ const setRendered = useStore(s => s.setRendered)
39
  const [isUpscaling, setUpscaling] = useState(false)
40
 
41
+ const downloadClap = useStore(s => s.downloadClap)
42
+
43
  const [hasGeneratedAtLeastOnce, setHasGeneratedAtLeastOnce] = useLocalStorage<boolean>(
44
  localStorageKeys.hasGeneratedAtLeastOnce,
45
  defaultSettings.hasGeneratedAtLeastOnce
 
152
  </Button>
153
  </div>
154
  */}
155
+ {canSeeBetaFeatures ? <Button
156
+ onClick={downloadClap}
157
+ disabled={!prompt?.length}
158
+ >
159
+ <span className="hidden md:inline">{
160
+ remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} panels βŒ›` : `Save .clap`
161
+ }</span>
162
+ <span className="inline md:hidden">{
163
+ remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} βŒ›` : `Save .clap`
164
+ }</span>
165
+ </Button> : null}
166
  <Button
167
  onClick={handlePrint}
168
  disabled={!prompt?.length}
src/app/main.tsx CHANGED
@@ -222,14 +222,17 @@ export default function Main() {
222
  // update the frontend
223
  // console.log("updating the frontend..")
224
  setCaptions(ref.current.newCaptions)
225
- setPanels(ref.current.newPanelsPrompts)
226
-
227
  setGeneratingStory(false)
 
 
 
228
  } catch (err) {
229
  console.log("main.tsx: LLM generation failed:", err)
230
  setGeneratingStory(false)
231
  break
232
  }
 
233
  if (currentPanel > (currentNbPanels / 2)) {
234
  console.log("main.tsx: we are halfway there, hold tight!")
235
  // setWaitABitMore(true)
 
222
  // update the frontend
223
  // console.log("updating the frontend..")
224
  setCaptions(ref.current.newCaptions)
225
+ setPanels(ref.current.newPanelsPrompts)
 
226
  setGeneratingStory(false)
227
+
228
+ // TODO generate the clap here
229
+
230
  } catch (err) {
231
  console.log("main.tsx: LLM generation failed:", err)
232
  setGeneratingStory(false)
233
  break
234
  }
235
+
236
  if (currentPanel > (currentNbPanels / 2)) {
237
  console.log("main.tsx: we are halfway there, hold tight!")
238
  // setWaitABitMore(true)
src/app/store/index.ts CHANGED
@@ -1,7 +1,7 @@
1
  "use client"
2
 
3
  import { create } from "zustand"
4
- import html2canvas from "html2canvas"
5
 
6
  import { FontName } from "@/lib/fonts"
7
  import { Preset, PresetName, defaultPreset, getPreset, getRandomPreset } from "@/app/engine/presets"
@@ -13,6 +13,7 @@ export const useStore = create<{
13
  prompt: string
14
  font: FontName
15
  preset: Preset
 
16
  currentNbPanelsPerPage: number
17
  maxNbPanelsPerPage: number
18
  currentNbPages: number
@@ -70,6 +71,10 @@ export const useStore = create<{
70
  // setPage: (page: HTMLDivElement) => void
71
 
72
  generate: (prompt: string, presetName: PresetName, layoutName: LayoutName) => void
 
 
 
 
73
  }>((set, get) => ({
74
  prompt:
75
  (getParam("stylePrompt", "") || getParam("storyPrompt", ""))
@@ -78,6 +83,7 @@ export const useStore = create<{
78
  font: "actionman",
79
  preset: getPreset(getParam("preset", defaultPreset)),
80
 
 
81
  currentNbPanelsPerPage: 4,
82
  maxNbPanelsPerPage: 4,
83
  currentNbPages: 1,
@@ -399,5 +405,114 @@ export const useStore = create<{
399
  layout: layouts[0],
400
  layouts,
401
  })
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
402
  }
403
  }))
 
1
  "use client"
2
 
3
  import { create } from "zustand"
4
+ import { ClapProject, newClap, newSegment, serializeClap } from "@aitube/clap"
5
 
6
  import { FontName } from "@/lib/fonts"
7
  import { Preset, PresetName, defaultPreset, getPreset, getRandomPreset } from "@/app/engine/presets"
 
13
  prompt: string
14
  font: FontName
15
  preset: Preset
16
+ currentClap?: ClapProject
17
  currentNbPanelsPerPage: number
18
  maxNbPanelsPerPage: number
19
  currentNbPages: number
 
71
  // setPage: (page: HTMLDivElement) => void
72
 
73
  generate: (prompt: string, presetName: PresetName, layoutName: LayoutName) => void
74
+
75
+ computeClap: () => Promise<ClapProject>
76
+
77
+ downloadClap: () => Promise<void>
78
  }>((set, get) => ({
79
  prompt:
80
  (getParam("stylePrompt", "") || getParam("storyPrompt", ""))
 
83
  font: "actionman",
84
  preset: getPreset(getParam("preset", defaultPreset)),
85
 
86
+ currentClap: undefined,
87
  currentNbPanelsPerPage: 4,
88
  maxNbPanelsPerPage: 4,
89
  currentNbPages: 1,
 
405
  layout: layouts[0],
406
  layouts,
407
  })
408
+ },
409
+ computeClap: async (): Promise<ClapProject> => {
410
+ const {
411
+ currentNbPanels,
412
+ prompt,
413
+ panels,
414
+ renderedScenes,
415
+ captions
416
+ } = get()
417
+
418
+ const defaultSegmentDurationInMs = 7000
419
+
420
+ let currentElapsedTimeInMs = 0
421
+
422
+
423
+ const clap: ClapProject = newClap({
424
+ meta: {
425
+ title: "Untitled", // we don't need a title actually
426
+ description: prompt,
427
+ prompt: prompt,
428
+ synopsis: "",
429
+ licence: "",
430
+ orientation: "landscape",
431
+ width: 512,
432
+ height: 288,
433
+ isInteractive: false,
434
+ isLoop: false,
435
+ durationInMs: panels.length * defaultSegmentDurationInMs,
436
+ defaultVideoModel: "SDXL",
437
+ }
438
+ })
439
+
440
+ for (let i = 0; i < panels.length; i++) {
441
+
442
+ const panel = panels[i]
443
+ const caption = panels[i]
444
+ const renderedScene = renderedScenes[`${i}`]
445
+
446
+ clap.segments.push(newSegment({
447
+ track: 1,
448
+ startTimeInMs: currentElapsedTimeInMs,
449
+ assetDurationInMs: defaultSegmentDurationInMs,
450
+ category: "storyboard",
451
+ prompt: panel,
452
+ outputType: "image"
453
+ }))
454
+
455
+ clap.segments.push(newSegment({
456
+ track: 2,
457
+ startTimeInMs: currentElapsedTimeInMs,
458
+ assetDurationInMs: defaultSegmentDurationInMs,
459
+ category: "interface",
460
+ prompt: caption,
461
+ // assetUrl: `data:text/plain;base64,${btoa(title)}`,
462
+ assetUrl: caption,
463
+ outputType: "text"
464
+ }))
465
+
466
+ clap.segments.push(newSegment({
467
+ track: 3,
468
+ startTimeInMs: currentElapsedTimeInMs,
469
+ assetDurationInMs: defaultSegmentDurationInMs,
470
+ category: "dialogue",
471
+ prompt: caption,
472
+ outputType: "audio"
473
+ }))
474
+
475
+ // the presence of a camera is mandatory
476
+ clap.segments.push(newSegment({
477
+ track: 4,
478
+ startTimeInMs: currentElapsedTimeInMs,
479
+ assetDurationInMs: defaultSegmentDurationInMs,
480
+ category: "camera",
481
+ prompt: "video",
482
+ outputType: "text"
483
+ }))
484
+
485
+ currentElapsedTimeInMs += defaultSegmentDurationInMs
486
+ }
487
+
488
+ set({ currentClap: clap })
489
+
490
+ return clap
491
+ },
492
+
493
+ downloadClap: async () => {
494
+ const { computeClap } = get()
495
+
496
+ const currentClap = await computeClap()
497
+
498
+ if (!currentClap) { throw new Error(`cannot save a clap.. if there is no clap`) }
499
+
500
+ const currentClapBlob: Blob = await serializeClap(currentClap)
501
+
502
+ // Create an object URL for the compressed clap blob
503
+ const objectUrl = URL.createObjectURL(currentClapBlob)
504
+
505
+ // Create an anchor element and force browser download
506
+ const anchor = document.createElement("a")
507
+ anchor.href = objectUrl
508
+
509
+ anchor.download = "my_ai_comic.clap"
510
+
511
+ document.body.appendChild(anchor) // Append to the body (could be removed once clicked)
512
+ anchor.click() // Trigger the download
513
+
514
+ // Cleanup: revoke the object URL and remove the anchor element
515
+ URL.revokeObjectURL(objectUrl)
516
+ document.body.removeChild(anchor)
517
  }
518
  }))
src/types.ts CHANGED
@@ -15,12 +15,8 @@ export interface RenderRequest {
15
  // actionnables are names of things like "chest", "key", "tree", "chair" etc
16
  actionnables: string[]
17
 
18
- // note: this is the number of frames for Zeroscope,
19
- // which is currently configured to only output 3 seconds, so:
20
- // nbFrames=8 -> 1 sec
21
- // nbFrames=16 -> 2 sec
22
- // nbFrames=24 -> 3 sec
23
- nbFrames: number // min: 1, max: 24
24
 
25
  nbSteps: number // min: 1, max: 50
26
 
@@ -52,6 +48,8 @@ export interface RenderRequest {
52
  wait: boolean // wait until the job is completed
53
 
54
  analyze: boolean // analyze the image to generate a caption (optional)
 
 
55
  }
56
 
57
  export interface ImageSegment {
 
15
  // actionnables are names of things like "chest", "key", "tree", "chair" etc
16
  actionnables: string[]
17
 
18
+ nbFrames: number
19
+ nbFPS: number
 
 
 
 
20
 
21
  nbSteps: number // min: 1, max: 50
22
 
 
48
  wait: boolean // wait until the job is completed
49
 
50
  analyze: boolean // analyze the image to generate a caption (optional)
51
+
52
+ identityImage: string // reference image for the main entity
53
  }
54
 
55
  export interface ImageSegment {