initial commit 🎬

.dockerignore

@@ -0,0 +1,6 @@
+node_modules
+npm-debug.log
+models
+sandbox
+audio.pipe
+video.pipe

.env

@@ -0,0 +1,3 @@
+
+# the secret micro service key used in various API spaces
+MICROSERVICE_API_SECRET_TOKEN=""

.gitignore

@@ -0,0 +1,13 @@
+samples
+node_modules
+*.log
+*.bin
+.DS_Store
+.venv
+*.mp4
+*.wav
+*.mp3
+*.webp
+sandbox
+scripts
+.env.local

.nvmrc

@@ -0,0 +1 @@
+v20.10.0

Dockerfile

@@ -0,0 +1,50 @@
+# And Node 20
+FROM node:20-alpine
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apk update
+
+RUN apk add alpine-sdk pkgconfig
+
+# For FFMPEG and gl concat
+RUN apk add curl python3 python3-dev libx11-dev libsm-dev libxrender libxext-dev mesa-dev xvfb libxi-dev glew-dev
+
+# For Puppeteer
+RUN apk add build-base gcompat udev ttf-opensans chromium
+
+RUN apk add ffmpeg
+
+# Set up a new user named "user" with user ID 1000
+RUN adduser --disabled-password --uid 1001 user
+
+# Switch to the "user" user
+USER user
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Install app dependencies
+# A wildcard is used to ensure both package.json AND package-lock.json are copied
+# where available (npm@5+)
+COPY --chown=user package*.json $HOME/app
+
+# make sure the .env is copied as well
+COPY --chown=user .env $HOME/app
+
+RUN ffmpeg -version
+
+RUN npm install
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+EXPOSE 7860
+
+# we can't use this (it time out)
+# CMD [ "xvfb-run", "-s", "-ac -screen 0 1920x1080x24", "npm", "run", "start" ]
+CMD [ "npm", "run", "start" ]

LICENSE.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,38 @@
+---
+title: AI Tube Clap Exporter
+emoji: 🍿🤖
+colorFrom: red
+colorTo: blue
+sdk: docker
+pinned: false
+app_port: 7860
+---
+
+Export a full .clap (with all its assets already in) to a video
+
+# Installation
+
+It is important that you make sure to use the correct version of Node (Node 20)
+
+1. `nvm use`
+2. `npm i`
+3. clone `.env` to `.env.local`
+4. edit `.env.local` to define the secrets / api access keys
+5. `npm run start`
+
+# Testing the Docker image
+
+Note: you need to install Docker, and it needs to be already running.
+
+You will also need to build it for *your* architecture.
+
+```bash
+docker build --platform linux/arm64 -t ai-tube-clap-exporter .
+docker run -it -p 7860:7860 ai-tube-clap-exporter
+```
+
+# Architecture
+
+AI Channels are just Hugging Face datasets.
+
+For now, we keep everything into one big JSON index, but don't worry we can migrate this to something more efficient, such as Redis (eg. using Upstash for convenience).

package.json

@@ -0,0 +1,40 @@
+{
+  "name": "ai-tube-clap-exporter",
+  "version": "1.0.0",
+  "description": "A service to convert a .clap (will all its assets) to a video file",
+  "main": "src/index.mts",
+  "scripts": {
+    "start": "tsx src/index.mts",
+    "dev": "tsx src/index.mts",
+    "docker": "npm run docker:build && npm run docker:run",
+    "docker:build": "docker build -t ai-tube-robot .",
+    "docker:run": "docker run -it -p 7860:7860 ai-tube-robot",
+    "alchemy:test": "tsx src/core/alchemy/test.mts"
+  },
+  "author": "Julian Bilcke <[email protected]>",
+  "license": "Apache License",
+  "dependencies": {
+    "@types/express": "^4.17.17",
+    "@types/fluent-ffmpeg": "^2.1.24",
+    "@types/uuid": "^9.0.2",
+    "dotenv": "^16.3.1",
+    "eventsource-parser": "^1.0.0",
+    "express": "^4.18.2",
+    "fluent-ffmpeg": "^2.1.2",
+    "fs-extra": "^11.1.1",
+    "mime-types": "^2.1.35",
+    "node-fetch": "^3.3.1",
+    "puppeteer": "^22.7.0",
+    "sharp": "^0.33.3",
+    "temp-dir": "^3.0.0",
+    "ts-node": "^10.9.1",
+    "type-fest": "^4.8.2",
+    "uuid": "^9.0.0",
+    "yaml": "^2.4.1"
+  },
+  "devDependencies": {
+    "@types/mime-types": "^2.1.4",
+    "@types/node": "^20.12.7",
+    "tsx": "^4.7.0"
+  }
+}

src/core/base64/addBase64.mts

@@ -0,0 +1,51 @@
+export function addBase64Header(
+  image?: string,
+  format?:
+    | "jpeg" | "jpg" | "png" | "webp" | "heic"
+    | "mp3" | "wav"
+    | "mp4" | "webm"
+    | string
+) {
+  
+  if (!image || typeof image !== "string" || image.length < 60) {
+    return ""
+  }
+
+  const ext = (`${format || ""}`.split(".").pop() || "").toLowerCase().trim()
+
+  let mime = ""
+  if (
+    ext === "jpeg" ||
+    ext === "jpg") {
+    mime = "image/jpeg"
+  } else if (
+    ext === "webp"
+  ) {
+    mime = "image/webp"
+  } else if (
+    ext === "png") {
+    mime = "image/png"
+  } else if (ext === "heic") {
+    mime = "image/heic"
+  } else if (ext === "mp3") {
+    mime = "audio/mp3"
+  } else if (ext === "mp4") {
+    mime = "video/mp4"
+  } else if (ext === "webm") {
+    mime = "video/webm"
+  } else if (ext === "wav") {
+    mime = "audio/wav"
+  } else {
+    throw new Error(`addBase64Header failed (unsupported format: ${format})`)
+  }
+
+  if (image.startsWith('data:')) {
+    if (image.startsWith(`data:${mime};base64,`)) {
+      return image
+    } else {
+      throw new Error(`addBase64Header failed (input string is NOT a ${mime} image)`)
+    }
+  } else {
+    return `data:${mime};base64,${image}`
+  }
+}

src/core/base64/dataUriToBlob.mts

@@ -0,0 +1,15 @@
+
+export function dataUriToBlob(dataURI = "", defaultContentType = ""): Blob {
+  dataURI = dataURI.replace(/^data:/, '');
+
+  const type = dataURI.match(/(?:image|application|video|audio|text)\/[^;]+/)?.[0] || defaultContentType;
+  const base64 = dataURI.replace(/^[^,]+,/, '');
+  const arrayBuffer = new ArrayBuffer(base64.length);
+  const typedArray = new Uint8Array(arrayBuffer);
+
+  for (let i = 0; i < base64.length; i++) {
+    typedArray[i] = base64.charCodeAt(i);
+  }
+
+  return new Blob([arrayBuffer], { type });
+}

src/core/base64/extractBase64.mts

@@ -0,0 +1,36 @@
+/**
+ * break a base64 string into sub-components
+ */
+export function extractBase64(base64: string = ""): {
+  mimetype: string;
+  extension: string;
+  data: string;
+  buffer: Buffer;
+  blob: Blob;
+} {
+  // console.log(`extractBase64(${base64.slice(0, 120)})`)
+  // Regular expression to extract the MIME type and the base64 data
+  const matches = base64.match(/^data:([A-Za-z-+/]+);base64,(.+)$/)
+
+  // console.log("matches:", matches)
+
+  if (!matches || matches.length !== 3) {
+    throw new Error("Invalid base64 string")
+  }
+   
+  const mimetype = matches[1] || ""
+  const data = matches[2] || ""
+  const buffer = Buffer.from(data, "base64")
+  const blob = new Blob([buffer])
+
+  // this should be enough for most media formats (jpeg, png, webp, mp4)
+  const extension = mimetype.split("/").pop() || ""
+
+  return {
+    mimetype,
+    extension,
+    data,
+    buffer,
+    blob,
+  }
+}

src/core/clap/getClapAssetSourceType.mts

@@ -0,0 +1,25 @@
+import { ClapAssetSource } from "./types.mts"
+
+export function getClapAssetSourceType(input: string = ""): ClapAssetSource {
+  
+  const str = `${input || ""}`.trim()
+
+  if (!str || !str.length) {
+    return "EMPTY"
+  }
+
+  if (str.startsWith("https://") || str.startsWith("http://")) {
+    return "REMOTE"
+  }
+
+  // note that "path" assets are potentially a security risk, they need to be treated with care
+  if (str.startsWith("/") || str.startsWith("../") || str.startsWith("./")) {
+    return "PATH"
+  }
+
+  if (str.startsWith("data:")) {
+    return "DATA"
+  }
+
+  return "PROMPT"
+}

src/core/clap/parseClap.mts

@@ -0,0 +1,320 @@
+
+import { v4 as uuidv4 } from "uuid"
+import YAML from "yaml"
+
+import { ClapHeader, ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment } from "./types.mts"
+import { getValidNumber } from "../parsers/getValidNumber.mts"
+import { dataUriToBlob } from "../base64/dataUriToBlob.mts"
+
+type StringOrBlob = string | Blob
+
+/**
+ * Import a clap file from various data sources into an ClapProject
+ * 
+ * Inputs can be:
+ * - a Clap project (which is an object)
+ * - an URL to a remote .clap file
+ * - a string containing a YAML array
+ * - a data uri containing a gzipped YAML array
+ * - a Blob containing a gzipped YAML array
+ * 
+ * note: it is not really async, because for some reason YAML.parse is a blocking call like for JSON,
+ * there is no async version although we are now in the 20s not 90s
+ */
+export async function parseClap(src?: ClapProject | string | Blob, debug = false): Promise<ClapProject> {
+
+  try {
+    if (
+      typeof src === "object" &&
+      Array.isArray( (src as any)?.scenes) &&
+      Array.isArray((src as any)?.models)
+    ) {
+      if (debug) {
+        console.log("parseClap: input is already a Clap file, nothing to do:", src)
+      }
+      // we can skip verification
+      return src as ClapProject
+    }
+  } catch (err) {
+    // well, this is not a clap project
+  }
+
+  let stringOrBlob = (src || "") as StringOrBlob
+
+  // both should work
+  const dataUriHeader1 = "data:application/x-gzip;base64,"
+  const dataUriHeader2 = "data:application/octet-stream;base64,"
+
+  const inputIsString = typeof stringOrBlob === "string"
+  const inputIsDataUri = typeof stringOrBlob === "string" ? stringOrBlob.startsWith(dataUriHeader1) || stringOrBlob.startsWith(dataUriHeader2) : false
+  const inputIsRemoteFile = typeof stringOrBlob === "string" ? (stringOrBlob.startsWith("http://") || stringOrBlob.startsWith("https://")) : false
+
+  let inputIsBlob = typeof stringOrBlob !== "string"
+
+  let inputYamlArrayString = ""
+
+  if (debug) {
+    console.log(`parseClap: pre-analysis: ${JSON.stringify({
+      inputIsString,
+      inputIsBlob,
+      inputIsDataUri,
+      inputIsRemoteFile
+    }, null, 2)}`)
+  }
+
+  if (typeof stringOrBlob === "string") {
+    if (debug) {
+      console.log("parseClap: input is a string ", stringOrBlob.slice(0, 120))
+    }
+    if (inputIsDataUri) {
+      if (debug) {
+        console.log(`parseClap: input is a data uri archive`)
+      }
+      stringOrBlob = dataUriToBlob(stringOrBlob, "application/x-gzip")
+      if (debug) {
+        console.log(`parseClap: inputBlob = `, stringOrBlob)
+      }
+      inputIsBlob = true
+    } else if (inputIsRemoteFile) {
+      try {
+        if (debug) {
+          console.log(`parseClap: input is a remote .clap file`)
+        }
+        const res = await fetch(stringOrBlob)
+        stringOrBlob = await res.blob()
+        if (!stringOrBlob) { throw new Error("blob is empty") }
+        inputIsBlob = true
+      } catch (err) {
+        // url seems invalid
+        throw new Error(`failed to download the .clap file (${err})`)
+      }
+    } else {
+      if (debug) {
+        console.log("parseClap: input is a text string containing a YAML array")
+      }
+      inputYamlArrayString = stringOrBlob
+      inputIsBlob = false
+    }
+  }
+
+  if (typeof stringOrBlob !== "string" && stringOrBlob) {
+    if (debug) {
+      console.log("parseClap: decompressing the blob..")
+    }
+    // Decompress the input blob using gzip
+    const decompressedStream = stringOrBlob.stream().pipeThrough(new DecompressionStream('gzip'))
+
+    try {
+      // Convert the stream to text using a Response object
+      const decompressedOutput = new Response(decompressedStream)
+      // decompressedOutput.headers.set("Content-Type", "application/x-gzip")
+      if (debug) {
+        console.log("parseClap: decompressedOutput: ", decompressedOutput)
+      }
+      // const blobAgain = await decompressedOutput.blob()
+      inputYamlArrayString = await decompressedOutput.text()
+
+      if (debug && inputYamlArrayString) {
+        console.log("parseClap: successfully decompressed the blob!")
+      }
+    } catch (err) {
+      const message = `parseClap: failed to decompress (${err})`
+      console.error(message)
+      throw new Error(message)
+    }
+  }
+
+  // we don't need this anymore I think
+  // new Blob([inputStringOrBlob], { type: "application/x-yaml" })
+
+  let maybeArray: any = {}
+  try {
+    if (debug) {
+      console.log("parseClap: parsing the YAML array..")
+    }
+    // Parse YAML string to raw data
+    maybeArray = YAML.parse(inputYamlArrayString)
+  } catch (err) {
+    throw new Error("invalid clap file (input string is not YAML)")
+  }
+
+  if (!Array.isArray(maybeArray) || maybeArray.length < 2) {
+    throw new Error("invalid clap file (need a clap format header block and project metadata block)")
+  }
+
+  if (debug) {
+    console.log("parseClap: the YAML seems okay, continuing decoding..")
+  }
+
+  const maybeClapHeader = maybeArray[0] as ClapHeader
+
+  if (maybeClapHeader.format !== "clap-0") {
+    throw new Error("invalid clap file (sorry, but you can't make up version numbers like that)")
+  }
+
+
+  const maybeClapMeta = maybeArray[1] as ClapMeta
+
+  const clapMeta: ClapMeta = {
+    id: typeof maybeClapMeta.title === "string" ? maybeClapMeta.id : uuidv4(),
+    title: typeof maybeClapMeta.title === "string" ? maybeClapMeta.title : "",
+    description: typeof maybeClapMeta.description === "string" ? maybeClapMeta.description : "",
+    synopsis: typeof maybeClapMeta.synopsis === "string" ? maybeClapMeta.synopsis : "",
+    licence: typeof maybeClapMeta.licence === "string" ? maybeClapMeta.licence : "",
+    orientation: maybeClapMeta.orientation === "portrait" ? "portrait" : maybeClapMeta.orientation === "square" ? "square" : "landscape",
+    durationInMs: getValidNumber(maybeClapMeta.durationInMs, 1000, Number.MAX_SAFE_INTEGER, 4000),
+    width: getValidNumber(maybeClapMeta.width, 128, 8192, 1024),
+    height: getValidNumber(maybeClapMeta.height, 128, 8192, 576),
+    defaultVideoModel: typeof maybeClapMeta.defaultVideoModel === "string" ? maybeClapMeta.defaultVideoModel : "SVD",
+    extraPositivePrompt: Array.isArray(maybeClapMeta.extraPositivePrompt) ? maybeClapMeta.extraPositivePrompt : [],
+    screenplay: typeof maybeClapMeta.screenplay === "string" ? maybeClapMeta.screenplay : "",
+    isLoop: typeof maybeClapMeta.isLoop === "boolean" ? maybeClapMeta.isLoop : false,
+    isInteractive: typeof maybeClapMeta.isInteractive === "boolean" ? maybeClapMeta.isInteractive : false,
+  }
+
+  /*
+  in case we want to support streaming (mix of models and segments etc), we could do it this way:
+
+  const maybeModelsOrSegments = rawData.slice(2)
+  maybeModelsOrSegments.forEach((unknownElement: any) => {
+    if (isValidNumber(unknownElement?.track)) {
+      maybeSegments.push(unknownElement as ClapSegment)
+    } else {
+      maybeModels.push(unknownElement as ClapModel)
+    }
+  })
+  */
+
+
+  const expectedNumberOfModels = maybeClapHeader.numberOfModels || 0
+  const expectedNumberOfScenes = maybeClapHeader.numberOfScenes || 0
+  const expectedNumberOfSegments = maybeClapHeader.numberOfSegments || 0
+
+  // note: we assume the order is strictly enforced!
+  // if you implement streaming (mix of models and segments) you will have to rewrite this!
+
+  const afterTheHeaders = 2
+  const afterTheModels = afterTheHeaders + expectedNumberOfModels
+
+  const afterTheScenes = afterTheModels + expectedNumberOfScenes
+
+  // note: if there are no expected models, maybeModels will be empty
+  const maybeModels = maybeArray.slice(afterTheHeaders, afterTheModels) as ClapModel[]
+
+  // note: if there are no expected scenes, maybeScenes will be empty
+  const maybeScenes = maybeArray.slice(afterTheModels, afterTheScenes) as ClapScene[]
+
+  const maybeSegments = maybeArray.slice(afterTheScenes) as ClapSegment[]
+
+  const clapModels: ClapModel[] = maybeModels.map(({
+    id,
+    category,
+    triggerName,
+    label,
+    description,
+    author,
+    thumbnailUrl,
+    seed,
+    assetSourceType,
+    assetUrl,
+    age,
+    gender,
+    region,
+    appearance,
+    voiceVendor,
+    voiceId,
+  }) => ({
+    // TODO: we should verify each of those, probably
+    id,
+    category,
+    triggerName,
+    label,
+    description,
+    author,
+    thumbnailUrl,
+    seed,
+    assetSourceType,
+    assetUrl,
+    age,
+    gender,
+    region,
+    appearance,
+    voiceVendor,
+    voiceId,
+  }))
+
+  const clapScenes: ClapScene[] = maybeScenes.map(({
+    id,
+    scene,
+    line,
+    rawLine,
+    sequenceFullText,
+    sequenceStartAtLine,
+    sequenceEndAtLine,
+    startAtLine,
+    endAtLine,
+    events,
+  }) => ({
+    id,
+    scene,
+    line,
+    rawLine,
+    sequenceFullText,
+    sequenceStartAtLine,
+    sequenceEndAtLine,
+    startAtLine,
+    endAtLine,
+    events: events.map(e => e)
+  }))
+  
+  const clapSegments: ClapSegment[] = maybeSegments.map(({
+    id,
+    track,
+    startTimeInMs,
+    endTimeInMs,
+    category,
+    modelId,
+    sceneId,
+    prompt,
+    label,
+    outputType,
+    renderId,
+    status,
+    assetUrl,
+    assetDurationInMs,
+    createdBy,
+    editedBy,
+    outputGain,
+    seed,
+  }) => ({
+    // TODO: we should verify each of those, probably
+    id,
+    track,
+    startTimeInMs,
+    endTimeInMs,
+    category,
+    modelId,
+    sceneId,
+    prompt,
+    label,
+    outputType,
+    renderId,
+    status,
+    assetUrl,
+    assetDurationInMs,
+    createdBy,
+    editedBy,
+    outputGain,
+    seed,
+  }))
+
+  if (debug) {
+    console.log(`parseClap: successfully parsed ${clapModels.length} models, ${clapScenes.length} scenes and ${clapSegments.length} segments`)
+  }
+  return {
+    meta: clapMeta,
+    models: clapModels,
+    scenes: clapScenes,
+    segments: clapSegments
+  }
+}

src/core/clap/types.mts

@@ -0,0 +1,203 @@
+
+export type ClapSegmentCategory =
+  | "splat"
+  | "mesh"
+  | "depth"
+  | "event"
+  | "interface"
+  | "phenomenon"
+  | "video"
+  | "storyboard"
+  | "transition"
+  | "characters"
+  | "location"
+  | "time"
+  | "era"
+  | "lighting"
+  | "weather"
+  | "action"
+  | "music"
+  | "sound"
+  | "dialogue"
+  | "style"
+  | "camera"
+  | "generic"
+
+export type ClapOutputType =
+  | "text"
+  | "animation"
+  | "interface"
+  | "event"
+  | "phenomenon"
+  | "transition"
+  | "image"
+  | "video"
+  | "audio"
+
+export type ClapSegmentStatus =
+  | "to_generate"
+  | "to_interpolate"
+  | "to_upscale"
+  | "completed"
+  | "error"
+
+export type ClapAuthor =
+  | "auto" // the element was edited automatically using basic if/else logical rules
+  | "ai" // the element was edited using a large language model
+  | "human" // the element was edited by a human
+
+export type ClapAssetSource =
+  | "REMOTE" // http:// or https://
+
+    // note that "path" assets are potentially a security risk, they need to be treated with care
+  | "PATH" // a file path eg. /path or ./path/to/ or ../path/to/
+
+  | "DATA" // a data URI, starting with data:
+
+  | "PROMPT" // by default, a plain text prompt
+
+  | "EMPTY"
+
+export type ClapModelGender =
+  | "male"
+  | "female"
+  | "person"
+  | "object"
+
+export type ClapModelAppearance = "serious" | "neutral" | "friendly" | "chill"
+
+// this is used for accent, style..
+export type ClapModelRegion =
+  | "american"
+  | "british"
+  | "australian"
+  | "canadian"
+  | "indian"
+  | "french"
+  | "italian"
+  | "german"
+  | "chinese"
+
+// note: this is all very subjective, so please use good judgment
+//
+// "deep" might indicate a deeper voice tone, thicker, rich in harmonics
+// in this context, it is used to indicate voices that could
+// be associated with African American (AADOS) characters
+//
+// "high" could be used for some other countries, eg. asia
+export type ClapModelTimbre = "high" | "neutral" | "deep"
+
+export type ClapVoiceVendor = "ElevenLabs" | "XTTS"
+
+export type ClapVoice = {
+  name: string
+  gender: ClapModelGender
+  age: number
+  region: ClapModelRegion
+  timbre: ClapModelTimbre
+  appearance: ClapModelAppearance
+  voiceVendor: ClapVoiceVendor
+  voiceId: string
+}
+
+export type ClapHeader = {
+  format: "clap-0"
+  numberOfModels: number
+  numberOfScenes: number
+  numberOfSegments: number
+}
+
+export type ClapMeta = {
+  id: string
+  title: string
+  description: string
+  synopsis: string
+  licence: string
+  orientation: string
+
+  // the default duration of the experience
+  // the real one might last longer if made interactive
+  durationInMs: number
+
+  width: number
+  height: number
+  defaultVideoModel: string
+  extraPositivePrompt: string[]
+  screenplay: string
+  isLoop: boolean
+  isInteractive: boolean
+}
+
+export type ClapSceneEvent = {
+  id: string
+  type: "description" | "dialogue" | "action"
+  character?: string
+  description: string
+  behavior: string
+  startAtLine: number
+  endAtLine: number
+}
+
+export type ClapScene = {
+  id: string
+  scene: string
+  line: string
+  rawLine: string
+  sequenceFullText: string
+  sequenceStartAtLine: number
+  sequenceEndAtLine: number
+  startAtLine: number
+  endAtLine: number
+  events: ClapSceneEvent[]
+}
+
+export type ClapSegment = {
+  id: string
+  track: number
+  startTimeInMs: number
+  endTimeInMs: number
+  category: ClapSegmentCategory
+  modelId: string
+  sceneId: string
+  prompt: string
+  label: string
+  outputType: ClapOutputType
+  renderId: string
+  status: ClapSegmentStatus
+  assetUrl: string
+  assetDurationInMs: number
+  createdBy: ClapAuthor
+  editedBy: ClapAuthor
+  outputGain: number
+  seed: number
+}
+
+export type ClapModel = {
+  id: string
+  category: ClapSegmentCategory
+  triggerName: string
+  label: string
+  description: string
+  author: string
+  thumbnailUrl: string
+  seed: number
+
+  assetSourceType: ClapAssetSource
+  assetUrl: string
+  
+  // those are only used by certain types of models
+  age: number
+  gender: ClapModelGender
+  region: ClapModelRegion
+  appearance: ClapModelAppearance
+  voiceVendor: ClapVoiceVendor
+  voiceId: string
+}
+
+export type ClapProject = {
+  meta: ClapMeta
+  models: ClapModel[]
+  scenes: ClapScene[]
+  segments: ClapSegment[]
+  // let's keep room for other stuff (screenplay etc)
+}

src/core/converters/blobToWebp.mts

@@ -0,0 +1,5 @@
+import { addBase64Header } from "../base64/addBase64.mts";
+
+export async function blobToWebp(blob: Blob) {
+  return addBase64Header(Buffer.from(await blob.text()).toString('base64'), "webp")
+}

src/core/converters/bufferToJpeg.mts

@@ -0,0 +1,5 @@
+import { addBase64Header } from "../base64/addBase64.mts";
+
+export async function bufferToJpeg(buffer: Buffer) {
+  return addBase64Header(buffer.toString('base64'), "jpeg")
+}

src/core/converters/bufferToMp3.mts

@@ -0,0 +1,5 @@
+import { addBase64Header } from "../base64/addBase64.mts";
+
+export async function bufferToMp3(buffer: Buffer) {
+  return addBase64Header(buffer.toString('base64'), "mp3")
+}

src/core/converters/bufferToMp4.mts

@@ -0,0 +1,5 @@
+import { addBase64Header } from "../base64/addBase64.mts";
+
+export async function bufferToMp4(buffer: Buffer) {
+  return addBase64Header(buffer.toString('base64'), "mp4")
+}

src/core/converters/bufferToPng.mts

@@ -0,0 +1,5 @@
+import { addBase64Header } from "../base64/addBase64.mts";
+
+export async function bufferToPng(buffer: Buffer) {
+  return addBase64Header(buffer.toString('base64'), "png")
+}

src/core/converters/bufferToWav.mts

@@ -0,0 +1,5 @@
+import { addBase64Header } from "../base64/addBase64.mts";
+
+export async function bufferToWav(buffer: Buffer) {
+  return addBase64Header(buffer.toString('base64'), "wav")
+}

src/core/converters/bufferToWebp.mts

@@ -0,0 +1,5 @@
+import { addBase64Header } from "../base64/addBase64.mts";
+
+export async function bufferToWebp(buffer: Buffer) {
+  return addBase64Header(buffer.toString('base64'), "webp")
+}

src/core/converters/convertImageTo.mts

@@ -0,0 +1,31 @@
+import { convertImageToJpeg } from "./convertImageToJpeg.mts"
+import { convertImageToPng } from "./convertImageToPng.mts"
+import { convertImageToWebp } from "./convertImageToWebp.mts"
+import { ImageFileExt } from "./imageFormats.mts"
+
+/**
+ * Convert an image to one of the supported file formats
+ * 
+ * @param imgBase64 
+ * @param outputFormat 
+ * @returns 
+ */
+export async function convertImageTo(imgBase64: string = "", outputFormat: ImageFileExt): Promise<string> {
+  const format = outputFormat.trim().toLowerCase() as ImageFileExt
+  if (!["jpeg", "jpg", "png", "webp"].includes(format)) {
+    throw new Error(`unsupported file format "${format}"`)
+  }
+
+  const isJpeg = format === "jpg" || format === "jpeg" 
+     
+
+  if (isJpeg) {
+    return convertImageToJpeg(imgBase64)
+  }
+
+  if (format === "webp") {
+    return convertImageToWebp(imgBase64)
+  }
+
+  return convertImageToPng(imgBase64)
+}

src/core/converters/convertImageToJpeg.mts

@@ -0,0 +1,27 @@
+import sharp from "sharp"
+
+export async function convertImageToJpeg(imgBase64: string = "", quality: number = 92): Promise<string> {
+
+  const base64WithoutHeader = imgBase64.split(";base64,")[1] || ""
+
+  if (!base64WithoutHeader) {
+    const slice = `${imgBase64 || ""}`.slice(0, 50)
+    throw new Error(`couldn't process input image "${slice}..."`)
+  }
+
+  // Convert base64 to buffer
+  const tmpBuffer = Buffer.from(base64WithoutHeader, 'base64')
+
+  // Resize the buffer to the target size
+  const newBuffer = await sharp(tmpBuffer)
+    .jpeg({
+      quality,
+      // we don't use progressive: true because we pre-load images anyway
+     })
+      .toBuffer()
+
+  // Convert the buffer back to base64
+  const newImageBase64 = newBuffer.toString('base64')
+
+  return `data:image/jpeg;base64,${newImageBase64}`
+}

src/core/converters/convertImageToOriginal.mts

@@ -0,0 +1,6 @@
+
+// you are reading it right: this function does.. nothing!
+// it is a NOOP conversion function
+export async function convertImageToOriginal(imgBase64: string = ""): Promise<string> {
+  return imgBase64
+}

src/core/converters/convertImageToPng.mts

@@ -0,0 +1,23 @@
+import sharp from "sharp"
+
+export async function convertImageToPng(imgBase64: string = ""): Promise<string> {
+
+  const base64WithoutHeader = imgBase64.split(";base64,")[1] || ""
+
+  if (!base64WithoutHeader) {
+    const slice = `${imgBase64 || ""}`.slice(0, 50)
+    throw new Error(`couldn't process input image "${slice}..."`)
+  }
+
+  // Convert base64 to buffer
+  const tmpBuffer = Buffer.from(base64WithoutHeader, 'base64')
+
+  const newBuffer = await sharp(tmpBuffer)
+    .png()
+    .toBuffer()
+
+  // Convert the buffer back to base64
+  const newImageBase64 = newBuffer.toString('base64')
+
+  return `data:image/png;base64,${newImageBase64}`
+}

src/core/converters/convertImageToWebp.mts

@@ -0,0 +1,41 @@
+import sharp from "sharp"
+
+export async function convertImageToWebp(imgBase64: string = ""): Promise<string> {
+
+  const base64WithoutHeader = imgBase64.split(";base64,")[1] || ""
+
+  if (!base64WithoutHeader) {
+    const slice = `${imgBase64 || ""}`.slice(0, 50)
+    throw new Error(`couldn't process input image "${slice}..."`)
+  }
+
+  // Convert base64 to buffer
+  const tmpBuffer = Buffer.from(base64WithoutHeader, 'base64')
+
+  // Resize the buffer to the target size
+  const newBuffer = await sharp(tmpBuffer)
+    .webp({
+      // for options please see https://sharp.pixelplumbing.com/api-output#webp
+
+      // preset: "photo",
+
+      // effort: 3,
+
+      // for a PNG-like quality
+      // lossless: true,
+
+      // by default it is quality 80
+      quality: 80,
+
+      // nearLossless: true,
+
+      // use high quality chroma subsampling
+      smartSubsample: true,
+     })
+      .toBuffer()
+
+  // Convert the buffer back to base64
+  const newImageBase64 = newBuffer.toString('base64')
+
+  return `data:image/webp;base64,${newImageBase64}`
+}

src/core/converters/htmlToBase64Png.mts

@@ -0,0 +1,78 @@
+import { promises as fs } from "node:fs"
+import os from "node:os"
+import path from "node:path"
+
+import { v4 as uuidv4 } from "uuid"
+import puppeteer from "puppeteer"
+
+export async function htmlToBase64Png({
+  outputImagePath,
+  html,
+  width = 800,
+  height = 600,
+}: {
+  outputImagePath?: string
+  html?: string
+  width?: number
+  height: number
+}): Promise<{
+  filePath: string
+  buffer: Buffer
+}> {
+
+  // If no output path is provided, create a temporary file for output
+  if (!outputImagePath) {
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), uuidv4()))
+
+    outputImagePath = path.join(tempDir, `${uuidv4()}.png`)
+  }
+
+  const browser = await puppeteer.launch({
+    headless: "new",
+
+    // apparently we need those, see:
+    // https://unix.stackexchange.com/questions/694734/puppeteer-in-alpine-docker-with-chromium-headless-dosent-seems-to-work
+    executablePath: '/usr/bin/chromium-browser',
+    args: [
+      '--no-sandbox',
+      '--headless',
+      '--disable-gpu',
+      '--disable-dev-shm-usage'
+    ]
+  })
+
+  const page = await browser.newPage()
+
+  page.setViewport({
+    width,
+    height,
+  })
+
+  try {
+    await page.setContent(html)
+
+    const content = await page.$("body")
+
+    const buffer = await content.screenshot({
+      path: outputImagePath,
+      omitBackground: true,
+      captureBeyondViewport: false,
+
+      // we must keep PNG here, if we want transparent backgrounds
+      type: "png",
+
+      // we should leave it to binary (the default value) if we save to a file
+      // encoding: "binary", // "base64",
+    })
+
+    return {
+      filePath: outputImagePath,
+      buffer
+    }
+  } catch (err) {
+    throw err
+  } finally {
+    await page.close()
+    await browser.close()
+  }
+};

src/core/converters/imageFormats.mts

@@ -0,0 +1 @@
+export type ImageFileExt = "png" | "jpeg" | "jpg" | "webp"

src/core/ffmpeg/addImageToVideo.mts

@@ -0,0 +1,50 @@
+import { promises as fs, existsSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import ffmpeg from "fluent-ffmpeg";
+import { v4 as uuidv4 } from "uuid";
+
+type AddImageToVideoParams = {
+  inputVideoPath: string;
+  inputImagePath: string;
+  outputVideoPath?: string;
+};
+
+export async function addImageToVideo({
+  inputVideoPath,
+  inputImagePath,
+  outputVideoPath,
+}: AddImageToVideoParams): Promise<string> {
+  // Verify that the input files exist
+  if (!existsSync(inputVideoPath)) {
+    throw new Error(`Input video file does not exist: ${inputVideoPath}`);
+  }
+  if (!existsSync(inputImagePath)) {
+    throw new Error(`Input image file does not exist: ${inputImagePath}`);
+  }
+
+  // If no output path is provided, create a temporary file for output
+  if (!outputVideoPath) {
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), uuidv4()));
+    outputVideoPath = path.join(tempDir, `${uuidv4()}.mp4`);
+  }
+
+  // Return a promise that resolves with the path to the output video
+  return new Promise((resolve, reject) => {
+    ffmpeg(inputVideoPath)
+      .input(inputImagePath)
+      .complexFilter([
+        {
+          filter: "overlay",
+          options: { x: "0", y: "0" }, // Overlay on the entire video frame
+        }
+      ])
+      .on("error", (err) => {
+        reject(new Error(`Error processing video: ${err.message}`));
+      })
+      .on("end", () => {
+        resolve(outputVideoPath);
+      })
+      .save(outputVideoPath);
+  });
+}

src/core/ffmpeg/addTextToVideo.mts

@@ -0,0 +1,23 @@
+import { createTextOverlayImage } from "./createTextOverlayImage.mts";
+import { addImageToVideo } from "./addImageToVideo.mts";
+
+export async function addTextToVideo() {
+  
+  const inputVideoPath = "/Users/jbilcke/Downloads/use_me.mp4"
+
+  const { filePath } = await createTextOverlayImage({
+    text: "This tech is hot 🥵",
+    width: 1024 ,
+    height: 576,
+  })
+  console.log("filePath:", filePath)
+
+  /*
+  const pathToVideo = await addImageToVideo({
+    inputVideoPath,
+    inputImagePath: filePath,
+  })
+
+  console.log("pathToVideo:", pathToVideo)
+  */
+}

src/core/ffmpeg/concatenateAudio.mts

@@ -0,0 +1,122 @@
+import { existsSync, promises as fs } from "node:fs"
+import os from "node:os"
+import path from "node:path"
+
+import { v4 as uuidv4 } from "uuid";
+import ffmpeg, { FfmpegCommand } from "fluent-ffmpeg";
+import { writeBase64ToFile } from "../files/writeBase64ToFile.mts";
+import { getMediaInfo } from "./getMediaInfo.mts";
+import { removeTemporaryFiles } from "../files/removeTmpFiles.mts";
+import { addBase64Header } from "../base64/addBase64.mts";
+
+export type ConcatenateAudioOptions = {
+  // those are base64 audio strings!
+  audioTracks?: string[]; // base64
+  audioFilePaths?: string[]; // path
+  crossfadeDurationInSec?: number;
+  outputFormat?: string; // "wav" or "mp3"
+  output?: string;
+}
+
+export type ConcatenateAudioOutput = {
+  filepath: string;
+  durationInSec: number;
+}
+
+export async function concatenateAudio({
+  output,
+  audioTracks = [],
+  audioFilePaths = [],
+  crossfadeDurationInSec = 10,
+  outputFormat = "wav"
+}: ConcatenateAudioOptions): Promise<ConcatenateAudioOutput> {
+  if (!Array.isArray(audioTracks)) {
+    throw new Error("Audios must be provided in an array");
+  }
+
+  const tempDir = path.join(os.tmpdir(), uuidv4());
+  await fs.mkdir(tempDir);
+
+  // console.log("  |- created tmp dir")
+
+  // trivial case: there is only one audio to concatenate!
+  if (audioTracks.length === 1 && audioTracks[0]) {
+    const audioTrack = audioTracks[0]
+    const outputFilePath = path.join(tempDir, `audio_0.${outputFormat}`);
+    await writeBase64ToFile(addBase64Header(audioTrack, "wav"), outputFilePath);
+
+    // console.log("  |- there is only one track! so.. returning that")
+    const { durationInSec } = await getMediaInfo(outputFilePath);
+    return { filepath: outputFilePath, durationInSec };
+  }
+
+  if (audioFilePaths.length === 1) {
+    throw new Error("concatenating a single audio file path is not implemented yet")
+  }
+  
+  try {
+
+    let i = 0
+    for (const track of audioTracks) {
+      if (!track) { continue }
+      const audioFilePath = path.join(tempDir, `audio_${++i}.wav`);
+      await writeBase64ToFile(addBase64Header(track, "wav"), audioFilePath);
+      audioFilePaths.push(audioFilePath);
+    }
+
+    audioFilePaths = audioFilePaths.filter((audio) => existsSync(audio))
+
+    const outputFilePath = output ?? path.join(tempDir, `${uuidv4()}.${outputFormat}`);
+  
+    let filterComplex = "";
+    let prevLabel = "0";
+
+    for (let i = 0; i < audioFilePaths.length - 1; i++) {
+      const nextLabel = `a${i}`;
+      filterComplex += `[${prevLabel}][${i + 1}]acrossfade=d=${crossfadeDurationInSec}:c1=tri:c2=tri[${nextLabel}];`;
+      prevLabel = nextLabel;
+    }
+
+  
+    console.log("  |- concatenateAudio(): DEBUG:", {
+      tempDir,
+      audioFilePaths,
+      outputFilePath,
+      filterComplex,
+      prevLabel
+    })
+
+    let cmd: FfmpegCommand = ffmpeg() // .outputOptions('-vn');
+
+    audioFilePaths.forEach((audio, i) => {
+      cmd = cmd.input(audio);
+    });
+
+
+    const promise = new Promise<ConcatenateAudioOutput>((resolve, reject) => {
+      cmd = cmd
+        .on('error', reject)
+        .on('end', async () => {
+          try {
+            const { durationInSec } = await getMediaInfo(outputFilePath);
+            // console.log("concatenation ended! see ->", outputFilePath)
+            resolve({ filepath: outputFilePath, durationInSec });
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .complexFilter(filterComplex, prevLabel)
+        .save(outputFilePath);
+    });
+
+    const result = await promise
+
+    return result
+  } catch (error) {
+    console.error(`Failed to assemble audio!`)
+    console.error(error)
+    throw new Error(`Failed to assemble audio: ${(error as Error)?.message || error}`);
+  } finally {
+    await removeTemporaryFiles(audioFilePaths)
+  }
+}

src/core/ffmpeg/concatenateVideos.mts

@@ -0,0 +1,61 @@
+import { existsSync, promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { v4 as uuidv4 } from "uuid";
+import ffmpeg, { FfmpegCommand } from "fluent-ffmpeg";
+
+import { getMediaInfo } from "./getMediaInfo.mts";
+
+export type ConcatenateVideoOutput = {
+  filepath: string;
+  durationInSec: number;
+}
+
+export async function concatenateVideos({
+  output,
+  videoFilePaths = [],
+}: {
+  output?: string;
+
+  // those are videos PATHs, not base64 strings!
+  videoFilePaths: string[];
+}): Promise<ConcatenateVideoOutput> {
+  if (!Array.isArray(videoFilePaths)) {
+    throw new Error("Videos must be provided in an array");
+  }
+
+  videoFilePaths = videoFilePaths.filter((videoPath) => existsSync(videoPath))
+
+  // Create a temporary working directory
+  const tempDir = path.join(os.tmpdir(), uuidv4());
+  await fs.mkdir(tempDir);
+
+  const filePath = output ? output : path.join(tempDir, `${uuidv4()}.mp4`);
+
+  if (!filePath) {
+    throw new Error("Failed to generate a valid temporary file path");
+  }
+
+  let cmd: FfmpegCommand = ffmpeg();
+
+  videoFilePaths.forEach((video) => {
+    cmd = cmd.addInput(video)
+  })
+
+  return new Promise<{ filepath: string; durationInSec: number }>(
+    (resolve, reject) => {
+      cmd
+        .on('error', reject)
+        .on('end', async () => {
+          try {
+            const { durationInSec } = await getMediaInfo(filePath);
+            resolve({ filepath: filePath, durationInSec });
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .mergeToFile(filePath, tempDir);
+    }
+  );
+};

src/core/ffmpeg/concatenateVideosAndMergeAudio.mts

@@ -0,0 +1,130 @@
+import { existsSync, promises as fs } from "node:fs"
+import os from "node:os"
+import path from "node:path"
+
+import { v4 as uuidv4 } from "uuid";
+import ffmpeg, { FfmpegCommand } from "fluent-ffmpeg";
+import { concatenateVideos } from "./concatenateVideos.mts";
+import { writeBase64ToFile } from "../files/writeBase64ToFile.mts";
+import { getMediaInfo } from "./getMediaInfo.mts";
+import { removeTemporaryFiles } from "../files/removeTmpFiles.mts";
+import { addBase64Header } from "../base64/addBase64.mts";
+
+type ConcatenateVideoAndMergeAudioOptions = {
+  output?: string;
+  audioTracks?: string[]; // base64
+  audioFilePaths?: string[]; // path
+  videoTracks?: string[]; // base64
+  videoFilePaths?: string[]; // path
+};
+
+export type ConcatenateVideoAndMergeAudioOutput = {
+  filepath: string;
+  durationInSec: number;
+}
+
+// note: the audio tracks will be fused together, as in "mixed"
+// this return a path to the file
+export const concatenateVideosAndMergeAudio = async ({
+  output,
+  audioTracks = [],
+  audioFilePaths = [],
+  videoTracks = [],
+  videoFilePaths = []
+}: ConcatenateVideoAndMergeAudioOptions): Promise<ConcatenateVideoAndMergeAudioOutput> => {
+
+  try {
+    // Prepare temporary directories
+    const tempDir = path.join(os.tmpdir(), uuidv4());
+    await fs.mkdir(tempDir);
+
+    let i = 0
+    for (const track of audioTracks) {
+      if (!track) { continue }
+      const audioFilePath = path.join(tempDir, `audio${++i}.wav`);
+      await writeBase64ToFile(addBase64Header(track, "wav"), audioFilePath);
+      audioFilePaths.push(audioFilePath);
+    }
+    audioFilePaths = audioFilePaths.filter((audio) => existsSync(audio))
+
+
+    // Decode and concatenate base64 video tracks to temporary file
+    i = 0
+    for (const track of videoTracks) {
+      if (!track) { continue }
+      const videoFilePath = path.join(tempDir, `video${++i}.mp4`);
+
+      await writeBase64ToFile(addBase64Header(track, "mp4"), videoFilePath);
+
+      videoFilePaths.push(videoFilePath);
+    }
+    videoFilePaths = videoFilePaths.filter((video) => existsSync(video))
+
+    // The final output file path
+    const finalOutputFilePath = output ? output : path.join(tempDir, `${uuidv4()}.mp4`);
+
+    /*
+    console.log("DEBUG:", {
+      tempDir,
+      audioFilePath,
+      audioTrack: audioTrack.slice(0, 40),
+      videoTracks: videoTracks.map(vid => vid.slice(0, 40)),
+      videoFilePaths,
+      finalOutputFilePath
+    })
+    */
+
+    // console.log("concatenating videos (without audio)..")
+    const tempFilePath = await concatenateVideos({
+      videoFilePaths,
+    })
+    // console.log("concatenated silent shots to: ", tempFilePath)
+    
+    // console.log("concatenating video + audio..")
+
+    // Add audio to the concatenated video file
+    const promise = new Promise<ConcatenateVideoAndMergeAudioOutput>((resolve, reject) => {
+      let cmd = ffmpeg().addInput(tempFilePath.filepath).outputOptions("-c:v copy");
+
+      for (const audioFilePath of audioFilePaths) {
+        cmd = cmd.addInput(audioFilePath);
+      }
+    
+      if (audioFilePaths.length) {
+        // Mix all audio tracks (if there are any) into a single stereo stream
+        const mixFilter = audioFilePaths.map((_, index) => `[${index + 1}:a]`).join('') + `amix=inputs=${audioFilePaths.length}:duration=first[outa]`;
+        cmd = cmd
+          .complexFilter(mixFilter)
+          .outputOptions([
+            "-map", "0:v:0", // Maps the video stream from the first input (index 0) as the output video stream
+            "-map", "[outa]", // Maps the labeled audio output from the complex filter (mixed audio) as the output audio stream
+            "-c:a aac", // Specifies the audio codec to be AAC (Advanced Audio Coding)
+            "-shortest" // Ensures the output file's duration equals the shortest input stream's duration
+          ]);
+      } else {
+        // If there are no audio tracks, just map the video
+        cmd = cmd.outputOptions(["-map", "0:v:0"]);
+      }    
+    
+      cmd = cmd
+        .on("error", reject)
+        .on('end', async () => {
+          try {
+            const { durationInSec } = await getMediaInfo(finalOutputFilePath);
+            resolve({ filepath: finalOutputFilePath, durationInSec });
+          } catch (err) {
+            reject(err);
+          }
+        })
+        .saveToFile(finalOutputFilePath);
+    });
+
+    const result = await promise;
+  
+    return result
+  } catch (error) {
+    throw new Error(`Failed to assemble video: ${(error as Error).message}`);
+  } finally {
+    await removeTemporaryFiles([...videoFilePaths, ...audioFilePaths])
+  }
+};

src/core/ffmpeg/concatenateVideosWithAudio.mts

@@ -0,0 +1,158 @@
+import { existsSync, promises as fs } from "node:fs"
+import os from "node:os"
+import path from "node:path"
+
+import { v4 as uuidv4 } from "uuid";
+import ffmpeg, { FfmpegCommand } from "fluent-ffmpeg";
+import { concatenateVideos } from "./concatenateVideos.mts";
+import { writeBase64ToFile } from "../files/writeBase64ToFile.mts";
+import { getMediaInfo } from "./getMediaInfo.mts";
+import { removeTemporaryFiles } from "../files/removeTmpFiles.mts";
+import { addBase64Header } from "../base64/addBase64.mts";
+
+type ConcatenateVideoWithAudioOptions = {
+  output?: string;
+  audioTrack?: string; // base64
+  audioFilePath?: string; // path
+  videoTracks?: string[]; // base64
+  videoFilePaths?: string[]; // path
+  videoTracksVolume?: number; // Represents the volume level of the original video track
+  audioTrackVolume?: number; // Represents the volume level of the additional audio track
+  asBase64?: boolean;
+};
+
+
+export const concatenateVideosWithAudio = async ({
+  output,
+  audioTrack = "",
+  audioFilePath = "",
+  videoTracks = [],
+  videoFilePaths = [],
+  videoTracksVolume = 0.5, // (1.0 = 100% volume)
+  audioTrackVolume = 0.5,
+  asBase64 = false,
+}: ConcatenateVideoWithAudioOptions): Promise<string> => {
+
+  try {
+    // Prepare temporary directories
+    const tempDir = path.join(os.tmpdir(), uuidv4());
+    await fs.mkdir(tempDir);
+
+    if (audioTrack) {
+      audioFilePath = path.join(tempDir, `audio.wav`);
+      await writeBase64ToFile(addBase64Header(audioTrack, "wav"), audioFilePath);
+    }
+
+    // Decode and concatenate base64 video tracks to temporary file
+    let i = 0
+    for (const track of videoTracks) {
+      if (!track) { continue }
+      const videoFilePath = path.join(tempDir, `video${++i}.mp4`);
+
+      await writeBase64ToFile(addBase64Header(track, "mp4"), videoFilePath);
+
+      videoFilePaths.push(videoFilePath);
+    }
+
+    videoFilePaths = videoFilePaths.filter((video) => existsSync(video))
+
+    // console.log("concatenating videos (without audio)..")
+    const tempFilePath = await concatenateVideos({
+      videoFilePaths,
+    })
+
+    // Check if the concatenated video has audio or not
+    const tempMediaInfo = await getMediaInfo(tempFilePath.filepath);
+    const hasOriginalAudio = tempMediaInfo.hasAudio;
+
+    const finalOutputFilePath = output || path.join(tempDir, `${uuidv4()}.mp4`);
+
+    // Begin ffmpeg command configuration
+    let cmd = ffmpeg();
+
+    // Add silent concatenated video
+    cmd = cmd.addInput(tempFilePath.filepath);
+ 
+    // If additional audio is provided, add audio to ffmpeg command
+    if (audioFilePath) {
+      cmd = cmd.addInput(audioFilePath);
+      // If the input video already has audio, we will mix it with additional audio
+      if (hasOriginalAudio) {
+        const filterComplex = `
+          [0:a]volume=${videoTracksVolume}[a0];
+          [1:a]volume=${audioTrackVolume}[a1];
+          [a0][a1]amix=inputs=2:duration=shortest[a]
+        `.trim();
+
+        cmd = cmd.outputOptions([
+          '-filter_complex', filterComplex,
+          '-map', '0:v',
+          '-map', '[a]',
+          '-c:v', 'copy',
+          '-c:a', 'aac',
+        ]);
+      } else {
+        // If the input video has no audio, just use the additional audio as is
+        cmd = cmd.outputOptions([
+          '-map', '0:v',
+          '-map', '1:a',
+          '-c:v', 'copy',
+          '-c:a', 'aac',
+        ]);
+      }
+    } else {
+      // If no additional audio is provided, simply copy the video stream
+      cmd = cmd.outputOptions([
+        '-c:v', 'copy',
+        hasOriginalAudio ? '-c:a' : '-an', // If original audio exists, copy it; otherwise, indicate no audio
+      ]);
+    }
+
+    /*
+    console.log("DEBUG:", {
+      videoTracksVolume,
+      audioTrackVolume,
+      videoFilePaths,
+      tempFilePath,
+      hasOriginalAudio,
+      // originalAudioVolume,
+      audioFilePath,
+      // additionalAudioVolume,
+      finalOutputFilePath
+     })
+     */
+  
+    // Set up event handlers for ffmpeg processing
+    const promise = new Promise<string>((resolve, reject) => {
+      cmd.on('error', (err) => {
+        console.error("    Error during ffmpeg processing:", err.message);
+        reject(err);
+      }).on('end', async () => {
+        // When ffmpeg finishes processing, resolve the promise with file info
+        try {
+          if (asBase64) {
+            try {
+              const outputBuffer = await fs.readFile(finalOutputFilePath);
+              const outputBase64 = addBase64Header(outputBuffer.toString("base64"), "mp4")
+              resolve(outputBase64);
+            } catch (error) {
+              reject(new Error(`Error reading output video file: ${(error as Error).message}`));
+            }
+          } else {
+            resolve(finalOutputFilePath)
+          }
+        } catch (err) {
+          reject(err);
+        }
+      }).save(finalOutputFilePath); // Provide the path where to save the file
+    });
+
+    // Wait for ffmpeg to complete the process
+    const result = await promise;
+    return result;
+  } catch (error) {
+    throw new Error(`Failed to assemble video: ${(error as Error).message}`);
+  } finally {
+    await removeTemporaryFiles([...videoFilePaths].concat(audioFilePath))
+  }
+};

src/core/ffmpeg/convertAudioToWav.mts

@@ -0,0 +1,69 @@
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import ffmpeg from "fluent-ffmpeg";
+import { Buffer } from "node:buffer";
+
+type ConvertAudioToWavParams = {
+  input: string;
+  outputAudioPath?: string;
+  asBase64?: boolean;
+};
+
+export async function convertAudioToWav({
+  input,
+  outputAudioPath,
+  asBase64 = false,
+}: ConvertAudioToWavParams): Promise<string> {
+  let inputAudioPath = input;
+
+  // Check if the input is a base64 string
+  if (input.startsWith("data:")) {
+    const matches = input.match(/^data:audio\/(mp3|wav);base64,(.+)$/);
+
+    if (!matches) {
+      throw new Error("Invalid base64 audio data");
+    }
+
+    const inputBuffer = Buffer.from(matches[2], "base64");
+    const inputFormat = matches[1]; // Either 'mp3' or 'wav'
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "ffmpeg-input-"));
+    inputAudioPath = path.join(tempDir, `temp.${inputFormat}`);
+
+    // Write the base64 data to the temporary file
+    await fs.writeFile(inputAudioPath, inputBuffer);
+  } else {
+    // Verify that the input file exists
+    if (!(await fs.stat(inputAudioPath)).isFile()) {
+      throw new Error(`Input audio file does not exist: ${inputAudioPath}`);
+    }
+  }
+
+  // If no output path is provided, create a temporary file for the output
+  if (!outputAudioPath) {
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "ffmpeg-output-"));
+    outputAudioPath = path.join(tempDir, `${path.parse(inputAudioPath).name}.wav`);
+  }
+
+  return new Promise((resolve, reject) => {
+    ffmpeg(inputAudioPath)
+      .toFormat("wav")
+      .on("error", (err) => {
+        reject(new Error(`Error converting audio to WAV: ${err.message}`));
+      })
+      .on("end", async () => {
+        if (asBase64) {
+          try {
+            const audioBuffer = await fs.readFile(outputAudioPath);
+            const audioBase64 = `data:audio/wav;base64,${audioBuffer.toString("base64")}`;
+            resolve(audioBase64);
+          } catch (error) {
+            reject(new Error(`Error reading audio file: ${(error as Error).message}`));
+          }
+        } else {
+          resolve(outputAudioPath);
+        }
+      })
+      .save(outputAudioPath);
+  });
+}

src/core/ffmpeg/convertMp4ToMp3.mts

@@ -0,0 +1,65 @@
+
+import { mkdtemp, stat, writeFile, readFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import ffmpeg from "fluent-ffmpeg";
+import { tmpdir } from "node:os";
+import { Buffer } from "node:buffer";
+
+export async function convertMp4ToMp3({
+  input,
+  outputAudioPath,
+  asBase64 = false,
+}: {
+  input: string;
+  outputAudioPath?: string;
+  asBase64?: boolean;
+}): Promise<string> {
+  let inputFilePath = input;
+
+  // Check if the input is a base64 string
+  if (input.startsWith("data:")) {
+    const base64Data = input.split(",")[1];
+    const inputBuffer = Buffer.from(base64Data, "base64");
+
+    // Create a temporary file for the input video
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), "ffmpeg-input-"));
+    inputFilePath = path.join(tempDir, "temp.mp4");
+
+    // Write the base64 data to the temporary file
+    await writeFile(inputFilePath, inputBuffer);
+  } else {
+    // Verify that the input file exists
+    if (!(await stat(inputFilePath)).isFile()) {
+      throw new Error(`Input video file does not exist: ${inputFilePath}`);
+    }
+  }
+
+  // If no output path is provided, create a temporary file for the output
+  if (!outputAudioPath) {
+    const tempDir = await mkdtemp(path.join(tmpdir(), "ffmpeg-output-"));
+    outputAudioPath = path.join(tempDir, `${path.parse(inputFilePath).name}.mp3`);
+  }
+
+  return new Promise((resolve, reject) => {
+    ffmpeg(inputFilePath)
+      .toFormat("mp3")
+      .on("error", (err) => {
+        reject(new Error(`Error converting video to audio: ${err.message}`));
+      })
+      .on("end", async () => {
+        if (asBase64) {
+          try {
+            const audioBuffer = await readFile(outputAudioPath);
+            const audioBase64 = `data:audio/mp3;base64,${audioBuffer.toString("base64")}`;
+            resolve(audioBase64);
+          } catch (error) {
+            reject(new Error(`Error reading audio file: ${(error as Error).message}`));
+          }
+        } else {
+          resolve(outputAudioPath);
+        }
+      })
+      .save(outputAudioPath);
+  });
+}

src/core/ffmpeg/convertMp4ToWebm.mts

@@ -0,0 +1,70 @@
+
+import { mkdtemp, stat, writeFile, readFile } from "node:fs/promises";
+import path from "node:path";
+
+import { tmpdir } from "node:os";
+import { Buffer } from "node:buffer";
+
+import ffmpeg from "fluent-ffmpeg";
+
+export async function convertMp4ToWebm({
+  input,
+  outputVideoPath,
+  asBase64 = false,
+}: {
+  input: string;
+  outputVideoPath?: string;
+  asBase64?: boolean;
+}): Promise<string> {
+  let inputFilePath = input;
+
+  // Check if the input is a base64 string
+  if (input.startsWith("data:")) {
+    const base64Data = input.split(",")[1];
+    const inputBuffer = Buffer.from(base64Data, "base64");
+
+    // Create a temporary file for the input video
+    const tempDir = await mkdtemp(path.join(tmpdir(), "ffmpeg-input-"));
+    inputFilePath = path.join(tempDir, "temp.mp4");
+
+    // Write the base64 data to the temporary file
+    await writeFile(inputFilePath, inputBuffer);
+  } else {
+    // Verify that the input file exists
+    const inputFileStats = await stat(inputFilePath);
+    if (!inputFileStats.isFile()) {
+      throw new Error(`Input video file does not exist: ${inputFilePath}`);
+    }
+  }
+
+  // If no output path is provided, create a temporary file for the output
+  if (!outputVideoPath) {
+    const tempDir = await mkdtemp(path.join(tmpdir(), "ffmpeg-output-"));
+    outputVideoPath = path.join(tempDir, `${path.parse(inputFilePath).name}.webm`);
+  }
+
+  return new Promise((resolve, reject) => {
+    ffmpeg(inputFilePath)
+      .toFormat("webm")
+      .videoCodec("libvpx")
+      .addOption("-b:v", "1000k") // ~ 400 kB for 3 seconds of video
+      .audioCodec("libvorbis")
+      .on("error", (err) => {
+        reject(new Error(`Error converting video to WebM: ${err.message}`));
+      })
+      .on("end", async () => {
+        if (asBase64) {
+          try {
+            const videoBuffer = await readFile(outputVideoPath);
+            const videoBase64 = `data:video/webm;base64,${videoBuffer.toString("base64")}`;
+            resolve(videoBase64);
+          } catch (error) {
+            reject(new Error(`Error reading video file: ${(error as Error).message}`));
+          }
+        } else {
+          resolve(outputVideoPath);
+        }
+      })
+      .save(outputVideoPath);
+  });
+}

src/core/ffmpeg/createTextOverlayImage.mts

@@ -0,0 +1,57 @@
+
+import { TextOverlayFont, TextOverlayFontWeight, TextOverlayStyle, getCssStyle } from "../utils/getCssStyle.mts"
+import { htmlToBase64Png } from "../converters/htmlToBase64Png.mts"
+
+// generate a PNG overlay using HTML
+export async function createTextOverlayImage({
+  text = "",
+  textStyle = "outline",
+  fontFamily = "Montserrat",
+  fontSize = 10,
+  fontWeight = 600,
+  rotation = 0,
+  width = 1024,
+  height = 576
+}: {
+  text?: string
+  textStyle?: TextOverlayStyle
+  fontFamily?: TextOverlayFont
+  fontSize?: number
+  fontWeight?: TextOverlayFontWeight
+  rotation?: number
+  width?: number
+  height?: number
+}): Promise<{
+  filePath: string
+  buffer: Buffer
+}> {
+
+
+  const html = `<html>
+  <head>${getCssStyle({
+    fontFamily,
+    fontSize,
+    fontWeight: 600,
+  })}</head>
+  <body>
+
+    <!-- main content block (will be center in the middle of the screen) -->
+    <div class="content">
+
+      <!-- main line of text -->
+      <p class="${textStyle}">
+        ${text}
+      </p>
+    </div>
+
+  </body>
+</html>`
+
+  const result = await htmlToBase64Png({
+    html,
+    width,
+    height,
+  })
+
+  return result;
+};

src/core/ffmpeg/createVideoFromFrames.mts

@@ -0,0 +1,173 @@
+import { promises as fs } from "node:fs"
+import { writeFile, readFile } from "node:fs/promises"
+import os from "node:os"
+import path from "node:path"
+
+import ffmpeg from "fluent-ffmpeg"
+import { v4 as uuidv4 } from "uuid"
+
+import { getMediaInfo } from "./getMediaInfo.mts"
+
+export async function createVideoFromFrames({
+  inputFramesDirectory,
+  framesFilePattern,
+  outputVideoPath,
+  framesPerSecond = 25,
+
+  // there isn't a lot of advantage for us to add film grain because:
+  // 1. I actually can't tell the different, probably because it's in HD, and so tiny
+  // 2. We want a neat "4K video from the 2020" look, not a quality from 30 years ago
+  // 3. grain has too much entropy and cannot be compressed, so it multiplies by 5 the size weight
+  grainAmount = 0, // Optional parameter for film grain (eg. 10)
+
+  inputVideoToUseAsAudio, // Optional parameter for audio input (need to be a mp4, but it can be a base64 data URI or a file path)
+
+  debug = false,
+
+  asBase64 = false,
+}: {
+  inputFramesDirectory: string;
+
+  // the ffmpeg file pattern to use
+  framesFilePattern?: string;
+
+  outputVideoPath?: string;
+  framesPerSecond?: number;
+  grainAmount?: number; // Values can range between 0 and higher for the desired amount
+  inputVideoToUseAsAudio?: string; //  Optional parameter for audio input (need to be a mp4, but it can be a base64 data URI or a file path)
+  debug?: boolean;
+  asBase64?: boolean;
+}): Promise<string> {
+  // Ensure the input directory exists
+  await fs.access(inputFramesDirectory);
+
+
+  // Construct the input frame pattern
+  const inputFramePattern = path.join(inputFramesDirectory, framesFilePattern);
+
+
+  // Create a temporary working directory
+  const tempDir = path.join(os.tmpdir(), uuidv4());
+  await fs.mkdir(tempDir);
+
+   
+  let inputVideoToUseAsAudioFilePath = "";
+  if (inputVideoToUseAsAudio.startsWith('data:')) {
+    // Extract the base64 content and decode it to a temporary file
+    const base64Content = inputVideoToUseAsAudio.split(';base64,').pop();
+    if (!base64Content) {
+      throw new Error('Invalid base64 input provided');
+    }
+    inputVideoToUseAsAudioFilePath = path.join(tempDir, `${uuidv4()}_audio_input.mp4`);
+    await writeFile(inputVideoToUseAsAudioFilePath, base64Content, 'base64');
+  } else {
+    inputVideoToUseAsAudioFilePath = inputVideoToUseAsAudio;
+  }
+
+  if (debug) {
+    console.log("      createVideoFromFraes(): inputVideoToUseAsAudioFilePath = ", inputVideoToUseAsAudioFilePath)
+  }
+
+
+  let canUseInputVideoForAudio = false
+  // Also, if provided, check that the audio source file exists
+  if (inputVideoToUseAsAudioFilePath) {
+    try {
+      await fs.access(inputVideoToUseAsAudioFilePath)
+      const info = await getMediaInfo(inputVideoToUseAsAudioFilePath)
+      if (info.hasAudio) {
+        canUseInputVideoForAudio = true
+      }
+    } catch (err) {
+      if (debug) {
+        console.log("      createVideoFromFrames(): warning: input video has no audio, so we are not gonna use that")
+      }
+    }
+  }
+
+  const outputVideoFilePath = outputVideoPath ?? path.join(tempDir, `${uuidv4()}.mp4`);
+
+  if (debug) {
+    console.log("      createVideoFromFrames(): outputOptions:", [
+      // by default ffmpeg doesn't tell us why it fails to convet
+      // so we need to force it to spit everything out
+      "-loglevel", "debug",
+
+      "-pix_fmt", "yuv420p",
+      "-c:v", "libx264",
+      "-r", `${framesPerSecond}`,
+
+      // from ffmpeg doc: "Consider 17 or 18 to be visually lossless or nearly so; 
+      // it should look the same or nearly the same as the input."
+      "-crf", "17",
+    ])
+  }
+
+  return new Promise<string>((resolve, reject) => {
+    const command = ffmpeg()
+      .input(inputFramePattern)
+      .inputFPS(framesPerSecond)
+      .outputOptions([
+        // by default ffmpeg doesn't tell us why it fails to convet
+        // so we need to force it to spit everything out
+        "-loglevel", "debug",
+
+        "-pix_fmt", "yuv420p",
+        "-c:v", "libx264",
+        "-r", `${framesPerSecond}`,
+        "-crf", "18",
+      ]);
+
+    
+    // If an input video for audio is provided, add it as an input for the ffmpeg command
+    if (canUseInputVideoForAudio) {
+      if (debug) {
+       console.log("      createVideoFromFrames(): adding audio as input:", inputVideoToUseAsAudioFilePath)
+      }
+      command.addInput(inputVideoToUseAsAudioFilePath);
+      command.outputOptions([
+        "-map", "0:v", // Map video from the frames
+        "-map", "1:a", // Map audio from the input video
+        "-shortest"    // Ensure output video duration is the shortest of the combined inputs
+      ]);
+    }
+
+    // Apply grain effect using the geq filter if grainAmount is specified
+    if (grainAmount != null && grainAmount > 0) {
+      if (debug) {
+        console.log("      createVideoFromFrames(): adding grain:", grainAmount)
+      }
+      command.complexFilter([
+        {
+          filter: "geq",
+          options: `lum='lum(X,Y)':cr='cr(X,Y)+(random(1)-0.5)*${grainAmount}':cb='cb(X,Y)+(random(1)-0.5)*${grainAmount}'`
+        }
+      ]);
+    }
+
+    command.save(outputVideoFilePath)
+      .on("error", (err) => reject(err))
+      .on("end", async () => {
+        if (debug) {
+          console.log("      createVideoFromFrames(): outputVideoFilePath: ", outputVideoFilePath)
+        }
+        if (!asBase64) {
+          resolve(outputVideoFilePath)
+          return
+        }
+        // Convert the output file to a base64 string
+        try {
+          const videoBuffer = await readFile(outputVideoFilePath);
+          const videoBase64 = `data:video/mp4;base64,${videoBuffer.toString('base64')}`;
+          console.log("      createVideoFromFrames(): output base64: ", videoBase64.slice(0, 120))
+          resolve(videoBase64);
+        } catch (error) {
+          reject(new Error(`Error loading the video file: ${error}`));
+        } finally {
+          // Clean up temporary files
+          await fs.rm(tempDir, { recursive: true });
+        }
+      });
+  });
+}
+

src/core/ffmpeg/cropBase64Video.mts

@@ -0,0 +1,65 @@
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import ffmpeg from "fluent-ffmpeg";
+
+export async function cropBase64Video({
+  base64Video,
+  width,
+  height,
+}: {
+  base64Video: string;
+  width: number;
+  height: number;
+}): Promise<string> {
+  // Create a buffer from the base64 string, skipping the data URI scheme
+  const base64Data = base64Video.replace(/^data:video\/mp4;base64,/, "");
+  const videoBuffer = Buffer.from(base64Data, "base64");
+
+  // Create a temporary file for the input video
+  const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "ffmpeg-crop-input-"));
+  const inputVideoPath = path.join(tempDir, `input.mp4`);
+  await fs.writeFile(inputVideoPath, videoBuffer);
+
+  // Create a temporary file for the output video
+  const outputTempDir = await fs.mkdtemp(path.join(os.tmpdir(), "ffmpeg-crop-output-"));
+  const outputVideoPath = path.join(outputTempDir, `output-cropped.mp4`);
+
+  // Return a promise that resolves with the path to the output cropped video file
+  return new Promise((resolve, reject) => {
+    ffmpeg(inputVideoPath)
+      .ffprobe((err, metadata) => {
+        if (err) {
+          reject(new Error(`Error reading video metadata: ${err.message}`));
+          return;
+        }
+
+        const videoStream = metadata.streams.find(s => s.codec_type === "video");
+        if (!videoStream) {
+          reject(new Error(`Cannot find video stream in file: ${inputVideoPath}`));
+          return;
+        }
+
+        const { width: inWidth, height: inHeight } = videoStream;
+        const x = Math.floor((inWidth - width) / 2);
+        const y = Math.floor((inHeight - height) / 2);
+
+        ffmpeg(inputVideoPath)
+          .outputOptions([
+            `-vf crop=${width}:${height}:${x}:${y}`
+          ])
+          .on("error", (err) => {
+            reject(new Error(`Error cropping video: ${err.message}`));
+          })
+          .on("end", () => {
+            resolve(outputVideoPath);
+          })
+          .on('codecData', (data) => {
+            console.log('Input is ' + data.audio + ' audio ' +
+              'with ' + data.video + ' video');
+          })
+          .save(outputVideoPath);
+      });
+  });
+}

src/core/ffmpeg/cropVideo.mts

@@ -0,0 +1,76 @@
+import { promises as fs } from "node:fs";
+// import { writeFile, readFile } from 'node:fs/promises';
+import os from "node:os";
+import path from "node:path";
+
+import ffmpeg from "fluent-ffmpeg";
+
+export async function cropVideo({
+  inputVideoPath,
+  width,
+  height,
+  debug = false,
+  asBase64 = false,
+}: {
+  inputVideoPath: string
+  width: number
+  height: number
+  debug?: boolean
+  asBase64?: boolean
+}): Promise<string> {
+  // Verify that the input file exists
+  if (!(await fs.stat(inputVideoPath)).isFile()) {
+    throw new Error(`Input video file does not exist: ${inputVideoPath}`);
+  }
+
+  // Create a temporary file for the output
+  const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "ffmpeg-crop-"));
+  const outputVideoPath = path.join(tempDir, `${path.parse(inputVideoPath).name}-cropped.mp4`);
+
+  // Return a promise that resolves with the path to the output cropped video file
+  return new Promise((resolve, reject) => {
+    ffmpeg(inputVideoPath)
+      .ffprobe((err, metadata) => {
+        if (err) {
+          reject(new Error(`Error reading video metadata: ${err.message}`));
+          return;
+        }
+
+        const videoStream = metadata.streams.find(s => s.codec_type === "video");
+        if (!videoStream) {
+          reject(new Error(`Cannot find video stream in file: ${inputVideoPath}`));
+          return;
+        }
+
+        const { width: inWidth, height: inHeight } = videoStream;
+        const x = Math.floor((inWidth - width) / 2);
+        const y = Math.floor((inHeight - height) / 2);
+
+        ffmpeg(inputVideoPath)
+          .outputOptions([
+            `-vf crop=${width}:${height}:${x}:${y}`
+          ])
+          .on("error", (err) => {
+            reject(new Error(`Error cropping video: ${err.message}`));
+          })
+          .on("end", async () => {
+            if (!asBase64) {
+              resolve(outputVideoPath)
+              return
+            }
+            // Convert the output file to a base64 string
+            try {
+              const videoBuffer = await fs.readFile(outputVideoPath);
+              const videoBase64 = `data:video/mp4;base64,${videoBuffer.toString('base64')}`;
+              resolve(videoBase64);
+            } catch (error) {
+              reject(new Error(`Error loading the video file: ${error}`));
+            } finally {
+              // Clean up temporary files
+              await fs.rm(tempDir, { recursive: true });
+            }
+          })
+          .save(outputVideoPath);
+      });
+  });
+}

src/core/ffmpeg/getMediaInfo.mts

@@ -0,0 +1,79 @@
+import ffmpeg from "fluent-ffmpeg";
+
+import { tmpdir } from "node:os";
+import { promises as fs } from "node:fs";
+import { join } from "node:path";
+
+export type MediaMetadata = {
+  durationInSec: number;
+  durationInMs: number;
+  hasAudio: boolean;
+};
+
+/**
+ * Get the media info of a base64 or file path
+ * @param input 
+ * @returns 
+ */
+export async function getMediaInfo(input: string): Promise<MediaMetadata> {
+  // If the input is a base64 string
+  if (input.startsWith("data:")) {
+    // Extract the base64 content
+    const base64Content = input.split(";base64,").pop();
+    if (!base64Content) {
+      throw new Error("Invalid base64 data");
+    }
+
+    // Decode the base64 content to a buffer
+    const buffer = Buffer.from(base64Content, 'base64');
+
+    // Generate a temporary file name
+    const tempFileName = join(tmpdir(), `temp-media-${Date.now()}`);
+
+    // Write the buffer to a temporary file
+    await fs.writeFile(tempFileName, buffer);
+
+    // Get metadata from the temporary file then delete the file
+    try {
+      return await getMetaDataFromPath(tempFileName);
+    } finally {
+      await fs.unlink(tempFileName);
+    }
+  }
+
+  // If the input is a path to the file
+  return await getMetaDataFromPath(input);
+}
+
+async function getMetaDataFromPath(filePath: string): Promise<MediaMetadata> {
+  return new Promise((resolve, reject) => {
+    ffmpeg.ffprobe(filePath, (err, metadata) => {
+
+      let results = {
+        durationInSec: 0,
+        durationInMs: 0,
+        hasAudio: false,
+      }
+
+      if (err) {
+        console.error("getMediaInfo(): failed to analyze the source (might happen with empty files)")
+        // reject(err);
+        resolve(results);
+        return;
+      }
+
+      try {
+        results.durationInSec = metadata?.format?.duration || 0;
+        results.durationInMs = results.durationInSec * 1000;
+        results.hasAudio = (metadata?.streams || []).some((stream) => stream.codec_type === 'audio');
+
+      } catch (err) {
+        console.error(`getMediaInfo(): failed to analyze the source (might happen with empty files)`)
+        results.durationInSec = 0
+        results.durationInMs = 0
+        results.hasAudio = false
+      }
+      resolve(results);
+    });
+  });
+}

src/core/ffmpeg/scaleVideo.mts

@@ -0,0 +1,90 @@
+import fs from 'node:fs/promises';
+import { writeFile, readFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+
+import { v4 as uuidv4 } from "uuid";
+import ffmpeg from 'fluent-ffmpeg';
+
+export type ScaleVideoParams = {
+  input: string;
+  height: number;
+  debug?: boolean;
+  asBase64?: boolean;
+}
+
+/**
+ * Rescale a video (either file or base 64) to a given height.
+ * This returns a base64 video.
+ * 
+ * Some essential things to note in this implementation:
+ * 
+ * If the input is a valid base64 string, it gets decoded and stored as a temporary .mp4 file.
+ * The ffmpeg.outputOptions includes the arguments for setting the output video height and keeping the aspect ratio intact. The -1 in scale=-1:${height} tells ffmpeg to preserve aspect ratio based on the height.
+ * The output is a libx264-encoded MP4 video, matching typical browser support standards.
+ * Upon completion, the temporary output file is read into a buffer, converted to a base64 string with the correct prefix, and then cleaned up by removing temporary files.
+ * To call this function with desired input and height, you'd use it similarly to the provided convertMp4ToMp3 function example, being mindful that input must be a file path or properly-formatted base64 string and height is a number representing the new height of the video.
+ * 
+ * Enter your message...
+ * 
+ * @param param0 
+ * @returns 
+ */
+export async function scaleVideo({
+  input,
+  height,
+  asBase64 = false,
+  debug = false
+}: ScaleVideoParams): Promise<string> {
+  const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "ffmpeg-"));
+  const tempOutPath = path.join(tempDir, `${uuidv4()}.mp4`);
+  
+  let inputPath;
+  if (input.startsWith('data:')) {
+    // Extract the base64 content and decode it to a temporary file
+    const base64Content = input.split(';base64,').pop();
+    if (!base64Content) {
+      throw new Error('Invalid base64 input provided');
+    }
+    inputPath = path.join(tempDir, `${uuidv4()}.mp4`);
+    await writeFile(inputPath, base64Content, 'base64');
+  } else {
+    inputPath = input;
+  }
+
+  if (debug) {
+    console.log("inputPath:", inputPath)
+  }
+  
+  // Return a promise that resolves with the base64 string of the output video
+  return new Promise((resolve, reject) => {
+    ffmpeg(inputPath)
+      .outputOptions([
+        '-vf', `scale=-1:${height}`,
+        '-c:v', 'libx264',
+        '-preset', 'fast',
+        '-crf', '22'
+      ])
+      .on('error', (err) => {
+        reject(new Error(`Error scaling the video: ${err.message}`));
+      })
+      .on('end', async () => {
+        if (!asBase64) {
+          resolve(tempOutPath)
+          return
+        }
+        // Convert the output file to a base64 string
+        try {
+          const videoBuffer = await readFile(tempOutPath);
+          const videoBase64 = `data:video/mp4;base64,${videoBuffer.toString('base64')}`;
+          resolve(videoBase64);
+        } catch (error) {
+          reject(new Error(`Error loading the video file: ${error}`));
+        } finally {
+          // Clean up temporary files
+          await fs.rm(tempDir, { recursive: true });
+        }
+      })
+      .save(tempOutPath);
+  });
+}

src/core/files/deleteFileWithName.mts

@@ -0,0 +1,17 @@
+import { promises as fs } from "node:fs"
+import path from "node:path"
+
+export const deleteFilesWithName = async (dir: string, name: string, debug?: boolean) => {
+  for (const file of await fs.readdir(dir)) {
+    if (file.includes(name)) {
+      const filePath = path.join(dir, file)
+      try {
+        await fs.unlink(filePath)
+      } catch (err) {
+        if (debug) {
+          console.error(`failed to unlink file in ${filePath}: ${err}`)
+        }
+      }
+    }
+  }
+}

src/core/files/downloadFileAsBase64.mts

@@ -0,0 +1,27 @@
+import { lookup } from "mime-types"
+
+export const downloadFileAsBase64 = async (remoteUrl: string): Promise<string> => {
+  // const controller = new AbortController()
+
+  // download the file
+  const response = await fetch(remoteUrl, {
+    // signal: controller.signal
+  })
+
+  // get as Buffer
+  const arrayBuffer = await response.arrayBuffer()
+  const buffer = Buffer.from(arrayBuffer)
+
+  // convert it to base64
+  const base64 = buffer.toString('base64')
+
+
+  const res = lookup(remoteUrl)
+  let contentType = res.toString()
+  if (typeof res === "boolean" && res === false) {
+    contentType = response.headers.get('content-type')
+  }
+
+  const assetUrl = `data:${contentType};base64,${base64}`
+  return assetUrl
+};

src/core/files/readJpegFileToBase64.mts

@@ -0,0 +1,18 @@
+import { promises as fs } from "fs"
+
+export async function readJpegFileToBase64(filePath: string): Promise<string> {
+  try {
+    // Read the file's content as a Buffer
+    const fileBuffer = await fs.readFile(filePath);
+
+    // Convert the buffer to a base64 string
+    const base64 = fileBuffer.toString('base64');
+
+    // Prefix the base64 string with the Data URI scheme for PNG images
+    return `data:image/jpeg;base64,${base64}`;
+  } catch (error) {
+    // Handle errors (e.g., file not found, no permissions, etc.)
+    console.error(error);
+    throw error;
+  }
+}

src/core/files/readMp3FileToBase64.mts

@@ -0,0 +1,18 @@
+import { promises as fs } from "fs"
+
+export async function readMp3FileToBase64(filePath: string): Promise<string> {
+  try {
+    // Read the file's content as a Buffer
+    const fileBuffer = await fs.readFile(filePath);
+
+    // Convert the buffer to a base64 string
+    const base64 = fileBuffer.toString('base64');
+
+    // Prefix the base64 string with the Data URI scheme for PNG images
+    return `data:audio/mp3;base64,${base64}`;
+  } catch (error) {
+    // Handle errors (e.g., file not found, no permissions, etc.)
+    console.error(error);
+    throw error;
+  }
+}

src/core/files/readMp4FileToBase64.mts

@@ -0,0 +1,18 @@
+import { promises as fs } from "fs"
+
+export async function readMp4FileToBase64(filePath: string): Promise<string> {
+  try {
+    // Read the file's content as a Buffer
+    const fileBuffer = await fs.readFile(filePath);
+
+    // Convert the buffer to a base64 string
+    const base64 = fileBuffer.toString('base64');
+
+    // Prefix the base64 string with the Data URI scheme for PNG images
+    return `data:video/mp4;base64,${base64}`;
+  } catch (error) {
+    // Handle errors (e.g., file not found, no permissions, etc.)
+    console.error(error);
+    throw error;
+  }
+}

src/core/files/readPlainText.mts

@@ -0,0 +1,13 @@
+import { promises as fs } from "fs"
+
+export async function readPlainText(filePath: string): Promise<string> {
+  try {
+    const plainText = await fs.readFile(filePath, "utf-8");
+
+    return plainText;
+  } catch (error) {
+    // Handle errors (e.g., file not found, no permissions, etc.)
+    console.error(error);
+    throw error;
+  }
+}