add support for OpenAI Dall-e 3 (well, it doesn't work yet, some kind of 400 error)

.env

@@ -3,6 +3,7 @@
 # - REPLICATE
 # - INFERENCE_ENDPOINT
 # - INFERENCE_API
+# - OPENAI
 RENDERING_ENGINE="INFERENCE_API"
 
 # Supported values:
@@ -32,7 +33,7 @@ AUTH_HF_API_TOKEN=
 AUTH_REPLICATE_API_TOKEN=
 
 # OpenAI.dom token: available for the LLM engine and the RENDERING engine
-AUTH_OPENAI_TOKEN=
+AUTH_OPENAI_API_KEY=
 
 # An experimental RENDERING engine (sorry it is not very documented yet, so you can use one of the other engines)
 AUTH_VIDEOCHAIN_API_TOKEN=
@@ -55,6 +56,9 @@ RENDERING_HF_INFERENCE_API_REFINER_MODEL="stabilityai/stable-diffusion-xl-refine
 # An experimental RENDERING engine (sorry it is not very documented yet, so you can use one of the other engines)
 RENDERING_VIDEOCHAIN_API_URL="http://localhost:7860"
 
+RENDERING_OPENAI_API_BASE_URL="https://api.openai.com/v1"
+RENDERING_OPENAI_API_MODEL="dall-e-3"
+
 # ------------- LLM API CONFIG ----------------
 
 # If you decided to use OpenAI for the LLM engine

README.md

@@ -24,7 +24,7 @@ If you try to duplicate the project, open the `.env` you will see it requires so
 
 Provider config:
 - `LLM_ENGINE`: can be one of: "INFERENCE_API", "INFERENCE_ENDPOINT", "OPENAI"
-- `RENDERING_ENGINE`: can be one of: "INFERENCE_API", "INFERENCE_ENDPOINT", "REPLICATE", "VIDEOCHAIN" for now, unless you code your custom solution
+- `RENDERING_ENGINE`: can be one of: "INFERENCE_API", "INFERENCE_ENDPOINT", "REPLICATE", "VIDEOCHAIN", "OPENAI" for now, unless you code your custom solution
 
 Auth config:
 - `AUTH_HF_API_TOKEN`: only if you decide to use OpenAI for the LLM engine necessary if you decide to use an inference api model or a custom inference endpoint

package-lock.json

@@ -44,7 +44,7 @@
         "konva": "^9.2.2",
         "lucide-react": "^0.260.0",
         "next": "13.4.10",
-        "openai": "^4.10.0",
+        "openai": "^4.15.4",
         "pick": "^0.0.1",
         "postcss": "8.4.26",
         "react": "18.2.0",
@@ -3907,9 +3907,9 @@
       "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ=="
     },
     "node_modules/fast-glob": {
-      "version": "3.3.1",
-      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.1.tgz",
-      "integrity": "sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==",
+      "version": "3.3.2",
+      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz",
+      "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==",
       "dependencies": {
         "@nodelib/fs.stat": "^2.0.2",
         "@nodelib/fs.walk": "^1.2.3",
@@ -5178,9 +5178,9 @@
       }
     },
     "node_modules/nanoid": {
-      "version": "3.3.6",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz",
-      "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==",
+      "version": "3.3.7",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz",
+      "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==",
       "funding": [
         {
           "type": "github",
@@ -5493,9 +5493,9 @@
       }
     },
     "node_modules/openai": {
-      "version": "4.15.3",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-4.15.3.tgz",
-      "integrity": "sha512-j2XSxxiOhF7lCiHUXmuDOGOgtKa36ia9pOQ2m9YCOMA2Ee4QTI+MzdHRoHlp6ewOsvW5NXkoT+xustSZljiGnA==",
+      "version": "4.15.4",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.15.4.tgz",
+      "integrity": "sha512-EnlSl1p8n7Q/HnBf4+VOEcYloBKKe23sKOFfH/WJcw+XVyWav4lwDK4wCmsUY1wS4RFOdbA2EwBUB2p5WEPmoQ==",
       "dependencies": {
         "@types/node": "^18.11.18",
         "@types/node-fetch": "^2.6.4",

package.json

@@ -45,7 +45,7 @@
     "konva": "^9.2.2",
     "lucide-react": "^0.260.0",
     "next": "13.4.10",
-    "openai": "^4.10.0",
+    "openai": "^4.15.4",
     "pick": "^0.0.1",
     "postcss": "8.4.26",
     "react": "18.2.0",

src/app/engine/render.ts

@@ -2,6 +2,7 @@
 
 import { v4 as uuidv4 } from "uuid"
 import Replicate from "replicate"
+import OpenAI from "openai"
 
 import { RenderRequest, RenderedScene, RenderingEngine } from "@/types"
 import { generateSeed } from "@/lib/generateSeed"
@@ -22,6 +23,10 @@ const replicateModelVersion = `${process.env.RENDERING_REPLICATE_API_MODEL_VERSI
 const videochainToken = `${process.env.AUTH_VIDEOCHAIN_API_TOKEN || ""}`
 const videochainApiUrl = `${process.env.RENDERING_VIDEOCHAIN_API_URL || ""}`
 
+const openaiApiKey = `${process.env.AUTH_OPENAI_API_KEY || ""}`
+const openaiApiBaseUrl = `${process.env.RENDERING_OPENAI_API_BASE_URL || "https://api.openai.com/v1"}`
+const openaiApiModel = `${process.env.RENDERING_OPENAI_API_MODEL || "dall-e-3"}`
+
 export async function newRender({
   prompt,
   // negativePrompt,
@@ -57,7 +62,69 @@ export async function newRender({
   const guidanceScale = 9
 
   try {
-    if (renderingEngine === "REPLICATE") {
+    if (renderingEngine === "OPENAI") {
+
+      /*
+      const openai = new OpenAI({
+        apiKey: openaiApiKey
+      });
+      */
+
+      // When using DALL·E 3, images can have a size of 1024x1024, 1024x1792 or 1792x1024 pixels.
+      // the improved resolution is nice, but the AI Comic Factory needs a special ratio
+      // anyway, let's see what we can do
+      
+      const size =
+        width > height ? '1792x1024' :
+        width < height ? '1024x1792' :
+        '1024x1024'
+
+      /*
+      const response = await openai.createImage({
+        model: "dall-e-3",
+        prompt,
+        n: 1,
+        size: size as any,
+        // quality: "standard",
+      })
+      */
+
+      const res = await fetch(`${openaiApiBaseUrl}/images/generations`, {
+        method: "POST",
+        headers: {
+          Accept: "application/json",
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${openaiApiKey}`,
+        },
+        body: JSON.stringify({
+          model: "dall-e-3",
+          prompt,
+          n: 1,
+          size,
+          // quality: "standard",
+        }),
+        cache: 'no-store',
+      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+      // next: { revalidate: 1 }
+      })
+
+      if (res.status !== 200) {
+        throw new Error('Failed to fetch data')
+      }
+      
+      const response = (await res.json()) as { data: { url: string }[] }
+
+      console.log("response:", response)
+      return {
+        renderId: uuidv4(),
+        status: "completed",
+        assetUrl: response.data[0].url || "", 
+        alt: prompt,
+        error: "",
+        maskUrl: "",
+        segments: []
+      } as RenderedScene
+    } else if (renderingEngine === "REPLICATE") {
       if (!replicateToken) {
         throw new Error(`you need to configure your REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)
       }