allow support for Inference API for SDXL

.env

@@ -1,35 +1,49 @@
-# ------------- IMAGE API CONFIG --------------
 # Supported values:
 # - VIDEOCHAIN
 # - REPLICATE
-RENDERING_ENGINE="REPLICATE"
-
-VIDEOCHAIN_API_URL="http://localhost:7860"
-VIDEOCHAIN_API_TOKEN=
-
-REPLICATE_API_TOKEN=
-REPLICATE_API_MODEL="stabilityai/sdxl"
-REPLICATE_API_MODEL_VERSION="da77bc59ee60423279fd632efb4795ab731d9e3ca9705ef3341091fb989b7eaf"
+# - INFERENCE_ENDPOINT
+# - INFERENCE_API
+RENDERING_ENGINE="INFERENCE_API"
 
-# ------------- LLM API CONFIG ----------------
 # Supported values:
 # - INFERENCE_ENDPOINT
 # - INFERENCE_API
-LLM_ENGINE="INFERENCE_ENDPOINT"
+LLM_ENGINE="INFERENCE_API"
+
+# ------------- PROVIDER AUTH ------------
 
-# Hugging Face token (if you choose to use a custom Inference Endpoint or an Inference API model)
+# Hugging Face token, if you plan to use the Inference API or Inference Endpoint
+# for the LLM or SDXL generation
 HF_API_TOKEN=
 
+# Replicate token, if you wish to use them as a provider for SDXL
+REPLICATE_API_TOKEN=
+
+# OpenAI is not supported yet
+OPENAI_TOKEN=
+
+# VideoChain is a custom API used for SDXL but you don't need it for the base features
+VIDEOCHAIN_API_TOKEN=
+
+# ------------- RENDERING API CONFIG --------------
+
+RENDERING_VIDEOCHAIN_API_URL="http://localhost:7860"
+
+RENDERING_REPLICATE_API_MODEL="stabilityai/sdxl"
+RENDERING_REPLICATE_API_MODEL_VERSION="da77bc59ee60423279fd632efb4795ab731d9e3ca9705ef3341091fb989b7eaf"
+
+RENDERING_HF_INFERENCE_ENDPOINT_URL="https://XXXXXXXXXX.endpoints.huggingface.cloud"
+RENDERING_HF_INFERENCE_API_MODEL="stabilityai/stable-diffusion-xl-base-1.0"
+
+# ------------- LLM API CONFIG ----------------
+
 # URL to a custom text-generation Inference Endpoint of your choice
 # -> You can leave it empty if you decide to use an Inference API Model instead
-HF_INFERENCE_ENDPOINT_URL=
+LLM_HF_INFERENCE_ENDPOINT_URL=
 
 # You can also use a model from the Inference API (not a custom inference endpoint)
 # -> You can leave it empty if you decide to use an Inference Endpoint URL instead
-HF_INFERENCE_API_MODEL="codellama/CodeLlama-7b-hf"
-
-# Not supported yet
-OPENAI_TOKEN=
+LLM_HF_INFERENCE_API_MODEL="codellama/CodeLlama-7b-hf"
 
 # ----------- COMMUNITY SHARING (OPTIONAL) -----------
 NEXT_PUBLIC_ENABLE_COMMUNITY_SHARING="false"

README.md

@@ -17,17 +17,28 @@ First, I would like to highlight that everything is open-source (see [here](http
 However the project isn't a monolithic Space that can be duplicated and ran immediately:
 it requires various components to run for the frontend, backend, LLM, SDXL etc.
 
-If you try to duplicate the project and open the `.env` you will see it requires some variables:
+If you try to duplicate the project, open the `.env` you will see it requires some variables.
 
+Provider config:
 - `LLM_ENGINE`: can be either "INFERENCE_API" or "INFERENCE_ENDPOINT"
-- `HF_API_TOKEN`: necessary if you decide to use an inference api model or a custom inference endpoint
-- `HF_INFERENCE_ENDPOINT_URL`: necessary if you decide to use a custom inference endpoint
 - `RENDERING_ENGINE`: can only be "VIDEOCHAIN" or "REPLICATE" for now, unless you code your custom solution
-- `VIDEOCHAIN_API_URL`: url to the VideoChain API server
+
+Auth config:
+- `HF_API_TOKEN`: necessary if you decide to use an inference api model or a custom inference endpoint
 - `VIDEOCHAIN_API_TOKEN`: secret token to access the VideoChain API server
 - `REPLICATE_API_TOKEN`: in case you want to use Replicate.com
-- `REPLICATE_API_MODEL`: optional, defaults to "stabilityai/sdxl"
-- `REPLICATE_API_MODEL_VERSION`: optional, in case you want to change the version
+
+Rendering config:
+- `RENDERING_HF_INFERENCE_ENDPOINT_URL`: necessary if you decide to use a custom inference endpoint
+- `RENDERING_REPLICATE_API_MODEL_VERSION`: url to the VideoChain API server
+- `RENDERING_HF_INFERENCE_ENDPOINT_URL`: optional, default to nothing
+- `RENDERING_HF_INFERENCE_API_MODEL`: optional, defaults to "stabilityai/stable-diffusion-xl-base-1.0"
+- `RENDERING_REPLICATE_API_MODEL`: optional, defaults to "stabilityai/sdxl"
+- `RENDERING_REPLICATE_API_MODEL_VERSION`: optional, in case you want to change the version
+
+Language model config:
+- `LLM_HF_INFERENCE_ENDPOINT_URL`: "https://llama-v2-70b-chat.ngrok.io"
+- `LLM_HF_INFERENCE_API_MODEL`: "codellama/CodeLlama-7b-hf"
 
 In addition, there are some community sharing variables that you can just ignore.
 Those variables are not required to run the AI Comic Factory on your own website or computer

src/app/engine/caption.ts

@@ -2,7 +2,7 @@
 
 import { ImageAnalysisRequest, ImageAnalysisResponse } from "@/types"
 
-const apiUrl = `${process.env.VIDEOCHAIN_API_URL || ""}`
+const apiUrl = `${process.env.RENDERING_VIDEOCHAIN_API_URL || ""}`
 
 export async function see({
   prompt,

src/app/engine/render.ts

@@ -1,6 +1,7 @@
 "use server"
 
-import Replicate, { Prediction } from "replicate"
+import { v4 as uuidv4 } from "uuid"
+import Replicate from "replicate"
 
 import { RenderRequest, RenderedScene, RenderingEngine } from "@/types"
 import { generateSeed } from "@/lib/generateSeed"
@@ -8,13 +9,15 @@ import { sleep } from "@/lib/sleep"
 
 const renderingEngine = `${process.env.RENDERING_ENGINE || ""}` as RenderingEngine
 
-const replicateToken = `${process.env.REPLICATE_API_TOKEN || ""}`
-const replicateModel = `${process.env.REPLICATE_API_MODEL || ""}`
-const replicateModelVersion = `${process.env.REPLICATE_API_MODEL_VERSION || ""}`
+const videochainApiUrl = `${process.env.RENDERING_VIDEOCHAIN_API_URL || "" }`
 
-// note: there is no / at the end in the variable
-// so we have to add it ourselves if needed
-const apiUrl = process.env.VIDEOCHAIN_API_URL
+const huggingFaceToken = `${process.env.HF_API_TOKEN || ""}`
+const huggingFaceInferenceEndpointUrl = `${process.env.RENDERING_HF_INFERENCE_ENDPOINT_URL || ""}`
+const huggingFaceInferenceApiModel = `${process.env.RENDERING_HF_INFERENCE_API_MODEL || ""}`
+
+const replicateToken = `${process.env.RENDERING_REPLICATE_API_TOKEN || ""}`
+const replicateModel = `${process.env.RENDERING_REPLICATE_API_MODEL || ""}`
+const replicateModelVersion = `${process.env.RENDERING_REPLICATE_API_MODEL_VERSION || ""}`
 
 export async function newRender({
   prompt,
@@ -79,9 +82,74 @@ export async function newRender({
         maskUrl: "",
         segments: []
       } as RenderedScene
+    } if (renderingEngine === "INFERENCE_ENDPOINT" || renderingEngine === "INFERENCE_API") {
+      if (!huggingFaceToken) {
+        throw new Error(`you need to configure your HF_API_TOKEN in order to use the ${renderingEngine} rendering engine`)
+      }
+      if (renderingEngine === "INFERENCE_ENDPOINT" && !huggingFaceInferenceEndpointUrl) {
+        throw new Error(`you need to configure your RENDERING_HF_INFERENCE_ENDPOINT_URL in order to use the INFERENCE_ENDPOINT rendering engine`)
+      }
+      if (renderingEngine === "INFERENCE_API" && !huggingFaceInferenceApiModel) {
+        throw new Error(`you need to configure your RENDERING_HF_INFERENCE_API_MODEL in order to use the INFERENCE_API rendering engine`)
+      }
+
+      const seed = generateSeed()
+
+      const url = renderingEngine === "INFERENCE_ENDPOINT"
+        ? huggingFaceInferenceEndpointUrl
+        : `https://api-inference.huggingface.co/models/${huggingFaceInferenceApiModel}`
+
+      const res = await fetch(url, {
+        method: "POST",
+        headers: {
+          // Accept: "application/json",
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${huggingFaceToken}`,
+        },
+        body: JSON.stringify({
+          inputs: [
+            "beautiful",
+            "intricate details",
+            prompt,
+            "award winning",
+            "high resolution"
+          ].join(", "),
+          parameters: {
+            num_inference_steps: 25,
+            guidance_scale: 8,
+            width,
+            height,
+
+          }
+        }),
+        cache: "no-store",
+        // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+        // next: { revalidate: 1 }
+      })
+  
+  
+      // Recommendation: handle errors
+      if (res.status !== 200) {
+        // This will activate the closest `error.js` Error Boundary
+        throw new Error('Failed to fetch data')
+      }
+  
+      // the result is a JSON-encoded string
+      const response = await res.json() as string
+      const assetUrl = `data:image/png;base64,${response}`
+
+      return {
+        renderId: uuidv4(),
+        status: "completed",
+        assetUrl, 
+        alt: prompt,
+        error: "",
+        maskUrl: "",
+        segments: []
+      } as RenderedScene
     } else {
       // console.log(`calling POST ${apiUrl}/render with prompt: ${prompt}`)
-      const res = await fetch(`${apiUrl}/render`, {
+      const res = await fetch(`${videochainApiUrl}/render`, {
         method: "POST",
         headers: {
           Accept: "application/json",
@@ -202,7 +270,7 @@ export async function getRender(renderId: string) {
       } as RenderedScene
     } else {
       // console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`)
-      const res = await fetch(`${apiUrl}/render/${renderId}`, {
+      const res = await fetch(`${videochainApiUrl}/render/${renderId}`, {
         method: "GET",
         headers: {
           Accept: "application/json",
@@ -255,7 +323,7 @@ export async function upscaleImage(image: string): Promise<{
 
   try {
     // console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`)
-    const res = await fetch(`${apiUrl}/upscale`, {
+    const res = await fetch(`${videochainApiUrl}/upscale`, {
       method: "POST",
       headers: {
         Accept: "application/json",

src/app/queries/predict.ts

@@ -8,8 +8,8 @@ const hf = new HfInference(process.env.HF_API_TOKEN)
 
 // note: we always try "inference endpoint" first
 const llmEngine = `${process.env.LLM_ENGINE || ""}` as LLMEngine
-const inferenceEndpoint = `${process.env.HF_INFERENCE_ENDPOINT_URL || ""}`
-const inferenceModel = `${process.env.HF_INFERENCE_API_MODEL || ""}`
+const inferenceEndpoint = `${process.env.LLM_HF_INFERENCE_ENDPOINT_URL || ""}`
+const inferenceModel = `${process.env.LLM_HF_INFERENCE_API_MODEL || ""}`
 
 let hfie: HfInferenceEndpoint
 

src/types.ts

@@ -91,6 +91,8 @@ export type RenderingEngine =
   | "VIDEOCHAIN"
   | "OPENAI"
   | "REPLICATE"
+  | "INFERENCE_API"
+  | "INFERENCE_ENDPOINT"
 
 export type PostVisibility =
   | "featured" // featured by admins