Upload 23 files

.env.local.example

@@ -0,0 +1,2 @@
+SUPABASE_URL=your-project-url
+SUPABASE_ANON_KEY=your-anon-key

.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

Dockerfile

@@ -0,0 +1,69 @@
+# syntax=docker/dockerfile:1.4
+
+# Adapted from https://github.com/vercel/next.js/blob/e60a1e747c3f521fc24dfd9ee2989e13afeb0a9b/examples/with-docker/Dockerfile
+# For more information, see https://nextjs.org/docs/pages/building-your-application/deploying#docker-image
+
+FROM node:18 AS base
+
+# Install dependencies only when needed
+FROM base AS deps
+WORKDIR /app
+
+# Install dependencies based on the preferred package manager
+COPY --link package.json yarn.lock* package-lock.json* pnpm-lock.yaml* ./
+RUN \
+  if [ -f yarn.lock ]; then yarn --frozen-lockfile; \
+  elif [ -f package-lock.json ]; then npm ci; \
+  elif [ -f pnpm-lock.yaml ]; then yarn global add pnpm && pnpm i --frozen-lockfile; \
+  else echo "Lockfile not found." && exit 1; \
+  fi
+
+
+# Rebuild the source code only when needed
+FROM base AS builder
+WORKDIR /app
+COPY --from=deps --link /app/node_modules ./node_modules
+COPY --link  . .
+
+# Next.js collects completely anonymous telemetry data about general usage.
+# Learn more here: https://nextjs.org/telemetry
+# Uncomment the following line in case you want to disable telemetry during the build.
+# ENV NEXT_TELEMETRY_DISABLED 1
+
+RUN npm run build
+
+# If using yarn comment out above and use below instead
+# RUN yarn build
+
+# Production image, copy all the files and run next
+FROM base AS runner
+WORKDIR /app
+
+ENV NODE_ENV production
+# Uncomment the following line in case you want to disable telemetry during runtime.
+# ENV NEXT_TELEMETRY_DISABLED 1
+
+RUN \
+  addgroup --system --gid 1001 nodejs; \
+  adduser --system --uid 1001 nextjs
+
+COPY --from=builder --link /app/public ./public
+
+# Automatically leverage output traces to reduce image size
+# https://nextjs.org/docs/advanced-features/output-file-tracing
+COPY --from=builder --link --chown=1001:1001 /app/.next/standalone ./
+COPY --from=builder --link --chown=1001:1001 /app/.next/static ./.next/static
+
+USER nextjs
+
+EXPOSE 3000
+
+ENV PORT 3000
+ENV HOSTNAME localhost
+
+# Allow the running process to write model files to the cache folder.
+# NOTE: In practice, you would probably want to pre-download the model files to avoid having to download them on-the-fly.
+RUN mkdir -p /app/node_modules/@xenova/.cache/
+RUN chmod 777 -R /app/node_modules/@xenova/
+
+CMD ["node", "server.js"]

README.md

@@ -1,10 +1,34 @@
----
-title: Semantic Image Search
-emoji: 📚
-colorFrom: green
-colorTo: indigo
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `app/page.js`. The page auto-updates as you edit the file.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

jsconfig.json

@@ -0,0 +1,7 @@
+{
+  "compilerOptions": {
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  }
+}

next.config.js

@@ -0,0 +1,18 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+    // (Optional) Export as a standalone site
+    // See https://nextjs.org/docs/pages/api-reference/next-config-js/output#automatically-copying-traced-files
+    output: 'standalone', // Feel free to modify/remove this option
+
+    // Indicate that these packages should not be bundled by webpack
+    experimental: {
+        serverComponentsExternalPackages: ['sharp', 'onnxruntime-node'],
+    },
+
+    // Define which domains we are allowed to load images from
+    images: {
+        domains: ['images.unsplash.com'],
+    },
+};
+
+module.exports = nextConfig;

package.json

@@ -0,0 +1,27 @@
+{
+  "name": "semantic-image-search",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "@xenova/transformers": "^2.5.0",
+    "@supabase/supabase-js": "^2.31.0",
+    "autoprefixer": "10.4.14",
+    "blurhash": "^2.0.5",
+    "eslint": "8.45.0",
+    "eslint-config-next": "13.4.12",
+    "next": "13.4.12",
+    "postcss": "8.4.27",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "tailwindcss": "3.3.3"
+  },
+  "overrides": {
+    "protobufjs": "^7.2.4"
+  }
+}

postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

scripts/update-database.mjs

@@ -0,0 +1,67 @@
+// Helper script to update the database with image embeddings
+
+import { AutoProcessor, RawImage, CLIPVisionModelWithProjection } from '@xenova/transformers';
+import { createClient } from '@supabase/supabase-js'
+
+if (!process.env.SUPABASE_SECRET_KEY) {
+    throw new Error('Missing `SUPABASE_SECRET_KEY` environment variable.')
+}
+
+// Create a single supabase client for interacting with your database
+const supabase = createClient(
+    process.env.SUPABASE_URL,
+    process.env.SUPABASE_SECRET_KEY,
+)
+
+let { data, error } = await supabase
+    .from('images')
+    .select('*')
+    .neq('ignore', true)
+    .is('image_embedding', null);
+
+if (error) {
+    throw error;
+}
+
+// Load processor and vision model
+const model_id = 'Xenova/clip-vit-base-patch16';
+const processor = await AutoProcessor.from_pretrained(model_id);
+const vision_model = await CLIPVisionModelWithProjection.from_pretrained(model_id, {
+    quantized: false,
+});
+
+for (const image_data of data) {
+    let image;
+    try {
+        image = await RawImage.read(image_data.photo_image_url);
+    } catch (e) {
+        // Unable to load image, so we ignore it
+        console.warn('Ignoring image due to error', e)
+        await supabase
+            .from('images')
+            .update({ ignore: true })
+            .eq('photo_id', image_data.photo_id)
+            .select()
+        continue;
+    }
+
+    // Read image and run processor
+    let image_inputs = await processor(image);
+
+    // Compute embeddings
+    const { image_embeds } = await vision_model(image_inputs);
+    const embed_as_list = image_embeds.tolist()[0];
+
+    // https://supabase.com/docs/guides/ai/vector-columns#storing-a-vector--embedding
+    const { data, error } = await supabase
+        .from('images')
+        .update({ image_embedding: embed_as_list })
+        .eq('photo_id', image_data.photo_id)
+        .select()
+
+    if (error) {
+        console.error('error', error)
+    } else {
+        console.log('success', image_data.photo_id)
+    }
+}

src/app/app.js

@@ -0,0 +1,51 @@
+import { AutoTokenizer, CLIPTextModelWithProjection } from "@xenova/transformers";
+import { createClient } from '@supabase/supabase-js'
+
+// Use the Singleton pattern to enable lazy construction of the pipeline.
+// NOTE: We wrap the class in a function to prevent code duplication (see below).
+const S = () => class ApplicationSingleton {
+    static model_id = 'Xenova/clip-vit-base-patch16';
+    static tokenizer = null;
+    static text_model = null;
+    static database = null;
+
+    static async getInstance() {
+        // Load tokenizer and text model
+        if (this.tokenizer === null) {
+            this.tokenizer = AutoTokenizer.from_pretrained(this.model_id);
+        }
+
+        if (this.text_model === null) {
+            this.text_model = CLIPTextModelWithProjection.from_pretrained(this.model_id, {
+                quantized: false,
+            });
+        }
+
+        if (this.database === null) {
+            this.database = createClient(
+                process.env.SUPABASE_URL,
+                process.env.SUPABASE_ANON_KEY,
+            )
+        }
+
+        return Promise.all([
+            this.tokenizer,
+            this.text_model,
+            this.database,
+        ]);
+    }
+}
+
+let ApplicationSingleton;
+if (process.env.NODE_ENV !== 'production') {
+    // When running in development mode, attach the pipeline to the
+    // global object so that it's preserved between hot reloads.
+    // For more information, see https://vercel.com/guides/nextjs-prisma-postgres
+    if (!global.ApplicationSingleton) {
+        global.ApplicationSingleton = S();
+    }
+    ApplicationSingleton = global.ApplicationSingleton;
+} else {
+    ApplicationSingleton = S();
+}
+export default ApplicationSingleton;

src/app/components/ImageGrid.jsx

@@ -0,0 +1,52 @@
+import Image from 'next/image'
+import { blurHashToDataURL } from '../utils.js'
+
+export function ImageGrid({ images, setCurrentImage }) {
+    return (
+        <div className="columns-2 gap-4 sm:columns-3 xl:columns-4 2xl:columns-5">
+            {images && images.map(({
+                photo_id,
+                photo_url,
+                photo_image_url,
+                photo_aspect_ratio,
+                photo_width,
+                photo_height,
+                blur_hash,
+                photo_description,
+                ai_description,
+                similarity,
+            }) => (
+                <div
+                    key={photo_id}
+                    href={photo_url}
+                    className='after:content group cursor-pointer relative mb-4 block w-full after:pointer-events-none after:absolute after:inset-0 after:rounded-lg after:shadow-highlight'
+                    onClick={() => {
+                        setCurrentImage({
+                            photo_id,
+                            photo_url,
+                            photo_image_url,
+                            photo_aspect_ratio,
+                            photo_width,
+                            photo_height,
+                            blur_hash,
+                            photo_description,
+                            ai_description,
+                            similarity,
+                        });
+                    }}
+                >
+                    <Image
+                        alt={photo_description || ai_description || ""}
+                        className="transform rounded-lg brightness-90 transition will-change-auto group-hover:brightness-110"
+                        style={{ transform: 'translate3d(0, 0, 0)' }}
+                        placeholder="blur"
+                        blurDataURL={blurHashToDataURL(blur_hash)}
+                        src={`${photo_image_url}?auto=format&fit=crop&w=480&q=80`}
+                        width={480}
+                        height={480 / photo_aspect_ratio}
+                        unoptimized={true}
+                    />
+                </div>
+            ))}
+        </div>)
+}

src/app/components/Modal.jsx

@@ -0,0 +1,60 @@
+import { downloadImage } from '../utils.js'
+import Image from 'next/image'
+
+export function Modal({ currentImage, setCurrentImage }) {
+    return (
+        <div
+            className='fixed inset-0 z-30 backdrop-blur-2xl w-full h-full bg-black top-0 left-0 transition'
+            style={{
+                backgroundColor: `rgba(0, 0, 0, ${currentImage ? 0.8 : 0})`,
+                opacity: currentImage ? 1 : 0,
+                pointerEvents: currentImage ? 'auto' : 'none',
+            }}
+        >
+            {currentImage && <>
+                <Image
+                    alt={currentImage.photo_description || currentImage.ai_description || ""}
+                    className="transform rounded-lg transition will-change-auto"
+                    style={
+                        { transform: 'translate3d(0, 0, 0)', }
+                    }
+                    layout={'fill'}
+                    objectFit={'contain'}
+                    src={currentImage.photo_image_url}
+                    unoptimized={true}
+                />
+                <div
+                    className='absolute top-0 left-0 flex items-center gap-2 p-3 text-white'
+                >
+                    <button
+                        onClick={() => setCurrentImage(null)}
+                        className="rounded-full bg-black/50 p-2 text-white/75 backdrop-blur-lg transition hover:bg-black/75 hover:text-white">
+                        <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" strokeWidth="1.5" stroke="currentColor" aria-hidden="true" className="h-5 w-5">
+                            <path strokeLinecap="round" strokeLinejoin="round" d="M6 18L18 6M6 6l12 12"></path>
+                        </svg>
+                    </button>
+                </div>
+                <div className="absolute top-0 right-0 flex items-center gap-2 p-3 text-white">
+                    <a
+                        href={currentImage.photo_url}
+                        className="rounded-full bg-black/50 p-2 text-white/75 backdrop-blur-lg transition hover:bg-black/75 hover:text-white"
+                        target="_blank" title="View on Unsplash"
+                        rel="noreferrer">
+                        <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" strokeWidth="1.5" stroke="currentColor" aria-hidden="true" className="h-5 w-5">
+                            <path strokeLinecap="round" strokeLinejoin="round" d="M13.5 6H5.25A2.25 2.25 0 003 8.25v10.5A2.25 2.25 0 005.25 21h10.5A2.25 2.25 0 0018 18.75V10.5m-10.5 6L21 3m0 0h-5.25M21 3v5.25"></path>
+                        </svg>
+                    </a>
+                    <button
+                        onClick={() => downloadImage(currentImage.photo_image_url, `${currentImage.photo_id}.png`)}
+                        className="rounded-full bg-black/50 p-2 text-white/75 backdrop-blur-lg transition hover:bg-black/75 hover:text-white" title="Download">
+                        <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" strokeWidth="1.5" stroke="currentColor" aria-hidden="true" className="h-5 w-5">
+                            <path strokeLinecap="round" strokeLinejoin="round" d="M3 16.5v2.25A2.25 2.25 0 005.25 21h13.5A2.25 2.25 0 0021 18.75V16.5M16.5 12L12 16.5m0 0L7.5 12m4.5 4.5V3">
+                            </path>
+                        </svg>
+                    </button>
+                </div>
+            </>
+            }
+
+        </div>)
+}

src/app/components/SearchBar.jsx

@@ -0,0 +1,32 @@
+
+export function SearchBar({ search }) {
+    return (<form
+        onSubmit={e => {
+            e.preventDefault();
+            const formData = new FormData(e.target);
+            const text = formData.get('text');
+            search(text);
+        }}
+        className='relative mb-2'
+    >
+        <div className="absolute inset-y-0 left-0 flex items-center pl-3 pointer-events-none">
+            <svg className="w-4 h-4 text-gray-500 dark:text-gray-400" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20">
+                <path stroke="currentColor" strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="m19 19-4-4m0-7A7 7 0 1 1 1 8a7 7 0 0 1 14 0Z" />
+            </svg>
+        </div>
+        <input
+            type="search"
+            name="text"
+            id="default-search"
+            className="block w-full p-4 pl-10 text-sm text-gray-900 border border-gray-300 rounded-lg bg-gray-50 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
+            placeholder="Search for images..."
+            required
+        />
+        <button
+            type="submit"
+            className="text-white absolute right-2.5 bottom-2.5 bg-blue-700 hover:bg-blue-800 focus:ring-4 focus:outline-none focus:ring-blue-300 font-medium rounded-lg text-sm px-4 py-2 dark:bg-blue-600 dark:hover:bg-blue-700 dark:focus:ring-blue-800"
+        >
+            Search
+        </button>
+    </form>)
+}

src/app/globals.css

@@ -0,0 +1,19 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+:root {
+  --foreground-rgb: 255, 255, 255;
+  --background-start-rgb: 0, 0, 0;
+  --background-end-rgb: 0, 0, 0;
+}
+
+body {
+  color: rgb(var(--foreground-rgb));
+  background: linear-gradient(
+      to bottom,
+      transparent,
+      rgb(var(--background-end-rgb))
+    )
+    rgb(var(--background-start-rgb));
+}

src/app/layout.js

@@ -0,0 +1,17 @@
+import './globals.css'
+import { Inter } from 'next/font/google'
+
+const inter = Inter({ subsets: ['latin'] })
+
+export const metadata = {
+  title: 'Semantic Image Search',
+  description: 'Search for images using text (built w/ Transformers.js and Supabase)',
+}
+
+export default function RootLayout({ children }) {
+  return (
+    <html lang="en">
+      <body className={inter.className}>{children}</body>
+    </html>
+  )
+}

src/app/page.js

@@ -0,0 +1,36 @@
+'use client'
+
+import { useState } from 'react'
+import { Modal } from './components/Modal';
+import { SearchBar } from './components/SearchBar';
+import { ImageGrid } from './components/ImageGrid';
+
+export default function Home() {
+
+  // Application state
+  const [images, setImages] = useState(null);
+  const [currentImage, setCurrentImage] = useState(null);
+
+  const search = async (text) => {
+    if (!text) return;
+
+    const params = new URLSearchParams();
+    params.append('text', text);
+    params.append('threshold', 0.1);
+    params.append('limit', 100);
+
+    // Make a request to the /classify route on the server.
+    const result = await fetch(`/search?${params.toString()}`);
+
+    const json = await result.json();
+    setImages(json);
+  };
+
+  return (
+    <main className="mx-auto max-w-[1960px] p-4 relative">
+      <Modal currentImage={currentImage} setCurrentImage={setCurrentImage} />
+      <SearchBar search={search} />
+      <ImageGrid images={images} setCurrentImage={setCurrentImage} />
+    </main>
+  )
+}

src/app/search/route.js

@@ -0,0 +1,73 @@
+// Create a custom request handler for the /classify route.
+// For more information, see https://nextjs.org/docs/app/building-your-application/routing/router-handlers
+
+import { NextResponse } from 'next/server'
+import ApplicationSingleton from '../app.js'
+
+const parseInputs = (searchParams) => {
+    const text = searchParams.get('text');
+    if (!text) {
+        return {
+            error: 'Missing text parameter',
+        };
+    }
+    const threshold = searchParams.get('threshold');
+    const match_threshold = Number(threshold ?? 0.1);
+    if (isNaN(match_threshold) || match_threshold < 0 || match_threshold > 1) {
+        return {
+            error: `Invalid threshold parameter "${threshold}" (should be a number between 0 and 1)`,
+        };
+    }
+
+    const limit = searchParams.get('limit');
+    const match_count = Number(limit ?? 25);
+    if (isNaN(match_count) || !Number.isInteger(match_count) || match_count < 0 || match_count > 1000) {
+        return {
+            error: `Invalid limit parameter "${limit}" (should be an integer between 0 and 1000)`,
+        };
+    }
+
+    return { text, match_threshold, match_count }
+}
+
+// TODO: add caching
+
+export async function GET(request) {
+    const parsedInputs = parseInputs(request.nextUrl.searchParams);
+    if (parsedInputs.error) {
+        return NextResponse.json({
+            error: parsedInputs.error,
+        }, { status: 400 });
+    }
+
+    // Valid inputs, so we can proceed
+    const { text, match_threshold, match_count } = parsedInputs;
+
+    // Get the tokenizer, model, and database singletons. When called for the first time,
+    // this will load the models and cache them for future use.
+    const [tokenizer, text_model, database] = await ApplicationSingleton.getInstance();
+
+    // Run tokenization
+    let text_inputs = tokenizer(text, { padding: true, truncation: true });
+
+    // Compute embeddings
+    const { text_embeds } = await text_model(text_inputs);
+    const query_embedding = text_embeds.tolist()[0];
+
+    // TODO add pagination?
+    let { data: images, error } = await database
+        .rpc('match_images', {
+            query_embedding,
+            match_threshold,
+            match_count,
+        });
+    if (error) {
+        console.warn('Error fetching images', error);
+        return NextResponse.json({
+            error: 'An error occurred while fetching images',
+        }, { status: 500 });
+    }
+
+
+    return NextResponse.json(images);
+}

src/app/utils.js

@@ -0,0 +1,52 @@
+
+import { decode } from "blurhash"
+
+const SIZE = 32;
+
+export function blurHashToDataURL(hash) {
+    if (!hash) return undefined
+
+    const pixels = decode(hash, SIZE, SIZE)
+
+    const canvas = document.createElement("canvas");
+    canvas.width = SIZE;
+    canvas.height = SIZE;
+
+    const ctx = canvas.getContext("2d");
+    const imageData = ctx.createImageData(SIZE, SIZE);
+    imageData.data.set(pixels);
+    ctx.putImageData(imageData, 0, 0);
+
+    return canvas.toDataURL();
+}
+
+function downloadData(url, filename) {
+
+    // Create an anchor element with the data URL as the href attribute
+    const downloadLink = document.createElement('a');
+    downloadLink.href = url;
+
+    // Set the download attribute to specify the desired filename for the downloaded image
+    downloadLink.download = filename;
+
+    // Trigger the download
+    downloadLink.click();
+
+    // Clean up: remove the anchor element from the DOM
+    downloadLink.remove();
+}
+
+export function downloadImage(url, filename) {
+    fetch(url, {
+        headers: new Headers({
+            Origin: location.origin,
+        }),
+        mode: 'cors',
+    })
+        .then((response) => response.blob())
+        .then((blob) => {
+            let blobUrl = window.URL.createObjectURL(blob)
+            downloadData(blobUrl, filename)
+        })
+        .catch((e) => console.error(e))
+}

tailwind.config.js

@@ -0,0 +1,16 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: [
+    './src/pages/**/*.{js,ts,jsx,tsx,mdx}',
+    './src/components/**/*.{js,ts,jsx,tsx,mdx}',
+    './src/app/**/*.{js,ts,jsx,tsx,mdx}',
+  ],
+  theme: {
+    extend: {
+      boxShadow: {
+        highlight: 'inset 0 0 0 1px rgba(255, 255, 255, 0.1)',
+      },
+    },
+  },
+  plugins: [],
+}