Upload 6 files

build/m.d.ts

@@ -14,11 +14,10 @@ export class Model {
 */
   set_image_embeddings(image_data: Uint8Array): void;
 /**
-* @param {number} x
-* @param {number} y
+* @param {any} input
 * @returns {any}
 */
-  mask_for_point(x: number, y: number): any;
+  mask_for_point(input: any): any;
 }
 
 export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
@@ -28,12 +27,13 @@ export interface InitOutput {
   readonly __wbg_model_free: (a: number) => void;
   readonly model_new: (a: number, b: number, c: number, d: number) => void;
   readonly model_set_image_embeddings: (a: number, b: number, c: number, d: number) => void;
-  readonly model_mask_for_point: (a: number, b: number, c: number, d: number) => void;
+  readonly model_mask_for_point: (a: number, b: number, c: number) => void;
   readonly main: (a: number, b: number) => number;
-  readonly __wbindgen_add_to_stack_pointer: (a: number) => number;
   readonly __wbindgen_malloc: (a: number, b: number) => number;
   readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
+  readonly __wbindgen_add_to_stack_pointer: (a: number) => number;
   readonly __wbindgen_free: (a: number, b: number, c: number) => void;
+  readonly __wbindgen_exn_store: (a: number) => void;
   readonly __wbindgen_start: () => void;
 }
 

build/m.js

@@ -47,13 +47,17 @@ function takeObject(idx) {
     return ret;
 }
 
-let WASM_VECTOR_LEN = 0;
+function isLikeNone(x) {
+    return x === undefined || x === null;
+}
 
-function passArray8ToWasm0(arg, malloc) {
-    const ptr = malloc(arg.length * 1, 1) >>> 0;
-    getUint8Memory0().set(arg, ptr / 1);
-    WASM_VECTOR_LEN = arg.length;
-    return ptr;
+let cachedFloat64Memory0 = null;
+
+function getFloat64Memory0() {
+    if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) {
+        cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer);
+    }
+    return cachedFloat64Memory0;
 }
 
 let cachedInt32Memory0 = null;
@@ -65,6 +69,8 @@ function getInt32Memory0() {
     return cachedInt32Memory0;
 }
 
+let WASM_VECTOR_LEN = 0;
+
 const cachedTextEncoder = (typeof TextEncoder !== 'undefined' ? new TextEncoder('utf-8') : { encode: () => { throw Error('TextEncoder not available') } } );
 
 const encodeString = (typeof cachedTextEncoder.encodeInto === 'function'
@@ -117,6 +123,86 @@ function passStringToWasm0(arg, malloc, realloc) {
     WASM_VECTOR_LEN = offset;
     return ptr;
 }
+
+function debugString(val) {
+    // primitive types
+    const type = typeof val;
+    if (type == 'number' || type == 'boolean' || val == null) {
+        return  `${val}`;
+    }
+    if (type == 'string') {
+        return `"${val}"`;
+    }
+    if (type == 'symbol') {
+        const description = val.description;
+        if (description == null) {
+            return 'Symbol';
+        } else {
+            return `Symbol(${description})`;
+        }
+    }
+    if (type == 'function') {
+        const name = val.name;
+        if (typeof name == 'string' && name.length > 0) {
+            return `Function(${name})`;
+        } else {
+            return 'Function';
+        }
+    }
+    // objects
+    if (Array.isArray(val)) {
+        const length = val.length;
+        let debug = '[';
+        if (length > 0) {
+            debug += debugString(val[0]);
+        }
+        for(let i = 1; i < length; i++) {
+            debug += ', ' + debugString(val[i]);
+        }
+        debug += ']';
+        return debug;
+    }
+    // Test for built-in
+    const builtInMatches = /\[object ([^\]]+)\]/.exec(toString.call(val));
+    let className;
+    if (builtInMatches.length > 1) {
+        className = builtInMatches[1];
+    } else {
+        // Failed to match the standard '[object ClassName]'
+        return toString.call(val);
+    }
+    if (className == 'Object') {
+        // we're a user defined class or Object
+        // JSON.stringify avoids problems with cycles, and is generally much
+        // easier than looping through ownProperties of `val`.
+        try {
+            return 'Object(' + JSON.stringify(val) + ')';
+        } catch (_) {
+            return 'Object';
+        }
+    }
+    // errors
+    if (val instanceof Error) {
+        return `${val.name}: ${val.message}\n${val.stack}`;
+    }
+    // TODO we could test for more things here, like `Set`s and `Map`s.
+    return className;
+}
+
+function passArray8ToWasm0(arg, malloc) {
+    const ptr = malloc(arg.length * 1, 1) >>> 0;
+    getUint8Memory0().set(arg, ptr / 1);
+    WASM_VECTOR_LEN = arg.length;
+    return ptr;
+}
+
+function handleError(f, args) {
+    try {
+        return f.apply(this, args);
+    } catch (e) {
+        wasm.__wbindgen_exn_store(addHeapObject(e));
+    }
+}
 /**
 */
 export class Model {
@@ -180,14 +266,13 @@ export class Model {
         }
     }
     /**
-    * @param {number} x
-    * @param {number} y
+    * @param {any} input
     * @returns {any}
     */
-    mask_for_point(x, y) {
+    mask_for_point(input) {
         try {
             const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
-            wasm.model_mask_for_point(retptr, this.__wbg_ptr, x, y);
+            wasm.model_mask_for_point(retptr, this.__wbg_ptr, addHeapObject(input));
             var r0 = getInt32Memory0()[retptr / 4 + 0];
             var r1 = getInt32Memory0()[retptr / 4 + 1];
             var r2 = getInt32Memory0()[retptr / 4 + 2];
@@ -242,10 +327,46 @@ function __wbg_get_imports() {
     imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
         takeObject(arg0);
     };
+    imports.wbg.__wbindgen_boolean_get = function(arg0) {
+        const v = getObject(arg0);
+        const ret = typeof(v) === 'boolean' ? (v ? 1 : 0) : 2;
+        return ret;
+    };
+    imports.wbg.__wbindgen_number_get = function(arg0, arg1) {
+        const obj = getObject(arg1);
+        const ret = typeof(obj) === 'number' ? obj : undefined;
+        getFloat64Memory0()[arg0 / 8 + 1] = isLikeNone(ret) ? 0 : ret;
+        getInt32Memory0()[arg0 / 4 + 0] = !isLikeNone(ret);
+    };
+    imports.wbg.__wbindgen_is_object = function(arg0) {
+        const val = getObject(arg0);
+        const ret = typeof(val) === 'object' && val !== null;
+        return ret;
+    };
+    imports.wbg.__wbindgen_is_undefined = function(arg0) {
+        const ret = getObject(arg0) === undefined;
+        return ret;
+    };
+    imports.wbg.__wbindgen_in = function(arg0, arg1) {
+        const ret = getObject(arg0) in getObject(arg1);
+        return ret;
+    };
     imports.wbg.__wbindgen_object_clone_ref = function(arg0) {
         const ret = getObject(arg0);
         return addHeapObject(ret);
     };
+    imports.wbg.__wbindgen_jsval_loose_eq = function(arg0, arg1) {
+        const ret = getObject(arg0) == getObject(arg1);
+        return ret;
+    };
+    imports.wbg.__wbindgen_string_get = function(arg0, arg1) {
+        const obj = getObject(arg1);
+        const ret = typeof(obj) === 'string' ? obj : undefined;
+        var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var len1 = WASM_VECTOR_LEN;
+        getInt32Memory0()[arg0 / 4 + 1] = len1;
+        getInt32Memory0()[arg0 / 4 + 0] = ptr1;
+    };
     imports.wbg.__wbg_String_4370c5505c674d30 = function(arg0, arg1) {
         const ret = String(getObject(arg1));
         const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
@@ -265,6 +386,10 @@ function __wbg_get_imports() {
         const ret = getStringFromWasm0(arg0, arg1);
         return addHeapObject(ret);
     };
+    imports.wbg.__wbg_getwithrefkey_d1f0d12f1f1b63ea = function(arg0, arg1) {
+        const ret = getObject(arg0)[getObject(arg1)];
+        return addHeapObject(ret);
+    };
     imports.wbg.__wbg_set_bd72c078edfa51ad = function(arg0, arg1, arg2) {
         getObject(arg0)[takeObject(arg1)] = takeObject(arg2);
     };
@@ -290,13 +415,53 @@ function __wbg_get_imports() {
             wasm.__wbindgen_free(deferred0_0, deferred0_1, 1);
         }
     };
-    imports.wbg.__wbg_log_fa2e12d24033131f = function(arg0, arg1) {
+    imports.wbg.__wbg_log_cfcb93115564df45 = function(arg0, arg1) {
         console.log(getStringFromWasm0(arg0, arg1));
     };
+    imports.wbg.__wbindgen_is_function = function(arg0) {
+        const ret = typeof(getObject(arg0)) === 'function';
+        return ret;
+    };
+    imports.wbg.__wbg_get_44be0491f933a435 = function(arg0, arg1) {
+        const ret = getObject(arg0)[arg1 >>> 0];
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_length_fff51ee6522a1a18 = function(arg0) {
+        const ret = getObject(arg0).length;
+        return ret;
+    };
     imports.wbg.__wbg_new_898a68150f225f2e = function() {
         const ret = new Array();
         return addHeapObject(ret);
     };
+    imports.wbg.__wbg_next_526fc47e980da008 = function(arg0) {
+        const ret = getObject(arg0).next;
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_next_ddb3312ca1c4e32a = function() { return handleError(function (arg0) {
+        const ret = getObject(arg0).next();
+        return addHeapObject(ret);
+    }, arguments) };
+    imports.wbg.__wbg_done_5c1f01fb660d73b5 = function(arg0) {
+        const ret = getObject(arg0).done;
+        return ret;
+    };
+    imports.wbg.__wbg_value_1695675138684bd5 = function(arg0) {
+        const ret = getObject(arg0).value;
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_iterator_97f0c81209c6c35a = function() {
+        const ret = Symbol.iterator;
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_get_97b561fb56f034b5 = function() { return handleError(function (arg0, arg1) {
+        const ret = Reflect.get(getObject(arg0), getObject(arg1));
+        return addHeapObject(ret);
+    }, arguments) };
+    imports.wbg.__wbg_call_cb65541d95d71282 = function() { return handleError(function (arg0, arg1) {
+        const ret = getObject(arg0).call(getObject(arg1));
+        return addHeapObject(ret);
+    }, arguments) };
     imports.wbg.__wbg_new_b51585de1b234aff = function() {
         const ret = new Object();
         return addHeapObject(ret);
@@ -304,9 +469,59 @@ function __wbg_get_imports() {
     imports.wbg.__wbg_set_502d29070ea18557 = function(arg0, arg1, arg2) {
         getObject(arg0)[arg1 >>> 0] = takeObject(arg2);
     };
+    imports.wbg.__wbg_isArray_4c24b343cb13cfb1 = function(arg0) {
+        const ret = Array.isArray(getObject(arg0));
+        return ret;
+    };
+    imports.wbg.__wbg_instanceof_ArrayBuffer_39ac22089b74fddb = function(arg0) {
+        let result;
+        try {
+            result = getObject(arg0) instanceof ArrayBuffer;
+        } catch {
+            result = false;
+        }
+        const ret = result;
+        return ret;
+    };
+    imports.wbg.__wbg_buffer_085ec1f694018c4f = function(arg0) {
+        const ret = getObject(arg0).buffer;
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_new_8125e318e6245eed = function(arg0) {
+        const ret = new Uint8Array(getObject(arg0));
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbg_set_5cf90238115182c3 = function(arg0, arg1, arg2) {
+        getObject(arg0).set(getObject(arg1), arg2 >>> 0);
+    };
+    imports.wbg.__wbg_length_72e2208bbc0efc61 = function(arg0) {
+        const ret = getObject(arg0).length;
+        return ret;
+    };
+    imports.wbg.__wbg_instanceof_Uint8Array_d8d9cb2b8e8ac1d4 = function(arg0) {
+        let result;
+        try {
+            result = getObject(arg0) instanceof Uint8Array;
+        } catch {
+            result = false;
+        }
+        const ret = result;
+        return ret;
+    };
+    imports.wbg.__wbindgen_debug_string = function(arg0, arg1) {
+        const ret = debugString(getObject(arg1));
+        const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len1 = WASM_VECTOR_LEN;
+        getInt32Memory0()[arg0 / 4 + 1] = len1;
+        getInt32Memory0()[arg0 / 4 + 0] = ptr1;
+    };
     imports.wbg.__wbindgen_throw = function(arg0, arg1) {
         throw new Error(getStringFromWasm0(arg0, arg1));
     };
+    imports.wbg.__wbindgen_memory = function() {
+        const ret = wasm.memory;
+        return addHeapObject(ret);
+    };
 
     return imports;
 }
@@ -318,6 +533,7 @@ function __wbg_init_memory(imports, maybe_memory) {
 function __wbg_finalize_init(instance, module) {
     wasm = instance.exports;
     __wbg_init.__wbindgen_wasm_module = module;
+    cachedFloat64Memory0 = null;
     cachedInt32Memory0 = null;
     cachedUint8Memory0 = null;
 

build/m_bg.wasm

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:acfbd107ec1c84a9ed5388cd1afd0e494bb99426d5e162e1d017c28cca325553
-size 2579585
+oid sha256:3aa7201ff46d314086d7c2aa49e7a415901414fc3c5e584a9762344dc759b6d6
+size 2667717

build/m_bg.wasm.d.ts

@@ -4,10 +4,11 @@ export const memory: WebAssembly.Memory;
 export function __wbg_model_free(a: number): void;
 export function model_new(a: number, b: number, c: number, d: number): void;
 export function model_set_image_embeddings(a: number, b: number, c: number, d: number): void;
-export function model_mask_for_point(a: number, b: number, c: number, d: number): void;
+export function model_mask_for_point(a: number, b: number, c: number): void;
 export function main(a: number, b: number): number;
-export function __wbindgen_add_to_stack_pointer(a: number): number;
 export function __wbindgen_malloc(a: number, b: number): number;
 export function __wbindgen_realloc(a: number, b: number, c: number, d: number): number;
+export function __wbindgen_add_to_stack_pointer(a: number): number;
 export function __wbindgen_free(a: number, b: number, c: number): void;
+export function __wbindgen_exn_store(a: number): void;
 export function __wbindgen_start(): void;

lib-example.html

@@ -0,0 +1,473 @@
+<html>
+  <head>
+    <meta content="text/html;charset=utf-8" http-equiv="Content-Type" />
+    <title>Candle Segment Anything Model (SAM) Rust/WASM</title>
+  </head>
+  <body></body>
+</html>
+
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <style>
+      @import url("https://fonts.googleapis.com/css2?family=Source+Code+Pro:wght@200;300;400&family=Source+Sans+3:wght@100;200;300;400;500;600;700;800;900&display=swap");
+      html,
+      body {
+        font-family: "Source Sans 3", sans-serif;
+      }
+    </style>
+    <script src="https://cdn.tailwindcss.com"></script>
+    <script type="module">
+      // base url for image examples
+      const MODEL_BASEURL =
+        "https://huggingface.co/lmz/candle-sam/resolve/main/";
+
+      // models base url
+      const MODELS = {
+        sam_mobile_tiny: {
+          url: "mobile_sam-tiny-vitt.safetensors",
+        },
+        sam_base: {
+          url: "sam_vit_b_01ec64.safetensors",
+        },
+      };
+      const samWorker = new Worker("./samWorker.js", { type: "module" });
+
+      async function segmentPoints(
+        modelURL, // URL to the weights file
+        modelID, // model ID
+        imageURL, // URL to the image file
+        points // {x, y} points to prompt image
+      ) {
+        return new Promise((resolve, reject) => {
+          function messageHandler(event) {
+            console.log(event.data);
+            if ("status" in event.data) {
+              updateStatus(event.data);
+            }
+            if ("error" in event.data) {
+              samWorker.removeEventListener("message", messageHandler);
+              reject(new Error(event.data.error));
+            }
+            if (event.data.status === "complete-embedding") {
+              samWorker.removeEventListener("message", messageHandler);
+              resolve();
+            }
+            if (event.data.status === "complete") {
+              samWorker.removeEventListener("message", messageHandler);
+              resolve(event.data.output);
+            }
+          }
+          samWorker.addEventListener("message", messageHandler);
+          samWorker.postMessage({
+            modelURL,
+            modelID,
+            imageURL,
+            points,
+          });
+        });
+      }
+      function updateStatus(statusMessage) {
+        statusOutput.innerText = event.data.message;
+      }
+
+      const clearBtn = document.querySelector("#clear-btn");
+      const maskBtn = document.querySelector("#mask-btn");
+      const undoBtn = document.querySelector("#undo-btn");
+      const canvas = document.querySelector("#canvas");
+      const mask = document.querySelector("#mask");
+      const ctxCanvas = canvas.getContext("2d");
+      const ctxMask = mask.getContext("2d");
+      const fileUpload = document.querySelector("#file-upload");
+      const dropArea = document.querySelector("#drop-area");
+      const dropButtons = document.querySelector("#drop-buttons");
+      const imagesExamples = document.querySelector("#image-select");
+      const modelSelection = document.querySelector("#model");
+      const statusOutput = document.querySelector("#output-status");
+
+      //add event listener to file input
+      fileUpload.addEventListener("change", (e) => {
+        const target = e.target;
+        if (target.files.length > 0) {
+          const href = URL.createObjectURL(target.files[0]);
+          cleanImageCanvas();
+          drawImageCanvas(href);
+          setImageEmbeddings(href);
+        }
+      });
+      // add event listener to drop-area
+      dropArea.addEventListener("dragenter", (e) => {
+        e.preventDefault();
+        dropArea.classList.add("border-blue-700");
+      });
+      dropArea.addEventListener("dragleave", (e) => {
+        e.preventDefault();
+        dropArea.classList.remove("border-blue-700");
+      });
+      dropArea.addEventListener("dragover", (e) => {
+        e.preventDefault();
+      });
+      dropArea.addEventListener("drop", (e) => {
+        e.preventDefault();
+        dropArea.classList.remove("border-blue-700");
+        const url = e.dataTransfer.getData("text/uri-list");
+        const files = e.dataTransfer.files;
+
+        if (files.length > 0) {
+          const href = URL.createObjectURL(files[0]);
+          cleanImageCanvas();
+          drawImageCanvas(href);
+          setImageEmbeddings(href);
+        } else if (url) {
+          cleanImageCanvas();
+          drawImageCanvas(url);
+          setImageEmbeddings(url);
+        }
+      });
+
+      let hasImage = false;
+      let isSegmenting = false;
+      let isEmbedding = false;
+      let currentImageURL = "";
+      let pointArr = [];
+      let bgPointMode = false;
+      //add event listener to image examples
+      imagesExamples.addEventListener("click", (e) => {
+        if (isEmbedding || isSegmenting) {
+          return;
+        }
+        const target = e.target;
+        if (target.nodeName === "IMG") {
+          const href = target.src;
+          cleanImageCanvas();
+          drawImageCanvas(href);
+          setImageEmbeddings(href);
+        }
+      });
+      //add event listener to mask button
+      maskBtn.addEventListener("click", () => {
+        togglePointMode();
+      });
+      //add event listener to clear button
+      clearBtn.addEventListener("click", () => {
+        cleanImageCanvas();
+        togglePointMode(false);
+        pointArr = [];
+      });
+      //add event listener to undo button
+      undoBtn.addEventListener("click", () => {
+        undoPoint();
+      });
+      //add click event to canvas
+      canvas.addEventListener("click", async (event) => {
+        if (!hasImage || isEmbedding || isSegmenting) {
+          return;
+        }
+        const targetBox = event.target.getBoundingClientRect();
+        const x = (event.clientX - targetBox.left) / targetBox.width;
+        const y = (event.clientY - targetBox.top) / targetBox.height;
+        isSegmenting = true;
+        pointArr = [...pointArr, [x, y, !bgPointMode]];
+        undoBtn.disabled = false;
+        const { maskURL } = await getSegmentationMask(pointArr);
+        isSegmenting = false;
+        drawMask(maskURL);
+      });
+      async function undoPoint() {
+        if (!hasImage || isEmbedding || isSegmenting) {
+          return;
+        }
+        if (pointArr.length === 0) {
+          return;
+        }
+        pointArr.pop();
+        if (pointArr.length === 0) {
+          ctxMask.clearRect(0, 0, canvas.width, canvas.height);
+          undoBtn.disabled = true;
+          return;
+        }
+        isSegmenting = true;
+        const { maskURL } = await getSegmentationMask(pointArr);
+        isSegmenting = false;
+        drawMask(maskURL);
+      }
+      function togglePointMode(mode) {
+        bgPointMode = mode ? mode : !bgPointMode;
+
+        maskBtn.querySelector("span").innerText = bgPointMode
+          ? "Background Point"
+          : "Mask Point";
+        if (bgPointMode) {
+          maskBtn.querySelector("#mask-circle").setAttribute("hidden", "");
+          maskBtn.querySelector("#unmask-circle").removeAttribute("hidden");
+        } else {
+          maskBtn.querySelector("#mask-circle").removeAttribute("hidden");
+          maskBtn.querySelector("#unmask-circle").setAttribute("hidden", "");
+        }
+      }
+
+      async function getSegmentationMask(points) {
+        const modelID = modelSelection.value;
+        const modelURL = MODEL_BASEURL + MODELS[modelID].url;
+        const imageURL = currentImageURL;
+        const { maskURL } = await segmentPoints(
+          modelURL,
+          modelID,
+          imageURL,
+          points
+        );
+        return { maskURL };
+      }
+      async function setImageEmbeddings(imageURL) {
+        if (isEmbedding) {
+          return;
+        }
+        canvas.classList.remove("cursor-pointer");
+        canvas.classList.add("cursor-wait");
+        clearBtn.disabled = true;
+        const modelID = modelSelection.value;
+        const modelURL = MODEL_BASEURL + MODELS[modelID].url;
+        isEmbedding = true;
+        await segmentPoints(modelURL, modelID, imageURL);
+        canvas.classList.remove("cursor-wait");
+        canvas.classList.add("cursor-pointer");
+        clearBtn.disabled = false;
+        isEmbedding = false;
+        currentImageURL = imageURL;
+      }
+
+      function cleanImageCanvas() {
+        ctxCanvas.clearRect(0, 0, canvas.width, canvas.height);
+        ctxMask.clearRect(0, 0, canvas.width, canvas.height);
+        hasImage = false;
+        isEmbedding = false;
+        isSegmenting = false;
+        currentImageURL = "";
+        clearBtn.disabled = true;
+        canvas.parentElement.style.height = "auto";
+        dropButtons.classList.remove("invisible");
+      }
+      function drawMask(maskURL) {
+        if (!maskURL) {
+          throw new Error("No mask URL provided");
+        }
+
+        const img = new Image();
+        img.crossOrigin = "anonymous";
+
+        img.onload = () => {
+          mask.width = canvas.width;
+          mask.height = canvas.height;
+          ctxMask.drawImage(canvas, 0, 0);
+          ctxMask.globalCompositeOperation = "source-atop";
+          ctxMask.fillStyle = "rgba(255, 0, 0, 0.6)";
+          ctxMask.fillRect(0, 0, canvas.width, canvas.height);
+          ctxMask.globalCompositeOperation = "destination-in";
+          ctxMask.drawImage(img, 0, 0);
+        };
+        img.src = maskURL;
+      }
+      function drawImageCanvas(imgURL) {
+        if (!imgURL) {
+          throw new Error("No image URL provided");
+        }
+
+        ctxCanvas.clearRect(0, 0, canvas.width, canvas.height);
+        ctxCanvas.clearRect(0, 0, canvas.width, canvas.height);
+
+        const img = new Image();
+        img.crossOrigin = "anonymous";
+
+        img.onload = () => {
+          canvas.width = img.width;
+          canvas.height = img.height;
+          ctxCanvas.drawImage(img, 0, 0);
+          canvas.parentElement.style.height = canvas.offsetHeight + "px";
+          hasImage = true;
+          clearBtn.disabled = false;
+          dropButtons.classList.add("invisible");
+        };
+        img.src = imgURL;
+      }
+
+      const observer = new ResizeObserver((entries) => {
+        for (let entry of entries) {
+          if (entry.target === canvas) {
+            canvas.parentElement.style.height = canvas.offsetHeight + "px";
+          }
+        }
+      });
+      observer.observe(canvas);
+    </script>
+  </head>
+  <body class="container max-w-4xl mx-auto p-4">
+    <main class="grid grid-cols-1 gap-8 relative">
+      <span class="absolute text-5xl -ml-[1em]">🕯️</span>
+      <div>
+        <h1 class="text-5xl font-bold">Candle Segment Anything</h1>
+        <h2 class="text-2xl font-bold">Rust/WASM Demo</h2>
+        <p class="max-w-lg">
+          Zero-shot image segmentation with
+          <a
+            href="https://segment-anything.com"
+            class="underline hover:text-blue-500 hover:no-underline"
+            target="_blank"
+            >Segment Anything Model (SAM)</a
+          >
+          and
+          <a
+            href="https://github.com/ChaoningZhang/MobileSAM"
+            class="underline hover:text-blue-500 hover:no-underline"
+            target="_blank"
+            >MobileSAM </a
+          >. It runs in the browser with a WASM runtime built with
+          <a
+            href="https://github.com/huggingface/candle/"
+            target="_blank"
+            class="underline hover:text-blue-500 hover:no-underline"
+            >Candle
+          </a>
+        </p>
+      </div>
+      <div>
+        <label for="model" class="font-medium">Models Options: </label>
+        <select
+          id="model"
+          class="border-2 border-gray-500 rounded-md font-light">
+          <option value="sam_mobile_tiny" selected>
+            Mobile SAM Tiny (40.6 MB)
+          </option>
+          <option value="sam_base">SAM Base (375 MB)</option>
+        </select>
+      </div>
+      <div>
+        <p class="text-xs italic max-w-lg">
+          <b>Note:</b>
+          The model's first run may take a few seconds as it loads and caches
+          the model in the browser, and then creates the image embeddings. Any
+          subsequent clicks on points will be significantly faster.
+        </p>
+      </div>
+      <div class="relative max-w-2xl">
+        <div class="flex justify-between items-center">
+          <div class="px-2 rounded-md inline text-xs">
+            <span id="output-status" class="m-auto font-light"></span>
+          </div>
+          <div class="flex gap-2">
+            <button
+              id="mask-btn"
+              title="Toggle Mask Point and Background Point"
+              class="text-xs bg-white rounded-md disabled:opacity-50 flex gap-1 items-center">
+              <span>Mask Point</span>
+              <svg
+                xmlns="http://www.w3.org/2000/svg"
+                height="1em"
+                viewBox="0 0 512 512">
+                <path
+                  id="mask-circle"
+                  d="M256 512a256 256 0 1 0 0-512 256 256 0 1 0 0 512z" />
+                <path
+                  id="unmask-circle"
+                  hidden
+                  d="M464 256a208 208 0 1 0-416 0 208 208 0 1 0 416 0zM0 256a256 256 0 1 1 512 0 256 256 0 1 1-512 0z" />
+              </svg>
+            </button>
+            <button
+              id="undo-btn"
+              disabled
+              title="Undo Last Point"
+              class="text-xs bg-white rounded-md disabled:opacity-50 flex gap-1 items-center">
+              <svg
+                xmlns="http://www.w3.org/2000/svg"
+                height="1em"
+                viewBox="0 0 512 512">
+                <path
+                  d="M48.5 224H40a24 24 0 0 1-24-24V72a24 24 0 0 1 41-17l41.6 41.6a224 224 0 1 1-1 317.8 32 32 0 0 1 45.3-45.3 160 160 0 1 0 1-227.3L185 183a24 24 0 0 1-17 41H48.5z" />
+              </svg>
+            </button>
+
+            <button
+              id="clear-btn"
+              disabled
+              title="Clear Image"
+              class="text-xs bg-white rounded-md disabled:opacity-50 flex gap-1 items-center">
+              <svg
+                class=""
+                xmlns="http://www.w3.org/2000/svg"
+                viewBox="0 0 13 12"
+                height="1em">
+                <path
+                  d="M1.6.7 12 11.1M12 .7 1.6 11.1"
+                  stroke="#2E3036"
+                  stroke-width="2" />
+              </svg>
+            </button>
+          </div>
+        </div>
+        <div
+          id="drop-area"
+          class="flex flex-col items-center justify-center border-2 border-gray-300 border-dashed rounded-xl relative p-20 w-full overflow-hidden">
+          <div
+            id="drop-buttons"
+            class="flex flex-col items-center justify-center space-y-1 text-center relative z-10">
+            <svg
+              width="25"
+              height="25"
+              viewBox="0 0 25 25"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg">
+              <path
+                d="M3.5 24.3a3 3 0 0 1-1.9-.8c-.5-.5-.8-1.2-.8-1.9V2.9c0-.7.3-1.3.8-1.9.6-.5 1.2-.7 2-.7h18.6c.7 0 1.3.2 1.9.7.5.6.7 1.2.7 2v18.6c0 .7-.2 1.4-.7 1.9a3 3 0 0 1-2 .8H3.6Zm0-2.7h18.7V2.9H3.5v18.7Zm2.7-2.7h13.3c.3 0 .5 0 .6-.3v-.7l-3.7-5a.6.6 0 0 0-.6-.2c-.2 0-.4 0-.5.3l-3.5 4.6-2.4-3.3a.6.6 0 0 0-.6-.3c-.2 0-.4.1-.5.3l-2.7 3.6c-.1.2-.2.4 0 .7.1.2.3.3.6.3Z"
+                fill="#000" />
+            </svg>
+            <div class="flex text-sm text-gray-600">
+              <label
+                for="file-upload"
+                class="relative cursor-pointer bg-white rounded-md font-medium text-blue-950 hover:text-blue-700">
+                <span>Drag and drop your image here</span>
+                <span class="block text-xs">or</span>
+                <span class="block text-xs">Click to upload</span>
+              </label>
+            </div>
+            <input
+              id="file-upload"
+              name="file-upload"
+              type="file"
+              class="sr-only" />
+          </div>
+          <canvas id="canvas" class="absolute w-full"></canvas>
+          <canvas
+            id="mask"
+            class="pointer-events-none absolute w-full"></canvas>
+        </div>
+        <div class="text-right py-2">
+          <button
+            id="share-btn"
+            class="bg-white rounded-md hover:outline outline-orange-200 disabled:opacity-50 invisible">
+            <img
+              src="https://huggingface.co/datasets/huggingface/badges/raw/main/share-to-community-sm.svg" />
+          </button>
+        </div>
+      </div>
+      <div>
+        <div
+          class="flex gap-3 items-center overflow-x-scroll"
+          id="image-select">
+          <h3 class="font-medium">Examples:</h3>
+
+          <img
+            src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/candle/examples/sf.jpg"
+            class="cursor-pointer w-24 h-24 object-cover" />
+          <img
+            src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/candle/examples/bike.jpeg"
+            class="cursor-pointer w-24 h-24 object-cover" />
+          <img
+            src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/candle/examples/000000000077.jpg"
+            class="cursor-pointer w-24 h-24 object-cover" />
+        </div>
+      </div>
+    </main>
+  </body>
+</html>

samWorker.js

@@ -141,7 +141,7 @@ self.addEventListener("message", async (event) => {
     }
 
     self.postMessage({ status: "segmenting", message: "Segmenting" });
-    const { mask, image } = sam.mask_for_point(points.x, points.y);
+    const { mask, image } = sam.mask_for_point({ points });
     const maskDataURL = await createImageCanvas(mask, image);
     // Send the segment back to the main thread as JSON
     self.postMessage({