Run SmolVLM-256M-Instruct in your browser

import { AutoProcessor, AutoModelForVision2Seq, load_image } from "@huggingface/transformers";

const processor = await AutoProcessor.from_pretrained("HuggingFaceTB/SmolVLM-256M-Instruct");
const model = await AutoModelForVision2Seq.from_pretrained("HuggingFaceTB/SmolVLM-256M-Instruct", {
  device: "webgpu",
  // fp32 on purpose: the q4f16 and fp16 builds generate garbled text over
  // WebGPU today (the official demo ships fp32 for the same reason).
  dtype: "fp32", // use "uint8" for the WASM build
});

const image = await load_image("https://your-image-url.jpg");
const messages = [
  { role: "user", content: [{ type: "image" }, { type: "text", text: "Describe this image." }] },
];
const text = processor.apply_chat_template(messages, { add_generation_prompt: true });
const inputs = await processor(text, [image]);

const output = await model.generate({ ...inputs, max_new_tokens: 256 });
const result = processor.batch_decode(output, { skip_special_tokens: true });