Run Florence-2-base-ft in your browser

import { Florence2ForConditionalGeneration, AutoProcessor, AutoTokenizer } from "@huggingface/transformers";

const model = await Florence2ForConditionalGeneration.from_pretrained("onnx-community/Florence-2-base-ft", {
  device: "webgpu",
  // Per-module dtype map: keep the language-model parts in fp16/fp32 for
  // quality, quantize the (much larger) vision encoder/decoder to q4.
  dtype: {
    embed_tokens: "fp16", // or "fp32" where WebGPU fp16 (shader-f16) isn't supported
    vision_encoder: "fp16", // or "fp32" where WebGPU fp16 isn't supported
    encoder_model: "q4",
    decoder_model_merged: "q4",
  },
});
const processor = await AutoProcessor.from_pretrained("onnx-community/Florence-2-base-ft");
const tokenizer = await AutoTokenizer.from_pretrained("onnx-community/Florence-2-base-ft");

const prompt = "<MORE_DETAILED_CAPTION>"; // task-prompt token, not free text
// const inputs = await processor(image, prompt);
// const generated = await model.generate({ ...inputs, max_new_tokens: 100 });
// const result = processor.batch_decode(generated, { skip_special_tokens: false });