Text→image AND image→image search over one shared CLIP vector space — embed a free-text query or a reference image and rank the ingested photo library with a top-K slider (default 20), a visible min-similarity threshold (getDefaultThreshold ?? 0.2), and CLIP-scale score bars. Own usePhotoLibrary instance + its own media-dropzone ingest. Split from ui/blocks/photo-search (Search tab; absorbs cross-modal-search + product-search).
src/app/blocks/photo/image-search/image-search.tsx
'use client';
/**
* @file image-search.tsx
* @description Self-sufficient CLIP image-search block — ingest a photo library, then run text→image and image→image search over one shared multimodal vector space.
*/
import { useState } from 'react';
import { Loader2, Search, Trash2, X } from 'lucide-react';
import {
usePhotoLibrary,
readFileAsDataUrl,
type PhotoEntry,
type RankedHit,
type PhotoLibrary,
} from '@localmode/react';
import { transformers, isModelCached } from '@localmode/transformers';
import { cn } from '@/lib/utils';
import { ModelSelector, type SelectableModel } from '@/components/model-selector';
import { ModelDownloader } from '@/components/model-downloader';
import { SegmentedModePicker } from '@/components/segmented-mode-picker';
import { ParameterSlider } from '@/components/parameter-slider';
import { MediaDropzone } from '@/components/media-dropzone';
import { ImageResultGallery, type ImageResultCard } from '@/components/image-result-gallery';
import { ScoredResultBarList } from '@/components/scored-result-bar-list';
interface PhotoSearchModel {
id: string;
name: string;
size: string;
dimensions: number;
}
const DEFAULT_MODEL_ID = 'Xenova/clip-vit-base-patch32';
const MODEL_CATALOG: PhotoSearchModel[] = [
{ id: 'Xenova/clip-vit-base-patch32', name: 'CLIP ViT-B/32', size: '~350 MB', dimensions: 512 },
{ id: 'Xenova/siglip-base-patch16-224', name: 'SigLIP Base', size: '~400 MB', dimensions: 768 },
];
function getModel(id: string): PhotoSearchModel {
return MODEL_CATALOG.find((m) => m.id === id) ?? MODEL_CATALOG[0];
}
const PHOTO_LABELS = [
'nature',
'people',
'animals',
'food',
'architecture',
'vehicles',
'art',
'technology',
'sports',
'other',
];
const PRODUCT_LABELS = [
'Electronics',
'Clothing',
'Home & Garden',
'Toys',
'Food & Beverage',
'Sports',
'Books',
'Automotive',
'Health',
'Other',
];
const LABEL_PRESETS: Record<string, { labels: string[] }> = {
photo: { labels: PHOTO_LABELS },
product: { labels: PRODUCT_LABELS },
};
const DEFAULT_TOP_K = 20;
const MIN_SIMILARITY_FALLBACK = 0.2;
function formatScore(score: number): string {
return `${Math.round(score * 100)}%`;
}
function scoreTone(score: number): 'strong' | 'medium' | 'weak' {
if (score >= 0.35) return 'strong';
if (score >= 0.2) return 'medium';
return 'weak';
}
type SearchMode = 'text' | 'image';
const ACCEPTED = ['image/png', 'image/jpeg', 'image/webp'];
const SELECTABLE_MODELS: SelectableModel[] = MODEL_CATALOG.map((m) => ({
id: m.id,
name: m.name,
backend: 'onnx',
category: 'Multimodal (CLIP)',
size: m.size,
vision: true,
}));
function toLibraryCard(photo: PhotoEntry): ImageResultCard {
const category = photo.processing
? 'Analyzing…'
: photo.category
? `${photo.category} · ${photo.similarCount} similar`
: undefined;
return {
id: photo.id,
src: photo.src,
label: photo.filename,
category,
score: photo.processing || photo.confidence === 0 ? undefined : photo.confidence,
processing: photo.processing,
};
}
export function ImageSearchBlock() {
const lib: PhotoLibrary = usePhotoLibrary({
modelId: DEFAULT_MODEL_ID,
createEmbeddingModel: (id, onProgress) =>
transformers.multimodalEmbedding(id, { onProgress: (p) => onProgress(p as any) }),
createZeroShotClassifier: (id) => transformers.zeroShotImageClassifier(id),
isModelCached: (id) => isModelCached(id),
labelPresets: { photo: { labels: LABEL_PRESETS.photo.labels }, product: { labels: LABEL_PRESETS.product.labels } },
getModelDimensions: (id) => getModel(id).dimensions,
defaultTopK: DEFAULT_TOP_K,
minSimilarityFallback: MIN_SIMILARITY_FALLBACK,
});
const model = getModel(lib.activeModelId);
const statusText = lib.switching
? lib.reindexProgress
? `re-indexing ${lib.reindexProgress.completed}/${lib.reindexProgress.total}…`
: 're-indexing library…'
: lib.modelStatus === 'loading'
? `loading ${model.name}… ${Math.round(lib.modelProgress * 100)}%`
: lib.ingestProgress
? `embedding ${lib.ingestProgress.completed}/${lib.ingestProgress.total}…`
: lib.error
? 'error'
: lib.modelReady
? `ready - ${lib.photos.length} photo${lib.photos.length === 1 ? '' : 's'} indexed`
: 'idle - load a model to start';
const [mode, setMode] = useState<SearchMode>('text');
const [query, setQuery] = useState('');
const [hits, setHits] = useState<RankedHit[] | null>(null);
const [refImage, setRefImage] = useState<string | null>(null);
const [searching, setSearching] = useState(false);
const searchDisabled = !lib.modelReady || lib.switching || lib.photos.length === 0;
const ingestDisabled = !lib.modelReady || lib.busy || lib.switching;
const hasPhotos = lib.photos.length > 0;
const clearSearch = () => {
setHits(null);
setQuery('');
setRefImage(null);
};
const runTextSearch = async () => {
const q = query.trim();
if (!q || searchDisabled) return;
setSearching(true);
try {
setHits(await lib.searchByText(q));
} finally {
setSearching(false);
}
};
const runImageSearch = async (file: File) => {
if (searchDisabled) return;
const dataUrl = await readFileAsDataUrl(file);
setRefImage(dataUrl);
setSearching(true);
try {
setHits(await lib.searchByImage(dataUrl));
} finally {
setSearching(false);
}
};
const cards: ImageResultCard[] = [];
for (const hit of hits ?? []) {
const photo = lib.getPhoto(hit.id);
if (!photo) continue;
cards.push({
id: photo.id,
src: photo.src,
label: photo.filename,
category: photo.category || undefined,
score: hit.score,
});
}
const scored = cards.map((c) => ({ label: c.label ?? c.id, score: c.score ?? 0 }));
const topFilename = cards[0]?.label ?? '';
return (
<div className="mx-auto flex max-w-5xl flex-col gap-4 p-4">
{}
<p
role="status"
aria-live="polite"
className="text-xs text-muted-foreground"
>
{statusText}
</p>
{lib.error && (
<div
className="flex items-center justify-between gap-3 rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-xs text-destructive"
>
<span>{lib.error}</span>
<button
type="button"
onClick={lib.clearError}
className="rounded px-2 py-0.5 font-medium hover:bg-destructive/20 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"
>
Dismiss
</button>
</div>
)}
{}
<div
data-status={lib.modelStatus}
data-model-id={lib.activeModelId}
role="group"
aria-label="CLIP model status"
className="flex flex-col gap-3 rounded-xl border border-border bg-muted/40 p-4 sm:flex-row sm:items-start"
>
<div className="w-full sm:max-w-sm">
<ModelSelector
models={SELECTABLE_MODELS}
selectedId={lib.activeModelId}
onSelect={(id) => lib.requestModel(id)}
/>
</div>
<div className="flex min-w-0 flex-1 flex-col gap-2">
{lib.modelStatus === 'idle' ? (
<>
<p className="text-sm text-muted-foreground">
<span className="font-medium text-foreground">{model.name}</span> ({model.size}) -
not loaded. It powers both search embeddings and categorization. Nothing downloads
until you press Load.
</p>
<button
type="button"
onClick={() => void lib.loadModel()}
className="inline-flex h-9 w-fit items-center rounded-md bg-primary px-4 text-sm font-medium text-primary-foreground transition-colors hover:bg-primary/90 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 focus-visible:ring-offset-2 focus-visible:ring-offset-background"
>
Load {model.name}
</button>
</>
) : (
<div>
<ModelDownloader
name={model.name}
size={model.size}
category="Multimodal (CLIP)"
progress={lib.modelProgressValue}
cached={lib.modelCached}
ready={lib.modelReady && !lib.switching}
className="max-w-sm"
/>
</div>
)}
{}
{lib.pendingModelId && (
<div
className="flex flex-col gap-2 rounded-lg border border-amber-500/40 bg-amber-500/10 p-3 text-xs"
>
<span className="text-foreground">
Switch to <span className="font-medium">{getModel(lib.pendingModelId).name}</span>?
The {getModel(lib.pendingModelId).dimensions}-dim vector space is incompatible - all{' '}
{lib.photos.length} photos will be re-embedded and re-categorized.
</span>
<div className="flex items-center gap-2">
<button
type="button"
onClick={lib.confirmModelSwitch}
className="inline-flex h-7 items-center rounded-md bg-primary px-3 font-medium text-primary-foreground hover:bg-primary/90 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 focus-visible:ring-offset-2 focus-visible:ring-offset-background"
>
Confirm switch
</button>
<button
type="button"
onClick={lib.cancelModelSwitch}
className="inline-flex h-7 items-center rounded-md border border-border px-3 font-medium hover:bg-accent focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"
>
Cancel
</button>
</div>
</div>
)}
{}
{lib.switching && lib.reindexProgress && (
<div
data-completed={lib.reindexProgress.completed}
data-total={lib.reindexProgress.total}
role="status"
aria-live="polite"
aria-label="Re-index progress"
className="text-xs tabular-nums text-muted-foreground"
>
Re-indexing {lib.reindexProgress.completed}/{lib.reindexProgress.total} photos through{' '}
{model.name}…
</div>
)}
</div>
</div>
{}
<section className="flex flex-col gap-3">
<h2 className="text-sm font-semibold text-foreground">1 · Build a library</h2>
<div role="group" aria-label="Photo library upload">
<MediaDropzone
accept={ACCEPTED}
multiple
disabled={ingestDisabled}
addAnother={hasPhotos}
processing={lib.ingestProgress != null}
processingLabel={
lib.ingestProgress
? `Embedding ${lib.ingestProgress.completed}/${lib.ingestProgress.total}…`
: 'Processing…'
}
title={lib.modelReady ? 'Drop photos here' : 'Load a model to start'}
subtitle="PNG, JPEG or WebP - or click to browse"
onFiles={(files) => void lib.ingest(files)}
onReject={(rejections) =>
lib.setRejection({
filename: rejections[0].file.name,
reason: rejections[0].reason,
})
}
/>
</div>
{}
{lib.rejection && (
<div
className="flex items-center justify-between gap-3 rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-sm text-destructive"
>
<span>
Rejected <span className="font-medium">{lib.rejection.filename}</span>:{' '}
{lib.rejection.reason}
</span>
<button
type="button"
onClick={() => lib.setRejection(null)}
className="rounded px-2 py-0.5 text-xs font-medium hover:bg-destructive/20 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"
>
Dismiss
</button>
</div>
)}
{}
{lib.ingestProgress && (
<div
data-completed={lib.ingestProgress.completed}
data-total={lib.ingestProgress.total}
role="status"
aria-live="polite"
aria-label="Ingest progress"
className="flex items-center justify-between gap-3 rounded-lg border border-border bg-card px-3 py-2 text-sm"
>
<span className="tabular-nums text-muted-foreground">
Embedding {lib.ingestProgress.completed}/{lib.ingestProgress.total}…
</span>
<button
type="button"
onClick={lib.cancelIngest}
className="rounded-md border border-border px-2.5 py-1 text-xs font-medium hover:bg-accent focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"
>
Cancel
</button>
</div>
)}
{}
{hasPhotos && (
<div className="flex flex-wrap items-center gap-2">
<span className="text-sm font-medium tabular-nums">
{lib.photos.length} photo{lib.photos.length === 1 ? '' : 's'} indexed
</span>
<button
type="button"
onClick={lib.clearAll}
disabled={lib.busy || lib.switching}
className="ml-auto inline-flex h-7 items-center gap-1 rounded-md border border-border px-2.5 text-xs font-medium text-muted-foreground transition-colors hover:border-destructive hover:text-destructive disabled:opacity-50 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"
>
<Trash2 className="size-3.5" />
Clear all
</button>
</div>
)}
{}
{hasPhotos ? (
<div>
<ImageResultGallery
cards={lib.photos.map(toLibraryCard)}
layout="grid"
onDelete={(id) => lib.deletePhoto(id)}
scoreThresholds={{ high: 0.35, medium: 0.2 }}
/>
</div>
) : lib.modelReady ? (
<p
className="rounded-lg border border-dashed border-border px-4 py-8 text-center text-sm text-muted-foreground"
>
No photos yet. Drop some images above to build a searchable library.
</p>
) : null}
{}
<ul aria-label="Indexed photos" className="sr-only">
{lib.photos.map((photo) => (
<li
key={photo.id}
data-id={photo.id}
data-filename={photo.filename}
data-category={photo.category}
data-confidence={photo.confidence.toFixed(4)}
data-embedded={photo.embedding !== null}
data-processing={photo.processing}
>
{photo.filename}: {photo.category} ({formatScore(photo.confidence)})
</li>
))}
</ul>
</section>
{}
<section className="flex flex-col gap-4 border-t border-border pt-4">
<h2 className="text-sm font-semibold text-foreground">
2 · Search the shared vector space
</h2>
{}
<div className="flex flex-wrap items-center gap-3">
<div data-mode={mode}>
<SegmentedModePicker<SearchMode>
items={[
{ id: 'text', label: 'Text' },
{ id: 'image', label: 'Image' },
]}
selectedId={mode}
onSelect={(m) => {
setMode(m);
clearSearch();
}}
aria-label="Search mode"
/>
</div>
<span
data-threshold={lib.minSimilarity}
className="ml-auto text-xs text-muted-foreground"
>
Minimum similarity: {formatScore(lib.minSimilarity)}
</span>
</div>
{}
<div data-value={lib.topK} className="max-w-xs">
<ParameterSlider
label="Results (top-K)"
value={lib.topK}
onChange={lib.setTopK}
min={1}
max={50}
step={1}
/>
</div>
{}
{mode === 'text' && (
<div className="flex flex-wrap items-center gap-2">
<div className="relative flex-1">
<Search className="pointer-events-none absolute left-2.5 top-1/2 size-4 -translate-y-1/2 text-muted-foreground" />
<input
type="text"
aria-label="Search query"
value={query}
disabled={searchDisabled}
placeholder="Describe a photo, e.g. “a photo of a dog”"
onChange={(e) => setQuery(e.target.value)}
onKeyDown={(e) => {
if (e.key === 'Enter') void runTextSearch();
}}
className="h-9 w-full rounded-md border border-border bg-background pl-8 pr-3 text-sm outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 disabled:opacity-50"
/>
</div>
<button
type="button"
onClick={() => void runTextSearch()}
disabled={searchDisabled || !query.trim() || searching}
className="inline-flex h-9 items-center gap-1.5 rounded-md bg-primary px-3 text-sm font-medium text-primary-foreground transition-colors hover:bg-primary/90 disabled:opacity-50 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 focus-visible:ring-offset-2 focus-visible:ring-offset-background"
>
{searching ? <Loader2 className="size-4 animate-spin" /> : <Search className="size-4" />}
Search
</button>
{hits && (
<button
type="button"
onClick={clearSearch}
className="inline-flex h-9 items-center gap-1 rounded-md border border-border px-2.5 text-xs font-medium text-muted-foreground hover:bg-accent focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"
>
<X className="size-3.5" />
Clear
</button>
)}
</div>
)}
{}
{mode === 'image' && (
<div className="flex flex-col gap-3 sm:flex-row sm:items-start">
<div
className="w-full sm:max-w-xs"
role="group"
aria-label="Reference image upload"
>
<MediaDropzone
accept={ACCEPTED}
multiple={false}
disabled={searchDisabled}
processing={searching}
processingLabel="Searching…"
title="Drop a reference image"
subtitle="Find visually similar photos"
onFiles={(files) => void runImageSearch(files[0])}
/>
</div>
{refImage && (
<div className="flex items-center gap-2">
{}
<img src={refImage} alt="Reference" className="size-20 rounded-md object-cover" />
<button
type="button"
onClick={clearSearch}
className="inline-flex h-8 items-center gap-1 rounded-md border border-border px-2.5 text-xs font-medium text-muted-foreground hover:bg-accent focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"
>
<X className="size-3.5" />
Clear
</button>
</div>
)}
</div>
)}
{}
{hits !== null &&
(cards.length > 0 ? (
<div className="flex flex-col gap-4">
<div
data-count={cards.length}
data-top={topFilename}
role="region"
aria-label="Search results"
>
<ImageResultGallery
cards={cards}
layout="grid"
scoreThresholds={{ high: 0.35, medium: 0.2 }}
/>
</div>
<ScoredResultBarList results={scored} sort={false} limit={10} />
{}
<ol aria-label="Search results ranking" className="sr-only">
{cards.map((c, i) => (
<li
key={c.id}
data-rank={i}
data-id={c.id}
data-filename={c.label}
data-score={(c.score ?? 0).toFixed(4)}
data-tone={scoreTone(c.score ?? 0)}
>
{c.label}: {formatScore(c.score ?? 0)}
</li>
))}
</ol>
</div>
) : (
<p
className="rounded-lg border border-dashed border-border px-4 py-10 text-center text-sm text-muted-foreground"
>
No photos at or above the {formatScore(lib.minSimilarity)} similarity threshold.
</p>
))}
{searchDisabled && lib.photos.length === 0 && (
<p className="text-sm text-muted-foreground">Ingest photos above first, then search them.</p>
)}
</section>
</div>
);
}