JezzWTF/phokus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add CLIP embeddings and similar-image search

Browse Source 
- add Candle + HF Hub CLIP image embedding pipeline with background embedding worker
- write image embeddings into sqlite-vec and expose similar-image lookup through a new backend command
- surface embedding progress and recovery in the UI, including retries for failed embeddings
- improve gallery/lightbox embedding UX and make similar-image actions directly accessible

Refs: #3, #4
This commit is contained in:
LyAhn
2026-04-06 00:54:14 +01:00
parent 76ec424167
commit 51e4c2c1f7
14 changed files with 2219 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/App.tsx
+5 -1
View File
@@ -9,11 +9,15 @@ import { Lightbox } from "./components/Lightbox";
export default function App() {
const loadFolders = useGalleryStore((state) => state.loadFolders);
const loadBackgroundJobProgress = useGalleryStore((state) => state.loadBackgroundJobProgress);
const loadImages = useGalleryStore((state) => state.loadImages);
const subscribeToProgress = useGalleryStore((state) => state.subscribeToProgress);
useEffect(() => {
loadFolders().then(() => loadImages(true));
loadFolders().then(() => {
void loadBackgroundJobProgress();
return loadImages(true);
});
let unlisten: (() => void) | undefined;
subscribeToProgress().then((fn) => {
unlisten = fn;
src/components/BackgroundTasks.tsx
+46 -4
View File
@@ -16,19 +16,27 @@ export function BackgroundTasks() {
const folders = useGalleryStore((state) => state.folders);
const indexingProgress = useGalleryStore((state) => state.indexingProgress);
const mediaJobProgress = useGalleryStore((state) => state.mediaJobProgress);
const retryFailedEmbeddings = useGalleryStore((state) => state.retryFailedEmbeddings);
const tasks = useMemo(() => {
return folders
.map((folder) => {
const index = indexingProgress[folder.id];
const jobs = mediaJobProgress[folder.id];
const pendingMediaWork = (jobs?.thumbnail_pending ?? 0) + (jobs?.metadata_pending ?? 0);
const pendingMediaWork =
(jobs?.thumbnail_pending ?? 0) +
(jobs?.metadata_pending ?? 0) +
(jobs?.embedding_pending ?? 0);
const embeddingProcessed = (jobs?.embedding_ready ?? 0) + (jobs?.embedding_failed ?? 0);
const embeddingTotal = embeddingProcessed + (jobs?.embedding_pending ?? 0);
const hasFailedEmbeddings = (jobs?.embedding_failed ?? 0) > 0;
if (!index && pendingMediaWork === 0) {
if (!index && pendingMediaWork === 0 && !hasFailedEmbeddings) {
return null;
}
const indexPercent = index && index.total > 0 ? (index.indexed / index.total) * 100 : 0;
const embeddingPercent = embeddingTotal > 0 ? (embeddingProcessed / embeddingTotal) * 100 : 0;
return {
id: folder.id,
name: folder.name,
@@ -36,6 +44,10 @@ export function BackgroundTasks() {
jobs,
pendingMediaWork,
indexPercent,
embeddingProcessed,
embeddingTotal,
embeddingPercent,
hasFailedEmbeddings,
};
})
.filter((task) => task !== null);
@@ -60,12 +72,20 @@ export function BackgroundTasks() {
<p className="text-[11px] text-gray-500">
{task.index && !task.index.done
? `${task.index.indexed.toLocaleString()} of ${task.index.total.toLocaleString()} scanned`
: `${task.pendingMediaWork.toLocaleString()} media jobs remaining`}
: task.hasFailedEmbeddings && task.pendingMediaWork === 0
? `Embedding failures require attention`
: `${task.pendingMediaWork.toLocaleString()} media jobs remaining`}
</p>
</div>
<div className="text-right text-[11px] text-gray-400">
{task.jobs?.thumbnail_pending ? <div>{task.jobs.thumbnail_pending.toLocaleString()} thumbnails</div> : null}
{task.jobs?.metadata_pending ? <div>{task.jobs.metadata_pending.toLocaleString()} metadata</div> : null}
{task.embeddingTotal > 0 ? (
<div>
{task.embeddingProcessed.toLocaleString()} / {task.embeddingTotal.toLocaleString()} embeddings
</div>
) : null}
{task.jobs?.embedding_failed ? <div>{task.jobs.embedding_failed.toLocaleString()} failed</div> : null}
</div>
</div>
@@ -74,10 +94,32 @@ export function BackgroundTasks() {
<ProgressBar value={task.indexPercent} />
<p className="truncate text-[11px] text-gray-500">{task.index.current_file || "Scanning..."}</p>
</div>
) : task.embeddingTotal > 0 && (task.jobs?.embedding_pending ?? 0) > 0 ? (
<div className="mt-2 space-y-1">
<ProgressBar value={task.embeddingPercent} />
<p className="text-[11px] text-gray-500">
{task.embeddingProcessed.toLocaleString()} completed, {task.jobs?.embedding_pending?.toLocaleString() ?? 0} remaining
</p>
</div>
) : task.hasFailedEmbeddings ? (
<div className="mt-2 space-y-1">
<ProgressBar value={100} />
<div className="flex items-center justify-between gap-3">
<p className="text-[11px] text-amber-300">
{task.jobs?.embedding_failed?.toLocaleString() ?? 0} embedding failures need attention
</p>
<button
className="rounded-full border border-amber-400/20 bg-amber-500/10 px-2.5 py-1 text-[11px] text-amber-200 hover:bg-amber-500/20"
onClick={() => void retryFailedEmbeddings(task.id)}
>
Retry
</button>
</div>
</div>
) : task.pendingMediaWork > 0 ? (
<div className="mt-2 space-y-1">
<ProgressBar value={0} />
<p className="text-[11px] text-gray-500">Processing thumbnails and metadata</p>
<p className="text-[11px] text-gray-500">Processing thumbnails, metadata, and embeddings</p>
</div>
) : null}
</div>
src/components/Gallery.tsx
+58 -10
View File
@@ -38,6 +38,19 @@ function formatDuration(durationMs: number | null): string | null {
return `${minutes}:${seconds.toString().padStart(2, "0")}`;
}
function embeddingLabel(image: ImageRecord): string {
if (image.embedding_status === "ready") {
return image.embedding_model ? `Embeddings ready` : "Embeddings ready";
}
if (image.embedding_status === "failed") {
return "Embeddings failed";
}
if (image.embedding_status === "processing") {
return "Embedding...";
}
return "Embedding queued";
}
function ContextMenu({
x,
y,
@@ -49,10 +62,13 @@ function ContextMenu({
image: ImageRecord;
onClose: () => void;
}) {
const { openImage, updateImageDetails } = useGalleryStore();
const openImage = useGalleryStore((state) => state.openImage);
const updateImageDetails = useGalleryStore((state) => state.updateImageDetails);
const loadSimilarImages = useGalleryStore((state) => state.loadSimilarImages);
return (
<div
data-gallery-context-menu
className="fixed z-40 min-w-56 rounded-2xl border border-white/10 bg-gray-950/95 p-2 shadow-2xl backdrop-blur"
style={{ left: x, top: y }}
onClick={(event) => event.stopPropagation()}
@@ -75,21 +91,37 @@ function ContextMenu({
>
{image.favorite ? "Remove Favorite" : "Add to Favorites"}
</button>
<button
className="w-full rounded-xl px-3 py-2 text-left text-sm text-gray-200 hover:bg-white/5"
onClick={async () => {
await loadSimilarImages(image.id);
onClose();
}}
>
Find Similar
</button>
<div className="my-2 h-px bg-white/5" />
<div className="px-3 pb-1 pt-1 text-[11px] uppercase tracking-[0.2em] text-gray-500">Rating</div>
<div className="flex gap-1 px-2 pb-1">
<div className="flex items-center gap-1 px-2 pb-1">
{Array.from({ length: 5 }, (_, index) => {
const rating = index + 1;
return (
<button
key={rating}
className={`rounded-lg px-2 py-1 text-sm ${rating <= image.rating ? "bg-amber-400/15 text-amber-300" : "bg-white/5 text-gray-400 hover:text-white"}`}
className="rounded-md p-1"
onClick={async () => {
await updateImageDetails(image.id, { rating });
onClose();
}}
title={`Set ${rating} star rating`}
>
{rating}
<svg
className={`h-5 w-5 ${rating <= image.rating ? "text-amber-300" : "text-white/20 hover:text-white/50"}`}
fill="currentColor"
viewBox="0 0 20 20"
>
<path d="M9.049 2.927c.3-.921 1.603-.921 1.902 0l1.07 3.292a1 1 0 00.95.69h3.462c.969 0 1.371 1.24.588 1.81l-2.8 2.034a1 1 0 00-.364 1.118l1.07 3.292c.3.921-.755 1.688-1.54 1.118l-2.8-2.034a1 1 0 00-1.176 0l-2.8 2.034c-.784.57-1.838-.197-1.539-1.118l1.07-3.292a1 1 0 00-.364-1.118L2.98 8.72c-.783-.57-.38-1.81.588-1.81H7.03a1 1 0 00.951-.69l1.07-3.292z" />
</svg>
</button>
);
})}
@@ -123,6 +155,7 @@ function ImageTile({
}) {
const [loaded, setLoaded] = useState(false);
const [errored, setErrored] = useState(false);
const loadSimilarImages = useGalleryStore((state) => state.loadSimilarImages);
const src = image.thumbnail_path
? convertFileSrc(image.thumbnail_path)
@@ -207,7 +240,19 @@ function ImageTile({
<p className="truncate text-sm font-medium text-white">{image.filename}</p>
<div className="mt-1 flex items-center justify-between gap-2 text-xs text-white/70">
<RatingStars rating={image.rating} />
<span>{image.embedding_status === "ready" ? "indexed" : image.embedding_status}</span>
<span>{embeddingLabel(image)}</span>
</div>
<div className="mt-2 flex items-center gap-2 opacity-0 transition-opacity group-hover:opacity-100">
<button
className="rounded-full border border-white/10 bg-black/40 px-2.5 py-1 text-[11px] text-white/85 hover:bg-black/60"
onClick={(event) => {
event.stopPropagation();
void loadSimilarImages(image.id);
}}
>
Find Similar
</button>
<span className="text-[11px] text-white/50">Right-click for more</span>
</div>
</div>
</button>
@@ -242,18 +287,21 @@ export function Gallery() {
}, [handleScroll]);
useEffect(() => {
const close = () => setContextMenu(null);
const close = (event: PointerEvent) => {
if ((event.target as HTMLElement | null)?.closest("[data-gallery-context-menu]")) {
return;
}
setContextMenu(null);
};
const handleKeyDown = (event: KeyboardEvent) => {
if (event.key === "Escape") {
setContextMenu(null);
}
};
window.addEventListener("click", close);
window.addEventListener("contextmenu", close);
window.addEventListener("pointerdown", close);
window.addEventListener("keydown", handleKeyDown);
return () => {
window.removeEventListener("click", close);
window.removeEventListener("contextmenu", close);
window.removeEventListener("pointerdown", close);
window.removeEventListener("keydown", handleKeyDown);
};
}, []);
src/components/Lightbox.tsx
+24 -1
View File
@@ -32,11 +32,25 @@ function formatDuration(durationMs: number | null): string {
return `${minutes}:${seconds.toString().padStart(2, "0")}`;
}
function embeddingLabel(status: string, model: string | null): string {
if (status === "ready") {
return model ? `Ready (${model})` : "Ready";
}
if (status === "failed") {
return "Failed";
}
if (status === "processing") {
return "Processing";
}
return "Queued";
}
export function Lightbox() {
const selectedImage = useGalleryStore((state) => state.selectedImage);
const closeImage = useGalleryStore((state) => state.closeImage);
const images = useGalleryStore((state) => state.images);
const openImage = useGalleryStore((state) => state.openImage);
const loadSimilarImages = useGalleryStore((state) => state.loadSimilarImages);
const updateImageDetails = useGalleryStore((state) => state.updateImageDetails);
const [zoom, setZoom] = useState(1);
const imageViewportRef = useRef<HTMLDivElement>(null);
@@ -184,6 +198,12 @@ export function Lightbox() {
<path d="M3.172 5.172a4 4 0 015.656 0L10 6.343l1.172-1.171a4 4 0 115.656 5.656L10 17.657l-6.828-6.829a4 4 0 010-5.656z" />
</svg>
</button>
<button
className="rounded-full border border-white/10 bg-white/5 px-3 py-1.5 text-xs text-gray-300 hover:text-white"
onClick={() => void loadSimilarImages(selectedImage.id)}
>
Similar
</button>
</div>
<button className="rounded p-1 text-gray-400 hover:text-white" onClick={closeImage}>
<svg className="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke="currentColor">
@@ -281,7 +301,10 @@ export function Lightbox() {
<div>
<p className="mb-1 text-xs uppercase tracking-wider text-gray-500">Embedding</p>
<p className="text-white">{selectedImage.embedding_status}</p>
<p className="text-white">{embeddingLabel(selectedImage.embedding_status, selectedImage.embedding_model)}</p>
{selectedImage.embedding_error ? (
<p className="mt-1 text-xs text-amber-300">{selectedImage.embedding_error}</p>
) : null}
</div>
<div>
src/components/Toolbar.tsx
+2 -1
View File
@@ -41,6 +41,7 @@ export function Toolbar() {
const totalImages = useGalleryStore((state) => state.totalImages);
const loadedCount = useGalleryStore((state) => state.loadedCount);
const selectedFolderId = useGalleryStore((state) => state.selectedFolderId);
const collectionTitle = useGalleryStore((state) => state.collectionTitle);
const folders = useGalleryStore((state) => state.folders);
const mediaFilter = useGalleryStore((state) => state.mediaFilter);
const setMediaFilter = useGalleryStore((state) => state.setMediaFilter);
@@ -53,7 +54,7 @@ export function Toolbar() {
const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const selectedFolder = folders.find((folder) => folder.id === selectedFolderId);
const title = selectedFolder ? selectedFolder.name : "All Media";
const title = collectionTitle ?? (selectedFolder ? selectedFolder.name : "All Media");
const tileSize = tileSizeForZoom(zoomPreset);
useEffect(() => {
src/store.ts
+47 -8
View File
@@ -53,6 +53,9 @@ export interface FolderJobProgress {
folder_id: number;
thumbnail_pending: number;
metadata_pending: number;
embedding_pending: number;
embedding_ready: number;
embedding_failed: number;
}
export interface MediaJobProgressEvent {
@@ -89,11 +92,13 @@ interface GalleryState {
favoritesOnly: boolean;
zoomPreset: ZoomPreset;
selectedImage: ImageRecord | null;
collectionTitle: string | null;
indexingProgress: Record<number, IndexProgress>;
mediaJobProgress: Record<number, FolderJobProgress>;
cacheDir: string;
loadFolders: () => Promise<void>;
loadBackgroundJobProgress: () => Promise<void>;
addFolder: (path: string) => Promise<void>;
removeFolder: (folderId: number) => Promise<void>;
reindexFolder: (folderId: number) => Promise<void>;
@@ -107,6 +112,8 @@ interface GalleryState {
setZoomPreset: (zoomPreset: ZoomPreset) => void;
openImage: (image: ImageRecord) => void;
closeImage: () => void;
loadSimilarImages: (imageId: number) => Promise<void>;
retryFailedEmbeddings: (folderId: number) => Promise<void>;
updateImageDetails: (imageId: number, updates: { favorite?: boolean; rating?: number }) => Promise<void>;
setCacheDir: (dir: string) => void;
subscribeToProgress: () => Promise<UnlistenFn>;
@@ -235,6 +242,7 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
favoritesOnly: false,
zoomPreset: "comfortable",
selectedImage: null,
collectionTitle: null,
indexingProgress: {},
mediaJobProgress: {},
cacheDir: "",
@@ -246,16 +254,25 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
set({ folders });
},
loadBackgroundJobProgress: async () => {
const progress = await invoke<FolderJobProgress[]>("get_background_job_progress");
set(() => ({
mediaJobProgress: Object.fromEntries(progress.map((entry) => [entry.folder_id, entry])),
}));
},
addFolder: async (path) => {
const { loadFolders } = get();
const { loadFolders, loadBackgroundJobProgress } = get();
await invoke("add_folder", { path });
await loadFolders();
await loadBackgroundJobProgress();
},
removeFolder: async (folderId) => {
await invoke("remove_folder", { folderId });
const { selectedFolderId, loadFolders, loadImages } = get();
const { selectedFolderId, loadFolders, loadImages, loadBackgroundJobProgress } = get();
await loadFolders();
await loadBackgroundJobProgress();
if (selectedFolderId === folderId) {
set({ selectedFolderId: null });
await loadImages(true);
@@ -263,13 +280,14 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
},
reindexFolder: async (folderId) => {
const { loadFolders } = get();
const { loadFolders, loadBackgroundJobProgress } = get();
await invoke("reindex_folder", { folderId });
await loadFolders();
await loadBackgroundJobProgress();
},
selectFolder: (folderId) => {
set({ selectedFolderId: folderId, images: [], loadedCount: 0 });
set({ selectedFolderId: folderId, images: [], loadedCount: 0, collectionTitle: null });
void get().loadImages(true);
},
@@ -301,6 +319,7 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
totalImages: result.total,
loadedCount: reset ? result.images.length : state.loadedCount + result.images.length,
loadingImages: false,
collectionTitle: reset ? null : state.collectionTitle,
}));
} catch (error) {
console.error("Failed to load media:", error);
@@ -315,22 +334,22 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
},
setSearch: (search) => {
set({ search, images: [], loadedCount: 0 });
set({ search, images: [], loadedCount: 0, collectionTitle: null });
void get().loadImages(true);
},
setSort: (sort) => {
set({ sort, images: [], loadedCount: 0 });
set({ sort, images: [], loadedCount: 0, collectionTitle: null });
void get().loadImages(true);
},
setMediaFilter: (mediaFilter) => {
set({ mediaFilter, images: [], loadedCount: 0 });
set({ mediaFilter, images: [], loadedCount: 0, collectionTitle: null });
void get().loadImages(true);
},
setFavoritesOnly: (favoritesOnly) => {
set({ favoritesOnly, images: [], loadedCount: 0 });
set({ favoritesOnly, images: [], loadedCount: 0, collectionTitle: null });
void get().loadImages(true);
},
@@ -339,6 +358,25 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
openImage: (image) => set({ selectedImage: image }),
closeImage: () => set({ selectedImage: null }),
loadSimilarImages: async (imageId) => {
const images = await invoke<ImageRecord[]>("find_similar_images", {
params: { image_id: imageId, limit: PAGE_SIZE },
});
set({
images,
totalImages: images.length,
loadedCount: images.length,
loadingImages: false,
collectionTitle: "Similar Images",
selectedFolderId: null,
});
},
retryFailedEmbeddings: async (folderId) => {
await invoke("retry_failed_embeddings", { params: { folder_id: folderId } });
await get().loadBackgroundJobProgress();
},
updateImageDetails: async (imageId, updates) => {
const updatedImage = await invoke<ImageRecord>("update_image_details", {
params: {
@@ -366,6 +404,7 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
if (progress.done) {
void get().loadFolders();
void get().loadBackgroundJobProgress();
void get().loadImages(true);
setTimeout(() => {