From ceb51f8fadf02fe8a6732bd089c061dffacaf12b Mon Sep 17 00:00:00 2001 From: LyAhn Date: Tue, 9 Jun 2026 01:16:12 +0100 Subject: [PATCH 1/9] feat: smart thumbnail cleanup and rename-aware file tracking Delete thumbnails alongside DB rows in all three removal paths (watcher detect, delete-from-disk command, remove-folder command) so orphans no longer accumulate passively. Intercept RenameMode::Both watcher events and handle them as in-place path updates: renames the thumbnail file to the new hash and updates the DB row without touching the embedding, avoiding a full rebuild on move. Falls back to thumbnail regeneration if the file rename fails. --- src-tauri/src/commands.rs | 8 +++ src-tauri/src/db.rs | 53 +++++++++++++++- src-tauri/src/indexer.rs | 123 ++++++++++++++++++++++++++++++++++---- src-tauri/src/lib.rs | 2 +- 4 files changed, 173 insertions(+), 13 deletions(-) diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 7be9948..db7639e 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -264,10 +264,15 @@ pub async fn remove_folder( .into_iter() .find(|f| f.id == folder_id) .map(|f| PathBuf::from(f.path)); + // Collect thumbnail paths before the cascade delete removes the rows. + let thumb_paths = db::get_thumbnail_paths_for_folder(&conn, folder_id).unwrap_or_default(); db::delete_folder(&conn, folder_id).map_err(|e| e.to_string())?; if let Some(path) = folder_path { watcher.remove_folder(&path); } + for thumb in &thumb_paths { + let _ = std::fs::remove_file(thumb); + } Ok(()) } @@ -1192,6 +1197,9 @@ pub async fn delete_images_from_disk( for r in records.into_iter().filter(|r| id_set.contains(&r.id)) { if std::fs::remove_file(&r.path).is_ok() { succeeded_ids.push(r.id); + if let Some(thumb) = &r.thumbnail_path { + let _ = std::fs::remove_file(thumb); + } } } if !succeeded_ids.is_empty() { diff --git a/src-tauri/src/db.rs b/src-tauri/src/db.rs index b2c02ba..2123d05 100644 --- a/src-tauri/src/db.rs +++ b/src-tauri/src/db.rs @@ -1676,6 +1676,7 @@ pub fn search_tags_autocomplete( pub struct ImagePathRecord { pub id: i64, pub path: String, + pub thumbnail_path: Option, } pub fn get_all_image_paths( @@ -1683,19 +1684,69 @@ pub fn get_all_image_paths( folder_id: Option, ) -> Result> { let mut stmt = conn.prepare( - "SELECT id, path FROM images WHERE (?1 IS NULL OR folder_id = ?1) ORDER BY id", + "SELECT id, path, thumbnail_path FROM images WHERE (?1 IS NULL OR folder_id = ?1) ORDER BY id", )?; let rows = stmt .query_map(params![folder_id], |row| { Ok(ImagePathRecord { id: row.get(0)?, path: row.get(1)?, + thumbnail_path: row.get(2)?, }) })? .collect::>>()?; Ok(rows) } +/// Returns (image_id, thumbnail_path) for the given path. Used by the watcher +/// delete branch so it can clean up the thumbnail file in the same step. +pub fn get_image_id_and_thumbnail_by_path( + conn: &Connection, + path: &str, +) -> Result)>> { + let result = conn.query_row( + "SELECT id, thumbnail_path FROM images WHERE path = ?1", + params![path], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)), + ); + match result { + Ok(v) => Ok(Some(v)), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(e.into()), + } +} + +/// Returns all non-null thumbnail_path values for images in a folder. +/// Called before folder deletion so callers can remove the files from disk. +pub fn get_thumbnail_paths_for_folder( + conn: &Connection, + folder_id: i64, +) -> Result> { + let mut stmt = conn.prepare( + "SELECT thumbnail_path FROM images WHERE folder_id = ?1 AND thumbnail_path IS NOT NULL", + )?; + let rows = stmt + .query_map([folder_id], |row| row.get::<_, String>(0))? + .collect::>>()?; + Ok(rows) +} + +/// Updates a moved/renamed image's path and thumbnail_path in-place. +/// Pass `new_thumbnail_path = None` to clear it (triggers regeneration). +pub fn update_image_path( + conn: &Connection, + image_id: i64, + new_path: &str, + new_filename: &str, + new_thumbnail_path: Option<&str>, +) -> Result<()> { + conn.execute( + "UPDATE images SET path = ?2, filename = ?3, thumbnail_path = ?4 WHERE id = ?1", + params![image_id, new_path, new_filename, new_thumbnail_path], + )?; + Ok(()) +} + pub fn get_explore_tags( conn: &Connection, folder_id: Option, diff --git a/src-tauri/src/indexer.rs b/src-tauri/src/indexer.rs index 704c466..95bd5ae 100644 --- a/src-tauri/src/indexer.rs +++ b/src-tauri/src/indexer.rs @@ -1347,7 +1347,7 @@ impl WatcherHandle { /// - `recv()` when no events are pending — zero CPU /// - `recv_timeout(earliest_deadline)` when events are pending — wakes exactly /// when the soonest debounce window expires, no busy-polling -pub fn start_watcher(app: AppHandle, pool: DbPool) -> WatcherHandle { +pub fn start_watcher(app: AppHandle, pool: DbPool, thumb_dir: PathBuf) -> WatcherHandle { let (tx, rx) = std::sync::mpsc::channel::>(); let raw_watcher = notify::recommended_watcher(move |result| { @@ -1392,17 +1392,19 @@ pub fn start_watcher(app: AppHandle, pool: DbPool) -> WatcherHandle { std::thread::spawn(move || { // path → deadline: the earliest instant at which this path should be processed. let mut pending: HashMap = HashMap::new(); + // old_path → new_path for rename events that carry both sides atomically. + let mut pending_renames: Vec<(PathBuf, PathBuf)> = Vec::new(); loop { // Adaptive blocking: block forever when idle, wake at the earliest // deadline when events are queued. - let received = if pending.is_empty() { + let received = if pending.is_empty() && pending_renames.is_empty() { match rx.recv() { Ok(e) => Some(e), Err(_) => break, // sender dropped — app is shutting down } } else { - let earliest = pending.values().copied().min().unwrap(); // safe: non-empty + let earliest = pending.values().copied().min().unwrap_or(Instant::now()); let timeout = earliest.saturating_duration_since(Instant::now()); match rx.recv_timeout(timeout) { Ok(e) => Some(e), @@ -1415,17 +1417,40 @@ pub fn start_watcher(app: AppHandle, pool: DbPool) -> WatcherHandle { // Absorb incoming event — coalesces rapid writes into one deadline. if let Some(Ok(event)) = received { - use notify::EventKind; - // Skip pure access events (reads); they never change file content. + use notify::{EventKind, event::{ModifyKind, RenameMode}}; if !matches!(event.kind, EventKind::Access(_)) { - for path in event.paths { - if is_supported_media(&path) { - pending.insert(path, now + WATCHER_DEBOUNCE); + // Intercept atomic rename events (old + new path in one event). + // Handle these separately to preserve embeddings and thumbnails. + if matches!( + event.kind, + EventKind::Modify(ModifyKind::Name(RenameMode::Both)) + ) && event.paths.len() == 2 + { + let old = event.paths[0].clone(); + let new = event.paths[1].clone(); + if is_supported_media(&old) || is_supported_media(&new) { + // Remove either side from regular pending so it isn't + // processed as an independent delete/create. + pending.remove(&old); + pending.remove(&new); + pending_renames.push((old, new)); + } + } else { + for path in event.paths { + if is_supported_media(&path) { + pending.insert(path, now + WATCHER_DEBOUNCE); + } } } } } + // Process renames immediately — they are atomic filesystem operations + // and do not benefit from debouncing. + for (old_path, new_path) in pending_renames.drain(..) { + process_watcher_rename(&app, &pool, &folder_map_thread, &thumb_dir, &old_path, &new_path); + } + // Process all paths whose debounce window has expired. let ready: Vec = pending .iter() @@ -1496,11 +1521,14 @@ fn process_watcher_path( Err(e) => eprintln!("Watcher: commit error for {:?}: {}", path, e), } } else { - // File removed from disk — clean up DB row. + // File removed from disk — clean up DB row and thumbnail. let path_str = path.to_string_lossy(); - match db::get_image_id_by_path(&conn, &path_str) { - Ok(Some(image_id)) => { + match db::get_image_id_and_thumbnail_by_path(&conn, &path_str) { + Ok(Some((image_id, thumb_path))) => { if db::delete_images_by_ids(&conn, &[image_id]).is_ok() { + if let Some(thumb) = thumb_path { + let _ = std::fs::remove_file(thumb); + } db::update_folder_count(&conn, folder_id).ok(); let _ = app.emit("watcher-deleted", vec![image_id]); let _ = app.emit("folder-counts-changed", ()); @@ -1511,3 +1539,76 @@ fn process_watcher_path( } } } + +/// Handles a filesystem rename/move event where both the old and new paths are +/// known. Updates the DB row in-place (preserving the embedding) and renames +/// the thumbnail file to match the new path hash. +fn process_watcher_rename( + app: &AppHandle, + pool: &DbPool, + folder_map: &Arc>>, + thumb_dir: &Path, + old_path: &Path, + new_path: &Path, +) { + // Resolve folder_id from the old path first, fall back to new path. + let folder_id = { + let map = folder_map.lock().unwrap(); + map.iter() + .find(|(fp, _)| old_path.starts_with(fp.as_path()) || new_path.starts_with(fp.as_path())) + .map(|(_, &id)| id) + }; + let Some(folder_id) = folder_id else { return }; + + let conn = match pool.get() { + Ok(c) => c, + Err(e) => { + eprintln!("Watcher rename: DB pool error: {}", e); + return; + } + }; + + let old_path_str = old_path.to_string_lossy(); + let new_path_str = new_path.to_string_lossy(); + + let (image_id, old_thumb) = match db::get_image_id_and_thumbnail_by_path(&conn, &old_path_str) { + Ok(Some(record)) => record, + Ok(None) => { + // Not yet indexed under the old path — treat as a brand-new file. + drop(conn); + process_watcher_path(app, pool, folder_map, new_path); + return; + } + Err(e) => { + eprintln!("Watcher rename: DB lookup error: {}", e); + return; + } + }; + + // Compute new thumbnail path and attempt to rename the file on disk. + let new_thumb_path = crate::thumbnail::thumb_path(thumb_dir, &new_path_str); + let new_thumb_str = new_thumb_path.to_string_lossy().into_owned(); + let thumb_ok = old_thumb + .as_deref() + .map(|old| std::fs::rename(old, &new_thumb_path).is_ok()) + .unwrap_or(false); + // If rename failed (e.g. old thumb never existed), clear thumbnail_path so + // the thumbnail worker regenerates it. + let effective_thumb: Option<&str> = if thumb_ok { Some(&new_thumb_str) } else { None }; + + let new_filename = new_path + .file_name() + .and_then(|f| f.to_str()) + .unwrap_or(""); + + if let Err(e) = db::update_image_path(&conn, image_id, &new_path_str, new_filename, effective_thumb) { + eprintln!("Watcher rename: DB update error: {}", e); + return; + } + + // Emit the updated record so the frontend refreshes without a full reload. + match db::get_image_by_id(&conn, image_id) { + Ok(record) => emit_images(app, &IndexedImagesBatch { folder_id, images: vec![record] }), + Err(e) => eprintln!("Watcher rename: post-update fetch error: {}", e), + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 00ba11a..02346f3 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -74,7 +74,7 @@ pub fn run() { // indexer::start_caption_worker(app.handle().clone(), pool.clone(), app_dir.clone()); indexer::start_tagging_worker(app.handle().clone(), pool.clone(), app_dir.clone()); - let watcher_handle = indexer::start_watcher(app.handle().clone(), pool.clone()); + let watcher_handle = indexer::start_watcher(app.handle().clone(), pool.clone(), thumb_dir.clone()); app.manage(pool); app.manage(media_tools); From b89e7406e9a48292b6a01be30e9eca371a91b848 Mon Sep 17 00:00:00 2001 From: LyAhn Date: Tue, 9 Jun 2026 06:45:24 +0100 Subject: [PATCH 2/9] fix(settings): improve thumbnail cleanup UX with live state and time warning Clear stale stats during cleanup, show zero when done, and warn when the file count is large enough to take a few minutes. --- src/components/SettingsModal.tsx | 40 +++++++++++++++++++------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/components/SettingsModal.tsx b/src/components/SettingsModal.tsx index a00accc..a246357 100644 --- a/src/components/SettingsModal.tsx +++ b/src/components/SettingsModal.tsx @@ -666,31 +666,39 @@ export function SettingsModal() {

Orphaned files

- {thumbnailCleanupResult - ? thumbnailCleanupResult.deleted_count.toLocaleString() - : thumbnailInfo - ? thumbnailInfo.count.toLocaleString() - : "—"} + {cleaningThumbnails + ? "—" + : thumbnailCleanupResult + ? "0" + : thumbnailInfo + ? thumbnailInfo.count.toLocaleString() + : "—"}

Reclaimable

- {thumbnailCleanupResult - ? `−${thumbnailCleanupResult.freed_mb.toFixed(1)} MB freed` - : thumbnailInfo - ? `${thumbnailInfo.size_mb.toFixed(1)} MB` - : "—"} + {cleaningThumbnails + ? "—" + : thumbnailCleanupResult + ? "0 MB" + : thumbnailInfo + ? `${thumbnailInfo.size_mb.toFixed(1)} MB` + : "—"}

- {thumbnailCleanupResult - ? `Removed ${thumbnailCleanupResult.deleted_count.toLocaleString()} orphaned thumbnail${thumbnailCleanupResult.deleted_count === 1 ? "" : "s"}.` - : thumbnailInfo && thumbnailInfo.count === 0 - ? "No orphaned thumbnails found." - : "Remove thumbnails no longer associated with any indexed image."} + {cleaningThumbnails + ? "Scanning and removing orphaned thumbnails…" + : thumbnailCleanupResult + ? `Removed ${thumbnailCleanupResult.deleted_count.toLocaleString()} file${thumbnailCleanupResult.deleted_count === 1 ? "" : "s"}, freed ${thumbnailCleanupResult.freed_mb.toFixed(1)} MB.` + : thumbnailInfo && thumbnailInfo.count === 0 + ? "No orphaned thumbnails found." + : thumbnailInfo && thumbnailInfo.count > 1000 + ? "Remove thumbnails no longer associated with any indexed image. This may take a few minutes for large collections." + : "Remove thumbnails no longer associated with any indexed image."}

From a34d38d9d3cc087c4cd2c0027660a600f4ead934 Mon Sep 17 00:00:00 2001 From: LyAhn Date: Thu, 11 Jun 2026 06:28:16 +0100 Subject: [PATCH 3/9] feat: notification batching, per-folder mute, and global pause Debounce embedding/tagging completion notifications per folder (6s window) so rapid file additions from downloaders fire one notification instead of one per file. Add per-folder mute via the sidebar context menu and a global pause toggle in Settings > General, both persisted across restarts. --- src-tauri/src/commands.rs | 65 +++++++++++++++++ src-tauri/src/lib.rs | 4 + src/App.tsx | 4 + src/components/SettingsModal.tsx | 22 ++++++ src/components/Sidebar.tsx | 11 ++- src/store.ts | 121 ++++++++++++++++++++++++------- 6 files changed, 200 insertions(+), 27 deletions(-) diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index db7639e..14f8465 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -1852,3 +1852,68 @@ pub async fn cleanup_orphaned_thumbnails( freed_mb: freed_bytes as f64 / 1_048_576.0, }) } + +const MUTED_FOLDER_IDS_FILE: &str = "settings/muted_folder_ids.txt"; +const NOTIFICATIONS_PAUSED_FILE: &str = "settings/notifications_paused.txt"; + +#[tauri::command] +pub async fn get_muted_folder_ids(app: AppHandle) -> Result, String> { + let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?; + let path = app_dir.join(MUTED_FOLDER_IDS_FILE); + if !path.exists() { + return Ok(Vec::new()); + } + let content = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; + Ok(content + .trim() + .split(',') + .filter(|s| !s.is_empty()) + .filter_map(|s| s.parse::().ok()) + .collect()) +} + +#[derive(serde::Deserialize)] +pub struct SetMutedFolderIdsParams { + pub folder_ids: Vec, +} + +#[tauri::command] +pub async fn set_muted_folder_ids( + app: AppHandle, + params: SetMutedFolderIdsParams, +) -> Result<(), String> { + let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?; + let settings_dir = app_dir.join("settings"); + std::fs::create_dir_all(&settings_dir).map_err(|e| e.to_string())?; + let content = params + .folder_ids + .iter() + .map(|id| id.to_string()) + .collect::>() + .join(","); + std::fs::write(settings_dir.join("muted_folder_ids.txt"), content) + .map_err(|e| e.to_string()) +} + +#[tauri::command] +pub async fn get_notifications_paused(app: AppHandle) -> Result { + let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?; + let path = app_dir.join(NOTIFICATIONS_PAUSED_FILE); + if !path.exists() { + return Ok(false); + } + let content = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; + Ok(content.trim() == "true") +} + +#[tauri::command] +pub async fn set_notifications_paused(app: AppHandle, paused: bool) -> Result<(), String> { + let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?; + let settings_dir = app_dir.join("settings"); + std::fs::create_dir_all(&settings_dir).map_err(|e| e.to_string())?; + std::fs::write( + settings_dir.join("notifications_paused.txt"), + if paused { "true" } else { "false" }, + ) + .map_err(|e| e.to_string()) +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 02346f3..b0e3ccb 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -148,6 +148,10 @@ pub fn run() { commands::vacuum_database, commands::get_orphaned_thumbnails_info, commands::cleanup_orphaned_thumbnails, + commands::get_muted_folder_ids, + commands::set_muted_folder_ids, + commands::get_notifications_paused, + commands::set_notifications_paused, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/src/App.tsx b/src/App.tsx index 84595a2..409da27 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -18,11 +18,15 @@ export default function App() { const loadImages = useGalleryStore((state) => state.loadImages); const loadCaptionModelStatus = useGalleryStore((state) => state.loadCaptionModelStatus); const loadDuplicateScanCache = useGalleryStore((state) => state.loadDuplicateScanCache); + const loadMutedFolderIds = useGalleryStore((state) => state.loadMutedFolderIds); + const loadNotificationsPaused = useGalleryStore((state) => state.loadNotificationsPaused); const subscribeToProgress = useGalleryStore((state) => state.subscribeToProgress); const activeView = useGalleryStore((state) => state.activeView); useEffect(() => { void initializeNotifications(); + void loadMutedFolderIds(); + void loadNotificationsPaused(); loadFolders().then(() => { void loadBackgroundJobProgress(); void loadCaptionModelStatus(); diff --git a/src/components/SettingsModal.tsx b/src/components/SettingsModal.tsx index a246357..190344f 100644 --- a/src/components/SettingsModal.tsx +++ b/src/components/SettingsModal.tsx @@ -162,6 +162,8 @@ export function SettingsModal() { const clearTaggingJobs = useGalleryStore((state) => state.clearTaggingJobs); const clearTaggingJobsForFolders = useGalleryStore((state) => state.clearTaggingJobsForFolders); const openAppDataFolder = useGalleryStore((state) => state.openAppDataFolder); + const notificationsPaused = useGalleryStore((state) => state.notificationsPaused); + const setNotificationsPaused = useGalleryStore((state) => state.setNotificationsPaused); const getDatabaseInfo = useGalleryStore((state) => state.getDatabaseInfo); const vacuumDatabase = useGalleryStore((state) => state.vacuumDatabase); const getOrphanedThumbnailsInfo = useGalleryStore((state) => state.getOrphanedThumbnailsInfo); @@ -601,6 +603,26 @@ export function SettingsModal() { + +
+
+

Pause all notifications

+

Suppress all indexing notifications until re-enabled.

+
+ +
+
+ void; onRename: () => void; onReindex: () => void; onLocate: () => void; + onToggleMute: () => void; onRemove: () => void; }) { const ref = useRef(null); @@ -61,6 +65,7 @@ function FolderContextMenu({ > {item("Reindex", onReindex)} {item("Rename", onRename)} + {item(isMuted ? "Unmute notifications" : "Mute notifications", onToggleMute)} {folder.scan_error && item("Locate Folder", onLocate)}
{item("Remove from app", onRemove, true)} @@ -77,7 +82,9 @@ function FolderItem({ selected: boolean; progress: IndexProgress | undefined; }) { - const { selectFolder, removeFolder, reindexFolder, renameFolder, updateFolderPath } = useGalleryStore(); + const { selectFolder, removeFolder, reindexFolder, renameFolder, updateFolderPath, toggleMutedFolder } = useGalleryStore(); + const mutedFolderIds = useGalleryStore((state) => state.mutedFolderIds); + const isMuted = mutedFolderIds.includes(folder.id); const isIndexing = progress && !progress.done; const isMissing = !!folder.scan_error && !isIndexing; @@ -250,10 +257,12 @@ function FolderItem({ setContextMenu(null)} onRename={() => setRenaming(true)} onReindex={() => void reindexFolder(folder.id)} onLocate={() => void handleLocateFolder()} + onToggleMute={() => toggleMutedFolder(folder.id)} onRemove={() => setConfirmingRemoval(true)} /> )} diff --git a/src/store.ts b/src/store.ts index b4a0090..479533d 100644 --- a/src/store.ts +++ b/src/store.ts @@ -4,6 +4,11 @@ import { listen, UnlistenFn } from "@tauri-apps/api/event"; import { appDataDir, join } from "@tauri-apps/api/path"; import { notifyTaskComplete } from "./notifications"; +// Per-folder debounce timers for batching notifications. +// Keyed as `${folderId}:embedding` or `${folderId}:tagging`. +const notificationTimers = new Map>(); +const NOTIFICATION_DEBOUNCE_MS = 6000; + export interface Folder { id: number; path: string; @@ -288,6 +293,8 @@ interface GalleryState { settingsOpen: boolean; taggingQueueScope: TaggingQueueScope; taggingQueueFolderIds: number[]; + mutedFolderIds: number[]; + notificationsPaused: boolean; taggerModelStatus: TaggerModelStatus | null; taggerModelPreparing: boolean; @@ -360,6 +367,10 @@ interface GalleryState { loadTaggingQueueFolderIds: () => Promise; toggleTaggingQueueFolder: (folderId: number) => void; setTaggingQueueFolderIds: (folderIds: number[]) => void; + loadMutedFolderIds: () => Promise; + toggleMutedFolder: (folderId: number) => void; + loadNotificationsPaused: () => Promise; + setNotificationsPaused: (paused: boolean) => void; openAppDataFolder: () => Promise; getDatabaseInfo: () => Promise; vacuumDatabase: () => Promise; @@ -635,6 +646,8 @@ export const useGalleryStore = create((set, get) => ({ settingsOpen: false, taggingQueueScope: "all", taggingQueueFolderIds: [], + mutedFolderIds: [], + notificationsPaused: false, taggerModelStatus: null, taggerModelPreparing: false, @@ -1371,6 +1384,39 @@ export const useGalleryStore = create((set, get) => ({ void invoke("set_tagging_queue_folder_ids", { folder_ids: taggingQueueFolderIds }).catch(() => {}); }, + loadMutedFolderIds: async () => { + try { + const folderIds = await invoke("get_muted_folder_ids"); + set({ mutedFolderIds: folderIds }); + } catch { + // fall back to in-memory default + } + }, + + toggleMutedFolder: (folderId) => { + set((state) => { + const next = state.mutedFolderIds.includes(folderId) + ? state.mutedFolderIds.filter((id) => id !== folderId) + : [...state.mutedFolderIds, folderId]; + void invoke("set_muted_folder_ids", { folder_ids: next }).catch(() => {}); + return { mutedFolderIds: next }; + }); + }, + + loadNotificationsPaused: async () => { + try { + const paused = await invoke("get_notifications_paused"); + set({ notificationsPaused: paused }); + } catch { + // fall back to in-memory default + } + }, + + setNotificationsPaused: (paused) => { + set({ notificationsPaused: paused }); + void invoke("set_notifications_paused", { paused }).catch(() => {}); + }, + openAppDataFolder: async () => { await invoke("open_app_data_folder"); }, @@ -1659,11 +1705,14 @@ export const useGalleryStore = create((set, get) => ({ progress.total > 0 && progress.indexed >= progress.total ) { - const folderName = get().folders.find((folder) => folder.id === progress.folder_id)?.name; - void notifyTaskComplete( - "Folder scan complete", - folderName ? `${folderName} has finished scanning.` : "A folder has finished scanning.", - ); + const { notificationsPaused, mutedFolderIds } = get(); + if (!notificationsPaused && !mutedFolderIds.includes(progress.folder_id)) { + const folderName = get().folders.find((folder) => folder.id === progress.folder_id)?.name; + void notifyTaskComplete( + "Folder scan complete", + folderName ? `${folderName} has finished scanning.` : "A folder has finished scanning.", + ); + } } void get().loadFolders(); void get().loadBackgroundJobProgress(); @@ -1688,32 +1737,52 @@ export const useGalleryStore = create((set, get) => ({ const previous = previousProgress[progress.folder_id]; if (!previous) continue; + const { notificationsPaused, mutedFolderIds } = get(); + const suppressed = notificationsPaused || mutedFolderIds.includes(progress.folder_id); const folderName = get().folders.find((folder) => folder.id === progress.folder_id)?.name ?? "Folder"; - if (previous.embedding_pending > 0 && progress.embedding_pending === 0) { - const failureDetail = - progress.embedding_failed > 0 - ? ` ${progress.embedding_failed.toLocaleString()} failed.` - : ""; - void notifyTaskComplete( - "Embeddings complete", - `${folderName} finished generating embeddings.${failureDetail}`, - ); + // Embeddings — debounced so rapid file additions don't fire per-file. + const embeddingKey = `${progress.folder_id}:embedding`; + if (!suppressed) { + if (previous.embedding_pending > 0 && progress.embedding_pending === 0) { + clearTimeout(notificationTimers.get(embeddingKey)); + const failureDetail = + progress.embedding_failed > 0 + ? ` ${progress.embedding_failed.toLocaleString()} failed.` + : ""; + const body = `${folderName} finished generating embeddings.${failureDetail}`; + notificationTimers.set(embeddingKey, setTimeout(() => { + notificationTimers.delete(embeddingKey); + void notifyTaskComplete("Embeddings complete", body); + }, NOTIFICATION_DEBOUNCE_MS)); + } else if (previous.embedding_pending === 0 && progress.embedding_pending > 0) { + // More jobs queued — cancel the pending notification. + clearTimeout(notificationTimers.get(embeddingKey)); + notificationTimers.delete(embeddingKey); + } } - if (previous.tagging_pending > 0 && progress.tagging_pending === 0) { - const failureDetail = - progress.tagging_failed > 0 - ? ` ${progress.tagging_failed.toLocaleString()} failed.` - : ""; - void notifyTaskComplete( - "AI tagging complete", - `${folderName} finished generating tags.${failureDetail}`, - ); - // New tags are now in the DB — invalidate the Explore tag cache so - // reopening Explore reflects the updated tag distribution. - set({ exploreTagsFolderId: undefined }); + // Tagging — same debounce pattern. + const taggingKey = `${progress.folder_id}:tagging`; + if (!suppressed) { + if (previous.tagging_pending > 0 && progress.tagging_pending === 0) { + clearTimeout(notificationTimers.get(taggingKey)); + const failureDetail = + progress.tagging_failed > 0 + ? ` ${progress.tagging_failed.toLocaleString()} failed.` + : ""; + const body = `${folderName} finished generating tags.${failureDetail}`; + notificationTimers.set(taggingKey, setTimeout(() => { + notificationTimers.delete(taggingKey); + void notifyTaskComplete("AI tagging complete", body); + // New tags landed — invalidate Explore tag cache. + set({ exploreTagsFolderId: undefined }); + }, NOTIFICATION_DEBOUNCE_MS)); + } else if (previous.tagging_pending === 0 && progress.tagging_pending > 0) { + clearTimeout(notificationTimers.get(taggingKey)); + notificationTimers.delete(taggingKey); + } } } From 665c315f56d6354698f311b07b30cf090986e770 Mon Sep 17 00:00:00 2001 From: LyAhn Date: Fri, 12 Jun 2026 08:12:49 +0100 Subject: [PATCH 4/9] feat(duplicates): phased scan progress with skipped-file reporting Replace the [scanned, total] tuple progress event with a structured {phase, processed, total, skipped} payload covering all three scan phases (checking, hashing, confirming). find_duplicates now returns a DuplicateScanResult with scanned/candidate/skipped counts, and the UI surfaces phase labels plus a warning when unreadable files were skipped. Also adds a clean:app script for cargo clean. --- CLAUDE.md | 2 +- package.json | 1 + src-tauri/src/commands.rs | 142 +++++++++++++++++++++++++---- src/components/BackgroundTasks.tsx | 16 +++- src/components/DuplicateFinder.tsx | 17 +++- src/store.ts | 61 +++++++++++-- 6 files changed, 200 insertions(+), 39 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e9febb9..b5786c8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,7 +80,7 @@ Database: SQLite with WAL mode, stored in the Tauri app data directory as `galle | `media-updated` | `ThumbnailBatch` | | `caption-model-progress` | `CaptionModelProgress` | | `tagger-model-progress` | `TaggerModelProgress` | -| `duplicate_scan_progress` | `[scanned, total]` | +| `duplicate_scan_progress` | `{ phase, processed, total, skipped }` | ### Key types diff --git a/package.json b/package.json index 1836830..542a05d 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "build:app": "tauri build", "build:vite": "tsc && vite build", + "clean:app": "cd src-tauri && cargo clean", "dev:app": "tauri dev", "dev:vite": "vite", "preview": "vite preview", diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 14f8465..3451f92 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -182,6 +182,22 @@ pub struct DuplicateGroup { pub images: Vec, } +#[derive(Clone, Serialize)] +pub struct DuplicateScanProgress { + pub phase: String, + pub processed: usize, + pub total: usize, + pub skipped: usize, +} + +#[derive(Serialize)] +pub struct DuplicateScanResult { + pub groups: Vec, + pub scanned_files: usize, + pub candidate_files: usize, + pub skipped_files: usize, +} + #[derive(Serialize)] pub struct DuplicateScanCache { pub groups: Vec, @@ -962,14 +978,19 @@ pub async fn find_duplicates( app: AppHandle, db: State<'_, DbState>, folder_id: Option, -) -> Result, String> { +) -> Result { let records = { let conn = db.get().map_err(|e| e.to_string())?; db::get_all_image_paths(&conn, folder_id).map_err(|e| e.to_string())? }; let total = records.len(); - let _ = app.emit("duplicate_scan_progress", (0usize, total)); + let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "checking".to_string(), + processed: 0, + total, + skipped: 0, + }); // Three-phase detection. // @@ -986,7 +1007,8 @@ pub async fn find_duplicates( // For sample-hash groups that contain files > 64 KB, compute a full-file // hash to confirm the match before presenting them as deletable duplicates. let app_hash = app.clone(); - let pairs: Vec<(u64, u64, i64, String)> = tokio::task::spawn_blocking(move || { + let (pairs, skipped_before_confirm, candidate_total): + (Vec<(u64, u64, i64, String)>, usize, usize) = tokio::task::spawn_blocking(move || { use memmap2::Mmap; use rayon::prelude::*; use std::collections::HashMap; @@ -1012,14 +1034,31 @@ pub async fn find_duplicates( } // ── Phase 1: stat ───────────────────────────────────────────────── + let stat_counter = AtomicUsize::new(0); + let stat_skipped = AtomicUsize::new(0); let sized: Vec<(i64, String, u64)> = records .par_iter() .filter_map(|r| { - let size = std::fs::metadata(&r.path).ok()?.len(); - if size == 0 { return None; } - Some((r.id, r.path.clone(), size)) + let result = match std::fs::metadata(&r.path) { + Ok(metadata) => Some((r.id, r.path.clone(), metadata.len())), + Err(_) => { + stat_skipped.fetch_add(1, Ordering::Relaxed); + None + } + }; + let done = stat_counter.fetch_add(1, Ordering::Relaxed) + 1; + if done % 100 == 0 || done == total { + let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "checking".to_string(), + processed: done, + total, + skipped: stat_skipped.load(Ordering::Relaxed), + }); + } + result }) .collect(); + let stat_skipped = stat_skipped.load(Ordering::Relaxed); let mut by_size: HashMap> = HashMap::new(); for (i, (_, _, size)) in sized.iter().enumerate() { @@ -1032,29 +1071,53 @@ pub async fn find_duplicates( .collect(); if candidates.is_empty() { - return vec![]; + return (vec![], stat_skipped, 0); } // ── Phase 2: sample hash ────────────────────────────────────────── let c_total = candidates.len(); - let _ = app_hash.emit("duplicate_scan_progress", (0usize, c_total)); + let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "hashing".to_string(), + processed: 0, + total: c_total, + skipped: stat_skipped, + }); let counter = AtomicUsize::new(0); + let hash_skipped = AtomicUsize::new(0); - candidates + let pairs = candidates .par_iter() .filter_map(|&idx| { let (id, path, size) = &sized[idx]; - let file = std::fs::File::open(path).ok()?; - // SAFETY: read-only; no external truncation expected during scan. - let mmap = unsafe { Mmap::map(&file).ok()? }; - let hash = sample(&mmap); + let result = if *size == 0 { + Some((xxh3_64(&[]), *size, *id, path.clone())) + } else { + std::fs::File::open(path) + .ok() + // SAFETY: read-only; no external truncation expected during scan. + .and_then(|file| unsafe { Mmap::map(&file).ok() }) + .map(|mmap| (sample(&mmap), *size, *id, path.clone())) + }; + if result.is_none() { + hash_skipped.fetch_add(1, Ordering::Relaxed); + } let done = counter.fetch_add(1, Ordering::Relaxed) + 1; if done % 100 == 0 || done == c_total { - let _ = app_hash.emit("duplicate_scan_progress", (done, c_total)); + let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "hashing".to_string(), + processed: done, + total: c_total, + skipped: stat_skipped + hash_skipped.load(Ordering::Relaxed), + }); } - Some((hash, *size, *id, path.clone())) + result }) - .collect() + .collect(); + ( + pairs, + stat_skipped + hash_skipped.load(Ordering::Relaxed), + c_total, + ) }) .await .map_err(|e| e.to_string())?; @@ -1071,13 +1134,30 @@ pub async fn find_duplicates( // hash becomes its own DuplicateGroup so disjoint sets (A,A vs B,B that // happened to share size and sample hash) are never merged. const COVERED: u64 = (16 * 1024 * 4) as u64; - let confirmed: Vec<(u64, u64, Vec)> = + let confirming_total: usize = size_hash_map + .iter() + .filter(|((_, file_size), entries)| *file_size > COVERED && entries.len() > 1) + .map(|(_, entries)| entries.len()) + .sum(); + if confirming_total > 0 { + let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "confirming".to_string(), + processed: 0, + total: confirming_total, + skipped: skipped_before_confirm, + }); + } + let app_confirm = app.clone(); + let (confirmed, skipped_files): (Vec<(u64, u64, Vec)>, usize) = tokio::task::spawn_blocking(move || { use memmap2::Mmap; use rayon::prelude::*; + use std::sync::atomic::{AtomicUsize, Ordering}; use xxhash_rust::xxh3::xxh3_64; - size_hash_map + let counter = AtomicUsize::new(0); + let confirm_skipped = AtomicUsize::new(0); + let confirmed = size_hash_map .into_iter() .filter(|(_, entries)| entries.len() > 1) .flat_map(|((sample_hash, file_size), entries)| { @@ -1094,6 +1174,19 @@ pub async fn find_duplicates( .ok() .and_then(|f| unsafe { Mmap::map(&f).ok() }) .map(|mmap| xxh3_64(&mmap)); + if hash.is_none() { + confirm_skipped.fetch_add(1, Ordering::Relaxed); + } + let done = counter.fetch_add(1, Ordering::Relaxed) + 1; + if done % 100 == 0 || done == confirming_total { + let _ = app_confirm.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "confirming".to_string(), + processed: done, + total: confirming_total, + skipped: skipped_before_confirm + + confirm_skipped.load(Ordering::Relaxed), + }); + } (*id, hash) }) .collect(); @@ -1111,7 +1204,11 @@ pub async fn find_duplicates( .map(|(full_hash, ids)| (full_hash, file_size, ids)) .collect() }) - .collect() + .collect(); + ( + confirmed, + skipped_before_confirm + confirm_skipped.load(Ordering::Relaxed), + ) }) .await .map_err(|e| e.to_string())?; @@ -1142,7 +1239,12 @@ pub async fn find_duplicates( let _ = db::set_duplicate_scan_cache(&conn, &folder_scope, &json); } - Ok(groups) + Ok(DuplicateScanResult { + groups, + scanned_files: total, + candidate_files: candidate_total, + skipped_files, + }) } #[tauri::command] diff --git a/src/components/BackgroundTasks.tsx b/src/components/BackgroundTasks.tsx index d34ceb2..8930943 100644 --- a/src/components/BackgroundTasks.tsx +++ b/src/components/BackgroundTasks.tsx @@ -278,12 +278,16 @@ export function BackgroundTasks() { id: -1, name: "Duplicate Scan", stages: [{ - label: "Hashing", + label: duplicateScanProgress?.phase === "checking" + ? "Checking" + : duplicateScanProgress?.phase === "confirming" + ? "Confirming" + : "Hashing", detail: duplicateScanProgress - ? `${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}` + ? `${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}` : "Starting…", progress: duplicateScanProgress && duplicateScanProgress.total > 0 - ? (duplicateScanProgress.scanned / duplicateScanProgress.total) * 100 + ? (duplicateScanProgress.processed / duplicateScanProgress.total) * 100 : null, failed: false, }], @@ -310,7 +314,8 @@ export function BackgroundTasks() { const embeddingStage = primary.stages.find((s) => s.label === "Embeddings"); const taggingStage = primary.stages.find((s) => s.label === "Tags"); const scanningStage = primary.stages.find((s) => s.label === "Scanning"); - const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? null; + const duplicateStage = primary.id === -1 ? primary.stages[0] : null; + const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? duplicateStage?.progress ?? null; return (
@@ -434,7 +439,8 @@ export function BackgroundTasks() { const taskEmbeddingStage = task.stages.find((s) => s.label === "Embeddings"); const taskTaggingStage = task.stages.find((s) => s.label === "Tags"); const taskScanningStage = task.stages.find((s) => s.label === "Scanning"); - const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? null; + const taskDuplicateStage = task.id === -1 ? task.stages[0] : null; + const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? taskDuplicateStage?.progress ?? null; const taskHasFailed = (task.hasFailedEmbeddings || task.hasFailedTagging || task.hasFailedCaptions) && task.pendingMediaWork === 0; return ( diff --git a/src/components/DuplicateFinder.tsx b/src/components/DuplicateFinder.tsx index cae277f..67b361b 100644 --- a/src/components/DuplicateFinder.tsx +++ b/src/components/DuplicateFinder.tsx @@ -117,6 +117,7 @@ export function DuplicateFinder() { const duplicateScanning = useGalleryStore((state) => state.duplicateScanning); const duplicateScanProgress = useGalleryStore((state) => state.duplicateScanProgress); const duplicateScanError = useGalleryStore((state) => state.duplicateScanError); + const duplicateScanWarning = useGalleryStore((state) => state.duplicateScanWarning); const duplicateSelectedIds = useGalleryStore((state) => state.duplicateSelectedIds); const duplicateLastScanned = useGalleryStore((state) => state.duplicateLastScanned); const selectedFolderId = useGalleryStore((state) => state.selectedFolderId); @@ -152,8 +153,15 @@ export function DuplicateFinder() { const progressPercent = duplicateScanProgress && duplicateScanProgress.total > 0 - ? Math.round((duplicateScanProgress.scanned / duplicateScanProgress.total) * 100) + ? Math.round((duplicateScanProgress.processed / duplicateScanProgress.total) * 100) : 0; + const progressLabel = duplicateScanProgress + ? duplicateScanProgress.phase === "checking" + ? "Checking file sizes" + : duplicateScanProgress.phase === "hashing" + ? "Hashing duplicate candidates" + : "Confirming exact matches" + : null; return (
@@ -165,7 +173,7 @@ export function DuplicateFinder() {

{duplicateScanning ? duplicateScanProgress - ? `Scanning… ${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}` + ? `${progressLabel}… ${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}` : "Starting scan…" : hasResults ? `${duplicateGroups.length} group${duplicateGroups.length === 1 ? "" : "s"} · ${formatBytes(totalWasted)} reclaimable` @@ -232,6 +240,9 @@ export function DuplicateFinder() { {duplicateScanError ? (

{duplicateScanError}

) : null} + {duplicateScanWarning ? ( +

{duplicateScanWarning}

+ ) : null} {deleteResult ? (

{deleteResult}

) : null} @@ -241,7 +252,7 @@ export function DuplicateFinder() { {duplicateScanning && !hasResults ? (
- Hashing files… + {progressLabel ? `${progressLabel}…` : "Preparing scan…"}
) : !hasScanned ? (
diff --git a/src/store.ts b/src/store.ts index 479533d..1fd5eb6 100644 --- a/src/store.ts +++ b/src/store.ts @@ -170,6 +170,20 @@ export interface DuplicateGroup { images: ImageRecord[]; } +export interface DuplicateScanProgress { + phase: "checking" | "hashing" | "confirming"; + processed: number; + total: number; + skipped: number; +} + +interface DuplicateScanResult { + groups: DuplicateGroup[]; + scanned_files: number; + candidate_files: number; + skipped_files: number; +} + export interface SimilarImagesPage { images: ImageRecord[]; offset: number; @@ -308,8 +322,9 @@ interface GalleryState { duplicateGroups: DuplicateGroup[]; duplicateScanning: boolean; - duplicateScanProgress: { scanned: number; total: number } | null; + duplicateScanProgress: DuplicateScanProgress | null; duplicateScanError: string | null; + duplicateScanWarning: string | null; duplicateSelectedIds: Set; duplicateLastScanned: number | null; // Unix timestamp (seconds) duplicateScanFolderId: number | null | undefined; // undefined = never scanned @@ -663,6 +678,7 @@ export const useGalleryStore = create((set, get) => ({ duplicateScanning: false, duplicateScanProgress: null, duplicateScanError: null, + duplicateScanWarning: null, duplicateSelectedIds: new Set(), duplicateLastScanned: null, duplicateScanFolderId: undefined, @@ -954,7 +970,13 @@ export const useGalleryStore = create((set, get) => ({ if (activeView === "duplicates") { const { selectedFolderId, duplicateScanFolderId } = get(); if (duplicateScanFolderId !== selectedFolderId) { - set({ activeView, duplicateGroups: [], duplicateLastScanned: null, duplicateScanFolderId: undefined }); + set({ + activeView, + duplicateGroups: [], + duplicateLastScanned: null, + duplicateScanFolderId: undefined, + duplicateScanWarning: null, + }); void get().loadDuplicateScanCache(selectedFolderId); return; } @@ -1583,23 +1605,42 @@ export const useGalleryStore = create((set, get) => ({ interface CacheResult { groups: DuplicateGroup[]; scanned_at: number } const cached = await invoke("load_duplicate_scan_cache", { folderId: folderId ?? null }); if (cached) { - set({ duplicateGroups: cached.groups, duplicateLastScanned: cached.scanned_at, duplicateScanFolderId: folderId }); + set({ + duplicateGroups: cached.groups, + duplicateLastScanned: cached.scanned_at, + duplicateScanFolderId: folderId, + duplicateScanWarning: null, + }); } }, scanDuplicates: async (folderId = null) => { const { listen } = await import("@tauri-apps/api/event"); - set({ duplicateScanning: true, duplicateGroups: [], duplicateScanProgress: null, duplicateScanError: null, duplicateSelectedIds: new Set() }); - const unlisten = await listen<[number, number]>("duplicate_scan_progress", (event) => { - const [scanned, total] = event.payload; - set({ duplicateScanProgress: { scanned, total } }); + set({ + duplicateScanning: true, + duplicateGroups: [], + duplicateScanProgress: null, + duplicateScanError: null, + duplicateScanWarning: null, + duplicateSelectedIds: new Set(), + }); + const unlisten = await listen("duplicate_scan_progress", (event) => { + set({ duplicateScanProgress: event.payload }); }); try { - const groups = await invoke("find_duplicates", { folderId: folderId ?? null }); - set({ duplicateGroups: groups, duplicateLastScanned: Math.floor(Date.now() / 1000), duplicateScanFolderId: folderId }); + const result = await invoke("find_duplicates", { folderId: folderId ?? null }); + const warning = result.skipped_files > 0 + ? `${result.skipped_files.toLocaleString()} file${result.skipped_files === 1 ? "" : "s"} could not be read and were skipped.` + : null; + set({ + duplicateGroups: result.groups, + duplicateLastScanned: Math.floor(Date.now() / 1000), + duplicateScanFolderId: folderId, + duplicateScanWarning: warning, + }); void notifyTaskComplete( "Duplicate scan complete", - groups.length === 1 ? "Found 1 duplicate group." : `Found ${groups.length.toLocaleString()} duplicate groups.`, + `${result.groups.length === 1 ? "Found 1 duplicate group." : `Found ${result.groups.length.toLocaleString()} duplicate groups.`}${warning ? ` ${warning}` : ""}`, ); } catch (e) { set({ duplicateScanError: String(e) }); From a4486547e8b5881f292af1b84dc8e63697afda6f Mon Sep 17 00:00:00 2001 From: LyAhn Date: Fri, 12 Jun 2026 08:13:12 +0100 Subject: [PATCH 5/9] perf(thumbnails): scaled JPEG decode and continuous worker batching Major speedups for large-folder thumbnail generation: - JPEGs decode through mozjpeg at the smallest DCT scale covering 320px instead of full resolution, with catch_unwind and a fallback to the image crate for anything mozjpeg rejects. EXIF orientation preserved in both paths. - RGB8 pipeline end to end (no RGBA round-trip) and CatmullRom resize in place of Lanczos3. - Video posters grab the seeked frame directly instead of running the ffmpeg thumbnail filter (which decoded ~100 frames), with decode threads capped at 2 per process. - Workers only sleep when the queue is empty rather than 250ms after every batch. Image batches commit before videos start, and videos run sequentially off the rayon pool (blocking ffmpeg waits were starving image decode) with per-item commits so progress keeps moving through slow stretches. --- src-tauri/Cargo.lock | 63 +++++++++++++++++ src-tauri/Cargo.toml | 5 ++ src-tauri/src/indexer.rs | 74 ++++++++++++-------- src-tauri/src/thumbnail.rs | 136 +++++++++++++++++++++++++++++++++---- 4 files changed, 236 insertions(+), 42 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 5e8dbe0..df4ef2c 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -143,6 +143,12 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-broadcast" version = "0.7.2" @@ -635,6 +641,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -3108,6 +3116,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.94" @@ -3559,6 +3577,31 @@ dependencies = [ "pxfm", ] +[[package]] +name = "mozjpeg" +version = "0.10.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7891b80aaa86097d38d276eb98b3805d6280708c4e0a1e6f6aed9380c51fec9" +dependencies = [ + "arrayvec", + "bytemuck", + "libc", + "mozjpeg-sys", + "rgb", +] + +[[package]] +name = "mozjpeg-sys" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f0dc668bf9bf888c88e2fb1ab16a406d2c380f1d082b20d51dd540ab2aa70c1" +dependencies = [ + "cc", + "dunce", + "libc", + "nasm-rs", +] + [[package]] name = "muda" version = "0.17.2" @@ -3586,6 +3629,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af" +[[package]] +name = "nasm-rs" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149" +dependencies = [ + "jobserver", + "log", +] + [[package]] name = "native-tls" version = "0.2.18" @@ -4388,6 +4441,7 @@ dependencies = [ "kamadak-exif", "log", "memmap2", + "mozjpeg", "notify", "ort", "r2d2", @@ -5102,6 +5156,15 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.17.14" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 6c201c7..84a8156 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -53,6 +53,7 @@ csv = "1" kamadak-exif = "0.5" notify = "6" tauri-plugin-notification = "2" +mozjpeg = "0.10.13" # ── Dev-mode performance ──────────────────────────────────────────────────── # opt-level=1 on the main crate keeps incremental compile short. @@ -81,6 +82,10 @@ opt-level = 3 opt-level = 3 [profile.dev.package.fast_image_resize] opt-level = 3 +[profile.dev.package.mozjpeg] +opt-level = 3 +[profile.dev.package.mozjpeg-sys] +opt-level = 3 # Parallel work scheduler [profile.dev.package.rayon] diff --git a/src-tauri/src/indexer.rs b/src-tauri/src/indexer.rs index 95bd5ae..35583c4 100644 --- a/src-tauri/src/indexer.rs +++ b/src-tauri/src/indexer.rs @@ -204,10 +204,16 @@ pub fn start_thumbnail_worker( cache_dir: PathBuf, ) { std::thread::spawn(move || loop { - if let Err(error) = process_thumbnail_batch(&app, &pool, &media_tools, &cache_dir) { - eprintln!("Thumbnail worker error: {}", error); + // Only back off when the queue is empty (or errored); while jobs are + // pending, claim the next batch immediately. + match process_thumbnail_batch(&app, &pool, &media_tools, &cache_dir) { + Ok(true) => {} + Ok(false) => std::thread::sleep(std::time::Duration::from_millis(250)), + Err(error) => { + eprintln!("Thumbnail worker error: {}", error); + std::thread::sleep(std::time::Duration::from_millis(250)); + } } - std::thread::sleep(std::time::Duration::from_millis(250)); }); } @@ -554,12 +560,14 @@ fn commit_batch(pool: &DbPool, records: &[ImageRecord]) -> Result Result<()> { +) -> Result { let jobs = { with_db_write_lock(|| { let mut conn = pool.get()?; @@ -578,7 +586,7 @@ fn process_thumbnail_batch( }; if jobs.is_empty() { - return Ok(()); + return Ok(false); } println!("Thumbnail batch claimed: {} items", jobs.len()); @@ -586,33 +594,39 @@ fn process_thumbnail_batch( let (image_jobs, video_jobs): (Vec<_>, Vec<_>) = jobs.into_iter().partition(|job| job.media_kind == "image"); - let mut results = image_jobs - .par_iter() - .map(|job| { - ( - job.image_id, - if job.media_kind == "image" { - thumbnail::generate_image_thumbnail(Path::new(&job.path), cache_dir).map(Some) - } else { - thumbnail::generate_video_thumbnail( - media_tools, - Path::new(&job.path), - cache_dir, - ) - .map(Some) - }, - ) - }) - .collect::>(); - - for job in video_jobs { - results.push(( - job.image_id, - thumbnail::generate_video_thumbnail(media_tools, Path::new(&job.path), cache_dir) - .map(Some), - )); + // Images: parallel decode, committed as one batch. + if !image_jobs.is_empty() { + let results = image_jobs + .par_iter() + .map(|job| { + ( + job.image_id, + thumbnail::generate_image_thumbnail(Path::new(&job.path), cache_dir).map(Some), + ) + }) + .collect::>(); + persist_thumbnail_results(app, pool, results)?; } + // Videos: sequential, off the rayon pool — each ffmpeg call blocks its + // thread, and a video-heavy batch on the shared pool would starve image + // decoding across all workers. Committed per item so progress keeps + // moving through slow stretches. + for job in &video_jobs { + let result = + thumbnail::generate_video_thumbnail(media_tools, Path::new(&job.path), cache_dir) + .map(Some); + persist_thumbnail_results(app, pool, vec![(job.image_id, result)])?; + } + + Ok(true) +} + +fn persist_thumbnail_results( + app: &AppHandle, + pool: &DbPool, + results: Vec<(i64, anyhow::Result>)>, +) -> Result<()> { let updated_images = { with_db_write_lock(|| { let mut conn = pool.get()?; diff --git a/src-tauri/src/thumbnail.rs b/src-tauri/src/thumbnail.rs index 4efcf2f..ce74bfb 100644 --- a/src-tauri/src/thumbnail.rs +++ b/src-tauri/src/thumbnail.rs @@ -31,29 +31,26 @@ pub fn generate_image_thumbnail(image_path: &Path, cache_dir: &Path) -> Result Result Result { + if is_jpeg(image_path) { + if let Some(img) = decode_jpeg_scaled(image_path) { + return Ok(img); + } + } + + let reader = image::ImageReader::open(image_path)?.with_guessed_format()?; + let mut decoder = reader.into_decoder()?; + let orientation = decoder.orientation()?; + let mut img = image::DynamicImage::from_decoder(decoder)?; + img.apply_orientation(orientation); + Ok(img) +} + +fn is_jpeg(path: &Path) -> bool { + path.extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext.eq_ignore_ascii_case("jpg") || ext.eq_ignore_ascii_case("jpeg")) +} + +/// Decodes a JPEG at the smallest DCT scale that still covers `THUMB_SIZE`. +/// Returns `None` on any failure (corrupt file, CMYK without conversion +/// support, etc.) so the caller can fall back to the generic decoder. +/// mozjpeg reports errors by panicking, hence the `catch_unwind`. +fn decode_jpeg_scaled(image_path: &Path) -> Option { + let path = image_path.to_path_buf(); + let decoded = std::panic::catch_unwind(move || -> std::io::Result<(Vec, u32, u32)> { + let mut decompress = mozjpeg::Decompress::new_path(&path)?; + let (width, height) = decompress.size(); + decompress.scale(scale_numerator(width.max(height))); + let mut started = decompress.rgb()?; + let (out_width, out_height) = (started.width() as u32, started.height() as u32); + let pixels = started.read_scanlines::()?; + started.finish()?; + Ok((pixels, out_width, out_height)) + }) + .ok()? + .ok()?; + + let (pixels, width, height) = decoded; + let buffer = image::RgbImage::from_raw(width, height, pixels)?; + let mut img = image::DynamicImage::ImageRgb8(buffer); + if let Some(orientation) = exif_orientation(image_path) { + img.apply_orientation(orientation); + } + Some(img) +} + +/// Smallest numerator (of /8) that keeps the longest side at or above +/// `THUMB_SIZE`, so the subsequent resize never upscales. +fn scale_numerator(max_dimension: usize) -> u8 { + for numerator in 1..=8u8 { + if max_dimension * numerator as usize / 8 >= THUMB_SIZE as usize { + return numerator; + } + } + 8 +} + +fn exif_orientation(path: &Path) -> Option { + let file = std::fs::File::open(path).ok()?; + let mut reader = std::io::BufReader::new(file); + let exif = exif::Reader::new().read_from_container(&mut reader).ok()?; + let field = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY)?; + let value = field.value.get_uint(0)?; + image::metadata::Orientation::from_exif(value as u8) +} + pub fn generate_video_thumbnail( tools: &MediaTools, video_path: &Path, @@ -88,6 +159,8 @@ pub fn generate_video_thumbnail( let attempts: [&[&str]; 3] = [ &[ "-y", + "-threads", + "2", "-ss", "00:00:00.000", "-i", @@ -95,13 +168,15 @@ pub fn generate_video_thumbnail( "-frames:v", "1", "-vf", - "thumbnail,scale=320:-1:force_original_aspect_ratio=decrease", + "scale=320:-1:force_original_aspect_ratio=decrease", "-q:v", "4", &output_path, ], &[ "-y", + "-threads", + "2", "-ss", "00:00:00.250", "-i", @@ -109,19 +184,21 @@ pub fn generate_video_thumbnail( "-frames:v", "1", "-vf", - "thumbnail,scale=320:-1:force_original_aspect_ratio=decrease", + "scale=320:-1:force_original_aspect_ratio=decrease", "-q:v", "4", &output_path, ], &[ "-y", + "-threads", + "2", "-i", path_str.as_ref(), "-frames:v", "1", "-vf", - "thumbnail,scale=320:-1:force_original_aspect_ratio=decrease", + "scale=320:-1:force_original_aspect_ratio=decrease", "-q:v", "4", &output_path, @@ -165,6 +242,41 @@ fn hash_path(s: &str) -> String { format!("{:016x}", xxhash_rust::xxh3::xxh3_64(s.as_bytes())) } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scale_numerator_picks_smallest_sufficient() { + assert_eq!(scale_numerator(6000), 1); + assert_eq!(scale_numerator(640), 4); + assert_eq!(scale_numerator(320), 8); + assert_eq!(scale_numerator(100), 8); + } + + #[test] + fn jpeg_fast_path_decodes_at_reduced_resolution() { + let dir = std::env::temp_dir().join("phokus-thumb-test"); + std::fs::create_dir_all(&dir).unwrap(); + let src_path = dir.join("large.jpg"); + let img = + image::RgbImage::from_fn(1600, 1200, |x, _| image::Rgb([(x % 256) as u8, 64, 128])); + img.save(&src_path).unwrap(); + + // 1600 max dim -> numerator 2 -> 400x300 decode output + let decoded = decode_jpeg_scaled(&src_path).expect("fast path should handle plain JPEG"); + assert_eq!((decoded.width(), decoded.height()), (400, 300)); + + let cache = dir.join("cache"); + let _ = std::fs::remove_file(thumb_path(&cache, &src_path.to_string_lossy())); + let result = generate_image_thumbnail(&src_path, &cache).unwrap(); + assert_eq!(result.width, Some(1600)); + assert_eq!(result.height, Some(1200)); + let (thumb_w, thumb_h) = image::image_dimensions(&result.path).unwrap(); + assert_eq!((thumb_w, thumb_h), (320, 240)); + } +} + fn fit_dimensions(width: u32, height: u32, max_size: u32) -> (u32, u32) { if width <= max_size && height <= max_size { return (width.max(1), height.max(1)); From 334ac54e003505ff53e43d99d933db7a68b73178 Mon Sep 17 00:00:00 2001 From: LyAhn Date: Fri, 12 Jun 2026 09:00:25 +0100 Subject: [PATCH 6/9] perf(workers): scaled embedding preprocessing and idle-only backoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Embedding and tagging workers now sleep their backoff interval (500ms/750ms) only when the queue is empty, the model is not ready, or a batch errored — previously every batch paid the sleep even with work pending. - CLIP preprocessing no longer decodes originals at full resolution: decode_for_thumbnail is generalized into decode_image_scaled with a cover mode (shortest side >= target) and the embedder decodes JPEGs at the smallest DCT scale covering its 224px fill-crop, in parallel via rayon. The forward pass, not image decode, is now the dominant embedding cost. - EXIF orientation is now applied before embedding, so rotated photos embed the way they are displayed (previously embedded sideways). Existing stored embeddings are unaffected. --- src-tauri/src/embedder.rs | 17 +++++++----- src-tauri/src/indexer.rs | 46 ++++++++++++++++++++----------- src-tauri/src/thumbnail.rs | 56 ++++++++++++++++++++++++++++---------- 3 files changed, 81 insertions(+), 38 deletions(-) diff --git a/src-tauri/src/embedder.rs b/src-tauri/src/embedder.rs index 3a977c7..f2a7551 100644 --- a/src-tauri/src/embedder.rs +++ b/src-tauri/src/embedder.rs @@ -191,9 +191,11 @@ fn resolve_device() -> Result { } fn load_image(path: &Path, image_size: usize) -> Result { - let image = image::ImageReader::open(path)? - .with_guessed_format()? - .decode()?; + // Scaled decode: CLIP only needs image_size² pixels, so decoding a large + // JPEG at full resolution is wasted work. Cover mode keeps the shortest + // side at or above image_size for the fill-crop below. Also applies EXIF + // orientation, so rotated photos embed the way they are displayed. + let image = crate::thumbnail::decode_image_scaled(path, image_size as u32, true)?; let image = image.resize_to_fill( image_size as u32, image_size as u32, @@ -208,10 +210,11 @@ fn load_image(path: &Path, image_size: usize) -> Result { } fn load_images(paths: &[PathBuf], image_size: usize) -> Result { - let mut images = Vec::with_capacity(paths.len()); - for path in paths { - images.push(load_image(path, image_size)?); - } + use rayon::prelude::*; + let images = paths + .par_iter() + .map(|path| load_image(path, image_size)) + .collect::>>()?; Ok(Tensor::stack(&images, 0)?) } diff --git a/src-tauri/src/indexer.rs b/src-tauri/src/indexer.rs index 35583c4..440dd7e 100644 --- a/src-tauri/src/indexer.rs +++ b/src-tauri/src/indexer.rs @@ -231,10 +231,16 @@ pub fn start_embedding_worker(app: AppHandle, pool: DbPool) { let mut embedder: Option = None; println!("Embedding worker started."); loop { - if let Err(error) = process_embedding_batch(&app, &pool, &mut embedder) { - eprintln!("Embedding worker error: {}", error); + // Only back off when the queue is empty (or errored); while jobs + // are pending, claim the next batch immediately. + match process_embedding_batch(&app, &pool, &mut embedder) { + Ok(true) => {} + Ok(false) => std::thread::sleep(std::time::Duration::from_millis(500)), + Err(error) => { + eprintln!("Embedding worker error: {}", error); + std::thread::sleep(std::time::Duration::from_millis(500)); + } } - std::thread::sleep(std::time::Duration::from_millis(500)); } }); } @@ -270,13 +276,17 @@ pub fn start_tagging_worker(app: AppHandle, pool: DbPool, app_data_dir: PathBuf) println!("Tagging worker: acceleration setting changed — resetting session."); tagger_instance = None; } - if let Err(error) = - process_tagging_batch(&app, &pool, &app_data_dir, &mut tagger_instance) - { - eprintln!("Tagging worker error: {}", error); - tagger_instance = None; + // Only back off when the queue is empty (or errored); while jobs + // are pending, claim the next batch immediately. + match process_tagging_batch(&app, &pool, &app_data_dir, &mut tagger_instance) { + Ok(true) => {} + Ok(false) => std::thread::sleep(std::time::Duration::from_millis(750)), + Err(error) => { + eprintln!("Tagging worker error: {}", error); + tagger_instance = None; + std::thread::sleep(std::time::Duration::from_millis(750)); + } } - std::thread::sleep(std::time::Duration::from_millis(750)); } }); } @@ -759,11 +769,13 @@ fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaToo Ok(()) } +/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if +/// the queue was empty. fn process_embedding_batch( app: &AppHandle, pool: &DbPool, embedder: &mut Option, -) -> Result<()> { +) -> Result { let batch_started_at = Instant::now(); let claim_started_at = Instant::now(); let paused_folders = paused_folder_ids("embedding"); @@ -774,7 +786,7 @@ fn process_embedding_batch( let claim_elapsed = claim_started_at.elapsed(); if jobs.is_empty() { - return Ok(()); + return Ok(false); } if embedder.is_none() { @@ -902,7 +914,7 @@ fn process_embedding_batch( EMBEDDING_BATCH_SIZE, claim_elapsed, infer_elapsed, write_elapsed, batch_elapsed ); - Ok(()) + Ok(true) } fn process_caption_batch( @@ -1007,14 +1019,16 @@ fn process_caption_batch( Ok(()) } +/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if +/// the queue was empty or the model is not ready. fn process_tagging_batch( app: &AppHandle, pool: &DbPool, app_data_dir: &Path, tagger_instance: &mut Option, -) -> Result<()> { +) -> Result { if !tagger::tagger_model_status(app_data_dir).ready { - return Ok(()); + return Ok(false); } let paused_folders = paused_folder_ids("tagging"); @@ -1025,7 +1039,7 @@ fn process_tagging_batch( })?; if jobs.is_empty() { - return Ok(()); + return Ok(false); } if tagger_instance.is_none() { @@ -1133,7 +1147,7 @@ fn process_tagging_batch( emit_folder_job_progress(app, pool, &folder_ids.into_iter().collect::>(), true); } - Ok(()) + Ok(true) } fn active_indexing_folders() -> HashSet { diff --git a/src-tauri/src/thumbnail.rs b/src-tauri/src/thumbnail.rs index ce74bfb..8b48765 100644 --- a/src-tauri/src/thumbnail.rs +++ b/src-tauri/src/thumbnail.rs @@ -62,13 +62,26 @@ pub fn generate_image_thumbnail(image_path: &Path, cache_dir: &Path) -> Result Result { + decode_image_scaled(image_path, THUMB_SIZE, false) +} + +/// Decodes an image at reduced resolution, with EXIF orientation applied. /// /// JPEGs take a fast path that decodes at reduced resolution (DCT scaling), /// which skips most of the IDCT/upsampling work for large photos. Anything -/// that path can't handle falls back to the `image` crate. -fn decode_for_thumbnail(image_path: &Path) -> Result { +/// that path can't handle falls back to the `image` crate at full size. +/// +/// `cover` selects which side must stay at or above `target`: `false` keeps +/// the longest side (for aspect-fit consumers), `true` keeps the shortest +/// side (for fill-crop consumers like the CLIP preprocessor). +pub fn decode_image_scaled( + image_path: &Path, + target: u32, + cover: bool, +) -> Result { if is_jpeg(image_path) { - if let Some(img) = decode_jpeg_scaled(image_path) { + if let Some(img) = decode_jpeg_scaled(image_path, target, cover) { return Ok(img); } } @@ -87,16 +100,21 @@ fn is_jpeg(path: &Path) -> bool { .is_some_and(|ext| ext.eq_ignore_ascii_case("jpg") || ext.eq_ignore_ascii_case("jpeg")) } -/// Decodes a JPEG at the smallest DCT scale that still covers `THUMB_SIZE`. +/// Decodes a JPEG at the smallest DCT scale that still covers `target`. /// Returns `None` on any failure (corrupt file, CMYK without conversion /// support, etc.) so the caller can fall back to the generic decoder. /// mozjpeg reports errors by panicking, hence the `catch_unwind`. -fn decode_jpeg_scaled(image_path: &Path) -> Option { +fn decode_jpeg_scaled(image_path: &Path, target: u32, cover: bool) -> Option { let path = image_path.to_path_buf(); let decoded = std::panic::catch_unwind(move || -> std::io::Result<(Vec, u32, u32)> { let mut decompress = mozjpeg::Decompress::new_path(&path)?; let (width, height) = decompress.size(); - decompress.scale(scale_numerator(width.max(height))); + let reference = if cover { + width.min(height) + } else { + width.max(height) + }; + decompress.scale(scale_numerator(reference, target)); let mut started = decompress.rgb()?; let (out_width, out_height) = (started.width() as u32, started.height() as u32); let pixels = started.read_scanlines::()?; @@ -115,11 +133,11 @@ fn decode_jpeg_scaled(image_path: &Path) -> Option { Some(img) } -/// Smallest numerator (of /8) that keeps the longest side at or above -/// `THUMB_SIZE`, so the subsequent resize never upscales. -fn scale_numerator(max_dimension: usize) -> u8 { +/// Smallest numerator (of /8) that keeps `reference` at or above `target`, +/// so the subsequent resize never upscales. +fn scale_numerator(reference: usize, target: u32) -> u8 { for numerator in 1..=8u8 { - if max_dimension * numerator as usize / 8 >= THUMB_SIZE as usize { + if reference * numerator as usize / 8 >= target as usize { return numerator; } } @@ -248,10 +266,12 @@ mod tests { #[test] fn scale_numerator_picks_smallest_sufficient() { - assert_eq!(scale_numerator(6000), 1); - assert_eq!(scale_numerator(640), 4); - assert_eq!(scale_numerator(320), 8); - assert_eq!(scale_numerator(100), 8); + assert_eq!(scale_numerator(6000, THUMB_SIZE), 1); + assert_eq!(scale_numerator(640, THUMB_SIZE), 4); + assert_eq!(scale_numerator(320, THUMB_SIZE), 8); + assert_eq!(scale_numerator(100, THUMB_SIZE), 8); + // Cover mode reference: shortest side must reach 224 for CLIP. + assert_eq!(scale_numerator(1200, 224), 2); } #[test] @@ -264,9 +284,15 @@ mod tests { img.save(&src_path).unwrap(); // 1600 max dim -> numerator 2 -> 400x300 decode output - let decoded = decode_jpeg_scaled(&src_path).expect("fast path should handle plain JPEG"); + let decoded = decode_jpeg_scaled(&src_path, THUMB_SIZE, false) + .expect("fast path should handle plain JPEG"); assert_eq!((decoded.width(), decoded.height()), (400, 300)); + // Cover mode: shortest side (1200) must stay >= 224 -> numerator 2 + let covered = decode_jpeg_scaled(&src_path, 224, true) + .expect("fast path should handle plain JPEG"); + assert_eq!((covered.width(), covered.height()), (400, 300)); + let cache = dir.join("cache"); let _ = std::fs::remove_file(thumb_path(&cache, &src_path.to_string_lossy())); let result = generate_image_thumbnail(&src_path, &cache).unwrap(); From 948a489a8a9ef31a6af7e5cc451af7bb6e936972 Mon Sep 17 00:00:00 2001 From: LyAhn Date: Fri, 12 Jun 2026 10:36:05 +0100 Subject: [PATCH 7/9] fix(indexer): keep embedding and tagging off folders mid-scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Embedding and tagging claims now exclude actively-indexing folders, matching the thumbnail and metadata workers. Previously the embedding worker started processing a folder while it was still being scanned, competing with the scanner for CPU (shared rayon pool), disk, and the DB writer — slowing scans and skewing the adaptive storage profile toward Conservative. Video embedding jobs claimed mid-scan also fail-fasted pointlessly since their thumbnails are deferred until indexing completes; the existing backfill at scan end covers requeue. --- src-tauri/src/indexer.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src-tauri/src/indexer.rs b/src-tauri/src/indexer.rs index 440dd7e..1b5fc15 100644 --- a/src-tauri/src/indexer.rs +++ b/src-tauri/src/indexer.rs @@ -778,10 +778,15 @@ fn process_embedding_batch( ) -> Result { let batch_started_at = Instant::now(); let claim_started_at = Instant::now(); - let paused_folders = paused_folder_ids("embedding"); + // Exclude folders that are actively indexing (matching the thumbnail and + // metadata workers): embedding mid-scan competes with the scanner for + // CPU, disk, and the DB writer, and video jobs would fail-fast anyway + // because their thumbnails are deferred until indexing completes. + let mut excluded_folders = paused_folder_ids("embedding"); + excluded_folders.extend(active_indexing_folders()); let jobs = with_db_write_lock(|| { let mut conn = pool.get()?; - db::claim_embedding_jobs(&mut conn, &paused_folders, EMBEDDING_BATCH_SIZE) + db::claim_embedding_jobs(&mut conn, &excluded_folders, EMBEDDING_BATCH_SIZE) })?; let claim_elapsed = claim_started_at.elapsed(); @@ -1031,11 +1036,14 @@ fn process_tagging_batch( return Ok(false); } - let paused_folders = paused_folder_ids("tagging"); + // Exclude actively-indexing folders for the same reason as the other + // workers: don't compete with a running scan. + let mut excluded_folders = paused_folder_ids("tagging"); + excluded_folders.extend(active_indexing_folders()); let batch_size = crate::tagger::tagger_batch_size(app_data_dir); let jobs = with_db_write_lock(|| { let mut conn = pool.get()?; - db::claim_tagging_jobs(&mut conn, &paused_folders, batch_size) + db::claim_tagging_jobs(&mut conn, &excluded_folders, batch_size) })?; if jobs.is_empty() { From cbfcbea96af5119ec4eff830f3635469e3435bce Mon Sep 17 00:00:00 2001 From: LyAhn Date: Fri, 12 Jun 2026 10:36:05 +0100 Subject: [PATCH 8/9] perf(metadata): idle-only backoff and per-item commits The metadata worker now sleeps its 250ms backoff only when the queue is empty or a batch errored, claiming the next batch immediately while work is pending. Each ffprobe result is committed and emitted individually (in its own transaction) instead of after the whole batch, so video metadata progress moves steadily rather than stalling for up to 16 sequential probes. --- src-tauri/src/indexer.rs | 92 +++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/src-tauri/src/indexer.rs b/src-tauri/src/indexer.rs index 1b5fc15..f34ddd0 100644 --- a/src-tauri/src/indexer.rs +++ b/src-tauri/src/indexer.rs @@ -219,10 +219,16 @@ pub fn start_thumbnail_worker( pub fn start_metadata_worker(app: AppHandle, pool: DbPool, media_tools: MediaTools) { std::thread::spawn(move || loop { - if let Err(error) = process_metadata_batch(&app, &pool, &media_tools) { - eprintln!("Metadata worker error: {}", error); + // Only back off when the queue is empty (or errored); while jobs are + // pending, claim the next batch immediately. + match process_metadata_batch(&app, &pool, &media_tools) { + Ok(true) => {} + Ok(false) => std::thread::sleep(std::time::Duration::from_millis(250)), + Err(error) => { + eprintln!("Metadata worker error: {}", error); + std::thread::sleep(std::time::Duration::from_millis(250)); + } } - std::thread::sleep(std::time::Duration::from_millis(250)); }); } @@ -689,7 +695,13 @@ fn persist_thumbnail_results( Ok(()) } -fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaTools) -> Result<()> { +/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if +/// the queue was empty. +fn process_metadata_batch( + app: &AppHandle, + pool: &DbPool, + media_tools: &MediaTools, +) -> Result { let jobs = { with_db_write_lock(|| { let mut conn = pool.get()?; @@ -708,65 +720,49 @@ fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaToo }; if jobs.is_empty() { - return Ok(()); + return Ok(false); } - let results = jobs - .into_iter() - .map(|job| { - ( - job.image_id, - probe_video_metadata(media_tools, Path::new(&job.path)), - ) - }) - .collect::>(); + // Probes run sequentially (each ffprobe blocks on its process); results + // are committed per item so progress keeps moving through slow stretches. + for job in jobs { + let metadata_result = probe_video_metadata(media_tools, Path::new(&job.path)); - let updated_images = { - with_db_write_lock(|| { + let updated_image = with_db_write_lock(|| { let mut conn = pool.get()?; let tx = conn.transaction()?; - let mut updated_images = Vec::new(); - - for (image_id, metadata_result) in results { - let metadata = match metadata_result { - Ok(metadata) => metadata, - Err(error) => { - db::mark_metadata_failed(&tx, image_id, &error.to_string())?; - continue; - } - }; - - updated_images.push(db::mark_metadata_ready( + let updated = match metadata_result { + Ok(metadata) => Some(db::mark_metadata_ready( &tx, - image_id, + job.image_id, metadata.duration_ms, metadata.width, metadata.height, metadata.video_codec.as_deref(), metadata.audio_codec.as_deref(), - )?); - } - + )?), + Err(error) => { + db::mark_metadata_failed(&tx, job.image_id, &error.to_string())?; + None + } + }; tx.commit()?; - Ok(updated_images) - })? - }; + Ok(updated) + })?; - if !updated_images.is_empty() { - let folder_ids = updated_images - .iter() - .map(|image| image.folder_id) - .collect::>(); - emit_media_updates( - app, - &MediaUpdateBatch { - images: updated_images, - }, - ); - emit_folder_job_progress(app, pool, &folder_ids.into_iter().collect::>(), true); + if let Some(image) = updated_image { + let folder_id = image.folder_id; + emit_media_updates( + app, + &MediaUpdateBatch { + images: vec![image], + }, + ); + emit_folder_job_progress(app, pool, &[folder_id], false); + } } - Ok(()) + Ok(true) } /// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if From d30fe47876d8dd730c1a7dd71c1310ba6f0b3813 Mon Sep 17 00:00:00 2001 From: LyAhn Date: Fri, 12 Jun 2026 10:45:12 +0100 Subject: [PATCH 9/9] perf(workers): strict priority pipeline across background workers Workers now form a priority pipeline (thumbnails -> metadata -> embeddings -> tagging): a worker only claims jobs when no higher-priority tier has claimable work, so stages drain one at a time at full speed instead of all workers contending for CPU (shared rayon pool), disk, and the DB writer simultaneously. Each tier gate is a single read-only EXISTS query per idle tick using the same exclusion set the corresponding claim would (that worker''s paused folders plus actively-indexing folders), so paused or mid-scan work never blocks lower tiers, and failed jobs stop gating once they leave pending. Deferred workers wake within one backoff interval (250-750ms) of the higher tier draining. A watcher-driven trickle of new files briefly pauses lower tiers by design. A user-initiated tag run during a large embedding backlog waits for it; pausing embeddings per folder overrides if tags are wanted first. --- src-tauri/src/db.rs | 51 ++++++++++++++++++++++++++++++++++++ src-tauri/src/indexer.rs | 56 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/src-tauri/src/db.rs b/src-tauri/src/db.rs index 2123d05..3542e5d 100644 --- a/src-tauri/src/db.rs +++ b/src-tauri/src/db.rs @@ -1152,6 +1152,57 @@ fn get_pending_thumbnail_jobs_excluding( Ok(rows.collect::>>()?) } +/// True if any claimable (pending, non-excluded) jobs exist in `job_table`. +/// Used by lower-priority workers to defer to higher-priority queues. +/// `extra_predicate` narrows the image join (e.g. videos only for metadata). +fn has_claimable_jobs( + conn: &Connection, + job_table: &str, + extra_predicate: &str, + excluded_folder_ids: &std::collections::HashSet, +) -> Result { + let sql = format!( + "SELECT EXISTS( + SELECT 1 + FROM {} j + JOIN images i ON i.id = j.image_id + WHERE j.status = 'pending' + {} + {} + )", + job_table, + extra_predicate, + folder_exclusion_clause("i", excluded_folder_ids) + ); + Ok(conn.query_row(&sql, [], |row| row.get::<_, i64>(0))? != 0) +} + +pub fn has_claimable_thumbnail_jobs( + conn: &Connection, + excluded_folder_ids: &std::collections::HashSet, +) -> Result { + has_claimable_jobs(conn, "thumbnail_jobs", "", excluded_folder_ids) +} + +pub fn has_claimable_metadata_jobs( + conn: &Connection, + excluded_folder_ids: &std::collections::HashSet, +) -> Result { + has_claimable_jobs( + conn, + "metadata_jobs", + "AND i.media_kind = 'video'", + excluded_folder_ids, + ) +} + +pub fn has_claimable_embedding_jobs( + conn: &Connection, + excluded_folder_ids: &std::collections::HashSet, +) -> Result { + has_claimable_jobs(conn, "embedding_jobs", "", excluded_folder_ids) +} + pub fn claim_thumbnail_jobs( conn: &mut Connection, active_folder_ids: &std::collections::HashSet, diff --git a/src-tauri/src/indexer.rs b/src-tauri/src/indexer.rs index f34ddd0..ed933c7 100644 --- a/src-tauri/src/indexer.rs +++ b/src-tauri/src/indexer.rs @@ -121,6 +121,50 @@ static FOLDER_STORAGE_PROFILES: OnceLock> = OnceLock::new(); const EMBEDDING_BATCH_SIZE: usize = 8; +/// Background workers form a strict priority pipeline: a worker only claims +/// jobs when no higher-priority claimable work exists, so stages drain one +/// at a time (thumbnails → metadata → embeddings → tagging) instead of all +/// workers contending for CPU, disk, and the DB writer at once. +#[derive(Clone, Copy, PartialEq, PartialOrd)] +enum WorkerTier { + Thumbnail = 0, + Metadata = 1, + Embedding = 2, + Tagging = 3, +} + +/// True if any tier above `own_tier` still has claimable work. Each check +/// uses the same exclusion set the corresponding claim would (that worker's +/// paused folders plus actively-indexing folders), so paused or mid-scan +/// work never gates lower tiers. +fn higher_priority_work_pending(pool: &DbPool, own_tier: WorkerTier) -> Result { + let conn = pool.get()?; + let active_folders = active_indexing_folders(); + + if own_tier > WorkerTier::Thumbnail { + let mut excluded = paused_folder_ids("thumbnail"); + excluded.extend(active_folders.iter().copied()); + if db::has_claimable_thumbnail_jobs(&conn, &excluded)? { + return Ok(true); + } + } + if own_tier > WorkerTier::Metadata { + let mut excluded = paused_folder_ids("metadata"); + excluded.extend(active_folders.iter().copied()); + if db::has_claimable_metadata_jobs(&conn, &excluded)? { + return Ok(true); + } + } + if own_tier > WorkerTier::Embedding { + let mut excluded = paused_folder_ids("embedding"); + excluded.extend(active_folders.iter().copied()); + if db::has_claimable_embedding_jobs(&conn, &excluded)? { + return Ok(true); + } + } + Ok(false) +} + #[derive(Clone, Serialize)] pub struct IndexProgress { pub folder_id: i64, @@ -702,6 +746,10 @@ fn process_metadata_batch( pool: &DbPool, media_tools: &MediaTools, ) -> Result { + if higher_priority_work_pending(pool, WorkerTier::Metadata)? { + return Ok(false); + } + let jobs = { with_db_write_lock(|| { let mut conn = pool.get()?; @@ -772,6 +820,10 @@ fn process_embedding_batch( pool: &DbPool, embedder: &mut Option, ) -> Result { + if higher_priority_work_pending(pool, WorkerTier::Embedding)? { + return Ok(false); + } + let batch_started_at = Instant::now(); let claim_started_at = Instant::now(); // Exclude folders that are actively indexing (matching the thumbnail and @@ -1032,6 +1084,10 @@ fn process_tagging_batch( return Ok(false); } + if higher_priority_work_pending(pool, WorkerTier::Tagging)? { + return Ok(false); + } + // Exclude actively-indexing folders for the same reason as the other // workers: don't compete with a running scan. let mut excluded_folders = paused_folder_ids("tagging");