diff --git a/CLAUDE.md b/CLAUDE.md index e9febb9..b5786c8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,7 +80,7 @@ Database: SQLite with WAL mode, stored in the Tauri app data directory as `galle | `media-updated` | `ThumbnailBatch` | | `caption-model-progress` | `CaptionModelProgress` | | `tagger-model-progress` | `TaggerModelProgress` | -| `duplicate_scan_progress` | `[scanned, total]` | +| `duplicate_scan_progress` | `{ phase, processed, total, skipped }` | ### Key types diff --git a/package.json b/package.json index 1836830..542a05d 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "build:app": "tauri build", "build:vite": "tsc && vite build", + "clean:app": "cd src-tauri && cargo clean", "dev:app": "tauri dev", "dev:vite": "vite", "preview": "vite preview", diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 5e8dbe0..df4ef2c 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -143,6 +143,12 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-broadcast" version = "0.7.2" @@ -635,6 +641,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -3108,6 +3116,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.94" @@ -3559,6 +3577,31 @@ dependencies = [ "pxfm", ] +[[package]] +name = "mozjpeg" +version = "0.10.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7891b80aaa86097d38d276eb98b3805d6280708c4e0a1e6f6aed9380c51fec9" +dependencies = [ + "arrayvec", + "bytemuck", + "libc", + "mozjpeg-sys", + "rgb", +] + +[[package]] +name = "mozjpeg-sys" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f0dc668bf9bf888c88e2fb1ab16a406d2c380f1d082b20d51dd540ab2aa70c1" +dependencies = [ + "cc", + "dunce", + "libc", + "nasm-rs", +] + [[package]] name = "muda" version = "0.17.2" @@ -3586,6 +3629,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af" +[[package]] +name = "nasm-rs" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149" +dependencies = [ + "jobserver", + "log", +] + [[package]] name = "native-tls" version = "0.2.18" @@ -4388,6 +4441,7 @@ dependencies = [ "kamadak-exif", "log", "memmap2", + "mozjpeg", "notify", "ort", "r2d2", @@ -5102,6 +5156,15 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.17.14" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 6c201c7..84a8156 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -53,6 +53,7 @@ csv = "1" kamadak-exif = "0.5" notify = "6" tauri-plugin-notification = "2" +mozjpeg = "0.10.13" # ── Dev-mode performance ──────────────────────────────────────────────────── # opt-level=1 on the main crate keeps incremental compile short. @@ -81,6 +82,10 @@ opt-level = 3 opt-level = 3 [profile.dev.package.fast_image_resize] opt-level = 3 +[profile.dev.package.mozjpeg] +opt-level = 3 +[profile.dev.package.mozjpeg-sys] +opt-level = 3 # Parallel work scheduler [profile.dev.package.rayon] diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 7be9948..3451f92 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -182,6 +182,22 @@ pub struct DuplicateGroup { pub images: Vec, } +#[derive(Clone, Serialize)] +pub struct DuplicateScanProgress { + pub phase: String, + pub processed: usize, + pub total: usize, + pub skipped: usize, +} + +#[derive(Serialize)] +pub struct DuplicateScanResult { + pub groups: Vec, + pub scanned_files: usize, + pub candidate_files: usize, + pub skipped_files: usize, +} + #[derive(Serialize)] pub struct DuplicateScanCache { pub groups: Vec, @@ -264,10 +280,15 @@ pub async fn remove_folder( .into_iter() .find(|f| f.id == folder_id) .map(|f| PathBuf::from(f.path)); + // Collect thumbnail paths before the cascade delete removes the rows. + let thumb_paths = db::get_thumbnail_paths_for_folder(&conn, folder_id).unwrap_or_default(); db::delete_folder(&conn, folder_id).map_err(|e| e.to_string())?; if let Some(path) = folder_path { watcher.remove_folder(&path); } + for thumb in &thumb_paths { + let _ = std::fs::remove_file(thumb); + } Ok(()) } @@ -957,14 +978,19 @@ pub async fn find_duplicates( app: AppHandle, db: State<'_, DbState>, folder_id: Option, -) -> Result, String> { +) -> Result { let records = { let conn = db.get().map_err(|e| e.to_string())?; db::get_all_image_paths(&conn, folder_id).map_err(|e| e.to_string())? }; let total = records.len(); - let _ = app.emit("duplicate_scan_progress", (0usize, total)); + let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "checking".to_string(), + processed: 0, + total, + skipped: 0, + }); // Three-phase detection. // @@ -981,7 +1007,8 @@ pub async fn find_duplicates( // For sample-hash groups that contain files > 64 KB, compute a full-file // hash to confirm the match before presenting them as deletable duplicates. let app_hash = app.clone(); - let pairs: Vec<(u64, u64, i64, String)> = tokio::task::spawn_blocking(move || { + let (pairs, skipped_before_confirm, candidate_total): + (Vec<(u64, u64, i64, String)>, usize, usize) = tokio::task::spawn_blocking(move || { use memmap2::Mmap; use rayon::prelude::*; use std::collections::HashMap; @@ -1007,14 +1034,31 @@ pub async fn find_duplicates( } // ── Phase 1: stat ───────────────────────────────────────────────── + let stat_counter = AtomicUsize::new(0); + let stat_skipped = AtomicUsize::new(0); let sized: Vec<(i64, String, u64)> = records .par_iter() .filter_map(|r| { - let size = std::fs::metadata(&r.path).ok()?.len(); - if size == 0 { return None; } - Some((r.id, r.path.clone(), size)) + let result = match std::fs::metadata(&r.path) { + Ok(metadata) => Some((r.id, r.path.clone(), metadata.len())), + Err(_) => { + stat_skipped.fetch_add(1, Ordering::Relaxed); + None + } + }; + let done = stat_counter.fetch_add(1, Ordering::Relaxed) + 1; + if done % 100 == 0 || done == total { + let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "checking".to_string(), + processed: done, + total, + skipped: stat_skipped.load(Ordering::Relaxed), + }); + } + result }) .collect(); + let stat_skipped = stat_skipped.load(Ordering::Relaxed); let mut by_size: HashMap> = HashMap::new(); for (i, (_, _, size)) in sized.iter().enumerate() { @@ -1027,29 +1071,53 @@ pub async fn find_duplicates( .collect(); if candidates.is_empty() { - return vec![]; + return (vec![], stat_skipped, 0); } // ── Phase 2: sample hash ────────────────────────────────────────── let c_total = candidates.len(); - let _ = app_hash.emit("duplicate_scan_progress", (0usize, c_total)); + let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "hashing".to_string(), + processed: 0, + total: c_total, + skipped: stat_skipped, + }); let counter = AtomicUsize::new(0); + let hash_skipped = AtomicUsize::new(0); - candidates + let pairs = candidates .par_iter() .filter_map(|&idx| { let (id, path, size) = &sized[idx]; - let file = std::fs::File::open(path).ok()?; - // SAFETY: read-only; no external truncation expected during scan. - let mmap = unsafe { Mmap::map(&file).ok()? }; - let hash = sample(&mmap); + let result = if *size == 0 { + Some((xxh3_64(&[]), *size, *id, path.clone())) + } else { + std::fs::File::open(path) + .ok() + // SAFETY: read-only; no external truncation expected during scan. + .and_then(|file| unsafe { Mmap::map(&file).ok() }) + .map(|mmap| (sample(&mmap), *size, *id, path.clone())) + }; + if result.is_none() { + hash_skipped.fetch_add(1, Ordering::Relaxed); + } let done = counter.fetch_add(1, Ordering::Relaxed) + 1; if done % 100 == 0 || done == c_total { - let _ = app_hash.emit("duplicate_scan_progress", (done, c_total)); + let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "hashing".to_string(), + processed: done, + total: c_total, + skipped: stat_skipped + hash_skipped.load(Ordering::Relaxed), + }); } - Some((hash, *size, *id, path.clone())) + result }) - .collect() + .collect(); + ( + pairs, + stat_skipped + hash_skipped.load(Ordering::Relaxed), + c_total, + ) }) .await .map_err(|e| e.to_string())?; @@ -1066,13 +1134,30 @@ pub async fn find_duplicates( // hash becomes its own DuplicateGroup so disjoint sets (A,A vs B,B that // happened to share size and sample hash) are never merged. const COVERED: u64 = (16 * 1024 * 4) as u64; - let confirmed: Vec<(u64, u64, Vec)> = + let confirming_total: usize = size_hash_map + .iter() + .filter(|((_, file_size), entries)| *file_size > COVERED && entries.len() > 1) + .map(|(_, entries)| entries.len()) + .sum(); + if confirming_total > 0 { + let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "confirming".to_string(), + processed: 0, + total: confirming_total, + skipped: skipped_before_confirm, + }); + } + let app_confirm = app.clone(); + let (confirmed, skipped_files): (Vec<(u64, u64, Vec)>, usize) = tokio::task::spawn_blocking(move || { use memmap2::Mmap; use rayon::prelude::*; + use std::sync::atomic::{AtomicUsize, Ordering}; use xxhash_rust::xxh3::xxh3_64; - size_hash_map + let counter = AtomicUsize::new(0); + let confirm_skipped = AtomicUsize::new(0); + let confirmed = size_hash_map .into_iter() .filter(|(_, entries)| entries.len() > 1) .flat_map(|((sample_hash, file_size), entries)| { @@ -1089,6 +1174,19 @@ pub async fn find_duplicates( .ok() .and_then(|f| unsafe { Mmap::map(&f).ok() }) .map(|mmap| xxh3_64(&mmap)); + if hash.is_none() { + confirm_skipped.fetch_add(1, Ordering::Relaxed); + } + let done = counter.fetch_add(1, Ordering::Relaxed) + 1; + if done % 100 == 0 || done == confirming_total { + let _ = app_confirm.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "confirming".to_string(), + processed: done, + total: confirming_total, + skipped: skipped_before_confirm + + confirm_skipped.load(Ordering::Relaxed), + }); + } (*id, hash) }) .collect(); @@ -1106,7 +1204,11 @@ pub async fn find_duplicates( .map(|(full_hash, ids)| (full_hash, file_size, ids)) .collect() }) - .collect() + .collect(); + ( + confirmed, + skipped_before_confirm + confirm_skipped.load(Ordering::Relaxed), + ) }) .await .map_err(|e| e.to_string())?; @@ -1137,7 +1239,12 @@ pub async fn find_duplicates( let _ = db::set_duplicate_scan_cache(&conn, &folder_scope, &json); } - Ok(groups) + Ok(DuplicateScanResult { + groups, + scanned_files: total, + candidate_files: candidate_total, + skipped_files, + }) } #[tauri::command] @@ -1192,6 +1299,9 @@ pub async fn delete_images_from_disk( for r in records.into_iter().filter(|r| id_set.contains(&r.id)) { if std::fs::remove_file(&r.path).is_ok() { succeeded_ids.push(r.id); + if let Some(thumb) = &r.thumbnail_path { + let _ = std::fs::remove_file(thumb); + } } } if !succeeded_ids.is_empty() { @@ -1844,3 +1954,68 @@ pub async fn cleanup_orphaned_thumbnails( freed_mb: freed_bytes as f64 / 1_048_576.0, }) } + +const MUTED_FOLDER_IDS_FILE: &str = "settings/muted_folder_ids.txt"; +const NOTIFICATIONS_PAUSED_FILE: &str = "settings/notifications_paused.txt"; + +#[tauri::command] +pub async fn get_muted_folder_ids(app: AppHandle) -> Result, String> { + let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?; + let path = app_dir.join(MUTED_FOLDER_IDS_FILE); + if !path.exists() { + return Ok(Vec::new()); + } + let content = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; + Ok(content + .trim() + .split(',') + .filter(|s| !s.is_empty()) + .filter_map(|s| s.parse::().ok()) + .collect()) +} + +#[derive(serde::Deserialize)] +pub struct SetMutedFolderIdsParams { + pub folder_ids: Vec, +} + +#[tauri::command] +pub async fn set_muted_folder_ids( + app: AppHandle, + params: SetMutedFolderIdsParams, +) -> Result<(), String> { + let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?; + let settings_dir = app_dir.join("settings"); + std::fs::create_dir_all(&settings_dir).map_err(|e| e.to_string())?; + let content = params + .folder_ids + .iter() + .map(|id| id.to_string()) + .collect::>() + .join(","); + std::fs::write(settings_dir.join("muted_folder_ids.txt"), content) + .map_err(|e| e.to_string()) +} + +#[tauri::command] +pub async fn get_notifications_paused(app: AppHandle) -> Result { + let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?; + let path = app_dir.join(NOTIFICATIONS_PAUSED_FILE); + if !path.exists() { + return Ok(false); + } + let content = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; + Ok(content.trim() == "true") +} + +#[tauri::command] +pub async fn set_notifications_paused(app: AppHandle, paused: bool) -> Result<(), String> { + let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?; + let settings_dir = app_dir.join("settings"); + std::fs::create_dir_all(&settings_dir).map_err(|e| e.to_string())?; + std::fs::write( + settings_dir.join("notifications_paused.txt"), + if paused { "true" } else { "false" }, + ) + .map_err(|e| e.to_string()) +} diff --git a/src-tauri/src/db.rs b/src-tauri/src/db.rs index b2c02ba..3542e5d 100644 --- a/src-tauri/src/db.rs +++ b/src-tauri/src/db.rs @@ -1152,6 +1152,57 @@ fn get_pending_thumbnail_jobs_excluding( Ok(rows.collect::>>()?) } +/// True if any claimable (pending, non-excluded) jobs exist in `job_table`. +/// Used by lower-priority workers to defer to higher-priority queues. +/// `extra_predicate` narrows the image join (e.g. videos only for metadata). +fn has_claimable_jobs( + conn: &Connection, + job_table: &str, + extra_predicate: &str, + excluded_folder_ids: &std::collections::HashSet, +) -> Result { + let sql = format!( + "SELECT EXISTS( + SELECT 1 + FROM {} j + JOIN images i ON i.id = j.image_id + WHERE j.status = 'pending' + {} + {} + )", + job_table, + extra_predicate, + folder_exclusion_clause("i", excluded_folder_ids) + ); + Ok(conn.query_row(&sql, [], |row| row.get::<_, i64>(0))? != 0) +} + +pub fn has_claimable_thumbnail_jobs( + conn: &Connection, + excluded_folder_ids: &std::collections::HashSet, +) -> Result { + has_claimable_jobs(conn, "thumbnail_jobs", "", excluded_folder_ids) +} + +pub fn has_claimable_metadata_jobs( + conn: &Connection, + excluded_folder_ids: &std::collections::HashSet, +) -> Result { + has_claimable_jobs( + conn, + "metadata_jobs", + "AND i.media_kind = 'video'", + excluded_folder_ids, + ) +} + +pub fn has_claimable_embedding_jobs( + conn: &Connection, + excluded_folder_ids: &std::collections::HashSet, +) -> Result { + has_claimable_jobs(conn, "embedding_jobs", "", excluded_folder_ids) +} + pub fn claim_thumbnail_jobs( conn: &mut Connection, active_folder_ids: &std::collections::HashSet, @@ -1676,6 +1727,7 @@ pub fn search_tags_autocomplete( pub struct ImagePathRecord { pub id: i64, pub path: String, + pub thumbnail_path: Option, } pub fn get_all_image_paths( @@ -1683,19 +1735,69 @@ pub fn get_all_image_paths( folder_id: Option, ) -> Result> { let mut stmt = conn.prepare( - "SELECT id, path FROM images WHERE (?1 IS NULL OR folder_id = ?1) ORDER BY id", + "SELECT id, path, thumbnail_path FROM images WHERE (?1 IS NULL OR folder_id = ?1) ORDER BY id", )?; let rows = stmt .query_map(params![folder_id], |row| { Ok(ImagePathRecord { id: row.get(0)?, path: row.get(1)?, + thumbnail_path: row.get(2)?, }) })? .collect::>>()?; Ok(rows) } +/// Returns (image_id, thumbnail_path) for the given path. Used by the watcher +/// delete branch so it can clean up the thumbnail file in the same step. +pub fn get_image_id_and_thumbnail_by_path( + conn: &Connection, + path: &str, +) -> Result)>> { + let result = conn.query_row( + "SELECT id, thumbnail_path FROM images WHERE path = ?1", + params![path], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)), + ); + match result { + Ok(v) => Ok(Some(v)), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(e.into()), + } +} + +/// Returns all non-null thumbnail_path values for images in a folder. +/// Called before folder deletion so callers can remove the files from disk. +pub fn get_thumbnail_paths_for_folder( + conn: &Connection, + folder_id: i64, +) -> Result> { + let mut stmt = conn.prepare( + "SELECT thumbnail_path FROM images WHERE folder_id = ?1 AND thumbnail_path IS NOT NULL", + )?; + let rows = stmt + .query_map([folder_id], |row| row.get::<_, String>(0))? + .collect::>>()?; + Ok(rows) +} + +/// Updates a moved/renamed image's path and thumbnail_path in-place. +/// Pass `new_thumbnail_path = None` to clear it (triggers regeneration). +pub fn update_image_path( + conn: &Connection, + image_id: i64, + new_path: &str, + new_filename: &str, + new_thumbnail_path: Option<&str>, +) -> Result<()> { + conn.execute( + "UPDATE images SET path = ?2, filename = ?3, thumbnail_path = ?4 WHERE id = ?1", + params![image_id, new_path, new_filename, new_thumbnail_path], + )?; + Ok(()) +} + pub fn get_explore_tags( conn: &Connection, folder_id: Option, diff --git a/src-tauri/src/embedder.rs b/src-tauri/src/embedder.rs index 3a977c7..f2a7551 100644 --- a/src-tauri/src/embedder.rs +++ b/src-tauri/src/embedder.rs @@ -191,9 +191,11 @@ fn resolve_device() -> Result { } fn load_image(path: &Path, image_size: usize) -> Result { - let image = image::ImageReader::open(path)? - .with_guessed_format()? - .decode()?; + // Scaled decode: CLIP only needs image_size² pixels, so decoding a large + // JPEG at full resolution is wasted work. Cover mode keeps the shortest + // side at or above image_size for the fill-crop below. Also applies EXIF + // orientation, so rotated photos embed the way they are displayed. + let image = crate::thumbnail::decode_image_scaled(path, image_size as u32, true)?; let image = image.resize_to_fill( image_size as u32, image_size as u32, @@ -208,10 +210,11 @@ fn load_image(path: &Path, image_size: usize) -> Result { } fn load_images(paths: &[PathBuf], image_size: usize) -> Result { - let mut images = Vec::with_capacity(paths.len()); - for path in paths { - images.push(load_image(path, image_size)?); - } + use rayon::prelude::*; + let images = paths + .par_iter() + .map(|path| load_image(path, image_size)) + .collect::>>()?; Ok(Tensor::stack(&images, 0)?) } diff --git a/src-tauri/src/indexer.rs b/src-tauri/src/indexer.rs index 704c466..ed933c7 100644 --- a/src-tauri/src/indexer.rs +++ b/src-tauri/src/indexer.rs @@ -121,6 +121,50 @@ static FOLDER_STORAGE_PROFILES: OnceLock> = OnceLock::new(); const EMBEDDING_BATCH_SIZE: usize = 8; +/// Background workers form a strict priority pipeline: a worker only claims +/// jobs when no higher-priority claimable work exists, so stages drain one +/// at a time (thumbnails → metadata → embeddings → tagging) instead of all +/// workers contending for CPU, disk, and the DB writer at once. +#[derive(Clone, Copy, PartialEq, PartialOrd)] +enum WorkerTier { + Thumbnail = 0, + Metadata = 1, + Embedding = 2, + Tagging = 3, +} + +/// True if any tier above `own_tier` still has claimable work. Each check +/// uses the same exclusion set the corresponding claim would (that worker's +/// paused folders plus actively-indexing folders), so paused or mid-scan +/// work never gates lower tiers. +fn higher_priority_work_pending(pool: &DbPool, own_tier: WorkerTier) -> Result { + let conn = pool.get()?; + let active_folders = active_indexing_folders(); + + if own_tier > WorkerTier::Thumbnail { + let mut excluded = paused_folder_ids("thumbnail"); + excluded.extend(active_folders.iter().copied()); + if db::has_claimable_thumbnail_jobs(&conn, &excluded)? { + return Ok(true); + } + } + if own_tier > WorkerTier::Metadata { + let mut excluded = paused_folder_ids("metadata"); + excluded.extend(active_folders.iter().copied()); + if db::has_claimable_metadata_jobs(&conn, &excluded)? { + return Ok(true); + } + } + if own_tier > WorkerTier::Embedding { + let mut excluded = paused_folder_ids("embedding"); + excluded.extend(active_folders.iter().copied()); + if db::has_claimable_embedding_jobs(&conn, &excluded)? { + return Ok(true); + } + } + Ok(false) +} + #[derive(Clone, Serialize)] pub struct IndexProgress { pub folder_id: i64, @@ -204,19 +248,31 @@ pub fn start_thumbnail_worker( cache_dir: PathBuf, ) { std::thread::spawn(move || loop { - if let Err(error) = process_thumbnail_batch(&app, &pool, &media_tools, &cache_dir) { - eprintln!("Thumbnail worker error: {}", error); + // Only back off when the queue is empty (or errored); while jobs are + // pending, claim the next batch immediately. + match process_thumbnail_batch(&app, &pool, &media_tools, &cache_dir) { + Ok(true) => {} + Ok(false) => std::thread::sleep(std::time::Duration::from_millis(250)), + Err(error) => { + eprintln!("Thumbnail worker error: {}", error); + std::thread::sleep(std::time::Duration::from_millis(250)); + } } - std::thread::sleep(std::time::Duration::from_millis(250)); }); } pub fn start_metadata_worker(app: AppHandle, pool: DbPool, media_tools: MediaTools) { std::thread::spawn(move || loop { - if let Err(error) = process_metadata_batch(&app, &pool, &media_tools) { - eprintln!("Metadata worker error: {}", error); + // Only back off when the queue is empty (or errored); while jobs are + // pending, claim the next batch immediately. + match process_metadata_batch(&app, &pool, &media_tools) { + Ok(true) => {} + Ok(false) => std::thread::sleep(std::time::Duration::from_millis(250)), + Err(error) => { + eprintln!("Metadata worker error: {}", error); + std::thread::sleep(std::time::Duration::from_millis(250)); + } } - std::thread::sleep(std::time::Duration::from_millis(250)); }); } @@ -225,10 +281,16 @@ pub fn start_embedding_worker(app: AppHandle, pool: DbPool) { let mut embedder: Option = None; println!("Embedding worker started."); loop { - if let Err(error) = process_embedding_batch(&app, &pool, &mut embedder) { - eprintln!("Embedding worker error: {}", error); + // Only back off when the queue is empty (or errored); while jobs + // are pending, claim the next batch immediately. + match process_embedding_batch(&app, &pool, &mut embedder) { + Ok(true) => {} + Ok(false) => std::thread::sleep(std::time::Duration::from_millis(500)), + Err(error) => { + eprintln!("Embedding worker error: {}", error); + std::thread::sleep(std::time::Duration::from_millis(500)); + } } - std::thread::sleep(std::time::Duration::from_millis(500)); } }); } @@ -264,13 +326,17 @@ pub fn start_tagging_worker(app: AppHandle, pool: DbPool, app_data_dir: PathBuf) println!("Tagging worker: acceleration setting changed — resetting session."); tagger_instance = None; } - if let Err(error) = - process_tagging_batch(&app, &pool, &app_data_dir, &mut tagger_instance) - { - eprintln!("Tagging worker error: {}", error); - tagger_instance = None; + // Only back off when the queue is empty (or errored); while jobs + // are pending, claim the next batch immediately. + match process_tagging_batch(&app, &pool, &app_data_dir, &mut tagger_instance) { + Ok(true) => {} + Ok(false) => std::thread::sleep(std::time::Duration::from_millis(750)), + Err(error) => { + eprintln!("Tagging worker error: {}", error); + tagger_instance = None; + std::thread::sleep(std::time::Duration::from_millis(750)); + } } - std::thread::sleep(std::time::Duration::from_millis(750)); } }); } @@ -554,12 +620,14 @@ fn commit_batch(pool: &DbPool, records: &[ImageRecord]) -> Result Result<()> { +) -> Result { let jobs = { with_db_write_lock(|| { let mut conn = pool.get()?; @@ -578,7 +646,7 @@ fn process_thumbnail_batch( }; if jobs.is_empty() { - return Ok(()); + return Ok(false); } println!("Thumbnail batch claimed: {} items", jobs.len()); @@ -586,33 +654,39 @@ fn process_thumbnail_batch( let (image_jobs, video_jobs): (Vec<_>, Vec<_>) = jobs.into_iter().partition(|job| job.media_kind == "image"); - let mut results = image_jobs - .par_iter() - .map(|job| { - ( - job.image_id, - if job.media_kind == "image" { - thumbnail::generate_image_thumbnail(Path::new(&job.path), cache_dir).map(Some) - } else { - thumbnail::generate_video_thumbnail( - media_tools, - Path::new(&job.path), - cache_dir, - ) - .map(Some) - }, - ) - }) - .collect::>(); - - for job in video_jobs { - results.push(( - job.image_id, - thumbnail::generate_video_thumbnail(media_tools, Path::new(&job.path), cache_dir) - .map(Some), - )); + // Images: parallel decode, committed as one batch. + if !image_jobs.is_empty() { + let results = image_jobs + .par_iter() + .map(|job| { + ( + job.image_id, + thumbnail::generate_image_thumbnail(Path::new(&job.path), cache_dir).map(Some), + ) + }) + .collect::>(); + persist_thumbnail_results(app, pool, results)?; } + // Videos: sequential, off the rayon pool — each ffmpeg call blocks its + // thread, and a video-heavy batch on the shared pool would starve image + // decoding across all workers. Committed per item so progress keeps + // moving through slow stretches. + for job in &video_jobs { + let result = + thumbnail::generate_video_thumbnail(media_tools, Path::new(&job.path), cache_dir) + .map(Some); + persist_thumbnail_results(app, pool, vec![(job.image_id, result)])?; + } + + Ok(true) +} + +fn persist_thumbnail_results( + app: &AppHandle, + pool: &DbPool, + results: Vec<(i64, anyhow::Result>)>, +) -> Result<()> { let updated_images = { with_db_write_lock(|| { let mut conn = pool.get()?; @@ -665,7 +739,17 @@ fn process_thumbnail_batch( Ok(()) } -fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaTools) -> Result<()> { +/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if +/// the queue was empty. +fn process_metadata_batch( + app: &AppHandle, + pool: &DbPool, + media_tools: &MediaTools, +) -> Result { + if higher_priority_work_pending(pool, WorkerTier::Metadata)? { + return Ok(false); + } + let jobs = { with_db_write_lock(|| { let mut conn = pool.get()?; @@ -684,83 +768,78 @@ fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaToo }; if jobs.is_empty() { - return Ok(()); + return Ok(false); } - let results = jobs - .into_iter() - .map(|job| { - ( - job.image_id, - probe_video_metadata(media_tools, Path::new(&job.path)), - ) - }) - .collect::>(); + // Probes run sequentially (each ffprobe blocks on its process); results + // are committed per item so progress keeps moving through slow stretches. + for job in jobs { + let metadata_result = probe_video_metadata(media_tools, Path::new(&job.path)); - let updated_images = { - with_db_write_lock(|| { + let updated_image = with_db_write_lock(|| { let mut conn = pool.get()?; let tx = conn.transaction()?; - let mut updated_images = Vec::new(); - - for (image_id, metadata_result) in results { - let metadata = match metadata_result { - Ok(metadata) => metadata, - Err(error) => { - db::mark_metadata_failed(&tx, image_id, &error.to_string())?; - continue; - } - }; - - updated_images.push(db::mark_metadata_ready( + let updated = match metadata_result { + Ok(metadata) => Some(db::mark_metadata_ready( &tx, - image_id, + job.image_id, metadata.duration_ms, metadata.width, metadata.height, metadata.video_codec.as_deref(), metadata.audio_codec.as_deref(), - )?); - } - + )?), + Err(error) => { + db::mark_metadata_failed(&tx, job.image_id, &error.to_string())?; + None + } + }; tx.commit()?; - Ok(updated_images) - })? - }; + Ok(updated) + })?; - if !updated_images.is_empty() { - let folder_ids = updated_images - .iter() - .map(|image| image.folder_id) - .collect::>(); - emit_media_updates( - app, - &MediaUpdateBatch { - images: updated_images, - }, - ); - emit_folder_job_progress(app, pool, &folder_ids.into_iter().collect::>(), true); + if let Some(image) = updated_image { + let folder_id = image.folder_id; + emit_media_updates( + app, + &MediaUpdateBatch { + images: vec![image], + }, + ); + emit_folder_job_progress(app, pool, &[folder_id], false); + } } - Ok(()) + Ok(true) } +/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if +/// the queue was empty. fn process_embedding_batch( app: &AppHandle, pool: &DbPool, embedder: &mut Option, -) -> Result<()> { +) -> Result { + if higher_priority_work_pending(pool, WorkerTier::Embedding)? { + return Ok(false); + } + let batch_started_at = Instant::now(); let claim_started_at = Instant::now(); - let paused_folders = paused_folder_ids("embedding"); + // Exclude folders that are actively indexing (matching the thumbnail and + // metadata workers): embedding mid-scan competes with the scanner for + // CPU, disk, and the DB writer, and video jobs would fail-fast anyway + // because their thumbnails are deferred until indexing completes. + let mut excluded_folders = paused_folder_ids("embedding"); + excluded_folders.extend(active_indexing_folders()); let jobs = with_db_write_lock(|| { let mut conn = pool.get()?; - db::claim_embedding_jobs(&mut conn, &paused_folders, EMBEDDING_BATCH_SIZE) + db::claim_embedding_jobs(&mut conn, &excluded_folders, EMBEDDING_BATCH_SIZE) })?; let claim_elapsed = claim_started_at.elapsed(); if jobs.is_empty() { - return Ok(()); + return Ok(false); } if embedder.is_none() { @@ -888,7 +967,7 @@ fn process_embedding_batch( EMBEDDING_BATCH_SIZE, claim_elapsed, infer_elapsed, write_elapsed, batch_elapsed ); - Ok(()) + Ok(true) } fn process_caption_batch( @@ -993,25 +1072,34 @@ fn process_caption_batch( Ok(()) } +/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if +/// the queue was empty or the model is not ready. fn process_tagging_batch( app: &AppHandle, pool: &DbPool, app_data_dir: &Path, tagger_instance: &mut Option, -) -> Result<()> { +) -> Result { if !tagger::tagger_model_status(app_data_dir).ready { - return Ok(()); + return Ok(false); } - let paused_folders = paused_folder_ids("tagging"); + if higher_priority_work_pending(pool, WorkerTier::Tagging)? { + return Ok(false); + } + + // Exclude actively-indexing folders for the same reason as the other + // workers: don't compete with a running scan. + let mut excluded_folders = paused_folder_ids("tagging"); + excluded_folders.extend(active_indexing_folders()); let batch_size = crate::tagger::tagger_batch_size(app_data_dir); let jobs = with_db_write_lock(|| { let mut conn = pool.get()?; - db::claim_tagging_jobs(&mut conn, &paused_folders, batch_size) + db::claim_tagging_jobs(&mut conn, &excluded_folders, batch_size) })?; if jobs.is_empty() { - return Ok(()); + return Ok(false); } if tagger_instance.is_none() { @@ -1119,7 +1207,7 @@ fn process_tagging_batch( emit_folder_job_progress(app, pool, &folder_ids.into_iter().collect::>(), true); } - Ok(()) + Ok(true) } fn active_indexing_folders() -> HashSet { @@ -1347,7 +1435,7 @@ impl WatcherHandle { /// - `recv()` when no events are pending — zero CPU /// - `recv_timeout(earliest_deadline)` when events are pending — wakes exactly /// when the soonest debounce window expires, no busy-polling -pub fn start_watcher(app: AppHandle, pool: DbPool) -> WatcherHandle { +pub fn start_watcher(app: AppHandle, pool: DbPool, thumb_dir: PathBuf) -> WatcherHandle { let (tx, rx) = std::sync::mpsc::channel::>(); let raw_watcher = notify::recommended_watcher(move |result| { @@ -1392,17 +1480,19 @@ pub fn start_watcher(app: AppHandle, pool: DbPool) -> WatcherHandle { std::thread::spawn(move || { // path → deadline: the earliest instant at which this path should be processed. let mut pending: HashMap = HashMap::new(); + // old_path → new_path for rename events that carry both sides atomically. + let mut pending_renames: Vec<(PathBuf, PathBuf)> = Vec::new(); loop { // Adaptive blocking: block forever when idle, wake at the earliest // deadline when events are queued. - let received = if pending.is_empty() { + let received = if pending.is_empty() && pending_renames.is_empty() { match rx.recv() { Ok(e) => Some(e), Err(_) => break, // sender dropped — app is shutting down } } else { - let earliest = pending.values().copied().min().unwrap(); // safe: non-empty + let earliest = pending.values().copied().min().unwrap_or(Instant::now()); let timeout = earliest.saturating_duration_since(Instant::now()); match rx.recv_timeout(timeout) { Ok(e) => Some(e), @@ -1415,17 +1505,40 @@ pub fn start_watcher(app: AppHandle, pool: DbPool) -> WatcherHandle { // Absorb incoming event — coalesces rapid writes into one deadline. if let Some(Ok(event)) = received { - use notify::EventKind; - // Skip pure access events (reads); they never change file content. + use notify::{EventKind, event::{ModifyKind, RenameMode}}; if !matches!(event.kind, EventKind::Access(_)) { - for path in event.paths { - if is_supported_media(&path) { - pending.insert(path, now + WATCHER_DEBOUNCE); + // Intercept atomic rename events (old + new path in one event). + // Handle these separately to preserve embeddings and thumbnails. + if matches!( + event.kind, + EventKind::Modify(ModifyKind::Name(RenameMode::Both)) + ) && event.paths.len() == 2 + { + let old = event.paths[0].clone(); + let new = event.paths[1].clone(); + if is_supported_media(&old) || is_supported_media(&new) { + // Remove either side from regular pending so it isn't + // processed as an independent delete/create. + pending.remove(&old); + pending.remove(&new); + pending_renames.push((old, new)); + } + } else { + for path in event.paths { + if is_supported_media(&path) { + pending.insert(path, now + WATCHER_DEBOUNCE); + } } } } } + // Process renames immediately — they are atomic filesystem operations + // and do not benefit from debouncing. + for (old_path, new_path) in pending_renames.drain(..) { + process_watcher_rename(&app, &pool, &folder_map_thread, &thumb_dir, &old_path, &new_path); + } + // Process all paths whose debounce window has expired. let ready: Vec = pending .iter() @@ -1496,11 +1609,14 @@ fn process_watcher_path( Err(e) => eprintln!("Watcher: commit error for {:?}: {}", path, e), } } else { - // File removed from disk — clean up DB row. + // File removed from disk — clean up DB row and thumbnail. let path_str = path.to_string_lossy(); - match db::get_image_id_by_path(&conn, &path_str) { - Ok(Some(image_id)) => { + match db::get_image_id_and_thumbnail_by_path(&conn, &path_str) { + Ok(Some((image_id, thumb_path))) => { if db::delete_images_by_ids(&conn, &[image_id]).is_ok() { + if let Some(thumb) = thumb_path { + let _ = std::fs::remove_file(thumb); + } db::update_folder_count(&conn, folder_id).ok(); let _ = app.emit("watcher-deleted", vec![image_id]); let _ = app.emit("folder-counts-changed", ()); @@ -1511,3 +1627,76 @@ fn process_watcher_path( } } } + +/// Handles a filesystem rename/move event where both the old and new paths are +/// known. Updates the DB row in-place (preserving the embedding) and renames +/// the thumbnail file to match the new path hash. +fn process_watcher_rename( + app: &AppHandle, + pool: &DbPool, + folder_map: &Arc>>, + thumb_dir: &Path, + old_path: &Path, + new_path: &Path, +) { + // Resolve folder_id from the old path first, fall back to new path. + let folder_id = { + let map = folder_map.lock().unwrap(); + map.iter() + .find(|(fp, _)| old_path.starts_with(fp.as_path()) || new_path.starts_with(fp.as_path())) + .map(|(_, &id)| id) + }; + let Some(folder_id) = folder_id else { return }; + + let conn = match pool.get() { + Ok(c) => c, + Err(e) => { + eprintln!("Watcher rename: DB pool error: {}", e); + return; + } + }; + + let old_path_str = old_path.to_string_lossy(); + let new_path_str = new_path.to_string_lossy(); + + let (image_id, old_thumb) = match db::get_image_id_and_thumbnail_by_path(&conn, &old_path_str) { + Ok(Some(record)) => record, + Ok(None) => { + // Not yet indexed under the old path — treat as a brand-new file. + drop(conn); + process_watcher_path(app, pool, folder_map, new_path); + return; + } + Err(e) => { + eprintln!("Watcher rename: DB lookup error: {}", e); + return; + } + }; + + // Compute new thumbnail path and attempt to rename the file on disk. + let new_thumb_path = crate::thumbnail::thumb_path(thumb_dir, &new_path_str); + let new_thumb_str = new_thumb_path.to_string_lossy().into_owned(); + let thumb_ok = old_thumb + .as_deref() + .map(|old| std::fs::rename(old, &new_thumb_path).is_ok()) + .unwrap_or(false); + // If rename failed (e.g. old thumb never existed), clear thumbnail_path so + // the thumbnail worker regenerates it. + let effective_thumb: Option<&str> = if thumb_ok { Some(&new_thumb_str) } else { None }; + + let new_filename = new_path + .file_name() + .and_then(|f| f.to_str()) + .unwrap_or(""); + + if let Err(e) = db::update_image_path(&conn, image_id, &new_path_str, new_filename, effective_thumb) { + eprintln!("Watcher rename: DB update error: {}", e); + return; + } + + // Emit the updated record so the frontend refreshes without a full reload. + match db::get_image_by_id(&conn, image_id) { + Ok(record) => emit_images(app, &IndexedImagesBatch { folder_id, images: vec![record] }), + Err(e) => eprintln!("Watcher rename: post-update fetch error: {}", e), + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 00ba11a..b0e3ccb 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -74,7 +74,7 @@ pub fn run() { // indexer::start_caption_worker(app.handle().clone(), pool.clone(), app_dir.clone()); indexer::start_tagging_worker(app.handle().clone(), pool.clone(), app_dir.clone()); - let watcher_handle = indexer::start_watcher(app.handle().clone(), pool.clone()); + let watcher_handle = indexer::start_watcher(app.handle().clone(), pool.clone(), thumb_dir.clone()); app.manage(pool); app.manage(media_tools); @@ -148,6 +148,10 @@ pub fn run() { commands::vacuum_database, commands::get_orphaned_thumbnails_info, commands::cleanup_orphaned_thumbnails, + commands::get_muted_folder_ids, + commands::set_muted_folder_ids, + commands::get_notifications_paused, + commands::set_notifications_paused, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/src-tauri/src/thumbnail.rs b/src-tauri/src/thumbnail.rs index 4efcf2f..8b48765 100644 --- a/src-tauri/src/thumbnail.rs +++ b/src-tauri/src/thumbnail.rs @@ -31,29 +31,26 @@ pub fn generate_image_thumbnail(image_path: &Path, cache_dir: &Path) -> Result Result Result { + decode_image_scaled(image_path, THUMB_SIZE, false) +} + +/// Decodes an image at reduced resolution, with EXIF orientation applied. +/// +/// JPEGs take a fast path that decodes at reduced resolution (DCT scaling), +/// which skips most of the IDCT/upsampling work for large photos. Anything +/// that path can't handle falls back to the `image` crate at full size. +/// +/// `cover` selects which side must stay at or above `target`: `false` keeps +/// the longest side (for aspect-fit consumers), `true` keeps the shortest +/// side (for fill-crop consumers like the CLIP preprocessor). +pub fn decode_image_scaled( + image_path: &Path, + target: u32, + cover: bool, +) -> Result { + if is_jpeg(image_path) { + if let Some(img) = decode_jpeg_scaled(image_path, target, cover) { + return Ok(img); + } + } + + let reader = image::ImageReader::open(image_path)?.with_guessed_format()?; + let mut decoder = reader.into_decoder()?; + let orientation = decoder.orientation()?; + let mut img = image::DynamicImage::from_decoder(decoder)?; + img.apply_orientation(orientation); + Ok(img) +} + +fn is_jpeg(path: &Path) -> bool { + path.extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext.eq_ignore_ascii_case("jpg") || ext.eq_ignore_ascii_case("jpeg")) +} + +/// Decodes a JPEG at the smallest DCT scale that still covers `target`. +/// Returns `None` on any failure (corrupt file, CMYK without conversion +/// support, etc.) so the caller can fall back to the generic decoder. +/// mozjpeg reports errors by panicking, hence the `catch_unwind`. +fn decode_jpeg_scaled(image_path: &Path, target: u32, cover: bool) -> Option { + let path = image_path.to_path_buf(); + let decoded = std::panic::catch_unwind(move || -> std::io::Result<(Vec, u32, u32)> { + let mut decompress = mozjpeg::Decompress::new_path(&path)?; + let (width, height) = decompress.size(); + let reference = if cover { + width.min(height) + } else { + width.max(height) + }; + decompress.scale(scale_numerator(reference, target)); + let mut started = decompress.rgb()?; + let (out_width, out_height) = (started.width() as u32, started.height() as u32); + let pixels = started.read_scanlines::()?; + started.finish()?; + Ok((pixels, out_width, out_height)) + }) + .ok()? + .ok()?; + + let (pixels, width, height) = decoded; + let buffer = image::RgbImage::from_raw(width, height, pixels)?; + let mut img = image::DynamicImage::ImageRgb8(buffer); + if let Some(orientation) = exif_orientation(image_path) { + img.apply_orientation(orientation); + } + Some(img) +} + +/// Smallest numerator (of /8) that keeps `reference` at or above `target`, +/// so the subsequent resize never upscales. +fn scale_numerator(reference: usize, target: u32) -> u8 { + for numerator in 1..=8u8 { + if reference * numerator as usize / 8 >= target as usize { + return numerator; + } + } + 8 +} + +fn exif_orientation(path: &Path) -> Option { + let file = std::fs::File::open(path).ok()?; + let mut reader = std::io::BufReader::new(file); + let exif = exif::Reader::new().read_from_container(&mut reader).ok()?; + let field = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY)?; + let value = field.value.get_uint(0)?; + image::metadata::Orientation::from_exif(value as u8) +} + pub fn generate_video_thumbnail( tools: &MediaTools, video_path: &Path, @@ -88,6 +177,8 @@ pub fn generate_video_thumbnail( let attempts: [&[&str]; 3] = [ &[ "-y", + "-threads", + "2", "-ss", "00:00:00.000", "-i", @@ -95,13 +186,15 @@ pub fn generate_video_thumbnail( "-frames:v", "1", "-vf", - "thumbnail,scale=320:-1:force_original_aspect_ratio=decrease", + "scale=320:-1:force_original_aspect_ratio=decrease", "-q:v", "4", &output_path, ], &[ "-y", + "-threads", + "2", "-ss", "00:00:00.250", "-i", @@ -109,19 +202,21 @@ pub fn generate_video_thumbnail( "-frames:v", "1", "-vf", - "thumbnail,scale=320:-1:force_original_aspect_ratio=decrease", + "scale=320:-1:force_original_aspect_ratio=decrease", "-q:v", "4", &output_path, ], &[ "-y", + "-threads", + "2", "-i", path_str.as_ref(), "-frames:v", "1", "-vf", - "thumbnail,scale=320:-1:force_original_aspect_ratio=decrease", + "scale=320:-1:force_original_aspect_ratio=decrease", "-q:v", "4", &output_path, @@ -165,6 +260,49 @@ fn hash_path(s: &str) -> String { format!("{:016x}", xxhash_rust::xxh3::xxh3_64(s.as_bytes())) } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scale_numerator_picks_smallest_sufficient() { + assert_eq!(scale_numerator(6000, THUMB_SIZE), 1); + assert_eq!(scale_numerator(640, THUMB_SIZE), 4); + assert_eq!(scale_numerator(320, THUMB_SIZE), 8); + assert_eq!(scale_numerator(100, THUMB_SIZE), 8); + // Cover mode reference: shortest side must reach 224 for CLIP. + assert_eq!(scale_numerator(1200, 224), 2); + } + + #[test] + fn jpeg_fast_path_decodes_at_reduced_resolution() { + let dir = std::env::temp_dir().join("phokus-thumb-test"); + std::fs::create_dir_all(&dir).unwrap(); + let src_path = dir.join("large.jpg"); + let img = + image::RgbImage::from_fn(1600, 1200, |x, _| image::Rgb([(x % 256) as u8, 64, 128])); + img.save(&src_path).unwrap(); + + // 1600 max dim -> numerator 2 -> 400x300 decode output + let decoded = decode_jpeg_scaled(&src_path, THUMB_SIZE, false) + .expect("fast path should handle plain JPEG"); + assert_eq!((decoded.width(), decoded.height()), (400, 300)); + + // Cover mode: shortest side (1200) must stay >= 224 -> numerator 2 + let covered = decode_jpeg_scaled(&src_path, 224, true) + .expect("fast path should handle plain JPEG"); + assert_eq!((covered.width(), covered.height()), (400, 300)); + + let cache = dir.join("cache"); + let _ = std::fs::remove_file(thumb_path(&cache, &src_path.to_string_lossy())); + let result = generate_image_thumbnail(&src_path, &cache).unwrap(); + assert_eq!(result.width, Some(1600)); + assert_eq!(result.height, Some(1200)); + let (thumb_w, thumb_h) = image::image_dimensions(&result.path).unwrap(); + assert_eq!((thumb_w, thumb_h), (320, 240)); + } +} + fn fit_dimensions(width: u32, height: u32, max_size: u32) -> (u32, u32) { if width <= max_size && height <= max_size { return (width.max(1), height.max(1)); diff --git a/src/App.tsx b/src/App.tsx index 84595a2..409da27 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -18,11 +18,15 @@ export default function App() { const loadImages = useGalleryStore((state) => state.loadImages); const loadCaptionModelStatus = useGalleryStore((state) => state.loadCaptionModelStatus); const loadDuplicateScanCache = useGalleryStore((state) => state.loadDuplicateScanCache); + const loadMutedFolderIds = useGalleryStore((state) => state.loadMutedFolderIds); + const loadNotificationsPaused = useGalleryStore((state) => state.loadNotificationsPaused); const subscribeToProgress = useGalleryStore((state) => state.subscribeToProgress); const activeView = useGalleryStore((state) => state.activeView); useEffect(() => { void initializeNotifications(); + void loadMutedFolderIds(); + void loadNotificationsPaused(); loadFolders().then(() => { void loadBackgroundJobProgress(); void loadCaptionModelStatus(); diff --git a/src/components/BackgroundTasks.tsx b/src/components/BackgroundTasks.tsx index d34ceb2..8930943 100644 --- a/src/components/BackgroundTasks.tsx +++ b/src/components/BackgroundTasks.tsx @@ -278,12 +278,16 @@ export function BackgroundTasks() { id: -1, name: "Duplicate Scan", stages: [{ - label: "Hashing", + label: duplicateScanProgress?.phase === "checking" + ? "Checking" + : duplicateScanProgress?.phase === "confirming" + ? "Confirming" + : "Hashing", detail: duplicateScanProgress - ? `${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}` + ? `${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}` : "Starting…", progress: duplicateScanProgress && duplicateScanProgress.total > 0 - ? (duplicateScanProgress.scanned / duplicateScanProgress.total) * 100 + ? (duplicateScanProgress.processed / duplicateScanProgress.total) * 100 : null, failed: false, }], @@ -310,7 +314,8 @@ export function BackgroundTasks() { const embeddingStage = primary.stages.find((s) => s.label === "Embeddings"); const taggingStage = primary.stages.find((s) => s.label === "Tags"); const scanningStage = primary.stages.find((s) => s.label === "Scanning"); - const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? null; + const duplicateStage = primary.id === -1 ? primary.stages[0] : null; + const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? duplicateStage?.progress ?? null; return (
@@ -434,7 +439,8 @@ export function BackgroundTasks() { const taskEmbeddingStage = task.stages.find((s) => s.label === "Embeddings"); const taskTaggingStage = task.stages.find((s) => s.label === "Tags"); const taskScanningStage = task.stages.find((s) => s.label === "Scanning"); - const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? null; + const taskDuplicateStage = task.id === -1 ? task.stages[0] : null; + const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? taskDuplicateStage?.progress ?? null; const taskHasFailed = (task.hasFailedEmbeddings || task.hasFailedTagging || task.hasFailedCaptions) && task.pendingMediaWork === 0; return ( diff --git a/src/components/DuplicateFinder.tsx b/src/components/DuplicateFinder.tsx index cae277f..67b361b 100644 --- a/src/components/DuplicateFinder.tsx +++ b/src/components/DuplicateFinder.tsx @@ -117,6 +117,7 @@ export function DuplicateFinder() { const duplicateScanning = useGalleryStore((state) => state.duplicateScanning); const duplicateScanProgress = useGalleryStore((state) => state.duplicateScanProgress); const duplicateScanError = useGalleryStore((state) => state.duplicateScanError); + const duplicateScanWarning = useGalleryStore((state) => state.duplicateScanWarning); const duplicateSelectedIds = useGalleryStore((state) => state.duplicateSelectedIds); const duplicateLastScanned = useGalleryStore((state) => state.duplicateLastScanned); const selectedFolderId = useGalleryStore((state) => state.selectedFolderId); @@ -152,8 +153,15 @@ export function DuplicateFinder() { const progressPercent = duplicateScanProgress && duplicateScanProgress.total > 0 - ? Math.round((duplicateScanProgress.scanned / duplicateScanProgress.total) * 100) + ? Math.round((duplicateScanProgress.processed / duplicateScanProgress.total) * 100) : 0; + const progressLabel = duplicateScanProgress + ? duplicateScanProgress.phase === "checking" + ? "Checking file sizes" + : duplicateScanProgress.phase === "hashing" + ? "Hashing duplicate candidates" + : "Confirming exact matches" + : null; return (
@@ -165,7 +173,7 @@ export function DuplicateFinder() {

{duplicateScanning ? duplicateScanProgress - ? `Scanning… ${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}` + ? `${progressLabel}… ${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}` : "Starting scan…" : hasResults ? `${duplicateGroups.length} group${duplicateGroups.length === 1 ? "" : "s"} · ${formatBytes(totalWasted)} reclaimable` @@ -232,6 +240,9 @@ export function DuplicateFinder() { {duplicateScanError ? (

{duplicateScanError}

) : null} + {duplicateScanWarning ? ( +

{duplicateScanWarning}

+ ) : null} {deleteResult ? (

{deleteResult}

) : null} @@ -241,7 +252,7 @@ export function DuplicateFinder() { {duplicateScanning && !hasResults ? (
- Hashing files… + {progressLabel ? `${progressLabel}…` : "Preparing scan…"}
) : !hasScanned ? (
diff --git a/src/components/SettingsModal.tsx b/src/components/SettingsModal.tsx index a00accc..190344f 100644 --- a/src/components/SettingsModal.tsx +++ b/src/components/SettingsModal.tsx @@ -162,6 +162,8 @@ export function SettingsModal() { const clearTaggingJobs = useGalleryStore((state) => state.clearTaggingJobs); const clearTaggingJobsForFolders = useGalleryStore((state) => state.clearTaggingJobsForFolders); const openAppDataFolder = useGalleryStore((state) => state.openAppDataFolder); + const notificationsPaused = useGalleryStore((state) => state.notificationsPaused); + const setNotificationsPaused = useGalleryStore((state) => state.setNotificationsPaused); const getDatabaseInfo = useGalleryStore((state) => state.getDatabaseInfo); const vacuumDatabase = useGalleryStore((state) => state.vacuumDatabase); const getOrphanedThumbnailsInfo = useGalleryStore((state) => state.getOrphanedThumbnailsInfo); @@ -601,6 +603,26 @@ export function SettingsModal() {
+ +
+
+

Pause all notifications

+

Suppress all indexing notifications until re-enabled.

+
+ +
+ +

Orphaned files

- {thumbnailCleanupResult - ? thumbnailCleanupResult.deleted_count.toLocaleString() - : thumbnailInfo - ? thumbnailInfo.count.toLocaleString() - : "—"} + {cleaningThumbnails + ? "—" + : thumbnailCleanupResult + ? "0" + : thumbnailInfo + ? thumbnailInfo.count.toLocaleString() + : "—"}

Reclaimable

- {thumbnailCleanupResult - ? `−${thumbnailCleanupResult.freed_mb.toFixed(1)} MB freed` - : thumbnailInfo - ? `${thumbnailInfo.size_mb.toFixed(1)} MB` - : "—"} + {cleaningThumbnails + ? "—" + : thumbnailCleanupResult + ? "0 MB" + : thumbnailInfo + ? `${thumbnailInfo.size_mb.toFixed(1)} MB` + : "—"}

- {thumbnailCleanupResult - ? `Removed ${thumbnailCleanupResult.deleted_count.toLocaleString()} orphaned thumbnail${thumbnailCleanupResult.deleted_count === 1 ? "" : "s"}.` - : thumbnailInfo && thumbnailInfo.count === 0 - ? "No orphaned thumbnails found." - : "Remove thumbnails no longer associated with any indexed image."} + {cleaningThumbnails + ? "Scanning and removing orphaned thumbnails…" + : thumbnailCleanupResult + ? `Removed ${thumbnailCleanupResult.deleted_count.toLocaleString()} file${thumbnailCleanupResult.deleted_count === 1 ? "" : "s"}, freed ${thumbnailCleanupResult.freed_mb.toFixed(1)} MB.` + : thumbnailInfo && thumbnailInfo.count === 0 + ? "No orphaned thumbnails found." + : thumbnailInfo && thumbnailInfo.count > 1000 + ? "Remove thumbnails no longer associated with any indexed image. This may take a few minutes for large collections." + : "Remove thumbnails no longer associated with any indexed image."}

diff --git a/src/components/Sidebar.tsx b/src/components/Sidebar.tsx index da84180..ba47dd6 100644 --- a/src/components/Sidebar.tsx +++ b/src/components/Sidebar.tsx @@ -11,18 +11,22 @@ interface ContextMenuState { function FolderContextMenu({ menu, folder, + isMuted, onClose, onRename, onReindex, onLocate, + onToggleMute, onRemove, }: { menu: ContextMenuState; folder: Folder; + isMuted: boolean; onClose: () => void; onRename: () => void; onReindex: () => void; onLocate: () => void; + onToggleMute: () => void; onRemove: () => void; }) { const ref = useRef(null); @@ -61,6 +65,7 @@ function FolderContextMenu({ > {item("Reindex", onReindex)} {item("Rename", onRename)} + {item(isMuted ? "Unmute notifications" : "Mute notifications", onToggleMute)} {folder.scan_error && item("Locate Folder", onLocate)}
{item("Remove from app", onRemove, true)} @@ -77,7 +82,9 @@ function FolderItem({ selected: boolean; progress: IndexProgress | undefined; }) { - const { selectFolder, removeFolder, reindexFolder, renameFolder, updateFolderPath } = useGalleryStore(); + const { selectFolder, removeFolder, reindexFolder, renameFolder, updateFolderPath, toggleMutedFolder } = useGalleryStore(); + const mutedFolderIds = useGalleryStore((state) => state.mutedFolderIds); + const isMuted = mutedFolderIds.includes(folder.id); const isIndexing = progress && !progress.done; const isMissing = !!folder.scan_error && !isIndexing; @@ -250,10 +257,12 @@ function FolderItem({ setContextMenu(null)} onRename={() => setRenaming(true)} onReindex={() => void reindexFolder(folder.id)} onLocate={() => void handleLocateFolder()} + onToggleMute={() => toggleMutedFolder(folder.id)} onRemove={() => setConfirmingRemoval(true)} /> )} diff --git a/src/store.ts b/src/store.ts index b4a0090..1fd5eb6 100644 --- a/src/store.ts +++ b/src/store.ts @@ -4,6 +4,11 @@ import { listen, UnlistenFn } from "@tauri-apps/api/event"; import { appDataDir, join } from "@tauri-apps/api/path"; import { notifyTaskComplete } from "./notifications"; +// Per-folder debounce timers for batching notifications. +// Keyed as `${folderId}:embedding` or `${folderId}:tagging`. +const notificationTimers = new Map>(); +const NOTIFICATION_DEBOUNCE_MS = 6000; + export interface Folder { id: number; path: string; @@ -165,6 +170,20 @@ export interface DuplicateGroup { images: ImageRecord[]; } +export interface DuplicateScanProgress { + phase: "checking" | "hashing" | "confirming"; + processed: number; + total: number; + skipped: number; +} + +interface DuplicateScanResult { + groups: DuplicateGroup[]; + scanned_files: number; + candidate_files: number; + skipped_files: number; +} + export interface SimilarImagesPage { images: ImageRecord[]; offset: number; @@ -288,6 +307,8 @@ interface GalleryState { settingsOpen: boolean; taggingQueueScope: TaggingQueueScope; taggingQueueFolderIds: number[]; + mutedFolderIds: number[]; + notificationsPaused: boolean; taggerModelStatus: TaggerModelStatus | null; taggerModelPreparing: boolean; @@ -301,8 +322,9 @@ interface GalleryState { duplicateGroups: DuplicateGroup[]; duplicateScanning: boolean; - duplicateScanProgress: { scanned: number; total: number } | null; + duplicateScanProgress: DuplicateScanProgress | null; duplicateScanError: string | null; + duplicateScanWarning: string | null; duplicateSelectedIds: Set; duplicateLastScanned: number | null; // Unix timestamp (seconds) duplicateScanFolderId: number | null | undefined; // undefined = never scanned @@ -360,6 +382,10 @@ interface GalleryState { loadTaggingQueueFolderIds: () => Promise; toggleTaggingQueueFolder: (folderId: number) => void; setTaggingQueueFolderIds: (folderIds: number[]) => void; + loadMutedFolderIds: () => Promise; + toggleMutedFolder: (folderId: number) => void; + loadNotificationsPaused: () => Promise; + setNotificationsPaused: (paused: boolean) => void; openAppDataFolder: () => Promise; getDatabaseInfo: () => Promise; vacuumDatabase: () => Promise; @@ -635,6 +661,8 @@ export const useGalleryStore = create((set, get) => ({ settingsOpen: false, taggingQueueScope: "all", taggingQueueFolderIds: [], + mutedFolderIds: [], + notificationsPaused: false, taggerModelStatus: null, taggerModelPreparing: false, @@ -650,6 +678,7 @@ export const useGalleryStore = create((set, get) => ({ duplicateScanning: false, duplicateScanProgress: null, duplicateScanError: null, + duplicateScanWarning: null, duplicateSelectedIds: new Set(), duplicateLastScanned: null, duplicateScanFolderId: undefined, @@ -941,7 +970,13 @@ export const useGalleryStore = create((set, get) => ({ if (activeView === "duplicates") { const { selectedFolderId, duplicateScanFolderId } = get(); if (duplicateScanFolderId !== selectedFolderId) { - set({ activeView, duplicateGroups: [], duplicateLastScanned: null, duplicateScanFolderId: undefined }); + set({ + activeView, + duplicateGroups: [], + duplicateLastScanned: null, + duplicateScanFolderId: undefined, + duplicateScanWarning: null, + }); void get().loadDuplicateScanCache(selectedFolderId); return; } @@ -1371,6 +1406,39 @@ export const useGalleryStore = create((set, get) => ({ void invoke("set_tagging_queue_folder_ids", { folder_ids: taggingQueueFolderIds }).catch(() => {}); }, + loadMutedFolderIds: async () => { + try { + const folderIds = await invoke("get_muted_folder_ids"); + set({ mutedFolderIds: folderIds }); + } catch { + // fall back to in-memory default + } + }, + + toggleMutedFolder: (folderId) => { + set((state) => { + const next = state.mutedFolderIds.includes(folderId) + ? state.mutedFolderIds.filter((id) => id !== folderId) + : [...state.mutedFolderIds, folderId]; + void invoke("set_muted_folder_ids", { folder_ids: next }).catch(() => {}); + return { mutedFolderIds: next }; + }); + }, + + loadNotificationsPaused: async () => { + try { + const paused = await invoke("get_notifications_paused"); + set({ notificationsPaused: paused }); + } catch { + // fall back to in-memory default + } + }, + + setNotificationsPaused: (paused) => { + set({ notificationsPaused: paused }); + void invoke("set_notifications_paused", { paused }).catch(() => {}); + }, + openAppDataFolder: async () => { await invoke("open_app_data_folder"); }, @@ -1537,23 +1605,42 @@ export const useGalleryStore = create((set, get) => ({ interface CacheResult { groups: DuplicateGroup[]; scanned_at: number } const cached = await invoke("load_duplicate_scan_cache", { folderId: folderId ?? null }); if (cached) { - set({ duplicateGroups: cached.groups, duplicateLastScanned: cached.scanned_at, duplicateScanFolderId: folderId }); + set({ + duplicateGroups: cached.groups, + duplicateLastScanned: cached.scanned_at, + duplicateScanFolderId: folderId, + duplicateScanWarning: null, + }); } }, scanDuplicates: async (folderId = null) => { const { listen } = await import("@tauri-apps/api/event"); - set({ duplicateScanning: true, duplicateGroups: [], duplicateScanProgress: null, duplicateScanError: null, duplicateSelectedIds: new Set() }); - const unlisten = await listen<[number, number]>("duplicate_scan_progress", (event) => { - const [scanned, total] = event.payload; - set({ duplicateScanProgress: { scanned, total } }); + set({ + duplicateScanning: true, + duplicateGroups: [], + duplicateScanProgress: null, + duplicateScanError: null, + duplicateScanWarning: null, + duplicateSelectedIds: new Set(), + }); + const unlisten = await listen("duplicate_scan_progress", (event) => { + set({ duplicateScanProgress: event.payload }); }); try { - const groups = await invoke("find_duplicates", { folderId: folderId ?? null }); - set({ duplicateGroups: groups, duplicateLastScanned: Math.floor(Date.now() / 1000), duplicateScanFolderId: folderId }); + const result = await invoke("find_duplicates", { folderId: folderId ?? null }); + const warning = result.skipped_files > 0 + ? `${result.skipped_files.toLocaleString()} file${result.skipped_files === 1 ? "" : "s"} could not be read and were skipped.` + : null; + set({ + duplicateGroups: result.groups, + duplicateLastScanned: Math.floor(Date.now() / 1000), + duplicateScanFolderId: folderId, + duplicateScanWarning: warning, + }); void notifyTaskComplete( "Duplicate scan complete", - groups.length === 1 ? "Found 1 duplicate group." : `Found ${groups.length.toLocaleString()} duplicate groups.`, + `${result.groups.length === 1 ? "Found 1 duplicate group." : `Found ${result.groups.length.toLocaleString()} duplicate groups.`}${warning ? ` ${warning}` : ""}`, ); } catch (e) { set({ duplicateScanError: String(e) }); @@ -1659,11 +1746,14 @@ export const useGalleryStore = create((set, get) => ({ progress.total > 0 && progress.indexed >= progress.total ) { - const folderName = get().folders.find((folder) => folder.id === progress.folder_id)?.name; - void notifyTaskComplete( - "Folder scan complete", - folderName ? `${folderName} has finished scanning.` : "A folder has finished scanning.", - ); + const { notificationsPaused, mutedFolderIds } = get(); + if (!notificationsPaused && !mutedFolderIds.includes(progress.folder_id)) { + const folderName = get().folders.find((folder) => folder.id === progress.folder_id)?.name; + void notifyTaskComplete( + "Folder scan complete", + folderName ? `${folderName} has finished scanning.` : "A folder has finished scanning.", + ); + } } void get().loadFolders(); void get().loadBackgroundJobProgress(); @@ -1688,32 +1778,52 @@ export const useGalleryStore = create((set, get) => ({ const previous = previousProgress[progress.folder_id]; if (!previous) continue; + const { notificationsPaused, mutedFolderIds } = get(); + const suppressed = notificationsPaused || mutedFolderIds.includes(progress.folder_id); const folderName = get().folders.find((folder) => folder.id === progress.folder_id)?.name ?? "Folder"; - if (previous.embedding_pending > 0 && progress.embedding_pending === 0) { - const failureDetail = - progress.embedding_failed > 0 - ? ` ${progress.embedding_failed.toLocaleString()} failed.` - : ""; - void notifyTaskComplete( - "Embeddings complete", - `${folderName} finished generating embeddings.${failureDetail}`, - ); + // Embeddings — debounced so rapid file additions don't fire per-file. + const embeddingKey = `${progress.folder_id}:embedding`; + if (!suppressed) { + if (previous.embedding_pending > 0 && progress.embedding_pending === 0) { + clearTimeout(notificationTimers.get(embeddingKey)); + const failureDetail = + progress.embedding_failed > 0 + ? ` ${progress.embedding_failed.toLocaleString()} failed.` + : ""; + const body = `${folderName} finished generating embeddings.${failureDetail}`; + notificationTimers.set(embeddingKey, setTimeout(() => { + notificationTimers.delete(embeddingKey); + void notifyTaskComplete("Embeddings complete", body); + }, NOTIFICATION_DEBOUNCE_MS)); + } else if (previous.embedding_pending === 0 && progress.embedding_pending > 0) { + // More jobs queued — cancel the pending notification. + clearTimeout(notificationTimers.get(embeddingKey)); + notificationTimers.delete(embeddingKey); + } } - if (previous.tagging_pending > 0 && progress.tagging_pending === 0) { - const failureDetail = - progress.tagging_failed > 0 - ? ` ${progress.tagging_failed.toLocaleString()} failed.` - : ""; - void notifyTaskComplete( - "AI tagging complete", - `${folderName} finished generating tags.${failureDetail}`, - ); - // New tags are now in the DB — invalidate the Explore tag cache so - // reopening Explore reflects the updated tag distribution. - set({ exploreTagsFolderId: undefined }); + // Tagging — same debounce pattern. + const taggingKey = `${progress.folder_id}:tagging`; + if (!suppressed) { + if (previous.tagging_pending > 0 && progress.tagging_pending === 0) { + clearTimeout(notificationTimers.get(taggingKey)); + const failureDetail = + progress.tagging_failed > 0 + ? ` ${progress.tagging_failed.toLocaleString()} failed.` + : ""; + const body = `${folderName} finished generating tags.${failureDetail}`; + notificationTimers.set(taggingKey, setTimeout(() => { + notificationTimers.delete(taggingKey); + void notifyTaskComplete("AI tagging complete", body); + // New tags landed — invalidate Explore tag cache. + set({ exploreTagsFolderId: undefined }); + }, NOTIFICATION_DEBOUNCE_MS)); + } else if (previous.tagging_pending === 0 && progress.tagging_pending > 0) { + clearTimeout(notificationTimers.get(taggingKey)); + notificationTimers.delete(taggingKey); + } } }