Merge feat/smart-file-tracking: file tracking, notifications, and worker performance

Delivers rename-aware file tracking plus a major background-worker
performance series:

- feat: smart thumbnail cleanup and rename-aware file tracking —
  watcher handles atomic renames in place (preserving thumbnails and
  embeddings); thumbnails deleted alongside DB rows in all removal paths
- feat: notification batching, per-folder mute, and global pause
- feat(duplicates): phased scan progress with skipped-file reporting
- fix(settings): thumbnail cleanup UX with live state and time warning
- perf(thumbnails): scaled JPEG decode (mozjpeg), RGB8 pipeline, direct
  video frame grabs, continuous worker batching
- perf(workers): scaled parallel embedding preprocessing, idle-only
  backoff, EXIF orientation applied before embedding
- fix(indexer): embedding/tagging no longer claim folders mid-scan
- perf(metadata): idle-only backoff and per-item commits
- perf(workers): strict priority pipeline (thumbnails -> metadata ->
  embeddings -> tagging)
This commit is contained in:
2026-06-12 11:05:29 +01:00
16 changed files with 1062 additions and 212 deletions
+1 -1
View File
@@ -80,7 +80,7 @@ Database: SQLite with WAL mode, stored in the Tauri app data directory as `galle
| `media-updated` | `ThumbnailBatch` | | `media-updated` | `ThumbnailBatch` |
| `caption-model-progress` | `CaptionModelProgress` | | `caption-model-progress` | `CaptionModelProgress` |
| `tagger-model-progress` | `TaggerModelProgress` | | `tagger-model-progress` | `TaggerModelProgress` |
| `duplicate_scan_progress` | `[scanned, total]` | | `duplicate_scan_progress` | `{ phase, processed, total, skipped }` |
### Key types ### Key types
+1
View File
@@ -6,6 +6,7 @@
"scripts": { "scripts": {
"build:app": "tauri build", "build:app": "tauri build",
"build:vite": "tsc && vite build", "build:vite": "tsc && vite build",
"clean:app": "cd src-tauri && cargo clean",
"dev:app": "tauri dev", "dev:app": "tauri dev",
"dev:vite": "vite", "dev:vite": "vite",
"preview": "vite preview", "preview": "vite preview",
+63
View File
@@ -143,6 +143,12 @@ dependencies = [
"derive_arbitrary", "derive_arbitrary",
] ]
[[package]]
name = "arrayvec"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]] [[package]]
name = "async-broadcast" name = "async-broadcast"
version = "0.7.2" version = "0.7.2"
@@ -635,6 +641,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283"
dependencies = [ dependencies = [
"find-msvc-tools", "find-msvc-tools",
"jobserver",
"libc",
"shlex", "shlex",
] ]
@@ -3108,6 +3116,16 @@ dependencies = [
"syn 2.0.117", "syn 2.0.117",
] ]
[[package]]
name = "jobserver"
version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
dependencies = [
"getrandom 0.3.4",
"libc",
]
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.94" version = "0.3.94"
@@ -3559,6 +3577,31 @@ dependencies = [
"pxfm", "pxfm",
] ]
[[package]]
name = "mozjpeg"
version = "0.10.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7891b80aaa86097d38d276eb98b3805d6280708c4e0a1e6f6aed9380c51fec9"
dependencies = [
"arrayvec",
"bytemuck",
"libc",
"mozjpeg-sys",
"rgb",
]
[[package]]
name = "mozjpeg-sys"
version = "2.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f0dc668bf9bf888c88e2fb1ab16a406d2c380f1d082b20d51dd540ab2aa70c1"
dependencies = [
"cc",
"dunce",
"libc",
"nasm-rs",
]
[[package]] [[package]]
name = "muda" name = "muda"
version = "0.17.2" version = "0.17.2"
@@ -3586,6 +3629,16 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af" checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af"
[[package]]
name = "nasm-rs"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149"
dependencies = [
"jobserver",
"log",
]
[[package]] [[package]]
name = "native-tls" name = "native-tls"
version = "0.2.18" version = "0.2.18"
@@ -4388,6 +4441,7 @@ dependencies = [
"kamadak-exif", "kamadak-exif",
"log", "log",
"memmap2", "memmap2",
"mozjpeg",
"notify", "notify",
"ort", "ort",
"r2d2", "r2d2",
@@ -5102,6 +5156,15 @@ dependencies = [
"windows-sys 0.60.2", "windows-sys 0.60.2",
] ]
[[package]]
name = "rgb"
version = "0.8.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4"
dependencies = [
"bytemuck",
]
[[package]] [[package]]
name = "ring" name = "ring"
version = "0.17.14" version = "0.17.14"
+5
View File
@@ -53,6 +53,7 @@ csv = "1"
kamadak-exif = "0.5" kamadak-exif = "0.5"
notify = "6" notify = "6"
tauri-plugin-notification = "2" tauri-plugin-notification = "2"
mozjpeg = "0.10.13"
# ── Dev-mode performance ──────────────────────────────────────────────────── # ── Dev-mode performance ────────────────────────────────────────────────────
# opt-level=1 on the main crate keeps incremental compile short. # opt-level=1 on the main crate keeps incremental compile short.
@@ -81,6 +82,10 @@ opt-level = 3
opt-level = 3 opt-level = 3
[profile.dev.package.fast_image_resize] [profile.dev.package.fast_image_resize]
opt-level = 3 opt-level = 3
[profile.dev.package.mozjpeg]
opt-level = 3
[profile.dev.package.mozjpeg-sys]
opt-level = 3
# Parallel work scheduler # Parallel work scheduler
[profile.dev.package.rayon] [profile.dev.package.rayon]
+195 -20
View File
@@ -182,6 +182,22 @@ pub struct DuplicateGroup {
pub images: Vec<ImageRecord>, pub images: Vec<ImageRecord>,
} }
#[derive(Clone, Serialize)]
pub struct DuplicateScanProgress {
pub phase: String,
pub processed: usize,
pub total: usize,
pub skipped: usize,
}
#[derive(Serialize)]
pub struct DuplicateScanResult {
pub groups: Vec<DuplicateGroup>,
pub scanned_files: usize,
pub candidate_files: usize,
pub skipped_files: usize,
}
#[derive(Serialize)] #[derive(Serialize)]
pub struct DuplicateScanCache { pub struct DuplicateScanCache {
pub groups: Vec<DuplicateGroup>, pub groups: Vec<DuplicateGroup>,
@@ -264,10 +280,15 @@ pub async fn remove_folder(
.into_iter() .into_iter()
.find(|f| f.id == folder_id) .find(|f| f.id == folder_id)
.map(|f| PathBuf::from(f.path)); .map(|f| PathBuf::from(f.path));
// Collect thumbnail paths before the cascade delete removes the rows.
let thumb_paths = db::get_thumbnail_paths_for_folder(&conn, folder_id).unwrap_or_default();
db::delete_folder(&conn, folder_id).map_err(|e| e.to_string())?; db::delete_folder(&conn, folder_id).map_err(|e| e.to_string())?;
if let Some(path) = folder_path { if let Some(path) = folder_path {
watcher.remove_folder(&path); watcher.remove_folder(&path);
} }
for thumb in &thumb_paths {
let _ = std::fs::remove_file(thumb);
}
Ok(()) Ok(())
} }
@@ -957,14 +978,19 @@ pub async fn find_duplicates(
app: AppHandle, app: AppHandle,
db: State<'_, DbState>, db: State<'_, DbState>,
folder_id: Option<i64>, folder_id: Option<i64>,
) -> Result<Vec<DuplicateGroup>, String> { ) -> Result<DuplicateScanResult, String> {
let records = { let records = {
let conn = db.get().map_err(|e| e.to_string())?; let conn = db.get().map_err(|e| e.to_string())?;
db::get_all_image_paths(&conn, folder_id).map_err(|e| e.to_string())? db::get_all_image_paths(&conn, folder_id).map_err(|e| e.to_string())?
}; };
let total = records.len(); let total = records.len();
let _ = app.emit("duplicate_scan_progress", (0usize, total)); let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress {
phase: "checking".to_string(),
processed: 0,
total,
skipped: 0,
});
// Three-phase detection. // Three-phase detection.
// //
@@ -981,7 +1007,8 @@ pub async fn find_duplicates(
// For sample-hash groups that contain files > 64 KB, compute a full-file // For sample-hash groups that contain files > 64 KB, compute a full-file
// hash to confirm the match before presenting them as deletable duplicates. // hash to confirm the match before presenting them as deletable duplicates.
let app_hash = app.clone(); let app_hash = app.clone();
let pairs: Vec<(u64, u64, i64, String)> = tokio::task::spawn_blocking(move || { let (pairs, skipped_before_confirm, candidate_total):
(Vec<(u64, u64, i64, String)>, usize, usize) = tokio::task::spawn_blocking(move || {
use memmap2::Mmap; use memmap2::Mmap;
use rayon::prelude::*; use rayon::prelude::*;
use std::collections::HashMap; use std::collections::HashMap;
@@ -1007,14 +1034,31 @@ pub async fn find_duplicates(
} }
// ── Phase 1: stat ───────────────────────────────────────────────── // ── Phase 1: stat ─────────────────────────────────────────────────
let stat_counter = AtomicUsize::new(0);
let stat_skipped = AtomicUsize::new(0);
let sized: Vec<(i64, String, u64)> = records let sized: Vec<(i64, String, u64)> = records
.par_iter() .par_iter()
.filter_map(|r| { .filter_map(|r| {
let size = std::fs::metadata(&r.path).ok()?.len(); let result = match std::fs::metadata(&r.path) {
if size == 0 { return None; } Ok(metadata) => Some((r.id, r.path.clone(), metadata.len())),
Some((r.id, r.path.clone(), size)) Err(_) => {
stat_skipped.fetch_add(1, Ordering::Relaxed);
None
}
};
let done = stat_counter.fetch_add(1, Ordering::Relaxed) + 1;
if done % 100 == 0 || done == total {
let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress {
phase: "checking".to_string(),
processed: done,
total,
skipped: stat_skipped.load(Ordering::Relaxed),
});
}
result
}) })
.collect(); .collect();
let stat_skipped = stat_skipped.load(Ordering::Relaxed);
let mut by_size: HashMap<u64, Vec<usize>> = HashMap::new(); let mut by_size: HashMap<u64, Vec<usize>> = HashMap::new();
for (i, (_, _, size)) in sized.iter().enumerate() { for (i, (_, _, size)) in sized.iter().enumerate() {
@@ -1027,29 +1071,53 @@ pub async fn find_duplicates(
.collect(); .collect();
if candidates.is_empty() { if candidates.is_empty() {
return vec![]; return (vec![], stat_skipped, 0);
} }
// ── Phase 2: sample hash ────────────────────────────────────────── // ── Phase 2: sample hash ──────────────────────────────────────────
let c_total = candidates.len(); let c_total = candidates.len();
let _ = app_hash.emit("duplicate_scan_progress", (0usize, c_total)); let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress {
phase: "hashing".to_string(),
processed: 0,
total: c_total,
skipped: stat_skipped,
});
let counter = AtomicUsize::new(0); let counter = AtomicUsize::new(0);
let hash_skipped = AtomicUsize::new(0);
candidates let pairs = candidates
.par_iter() .par_iter()
.filter_map(|&idx| { .filter_map(|&idx| {
let (id, path, size) = &sized[idx]; let (id, path, size) = &sized[idx];
let file = std::fs::File::open(path).ok()?; let result = if *size == 0 {
// SAFETY: read-only; no external truncation expected during scan. Some((xxh3_64(&[]), *size, *id, path.clone()))
let mmap = unsafe { Mmap::map(&file).ok()? }; } else {
let hash = sample(&mmap); std::fs::File::open(path)
.ok()
// SAFETY: read-only; no external truncation expected during scan.
.and_then(|file| unsafe { Mmap::map(&file).ok() })
.map(|mmap| (sample(&mmap), *size, *id, path.clone()))
};
if result.is_none() {
hash_skipped.fetch_add(1, Ordering::Relaxed);
}
let done = counter.fetch_add(1, Ordering::Relaxed) + 1; let done = counter.fetch_add(1, Ordering::Relaxed) + 1;
if done % 100 == 0 || done == c_total { if done % 100 == 0 || done == c_total {
let _ = app_hash.emit("duplicate_scan_progress", (done, c_total)); let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress {
phase: "hashing".to_string(),
processed: done,
total: c_total,
skipped: stat_skipped + hash_skipped.load(Ordering::Relaxed),
});
} }
Some((hash, *size, *id, path.clone())) result
}) })
.collect() .collect();
(
pairs,
stat_skipped + hash_skipped.load(Ordering::Relaxed),
c_total,
)
}) })
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
@@ -1066,13 +1134,30 @@ pub async fn find_duplicates(
// hash becomes its own DuplicateGroup so disjoint sets (A,A vs B,B that // hash becomes its own DuplicateGroup so disjoint sets (A,A vs B,B that
// happened to share size and sample hash) are never merged. // happened to share size and sample hash) are never merged.
const COVERED: u64 = (16 * 1024 * 4) as u64; const COVERED: u64 = (16 * 1024 * 4) as u64;
let confirmed: Vec<(u64, u64, Vec<i64>)> = let confirming_total: usize = size_hash_map
.iter()
.filter(|((_, file_size), entries)| *file_size > COVERED && entries.len() > 1)
.map(|(_, entries)| entries.len())
.sum();
if confirming_total > 0 {
let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress {
phase: "confirming".to_string(),
processed: 0,
total: confirming_total,
skipped: skipped_before_confirm,
});
}
let app_confirm = app.clone();
let (confirmed, skipped_files): (Vec<(u64, u64, Vec<i64>)>, usize) =
tokio::task::spawn_blocking(move || { tokio::task::spawn_blocking(move || {
use memmap2::Mmap; use memmap2::Mmap;
use rayon::prelude::*; use rayon::prelude::*;
use std::sync::atomic::{AtomicUsize, Ordering};
use xxhash_rust::xxh3::xxh3_64; use xxhash_rust::xxh3::xxh3_64;
size_hash_map let counter = AtomicUsize::new(0);
let confirm_skipped = AtomicUsize::new(0);
let confirmed = size_hash_map
.into_iter() .into_iter()
.filter(|(_, entries)| entries.len() > 1) .filter(|(_, entries)| entries.len() > 1)
.flat_map(|((sample_hash, file_size), entries)| { .flat_map(|((sample_hash, file_size), entries)| {
@@ -1089,6 +1174,19 @@ pub async fn find_duplicates(
.ok() .ok()
.and_then(|f| unsafe { Mmap::map(&f).ok() }) .and_then(|f| unsafe { Mmap::map(&f).ok() })
.map(|mmap| xxh3_64(&mmap)); .map(|mmap| xxh3_64(&mmap));
if hash.is_none() {
confirm_skipped.fetch_add(1, Ordering::Relaxed);
}
let done = counter.fetch_add(1, Ordering::Relaxed) + 1;
if done % 100 == 0 || done == confirming_total {
let _ = app_confirm.emit("duplicate_scan_progress", DuplicateScanProgress {
phase: "confirming".to_string(),
processed: done,
total: confirming_total,
skipped: skipped_before_confirm
+ confirm_skipped.load(Ordering::Relaxed),
});
}
(*id, hash) (*id, hash)
}) })
.collect(); .collect();
@@ -1106,7 +1204,11 @@ pub async fn find_duplicates(
.map(|(full_hash, ids)| (full_hash, file_size, ids)) .map(|(full_hash, ids)| (full_hash, file_size, ids))
.collect() .collect()
}) })
.collect() .collect();
(
confirmed,
skipped_before_confirm + confirm_skipped.load(Ordering::Relaxed),
)
}) })
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
@@ -1137,7 +1239,12 @@ pub async fn find_duplicates(
let _ = db::set_duplicate_scan_cache(&conn, &folder_scope, &json); let _ = db::set_duplicate_scan_cache(&conn, &folder_scope, &json);
} }
Ok(groups) Ok(DuplicateScanResult {
groups,
scanned_files: total,
candidate_files: candidate_total,
skipped_files,
})
} }
#[tauri::command] #[tauri::command]
@@ -1192,6 +1299,9 @@ pub async fn delete_images_from_disk(
for r in records.into_iter().filter(|r| id_set.contains(&r.id)) { for r in records.into_iter().filter(|r| id_set.contains(&r.id)) {
if std::fs::remove_file(&r.path).is_ok() { if std::fs::remove_file(&r.path).is_ok() {
succeeded_ids.push(r.id); succeeded_ids.push(r.id);
if let Some(thumb) = &r.thumbnail_path {
let _ = std::fs::remove_file(thumb);
}
} }
} }
if !succeeded_ids.is_empty() { if !succeeded_ids.is_empty() {
@@ -1844,3 +1954,68 @@ pub async fn cleanup_orphaned_thumbnails(
freed_mb: freed_bytes as f64 / 1_048_576.0, freed_mb: freed_bytes as f64 / 1_048_576.0,
}) })
} }
const MUTED_FOLDER_IDS_FILE: &str = "settings/muted_folder_ids.txt";
const NOTIFICATIONS_PAUSED_FILE: &str = "settings/notifications_paused.txt";
#[tauri::command]
pub async fn get_muted_folder_ids(app: AppHandle) -> Result<Vec<i64>, String> {
let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?;
let path = app_dir.join(MUTED_FOLDER_IDS_FILE);
if !path.exists() {
return Ok(Vec::new());
}
let content = std::fs::read_to_string(&path).map_err(|e| e.to_string())?;
Ok(content
.trim()
.split(',')
.filter(|s| !s.is_empty())
.filter_map(|s| s.parse::<i64>().ok())
.collect())
}
#[derive(serde::Deserialize)]
pub struct SetMutedFolderIdsParams {
pub folder_ids: Vec<i64>,
}
#[tauri::command]
pub async fn set_muted_folder_ids(
app: AppHandle,
params: SetMutedFolderIdsParams,
) -> Result<(), String> {
let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?;
let settings_dir = app_dir.join("settings");
std::fs::create_dir_all(&settings_dir).map_err(|e| e.to_string())?;
let content = params
.folder_ids
.iter()
.map(|id| id.to_string())
.collect::<Vec<_>>()
.join(",");
std::fs::write(settings_dir.join("muted_folder_ids.txt"), content)
.map_err(|e| e.to_string())
}
#[tauri::command]
pub async fn get_notifications_paused(app: AppHandle) -> Result<bool, String> {
let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?;
let path = app_dir.join(NOTIFICATIONS_PAUSED_FILE);
if !path.exists() {
return Ok(false);
}
let content = std::fs::read_to_string(&path).map_err(|e| e.to_string())?;
Ok(content.trim() == "true")
}
#[tauri::command]
pub async fn set_notifications_paused(app: AppHandle, paused: bool) -> Result<(), String> {
let app_dir = app.path().app_data_dir().map_err(|e| e.to_string())?;
let settings_dir = app_dir.join("settings");
std::fs::create_dir_all(&settings_dir).map_err(|e| e.to_string())?;
std::fs::write(
settings_dir.join("notifications_paused.txt"),
if paused { "true" } else { "false" },
)
.map_err(|e| e.to_string())
}
+103 -1
View File
@@ -1152,6 +1152,57 @@ fn get_pending_thumbnail_jobs_excluding(
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?) Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
} }
/// True if any claimable (pending, non-excluded) jobs exist in `job_table`.
/// Used by lower-priority workers to defer to higher-priority queues.
/// `extra_predicate` narrows the image join (e.g. videos only for metadata).
fn has_claimable_jobs(
conn: &Connection,
job_table: &str,
extra_predicate: &str,
excluded_folder_ids: &std::collections::HashSet<i64>,
) -> Result<bool> {
let sql = format!(
"SELECT EXISTS(
SELECT 1
FROM {} j
JOIN images i ON i.id = j.image_id
WHERE j.status = 'pending'
{}
{}
)",
job_table,
extra_predicate,
folder_exclusion_clause("i", excluded_folder_ids)
);
Ok(conn.query_row(&sql, [], |row| row.get::<_, i64>(0))? != 0)
}
pub fn has_claimable_thumbnail_jobs(
conn: &Connection,
excluded_folder_ids: &std::collections::HashSet<i64>,
) -> Result<bool> {
has_claimable_jobs(conn, "thumbnail_jobs", "", excluded_folder_ids)
}
pub fn has_claimable_metadata_jobs(
conn: &Connection,
excluded_folder_ids: &std::collections::HashSet<i64>,
) -> Result<bool> {
has_claimable_jobs(
conn,
"metadata_jobs",
"AND i.media_kind = 'video'",
excluded_folder_ids,
)
}
pub fn has_claimable_embedding_jobs(
conn: &Connection,
excluded_folder_ids: &std::collections::HashSet<i64>,
) -> Result<bool> {
has_claimable_jobs(conn, "embedding_jobs", "", excluded_folder_ids)
}
pub fn claim_thumbnail_jobs( pub fn claim_thumbnail_jobs(
conn: &mut Connection, conn: &mut Connection,
active_folder_ids: &std::collections::HashSet<i64>, active_folder_ids: &std::collections::HashSet<i64>,
@@ -1676,6 +1727,7 @@ pub fn search_tags_autocomplete(
pub struct ImagePathRecord { pub struct ImagePathRecord {
pub id: i64, pub id: i64,
pub path: String, pub path: String,
pub thumbnail_path: Option<String>,
} }
pub fn get_all_image_paths( pub fn get_all_image_paths(
@@ -1683,19 +1735,69 @@ pub fn get_all_image_paths(
folder_id: Option<i64>, folder_id: Option<i64>,
) -> Result<Vec<ImagePathRecord>> { ) -> Result<Vec<ImagePathRecord>> {
let mut stmt = conn.prepare( let mut stmt = conn.prepare(
"SELECT id, path FROM images WHERE (?1 IS NULL OR folder_id = ?1) ORDER BY id", "SELECT id, path, thumbnail_path FROM images WHERE (?1 IS NULL OR folder_id = ?1) ORDER BY id",
)?; )?;
let rows = stmt let rows = stmt
.query_map(params![folder_id], |row| { .query_map(params![folder_id], |row| {
Ok(ImagePathRecord { Ok(ImagePathRecord {
id: row.get(0)?, id: row.get(0)?,
path: row.get(1)?, path: row.get(1)?,
thumbnail_path: row.get(2)?,
}) })
})? })?
.collect::<rusqlite::Result<Vec<_>>>()?; .collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows) Ok(rows)
} }
/// Returns (image_id, thumbnail_path) for the given path. Used by the watcher
/// delete branch so it can clean up the thumbnail file in the same step.
pub fn get_image_id_and_thumbnail_by_path(
conn: &Connection,
path: &str,
) -> Result<Option<(i64, Option<String>)>> {
let result = conn.query_row(
"SELECT id, thumbnail_path FROM images WHERE path = ?1",
params![path],
|row| Ok((row.get::<_, i64>(0)?, row.get::<_, Option<String>>(1)?)),
);
match result {
Ok(v) => Ok(Some(v)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(e.into()),
}
}
/// Returns all non-null thumbnail_path values for images in a folder.
/// Called before folder deletion so callers can remove the files from disk.
pub fn get_thumbnail_paths_for_folder(
conn: &Connection,
folder_id: i64,
) -> Result<Vec<String>> {
let mut stmt = conn.prepare(
"SELECT thumbnail_path FROM images WHERE folder_id = ?1 AND thumbnail_path IS NOT NULL",
)?;
let rows = stmt
.query_map([folder_id], |row| row.get::<_, String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows)
}
/// Updates a moved/renamed image's path and thumbnail_path in-place.
/// Pass `new_thumbnail_path = None` to clear it (triggers regeneration).
pub fn update_image_path(
conn: &Connection,
image_id: i64,
new_path: &str,
new_filename: &str,
new_thumbnail_path: Option<&str>,
) -> Result<()> {
conn.execute(
"UPDATE images SET path = ?2, filename = ?3, thumbnail_path = ?4 WHERE id = ?1",
params![image_id, new_path, new_filename, new_thumbnail_path],
)?;
Ok(())
}
pub fn get_explore_tags( pub fn get_explore_tags(
conn: &Connection, conn: &Connection,
folder_id: Option<i64>, folder_id: Option<i64>,
+10 -7
View File
@@ -191,9 +191,11 @@ fn resolve_device() -> Result<Device> {
} }
fn load_image(path: &Path, image_size: usize) -> Result<Tensor> { fn load_image(path: &Path, image_size: usize) -> Result<Tensor> {
let image = image::ImageReader::open(path)? // Scaled decode: CLIP only needs image_size² pixels, so decoding a large
.with_guessed_format()? // JPEG at full resolution is wasted work. Cover mode keeps the shortest
.decode()?; // side at or above image_size for the fill-crop below. Also applies EXIF
// orientation, so rotated photos embed the way they are displayed.
let image = crate::thumbnail::decode_image_scaled(path, image_size as u32, true)?;
let image = image.resize_to_fill( let image = image.resize_to_fill(
image_size as u32, image_size as u32,
image_size as u32, image_size as u32,
@@ -208,10 +210,11 @@ fn load_image(path: &Path, image_size: usize) -> Result<Tensor> {
} }
fn load_images(paths: &[PathBuf], image_size: usize) -> Result<Tensor> { fn load_images(paths: &[PathBuf], image_size: usize) -> Result<Tensor> {
let mut images = Vec::with_capacity(paths.len()); use rayon::prelude::*;
for path in paths { let images = paths
images.push(load_image(path, image_size)?); .par_iter()
} .map(|path| load_image(path, image_size))
.collect::<Result<Vec<_>>>()?;
Ok(Tensor::stack(&images, 0)?) Ok(Tensor::stack(&images, 0)?)
} }
+298 -109
View File
@@ -121,6 +121,50 @@ static FOLDER_STORAGE_PROFILES: OnceLock<Mutex<HashMap<i64, RuntimeAdaptiveProfi
static DB_WRITE_LOCK: OnceLock<Mutex<()>> = OnceLock::new(); static DB_WRITE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
const EMBEDDING_BATCH_SIZE: usize = 8; const EMBEDDING_BATCH_SIZE: usize = 8;
/// Background workers form a strict priority pipeline: a worker only claims
/// jobs when no higher-priority claimable work exists, so stages drain one
/// at a time (thumbnails → metadata → embeddings → tagging) instead of all
/// workers contending for CPU, disk, and the DB writer at once.
#[derive(Clone, Copy, PartialEq, PartialOrd)]
enum WorkerTier {
Thumbnail = 0,
Metadata = 1,
Embedding = 2,
Tagging = 3,
}
/// True if any tier above `own_tier` still has claimable work. Each check
/// uses the same exclusion set the corresponding claim would (that worker's
/// paused folders plus actively-indexing folders), so paused or mid-scan
/// work never gates lower tiers.
fn higher_priority_work_pending(pool: &DbPool, own_tier: WorkerTier) -> Result<bool> {
let conn = pool.get()?;
let active_folders = active_indexing_folders();
if own_tier > WorkerTier::Thumbnail {
let mut excluded = paused_folder_ids("thumbnail");
excluded.extend(active_folders.iter().copied());
if db::has_claimable_thumbnail_jobs(&conn, &excluded)? {
return Ok(true);
}
}
if own_tier > WorkerTier::Metadata {
let mut excluded = paused_folder_ids("metadata");
excluded.extend(active_folders.iter().copied());
if db::has_claimable_metadata_jobs(&conn, &excluded)? {
return Ok(true);
}
}
if own_tier > WorkerTier::Embedding {
let mut excluded = paused_folder_ids("embedding");
excluded.extend(active_folders.iter().copied());
if db::has_claimable_embedding_jobs(&conn, &excluded)? {
return Ok(true);
}
}
Ok(false)
}
#[derive(Clone, Serialize)] #[derive(Clone, Serialize)]
pub struct IndexProgress { pub struct IndexProgress {
pub folder_id: i64, pub folder_id: i64,
@@ -204,19 +248,31 @@ pub fn start_thumbnail_worker(
cache_dir: PathBuf, cache_dir: PathBuf,
) { ) {
std::thread::spawn(move || loop { std::thread::spawn(move || loop {
if let Err(error) = process_thumbnail_batch(&app, &pool, &media_tools, &cache_dir) { // Only back off when the queue is empty (or errored); while jobs are
eprintln!("Thumbnail worker error: {}", error); // pending, claim the next batch immediately.
match process_thumbnail_batch(&app, &pool, &media_tools, &cache_dir) {
Ok(true) => {}
Ok(false) => std::thread::sleep(std::time::Duration::from_millis(250)),
Err(error) => {
eprintln!("Thumbnail worker error: {}", error);
std::thread::sleep(std::time::Duration::from_millis(250));
}
} }
std::thread::sleep(std::time::Duration::from_millis(250));
}); });
} }
pub fn start_metadata_worker(app: AppHandle, pool: DbPool, media_tools: MediaTools) { pub fn start_metadata_worker(app: AppHandle, pool: DbPool, media_tools: MediaTools) {
std::thread::spawn(move || loop { std::thread::spawn(move || loop {
if let Err(error) = process_metadata_batch(&app, &pool, &media_tools) { // Only back off when the queue is empty (or errored); while jobs are
eprintln!("Metadata worker error: {}", error); // pending, claim the next batch immediately.
match process_metadata_batch(&app, &pool, &media_tools) {
Ok(true) => {}
Ok(false) => std::thread::sleep(std::time::Duration::from_millis(250)),
Err(error) => {
eprintln!("Metadata worker error: {}", error);
std::thread::sleep(std::time::Duration::from_millis(250));
}
} }
std::thread::sleep(std::time::Duration::from_millis(250));
}); });
} }
@@ -225,10 +281,16 @@ pub fn start_embedding_worker(app: AppHandle, pool: DbPool) {
let mut embedder: Option<ClipImageEmbedder> = None; let mut embedder: Option<ClipImageEmbedder> = None;
println!("Embedding worker started."); println!("Embedding worker started.");
loop { loop {
if let Err(error) = process_embedding_batch(&app, &pool, &mut embedder) { // Only back off when the queue is empty (or errored); while jobs
eprintln!("Embedding worker error: {}", error); // are pending, claim the next batch immediately.
match process_embedding_batch(&app, &pool, &mut embedder) {
Ok(true) => {}
Ok(false) => std::thread::sleep(std::time::Duration::from_millis(500)),
Err(error) => {
eprintln!("Embedding worker error: {}", error);
std::thread::sleep(std::time::Duration::from_millis(500));
}
} }
std::thread::sleep(std::time::Duration::from_millis(500));
} }
}); });
} }
@@ -264,13 +326,17 @@ pub fn start_tagging_worker(app: AppHandle, pool: DbPool, app_data_dir: PathBuf)
println!("Tagging worker: acceleration setting changed — resetting session."); println!("Tagging worker: acceleration setting changed — resetting session.");
tagger_instance = None; tagger_instance = None;
} }
if let Err(error) = // Only back off when the queue is empty (or errored); while jobs
process_tagging_batch(&app, &pool, &app_data_dir, &mut tagger_instance) // are pending, claim the next batch immediately.
{ match process_tagging_batch(&app, &pool, &app_data_dir, &mut tagger_instance) {
eprintln!("Tagging worker error: {}", error); Ok(true) => {}
tagger_instance = None; Ok(false) => std::thread::sleep(std::time::Duration::from_millis(750)),
Err(error) => {
eprintln!("Tagging worker error: {}", error);
tagger_instance = None;
std::thread::sleep(std::time::Duration::from_millis(750));
}
} }
std::thread::sleep(std::time::Duration::from_millis(750));
} }
}); });
} }
@@ -554,12 +620,14 @@ fn commit_batch(pool: &DbPool, records: &[ImageRecord]) -> Result<Vec<ImageRecor
Ok(committed) Ok(committed)
} }
/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
/// the queue was empty.
fn process_thumbnail_batch( fn process_thumbnail_batch(
app: &AppHandle, app: &AppHandle,
pool: &DbPool, pool: &DbPool,
media_tools: &MediaTools, media_tools: &MediaTools,
cache_dir: &Path, cache_dir: &Path,
) -> Result<()> { ) -> Result<bool> {
let jobs = { let jobs = {
with_db_write_lock(|| { with_db_write_lock(|| {
let mut conn = pool.get()?; let mut conn = pool.get()?;
@@ -578,7 +646,7 @@ fn process_thumbnail_batch(
}; };
if jobs.is_empty() { if jobs.is_empty() {
return Ok(()); return Ok(false);
} }
println!("Thumbnail batch claimed: {} items", jobs.len()); println!("Thumbnail batch claimed: {} items", jobs.len());
@@ -586,33 +654,39 @@ fn process_thumbnail_batch(
let (image_jobs, video_jobs): (Vec<_>, Vec<_>) = let (image_jobs, video_jobs): (Vec<_>, Vec<_>) =
jobs.into_iter().partition(|job| job.media_kind == "image"); jobs.into_iter().partition(|job| job.media_kind == "image");
let mut results = image_jobs // Images: parallel decode, committed as one batch.
.par_iter() if !image_jobs.is_empty() {
.map(|job| { let results = image_jobs
( .par_iter()
job.image_id, .map(|job| {
if job.media_kind == "image" { (
thumbnail::generate_image_thumbnail(Path::new(&job.path), cache_dir).map(Some) job.image_id,
} else { thumbnail::generate_image_thumbnail(Path::new(&job.path), cache_dir).map(Some),
thumbnail::generate_video_thumbnail( )
media_tools, })
Path::new(&job.path), .collect::<Vec<_>>();
cache_dir, persist_thumbnail_results(app, pool, results)?;
)
.map(Some)
},
)
})
.collect::<Vec<_>>();
for job in video_jobs {
results.push((
job.image_id,
thumbnail::generate_video_thumbnail(media_tools, Path::new(&job.path), cache_dir)
.map(Some),
));
} }
// Videos: sequential, off the rayon pool — each ffmpeg call blocks its
// thread, and a video-heavy batch on the shared pool would starve image
// decoding across all workers. Committed per item so progress keeps
// moving through slow stretches.
for job in &video_jobs {
let result =
thumbnail::generate_video_thumbnail(media_tools, Path::new(&job.path), cache_dir)
.map(Some);
persist_thumbnail_results(app, pool, vec![(job.image_id, result)])?;
}
Ok(true)
}
fn persist_thumbnail_results(
app: &AppHandle,
pool: &DbPool,
results: Vec<(i64, anyhow::Result<Option<thumbnail::GeneratedThumbnail>>)>,
) -> Result<()> {
let updated_images = { let updated_images = {
with_db_write_lock(|| { with_db_write_lock(|| {
let mut conn = pool.get()?; let mut conn = pool.get()?;
@@ -665,7 +739,17 @@ fn process_thumbnail_batch(
Ok(()) Ok(())
} }
fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaTools) -> Result<()> { /// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
/// the queue was empty.
fn process_metadata_batch(
app: &AppHandle,
pool: &DbPool,
media_tools: &MediaTools,
) -> Result<bool> {
if higher_priority_work_pending(pool, WorkerTier::Metadata)? {
return Ok(false);
}
let jobs = { let jobs = {
with_db_write_lock(|| { with_db_write_lock(|| {
let mut conn = pool.get()?; let mut conn = pool.get()?;
@@ -684,83 +768,78 @@ fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaToo
}; };
if jobs.is_empty() { if jobs.is_empty() {
return Ok(()); return Ok(false);
} }
let results = jobs // Probes run sequentially (each ffprobe blocks on its process); results
.into_iter() // are committed per item so progress keeps moving through slow stretches.
.map(|job| { for job in jobs {
( let metadata_result = probe_video_metadata(media_tools, Path::new(&job.path));
job.image_id,
probe_video_metadata(media_tools, Path::new(&job.path)),
)
})
.collect::<Vec<_>>();
let updated_images = { let updated_image = with_db_write_lock(|| {
with_db_write_lock(|| {
let mut conn = pool.get()?; let mut conn = pool.get()?;
let tx = conn.transaction()?; let tx = conn.transaction()?;
let mut updated_images = Vec::new(); let updated = match metadata_result {
Ok(metadata) => Some(db::mark_metadata_ready(
for (image_id, metadata_result) in results {
let metadata = match metadata_result {
Ok(metadata) => metadata,
Err(error) => {
db::mark_metadata_failed(&tx, image_id, &error.to_string())?;
continue;
}
};
updated_images.push(db::mark_metadata_ready(
&tx, &tx,
image_id, job.image_id,
metadata.duration_ms, metadata.duration_ms,
metadata.width, metadata.width,
metadata.height, metadata.height,
metadata.video_codec.as_deref(), metadata.video_codec.as_deref(),
metadata.audio_codec.as_deref(), metadata.audio_codec.as_deref(),
)?); )?),
} Err(error) => {
db::mark_metadata_failed(&tx, job.image_id, &error.to_string())?;
None
}
};
tx.commit()?; tx.commit()?;
Ok(updated_images) Ok(updated)
})? })?;
};
if !updated_images.is_empty() { if let Some(image) = updated_image {
let folder_ids = updated_images let folder_id = image.folder_id;
.iter() emit_media_updates(
.map(|image| image.folder_id) app,
.collect::<HashSet<_>>(); &MediaUpdateBatch {
emit_media_updates( images: vec![image],
app, },
&MediaUpdateBatch { );
images: updated_images, emit_folder_job_progress(app, pool, &[folder_id], false);
}, }
);
emit_folder_job_progress(app, pool, &folder_ids.into_iter().collect::<Vec<_>>(), true);
} }
Ok(()) Ok(true)
} }
/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
/// the queue was empty.
fn process_embedding_batch( fn process_embedding_batch(
app: &AppHandle, app: &AppHandle,
pool: &DbPool, pool: &DbPool,
embedder: &mut Option<ClipImageEmbedder>, embedder: &mut Option<ClipImageEmbedder>,
) -> Result<()> { ) -> Result<bool> {
if higher_priority_work_pending(pool, WorkerTier::Embedding)? {
return Ok(false);
}
let batch_started_at = Instant::now(); let batch_started_at = Instant::now();
let claim_started_at = Instant::now(); let claim_started_at = Instant::now();
let paused_folders = paused_folder_ids("embedding"); // Exclude folders that are actively indexing (matching the thumbnail and
// metadata workers): embedding mid-scan competes with the scanner for
// CPU, disk, and the DB writer, and video jobs would fail-fast anyway
// because their thumbnails are deferred until indexing completes.
let mut excluded_folders = paused_folder_ids("embedding");
excluded_folders.extend(active_indexing_folders());
let jobs = with_db_write_lock(|| { let jobs = with_db_write_lock(|| {
let mut conn = pool.get()?; let mut conn = pool.get()?;
db::claim_embedding_jobs(&mut conn, &paused_folders, EMBEDDING_BATCH_SIZE) db::claim_embedding_jobs(&mut conn, &excluded_folders, EMBEDDING_BATCH_SIZE)
})?; })?;
let claim_elapsed = claim_started_at.elapsed(); let claim_elapsed = claim_started_at.elapsed();
if jobs.is_empty() { if jobs.is_empty() {
return Ok(()); return Ok(false);
} }
if embedder.is_none() { if embedder.is_none() {
@@ -888,7 +967,7 @@ fn process_embedding_batch(
EMBEDDING_BATCH_SIZE, claim_elapsed, infer_elapsed, write_elapsed, batch_elapsed EMBEDDING_BATCH_SIZE, claim_elapsed, infer_elapsed, write_elapsed, batch_elapsed
); );
Ok(()) Ok(true)
} }
fn process_caption_batch( fn process_caption_batch(
@@ -993,25 +1072,34 @@ fn process_caption_batch(
Ok(()) Ok(())
} }
/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
/// the queue was empty or the model is not ready.
fn process_tagging_batch( fn process_tagging_batch(
app: &AppHandle, app: &AppHandle,
pool: &DbPool, pool: &DbPool,
app_data_dir: &Path, app_data_dir: &Path,
tagger_instance: &mut Option<WdTagger>, tagger_instance: &mut Option<WdTagger>,
) -> Result<()> { ) -> Result<bool> {
if !tagger::tagger_model_status(app_data_dir).ready { if !tagger::tagger_model_status(app_data_dir).ready {
return Ok(()); return Ok(false);
} }
let paused_folders = paused_folder_ids("tagging"); if higher_priority_work_pending(pool, WorkerTier::Tagging)? {
return Ok(false);
}
// Exclude actively-indexing folders for the same reason as the other
// workers: don't compete with a running scan.
let mut excluded_folders = paused_folder_ids("tagging");
excluded_folders.extend(active_indexing_folders());
let batch_size = crate::tagger::tagger_batch_size(app_data_dir); let batch_size = crate::tagger::tagger_batch_size(app_data_dir);
let jobs = with_db_write_lock(|| { let jobs = with_db_write_lock(|| {
let mut conn = pool.get()?; let mut conn = pool.get()?;
db::claim_tagging_jobs(&mut conn, &paused_folders, batch_size) db::claim_tagging_jobs(&mut conn, &excluded_folders, batch_size)
})?; })?;
if jobs.is_empty() { if jobs.is_empty() {
return Ok(()); return Ok(false);
} }
if tagger_instance.is_none() { if tagger_instance.is_none() {
@@ -1119,7 +1207,7 @@ fn process_tagging_batch(
emit_folder_job_progress(app, pool, &folder_ids.into_iter().collect::<Vec<_>>(), true); emit_folder_job_progress(app, pool, &folder_ids.into_iter().collect::<Vec<_>>(), true);
} }
Ok(()) Ok(true)
} }
fn active_indexing_folders() -> HashSet<i64> { fn active_indexing_folders() -> HashSet<i64> {
@@ -1347,7 +1435,7 @@ impl WatcherHandle {
/// - `recv()` when no events are pending — zero CPU /// - `recv()` when no events are pending — zero CPU
/// - `recv_timeout(earliest_deadline)` when events are pending — wakes exactly /// - `recv_timeout(earliest_deadline)` when events are pending — wakes exactly
/// when the soonest debounce window expires, no busy-polling /// when the soonest debounce window expires, no busy-polling
pub fn start_watcher(app: AppHandle, pool: DbPool) -> WatcherHandle { pub fn start_watcher(app: AppHandle, pool: DbPool, thumb_dir: PathBuf) -> WatcherHandle {
let (tx, rx) = std::sync::mpsc::channel::<notify::Result<notify::Event>>(); let (tx, rx) = std::sync::mpsc::channel::<notify::Result<notify::Event>>();
let raw_watcher = notify::recommended_watcher(move |result| { let raw_watcher = notify::recommended_watcher(move |result| {
@@ -1392,17 +1480,19 @@ pub fn start_watcher(app: AppHandle, pool: DbPool) -> WatcherHandle {
std::thread::spawn(move || { std::thread::spawn(move || {
// path → deadline: the earliest instant at which this path should be processed. // path → deadline: the earliest instant at which this path should be processed.
let mut pending: HashMap<PathBuf, Instant> = HashMap::new(); let mut pending: HashMap<PathBuf, Instant> = HashMap::new();
// old_path → new_path for rename events that carry both sides atomically.
let mut pending_renames: Vec<(PathBuf, PathBuf)> = Vec::new();
loop { loop {
// Adaptive blocking: block forever when idle, wake at the earliest // Adaptive blocking: block forever when idle, wake at the earliest
// deadline when events are queued. // deadline when events are queued.
let received = if pending.is_empty() { let received = if pending.is_empty() && pending_renames.is_empty() {
match rx.recv() { match rx.recv() {
Ok(e) => Some(e), Ok(e) => Some(e),
Err(_) => break, // sender dropped — app is shutting down Err(_) => break, // sender dropped — app is shutting down
} }
} else { } else {
let earliest = pending.values().copied().min().unwrap(); // safe: non-empty let earliest = pending.values().copied().min().unwrap_or(Instant::now());
let timeout = earliest.saturating_duration_since(Instant::now()); let timeout = earliest.saturating_duration_since(Instant::now());
match rx.recv_timeout(timeout) { match rx.recv_timeout(timeout) {
Ok(e) => Some(e), Ok(e) => Some(e),
@@ -1415,17 +1505,40 @@ pub fn start_watcher(app: AppHandle, pool: DbPool) -> WatcherHandle {
// Absorb incoming event — coalesces rapid writes into one deadline. // Absorb incoming event — coalesces rapid writes into one deadline.
if let Some(Ok(event)) = received { if let Some(Ok(event)) = received {
use notify::EventKind; use notify::{EventKind, event::{ModifyKind, RenameMode}};
// Skip pure access events (reads); they never change file content.
if !matches!(event.kind, EventKind::Access(_)) { if !matches!(event.kind, EventKind::Access(_)) {
for path in event.paths { // Intercept atomic rename events (old + new path in one event).
if is_supported_media(&path) { // Handle these separately to preserve embeddings and thumbnails.
pending.insert(path, now + WATCHER_DEBOUNCE); if matches!(
event.kind,
EventKind::Modify(ModifyKind::Name(RenameMode::Both))
) && event.paths.len() == 2
{
let old = event.paths[0].clone();
let new = event.paths[1].clone();
if is_supported_media(&old) || is_supported_media(&new) {
// Remove either side from regular pending so it isn't
// processed as an independent delete/create.
pending.remove(&old);
pending.remove(&new);
pending_renames.push((old, new));
}
} else {
for path in event.paths {
if is_supported_media(&path) {
pending.insert(path, now + WATCHER_DEBOUNCE);
}
} }
} }
} }
} }
// Process renames immediately — they are atomic filesystem operations
// and do not benefit from debouncing.
for (old_path, new_path) in pending_renames.drain(..) {
process_watcher_rename(&app, &pool, &folder_map_thread, &thumb_dir, &old_path, &new_path);
}
// Process all paths whose debounce window has expired. // Process all paths whose debounce window has expired.
let ready: Vec<PathBuf> = pending let ready: Vec<PathBuf> = pending
.iter() .iter()
@@ -1496,11 +1609,14 @@ fn process_watcher_path(
Err(e) => eprintln!("Watcher: commit error for {:?}: {}", path, e), Err(e) => eprintln!("Watcher: commit error for {:?}: {}", path, e),
} }
} else { } else {
// File removed from disk — clean up DB row. // File removed from disk — clean up DB row and thumbnail.
let path_str = path.to_string_lossy(); let path_str = path.to_string_lossy();
match db::get_image_id_by_path(&conn, &path_str) { match db::get_image_id_and_thumbnail_by_path(&conn, &path_str) {
Ok(Some(image_id)) => { Ok(Some((image_id, thumb_path))) => {
if db::delete_images_by_ids(&conn, &[image_id]).is_ok() { if db::delete_images_by_ids(&conn, &[image_id]).is_ok() {
if let Some(thumb) = thumb_path {
let _ = std::fs::remove_file(thumb);
}
db::update_folder_count(&conn, folder_id).ok(); db::update_folder_count(&conn, folder_id).ok();
let _ = app.emit("watcher-deleted", vec![image_id]); let _ = app.emit("watcher-deleted", vec![image_id]);
let _ = app.emit("folder-counts-changed", ()); let _ = app.emit("folder-counts-changed", ());
@@ -1511,3 +1627,76 @@ fn process_watcher_path(
} }
} }
} }
/// Handles a filesystem rename/move event where both the old and new paths are
/// known. Updates the DB row in-place (preserving the embedding) and renames
/// the thumbnail file to match the new path hash.
fn process_watcher_rename(
app: &AppHandle,
pool: &DbPool,
folder_map: &Arc<Mutex<HashMap<PathBuf, i64>>>,
thumb_dir: &Path,
old_path: &Path,
new_path: &Path,
) {
// Resolve folder_id from the old path first, fall back to new path.
let folder_id = {
let map = folder_map.lock().unwrap();
map.iter()
.find(|(fp, _)| old_path.starts_with(fp.as_path()) || new_path.starts_with(fp.as_path()))
.map(|(_, &id)| id)
};
let Some(folder_id) = folder_id else { return };
let conn = match pool.get() {
Ok(c) => c,
Err(e) => {
eprintln!("Watcher rename: DB pool error: {}", e);
return;
}
};
let old_path_str = old_path.to_string_lossy();
let new_path_str = new_path.to_string_lossy();
let (image_id, old_thumb) = match db::get_image_id_and_thumbnail_by_path(&conn, &old_path_str) {
Ok(Some(record)) => record,
Ok(None) => {
// Not yet indexed under the old path — treat as a brand-new file.
drop(conn);
process_watcher_path(app, pool, folder_map, new_path);
return;
}
Err(e) => {
eprintln!("Watcher rename: DB lookup error: {}", e);
return;
}
};
// Compute new thumbnail path and attempt to rename the file on disk.
let new_thumb_path = crate::thumbnail::thumb_path(thumb_dir, &new_path_str);
let new_thumb_str = new_thumb_path.to_string_lossy().into_owned();
let thumb_ok = old_thumb
.as_deref()
.map(|old| std::fs::rename(old, &new_thumb_path).is_ok())
.unwrap_or(false);
// If rename failed (e.g. old thumb never existed), clear thumbnail_path so
// the thumbnail worker regenerates it.
let effective_thumb: Option<&str> = if thumb_ok { Some(&new_thumb_str) } else { None };
let new_filename = new_path
.file_name()
.and_then(|f| f.to_str())
.unwrap_or("");
if let Err(e) = db::update_image_path(&conn, image_id, &new_path_str, new_filename, effective_thumb) {
eprintln!("Watcher rename: DB update error: {}", e);
return;
}
// Emit the updated record so the frontend refreshes without a full reload.
match db::get_image_by_id(&conn, image_id) {
Ok(record) => emit_images(app, &IndexedImagesBatch { folder_id, images: vec![record] }),
Err(e) => eprintln!("Watcher rename: post-update fetch error: {}", e),
}
}
+5 -1
View File
@@ -74,7 +74,7 @@ pub fn run() {
// indexer::start_caption_worker(app.handle().clone(), pool.clone(), app_dir.clone()); // indexer::start_caption_worker(app.handle().clone(), pool.clone(), app_dir.clone());
indexer::start_tagging_worker(app.handle().clone(), pool.clone(), app_dir.clone()); indexer::start_tagging_worker(app.handle().clone(), pool.clone(), app_dir.clone());
let watcher_handle = indexer::start_watcher(app.handle().clone(), pool.clone()); let watcher_handle = indexer::start_watcher(app.handle().clone(), pool.clone(), thumb_dir.clone());
app.manage(pool); app.manage(pool);
app.manage(media_tools); app.manage(media_tools);
@@ -148,6 +148,10 @@ pub fn run() {
commands::vacuum_database, commands::vacuum_database,
commands::get_orphaned_thumbnails_info, commands::get_orphaned_thumbnails_info,
commands::cleanup_orphaned_thumbnails, commands::cleanup_orphaned_thumbnails,
commands::get_muted_folder_ids,
commands::set_muted_folder_ids,
commands::get_notifications_paused,
commands::set_notifications_paused,
]) ])
.run(tauri::generate_context!()) .run(tauri::generate_context!())
.expect("error while running tauri application"); .expect("error while running tauri application");
+150 -12
View File
@@ -31,29 +31,26 @@ pub fn generate_image_thumbnail(image_path: &Path, cache_dir: &Path) -> Result<G
}); });
} }
let reader = image::ImageReader::open(image_path)?.with_guessed_format()?; let img = decode_for_thumbnail(image_path)?;
let mut decoder = reader.into_decoder()?;
let orientation = decoder.orientation()?;
let mut img = image::DynamicImage::from_decoder(decoder)?;
img.apply_orientation(orientation);
let src = image::DynamicImage::ImageRgba8(img.into_rgba8()); // RGB8 throughout: JPEG output has no alpha, so decoding/resizing RGBA
// only to flatten at encode time wastes a third of the bandwidth.
let src = image::DynamicImage::ImageRgb8(img.into_rgb8());
let (dst_width, dst_height) = fit_dimensions(src.width(), src.height(), THUMB_SIZE); let (dst_width, dst_height) = fit_dimensions(src.width(), src.height(), THUMB_SIZE);
let mut dst = fir::images::Image::new(dst_width, dst_height, src.pixel_type().unwrap()); let mut dst = fir::images::Image::new(dst_width, dst_height, src.pixel_type().unwrap());
let mut resizer = fir::Resizer::new(); let mut resizer = fir::Resizer::new();
let options = fir::ResizeOptions::new() let options = fir::ResizeOptions::new()
.resize_alg(fir::ResizeAlg::Convolution(fir::FilterType::Lanczos3)); .resize_alg(fir::ResizeAlg::Convolution(fir::FilterType::CatmullRom));
resizer.resize(&src, &mut dst, Some(&options))?; resizer.resize(&src, &mut dst, Some(&options))?;
let thumb = image::RgbaImage::from_raw(dst_width, dst_height, dst.buffer().to_vec()) let thumb = image::RgbImage::from_raw(dst_width, dst_height, dst.buffer().to_vec())
.ok_or_else(|| anyhow!("failed to construct resized thumbnail buffer"))?; .ok_or_else(|| anyhow!("failed to construct resized thumbnail buffer"))?;
if let Some(parent) = out_path.parent() { if let Some(parent) = out_path.parent() {
std::fs::create_dir_all(parent)?; std::fs::create_dir_all(parent)?;
} }
let thumb = image::DynamicImage::ImageRgba8(thumb).into_rgb8();
let mut output_file = std::fs::File::create(&out_path)?; let mut output_file = std::fs::File::create(&out_path)?;
let mut encoder = JpegEncoder::new_with_quality(&mut output_file, 82); let mut encoder = JpegEncoder::new_with_quality(&mut output_file, 82);
encoder.encode_image(&thumb)?; encoder.encode_image(&thumb)?;
@@ -64,6 +61,98 @@ pub fn generate_image_thumbnail(image_path: &Path, cache_dir: &Path) -> Result<G
}) })
} }
/// Decodes an image for thumbnailing, with EXIF orientation already applied.
fn decode_for_thumbnail(image_path: &Path) -> Result<image::DynamicImage> {
decode_image_scaled(image_path, THUMB_SIZE, false)
}
/// Decodes an image at reduced resolution, with EXIF orientation applied.
///
/// JPEGs take a fast path that decodes at reduced resolution (DCT scaling),
/// which skips most of the IDCT/upsampling work for large photos. Anything
/// that path can't handle falls back to the `image` crate at full size.
///
/// `cover` selects which side must stay at or above `target`: `false` keeps
/// the longest side (for aspect-fit consumers), `true` keeps the shortest
/// side (for fill-crop consumers like the CLIP preprocessor).
pub fn decode_image_scaled(
image_path: &Path,
target: u32,
cover: bool,
) -> Result<image::DynamicImage> {
if is_jpeg(image_path) {
if let Some(img) = decode_jpeg_scaled(image_path, target, cover) {
return Ok(img);
}
}
let reader = image::ImageReader::open(image_path)?.with_guessed_format()?;
let mut decoder = reader.into_decoder()?;
let orientation = decoder.orientation()?;
let mut img = image::DynamicImage::from_decoder(decoder)?;
img.apply_orientation(orientation);
Ok(img)
}
fn is_jpeg(path: &Path) -> bool {
path.extension()
.and_then(|ext| ext.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("jpg") || ext.eq_ignore_ascii_case("jpeg"))
}
/// Decodes a JPEG at the smallest DCT scale that still covers `target`.
/// Returns `None` on any failure (corrupt file, CMYK without conversion
/// support, etc.) so the caller can fall back to the generic decoder.
/// mozjpeg reports errors by panicking, hence the `catch_unwind`.
fn decode_jpeg_scaled(image_path: &Path, target: u32, cover: bool) -> Option<image::DynamicImage> {
let path = image_path.to_path_buf();
let decoded = std::panic::catch_unwind(move || -> std::io::Result<(Vec<u8>, u32, u32)> {
let mut decompress = mozjpeg::Decompress::new_path(&path)?;
let (width, height) = decompress.size();
let reference = if cover {
width.min(height)
} else {
width.max(height)
};
decompress.scale(scale_numerator(reference, target));
let mut started = decompress.rgb()?;
let (out_width, out_height) = (started.width() as u32, started.height() as u32);
let pixels = started.read_scanlines::<u8>()?;
started.finish()?;
Ok((pixels, out_width, out_height))
})
.ok()?
.ok()?;
let (pixels, width, height) = decoded;
let buffer = image::RgbImage::from_raw(width, height, pixels)?;
let mut img = image::DynamicImage::ImageRgb8(buffer);
if let Some(orientation) = exif_orientation(image_path) {
img.apply_orientation(orientation);
}
Some(img)
}
/// Smallest numerator (of /8) that keeps `reference` at or above `target`,
/// so the subsequent resize never upscales.
fn scale_numerator(reference: usize, target: u32) -> u8 {
for numerator in 1..=8u8 {
if reference * numerator as usize / 8 >= target as usize {
return numerator;
}
}
8
}
fn exif_orientation(path: &Path) -> Option<image::metadata::Orientation> {
let file = std::fs::File::open(path).ok()?;
let mut reader = std::io::BufReader::new(file);
let exif = exif::Reader::new().read_from_container(&mut reader).ok()?;
let field = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY)?;
let value = field.value.get_uint(0)?;
image::metadata::Orientation::from_exif(value as u8)
}
pub fn generate_video_thumbnail( pub fn generate_video_thumbnail(
tools: &MediaTools, tools: &MediaTools,
video_path: &Path, video_path: &Path,
@@ -88,6 +177,8 @@ pub fn generate_video_thumbnail(
let attempts: [&[&str]; 3] = [ let attempts: [&[&str]; 3] = [
&[ &[
"-y", "-y",
"-threads",
"2",
"-ss", "-ss",
"00:00:00.000", "00:00:00.000",
"-i", "-i",
@@ -95,13 +186,15 @@ pub fn generate_video_thumbnail(
"-frames:v", "-frames:v",
"1", "1",
"-vf", "-vf",
"thumbnail,scale=320:-1:force_original_aspect_ratio=decrease", "scale=320:-1:force_original_aspect_ratio=decrease",
"-q:v", "-q:v",
"4", "4",
&output_path, &output_path,
], ],
&[ &[
"-y", "-y",
"-threads",
"2",
"-ss", "-ss",
"00:00:00.250", "00:00:00.250",
"-i", "-i",
@@ -109,19 +202,21 @@ pub fn generate_video_thumbnail(
"-frames:v", "-frames:v",
"1", "1",
"-vf", "-vf",
"thumbnail,scale=320:-1:force_original_aspect_ratio=decrease", "scale=320:-1:force_original_aspect_ratio=decrease",
"-q:v", "-q:v",
"4", "4",
&output_path, &output_path,
], ],
&[ &[
"-y", "-y",
"-threads",
"2",
"-i", "-i",
path_str.as_ref(), path_str.as_ref(),
"-frames:v", "-frames:v",
"1", "1",
"-vf", "-vf",
"thumbnail,scale=320:-1:force_original_aspect_ratio=decrease", "scale=320:-1:force_original_aspect_ratio=decrease",
"-q:v", "-q:v",
"4", "4",
&output_path, &output_path,
@@ -165,6 +260,49 @@ fn hash_path(s: &str) -> String {
format!("{:016x}", xxhash_rust::xxh3::xxh3_64(s.as_bytes())) format!("{:016x}", xxhash_rust::xxh3::xxh3_64(s.as_bytes()))
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn scale_numerator_picks_smallest_sufficient() {
assert_eq!(scale_numerator(6000, THUMB_SIZE), 1);
assert_eq!(scale_numerator(640, THUMB_SIZE), 4);
assert_eq!(scale_numerator(320, THUMB_SIZE), 8);
assert_eq!(scale_numerator(100, THUMB_SIZE), 8);
// Cover mode reference: shortest side must reach 224 for CLIP.
assert_eq!(scale_numerator(1200, 224), 2);
}
#[test]
fn jpeg_fast_path_decodes_at_reduced_resolution() {
let dir = std::env::temp_dir().join("phokus-thumb-test");
std::fs::create_dir_all(&dir).unwrap();
let src_path = dir.join("large.jpg");
let img =
image::RgbImage::from_fn(1600, 1200, |x, _| image::Rgb([(x % 256) as u8, 64, 128]));
img.save(&src_path).unwrap();
// 1600 max dim -> numerator 2 -> 400x300 decode output
let decoded = decode_jpeg_scaled(&src_path, THUMB_SIZE, false)
.expect("fast path should handle plain JPEG");
assert_eq!((decoded.width(), decoded.height()), (400, 300));
// Cover mode: shortest side (1200) must stay >= 224 -> numerator 2
let covered = decode_jpeg_scaled(&src_path, 224, true)
.expect("fast path should handle plain JPEG");
assert_eq!((covered.width(), covered.height()), (400, 300));
let cache = dir.join("cache");
let _ = std::fs::remove_file(thumb_path(&cache, &src_path.to_string_lossy()));
let result = generate_image_thumbnail(&src_path, &cache).unwrap();
assert_eq!(result.width, Some(1600));
assert_eq!(result.height, Some(1200));
let (thumb_w, thumb_h) = image::image_dimensions(&result.path).unwrap();
assert_eq!((thumb_w, thumb_h), (320, 240));
}
}
fn fit_dimensions(width: u32, height: u32, max_size: u32) -> (u32, u32) { fn fit_dimensions(width: u32, height: u32, max_size: u32) -> (u32, u32) {
if width <= max_size && height <= max_size { if width <= max_size && height <= max_size {
return (width.max(1), height.max(1)); return (width.max(1), height.max(1));
+4
View File
@@ -18,11 +18,15 @@ export default function App() {
const loadImages = useGalleryStore((state) => state.loadImages); const loadImages = useGalleryStore((state) => state.loadImages);
const loadCaptionModelStatus = useGalleryStore((state) => state.loadCaptionModelStatus); const loadCaptionModelStatus = useGalleryStore((state) => state.loadCaptionModelStatus);
const loadDuplicateScanCache = useGalleryStore((state) => state.loadDuplicateScanCache); const loadDuplicateScanCache = useGalleryStore((state) => state.loadDuplicateScanCache);
const loadMutedFolderIds = useGalleryStore((state) => state.loadMutedFolderIds);
const loadNotificationsPaused = useGalleryStore((state) => state.loadNotificationsPaused);
const subscribeToProgress = useGalleryStore((state) => state.subscribeToProgress); const subscribeToProgress = useGalleryStore((state) => state.subscribeToProgress);
const activeView = useGalleryStore((state) => state.activeView); const activeView = useGalleryStore((state) => state.activeView);
useEffect(() => { useEffect(() => {
void initializeNotifications(); void initializeNotifications();
void loadMutedFolderIds();
void loadNotificationsPaused();
loadFolders().then(() => { loadFolders().then(() => {
void loadBackgroundJobProgress(); void loadBackgroundJobProgress();
void loadCaptionModelStatus(); void loadCaptionModelStatus();
+11 -5
View File
@@ -278,12 +278,16 @@ export function BackgroundTasks() {
id: -1, id: -1,
name: "Duplicate Scan", name: "Duplicate Scan",
stages: [{ stages: [{
label: "Hashing", label: duplicateScanProgress?.phase === "checking"
? "Checking"
: duplicateScanProgress?.phase === "confirming"
? "Confirming"
: "Hashing",
detail: duplicateScanProgress detail: duplicateScanProgress
? `${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}` ? `${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}`
: "Starting…", : "Starting…",
progress: duplicateScanProgress && duplicateScanProgress.total > 0 progress: duplicateScanProgress && duplicateScanProgress.total > 0
? (duplicateScanProgress.scanned / duplicateScanProgress.total) * 100 ? (duplicateScanProgress.processed / duplicateScanProgress.total) * 100
: null, : null,
failed: false, failed: false,
}], }],
@@ -310,7 +314,8 @@ export function BackgroundTasks() {
const embeddingStage = primary.stages.find((s) => s.label === "Embeddings"); const embeddingStage = primary.stages.find((s) => s.label === "Embeddings");
const taggingStage = primary.stages.find((s) => s.label === "Tags"); const taggingStage = primary.stages.find((s) => s.label === "Tags");
const scanningStage = primary.stages.find((s) => s.label === "Scanning"); const scanningStage = primary.stages.find((s) => s.label === "Scanning");
const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? null; const duplicateStage = primary.id === -1 ? primary.stages[0] : null;
const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? duplicateStage?.progress ?? null;
return ( return (
<div className="shrink-0 border-b border-white/[0.06]"> <div className="shrink-0 border-b border-white/[0.06]">
@@ -434,7 +439,8 @@ export function BackgroundTasks() {
const taskEmbeddingStage = task.stages.find((s) => s.label === "Embeddings"); const taskEmbeddingStage = task.stages.find((s) => s.label === "Embeddings");
const taskTaggingStage = task.stages.find((s) => s.label === "Tags"); const taskTaggingStage = task.stages.find((s) => s.label === "Tags");
const taskScanningStage = task.stages.find((s) => s.label === "Scanning"); const taskScanningStage = task.stages.find((s) => s.label === "Scanning");
const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? null; const taskDuplicateStage = task.id === -1 ? task.stages[0] : null;
const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? taskDuplicateStage?.progress ?? null;
const taskHasFailed = (task.hasFailedEmbeddings || task.hasFailedTagging || task.hasFailedCaptions) && task.pendingMediaWork === 0; const taskHasFailed = (task.hasFailedEmbeddings || task.hasFailedTagging || task.hasFailedCaptions) && task.pendingMediaWork === 0;
return ( return (
+14 -3
View File
@@ -117,6 +117,7 @@ export function DuplicateFinder() {
const duplicateScanning = useGalleryStore((state) => state.duplicateScanning); const duplicateScanning = useGalleryStore((state) => state.duplicateScanning);
const duplicateScanProgress = useGalleryStore((state) => state.duplicateScanProgress); const duplicateScanProgress = useGalleryStore((state) => state.duplicateScanProgress);
const duplicateScanError = useGalleryStore((state) => state.duplicateScanError); const duplicateScanError = useGalleryStore((state) => state.duplicateScanError);
const duplicateScanWarning = useGalleryStore((state) => state.duplicateScanWarning);
const duplicateSelectedIds = useGalleryStore((state) => state.duplicateSelectedIds); const duplicateSelectedIds = useGalleryStore((state) => state.duplicateSelectedIds);
const duplicateLastScanned = useGalleryStore((state) => state.duplicateLastScanned); const duplicateLastScanned = useGalleryStore((state) => state.duplicateLastScanned);
const selectedFolderId = useGalleryStore((state) => state.selectedFolderId); const selectedFolderId = useGalleryStore((state) => state.selectedFolderId);
@@ -152,8 +153,15 @@ export function DuplicateFinder() {
const progressPercent = const progressPercent =
duplicateScanProgress && duplicateScanProgress.total > 0 duplicateScanProgress && duplicateScanProgress.total > 0
? Math.round((duplicateScanProgress.scanned / duplicateScanProgress.total) * 100) ? Math.round((duplicateScanProgress.processed / duplicateScanProgress.total) * 100)
: 0; : 0;
const progressLabel = duplicateScanProgress
? duplicateScanProgress.phase === "checking"
? "Checking file sizes"
: duplicateScanProgress.phase === "hashing"
? "Hashing duplicate candidates"
: "Confirming exact matches"
: null;
return ( return (
<div className="flex min-h-0 flex-1 flex-col overflow-hidden bg-[#07080f]"> <div className="flex min-h-0 flex-1 flex-col overflow-hidden bg-[#07080f]">
@@ -165,7 +173,7 @@ export function DuplicateFinder() {
<p className="mt-0.5 text-[11px] text-white/30"> <p className="mt-0.5 text-[11px] text-white/30">
{duplicateScanning {duplicateScanning
? duplicateScanProgress ? duplicateScanProgress
? `Scanning${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}` ? `${progressLabel}${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}`
: "Starting scan…" : "Starting scan…"
: hasResults : hasResults
? `${duplicateGroups.length} group${duplicateGroups.length === 1 ? "" : "s"} · ${formatBytes(totalWasted)} reclaimable` ? `${duplicateGroups.length} group${duplicateGroups.length === 1 ? "" : "s"} · ${formatBytes(totalWasted)} reclaimable`
@@ -232,6 +240,9 @@ export function DuplicateFinder() {
{duplicateScanError ? ( {duplicateScanError ? (
<p className="mt-2 text-[11px] text-red-400/80">{duplicateScanError}</p> <p className="mt-2 text-[11px] text-red-400/80">{duplicateScanError}</p>
) : null} ) : null}
{duplicateScanWarning ? (
<p className="mt-2 text-[11px] text-amber-300/70">{duplicateScanWarning}</p>
) : null}
{deleteResult ? ( {deleteResult ? (
<p className="mt-2 text-[11px] text-white/40">{deleteResult}</p> <p className="mt-2 text-[11px] text-white/40">{deleteResult}</p>
) : null} ) : null}
@@ -241,7 +252,7 @@ export function DuplicateFinder() {
{duplicateScanning && !hasResults ? ( {duplicateScanning && !hasResults ? (
<div className="flex flex-1 items-center justify-center gap-3 text-white/25"> <div className="flex flex-1 items-center justify-center gap-3 text-white/25">
<div className="h-5 w-5 animate-spin rounded-full border-2 border-white/15 border-t-white/50" /> <div className="h-5 w-5 animate-spin rounded-full border-2 border-white/15 border-t-white/50" />
<span className="text-sm">Hashing files</span> <span className="text-sm">{progressLabel ? `${progressLabel}` : "Preparing scan…"}</span>
</div> </div>
) : !hasScanned ? ( ) : !hasScanned ? (
<div className="flex flex-1 items-center justify-center px-8"> <div className="flex flex-1 items-center justify-center px-8">
+46 -16
View File
@@ -162,6 +162,8 @@ export function SettingsModal() {
const clearTaggingJobs = useGalleryStore((state) => state.clearTaggingJobs); const clearTaggingJobs = useGalleryStore((state) => state.clearTaggingJobs);
const clearTaggingJobsForFolders = useGalleryStore((state) => state.clearTaggingJobsForFolders); const clearTaggingJobsForFolders = useGalleryStore((state) => state.clearTaggingJobsForFolders);
const openAppDataFolder = useGalleryStore((state) => state.openAppDataFolder); const openAppDataFolder = useGalleryStore((state) => state.openAppDataFolder);
const notificationsPaused = useGalleryStore((state) => state.notificationsPaused);
const setNotificationsPaused = useGalleryStore((state) => state.setNotificationsPaused);
const getDatabaseInfo = useGalleryStore((state) => state.getDatabaseInfo); const getDatabaseInfo = useGalleryStore((state) => state.getDatabaseInfo);
const vacuumDatabase = useGalleryStore((state) => state.vacuumDatabase); const vacuumDatabase = useGalleryStore((state) => state.vacuumDatabase);
const getOrphanedThumbnailsInfo = useGalleryStore((state) => state.getOrphanedThumbnailsInfo); const getOrphanedThumbnailsInfo = useGalleryStore((state) => state.getOrphanedThumbnailsInfo);
@@ -601,6 +603,26 @@ export function SettingsModal() {
</div> </div>
</SettingsCard> </SettingsCard>
<SettingsCard
title="Notifications"
description="Notifications are batched per folder — a single alert fires once activity settles. Mute individual folders from their right-click menu."
>
<div className="flex items-center justify-between gap-4 rounded-xl border border-white/[0.07] bg-black/20 p-4">
<div>
<p className="text-sm font-medium text-white">Pause all notifications</p>
<p className="mt-0.5 text-xs text-gray-500">Suppress all indexing notifications until re-enabled.</p>
</div>
<button
role="switch"
aria-checked={notificationsPaused}
className={`relative inline-flex h-5 w-9 shrink-0 cursor-pointer rounded-full border-2 border-transparent transition-colors focus:outline-none ${notificationsPaused ? "bg-sky-500" : "bg-white/15"}`}
onClick={() => setNotificationsPaused(!notificationsPaused)}
>
<span className={`inline-block h-4 w-4 transform rounded-full bg-white shadow transition-transform ${notificationsPaused ? "translate-x-4" : "translate-x-0"}`} />
</button>
</div>
</SettingsCard>
<SettingsCard <SettingsCard
title="Compact database" title="Compact database"
description="Reclaims wasted space left behind when images or tags are deleted. Safe to run at any time." description="Reclaims wasted space left behind when images or tags are deleted. Safe to run at any time."
@@ -666,31 +688,39 @@ export function SettingsModal() {
<div className="rounded-xl border border-white/[0.07] bg-black/20 p-4"> <div className="rounded-xl border border-white/[0.07] bg-black/20 p-4">
<p className="text-[11px] uppercase tracking-[0.14em] text-gray-600">Orphaned files</p> <p className="text-[11px] uppercase tracking-[0.14em] text-gray-600">Orphaned files</p>
<p className="mt-2 text-2xl font-semibold text-white"> <p className="mt-2 text-2xl font-semibold text-white">
{thumbnailCleanupResult {cleaningThumbnails
? thumbnailCleanupResult.deleted_count.toLocaleString() ? "—"
: thumbnailInfo : thumbnailCleanupResult
? thumbnailInfo.count.toLocaleString() ? "0"
: "—"} : thumbnailInfo
? thumbnailInfo.count.toLocaleString()
: "—"}
</p> </p>
</div> </div>
<div className="rounded-xl border border-white/[0.07] bg-black/20 p-4"> <div className="rounded-xl border border-white/[0.07] bg-black/20 p-4">
<p className="text-[11px] uppercase tracking-[0.14em] text-gray-600">Reclaimable</p> <p className="text-[11px] uppercase tracking-[0.14em] text-gray-600">Reclaimable</p>
<p className={`mt-2 text-2xl font-semibold ${thumbnailCleanupResult ? "text-emerald-300" : "text-white"}`}> <p className={`mt-2 text-2xl font-semibold ${thumbnailCleanupResult ? "text-emerald-300" : "text-white"}`}>
{thumbnailCleanupResult {cleaningThumbnails
? `${thumbnailCleanupResult.freed_mb.toFixed(1)} MB freed` ? "—"
: thumbnailInfo : thumbnailCleanupResult
? `${thumbnailInfo.size_mb.toFixed(1)} MB` ? "0 MB"
: "—"} : thumbnailInfo
? `${thumbnailInfo.size_mb.toFixed(1)} MB`
: "—"}
</p> </p>
</div> </div>
</div> </div>
<div className="flex items-center justify-between gap-4 rounded-xl border border-white/[0.07] bg-black/20 p-4"> <div className="flex items-center justify-between gap-4 rounded-xl border border-white/[0.07] bg-black/20 p-4">
<p className="text-sm text-gray-400"> <p className="text-sm text-gray-400">
{thumbnailCleanupResult {cleaningThumbnails
? `Removed ${thumbnailCleanupResult.deleted_count.toLocaleString()} orphaned thumbnail${thumbnailCleanupResult.deleted_count === 1 ? "" : "s"}.` ? "Scanning and removing orphaned thumbnails…"
: thumbnailInfo && thumbnailInfo.count === 0 : thumbnailCleanupResult
? "No orphaned thumbnails found." ? `Removed ${thumbnailCleanupResult.deleted_count.toLocaleString()} file${thumbnailCleanupResult.deleted_count === 1 ? "" : "s"}, freed ${thumbnailCleanupResult.freed_mb.toFixed(1)} MB.`
: "Remove thumbnails no longer associated with any indexed image."} : thumbnailInfo && thumbnailInfo.count === 0
? "No orphaned thumbnails found."
: thumbnailInfo && thumbnailInfo.count > 1000
? "Remove thumbnails no longer associated with any indexed image. This may take a few minutes for large collections."
: "Remove thumbnails no longer associated with any indexed image."}
</p> </p>
<button <button
className="shrink-0 rounded-lg bg-white/[0.07] px-4 py-2 text-sm font-medium text-white transition-colors hover:bg-white/[0.11] disabled:cursor-not-allowed disabled:opacity-40" className="shrink-0 rounded-lg bg-white/[0.07] px-4 py-2 text-sm font-medium text-white transition-colors hover:bg-white/[0.11] disabled:cursor-not-allowed disabled:opacity-40"
@@ -706,7 +736,7 @@ export function SettingsModal() {
}} }}
disabled={cleaningThumbnails || thumbnailCleanupResult !== null || (thumbnailInfo !== null && thumbnailInfo.count === 0)} disabled={cleaningThumbnails || thumbnailCleanupResult !== null || (thumbnailInfo !== null && thumbnailInfo.count === 0)}
> >
{cleaningThumbnails ? "Cleaning..." : "Clean up"} {cleaningThumbnails ? "Cleaning" : "Clean up"}
</button> </button>
</div> </div>
</div> </div>
+10 -1
View File
@@ -11,18 +11,22 @@ interface ContextMenuState {
function FolderContextMenu({ function FolderContextMenu({
menu, menu,
folder, folder,
isMuted,
onClose, onClose,
onRename, onRename,
onReindex, onReindex,
onLocate, onLocate,
onToggleMute,
onRemove, onRemove,
}: { }: {
menu: ContextMenuState; menu: ContextMenuState;
folder: Folder; folder: Folder;
isMuted: boolean;
onClose: () => void; onClose: () => void;
onRename: () => void; onRename: () => void;
onReindex: () => void; onReindex: () => void;
onLocate: () => void; onLocate: () => void;
onToggleMute: () => void;
onRemove: () => void; onRemove: () => void;
}) { }) {
const ref = useRef<HTMLDivElement>(null); const ref = useRef<HTMLDivElement>(null);
@@ -61,6 +65,7 @@ function FolderContextMenu({
> >
{item("Reindex", onReindex)} {item("Reindex", onReindex)}
{item("Rename", onRename)} {item("Rename", onRename)}
{item(isMuted ? "Unmute notifications" : "Mute notifications", onToggleMute)}
{folder.scan_error && item("Locate Folder", onLocate)} {folder.scan_error && item("Locate Folder", onLocate)}
<div className="my-1 border-t border-white/[0.06]" /> <div className="my-1 border-t border-white/[0.06]" />
{item("Remove from app", onRemove, true)} {item("Remove from app", onRemove, true)}
@@ -77,7 +82,9 @@ function FolderItem({
selected: boolean; selected: boolean;
progress: IndexProgress | undefined; progress: IndexProgress | undefined;
}) { }) {
const { selectFolder, removeFolder, reindexFolder, renameFolder, updateFolderPath } = useGalleryStore(); const { selectFolder, removeFolder, reindexFolder, renameFolder, updateFolderPath, toggleMutedFolder } = useGalleryStore();
const mutedFolderIds = useGalleryStore((state) => state.mutedFolderIds);
const isMuted = mutedFolderIds.includes(folder.id);
const isIndexing = progress && !progress.done; const isIndexing = progress && !progress.done;
const isMissing = !!folder.scan_error && !isIndexing; const isMissing = !!folder.scan_error && !isIndexing;
@@ -250,10 +257,12 @@ function FolderItem({
<FolderContextMenu <FolderContextMenu
menu={contextMenu} menu={contextMenu}
folder={folder} folder={folder}
isMuted={isMuted}
onClose={() => setContextMenu(null)} onClose={() => setContextMenu(null)}
onRename={() => setRenaming(true)} onRename={() => setRenaming(true)}
onReindex={() => void reindexFolder(folder.id)} onReindex={() => void reindexFolder(folder.id)}
onLocate={() => void handleLocateFolder()} onLocate={() => void handleLocateFolder()}
onToggleMute={() => toggleMutedFolder(folder.id)}
onRemove={() => setConfirmingRemoval(true)} onRemove={() => setConfirmingRemoval(true)}
/> />
)} )}
+146 -36
View File
@@ -4,6 +4,11 @@ import { listen, UnlistenFn } from "@tauri-apps/api/event";
import { appDataDir, join } from "@tauri-apps/api/path"; import { appDataDir, join } from "@tauri-apps/api/path";
import { notifyTaskComplete } from "./notifications"; import { notifyTaskComplete } from "./notifications";
// Per-folder debounce timers for batching notifications.
// Keyed as `${folderId}:embedding` or `${folderId}:tagging`.
const notificationTimers = new Map<string, ReturnType<typeof setTimeout>>();
const NOTIFICATION_DEBOUNCE_MS = 6000;
export interface Folder { export interface Folder {
id: number; id: number;
path: string; path: string;
@@ -165,6 +170,20 @@ export interface DuplicateGroup {
images: ImageRecord[]; images: ImageRecord[];
} }
export interface DuplicateScanProgress {
phase: "checking" | "hashing" | "confirming";
processed: number;
total: number;
skipped: number;
}
interface DuplicateScanResult {
groups: DuplicateGroup[];
scanned_files: number;
candidate_files: number;
skipped_files: number;
}
export interface SimilarImagesPage { export interface SimilarImagesPage {
images: ImageRecord[]; images: ImageRecord[];
offset: number; offset: number;
@@ -288,6 +307,8 @@ interface GalleryState {
settingsOpen: boolean; settingsOpen: boolean;
taggingQueueScope: TaggingQueueScope; taggingQueueScope: TaggingQueueScope;
taggingQueueFolderIds: number[]; taggingQueueFolderIds: number[];
mutedFolderIds: number[];
notificationsPaused: boolean;
taggerModelStatus: TaggerModelStatus | null; taggerModelStatus: TaggerModelStatus | null;
taggerModelPreparing: boolean; taggerModelPreparing: boolean;
@@ -301,8 +322,9 @@ interface GalleryState {
duplicateGroups: DuplicateGroup[]; duplicateGroups: DuplicateGroup[];
duplicateScanning: boolean; duplicateScanning: boolean;
duplicateScanProgress: { scanned: number; total: number } | null; duplicateScanProgress: DuplicateScanProgress | null;
duplicateScanError: string | null; duplicateScanError: string | null;
duplicateScanWarning: string | null;
duplicateSelectedIds: Set<number>; duplicateSelectedIds: Set<number>;
duplicateLastScanned: number | null; // Unix timestamp (seconds) duplicateLastScanned: number | null; // Unix timestamp (seconds)
duplicateScanFolderId: number | null | undefined; // undefined = never scanned duplicateScanFolderId: number | null | undefined; // undefined = never scanned
@@ -360,6 +382,10 @@ interface GalleryState {
loadTaggingQueueFolderIds: () => Promise<void>; loadTaggingQueueFolderIds: () => Promise<void>;
toggleTaggingQueueFolder: (folderId: number) => void; toggleTaggingQueueFolder: (folderId: number) => void;
setTaggingQueueFolderIds: (folderIds: number[]) => void; setTaggingQueueFolderIds: (folderIds: number[]) => void;
loadMutedFolderIds: () => Promise<void>;
toggleMutedFolder: (folderId: number) => void;
loadNotificationsPaused: () => Promise<void>;
setNotificationsPaused: (paused: boolean) => void;
openAppDataFolder: () => Promise<void>; openAppDataFolder: () => Promise<void>;
getDatabaseInfo: () => Promise<DatabaseInfo>; getDatabaseInfo: () => Promise<DatabaseInfo>;
vacuumDatabase: () => Promise<VacuumResult>; vacuumDatabase: () => Promise<VacuumResult>;
@@ -635,6 +661,8 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
settingsOpen: false, settingsOpen: false,
taggingQueueScope: "all", taggingQueueScope: "all",
taggingQueueFolderIds: [], taggingQueueFolderIds: [],
mutedFolderIds: [],
notificationsPaused: false,
taggerModelStatus: null, taggerModelStatus: null,
taggerModelPreparing: false, taggerModelPreparing: false,
@@ -650,6 +678,7 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
duplicateScanning: false, duplicateScanning: false,
duplicateScanProgress: null, duplicateScanProgress: null,
duplicateScanError: null, duplicateScanError: null,
duplicateScanWarning: null,
duplicateSelectedIds: new Set(), duplicateSelectedIds: new Set(),
duplicateLastScanned: null, duplicateLastScanned: null,
duplicateScanFolderId: undefined, duplicateScanFolderId: undefined,
@@ -941,7 +970,13 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
if (activeView === "duplicates") { if (activeView === "duplicates") {
const { selectedFolderId, duplicateScanFolderId } = get(); const { selectedFolderId, duplicateScanFolderId } = get();
if (duplicateScanFolderId !== selectedFolderId) { if (duplicateScanFolderId !== selectedFolderId) {
set({ activeView, duplicateGroups: [], duplicateLastScanned: null, duplicateScanFolderId: undefined }); set({
activeView,
duplicateGroups: [],
duplicateLastScanned: null,
duplicateScanFolderId: undefined,
duplicateScanWarning: null,
});
void get().loadDuplicateScanCache(selectedFolderId); void get().loadDuplicateScanCache(selectedFolderId);
return; return;
} }
@@ -1371,6 +1406,39 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
void invoke("set_tagging_queue_folder_ids", { folder_ids: taggingQueueFolderIds }).catch(() => {}); void invoke("set_tagging_queue_folder_ids", { folder_ids: taggingQueueFolderIds }).catch(() => {});
}, },
loadMutedFolderIds: async () => {
try {
const folderIds = await invoke<number[]>("get_muted_folder_ids");
set({ mutedFolderIds: folderIds });
} catch {
// fall back to in-memory default
}
},
toggleMutedFolder: (folderId) => {
set((state) => {
const next = state.mutedFolderIds.includes(folderId)
? state.mutedFolderIds.filter((id) => id !== folderId)
: [...state.mutedFolderIds, folderId];
void invoke("set_muted_folder_ids", { folder_ids: next }).catch(() => {});
return { mutedFolderIds: next };
});
},
loadNotificationsPaused: async () => {
try {
const paused = await invoke<boolean>("get_notifications_paused");
set({ notificationsPaused: paused });
} catch {
// fall back to in-memory default
}
},
setNotificationsPaused: (paused) => {
set({ notificationsPaused: paused });
void invoke("set_notifications_paused", { paused }).catch(() => {});
},
openAppDataFolder: async () => { openAppDataFolder: async () => {
await invoke("open_app_data_folder"); await invoke("open_app_data_folder");
}, },
@@ -1537,23 +1605,42 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
interface CacheResult { groups: DuplicateGroup[]; scanned_at: number } interface CacheResult { groups: DuplicateGroup[]; scanned_at: number }
const cached = await invoke<CacheResult | null>("load_duplicate_scan_cache", { folderId: folderId ?? null }); const cached = await invoke<CacheResult | null>("load_duplicate_scan_cache", { folderId: folderId ?? null });
if (cached) { if (cached) {
set({ duplicateGroups: cached.groups, duplicateLastScanned: cached.scanned_at, duplicateScanFolderId: folderId }); set({
duplicateGroups: cached.groups,
duplicateLastScanned: cached.scanned_at,
duplicateScanFolderId: folderId,
duplicateScanWarning: null,
});
} }
}, },
scanDuplicates: async (folderId = null) => { scanDuplicates: async (folderId = null) => {
const { listen } = await import("@tauri-apps/api/event"); const { listen } = await import("@tauri-apps/api/event");
set({ duplicateScanning: true, duplicateGroups: [], duplicateScanProgress: null, duplicateScanError: null, duplicateSelectedIds: new Set() }); set({
const unlisten = await listen<[number, number]>("duplicate_scan_progress", (event) => { duplicateScanning: true,
const [scanned, total] = event.payload; duplicateGroups: [],
set({ duplicateScanProgress: { scanned, total } }); duplicateScanProgress: null,
duplicateScanError: null,
duplicateScanWarning: null,
duplicateSelectedIds: new Set(),
});
const unlisten = await listen<DuplicateScanProgress>("duplicate_scan_progress", (event) => {
set({ duplicateScanProgress: event.payload });
}); });
try { try {
const groups = await invoke<DuplicateGroup[]>("find_duplicates", { folderId: folderId ?? null }); const result = await invoke<DuplicateScanResult>("find_duplicates", { folderId: folderId ?? null });
set({ duplicateGroups: groups, duplicateLastScanned: Math.floor(Date.now() / 1000), duplicateScanFolderId: folderId }); const warning = result.skipped_files > 0
? `${result.skipped_files.toLocaleString()} file${result.skipped_files === 1 ? "" : "s"} could not be read and were skipped.`
: null;
set({
duplicateGroups: result.groups,
duplicateLastScanned: Math.floor(Date.now() / 1000),
duplicateScanFolderId: folderId,
duplicateScanWarning: warning,
});
void notifyTaskComplete( void notifyTaskComplete(
"Duplicate scan complete", "Duplicate scan complete",
groups.length === 1 ? "Found 1 duplicate group." : `Found ${groups.length.toLocaleString()} duplicate groups.`, `${result.groups.length === 1 ? "Found 1 duplicate group." : `Found ${result.groups.length.toLocaleString()} duplicate groups.`}${warning ? ` ${warning}` : ""}`,
); );
} catch (e) { } catch (e) {
set({ duplicateScanError: String(e) }); set({ duplicateScanError: String(e) });
@@ -1659,11 +1746,14 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
progress.total > 0 && progress.total > 0 &&
progress.indexed >= progress.total progress.indexed >= progress.total
) { ) {
const folderName = get().folders.find((folder) => folder.id === progress.folder_id)?.name; const { notificationsPaused, mutedFolderIds } = get();
void notifyTaskComplete( if (!notificationsPaused && !mutedFolderIds.includes(progress.folder_id)) {
"Folder scan complete", const folderName = get().folders.find((folder) => folder.id === progress.folder_id)?.name;
folderName ? `${folderName} has finished scanning.` : "A folder has finished scanning.", void notifyTaskComplete(
); "Folder scan complete",
folderName ? `${folderName} has finished scanning.` : "A folder has finished scanning.",
);
}
} }
void get().loadFolders(); void get().loadFolders();
void get().loadBackgroundJobProgress(); void get().loadBackgroundJobProgress();
@@ -1688,32 +1778,52 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
const previous = previousProgress[progress.folder_id]; const previous = previousProgress[progress.folder_id];
if (!previous) continue; if (!previous) continue;
const { notificationsPaused, mutedFolderIds } = get();
const suppressed = notificationsPaused || mutedFolderIds.includes(progress.folder_id);
const folderName = const folderName =
get().folders.find((folder) => folder.id === progress.folder_id)?.name ?? "Folder"; get().folders.find((folder) => folder.id === progress.folder_id)?.name ?? "Folder";
if (previous.embedding_pending > 0 && progress.embedding_pending === 0) { // Embeddings — debounced so rapid file additions don't fire per-file.
const failureDetail = const embeddingKey = `${progress.folder_id}:embedding`;
progress.embedding_failed > 0 if (!suppressed) {
? ` ${progress.embedding_failed.toLocaleString()} failed.` if (previous.embedding_pending > 0 && progress.embedding_pending === 0) {
: ""; clearTimeout(notificationTimers.get(embeddingKey));
void notifyTaskComplete( const failureDetail =
"Embeddings complete", progress.embedding_failed > 0
`${folderName} finished generating embeddings.${failureDetail}`, ? ` ${progress.embedding_failed.toLocaleString()} failed.`
); : "";
const body = `${folderName} finished generating embeddings.${failureDetail}`;
notificationTimers.set(embeddingKey, setTimeout(() => {
notificationTimers.delete(embeddingKey);
void notifyTaskComplete("Embeddings complete", body);
}, NOTIFICATION_DEBOUNCE_MS));
} else if (previous.embedding_pending === 0 && progress.embedding_pending > 0) {
// More jobs queued — cancel the pending notification.
clearTimeout(notificationTimers.get(embeddingKey));
notificationTimers.delete(embeddingKey);
}
} }
if (previous.tagging_pending > 0 && progress.tagging_pending === 0) { // Tagging — same debounce pattern.
const failureDetail = const taggingKey = `${progress.folder_id}:tagging`;
progress.tagging_failed > 0 if (!suppressed) {
? ` ${progress.tagging_failed.toLocaleString()} failed.` if (previous.tagging_pending > 0 && progress.tagging_pending === 0) {
: ""; clearTimeout(notificationTimers.get(taggingKey));
void notifyTaskComplete( const failureDetail =
"AI tagging complete", progress.tagging_failed > 0
`${folderName} finished generating tags.${failureDetail}`, ? ` ${progress.tagging_failed.toLocaleString()} failed.`
); : "";
// New tags are now in the DB — invalidate the Explore tag cache so const body = `${folderName} finished generating tags.${failureDetail}`;
// reopening Explore reflects the updated tag distribution. notificationTimers.set(taggingKey, setTimeout(() => {
set({ exploreTagsFolderId: undefined }); notificationTimers.delete(taggingKey);
void notifyTaskComplete("AI tagging complete", body);
// New tags landed — invalidate Explore tag cache.
set({ exploreTagsFolderId: undefined });
}, NOTIFICATION_DEBOUNCE_MS));
} else if (previous.tagging_pending === 0 && progress.tagging_pending > 0) {
clearTimeout(notificationTimers.get(taggingKey));
notificationTimers.delete(taggingKey);
}
} }
} }