From 665c315f56d6354698f311b07b30cf090986e770 Mon Sep 17 00:00:00 2001 From: LyAhn Date: Fri, 12 Jun 2026 08:12:49 +0100 Subject: [PATCH] feat(duplicates): phased scan progress with skipped-file reporting Replace the [scanned, total] tuple progress event with a structured {phase, processed, total, skipped} payload covering all three scan phases (checking, hashing, confirming). find_duplicates now returns a DuplicateScanResult with scanned/candidate/skipped counts, and the UI surfaces phase labels plus a warning when unreadable files were skipped. Also adds a clean:app script for cargo clean. --- CLAUDE.md | 2 +- package.json | 1 + src-tauri/src/commands.rs | 142 +++++++++++++++++++++++++---- src/components/BackgroundTasks.tsx | 16 +++- src/components/DuplicateFinder.tsx | 17 +++- src/store.ts | 61 +++++++++++-- 6 files changed, 200 insertions(+), 39 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e9febb9..b5786c8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,7 +80,7 @@ Database: SQLite with WAL mode, stored in the Tauri app data directory as `galle | `media-updated` | `ThumbnailBatch` | | `caption-model-progress` | `CaptionModelProgress` | | `tagger-model-progress` | `TaggerModelProgress` | -| `duplicate_scan_progress` | `[scanned, total]` | +| `duplicate_scan_progress` | `{ phase, processed, total, skipped }` | ### Key types diff --git a/package.json b/package.json index 1836830..542a05d 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "build:app": "tauri build", "build:vite": "tsc && vite build", + "clean:app": "cd src-tauri && cargo clean", "dev:app": "tauri dev", "dev:vite": "vite", "preview": "vite preview", diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 14f8465..3451f92 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -182,6 +182,22 @@ pub struct DuplicateGroup { pub images: Vec, } +#[derive(Clone, Serialize)] +pub struct DuplicateScanProgress { + pub phase: String, + pub processed: usize, + pub total: usize, + pub skipped: usize, +} + +#[derive(Serialize)] +pub struct DuplicateScanResult { + pub groups: Vec, + pub scanned_files: usize, + pub candidate_files: usize, + pub skipped_files: usize, +} + #[derive(Serialize)] pub struct DuplicateScanCache { pub groups: Vec, @@ -962,14 +978,19 @@ pub async fn find_duplicates( app: AppHandle, db: State<'_, DbState>, folder_id: Option, -) -> Result, String> { +) -> Result { let records = { let conn = db.get().map_err(|e| e.to_string())?; db::get_all_image_paths(&conn, folder_id).map_err(|e| e.to_string())? }; let total = records.len(); - let _ = app.emit("duplicate_scan_progress", (0usize, total)); + let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "checking".to_string(), + processed: 0, + total, + skipped: 0, + }); // Three-phase detection. // @@ -986,7 +1007,8 @@ pub async fn find_duplicates( // For sample-hash groups that contain files > 64 KB, compute a full-file // hash to confirm the match before presenting them as deletable duplicates. let app_hash = app.clone(); - let pairs: Vec<(u64, u64, i64, String)> = tokio::task::spawn_blocking(move || { + let (pairs, skipped_before_confirm, candidate_total): + (Vec<(u64, u64, i64, String)>, usize, usize) = tokio::task::spawn_blocking(move || { use memmap2::Mmap; use rayon::prelude::*; use std::collections::HashMap; @@ -1012,14 +1034,31 @@ pub async fn find_duplicates( } // ── Phase 1: stat ───────────────────────────────────────────────── + let stat_counter = AtomicUsize::new(0); + let stat_skipped = AtomicUsize::new(0); let sized: Vec<(i64, String, u64)> = records .par_iter() .filter_map(|r| { - let size = std::fs::metadata(&r.path).ok()?.len(); - if size == 0 { return None; } - Some((r.id, r.path.clone(), size)) + let result = match std::fs::metadata(&r.path) { + Ok(metadata) => Some((r.id, r.path.clone(), metadata.len())), + Err(_) => { + stat_skipped.fetch_add(1, Ordering::Relaxed); + None + } + }; + let done = stat_counter.fetch_add(1, Ordering::Relaxed) + 1; + if done % 100 == 0 || done == total { + let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "checking".to_string(), + processed: done, + total, + skipped: stat_skipped.load(Ordering::Relaxed), + }); + } + result }) .collect(); + let stat_skipped = stat_skipped.load(Ordering::Relaxed); let mut by_size: HashMap> = HashMap::new(); for (i, (_, _, size)) in sized.iter().enumerate() { @@ -1032,29 +1071,53 @@ pub async fn find_duplicates( .collect(); if candidates.is_empty() { - return vec![]; + return (vec![], stat_skipped, 0); } // ── Phase 2: sample hash ────────────────────────────────────────── let c_total = candidates.len(); - let _ = app_hash.emit("duplicate_scan_progress", (0usize, c_total)); + let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "hashing".to_string(), + processed: 0, + total: c_total, + skipped: stat_skipped, + }); let counter = AtomicUsize::new(0); + let hash_skipped = AtomicUsize::new(0); - candidates + let pairs = candidates .par_iter() .filter_map(|&idx| { let (id, path, size) = &sized[idx]; - let file = std::fs::File::open(path).ok()?; - // SAFETY: read-only; no external truncation expected during scan. - let mmap = unsafe { Mmap::map(&file).ok()? }; - let hash = sample(&mmap); + let result = if *size == 0 { + Some((xxh3_64(&[]), *size, *id, path.clone())) + } else { + std::fs::File::open(path) + .ok() + // SAFETY: read-only; no external truncation expected during scan. + .and_then(|file| unsafe { Mmap::map(&file).ok() }) + .map(|mmap| (sample(&mmap), *size, *id, path.clone())) + }; + if result.is_none() { + hash_skipped.fetch_add(1, Ordering::Relaxed); + } let done = counter.fetch_add(1, Ordering::Relaxed) + 1; if done % 100 == 0 || done == c_total { - let _ = app_hash.emit("duplicate_scan_progress", (done, c_total)); + let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "hashing".to_string(), + processed: done, + total: c_total, + skipped: stat_skipped + hash_skipped.load(Ordering::Relaxed), + }); } - Some((hash, *size, *id, path.clone())) + result }) - .collect() + .collect(); + ( + pairs, + stat_skipped + hash_skipped.load(Ordering::Relaxed), + c_total, + ) }) .await .map_err(|e| e.to_string())?; @@ -1071,13 +1134,30 @@ pub async fn find_duplicates( // hash becomes its own DuplicateGroup so disjoint sets (A,A vs B,B that // happened to share size and sample hash) are never merged. const COVERED: u64 = (16 * 1024 * 4) as u64; - let confirmed: Vec<(u64, u64, Vec)> = + let confirming_total: usize = size_hash_map + .iter() + .filter(|((_, file_size), entries)| *file_size > COVERED && entries.len() > 1) + .map(|(_, entries)| entries.len()) + .sum(); + if confirming_total > 0 { + let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "confirming".to_string(), + processed: 0, + total: confirming_total, + skipped: skipped_before_confirm, + }); + } + let app_confirm = app.clone(); + let (confirmed, skipped_files): (Vec<(u64, u64, Vec)>, usize) = tokio::task::spawn_blocking(move || { use memmap2::Mmap; use rayon::prelude::*; + use std::sync::atomic::{AtomicUsize, Ordering}; use xxhash_rust::xxh3::xxh3_64; - size_hash_map + let counter = AtomicUsize::new(0); + let confirm_skipped = AtomicUsize::new(0); + let confirmed = size_hash_map .into_iter() .filter(|(_, entries)| entries.len() > 1) .flat_map(|((sample_hash, file_size), entries)| { @@ -1094,6 +1174,19 @@ pub async fn find_duplicates( .ok() .and_then(|f| unsafe { Mmap::map(&f).ok() }) .map(|mmap| xxh3_64(&mmap)); + if hash.is_none() { + confirm_skipped.fetch_add(1, Ordering::Relaxed); + } + let done = counter.fetch_add(1, Ordering::Relaxed) + 1; + if done % 100 == 0 || done == confirming_total { + let _ = app_confirm.emit("duplicate_scan_progress", DuplicateScanProgress { + phase: "confirming".to_string(), + processed: done, + total: confirming_total, + skipped: skipped_before_confirm + + confirm_skipped.load(Ordering::Relaxed), + }); + } (*id, hash) }) .collect(); @@ -1111,7 +1204,11 @@ pub async fn find_duplicates( .map(|(full_hash, ids)| (full_hash, file_size, ids)) .collect() }) - .collect() + .collect(); + ( + confirmed, + skipped_before_confirm + confirm_skipped.load(Ordering::Relaxed), + ) }) .await .map_err(|e| e.to_string())?; @@ -1142,7 +1239,12 @@ pub async fn find_duplicates( let _ = db::set_duplicate_scan_cache(&conn, &folder_scope, &json); } - Ok(groups) + Ok(DuplicateScanResult { + groups, + scanned_files: total, + candidate_files: candidate_total, + skipped_files, + }) } #[tauri::command] diff --git a/src/components/BackgroundTasks.tsx b/src/components/BackgroundTasks.tsx index d34ceb2..8930943 100644 --- a/src/components/BackgroundTasks.tsx +++ b/src/components/BackgroundTasks.tsx @@ -278,12 +278,16 @@ export function BackgroundTasks() { id: -1, name: "Duplicate Scan", stages: [{ - label: "Hashing", + label: duplicateScanProgress?.phase === "checking" + ? "Checking" + : duplicateScanProgress?.phase === "confirming" + ? "Confirming" + : "Hashing", detail: duplicateScanProgress - ? `${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}` + ? `${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}` : "Starting…", progress: duplicateScanProgress && duplicateScanProgress.total > 0 - ? (duplicateScanProgress.scanned / duplicateScanProgress.total) * 100 + ? (duplicateScanProgress.processed / duplicateScanProgress.total) * 100 : null, failed: false, }], @@ -310,7 +314,8 @@ export function BackgroundTasks() { const embeddingStage = primary.stages.find((s) => s.label === "Embeddings"); const taggingStage = primary.stages.find((s) => s.label === "Tags"); const scanningStage = primary.stages.find((s) => s.label === "Scanning"); - const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? null; + const duplicateStage = primary.id === -1 ? primary.stages[0] : null; + const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? duplicateStage?.progress ?? null; return (
@@ -434,7 +439,8 @@ export function BackgroundTasks() { const taskEmbeddingStage = task.stages.find((s) => s.label === "Embeddings"); const taskTaggingStage = task.stages.find((s) => s.label === "Tags"); const taskScanningStage = task.stages.find((s) => s.label === "Scanning"); - const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? null; + const taskDuplicateStage = task.id === -1 ? task.stages[0] : null; + const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? taskDuplicateStage?.progress ?? null; const taskHasFailed = (task.hasFailedEmbeddings || task.hasFailedTagging || task.hasFailedCaptions) && task.pendingMediaWork === 0; return ( diff --git a/src/components/DuplicateFinder.tsx b/src/components/DuplicateFinder.tsx index cae277f..67b361b 100644 --- a/src/components/DuplicateFinder.tsx +++ b/src/components/DuplicateFinder.tsx @@ -117,6 +117,7 @@ export function DuplicateFinder() { const duplicateScanning = useGalleryStore((state) => state.duplicateScanning); const duplicateScanProgress = useGalleryStore((state) => state.duplicateScanProgress); const duplicateScanError = useGalleryStore((state) => state.duplicateScanError); + const duplicateScanWarning = useGalleryStore((state) => state.duplicateScanWarning); const duplicateSelectedIds = useGalleryStore((state) => state.duplicateSelectedIds); const duplicateLastScanned = useGalleryStore((state) => state.duplicateLastScanned); const selectedFolderId = useGalleryStore((state) => state.selectedFolderId); @@ -152,8 +153,15 @@ export function DuplicateFinder() { const progressPercent = duplicateScanProgress && duplicateScanProgress.total > 0 - ? Math.round((duplicateScanProgress.scanned / duplicateScanProgress.total) * 100) + ? Math.round((duplicateScanProgress.processed / duplicateScanProgress.total) * 100) : 0; + const progressLabel = duplicateScanProgress + ? duplicateScanProgress.phase === "checking" + ? "Checking file sizes" + : duplicateScanProgress.phase === "hashing" + ? "Hashing duplicate candidates" + : "Confirming exact matches" + : null; return (
@@ -165,7 +173,7 @@ export function DuplicateFinder() {

{duplicateScanning ? duplicateScanProgress - ? `Scanning… ${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}` + ? `${progressLabel}… ${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}` : "Starting scan…" : hasResults ? `${duplicateGroups.length} group${duplicateGroups.length === 1 ? "" : "s"} · ${formatBytes(totalWasted)} reclaimable` @@ -232,6 +240,9 @@ export function DuplicateFinder() { {duplicateScanError ? (

{duplicateScanError}

) : null} + {duplicateScanWarning ? ( +

{duplicateScanWarning}

+ ) : null} {deleteResult ? (

{deleteResult}

) : null} @@ -241,7 +252,7 @@ export function DuplicateFinder() { {duplicateScanning && !hasResults ? (
- Hashing files… + {progressLabel ? `${progressLabel}…` : "Preparing scan…"}
) : !hasScanned ? (
diff --git a/src/store.ts b/src/store.ts index 479533d..1fd5eb6 100644 --- a/src/store.ts +++ b/src/store.ts @@ -170,6 +170,20 @@ export interface DuplicateGroup { images: ImageRecord[]; } +export interface DuplicateScanProgress { + phase: "checking" | "hashing" | "confirming"; + processed: number; + total: number; + skipped: number; +} + +interface DuplicateScanResult { + groups: DuplicateGroup[]; + scanned_files: number; + candidate_files: number; + skipped_files: number; +} + export interface SimilarImagesPage { images: ImageRecord[]; offset: number; @@ -308,8 +322,9 @@ interface GalleryState { duplicateGroups: DuplicateGroup[]; duplicateScanning: boolean; - duplicateScanProgress: { scanned: number; total: number } | null; + duplicateScanProgress: DuplicateScanProgress | null; duplicateScanError: string | null; + duplicateScanWarning: string | null; duplicateSelectedIds: Set; duplicateLastScanned: number | null; // Unix timestamp (seconds) duplicateScanFolderId: number | null | undefined; // undefined = never scanned @@ -663,6 +678,7 @@ export const useGalleryStore = create((set, get) => ({ duplicateScanning: false, duplicateScanProgress: null, duplicateScanError: null, + duplicateScanWarning: null, duplicateSelectedIds: new Set(), duplicateLastScanned: null, duplicateScanFolderId: undefined, @@ -954,7 +970,13 @@ export const useGalleryStore = create((set, get) => ({ if (activeView === "duplicates") { const { selectedFolderId, duplicateScanFolderId } = get(); if (duplicateScanFolderId !== selectedFolderId) { - set({ activeView, duplicateGroups: [], duplicateLastScanned: null, duplicateScanFolderId: undefined }); + set({ + activeView, + duplicateGroups: [], + duplicateLastScanned: null, + duplicateScanFolderId: undefined, + duplicateScanWarning: null, + }); void get().loadDuplicateScanCache(selectedFolderId); return; } @@ -1583,23 +1605,42 @@ export const useGalleryStore = create((set, get) => ({ interface CacheResult { groups: DuplicateGroup[]; scanned_at: number } const cached = await invoke("load_duplicate_scan_cache", { folderId: folderId ?? null }); if (cached) { - set({ duplicateGroups: cached.groups, duplicateLastScanned: cached.scanned_at, duplicateScanFolderId: folderId }); + set({ + duplicateGroups: cached.groups, + duplicateLastScanned: cached.scanned_at, + duplicateScanFolderId: folderId, + duplicateScanWarning: null, + }); } }, scanDuplicates: async (folderId = null) => { const { listen } = await import("@tauri-apps/api/event"); - set({ duplicateScanning: true, duplicateGroups: [], duplicateScanProgress: null, duplicateScanError: null, duplicateSelectedIds: new Set() }); - const unlisten = await listen<[number, number]>("duplicate_scan_progress", (event) => { - const [scanned, total] = event.payload; - set({ duplicateScanProgress: { scanned, total } }); + set({ + duplicateScanning: true, + duplicateGroups: [], + duplicateScanProgress: null, + duplicateScanError: null, + duplicateScanWarning: null, + duplicateSelectedIds: new Set(), + }); + const unlisten = await listen("duplicate_scan_progress", (event) => { + set({ duplicateScanProgress: event.payload }); }); try { - const groups = await invoke("find_duplicates", { folderId: folderId ?? null }); - set({ duplicateGroups: groups, duplicateLastScanned: Math.floor(Date.now() / 1000), duplicateScanFolderId: folderId }); + const result = await invoke("find_duplicates", { folderId: folderId ?? null }); + const warning = result.skipped_files > 0 + ? `${result.skipped_files.toLocaleString()} file${result.skipped_files === 1 ? "" : "s"} could not be read and were skipped.` + : null; + set({ + duplicateGroups: result.groups, + duplicateLastScanned: Math.floor(Date.now() / 1000), + duplicateScanFolderId: folderId, + duplicateScanWarning: warning, + }); void notifyTaskComplete( "Duplicate scan complete", - groups.length === 1 ? "Found 1 duplicate group." : `Found ${groups.length.toLocaleString()} duplicate groups.`, + `${result.groups.length === 1 ? "Found 1 duplicate group." : `Found ${result.groups.length.toLocaleString()} duplicate groups.`}${warning ? ` ${warning}` : ""}`, ); } catch (e) { set({ duplicateScanError: String(e) });