feat(duplicates): phased scan progress with skipped-file reporting
Replace the [scanned, total] tuple progress event with a structured
{phase, processed, total, skipped} payload covering all three scan
phases (checking, hashing, confirming). find_duplicates now returns a
DuplicateScanResult with scanned/candidate/skipped counts, and the UI
surfaces phase labels plus a warning when unreadable files were
skipped. Also adds a clean:app script for cargo clean.
This commit is contained in:
@@ -80,7 +80,7 @@ Database: SQLite with WAL mode, stored in the Tauri app data directory as `galle
|
||||
| `media-updated` | `ThumbnailBatch` |
|
||||
| `caption-model-progress` | `CaptionModelProgress` |
|
||||
| `tagger-model-progress` | `TaggerModelProgress` |
|
||||
| `duplicate_scan_progress` | `[scanned, total]` |
|
||||
| `duplicate_scan_progress` | `{ phase, processed, total, skipped }` |
|
||||
|
||||
### Key types
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
"scripts": {
|
||||
"build:app": "tauri build",
|
||||
"build:vite": "tsc && vite build",
|
||||
"clean:app": "cd src-tauri && cargo clean",
|
||||
"dev:app": "tauri dev",
|
||||
"dev:vite": "vite",
|
||||
"preview": "vite preview",
|
||||
|
||||
+121
-19
@@ -182,6 +182,22 @@ pub struct DuplicateGroup {
|
||||
pub images: Vec<ImageRecord>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
pub struct DuplicateScanProgress {
|
||||
pub phase: String,
|
||||
pub processed: usize,
|
||||
pub total: usize,
|
||||
pub skipped: usize,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct DuplicateScanResult {
|
||||
pub groups: Vec<DuplicateGroup>,
|
||||
pub scanned_files: usize,
|
||||
pub candidate_files: usize,
|
||||
pub skipped_files: usize,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct DuplicateScanCache {
|
||||
pub groups: Vec<DuplicateGroup>,
|
||||
@@ -962,14 +978,19 @@ pub async fn find_duplicates(
|
||||
app: AppHandle,
|
||||
db: State<'_, DbState>,
|
||||
folder_id: Option<i64>,
|
||||
) -> Result<Vec<DuplicateGroup>, String> {
|
||||
) -> Result<DuplicateScanResult, String> {
|
||||
let records = {
|
||||
let conn = db.get().map_err(|e| e.to_string())?;
|
||||
db::get_all_image_paths(&conn, folder_id).map_err(|e| e.to_string())?
|
||||
};
|
||||
|
||||
let total = records.len();
|
||||
let _ = app.emit("duplicate_scan_progress", (0usize, total));
|
||||
let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||
phase: "checking".to_string(),
|
||||
processed: 0,
|
||||
total,
|
||||
skipped: 0,
|
||||
});
|
||||
|
||||
// Three-phase detection.
|
||||
//
|
||||
@@ -986,7 +1007,8 @@ pub async fn find_duplicates(
|
||||
// For sample-hash groups that contain files > 64 KB, compute a full-file
|
||||
// hash to confirm the match before presenting them as deletable duplicates.
|
||||
let app_hash = app.clone();
|
||||
let pairs: Vec<(u64, u64, i64, String)> = tokio::task::spawn_blocking(move || {
|
||||
let (pairs, skipped_before_confirm, candidate_total):
|
||||
(Vec<(u64, u64, i64, String)>, usize, usize) = tokio::task::spawn_blocking(move || {
|
||||
use memmap2::Mmap;
|
||||
use rayon::prelude::*;
|
||||
use std::collections::HashMap;
|
||||
@@ -1012,14 +1034,31 @@ pub async fn find_duplicates(
|
||||
}
|
||||
|
||||
// ── Phase 1: stat ─────────────────────────────────────────────────
|
||||
let stat_counter = AtomicUsize::new(0);
|
||||
let stat_skipped = AtomicUsize::new(0);
|
||||
let sized: Vec<(i64, String, u64)> = records
|
||||
.par_iter()
|
||||
.filter_map(|r| {
|
||||
let size = std::fs::metadata(&r.path).ok()?.len();
|
||||
if size == 0 { return None; }
|
||||
Some((r.id, r.path.clone(), size))
|
||||
let result = match std::fs::metadata(&r.path) {
|
||||
Ok(metadata) => Some((r.id, r.path.clone(), metadata.len())),
|
||||
Err(_) => {
|
||||
stat_skipped.fetch_add(1, Ordering::Relaxed);
|
||||
None
|
||||
}
|
||||
};
|
||||
let done = stat_counter.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if done % 100 == 0 || done == total {
|
||||
let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||
phase: "checking".to_string(),
|
||||
processed: done,
|
||||
total,
|
||||
skipped: stat_skipped.load(Ordering::Relaxed),
|
||||
});
|
||||
}
|
||||
result
|
||||
})
|
||||
.collect();
|
||||
let stat_skipped = stat_skipped.load(Ordering::Relaxed);
|
||||
|
||||
let mut by_size: HashMap<u64, Vec<usize>> = HashMap::new();
|
||||
for (i, (_, _, size)) in sized.iter().enumerate() {
|
||||
@@ -1032,29 +1071,53 @@ pub async fn find_duplicates(
|
||||
.collect();
|
||||
|
||||
if candidates.is_empty() {
|
||||
return vec![];
|
||||
return (vec![], stat_skipped, 0);
|
||||
}
|
||||
|
||||
// ── Phase 2: sample hash ──────────────────────────────────────────
|
||||
let c_total = candidates.len();
|
||||
let _ = app_hash.emit("duplicate_scan_progress", (0usize, c_total));
|
||||
let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||
phase: "hashing".to_string(),
|
||||
processed: 0,
|
||||
total: c_total,
|
||||
skipped: stat_skipped,
|
||||
});
|
||||
let counter = AtomicUsize::new(0);
|
||||
let hash_skipped = AtomicUsize::new(0);
|
||||
|
||||
candidates
|
||||
let pairs = candidates
|
||||
.par_iter()
|
||||
.filter_map(|&idx| {
|
||||
let (id, path, size) = &sized[idx];
|
||||
let file = std::fs::File::open(path).ok()?;
|
||||
let result = if *size == 0 {
|
||||
Some((xxh3_64(&[]), *size, *id, path.clone()))
|
||||
} else {
|
||||
std::fs::File::open(path)
|
||||
.ok()
|
||||
// SAFETY: read-only; no external truncation expected during scan.
|
||||
let mmap = unsafe { Mmap::map(&file).ok()? };
|
||||
let hash = sample(&mmap);
|
||||
.and_then(|file| unsafe { Mmap::map(&file).ok() })
|
||||
.map(|mmap| (sample(&mmap), *size, *id, path.clone()))
|
||||
};
|
||||
if result.is_none() {
|
||||
hash_skipped.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
let done = counter.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if done % 100 == 0 || done == c_total {
|
||||
let _ = app_hash.emit("duplicate_scan_progress", (done, c_total));
|
||||
let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||
phase: "hashing".to_string(),
|
||||
processed: done,
|
||||
total: c_total,
|
||||
skipped: stat_skipped + hash_skipped.load(Ordering::Relaxed),
|
||||
});
|
||||
}
|
||||
Some((hash, *size, *id, path.clone()))
|
||||
result
|
||||
})
|
||||
.collect()
|
||||
.collect();
|
||||
(
|
||||
pairs,
|
||||
stat_skipped + hash_skipped.load(Ordering::Relaxed),
|
||||
c_total,
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
@@ -1071,13 +1134,30 @@ pub async fn find_duplicates(
|
||||
// hash becomes its own DuplicateGroup so disjoint sets (A,A vs B,B that
|
||||
// happened to share size and sample hash) are never merged.
|
||||
const COVERED: u64 = (16 * 1024 * 4) as u64;
|
||||
let confirmed: Vec<(u64, u64, Vec<i64>)> =
|
||||
let confirming_total: usize = size_hash_map
|
||||
.iter()
|
||||
.filter(|((_, file_size), entries)| *file_size > COVERED && entries.len() > 1)
|
||||
.map(|(_, entries)| entries.len())
|
||||
.sum();
|
||||
if confirming_total > 0 {
|
||||
let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||
phase: "confirming".to_string(),
|
||||
processed: 0,
|
||||
total: confirming_total,
|
||||
skipped: skipped_before_confirm,
|
||||
});
|
||||
}
|
||||
let app_confirm = app.clone();
|
||||
let (confirmed, skipped_files): (Vec<(u64, u64, Vec<i64>)>, usize) =
|
||||
tokio::task::spawn_blocking(move || {
|
||||
use memmap2::Mmap;
|
||||
use rayon::prelude::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use xxhash_rust::xxh3::xxh3_64;
|
||||
|
||||
size_hash_map
|
||||
let counter = AtomicUsize::new(0);
|
||||
let confirm_skipped = AtomicUsize::new(0);
|
||||
let confirmed = size_hash_map
|
||||
.into_iter()
|
||||
.filter(|(_, entries)| entries.len() > 1)
|
||||
.flat_map(|((sample_hash, file_size), entries)| {
|
||||
@@ -1094,6 +1174,19 @@ pub async fn find_duplicates(
|
||||
.ok()
|
||||
.and_then(|f| unsafe { Mmap::map(&f).ok() })
|
||||
.map(|mmap| xxh3_64(&mmap));
|
||||
if hash.is_none() {
|
||||
confirm_skipped.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
let done = counter.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if done % 100 == 0 || done == confirming_total {
|
||||
let _ = app_confirm.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||
phase: "confirming".to_string(),
|
||||
processed: done,
|
||||
total: confirming_total,
|
||||
skipped: skipped_before_confirm
|
||||
+ confirm_skipped.load(Ordering::Relaxed),
|
||||
});
|
||||
}
|
||||
(*id, hash)
|
||||
})
|
||||
.collect();
|
||||
@@ -1111,7 +1204,11 @@ pub async fn find_duplicates(
|
||||
.map(|(full_hash, ids)| (full_hash, file_size, ids))
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
.collect();
|
||||
(
|
||||
confirmed,
|
||||
skipped_before_confirm + confirm_skipped.load(Ordering::Relaxed),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
@@ -1142,7 +1239,12 @@ pub async fn find_duplicates(
|
||||
let _ = db::set_duplicate_scan_cache(&conn, &folder_scope, &json);
|
||||
}
|
||||
|
||||
Ok(groups)
|
||||
Ok(DuplicateScanResult {
|
||||
groups,
|
||||
scanned_files: total,
|
||||
candidate_files: candidate_total,
|
||||
skipped_files,
|
||||
})
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
|
||||
@@ -278,12 +278,16 @@ export function BackgroundTasks() {
|
||||
id: -1,
|
||||
name: "Duplicate Scan",
|
||||
stages: [{
|
||||
label: "Hashing",
|
||||
label: duplicateScanProgress?.phase === "checking"
|
||||
? "Checking"
|
||||
: duplicateScanProgress?.phase === "confirming"
|
||||
? "Confirming"
|
||||
: "Hashing",
|
||||
detail: duplicateScanProgress
|
||||
? `${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}`
|
||||
? `${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}`
|
||||
: "Starting…",
|
||||
progress: duplicateScanProgress && duplicateScanProgress.total > 0
|
||||
? (duplicateScanProgress.scanned / duplicateScanProgress.total) * 100
|
||||
? (duplicateScanProgress.processed / duplicateScanProgress.total) * 100
|
||||
: null,
|
||||
failed: false,
|
||||
}],
|
||||
@@ -310,7 +314,8 @@ export function BackgroundTasks() {
|
||||
const embeddingStage = primary.stages.find((s) => s.label === "Embeddings");
|
||||
const taggingStage = primary.stages.find((s) => s.label === "Tags");
|
||||
const scanningStage = primary.stages.find((s) => s.label === "Scanning");
|
||||
const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? null;
|
||||
const duplicateStage = primary.id === -1 ? primary.stages[0] : null;
|
||||
const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? duplicateStage?.progress ?? null;
|
||||
|
||||
return (
|
||||
<div className="shrink-0 border-b border-white/[0.06]">
|
||||
@@ -434,7 +439,8 @@ export function BackgroundTasks() {
|
||||
const taskEmbeddingStage = task.stages.find((s) => s.label === "Embeddings");
|
||||
const taskTaggingStage = task.stages.find((s) => s.label === "Tags");
|
||||
const taskScanningStage = task.stages.find((s) => s.label === "Scanning");
|
||||
const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? null;
|
||||
const taskDuplicateStage = task.id === -1 ? task.stages[0] : null;
|
||||
const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? taskDuplicateStage?.progress ?? null;
|
||||
const taskHasFailed = (task.hasFailedEmbeddings || task.hasFailedTagging || task.hasFailedCaptions) && task.pendingMediaWork === 0;
|
||||
|
||||
return (
|
||||
|
||||
@@ -117,6 +117,7 @@ export function DuplicateFinder() {
|
||||
const duplicateScanning = useGalleryStore((state) => state.duplicateScanning);
|
||||
const duplicateScanProgress = useGalleryStore((state) => state.duplicateScanProgress);
|
||||
const duplicateScanError = useGalleryStore((state) => state.duplicateScanError);
|
||||
const duplicateScanWarning = useGalleryStore((state) => state.duplicateScanWarning);
|
||||
const duplicateSelectedIds = useGalleryStore((state) => state.duplicateSelectedIds);
|
||||
const duplicateLastScanned = useGalleryStore((state) => state.duplicateLastScanned);
|
||||
const selectedFolderId = useGalleryStore((state) => state.selectedFolderId);
|
||||
@@ -152,8 +153,15 @@ export function DuplicateFinder() {
|
||||
|
||||
const progressPercent =
|
||||
duplicateScanProgress && duplicateScanProgress.total > 0
|
||||
? Math.round((duplicateScanProgress.scanned / duplicateScanProgress.total) * 100)
|
||||
? Math.round((duplicateScanProgress.processed / duplicateScanProgress.total) * 100)
|
||||
: 0;
|
||||
const progressLabel = duplicateScanProgress
|
||||
? duplicateScanProgress.phase === "checking"
|
||||
? "Checking file sizes"
|
||||
: duplicateScanProgress.phase === "hashing"
|
||||
? "Hashing duplicate candidates"
|
||||
: "Confirming exact matches"
|
||||
: null;
|
||||
|
||||
return (
|
||||
<div className="flex min-h-0 flex-1 flex-col overflow-hidden bg-[#07080f]">
|
||||
@@ -165,7 +173,7 @@ export function DuplicateFinder() {
|
||||
<p className="mt-0.5 text-[11px] text-white/30">
|
||||
{duplicateScanning
|
||||
? duplicateScanProgress
|
||||
? `Scanning… ${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}`
|
||||
? `${progressLabel}… ${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}`
|
||||
: "Starting scan…"
|
||||
: hasResults
|
||||
? `${duplicateGroups.length} group${duplicateGroups.length === 1 ? "" : "s"} · ${formatBytes(totalWasted)} reclaimable`
|
||||
@@ -232,6 +240,9 @@ export function DuplicateFinder() {
|
||||
{duplicateScanError ? (
|
||||
<p className="mt-2 text-[11px] text-red-400/80">{duplicateScanError}</p>
|
||||
) : null}
|
||||
{duplicateScanWarning ? (
|
||||
<p className="mt-2 text-[11px] text-amber-300/70">{duplicateScanWarning}</p>
|
||||
) : null}
|
||||
{deleteResult ? (
|
||||
<p className="mt-2 text-[11px] text-white/40">{deleteResult}</p>
|
||||
) : null}
|
||||
@@ -241,7 +252,7 @@ export function DuplicateFinder() {
|
||||
{duplicateScanning && !hasResults ? (
|
||||
<div className="flex flex-1 items-center justify-center gap-3 text-white/25">
|
||||
<div className="h-5 w-5 animate-spin rounded-full border-2 border-white/15 border-t-white/50" />
|
||||
<span className="text-sm">Hashing files…</span>
|
||||
<span className="text-sm">{progressLabel ? `${progressLabel}…` : "Preparing scan…"}</span>
|
||||
</div>
|
||||
) : !hasScanned ? (
|
||||
<div className="flex flex-1 items-center justify-center px-8">
|
||||
|
||||
+51
-10
@@ -170,6 +170,20 @@ export interface DuplicateGroup {
|
||||
images: ImageRecord[];
|
||||
}
|
||||
|
||||
export interface DuplicateScanProgress {
|
||||
phase: "checking" | "hashing" | "confirming";
|
||||
processed: number;
|
||||
total: number;
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
interface DuplicateScanResult {
|
||||
groups: DuplicateGroup[];
|
||||
scanned_files: number;
|
||||
candidate_files: number;
|
||||
skipped_files: number;
|
||||
}
|
||||
|
||||
export interface SimilarImagesPage {
|
||||
images: ImageRecord[];
|
||||
offset: number;
|
||||
@@ -308,8 +322,9 @@ interface GalleryState {
|
||||
|
||||
duplicateGroups: DuplicateGroup[];
|
||||
duplicateScanning: boolean;
|
||||
duplicateScanProgress: { scanned: number; total: number } | null;
|
||||
duplicateScanProgress: DuplicateScanProgress | null;
|
||||
duplicateScanError: string | null;
|
||||
duplicateScanWarning: string | null;
|
||||
duplicateSelectedIds: Set<number>;
|
||||
duplicateLastScanned: number | null; // Unix timestamp (seconds)
|
||||
duplicateScanFolderId: number | null | undefined; // undefined = never scanned
|
||||
@@ -663,6 +678,7 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
|
||||
duplicateScanning: false,
|
||||
duplicateScanProgress: null,
|
||||
duplicateScanError: null,
|
||||
duplicateScanWarning: null,
|
||||
duplicateSelectedIds: new Set(),
|
||||
duplicateLastScanned: null,
|
||||
duplicateScanFolderId: undefined,
|
||||
@@ -954,7 +970,13 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
|
||||
if (activeView === "duplicates") {
|
||||
const { selectedFolderId, duplicateScanFolderId } = get();
|
||||
if (duplicateScanFolderId !== selectedFolderId) {
|
||||
set({ activeView, duplicateGroups: [], duplicateLastScanned: null, duplicateScanFolderId: undefined });
|
||||
set({
|
||||
activeView,
|
||||
duplicateGroups: [],
|
||||
duplicateLastScanned: null,
|
||||
duplicateScanFolderId: undefined,
|
||||
duplicateScanWarning: null,
|
||||
});
|
||||
void get().loadDuplicateScanCache(selectedFolderId);
|
||||
return;
|
||||
}
|
||||
@@ -1583,23 +1605,42 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
|
||||
interface CacheResult { groups: DuplicateGroup[]; scanned_at: number }
|
||||
const cached = await invoke<CacheResult | null>("load_duplicate_scan_cache", { folderId: folderId ?? null });
|
||||
if (cached) {
|
||||
set({ duplicateGroups: cached.groups, duplicateLastScanned: cached.scanned_at, duplicateScanFolderId: folderId });
|
||||
set({
|
||||
duplicateGroups: cached.groups,
|
||||
duplicateLastScanned: cached.scanned_at,
|
||||
duplicateScanFolderId: folderId,
|
||||
duplicateScanWarning: null,
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
scanDuplicates: async (folderId = null) => {
|
||||
const { listen } = await import("@tauri-apps/api/event");
|
||||
set({ duplicateScanning: true, duplicateGroups: [], duplicateScanProgress: null, duplicateScanError: null, duplicateSelectedIds: new Set() });
|
||||
const unlisten = await listen<[number, number]>("duplicate_scan_progress", (event) => {
|
||||
const [scanned, total] = event.payload;
|
||||
set({ duplicateScanProgress: { scanned, total } });
|
||||
set({
|
||||
duplicateScanning: true,
|
||||
duplicateGroups: [],
|
||||
duplicateScanProgress: null,
|
||||
duplicateScanError: null,
|
||||
duplicateScanWarning: null,
|
||||
duplicateSelectedIds: new Set(),
|
||||
});
|
||||
const unlisten = await listen<DuplicateScanProgress>("duplicate_scan_progress", (event) => {
|
||||
set({ duplicateScanProgress: event.payload });
|
||||
});
|
||||
try {
|
||||
const groups = await invoke<DuplicateGroup[]>("find_duplicates", { folderId: folderId ?? null });
|
||||
set({ duplicateGroups: groups, duplicateLastScanned: Math.floor(Date.now() / 1000), duplicateScanFolderId: folderId });
|
||||
const result = await invoke<DuplicateScanResult>("find_duplicates", { folderId: folderId ?? null });
|
||||
const warning = result.skipped_files > 0
|
||||
? `${result.skipped_files.toLocaleString()} file${result.skipped_files === 1 ? "" : "s"} could not be read and were skipped.`
|
||||
: null;
|
||||
set({
|
||||
duplicateGroups: result.groups,
|
||||
duplicateLastScanned: Math.floor(Date.now() / 1000),
|
||||
duplicateScanFolderId: folderId,
|
||||
duplicateScanWarning: warning,
|
||||
});
|
||||
void notifyTaskComplete(
|
||||
"Duplicate scan complete",
|
||||
groups.length === 1 ? "Found 1 duplicate group." : `Found ${groups.length.toLocaleString()} duplicate groups.`,
|
||||
`${result.groups.length === 1 ? "Found 1 duplicate group." : `Found ${result.groups.length.toLocaleString()} duplicate groups.`}${warning ? ` ${warning}` : ""}`,
|
||||
);
|
||||
} catch (e) {
|
||||
set({ duplicateScanError: String(e) });
|
||||
|
||||
Reference in New Issue
Block a user