fix: four more correctness bugs from PR review

- Duplicate scanner now emits one DuplicateGroup per distinct full hash
  within a sample-hash bucket, preventing disjoint sets (A,A vs B,B)
  from being merged and presented as a single deletable group
- Tagger model download now installs the shared ONNX runtime DLLs via
  ensure_onnx_runtime so the tagger is ready on a clean install without
  requiring the caption model to have been downloaded first
- Caption queue clear now follows the tagging worker pattern: marks
  in-flight rows as cancelled and deletes only non-processing rows;
  the caption worker skips writing results for cancelled jobs; startup
  cleanup removes any leftover cancelled caption rows
- Region search pagination now stores the crop rect in state and uses
  find_similar_by_region for subsequent pages instead of falling through
  to loadImages which appended unrelated folder results
This commit is contained in:
2026-06-06 22:00:20 +01:00
parent f82002129a
commit de0c2ab12d
5 changed files with 91 additions and 31 deletions
+16 -24
View File
@@ -953,9 +953,11 @@ pub async fn find_duplicates(
}
// Phase 3: for large-file groups (> 64 KB) the sample hash is not exact;
// re-hash the full file contents to avoid false positives before deletion.
// re-hash the full file contents to confirm matches. Each distinct full
// hash becomes its own DuplicateGroup so disjoint sets (A,A vs B,B that
// happened to share size and sample hash) are never merged.
const COVERED: u64 = (16 * 1024 * 4) as u64;
let confirmed: std::collections::HashMap<(u64, u64), Vec<i64>> =
let confirmed: Vec<(u64, u64, Vec<i64>)> =
tokio::task::spawn_blocking(move || {
use memmap2::Mmap;
use rayon::prelude::*;
@@ -964,13 +966,13 @@ pub async fn find_duplicates(
size_hash_map
.into_iter()
.filter(|(_, entries)| entries.len() > 1)
.filter_map(|((sample_hash, file_size), entries)| {
.flat_map(|((sample_hash, file_size), entries)| {
if file_size <= COVERED {
// Sample was already a full hash — no re-read needed.
let ids = entries.into_iter().map(|(id, _)| id).collect();
return Some(((sample_hash, file_size), ids));
// Sample was already a full hash — one group, no re-read needed.
return vec![(sample_hash, file_size, entries.into_iter().map(|(id, _)| id).collect())];
}
// Full-file hash pass to confirm.
// Full-file hash pass. Group by full hash so that two sets of
// files that collide only in the sample produce separate groups.
let full_hashes: Vec<(i64, Option<u64>)> = entries
.par_iter()
.map(|(id, path)| {
@@ -981,8 +983,6 @@ pub async fn find_duplicates(
(*id, hash)
})
.collect();
// Group by full hash; keep only groups where every file hashed
// successfully and at least two share the same hash.
let mut by_full: std::collections::HashMap<u64, Vec<i64>> =
std::collections::HashMap::new();
for (id, maybe_hash) in full_hashes {
@@ -990,20 +990,12 @@ pub async fn find_duplicates(
by_full.entry(h).or_default().push(id);
}
}
// Return the largest confirmed group (same sample_hash/size bucket).
// In practice a single full hash will dominate; emit each sub-group.
// We flatten into one entry using the sample_hash key — callers only
// need confirmed IDs, not the exact full hash.
let confirmed_ids: Vec<i64> = by_full
.into_values()
.filter(|g| g.len() > 1)
.flatten()
.collect();
if confirmed_ids.is_empty() {
None
} else {
Some(((sample_hash, file_size), confirmed_ids))
}
// Emit one entry per distinct full hash that has ≥ 2 members.
by_full
.into_iter()
.filter(|(_, ids)| ids.len() > 1)
.map(|(full_hash, ids)| (full_hash, file_size, ids))
.collect()
})
.collect()
})
@@ -1014,7 +1006,7 @@ pub async fn find_duplicates(
let conn = db.get().map_err(|e| e.to_string())?;
let mut groups: Vec<DuplicateGroup> = confirmed
.into_iter()
.filter_map(|((hash, file_size), ids)| {
.filter_map(|(hash, file_size, ids)| {
let images = db::get_images_by_ids(&conn, &ids).ok()?;
Some(DuplicateGroup {
file_hash: format!("{:016x}", hash),