Compare commits
6 Commits
a34d38d9d3
...
d30fe47876
| Author | SHA1 | Date | |
|---|---|---|---|
| d30fe47876 | |||
| cbfcbea96a | |||
| 948a489a8a | |||
| 334ac54e00 | |||
| a4486547e8 | |||
| 665c315f56 |
@@ -80,7 +80,7 @@ Database: SQLite with WAL mode, stored in the Tauri app data directory as `galle
|
|||||||
| `media-updated` | `ThumbnailBatch` |
|
| `media-updated` | `ThumbnailBatch` |
|
||||||
| `caption-model-progress` | `CaptionModelProgress` |
|
| `caption-model-progress` | `CaptionModelProgress` |
|
||||||
| `tagger-model-progress` | `TaggerModelProgress` |
|
| `tagger-model-progress` | `TaggerModelProgress` |
|
||||||
| `duplicate_scan_progress` | `[scanned, total]` |
|
| `duplicate_scan_progress` | `{ phase, processed, total, skipped }` |
|
||||||
|
|
||||||
### Key types
|
### Key types
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"build:app": "tauri build",
|
"build:app": "tauri build",
|
||||||
"build:vite": "tsc && vite build",
|
"build:vite": "tsc && vite build",
|
||||||
|
"clean:app": "cd src-tauri && cargo clean",
|
||||||
"dev:app": "tauri dev",
|
"dev:app": "tauri dev",
|
||||||
"dev:vite": "vite",
|
"dev:vite": "vite",
|
||||||
"preview": "vite preview",
|
"preview": "vite preview",
|
||||||
|
|||||||
Generated
+63
@@ -143,6 +143,12 @@ dependencies = [
|
|||||||
"derive_arbitrary",
|
"derive_arbitrary",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "arrayvec"
|
||||||
|
version = "0.7.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "async-broadcast"
|
name = "async-broadcast"
|
||||||
version = "0.7.2"
|
version = "0.7.2"
|
||||||
@@ -635,6 +641,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283"
|
checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"find-msvc-tools",
|
"find-msvc-tools",
|
||||||
|
"jobserver",
|
||||||
|
"libc",
|
||||||
"shlex",
|
"shlex",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -3108,6 +3116,16 @@ dependencies = [
|
|||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "jobserver"
|
||||||
|
version = "0.1.34"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom 0.3.4",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "js-sys"
|
name = "js-sys"
|
||||||
version = "0.3.94"
|
version = "0.3.94"
|
||||||
@@ -3559,6 +3577,31 @@ dependencies = [
|
|||||||
"pxfm",
|
"pxfm",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "mozjpeg"
|
||||||
|
version = "0.10.13"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b7891b80aaa86097d38d276eb98b3805d6280708c4e0a1e6f6aed9380c51fec9"
|
||||||
|
dependencies = [
|
||||||
|
"arrayvec",
|
||||||
|
"bytemuck",
|
||||||
|
"libc",
|
||||||
|
"mozjpeg-sys",
|
||||||
|
"rgb",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "mozjpeg-sys"
|
||||||
|
version = "2.2.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7f0dc668bf9bf888c88e2fb1ab16a406d2c380f1d082b20d51dd540ab2aa70c1"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
"dunce",
|
||||||
|
"libc",
|
||||||
|
"nasm-rs",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "muda"
|
name = "muda"
|
||||||
version = "0.17.2"
|
version = "0.17.2"
|
||||||
@@ -3586,6 +3629,16 @@ version = "0.1.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af"
|
checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nasm-rs"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149"
|
||||||
|
dependencies = [
|
||||||
|
"jobserver",
|
||||||
|
"log",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "native-tls"
|
name = "native-tls"
|
||||||
version = "0.2.18"
|
version = "0.2.18"
|
||||||
@@ -4388,6 +4441,7 @@ dependencies = [
|
|||||||
"kamadak-exif",
|
"kamadak-exif",
|
||||||
"log",
|
"log",
|
||||||
"memmap2",
|
"memmap2",
|
||||||
|
"mozjpeg",
|
||||||
"notify",
|
"notify",
|
||||||
"ort",
|
"ort",
|
||||||
"r2d2",
|
"r2d2",
|
||||||
@@ -5102,6 +5156,15 @@ dependencies = [
|
|||||||
"windows-sys 0.60.2",
|
"windows-sys 0.60.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rgb"
|
||||||
|
version = "0.8.53"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4"
|
||||||
|
dependencies = [
|
||||||
|
"bytemuck",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ring"
|
name = "ring"
|
||||||
version = "0.17.14"
|
version = "0.17.14"
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ csv = "1"
|
|||||||
kamadak-exif = "0.5"
|
kamadak-exif = "0.5"
|
||||||
notify = "6"
|
notify = "6"
|
||||||
tauri-plugin-notification = "2"
|
tauri-plugin-notification = "2"
|
||||||
|
mozjpeg = "0.10.13"
|
||||||
|
|
||||||
# ── Dev-mode performance ────────────────────────────────────────────────────
|
# ── Dev-mode performance ────────────────────────────────────────────────────
|
||||||
# opt-level=1 on the main crate keeps incremental compile short.
|
# opt-level=1 on the main crate keeps incremental compile short.
|
||||||
@@ -81,6 +82,10 @@ opt-level = 3
|
|||||||
opt-level = 3
|
opt-level = 3
|
||||||
[profile.dev.package.fast_image_resize]
|
[profile.dev.package.fast_image_resize]
|
||||||
opt-level = 3
|
opt-level = 3
|
||||||
|
[profile.dev.package.mozjpeg]
|
||||||
|
opt-level = 3
|
||||||
|
[profile.dev.package.mozjpeg-sys]
|
||||||
|
opt-level = 3
|
||||||
|
|
||||||
# Parallel work scheduler
|
# Parallel work scheduler
|
||||||
[profile.dev.package.rayon]
|
[profile.dev.package.rayon]
|
||||||
|
|||||||
+121
-19
@@ -182,6 +182,22 @@ pub struct DuplicateGroup {
|
|||||||
pub images: Vec<ImageRecord>,
|
pub images: Vec<ImageRecord>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub struct DuplicateScanProgress {
|
||||||
|
pub phase: String,
|
||||||
|
pub processed: usize,
|
||||||
|
pub total: usize,
|
||||||
|
pub skipped: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct DuplicateScanResult {
|
||||||
|
pub groups: Vec<DuplicateGroup>,
|
||||||
|
pub scanned_files: usize,
|
||||||
|
pub candidate_files: usize,
|
||||||
|
pub skipped_files: usize,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub struct DuplicateScanCache {
|
pub struct DuplicateScanCache {
|
||||||
pub groups: Vec<DuplicateGroup>,
|
pub groups: Vec<DuplicateGroup>,
|
||||||
@@ -962,14 +978,19 @@ pub async fn find_duplicates(
|
|||||||
app: AppHandle,
|
app: AppHandle,
|
||||||
db: State<'_, DbState>,
|
db: State<'_, DbState>,
|
||||||
folder_id: Option<i64>,
|
folder_id: Option<i64>,
|
||||||
) -> Result<Vec<DuplicateGroup>, String> {
|
) -> Result<DuplicateScanResult, String> {
|
||||||
let records = {
|
let records = {
|
||||||
let conn = db.get().map_err(|e| e.to_string())?;
|
let conn = db.get().map_err(|e| e.to_string())?;
|
||||||
db::get_all_image_paths(&conn, folder_id).map_err(|e| e.to_string())?
|
db::get_all_image_paths(&conn, folder_id).map_err(|e| e.to_string())?
|
||||||
};
|
};
|
||||||
|
|
||||||
let total = records.len();
|
let total = records.len();
|
||||||
let _ = app.emit("duplicate_scan_progress", (0usize, total));
|
let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||||
|
phase: "checking".to_string(),
|
||||||
|
processed: 0,
|
||||||
|
total,
|
||||||
|
skipped: 0,
|
||||||
|
});
|
||||||
|
|
||||||
// Three-phase detection.
|
// Three-phase detection.
|
||||||
//
|
//
|
||||||
@@ -986,7 +1007,8 @@ pub async fn find_duplicates(
|
|||||||
// For sample-hash groups that contain files > 64 KB, compute a full-file
|
// For sample-hash groups that contain files > 64 KB, compute a full-file
|
||||||
// hash to confirm the match before presenting them as deletable duplicates.
|
// hash to confirm the match before presenting them as deletable duplicates.
|
||||||
let app_hash = app.clone();
|
let app_hash = app.clone();
|
||||||
let pairs: Vec<(u64, u64, i64, String)> = tokio::task::spawn_blocking(move || {
|
let (pairs, skipped_before_confirm, candidate_total):
|
||||||
|
(Vec<(u64, u64, i64, String)>, usize, usize) = tokio::task::spawn_blocking(move || {
|
||||||
use memmap2::Mmap;
|
use memmap2::Mmap;
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@@ -1012,14 +1034,31 @@ pub async fn find_duplicates(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ── Phase 1: stat ─────────────────────────────────────────────────
|
// ── Phase 1: stat ─────────────────────────────────────────────────
|
||||||
|
let stat_counter = AtomicUsize::new(0);
|
||||||
|
let stat_skipped = AtomicUsize::new(0);
|
||||||
let sized: Vec<(i64, String, u64)> = records
|
let sized: Vec<(i64, String, u64)> = records
|
||||||
.par_iter()
|
.par_iter()
|
||||||
.filter_map(|r| {
|
.filter_map(|r| {
|
||||||
let size = std::fs::metadata(&r.path).ok()?.len();
|
let result = match std::fs::metadata(&r.path) {
|
||||||
if size == 0 { return None; }
|
Ok(metadata) => Some((r.id, r.path.clone(), metadata.len())),
|
||||||
Some((r.id, r.path.clone(), size))
|
Err(_) => {
|
||||||
|
stat_skipped.fetch_add(1, Ordering::Relaxed);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let done = stat_counter.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
|
if done % 100 == 0 || done == total {
|
||||||
|
let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||||
|
phase: "checking".to_string(),
|
||||||
|
processed: done,
|
||||||
|
total,
|
||||||
|
skipped: stat_skipped.load(Ordering::Relaxed),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
result
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
let stat_skipped = stat_skipped.load(Ordering::Relaxed);
|
||||||
|
|
||||||
let mut by_size: HashMap<u64, Vec<usize>> = HashMap::new();
|
let mut by_size: HashMap<u64, Vec<usize>> = HashMap::new();
|
||||||
for (i, (_, _, size)) in sized.iter().enumerate() {
|
for (i, (_, _, size)) in sized.iter().enumerate() {
|
||||||
@@ -1032,29 +1071,53 @@ pub async fn find_duplicates(
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
if candidates.is_empty() {
|
if candidates.is_empty() {
|
||||||
return vec![];
|
return (vec![], stat_skipped, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Phase 2: sample hash ──────────────────────────────────────────
|
// ── Phase 2: sample hash ──────────────────────────────────────────
|
||||||
let c_total = candidates.len();
|
let c_total = candidates.len();
|
||||||
let _ = app_hash.emit("duplicate_scan_progress", (0usize, c_total));
|
let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||||
|
phase: "hashing".to_string(),
|
||||||
|
processed: 0,
|
||||||
|
total: c_total,
|
||||||
|
skipped: stat_skipped,
|
||||||
|
});
|
||||||
let counter = AtomicUsize::new(0);
|
let counter = AtomicUsize::new(0);
|
||||||
|
let hash_skipped = AtomicUsize::new(0);
|
||||||
|
|
||||||
candidates
|
let pairs = candidates
|
||||||
.par_iter()
|
.par_iter()
|
||||||
.filter_map(|&idx| {
|
.filter_map(|&idx| {
|
||||||
let (id, path, size) = &sized[idx];
|
let (id, path, size) = &sized[idx];
|
||||||
let file = std::fs::File::open(path).ok()?;
|
let result = if *size == 0 {
|
||||||
|
Some((xxh3_64(&[]), *size, *id, path.clone()))
|
||||||
|
} else {
|
||||||
|
std::fs::File::open(path)
|
||||||
|
.ok()
|
||||||
// SAFETY: read-only; no external truncation expected during scan.
|
// SAFETY: read-only; no external truncation expected during scan.
|
||||||
let mmap = unsafe { Mmap::map(&file).ok()? };
|
.and_then(|file| unsafe { Mmap::map(&file).ok() })
|
||||||
let hash = sample(&mmap);
|
.map(|mmap| (sample(&mmap), *size, *id, path.clone()))
|
||||||
|
};
|
||||||
|
if result.is_none() {
|
||||||
|
hash_skipped.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
let done = counter.fetch_add(1, Ordering::Relaxed) + 1;
|
let done = counter.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
if done % 100 == 0 || done == c_total {
|
if done % 100 == 0 || done == c_total {
|
||||||
let _ = app_hash.emit("duplicate_scan_progress", (done, c_total));
|
let _ = app_hash.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||||
|
phase: "hashing".to_string(),
|
||||||
|
processed: done,
|
||||||
|
total: c_total,
|
||||||
|
skipped: stat_skipped + hash_skipped.load(Ordering::Relaxed),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
Some((hash, *size, *id, path.clone()))
|
result
|
||||||
})
|
})
|
||||||
.collect()
|
.collect();
|
||||||
|
(
|
||||||
|
pairs,
|
||||||
|
stat_skipped + hash_skipped.load(Ordering::Relaxed),
|
||||||
|
c_total,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|e| e.to_string())?;
|
.map_err(|e| e.to_string())?;
|
||||||
@@ -1071,13 +1134,30 @@ pub async fn find_duplicates(
|
|||||||
// hash becomes its own DuplicateGroup so disjoint sets (A,A vs B,B that
|
// hash becomes its own DuplicateGroup so disjoint sets (A,A vs B,B that
|
||||||
// happened to share size and sample hash) are never merged.
|
// happened to share size and sample hash) are never merged.
|
||||||
const COVERED: u64 = (16 * 1024 * 4) as u64;
|
const COVERED: u64 = (16 * 1024 * 4) as u64;
|
||||||
let confirmed: Vec<(u64, u64, Vec<i64>)> =
|
let confirming_total: usize = size_hash_map
|
||||||
|
.iter()
|
||||||
|
.filter(|((_, file_size), entries)| *file_size > COVERED && entries.len() > 1)
|
||||||
|
.map(|(_, entries)| entries.len())
|
||||||
|
.sum();
|
||||||
|
if confirming_total > 0 {
|
||||||
|
let _ = app.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||||
|
phase: "confirming".to_string(),
|
||||||
|
processed: 0,
|
||||||
|
total: confirming_total,
|
||||||
|
skipped: skipped_before_confirm,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let app_confirm = app.clone();
|
||||||
|
let (confirmed, skipped_files): (Vec<(u64, u64, Vec<i64>)>, usize) =
|
||||||
tokio::task::spawn_blocking(move || {
|
tokio::task::spawn_blocking(move || {
|
||||||
use memmap2::Mmap;
|
use memmap2::Mmap;
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
use xxhash_rust::xxh3::xxh3_64;
|
use xxhash_rust::xxh3::xxh3_64;
|
||||||
|
|
||||||
size_hash_map
|
let counter = AtomicUsize::new(0);
|
||||||
|
let confirm_skipped = AtomicUsize::new(0);
|
||||||
|
let confirmed = size_hash_map
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|(_, entries)| entries.len() > 1)
|
.filter(|(_, entries)| entries.len() > 1)
|
||||||
.flat_map(|((sample_hash, file_size), entries)| {
|
.flat_map(|((sample_hash, file_size), entries)| {
|
||||||
@@ -1094,6 +1174,19 @@ pub async fn find_duplicates(
|
|||||||
.ok()
|
.ok()
|
||||||
.and_then(|f| unsafe { Mmap::map(&f).ok() })
|
.and_then(|f| unsafe { Mmap::map(&f).ok() })
|
||||||
.map(|mmap| xxh3_64(&mmap));
|
.map(|mmap| xxh3_64(&mmap));
|
||||||
|
if hash.is_none() {
|
||||||
|
confirm_skipped.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
let done = counter.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
|
if done % 100 == 0 || done == confirming_total {
|
||||||
|
let _ = app_confirm.emit("duplicate_scan_progress", DuplicateScanProgress {
|
||||||
|
phase: "confirming".to_string(),
|
||||||
|
processed: done,
|
||||||
|
total: confirming_total,
|
||||||
|
skipped: skipped_before_confirm
|
||||||
|
+ confirm_skipped.load(Ordering::Relaxed),
|
||||||
|
});
|
||||||
|
}
|
||||||
(*id, hash)
|
(*id, hash)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@@ -1111,7 +1204,11 @@ pub async fn find_duplicates(
|
|||||||
.map(|(full_hash, ids)| (full_hash, file_size, ids))
|
.map(|(full_hash, ids)| (full_hash, file_size, ids))
|
||||||
.collect()
|
.collect()
|
||||||
})
|
})
|
||||||
.collect()
|
.collect();
|
||||||
|
(
|
||||||
|
confirmed,
|
||||||
|
skipped_before_confirm + confirm_skipped.load(Ordering::Relaxed),
|
||||||
|
)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|e| e.to_string())?;
|
.map_err(|e| e.to_string())?;
|
||||||
@@ -1142,7 +1239,12 @@ pub async fn find_duplicates(
|
|||||||
let _ = db::set_duplicate_scan_cache(&conn, &folder_scope, &json);
|
let _ = db::set_duplicate_scan_cache(&conn, &folder_scope, &json);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(groups)
|
Ok(DuplicateScanResult {
|
||||||
|
groups,
|
||||||
|
scanned_files: total,
|
||||||
|
candidate_files: candidate_total,
|
||||||
|
skipped_files,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tauri::command]
|
#[tauri::command]
|
||||||
|
|||||||
@@ -1152,6 +1152,57 @@ fn get_pending_thumbnail_jobs_excluding(
|
|||||||
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
|
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// True if any claimable (pending, non-excluded) jobs exist in `job_table`.
|
||||||
|
/// Used by lower-priority workers to defer to higher-priority queues.
|
||||||
|
/// `extra_predicate` narrows the image join (e.g. videos only for metadata).
|
||||||
|
fn has_claimable_jobs(
|
||||||
|
conn: &Connection,
|
||||||
|
job_table: &str,
|
||||||
|
extra_predicate: &str,
|
||||||
|
excluded_folder_ids: &std::collections::HashSet<i64>,
|
||||||
|
) -> Result<bool> {
|
||||||
|
let sql = format!(
|
||||||
|
"SELECT EXISTS(
|
||||||
|
SELECT 1
|
||||||
|
FROM {} j
|
||||||
|
JOIN images i ON i.id = j.image_id
|
||||||
|
WHERE j.status = 'pending'
|
||||||
|
{}
|
||||||
|
{}
|
||||||
|
)",
|
||||||
|
job_table,
|
||||||
|
extra_predicate,
|
||||||
|
folder_exclusion_clause("i", excluded_folder_ids)
|
||||||
|
);
|
||||||
|
Ok(conn.query_row(&sql, [], |row| row.get::<_, i64>(0))? != 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn has_claimable_thumbnail_jobs(
|
||||||
|
conn: &Connection,
|
||||||
|
excluded_folder_ids: &std::collections::HashSet<i64>,
|
||||||
|
) -> Result<bool> {
|
||||||
|
has_claimable_jobs(conn, "thumbnail_jobs", "", excluded_folder_ids)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn has_claimable_metadata_jobs(
|
||||||
|
conn: &Connection,
|
||||||
|
excluded_folder_ids: &std::collections::HashSet<i64>,
|
||||||
|
) -> Result<bool> {
|
||||||
|
has_claimable_jobs(
|
||||||
|
conn,
|
||||||
|
"metadata_jobs",
|
||||||
|
"AND i.media_kind = 'video'",
|
||||||
|
excluded_folder_ids,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn has_claimable_embedding_jobs(
|
||||||
|
conn: &Connection,
|
||||||
|
excluded_folder_ids: &std::collections::HashSet<i64>,
|
||||||
|
) -> Result<bool> {
|
||||||
|
has_claimable_jobs(conn, "embedding_jobs", "", excluded_folder_ids)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn claim_thumbnail_jobs(
|
pub fn claim_thumbnail_jobs(
|
||||||
conn: &mut Connection,
|
conn: &mut Connection,
|
||||||
active_folder_ids: &std::collections::HashSet<i64>,
|
active_folder_ids: &std::collections::HashSet<i64>,
|
||||||
|
|||||||
@@ -191,9 +191,11 @@ fn resolve_device() -> Result<Device> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn load_image(path: &Path, image_size: usize) -> Result<Tensor> {
|
fn load_image(path: &Path, image_size: usize) -> Result<Tensor> {
|
||||||
let image = image::ImageReader::open(path)?
|
// Scaled decode: CLIP only needs image_size² pixels, so decoding a large
|
||||||
.with_guessed_format()?
|
// JPEG at full resolution is wasted work. Cover mode keeps the shortest
|
||||||
.decode()?;
|
// side at or above image_size for the fill-crop below. Also applies EXIF
|
||||||
|
// orientation, so rotated photos embed the way they are displayed.
|
||||||
|
let image = crate::thumbnail::decode_image_scaled(path, image_size as u32, true)?;
|
||||||
let image = image.resize_to_fill(
|
let image = image.resize_to_fill(
|
||||||
image_size as u32,
|
image_size as u32,
|
||||||
image_size as u32,
|
image_size as u32,
|
||||||
@@ -208,10 +210,11 @@ fn load_image(path: &Path, image_size: usize) -> Result<Tensor> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn load_images(paths: &[PathBuf], image_size: usize) -> Result<Tensor> {
|
fn load_images(paths: &[PathBuf], image_size: usize) -> Result<Tensor> {
|
||||||
let mut images = Vec::with_capacity(paths.len());
|
use rayon::prelude::*;
|
||||||
for path in paths {
|
let images = paths
|
||||||
images.push(load_image(path, image_size)?);
|
.par_iter()
|
||||||
}
|
.map(|path| load_image(path, image_size))
|
||||||
|
.collect::<Result<Vec<_>>>()?;
|
||||||
Ok(Tensor::stack(&images, 0)?)
|
Ok(Tensor::stack(&images, 0)?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+168
-80
@@ -121,6 +121,50 @@ static FOLDER_STORAGE_PROFILES: OnceLock<Mutex<HashMap<i64, RuntimeAdaptiveProfi
|
|||||||
static DB_WRITE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
static DB_WRITE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||||
const EMBEDDING_BATCH_SIZE: usize = 8;
|
const EMBEDDING_BATCH_SIZE: usize = 8;
|
||||||
|
|
||||||
|
/// Background workers form a strict priority pipeline: a worker only claims
|
||||||
|
/// jobs when no higher-priority claimable work exists, so stages drain one
|
||||||
|
/// at a time (thumbnails → metadata → embeddings → tagging) instead of all
|
||||||
|
/// workers contending for CPU, disk, and the DB writer at once.
|
||||||
|
#[derive(Clone, Copy, PartialEq, PartialOrd)]
|
||||||
|
enum WorkerTier {
|
||||||
|
Thumbnail = 0,
|
||||||
|
Metadata = 1,
|
||||||
|
Embedding = 2,
|
||||||
|
Tagging = 3,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// True if any tier above `own_tier` still has claimable work. Each check
|
||||||
|
/// uses the same exclusion set the corresponding claim would (that worker's
|
||||||
|
/// paused folders plus actively-indexing folders), so paused or mid-scan
|
||||||
|
/// work never gates lower tiers.
|
||||||
|
fn higher_priority_work_pending(pool: &DbPool, own_tier: WorkerTier) -> Result<bool> {
|
||||||
|
let conn = pool.get()?;
|
||||||
|
let active_folders = active_indexing_folders();
|
||||||
|
|
||||||
|
if own_tier > WorkerTier::Thumbnail {
|
||||||
|
let mut excluded = paused_folder_ids("thumbnail");
|
||||||
|
excluded.extend(active_folders.iter().copied());
|
||||||
|
if db::has_claimable_thumbnail_jobs(&conn, &excluded)? {
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if own_tier > WorkerTier::Metadata {
|
||||||
|
let mut excluded = paused_folder_ids("metadata");
|
||||||
|
excluded.extend(active_folders.iter().copied());
|
||||||
|
if db::has_claimable_metadata_jobs(&conn, &excluded)? {
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if own_tier > WorkerTier::Embedding {
|
||||||
|
let mut excluded = paused_folder_ids("embedding");
|
||||||
|
excluded.extend(active_folders.iter().copied());
|
||||||
|
if db::has_claimable_embedding_jobs(&conn, &excluded)? {
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Serialize)]
|
#[derive(Clone, Serialize)]
|
||||||
pub struct IndexProgress {
|
pub struct IndexProgress {
|
||||||
pub folder_id: i64,
|
pub folder_id: i64,
|
||||||
@@ -204,19 +248,31 @@ pub fn start_thumbnail_worker(
|
|||||||
cache_dir: PathBuf,
|
cache_dir: PathBuf,
|
||||||
) {
|
) {
|
||||||
std::thread::spawn(move || loop {
|
std::thread::spawn(move || loop {
|
||||||
if let Err(error) = process_thumbnail_batch(&app, &pool, &media_tools, &cache_dir) {
|
// Only back off when the queue is empty (or errored); while jobs are
|
||||||
|
// pending, claim the next batch immediately.
|
||||||
|
match process_thumbnail_batch(&app, &pool, &media_tools, &cache_dir) {
|
||||||
|
Ok(true) => {}
|
||||||
|
Ok(false) => std::thread::sleep(std::time::Duration::from_millis(250)),
|
||||||
|
Err(error) => {
|
||||||
eprintln!("Thumbnail worker error: {}", error);
|
eprintln!("Thumbnail worker error: {}", error);
|
||||||
}
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(250));
|
std::thread::sleep(std::time::Duration::from_millis(250));
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn start_metadata_worker(app: AppHandle, pool: DbPool, media_tools: MediaTools) {
|
pub fn start_metadata_worker(app: AppHandle, pool: DbPool, media_tools: MediaTools) {
|
||||||
std::thread::spawn(move || loop {
|
std::thread::spawn(move || loop {
|
||||||
if let Err(error) = process_metadata_batch(&app, &pool, &media_tools) {
|
// Only back off when the queue is empty (or errored); while jobs are
|
||||||
|
// pending, claim the next batch immediately.
|
||||||
|
match process_metadata_batch(&app, &pool, &media_tools) {
|
||||||
|
Ok(true) => {}
|
||||||
|
Ok(false) => std::thread::sleep(std::time::Duration::from_millis(250)),
|
||||||
|
Err(error) => {
|
||||||
eprintln!("Metadata worker error: {}", error);
|
eprintln!("Metadata worker error: {}", error);
|
||||||
}
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(250));
|
std::thread::sleep(std::time::Duration::from_millis(250));
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -225,11 +281,17 @@ pub fn start_embedding_worker(app: AppHandle, pool: DbPool) {
|
|||||||
let mut embedder: Option<ClipImageEmbedder> = None;
|
let mut embedder: Option<ClipImageEmbedder> = None;
|
||||||
println!("Embedding worker started.");
|
println!("Embedding worker started.");
|
||||||
loop {
|
loop {
|
||||||
if let Err(error) = process_embedding_batch(&app, &pool, &mut embedder) {
|
// Only back off when the queue is empty (or errored); while jobs
|
||||||
|
// are pending, claim the next batch immediately.
|
||||||
|
match process_embedding_batch(&app, &pool, &mut embedder) {
|
||||||
|
Ok(true) => {}
|
||||||
|
Ok(false) => std::thread::sleep(std::time::Duration::from_millis(500)),
|
||||||
|
Err(error) => {
|
||||||
eprintln!("Embedding worker error: {}", error);
|
eprintln!("Embedding worker error: {}", error);
|
||||||
}
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -264,14 +326,18 @@ pub fn start_tagging_worker(app: AppHandle, pool: DbPool, app_data_dir: PathBuf)
|
|||||||
println!("Tagging worker: acceleration setting changed — resetting session.");
|
println!("Tagging worker: acceleration setting changed — resetting session.");
|
||||||
tagger_instance = None;
|
tagger_instance = None;
|
||||||
}
|
}
|
||||||
if let Err(error) =
|
// Only back off when the queue is empty (or errored); while jobs
|
||||||
process_tagging_batch(&app, &pool, &app_data_dir, &mut tagger_instance)
|
// are pending, claim the next batch immediately.
|
||||||
{
|
match process_tagging_batch(&app, &pool, &app_data_dir, &mut tagger_instance) {
|
||||||
|
Ok(true) => {}
|
||||||
|
Ok(false) => std::thread::sleep(std::time::Duration::from_millis(750)),
|
||||||
|
Err(error) => {
|
||||||
eprintln!("Tagging worker error: {}", error);
|
eprintln!("Tagging worker error: {}", error);
|
||||||
tagger_instance = None;
|
tagger_instance = None;
|
||||||
}
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(750));
|
std::thread::sleep(std::time::Duration::from_millis(750));
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -554,12 +620,14 @@ fn commit_batch(pool: &DbPool, records: &[ImageRecord]) -> Result<Vec<ImageRecor
|
|||||||
Ok(committed)
|
Ok(committed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
|
||||||
|
/// the queue was empty.
|
||||||
fn process_thumbnail_batch(
|
fn process_thumbnail_batch(
|
||||||
app: &AppHandle,
|
app: &AppHandle,
|
||||||
pool: &DbPool,
|
pool: &DbPool,
|
||||||
media_tools: &MediaTools,
|
media_tools: &MediaTools,
|
||||||
cache_dir: &Path,
|
cache_dir: &Path,
|
||||||
) -> Result<()> {
|
) -> Result<bool> {
|
||||||
let jobs = {
|
let jobs = {
|
||||||
with_db_write_lock(|| {
|
with_db_write_lock(|| {
|
||||||
let mut conn = pool.get()?;
|
let mut conn = pool.get()?;
|
||||||
@@ -578,7 +646,7 @@ fn process_thumbnail_batch(
|
|||||||
};
|
};
|
||||||
|
|
||||||
if jobs.is_empty() {
|
if jobs.is_empty() {
|
||||||
return Ok(());
|
return Ok(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
println!("Thumbnail batch claimed: {} items", jobs.len());
|
println!("Thumbnail batch claimed: {} items", jobs.len());
|
||||||
@@ -586,33 +654,39 @@ fn process_thumbnail_batch(
|
|||||||
let (image_jobs, video_jobs): (Vec<_>, Vec<_>) =
|
let (image_jobs, video_jobs): (Vec<_>, Vec<_>) =
|
||||||
jobs.into_iter().partition(|job| job.media_kind == "image");
|
jobs.into_iter().partition(|job| job.media_kind == "image");
|
||||||
|
|
||||||
let mut results = image_jobs
|
// Images: parallel decode, committed as one batch.
|
||||||
|
if !image_jobs.is_empty() {
|
||||||
|
let results = image_jobs
|
||||||
.par_iter()
|
.par_iter()
|
||||||
.map(|job| {
|
.map(|job| {
|
||||||
(
|
(
|
||||||
job.image_id,
|
job.image_id,
|
||||||
if job.media_kind == "image" {
|
thumbnail::generate_image_thumbnail(Path::new(&job.path), cache_dir).map(Some),
|
||||||
thumbnail::generate_image_thumbnail(Path::new(&job.path), cache_dir).map(Some)
|
|
||||||
} else {
|
|
||||||
thumbnail::generate_video_thumbnail(
|
|
||||||
media_tools,
|
|
||||||
Path::new(&job.path),
|
|
||||||
cache_dir,
|
|
||||||
)
|
|
||||||
.map(Some)
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
persist_thumbnail_results(app, pool, results)?;
|
||||||
for job in video_jobs {
|
|
||||||
results.push((
|
|
||||||
job.image_id,
|
|
||||||
thumbnail::generate_video_thumbnail(media_tools, Path::new(&job.path), cache_dir)
|
|
||||||
.map(Some),
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Videos: sequential, off the rayon pool — each ffmpeg call blocks its
|
||||||
|
// thread, and a video-heavy batch on the shared pool would starve image
|
||||||
|
// decoding across all workers. Committed per item so progress keeps
|
||||||
|
// moving through slow stretches.
|
||||||
|
for job in &video_jobs {
|
||||||
|
let result =
|
||||||
|
thumbnail::generate_video_thumbnail(media_tools, Path::new(&job.path), cache_dir)
|
||||||
|
.map(Some);
|
||||||
|
persist_thumbnail_results(app, pool, vec![(job.image_id, result)])?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn persist_thumbnail_results(
|
||||||
|
app: &AppHandle,
|
||||||
|
pool: &DbPool,
|
||||||
|
results: Vec<(i64, anyhow::Result<Option<thumbnail::GeneratedThumbnail>>)>,
|
||||||
|
) -> Result<()> {
|
||||||
let updated_images = {
|
let updated_images = {
|
||||||
with_db_write_lock(|| {
|
with_db_write_lock(|| {
|
||||||
let mut conn = pool.get()?;
|
let mut conn = pool.get()?;
|
||||||
@@ -665,7 +739,17 @@ fn process_thumbnail_batch(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaTools) -> Result<()> {
|
/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
|
||||||
|
/// the queue was empty.
|
||||||
|
fn process_metadata_batch(
|
||||||
|
app: &AppHandle,
|
||||||
|
pool: &DbPool,
|
||||||
|
media_tools: &MediaTools,
|
||||||
|
) -> Result<bool> {
|
||||||
|
if higher_priority_work_pending(pool, WorkerTier::Metadata)? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
let jobs = {
|
let jobs = {
|
||||||
with_db_write_lock(|| {
|
with_db_write_lock(|| {
|
||||||
let mut conn = pool.get()?;
|
let mut conn = pool.get()?;
|
||||||
@@ -684,83 +768,78 @@ fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaToo
|
|||||||
};
|
};
|
||||||
|
|
||||||
if jobs.is_empty() {
|
if jobs.is_empty() {
|
||||||
return Ok(());
|
return Ok(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
let results = jobs
|
// Probes run sequentially (each ffprobe blocks on its process); results
|
||||||
.into_iter()
|
// are committed per item so progress keeps moving through slow stretches.
|
||||||
.map(|job| {
|
for job in jobs {
|
||||||
(
|
let metadata_result = probe_video_metadata(media_tools, Path::new(&job.path));
|
||||||
job.image_id,
|
|
||||||
probe_video_metadata(media_tools, Path::new(&job.path)),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
let updated_images = {
|
let updated_image = with_db_write_lock(|| {
|
||||||
with_db_write_lock(|| {
|
|
||||||
let mut conn = pool.get()?;
|
let mut conn = pool.get()?;
|
||||||
let tx = conn.transaction()?;
|
let tx = conn.transaction()?;
|
||||||
let mut updated_images = Vec::new();
|
let updated = match metadata_result {
|
||||||
|
Ok(metadata) => Some(db::mark_metadata_ready(
|
||||||
for (image_id, metadata_result) in results {
|
|
||||||
let metadata = match metadata_result {
|
|
||||||
Ok(metadata) => metadata,
|
|
||||||
Err(error) => {
|
|
||||||
db::mark_metadata_failed(&tx, image_id, &error.to_string())?;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
updated_images.push(db::mark_metadata_ready(
|
|
||||||
&tx,
|
&tx,
|
||||||
image_id,
|
job.image_id,
|
||||||
metadata.duration_ms,
|
metadata.duration_ms,
|
||||||
metadata.width,
|
metadata.width,
|
||||||
metadata.height,
|
metadata.height,
|
||||||
metadata.video_codec.as_deref(),
|
metadata.video_codec.as_deref(),
|
||||||
metadata.audio_codec.as_deref(),
|
metadata.audio_codec.as_deref(),
|
||||||
)?);
|
)?),
|
||||||
|
Err(error) => {
|
||||||
|
db::mark_metadata_failed(&tx, job.image_id, &error.to_string())?;
|
||||||
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
tx.commit()?;
|
|
||||||
Ok(updated_images)
|
|
||||||
})?
|
|
||||||
};
|
};
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(updated)
|
||||||
|
})?;
|
||||||
|
|
||||||
if !updated_images.is_empty() {
|
if let Some(image) = updated_image {
|
||||||
let folder_ids = updated_images
|
let folder_id = image.folder_id;
|
||||||
.iter()
|
|
||||||
.map(|image| image.folder_id)
|
|
||||||
.collect::<HashSet<_>>();
|
|
||||||
emit_media_updates(
|
emit_media_updates(
|
||||||
app,
|
app,
|
||||||
&MediaUpdateBatch {
|
&MediaUpdateBatch {
|
||||||
images: updated_images,
|
images: vec![image],
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
emit_folder_job_progress(app, pool, &folder_ids.into_iter().collect::<Vec<_>>(), true);
|
emit_folder_job_progress(app, pool, &[folder_id], false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
|
||||||
|
/// the queue was empty.
|
||||||
fn process_embedding_batch(
|
fn process_embedding_batch(
|
||||||
app: &AppHandle,
|
app: &AppHandle,
|
||||||
pool: &DbPool,
|
pool: &DbPool,
|
||||||
embedder: &mut Option<ClipImageEmbedder>,
|
embedder: &mut Option<ClipImageEmbedder>,
|
||||||
) -> Result<()> {
|
) -> Result<bool> {
|
||||||
|
if higher_priority_work_pending(pool, WorkerTier::Embedding)? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
let batch_started_at = Instant::now();
|
let batch_started_at = Instant::now();
|
||||||
let claim_started_at = Instant::now();
|
let claim_started_at = Instant::now();
|
||||||
let paused_folders = paused_folder_ids("embedding");
|
// Exclude folders that are actively indexing (matching the thumbnail and
|
||||||
|
// metadata workers): embedding mid-scan competes with the scanner for
|
||||||
|
// CPU, disk, and the DB writer, and video jobs would fail-fast anyway
|
||||||
|
// because their thumbnails are deferred until indexing completes.
|
||||||
|
let mut excluded_folders = paused_folder_ids("embedding");
|
||||||
|
excluded_folders.extend(active_indexing_folders());
|
||||||
let jobs = with_db_write_lock(|| {
|
let jobs = with_db_write_lock(|| {
|
||||||
let mut conn = pool.get()?;
|
let mut conn = pool.get()?;
|
||||||
db::claim_embedding_jobs(&mut conn, &paused_folders, EMBEDDING_BATCH_SIZE)
|
db::claim_embedding_jobs(&mut conn, &excluded_folders, EMBEDDING_BATCH_SIZE)
|
||||||
})?;
|
})?;
|
||||||
let claim_elapsed = claim_started_at.elapsed();
|
let claim_elapsed = claim_started_at.elapsed();
|
||||||
|
|
||||||
if jobs.is_empty() {
|
if jobs.is_empty() {
|
||||||
return Ok(());
|
return Ok(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
if embedder.is_none() {
|
if embedder.is_none() {
|
||||||
@@ -888,7 +967,7 @@ fn process_embedding_batch(
|
|||||||
EMBEDDING_BATCH_SIZE, claim_elapsed, infer_elapsed, write_elapsed, batch_elapsed
|
EMBEDDING_BATCH_SIZE, claim_elapsed, infer_elapsed, write_elapsed, batch_elapsed
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok(())
|
Ok(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn process_caption_batch(
|
fn process_caption_batch(
|
||||||
@@ -993,25 +1072,34 @@ fn process_caption_batch(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
|
||||||
|
/// the queue was empty or the model is not ready.
|
||||||
fn process_tagging_batch(
|
fn process_tagging_batch(
|
||||||
app: &AppHandle,
|
app: &AppHandle,
|
||||||
pool: &DbPool,
|
pool: &DbPool,
|
||||||
app_data_dir: &Path,
|
app_data_dir: &Path,
|
||||||
tagger_instance: &mut Option<WdTagger>,
|
tagger_instance: &mut Option<WdTagger>,
|
||||||
) -> Result<()> {
|
) -> Result<bool> {
|
||||||
if !tagger::tagger_model_status(app_data_dir).ready {
|
if !tagger::tagger_model_status(app_data_dir).ready {
|
||||||
return Ok(());
|
return Ok(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
let paused_folders = paused_folder_ids("tagging");
|
if higher_priority_work_pending(pool, WorkerTier::Tagging)? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exclude actively-indexing folders for the same reason as the other
|
||||||
|
// workers: don't compete with a running scan.
|
||||||
|
let mut excluded_folders = paused_folder_ids("tagging");
|
||||||
|
excluded_folders.extend(active_indexing_folders());
|
||||||
let batch_size = crate::tagger::tagger_batch_size(app_data_dir);
|
let batch_size = crate::tagger::tagger_batch_size(app_data_dir);
|
||||||
let jobs = with_db_write_lock(|| {
|
let jobs = with_db_write_lock(|| {
|
||||||
let mut conn = pool.get()?;
|
let mut conn = pool.get()?;
|
||||||
db::claim_tagging_jobs(&mut conn, &paused_folders, batch_size)
|
db::claim_tagging_jobs(&mut conn, &excluded_folders, batch_size)
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
if jobs.is_empty() {
|
if jobs.is_empty() {
|
||||||
return Ok(());
|
return Ok(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
if tagger_instance.is_none() {
|
if tagger_instance.is_none() {
|
||||||
@@ -1119,7 +1207,7 @@ fn process_tagging_batch(
|
|||||||
emit_folder_job_progress(app, pool, &folder_ids.into_iter().collect::<Vec<_>>(), true);
|
emit_folder_job_progress(app, pool, &folder_ids.into_iter().collect::<Vec<_>>(), true);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn active_indexing_folders() -> HashSet<i64> {
|
fn active_indexing_folders() -> HashSet<i64> {
|
||||||
|
|||||||
+150
-12
@@ -31,29 +31,26 @@ pub fn generate_image_thumbnail(image_path: &Path, cache_dir: &Path) -> Result<G
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let reader = image::ImageReader::open(image_path)?.with_guessed_format()?;
|
let img = decode_for_thumbnail(image_path)?;
|
||||||
let mut decoder = reader.into_decoder()?;
|
|
||||||
let orientation = decoder.orientation()?;
|
|
||||||
let mut img = image::DynamicImage::from_decoder(decoder)?;
|
|
||||||
img.apply_orientation(orientation);
|
|
||||||
|
|
||||||
let src = image::DynamicImage::ImageRgba8(img.into_rgba8());
|
// RGB8 throughout: JPEG output has no alpha, so decoding/resizing RGBA
|
||||||
|
// only to flatten at encode time wastes a third of the bandwidth.
|
||||||
|
let src = image::DynamicImage::ImageRgb8(img.into_rgb8());
|
||||||
let (dst_width, dst_height) = fit_dimensions(src.width(), src.height(), THUMB_SIZE);
|
let (dst_width, dst_height) = fit_dimensions(src.width(), src.height(), THUMB_SIZE);
|
||||||
|
|
||||||
let mut dst = fir::images::Image::new(dst_width, dst_height, src.pixel_type().unwrap());
|
let mut dst = fir::images::Image::new(dst_width, dst_height, src.pixel_type().unwrap());
|
||||||
let mut resizer = fir::Resizer::new();
|
let mut resizer = fir::Resizer::new();
|
||||||
let options = fir::ResizeOptions::new()
|
let options = fir::ResizeOptions::new()
|
||||||
.resize_alg(fir::ResizeAlg::Convolution(fir::FilterType::Lanczos3));
|
.resize_alg(fir::ResizeAlg::Convolution(fir::FilterType::CatmullRom));
|
||||||
resizer.resize(&src, &mut dst, Some(&options))?;
|
resizer.resize(&src, &mut dst, Some(&options))?;
|
||||||
|
|
||||||
let thumb = image::RgbaImage::from_raw(dst_width, dst_height, dst.buffer().to_vec())
|
let thumb = image::RgbImage::from_raw(dst_width, dst_height, dst.buffer().to_vec())
|
||||||
.ok_or_else(|| anyhow!("failed to construct resized thumbnail buffer"))?;
|
.ok_or_else(|| anyhow!("failed to construct resized thumbnail buffer"))?;
|
||||||
|
|
||||||
if let Some(parent) = out_path.parent() {
|
if let Some(parent) = out_path.parent() {
|
||||||
std::fs::create_dir_all(parent)?;
|
std::fs::create_dir_all(parent)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let thumb = image::DynamicImage::ImageRgba8(thumb).into_rgb8();
|
|
||||||
let mut output_file = std::fs::File::create(&out_path)?;
|
let mut output_file = std::fs::File::create(&out_path)?;
|
||||||
let mut encoder = JpegEncoder::new_with_quality(&mut output_file, 82);
|
let mut encoder = JpegEncoder::new_with_quality(&mut output_file, 82);
|
||||||
encoder.encode_image(&thumb)?;
|
encoder.encode_image(&thumb)?;
|
||||||
@@ -64,6 +61,98 @@ pub fn generate_image_thumbnail(image_path: &Path, cache_dir: &Path) -> Result<G
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Decodes an image for thumbnailing, with EXIF orientation already applied.
|
||||||
|
fn decode_for_thumbnail(image_path: &Path) -> Result<image::DynamicImage> {
|
||||||
|
decode_image_scaled(image_path, THUMB_SIZE, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decodes an image at reduced resolution, with EXIF orientation applied.
|
||||||
|
///
|
||||||
|
/// JPEGs take a fast path that decodes at reduced resolution (DCT scaling),
|
||||||
|
/// which skips most of the IDCT/upsampling work for large photos. Anything
|
||||||
|
/// that path can't handle falls back to the `image` crate at full size.
|
||||||
|
///
|
||||||
|
/// `cover` selects which side must stay at or above `target`: `false` keeps
|
||||||
|
/// the longest side (for aspect-fit consumers), `true` keeps the shortest
|
||||||
|
/// side (for fill-crop consumers like the CLIP preprocessor).
|
||||||
|
pub fn decode_image_scaled(
|
||||||
|
image_path: &Path,
|
||||||
|
target: u32,
|
||||||
|
cover: bool,
|
||||||
|
) -> Result<image::DynamicImage> {
|
||||||
|
if is_jpeg(image_path) {
|
||||||
|
if let Some(img) = decode_jpeg_scaled(image_path, target, cover) {
|
||||||
|
return Ok(img);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let reader = image::ImageReader::open(image_path)?.with_guessed_format()?;
|
||||||
|
let mut decoder = reader.into_decoder()?;
|
||||||
|
let orientation = decoder.orientation()?;
|
||||||
|
let mut img = image::DynamicImage::from_decoder(decoder)?;
|
||||||
|
img.apply_orientation(orientation);
|
||||||
|
Ok(img)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_jpeg(path: &Path) -> bool {
|
||||||
|
path.extension()
|
||||||
|
.and_then(|ext| ext.to_str())
|
||||||
|
.is_some_and(|ext| ext.eq_ignore_ascii_case("jpg") || ext.eq_ignore_ascii_case("jpeg"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decodes a JPEG at the smallest DCT scale that still covers `target`.
|
||||||
|
/// Returns `None` on any failure (corrupt file, CMYK without conversion
|
||||||
|
/// support, etc.) so the caller can fall back to the generic decoder.
|
||||||
|
/// mozjpeg reports errors by panicking, hence the `catch_unwind`.
|
||||||
|
fn decode_jpeg_scaled(image_path: &Path, target: u32, cover: bool) -> Option<image::DynamicImage> {
|
||||||
|
let path = image_path.to_path_buf();
|
||||||
|
let decoded = std::panic::catch_unwind(move || -> std::io::Result<(Vec<u8>, u32, u32)> {
|
||||||
|
let mut decompress = mozjpeg::Decompress::new_path(&path)?;
|
||||||
|
let (width, height) = decompress.size();
|
||||||
|
let reference = if cover {
|
||||||
|
width.min(height)
|
||||||
|
} else {
|
||||||
|
width.max(height)
|
||||||
|
};
|
||||||
|
decompress.scale(scale_numerator(reference, target));
|
||||||
|
let mut started = decompress.rgb()?;
|
||||||
|
let (out_width, out_height) = (started.width() as u32, started.height() as u32);
|
||||||
|
let pixels = started.read_scanlines::<u8>()?;
|
||||||
|
started.finish()?;
|
||||||
|
Ok((pixels, out_width, out_height))
|
||||||
|
})
|
||||||
|
.ok()?
|
||||||
|
.ok()?;
|
||||||
|
|
||||||
|
let (pixels, width, height) = decoded;
|
||||||
|
let buffer = image::RgbImage::from_raw(width, height, pixels)?;
|
||||||
|
let mut img = image::DynamicImage::ImageRgb8(buffer);
|
||||||
|
if let Some(orientation) = exif_orientation(image_path) {
|
||||||
|
img.apply_orientation(orientation);
|
||||||
|
}
|
||||||
|
Some(img)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Smallest numerator (of /8) that keeps `reference` at or above `target`,
|
||||||
|
/// so the subsequent resize never upscales.
|
||||||
|
fn scale_numerator(reference: usize, target: u32) -> u8 {
|
||||||
|
for numerator in 1..=8u8 {
|
||||||
|
if reference * numerator as usize / 8 >= target as usize {
|
||||||
|
return numerator;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
8
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exif_orientation(path: &Path) -> Option<image::metadata::Orientation> {
|
||||||
|
let file = std::fs::File::open(path).ok()?;
|
||||||
|
let mut reader = std::io::BufReader::new(file);
|
||||||
|
let exif = exif::Reader::new().read_from_container(&mut reader).ok()?;
|
||||||
|
let field = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY)?;
|
||||||
|
let value = field.value.get_uint(0)?;
|
||||||
|
image::metadata::Orientation::from_exif(value as u8)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn generate_video_thumbnail(
|
pub fn generate_video_thumbnail(
|
||||||
tools: &MediaTools,
|
tools: &MediaTools,
|
||||||
video_path: &Path,
|
video_path: &Path,
|
||||||
@@ -88,6 +177,8 @@ pub fn generate_video_thumbnail(
|
|||||||
let attempts: [&[&str]; 3] = [
|
let attempts: [&[&str]; 3] = [
|
||||||
&[
|
&[
|
||||||
"-y",
|
"-y",
|
||||||
|
"-threads",
|
||||||
|
"2",
|
||||||
"-ss",
|
"-ss",
|
||||||
"00:00:00.000",
|
"00:00:00.000",
|
||||||
"-i",
|
"-i",
|
||||||
@@ -95,13 +186,15 @@ pub fn generate_video_thumbnail(
|
|||||||
"-frames:v",
|
"-frames:v",
|
||||||
"1",
|
"1",
|
||||||
"-vf",
|
"-vf",
|
||||||
"thumbnail,scale=320:-1:force_original_aspect_ratio=decrease",
|
"scale=320:-1:force_original_aspect_ratio=decrease",
|
||||||
"-q:v",
|
"-q:v",
|
||||||
"4",
|
"4",
|
||||||
&output_path,
|
&output_path,
|
||||||
],
|
],
|
||||||
&[
|
&[
|
||||||
"-y",
|
"-y",
|
||||||
|
"-threads",
|
||||||
|
"2",
|
||||||
"-ss",
|
"-ss",
|
||||||
"00:00:00.250",
|
"00:00:00.250",
|
||||||
"-i",
|
"-i",
|
||||||
@@ -109,19 +202,21 @@ pub fn generate_video_thumbnail(
|
|||||||
"-frames:v",
|
"-frames:v",
|
||||||
"1",
|
"1",
|
||||||
"-vf",
|
"-vf",
|
||||||
"thumbnail,scale=320:-1:force_original_aspect_ratio=decrease",
|
"scale=320:-1:force_original_aspect_ratio=decrease",
|
||||||
"-q:v",
|
"-q:v",
|
||||||
"4",
|
"4",
|
||||||
&output_path,
|
&output_path,
|
||||||
],
|
],
|
||||||
&[
|
&[
|
||||||
"-y",
|
"-y",
|
||||||
|
"-threads",
|
||||||
|
"2",
|
||||||
"-i",
|
"-i",
|
||||||
path_str.as_ref(),
|
path_str.as_ref(),
|
||||||
"-frames:v",
|
"-frames:v",
|
||||||
"1",
|
"1",
|
||||||
"-vf",
|
"-vf",
|
||||||
"thumbnail,scale=320:-1:force_original_aspect_ratio=decrease",
|
"scale=320:-1:force_original_aspect_ratio=decrease",
|
||||||
"-q:v",
|
"-q:v",
|
||||||
"4",
|
"4",
|
||||||
&output_path,
|
&output_path,
|
||||||
@@ -165,6 +260,49 @@ fn hash_path(s: &str) -> String {
|
|||||||
format!("{:016x}", xxhash_rust::xxh3::xxh3_64(s.as_bytes()))
|
format!("{:016x}", xxhash_rust::xxh3::xxh3_64(s.as_bytes()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scale_numerator_picks_smallest_sufficient() {
|
||||||
|
assert_eq!(scale_numerator(6000, THUMB_SIZE), 1);
|
||||||
|
assert_eq!(scale_numerator(640, THUMB_SIZE), 4);
|
||||||
|
assert_eq!(scale_numerator(320, THUMB_SIZE), 8);
|
||||||
|
assert_eq!(scale_numerator(100, THUMB_SIZE), 8);
|
||||||
|
// Cover mode reference: shortest side must reach 224 for CLIP.
|
||||||
|
assert_eq!(scale_numerator(1200, 224), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn jpeg_fast_path_decodes_at_reduced_resolution() {
|
||||||
|
let dir = std::env::temp_dir().join("phokus-thumb-test");
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let src_path = dir.join("large.jpg");
|
||||||
|
let img =
|
||||||
|
image::RgbImage::from_fn(1600, 1200, |x, _| image::Rgb([(x % 256) as u8, 64, 128]));
|
||||||
|
img.save(&src_path).unwrap();
|
||||||
|
|
||||||
|
// 1600 max dim -> numerator 2 -> 400x300 decode output
|
||||||
|
let decoded = decode_jpeg_scaled(&src_path, THUMB_SIZE, false)
|
||||||
|
.expect("fast path should handle plain JPEG");
|
||||||
|
assert_eq!((decoded.width(), decoded.height()), (400, 300));
|
||||||
|
|
||||||
|
// Cover mode: shortest side (1200) must stay >= 224 -> numerator 2
|
||||||
|
let covered = decode_jpeg_scaled(&src_path, 224, true)
|
||||||
|
.expect("fast path should handle plain JPEG");
|
||||||
|
assert_eq!((covered.width(), covered.height()), (400, 300));
|
||||||
|
|
||||||
|
let cache = dir.join("cache");
|
||||||
|
let _ = std::fs::remove_file(thumb_path(&cache, &src_path.to_string_lossy()));
|
||||||
|
let result = generate_image_thumbnail(&src_path, &cache).unwrap();
|
||||||
|
assert_eq!(result.width, Some(1600));
|
||||||
|
assert_eq!(result.height, Some(1200));
|
||||||
|
let (thumb_w, thumb_h) = image::image_dimensions(&result.path).unwrap();
|
||||||
|
assert_eq!((thumb_w, thumb_h), (320, 240));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn fit_dimensions(width: u32, height: u32, max_size: u32) -> (u32, u32) {
|
fn fit_dimensions(width: u32, height: u32, max_size: u32) -> (u32, u32) {
|
||||||
if width <= max_size && height <= max_size {
|
if width <= max_size && height <= max_size {
|
||||||
return (width.max(1), height.max(1));
|
return (width.max(1), height.max(1));
|
||||||
|
|||||||
@@ -278,12 +278,16 @@ export function BackgroundTasks() {
|
|||||||
id: -1,
|
id: -1,
|
||||||
name: "Duplicate Scan",
|
name: "Duplicate Scan",
|
||||||
stages: [{
|
stages: [{
|
||||||
label: "Hashing",
|
label: duplicateScanProgress?.phase === "checking"
|
||||||
|
? "Checking"
|
||||||
|
: duplicateScanProgress?.phase === "confirming"
|
||||||
|
? "Confirming"
|
||||||
|
: "Hashing",
|
||||||
detail: duplicateScanProgress
|
detail: duplicateScanProgress
|
||||||
? `${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}`
|
? `${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}`
|
||||||
: "Starting…",
|
: "Starting…",
|
||||||
progress: duplicateScanProgress && duplicateScanProgress.total > 0
|
progress: duplicateScanProgress && duplicateScanProgress.total > 0
|
||||||
? (duplicateScanProgress.scanned / duplicateScanProgress.total) * 100
|
? (duplicateScanProgress.processed / duplicateScanProgress.total) * 100
|
||||||
: null,
|
: null,
|
||||||
failed: false,
|
failed: false,
|
||||||
}],
|
}],
|
||||||
@@ -310,7 +314,8 @@ export function BackgroundTasks() {
|
|||||||
const embeddingStage = primary.stages.find((s) => s.label === "Embeddings");
|
const embeddingStage = primary.stages.find((s) => s.label === "Embeddings");
|
||||||
const taggingStage = primary.stages.find((s) => s.label === "Tags");
|
const taggingStage = primary.stages.find((s) => s.label === "Tags");
|
||||||
const scanningStage = primary.stages.find((s) => s.label === "Scanning");
|
const scanningStage = primary.stages.find((s) => s.label === "Scanning");
|
||||||
const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? null;
|
const duplicateStage = primary.id === -1 ? primary.stages[0] : null;
|
||||||
|
const barProgress = embeddingStage?.progress ?? taggingStage?.progress ?? scanningStage?.progress ?? duplicateStage?.progress ?? null;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="shrink-0 border-b border-white/[0.06]">
|
<div className="shrink-0 border-b border-white/[0.06]">
|
||||||
@@ -434,7 +439,8 @@ export function BackgroundTasks() {
|
|||||||
const taskEmbeddingStage = task.stages.find((s) => s.label === "Embeddings");
|
const taskEmbeddingStage = task.stages.find((s) => s.label === "Embeddings");
|
||||||
const taskTaggingStage = task.stages.find((s) => s.label === "Tags");
|
const taskTaggingStage = task.stages.find((s) => s.label === "Tags");
|
||||||
const taskScanningStage = task.stages.find((s) => s.label === "Scanning");
|
const taskScanningStage = task.stages.find((s) => s.label === "Scanning");
|
||||||
const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? null;
|
const taskDuplicateStage = task.id === -1 ? task.stages[0] : null;
|
||||||
|
const taskBarProgress = taskEmbeddingStage?.progress ?? taskTaggingStage?.progress ?? taskScanningStage?.progress ?? taskDuplicateStage?.progress ?? null;
|
||||||
const taskHasFailed = (task.hasFailedEmbeddings || task.hasFailedTagging || task.hasFailedCaptions) && task.pendingMediaWork === 0;
|
const taskHasFailed = (task.hasFailedEmbeddings || task.hasFailedTagging || task.hasFailedCaptions) && task.pendingMediaWork === 0;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
|||||||
@@ -117,6 +117,7 @@ export function DuplicateFinder() {
|
|||||||
const duplicateScanning = useGalleryStore((state) => state.duplicateScanning);
|
const duplicateScanning = useGalleryStore((state) => state.duplicateScanning);
|
||||||
const duplicateScanProgress = useGalleryStore((state) => state.duplicateScanProgress);
|
const duplicateScanProgress = useGalleryStore((state) => state.duplicateScanProgress);
|
||||||
const duplicateScanError = useGalleryStore((state) => state.duplicateScanError);
|
const duplicateScanError = useGalleryStore((state) => state.duplicateScanError);
|
||||||
|
const duplicateScanWarning = useGalleryStore((state) => state.duplicateScanWarning);
|
||||||
const duplicateSelectedIds = useGalleryStore((state) => state.duplicateSelectedIds);
|
const duplicateSelectedIds = useGalleryStore((state) => state.duplicateSelectedIds);
|
||||||
const duplicateLastScanned = useGalleryStore((state) => state.duplicateLastScanned);
|
const duplicateLastScanned = useGalleryStore((state) => state.duplicateLastScanned);
|
||||||
const selectedFolderId = useGalleryStore((state) => state.selectedFolderId);
|
const selectedFolderId = useGalleryStore((state) => state.selectedFolderId);
|
||||||
@@ -152,8 +153,15 @@ export function DuplicateFinder() {
|
|||||||
|
|
||||||
const progressPercent =
|
const progressPercent =
|
||||||
duplicateScanProgress && duplicateScanProgress.total > 0
|
duplicateScanProgress && duplicateScanProgress.total > 0
|
||||||
? Math.round((duplicateScanProgress.scanned / duplicateScanProgress.total) * 100)
|
? Math.round((duplicateScanProgress.processed / duplicateScanProgress.total) * 100)
|
||||||
: 0;
|
: 0;
|
||||||
|
const progressLabel = duplicateScanProgress
|
||||||
|
? duplicateScanProgress.phase === "checking"
|
||||||
|
? "Checking file sizes"
|
||||||
|
: duplicateScanProgress.phase === "hashing"
|
||||||
|
? "Hashing duplicate candidates"
|
||||||
|
: "Confirming exact matches"
|
||||||
|
: null;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="flex min-h-0 flex-1 flex-col overflow-hidden bg-[#07080f]">
|
<div className="flex min-h-0 flex-1 flex-col overflow-hidden bg-[#07080f]">
|
||||||
@@ -165,7 +173,7 @@ export function DuplicateFinder() {
|
|||||||
<p className="mt-0.5 text-[11px] text-white/30">
|
<p className="mt-0.5 text-[11px] text-white/30">
|
||||||
{duplicateScanning
|
{duplicateScanning
|
||||||
? duplicateScanProgress
|
? duplicateScanProgress
|
||||||
? `Scanning… ${duplicateScanProgress.scanned.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}`
|
? `${progressLabel}… ${duplicateScanProgress.processed.toLocaleString()} / ${duplicateScanProgress.total.toLocaleString()}${duplicateScanProgress.skipped > 0 ? ` · ${duplicateScanProgress.skipped.toLocaleString()} skipped` : ""}`
|
||||||
: "Starting scan…"
|
: "Starting scan…"
|
||||||
: hasResults
|
: hasResults
|
||||||
? `${duplicateGroups.length} group${duplicateGroups.length === 1 ? "" : "s"} · ${formatBytes(totalWasted)} reclaimable`
|
? `${duplicateGroups.length} group${duplicateGroups.length === 1 ? "" : "s"} · ${formatBytes(totalWasted)} reclaimable`
|
||||||
@@ -232,6 +240,9 @@ export function DuplicateFinder() {
|
|||||||
{duplicateScanError ? (
|
{duplicateScanError ? (
|
||||||
<p className="mt-2 text-[11px] text-red-400/80">{duplicateScanError}</p>
|
<p className="mt-2 text-[11px] text-red-400/80">{duplicateScanError}</p>
|
||||||
) : null}
|
) : null}
|
||||||
|
{duplicateScanWarning ? (
|
||||||
|
<p className="mt-2 text-[11px] text-amber-300/70">{duplicateScanWarning}</p>
|
||||||
|
) : null}
|
||||||
{deleteResult ? (
|
{deleteResult ? (
|
||||||
<p className="mt-2 text-[11px] text-white/40">{deleteResult}</p>
|
<p className="mt-2 text-[11px] text-white/40">{deleteResult}</p>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -241,7 +252,7 @@ export function DuplicateFinder() {
|
|||||||
{duplicateScanning && !hasResults ? (
|
{duplicateScanning && !hasResults ? (
|
||||||
<div className="flex flex-1 items-center justify-center gap-3 text-white/25">
|
<div className="flex flex-1 items-center justify-center gap-3 text-white/25">
|
||||||
<div className="h-5 w-5 animate-spin rounded-full border-2 border-white/15 border-t-white/50" />
|
<div className="h-5 w-5 animate-spin rounded-full border-2 border-white/15 border-t-white/50" />
|
||||||
<span className="text-sm">Hashing files…</span>
|
<span className="text-sm">{progressLabel ? `${progressLabel}…` : "Preparing scan…"}</span>
|
||||||
</div>
|
</div>
|
||||||
) : !hasScanned ? (
|
) : !hasScanned ? (
|
||||||
<div className="flex flex-1 items-center justify-center px-8">
|
<div className="flex flex-1 items-center justify-center px-8">
|
||||||
|
|||||||
+51
-10
@@ -170,6 +170,20 @@ export interface DuplicateGroup {
|
|||||||
images: ImageRecord[];
|
images: ImageRecord[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface DuplicateScanProgress {
|
||||||
|
phase: "checking" | "hashing" | "confirming";
|
||||||
|
processed: number;
|
||||||
|
total: number;
|
||||||
|
skipped: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DuplicateScanResult {
|
||||||
|
groups: DuplicateGroup[];
|
||||||
|
scanned_files: number;
|
||||||
|
candidate_files: number;
|
||||||
|
skipped_files: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface SimilarImagesPage {
|
export interface SimilarImagesPage {
|
||||||
images: ImageRecord[];
|
images: ImageRecord[];
|
||||||
offset: number;
|
offset: number;
|
||||||
@@ -308,8 +322,9 @@ interface GalleryState {
|
|||||||
|
|
||||||
duplicateGroups: DuplicateGroup[];
|
duplicateGroups: DuplicateGroup[];
|
||||||
duplicateScanning: boolean;
|
duplicateScanning: boolean;
|
||||||
duplicateScanProgress: { scanned: number; total: number } | null;
|
duplicateScanProgress: DuplicateScanProgress | null;
|
||||||
duplicateScanError: string | null;
|
duplicateScanError: string | null;
|
||||||
|
duplicateScanWarning: string | null;
|
||||||
duplicateSelectedIds: Set<number>;
|
duplicateSelectedIds: Set<number>;
|
||||||
duplicateLastScanned: number | null; // Unix timestamp (seconds)
|
duplicateLastScanned: number | null; // Unix timestamp (seconds)
|
||||||
duplicateScanFolderId: number | null | undefined; // undefined = never scanned
|
duplicateScanFolderId: number | null | undefined; // undefined = never scanned
|
||||||
@@ -663,6 +678,7 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
|
|||||||
duplicateScanning: false,
|
duplicateScanning: false,
|
||||||
duplicateScanProgress: null,
|
duplicateScanProgress: null,
|
||||||
duplicateScanError: null,
|
duplicateScanError: null,
|
||||||
|
duplicateScanWarning: null,
|
||||||
duplicateSelectedIds: new Set(),
|
duplicateSelectedIds: new Set(),
|
||||||
duplicateLastScanned: null,
|
duplicateLastScanned: null,
|
||||||
duplicateScanFolderId: undefined,
|
duplicateScanFolderId: undefined,
|
||||||
@@ -954,7 +970,13 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
|
|||||||
if (activeView === "duplicates") {
|
if (activeView === "duplicates") {
|
||||||
const { selectedFolderId, duplicateScanFolderId } = get();
|
const { selectedFolderId, duplicateScanFolderId } = get();
|
||||||
if (duplicateScanFolderId !== selectedFolderId) {
|
if (duplicateScanFolderId !== selectedFolderId) {
|
||||||
set({ activeView, duplicateGroups: [], duplicateLastScanned: null, duplicateScanFolderId: undefined });
|
set({
|
||||||
|
activeView,
|
||||||
|
duplicateGroups: [],
|
||||||
|
duplicateLastScanned: null,
|
||||||
|
duplicateScanFolderId: undefined,
|
||||||
|
duplicateScanWarning: null,
|
||||||
|
});
|
||||||
void get().loadDuplicateScanCache(selectedFolderId);
|
void get().loadDuplicateScanCache(selectedFolderId);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1583,23 +1605,42 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
|
|||||||
interface CacheResult { groups: DuplicateGroup[]; scanned_at: number }
|
interface CacheResult { groups: DuplicateGroup[]; scanned_at: number }
|
||||||
const cached = await invoke<CacheResult | null>("load_duplicate_scan_cache", { folderId: folderId ?? null });
|
const cached = await invoke<CacheResult | null>("load_duplicate_scan_cache", { folderId: folderId ?? null });
|
||||||
if (cached) {
|
if (cached) {
|
||||||
set({ duplicateGroups: cached.groups, duplicateLastScanned: cached.scanned_at, duplicateScanFolderId: folderId });
|
set({
|
||||||
|
duplicateGroups: cached.groups,
|
||||||
|
duplicateLastScanned: cached.scanned_at,
|
||||||
|
duplicateScanFolderId: folderId,
|
||||||
|
duplicateScanWarning: null,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
scanDuplicates: async (folderId = null) => {
|
scanDuplicates: async (folderId = null) => {
|
||||||
const { listen } = await import("@tauri-apps/api/event");
|
const { listen } = await import("@tauri-apps/api/event");
|
||||||
set({ duplicateScanning: true, duplicateGroups: [], duplicateScanProgress: null, duplicateScanError: null, duplicateSelectedIds: new Set() });
|
set({
|
||||||
const unlisten = await listen<[number, number]>("duplicate_scan_progress", (event) => {
|
duplicateScanning: true,
|
||||||
const [scanned, total] = event.payload;
|
duplicateGroups: [],
|
||||||
set({ duplicateScanProgress: { scanned, total } });
|
duplicateScanProgress: null,
|
||||||
|
duplicateScanError: null,
|
||||||
|
duplicateScanWarning: null,
|
||||||
|
duplicateSelectedIds: new Set(),
|
||||||
|
});
|
||||||
|
const unlisten = await listen<DuplicateScanProgress>("duplicate_scan_progress", (event) => {
|
||||||
|
set({ duplicateScanProgress: event.payload });
|
||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
const groups = await invoke<DuplicateGroup[]>("find_duplicates", { folderId: folderId ?? null });
|
const result = await invoke<DuplicateScanResult>("find_duplicates", { folderId: folderId ?? null });
|
||||||
set({ duplicateGroups: groups, duplicateLastScanned: Math.floor(Date.now() / 1000), duplicateScanFolderId: folderId });
|
const warning = result.skipped_files > 0
|
||||||
|
? `${result.skipped_files.toLocaleString()} file${result.skipped_files === 1 ? "" : "s"} could not be read and were skipped.`
|
||||||
|
: null;
|
||||||
|
set({
|
||||||
|
duplicateGroups: result.groups,
|
||||||
|
duplicateLastScanned: Math.floor(Date.now() / 1000),
|
||||||
|
duplicateScanFolderId: folderId,
|
||||||
|
duplicateScanWarning: warning,
|
||||||
|
});
|
||||||
void notifyTaskComplete(
|
void notifyTaskComplete(
|
||||||
"Duplicate scan complete",
|
"Duplicate scan complete",
|
||||||
groups.length === 1 ? "Found 1 duplicate group." : `Found ${groups.length.toLocaleString()} duplicate groups.`,
|
`${result.groups.length === 1 ? "Found 1 duplicate group." : `Found ${result.groups.length.toLocaleString()} duplicate groups.`}${warning ? ` ${warning}` : ""}`,
|
||||||
);
|
);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
set({ duplicateScanError: String(e) });
|
set({ duplicateScanError: String(e) });
|
||||||
|
|||||||
Reference in New Issue
Block a user