Optimize thumbnail workers and adaptive indexing
This commit is contained in:
Generated
+97
-12
@@ -2450,6 +2450,15 @@ version = "0.1.14"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
|
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ntapi"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae"
|
||||||
|
dependencies = [
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-conv"
|
name = "num-conv"
|
||||||
version = "0.2.1"
|
version = "0.2.1"
|
||||||
@@ -2562,6 +2571,16 @@ dependencies = [
|
|||||||
"objc2-core-foundation",
|
"objc2-core-foundation",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "objc2-io-kit"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"objc2-core-foundation",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "objc2-io-surface"
|
name = "objc2-io-surface"
|
||||||
version = "0.3.2"
|
version = "0.3.2"
|
||||||
@@ -2915,6 +2934,7 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sqlite-vec",
|
"sqlite-vec",
|
||||||
|
"sysinfo",
|
||||||
"tauri",
|
"tauri",
|
||||||
"tauri-build",
|
"tauri-build",
|
||||||
"tauri-plugin-dialog",
|
"tauri-plugin-dialog",
|
||||||
@@ -4070,6 +4090,20 @@ dependencies = [
|
|||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sysinfo"
|
||||||
|
version = "0.38.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"memchr",
|
||||||
|
"ntapi",
|
||||||
|
"objc2-core-foundation",
|
||||||
|
"objc2-io-kit",
|
||||||
|
"windows 0.62.2",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "system-deps"
|
name = "system-deps"
|
||||||
version = "6.2.2"
|
version = "6.2.2"
|
||||||
@@ -4115,7 +4149,7 @@ dependencies = [
|
|||||||
"tao-macros",
|
"tao-macros",
|
||||||
"unicode-segmentation",
|
"unicode-segmentation",
|
||||||
"url",
|
"url",
|
||||||
"windows",
|
"windows 0.61.3",
|
||||||
"windows-core 0.61.2",
|
"windows-core 0.61.2",
|
||||||
"windows-version",
|
"windows-version",
|
||||||
"x11-dl",
|
"x11-dl",
|
||||||
@@ -4198,7 +4232,7 @@ dependencies = [
|
|||||||
"webkit2gtk",
|
"webkit2gtk",
|
||||||
"webview2-com",
|
"webview2-com",
|
||||||
"window-vibrancy",
|
"window-vibrancy",
|
||||||
"windows",
|
"windows 0.61.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -4341,7 +4375,7 @@ dependencies = [
|
|||||||
"tauri-plugin",
|
"tauri-plugin",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
"url",
|
"url",
|
||||||
"windows",
|
"windows 0.61.3",
|
||||||
"zbus",
|
"zbus",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -4367,7 +4401,7 @@ dependencies = [
|
|||||||
"url",
|
"url",
|
||||||
"webkit2gtk",
|
"webkit2gtk",
|
||||||
"webview2-com",
|
"webview2-com",
|
||||||
"windows",
|
"windows 0.61.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -4392,7 +4426,7 @@ dependencies = [
|
|||||||
"url",
|
"url",
|
||||||
"webkit2gtk",
|
"webkit2gtk",
|
||||||
"webview2-com",
|
"webview2-com",
|
||||||
"windows",
|
"windows 0.61.3",
|
||||||
"wry",
|
"wry",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -5269,7 +5303,7 @@ checksum = "7130243a7a5b33c54a444e54842e6a9e133de08b5ad7b5861cd8ed9a6a5bc96a"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"webview2-com-macros",
|
"webview2-com-macros",
|
||||||
"webview2-com-sys",
|
"webview2-com-sys",
|
||||||
"windows",
|
"windows 0.61.3",
|
||||||
"windows-core 0.61.2",
|
"windows-core 0.61.2",
|
||||||
"windows-implement",
|
"windows-implement",
|
||||||
"windows-interface",
|
"windows-interface",
|
||||||
@@ -5293,7 +5327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "381336cfffd772377d291702245447a5251a2ffa5bad679c99e61bc48bacbf9c"
|
checksum = "381336cfffd772377d291702245447a5251a2ffa5bad679c99e61bc48bacbf9c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
"windows",
|
"windows 0.61.3",
|
||||||
"windows-core 0.61.2",
|
"windows-core 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -5355,11 +5389,23 @@ version = "0.61.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
|
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"windows-collections",
|
"windows-collections 0.2.0",
|
||||||
"windows-core 0.61.2",
|
"windows-core 0.61.2",
|
||||||
"windows-future",
|
"windows-future 0.2.1",
|
||||||
"windows-link 0.1.3",
|
"windows-link 0.1.3",
|
||||||
"windows-numerics",
|
"windows-numerics 0.2.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows"
|
||||||
|
version = "0.62.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580"
|
||||||
|
dependencies = [
|
||||||
|
"windows-collections 0.3.2",
|
||||||
|
"windows-core 0.62.2",
|
||||||
|
"windows-future 0.3.2",
|
||||||
|
"windows-numerics 0.3.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -5371,6 +5417,15 @@ dependencies = [
|
|||||||
"windows-core 0.61.2",
|
"windows-core 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-collections"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610"
|
||||||
|
dependencies = [
|
||||||
|
"windows-core 0.62.2",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-core"
|
name = "windows-core"
|
||||||
version = "0.61.2"
|
version = "0.61.2"
|
||||||
@@ -5405,7 +5460,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"windows-core 0.61.2",
|
"windows-core 0.61.2",
|
||||||
"windows-link 0.1.3",
|
"windows-link 0.1.3",
|
||||||
"windows-threading",
|
"windows-threading 0.1.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-future"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb"
|
||||||
|
dependencies = [
|
||||||
|
"windows-core 0.62.2",
|
||||||
|
"windows-link 0.2.1",
|
||||||
|
"windows-threading 0.2.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -5452,6 +5518,16 @@ dependencies = [
|
|||||||
"windows-link 0.1.3",
|
"windows-link 0.1.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-numerics"
|
||||||
|
version = "0.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26"
|
||||||
|
dependencies = [
|
||||||
|
"windows-core 0.62.2",
|
||||||
|
"windows-link 0.2.1",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-result"
|
name = "windows-result"
|
||||||
version = "0.3.4"
|
version = "0.3.4"
|
||||||
@@ -5590,6 +5666,15 @@ dependencies = [
|
|||||||
"windows-link 0.1.3",
|
"windows-link 0.1.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-threading"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37"
|
||||||
|
dependencies = [
|
||||||
|
"windows-link 0.2.1",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-version"
|
name = "windows-version"
|
||||||
version = "0.1.7"
|
version = "0.1.7"
|
||||||
@@ -5906,7 +5991,7 @@ dependencies = [
|
|||||||
"webkit2gtk",
|
"webkit2gtk",
|
||||||
"webkit2gtk-sys",
|
"webkit2gtk-sys",
|
||||||
"webview2-com",
|
"webview2-com",
|
||||||
"windows",
|
"windows 0.61.3",
|
||||||
"windows-core 0.61.2",
|
"windows-core 0.61.2",
|
||||||
"windows-version",
|
"windows-version",
|
||||||
"x11-dl",
|
"x11-dl",
|
||||||
|
|||||||
@@ -34,3 +34,4 @@ anyhow = "1"
|
|||||||
log = "0.4"
|
log = "0.4"
|
||||||
ffmpeg-sidecar = "2.5.0"
|
ffmpeg-sidecar = "2.5.0"
|
||||||
xxhash-rust = { version = "0.8", features = ["xxh3"] }
|
xxhash-rust = { version = "0.8", features = ["xxh3"] }
|
||||||
|
sysinfo = "0.38.4"
|
||||||
|
|||||||
+66
-16
@@ -453,6 +453,41 @@ pub fn get_pending_thumbnail_jobs(conn: &Connection, limit: usize) -> Result<Vec
|
|||||||
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
|
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn claim_thumbnail_jobs(
|
||||||
|
conn: &mut Connection,
|
||||||
|
active_folder_ids: &std::collections::HashSet<i64>,
|
||||||
|
fetch_limit: usize,
|
||||||
|
claim_limit: usize,
|
||||||
|
) -> Result<Vec<ThumbnailJob>> {
|
||||||
|
let tx = conn.transaction()?;
|
||||||
|
let candidates = get_pending_thumbnail_jobs(&tx, fetch_limit)?;
|
||||||
|
let mut claimed = Vec::with_capacity(claim_limit);
|
||||||
|
|
||||||
|
for job in candidates {
|
||||||
|
if active_folder_ids.contains(&job.folder_id) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let updated = tx.execute(
|
||||||
|
"UPDATE thumbnail_jobs
|
||||||
|
SET status = 'processing', attempts = attempts + 1, updated_at = datetime('now')
|
||||||
|
WHERE image_id = ?1 AND status = 'pending'",
|
||||||
|
[job.image_id],
|
||||||
|
)?;
|
||||||
|
|
||||||
|
if updated == 1 {
|
||||||
|
claimed.push(job);
|
||||||
|
}
|
||||||
|
|
||||||
|
if claimed.len() >= claim_limit {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(claimed)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get_pending_metadata_jobs(conn: &Connection, limit: usize) -> Result<Vec<MetadataJob>> {
|
pub fn get_pending_metadata_jobs(conn: &Connection, limit: usize) -> Result<Vec<MetadataJob>> {
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(
|
||||||
"SELECT j.image_id, i.folder_id, i.path
|
"SELECT j.image_id, i.folder_id, i.path
|
||||||
@@ -472,14 +507,39 @@ pub fn get_pending_metadata_jobs(conn: &Connection, limit: usize) -> Result<Vec<
|
|||||||
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
|
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn mark_thumbnail_job_processing(conn: &Connection, image_id: i64) -> Result<()> {
|
pub fn claim_metadata_jobs(
|
||||||
conn.execute(
|
conn: &mut Connection,
|
||||||
"UPDATE thumbnail_jobs
|
active_folder_ids: &std::collections::HashSet<i64>,
|
||||||
|
fetch_limit: usize,
|
||||||
|
claim_limit: usize,
|
||||||
|
) -> Result<Vec<MetadataJob>> {
|
||||||
|
let tx = conn.transaction()?;
|
||||||
|
let candidates = get_pending_metadata_jobs(&tx, fetch_limit)?;
|
||||||
|
let mut claimed = Vec::with_capacity(claim_limit);
|
||||||
|
|
||||||
|
for job in candidates {
|
||||||
|
if active_folder_ids.contains(&job.folder_id) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let updated = tx.execute(
|
||||||
|
"UPDATE metadata_jobs
|
||||||
SET status = 'processing', attempts = attempts + 1, updated_at = datetime('now')
|
SET status = 'processing', attempts = attempts + 1, updated_at = datetime('now')
|
||||||
WHERE image_id = ?1",
|
WHERE image_id = ?1 AND status = 'pending'",
|
||||||
[image_id],
|
[job.image_id],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
|
||||||
|
if updated == 1 {
|
||||||
|
claimed.push(job);
|
||||||
|
}
|
||||||
|
|
||||||
|
if claimed.len() >= claim_limit {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(claimed)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn mark_thumbnail_ready(
|
pub fn mark_thumbnail_ready(
|
||||||
@@ -511,16 +571,6 @@ pub fn mark_thumbnail_failed(conn: &Connection, image_id: i64, error: &str) -> R
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn mark_metadata_job_processing(conn: &Connection, image_id: i64) -> Result<()> {
|
|
||||||
conn.execute(
|
|
||||||
"UPDATE metadata_jobs
|
|
||||||
SET status = 'processing', attempts = attempts + 1, updated_at = datetime('now')
|
|
||||||
WHERE image_id = ?1",
|
|
||||||
[image_id],
|
|
||||||
)?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn mark_metadata_ready(
|
pub fn mark_metadata_ready(
|
||||||
conn: &Connection,
|
conn: &Connection,
|
||||||
image_id: i64,
|
image_id: i64,
|
||||||
|
|||||||
+131
-44
@@ -1,5 +1,6 @@
|
|||||||
use crate::db::{self, DbPool, FolderJobProgress, ImageRecord, IndexedMediaEntry};
|
use crate::db::{self, DbPool, FolderJobProgress, ImageRecord, IndexedMediaEntry};
|
||||||
use crate::media::{probe_video_metadata, MediaTools};
|
use crate::media::{probe_video_metadata, MediaTools};
|
||||||
|
use crate::storage::{detect_storage_profile, RuntimeAdaptiveProfile, StorageProfile};
|
||||||
use crate::thumbnail;
|
use crate::thumbnail;
|
||||||
use crate::vector;
|
use crate::vector;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
@@ -18,13 +19,13 @@ const IMAGE_EXTENSIONS: &[&str] = &[
|
|||||||
|
|
||||||
const VIDEO_EXTENSIONS: &[&str] = &["mp4", "mov", "m4v", "webm"];
|
const VIDEO_EXTENSIONS: &[&str] = &["mp4", "mov", "m4v", "webm"];
|
||||||
|
|
||||||
const INDEX_BATCH_SIZE: usize = 250;
|
|
||||||
const WORKER_BATCH_SIZE: usize = 12;
|
|
||||||
const WORKER_FETCH_SIZE: usize = 64;
|
|
||||||
const JOB_PROGRESS_EMIT_INTERVAL: Duration = Duration::from_millis(750);
|
const JOB_PROGRESS_EMIT_INTERVAL: Duration = Duration::from_millis(750);
|
||||||
|
|
||||||
static LAST_JOB_PROGRESS_EMIT: OnceLock<Mutex<HashMap<i64, Instant>>> = OnceLock::new();
|
static LAST_JOB_PROGRESS_EMIT: OnceLock<Mutex<HashMap<i64, Instant>>> = OnceLock::new();
|
||||||
static ACTIVE_INDEXING_FOLDERS: OnceLock<Mutex<HashSet<i64>>> = OnceLock::new();
|
static ACTIVE_INDEXING_FOLDERS: OnceLock<Mutex<HashSet<i64>>> = OnceLock::new();
|
||||||
|
static FOLDER_STORAGE_PROFILES: OnceLock<Mutex<HashMap<i64, RuntimeAdaptiveProfile>>> =
|
||||||
|
OnceLock::new();
|
||||||
|
static DB_WRITE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||||
|
|
||||||
#[derive(Clone, Serialize)]
|
#[derive(Clone, Serialize)]
|
||||||
pub struct IndexProgress {
|
pub struct IndexProgress {
|
||||||
@@ -53,6 +54,8 @@ pub struct MediaJobProgressEvent {
|
|||||||
|
|
||||||
pub fn index_folder(app: AppHandle, pool: DbPool, folder_id: i64, folder_path: PathBuf) {
|
pub fn index_folder(app: AppHandle, pool: DbPool, folder_id: i64, folder_path: PathBuf) {
|
||||||
std::thread::spawn(move || {
|
std::thread::spawn(move || {
|
||||||
|
let storage_profile = detect_storage_profile(&folder_path);
|
||||||
|
set_folder_storage_profile(folder_id, RuntimeAdaptiveProfile::new(storage_profile));
|
||||||
set_folder_indexing_state(folder_id, true);
|
set_folder_indexing_state(folder_id, true);
|
||||||
if let Err(error) = do_index(app, pool, folder_id, folder_path) {
|
if let Err(error) = do_index(app, pool, folder_id, folder_path) {
|
||||||
eprintln!("Indexing error: {}", error);
|
eprintln!("Indexing error: {}", error);
|
||||||
@@ -119,7 +122,12 @@ fn do_index(app: AppHandle, pool: DbPool, folder_id: i64, folder_path: PathBuf)
|
|||||||
let mut seen_paths = HashSet::with_capacity(total);
|
let mut seen_paths = HashSet::with_capacity(total);
|
||||||
let mut processed = 0usize;
|
let mut processed = 0usize;
|
||||||
|
|
||||||
for path_chunk in media_paths.chunks(INDEX_BATCH_SIZE) {
|
let mut offset = 0usize;
|
||||||
|
while offset < media_paths.len() {
|
||||||
|
let storage_profile = folder_storage_profile(folder_id);
|
||||||
|
let end = (offset + storage_profile.index_batch_size()).min(media_paths.len());
|
||||||
|
let path_chunk = &media_paths[offset..end];
|
||||||
|
let batch_start = Instant::now();
|
||||||
let records: Vec<ImageRecord> = path_chunk
|
let records: Vec<ImageRecord> = path_chunk
|
||||||
.par_iter()
|
.par_iter()
|
||||||
.filter_map(|path| {
|
.filter_map(|path| {
|
||||||
@@ -146,6 +154,7 @@ fn do_index(app: AppHandle, pool: DbPool, folder_id: i64, folder_path: PathBuf)
|
|||||||
}
|
}
|
||||||
|
|
||||||
processed += path_chunk.len();
|
processed += path_chunk.len();
|
||||||
|
observe_folder_scan_batch(folder_id, path_chunk.len(), batch_start.elapsed());
|
||||||
let current_file = path_chunk
|
let current_file = path_chunk
|
||||||
.last()
|
.last()
|
||||||
.and_then(|path| path.file_name())
|
.and_then(|path| path.file_name())
|
||||||
@@ -162,6 +171,7 @@ fn do_index(app: AppHandle, pool: DbPool, folder_id: i64, folder_path: PathBuf)
|
|||||||
done: false,
|
done: false,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
offset = end;
|
||||||
}
|
}
|
||||||
|
|
||||||
let missing_ids = existing_by_path
|
let missing_ids = existing_by_path
|
||||||
@@ -275,24 +285,24 @@ fn process_thumbnail_batch(
|
|||||||
cache_dir: &Path,
|
cache_dir: &Path,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let jobs = {
|
let jobs = {
|
||||||
let conn = pool.get()?;
|
with_db_write_lock(|| {
|
||||||
|
let mut conn = pool.get()?;
|
||||||
let active_folders = active_indexing_folders();
|
let active_folders = active_indexing_folders();
|
||||||
db::get_pending_thumbnail_jobs(&conn, WORKER_FETCH_SIZE)?
|
let worker_batch_size = max_worker_batch_size(&active_folders);
|
||||||
.into_iter()
|
let worker_fetch_size = max_worker_fetch_size(&active_folders);
|
||||||
.filter(|job| !active_folders.contains(&job.folder_id))
|
db::claim_thumbnail_jobs(
|
||||||
.take(WORKER_BATCH_SIZE)
|
&mut conn,
|
||||||
.collect::<Vec<_>>()
|
&active_folders,
|
||||||
|
worker_fetch_size,
|
||||||
|
worker_batch_size,
|
||||||
|
)
|
||||||
|
})?
|
||||||
};
|
};
|
||||||
|
|
||||||
if jobs.is_empty() {
|
if jobs.is_empty() {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
for job in &jobs {
|
|
||||||
let conn = pool.get()?;
|
|
||||||
db::mark_thumbnail_job_processing(&conn, job.image_id)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
let (image_jobs, video_jobs): (Vec<_>, Vec<_>) =
|
let (image_jobs, video_jobs): (Vec<_>, Vec<_>) =
|
||||||
jobs.into_iter().partition(|job| job.media_kind == "image");
|
jobs.into_iter().partition(|job| job.media_kind == "image");
|
||||||
|
|
||||||
@@ -323,13 +333,17 @@ fn process_thumbnail_batch(
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let updated_images = {
|
||||||
|
with_db_write_lock(|| {
|
||||||
|
let mut conn = pool.get()?;
|
||||||
|
let tx = conn.transaction()?;
|
||||||
let mut updated_images = Vec::new();
|
let mut updated_images = Vec::new();
|
||||||
|
|
||||||
for (image_id, thumbnail_result) in results {
|
for (image_id, thumbnail_result) in results {
|
||||||
let generated = match thumbnail_result {
|
let generated = match thumbnail_result {
|
||||||
Ok(path) => path,
|
Ok(path) => path,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
let conn = pool.get()?;
|
db::mark_thumbnail_failed(&tx, image_id, &error.to_string())?;
|
||||||
db::mark_thumbnail_failed(&conn, image_id, &error.to_string())?;
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -340,13 +354,20 @@ fn process_thumbnail_batch(
|
|||||||
let width = generated.as_ref().and_then(|thumb| thumb.width);
|
let width = generated.as_ref().and_then(|thumb| thumb.width);
|
||||||
let height = generated.as_ref().and_then(|thumb| thumb.height);
|
let height = generated.as_ref().and_then(|thumb| thumb.height);
|
||||||
|
|
||||||
let updated_image = {
|
updated_images.push(db::mark_thumbnail_ready(
|
||||||
let conn = pool.get()?;
|
&tx,
|
||||||
db::mark_thumbnail_ready(&conn, image_id, thumbnail_path.as_deref(), width, height)?
|
image_id,
|
||||||
};
|
thumbnail_path.as_deref(),
|
||||||
updated_images.push(updated_image);
|
width,
|
||||||
|
height,
|
||||||
|
)?);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(updated_images)
|
||||||
|
})?
|
||||||
|
};
|
||||||
|
|
||||||
if !updated_images.is_empty() {
|
if !updated_images.is_empty() {
|
||||||
let folder_ids = updated_images
|
let folder_ids = updated_images
|
||||||
.iter()
|
.iter()
|
||||||
@@ -366,51 +387,65 @@ fn process_thumbnail_batch(
|
|||||||
|
|
||||||
fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaTools) -> Result<()> {
|
fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaTools) -> Result<()> {
|
||||||
let jobs = {
|
let jobs = {
|
||||||
let conn = pool.get()?;
|
with_db_write_lock(|| {
|
||||||
|
let mut conn = pool.get()?;
|
||||||
let active_folders = active_indexing_folders();
|
let active_folders = active_indexing_folders();
|
||||||
db::get_pending_metadata_jobs(&conn, WORKER_FETCH_SIZE)?
|
let worker_batch_size = max_worker_batch_size(&active_folders);
|
||||||
.into_iter()
|
let worker_fetch_size = max_worker_fetch_size(&active_folders);
|
||||||
.filter(|job| !active_folders.contains(&job.folder_id))
|
db::claim_metadata_jobs(
|
||||||
.take(WORKER_BATCH_SIZE)
|
&mut conn,
|
||||||
.collect::<Vec<_>>()
|
&active_folders,
|
||||||
|
worker_fetch_size,
|
||||||
|
worker_batch_size,
|
||||||
|
)
|
||||||
|
})?
|
||||||
};
|
};
|
||||||
|
|
||||||
if jobs.is_empty() {
|
if jobs.is_empty() {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
for job in &jobs {
|
let results = jobs
|
||||||
let conn = pool.get()?;
|
.into_iter()
|
||||||
db::mark_metadata_job_processing(&conn, job.image_id)?;
|
.map(|job| {
|
||||||
}
|
(
|
||||||
|
job.image_id,
|
||||||
|
probe_video_metadata(media_tools, Path::new(&job.path)),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
let updated_images = {
|
||||||
|
with_db_write_lock(|| {
|
||||||
|
let mut conn = pool.get()?;
|
||||||
|
let tx = conn.transaction()?;
|
||||||
let mut updated_images = Vec::new();
|
let mut updated_images = Vec::new();
|
||||||
|
|
||||||
for job in jobs {
|
for (image_id, metadata_result) in results {
|
||||||
let metadata = match probe_video_metadata(media_tools, Path::new(&job.path)) {
|
let metadata = match metadata_result {
|
||||||
Ok(metadata) => metadata,
|
Ok(metadata) => metadata,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
let conn = pool.get()?;
|
db::mark_metadata_failed(&tx, image_id, &error.to_string())?;
|
||||||
db::mark_metadata_failed(&conn, job.image_id, &error.to_string())?;
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let updated_image = {
|
updated_images.push(db::mark_metadata_ready(
|
||||||
let conn = pool.get()?;
|
&tx,
|
||||||
db::mark_metadata_ready(
|
image_id,
|
||||||
&conn,
|
|
||||||
job.image_id,
|
|
||||||
metadata.duration_ms,
|
metadata.duration_ms,
|
||||||
metadata.width,
|
metadata.width,
|
||||||
metadata.height,
|
metadata.height,
|
||||||
metadata.video_codec.as_deref(),
|
metadata.video_codec.as_deref(),
|
||||||
metadata.audio_codec.as_deref(),
|
metadata.audio_codec.as_deref(),
|
||||||
)?
|
)?);
|
||||||
};
|
|
||||||
updated_images.push(updated_image);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(updated_images)
|
||||||
|
})?
|
||||||
|
};
|
||||||
|
|
||||||
if !updated_images.is_empty() {
|
if !updated_images.is_empty() {
|
||||||
let folder_ids = updated_images
|
let folder_ids = updated_images
|
||||||
.iter()
|
.iter()
|
||||||
@@ -449,6 +484,58 @@ fn set_folder_indexing_state(folder_id: i64, is_active: bool) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn set_folder_storage_profile(folder_id: i64, profile: RuntimeAdaptiveProfile) {
|
||||||
|
if let Ok(mut profiles) = FOLDER_STORAGE_PROFILES
|
||||||
|
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||||
|
.lock()
|
||||||
|
{
|
||||||
|
profiles.insert(folder_id, profile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn folder_storage_profile(folder_id: i64) -> StorageProfile {
|
||||||
|
FOLDER_STORAGE_PROFILES
|
||||||
|
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||||
|
.lock()
|
||||||
|
.ok()
|
||||||
|
.and_then(|profiles| profiles.get(&folder_id).copied())
|
||||||
|
.map(|profile| profile.profile())
|
||||||
|
.unwrap_or(StorageProfile::Balanced)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn observe_folder_scan_batch(folder_id: i64, item_count: usize, elapsed: Duration) {
|
||||||
|
if let Ok(mut profiles) = FOLDER_STORAGE_PROFILES
|
||||||
|
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||||
|
.lock()
|
||||||
|
{
|
||||||
|
if let Some(profile) = profiles.get_mut(&folder_id) {
|
||||||
|
profile.observe_scan_batch(item_count, elapsed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn max_worker_batch_size(active_folders: &HashSet<i64>) -> usize {
|
||||||
|
active_folders
|
||||||
|
.iter()
|
||||||
|
.map(|folder_id| folder_storage_profile(*folder_id).worker_batch_size())
|
||||||
|
.min()
|
||||||
|
.unwrap_or(StorageProfile::Balanced.worker_batch_size())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn max_worker_fetch_size(active_folders: &HashSet<i64>) -> usize {
|
||||||
|
active_folders
|
||||||
|
.iter()
|
||||||
|
.map(|folder_id| folder_storage_profile(*folder_id).worker_fetch_size())
|
||||||
|
.min()
|
||||||
|
.unwrap_or(StorageProfile::Balanced.worker_fetch_size())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_db_write_lock<T>(operation: impl FnOnce() -> Result<T>) -> Result<T> {
|
||||||
|
let lock = DB_WRITE_LOCK.get_or_init(|| Mutex::new(()));
|
||||||
|
let _guard = lock.lock().unwrap();
|
||||||
|
operation()
|
||||||
|
}
|
||||||
|
|
||||||
fn emit_progress(app: &AppHandle, progress: &IndexProgress) {
|
fn emit_progress(app: &AppHandle, progress: &IndexProgress) {
|
||||||
let _ = app.emit("index-progress", progress);
|
let _ = app.emit("index-progress", progress);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,10 +2,12 @@ mod commands;
|
|||||||
mod db;
|
mod db;
|
||||||
mod indexer;
|
mod indexer;
|
||||||
mod media;
|
mod media;
|
||||||
|
mod storage;
|
||||||
mod thumbnail;
|
mod thumbnail;
|
||||||
mod vector;
|
mod vector;
|
||||||
|
|
||||||
use tauri::Manager;
|
use tauri::Manager;
|
||||||
|
use crate::storage::StorageProfile;
|
||||||
|
|
||||||
#[cfg_attr(mobile, tauri::mobile_entry_point)]
|
#[cfg_attr(mobile, tauri::mobile_entry_point)]
|
||||||
pub fn run() {
|
pub fn run() {
|
||||||
@@ -35,12 +37,18 @@ pub fn run() {
|
|||||||
let thumb_dir = app_dir.join("thumbnails");
|
let thumb_dir = app_dir.join("thumbnails");
|
||||||
std::fs::create_dir_all(&thumb_dir).expect("Failed to create thumbnail dir");
|
std::fs::create_dir_all(&thumb_dir).expect("Failed to create thumbnail dir");
|
||||||
|
|
||||||
|
let thumbnail_worker_count = std::thread::available_parallelism()
|
||||||
|
.map(|parallelism| StorageProfile::Balanced.thumbnail_workers(parallelism.get()))
|
||||||
|
.unwrap_or(2);
|
||||||
|
|
||||||
|
for _ in 0..thumbnail_worker_count {
|
||||||
indexer::start_thumbnail_worker(
|
indexer::start_thumbnail_worker(
|
||||||
app.handle().clone(),
|
app.handle().clone(),
|
||||||
pool.clone(),
|
pool.clone(),
|
||||||
media_tools.clone(),
|
media_tools.clone(),
|
||||||
thumb_dir,
|
thumb_dir.clone(),
|
||||||
);
|
);
|
||||||
|
}
|
||||||
indexer::start_metadata_worker(app.handle().clone(), pool.clone(), media_tools.clone());
|
indexer::start_metadata_worker(app.handle().clone(), pool.clone(), media_tools.clone());
|
||||||
|
|
||||||
app.manage(pool);
|
app.manage(pool);
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use sysinfo::{DiskKind, Disks};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum StorageProfile {
|
||||||
|
Fast,
|
||||||
|
Balanced,
|
||||||
|
Conservative,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub struct RuntimeAdaptiveProfile {
|
||||||
|
profile: StorageProfile,
|
||||||
|
ema_ms_per_item: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StorageProfile {
|
||||||
|
pub fn index_batch_size(self) -> usize {
|
||||||
|
match self {
|
||||||
|
StorageProfile::Fast => 300,
|
||||||
|
StorageProfile::Balanced => 180,
|
||||||
|
StorageProfile::Conservative => 90,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn worker_batch_size(self) -> usize {
|
||||||
|
match self {
|
||||||
|
StorageProfile::Fast => 16,
|
||||||
|
StorageProfile::Balanced => 10,
|
||||||
|
StorageProfile::Conservative => 6,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn worker_fetch_size(self) -> usize {
|
||||||
|
match self {
|
||||||
|
StorageProfile::Fast => 96,
|
||||||
|
StorageProfile::Balanced => 48,
|
||||||
|
StorageProfile::Conservative => 24,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn thumbnail_workers(self, available_parallelism: usize) -> usize {
|
||||||
|
match self {
|
||||||
|
StorageProfile::Fast => (available_parallelism / 3).clamp(2, 4),
|
||||||
|
StorageProfile::Balanced => (available_parallelism / 4).clamp(2, 3),
|
||||||
|
StorageProfile::Conservative => 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RuntimeAdaptiveProfile {
|
||||||
|
pub fn new(initial_profile: StorageProfile) -> Self {
|
||||||
|
Self {
|
||||||
|
profile: initial_profile,
|
||||||
|
ema_ms_per_item: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn profile(self) -> StorageProfile {
|
||||||
|
self.profile
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn observe_scan_batch(&mut self, item_count: usize, elapsed: std::time::Duration) {
|
||||||
|
if item_count == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let ms_per_item = elapsed.as_secs_f64() * 1000.0 / item_count as f64;
|
||||||
|
self.ema_ms_per_item = Some(match self.ema_ms_per_item {
|
||||||
|
Some(existing) => (existing * 0.7) + (ms_per_item * 0.3),
|
||||||
|
None => ms_per_item,
|
||||||
|
});
|
||||||
|
|
||||||
|
let ema = self.ema_ms_per_item.unwrap_or(ms_per_item);
|
||||||
|
self.profile = if ema >= 8.0 {
|
||||||
|
StorageProfile::Conservative
|
||||||
|
} else if ema >= 3.0 {
|
||||||
|
StorageProfile::Balanced
|
||||||
|
} else {
|
||||||
|
StorageProfile::Fast
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn detect_storage_profile(path: &Path) -> StorageProfile {
|
||||||
|
let canonical = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());
|
||||||
|
let disks = Disks::new_with_refreshed_list();
|
||||||
|
|
||||||
|
let best_match = disks
|
||||||
|
.list()
|
||||||
|
.iter()
|
||||||
|
.filter(|disk| canonical.starts_with(disk.mount_point()))
|
||||||
|
.max_by_key(|disk| disk.mount_point().as_os_str().len());
|
||||||
|
|
||||||
|
let Some(disk) = best_match else {
|
||||||
|
return fallback_profile_for_path(&canonical);
|
||||||
|
};
|
||||||
|
|
||||||
|
if disk.is_removable() {
|
||||||
|
return StorageProfile::Conservative;
|
||||||
|
}
|
||||||
|
|
||||||
|
match disk.kind() {
|
||||||
|
DiskKind::SSD => StorageProfile::Fast,
|
||||||
|
DiskKind::HDD => StorageProfile::Conservative,
|
||||||
|
DiskKind::Unknown(_) => fallback_profile_for_path(&canonical),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fallback_profile_for_path(path: &PathBuf) -> StorageProfile {
|
||||||
|
let path_str = path.to_string_lossy().to_lowercase();
|
||||||
|
if path_str.starts_with("\\\\") {
|
||||||
|
return StorageProfile::Conservative;
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg!(target_os = "windows") {
|
||||||
|
let drive = path_str.chars().next();
|
||||||
|
if drive.is_some_and(|letter| !matches!(letter, 'c' | 'd')) {
|
||||||
|
return StorageProfile::Balanced;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
StorageProfile::Balanced
|
||||||
|
}
|
||||||
@@ -2,8 +2,8 @@ use crate::media::MediaTools;
|
|||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
use fast_image_resize as fir;
|
use fast_image_resize as fir;
|
||||||
use fast_image_resize::IntoImageView;
|
use fast_image_resize::IntoImageView;
|
||||||
|
use image::codecs::jpeg::JpegEncoder;
|
||||||
use image::ImageDecoder;
|
use image::ImageDecoder;
|
||||||
use image::ImageFormat;
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
pub const THUMB_SIZE: u32 = 320;
|
pub const THUMB_SIZE: u32 = 320;
|
||||||
@@ -53,7 +53,10 @@ pub fn generate_image_thumbnail(image_path: &Path, cache_dir: &Path) -> Result<G
|
|||||||
std::fs::create_dir_all(parent)?;
|
std::fs::create_dir_all(parent)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
image::DynamicImage::ImageRgba8(thumb).save_with_format(&out_path, ImageFormat::WebP)?;
|
let thumb = image::DynamicImage::ImageRgba8(thumb).into_rgb8();
|
||||||
|
let mut output_file = std::fs::File::create(&out_path)?;
|
||||||
|
let mut encoder = JpegEncoder::new_with_quality(&mut output_file, 82);
|
||||||
|
encoder.encode_image(&thumb)?;
|
||||||
Ok(GeneratedThumbnail {
|
Ok(GeneratedThumbnail {
|
||||||
path: out_path,
|
path: out_path,
|
||||||
width: original_dimensions.0,
|
width: original_dimensions.0,
|
||||||
@@ -146,7 +149,7 @@ pub fn generate_video_thumbnail(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn thumb_path(cache_dir: &Path, image_path: &str) -> PathBuf {
|
pub fn thumb_path(cache_dir: &Path, image_path: &str) -> PathBuf {
|
||||||
thumb_path_with_ext(cache_dir, image_path, "webp")
|
thumb_path_with_ext(cache_dir, image_path, "jpg")
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn video_poster_path(cache_dir: &Path, video_path: &str) -> PathBuf {
|
pub fn video_poster_path(cache_dir: &Path, video_path: &str) -> PathBuf {
|
||||||
|
|||||||
Reference in New Issue
Block a user