Optimize thumbnail workers and adaptive indexing
This commit is contained in:
Generated
+97
-12
@@ -2450,6 +2450,15 @@ version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
|
||||
|
||||
[[package]]
|
||||
name = "ntapi"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.1"
|
||||
@@ -2562,6 +2571,16 @@ dependencies = [
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-io-kit"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-io-surface"
|
||||
version = "0.3.2"
|
||||
@@ -2915,6 +2934,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sqlite-vec",
|
||||
"sysinfo",
|
||||
"tauri",
|
||||
"tauri-build",
|
||||
"tauri-plugin-dialog",
|
||||
@@ -4070,6 +4090,20 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.38.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
"ntapi",
|
||||
"objc2-core-foundation",
|
||||
"objc2-io-kit",
|
||||
"windows 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system-deps"
|
||||
version = "6.2.2"
|
||||
@@ -4115,7 +4149,7 @@ dependencies = [
|
||||
"tao-macros",
|
||||
"unicode-segmentation",
|
||||
"url",
|
||||
"windows",
|
||||
"windows 0.61.3",
|
||||
"windows-core 0.61.2",
|
||||
"windows-version",
|
||||
"x11-dl",
|
||||
@@ -4198,7 +4232,7 @@ dependencies = [
|
||||
"webkit2gtk",
|
||||
"webview2-com",
|
||||
"window-vibrancy",
|
||||
"windows",
|
||||
"windows 0.61.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4341,7 +4375,7 @@ dependencies = [
|
||||
"tauri-plugin",
|
||||
"thiserror 2.0.18",
|
||||
"url",
|
||||
"windows",
|
||||
"windows 0.61.3",
|
||||
"zbus",
|
||||
]
|
||||
|
||||
@@ -4367,7 +4401,7 @@ dependencies = [
|
||||
"url",
|
||||
"webkit2gtk",
|
||||
"webview2-com",
|
||||
"windows",
|
||||
"windows 0.61.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4392,7 +4426,7 @@ dependencies = [
|
||||
"url",
|
||||
"webkit2gtk",
|
||||
"webview2-com",
|
||||
"windows",
|
||||
"windows 0.61.3",
|
||||
"wry",
|
||||
]
|
||||
|
||||
@@ -5269,7 +5303,7 @@ checksum = "7130243a7a5b33c54a444e54842e6a9e133de08b5ad7b5861cd8ed9a6a5bc96a"
|
||||
dependencies = [
|
||||
"webview2-com-macros",
|
||||
"webview2-com-sys",
|
||||
"windows",
|
||||
"windows 0.61.3",
|
||||
"windows-core 0.61.2",
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
@@ -5293,7 +5327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "381336cfffd772377d291702245447a5251a2ffa5bad679c99e61bc48bacbf9c"
|
||||
dependencies = [
|
||||
"thiserror 2.0.18",
|
||||
"windows",
|
||||
"windows 0.61.3",
|
||||
"windows-core 0.61.2",
|
||||
]
|
||||
|
||||
@@ -5355,11 +5389,23 @@ version = "0.61.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
|
||||
dependencies = [
|
||||
"windows-collections",
|
||||
"windows-collections 0.2.0",
|
||||
"windows-core 0.61.2",
|
||||
"windows-future",
|
||||
"windows-future 0.2.1",
|
||||
"windows-link 0.1.3",
|
||||
"windows-numerics",
|
||||
"windows-numerics 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580"
|
||||
dependencies = [
|
||||
"windows-collections 0.3.2",
|
||||
"windows-core 0.62.2",
|
||||
"windows-future 0.3.2",
|
||||
"windows-numerics 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5371,6 +5417,15 @@ dependencies = [
|
||||
"windows-core 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-collections"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.61.2"
|
||||
@@ -5405,7 +5460,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
|
||||
dependencies = [
|
||||
"windows-core 0.61.2",
|
||||
"windows-link 0.1.3",
|
||||
"windows-threading",
|
||||
"windows-threading 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-future"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
"windows-link 0.2.1",
|
||||
"windows-threading 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5452,6 +5518,16 @@ dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-numerics"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.3.4"
|
||||
@@ -5590,6 +5666,15 @@ dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-threading"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-version"
|
||||
version = "0.1.7"
|
||||
@@ -5906,7 +5991,7 @@ dependencies = [
|
||||
"webkit2gtk",
|
||||
"webkit2gtk-sys",
|
||||
"webview2-com",
|
||||
"windows",
|
||||
"windows 0.61.3",
|
||||
"windows-core 0.61.2",
|
||||
"windows-version",
|
||||
"x11-dl",
|
||||
|
||||
@@ -34,3 +34,4 @@ anyhow = "1"
|
||||
log = "0.4"
|
||||
ffmpeg-sidecar = "2.5.0"
|
||||
xxhash-rust = { version = "0.8", features = ["xxh3"] }
|
||||
sysinfo = "0.38.4"
|
||||
|
||||
+66
-16
@@ -453,6 +453,41 @@ pub fn get_pending_thumbnail_jobs(conn: &Connection, limit: usize) -> Result<Vec
|
||||
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
|
||||
}
|
||||
|
||||
pub fn claim_thumbnail_jobs(
|
||||
conn: &mut Connection,
|
||||
active_folder_ids: &std::collections::HashSet<i64>,
|
||||
fetch_limit: usize,
|
||||
claim_limit: usize,
|
||||
) -> Result<Vec<ThumbnailJob>> {
|
||||
let tx = conn.transaction()?;
|
||||
let candidates = get_pending_thumbnail_jobs(&tx, fetch_limit)?;
|
||||
let mut claimed = Vec::with_capacity(claim_limit);
|
||||
|
||||
for job in candidates {
|
||||
if active_folder_ids.contains(&job.folder_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let updated = tx.execute(
|
||||
"UPDATE thumbnail_jobs
|
||||
SET status = 'processing', attempts = attempts + 1, updated_at = datetime('now')
|
||||
WHERE image_id = ?1 AND status = 'pending'",
|
||||
[job.image_id],
|
||||
)?;
|
||||
|
||||
if updated == 1 {
|
||||
claimed.push(job);
|
||||
}
|
||||
|
||||
if claimed.len() >= claim_limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
tx.commit()?;
|
||||
Ok(claimed)
|
||||
}
|
||||
|
||||
pub fn get_pending_metadata_jobs(conn: &Connection, limit: usize) -> Result<Vec<MetadataJob>> {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT j.image_id, i.folder_id, i.path
|
||||
@@ -472,14 +507,39 @@ pub fn get_pending_metadata_jobs(conn: &Connection, limit: usize) -> Result<Vec<
|
||||
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
|
||||
}
|
||||
|
||||
pub fn mark_thumbnail_job_processing(conn: &Connection, image_id: i64) -> Result<()> {
|
||||
conn.execute(
|
||||
"UPDATE thumbnail_jobs
|
||||
pub fn claim_metadata_jobs(
|
||||
conn: &mut Connection,
|
||||
active_folder_ids: &std::collections::HashSet<i64>,
|
||||
fetch_limit: usize,
|
||||
claim_limit: usize,
|
||||
) -> Result<Vec<MetadataJob>> {
|
||||
let tx = conn.transaction()?;
|
||||
let candidates = get_pending_metadata_jobs(&tx, fetch_limit)?;
|
||||
let mut claimed = Vec::with_capacity(claim_limit);
|
||||
|
||||
for job in candidates {
|
||||
if active_folder_ids.contains(&job.folder_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let updated = tx.execute(
|
||||
"UPDATE metadata_jobs
|
||||
SET status = 'processing', attempts = attempts + 1, updated_at = datetime('now')
|
||||
WHERE image_id = ?1",
|
||||
[image_id],
|
||||
WHERE image_id = ?1 AND status = 'pending'",
|
||||
[job.image_id],
|
||||
)?;
|
||||
Ok(())
|
||||
|
||||
if updated == 1 {
|
||||
claimed.push(job);
|
||||
}
|
||||
|
||||
if claimed.len() >= claim_limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
tx.commit()?;
|
||||
Ok(claimed)
|
||||
}
|
||||
|
||||
pub fn mark_thumbnail_ready(
|
||||
@@ -511,16 +571,6 @@ pub fn mark_thumbnail_failed(conn: &Connection, image_id: i64, error: &str) -> R
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn mark_metadata_job_processing(conn: &Connection, image_id: i64) -> Result<()> {
|
||||
conn.execute(
|
||||
"UPDATE metadata_jobs
|
||||
SET status = 'processing', attempts = attempts + 1, updated_at = datetime('now')
|
||||
WHERE image_id = ?1",
|
||||
[image_id],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn mark_metadata_ready(
|
||||
conn: &Connection,
|
||||
image_id: i64,
|
||||
|
||||
+131
-44
@@ -1,5 +1,6 @@
|
||||
use crate::db::{self, DbPool, FolderJobProgress, ImageRecord, IndexedMediaEntry};
|
||||
use crate::media::{probe_video_metadata, MediaTools};
|
||||
use crate::storage::{detect_storage_profile, RuntimeAdaptiveProfile, StorageProfile};
|
||||
use crate::thumbnail;
|
||||
use crate::vector;
|
||||
use anyhow::Result;
|
||||
@@ -18,13 +19,13 @@ const IMAGE_EXTENSIONS: &[&str] = &[
|
||||
|
||||
const VIDEO_EXTENSIONS: &[&str] = &["mp4", "mov", "m4v", "webm"];
|
||||
|
||||
const INDEX_BATCH_SIZE: usize = 250;
|
||||
const WORKER_BATCH_SIZE: usize = 12;
|
||||
const WORKER_FETCH_SIZE: usize = 64;
|
||||
const JOB_PROGRESS_EMIT_INTERVAL: Duration = Duration::from_millis(750);
|
||||
|
||||
static LAST_JOB_PROGRESS_EMIT: OnceLock<Mutex<HashMap<i64, Instant>>> = OnceLock::new();
|
||||
static ACTIVE_INDEXING_FOLDERS: OnceLock<Mutex<HashSet<i64>>> = OnceLock::new();
|
||||
static FOLDER_STORAGE_PROFILES: OnceLock<Mutex<HashMap<i64, RuntimeAdaptiveProfile>>> =
|
||||
OnceLock::new();
|
||||
static DB_WRITE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
pub struct IndexProgress {
|
||||
@@ -53,6 +54,8 @@ pub struct MediaJobProgressEvent {
|
||||
|
||||
pub fn index_folder(app: AppHandle, pool: DbPool, folder_id: i64, folder_path: PathBuf) {
|
||||
std::thread::spawn(move || {
|
||||
let storage_profile = detect_storage_profile(&folder_path);
|
||||
set_folder_storage_profile(folder_id, RuntimeAdaptiveProfile::new(storage_profile));
|
||||
set_folder_indexing_state(folder_id, true);
|
||||
if let Err(error) = do_index(app, pool, folder_id, folder_path) {
|
||||
eprintln!("Indexing error: {}", error);
|
||||
@@ -119,7 +122,12 @@ fn do_index(app: AppHandle, pool: DbPool, folder_id: i64, folder_path: PathBuf)
|
||||
let mut seen_paths = HashSet::with_capacity(total);
|
||||
let mut processed = 0usize;
|
||||
|
||||
for path_chunk in media_paths.chunks(INDEX_BATCH_SIZE) {
|
||||
let mut offset = 0usize;
|
||||
while offset < media_paths.len() {
|
||||
let storage_profile = folder_storage_profile(folder_id);
|
||||
let end = (offset + storage_profile.index_batch_size()).min(media_paths.len());
|
||||
let path_chunk = &media_paths[offset..end];
|
||||
let batch_start = Instant::now();
|
||||
let records: Vec<ImageRecord> = path_chunk
|
||||
.par_iter()
|
||||
.filter_map(|path| {
|
||||
@@ -146,6 +154,7 @@ fn do_index(app: AppHandle, pool: DbPool, folder_id: i64, folder_path: PathBuf)
|
||||
}
|
||||
|
||||
processed += path_chunk.len();
|
||||
observe_folder_scan_batch(folder_id, path_chunk.len(), batch_start.elapsed());
|
||||
let current_file = path_chunk
|
||||
.last()
|
||||
.and_then(|path| path.file_name())
|
||||
@@ -162,6 +171,7 @@ fn do_index(app: AppHandle, pool: DbPool, folder_id: i64, folder_path: PathBuf)
|
||||
done: false,
|
||||
},
|
||||
);
|
||||
offset = end;
|
||||
}
|
||||
|
||||
let missing_ids = existing_by_path
|
||||
@@ -275,24 +285,24 @@ fn process_thumbnail_batch(
|
||||
cache_dir: &Path,
|
||||
) -> Result<()> {
|
||||
let jobs = {
|
||||
let conn = pool.get()?;
|
||||
with_db_write_lock(|| {
|
||||
let mut conn = pool.get()?;
|
||||
let active_folders = active_indexing_folders();
|
||||
db::get_pending_thumbnail_jobs(&conn, WORKER_FETCH_SIZE)?
|
||||
.into_iter()
|
||||
.filter(|job| !active_folders.contains(&job.folder_id))
|
||||
.take(WORKER_BATCH_SIZE)
|
||||
.collect::<Vec<_>>()
|
||||
let worker_batch_size = max_worker_batch_size(&active_folders);
|
||||
let worker_fetch_size = max_worker_fetch_size(&active_folders);
|
||||
db::claim_thumbnail_jobs(
|
||||
&mut conn,
|
||||
&active_folders,
|
||||
worker_fetch_size,
|
||||
worker_batch_size,
|
||||
)
|
||||
})?
|
||||
};
|
||||
|
||||
if jobs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for job in &jobs {
|
||||
let conn = pool.get()?;
|
||||
db::mark_thumbnail_job_processing(&conn, job.image_id)?;
|
||||
}
|
||||
|
||||
let (image_jobs, video_jobs): (Vec<_>, Vec<_>) =
|
||||
jobs.into_iter().partition(|job| job.media_kind == "image");
|
||||
|
||||
@@ -323,13 +333,17 @@ fn process_thumbnail_batch(
|
||||
));
|
||||
}
|
||||
|
||||
let updated_images = {
|
||||
with_db_write_lock(|| {
|
||||
let mut conn = pool.get()?;
|
||||
let tx = conn.transaction()?;
|
||||
let mut updated_images = Vec::new();
|
||||
|
||||
for (image_id, thumbnail_result) in results {
|
||||
let generated = match thumbnail_result {
|
||||
Ok(path) => path,
|
||||
Err(error) => {
|
||||
let conn = pool.get()?;
|
||||
db::mark_thumbnail_failed(&conn, image_id, &error.to_string())?;
|
||||
db::mark_thumbnail_failed(&tx, image_id, &error.to_string())?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -340,13 +354,20 @@ fn process_thumbnail_batch(
|
||||
let width = generated.as_ref().and_then(|thumb| thumb.width);
|
||||
let height = generated.as_ref().and_then(|thumb| thumb.height);
|
||||
|
||||
let updated_image = {
|
||||
let conn = pool.get()?;
|
||||
db::mark_thumbnail_ready(&conn, image_id, thumbnail_path.as_deref(), width, height)?
|
||||
};
|
||||
updated_images.push(updated_image);
|
||||
updated_images.push(db::mark_thumbnail_ready(
|
||||
&tx,
|
||||
image_id,
|
||||
thumbnail_path.as_deref(),
|
||||
width,
|
||||
height,
|
||||
)?);
|
||||
}
|
||||
|
||||
tx.commit()?;
|
||||
Ok(updated_images)
|
||||
})?
|
||||
};
|
||||
|
||||
if !updated_images.is_empty() {
|
||||
let folder_ids = updated_images
|
||||
.iter()
|
||||
@@ -366,51 +387,65 @@ fn process_thumbnail_batch(
|
||||
|
||||
fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaTools) -> Result<()> {
|
||||
let jobs = {
|
||||
let conn = pool.get()?;
|
||||
with_db_write_lock(|| {
|
||||
let mut conn = pool.get()?;
|
||||
let active_folders = active_indexing_folders();
|
||||
db::get_pending_metadata_jobs(&conn, WORKER_FETCH_SIZE)?
|
||||
.into_iter()
|
||||
.filter(|job| !active_folders.contains(&job.folder_id))
|
||||
.take(WORKER_BATCH_SIZE)
|
||||
.collect::<Vec<_>>()
|
||||
let worker_batch_size = max_worker_batch_size(&active_folders);
|
||||
let worker_fetch_size = max_worker_fetch_size(&active_folders);
|
||||
db::claim_metadata_jobs(
|
||||
&mut conn,
|
||||
&active_folders,
|
||||
worker_fetch_size,
|
||||
worker_batch_size,
|
||||
)
|
||||
})?
|
||||
};
|
||||
|
||||
if jobs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for job in &jobs {
|
||||
let conn = pool.get()?;
|
||||
db::mark_metadata_job_processing(&conn, job.image_id)?;
|
||||
}
|
||||
let results = jobs
|
||||
.into_iter()
|
||||
.map(|job| {
|
||||
(
|
||||
job.image_id,
|
||||
probe_video_metadata(media_tools, Path::new(&job.path)),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let updated_images = {
|
||||
with_db_write_lock(|| {
|
||||
let mut conn = pool.get()?;
|
||||
let tx = conn.transaction()?;
|
||||
let mut updated_images = Vec::new();
|
||||
|
||||
for job in jobs {
|
||||
let metadata = match probe_video_metadata(media_tools, Path::new(&job.path)) {
|
||||
for (image_id, metadata_result) in results {
|
||||
let metadata = match metadata_result {
|
||||
Ok(metadata) => metadata,
|
||||
Err(error) => {
|
||||
let conn = pool.get()?;
|
||||
db::mark_metadata_failed(&conn, job.image_id, &error.to_string())?;
|
||||
db::mark_metadata_failed(&tx, image_id, &error.to_string())?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let updated_image = {
|
||||
let conn = pool.get()?;
|
||||
db::mark_metadata_ready(
|
||||
&conn,
|
||||
job.image_id,
|
||||
updated_images.push(db::mark_metadata_ready(
|
||||
&tx,
|
||||
image_id,
|
||||
metadata.duration_ms,
|
||||
metadata.width,
|
||||
metadata.height,
|
||||
metadata.video_codec.as_deref(),
|
||||
metadata.audio_codec.as_deref(),
|
||||
)?
|
||||
};
|
||||
updated_images.push(updated_image);
|
||||
)?);
|
||||
}
|
||||
|
||||
tx.commit()?;
|
||||
Ok(updated_images)
|
||||
})?
|
||||
};
|
||||
|
||||
if !updated_images.is_empty() {
|
||||
let folder_ids = updated_images
|
||||
.iter()
|
||||
@@ -449,6 +484,58 @@ fn set_folder_indexing_state(folder_id: i64, is_active: bool) {
|
||||
}
|
||||
}
|
||||
|
||||
fn set_folder_storage_profile(folder_id: i64, profile: RuntimeAdaptiveProfile) {
|
||||
if let Ok(mut profiles) = FOLDER_STORAGE_PROFILES
|
||||
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
.lock()
|
||||
{
|
||||
profiles.insert(folder_id, profile);
|
||||
}
|
||||
}
|
||||
|
||||
fn folder_storage_profile(folder_id: i64) -> StorageProfile {
|
||||
FOLDER_STORAGE_PROFILES
|
||||
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
.lock()
|
||||
.ok()
|
||||
.and_then(|profiles| profiles.get(&folder_id).copied())
|
||||
.map(|profile| profile.profile())
|
||||
.unwrap_or(StorageProfile::Balanced)
|
||||
}
|
||||
|
||||
fn observe_folder_scan_batch(folder_id: i64, item_count: usize, elapsed: Duration) {
|
||||
if let Ok(mut profiles) = FOLDER_STORAGE_PROFILES
|
||||
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
.lock()
|
||||
{
|
||||
if let Some(profile) = profiles.get_mut(&folder_id) {
|
||||
profile.observe_scan_batch(item_count, elapsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn max_worker_batch_size(active_folders: &HashSet<i64>) -> usize {
|
||||
active_folders
|
||||
.iter()
|
||||
.map(|folder_id| folder_storage_profile(*folder_id).worker_batch_size())
|
||||
.min()
|
||||
.unwrap_or(StorageProfile::Balanced.worker_batch_size())
|
||||
}
|
||||
|
||||
fn max_worker_fetch_size(active_folders: &HashSet<i64>) -> usize {
|
||||
active_folders
|
||||
.iter()
|
||||
.map(|folder_id| folder_storage_profile(*folder_id).worker_fetch_size())
|
||||
.min()
|
||||
.unwrap_or(StorageProfile::Balanced.worker_fetch_size())
|
||||
}
|
||||
|
||||
fn with_db_write_lock<T>(operation: impl FnOnce() -> Result<T>) -> Result<T> {
|
||||
let lock = DB_WRITE_LOCK.get_or_init(|| Mutex::new(()));
|
||||
let _guard = lock.lock().unwrap();
|
||||
operation()
|
||||
}
|
||||
|
||||
fn emit_progress(app: &AppHandle, progress: &IndexProgress) {
|
||||
let _ = app.emit("index-progress", progress);
|
||||
}
|
||||
|
||||
@@ -2,10 +2,12 @@ mod commands;
|
||||
mod db;
|
||||
mod indexer;
|
||||
mod media;
|
||||
mod storage;
|
||||
mod thumbnail;
|
||||
mod vector;
|
||||
|
||||
use tauri::Manager;
|
||||
use crate::storage::StorageProfile;
|
||||
|
||||
#[cfg_attr(mobile, tauri::mobile_entry_point)]
|
||||
pub fn run() {
|
||||
@@ -35,12 +37,18 @@ pub fn run() {
|
||||
let thumb_dir = app_dir.join("thumbnails");
|
||||
std::fs::create_dir_all(&thumb_dir).expect("Failed to create thumbnail dir");
|
||||
|
||||
let thumbnail_worker_count = std::thread::available_parallelism()
|
||||
.map(|parallelism| StorageProfile::Balanced.thumbnail_workers(parallelism.get()))
|
||||
.unwrap_or(2);
|
||||
|
||||
for _ in 0..thumbnail_worker_count {
|
||||
indexer::start_thumbnail_worker(
|
||||
app.handle().clone(),
|
||||
pool.clone(),
|
||||
media_tools.clone(),
|
||||
thumb_dir,
|
||||
thumb_dir.clone(),
|
||||
);
|
||||
}
|
||||
indexer::start_metadata_worker(app.handle().clone(), pool.clone(), media_tools.clone());
|
||||
|
||||
app.manage(pool);
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use sysinfo::{DiskKind, Disks};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum StorageProfile {
|
||||
Fast,
|
||||
Balanced,
|
||||
Conservative,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct RuntimeAdaptiveProfile {
|
||||
profile: StorageProfile,
|
||||
ema_ms_per_item: Option<f64>,
|
||||
}
|
||||
|
||||
impl StorageProfile {
|
||||
pub fn index_batch_size(self) -> usize {
|
||||
match self {
|
||||
StorageProfile::Fast => 300,
|
||||
StorageProfile::Balanced => 180,
|
||||
StorageProfile::Conservative => 90,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn worker_batch_size(self) -> usize {
|
||||
match self {
|
||||
StorageProfile::Fast => 16,
|
||||
StorageProfile::Balanced => 10,
|
||||
StorageProfile::Conservative => 6,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn worker_fetch_size(self) -> usize {
|
||||
match self {
|
||||
StorageProfile::Fast => 96,
|
||||
StorageProfile::Balanced => 48,
|
||||
StorageProfile::Conservative => 24,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn thumbnail_workers(self, available_parallelism: usize) -> usize {
|
||||
match self {
|
||||
StorageProfile::Fast => (available_parallelism / 3).clamp(2, 4),
|
||||
StorageProfile::Balanced => (available_parallelism / 4).clamp(2, 3),
|
||||
StorageProfile::Conservative => 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RuntimeAdaptiveProfile {
|
||||
pub fn new(initial_profile: StorageProfile) -> Self {
|
||||
Self {
|
||||
profile: initial_profile,
|
||||
ema_ms_per_item: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn profile(self) -> StorageProfile {
|
||||
self.profile
|
||||
}
|
||||
|
||||
pub fn observe_scan_batch(&mut self, item_count: usize, elapsed: std::time::Duration) {
|
||||
if item_count == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let ms_per_item = elapsed.as_secs_f64() * 1000.0 / item_count as f64;
|
||||
self.ema_ms_per_item = Some(match self.ema_ms_per_item {
|
||||
Some(existing) => (existing * 0.7) + (ms_per_item * 0.3),
|
||||
None => ms_per_item,
|
||||
});
|
||||
|
||||
let ema = self.ema_ms_per_item.unwrap_or(ms_per_item);
|
||||
self.profile = if ema >= 8.0 {
|
||||
StorageProfile::Conservative
|
||||
} else if ema >= 3.0 {
|
||||
StorageProfile::Balanced
|
||||
} else {
|
||||
StorageProfile::Fast
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
pub fn detect_storage_profile(path: &Path) -> StorageProfile {
|
||||
let canonical = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());
|
||||
let disks = Disks::new_with_refreshed_list();
|
||||
|
||||
let best_match = disks
|
||||
.list()
|
||||
.iter()
|
||||
.filter(|disk| canonical.starts_with(disk.mount_point()))
|
||||
.max_by_key(|disk| disk.mount_point().as_os_str().len());
|
||||
|
||||
let Some(disk) = best_match else {
|
||||
return fallback_profile_for_path(&canonical);
|
||||
};
|
||||
|
||||
if disk.is_removable() {
|
||||
return StorageProfile::Conservative;
|
||||
}
|
||||
|
||||
match disk.kind() {
|
||||
DiskKind::SSD => StorageProfile::Fast,
|
||||
DiskKind::HDD => StorageProfile::Conservative,
|
||||
DiskKind::Unknown(_) => fallback_profile_for_path(&canonical),
|
||||
}
|
||||
}
|
||||
|
||||
fn fallback_profile_for_path(path: &PathBuf) -> StorageProfile {
|
||||
let path_str = path.to_string_lossy().to_lowercase();
|
||||
if path_str.starts_with("\\\\") {
|
||||
return StorageProfile::Conservative;
|
||||
}
|
||||
|
||||
if cfg!(target_os = "windows") {
|
||||
let drive = path_str.chars().next();
|
||||
if drive.is_some_and(|letter| !matches!(letter, 'c' | 'd')) {
|
||||
return StorageProfile::Balanced;
|
||||
}
|
||||
}
|
||||
|
||||
StorageProfile::Balanced
|
||||
}
|
||||
@@ -2,8 +2,8 @@ use crate::media::MediaTools;
|
||||
use anyhow::{anyhow, Result};
|
||||
use fast_image_resize as fir;
|
||||
use fast_image_resize::IntoImageView;
|
||||
use image::codecs::jpeg::JpegEncoder;
|
||||
use image::ImageDecoder;
|
||||
use image::ImageFormat;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub const THUMB_SIZE: u32 = 320;
|
||||
@@ -53,7 +53,10 @@ pub fn generate_image_thumbnail(image_path: &Path, cache_dir: &Path) -> Result<G
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
image::DynamicImage::ImageRgba8(thumb).save_with_format(&out_path, ImageFormat::WebP)?;
|
||||
let thumb = image::DynamicImage::ImageRgba8(thumb).into_rgb8();
|
||||
let mut output_file = std::fs::File::create(&out_path)?;
|
||||
let mut encoder = JpegEncoder::new_with_quality(&mut output_file, 82);
|
||||
encoder.encode_image(&thumb)?;
|
||||
Ok(GeneratedThumbnail {
|
||||
path: out_path,
|
||||
width: original_dimensions.0,
|
||||
@@ -146,7 +149,7 @@ pub fn generate_video_thumbnail(
|
||||
}
|
||||
|
||||
pub fn thumb_path(cache_dir: &Path, image_path: &str) -> PathBuf {
|
||||
thumb_path_with_ext(cache_dir, image_path, "webp")
|
||||
thumb_path_with_ext(cache_dir, image_path, "jpg")
|
||||
}
|
||||
|
||||
pub fn video_poster_path(cache_dir: &Path, video_path: &str) -> PathBuf {
|
||||
|
||||
Reference in New Issue
Block a user