perf(metadata): idle-only backoff and per-item commits
The metadata worker now sleeps its 250ms backoff only when the queue is empty or a batch errored, claiming the next batch immediately while work is pending. Each ffprobe result is committed and emitted individually (in its own transaction) instead of after the whole batch, so video metadata progress moves steadily rather than stalling for up to 16 sequential probes.
This commit is contained in:
+37
-41
@@ -219,10 +219,16 @@ pub fn start_thumbnail_worker(
|
|||||||
|
|
||||||
pub fn start_metadata_worker(app: AppHandle, pool: DbPool, media_tools: MediaTools) {
|
pub fn start_metadata_worker(app: AppHandle, pool: DbPool, media_tools: MediaTools) {
|
||||||
std::thread::spawn(move || loop {
|
std::thread::spawn(move || loop {
|
||||||
if let Err(error) = process_metadata_batch(&app, &pool, &media_tools) {
|
// Only back off when the queue is empty (or errored); while jobs are
|
||||||
|
// pending, claim the next batch immediately.
|
||||||
|
match process_metadata_batch(&app, &pool, &media_tools) {
|
||||||
|
Ok(true) => {}
|
||||||
|
Ok(false) => std::thread::sleep(std::time::Duration::from_millis(250)),
|
||||||
|
Err(error) => {
|
||||||
eprintln!("Metadata worker error: {}", error);
|
eprintln!("Metadata worker error: {}", error);
|
||||||
}
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(250));
|
std::thread::sleep(std::time::Duration::from_millis(250));
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -689,7 +695,13 @@ fn persist_thumbnail_results(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaTools) -> Result<()> {
|
/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
|
||||||
|
/// the queue was empty.
|
||||||
|
fn process_metadata_batch(
|
||||||
|
app: &AppHandle,
|
||||||
|
pool: &DbPool,
|
||||||
|
media_tools: &MediaTools,
|
||||||
|
) -> Result<bool> {
|
||||||
let jobs = {
|
let jobs = {
|
||||||
with_db_write_lock(|| {
|
with_db_write_lock(|| {
|
||||||
let mut conn = pool.get()?;
|
let mut conn = pool.get()?;
|
||||||
@@ -708,65 +720,49 @@ fn process_metadata_batch(app: &AppHandle, pool: &DbPool, media_tools: &MediaToo
|
|||||||
};
|
};
|
||||||
|
|
||||||
if jobs.is_empty() {
|
if jobs.is_empty() {
|
||||||
return Ok(());
|
return Ok(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
let results = jobs
|
// Probes run sequentially (each ffprobe blocks on its process); results
|
||||||
.into_iter()
|
// are committed per item so progress keeps moving through slow stretches.
|
||||||
.map(|job| {
|
for job in jobs {
|
||||||
(
|
let metadata_result = probe_video_metadata(media_tools, Path::new(&job.path));
|
||||||
job.image_id,
|
|
||||||
probe_video_metadata(media_tools, Path::new(&job.path)),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
let updated_images = {
|
let updated_image = with_db_write_lock(|| {
|
||||||
with_db_write_lock(|| {
|
|
||||||
let mut conn = pool.get()?;
|
let mut conn = pool.get()?;
|
||||||
let tx = conn.transaction()?;
|
let tx = conn.transaction()?;
|
||||||
let mut updated_images = Vec::new();
|
let updated = match metadata_result {
|
||||||
|
Ok(metadata) => Some(db::mark_metadata_ready(
|
||||||
for (image_id, metadata_result) in results {
|
|
||||||
let metadata = match metadata_result {
|
|
||||||
Ok(metadata) => metadata,
|
|
||||||
Err(error) => {
|
|
||||||
db::mark_metadata_failed(&tx, image_id, &error.to_string())?;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
updated_images.push(db::mark_metadata_ready(
|
|
||||||
&tx,
|
&tx,
|
||||||
image_id,
|
job.image_id,
|
||||||
metadata.duration_ms,
|
metadata.duration_ms,
|
||||||
metadata.width,
|
metadata.width,
|
||||||
metadata.height,
|
metadata.height,
|
||||||
metadata.video_codec.as_deref(),
|
metadata.video_codec.as_deref(),
|
||||||
metadata.audio_codec.as_deref(),
|
metadata.audio_codec.as_deref(),
|
||||||
)?);
|
)?),
|
||||||
|
Err(error) => {
|
||||||
|
db::mark_metadata_failed(&tx, job.image_id, &error.to_string())?;
|
||||||
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
tx.commit()?;
|
|
||||||
Ok(updated_images)
|
|
||||||
})?
|
|
||||||
};
|
};
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(updated)
|
||||||
|
})?;
|
||||||
|
|
||||||
if !updated_images.is_empty() {
|
if let Some(image) = updated_image {
|
||||||
let folder_ids = updated_images
|
let folder_id = image.folder_id;
|
||||||
.iter()
|
|
||||||
.map(|image| image.folder_id)
|
|
||||||
.collect::<HashSet<_>>();
|
|
||||||
emit_media_updates(
|
emit_media_updates(
|
||||||
app,
|
app,
|
||||||
&MediaUpdateBatch {
|
&MediaUpdateBatch {
|
||||||
images: updated_images,
|
images: vec![image],
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
emit_folder_job_progress(app, pool, &folder_ids.into_iter().collect::<Vec<_>>(), true);
|
emit_folder_job_progress(app, pool, &[folder_id], false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
|
/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
|
||||||
|
|||||||
Reference in New Issue
Block a user