fix: support AVIF thumbnail processing
github/actions/ci GitHub Actions CI finished: success

Route AVIF thumbnail generation through the bundled FFmpeg path instead of the Rust image decoder, avoiding unsupported-format failures without requiring system dav1d dependencies.

Requeue existing AVIF jobs that previously failed with unsupported-format errors and feed generated JPEG derivatives to embedding/tagging preprocessing while leaving lightbox display on the original AVIF file.
This commit is contained in:
2026-06-22 20:43:46 +01:00
parent a06e76c7a7
commit 4f9ab0b821
5 changed files with 183 additions and 17 deletions
+54 -7
View File
@@ -114,6 +114,7 @@ pub struct TaggingJob {
pub image_id: i64, pub image_id: i64,
pub folder_id: i64, pub folder_id: i64,
pub path: String, pub path: String,
pub thumbnail_path: Option<String>,
} }
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -1202,14 +1203,14 @@ fn get_pending_thumbnail_jobs_excluding(
Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?) Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
} }
/// Video jobs need FFmpeg; while it isn't provisioned they must be invisible /// Video and AVIF thumbnail jobs need FFmpeg; while it isn't provisioned they
/// to both claiming and the tier-priority checks, or pending video jobs would /// must be invisible to both claiming and tier-priority checks, or pending
/// stall every lower tier indefinitely. /// FFmpeg-backed jobs would stall every lower tier indefinitely.
fn media_kind_clause(include_videos: bool) -> &'static str { fn media_kind_clause(include_videos: bool) -> &'static str {
if include_videos { if include_videos {
"" ""
} else { } else {
"AND i.media_kind = 'image'" "AND i.media_kind = 'image' AND lower(i.path) NOT GLOB '*.avif'"
} }
} }
@@ -1593,7 +1594,7 @@ pub fn repair_deferred_embedding_jobs(conn: &Connection) -> Result<usize> {
WHERE status = 'failed' WHERE status = 'failed'
AND last_error LIKE ?1 AND last_error LIKE ?1
AND image_id IN ( AND image_id IN (
SELECT id FROM images WHERE media_kind = 'video' SELECT id FROM images WHERE media_kind = 'video' OR lower(path) GLOB '*.avif'
)", )",
[pattern], [pattern],
)?; )?;
@@ -1602,12 +1603,56 @@ pub fn repair_deferred_embedding_jobs(conn: &Connection) -> Result<usize> {
SET embedding_status = 'pending', embedding_error = NULL SET embedding_status = 'pending', embedding_error = NULL
WHERE embedding_status = 'failed' WHERE embedding_status = 'failed'
AND embedding_error LIKE ?1 AND embedding_error LIKE ?1
AND media_kind = 'video'", AND (media_kind = 'video' OR lower(path) GLOB '*.avif')",
[pattern], [pattern],
)?; )?;
Ok(repaired) Ok(repaired)
} }
pub fn repair_avif_jobs(conn: &Connection) -> Result<usize> {
let unsupported_pattern = "%Avif%not supported%";
let thumbnail_repaired = conn.execute(
"UPDATE thumbnail_jobs
SET status = 'pending', last_error = NULL, updated_at = datetime('now')
WHERE status = 'failed'
AND image_id IN (SELECT id FROM images WHERE lower(path) GLOB '*.avif')
AND last_error LIKE ?1",
[unsupported_pattern],
)?;
let embedding_repaired = conn.execute(
"UPDATE embedding_jobs
SET status = 'pending', last_error = NULL, updated_at = datetime('now')
WHERE status = 'failed'
AND image_id IN (SELECT id FROM images WHERE lower(path) GLOB '*.avif')
AND last_error LIKE ?1",
[unsupported_pattern],
)?;
conn.execute(
"UPDATE images
SET embedding_status = 'pending', embedding_error = NULL
WHERE embedding_status = 'failed'
AND lower(path) GLOB '*.avif'
AND embedding_error LIKE ?1",
[unsupported_pattern],
)?;
let tagging_repaired = conn.execute(
"UPDATE tagging_jobs
SET status = 'pending', last_error = NULL, updated_at = datetime('now')
WHERE status = 'failed'
AND image_id IN (SELECT id FROM images WHERE lower(path) GLOB '*.avif')
AND last_error LIKE ?1",
[unsupported_pattern],
)?;
conn.execute(
"UPDATE images
SET ai_tagger_error = NULL
WHERE lower(path) GLOB '*.avif'
AND ai_tagger_error LIKE ?1",
[unsupported_pattern],
)?;
Ok(thumbnail_repaired + embedding_repaired + tagging_repaired)
}
pub fn rename_folder(conn: &Connection, folder_id: i64, new_name: &str) -> Result<()> { pub fn rename_folder(conn: &Connection, folder_id: i64, new_name: &str) -> Result<()> {
conn.execute( conn.execute(
"UPDATE folders SET name = ?2 WHERE id = ?1", "UPDATE folders SET name = ?2 WHERE id = ?1",
@@ -2102,10 +2147,11 @@ fn get_pending_tagging_jobs_excluding(
limit: usize, limit: usize,
) -> Result<Vec<TaggingJob>> { ) -> Result<Vec<TaggingJob>> {
let sql = format!( let sql = format!(
"SELECT j.image_id, i.folder_id, i.path "SELECT j.image_id, i.folder_id, i.path, i.thumbnail_path
FROM tagging_jobs j FROM tagging_jobs j
JOIN images i ON i.id = j.image_id JOIN images i ON i.id = j.image_id
WHERE j.status = 'pending' WHERE j.status = 'pending'
AND NOT (lower(i.path) GLOB '*.avif' AND i.thumbnail_path IS NULL)
{} {}
ORDER BY j.created_at ASC ORDER BY j.created_at ASC
LIMIT ?1", LIMIT ?1",
@@ -2118,6 +2164,7 @@ fn get_pending_tagging_jobs_excluding(
image_id: row.get(0)?, image_id: row.get(0)?,
folder_id: row.get(1)?, folder_id: row.get(1)?,
path: row.get(2)?, path: row.get(2)?,
thumbnail_path: row.get(3)?,
}) })
})? })?
.collect::<rusqlite::Result<Vec<_>>>()?; .collect::<rusqlite::Result<Vec<_>>>()?;
+12 -5
View File
@@ -220,19 +220,19 @@ fn load_images(paths: &[PathBuf], image_size: usize) -> Result<Tensor> {
/// Returns the path that should be fed to the CLIP image embedder for a given media file. /// Returns the path that should be fed to the CLIP image embedder for a given media file.
/// ///
/// For videos the thumbnail image is used (because CLIP only understands still images). /// For videos and AVIFs the thumbnail image is used because CLIP preprocessing
/// Video jobs without thumbnails are excluded when jobs are claimed. The error remains /// only uses decoders from the `image` crate, while AVIF is decoded through
/// as a guard against a race where a thumbnail disappears after a job is claimed. /// FFmpeg into a JPEG thumbnail.
pub fn embedding_source_path( pub fn embedding_source_path(
path: &str, path: &str,
thumbnail_path: Option<&str>, thumbnail_path: Option<&str>,
media_kind: &str, media_kind: &str,
) -> Result<PathBuf> { ) -> Result<PathBuf> {
if media_kind == "video" { if media_kind == "video" || is_avif_path(path) {
match thumbnail_path { match thumbnail_path {
Some(thumb) => Ok(PathBuf::from(thumb)), Some(thumb) => Ok(PathBuf::from(thumb)),
None => Err(anyhow::anyhow!( None => Err(anyhow::anyhow!(
"No thumbnail available yet for video '{}' — embedding deferred until thumbnail is generated", "No thumbnail available yet for '{}' — embedding deferred until thumbnail is generated",
std::path::Path::new(path) std::path::Path::new(path)
.file_name() .file_name()
.map(|n| n.to_string_lossy()) .map(|n| n.to_string_lossy())
@@ -243,3 +243,10 @@ pub fn embedding_source_path(
Ok(PathBuf::from(path)) Ok(PathBuf::from(path))
} }
} }
fn is_avif_path(path: &str) -> bool {
std::path::Path::new(path)
.extension()
.and_then(|ext| ext.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("avif"))
}
+26 -3
View File
@@ -663,10 +663,13 @@ fn process_thumbnail_batch(
let (image_jobs, video_jobs): (Vec<_>, Vec<_>) = let (image_jobs, video_jobs): (Vec<_>, Vec<_>) =
jobs.into_iter().partition(|job| job.media_kind == "image"); jobs.into_iter().partition(|job| job.media_kind == "image");
let (avif_jobs, raster_jobs): (Vec<_>, Vec<_>) = image_jobs
.into_iter()
.partition(|job| is_avif_path(Path::new(&job.path)));
// Images: parallel decode, committed as one batch. // Images: parallel decode, committed as one batch.
if !image_jobs.is_empty() { if !raster_jobs.is_empty() {
let results = image_jobs let results = raster_jobs
.par_iter() .par_iter()
.map(|job| { .map(|job| {
( (
@@ -678,6 +681,15 @@ fn process_thumbnail_batch(
persist_thumbnail_results(app, pool, results)?; persist_thumbnail_results(app, pool, results)?;
} }
// AVIF: FFmpeg-backed decode, like videos. Keep this off the shared rayon
// pool because each subprocess blocks its worker thread.
for job in &avif_jobs {
let result =
thumbnail::generate_avif_thumbnail(media_tools, Path::new(&job.path), cache_dir)
.map(Some);
persist_thumbnail_results(app, pool, vec![(job.image_id, result)])?;
}
// Videos: sequential, off the rayon pool — each ffmpeg call blocks its // Videos: sequential, off the rayon pool — each ffmpeg call blocks its
// thread, and a video-heavy batch on the shared pool would starve image // thread, and a video-heavy batch on the shared pool would starve image
// decoding across all workers. Committed per item so progress keeps // decoding across all workers. Committed per item so progress keeps
@@ -749,6 +761,12 @@ fn persist_thumbnail_results(
Ok(()) Ok(())
} }
fn is_avif_path(path: &Path) -> bool {
path.extension()
.and_then(|ext| ext.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("avif"))
}
/// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if /// Returns `Ok(true)` if a batch was claimed and processed, `Ok(false)` if
/// the queue was empty. /// the queue was empty.
fn process_metadata_batch( fn process_metadata_batch(
@@ -1142,9 +1160,14 @@ fn process_tagging_batch(
let tag_results = jobs let tag_results = jobs
.iter() .iter()
.map(|job| { .map(|job| {
let source_path = if is_avif_path(Path::new(&job.path)) {
job.thumbnail_path.as_deref().unwrap_or(&job.path)
} else {
&job.path
};
( (
job.clone(), job.clone(),
tagger_ref.run(Path::new(&job.path), tagger::DEFAULT_MAX_TAGS), tagger_ref.run(Path::new(source_path), tagger::DEFAULT_MAX_TAGS),
) )
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
+7 -2
View File
@@ -53,7 +53,7 @@ pub fn run() {
std::fs::create_dir_all(&app_dir).expect("Failed to create app data dir"); std::fs::create_dir_all(&app_dir).expect("Failed to create app data dir");
// FFmpeg provisioning happens in the background so the window // FFmpeg provisioning happens in the background so the window
// appears immediately; workers gate video jobs on readiness and // appears immediately; workers gate video/AVIF jobs on readiness and
// the onboarding/Settings UI shows progress and retry. // the onboarding/Settings UI shows progress and retry.
media::spawn_ffmpeg_provision(app.handle().clone()); media::spawn_ffmpeg_provision(app.handle().clone());
@@ -68,7 +68,12 @@ pub fn run() {
let repaired_deferred = db::repair_deferred_embedding_jobs(&conn) let repaired_deferred = db::repair_deferred_embedding_jobs(&conn)
.expect("Failed to repair deferred embedding jobs"); .expect("Failed to repair deferred embedding jobs");
if repaired_deferred > 0 { if repaired_deferred > 0 {
log::info!("Requeued {repaired_deferred} deferred video embedding jobs."); log::info!("Requeued {repaired_deferred} deferred embedding jobs.");
}
let repaired_avif =
db::repair_avif_jobs(&conn).expect("Failed to repair AVIF jobs");
if repaired_avif > 0 {
log::info!("Requeued {repaired_avif} AVIF jobs.");
} }
let backfilled = let backfilled =
db::backfill_embedding_jobs(&conn).expect("Failed to backfill embedding jobs"); db::backfill_embedding_jobs(&conn).expect("Failed to backfill embedding jobs");
+84
View File
@@ -241,6 +241,90 @@ pub fn generate_video_thumbnail(
)) ))
} }
pub fn generate_avif_thumbnail(
tools: &MediaTools,
image_path: &Path,
cache_dir: &Path,
) -> Result<GeneratedThumbnail> {
let path_str = image_path.to_string_lossy();
let out_path = thumb_path(cache_dir, &path_str);
let original_dimensions = ffprobe_dimensions(tools, image_path);
if out_path.exists() {
return Ok(GeneratedThumbnail {
path: out_path,
width: original_dimensions.0,
height: original_dimensions.1,
});
}
if let Some(parent) = out_path.parent() {
std::fs::create_dir_all(parent)?;
}
let output_path = out_path.to_string_lossy().into_owned();
let output = tools
.ffmpeg_command()
.args([
"-y",
"-threads",
"2",
"-i",
path_str.as_ref(),
"-frames:v",
"1",
"-vf",
"scale=320:-1:force_original_aspect_ratio=decrease",
"-q:v",
"4",
&output_path,
])
.output()?;
if output.status.success() && out_path.exists() {
return Ok(GeneratedThumbnail {
path: out_path,
width: original_dimensions.0,
height: original_dimensions.1,
});
}
Err(anyhow!(
"ffmpeg failed generating AVIF thumbnail for {path_str}: {}",
String::from_utf8_lossy(&output.stderr)
))
}
fn ffprobe_dimensions(tools: &MediaTools, image_path: &Path) -> (Option<i64>, Option<i64>) {
let output = tools
.ffprobe_command()
.args([
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=width,height",
"-of",
"csv=p=0:s=x",
&image_path.to_string_lossy(),
])
.output();
let Ok(output) = output else {
return (None, None);
};
if !output.status.success() {
return (None, None);
}
let stdout = String::from_utf8_lossy(&output.stdout);
let mut parts = stdout.trim().split('x');
let width = parts.next().and_then(|part| part.parse::<i64>().ok());
let height = parts.next().and_then(|part| part.parse::<i64>().ok());
(width, height)
}
pub fn thumb_path(cache_dir: &Path, image_path: &str) -> PathBuf { pub fn thumb_path(cache_dir: &Path, image_path: &str) -> PathBuf {
thumb_path_with_ext(cache_dir, image_path, "jpg") thumb_path_with_ext(cache_dir, image_path, "jpg")
} }