diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 65de8fc..fb16359 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -4,7 +4,7 @@ use crate::indexer; use crate::vector; use serde::{Deserialize, Serialize}; use std::path::PathBuf; -use tauri::{AppHandle, Manager, State}; +use tauri::{AppHandle, State}; pub type DbState = DbPool; @@ -22,6 +22,7 @@ pub struct GetImagesParams { pub search: Option, pub media_kind: Option, pub favorites_only: Option, + pub embedding_failed_only: Option, pub sort: Option, pub offset: Option, pub limit: Option, @@ -124,8 +125,9 @@ pub async fn get_images( let search = params.search.as_deref(); let media_kind = params.media_kind.as_deref(); let favorites_only = params.favorites_only.unwrap_or(false); + let embedding_failed_only = params.embedding_failed_only.unwrap_or(false); - let total = db::count_images(&conn, params.folder_id, search, media_kind, favorites_only) + let total = db::count_images(&conn, params.folder_id, search, media_kind, favorites_only, embedding_failed_only) .map_err(|e| e.to_string())?; let images = db::get_images( @@ -134,6 +136,7 @@ pub async fn get_images( search, media_kind, favorites_only, + embedding_failed_only, sort, offset, limit, @@ -224,77 +227,109 @@ pub async fn semantic_search_images( Ok(images) } -#[derive(Serialize)] +#[derive(Serialize, Deserialize)] pub struct TagCloudEntry { - pub label: String, pub count: usize, + pub representative_image_id: i64, + pub thumbnail_path: Option, } -/// Clusters the library's image embeddings with k-means, then labels each cluster by -/// finding the closest word in the vocabulary. The vocabulary is loaded from -/// `{app_data_dir}/vocabulary.txt` if present, otherwise from the bundled default. -/// Vocabulary embeddings are cached to disk — only recomputed when the vocabulary changes. +fn fnv_hash_ids(ids: &[i64]) -> u64 { + let mut h: u64 = 0xcbf29ce484222325; + for &id in ids { + for b in id.to_le_bytes() { + h = h.wrapping_mul(0x100000001b3) ^ (b as u64); + } + } + h +} + +/// Clusters the library's image embeddings with k-means and returns one representative +/// image per cluster — the member whose embedding is closest to its cluster centroid. +/// Results are cached in SQLite keyed by a hash of the embedded image IDs, so repeated +/// calls (including across app restarts) return instantly when the library hasn't changed. #[tauri::command] pub async fn get_tag_cloud( - app: AppHandle, db: State<'_, DbState>, folder_id: Option, ) -> Result, String> { - let app_data_dir = app.path().app_data_dir().map_err(|e| e.to_string())?; - - let image_embeddings = { + let embeddings_with_ids = { let conn = db.get().map_err(|e| e.to_string())?; - vector::get_all_image_embeddings(&conn, folder_id).map_err(|e| e.to_string())? + vector::get_all_image_embeddings_with_ids(&conn, folder_id).map_err(|e| e.to_string())? }; - let n = image_embeddings.len(); + let n = embeddings_with_ids.len(); if n < 5 { return Ok(vec![]); } - // Load vocabulary (custom file > bundled default) - let vocab = embedder::load_vocabulary(&app_data_dir); + // Compute a hash of the current embedded image IDs (sorted for stability) + let mut sorted_ids: Vec = embeddings_with_ids.iter().map(|(id, _)| *id).collect(); + sorted_ids.sort_unstable(); + let current_hash = fnv_hash_ids(&sorted_ids); - // Embed vocabulary — disk-cached, only recomputes when vocabulary changes - let vocab_refs: Vec<&str> = vocab.iter().map(|s| s.as_str()).collect(); - let vocab_embeddings = embedder::embed_vocab_cached(&vocab, &app_data_dir) - .map_err(|e| e.to_string())?; + let folder_scope = match folder_id { + Some(id) => format!("folder_{}", id), + None => "all".to_string(), + }; - // Choose k proportional to library size, capped at 30 - let k = (n / 20).clamp(5, 30); - - // Cluster image embeddings - let (centroids, cluster_counts) = kmeans_cosine(&image_embeddings, k, 40); - - // Label each cluster with the nearest vocabulary word - let mut entries: Vec = Vec::new(); - let mut used_labels = std::collections::HashSet::new(); - - let mut order: Vec = (0..k).collect(); - order.sort_unstable_by(|&a, &b| cluster_counts[b].cmp(&cluster_counts[a])); - - for ci in order { - let count = cluster_counts[ci]; - if count == 0 { continue; } - - let centroid = ¢roids[ci]; - let label = vocab_refs - .iter() - .zip(vocab_embeddings.iter()) - .map(|(&word, emb)| { - let sim: f32 = centroid.iter().zip(emb.iter()).map(|(a, b)| a * b).sum(); - (word, sim) - }) - .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)) - .map(|(word, _)| word.to_string()) - .unwrap_or_else(|| "other".to_string()); - - if used_labels.insert(label.clone()) { - entries.push(TagCloudEntry { label, count }); + // Try to return a valid SQLite cache + { + let conn = db.get().map_err(|e| e.to_string())?; + if let Some(json) = db::get_tag_cloud_cache(&conn, &folder_scope, current_hash) + .map_err(|e| e.to_string())? + { + if let Ok(entries) = serde_json::from_str::>(&json) { + return Ok(entries); + } } } - entries.sort_unstable_by(|a, b| b.count.cmp(&a.count)); + // Cache miss — run k-means + let ids: Vec = embeddings_with_ids.iter().map(|(id, _)| *id).collect(); + let points: Vec> = embeddings_with_ids.into_iter().map(|(_, emb)| emb).collect(); + + let k = (n / 20).clamp(5, 30); + let (centroids, cluster_counts, assignments) = kmeans_cosine(&points, k, 40); + + let mut entries: Vec = Vec::new(); + let mut order: Vec = (0..k).collect(); + order.sort_unstable_by(|&a, &b| cluster_counts[b].cmp(&cluster_counts[a])); + + let conn = db.get().map_err(|e| e.to_string())?; + + for ci in order { + let count = cluster_counts[ci]; + if count == 0 { + continue; + } + + let centroid = ¢roids[ci]; + let best_id = points + .iter() + .enumerate() + .filter(|(i, _)| assignments[*i] == ci) + .map(|(i, p)| (ids[i], dot(centroid, p))) + .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)) + .map(|(id, _)| id) + .unwrap_or(0); + + let thumbnail_path = db::get_image_by_id(&conn, best_id) + .ok() + .and_then(|img| img.thumbnail_path); + + entries.push(TagCloudEntry { + count, + representative_image_id: best_id, + thumbnail_path, + }); + } + + // Persist to SQLite — ignore write errors (cache is best-effort) + if let Ok(json) = serde_json::to_string(&entries) { + let _ = db::set_tag_cloud_cache(&conn, &folder_scope, current_hash, &json); + } + Ok(entries) } @@ -315,7 +350,7 @@ fn kmeans_cosine( points: &[Vec], k: usize, max_iter: usize, -) -> (Vec>, Vec) { +) -> (Vec>, Vec, Vec) { let n = points.len(); let dim = points[0].len(); @@ -375,7 +410,31 @@ fn kmeans_cosine( let mut counts = vec![0usize; k]; for &a in &assignments { counts[a] += 1; } - (centroids, counts) + (centroids, counts, assignments) +} + +#[derive(Serialize)] +pub struct FailedEmbeddingItem { + pub image_id: i64, + pub filename: String, + pub error: Option, +} + +#[tauri::command] +pub async fn get_failed_embedding_images( + db: State<'_, DbState>, + folder_id: i64, +) -> Result, String> { + let conn = db.get().map_err(|e| e.to_string())?; + let rows = db::get_failed_embedding_images(&conn, folder_id).map_err(|e| e.to_string())?; + Ok(rows + .into_iter() + .map(|(image_id, filename, error)| FailedEmbeddingItem { + image_id, + filename, + error, + }) + .collect()) } #[derive(Serialize)] diff --git a/src-tauri/src/db.rs b/src-tauri/src/db.rs index 3c44d2d..7fe7ead 100644 --- a/src-tauri/src/db.rs +++ b/src-tauri/src/db.rs @@ -172,6 +172,13 @@ pub fn migrate(conn: &Connection) -> Result<()> { updated_at TEXT NOT NULL DEFAULT (datetime('now')) ); + CREATE TABLE IF NOT EXISTS tag_cloud_cache ( + folder_scope TEXT PRIMARY KEY, + image_ids_hash INTEGER NOT NULL, + entries_json TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_images_folder_id ON images(folder_id); CREATE INDEX IF NOT EXISTS idx_images_modified_at ON images(modified_at); CREATE INDEX IF NOT EXISTS idx_embedding_jobs_status ON embedding_jobs(status); @@ -301,11 +308,16 @@ pub fn backfill_embedding_jobs(conn: &Connection) -> Result { } pub fn retry_failed_embedding_jobs(conn: &Connection, folder_id: i64) -> Result { + // Only re-queue images that are actually embeddable right now. + // Videos without a thumbnail would just fail again immediately, so skip them — + // they will be re-queued automatically once their thumbnail is generated. let updated = conn.execute( "INSERT INTO embedding_jobs (image_id, status, attempts, last_error, created_at, updated_at) SELECT id, 'pending', 0, NULL, datetime('now'), datetime('now') FROM images - WHERE folder_id = ?1 AND embedding_status = 'failed' + WHERE folder_id = ?1 + AND embedding_status = 'failed' + AND NOT (media_kind = 'video' AND thumbnail_path IS NULL) ON CONFLICT(image_id) DO UPDATE SET status = 'pending', last_error = NULL, @@ -315,7 +327,9 @@ pub fn retry_failed_embedding_jobs(conn: &Connection, folder_id: i64) -> Result< conn.execute( "UPDATE images SET embedding_status = 'pending', embedding_error = NULL - WHERE folder_id = ?1 AND embedding_status = 'failed'", + WHERE folder_id = ?1 + AND embedding_status = 'failed' + AND NOT (media_kind = 'video' AND thumbnail_path IS NULL)", [folder_id], )?; Ok(updated) @@ -802,6 +816,7 @@ pub fn get_images( search: Option<&str>, media_kind: Option<&str>, favorites_only: bool, + embedding_failed_only: bool, sort: &str, offset: i64, limit: i64, @@ -820,6 +835,7 @@ pub fn get_images( let search_pattern = search.map(|value| format!("%{}%", value)); let favorites_flag = i64::from(favorites_only); + let embedding_failed_flag = i64::from(embedding_failed_only); let sql = format!( "SELECT id, folder_id, path, filename, thumbnail_path, width, height, file_size, created_at, modified_at, mime_type, media_kind, duration_ms, video_codec, audio_codec, metadata_updated_at, metadata_error, @@ -829,8 +845,9 @@ pub fn get_images( AND (?2 IS NULL OR filename LIKE ?2) AND (?3 IS NULL OR media_kind = ?3) AND (?4 = 0 OR favorite = 1) + AND (?5 = 0 OR embedding_status = 'failed') ORDER BY {} - LIMIT ?5 OFFSET ?6", + LIMIT ?6 OFFSET ?7", order ); let mut stmt = conn.prepare(&sql)?; @@ -840,6 +857,7 @@ pub fn get_images( search_pattern, media_kind, favorites_flag, + embedding_failed_flag, limit, offset ], @@ -854,23 +872,52 @@ pub fn count_images( search: Option<&str>, media_kind: Option<&str>, favorites_only: bool, + embedding_failed_only: bool, ) -> Result { let search_pattern = search.map(|value| format!("%{}%", value)); let favorites_flag = i64::from(favorites_only); + let embedding_failed_flag = i64::from(embedding_failed_only); let count = conn.query_row( "SELECT COUNT(*) FROM images WHERE (?1 IS NULL OR folder_id = ?1) AND (?2 IS NULL OR filename LIKE ?2) AND (?3 IS NULL OR media_kind = ?3) - AND (?4 = 0 OR favorite = 1)", - params![folder_id, search_pattern, media_kind, favorites_flag], + AND (?4 = 0 OR favorite = 1) + AND (?5 = 0 OR embedding_status = 'failed')", + params![ + folder_id, + search_pattern, + media_kind, + favorites_flag, + embedding_failed_flag + ], |row| row.get(0), )?; Ok(count) } +pub fn get_failed_embedding_images( + conn: &Connection, + folder_id: i64, +) -> Result)>> { + let mut stmt = conn.prepare( + "SELECT id, filename, embedding_error + FROM images + WHERE folder_id = ?1 AND embedding_status = 'failed' + ORDER BY filename", + )?; + let rows = stmt.query_map([folder_id], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + )) + })?; + Ok(rows.collect::>>()?) +} + pub fn delete_folder(conn: &Connection, folder_id: i64) -> Result<()> { conn.execute("DELETE FROM folders WHERE id = ?1", params![folder_id])?; Ok(()) @@ -904,6 +951,44 @@ fn map_image_row(row: &Row<'_>) -> rusqlite::Result { }) } +/// Returns cached tag-cloud entries if the stored hash matches `current_hash`. +pub fn get_tag_cloud_cache( + conn: &Connection, + folder_scope: &str, + current_hash: u64, +) -> Result> { + let result: rusqlite::Result<(i64, String)> = conn.query_row( + "SELECT image_ids_hash, entries_json FROM tag_cloud_cache WHERE folder_scope = ?1", + params![folder_scope], + |row| Ok((row.get(0)?, row.get(1)?)), + ); + match result { + Ok((stored_hash, json)) if stored_hash as u64 == current_hash => Ok(Some(json)), + Ok(_) => Ok(None), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(e.into()), + } +} + +/// Upserts the tag-cloud cache for the given scope. +pub fn set_tag_cloud_cache( + conn: &Connection, + folder_scope: &str, + image_ids_hash: u64, + entries_json: &str, +) -> Result<()> { + conn.execute( + "INSERT INTO tag_cloud_cache (folder_scope, image_ids_hash, entries_json, created_at) + VALUES (?1, ?2, ?3, datetime('now')) + ON CONFLICT(folder_scope) DO UPDATE SET + image_ids_hash = excluded.image_ids_hash, + entries_json = excluded.entries_json, + created_at = excluded.created_at", + params![folder_scope, image_ids_hash as i64, entries_json], + )?; + Ok(()) +} + fn ensure_column(conn: &Connection, table: &str, column: &str, definition: &str) -> Result<()> { let mut stmt = conn.prepare(&format!("PRAGMA table_info({})", table))?; let mut rows = stmt.query([])?; diff --git a/src-tauri/src/embedder.rs b/src-tauri/src/embedder.rs index ac0c104..532417d 100644 --- a/src-tauri/src/embedder.rs +++ b/src-tauri/src/embedder.rs @@ -1,173 +1,24 @@ -use anyhow::{Context, Result}; +use anyhow::Result; use candle_core::{DType, Device, Tensor}; use candle_nn::VarBuilder; use candle_transformers::models::clip::{self, ClipModel}; use hf_hub::{api::sync::Api, Repo, RepoType}; -use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::sync::{Mutex, OnceLock}; use tokenizers::Tokenizer; static TEXT_SEARCH_EMBEDDER: OnceLock>> = OnceLock::new(); -/// In-process cache so the disk file is only read/written once per session. -static VOCAB_EMBED_CACHE: OnceLock>>>> = OnceLock::new(); - -/// Default vocabulary bundled with the binary. -pub const DEFAULT_VOCABULARY: &str = include_str!("../data/vocabulary.txt"); /// Embed a text query using a lazily-initialized, cached CLIP embedder. pub fn embed_text_query(query: &str) -> Result> { with_text_embedder(|e| e.embed_text(query)) } -/// Parse vocabulary text: strip comment lines (starting with `#`) and blank lines. -pub fn parse_vocabulary(text: &str) -> Vec { - text.lines() - .map(|l| l.trim()) - .filter(|l| !l.is_empty() && !l.starts_with('#')) - .map(|l| l.to_string()) - .collect() -} - -/// Load the vocabulary from `{cache_dir}/vocabulary.txt` if it exists, -/// otherwise fall back to the binary-bundled default. -pub fn load_vocabulary(cache_dir: &Path) -> Vec { - let custom_path = cache_dir.join("vocabulary.txt"); - if custom_path.exists() { - if let Ok(text) = std::fs::read_to_string(&custom_path) { - let words = parse_vocabulary(&text); - if !words.is_empty() { - println!("Using custom vocabulary from {:?} ({} words)", custom_path, words.len()); - return words; - } - } - } - parse_vocabulary(DEFAULT_VOCABULARY) -} - -/// Embed the vocabulary, using a disk cache so CLIP is only called when the vocabulary -/// actually changes. Cache file: `{cache_dir}/vocab_embeddings.bin`. -/// -/// Format: `[u64 hash][u32 n_words][u32 dim][(n_words × dim) × f32 LE]` -pub fn embed_vocab_cached(vocab: &[String], cache_dir: &Path) -> Result>> { - // In-process cache hit - { - let guard = VOCAB_EMBED_CACHE - .get_or_init(|| Mutex::new(None)) - .lock() - .map_err(|_| anyhow::anyhow!("Vocab cache lock poisoned"))?; - if let Some(cached) = guard.as_ref() { - return Ok(cached.clone()); - } - } - - let vocab_hash = fnv_hash(vocab); - let disk_path = cache_dir.join("vocab_embeddings.bin"); - - // Try loading from disk - if let Ok(embeddings) = load_disk_cache(&disk_path, vocab_hash) { - let mut guard = VOCAB_EMBED_CACHE - .get_or_init(|| Mutex::new(None)) - .lock() - .map_err(|_| anyhow::anyhow!("Vocab cache lock poisoned"))?; - *guard = Some(embeddings.clone()); - println!("Vocabulary embeddings loaded from disk cache ({} words).", embeddings.len()); - return Ok(embeddings); - } - - // Compute embeddings - println!("Computing vocabulary embeddings ({} words) — this is cached after the first run.", vocab.len()); - let prompts: Vec = vocab.iter().map(|w| format!("a photo of {}", w)).collect(); - let prompt_refs: Vec<&str> = prompts.iter().map(|s| s.as_str()).collect(); - - let embeddings = with_text_embedder(|e| { - let mut all = Vec::with_capacity(vocab.len()); - for chunk in prompt_refs.chunks(64) { - all.extend(e.embed_texts_batch(chunk)?); - } - Ok(all) - })?; - - // Save to disk - if let Err(e) = save_disk_cache(&disk_path, vocab_hash, &embeddings) { - eprintln!("Warning: could not write vocab cache: {e}"); - } - - let mut guard = VOCAB_EMBED_CACHE - .get_or_init(|| Mutex::new(None)) - .lock() - .map_err(|_| anyhow::anyhow!("Vocab cache lock poisoned"))?; - *guard = Some(embeddings.clone()); - Ok(embeddings) -} - -fn fnv_hash(words: &[String]) -> u64 { - let mut h: u64 = 0xcbf29ce484222325; - for w in words { - for b in w.bytes() { - h = h.wrapping_mul(0x100000001b3) ^ (b as u64); - } - h = h.wrapping_mul(0x100000001b3) ^ 0x0A; // newline separator - } - h -} - -fn load_disk_cache(path: &Path, expected_hash: u64) -> Result>> { - let mut f = std::fs::File::open(path).context("no cache file")?; - let mut buf = Vec::new(); - f.read_to_end(&mut buf)?; - - if buf.len() < 16 { - anyhow::bail!("cache too short"); - } - - let hash = u64::from_le_bytes(buf[0..8].try_into()?); - if hash != expected_hash { - anyhow::bail!("vocab hash mismatch — recomputing"); - } - let n = u32::from_le_bytes(buf[8..12].try_into()?) as usize; - let dim = u32::from_le_bytes(buf[12..16].try_into()?) as usize; - - let expected_len = 16 + n * dim * 4; - if buf.len() != expected_len { - anyhow::bail!("cache size mismatch"); - } - - let mut embeddings = Vec::with_capacity(n); - let data = &buf[16..]; - for i in 0..n { - let start = i * dim * 4; - let emb: Vec = data[start..start + dim * 4] - .chunks_exact(4) - .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) - .collect(); - embeddings.push(emb); - } - Ok(embeddings) -} - -fn save_disk_cache(path: &Path, hash: u64, embeddings: &[Vec]) -> Result<()> { - let n = embeddings.len(); - let dim = embeddings.first().map(|e| e.len()).unwrap_or(0); - - let mut buf = Vec::with_capacity(16 + n * dim * 4); - buf.extend_from_slice(&hash.to_le_bytes()); - buf.extend_from_slice(&(n as u32).to_le_bytes()); - buf.extend_from_slice(&(dim as u32).to_le_bytes()); - for emb in embeddings { - for &v in emb { - buf.extend_from_slice(&v.to_le_bytes()); - } - } - - std::fs::write(path, buf)?; - println!("Vocabulary embeddings cached to disk ({n} words, {dim} dims)."); - Ok(()) -} - fn with_text_embedder(f: impl FnOnce(&ClipImageEmbedder) -> Result) -> Result { let lock = TEXT_SEARCH_EMBEDDER.get_or_init(|| Mutex::new(None)); - let mut guard = lock.lock().map_err(|_| anyhow::anyhow!("Text embedder lock poisoned"))?; + let mut guard = lock + .lock() + .map_err(|_| anyhow::anyhow!("Text embedder lock poisoned"))?; if guard.is_none() { println!("Initializing CLIP text embedder..."); *guard = Some(ClipImageEmbedder::new()?); @@ -319,16 +170,28 @@ fn load_images(paths: &[PathBuf], image_size: usize) -> Result { Ok(Tensor::stack(&images, 0)?) } +/// Returns the path that should be fed to the CLIP image embedder for a given media file. +/// +/// For videos the thumbnail image is used (because CLIP only understands still images). +/// If a video has no thumbnail yet, an error is returned — the caller should mark the +/// embedding job as failed rather than trying to decode the raw video file. pub fn embedding_source_path( path: &str, thumbnail_path: Option<&str>, media_kind: &str, -) -> PathBuf { +) -> Result { if media_kind == "video" { - thumbnail_path - .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from(path)) + match thumbnail_path { + Some(thumb) => Ok(PathBuf::from(thumb)), + None => Err(anyhow::anyhow!( + "No thumbnail available yet for video '{}' — embedding deferred until thumbnail is generated", + std::path::Path::new(path) + .file_name() + .map(|n| n.to_string_lossy()) + .unwrap_or_default() + )), + } } else { - PathBuf::from(path) + Ok(PathBuf::from(path)) } } diff --git a/src-tauri/src/indexer.rs b/src-tauri/src/indexer.rs index 34ed185..cd7e8a2 100644 --- a/src-tauri/src/indexer.rs +++ b/src-tauri/src/indexer.rs @@ -1,4 +1,4 @@ -use crate::db::{self, DbPool, FolderJobProgress, ImageRecord, IndexedMediaEntry}; +use crate::db::{self, DbPool, EmbeddingJob, FolderJobProgress, ImageRecord, IndexedMediaEntry}; use crate::embedder::{embedding_source_path, ClipImageEmbedder}; use crate::media::{probe_video_metadata, MediaTools}; use crate::storage::{detect_storage_profile, RuntimeAdaptiveProfile, StorageProfile}; @@ -539,27 +539,59 @@ fn process_embedding_batch( let embedder = embedder.as_ref().expect("embedder should be initialized"); let infer_started_at = Instant::now(); - let source_paths = jobs + // Resolve the source path for each job. Videos without a thumbnail produce an Err + // here — those jobs are marked failed immediately without going to the embedder. + let source_results: Vec> = jobs .iter() .map(|job| embedding_source_path(&job.path, job.thumbnail_path.as_deref(), &job.media_kind)) - .collect::>(); + .collect(); - let results = match embedder.embed_images(&source_paths) { - Ok(embeddings) => jobs - .into_iter() - .zip(embeddings.into_iter().map(Ok)) - .collect::>(), - Err(batch_error) => { - eprintln!( - "Embedding batch fallback to per-image mode: {}", - batch_error - ); - jobs.into_iter() - .zip(source_paths.into_iter()) - .map(|(job, source_path)| (job, embedder.embed_image(&source_path))) - .collect::>() + // Separate jobs with a valid source from those that fail early (e.g. video with no thumbnail). + let mut embeddable_indices: Vec = Vec::new(); + let mut embeddable_paths: Vec = Vec::new(); + // image_id -> early error message for jobs that cannot be embedded yet + let mut pre_failed: HashMap = HashMap::new(); + + for (i, (job, result)) in jobs.iter().zip(source_results.into_iter()).enumerate() { + match result { + Ok(path) => { + embeddable_indices.push(i); + embeddable_paths.push(path); + } + Err(e) => { + pre_failed.insert(job.image_id, e.to_string()); + } } - }; + } + + // Run CLIP only on the jobs that have a valid source image. + // image_id -> embedding result + let mut embed_results: HashMap>> = HashMap::new(); + + if !embeddable_indices.is_empty() { + let embeddable_jobs: Vec<&EmbeddingJob> = + embeddable_indices.iter().map(|&i| &jobs[i]).collect(); + + match embedder.embed_images(&embeddable_paths) { + Ok(embeddings) => { + for (job, embedding) in embeddable_jobs.iter().zip(embeddings.into_iter()) { + embed_results.insert(job.image_id, Ok(embedding)); + } + } + Err(batch_error) => { + eprintln!( + "Embedding batch fallback to per-image mode: {}", + batch_error + ); + for (job, source_path) in embeddable_jobs + .into_iter() + .zip(embeddable_paths.into_iter()) + { + embed_results.insert(job.image_id, embedder.embed_image(&source_path)); + } + } + } + } let infer_elapsed = infer_started_at.elapsed(); let write_started_at = Instant::now(); @@ -568,7 +600,16 @@ fn process_embedding_batch( let tx = conn.transaction()?; let mut updated_images = Vec::new(); - for (job, embedding_result) in results { + for job in &jobs { + let embedding_result: Result> = + if let Some(err) = pre_failed.remove(&job.image_id) { + Err(anyhow::anyhow!("{}", err)) + } else if let Some(r) = embed_results.remove(&job.image_id) { + r + } else { + Err(anyhow::anyhow!("no result for image {}", job.image_id)) + }; + match embedding_result { Ok(embedding) => { vector::upsert_embedding(&tx, job.image_id, &embedding)?; diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index de5b311..3592fbf 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -78,6 +78,7 @@ pub fn run() { commands::set_worker_paused, commands::get_worker_states, commands::get_tag_cloud, + commands::get_failed_embedding_images, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/src-tauri/src/vector.rs b/src-tauri/src/vector.rs index befeae8..61217d5 100644 --- a/src-tauri/src/vector.rs +++ b/src-tauri/src/vector.rs @@ -78,32 +78,42 @@ pub fn find_similar_image_ids(conn: &Connection, image_id: i64, limit: usize) -> Ok(ids) } -/// Returns all stored image embeddings, optionally filtered to one folder. -/// Each embedding is returned as a normalized f32 vector. -pub fn get_all_image_embeddings(conn: &Connection, folder_id: Option) -> Result>> { - let packed_rows: Vec> = match folder_id { +/// Returns all stored image embeddings with their image IDs, optionally filtered to one folder. +/// Each entry is `(image_id, normalized_f32_embedding)`. +pub fn get_all_image_embeddings_with_ids( + conn: &Connection, + folder_id: Option, +) -> Result)>> { + let packed_rows: Vec<(i64, Vec)> = match folder_id { Some(fid) => { let mut stmt = conn.prepare( - "SELECT embedding FROM image_vec + "SELECT image_id, embedding FROM image_vec WHERE image_id IN (SELECT id FROM images WHERE folder_id = ?1)", )?; - let rows: Vec> = stmt - .query_map([fid], |row| row.get::<_, Vec>(0))? + let rows: Vec<(i64, Vec)> = stmt + .query_map([fid], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) + })? .filter_map(|r| r.ok()) .collect(); rows } None => { - let mut stmt = conn.prepare("SELECT embedding FROM image_vec")?; - let rows: Vec> = stmt - .query_map([], |row| row.get::<_, Vec>(0))? + let mut stmt = conn.prepare("SELECT image_id, embedding FROM image_vec")?; + let rows: Vec<(i64, Vec)> = stmt + .query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) + })? .filter_map(|r| r.ok()) .collect(); rows } }; - Ok(packed_rows.iter().map(|b| unpack_f32(b)).collect()) + Ok(packed_rows + .into_iter() + .map(|(id, b)| (id, unpack_f32(&b))) + .collect()) } fn unpack_f32(bytes: &[u8]) -> Vec { diff --git a/src/components/BackgroundTasks.tsx b/src/components/BackgroundTasks.tsx index eb8232e..0433f63 100644 --- a/src/components/BackgroundTasks.tsx +++ b/src/components/BackgroundTasks.tsx @@ -29,6 +29,12 @@ interface Task { snapshot: string; } +interface FailedEmbeddingItem { + image_id: number; + filename: string; + error: string | null; +} + export function BackgroundTasks() { const folders = useGalleryStore((state) => state.folders); const indexingProgress = useGalleryStore((state) => state.indexingProgress); @@ -41,6 +47,7 @@ export function BackgroundTasks() { metadata: false, embedding: false, }); + const [failedItems, setFailedItems] = useState>({}); useEffect(() => { invoke<{ thumbnail_paused: boolean; metadata_paused: boolean; embedding_paused: boolean }>( @@ -54,6 +61,28 @@ export function BackgroundTasks() { }); }, []); + // Fetch failed embedding filenames whenever the expanded panel opens or failure counts change. + const failedCounts = useMemo( + () => + Object.fromEntries( + Object.entries(mediaJobProgress).map(([id, p]) => [id, p?.embedding_failed ?? 0]), + ), + [mediaJobProgress], + ); + + useEffect(() => { + if (!expanded) return; + for (const [folderId, count] of Object.entries(failedCounts)) { + if (count > 0) { + invoke("get_failed_embedding_images", { + folderId: Number(folderId), + }) + .then((items) => setFailedItems((prev) => ({ ...prev, [folderId]: items }))) + .catch(() => undefined); + } + } + }, [expanded, failedCounts]); + const toggleWorker = (worker: WorkerKey) => { const next = !paused[worker]; setPaused((prev) => ({ ...prev, [worker]: next })); @@ -374,6 +403,26 @@ export function BackgroundTasks() { {task.currentFile}

)} + + {/* Failed embedding file list */} + {taskHasFailed && failedItems[task.id] && failedItems[task.id].length > 0 && ( +
+ {failedItems[task.id].map((item) => ( +
+ + + +
+

{item.filename}

+ {item.error && ( +

{item.error}

+ )} +
+
+ ))} +
+ )} ); })} diff --git a/src/components/Gallery.tsx b/src/components/Gallery.tsx index 05735d0..95a5710 100644 --- a/src/components/Gallery.tsx +++ b/src/components/Gallery.tsx @@ -188,6 +188,21 @@ function ImageTile({ )} + {/* Embedding failed badge — top-left */} + {image.embedding_status === "failed" && ( +
+
+ + + +
+
+ )} + {/* Hover overlay — slides up from bottom */}
diff --git a/src/components/TagCloud.tsx b/src/components/TagCloud.tsx index 5234622..7c6d3f6 100644 --- a/src/components/TagCloud.tsx +++ b/src/components/TagCloud.tsx @@ -1,17 +1,18 @@ import { useEffect } from "react"; import { motion } from "framer-motion"; +import { convertFileSrc } from "@tauri-apps/api/core"; import { useGalleryStore, TagCloudEntry } from "../store"; -// Accent color pairs: [rest, hover, glow] -const ACCENTS: [string, string, string][] = [ - ["rgba(96,165,250,0.5)", "#93c5fd", "rgba(59,130,246,0.3)"], - ["rgba(192,132,252,0.5)", "#d8b4fe", "rgba(168,85,247,0.3)"], - ["rgba(52,211,153,0.5)", "#6ee7b7", "rgba(16,185,129,0.3)"], - ["rgba(251,191,36,0.5)", "#fcd34d", "rgba(245,158,11,0.3)"], - ["rgba(249,168,212,0.5)", "#fbcfe8", "rgba(236,72,153,0.3)"], - ["rgba(103,232,249,0.5)", "#a5f3fc", "rgba(6,182,212,0.3)"], - ["rgba(253,186,116,0.5)", "#fed7aa", "rgba(249,115,22,0.3)"], - ["rgba(167,243,208,0.5)", "#bbf7d0", "rgba(34,197,94,0.3)"], +// Accent glow colours for the hover ring — cycled by index +const GLOWS: string[] = [ + "rgba(59,130,246,0.5)", + "rgba(168,85,247,0.5)", + "rgba(16,185,129,0.5)", + "rgba(245,158,11,0.5)", + "rgba(236,72,153,0.5)", + "rgba(6,182,212,0.5)", + "rgba(249,115,22,0.5)", + "rgba(34,197,94,0.5)", ]; function pseudoRandom(seed: number): number { @@ -19,24 +20,17 @@ function pseudoRandom(seed: number): number { return x - Math.floor(x); } -function getWeight(count: number, maxCount: number): 1 | 2 | 3 | 4 | 5 { - if (maxCount === 0) return 1; +// Map cluster size to a tile size bucket (px) +function getTileSize(count: number, maxCount: number): number { + if (maxCount === 0) return 72; const ratio = count / maxCount; - if (ratio > 0.75) return 5; - if (ratio > 0.45) return 4; - if (ratio > 0.22) return 3; - if (ratio > 0.08) return 2; - return 1; + if (ratio > 0.75) return 160; + if (ratio > 0.45) return 128; + if (ratio > 0.22) return 104; + if (ratio > 0.08) return 88; + return 72; } -const FONT_SIZE: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 11, 2: 14, 3: 19, 4: 30, 5: 46 }; -const FONT_WEIGHT: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 400, 2: 400, 3: 500, 4: 700, 5: 800 }; -const LETTER_SPACING: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 0.8, 2: 0.4, 3: 0, 4: -0.5, 5: -1.5 }; -const PADDING: Record<1 | 2 | 3 | 4 | 5, string> = { - 1: "3px 8px", 2: "4px 10px", 3: "5px 13px", 4: "8px 18px", 5: "10px 22px", -}; -const MAX_ROTATION: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 14, 2: 11, 3: 7, 4: 3, 5: 0 }; - function TagButton({ entry, index, @@ -46,66 +40,108 @@ function TagButton({ entry: TagCloudEntry; index: number; maxCount: number; - onSearch: (label: string) => void; + onSearch: (imageId: number) => void; }) { - const weight = getWeight(entry.count, maxCount); - const accentIndex = (index * 3 + weight) % ACCENTS.length; - const [restColor, hoverColor, glowColor] = ACCENTS[accentIndex]; - const rotation = (pseudoRandom(index * 7) - 0.5) * 2 * MAX_ROTATION[weight]; + const size = getTileSize(entry.count, maxCount); + const glow = GLOWS[index % GLOWS.length]; - const mt = Math.floor(pseudoRandom(index * 3) * 14) + 3; - const mr = Math.floor(pseudoRandom(index * 5) * 20) + 6; - const mb = Math.floor(pseudoRandom(index * 11) * 14) + 3; - const ml = Math.floor(pseudoRandom(index * 13) * 20) + 6; + // Small random rotation for organic feel — larger tiles stay flatter + const maxRot = size >= 128 ? 0 : size >= 104 ? 3 : size >= 88 ? 6 : 10; + const rotation = (pseudoRandom(index * 7) - 0.5) * 2 * maxRot; + + const mt = Math.floor(pseudoRandom(index * 3) * 10) + 4; + const mr = Math.floor(pseudoRandom(index * 5) * 12) + 4; + const mb = Math.floor(pseudoRandom(index * 11) * 10) + 4; + const ml = Math.floor(pseudoRandom(index * 13) * 12) + 4; + + const src = entry.thumbnail_path ? convertFileSrc(entry.thumbnail_path) : null; return ( onSearch(entry.representative_image_id)} + title={`${entry.count} similar ${entry.count === 1 ? "photo" : "photos"}`} style={{ - fontSize: FONT_SIZE[weight], - fontWeight: FONT_WEIGHT[weight], - letterSpacing: LETTER_SPACING[weight], - padding: PADDING[weight], + width: size, + height: size, margin: `${mt}px ${mr}px ${mb}px ${ml}px`, - color: restColor, - borderRadius: 10, - border: "1px solid transparent", - background: "transparent", + borderRadius: 12, + border: "2px solid rgba(255,255,255,0.08)", + background: "rgba(255,255,255,0.04)", cursor: "pointer", - userSelect: "none", - transition: "color 0.15s, text-shadow 0.15s, background 0.15s, border-color 0.15s", + padding: 0, + overflow: "hidden", + position: "relative", + flexShrink: 0, + boxShadow: "none", + transition: "border-color 0.15s, box-shadow 0.15s", }} onMouseEnter={(e) => { const el = e.currentTarget; - el.style.color = hoverColor; - el.style.textShadow = `0 0 20px ${glowColor}, 0 0 40px ${glowColor}`; - el.style.background = glowColor.replace("0.3", "0.1"); - el.style.borderColor = glowColor.replace("0.3", "0.25"); + el.style.borderColor = glow; + el.style.boxShadow = `0 0 16px ${glow}, 0 0 32px ${glow.replace("0.5", "0.25")}`; }} onMouseLeave={(e) => { const el = e.currentTarget; - el.style.color = restColor; - el.style.textShadow = "none"; - el.style.background = "transparent"; - el.style.borderColor = "transparent"; + el.style.borderColor = "rgba(255,255,255,0.08)"; + el.style.boxShadow = "none"; }} - onClick={() => onSearch(entry.label)} - title={`${entry.count} matching ${entry.count === 1 ? "photo" : "photos"}`} > - {entry.label} + {src ? ( + + ) : ( + // Fallback placeholder when no thumbnail exists yet +
+ )} + + {/* Count badge — bottom-right corner */} +
+ {entry.count} +
); } @@ -121,9 +157,10 @@ export function TagCloud() { void loadTagCloud(); }, [selectedFolderId]); - const maxCount = tagCloudEntries.length > 0 - ? Math.max(...tagCloudEntries.map((e) => e.count)) - : 1; + const maxCount = + tagCloudEntries.length > 0 + ? Math.max(...tagCloudEntries.map((e) => e.count)) + : 1; return (
@@ -138,7 +175,7 @@ export function TagCloud() { Explore your library

- Topics found in your photos — sized by how many match + Visual clusters from your photos — sized by how many match

@@ -152,10 +189,22 @@ export function TagCloud() { animate={{ rotate: 360 }} transition={{ duration: 1, repeat: Infinity, ease: "linear" }} > - - + + -

Analysing your library with CLIP…

+

Clustering your library…

)} @@ -163,18 +212,18 @@ export function TagCloud() { {!tagCloudLoading && tagCloudEntries.length === 0 && (

- No embeddings yet. Add a folder and wait for the embedding worker to finish, - then come back here. + No embeddings yet. Add a folder and wait for the embedding worker to + finish, then come back here.

)} - {/* Cloud */} + {/* Cluster grid */} {!tagCloudLoading && tagCloudEntries.length > 0 && (
{tagCloudEntries.map((entry, index) => ( 0 && (

- Ranked by visual similarity · CLIP ViT-B/32 + Grouped by visual similarity · CLIP ViT-B/32

)}
diff --git a/src/components/Toolbar.tsx b/src/components/Toolbar.tsx index a34e272..88cedee 100644 --- a/src/components/Toolbar.tsx +++ b/src/components/Toolbar.tsx @@ -91,16 +91,22 @@ function FilterPill({ label, active, onClick, + variant = "default", }: { label: string; active: boolean; onClick: () => void; + variant?: "default" | "amber"; }) { + const activeClass = + variant === "amber" + ? "bg-amber-500/15 text-amber-300 border border-amber-500/30" + : "bg-white/10 text-white"; return (
); diff --git a/src/store.ts b/src/store.ts index 2f18bc7..47ba35d 100644 --- a/src/store.ts +++ b/src/store.ts @@ -75,8 +75,9 @@ export interface ThumbnailBatch { export type ActiveView = "gallery" | "explore"; export interface TagCloudEntry { - label: string; count: number; + representative_image_id: number; + thumbnail_path: string | null; } export type SortOrder = @@ -101,12 +102,14 @@ interface GalleryState { sort: SortOrder; mediaFilter: MediaFilter; favoritesOnly: boolean; + failedEmbeddingsOnly: boolean; zoomPreset: ZoomPreset; selectedImage: ImageRecord | null; collectionTitle: string | null; activeView: ActiveView; tagCloudEntries: TagCloudEntry[]; tagCloudLoading: boolean; + tagCloudFolderId: number | null | undefined; // undefined = never loaded indexingProgress: Record; mediaJobProgress: Record; cacheDir: string; @@ -126,12 +129,13 @@ interface GalleryState { setSort: (sort: SortOrder) => void; setMediaFilter: (filter: MediaFilter) => void; setFavoritesOnly: (favoritesOnly: boolean) => void; + setFailedEmbeddingsOnly: (failedEmbeddingsOnly: boolean) => void; setZoomPreset: (zoomPreset: ZoomPreset) => void; openImage: (image: ImageRecord) => void; closeImage: () => void; setView: (view: ActiveView) => void; loadTagCloud: () => Promise; - searchByTag: (tag: string) => void; + searchByTag: (imageId: number) => void; loadSimilarImages: (imageId: number) => Promise; retryFailedEmbeddings: (folderId: number) => Promise; updateImageDetails: (imageId: number, updates: { favorite?: boolean; rating?: number }) => Promise; @@ -161,12 +165,14 @@ function matchesFilters( selectedFolderId: number | null, mediaFilter: MediaFilter, favoritesOnly: boolean, + failedEmbeddingsOnly: boolean, search: string, ): boolean { const matchesFolder = selectedFolderId === null || image.folder_id === selectedFolderId; const matchesMedia = mediaFilter === "all" || image.media_kind === mediaFilter; const matchesFavorite = !favoritesOnly || image.favorite; - return matchesFolder && matchesMedia && matchesFavorite && matchesSearch(image, search); + const matchesFailedEmbedding = !failedEmbeddingsOnly || image.embedding_status === "failed"; + return matchesFolder && matchesMedia && matchesFavorite && matchesFailedEmbedding && matchesSearch(image, search); } function compareNullableNumber(a: number | null, b: number | null): number { @@ -265,12 +271,14 @@ export const useGalleryStore = create((set, get) => ({ sort: "date_desc", mediaFilter: "all", favoritesOnly: false, + failedEmbeddingsOnly: false, zoomPreset: "comfortable", selectedImage: null, collectionTitle: null, activeView: "gallery", tagCloudEntries: [], tagCloudLoading: false, + tagCloudFolderId: undefined, indexingProgress: {}, mediaJobProgress: {}, cacheDir: "", @@ -301,6 +309,8 @@ export const useGalleryStore = create((set, get) => ({ const { selectedFolderId, loadFolders, loadImages, loadBackgroundJobProgress } = get(); await loadFolders(); await loadBackgroundJobProgress(); + // Invalidate tag cloud cache since library content changed + set({ tagCloudFolderId: undefined, tagCloudEntries: [] }); if (selectedFolderId === folderId) { set({ selectedFolderId: null }); await loadImages(true); @@ -311,16 +321,18 @@ export const useGalleryStore = create((set, get) => ({ const { loadFolders, loadBackgroundJobProgress } = get(); await invoke("reindex_folder", { folderId }); await loadFolders(); + // Invalidate tag cloud cache since embeddings will be regenerated + set({ tagCloudFolderId: undefined, tagCloudEntries: [] }); await loadBackgroundJobProgress(); }, selectFolder: (folderId) => { - set({ selectedFolderId: folderId, images: [], loadedCount: 0, collectionTitle: null, activeView: "gallery" }); + set({ selectedFolderId: folderId, images: [], loadedCount: 0, collectionTitle: null, activeView: "gallery", failedEmbeddingsOnly: false }); void get().loadImages(true); }, loadImages: async (reset = false) => { - const { selectedFolderId, search, searchMode, sort, loadedCount, mediaFilter, favoritesOnly } = get(); + const { selectedFolderId, search, searchMode, sort, loadedCount, mediaFilter, favoritesOnly, failedEmbeddingsOnly } = get(); set({ loadingImages: true }); try { @@ -357,6 +369,7 @@ export const useGalleryStore = create((set, get) => ({ search: search || null, media_kind: mediaFilter === "all" ? null : mediaFilter, favorites_only: favoritesOnly, + embedding_failed_only: failedEmbeddingsOnly, sort, offset, limit: PAGE_SIZE, @@ -417,6 +430,11 @@ export const useGalleryStore = create((set, get) => ({ void get().loadImages(true); }, + setFailedEmbeddingsOnly: (failedEmbeddingsOnly) => { + set({ failedEmbeddingsOnly, images: [], loadedCount: 0, collectionTitle: null }); + void get().loadImages(true); + }, + setZoomPreset: (zoomPreset) => set({ zoomPreset }), openImage: (image) => set({ selectedImage: image }), @@ -425,8 +443,12 @@ export const useGalleryStore = create((set, get) => ({ setView: (activeView) => set({ activeView }), loadTagCloud: async () => { - const { selectedFolderId } = get(); - set({ tagCloudLoading: true }); + const { selectedFolderId, tagCloudFolderId, tagCloudLoading } = get(); + // Skip if already loaded for this folder and not currently loading + if (!tagCloudLoading && tagCloudFolderId !== undefined && tagCloudFolderId === selectedFolderId) { + return; + } + set({ tagCloudLoading: true, tagCloudFolderId: selectedFolderId }); try { const entries = await invoke("get_tag_cloud", { folderId: selectedFolderId, @@ -438,19 +460,13 @@ export const useGalleryStore = create((set, get) => ({ } }, - searchByTag: (tag) => { - set({ - activeView: "gallery", - search: tag, - searchMode: "semantic", - images: [], - loadedCount: 0, - collectionTitle: `Exploring: ${tag}`, - }); - void get().loadImages(true); + searchByTag: (imageId) => { + set({ activeView: "gallery", images: [], loadedCount: 0, loadingImages: true, collectionTitle: "Similar Images" }); + void get().loadSimilarImages(imageId); }, loadSimilarImages: async (imageId) => { + set({ images: [], loadedCount: 0, loadingImages: true, collectionTitle: "Similar Images" }); const images = await invoke("find_similar_images", { params: { image_id: imageId, limit: PAGE_SIZE }, }); @@ -530,6 +546,7 @@ export const useGalleryStore = create((set, get) => ({ state.selectedFolderId, state.mediaFilter, state.favoritesOnly, + state.failedEmbeddingsOnly, state.search, ), ); @@ -559,6 +576,7 @@ export const useGalleryStore = create((set, get) => ({ state.selectedFolderId, state.mediaFilter, state.favoritesOnly, + state.failedEmbeddingsOnly, state.search, ), );