diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index c810d72..b9daa8a 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -39,8 +39,8 @@ log = "0.4" ffmpeg-sidecar = "2.5.0" xxhash-rust = { version = "0.8", features = ["xxh3"] } sysinfo = "0.38.4" -candle-core = "0.10.2" -candle-nn = "0.10.2" -candle-transformers = "0.10.2" +candle-core = { version = "0.10.2", features = ["cuda"] } +candle-nn = { version = "0.10.2", features = ["cuda"] } +candle-transformers = { version = "0.10.2", features = ["cuda"] } hf-hub = { version = "0.5.0", default-features = false, features = ["ureq", "native-tls"] } tokenizers = "0.22.1" diff --git a/src-tauri/build.rs b/src-tauri/build.rs index d860e1e..f6be0ab 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -1,3 +1,28 @@ fn main() { - tauri_build::build() + tauri_build::build(); + + // When the candle-cuda feature is enabled, verify the CUDA Toolkit is present + // so the developer gets a clear message rather than an obscure cudarc/cc error. + if std::env::var("CARGO_FEATURE_CANDLE_CUDA").is_ok() { + let cuda_path = std::env::var("CUDA_PATH"); // Set by the CUDA Toolkit installer on Windows + let nvcc_found = std::process::Command::new("nvcc") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + + if cuda_path.is_ok() || nvcc_found { + if let Ok(path) = &cuda_path { + println!("cargo:warning=CUDA Toolkit found at {path} — GPU acceleration enabled."); + } else { + println!("cargo:warning=CUDA Toolkit (nvcc) found in PATH — GPU acceleration enabled."); + } + } else { + println!("cargo:warning=━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"); + println!("cargo:warning= candle-cuda feature is enabled but no CUDA Toolkit found."); + println!("cargo:warning= Install CUDA Toolkit from https://developer.nvidia.com/cuda-downloads"); + println!("cargo:warning= Or build without GPU support: cargo build --no-default-features"); + println!("cargo:warning=━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"); + } + } } diff --git a/src-tauri/data/vocabulary.txt b/src-tauri/data/vocabulary.txt new file mode 100644 index 0000000..d08aec4 --- /dev/null +++ b/src-tauri/data/vocabulary.txt @@ -0,0 +1,535 @@ +# Visual concept vocabulary for tag cloud labelling +# Each non-empty, non-comment line is embedded as "a photo of " +# and used to name clusters found in your library. +# Add your own words — changes take effect after restarting the app. + +# People +person +man +woman +boy +girl +baby +child +teenager +elderly person +couple +family +friends +crowd +portrait +face +smile +athlete +soldier +chef +bride +groom +wedding +monk +dancer +musician +street performer +protester + +# Animals +cat +dog +bird +horse +cow +pig +sheep +chicken +duck +rabbit +fox +wolf +bear +deer +elk +moose +elephant +lion +tiger +leopard +cheetah +giraffe +zebra +monkey +gorilla +chimpanzee +kangaroo +koala +panda +polar bear +penguin +flamingo +parrot +owl +eagle +hawk +falcon +hummingbird +peacock +crane +seagull +pelican +snake +lizard +iguana +crocodile +turtle +frog +fish +shark +whale +dolphin +seal +otter +octopus +jellyfish +crab +lobster +bee +butterfly +dragonfly +ant +spider +ladybug +beetle +squirrel +raccoon +bat +hedgehog + +# Nature — water & coast +ocean +sea +beach +waves +coastline +cliff +tide pool +lake +river +stream +creek +waterfall +swamp +marsh +wetland +pond +reservoir +fjord +lagoon +bay +harbour +reef +coral +iceberg +glacier + +# Nature — land & terrain +mountain +mountains +valley +canyon +gorge +cliff face +hill +plain +meadow +field +prairie +farmland +vineyard +orchard +forest +rainforest +jungle +woodland +mangrove +desert +sand dunes +salt flats +savanna +steppe +tundra +arctic +volcano +lava +cave +stalactite +hot spring + +# Nature — plants & vegetation +tree +trees +pine tree +oak tree +palm tree +bamboo +grass +wildflowers +flower +rose +sunflower +tulip +lavender +cherry blossom +dandelion +clover +fern +moss +lichen +mushroom +cactus +succulent +seaweed +lily pad +autumn leaves +fallen leaves +vineyard + +# Sky & weather +sky +blue sky +clouds +storm clouds +thunderstorm +lightning +rainbow +fog +mist +haze +smoke +sunrise +sunset +golden hour +blue hour +dusk +dawn +night sky +stars +milky way +moon +full moon +crescent moon +northern lights +aurora +snow +blizzard +rain +puddle +ice +frost +dew drops + +# Urban — streets & infrastructure +city +cityscape +skyline +skyscraper +building +apartment block +neighbourhood +street +alley +road +highway +motorway +crossroads +traffic +car park +bridge +overpass +tunnel +fence +wall +power lines +crane +construction site +fire hydrant +street lamp +billboard +bus stop +telephone booth +postbox + +# Urban — buildings & architecture +architecture +church +cathedral +mosque +temple +synagogue +pagoda +monastery +castle +palace +fortress +ruins +museum +library +stadium +arena +theatre +cinema +school +university +hospital +factory +warehouse +barn +farmhouse +cottage +villa +lighthouse +windmill +water tower +dam + +# Urban — interior spaces +bedroom +living room +kitchen +bathroom +dining room +office +corridor +staircase +attic +basement +garage +studio +classroom +laboratory +workshop +greenhouse +gymnasium + +# Food & drink +food +meal +bread +pastry +cake +pie +cookie +pizza +pasta +noodles +rice +soup +salad +sandwich +burger +hot dog +taco +sushi +sashimi +seafood +steak +chicken +barbecue +breakfast +brunch +fruit +vegetables +herbs +spices +cheese +eggs +coffee +espresso +tea +wine +beer +cocktail +juice +ice cream +dessert +chocolate +candy +market stall + +# Transport +car +sports car +vintage car +truck +van +bus +tram +motorcycle +scooter +bicycle +skateboard +train +metro +subway +tram +steam train +aeroplane +helicopter +hot air balloon +drone +speedboat +sailing boat +yacht +cruise ship +container ship +submarine +kayak +canoe +surfboard + +# Activities & sport +running +jogging +walking +hiking +cycling +swimming +surfing +diving +snorkelling +sailing +rowing +kayaking +climbing +abseiling +skiing +snowboarding +ice skating +skateboarding +horse riding +football +basketball +tennis +golf +baseball +cricket +rugby +boxing +wrestling +martial arts +gymnastics +yoga +meditation +stretching +weight training +dancing +ballet +breakdancing +cheerleading +fishing + +# Arts, culture & leisure +painting +drawing +sketch +sculpture +mural +graffiti +street art +calligraphy +pottery +glasswork +photography +concert +live music +festival +carnival +parade +fireworks +ceremony +graduation +protest +market +fair +circus +magic show +puppet show +gaming +reading +writing +chess +board game + +# Mood, lighting & photographic style +silhouette +reflection +shadow +long exposure +light trails +bokeh +macro photography +close-up +aerial view +bird's eye view +underwater +black and white +vintage +film grain +neon lights +candlelight +backlight +rim light +dramatic lighting +soft light +misty +moody +minimalist +symmetry +geometry +abstract +pattern +texture +repetition +colour splash +double exposure + +# Miscellaneous visual subjects +window +door +gate +staircase +mirror +candle +clock +book +map +flag +trophy +statue +monument +graveyard +cemetery +prison +lighthouse +windmill +water mill +solar panels +satellite dish +telescope +microscope +laboratory equipment +musical instrument +guitar +piano +violin +drums +trumpet +painting on wall +cinema screen +television +old photograph +typewriter +sewing +knitting +gardening +woodworking +welding +pottery wheel diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index cc57607..65de8fc 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -1,10 +1,10 @@ use crate::db::{self, DbPool, Folder, FolderJobProgress, ImageRecord}; -use crate::embedder::ClipImageEmbedder; +use crate::embedder; use crate::indexer; use crate::vector; use serde::{Deserialize, Serialize}; use std::path::PathBuf; -use tauri::{AppHandle, State}; +use tauri::{AppHandle, Manager, State}; pub type DbState = DbPool; @@ -204,8 +204,7 @@ pub async fn semantic_search_images( db: State<'_, DbState>, params: SemanticSearchParams, ) -> Result, String> { - let embedder = ClipImageEmbedder::new().map_err(|e| e.to_string())?; - let embedding = embedder.embed_text(¶ms.query).map_err(|e| e.to_string())?; + let embedding = embedder::embed_text_query(¶ms.query).map_err(|e| e.to_string())?; let conn = db.get().map_err(|e| e.to_string())?; let limit = params.limit.unwrap_or(64); @@ -225,6 +224,160 @@ pub async fn semantic_search_images( Ok(images) } +#[derive(Serialize)] +pub struct TagCloudEntry { + pub label: String, + pub count: usize, +} + +/// Clusters the library's image embeddings with k-means, then labels each cluster by +/// finding the closest word in the vocabulary. The vocabulary is loaded from +/// `{app_data_dir}/vocabulary.txt` if present, otherwise from the bundled default. +/// Vocabulary embeddings are cached to disk — only recomputed when the vocabulary changes. +#[tauri::command] +pub async fn get_tag_cloud( + app: AppHandle, + db: State<'_, DbState>, + folder_id: Option, +) -> Result, String> { + let app_data_dir = app.path().app_data_dir().map_err(|e| e.to_string())?; + + let image_embeddings = { + let conn = db.get().map_err(|e| e.to_string())?; + vector::get_all_image_embeddings(&conn, folder_id).map_err(|e| e.to_string())? + }; + + let n = image_embeddings.len(); + if n < 5 { + return Ok(vec![]); + } + + // Load vocabulary (custom file > bundled default) + let vocab = embedder::load_vocabulary(&app_data_dir); + + // Embed vocabulary — disk-cached, only recomputes when vocabulary changes + let vocab_refs: Vec<&str> = vocab.iter().map(|s| s.as_str()).collect(); + let vocab_embeddings = embedder::embed_vocab_cached(&vocab, &app_data_dir) + .map_err(|e| e.to_string())?; + + // Choose k proportional to library size, capped at 30 + let k = (n / 20).clamp(5, 30); + + // Cluster image embeddings + let (centroids, cluster_counts) = kmeans_cosine(&image_embeddings, k, 40); + + // Label each cluster with the nearest vocabulary word + let mut entries: Vec = Vec::new(); + let mut used_labels = std::collections::HashSet::new(); + + let mut order: Vec = (0..k).collect(); + order.sort_unstable_by(|&a, &b| cluster_counts[b].cmp(&cluster_counts[a])); + + for ci in order { + let count = cluster_counts[ci]; + if count == 0 { continue; } + + let centroid = ¢roids[ci]; + let label = vocab_refs + .iter() + .zip(vocab_embeddings.iter()) + .map(|(&word, emb)| { + let sim: f32 = centroid.iter().zip(emb.iter()).map(|(a, b)| a * b).sum(); + (word, sim) + }) + .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)) + .map(|(word, _)| word.to_string()) + .unwrap_or_else(|| "other".to_string()); + + if used_labels.insert(label.clone()) { + entries.push(TagCloudEntry { label, count }); + } + } + + entries.sort_unstable_by(|a, b| b.count.cmp(&a.count)); + Ok(entries) +} + +// ── k-means with cosine similarity (all vectors assumed to be unit-normalized) ── + +fn dot(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b.iter()).map(|(x, y)| x * y).sum() +} + +fn normalize(v: &mut Vec) { + let norm: f32 = v.iter().map(|x| x * x).sum::().sqrt(); + if norm > 1e-10 { + v.iter_mut().for_each(|x| *x /= norm); + } +} + +fn kmeans_cosine( + points: &[Vec], + k: usize, + max_iter: usize, +) -> (Vec>, Vec) { + let n = points.len(); + let dim = points[0].len(); + + // Deterministic k-means++ init: spread centroids as far apart as possible + let mut centroids: Vec> = Vec::with_capacity(k); + centroids.push(points[n / 2].clone()); + for _ in 1..k { + let next = points + .iter() + .map(|p| { + let best_sim = centroids.iter().map(|c| dot(p, c)).fold(f32::NEG_INFINITY, f32::max); + 1.0 - best_sim // distance = 1 - cosine_similarity + }) + .enumerate() + .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) + .map(|(i, _)| i) + .unwrap_or(0); + centroids.push(points[next].clone()); + } + + let mut assignments = vec![0usize; n]; + + for _ in 0..max_iter { + // Assignment step + let mut changed = false; + for (i, p) in points.iter().enumerate() { + let best = centroids + .iter() + .enumerate() + .map(|(j, c)| (j, dot(p, c))) + .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)) + .map(|(j, _)| j) + .unwrap_or(0); + if assignments[i] != best { + assignments[i] = best; + changed = true; + } + } + if !changed { break; } + + // Update step: mean of assigned points, then normalize + let mut sums = vec![vec![0.0f32; dim]; k]; + let mut counts = vec![0usize; k]; + for (p, &c) in points.iter().zip(assignments.iter()) { + sums[c].iter_mut().zip(p.iter()).for_each(|(s, v)| *s += v); + counts[c] += 1; + } + for (centroid, (sum, &count)) in centroids.iter_mut().zip(sums.iter_mut().zip(counts.iter())) { + if count > 0 { + sum.iter_mut().for_each(|v| *v /= count as f32); + normalize(sum); + *centroid = sum.clone(); + } + } + } + + let mut counts = vec![0usize; k]; + for &a in &assignments { counts[a] += 1; } + + (centroids, counts) +} + #[derive(Serialize)] pub struct WorkerStates { pub thumbnail_paused: bool, diff --git a/src-tauri/src/embedder.rs b/src-tauri/src/embedder.rs index f5315de..ac0c104 100644 --- a/src-tauri/src/embedder.rs +++ b/src-tauri/src/embedder.rs @@ -1,11 +1,180 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use candle_core::{DType, Device, Tensor}; use candle_nn::VarBuilder; use candle_transformers::models::clip::{self, ClipModel}; use hf_hub::{api::sync::Api, Repo, RepoType}; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock}; use tokenizers::Tokenizer; +static TEXT_SEARCH_EMBEDDER: OnceLock>> = OnceLock::new(); +/// In-process cache so the disk file is only read/written once per session. +static VOCAB_EMBED_CACHE: OnceLock>>>> = OnceLock::new(); + +/// Default vocabulary bundled with the binary. +pub const DEFAULT_VOCABULARY: &str = include_str!("../data/vocabulary.txt"); + +/// Embed a text query using a lazily-initialized, cached CLIP embedder. +pub fn embed_text_query(query: &str) -> Result> { + with_text_embedder(|e| e.embed_text(query)) +} + +/// Parse vocabulary text: strip comment lines (starting with `#`) and blank lines. +pub fn parse_vocabulary(text: &str) -> Vec { + text.lines() + .map(|l| l.trim()) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .map(|l| l.to_string()) + .collect() +} + +/// Load the vocabulary from `{cache_dir}/vocabulary.txt` if it exists, +/// otherwise fall back to the binary-bundled default. +pub fn load_vocabulary(cache_dir: &Path) -> Vec { + let custom_path = cache_dir.join("vocabulary.txt"); + if custom_path.exists() { + if let Ok(text) = std::fs::read_to_string(&custom_path) { + let words = parse_vocabulary(&text); + if !words.is_empty() { + println!("Using custom vocabulary from {:?} ({} words)", custom_path, words.len()); + return words; + } + } + } + parse_vocabulary(DEFAULT_VOCABULARY) +} + +/// Embed the vocabulary, using a disk cache so CLIP is only called when the vocabulary +/// actually changes. Cache file: `{cache_dir}/vocab_embeddings.bin`. +/// +/// Format: `[u64 hash][u32 n_words][u32 dim][(n_words × dim) × f32 LE]` +pub fn embed_vocab_cached(vocab: &[String], cache_dir: &Path) -> Result>> { + // In-process cache hit + { + let guard = VOCAB_EMBED_CACHE + .get_or_init(|| Mutex::new(None)) + .lock() + .map_err(|_| anyhow::anyhow!("Vocab cache lock poisoned"))?; + if let Some(cached) = guard.as_ref() { + return Ok(cached.clone()); + } + } + + let vocab_hash = fnv_hash(vocab); + let disk_path = cache_dir.join("vocab_embeddings.bin"); + + // Try loading from disk + if let Ok(embeddings) = load_disk_cache(&disk_path, vocab_hash) { + let mut guard = VOCAB_EMBED_CACHE + .get_or_init(|| Mutex::new(None)) + .lock() + .map_err(|_| anyhow::anyhow!("Vocab cache lock poisoned"))?; + *guard = Some(embeddings.clone()); + println!("Vocabulary embeddings loaded from disk cache ({} words).", embeddings.len()); + return Ok(embeddings); + } + + // Compute embeddings + println!("Computing vocabulary embeddings ({} words) — this is cached after the first run.", vocab.len()); + let prompts: Vec = vocab.iter().map(|w| format!("a photo of {}", w)).collect(); + let prompt_refs: Vec<&str> = prompts.iter().map(|s| s.as_str()).collect(); + + let embeddings = with_text_embedder(|e| { + let mut all = Vec::with_capacity(vocab.len()); + for chunk in prompt_refs.chunks(64) { + all.extend(e.embed_texts_batch(chunk)?); + } + Ok(all) + })?; + + // Save to disk + if let Err(e) = save_disk_cache(&disk_path, vocab_hash, &embeddings) { + eprintln!("Warning: could not write vocab cache: {e}"); + } + + let mut guard = VOCAB_EMBED_CACHE + .get_or_init(|| Mutex::new(None)) + .lock() + .map_err(|_| anyhow::anyhow!("Vocab cache lock poisoned"))?; + *guard = Some(embeddings.clone()); + Ok(embeddings) +} + +fn fnv_hash(words: &[String]) -> u64 { + let mut h: u64 = 0xcbf29ce484222325; + for w in words { + for b in w.bytes() { + h = h.wrapping_mul(0x100000001b3) ^ (b as u64); + } + h = h.wrapping_mul(0x100000001b3) ^ 0x0A; // newline separator + } + h +} + +fn load_disk_cache(path: &Path, expected_hash: u64) -> Result>> { + let mut f = std::fs::File::open(path).context("no cache file")?; + let mut buf = Vec::new(); + f.read_to_end(&mut buf)?; + + if buf.len() < 16 { + anyhow::bail!("cache too short"); + } + + let hash = u64::from_le_bytes(buf[0..8].try_into()?); + if hash != expected_hash { + anyhow::bail!("vocab hash mismatch — recomputing"); + } + let n = u32::from_le_bytes(buf[8..12].try_into()?) as usize; + let dim = u32::from_le_bytes(buf[12..16].try_into()?) as usize; + + let expected_len = 16 + n * dim * 4; + if buf.len() != expected_len { + anyhow::bail!("cache size mismatch"); + } + + let mut embeddings = Vec::with_capacity(n); + let data = &buf[16..]; + for i in 0..n { + let start = i * dim * 4; + let emb: Vec = data[start..start + dim * 4] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + embeddings.push(emb); + } + Ok(embeddings) +} + +fn save_disk_cache(path: &Path, hash: u64, embeddings: &[Vec]) -> Result<()> { + let n = embeddings.len(); + let dim = embeddings.first().map(|e| e.len()).unwrap_or(0); + + let mut buf = Vec::with_capacity(16 + n * dim * 4); + buf.extend_from_slice(&hash.to_le_bytes()); + buf.extend_from_slice(&(n as u32).to_le_bytes()); + buf.extend_from_slice(&(dim as u32).to_le_bytes()); + for emb in embeddings { + for &v in emb { + buf.extend_from_slice(&v.to_le_bytes()); + } + } + + std::fs::write(path, buf)?; + println!("Vocabulary embeddings cached to disk ({n} words, {dim} dims)."); + Ok(()) +} + +fn with_text_embedder(f: impl FnOnce(&ClipImageEmbedder) -> Result) -> Result { + let lock = TEXT_SEARCH_EMBEDDER.get_or_init(|| Mutex::new(None)); + let mut guard = lock.lock().map_err(|_| anyhow::anyhow!("Text embedder lock poisoned"))?; + if guard.is_none() { + println!("Initializing CLIP text embedder..."); + *guard = Some(ClipImageEmbedder::new()?); + } + f(guard.as_ref().unwrap()) +} + pub struct ClipImageEmbedder { model: ClipModel, tokenizer: Tokenizer, @@ -67,36 +236,61 @@ impl ClipImageEmbedder { } pub fn embed_text(&self, query: &str) -> Result> { - let encoding = self - .tokenizer - .encode(query, true) - .map_err(anyhow::Error::msg)?; - let token_ids = encoding - .get_ids() + Ok(self.embed_texts_batch(&[query])?.remove(0)) + } + + /// Embed multiple text queries in a single CLIP forward pass. + /// All sequences are padded to the same length (max 77 tokens). + pub fn embed_texts_batch(&self, queries: &[&str]) -> Result>> { + if queries.is_empty() { + return Ok(vec![]); + } + + let encodings: Vec<_> = queries .iter() - .map(|token| *token as u32) - .collect::>(); - let input_ids = Tensor::new(vec![token_ids], &self.device)?; + .map(|q| self.tokenizer.encode(*q, true).map_err(anyhow::Error::msg)) + .collect::>()?; + + let max_len = encodings + .iter() + .map(|e| e.get_ids().len()) + .max() + .unwrap_or(10) + .min(77); + + let n = queries.len(); + let mut flat = vec![0u32; n * max_len]; + for (i, enc) in encodings.iter().enumerate() { + let ids = enc.get_ids(); + let len = ids.len().min(max_len); + for j in 0..len { + flat[i * max_len + j] = ids[j] as u32; + } + } + + let input_ids = Tensor::new(flat, &self.device)?.reshape((n, max_len))?; let features = self.model.get_text_features(&input_ids)?; let normalized = clip::div_l2_norm(&features)?; - Ok(normalized.flatten_all()?.to_vec1::()?) + + (0..n) + .map(|i| Ok(normalized.get(i)?.flatten_all()?.to_vec1::()?)) + .collect() } } fn resolve_device() -> Result { - #[cfg(feature = "candle-cuda")] - { - let cuda_device = Device::cuda_if_available(0)?; - if cuda_device.is_cuda() { - println!("CLIP embedder using CUDA device."); - return Ok(cuda_device); + match Device::cuda_if_available(0) { + Ok(device) if device.is_cuda() => { + println!("CLIP embedder: using CUDA GPU (device 0)."); + return Ok(device); + } + Ok(_) => { + println!("CLIP embedder: no compatible CUDA GPU found — using CPU."); + } + Err(e) => { + println!("CLIP embedder: CUDA init failed ({e}) — using CPU."); } } - - #[cfg(not(feature = "candle-cuda"))] - println!("CLIP embedder built without CUDA support."); - - println!("CLIP embedder using CPU device."); Ok(Device::Cpu) } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index a755288..de5b311 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -77,6 +77,7 @@ pub fn run() { commands::semantic_search_images, commands::set_worker_paused, commands::get_worker_states, + commands::get_tag_cloud, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/src-tauri/src/vector.rs b/src-tauri/src/vector.rs index 5679c83..befeae8 100644 --- a/src-tauri/src/vector.rs +++ b/src-tauri/src/vector.rs @@ -78,6 +78,41 @@ pub fn find_similar_image_ids(conn: &Connection, image_id: i64, limit: usize) -> Ok(ids) } +/// Returns all stored image embeddings, optionally filtered to one folder. +/// Each embedding is returned as a normalized f32 vector. +pub fn get_all_image_embeddings(conn: &Connection, folder_id: Option) -> Result>> { + let packed_rows: Vec> = match folder_id { + Some(fid) => { + let mut stmt = conn.prepare( + "SELECT embedding FROM image_vec + WHERE image_id IN (SELECT id FROM images WHERE folder_id = ?1)", + )?; + let rows: Vec> = stmt + .query_map([fid], |row| row.get::<_, Vec>(0))? + .filter_map(|r| r.ok()) + .collect(); + rows + } + None => { + let mut stmt = conn.prepare("SELECT embedding FROM image_vec")?; + let rows: Vec> = stmt + .query_map([], |row| row.get::<_, Vec>(0))? + .filter_map(|r| r.ok()) + .collect(); + rows + } + }; + + Ok(packed_rows.iter().map(|b| unpack_f32(b)).collect()) +} + +fn unpack_f32(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect() +} + pub fn search_image_ids_by_embedding( conn: &Connection, embedding: &[f32], diff --git a/src/App.tsx b/src/App.tsx index de6eb4b..cd5fa58 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -5,12 +5,14 @@ import { BackgroundTasks } from "./components/BackgroundTasks"; import { Toolbar } from "./components/Toolbar"; import { Gallery } from "./components/Gallery"; import { Lightbox } from "./components/Lightbox"; +import { TagCloud } from "./components/TagCloud"; export default function App() { const loadFolders = useGalleryStore((state) => state.loadFolders); const loadBackgroundJobProgress = useGalleryStore((state) => state.loadBackgroundJobProgress); const loadImages = useGalleryStore((state) => state.loadImages); const subscribeToProgress = useGalleryStore((state) => state.subscribeToProgress); + const activeView = useGalleryStore((state) => state.activeView); useEffect(() => { loadFolders().then(() => { @@ -30,9 +32,18 @@ export default function App() {
- - - + {activeView === "explore" ? ( + <> + + + + ) : ( + <> + + + + + )}
diff --git a/src/components/Sidebar.tsx b/src/components/Sidebar.tsx index daee831..4a4a33f 100644 --- a/src/components/Sidebar.tsx +++ b/src/components/Sidebar.tsx @@ -77,6 +77,8 @@ export function Sidebar() { const addFolder = useGalleryStore((state) => state.addFolder); const indexingProgress = useGalleryStore((state) => state.indexingProgress); const selectFolder = useGalleryStore((state) => state.selectFolder); + const activeView = useGalleryStore((state) => state.activeView); + const setView = useGalleryStore((state) => state.setView); const handleAddFolder = async () => { const selected = await open({ directory: true, multiple: false, title: "Select Media Folder" }); @@ -102,10 +104,10 @@ export function Sidebar() { {/* Nav */} -
+
- + All Media
+ +
setView("explore")} + > + + + + + Explore + +
{/* Section label */} diff --git a/src/components/TagCloud.tsx b/src/components/TagCloud.tsx new file mode 100644 index 0000000..5234622 --- /dev/null +++ b/src/components/TagCloud.tsx @@ -0,0 +1,194 @@ +import { useEffect } from "react"; +import { motion } from "framer-motion"; +import { useGalleryStore, TagCloudEntry } from "../store"; + +// Accent color pairs: [rest, hover, glow] +const ACCENTS: [string, string, string][] = [ + ["rgba(96,165,250,0.5)", "#93c5fd", "rgba(59,130,246,0.3)"], + ["rgba(192,132,252,0.5)", "#d8b4fe", "rgba(168,85,247,0.3)"], + ["rgba(52,211,153,0.5)", "#6ee7b7", "rgba(16,185,129,0.3)"], + ["rgba(251,191,36,0.5)", "#fcd34d", "rgba(245,158,11,0.3)"], + ["rgba(249,168,212,0.5)", "#fbcfe8", "rgba(236,72,153,0.3)"], + ["rgba(103,232,249,0.5)", "#a5f3fc", "rgba(6,182,212,0.3)"], + ["rgba(253,186,116,0.5)", "#fed7aa", "rgba(249,115,22,0.3)"], + ["rgba(167,243,208,0.5)", "#bbf7d0", "rgba(34,197,94,0.3)"], +]; + +function pseudoRandom(seed: number): number { + const x = Math.sin(seed + 1) * 10000; + return x - Math.floor(x); +} + +function getWeight(count: number, maxCount: number): 1 | 2 | 3 | 4 | 5 { + if (maxCount === 0) return 1; + const ratio = count / maxCount; + if (ratio > 0.75) return 5; + if (ratio > 0.45) return 4; + if (ratio > 0.22) return 3; + if (ratio > 0.08) return 2; + return 1; +} + +const FONT_SIZE: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 11, 2: 14, 3: 19, 4: 30, 5: 46 }; +const FONT_WEIGHT: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 400, 2: 400, 3: 500, 4: 700, 5: 800 }; +const LETTER_SPACING: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 0.8, 2: 0.4, 3: 0, 4: -0.5, 5: -1.5 }; +const PADDING: Record<1 | 2 | 3 | 4 | 5, string> = { + 1: "3px 8px", 2: "4px 10px", 3: "5px 13px", 4: "8px 18px", 5: "10px 22px", +}; +const MAX_ROTATION: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 14, 2: 11, 3: 7, 4: 3, 5: 0 }; + +function TagButton({ + entry, + index, + maxCount, + onSearch, +}: { + entry: TagCloudEntry; + index: number; + maxCount: number; + onSearch: (label: string) => void; +}) { + const weight = getWeight(entry.count, maxCount); + const accentIndex = (index * 3 + weight) % ACCENTS.length; + const [restColor, hoverColor, glowColor] = ACCENTS[accentIndex]; + const rotation = (pseudoRandom(index * 7) - 0.5) * 2 * MAX_ROTATION[weight]; + + const mt = Math.floor(pseudoRandom(index * 3) * 14) + 3; + const mr = Math.floor(pseudoRandom(index * 5) * 20) + 6; + const mb = Math.floor(pseudoRandom(index * 11) * 14) + 3; + const ml = Math.floor(pseudoRandom(index * 13) * 20) + 6; + + return ( + { + const el = e.currentTarget; + el.style.color = hoverColor; + el.style.textShadow = `0 0 20px ${glowColor}, 0 0 40px ${glowColor}`; + el.style.background = glowColor.replace("0.3", "0.1"); + el.style.borderColor = glowColor.replace("0.3", "0.25"); + }} + onMouseLeave={(e) => { + const el = e.currentTarget; + el.style.color = restColor; + el.style.textShadow = "none"; + el.style.background = "transparent"; + el.style.borderColor = "transparent"; + }} + onClick={() => onSearch(entry.label)} + title={`${entry.count} matching ${entry.count === 1 ? "photo" : "photos"}`} + > + {entry.label} + + ); +} + +export function TagCloud() { + const tagCloudEntries = useGalleryStore((state) => state.tagCloudEntries); + const tagCloudLoading = useGalleryStore((state) => state.tagCloudLoading); + const loadTagCloud = useGalleryStore((state) => state.loadTagCloud); + const searchByTag = useGalleryStore((state) => state.searchByTag); + const selectedFolderId = useGalleryStore((state) => state.selectedFolderId); + + useEffect(() => { + void loadTagCloud(); + }, [selectedFolderId]); + + const maxCount = tagCloudEntries.length > 0 + ? Math.max(...tagCloudEntries.map((e) => e.count)) + : 1; + + return ( +
+ {/* Header */} + +

+ Explore your library +

+

+ Topics found in your photos — sized by how many match +

+ + + {/* Loading */} + {tagCloudLoading && ( +
+ + + + +

Analysing your library with CLIP…

+
+ )} + + {/* Empty state */} + {!tagCloudLoading && tagCloudEntries.length === 0 && ( +
+

+ No embeddings yet. Add a folder and wait for the embedding worker to finish, + then come back here. +

+
+ )} + + {/* Cloud */} + {!tagCloudLoading && tagCloudEntries.length > 0 && ( +
+ {tagCloudEntries.map((entry, index) => ( + + ))} +
+ )} + + {!tagCloudLoading && tagCloudEntries.length > 0 && ( +

+ Ranked by visual similarity · CLIP ViT-B/32 +

+ )} +
+ ); +} diff --git a/src/store.ts b/src/store.ts index 610790c..2f18bc7 100644 --- a/src/store.ts +++ b/src/store.ts @@ -72,6 +72,13 @@ export interface ThumbnailBatch { images: ImageRecord[]; } +export type ActiveView = "gallery" | "explore"; + +export interface TagCloudEntry { + label: string; + count: number; +} + export type SortOrder = | "date_desc" | "date_asc" @@ -97,6 +104,9 @@ interface GalleryState { zoomPreset: ZoomPreset; selectedImage: ImageRecord | null; collectionTitle: string | null; + activeView: ActiveView; + tagCloudEntries: TagCloudEntry[]; + tagCloudLoading: boolean; indexingProgress: Record; mediaJobProgress: Record; cacheDir: string; @@ -119,6 +129,9 @@ interface GalleryState { setZoomPreset: (zoomPreset: ZoomPreset) => void; openImage: (image: ImageRecord) => void; closeImage: () => void; + setView: (view: ActiveView) => void; + loadTagCloud: () => Promise; + searchByTag: (tag: string) => void; loadSimilarImages: (imageId: number) => Promise; retryFailedEmbeddings: (folderId: number) => Promise; updateImageDetails: (imageId: number, updates: { favorite?: boolean; rating?: number }) => Promise; @@ -255,6 +268,9 @@ export const useGalleryStore = create((set, get) => ({ zoomPreset: "comfortable", selectedImage: null, collectionTitle: null, + activeView: "gallery", + tagCloudEntries: [], + tagCloudLoading: false, indexingProgress: {}, mediaJobProgress: {}, cacheDir: "", @@ -299,7 +315,7 @@ export const useGalleryStore = create((set, get) => ({ }, selectFolder: (folderId) => { - set({ selectedFolderId: folderId, images: [], loadedCount: 0, collectionTitle: null }); + set({ selectedFolderId: folderId, images: [], loadedCount: 0, collectionTitle: null, activeView: "gallery" }); void get().loadImages(true); }, @@ -406,6 +422,34 @@ export const useGalleryStore = create((set, get) => ({ openImage: (image) => set({ selectedImage: image }), closeImage: () => set({ selectedImage: null }), + setView: (activeView) => set({ activeView }), + + loadTagCloud: async () => { + const { selectedFolderId } = get(); + set({ tagCloudLoading: true }); + try { + const entries = await invoke("get_tag_cloud", { + folderId: selectedFolderId, + }); + set({ tagCloudEntries: entries, tagCloudLoading: false }); + } catch (error) { + console.error("Failed to load tag cloud:", error); + set({ tagCloudLoading: false }); + } + }, + + searchByTag: (tag) => { + set({ + activeView: "gallery", + search: tag, + searchMode: "semantic", + images: [], + loadedCount: 0, + collectionTitle: `Exploring: ${tag}`, + }); + void get().loadImages(true); + }, + loadSimilarImages: async (imageId) => { const images = await invoke("find_similar_images", { params: { image_id: imageId, limit: PAGE_SIZE },