Add tag cloud feature with k-means clustering and CUDA support

Introduces an Explore view with a tag cloud that clusters image embeddings
using cosine k-means and labels clusters via vocabulary-nearest-neighbour
CLIP matching. Vocabulary embeddings are disk-cached (FNV hash-keyed) to
avoid redundant inference. Enables CUDA for candle dependencies and adds a
build.rs check that surfaces a clear error when the toolkit is missing.
This commit is contained in:
2026-04-06 12:43:44 +01:00
parent 35c1dafd65
commit 6c3fd449ce
11 changed files with 1248 additions and 37 deletions
+3 -3
View File
@@ -39,8 +39,8 @@ log = "0.4"
ffmpeg-sidecar = "2.5.0" ffmpeg-sidecar = "2.5.0"
xxhash-rust = { version = "0.8", features = ["xxh3"] } xxhash-rust = { version = "0.8", features = ["xxh3"] }
sysinfo = "0.38.4" sysinfo = "0.38.4"
candle-core = "0.10.2" candle-core = { version = "0.10.2", features = ["cuda"] }
candle-nn = "0.10.2" candle-nn = { version = "0.10.2", features = ["cuda"] }
candle-transformers = "0.10.2" candle-transformers = { version = "0.10.2", features = ["cuda"] }
hf-hub = { version = "0.5.0", default-features = false, features = ["ureq", "native-tls"] } hf-hub = { version = "0.5.0", default-features = false, features = ["ureq", "native-tls"] }
tokenizers = "0.22.1" tokenizers = "0.22.1"
+26 -1
View File
@@ -1,3 +1,28 @@
fn main() { fn main() {
tauri_build::build() tauri_build::build();
// When the candle-cuda feature is enabled, verify the CUDA Toolkit is present
// so the developer gets a clear message rather than an obscure cudarc/cc error.
if std::env::var("CARGO_FEATURE_CANDLE_CUDA").is_ok() {
let cuda_path = std::env::var("CUDA_PATH"); // Set by the CUDA Toolkit installer on Windows
let nvcc_found = std::process::Command::new("nvcc")
.arg("--version")
.output()
.map(|o| o.status.success())
.unwrap_or(false);
if cuda_path.is_ok() || nvcc_found {
if let Ok(path) = &cuda_path {
println!("cargo:warning=CUDA Toolkit found at {path} — GPU acceleration enabled.");
} else {
println!("cargo:warning=CUDA Toolkit (nvcc) found in PATH — GPU acceleration enabled.");
}
} else {
println!("cargo:warning=━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!("cargo:warning= candle-cuda feature is enabled but no CUDA Toolkit found.");
println!("cargo:warning= Install CUDA Toolkit from https://developer.nvidia.com/cuda-downloads");
println!("cargo:warning= Or build without GPU support: cargo build --no-default-features");
println!("cargo:warning=━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
}
}
} }
+535
View File
@@ -0,0 +1,535 @@
# Visual concept vocabulary for tag cloud labelling
# Each non-empty, non-comment line is embedded as "a photo of <word>"
# and used to name clusters found in your library.
# Add your own words — changes take effect after restarting the app.
# People
person
man
woman
boy
girl
baby
child
teenager
elderly person
couple
family
friends
crowd
portrait
face
smile
athlete
soldier
chef
bride
groom
wedding
monk
dancer
musician
street performer
protester
# Animals
cat
dog
bird
horse
cow
pig
sheep
chicken
duck
rabbit
fox
wolf
bear
deer
elk
moose
elephant
lion
tiger
leopard
cheetah
giraffe
zebra
monkey
gorilla
chimpanzee
kangaroo
koala
panda
polar bear
penguin
flamingo
parrot
owl
eagle
hawk
falcon
hummingbird
peacock
crane
seagull
pelican
snake
lizard
iguana
crocodile
turtle
frog
fish
shark
whale
dolphin
seal
otter
octopus
jellyfish
crab
lobster
bee
butterfly
dragonfly
ant
spider
ladybug
beetle
squirrel
raccoon
bat
hedgehog
# Nature — water & coast
ocean
sea
beach
waves
coastline
cliff
tide pool
lake
river
stream
creek
waterfall
swamp
marsh
wetland
pond
reservoir
fjord
lagoon
bay
harbour
reef
coral
iceberg
glacier
# Nature — land & terrain
mountain
mountains
valley
canyon
gorge
cliff face
hill
plain
meadow
field
prairie
farmland
vineyard
orchard
forest
rainforest
jungle
woodland
mangrove
desert
sand dunes
salt flats
savanna
steppe
tundra
arctic
volcano
lava
cave
stalactite
hot spring
# Nature — plants & vegetation
tree
trees
pine tree
oak tree
palm tree
bamboo
grass
wildflowers
flower
rose
sunflower
tulip
lavender
cherry blossom
dandelion
clover
fern
moss
lichen
mushroom
cactus
succulent
seaweed
lily pad
autumn leaves
fallen leaves
vineyard
# Sky & weather
sky
blue sky
clouds
storm clouds
thunderstorm
lightning
rainbow
fog
mist
haze
smoke
sunrise
sunset
golden hour
blue hour
dusk
dawn
night sky
stars
milky way
moon
full moon
crescent moon
northern lights
aurora
snow
blizzard
rain
puddle
ice
frost
dew drops
# Urban — streets & infrastructure
city
cityscape
skyline
skyscraper
building
apartment block
neighbourhood
street
alley
road
highway
motorway
crossroads
traffic
car park
bridge
overpass
tunnel
fence
wall
power lines
crane
construction site
fire hydrant
street lamp
billboard
bus stop
telephone booth
postbox
# Urban — buildings & architecture
architecture
church
cathedral
mosque
temple
synagogue
pagoda
monastery
castle
palace
fortress
ruins
museum
library
stadium
arena
theatre
cinema
school
university
hospital
factory
warehouse
barn
farmhouse
cottage
villa
lighthouse
windmill
water tower
dam
# Urban — interior spaces
bedroom
living room
kitchen
bathroom
dining room
office
corridor
staircase
attic
basement
garage
studio
classroom
laboratory
workshop
greenhouse
gymnasium
# Food & drink
food
meal
bread
pastry
cake
pie
cookie
pizza
pasta
noodles
rice
soup
salad
sandwich
burger
hot dog
taco
sushi
sashimi
seafood
steak
chicken
barbecue
breakfast
brunch
fruit
vegetables
herbs
spices
cheese
eggs
coffee
espresso
tea
wine
beer
cocktail
juice
ice cream
dessert
chocolate
candy
market stall
# Transport
car
sports car
vintage car
truck
van
bus
tram
motorcycle
scooter
bicycle
skateboard
train
metro
subway
tram
steam train
aeroplane
helicopter
hot air balloon
drone
speedboat
sailing boat
yacht
cruise ship
container ship
submarine
kayak
canoe
surfboard
# Activities & sport
running
jogging
walking
hiking
cycling
swimming
surfing
diving
snorkelling
sailing
rowing
kayaking
climbing
abseiling
skiing
snowboarding
ice skating
skateboarding
horse riding
football
basketball
tennis
golf
baseball
cricket
rugby
boxing
wrestling
martial arts
gymnastics
yoga
meditation
stretching
weight training
dancing
ballet
breakdancing
cheerleading
fishing
# Arts, culture & leisure
painting
drawing
sketch
sculpture
mural
graffiti
street art
calligraphy
pottery
glasswork
photography
concert
live music
festival
carnival
parade
fireworks
ceremony
graduation
protest
market
fair
circus
magic show
puppet show
gaming
reading
writing
chess
board game
# Mood, lighting & photographic style
silhouette
reflection
shadow
long exposure
light trails
bokeh
macro photography
close-up
aerial view
bird's eye view
underwater
black and white
vintage
film grain
neon lights
candlelight
backlight
rim light
dramatic lighting
soft light
misty
moody
minimalist
symmetry
geometry
abstract
pattern
texture
repetition
colour splash
double exposure
# Miscellaneous visual subjects
window
door
gate
staircase
mirror
candle
clock
book
map
flag
trophy
statue
monument
graveyard
cemetery
prison
lighthouse
windmill
water mill
solar panels
satellite dish
telescope
microscope
laboratory equipment
musical instrument
guitar
piano
violin
drums
trumpet
painting on wall
cinema screen
television
old photograph
typewriter
sewing
knitting
gardening
woodworking
welding
pottery wheel
+157 -4
View File
@@ -1,10 +1,10 @@
use crate::db::{self, DbPool, Folder, FolderJobProgress, ImageRecord}; use crate::db::{self, DbPool, Folder, FolderJobProgress, ImageRecord};
use crate::embedder::ClipImageEmbedder; use crate::embedder;
use crate::indexer; use crate::indexer;
use crate::vector; use crate::vector;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::path::PathBuf; use std::path::PathBuf;
use tauri::{AppHandle, State}; use tauri::{AppHandle, Manager, State};
pub type DbState = DbPool; pub type DbState = DbPool;
@@ -204,8 +204,7 @@ pub async fn semantic_search_images(
db: State<'_, DbState>, db: State<'_, DbState>,
params: SemanticSearchParams, params: SemanticSearchParams,
) -> Result<Vec<ImageRecord>, String> { ) -> Result<Vec<ImageRecord>, String> {
let embedder = ClipImageEmbedder::new().map_err(|e| e.to_string())?; let embedding = embedder::embed_text_query(&params.query).map_err(|e| e.to_string())?;
let embedding = embedder.embed_text(&params.query).map_err(|e| e.to_string())?;
let conn = db.get().map_err(|e| e.to_string())?; let conn = db.get().map_err(|e| e.to_string())?;
let limit = params.limit.unwrap_or(64); let limit = params.limit.unwrap_or(64);
@@ -225,6 +224,160 @@ pub async fn semantic_search_images(
Ok(images) Ok(images)
} }
#[derive(Serialize)]
pub struct TagCloudEntry {
pub label: String,
pub count: usize,
}
/// Clusters the library's image embeddings with k-means, then labels each cluster by
/// finding the closest word in the vocabulary. The vocabulary is loaded from
/// `{app_data_dir}/vocabulary.txt` if present, otherwise from the bundled default.
/// Vocabulary embeddings are cached to disk — only recomputed when the vocabulary changes.
#[tauri::command]
pub async fn get_tag_cloud(
app: AppHandle,
db: State<'_, DbState>,
folder_id: Option<i64>,
) -> Result<Vec<TagCloudEntry>, String> {
let app_data_dir = app.path().app_data_dir().map_err(|e| e.to_string())?;
let image_embeddings = {
let conn = db.get().map_err(|e| e.to_string())?;
vector::get_all_image_embeddings(&conn, folder_id).map_err(|e| e.to_string())?
};
let n = image_embeddings.len();
if n < 5 {
return Ok(vec![]);
}
// Load vocabulary (custom file > bundled default)
let vocab = embedder::load_vocabulary(&app_data_dir);
// Embed vocabulary — disk-cached, only recomputes when vocabulary changes
let vocab_refs: Vec<&str> = vocab.iter().map(|s| s.as_str()).collect();
let vocab_embeddings = embedder::embed_vocab_cached(&vocab, &app_data_dir)
.map_err(|e| e.to_string())?;
// Choose k proportional to library size, capped at 30
let k = (n / 20).clamp(5, 30);
// Cluster image embeddings
let (centroids, cluster_counts) = kmeans_cosine(&image_embeddings, k, 40);
// Label each cluster with the nearest vocabulary word
let mut entries: Vec<TagCloudEntry> = Vec::new();
let mut used_labels = std::collections::HashSet::new();
let mut order: Vec<usize> = (0..k).collect();
order.sort_unstable_by(|&a, &b| cluster_counts[b].cmp(&cluster_counts[a]));
for ci in order {
let count = cluster_counts[ci];
if count == 0 { continue; }
let centroid = &centroids[ci];
let label = vocab_refs
.iter()
.zip(vocab_embeddings.iter())
.map(|(&word, emb)| {
let sim: f32 = centroid.iter().zip(emb.iter()).map(|(a, b)| a * b).sum();
(word, sim)
})
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
.map(|(word, _)| word.to_string())
.unwrap_or_else(|| "other".to_string());
if used_labels.insert(label.clone()) {
entries.push(TagCloudEntry { label, count });
}
}
entries.sort_unstable_by(|a, b| b.count.cmp(&a.count));
Ok(entries)
}
// ── k-means with cosine similarity (all vectors assumed to be unit-normalized) ──
fn dot(a: &[f32], b: &[f32]) -> f32 {
a.iter().zip(b.iter()).map(|(x, y)| x * y).sum()
}
fn normalize(v: &mut Vec<f32>) {
let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm > 1e-10 {
v.iter_mut().for_each(|x| *x /= norm);
}
}
fn kmeans_cosine(
points: &[Vec<f32>],
k: usize,
max_iter: usize,
) -> (Vec<Vec<f32>>, Vec<usize>) {
let n = points.len();
let dim = points[0].len();
// Deterministic k-means++ init: spread centroids as far apart as possible
let mut centroids: Vec<Vec<f32>> = Vec::with_capacity(k);
centroids.push(points[n / 2].clone());
for _ in 1..k {
let next = points
.iter()
.map(|p| {
let best_sim = centroids.iter().map(|c| dot(p, c)).fold(f32::NEG_INFINITY, f32::max);
1.0 - best_sim // distance = 1 - cosine_similarity
})
.enumerate()
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
.map(|(i, _)| i)
.unwrap_or(0);
centroids.push(points[next].clone());
}
let mut assignments = vec![0usize; n];
for _ in 0..max_iter {
// Assignment step
let mut changed = false;
for (i, p) in points.iter().enumerate() {
let best = centroids
.iter()
.enumerate()
.map(|(j, c)| (j, dot(p, c)))
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
.map(|(j, _)| j)
.unwrap_or(0);
if assignments[i] != best {
assignments[i] = best;
changed = true;
}
}
if !changed { break; }
// Update step: mean of assigned points, then normalize
let mut sums = vec![vec![0.0f32; dim]; k];
let mut counts = vec![0usize; k];
for (p, &c) in points.iter().zip(assignments.iter()) {
sums[c].iter_mut().zip(p.iter()).for_each(|(s, v)| *s += v);
counts[c] += 1;
}
for (centroid, (sum, &count)) in centroids.iter_mut().zip(sums.iter_mut().zip(counts.iter())) {
if count > 0 {
sum.iter_mut().for_each(|v| *v /= count as f32);
normalize(sum);
*centroid = sum.clone();
}
}
}
let mut counts = vec![0usize; k];
for &a in &assignments { counts[a] += 1; }
(centroids, counts)
}
#[derive(Serialize)] #[derive(Serialize)]
pub struct WorkerStates { pub struct WorkerStates {
pub thumbnail_paused: bool, pub thumbnail_paused: bool,
+216 -22
View File
@@ -1,11 +1,180 @@
use anyhow::Result; use anyhow::{Context, Result};
use candle_core::{DType, Device, Tensor}; use candle_core::{DType, Device, Tensor};
use candle_nn::VarBuilder; use candle_nn::VarBuilder;
use candle_transformers::models::clip::{self, ClipModel}; use candle_transformers::models::clip::{self, ClipModel};
use hf_hub::{api::sync::Api, Repo, RepoType}; use hf_hub::{api::sync::Api, Repo, RepoType};
use std::io::{Read, Write};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::{Mutex, OnceLock};
use tokenizers::Tokenizer; use tokenizers::Tokenizer;
static TEXT_SEARCH_EMBEDDER: OnceLock<Mutex<Option<ClipImageEmbedder>>> = OnceLock::new();
/// In-process cache so the disk file is only read/written once per session.
static VOCAB_EMBED_CACHE: OnceLock<Mutex<Option<Vec<Vec<f32>>>>> = OnceLock::new();
/// Default vocabulary bundled with the binary.
pub const DEFAULT_VOCABULARY: &str = include_str!("../data/vocabulary.txt");
/// Embed a text query using a lazily-initialized, cached CLIP embedder.
pub fn embed_text_query(query: &str) -> Result<Vec<f32>> {
with_text_embedder(|e| e.embed_text(query))
}
/// Parse vocabulary text: strip comment lines (starting with `#`) and blank lines.
pub fn parse_vocabulary(text: &str) -> Vec<String> {
text.lines()
.map(|l| l.trim())
.filter(|l| !l.is_empty() && !l.starts_with('#'))
.map(|l| l.to_string())
.collect()
}
/// Load the vocabulary from `{cache_dir}/vocabulary.txt` if it exists,
/// otherwise fall back to the binary-bundled default.
pub fn load_vocabulary(cache_dir: &Path) -> Vec<String> {
let custom_path = cache_dir.join("vocabulary.txt");
if custom_path.exists() {
if let Ok(text) = std::fs::read_to_string(&custom_path) {
let words = parse_vocabulary(&text);
if !words.is_empty() {
println!("Using custom vocabulary from {:?} ({} words)", custom_path, words.len());
return words;
}
}
}
parse_vocabulary(DEFAULT_VOCABULARY)
}
/// Embed the vocabulary, using a disk cache so CLIP is only called when the vocabulary
/// actually changes. Cache file: `{cache_dir}/vocab_embeddings.bin`.
///
/// Format: `[u64 hash][u32 n_words][u32 dim][(n_words × dim) × f32 LE]`
pub fn embed_vocab_cached(vocab: &[String], cache_dir: &Path) -> Result<Vec<Vec<f32>>> {
// In-process cache hit
{
let guard = VOCAB_EMBED_CACHE
.get_or_init(|| Mutex::new(None))
.lock()
.map_err(|_| anyhow::anyhow!("Vocab cache lock poisoned"))?;
if let Some(cached) = guard.as_ref() {
return Ok(cached.clone());
}
}
let vocab_hash = fnv_hash(vocab);
let disk_path = cache_dir.join("vocab_embeddings.bin");
// Try loading from disk
if let Ok(embeddings) = load_disk_cache(&disk_path, vocab_hash) {
let mut guard = VOCAB_EMBED_CACHE
.get_or_init(|| Mutex::new(None))
.lock()
.map_err(|_| anyhow::anyhow!("Vocab cache lock poisoned"))?;
*guard = Some(embeddings.clone());
println!("Vocabulary embeddings loaded from disk cache ({} words).", embeddings.len());
return Ok(embeddings);
}
// Compute embeddings
println!("Computing vocabulary embeddings ({} words) — this is cached after the first run.", vocab.len());
let prompts: Vec<String> = vocab.iter().map(|w| format!("a photo of {}", w)).collect();
let prompt_refs: Vec<&str> = prompts.iter().map(|s| s.as_str()).collect();
let embeddings = with_text_embedder(|e| {
let mut all = Vec::with_capacity(vocab.len());
for chunk in prompt_refs.chunks(64) {
all.extend(e.embed_texts_batch(chunk)?);
}
Ok(all)
})?;
// Save to disk
if let Err(e) = save_disk_cache(&disk_path, vocab_hash, &embeddings) {
eprintln!("Warning: could not write vocab cache: {e}");
}
let mut guard = VOCAB_EMBED_CACHE
.get_or_init(|| Mutex::new(None))
.lock()
.map_err(|_| anyhow::anyhow!("Vocab cache lock poisoned"))?;
*guard = Some(embeddings.clone());
Ok(embeddings)
}
fn fnv_hash(words: &[String]) -> u64 {
let mut h: u64 = 0xcbf29ce484222325;
for w in words {
for b in w.bytes() {
h = h.wrapping_mul(0x100000001b3) ^ (b as u64);
}
h = h.wrapping_mul(0x100000001b3) ^ 0x0A; // newline separator
}
h
}
fn load_disk_cache(path: &Path, expected_hash: u64) -> Result<Vec<Vec<f32>>> {
let mut f = std::fs::File::open(path).context("no cache file")?;
let mut buf = Vec::new();
f.read_to_end(&mut buf)?;
if buf.len() < 16 {
anyhow::bail!("cache too short");
}
let hash = u64::from_le_bytes(buf[0..8].try_into()?);
if hash != expected_hash {
anyhow::bail!("vocab hash mismatch — recomputing");
}
let n = u32::from_le_bytes(buf[8..12].try_into()?) as usize;
let dim = u32::from_le_bytes(buf[12..16].try_into()?) as usize;
let expected_len = 16 + n * dim * 4;
if buf.len() != expected_len {
anyhow::bail!("cache size mismatch");
}
let mut embeddings = Vec::with_capacity(n);
let data = &buf[16..];
for i in 0..n {
let start = i * dim * 4;
let emb: Vec<f32> = data[start..start + dim * 4]
.chunks_exact(4)
.map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
.collect();
embeddings.push(emb);
}
Ok(embeddings)
}
fn save_disk_cache(path: &Path, hash: u64, embeddings: &[Vec<f32>]) -> Result<()> {
let n = embeddings.len();
let dim = embeddings.first().map(|e| e.len()).unwrap_or(0);
let mut buf = Vec::with_capacity(16 + n * dim * 4);
buf.extend_from_slice(&hash.to_le_bytes());
buf.extend_from_slice(&(n as u32).to_le_bytes());
buf.extend_from_slice(&(dim as u32).to_le_bytes());
for emb in embeddings {
for &v in emb {
buf.extend_from_slice(&v.to_le_bytes());
}
}
std::fs::write(path, buf)?;
println!("Vocabulary embeddings cached to disk ({n} words, {dim} dims).");
Ok(())
}
fn with_text_embedder<T>(f: impl FnOnce(&ClipImageEmbedder) -> Result<T>) -> Result<T> {
let lock = TEXT_SEARCH_EMBEDDER.get_or_init(|| Mutex::new(None));
let mut guard = lock.lock().map_err(|_| anyhow::anyhow!("Text embedder lock poisoned"))?;
if guard.is_none() {
println!("Initializing CLIP text embedder...");
*guard = Some(ClipImageEmbedder::new()?);
}
f(guard.as_ref().unwrap())
}
pub struct ClipImageEmbedder { pub struct ClipImageEmbedder {
model: ClipModel, model: ClipModel,
tokenizer: Tokenizer, tokenizer: Tokenizer,
@@ -67,36 +236,61 @@ impl ClipImageEmbedder {
} }
pub fn embed_text(&self, query: &str) -> Result<Vec<f32>> { pub fn embed_text(&self, query: &str) -> Result<Vec<f32>> {
let encoding = self Ok(self.embed_texts_batch(&[query])?.remove(0))
.tokenizer }
.encode(query, true)
.map_err(anyhow::Error::msg)?; /// Embed multiple text queries in a single CLIP forward pass.
let token_ids = encoding /// All sequences are padded to the same length (max 77 tokens).
.get_ids() pub fn embed_texts_batch(&self, queries: &[&str]) -> Result<Vec<Vec<f32>>> {
if queries.is_empty() {
return Ok(vec![]);
}
let encodings: Vec<_> = queries
.iter() .iter()
.map(|token| *token as u32) .map(|q| self.tokenizer.encode(*q, true).map_err(anyhow::Error::msg))
.collect::<Vec<_>>(); .collect::<Result<_>>()?;
let input_ids = Tensor::new(vec![token_ids], &self.device)?;
let max_len = encodings
.iter()
.map(|e| e.get_ids().len())
.max()
.unwrap_or(10)
.min(77);
let n = queries.len();
let mut flat = vec![0u32; n * max_len];
for (i, enc) in encodings.iter().enumerate() {
let ids = enc.get_ids();
let len = ids.len().min(max_len);
for j in 0..len {
flat[i * max_len + j] = ids[j] as u32;
}
}
let input_ids = Tensor::new(flat, &self.device)?.reshape((n, max_len))?;
let features = self.model.get_text_features(&input_ids)?; let features = self.model.get_text_features(&input_ids)?;
let normalized = clip::div_l2_norm(&features)?; let normalized = clip::div_l2_norm(&features)?;
Ok(normalized.flatten_all()?.to_vec1::<f32>()?)
(0..n)
.map(|i| Ok(normalized.get(i)?.flatten_all()?.to_vec1::<f32>()?))
.collect()
} }
} }
fn resolve_device() -> Result<Device> { fn resolve_device() -> Result<Device> {
#[cfg(feature = "candle-cuda")] match Device::cuda_if_available(0) {
{ Ok(device) if device.is_cuda() => {
let cuda_device = Device::cuda_if_available(0)?; println!("CLIP embedder: using CUDA GPU (device 0).");
if cuda_device.is_cuda() { return Ok(device);
println!("CLIP embedder using CUDA device."); }
return Ok(cuda_device); Ok(_) => {
println!("CLIP embedder: no compatible CUDA GPU found — using CPU.");
}
Err(e) => {
println!("CLIP embedder: CUDA init failed ({e}) — using CPU.");
} }
} }
#[cfg(not(feature = "candle-cuda"))]
println!("CLIP embedder built without CUDA support.");
println!("CLIP embedder using CPU device.");
Ok(Device::Cpu) Ok(Device::Cpu)
} }
+1
View File
@@ -77,6 +77,7 @@ pub fn run() {
commands::semantic_search_images, commands::semantic_search_images,
commands::set_worker_paused, commands::set_worker_paused,
commands::get_worker_states, commands::get_worker_states,
commands::get_tag_cloud,
]) ])
.run(tauri::generate_context!()) .run(tauri::generate_context!())
.expect("error while running tauri application"); .expect("error while running tauri application");
+35
View File
@@ -78,6 +78,41 @@ pub fn find_similar_image_ids(conn: &Connection, image_id: i64, limit: usize) ->
Ok(ids) Ok(ids)
} }
/// Returns all stored image embeddings, optionally filtered to one folder.
/// Each embedding is returned as a normalized f32 vector.
pub fn get_all_image_embeddings(conn: &Connection, folder_id: Option<i64>) -> Result<Vec<Vec<f32>>> {
let packed_rows: Vec<Vec<u8>> = match folder_id {
Some(fid) => {
let mut stmt = conn.prepare(
"SELECT embedding FROM image_vec
WHERE image_id IN (SELECT id FROM images WHERE folder_id = ?1)",
)?;
let rows: Vec<Vec<u8>> = stmt
.query_map([fid], |row| row.get::<_, Vec<u8>>(0))?
.filter_map(|r| r.ok())
.collect();
rows
}
None => {
let mut stmt = conn.prepare("SELECT embedding FROM image_vec")?;
let rows: Vec<Vec<u8>> = stmt
.query_map([], |row| row.get::<_, Vec<u8>>(0))?
.filter_map(|r| r.ok())
.collect();
rows
}
};
Ok(packed_rows.iter().map(|b| unpack_f32(b)).collect())
}
fn unpack_f32(bytes: &[u8]) -> Vec<f32> {
bytes
.chunks_exact(4)
.map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
.collect()
}
pub fn search_image_ids_by_embedding( pub fn search_image_ids_by_embedding(
conn: &Connection, conn: &Connection,
embedding: &[f32], embedding: &[f32],
+14 -3
View File
@@ -5,12 +5,14 @@ import { BackgroundTasks } from "./components/BackgroundTasks";
import { Toolbar } from "./components/Toolbar"; import { Toolbar } from "./components/Toolbar";
import { Gallery } from "./components/Gallery"; import { Gallery } from "./components/Gallery";
import { Lightbox } from "./components/Lightbox"; import { Lightbox } from "./components/Lightbox";
import { TagCloud } from "./components/TagCloud";
export default function App() { export default function App() {
const loadFolders = useGalleryStore((state) => state.loadFolders); const loadFolders = useGalleryStore((state) => state.loadFolders);
const loadBackgroundJobProgress = useGalleryStore((state) => state.loadBackgroundJobProgress); const loadBackgroundJobProgress = useGalleryStore((state) => state.loadBackgroundJobProgress);
const loadImages = useGalleryStore((state) => state.loadImages); const loadImages = useGalleryStore((state) => state.loadImages);
const subscribeToProgress = useGalleryStore((state) => state.subscribeToProgress); const subscribeToProgress = useGalleryStore((state) => state.subscribeToProgress);
const activeView = useGalleryStore((state) => state.activeView);
useEffect(() => { useEffect(() => {
loadFolders().then(() => { loadFolders().then(() => {
@@ -30,9 +32,18 @@ export default function App() {
<div className="flex h-screen bg-gray-950 text-white overflow-hidden select-none"> <div className="flex h-screen bg-gray-950 text-white overflow-hidden select-none">
<Sidebar /> <Sidebar />
<main className="flex-1 flex flex-col min-w-0"> <main className="flex-1 flex flex-col min-w-0">
<Toolbar /> {activeView === "explore" ? (
<BackgroundTasks /> <>
<Gallery /> <BackgroundTasks />
<TagCloud />
</>
) : (
<>
<Toolbar />
<BackgroundTasks />
<Gallery />
</>
)}
</main> </main>
<Lightbox /> <Lightbox />
</div> </div>
+22 -3
View File
@@ -77,6 +77,8 @@ export function Sidebar() {
const addFolder = useGalleryStore((state) => state.addFolder); const addFolder = useGalleryStore((state) => state.addFolder);
const indexingProgress = useGalleryStore((state) => state.indexingProgress); const indexingProgress = useGalleryStore((state) => state.indexingProgress);
const selectFolder = useGalleryStore((state) => state.selectFolder); const selectFolder = useGalleryStore((state) => state.selectFolder);
const activeView = useGalleryStore((state) => state.activeView);
const setView = useGalleryStore((state) => state.setView);
const handleAddFolder = async () => { const handleAddFolder = async () => {
const selected = await open({ directory: true, multiple: false, title: "Select Media Folder" }); const selected = await open({ directory: true, multiple: false, title: "Select Media Folder" });
@@ -102,10 +104,10 @@ export function Sidebar() {
</div> </div>
{/* Nav */} {/* Nav */}
<div className="px-2 pt-2 pb-1"> <div className="px-2 pt-2 pb-1 space-y-px">
<div <div
className={`flex items-center gap-2.5 px-3 py-2 rounded-lg cursor-pointer transition-all duration-150 ${ className={`flex items-center gap-2.5 px-3 py-2 rounded-lg cursor-pointer transition-all duration-150 ${
selectedFolderId === null activeView === "gallery" && selectedFolderId === null
? "bg-white/8 text-white" ? "bg-white/8 text-white"
: "text-gray-500 hover:text-gray-200 hover:bg-white/5" : "text-gray-500 hover:text-gray-200 hover:bg-white/5"
}`} }`}
@@ -115,10 +117,27 @@ export function Sidebar() {
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={1.5} <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={1.5}
d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" /> d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" />
</svg> </svg>
<span className={`text-[13px] font-medium ${selectedFolderId === null ? "text-white" : ""}`}> <span className={`text-[13px] font-medium ${activeView === "gallery" && selectedFolderId === null ? "text-white" : ""}`}>
All Media All Media
</span> </span>
</div> </div>
<div
className={`flex items-center gap-2.5 px-3 py-2 rounded-lg cursor-pointer transition-all duration-150 ${
activeView === "explore"
? "bg-white/8 text-white"
: "text-gray-500 hover:text-gray-200 hover:bg-white/5"
}`}
onClick={() => setView("explore")}
>
<svg className="w-3.5 h-3.5 shrink-0" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={1.5}
d="M7 20l4-16m2 16l4-16M6 9h14M4 15h14" />
</svg>
<span className={`text-[13px] font-medium ${activeView === "explore" ? "text-white" : ""}`}>
Explore
</span>
</div>
</div> </div>
{/* Section label */} {/* Section label */}
+194
View File
@@ -0,0 +1,194 @@
import { useEffect } from "react";
import { motion } from "framer-motion";
import { useGalleryStore, TagCloudEntry } from "../store";
// Accent color pairs: [rest, hover, glow]
const ACCENTS: [string, string, string][] = [
["rgba(96,165,250,0.5)", "#93c5fd", "rgba(59,130,246,0.3)"],
["rgba(192,132,252,0.5)", "#d8b4fe", "rgba(168,85,247,0.3)"],
["rgba(52,211,153,0.5)", "#6ee7b7", "rgba(16,185,129,0.3)"],
["rgba(251,191,36,0.5)", "#fcd34d", "rgba(245,158,11,0.3)"],
["rgba(249,168,212,0.5)", "#fbcfe8", "rgba(236,72,153,0.3)"],
["rgba(103,232,249,0.5)", "#a5f3fc", "rgba(6,182,212,0.3)"],
["rgba(253,186,116,0.5)", "#fed7aa", "rgba(249,115,22,0.3)"],
["rgba(167,243,208,0.5)", "#bbf7d0", "rgba(34,197,94,0.3)"],
];
function pseudoRandom(seed: number): number {
const x = Math.sin(seed + 1) * 10000;
return x - Math.floor(x);
}
function getWeight(count: number, maxCount: number): 1 | 2 | 3 | 4 | 5 {
if (maxCount === 0) return 1;
const ratio = count / maxCount;
if (ratio > 0.75) return 5;
if (ratio > 0.45) return 4;
if (ratio > 0.22) return 3;
if (ratio > 0.08) return 2;
return 1;
}
const FONT_SIZE: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 11, 2: 14, 3: 19, 4: 30, 5: 46 };
const FONT_WEIGHT: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 400, 2: 400, 3: 500, 4: 700, 5: 800 };
const LETTER_SPACING: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 0.8, 2: 0.4, 3: 0, 4: -0.5, 5: -1.5 };
const PADDING: Record<1 | 2 | 3 | 4 | 5, string> = {
1: "3px 8px", 2: "4px 10px", 3: "5px 13px", 4: "8px 18px", 5: "10px 22px",
};
const MAX_ROTATION: Record<1 | 2 | 3 | 4 | 5, number> = { 1: 14, 2: 11, 3: 7, 4: 3, 5: 0 };
function TagButton({
entry,
index,
maxCount,
onSearch,
}: {
entry: TagCloudEntry;
index: number;
maxCount: number;
onSearch: (label: string) => void;
}) {
const weight = getWeight(entry.count, maxCount);
const accentIndex = (index * 3 + weight) % ACCENTS.length;
const [restColor, hoverColor, glowColor] = ACCENTS[accentIndex];
const rotation = (pseudoRandom(index * 7) - 0.5) * 2 * MAX_ROTATION[weight];
const mt = Math.floor(pseudoRandom(index * 3) * 14) + 3;
const mr = Math.floor(pseudoRandom(index * 5) * 20) + 6;
const mb = Math.floor(pseudoRandom(index * 11) * 14) + 3;
const ml = Math.floor(pseudoRandom(index * 13) * 20) + 6;
return (
<motion.button
initial={{ opacity: 0, scale: 0.4, rotate: rotation * 2 }}
animate={{ opacity: 1, scale: 1, rotate: rotation }}
transition={{
delay: index * 0.014,
type: "spring",
stiffness: 180,
damping: 16,
}}
whileHover={{
scale: 1.2,
rotate: 0,
transition: { type: "spring", stiffness: 400, damping: 22 },
}}
whileTap={{ scale: 0.9 }}
style={{
fontSize: FONT_SIZE[weight],
fontWeight: FONT_WEIGHT[weight],
letterSpacing: LETTER_SPACING[weight],
padding: PADDING[weight],
margin: `${mt}px ${mr}px ${mb}px ${ml}px`,
color: restColor,
borderRadius: 10,
border: "1px solid transparent",
background: "transparent",
cursor: "pointer",
userSelect: "none",
transition: "color 0.15s, text-shadow 0.15s, background 0.15s, border-color 0.15s",
}}
onMouseEnter={(e) => {
const el = e.currentTarget;
el.style.color = hoverColor;
el.style.textShadow = `0 0 20px ${glowColor}, 0 0 40px ${glowColor}`;
el.style.background = glowColor.replace("0.3", "0.1");
el.style.borderColor = glowColor.replace("0.3", "0.25");
}}
onMouseLeave={(e) => {
const el = e.currentTarget;
el.style.color = restColor;
el.style.textShadow = "none";
el.style.background = "transparent";
el.style.borderColor = "transparent";
}}
onClick={() => onSearch(entry.label)}
title={`${entry.count} matching ${entry.count === 1 ? "photo" : "photos"}`}
>
{entry.label}
</motion.button>
);
}
export function TagCloud() {
const tagCloudEntries = useGalleryStore((state) => state.tagCloudEntries);
const tagCloudLoading = useGalleryStore((state) => state.tagCloudLoading);
const loadTagCloud = useGalleryStore((state) => state.loadTagCloud);
const searchByTag = useGalleryStore((state) => state.searchByTag);
const selectedFolderId = useGalleryStore((state) => state.selectedFolderId);
useEffect(() => {
void loadTagCloud();
}, [selectedFolderId]);
const maxCount = tagCloudEntries.length > 0
? Math.max(...tagCloudEntries.map((e) => e.count))
: 1;
return (
<div className="flex-1 flex flex-col items-center min-h-0 overflow-y-auto">
{/* Header */}
<motion.div
className="text-center pt-14 pb-8 shrink-0"
initial={{ opacity: 0, y: -10 }}
animate={{ opacity: 1, y: 0 }}
transition={{ duration: 0.35 }}
>
<h2 className="text-[22px] font-semibold text-white/70 tracking-tight mb-2">
Explore your library
</h2>
<p className="text-[13px] text-white/25">
Topics found in your photos sized by how many match
</p>
</motion.div>
{/* Loading */}
{tagCloudLoading && (
<div className="flex-1 flex flex-col items-center justify-center gap-4">
<motion.svg
className="w-8 h-8 text-white/20"
viewBox="0 0 24 24"
fill="none"
animate={{ rotate: 360 }}
transition={{ duration: 1, repeat: Infinity, ease: "linear" }}
>
<circle cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="2" strokeOpacity="0.2" />
<path d="M4 12a8 8 0 018-8" stroke="currentColor" strokeWidth="2" strokeLinecap="round" />
</motion.svg>
<p className="text-[12px] text-white/20">Analysing your library with CLIP</p>
</div>
)}
{/* Empty state */}
{!tagCloudLoading && tagCloudEntries.length === 0 && (
<div className="flex-1 flex items-center justify-center">
<p className="text-[13px] text-white/25 text-center max-w-xs leading-relaxed">
No embeddings yet. Add a folder and wait for the embedding worker to finish,
then come back here.
</p>
</div>
)}
{/* Cloud */}
{!tagCloudLoading && tagCloudEntries.length > 0 && (
<div className="flex flex-wrap justify-center px-12 pb-16 max-w-5xl w-full">
{tagCloudEntries.map((entry, index) => (
<TagButton
key={entry.label}
entry={entry}
index={index}
maxCount={maxCount}
onSearch={searchByTag}
/>
))}
</div>
)}
{!tagCloudLoading && tagCloudEntries.length > 0 && (
<p className="shrink-0 pb-8 text-[11px] text-white/12 text-center">
Ranked by visual similarity · CLIP ViT-B/32
</p>
)}
</div>
);
}
+45 -1
View File
@@ -72,6 +72,13 @@ export interface ThumbnailBatch {
images: ImageRecord[]; images: ImageRecord[];
} }
export type ActiveView = "gallery" | "explore";
export interface TagCloudEntry {
label: string;
count: number;
}
export type SortOrder = export type SortOrder =
| "date_desc" | "date_desc"
| "date_asc" | "date_asc"
@@ -97,6 +104,9 @@ interface GalleryState {
zoomPreset: ZoomPreset; zoomPreset: ZoomPreset;
selectedImage: ImageRecord | null; selectedImage: ImageRecord | null;
collectionTitle: string | null; collectionTitle: string | null;
activeView: ActiveView;
tagCloudEntries: TagCloudEntry[];
tagCloudLoading: boolean;
indexingProgress: Record<number, IndexProgress>; indexingProgress: Record<number, IndexProgress>;
mediaJobProgress: Record<number, FolderJobProgress>; mediaJobProgress: Record<number, FolderJobProgress>;
cacheDir: string; cacheDir: string;
@@ -119,6 +129,9 @@ interface GalleryState {
setZoomPreset: (zoomPreset: ZoomPreset) => void; setZoomPreset: (zoomPreset: ZoomPreset) => void;
openImage: (image: ImageRecord) => void; openImage: (image: ImageRecord) => void;
closeImage: () => void; closeImage: () => void;
setView: (view: ActiveView) => void;
loadTagCloud: () => Promise<void>;
searchByTag: (tag: string) => void;
loadSimilarImages: (imageId: number) => Promise<void>; loadSimilarImages: (imageId: number) => Promise<void>;
retryFailedEmbeddings: (folderId: number) => Promise<void>; retryFailedEmbeddings: (folderId: number) => Promise<void>;
updateImageDetails: (imageId: number, updates: { favorite?: boolean; rating?: number }) => Promise<void>; updateImageDetails: (imageId: number, updates: { favorite?: boolean; rating?: number }) => Promise<void>;
@@ -255,6 +268,9 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
zoomPreset: "comfortable", zoomPreset: "comfortable",
selectedImage: null, selectedImage: null,
collectionTitle: null, collectionTitle: null,
activeView: "gallery",
tagCloudEntries: [],
tagCloudLoading: false,
indexingProgress: {}, indexingProgress: {},
mediaJobProgress: {}, mediaJobProgress: {},
cacheDir: "", cacheDir: "",
@@ -299,7 +315,7 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
}, },
selectFolder: (folderId) => { selectFolder: (folderId) => {
set({ selectedFolderId: folderId, images: [], loadedCount: 0, collectionTitle: null }); set({ selectedFolderId: folderId, images: [], loadedCount: 0, collectionTitle: null, activeView: "gallery" });
void get().loadImages(true); void get().loadImages(true);
}, },
@@ -406,6 +422,34 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
openImage: (image) => set({ selectedImage: image }), openImage: (image) => set({ selectedImage: image }),
closeImage: () => set({ selectedImage: null }), closeImage: () => set({ selectedImage: null }),
setView: (activeView) => set({ activeView }),
loadTagCloud: async () => {
const { selectedFolderId } = get();
set({ tagCloudLoading: true });
try {
const entries = await invoke<TagCloudEntry[]>("get_tag_cloud", {
folderId: selectedFolderId,
});
set({ tagCloudEntries: entries, tagCloudLoading: false });
} catch (error) {
console.error("Failed to load tag cloud:", error);
set({ tagCloudLoading: false });
}
},
searchByTag: (tag) => {
set({
activeView: "gallery",
search: tag,
searchMode: "semantic",
images: [],
loadedCount: 0,
collectionTitle: `Exploring: ${tag}`,
});
void get().loadImages(true);
},
loadSimilarImages: async (imageId) => { loadSimilarImages: async (imageId) => {
const images = await invoke<ImageRecord[]>("find_similar_images", { const images = await invoke<ImageRecord[]>("find_similar_images", {
params: { image_id: imageId, limit: PAGE_SIZE }, params: { image_id: imageId, limit: PAGE_SIZE },