Polish search and embedding UX

- add semantic text search with toolbar mode switching and sqlite-vec query support
- improve embedding progress visibility, failure recovery, and similar-image affordances
- add search clearing and keyboard controls for filename vs semantic search modes
- refine background task interactions and gallery/lightbox embedding states

Refs: #4
This commit is contained in:
2026-04-06 01:45:25 +01:00
parent 51e4c2c1f7
commit c6a66d1ba9
16 changed files with 978 additions and 415 deletions
+59
View File
@@ -1,4 +1,5 @@
use crate::db::{self, DbPool, Folder, FolderJobProgress, ImageRecord};
use crate::embedder::ClipImageEmbedder;
use crate::indexer;
use crate::vector;
use serde::{Deserialize, Serialize};
@@ -44,6 +45,15 @@ pub struct RetryFailedEmbeddingsParams {
pub folder_id: i64,
}
#[derive(Deserialize)]
pub struct SemanticSearchParams {
pub query: String,
pub folder_id: Option<i64>,
pub media_kind: Option<String>,
pub favorites_only: Option<bool>,
pub limit: Option<usize>,
}
#[tauri::command]
pub async fn add_folder(
app: AppHandle,
@@ -188,3 +198,52 @@ pub async fn retry_failed_embeddings(
let conn = db.get().map_err(|e| e.to_string())?;
db::retry_failed_embedding_jobs(&conn, params.folder_id).map_err(|e| e.to_string())
}
#[tauri::command]
pub async fn semantic_search_images(
db: State<'_, DbState>,
params: SemanticSearchParams,
) -> Result<Vec<ImageRecord>, String> {
let embedder = ClipImageEmbedder::new().map_err(|e| e.to_string())?;
let embedding = embedder.embed_text(&params.query).map_err(|e| e.to_string())?;
let conn = db.get().map_err(|e| e.to_string())?;
let limit = params.limit.unwrap_or(64);
let ids = vector::search_image_ids_by_embedding(&conn, &embedding, limit).map_err(|e| e.to_string())?;
let mut images = db::get_images_by_ids(&conn, &ids).map_err(|e| e.to_string())?;
if let Some(folder_id) = params.folder_id {
images.retain(|image| image.folder_id == folder_id);
}
if let Some(media_kind) = params.media_kind.as_deref() {
images.retain(|image| image.media_kind == media_kind);
}
if params.favorites_only.unwrap_or(false) {
images.retain(|image| image.favorite);
}
Ok(images)
}
#[derive(Serialize)]
pub struct WorkerStates {
pub thumbnail_paused: bool,
pub metadata_paused: bool,
pub embedding_paused: bool,
}
#[tauri::command]
pub async fn set_worker_paused(worker: String, paused: bool) -> Result<(), String> {
indexer::set_worker_paused(&worker, paused);
Ok(())
}
#[tauri::command]
pub async fn get_worker_states() -> Result<WorkerStates, String> {
let states = indexer::get_worker_paused_states();
Ok(WorkerStates {
thumbnail_paused: states[0],
metadata_paused: states[1],
embedding_paused: states[2],
})
}
+2
View File
@@ -813,6 +813,8 @@ pub fn get_images(
"date_desc" => "modified_at DESC NULLS LAST",
"size_asc" => "file_size ASC",
"size_desc" => "file_size DESC",
"duration_asc" => "duration_ms ASC NULLS LAST",
"duration_desc" => "duration_ms DESC NULLS LAST",
_ => "modified_at DESC NULLS LAST",
};
+25
View File
@@ -4,9 +4,11 @@ use candle_nn::VarBuilder;
use candle_transformers::models::clip::{self, ClipModel};
use hf_hub::{api::sync::Api, Repo, RepoType};
use std::path::{Path, PathBuf};
use tokenizers::Tokenizer;
pub struct ClipImageEmbedder {
model: ClipModel,
tokenizer: Tokenizer,
device: Device,
image_size: usize,
}
@@ -21,6 +23,11 @@ impl ClipImageEmbedder {
));
println!("Resolving CLIP model weights from Hugging Face cache...");
let model_path = repo.get("model.safetensors")?;
let tokenizer_repo = api.repo(Repo::new(
"openai/clip-vit-base-patch32".to_string(),
RepoType::Model,
));
let tokenizer_path = tokenizer_repo.get("tokenizer.json")?;
let config = clip::ClipConfig::vit_base_patch32();
let device = resolve_device()?;
@@ -32,10 +39,12 @@ impl ClipImageEmbedder {
)?
};
let model = ClipModel::new(vb, &config)?;
let tokenizer = Tokenizer::from_file(tokenizer_path).map_err(anyhow::Error::msg)?;
println!("CLIP image embedder ready.");
Ok(Self {
model,
tokenizer,
device,
image_size: config.image_size,
})
@@ -56,6 +65,22 @@ impl ClipImageEmbedder {
}
Ok(embeddings)
}
pub fn embed_text(&self, query: &str) -> Result<Vec<f32>> {
let encoding = self
.tokenizer
.encode(query, true)
.map_err(anyhow::Error::msg)?;
let token_ids = encoding
.get_ids()
.iter()
.map(|token| *token as u32)
.collect::<Vec<_>>();
let input_ids = Tensor::new(vec![token_ids], &self.device)?;
let features = self.model.get_text_features(&input_ids)?;
let normalized = clip::div_l2_norm(&features)?;
Ok(normalized.flatten_all()?.to_vec1::<f32>()?)
}
}
fn resolve_device() -> Result<Device> {
+35 -4
View File
@@ -9,6 +9,7 @@ use rayon::prelude::*;
use serde::Serialize;
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};
use tauri::{AppHandle, Emitter};
@@ -24,6 +25,27 @@ const JOB_PROGRESS_EMIT_INTERVAL: Duration = Duration::from_millis(750);
static LAST_JOB_PROGRESS_EMIT: OnceLock<Mutex<HashMap<i64, Instant>>> = OnceLock::new();
static ACTIVE_INDEXING_FOLDERS: OnceLock<Mutex<HashSet<i64>>> = OnceLock::new();
static THUMBNAIL_WORKER_PAUSED: AtomicBool = AtomicBool::new(false);
static METADATA_WORKER_PAUSED: AtomicBool = AtomicBool::new(false);
static EMBEDDING_WORKER_PAUSED: AtomicBool = AtomicBool::new(false);
pub fn set_worker_paused(worker: &str, paused: bool) {
match worker {
"thumbnail" => THUMBNAIL_WORKER_PAUSED.store(paused, Ordering::Relaxed),
"metadata" => METADATA_WORKER_PAUSED.store(paused, Ordering::Relaxed),
"embedding" => EMBEDDING_WORKER_PAUSED.store(paused, Ordering::Relaxed),
_ => {}
}
}
pub fn get_worker_paused_states() -> [bool; 3] {
[
THUMBNAIL_WORKER_PAUSED.load(Ordering::Relaxed),
METADATA_WORKER_PAUSED.load(Ordering::Relaxed),
EMBEDDING_WORKER_PAUSED.load(Ordering::Relaxed),
]
}
static FOLDER_STORAGE_PROFILES: OnceLock<Mutex<HashMap<i64, RuntimeAdaptiveProfile>>> =
OnceLock::new();
static DB_WRITE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
@@ -73,20 +95,26 @@ pub fn start_thumbnail_worker(
cache_dir: PathBuf,
) {
std::thread::spawn(move || loop {
if THUMBNAIL_WORKER_PAUSED.load(Ordering::Relaxed) {
std::thread::sleep(std::time::Duration::from_millis(500));
continue;
}
if let Err(error) = process_thumbnail_batch(&app, &pool, &media_tools, &cache_dir) {
eprintln!("Thumbnail worker error: {}", error);
}
std::thread::sleep(std::time::Duration::from_millis(250));
});
}
pub fn start_metadata_worker(app: AppHandle, pool: DbPool, media_tools: MediaTools) {
std::thread::spawn(move || loop {
if METADATA_WORKER_PAUSED.load(Ordering::Relaxed) {
std::thread::sleep(std::time::Duration::from_millis(500));
continue;
}
if let Err(error) = process_metadata_batch(&app, &pool, &media_tools) {
eprintln!("Metadata worker error: {}", error);
}
std::thread::sleep(std::time::Duration::from_millis(250));
});
}
@@ -96,10 +124,13 @@ pub fn start_embedding_worker(app: AppHandle, pool: DbPool) {
let mut embedder: Option<ClipImageEmbedder> = None;
println!("Embedding worker started.");
loop {
if EMBEDDING_WORKER_PAUSED.load(Ordering::Relaxed) {
std::thread::sleep(std::time::Duration::from_millis(500));
continue;
}
if let Err(error) = process_embedding_batch(&app, &pool, &mut embedder) {
eprintln!("Embedding worker error: {}", error);
}
std::thread::sleep(std::time::Duration::from_millis(500));
}
});
@@ -319,7 +350,7 @@ fn process_thumbnail_batch(
return Ok(());
}
println!("Embedding batch claimed: {} items", jobs.len());
println!("Thumbnail batch claimed: {} items", jobs.len());
let (image_jobs, video_jobs): (Vec<_>, Vec<_>) =
jobs.into_iter().partition(|job| job.media_kind == "image");
+3
View File
@@ -74,6 +74,9 @@ pub fn run() {
commands::update_image_details,
commands::find_similar_images,
commands::retry_failed_embeddings,
commands::semantic_search_images,
commands::set_worker_paused,
commands::get_worker_states,
])
.run(tauri::generate_context!())
.expect("error while running tauri application");
+19 -2
View File
@@ -6,6 +6,13 @@ use serde::Deserialize;
use std::path::{Path, PathBuf};
use std::process::Command;
// On Windows, GUI apps spawn subprocesses with a visible console window by default.
// CREATE_NO_WINDOW suppresses that for every ffmpeg/ffprobe invocation.
#[cfg(target_os = "windows")]
use std::os::windows::process::CommandExt;
#[cfg(target_os = "windows")]
const CREATE_NO_WINDOW: u32 = 0x08000000;
#[derive(Debug, Clone)]
pub struct MediaTools {
ffmpeg_path: PathBuf,
@@ -30,6 +37,10 @@ impl MediaTools {
}
pub fn ensure_installed() -> Result<()> {
// Skip download entirely if both binaries are already present.
if ffmpeg_path().exists() && ffprobe_path().exists() {
return Ok(());
}
auto_download_with_progress(|event| match event {
FfmpegDownloadProgressEvent::Starting => {
println!("Downloading bundled FFmpeg...");
@@ -53,11 +64,17 @@ impl MediaTools {
}
pub fn ffmpeg_command(&self) -> Command {
Command::new(&self.ffmpeg_path)
let mut cmd = Command::new(&self.ffmpeg_path);
#[cfg(target_os = "windows")]
cmd.creation_flags(CREATE_NO_WINDOW);
cmd
}
pub fn ffprobe_command(&self) -> Command {
Command::new(&self.ffprobe_path)
let mut cmd = Command::new(&self.ffprobe_path);
#[cfg(target_os = "windows")]
cmd.creation_flags(CREATE_NO_WINDOW);
cmd
}
}
+32
View File
@@ -78,6 +78,38 @@ pub fn find_similar_image_ids(conn: &Connection, image_id: i64, limit: usize) ->
Ok(ids)
}
pub fn search_image_ids_by_embedding(
conn: &Connection,
embedding: &[f32],
limit: usize,
) -> Result<Vec<i64>> {
if embedding.len() != CLIP_VECTOR_DIM {
return Err(anyhow!(
"expected {}-dimensional embedding, got {}",
CLIP_VECTOR_DIM,
embedding.len()
));
}
let packed = pack_f32(embedding);
let mut stmt = conn.prepare(
"SELECT image_id
FROM image_vec
WHERE embedding MATCH vec_f32(?1)
AND k = ?2",
)?;
let rows = stmt.query_map((&packed, limit as i64), |row| row.get::<_, i64>(0))?;
let mut ids = Vec::new();
for row in rows {
ids.push(row?);
if ids.len() >= limit {
break;
}
}
Ok(ids)
}
#[allow(dead_code)]
fn pack_f32(values: &[f32]) -> Vec<u8> {
let mut out = Vec::with_capacity(values.len() * std::mem::size_of::<f32>());