feat: add region-based similarity search

Adds a "Search within image" button to the lightbox that lets the user
draw a crop region on an image and find visually similar results using
that crop's embedding. The crop is embedded in-memory without a temp
file via a new embed_image_crop method on ClipImageEmbedder.

Introduces a folder-scoped cosine search (search_image_ids_by_embedding_in_folder)
to support the current-folder scope option, and wires up the new
find_similar_by_region Tauri command end-to-end from Rust through to
the Zustand store and Lightbox UI.
This commit is contained in:
2026-06-02 19:39:31 +01:00
parent ff4a568b57
commit df17497808
6 changed files with 477 additions and 24 deletions
+45
View File
@@ -74,6 +74,51 @@ impl ClipImageEmbedder {
Ok(self.embed_images(&[path.to_path_buf()])?.remove(0))
}
/// Embed a cropped region of an image without writing a temp file to disk.
/// `crop_x`, `crop_y`, `crop_w`, `crop_h` are normalized 0.01.0 coordinates.
pub fn embed_image_crop(
&self,
path: &Path,
crop_x: f32,
crop_y: f32,
crop_w: f32,
crop_h: f32,
) -> Result<Vec<f32>> {
let img = image::ImageReader::open(path)?
.with_guessed_format()?
.decode()?;
let img_w = img.width() as f32;
let img_h = img.height() as f32;
let x = ((crop_x * img_w) as u32).min(img.width().saturating_sub(1));
let y = ((crop_y * img_h) as u32).min(img.height().saturating_sub(1));
let w = ((crop_w * img_w) as u32).max(1).min(img.width() - x);
let h = ((crop_h * img_h) as u32).max(1).min(img.height() - y);
let cropped = img.crop_imm(x, y, w, h);
let resized = cropped.resize_to_fill(
self.image_size as u32,
self.image_size as u32,
image::imageops::FilterType::Triangle,
);
let raw = resized.to_rgb8().into_raw();
let tensor = candle_core::Tensor::from_vec(
raw,
(self.image_size, self.image_size, 3),
&candle_core::Device::Cpu,
)?
.permute((2, 0, 1))?
.to_dtype(candle_core::DType::F32)?
.affine(2.0 / 255.0, -1.0)?;
let batch = tensor.unsqueeze(0)?.to_device(&self.device)?;
let features = self.model.get_image_features(&batch)?;
let normalized = candle_transformers::models::clip::div_l2_norm(&features)?;
Ok(normalized.get(0)?.flatten_all()?.to_vec1::<f32>()?)
}
pub fn embed_images(&self, paths: &[PathBuf]) -> Result<Vec<Vec<f32>>> {
let images = load_images(paths, self.image_size)?.to_device(&self.device)?;
let features = self.model.get_image_features(&images)?;