feat(settings): add "Rebuild semantic index" maintenance action
Drops and recreates the sqlite-vec tables at the current model dimension, then re-queues every image for embedding. Fixes "dimension mismatch" search errors that occur when the vector table was built for a different model/dimension (e.g. after experimenting with 768-dim models against this 512-dim build). The rebuild runs under the embedding worker's DB write lock and resets the job queue inside a single transaction, so it can't interleave with an in-flight embedding batch (per code review).
This commit is contained in:
@@ -1987,6 +1987,14 @@ pub async fn vacuum_database(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tauri::command]
|
||||||
|
pub async fn rebuild_semantic_index(db: State<'_, DbState>) -> Result<usize, String> {
|
||||||
|
let conn = db.get().map_err(|e| e.to_string())?;
|
||||||
|
// Serialize against the embedding worker (which writes under the same lock)
|
||||||
|
// so the rebuild can't interleave with an in-flight embedding batch.
|
||||||
|
indexer::with_db_write_lock(|| db::reset_all_embeddings(&conn)).map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
pub struct OrphanedThumbnailsInfo {
|
pub struct OrphanedThumbnailsInfo {
|
||||||
pub count: u64,
|
pub count: u64,
|
||||||
|
|||||||
@@ -496,6 +496,41 @@ pub fn repair_embedding_consistency(conn: &Connection) -> Result<(usize, usize)>
|
|||||||
Ok((orphaned_vectors, missing_vector_ids.len()))
|
Ok((orphaned_vectors, missing_vector_ids.len()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Full semantic-index rebuild: recreate the vector tables at the current model
|
||||||
|
/// dimension and re-queue every image for embedding. Used by the maintenance
|
||||||
|
/// action when the index is stale or its dimension no longer matches the model
|
||||||
|
/// (which otherwise surfaces as a "dimension mismatch" search error). Returns the
|
||||||
|
/// number of images re-queued.
|
||||||
|
pub fn reset_all_embeddings(conn: &Connection) -> Result<usize> {
|
||||||
|
// Recreate the vector tables at the current model dimension first (DDL, kept
|
||||||
|
// out of the transaction below). The caller holds the DB write lock, so the
|
||||||
|
// embedding worker can't interleave a write between this and the queue reset.
|
||||||
|
vector::rebuild_tables(conn)?;
|
||||||
|
let tx = conn.unchecked_transaction()?;
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE images
|
||||||
|
SET embedding_status = 'pending',
|
||||||
|
embedding_model = NULL,
|
||||||
|
embedding_updated_at = NULL,
|
||||||
|
embedding_error = NULL",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
tx.execute("DELETE FROM embedding_jobs", [])?;
|
||||||
|
let queued = tx.execute(
|
||||||
|
"INSERT INTO embedding_jobs (image_id, status, attempts, last_error, created_at, updated_at)
|
||||||
|
SELECT id, 'pending', 0, NULL, datetime('now'), datetime('now')
|
||||||
|
FROM images",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO app_kv (key, value) VALUES ('embedding_revision', 1)
|
||||||
|
ON CONFLICT(key) DO UPDATE SET value = value + 1",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(queued)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn retry_failed_embedding_jobs(conn: &Connection, folder_id: i64) -> Result<usize> {
|
pub fn retry_failed_embedding_jobs(conn: &Connection, folder_id: i64) -> Result<usize> {
|
||||||
// Only re-queue images that are actually embeddable right now.
|
// Only re-queue images that are actually embeddable right now.
|
||||||
// Videos without a thumbnail would just fail again immediately, so skip them —
|
// Videos without a thumbnail would just fail again immediately, so skip them —
|
||||||
|
|||||||
@@ -1288,7 +1288,7 @@ fn max_worker_fetch_size(active_folders: &HashSet<i64>) -> usize {
|
|||||||
.unwrap_or(StorageProfile::Balanced.worker_fetch_size())
|
.unwrap_or(StorageProfile::Balanced.worker_fetch_size())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn with_db_write_lock<T>(operation: impl FnOnce() -> Result<T>) -> Result<T> {
|
pub fn with_db_write_lock<T>(operation: impl FnOnce() -> Result<T>) -> Result<T> {
|
||||||
let lock = DB_WRITE_LOCK.get_or_init(|| Mutex::new(()));
|
let lock = DB_WRITE_LOCK.get_or_init(|| Mutex::new(()));
|
||||||
let _guard = lock.lock().unwrap();
|
let _guard = lock.lock().unwrap();
|
||||||
operation()
|
operation()
|
||||||
|
|||||||
@@ -192,6 +192,7 @@ pub fn run() {
|
|||||||
commands::open_app_data_folder,
|
commands::open_app_data_folder,
|
||||||
commands::get_database_info,
|
commands::get_database_info,
|
||||||
commands::vacuum_database,
|
commands::vacuum_database,
|
||||||
|
commands::rebuild_semantic_index,
|
||||||
commands::get_orphaned_thumbnails_info,
|
commands::get_orphaned_thumbnails_info,
|
||||||
commands::cleanup_orphaned_thumbnails,
|
commands::cleanup_orphaned_thumbnails,
|
||||||
commands::get_muted_folder_ids,
|
commands::get_muted_folder_ids,
|
||||||
|
|||||||
@@ -36,6 +36,18 @@ pub fn migrate(conn: &Connection) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Drop and recreate the vector tables at the current `CLIP_VECTOR_DIM`. Used by
|
||||||
|
/// the "Rebuild semantic index" maintenance action when stored vectors no longer
|
||||||
|
/// match the active model's dimension (e.g. after switching embedding models),
|
||||||
|
/// so the columns are rebuilt to the right size before embeddings regenerate.
|
||||||
|
pub fn rebuild_tables(conn: &Connection) -> Result<()> {
|
||||||
|
conn.execute_batch(
|
||||||
|
"DROP TABLE IF EXISTS image_vec;
|
||||||
|
DROP TABLE IF EXISTS caption_vec;",
|
||||||
|
)?;
|
||||||
|
migrate(conn)
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub fn delete_embedding(conn: &Connection, image_id: i64) -> Result<()> {
|
pub fn delete_embedding(conn: &Connection, image_id: i64) -> Result<()> {
|
||||||
conn.execute("DELETE FROM image_vec WHERE image_id = ?1", [image_id])?;
|
conn.execute("DELETE FROM image_vec WHERE image_id = ?1", [image_id])?;
|
||||||
|
|||||||
@@ -138,6 +138,8 @@ export function SettingsModal() {
|
|||||||
const [dbInfo, setDbInfo] = useState<DatabaseInfo | null>(null);
|
const [dbInfo, setDbInfo] = useState<DatabaseInfo | null>(null);
|
||||||
const [vacuuming, setVacuuming] = useState(false);
|
const [vacuuming, setVacuuming] = useState(false);
|
||||||
const [vacuumResult, setVacuumResult] = useState<VacuumResult | null>(null);
|
const [vacuumResult, setVacuumResult] = useState<VacuumResult | null>(null);
|
||||||
|
const [rebuildingIndex, setRebuildingIndex] = useState(false);
|
||||||
|
const [rebuildIndexResult, setRebuildIndexResult] = useState<string | null>(null);
|
||||||
const [thumbnailInfo, setThumbnailInfo] = useState<OrphanedThumbnailsInfo | null>(null);
|
const [thumbnailInfo, setThumbnailInfo] = useState<OrphanedThumbnailsInfo | null>(null);
|
||||||
const [cleaningThumbnails, setCleaningThumbnails] = useState(false);
|
const [cleaningThumbnails, setCleaningThumbnails] = useState(false);
|
||||||
const [thumbnailCleanupResult, setThumbnailCleanupResult] = useState<CleanupOrphanedThumbnailsResult | null>(null);
|
const [thumbnailCleanupResult, setThumbnailCleanupResult] = useState<CleanupOrphanedThumbnailsResult | null>(null);
|
||||||
@@ -184,6 +186,7 @@ export function SettingsModal() {
|
|||||||
const setNotificationsPaused = useGalleryStore((state) => state.setNotificationsPaused);
|
const setNotificationsPaused = useGalleryStore((state) => state.setNotificationsPaused);
|
||||||
const getDatabaseInfo = useGalleryStore((state) => state.getDatabaseInfo);
|
const getDatabaseInfo = useGalleryStore((state) => state.getDatabaseInfo);
|
||||||
const vacuumDatabase = useGalleryStore((state) => state.vacuumDatabase);
|
const vacuumDatabase = useGalleryStore((state) => state.vacuumDatabase);
|
||||||
|
const rebuildSemanticIndex = useGalleryStore((state) => state.rebuildSemanticIndex);
|
||||||
const getOrphanedThumbnailsInfo = useGalleryStore((state) => state.getOrphanedThumbnailsInfo);
|
const getOrphanedThumbnailsInfo = useGalleryStore((state) => state.getOrphanedThumbnailsInfo);
|
||||||
const cleanupOrphanedThumbnails = useGalleryStore((state) => state.cleanupOrphanedThumbnails);
|
const cleanupOrphanedThumbnails = useGalleryStore((state) => state.cleanupOrphanedThumbnails);
|
||||||
const appVersion = useGalleryStore((state) => state.appVersion);
|
const appVersion = useGalleryStore((state) => state.appVersion);
|
||||||
@@ -799,6 +802,42 @@ export function SettingsModal() {
|
|||||||
</button>
|
</button>
|
||||||
</SettingsItem>
|
</SettingsItem>
|
||||||
|
|
||||||
|
<SettingsItem
|
||||||
|
label="Rebuild semantic index"
|
||||||
|
description={
|
||||||
|
<>
|
||||||
|
<span>
|
||||||
|
Recreates the visual-embedding index and re-embeds every image in the
|
||||||
|
background. Use this if semantic or similar-image search reports a
|
||||||
|
dimension-mismatch error (for example after experimenting with a
|
||||||
|
different embedding model).
|
||||||
|
</span>
|
||||||
|
{rebuildIndexResult !== null ? (
|
||||||
|
<span className="mt-2 block text-gray-600">{rebuildIndexResult}</span>
|
||||||
|
) : null}
|
||||||
|
</>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
className="rounded-md border border-white/10 bg-white/[0.055] px-3 py-1.5 text-xs text-gray-300 transition-colors hover:bg-white/10 hover:text-white disabled:cursor-not-allowed disabled:opacity-45 light-theme:border-gray-700/50 light-theme:bg-gray-900 light-theme:text-white light-theme:hover:bg-gray-800 light-theme:hover:text-white"
|
||||||
|
onClick={() => {
|
||||||
|
setRebuildingIndex(true);
|
||||||
|
setRebuildIndexResult(null);
|
||||||
|
void rebuildSemanticIndex()
|
||||||
|
.then((count) =>
|
||||||
|
setRebuildIndexResult(
|
||||||
|
`Re-queued ${count.toLocaleString()} image${count === 1 ? "" : "s"} for embedding.`,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.catch((error) => setRebuildIndexResult(String(error)))
|
||||||
|
.finally(() => setRebuildingIndex(false));
|
||||||
|
}}
|
||||||
|
disabled={rebuildingIndex}
|
||||||
|
>
|
||||||
|
{rebuildingIndex ? "Rebuilding…" : "Rebuild index"}
|
||||||
|
</button>
|
||||||
|
</SettingsItem>
|
||||||
|
|
||||||
<SettingsItem
|
<SettingsItem
|
||||||
label="Thumbnail cache"
|
label="Thumbnail cache"
|
||||||
description={
|
description={
|
||||||
|
|||||||
@@ -467,6 +467,7 @@ interface GalleryState {
|
|||||||
openAppDataFolder: () => Promise<void>;
|
openAppDataFolder: () => Promise<void>;
|
||||||
getDatabaseInfo: () => Promise<DatabaseInfo>;
|
getDatabaseInfo: () => Promise<DatabaseInfo>;
|
||||||
vacuumDatabase: () => Promise<VacuumResult>;
|
vacuumDatabase: () => Promise<VacuumResult>;
|
||||||
|
rebuildSemanticIndex: () => Promise<number>;
|
||||||
getOrphanedThumbnailsInfo: () => Promise<OrphanedThumbnailsInfo>;
|
getOrphanedThumbnailsInfo: () => Promise<OrphanedThumbnailsInfo>;
|
||||||
cleanupOrphanedThumbnails: () => Promise<CleanupOrphanedThumbnailsResult>;
|
cleanupOrphanedThumbnails: () => Promise<CleanupOrphanedThumbnailsResult>;
|
||||||
retryFailedEmbeddings: (folderId: number) => Promise<void>;
|
retryFailedEmbeddings: (folderId: number) => Promise<void>;
|
||||||
@@ -1858,6 +1859,8 @@ export const useGalleryStore = create<GalleryState>((set, get) => ({
|
|||||||
|
|
||||||
vacuumDatabase: () => invoke<VacuumResult>("vacuum_database"),
|
vacuumDatabase: () => invoke<VacuumResult>("vacuum_database"),
|
||||||
|
|
||||||
|
rebuildSemanticIndex: () => invoke<number>("rebuild_semantic_index"),
|
||||||
|
|
||||||
getOrphanedThumbnailsInfo: () => invoke<OrphanedThumbnailsInfo>("get_orphaned_thumbnails_info"),
|
getOrphanedThumbnailsInfo: () => invoke<OrphanedThumbnailsInfo>("get_orphaned_thumbnails_info"),
|
||||||
|
|
||||||
cleanupOrphanedThumbnails: () => invoke<CleanupOrphanedThumbnailsResult>("cleanup_orphaned_thumbnails"),
|
cleanupOrphanedThumbnails: () => invoke<CleanupOrphanedThumbnailsResult>("cleanup_orphaned_thumbnails"),
|
||||||
|
|||||||
Reference in New Issue
Block a user