Compare commits
3 Commits
v0.2.0
..
f93a80bc87
| Author | SHA1 | Date | |
|---|---|---|---|
| f93a80bc87 | |||
| a9dd2b2797 | |||
| bee6adc61a |
@@ -1,22 +0,0 @@
|
|||||||
# Keep LF in the working copy for all text files (prettier enforces LF)
|
|
||||||
* text=auto eol=lf
|
|
||||||
|
|
||||||
# Windows scripts that genuinely need CRLF
|
|
||||||
*.bat text eol=crlf
|
|
||||||
*.cmd text eol=crlf
|
|
||||||
*.ps1 text eol=crlf
|
|
||||||
|
|
||||||
# Binary assets — never touch line endings
|
|
||||||
*.png binary
|
|
||||||
*.jpg binary
|
|
||||||
*.jpeg binary
|
|
||||||
*.webp binary
|
|
||||||
*.gif binary
|
|
||||||
*.ico binary
|
|
||||||
*.icns binary
|
|
||||||
*.woff binary
|
|
||||||
*.woff2 binary
|
|
||||||
*.ttf binary
|
|
||||||
*.otf binary
|
|
||||||
*.mp4 binary
|
|
||||||
*.onnx binary
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
# External CI
|
|
||||||
|
|
||||||
CI and release builds run on the GitHub mirror because they require Windows
|
|
||||||
runner capacity. The GitHub workflows report their state back to this Gitea
|
|
||||||
repository through the commit status API.
|
|
||||||
|
|
||||||
Keep this directory in the repository. Gitea checks `.gitea/workflows` before
|
|
||||||
falling back to `.github/workflows`; an existing directory with no workflow
|
|
||||||
files prevents the GitHub-only workflows from being queued by Gitea Actions.
|
|
||||||
|
|
||||||
## Setup
|
|
||||||
|
|
||||||
1. In Gitea, create an access token with `write:repository` permission for an
|
|
||||||
account that can update `JezzWTF/phokus`.
|
|
||||||
2. In the GitHub repository, add that token as the Actions repository secret
|
|
||||||
`GITEA_STATUS_TOKEN`.
|
|
||||||
|
|
||||||
The status steps are non-blocking. If Gitea is temporarily unavailable, the
|
|
||||||
GitHub build result is preserved and the failed status update remains visible
|
|
||||||
in the workflow log.
|
|
||||||
@@ -1,149 +0,0 @@
|
|||||||
name: CI
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
paths:
|
|
||||||
- '.github/workflows/ci.yml'
|
|
||||||
# CHANGELOG.md is a build input: src/changelog.ts imports it ?raw for
|
|
||||||
# the What's New modal, and changelog.test.ts asserts on its content.
|
|
||||||
- 'CHANGELOG.md'
|
|
||||||
- 'index.html'
|
|
||||||
- 'package.json'
|
|
||||||
- 'pnpm-lock.yaml'
|
|
||||||
- 'pnpm-workspace.yaml'
|
|
||||||
- 'src/**'
|
|
||||||
- 'src-tauri/**'
|
|
||||||
- 'tsconfig*.json'
|
|
||||||
- 'vite.config.ts'
|
|
||||||
pull_request:
|
|
||||||
paths:
|
|
||||||
- '.github/workflows/ci.yml'
|
|
||||||
# CHANGELOG.md is a build input: src/changelog.ts imports it ?raw for
|
|
||||||
# the What's New modal, and changelog.test.ts asserts on its content.
|
|
||||||
- 'CHANGELOG.md'
|
|
||||||
- 'index.html'
|
|
||||||
- 'package.json'
|
|
||||||
- 'pnpm-lock.yaml'
|
|
||||||
- 'pnpm-workspace.yaml'
|
|
||||||
- 'src/**'
|
|
||||||
- 'src-tauri/**'
|
|
||||||
- 'tsconfig*.json'
|
|
||||||
- 'vite.config.ts'
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: ${{ github.workflow }}-${{ github.ref }}
|
|
||||||
cancel-in-progress: true
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
check:
|
|
||||||
# windows-latest to match the release target — the ML crates (ort, candle)
|
|
||||||
# and the NSIS bundle only ever ship from Windows.
|
|
||||||
runs-on: windows-latest
|
|
||||||
timeout-minutes: 60
|
|
||||||
env:
|
|
||||||
CARGO_INCREMENTAL: 0
|
|
||||||
GITEA_STATUS_TOKEN: ${{ secrets.GITEA_STATUS_TOKEN }}
|
|
||||||
steps:
|
|
||||||
- name: Report pending status to Gitea
|
|
||||||
if: github.event_name != 'pull_request' && env.GITEA_STATUS_TOKEN != ''
|
|
||||||
continue-on-error: true
|
|
||||||
shell: pwsh
|
|
||||||
env:
|
|
||||||
GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
||||||
run: |
|
|
||||||
$headers = @{
|
|
||||||
Authorization = "token $env:GITEA_STATUS_TOKEN"
|
|
||||||
Accept = "application/json"
|
|
||||||
}
|
|
||||||
$body = @{
|
|
||||||
state = "pending"
|
|
||||||
context = "github/actions/ci"
|
|
||||||
description = "GitHub Actions CI is running"
|
|
||||||
target_url = $env:GITHUB_RUN_URL
|
|
||||||
} | ConvertTo-Json
|
|
||||||
Invoke-RestMethod `
|
|
||||||
-Method Post `
|
|
||||||
-Uri "https://git.jezz.wtf/api/v1/repos/JezzWTF/phokus/statuses/$env:GITHUB_SHA" `
|
|
||||||
-Headers $headers `
|
|
||||||
-ContentType "application/json" `
|
|
||||||
-Body $body
|
|
||||||
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- uses: pnpm/action-setup@v4
|
|
||||||
with:
|
|
||||||
version: 11
|
|
||||||
|
|
||||||
- uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: 22
|
|
||||||
cache: pnpm
|
|
||||||
cache-dependency-path: pnpm-lock.yaml
|
|
||||||
|
|
||||||
- name: Install frontend dependencies
|
|
||||||
run: pnpm install --frozen-lockfile
|
|
||||||
|
|
||||||
- name: Unit tests
|
|
||||||
run: pnpm test:unit
|
|
||||||
|
|
||||||
# tsc + vite build; also produces dist/ which tauri's build script expects
|
|
||||||
- name: Type-check and build frontend
|
|
||||||
run: pnpm build:vite
|
|
||||||
|
|
||||||
- name: Install Rust stable
|
|
||||||
uses: dtolnay/rust-toolchain@stable
|
|
||||||
with:
|
|
||||||
components: clippy, rustfmt
|
|
||||||
|
|
||||||
- uses: Swatinem/rust-cache@v2
|
|
||||||
with:
|
|
||||||
workspaces: src-tauri
|
|
||||||
|
|
||||||
- name: Rustfmt
|
|
||||||
working-directory: src-tauri
|
|
||||||
run: cargo fmt --check
|
|
||||||
|
|
||||||
# --no-default-features: default enables candle-cuda for the main dev
|
|
||||||
# machine; CI runners have no CUDA toolkit and must build CPU-only.
|
|
||||||
- name: Clippy
|
|
||||||
working-directory: src-tauri
|
|
||||||
run: cargo clippy --all-targets --locked --no-default-features -- -D warnings
|
|
||||||
|
|
||||||
- name: Rust tests
|
|
||||||
working-directory: src-tauri
|
|
||||||
run: cargo test --locked --no-default-features
|
|
||||||
|
|
||||||
- name: Report final status to Gitea
|
|
||||||
if: always() && github.event_name != 'pull_request' && env.GITEA_STATUS_TOKEN != ''
|
|
||||||
continue-on-error: true
|
|
||||||
shell: pwsh
|
|
||||||
env:
|
|
||||||
GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
||||||
JOB_STATUS: ${{ job.status }}
|
|
||||||
run: |
|
|
||||||
$state = switch ($env:JOB_STATUS) {
|
|
||||||
"success" { "success" }
|
|
||||||
"failure" { "failure" }
|
|
||||||
default { "error" }
|
|
||||||
}
|
|
||||||
$headers = @{
|
|
||||||
Authorization = "token $env:GITEA_STATUS_TOKEN"
|
|
||||||
Accept = "application/json"
|
|
||||||
}
|
|
||||||
$body = @{
|
|
||||||
state = $state
|
|
||||||
context = "github/actions/ci"
|
|
||||||
description = "GitHub Actions CI finished: $env:JOB_STATUS"
|
|
||||||
target_url = $env:GITHUB_RUN_URL
|
|
||||||
} | ConvertTo-Json
|
|
||||||
Invoke-RestMethod `
|
|
||||||
-Method Post `
|
|
||||||
-Uri "https://git.jezz.wtf/api/v1/repos/JezzWTF/phokus/statuses/$env:GITHUB_SHA" `
|
|
||||||
-Headers $headers `
|
|
||||||
-ContentType "application/json" `
|
|
||||||
-Body $body
|
|
||||||
@@ -1,158 +0,0 @@
|
|||||||
name: Release
|
|
||||||
|
|
||||||
# Tag pushes mirrored from Gitea (git.jezz.wtf) trigger this on the GitHub
|
|
||||||
# mirror. Builds the NSIS installer and attaches it to a DRAFT GitHub Release —
|
|
||||||
# review and publish manually.
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
tags:
|
|
||||||
- 'v*'
|
|
||||||
workflow_dispatch:
|
|
||||||
inputs:
|
|
||||||
tag:
|
|
||||||
description: 'Existing release tag to build, for example v0.1.1'
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
|
||||||
cancel-in-progress: false
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
release:
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
runs-on: windows-latest
|
|
||||||
timeout-minutes: 120
|
|
||||||
env:
|
|
||||||
CARGO_INCREMENTAL: 0
|
|
||||||
GITEA_STATUS_TOKEN: ${{ secrets.GITEA_STATUS_TOKEN }}
|
|
||||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
|
||||||
steps:
|
|
||||||
# refs/tags/ qualification: a branch with a tag-like name must never win
|
|
||||||
# ref resolution. Works for tag pushes too — RELEASE_TAG is the tag name.
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
ref: refs/tags/${{ env.RELEASE_TAG }}
|
|
||||||
|
|
||||||
# On workflow_dispatch GITHUB_SHA is the dispatched branch head, not the
|
|
||||||
# tag's commit — resolve the real one for Gitea commit statuses.
|
|
||||||
- name: Resolve release commit
|
|
||||||
shell: pwsh
|
|
||||||
run: |
|
|
||||||
$sha = git rev-parse HEAD
|
|
||||||
"RELEASE_SHA=$sha" | Out-File -FilePath $env:GITHUB_ENV -Append
|
|
||||||
|
|
||||||
- name: Verify release version
|
|
||||||
shell: pwsh
|
|
||||||
run: |
|
|
||||||
if ($env:RELEASE_TAG -notmatch '^v\d+\.\d+\.\d+(-.+)?$') {
|
|
||||||
throw "Release tag '$env:RELEASE_TAG' must look like v0.1.1"
|
|
||||||
}
|
|
||||||
|
|
||||||
$packageVersion = (Get-Content package.json -Raw | ConvertFrom-Json).version
|
|
||||||
$cargoVersion = (Select-String -Path src-tauri/Cargo.toml -Pattern '^version\s*=\s*"(.+)"').Matches[0].Groups[1].Value
|
|
||||||
|
|
||||||
if ($packageVersion -ne $cargoVersion) {
|
|
||||||
throw "package.json version ($packageVersion) does not match Cargo.toml version ($cargoVersion)"
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($env:RELEASE_TAG -ne "v$packageVersion") {
|
|
||||||
throw "Release tag '$env:RELEASE_TAG' does not match project version v$packageVersion"
|
|
||||||
}
|
|
||||||
|
|
||||||
- name: Report pending status to Gitea
|
|
||||||
if: env.GITEA_STATUS_TOKEN != ''
|
|
||||||
continue-on-error: true
|
|
||||||
shell: pwsh
|
|
||||||
env:
|
|
||||||
GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
||||||
run: |
|
|
||||||
$headers = @{
|
|
||||||
Authorization = "token $env:GITEA_STATUS_TOKEN"
|
|
||||||
Accept = "application/json"
|
|
||||||
}
|
|
||||||
$body = @{
|
|
||||||
state = "pending"
|
|
||||||
context = "github/actions/release"
|
|
||||||
description = "GitHub Actions release is running"
|
|
||||||
target_url = $env:GITHUB_RUN_URL
|
|
||||||
} | ConvertTo-Json
|
|
||||||
Invoke-RestMethod `
|
|
||||||
-Method Post `
|
|
||||||
-Uri "https://git.jezz.wtf/api/v1/repos/JezzWTF/phokus/statuses/$env:RELEASE_SHA" `
|
|
||||||
-Headers $headers `
|
|
||||||
-ContentType "application/json" `
|
|
||||||
-Body $body
|
|
||||||
|
|
||||||
- uses: pnpm/action-setup@v4
|
|
||||||
with:
|
|
||||||
version: 11
|
|
||||||
|
|
||||||
- uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: 22
|
|
||||||
cache: pnpm
|
|
||||||
cache-dependency-path: pnpm-lock.yaml
|
|
||||||
|
|
||||||
- name: Install Rust stable
|
|
||||||
uses: dtolnay/rust-toolchain@stable
|
|
||||||
|
|
||||||
- uses: Swatinem/rust-cache@v2
|
|
||||||
with:
|
|
||||||
workspaces: src-tauri
|
|
||||||
|
|
||||||
- name: Install frontend dependencies
|
|
||||||
run: pnpm install --frozen-lockfile
|
|
||||||
|
|
||||||
- name: Build and create draft release
|
|
||||||
uses: tauri-apps/tauri-action@v0.6.2
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
# Updater artifact signing (Phase 3) — set these repo secrets once
|
|
||||||
# the updater keypair exists; empty values are ignored until then.
|
|
||||||
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
|
|
||||||
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
|
|
||||||
with:
|
|
||||||
tagName: ${{ env.RELEASE_TAG }}
|
|
||||||
releaseName: 'Phokus v__VERSION__'
|
|
||||||
releaseBody: 'See the assets below to download and install this version.'
|
|
||||||
releaseDraft: true
|
|
||||||
prerelease: false
|
|
||||||
includeUpdaterJson: true
|
|
||||||
updaterJsonPreferNsis: true
|
|
||||||
# Cargo args after `--`: ship the CPU/DirectML build — default
|
|
||||||
# features enable candle-cuda, which runners (and most users) lack.
|
|
||||||
args: '-- --no-default-features'
|
|
||||||
|
|
||||||
- name: Report final status to Gitea
|
|
||||||
if: always() && env.GITEA_STATUS_TOKEN != ''
|
|
||||||
continue-on-error: true
|
|
||||||
shell: pwsh
|
|
||||||
env:
|
|
||||||
GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
||||||
JOB_STATUS: ${{ job.status }}
|
|
||||||
run: |
|
|
||||||
$state = switch ($env:JOB_STATUS) {
|
|
||||||
"success" { "success" }
|
|
||||||
"failure" { "failure" }
|
|
||||||
default { "error" }
|
|
||||||
}
|
|
||||||
$headers = @{
|
|
||||||
Authorization = "token $env:GITEA_STATUS_TOKEN"
|
|
||||||
Accept = "application/json"
|
|
||||||
}
|
|
||||||
$body = @{
|
|
||||||
state = $state
|
|
||||||
context = "github/actions/release"
|
|
||||||
description = "GitHub Actions release finished: $env:JOB_STATUS"
|
|
||||||
target_url = $env:GITHUB_RUN_URL
|
|
||||||
} | ConvertTo-Json
|
|
||||||
# RELEASE_SHA is unset if the job died before checkout; fall back.
|
|
||||||
$sha = if ($env:RELEASE_SHA) { $env:RELEASE_SHA } else { $env:GITHUB_SHA }
|
|
||||||
Invoke-RestMethod `
|
|
||||||
-Method Post `
|
|
||||||
-Uri "https://git.jezz.wtf/api/v1/repos/JezzWTF/phokus/statuses/$sha" `
|
|
||||||
-Headers $headers `
|
|
||||||
-ContentType "application/json" `
|
|
||||||
-Body $body
|
|
||||||
@@ -29,18 +29,3 @@ dist-ssr
|
|||||||
|
|
||||||
# Local staging area
|
# Local staging area
|
||||||
/staging
|
/staging
|
||||||
|
|
||||||
# Misc
|
|
||||||
*.py
|
|
||||||
*.pyc
|
|
||||||
|
|
||||||
# Bundled CUDA runtime DLLs for the CUDA build (copied from the toolkit
|
|
||||||
# locally; ~600 MB, never committed).
|
|
||||||
src-tauri/cuda-redist/
|
|
||||||
|
|
||||||
# Playwright
|
|
||||||
/test-results/
|
|
||||||
/playwright-report/
|
|
||||||
/blob-report/
|
|
||||||
/playwright/.cache/
|
|
||||||
/playwright/.auth/
|
|
||||||
|
|||||||
@@ -1,30 +0,0 @@
|
|||||||
# Build outputs
|
|
||||||
dist/
|
|
||||||
build/
|
|
||||||
|
|
||||||
# Tauri / Rust (handled by cargo fmt)
|
|
||||||
src-tauri/
|
|
||||||
|
|
||||||
# Lock files & generated
|
|
||||||
pnpm-lock.yaml
|
|
||||||
*.lock
|
|
||||||
|
|
||||||
# Generated
|
|
||||||
src/vite-env.d.ts
|
|
||||||
|
|
||||||
# Public assets
|
|
||||||
public/
|
|
||||||
|
|
||||||
# Website subpackage (has its own config if needed)
|
|
||||||
website/
|
|
||||||
|
|
||||||
# Markdown & docs (hand-written or tool-generated, keep diffs quiet)
|
|
||||||
*.md
|
|
||||||
docs/
|
|
||||||
|
|
||||||
# CI workflows
|
|
||||||
.github/
|
|
||||||
.gitea/
|
|
||||||
|
|
||||||
# Tool-managed config
|
|
||||||
.claude/
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
{
|
|
||||||
"semi": false,
|
|
||||||
"singleQuote": true,
|
|
||||||
"jsxSingleQuote": false,
|
|
||||||
"trailingComma": "es5",
|
|
||||||
"printWidth": 100,
|
|
||||||
"tabWidth": 2,
|
|
||||||
"useTabs": false,
|
|
||||||
"bracketSpacing": true,
|
|
||||||
"bracketSameLine": false,
|
|
||||||
"arrowParens": "always",
|
|
||||||
"endOfLine": "lf",
|
|
||||||
"plugins": ["prettier-plugin-tailwindcss"]
|
|
||||||
}
|
|
||||||
@@ -1,269 +0,0 @@
|
|||||||
# Changelog
|
|
||||||
|
|
||||||
All notable changes to Phokus are documented here. The format is based on
|
|
||||||
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
|
|
||||||
aims to follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
|
|
||||||
(0.x: anything may change between minor versions).
|
|
||||||
|
|
||||||
## [0.2.0] — 2026-07-11
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- **What's New** — after updating, Phokus now greets you with a "What's new"
|
|
||||||
toast that opens a tidy in-app tour of the new version. Added, Changed, and
|
|
||||||
Fixed notes are grouped into collapsible sections, so you can skim the good
|
|
||||||
bits without playing "spot the difference".
|
|
||||||
- **Quick theme switch** — right-click the settings cog in the title bar to
|
|
||||||
swap between Phokus, Subtle Light, and Conventional Dark instantly. No Settings
|
|
||||||
detour required.
|
|
||||||
- **Albums** — make your own cross-folder collections without moving a single
|
|
||||||
file. Albums live in their own sidebar section with cover thumbnails, can be
|
|
||||||
created, renamed, reordered, opened, and cleaned up in Manage mode, and deleting
|
|
||||||
one only removes the grouping.
|
|
||||||
- **Gallery multi-select** — hover a thumbnail's top-left corner to start
|
|
||||||
selecting, then use the floating action bar to tag, rate, favorite, add to an
|
|
||||||
album, or delete a whole batch at once. It also works in similar-image, region,
|
|
||||||
and album views, because bulk work should not disappear the moment you need it.
|
|
||||||
- **Colour search** — narrow the Gallery, Timeline, or tag results by dominant
|
|
||||||
colour using toolbar swatches or a custom picker. Great for those "I know it
|
|
||||||
was mostly blue" moments.
|
|
||||||
- **Album-aware similar search** — similar-image and region searches started from
|
|
||||||
an album can now stay inside that album, jump back to the source folder, or
|
|
||||||
search everything.
|
|
||||||
- **Tag manager** — Explore's Tag Cloud now has a Manage mode for renaming,
|
|
||||||
merging, and deleting tags across the whole library.
|
|
||||||
- **Camera info in the lightbox** — the info panel now shows EXIF details like
|
|
||||||
camera, lens, aperture, shutter speed, ISO, and focal length. Geotagged photos
|
|
||||||
also get a browser link for their GPS coordinates, and already-indexed images
|
|
||||||
do not need a re-index.
|
|
||||||
- **Build badge in Settings** — Settings -> Updates now shows whether you are
|
|
||||||
running the CPU build or the CUDA build.
|
|
||||||
- **Choose your tagging model** — Settings -> AI Workspace now lets you pick
|
|
||||||
between the anime-focused WD tagger and JoyTag, which is better suited to photo
|
|
||||||
libraries and stronger on NSFW concepts (if that's your thing, we don't judge).
|
|
||||||
New users get the same choice during the welcome tour, and each model keeps its
|
|
||||||
own confidence threshold instead of sharing one.
|
|
||||||
- **Reset AI tags** — a new reset action in Settings -> AI Workspace and the Tag
|
|
||||||
manager wipes AI-generated tags for a folder or the whole library, cancelling
|
|
||||||
any tagging still in flight. Tag manager rows now show which tags came from the
|
|
||||||
AI, so you know what you are about to lose before you lose it. Manually-added
|
|
||||||
tags are never touched.
|
|
||||||
- **Related tags in Explore** — Hover over a tag in the Tag Cloud to see the tags
|
|
||||||
that most often appear with it, complete with connection lines and image counts.
|
|
||||||
Handy for finding little clusters you did not know were there.
|
|
||||||
- **Pause workers for longer** — Settings -> General can now remember per-folder
|
|
||||||
worker pauses across app restarts, useful for folders you want to keep in the
|
|
||||||
library but leave out of background processing for now.
|
|
||||||
- **Editable folder path** — the folder picker now has an address bar, so you can
|
|
||||||
paste a path directly while still using breadcrumbs for quick jumps.
|
|
||||||
- **Slideshow mode** — turn the lightbox into a fullscreen, image-only slideshow
|
|
||||||
from whatever collection you are already browsing.
|
|
||||||
- **Add to album from the right-click menu** — right-click any image in the
|
|
||||||
Gallery or Timeline and file it straight into an album from the new "Add to
|
|
||||||
Album" submenu. One image, one click, zero ceremony.
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
|
|
||||||
- **Settings got a spring clean** — preferences are now organised into General,
|
|
||||||
Media, Updates & Setup, Storage, and AI Workspace pages, so update and
|
|
||||||
maintenance controls no longer hide at the bottom of unrelated sections.
|
|
||||||
- **Menus got their act together** — right-click menus (images, folders,
|
|
||||||
albums, the theme switcher) and every dropdown (sort, folder scope, settings,
|
|
||||||
sidebar) now share one style with one set of manners: they stay on screen
|
|
||||||
instead of wandering off the edge, all close on Escape, and right-click menus
|
|
||||||
can do proper submenus now. Subtle Light dresses them all the same way too,
|
|
||||||
instead of saving the nice outfit for one dropdown.
|
|
||||||
- **Neater lightbox details** — image and video metadata now sits in two columns,
|
|
||||||
so the info panel shows more at a glance with less scrolling.
|
|
||||||
- **Faster Explore revisits** — returning to a folder's visual clusters should
|
|
||||||
feel much faster now, even in big libraries.
|
|
||||||
- **Calmer Tag Cloud during AI tagging** — Explore no longer keeps hammering the
|
|
||||||
tag list while a folder is actively being tagged, so tagging stays smoother and
|
|
||||||
the cloud catches up once the work settles.
|
|
||||||
- **Faster first-time clustering** — large libraries build their first visual
|
|
||||||
clusters much more quickly, while still keeping the groups nicely balanced.
|
|
||||||
- **Better tag browsing** — the Tag manager now has live search, sorting
|
|
||||||
(most-used, least-used, A-Z, and Z-A), smooth scrolling for huge tag lists, and
|
|
||||||
it keeps your filter/sort in place while you edit.
|
|
||||||
- **Safer deletion** — deleting media now asks for confirmation and clearly says
|
|
||||||
the file is being removed from disk. This covers gallery bulk delete and the
|
|
||||||
Duplicate Finder.
|
|
||||||
- **Clearer update progress** — Settings -> Updates now shows a real download
|
|
||||||
progress bar with a percentage instead of the old lonely "Downloading" label.
|
|
||||||
- **Better narrow-window layout** — the toolbar, filters, search box, colour
|
|
||||||
picker, sidebar, and lightbox info panel now adapt more gracefully when the
|
|
||||||
window is short on space.
|
|
||||||
- **Tidier Explore clusters** — busier clusters get more room, dense groups
|
|
||||||
overlap less, and everything should stay easier to read and click.
|
|
||||||
- **Faster CPU tagging** — CPU-only AI tagging can now use multiple cores while
|
|
||||||
leaving some breathing room for the rest of the app. GPU tagging is unchanged.
|
|
||||||
- **Smoother tooltips** — Phokus now uses its custom tooltip style across more of
|
|
||||||
the app instead of falling back to the native browser tooltip.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
|
|
||||||
- **Explore no longer flashes the last folder** — switching folders now clears
|
|
||||||
the old clusters/tags and shows a loading state while the new folder catches
|
|
||||||
up.
|
|
||||||
- **Ratings keep your search order** — if you ever rated an image mid-search and
|
|
||||||
watched your results reshuffle themselves, that's over. Similar-image, region,
|
|
||||||
semantic, tag, and album results now stay put.
|
|
||||||
- **Update progress comes back when you need it** — if you dismiss the update
|
|
||||||
prompt and later start the update from the title bar or Settings, the progress
|
|
||||||
toast now reappears instead of hiding away in Settings.
|
|
||||||
- **Subtle Light cleanup** — fixed dark or hard-to-read surfaces, hover states,
|
|
||||||
dialogs, updater buttons, onboarding controls, and green action buttons in the
|
|
||||||
light theme.
|
|
||||||
- **No more self-indexing loops** — adding a broad folder like your whole user
|
|
||||||
profile used to send Phokus off indexing its own thumbnail cache, generating
|
|
||||||
thumbnails of thumbnails until the end of time. It now skips its own app-data
|
|
||||||
directory.
|
|
||||||
- **Background tasks show the active work first** — when one folder is paused and
|
|
||||||
another is processing, the active folder gets the main spot in the background
|
|
||||||
tasks bar.
|
|
||||||
- **First launch fits smaller screens** — on 1366x768-style displays, fresh
|
|
||||||
installs used to open with part of the app tucked below the taskbar. The window
|
|
||||||
now clamps itself to the usable monitor area.
|
|
||||||
- **Explore is clearer in Subtle Light** — cluster captions, buttons, cloud
|
|
||||||
words, hover glows, and the new connection lines now use stronger light-theme
|
|
||||||
colours.
|
|
||||||
- **Explore got a few sharp edges sanded down** — Cluster Cloud uses the in-app
|
|
||||||
tooltip, singular counts now say "1 image", and the folder-scope dropdown no
|
|
||||||
longer hides behind cluster cards.
|
|
||||||
- **AI tagging stays responsive** — starting a big tagging job used to turn the
|
|
||||||
rest of the app into a slideshow. GPU tagging now works in smaller bursts with
|
|
||||||
brief pauses between them, so the UI keeps moving and the first results land
|
|
||||||
sooner.
|
|
||||||
- **Lightbox AI tags wake up on time** — the lightbox's AI tags action no longer
|
|
||||||
stays disabled until you happen to open Settings, and it switches on as soon as
|
|
||||||
a model download finishes.
|
|
||||||
- **Noisy AI tags get cleaned up** — generic low-signal tags from WD and JoyTag
|
|
||||||
are filtered before they are saved, and matching older generated tags are
|
|
||||||
cleaned up on startup. Your manually-added tags are left alone.
|
|
||||||
- **Selected Folders starts empty** — choosing "Selected Folders" for AI tagging
|
|
||||||
no longer pre-selects the first folder. You decide exactly what gets queued.
|
|
||||||
- **A handful of tiny UI papercuts are gone** — the zoom buttons now show the
|
|
||||||
right tile size when you hover them, the folder picker no longer adds an odd
|
|
||||||
trailing slash to Windows drive breadcrumbs, the search field's clear button
|
|
||||||
no longer sits a few pixels too high, and menu labels no longer highlight like
|
|
||||||
text when you drag across them.
|
|
||||||
|
|
||||||
## [0.1.1] — 2026-06-23
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- **Custom multi-folder picker** — replaces the native OS dialog with an
|
|
||||||
in-app folder browser that lets you navigate, stage folders from multiple
|
|
||||||
locations, and add them all in one go. Duplicate roots are skipped
|
|
||||||
automatically; partially-failed batches remove successfully-added entries
|
|
||||||
from the staging panel so only failed folders remain to retry.
|
|
||||||
- **Rebuild semantic index** maintenance action in Settings — drops and
|
|
||||||
recreates the vector tables at the current model dimension, then re-queues
|
|
||||||
every image for embedding. Fixes "dimension mismatch" search errors that
|
|
||||||
occur after switching between CLIP models with different output sizes.
|
|
||||||
- **Video playback settings** — new Video Playback group in Settings with two
|
|
||||||
persisted toggles: "Autoplay in lightbox" (default on) and "Start muted"
|
|
||||||
(default off). Settings apply to the next opened video rather than the
|
|
||||||
current one.
|
|
||||||
- **Timeline scrubber** — a year/month rail on the Timeline view that jumps to
|
|
||||||
any period in the library. Timeline now loads the full filtered set so the
|
|
||||||
scrubber spans the whole library instead of just the first page.
|
|
||||||
- **Folder reordering** in the sidebar — drag-and-drop (with edge auto-scroll)
|
|
||||||
or keyboard (↑/↓ on the drag handle), with the custom order persisted across
|
|
||||||
sessions; the Libraries list also gains A–Z / Z–A / Custom sort.
|
|
||||||
- Failed AI-tagging jobs can now be located from the background worker prompt,
|
|
||||||
including a gallery filter for images with failed tags and an expanded list
|
|
||||||
of failed filenames/errors.
|
|
||||||
- A new theme system adds Phokus, Subtle Light, and Conventional Dark chrome
|
|
||||||
options across the app.
|
|
||||||
- First-run onboarding now includes an inline theme picker so new users can
|
|
||||||
choose their preferred app chrome before continuing the tour.
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
|
|
||||||
- Settings sections are reordered — General is now the first and default
|
|
||||||
section instead of AI Workspace.
|
|
||||||
- The Duplicate Finder group list is now virtualised — only on-screen cards
|
|
||||||
mount, so large result sets (e.g. 5,000+ pairs) scroll without lag rather
|
|
||||||
than mounting every thumbnail at once.
|
|
||||||
- The gallery grid is now row-virtualised, so very large libraries scroll
|
|
||||||
smoothly and only on-screen thumbnails are rendered.
|
|
||||||
- Polished the new theme surfaces before release, including readable
|
|
||||||
subtle-light secondary buttons, failed-worker action buttons, and onboarding
|
|
||||||
controls.
|
|
||||||
- Onboarding preview media keeps the dark gallery/media surface regardless of
|
|
||||||
the active chrome theme.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
|
|
||||||
- **AVIF thumbnails** — AVIF files are now processed correctly by routing
|
|
||||||
thumbnail generation through the bundled FFmpeg path instead of the Rust
|
|
||||||
image decoder (which has no dav1d dependency). Previously-failed AVIF jobs
|
|
||||||
are requeued on startup; JPEG derivatives are fed to the embedding and
|
|
||||||
tagging pipeline while the lightbox continues to display the original file.
|
|
||||||
- Accent text is now readable in the Subtle Light theme.
|
|
||||||
- Folder picker chevron tooltip now correctly shows "No subfolders" for leaf
|
|
||||||
entries instead of "Open folder" in both branches.
|
|
||||||
- Folder picker Unix breadcrumb root now shows "/" instead of always "Home"
|
|
||||||
for non-home paths such as `/mnt/data`.
|
|
||||||
- Video embedding jobs are no longer claimed before their thumbnail exists, and
|
|
||||||
any that previously failed for that reason are requeued on startup — videos no
|
|
||||||
longer churn through failed embeddings.
|
|
||||||
- Subtle Light theme consistency — the lightbox metadata panel now follows the
|
|
||||||
light chrome while the image canvas stays dark (matching Conventional Dark),
|
|
||||||
and gallery/timeline media badges, duplicate-finder thumbnails, and the window
|
|
||||||
restore icon now theme correctly instead of staying Phokus-dark.
|
|
||||||
- Timeline scrolling is now smooth on large libraries — it virtualizes per row
|
|
||||||
of tiles instead of per month, so a month with thousands of photos no longer
|
|
||||||
mounts every tile at once (thumbnails now load in incrementally as you scroll,
|
|
||||||
matching the All Media grid).
|
|
||||||
- Background worker updates (thumbnails, metadata, embeddings, tags) no longer
|
|
||||||
re-sort the entire loaded image set on every batch. In Timeline, which loads
|
|
||||||
the whole library, this re-sort caused severe lag and could crash the app
|
|
||||||
during background indexing.
|
|
||||||
|
|
||||||
## [0.1.0] — 2026-06-14
|
|
||||||
|
|
||||||
First public release. Windows desktop, distributed as an unsigned NSIS
|
|
||||||
installer with a built-in updater.
|
|
||||||
|
|
||||||
### Added
|
|
||||||
|
|
||||||
- **Local media library** — add folders, recursive background indexing with
|
|
||||||
live progress, and a filesystem watcher that keeps the library in sync as
|
|
||||||
files are added, edited, moved, renamed, or removed (thumbnails and
|
|
||||||
embeddings are preserved across renames).
|
|
||||||
- **Gallery** — virtualized grid (handles very large libraries), favorites,
|
|
||||||
star ratings, video durations; filter by folder, type, favorites, or
|
|
||||||
rating; sort by date added, date taken (EXIF), name, size, rating, or
|
|
||||||
duration; compact / comfortable / detail density.
|
|
||||||
- **Search** — filename, semantic (`/s`, via CLIP visual embeddings), and tag
|
|
||||||
(`/t`) search from one prefix-aware search bar.
|
|
||||||
- **Discovery** — similar-image search (by image or selected region), an
|
|
||||||
Explore view with a visual cluster map and tag cloud, and a Timeline grouped
|
|
||||||
by EXIF capture date. Explore, Timeline, and Duplicates are folder-scopable
|
|
||||||
from their headers.
|
|
||||||
- **Lightbox** — keyboard navigation, zoom, pan, inline tag editing, and
|
|
||||||
rating controls, plus a custom edge-to-edge video player (scrubbing, volume,
|
|
||||||
speed, loop, fullscreen, keyboard support).
|
|
||||||
- **AI tagging** — WD tagger (ONNX, CPU/DirectML) with adjustable confidence
|
|
||||||
threshold, batch size, and per-folder queue targeting. Optional.
|
|
||||||
- **Duplicate finder** — three-phase exact-duplicate scan
|
|
||||||
(size → sample hash → full hash) with live progress and bulk delete.
|
|
||||||
- **Background pipeline** — strict-priority workers (thumbnails → metadata →
|
|
||||||
embeddings → tags) with per-folder pausing from the sidebar context menu or
|
|
||||||
the background-tasks bar.
|
|
||||||
- **Guided first-run onboarding** — background FFmpeg provisioning with live
|
|
||||||
progress and retry, a walkthrough of the library, pipeline, search modes,
|
|
||||||
views, and updates, plus an optional AI-tagger download. Re-runnable from
|
|
||||||
Settings.
|
|
||||||
- **Updater** — checks GitHub Releases on launch and from Settings; a title-bar
|
|
||||||
indicator lights up when a new version is ready, and one click downloads,
|
|
||||||
installs the signed update, and relaunches.
|
|
||||||
- **Maintenance** — database compaction and orphaned-thumbnail cleanup from
|
|
||||||
Settings, with live size/reclaimable stats.
|
|
||||||
- **Window state** persistence and single-instance handling.
|
|
||||||
|
|
||||||
[0.2.0]: https://github.com/JezzWTF/phokus/releases/tag/v0.2.0
|
|
||||||
[0.1.1]: https://github.com/JezzWTF/phokus/releases/tag/v0.1.1
|
|
||||||
[0.1.0]: https://github.com/JezzWTF/phokus/releases/tag/v0.1.0
|
|
||||||
@@ -1,133 +0,0 @@
|
|||||||
# CLAUDE.md
|
|
||||||
|
|
||||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
||||||
|
|
||||||
## Project overview
|
|
||||||
|
|
||||||
**Phokus** — a Tauri v2 desktop image gallery app with a React/TypeScript frontend and a Rust backend. The app indexes local media folders, generates thumbnails via FFmpeg, computes visual embeddings for semantic search and similarity, and AI-tags images using the WD tagger (ONNX via `ort`). AI captioning code exists in the backend but the UI surface has been removed; the worker is commented out in `lib.rs`.
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Development (Vite hot-reload + Rust auto-rebuild)
|
|
||||||
pnpm dev:app
|
|
||||||
|
|
||||||
# Frontend only (no Tauri window)
|
|
||||||
pnpm dev:vite
|
|
||||||
|
|
||||||
# UI Lab — browser-only frontend with mocked Tauri backend (http://127.0.0.1:1422)
|
|
||||||
pnpm dev:ui
|
|
||||||
|
|
||||||
# Production build (CPU)
|
|
||||||
pnpm build:app:cpu
|
|
||||||
|
|
||||||
# Production build (CUDA / GPU-accelerated)
|
|
||||||
pnpm build:app:cuda
|
|
||||||
|
|
||||||
# Type-check frontend
|
|
||||||
pnpm build:vite
|
|
||||||
|
|
||||||
# Frontend unit tests (Vitest; only picks up src/**/*.test.ts)
|
|
||||||
pnpm test:unit
|
|
||||||
pnpm test:unit:watch
|
|
||||||
|
|
||||||
# Rust unit tests (in-memory SQLite; --no-default-features skips CUDA)
|
|
||||||
pnpm test:rust
|
|
||||||
|
|
||||||
# E2E tests (Playwright against the UI Lab; auto-starts the server)
|
|
||||||
pnpm test:e2e
|
|
||||||
pnpm exec playwright test tests/ui-lab.spec.ts # single file
|
|
||||||
pnpm exec playwright test -g "filename search" # single test by name
|
|
||||||
|
|
||||||
# Formatting (Prettier + prettier-plugin-tailwindcss; cargo fmt for Rust)
|
|
||||||
pnpm format:all
|
|
||||||
```
|
|
||||||
|
|
||||||
Use **pnpm** — never npm.
|
|
||||||
|
|
||||||
Three test layers, none of which exercise the real Tauri window:
|
|
||||||
- **Vitest unit tests** — co-located `*.test.ts` next to the pure logic they cover (`src/store/helpers.ts`, formatters, path utils); fixture factories in `src/test/factories.ts`.
|
|
||||||
- **Rust unit tests** — inline `#[cfg(test)]` modules; DB tests run against in-memory SQLite via the shared `db::test_support` fixture (`test_conn()` + `test_image()`), which registers sqlite-vec and applies both migrations. Reuse it for any new DB-touching tests.
|
|
||||||
- **Playwright e2e smoke tests** in `tests/` — run against the UI Lab (browser mocks).
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
### Frontend (`src/`)
|
|
||||||
|
|
||||||
- **`src/store/`** — single Zustand store (`useGalleryStore`) that owns all app state and all `invoke()` calls to the Tauri backend, split into per-feature slices combined in `index.ts` (which exports `useGalleryStore` and the `GalleryStore` type). `types.ts` holds all interfaces/type unions (including `ImageRecord`); `helpers.ts` holds pure functions and cross-slice module state (search parsing, image sort/merge, request-token guards). Slices: `librarySlice` (folders), `gallerySlice` (image paging/filters/bulk actions), `searchSlice` (search + similar-images), `exploreSlice` (visual clusters, tag cloud, tags), `albumSlice`, `duplicateSlice`, `taggerSlice`, `captionSlice`, `settingsSlice`, `appSlice` (updates, onboarding, ffmpeg, worker pauses); `events.ts` wires the Tauri event listeners (`subscribeToProgress`). Components still call `useGalleryStore(s => s.field)` against one flat state object — the slice split is internal. React components are thin consumers.
|
|
||||||
- **`App.tsx`** — sets up Tauri event listeners (`subscribeToProgress`) and renders the top-level layout (sidebar + active view).
|
|
||||||
- **`src/components/`** — UI components: `Gallery`, `Lightbox`, `Sidebar`, `Toolbar`, `Timeline`, `ExploreView`, `DuplicateFinder`, `BackgroundTasks`, `SettingsModal`, `TitleBar`.
|
|
||||||
- **`src/components/menu/`** — shared floating-UI primitives: `useDismissable`, `MenuPanel`/`MenuItem`/`SubMenu`, the portal-based `ContextMenu`, and the app-wide `Dropdown`. Build menus, dropdowns, and popovers on these instead of hand-rolling.
|
|
||||||
- State management: Zustand v5, no selectors library — components call `useGalleryStore(s => s.field)` directly.
|
|
||||||
- Styling: Tailwind CSS v4 (Vite plugin, no config file).
|
|
||||||
- Virtualized gallery grid: `@tanstack/react-virtual`.
|
|
||||||
- Animation: `framer-motion`.
|
|
||||||
|
|
||||||
### UI Lab (`src/dev/`)
|
|
||||||
|
|
||||||
`pnpm dev:ui` runs the real frontend (same `App.tsx`, store, components, CSS) in a plain browser with Tauri fully mocked — no Rust backend or Tauri window. `src/main.tsx` imports `src/dev/setupMockTauri.ts` before `App` in `ui` mode; `mockBackend.ts` implements an in-memory command backend, with fixtures in `mockFixtures.ts`/`mockScenarios.ts`. Pick a seeded state via `?scenario=` (`rich` default; also `empty`, `new-user`, `just-updated`, `busy`, `duplicates`, `album`, `errors`, `huge`) and a What's New entry via `?changelog=`. `Ctrl+Shift+D` opens the demo panel. Full guide: `docs/ui-lab.md`.
|
|
||||||
|
|
||||||
Rules that keep UI Lab working:
|
|
||||||
- Components that render thumbnails/covers/posters must use `mediaSrc(...)` from `src/lib/mediaSrc.ts`, never `convertFileSrc(...)` directly.
|
|
||||||
- New Tauri commands invoked from the frontend need a mock in `src/dev/mockBackend.ts` (unmocked commands log console errors, which fail the e2e tests).
|
|
||||||
|
|
||||||
UI Lab is for visual/layout work and agent browser inspection. Native behavior (file pickers, real thumbnails, window controls, updater) must still be validated in `pnpm dev:app`.
|
|
||||||
|
|
||||||
### Search modes
|
|
||||||
|
|
||||||
The search bar supports prefix syntax parsed by `parseSearchValue` in `src/store/helpers.ts`:
|
|
||||||
- No prefix / `f:` — filename search (paginated, DB-backed)
|
|
||||||
- `/s <query>` or `s: <query>` — semantic (embedding) search
|
|
||||||
- `/t <tag>` or `t: <tag>` — tag search
|
|
||||||
|
|
||||||
### Backend (`src-tauri/src/`)
|
|
||||||
|
|
||||||
Workers are started in `lib.rs` and run as background threads throughout the app lifetime:
|
|
||||||
- **thumbnail worker** (multiple threads, count from `StorageProfile::Balanced`)
|
|
||||||
- **metadata worker** — FFmpeg probe for video files
|
|
||||||
- **embedding worker** — generates CLIP-style visual embeddings (candle, HuggingFace hub)
|
|
||||||
- **tagging worker** — WD tagger via ONNX Runtime (`ort`), DirectML/CPU acceleration
|
|
||||||
|
|
||||||
Key modules:
|
|
||||||
| File | Purpose |
|
|
||||||
|------|---------|
|
|
||||||
| `db.rs` | SQLite pool (r2d2 + rusqlite), schema migrations, all query functions |
|
|
||||||
| `commands.rs` | All `#[tauri::command]` handlers — one-to-one with frontend `invoke()` calls |
|
|
||||||
| `indexer.rs` | Worker thread launchers and job dispatch |
|
|
||||||
| `embedder.rs` | Visual embedding generation (candle + HF hub models) |
|
|
||||||
| `vector.rs` | sqlite-vec integration + HNSW index for ANN search |
|
|
||||||
| `hnsw_index.rs` | In-memory HNSW index wrapper (hnsw_rs) |
|
|
||||||
| `tagger.rs` | WD tagger: ONNX model download, inference, CSV tag loading |
|
|
||||||
| `captioner.rs` | AI captioning (ONNX, disabled in workers but code intact) |
|
|
||||||
| `download.rs` | Resilient file downloads via the system `curl` (resume, stall detection) |
|
|
||||||
| `onnx_runtime.rs` | Shared ONNX Runtime DLL provisioning + `ort` init (used by tagger and captioner; DLLs live in the caption model dir for legacy reasons) |
|
|
||||||
| `thumbnail.rs` | Thumbnail generation (image crate + fast_image_resize, FFmpeg for video) |
|
|
||||||
| `media.rs` | FFmpeg sidecar provisioning and probing |
|
|
||||||
| `storage.rs` | `StorageProfile` for tuning worker counts |
|
|
||||||
|
|
||||||
Database: SQLite with WAL mode, stored in the Tauri app data directory as `gallery.db`. Thumbnails stored alongside as `thumbnails/`.
|
|
||||||
|
|
||||||
### Tauri events (backend → frontend)
|
|
||||||
|
|
||||||
| Event | Payload |
|
|
||||||
|-------|---------|
|
|
||||||
| `index-progress` | `IndexProgress` |
|
|
||||||
| `media-job-progress` | `MediaJobProgressEvent` |
|
|
||||||
| `indexed-images` | `IndexedImagesBatch` |
|
|
||||||
| `media-updated` | `ThumbnailBatch` |
|
|
||||||
| `caption-model-progress` | `CaptionModelProgress` |
|
|
||||||
| `tagger-model-progress` | `TaggerModelProgress` |
|
|
||||||
| `duplicate_scan_progress` | `{ phase, processed, total, skipped }` |
|
|
||||||
|
|
||||||
### Key types
|
|
||||||
|
|
||||||
`ImageRecord` (mirrored in `src/store/types.ts` and `db.rs`) is the central data type. It carries embedding status, tagging status, caption data, and media metadata. The frontend type must stay in sync with the Rust struct serialization.
|
|
||||||
|
|
||||||
## Development notes
|
|
||||||
|
|
||||||
- Hot Reload is active during `dev:app` — do not restart the server for frontend changes. Restart only when adding Rust crates or changing Vite config.
|
|
||||||
- ML inference crates (`candle-*`, `ort`, `image`, `rayon`, `tokenizers`, `xxhash-rust`, `rusqlite`) use `opt-level = 3` in dev profile to keep inference performance acceptable.
|
|
||||||
- The caption worker is intentionally disabled (`lib.rs:73`) — the backend code is intact for future re-enabling.
|
|
||||||
- **Never use `any` type** in TypeScript — look up correct types.
|
|
||||||
- `website/` is a separate Vite project for the marketing site (phokus.jezz.wtf): `pnpm dev:web` / `pnpm build:web`. It is not part of the app build.
|
|
||||||
- `pnpm changelog:add -- --type fixed --message "..."` appends an entry to the `[Unreleased]` section of `CHANGELOG.md`, which also feeds the in-app What's New modal (types: added, changed, deprecated, removed, fixed, security).
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2026 JezzWTF
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
@@ -1,135 +1,116 @@
|
|||||||
# Phokus
|
# Phokus
|
||||||
|
|
||||||
A local-first desktop media library for browsing, filtering, and curating image and video folders.
|
## Overview
|
||||||
|
|
||||||
## Features
|
Phokus is a Tauri desktop app for building a fast, local media library from folders on disk. It indexes images and videos, stores metadata in SQLite, and gives you a dense browsing workflow with filtering, favorites, ratings, and a lightbox preview.
|
||||||
|
|
||||||
### Library
|
The current app is optimized for:
|
||||||
- Add and remove media folders; background indexing with live progress
|
|
||||||
- **Live file tracking** — a filesystem watcher keeps the library in sync as files are added, changed, or removed; renames and moves are handled in place, preserving thumbnails and embeddings
|
|
||||||
- Browse all media or filter by folder, type (image/video), favorites, or star rating
|
|
||||||
- Sort by date added, date taken (EXIF), name, size, rating, or duration
|
|
||||||
- Grid density controls (compact / comfortable / detail)
|
|
||||||
- Desktop notifications, batched per folder, with per-folder mute and a global pause
|
|
||||||
|
|
||||||
### Search & discovery
|
- local folders instead of cloud import flows
|
||||||
- **Filename search**, **semantic search** (`/s query`), and **tag search** (`/t tag`)
|
- large visual libraries
|
||||||
- **Similar image search** — find visually similar media by image or a selected region
|
- quick review and curation
|
||||||
- **Explore view** — visual cluster map and tag cloud for browsing by theme
|
- mixed image and video browsing
|
||||||
- **Timeline view** — media grouped chronologically by capture date (EXIF)
|
|
||||||
- **Duplicate finder** — three-phase exact-duplicate scan (size → sample hash → full hash) with live progress and bulk delete
|
|
||||||
- Explore, Timeline, and Duplicates are folder-scopable directly from their headers — no need to bounce through the sidebar
|
|
||||||
|
|
||||||
### Viewing & curation
|
## Current features
|
||||||
- Lightbox with keyboard navigation, zoom, inline tag editing, and rating controls
|
|
||||||
- **Custom video player** — immersive edge-to-edge playback with scrubbing, volume, playback speed, loop, and fullscreen; auto-hiding controls and full keyboard support
|
|
||||||
- **AI tagging** via WD tagger (ONNX, CPU/DirectML) with confidence threshold, batch size, and per-folder queue targeting
|
|
||||||
|
|
||||||
### Maintenance
|
- Add and remove media folders
|
||||||
- Database compaction and orphaned-thumbnail cleanup from Settings, with live size/reclaimable stats
|
- Background indexing with progress updates
|
||||||
- Per-folder pausing of background work (thumbnails, metadata, embeddings, tagging)
|
- Browse all media or filter by folder
|
||||||
|
- Search by filename
|
||||||
|
- Filter by images, videos, or favorites
|
||||||
|
- Sort by modified date, name, or file size
|
||||||
|
- Grid density controls
|
||||||
|
- Lightbox preview with keyboard navigation
|
||||||
|
- Favorite and star-rating metadata saved in SQLite
|
||||||
|
- Virtualized/local-first architecture built on Tauri + React
|
||||||
|
|
||||||
## Supported formats
|
## Supported formats
|
||||||
|
|
||||||
| Images | Videos |
|
Images:
|
||||||
|--------|--------|
|
|
||||||
| jpg, jpeg, png, gif, bmp | mp4, mov, m4v |
|
|
||||||
| tiff, tif, webp, avif | webm |
|
|
||||||
|
|
||||||
## Installation
|
- `jpg`
|
||||||
|
- `jpeg`
|
||||||
|
- `png`
|
||||||
|
- `gif`
|
||||||
|
- `bmp`
|
||||||
|
- `tiff`
|
||||||
|
- `tif`
|
||||||
|
- `webp`
|
||||||
|
- `avif`
|
||||||
|
- `heic`
|
||||||
|
- `heif`
|
||||||
|
|
||||||
Phokus is a **Windows desktop app**. Download the latest installer from the
|
Videos:
|
||||||
[Releases page](https://github.com/JezzWTF/phokus/releases/latest) and run it.
|
|
||||||
|
|
||||||
**Requirements:** Windows 10 or 11. The installer will fetch the WebView2
|
- `mp4`
|
||||||
runtime automatically if it isn't already present (it ships with Windows 11).
|
- `mov`
|
||||||
|
- `m4v`
|
||||||
### A note on the unsigned build
|
- `webm`
|
||||||
|
|
||||||
Phokus 0.1.0 installers are **not code-signed**. Code-signing certificates are
|
|
||||||
an ongoing expense that's hard to justify for an unfunded open-source project,
|
|
||||||
so signing is on the roadmap rather than in place today.
|
|
||||||
|
|
||||||
In practice this means Windows SmartScreen will show a blue
|
|
||||||
**"Windows protected your PC"** warning the first time you run the installer.
|
|
||||||
To proceed: click **More info → Run anyway**. If a release publishes a SHA-256
|
|
||||||
checksum, you can verify the download against it first.
|
|
||||||
|
|
||||||
### First run
|
|
||||||
|
|
||||||
On first launch Phokus downloads a few tools and models — all one-time, and all
|
|
||||||
processed on your machine:
|
|
||||||
|
|
||||||
- **FFmpeg** (~tens of MB) for video thumbnails and metadata — downloaded in the
|
|
||||||
background; the guided first-run tour shows progress.
|
|
||||||
- **WD tagger model** (~1.3 GB) — **optional**, only if you enable AI tagging.
|
|
||||||
- **CLIP embedding model** (~580 MB) — downloaded automatically the first time
|
|
||||||
visual embeddings run (powers semantic search and similar images).
|
|
||||||
|
|
||||||
## Privacy & local-first
|
|
||||||
|
|
||||||
Phokus is local-first by design. **Your media and everything derived from it —
|
|
||||||
thumbnails, embeddings, tags, ratings — never leave your machine.** There is no
|
|
||||||
account, no telemetry, and no cloud sync.
|
|
||||||
|
|
||||||
The only network activity is:
|
|
||||||
|
|
||||||
- **One-time tool/model downloads** on first use (FFmpeg, the CLIP and WD
|
|
||||||
models, and the ONNX runtime), fetched from their official sources
|
|
||||||
(FFmpeg builds, Hugging Face, NuGet). These pull *tools to your machine* —
|
|
||||||
none of your images are uploaded.
|
|
||||||
- **Update checks** against the GitHub Releases page, so the app can offer new
|
|
||||||
versions. This can be ignored if you never update.
|
|
||||||
|
|
||||||
Everything Phokus stores lives in its app-data directory (`gallery.db`,
|
|
||||||
`thumbnails/`, `models/`, `settings/`). Removing that directory resets the app
|
|
||||||
completely; your original media folders are never modified.
|
|
||||||
|
|
||||||
## Stack
|
## Stack
|
||||||
|
|
||||||
- Tauri 2 + Rust backend
|
- Tauri 2
|
||||||
- React 19 + TypeScript + Zustand
|
- React 19
|
||||||
- SQLite + `sqlite-vec` (vector search) + HNSW index
|
- TypeScript
|
||||||
- ONNX Runtime (`ort`) for AI tagging
|
- Zustand
|
||||||
- Candle (Rust ML) for CLIP visual embeddings
|
- Rust
|
||||||
- mozjpeg for fast scaled JPEG decoding
|
- SQLite + `sqlite-vec`
|
||||||
- FFmpeg sidecar for video thumbnails and metadata
|
- Vite
|
||||||
- Vite + Tailwind CSS v4
|
|
||||||
|
## Project structure
|
||||||
|
|
||||||
|
- `src/`: React UI, state, and components
|
||||||
|
- `src-tauri/src/commands.rs`: Tauri command surface
|
||||||
|
- `src-tauri/src/db.rs`: SQLite schema and queries
|
||||||
|
- `src-tauri/src/indexer.rs`: folder crawling and batch indexing
|
||||||
|
- `src-tauri/src/vector.rs`: vector table setup for future semantic workflows
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
**Prerequisites:** Node.js 20+, pnpm, Rust toolchain, Tauri system prerequisites for Windows.
|
### Prerequisites
|
||||||
|
|
||||||
|
- Node.js 20+
|
||||||
|
- `pnpm`
|
||||||
|
- Rust toolchain
|
||||||
|
- Tauri system prerequisites for Windows
|
||||||
|
|
||||||
|
### Install
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pnpm install
|
pnpm install
|
||||||
|
|
||||||
# Run with hot-reload (frontend + Rust)
|
|
||||||
pnpm dev:app
|
|
||||||
|
|
||||||
# Frontend only
|
|
||||||
pnpm dev:vite
|
|
||||||
|
|
||||||
# Browser-only UI Lab with mocked Tauri APIs
|
|
||||||
pnpm dev:ui
|
|
||||||
|
|
||||||
# Production build (CPU)
|
|
||||||
pnpm build:app:cpu
|
|
||||||
|
|
||||||
# Production build (CUDA / GPU-accelerated)
|
|
||||||
pnpm build:app:cuda
|
|
||||||
|
|
||||||
# Type-check the frontend
|
|
||||||
pnpm build:vite
|
|
||||||
```
|
```
|
||||||
|
|
||||||
For visual frontend work without launching Tauri or the Rust backend, see
|
### Run in development
|
||||||
[Phokus UI Lab](docs/ui-lab.md).
|
|
||||||
|
```bash
|
||||||
|
pnpm tauri dev
|
||||||
|
```
|
||||||
|
|
||||||
|
### Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm tauri build
|
||||||
|
```
|
||||||
|
|
||||||
## How it works
|
## How it works
|
||||||
|
|
||||||
1. Add a folder from the sidebar — the Rust indexer walks it recursively.
|
1. Add a folder from the sidebar or Library menu.
|
||||||
2. Supported files are written to SQLite with metadata (path, dimensions, media type, EXIF capture date, etc.).
|
2. The Rust indexer walks the directory recursively.
|
||||||
3. Background workers process the queue as a strict priority pipeline — thumbnails first, then video metadata, then visual embeddings, then AI tags — so each stage runs at full speed instead of competing for CPU, disk, and the database.
|
3. Supported files are written into SQLite with metadata such as path, size, dimensions, media type, rating, and favorite state.
|
||||||
4. Progress events stream back to the UI while the gallery updates incrementally.
|
4. Progress events stream back to the UI while the gallery updates incrementally.
|
||||||
5. A filesystem watcher picks up later changes (new files, edits, deletions, renames) and keeps the index current without rescans.
|
5. The gallery view loads media in pages and opens items in a lightbox for review.
|
||||||
6. Embeddings power semantic search and the similar-images feature via an HNSW index.
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- This is currently a local desktop library, not a sync product.
|
||||||
|
- Search is filename-based right now.
|
||||||
|
- The vector table and embedding fields exist, but semantic search is not wired into the UI yet.
|
||||||
|
- Some visible UI copy may still use the old working name until the frontend text is updated.
|
||||||
|
|
||||||
|
## Positioning
|
||||||
|
|
||||||
|
The clearest product description today is:
|
||||||
|
|
||||||
|
> A local-first desktop media library for browsing, filtering, and curating image and video folders.
|
||||||
|
|
||||||
|
That description is more accurate than "gallery" alone and gives you a better base for future branding, onboarding copy, and a landing page.
|
||||||
|
|||||||
@@ -1,43 +0,0 @@
|
|||||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256" width="256" height="256" role="img" aria-label="Phokus">
|
|
||||||
<title>Phokus</title>
|
|
||||||
|
|
||||||
<!--
|
|
||||||
Phokus app mark — a camera iris.
|
|
||||||
The hexagon in the middle is the lens OPENING; each blade edge sweeps from a
|
|
||||||
corner of the opening out to the rim, all leaning the same way (the pinwheel).
|
|
||||||
|
|
||||||
Tuning knobs (edit and re-open in any browser/Figma):
|
|
||||||
* opening roundness -> the "52" radius in the opening <path> arcs.
|
|
||||||
smaller (toward 30) = rounder/softer hole; larger (toward 90) = flatter, sharper.
|
|
||||||
* blade curve -> the "110" radius in each blade <path>.
|
|
||||||
smaller = more pronounced curve; larger = straighter blade.
|
|
||||||
* blade sweep amount -> the +40deg baked into the blade endpoint (70.71,-84.27).
|
|
||||||
* curve DIRECTION -> the final flag in each "A r r 0 0 X" command (0<->1 flips the bow).
|
|
||||||
* weight -> stroke-width on the <g>.
|
|
||||||
* color -> stroke on the <g>; swap "#e9e9ec" for currentColor to inherit.
|
|
||||||
A brand-accent focal dot is provided at the bottom (commented out).
|
|
||||||
-->
|
|
||||||
|
|
||||||
<rect x="8" y="8" width="240" height="240" rx="52" fill="#0e0f14"/>
|
|
||||||
|
|
||||||
<g transform="translate(128 128)" fill="none" stroke="#e9e9ec" stroke-width="9"
|
|
||||||
stroke-linecap="round" stroke-linejoin="round">
|
|
||||||
|
|
||||||
<!-- outer rim -->
|
|
||||||
<circle r="110"/>
|
|
||||||
|
|
||||||
<!-- lens opening: soft, arc-rounded hexagon -->
|
|
||||||
<path d="M0,-46 A52,52 0 0 0 39.84,-23 A52,52 0 0 0 39.84,23 A52,52 0 0 0 0,46 A52,52 0 0 0 -39.84,23 A52,52 0 0 0 -39.84,-23 A52,52 0 0 0 0,-46 Z"/>
|
|
||||||
|
|
||||||
<!-- blades: one curved edge, swept six times -->
|
|
||||||
<g transform="rotate(0)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
<g transform="rotate(60)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
<g transform="rotate(120)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
<g transform="rotate(180)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
<g transform="rotate(240)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
<g transform="rotate(300)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
</g>
|
|
||||||
|
|
||||||
<!-- optional brand focal point -->
|
|
||||||
<!-- <circle cx="128" cy="128" r="9" fill="#e6a23c"/> -->
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 2.2 KiB |
@@ -1,59 +0,0 @@
|
|||||||
# Phokus v0.1.0
|
|
||||||
|
|
||||||
> Draft for the GitHub Release body. Paste into the release, fill in the
|
|
||||||
> checksum, and trim as needed.
|
|
||||||
|
|
||||||
**Phokus is a local-first desktop media library for Windows** — point it at
|
|
||||||
your image and video folders and it builds a fast, searchable gallery with
|
|
||||||
thumbnails, semantic search, visual discovery, AI tagging, and duplicate
|
|
||||||
cleanup. Everything is processed on your machine; nothing is uploaded.
|
|
||||||
|
|
||||||
This is the **first public release**. Expect rough edges, and please file
|
|
||||||
issues.
|
|
||||||
|
|
||||||
## Install
|
|
||||||
|
|
||||||
1. Download `Phokus_0.1.0_x64-setup.exe` below.
|
|
||||||
2. Run it. **Windows SmartScreen will warn** that the publisher is
|
|
||||||
unrecognized — this build is **not code-signed** (cost; signing is on the
|
|
||||||
roadmap). Click **More info → Run anyway** to proceed.
|
|
||||||
3. Requires **Windows 10/11**. The installer fetches the WebView2 runtime
|
|
||||||
automatically if needed.
|
|
||||||
|
|
||||||
On first launch, Phokus runs a short guided tour and downloads FFmpeg in the
|
|
||||||
background (one-time, ~tens of MB). AI tagging (~1.3 GB) is optional; the CLIP
|
|
||||||
model for semantic search (~580 MB) downloads automatically the first time
|
|
||||||
embeddings run. **All of this stays on your machine.**
|
|
||||||
|
|
||||||
## Highlights
|
|
||||||
|
|
||||||
- Virtualized gallery for very large libraries; favorites, ratings, filters, sorts
|
|
||||||
- Filename, semantic (`/s`), and tag (`/t`) search
|
|
||||||
- Similar-image search, Explore clusters + tag cloud, EXIF Timeline
|
|
||||||
- Lightbox with zoom/pan and a custom video player
|
|
||||||
- Optional on-device AI tagging (WD tagger)
|
|
||||||
- Exact-duplicate finder with bulk delete
|
|
||||||
- Built-in updater
|
|
||||||
|
|
||||||
See the [changelog](https://github.com/JezzWTF/phokus/blob/main/CHANGELOG.md)
|
|
||||||
for the full list.
|
|
||||||
|
|
||||||
## Privacy
|
|
||||||
|
|
||||||
Your media and all derived data (thumbnails, embeddings, tags, ratings) never
|
|
||||||
leave your machine. No account, no telemetry, no cloud. The only network
|
|
||||||
activity is the one-time tool/model downloads above and update checks against
|
|
||||||
this Releases page.
|
|
||||||
|
|
||||||
## Known limitations
|
|
||||||
|
|
||||||
- **Unsigned installer** — SmartScreen warning as described above.
|
|
||||||
- **Windows only** for now.
|
|
||||||
- CPU/DirectML inference in the shipped build; very large libraries take time
|
|
||||||
to embed on first index.
|
|
||||||
|
|
||||||
## Verify your download (optional)
|
|
||||||
|
|
||||||
```
|
|
||||||
SHA-256 (Phokus_0.1.0_x64-setup.exe) = <fill in at publish time>
|
|
||||||
```
|
|
||||||
@@ -1,64 +0,0 @@
|
|||||||
# Phokus v0.1.1
|
|
||||||
|
|
||||||
> Draft for the GitHub Release body. Fill in the checksums and trim as needed.
|
|
||||||
|
|
||||||
**Phokus is a local-first desktop media library for Windows** — point it at
|
|
||||||
your image and video folders and it builds a fast, searchable gallery with
|
|
||||||
thumbnails, semantic search, visual discovery, AI tagging, and duplicate
|
|
||||||
cleanup. Everything is processed on your machine; nothing is uploaded.
|
|
||||||
|
|
||||||
This is a quality-of-life release on top of 0.1.0: a new in-app folder
|
|
||||||
picker, themes, smoother scrolling on large libraries, and a batch of fixes.
|
|
||||||
If you're updating from 0.1.0, the built-in updater will fetch this for you.
|
|
||||||
|
|
||||||
## Install / update
|
|
||||||
|
|
||||||
- **Updating from 0.1.0:** the in-app updater will offer 0.1.1 on launch — one
|
|
||||||
click downloads, installs, and relaunches.
|
|
||||||
- **Fresh install:** download `Phokus_0.1.1_x64-setup.exe` below and run it.
|
|
||||||
**Windows SmartScreen will warn** that the publisher is unrecognized — this
|
|
||||||
build is **not code-signed**. Click **More info → Run anyway**.
|
|
||||||
- Requires **Windows 10/11**. WebView2 is fetched automatically if missing.
|
|
||||||
|
|
||||||
NVIDIA users wanting GPU embedding speed can use the
|
|
||||||
`Phokus_0.1.1_x64-cuda-setup.exe` variant instead (larger download; bundles the
|
|
||||||
CUDA runtime DLLs).
|
|
||||||
|
|
||||||
## Highlights
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- **Custom multi-folder picker** — navigate and stage folders from multiple
|
|
||||||
locations and add them in one go, replacing the native OS dialog.
|
|
||||||
- **Themes** — Phokus, Subtle Light, and Conventional Dark chrome, with an
|
|
||||||
inline theme picker in first-run onboarding.
|
|
||||||
- **Timeline scrubber** — a year/month rail to jump anywhere in the library.
|
|
||||||
- **Folder reordering** in the sidebar (drag-and-drop or keyboard), persisted,
|
|
||||||
plus A–Z / Z–A / Custom sort for the Libraries list.
|
|
||||||
- **Video playback settings** — autoplay-in-lightbox and start-muted toggles.
|
|
||||||
- **Rebuild semantic index** maintenance action — fixes "dimension mismatch"
|
|
||||||
search errors after switching CLIP models.
|
|
||||||
- Locate and filter images with failed AI-tagging jobs.
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- Gallery grid and Duplicate Finder list are now row-virtualised — large
|
|
||||||
libraries and large result sets (5,000+ pairs) scroll without lag.
|
|
||||||
- Settings now open on General by default.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- **AVIF thumbnails** now generate correctly (routed through bundled FFmpeg);
|
|
||||||
previously-failed AVIF jobs are requeued on startup.
|
|
||||||
- Video embedding jobs no longer churn through failed states before their
|
|
||||||
thumbnail exists; previously-failed ones are requeued.
|
|
||||||
- Timeline scrolling is smooth on large libraries (per-row virtualisation);
|
|
||||||
background batches no longer re-sort the whole loaded set.
|
|
||||||
- Numerous Subtle Light theme readability/parity fixes.
|
|
||||||
|
|
||||||
See the [changelog](https://github.com/JezzWTF/phokus/blob/main/CHANGELOG.md)
|
|
||||||
for the full list.
|
|
||||||
|
|
||||||
## Checksums
|
|
||||||
|
|
||||||
```
|
|
||||||
SHA-256 (Phokus_0.1.1_x64-setup.exe) = 1c19cbeb77f38a44149380c42c76b633add65777d317e1f3ff7e45d96d12d287
|
|
||||||
SHA-256 (Phokus_0.1.1_x64-cuda-setup.exe) = a7337ef5ee0478a785b48acc8012e8fc5c957341161d6103409213ad78eb845f
|
|
||||||
```
|
|
||||||
@@ -1,85 +0,0 @@
|
|||||||
# Phokus v0.2.0
|
|
||||||
|
|
||||||
> Draft for the GitHub Release body. Fill in the checksums and trim as needed.
|
|
||||||
|
|
||||||
**Phokus is a local-first desktop media library for Windows** — point it at
|
|
||||||
your image and video folders and it builds a fast, searchable gallery with
|
|
||||||
thumbnails, semantic search, visual discovery, AI tagging, and duplicate
|
|
||||||
cleanup. Everything is processed on your machine; nothing is uploaded.
|
|
||||||
|
|
||||||
0.2.0 is the biggest update since launch: albums, gallery multi-select with
|
|
||||||
bulk actions, colour search, a second AI tagging model (JoyTag), a full tag
|
|
||||||
manager, camera EXIF details in the lightbox, a slideshow mode, and a large
|
|
||||||
batch of performance and theme fixes. Updating from 0.1.x? The built-in
|
|
||||||
updater will fetch this for you — and afterwards, a new in-app "What's New"
|
|
||||||
tour shows you around.
|
|
||||||
|
|
||||||
## Install / update
|
|
||||||
|
|
||||||
- **Updating from 0.1.x:** the in-app updater will offer 0.2.0 on launch — one
|
|
||||||
click downloads, installs, and relaunches.
|
|
||||||
- **Fresh install:** download `Phokus_0.2.0_x64-setup.exe` below and run it.
|
|
||||||
**Windows SmartScreen will warn** that the publisher is unrecognized — this
|
|
||||||
build is **not code-signed**. Click **More info → Run anyway**.
|
|
||||||
- Requires **Windows 10/11**. WebView2 is fetched automatically if missing.
|
|
||||||
|
|
||||||
NVIDIA users wanting GPU embedding/tagging speed can use the
|
|
||||||
`Phokus_0.2.0_x64-cuda-setup.exe` variant instead (larger download; bundles the
|
|
||||||
CUDA runtime DLLs). Settings → Updates now shows which build you are running.
|
|
||||||
|
|
||||||
## Highlights
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- **Albums** — cross-folder collections without moving files: their own
|
|
||||||
sidebar section with cover thumbnails, create/rename/reorder/manage, and
|
|
||||||
album-aware similar-image search.
|
|
||||||
- **Gallery multi-select** — hover a thumbnail's corner to start selecting,
|
|
||||||
then bulk tag, rate, favorite, add to an album, or delete from a floating
|
|
||||||
action bar. Works in similar-image, region, and album views too.
|
|
||||||
- **Colour search** — filter the Gallery, Timeline, or tag results by dominant
|
|
||||||
colour via toolbar swatches or a custom picker.
|
|
||||||
- **Choose your tagging model** — pick between the anime-focused WD tagger and
|
|
||||||
**JoyTag** (better for photo libraries), each with its own confidence
|
|
||||||
threshold, plus a **Reset AI tags** action that never touches manual tags.
|
|
||||||
- **Tag manager** — rename, merge, and delete tags across the library, with
|
|
||||||
live search and sorting; Explore's Tag Cloud also shows **related tags** on
|
|
||||||
hover.
|
|
||||||
- **Camera info in the lightbox** — EXIF camera, lens, aperture, shutter
|
|
||||||
speed, ISO, focal length, and a map link for geotagged photos.
|
|
||||||
- **Slideshow mode**, **What's New tour after updates**, **quick theme switch**
|
|
||||||
from the title bar, an editable path bar in the folder picker, persistent
|
|
||||||
per-folder worker pauses, and add-to-album from the right-click menu.
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- **Settings reorganised** into General, Media, Updates & Setup, Storage, and
|
|
||||||
AI Workspace pages.
|
|
||||||
- **Menus unified** — all context menus and dropdowns share one style, stay on
|
|
||||||
screen, close on Escape, and support submenus.
|
|
||||||
- **Faster Explore** — quicker cluster revisits, much faster first-time
|
|
||||||
clustering on large libraries, and a calmer Tag Cloud during active tagging.
|
|
||||||
- **Faster CPU tagging** (multi-core with headroom) and smoother GPU tagging.
|
|
||||||
- **Safer deletion** — deleting media now asks for confirmation and says
|
|
||||||
clearly that files are removed from disk.
|
|
||||||
- Better narrow-window layouts, a real update download progress bar, and a
|
|
||||||
two-column lightbox info panel.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- **No more self-indexing loops** — Phokus now skips its own thumbnail cache
|
|
||||||
when you add a broad folder like your user profile.
|
|
||||||
- **Ratings keep your search order** — rating an image mid-search no longer
|
|
||||||
reshuffles similar-image, region, semantic, tag, or album results.
|
|
||||||
- **AI tagging stays responsive** — big GPU tagging jobs no longer freeze the
|
|
||||||
UI, and noisy low-signal tags are filtered (older ones cleaned up on
|
|
||||||
startup).
|
|
||||||
- First launch now fits smaller screens, Explore no longer flashes the
|
|
||||||
previous folder, and a long list of Subtle Light readability fixes.
|
|
||||||
|
|
||||||
See the [changelog](https://github.com/JezzWTF/phokus/blob/main/CHANGELOG.md)
|
|
||||||
for the full list.
|
|
||||||
|
|
||||||
## Checksums
|
|
||||||
|
|
||||||
```
|
|
||||||
SHA-256 (Phokus_0.2.0_x64-setup.exe) = TBD
|
|
||||||
SHA-256 (Phokus_0.2.0_x64-cuda-setup.exe) = TBD
|
|
||||||
```
|
|
||||||
@@ -1,201 +0,0 @@
|
|||||||
# Phokus UI Lab
|
|
||||||
|
|
||||||
Phokus UI Lab is a browser-only development mode for visual work on the real
|
|
||||||
Phokus frontend. It runs the same `App.tsx`, Zustand store, components, and CSS
|
|
||||||
as the Tauri app, but installs Tauri JavaScript mocks before the app imports.
|
|
||||||
|
|
||||||
This gives UI agents and contributors a stable browser target without launching
|
|
||||||
the Rust backend or a Tauri window.
|
|
||||||
|
|
||||||
## Run It
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pnpm dev:ui
|
|
||||||
```
|
|
||||||
|
|
||||||
Then open:
|
|
||||||
|
|
||||||
```text
|
|
||||||
http://127.0.0.1:1422
|
|
||||||
```
|
|
||||||
|
|
||||||
The script runs Vite in the custom `ui` mode:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vite --mode ui --host 127.0.0.1 --port 1422 --strictPort
|
|
||||||
```
|
|
||||||
|
|
||||||
The normal app development commands are unchanged:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pnpm dev:app # Tauri app + Rust backend
|
|
||||||
pnpm dev:vite # frontend dev server for Tauri dev
|
|
||||||
```
|
|
||||||
|
|
||||||
## Scenarios
|
|
||||||
|
|
||||||
UI Lab reads `?scenario=` from the URL. If no scenario is provided, it uses
|
|
||||||
`rich`.
|
|
||||||
|
|
||||||
| URL | Purpose |
|
|
||||||
| --- | --- |
|
|
||||||
| `/?scenario=rich` | Default realistic library with folders, albums, ratings, favorites, tags, images, and videos |
|
|
||||||
| `/?scenario=empty` | Empty library with no folders or media (onboarding already completed) |
|
|
||||||
| `/?scenario=new-user` | True first run: onboarding tour open, empty library, no tagger model downloaded |
|
|
||||||
| `/?scenario=just-updated` | Rich library that has just been updated — the "What's new" toast fires on launch |
|
|
||||||
| `/?scenario=busy` | Background workers with pending thumbnail, metadata, embedding, caption, and tagging jobs |
|
|
||||||
| `/?scenario=duplicates` | Duplicate Finder opened with duplicate groups already available |
|
|
||||||
| `/?scenario=album` | Gallery opened directly into an album |
|
|
||||||
| `/?scenario=errors` | Broken thumbnails, folder scan errors, failed embeddings, failed tagging, and metadata issues |
|
|
||||||
| `/?scenario=huge` | Large mock library for virtualization and layout checks |
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
|
|
||||||
```text
|
|
||||||
http://127.0.0.1:1422/?scenario=duplicates
|
|
||||||
http://127.0.0.1:1422/?scenario=errors
|
|
||||||
http://127.0.0.1:1422/?scenario=huge
|
|
||||||
```
|
|
||||||
|
|
||||||
## Changelog Previews
|
|
||||||
|
|
||||||
The What's New modal adapts its layout to release size (compact single column
|
|
||||||
for small releases, a two-pane section rail for large ones). UI Lab reads
|
|
||||||
`?changelog=` to override which entry the modal shows:
|
|
||||||
|
|
||||||
| URL | Purpose |
|
|
||||||
| --- | --- |
|
|
||||||
| `/?changelog=unreleased` | The in-progress `[Unreleased]` notes — large release, rail layout |
|
|
||||||
| `/?changelog=small` | Synthetic hotfix-sized entry — compact single-column layout |
|
|
||||||
| `/?changelog=0.1.1` | Any specific released version |
|
|
||||||
|
|
||||||
Open the modal via the demo panel: `Ctrl+Shift+D` → Open "What's new" modal.
|
|
||||||
|
|
||||||
Combine with the scenario above to walk the whole post-update greeting for the
|
|
||||||
next release: `/?scenario=just-updated&changelog=unreleased`.
|
|
||||||
|
|
||||||
## How It Works
|
|
||||||
|
|
||||||
`src/main.tsx` bootstraps the app asynchronously. In `ui` mode it imports
|
|
||||||
`src/dev/setupMockTauri.ts` first, then imports the real `App`.
|
|
||||||
|
|
||||||
That order matters because static imports are hoisted. The Tauri mocks must be
|
|
||||||
installed before `App`, the store, the title bar, or visual components import
|
|
||||||
Tauri APIs.
|
|
||||||
|
|
||||||
The mock setup installs:
|
|
||||||
|
|
||||||
- `mockWindows("main")` for window APIs used by the title bar
|
|
||||||
- `mockConvertFileSrc("windows")` as a baseline Tauri file URL mock
|
|
||||||
- `mockIPC(..., { shouldMockEvents: true })` for `invoke`, `listen`, and `emit`
|
|
||||||
|
|
||||||
The in-memory backend lives in:
|
|
||||||
|
|
||||||
```text
|
|
||||||
src/dev/mockBackend.ts
|
|
||||||
src/dev/mockFixtures.ts
|
|
||||||
src/dev/mockScenarios.ts
|
|
||||||
src/dev/applyMockScenario.ts
|
|
||||||
```
|
|
||||||
|
|
||||||
Happy-path fixture media is generated from existing onboarding images and
|
|
||||||
website screenshots into:
|
|
||||||
|
|
||||||
```text
|
|
||||||
public/dev-media/fixture-*.webp
|
|
||||||
```
|
|
||||||
|
|
||||||
The pack is deliberately small and deterministic, but varied enough for browser
|
|
||||||
screenshots: square, portrait, landscape, monochrome, tinted, framed, and
|
|
||||||
interface-like crops. Synthetic or deliberately broken fixture paths can still
|
|
||||||
use the `mock://` scheme described below.
|
|
||||||
|
|
||||||
## Mock Media
|
|
||||||
|
|
||||||
Visual components should use `mediaSrc(...)` instead of calling
|
|
||||||
`convertFileSrc(...)` directly:
|
|
||||||
|
|
||||||
```ts
|
|
||||||
import { mediaSrc } from "../lib/mediaSrc";
|
|
||||||
|
|
||||||
const src = mediaSrc(image.thumbnail_path);
|
|
||||||
```
|
|
||||||
|
|
||||||
In normal Tauri modes, `mediaSrc` delegates to `convertFileSrc`. In UI Lab,
|
|
||||||
root-relative asset URLs such as `/dev-media/fixture-01.webp` are returned
|
|
||||||
as-is. Paths beginning with `mock://` are also served from `public/dev-media`:
|
|
||||||
|
|
||||||
```text
|
|
||||||
mock://thumb-1.svg -> /dev-media/thumb-1.svg
|
|
||||||
```
|
|
||||||
|
|
||||||
Use `mediaSrc` when adding components that render thumbnails, album covers,
|
|
||||||
preview images, or video posters.
|
|
||||||
|
|
||||||
## Adding A Mock Command
|
|
||||||
|
|
||||||
If the browser console logs an unmocked command, add it to
|
|
||||||
`src/dev/mockBackend.ts`.
|
|
||||||
|
|
||||||
Prefer returning realistic data for commands that affect visible UI. For actions
|
|
||||||
that would touch the machine, return a safe no-op:
|
|
||||||
|
|
||||||
```ts
|
|
||||||
case "open_app_data_folder":
|
|
||||||
return null;
|
|
||||||
```
|
|
||||||
|
|
||||||
When adding fixture data, keep it shaped like the real store/Rust contracts.
|
|
||||||
Most frontend-facing types are exported from `src/store.ts`, so the mocks can
|
|
||||||
stay type-checked against the real UI.
|
|
||||||
|
|
||||||
## Adding A Scenario
|
|
||||||
|
|
||||||
1. Add the scenario name to `MockScenario` and `SCENARIOS` in
|
|
||||||
`src/dev/mockScenarios.ts`.
|
|
||||||
2. Adjust fixture creation in `src/dev/mockFixtures.ts`.
|
|
||||||
3. If the scenario should open a specific view, add that behavior in
|
|
||||||
`src/dev/applyMockScenario.ts`.
|
|
||||||
4. Visit `http://127.0.0.1:1422/?scenario=your-scenario` and check for console
|
|
||||||
errors.
|
|
||||||
|
|
||||||
## What UI Lab Is For
|
|
||||||
|
|
||||||
UI Lab is intended for:
|
|
||||||
|
|
||||||
- visual iteration on the real app shell
|
|
||||||
- layout and responsive checks
|
|
||||||
- state-heavy views like Explore, Timeline, Duplicates, albums, and Settings
|
|
||||||
- AI-agent screenshot/browser inspection
|
|
||||||
- safe UI work without filesystem or Rust-side effects
|
|
||||||
|
|
||||||
It is not a replacement for Tauri app testing. Validate native behavior with
|
|
||||||
`pnpm dev:app` when touching:
|
|
||||||
|
|
||||||
- file or folder pickers
|
|
||||||
- filesystem permissions
|
|
||||||
- real thumbnail/video loading
|
|
||||||
- window drag, minimize, maximize, or close behavior
|
|
||||||
- Rust backend performance
|
|
||||||
- updater installation
|
|
||||||
- WebView2-specific behavior
|
|
||||||
|
|
||||||
## Verification
|
|
||||||
|
|
||||||
Useful checks after changing UI Lab:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pnpm exec tsc --noEmit
|
|
||||||
pnpm build:vite
|
|
||||||
pnpm dev:ui
|
|
||||||
```
|
|
||||||
|
|
||||||
Then smoke-test at least:
|
|
||||||
|
|
||||||
```text
|
|
||||||
http://127.0.0.1:1422/?scenario=rich
|
|
||||||
http://127.0.0.1:1422/?scenario=empty
|
|
||||||
http://127.0.0.1:1422/?scenario=duplicates
|
|
||||||
http://127.0.0.1:1422/?scenario=errors
|
|
||||||
http://127.0.0.1:1422/?scenario=huge
|
|
||||||
```
|
|
||||||
@@ -2,9 +2,11 @@
|
|||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8" />
|
<meta charset="UTF-8" />
|
||||||
|
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<title>Phokus</title>
|
<title>Tauri + React + Typescript</title>
|
||||||
</head>
|
</head>
|
||||||
|
|
||||||
<body>
|
<body>
|
||||||
<div id="root"></div>
|
<div id="root"></div>
|
||||||
<script type="module" src="/src/main.tsx"></script>
|
<script type="module" src="/src/main.tsx"></script>
|
||||||
|
|||||||
@@ -1,62 +1,33 @@
|
|||||||
{
|
{
|
||||||
"name": "phokus",
|
"name": "phokus",
|
||||||
"private": true,
|
"private": true,
|
||||||
"version": "0.2.0",
|
"version": "0.1.0",
|
||||||
"license": "MIT",
|
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build:app:cpu": "tauri build -- --no-default-features",
|
"dev": "vite",
|
||||||
"build:app:cuda": "tauri build --config src-tauri/tauri.cuda.conf.json",
|
"build": "tsc && vite build",
|
||||||
"build:vite": "tsc && vite build",
|
|
||||||
"build:web": "cd website && tsc && vite build",
|
|
||||||
"changelog:add": "node tools/changelog-add.mjs",
|
|
||||||
"clean:app": "cd src-tauri && cargo clean",
|
|
||||||
"dev:app": "tauri dev",
|
|
||||||
"dev:app:cpu": "tauri dev -- --no-default-features",
|
|
||||||
"dev:ui": "vite --mode ui --host 127.0.0.1 --port 1422 --strictPort",
|
|
||||||
"dev:vite": "vite",
|
|
||||||
"dev:web": "cd website && pnpm dev",
|
|
||||||
"format": "prettier --write .",
|
|
||||||
"format:all": "pnpm format && pnpm format:rust",
|
|
||||||
"format:check": "prettier --check .",
|
|
||||||
"format:rust": "cd src-tauri && cargo fmt",
|
|
||||||
"format:rust:check": "cd src-tauri && cargo fmt --check",
|
|
||||||
"preview": "vite preview",
|
"preview": "vite preview",
|
||||||
"tauri": "tauri",
|
"tauri": "tauri"
|
||||||
"test:e2e": "playwright test",
|
|
||||||
"test:rust": "cd src-tauri && cargo test --no-default-features",
|
|
||||||
"test:unit": "vitest run",
|
|
||||||
"test:unit:watch": "vitest"
|
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@tanstack/react-virtual": "^3.13.23",
|
"@tanstack/react-virtual": "^3.13.23",
|
||||||
"@tauri-apps/api": "^2",
|
"@tauri-apps/api": "^2",
|
||||||
"@tauri-apps/plugin-dialog": "^2.7.0",
|
"@tauri-apps/plugin-dialog": "^2.7.0",
|
||||||
"@tauri-apps/plugin-fs": "^2.5.0",
|
"@tauri-apps/plugin-fs": "^2.5.0",
|
||||||
"@tauri-apps/plugin-notification": "^2.3.3",
|
|
||||||
"@tauri-apps/plugin-opener": "^2",
|
"@tauri-apps/plugin-opener": "^2",
|
||||||
"@tauri-apps/plugin-process": "^2.3.1",
|
|
||||||
"@tauri-apps/plugin-updater": "^2.10.1",
|
|
||||||
"d3-force": "^3.0.0",
|
|
||||||
"framer-motion": "^12.38.0",
|
"framer-motion": "^12.38.0",
|
||||||
"react": "^19.1.0",
|
"react": "^19.1.0",
|
||||||
"react-dom": "^19.1.0",
|
"react-dom": "^19.1.0",
|
||||||
"zustand": "^5.0.12"
|
"zustand": "^5.0.12"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@playwright/test": "^1.61.1",
|
|
||||||
"@tailwindcss/vite": "^4.2.2",
|
"@tailwindcss/vite": "^4.2.2",
|
||||||
"@tauri-apps/cli": "^2",
|
"@tauri-apps/cli": "^2",
|
||||||
"@types/d3-force": "^3.0.10",
|
|
||||||
"@types/node": "^26.0.1",
|
|
||||||
"@types/react": "^19.1.8",
|
"@types/react": "^19.1.8",
|
||||||
"@types/react-dom": "^19.1.6",
|
"@types/react-dom": "^19.1.6",
|
||||||
"@vitejs/plugin-react": "^4.6.0",
|
"@vitejs/plugin-react": "^4.6.0",
|
||||||
"prettier": "^3.9.4",
|
|
||||||
"prettier-plugin-tailwindcss": "^0.8.0",
|
|
||||||
"tailwindcss": "^4.2.2",
|
"tailwindcss": "^4.2.2",
|
||||||
"typescript": "~5.8.3",
|
"typescript": "~5.8.3",
|
||||||
"vite": "^7.0.4",
|
"vite": "^7.0.4"
|
||||||
"vitest": "^4.1.9"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,75 +0,0 @@
|
|||||||
import { defineConfig, devices } from '@playwright/test'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Read environment variables from file.
|
|
||||||
* https://github.com/motdotla/dotenv
|
|
||||||
*/
|
|
||||||
// import dotenv from 'dotenv';
|
|
||||||
// import path from 'path';
|
|
||||||
// dotenv.config({ path: path.resolve(__dirname, '.env') });
|
|
||||||
|
|
||||||
/**
|
|
||||||
* See https://playwright.dev/docs/test-configuration.
|
|
||||||
*/
|
|
||||||
export default defineConfig({
|
|
||||||
testDir: './tests',
|
|
||||||
/* Run tests in files in parallel */
|
|
||||||
fullyParallel: true,
|
|
||||||
/* Fail the build on CI if you accidentally left test.only in the source code. */
|
|
||||||
forbidOnly: !!process.env.CI,
|
|
||||||
/* Retry on CI only */
|
|
||||||
retries: process.env.CI ? 2 : 0,
|
|
||||||
/* Opt out of parallel tests on CI. */
|
|
||||||
workers: process.env.CI ? 1 : 4,
|
|
||||||
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
|
|
||||||
reporter: 'html',
|
|
||||||
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
|
|
||||||
use: {
|
|
||||||
/* Base URL to use in actions like `await page.goto('')`. */
|
|
||||||
baseURL: 'http://127.0.0.1:1422',
|
|
||||||
|
|
||||||
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
|
|
||||||
trace: 'on-first-retry',
|
|
||||||
},
|
|
||||||
|
|
||||||
/* Configure projects for major browsers */
|
|
||||||
projects: [
|
|
||||||
{
|
|
||||||
name: 'chromium',
|
|
||||||
use: { ...devices['Desktop Chrome'] },
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
name: 'webkit',
|
|
||||||
use: { ...devices['Desktop Safari'] },
|
|
||||||
},
|
|
||||||
|
|
||||||
/* Test against mobile viewports. */
|
|
||||||
// {
|
|
||||||
// name: 'Mobile Chrome',
|
|
||||||
// use: { ...devices['Pixel 5'] },
|
|
||||||
// },
|
|
||||||
// {
|
|
||||||
// name: 'Mobile Safari',
|
|
||||||
// use: { ...devices['iPhone 12'] },
|
|
||||||
// },
|
|
||||||
|
|
||||||
/* Test against branded browsers. */
|
|
||||||
// {
|
|
||||||
// name: 'Microsoft Edge',
|
|
||||||
// use: { ...devices['Desktop Edge'], channel: 'msedge' },
|
|
||||||
// },
|
|
||||||
// {
|
|
||||||
// name: 'Google Chrome',
|
|
||||||
// use: { ...devices['Desktop Chrome'], channel: 'chrome' },
|
|
||||||
// },
|
|
||||||
],
|
|
||||||
|
|
||||||
/* Run the Phokus UI Lab (browser-only dev mode, see docs/ui-lab.md) before starting the tests */
|
|
||||||
webServer: {
|
|
||||||
command: 'pnpm exec vite --mode ui --host 127.0.0.1 --port 1422 --strictPort',
|
|
||||||
url: 'http://127.0.0.1:1422',
|
|
||||||
reuseExistingServer: !process.env.CI,
|
|
||||||
timeout: 120_000,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
packages:
|
|
||||||
- website
|
|
||||||
|
|
||||||
allowBuilds:
|
|
||||||
esbuild: true
|
|
||||||
sharp: true
|
|
||||||
|
Before Width: | Height: | Size: 27 KiB |
|
Before Width: | Height: | Size: 14 KiB |
|
Before Width: | Height: | Size: 24 KiB |
|
Before Width: | Height: | Size: 25 KiB |
|
Before Width: | Height: | Size: 30 KiB |
|
Before Width: | Height: | Size: 70 KiB |
|
Before Width: | Height: | Size: 60 KiB |
|
Before Width: | Height: | Size: 24 KiB |
|
Before Width: | Height: | Size: 29 KiB |
|
Before Width: | Height: | Size: 28 KiB |
|
Before Width: | Height: | Size: 78 KiB |
|
Before Width: | Height: | Size: 27 KiB |
|
Before Width: | Height: | Size: 23 KiB |
|
Before Width: | Height: | Size: 35 KiB |
|
Before Width: | Height: | Size: 40 KiB |
|
Before Width: | Height: | Size: 18 KiB |
|
Before Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 81 KiB |
|
Before Width: | Height: | Size: 25 KiB |
|
Before Width: | Height: | Size: 44 KiB |
|
Before Width: | Height: | Size: 81 KiB |
|
Before Width: | Height: | Size: 45 KiB |
|
Before Width: | Height: | Size: 27 KiB |
|
Before Width: | Height: | Size: 12 KiB |
|
Before Width: | Height: | Size: 22 KiB |
|
Before Width: | Height: | Size: 23 KiB |
|
Before Width: | Height: | Size: 44 KiB |
|
Before Width: | Height: | Size: 105 KiB |
|
Before Width: | Height: | Size: 35 KiB |
|
Before Width: | Height: | Size: 17 KiB |
|
Before Width: | Height: | Size: 33 KiB |
|
Before Width: | Height: | Size: 40 KiB |
|
Before Width: | Height: | Size: 43 KiB |
|
Before Width: | Height: | Size: 21 KiB |
|
Before Width: | Height: | Size: 42 KiB |
|
Before Width: | Height: | Size: 34 KiB |
|
Before Width: | Height: | Size: 55 KiB |
|
Before Width: | Height: | Size: 7.1 KiB |
|
Before Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 47 KiB |
|
Before Width: | Height: | Size: 30 KiB |
|
Before Width: | Height: | Size: 106 KiB |
|
Before Width: | Height: | Size: 48 KiB |
|
Before Width: | Height: | Size: 37 KiB |
|
Before Width: | Height: | Size: 24 KiB |
|
Before Width: | Height: | Size: 20 KiB |
|
Before Width: | Height: | Size: 18 KiB |
|
Before Width: | Height: | Size: 21 KiB |
@@ -1,43 +0,0 @@
|
|||||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256" width="256" height="256" role="img" aria-label="Phokus">
|
|
||||||
<title>Phokus</title>
|
|
||||||
|
|
||||||
<!--
|
|
||||||
Phokus app mark — a camera iris.
|
|
||||||
The hexagon in the middle is the lens OPENING; each blade edge sweeps from a
|
|
||||||
corner of the opening out to the rim, all leaning the same way (the pinwheel).
|
|
||||||
|
|
||||||
Tuning knobs (edit and re-open in any browser/Figma):
|
|
||||||
* opening roundness -> the "52" radius in the opening <path> arcs.
|
|
||||||
smaller (toward 30) = rounder/softer hole; larger (toward 90) = flatter, sharper.
|
|
||||||
* blade curve -> the "110" radius in each blade <path>.
|
|
||||||
smaller = more pronounced curve; larger = straighter blade.
|
|
||||||
* blade sweep amount -> the +40deg baked into the blade endpoint (70.71,-84.27).
|
|
||||||
* curve DIRECTION -> the final flag in each "A r r 0 0 X" command (0<->1 flips the bow).
|
|
||||||
* weight -> stroke-width on the <g>.
|
|
||||||
* color -> stroke on the <g>; swap "#e9e9ec" for currentColor to inherit.
|
|
||||||
A brand-accent focal dot is provided at the bottom (commented out).
|
|
||||||
-->
|
|
||||||
|
|
||||||
<rect x="8" y="8" width="240" height="240" rx="52" fill="#0e0f14"/>
|
|
||||||
|
|
||||||
<g transform="translate(128 128)" fill="none" stroke="#e9e9ec" stroke-width="9"
|
|
||||||
stroke-linecap="round" stroke-linejoin="round">
|
|
||||||
|
|
||||||
<!-- outer rim -->
|
|
||||||
<circle r="110"/>
|
|
||||||
|
|
||||||
<!-- lens opening: soft, arc-rounded hexagon -->
|
|
||||||
<path d="M0,-46 A52,52 0 0 0 39.84,-23 A52,52 0 0 0 39.84,23 A52,52 0 0 0 0,46 A52,52 0 0 0 -39.84,23 A52,52 0 0 0 -39.84,-23 A52,52 0 0 0 0,-46 Z"/>
|
|
||||||
|
|
||||||
<!-- blades: one curved edge, swept six times -->
|
|
||||||
<g transform="rotate(0)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
<g transform="rotate(60)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
<g transform="rotate(120)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
<g transform="rotate(180)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
<g transform="rotate(240)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
<g transform="rotate(300)"><path d="M0,-46 A110,110 0 0 1 70.71,-84.27"/></g>
|
|
||||||
</g>
|
|
||||||
|
|
||||||
<!-- optional brand focal point -->
|
|
||||||
<!-- <circle cx="128" cy="128" r="9" fill="#e6a23c"/> -->
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 2.2 KiB |
@@ -1,189 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -Eeuo pipefail
|
|
||||||
|
|
||||||
# Codex Cloud setup for Phokus.
|
|
||||||
# Paste this script into the Codex Cloud environment setup field, or run it from
|
|
||||||
# the repo root with: bash scripts/codex-cloud-setup.sh
|
|
||||||
#
|
|
||||||
# Goals:
|
|
||||||
# - install Linux packages needed by Tauri/WebKit and native Rust crates
|
|
||||||
# - install JS dependencies with pnpm using the lockfile
|
|
||||||
# - pre-fetch Rust dependencies while setup still has internet access
|
|
||||||
# - keep the environment CPU-safe by avoiding Phokus' default CUDA feature set
|
|
||||||
# - leave Codex with clear verification commands for UI and Rust work
|
|
||||||
|
|
||||||
log() {
|
|
||||||
printf '\n\033[1;36m[phokus-codex]\033[0m %s\n' "$*"
|
|
||||||
}
|
|
||||||
|
|
||||||
warn() {
|
|
||||||
printf '\n\033[1;33m[phokus-codex warning]\033[0m %s\n' "$*" >&2
|
|
||||||
}
|
|
||||||
|
|
||||||
repo_root="$(pwd)"
|
|
||||||
|
|
||||||
if [[ ! -f "package.json" || ! -d "src-tauri" ]]; then
|
|
||||||
warn "This script should be run from the Phokus repository root. Current directory: ${repo_root}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
export CI=1
|
|
||||||
export PNPM_HOME="${PNPM_HOME:-$HOME/.local/share/pnpm}"
|
|
||||||
export PATH="$PNPM_HOME:$HOME/.cargo/bin:$PATH"
|
|
||||||
|
|
||||||
# Persist useful shell defaults for the later Codex agent phase. Codex setup runs
|
|
||||||
# in a separate Bash session, so exports here alone would not survive.
|
|
||||||
if ! grep -q "# Phokus Codex Cloud" "$HOME/.bashrc" 2>/dev/null; then
|
|
||||||
cat >> "$HOME/.bashrc" <<'BASHRC'
|
|
||||||
|
|
||||||
# Phokus Codex Cloud
|
|
||||||
export PNPM_HOME="${PNPM_HOME:-$HOME/.local/share/pnpm}"
|
|
||||||
export PATH="$PNPM_HOME:$HOME/.cargo/bin:$PATH"
|
|
||||||
export CI=1
|
|
||||||
BASHRC
|
|
||||||
fi
|
|
||||||
|
|
||||||
install_apt_packages() {
|
|
||||||
if ! command -v apt-get >/dev/null 2>&1; then
|
|
||||||
warn "apt-get not found; skipping system package installation."
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Installing Linux system dependencies for Tauri, WebKit, SQLite/native crates, and browser tooling"
|
|
||||||
|
|
||||||
sudo apt-get update
|
|
||||||
|
|
||||||
# Tauri v2 Linux builds need the WebKitGTK/AppIndicator/Rsvg stack. Some base
|
|
||||||
# images expose either the 4.1 or 4.0 WebKit development package, so try the
|
|
||||||
# modern package first and gracefully fall back.
|
|
||||||
local common_packages=(
|
|
||||||
build-essential
|
|
||||||
curl
|
|
||||||
wget
|
|
||||||
file
|
|
||||||
pkg-config
|
|
||||||
libssl-dev
|
|
||||||
libgtk-3-dev
|
|
||||||
libayatana-appindicator3-dev
|
|
||||||
librsvg2-dev
|
|
||||||
patchelf
|
|
||||||
ca-certificates
|
|
||||||
)
|
|
||||||
|
|
||||||
if apt-cache show libwebkit2gtk-4.1-dev >/dev/null 2>&1; then
|
|
||||||
sudo apt-get install -y --no-install-recommends "${common_packages[@]}" libwebkit2gtk-4.1-dev
|
|
||||||
else
|
|
||||||
sudo apt-get install -y --no-install-recommends "${common_packages[@]}" libwebkit2gtk-4.0-dev
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
ensure_node_and_pnpm() {
|
|
||||||
log "Preparing Node/pnpm"
|
|
||||||
|
|
||||||
if ! command -v node >/dev/null 2>&1; then
|
|
||||||
warn "Node.js is not available in this image. Pin Node.js 20+ in the Codex environment settings or use a Codex universal image with Node installed."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
node_major="$(node -p "process.versions.node.split('.')[0]")"
|
|
||||||
if [[ "$node_major" -lt 20 ]]; then
|
|
||||||
warn "Phokus expects Node.js 20+. Current version: $(node --version). Pin Node.js 20+ in Codex environment settings."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
mkdir -p "$PNPM_HOME"
|
|
||||||
|
|
||||||
if command -v corepack >/dev/null 2>&1; then
|
|
||||||
corepack enable
|
|
||||||
corepack prepare pnpm@latest --activate
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! command -v pnpm >/dev/null 2>&1; then
|
|
||||||
npm install -g pnpm
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Node: $(node --version)"
|
|
||||||
log "pnpm: $(pnpm --version)"
|
|
||||||
}
|
|
||||||
|
|
||||||
ensure_rust() {
|
|
||||||
log "Preparing Rust toolchain"
|
|
||||||
|
|
||||||
if ! command -v rustup >/dev/null 2>&1; then
|
|
||||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "$HOME/.cargo/env"
|
|
||||||
fi
|
|
||||||
|
|
||||||
rustup toolchain install stable --profile minimal
|
|
||||||
rustup default stable
|
|
||||||
rustup component add rustfmt clippy
|
|
||||||
|
|
||||||
log "rustc: $(rustc --version)"
|
|
||||||
log "cargo: $(cargo --version)"
|
|
||||||
}
|
|
||||||
|
|
||||||
install_js_dependencies() {
|
|
||||||
log "Installing frontend dependencies"
|
|
||||||
pnpm install --frozen-lockfile
|
|
||||||
}
|
|
||||||
|
|
||||||
prefetch_rust_dependencies() {
|
|
||||||
log "Pre-fetching Rust dependencies for CPU-safe Tauri checks"
|
|
||||||
|
|
||||||
# Phokus enables candle-cuda by default in Cargo.toml. Codex Cloud usually runs
|
|
||||||
# in a CPU Linux container, so use --no-default-features for checks/builds
|
|
||||||
# unless you intentionally configure a CUDA-capable environment.
|
|
||||||
cargo fetch --manifest-path src-tauri/Cargo.toml --locked
|
|
||||||
|
|
||||||
# This is intentionally a check, not a full release build. It warms the Cargo
|
|
||||||
# cache and catches missing native packages without making setup painfully slow.
|
|
||||||
cargo check --manifest-path src-tauri/Cargo.toml --locked --no-default-features
|
|
||||||
}
|
|
||||||
|
|
||||||
install_playwright_browsers() {
|
|
||||||
log "Installing Playwright Chromium dependencies for UI Lab/browser screenshots"
|
|
||||||
|
|
||||||
# Safe even if no Playwright tests are present yet. Useful for Codex browser
|
|
||||||
# inspection against `pnpm dev:ui`.
|
|
||||||
pnpm exec playwright install --with-deps chromium || warn "Playwright browser install failed; UI work may still run, but browser automation/screenshots may need manual setup."
|
|
||||||
}
|
|
||||||
|
|
||||||
print_next_steps() {
|
|
||||||
cat <<'EOF'
|
|
||||||
|
|
||||||
[phokus-codex] Setup complete.
|
|
||||||
|
|
||||||
Recommended Codex verification commands:
|
|
||||||
pnpm exec tsc --noEmit
|
|
||||||
pnpm build:vite
|
|
||||||
cargo check --manifest-path src-tauri/Cargo.toml --locked --no-default-features
|
|
||||||
|
|
||||||
For visual UI work in Codex/browser environments:
|
|
||||||
pnpm dev:ui
|
|
||||||
open http://127.0.0.1:1422/?scenario=rich
|
|
||||||
|
|
||||||
Other useful UI Lab scenarios:
|
|
||||||
http://127.0.0.1:1422/?scenario=empty
|
|
||||||
http://127.0.0.1:1422/?scenario=duplicates
|
|
||||||
http://127.0.0.1:1422/?scenario=errors
|
|
||||||
http://127.0.0.1:1422/?scenario=huge
|
|
||||||
|
|
||||||
Avoid the below in standard CPU-only Codex Cloud unless you have configured CUDA:
|
|
||||||
pnpm dev:app
|
|
||||||
pnpm build:app:cuda
|
|
||||||
cargo check --manifest-path src-tauri/Cargo.toml
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
install_apt_packages
|
|
||||||
ensure_node_and_pnpm
|
|
||||||
ensure_rust
|
|
||||||
install_js_dependencies
|
|
||||||
prefetch_rust_dependencies
|
|
||||||
install_playwright_browsers
|
|
||||||
print_next_steps
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
[target.x86_64-pc-windows-msvc]
|
|
||||||
rustflags = [
|
|
||||||
# Disable MSVC incremental linking — avoids .ilk file corruption
|
|
||||||
# and removes one source of link failure on restart.
|
|
||||||
"-C", "link-arg=/INCREMENTAL:NO",
|
|
||||||
# Skip PDB generation entirely in dev builds.
|
|
||||||
# The .pdb file is the most common reason the linker fails after
|
|
||||||
# an unclean shutdown: the previous phokus.exe process holds the
|
|
||||||
# file open, so link.exe cannot write a new one → LNK error.
|
|
||||||
# Rust backtraces still work without MSVC PDBs.
|
|
||||||
"-C", "link-arg=/DEBUG:NONE",
|
|
||||||
]
|
|
||||||
@@ -1,9 +1,8 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "phokus"
|
name = "phokus"
|
||||||
version = "0.2.0"
|
version = "0.1.0"
|
||||||
description = "Local-first desktop media library"
|
description = "A performant image gallery application"
|
||||||
authors = ["JezzWTF"]
|
authors = ["JezzWTF"]
|
||||||
license = "MIT"
|
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
@@ -14,9 +13,7 @@ crate-type = ["staticlib", "cdylib", "rlib"]
|
|||||||
tauri-build = { version = "2", features = [] }
|
tauri-build = { version = "2", features = [] }
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
# CUDA is on by default for the main dev machine; build with
|
default = []
|
||||||
# `--no-default-features` (pnpm dev:app:cpu) on machines without the toolkit.
|
|
||||||
default = ["candle-cuda"]
|
|
||||||
candle-cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"]
|
candle-cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
@@ -30,86 +27,21 @@ rusqlite = { version = "0.32", features = ["bundled"] }
|
|||||||
r2d2 = "0.8"
|
r2d2 = "0.8"
|
||||||
r2d2_sqlite = "0.25"
|
r2d2_sqlite = "0.25"
|
||||||
sqlite-vec = "=0.1.9"
|
sqlite-vec = "=0.1.9"
|
||||||
hnsw_rs = "0.3.4"
|
|
||||||
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp", "gif", "bmp", "tiff"] }
|
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp", "gif", "bmp", "tiff"] }
|
||||||
fast_image_resize = { version = "6.0.0", features = ["image"] }
|
fast_image_resize = { version = "6.0.0", features = ["image"] }
|
||||||
walkdir = "2"
|
walkdir = "2"
|
||||||
rayon = "1"
|
rayon = "1"
|
||||||
tokio = { version = "1", features = ["rt-multi-thread"] }
|
tokio = { version = "1", features = ["full"] }
|
||||||
chrono = "0.4"
|
chrono = { version = "0.4", features = ["serde"] }
|
||||||
|
uuid = { version = "1", features = ["v4"] }
|
||||||
anyhow = "1"
|
anyhow = "1"
|
||||||
|
log = "0.4"
|
||||||
ffmpeg-sidecar = "2.5.0"
|
ffmpeg-sidecar = "2.5.0"
|
||||||
xxhash-rust = { version = "0.8", features = ["xxh3"] }
|
xxhash-rust = { version = "0.8", features = ["xxh3"] }
|
||||||
memmap2 = "0.9"
|
|
||||||
sysinfo = "0.38.4"
|
sysinfo = "0.38.4"
|
||||||
candle-core = "0.10.2"
|
candle-core = { version = "0.10.2", features = ["cuda"] }
|
||||||
candle-nn = "0.10.2"
|
candle-nn = { version = "0.10.2", features = ["cuda"] }
|
||||||
candle-transformers = "0.10.2"
|
candle-transformers = { version = "0.10.2", features = ["cuda"] }
|
||||||
hf-hub = { version = "0.5.0", default-features = false, features = ["ureq", "native-tls"] }
|
hf-hub = { version = "0.5.0", default-features = false, features = ["ureq", "native-tls"] }
|
||||||
tokenizers = "0.22.1"
|
tokenizers = "0.22.1"
|
||||||
ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "ndarray", "download-binaries", "copy-dylibs", "load-dynamic", "directml", "api-24", "tls-native"] }
|
ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "ndarray", "download-binaries", "copy-dylibs", "load-dynamic", "api-24", "tls-native"] }
|
||||||
ureq = { version = "3.3.0", default-features = false, features = ["native-tls"] }
|
|
||||||
zip = { version = "4.6.1", default-features = false, features = ["deflate"] }
|
|
||||||
csv = "1"
|
|
||||||
kamadak-exif = "0.5"
|
|
||||||
notify = "6"
|
|
||||||
tauri-plugin-notification = "2"
|
|
||||||
mozjpeg = "0.10.13"
|
|
||||||
tauri-plugin-updater = "2"
|
|
||||||
tauri-plugin-process = "2"
|
|
||||||
tauri-plugin-log = "2"
|
|
||||||
tauri-plugin-single-instance = "2"
|
|
||||||
tauri-plugin-window-state = "2"
|
|
||||||
log = "0.4"
|
|
||||||
|
|
||||||
# ── Dev-mode performance ────────────────────────────────────────────────────
|
|
||||||
# opt-level=1 on the main crate keeps incremental compile short.
|
|
||||||
# Only the packages that are genuine hot-path bottlenecks in dev get opt-level=3
|
|
||||||
# (using "*" caused cargo to recheck all dependency fingerprints too aggressively,
|
|
||||||
# which caused spurious full rebuilds and linker conflicts).
|
|
||||||
[profile.dev]
|
|
||||||
opt-level = 1
|
|
||||||
|
|
||||||
# ML inference — without opt these run 20-50× slower than release
|
|
||||||
[profile.dev.package.candle-core]
|
|
||||||
opt-level = 3
|
|
||||||
[profile.dev.package.candle-nn]
|
|
||||||
opt-level = 3
|
|
||||||
[profile.dev.package.candle-transformers]
|
|
||||||
opt-level = 3
|
|
||||||
|
|
||||||
# ONNX runtime (WD tagger)
|
|
||||||
[profile.dev.package.ort]
|
|
||||||
opt-level = 3
|
|
||||||
[profile.dev.package.ort-sys]
|
|
||||||
opt-level = 3
|
|
||||||
|
|
||||||
# Image decode/resize workers
|
|
||||||
[profile.dev.package.image]
|
|
||||||
opt-level = 3
|
|
||||||
[profile.dev.package.fast_image_resize]
|
|
||||||
opt-level = 3
|
|
||||||
[profile.dev.package.mozjpeg]
|
|
||||||
opt-level = 3
|
|
||||||
[profile.dev.package.mozjpeg-sys]
|
|
||||||
opt-level = 3
|
|
||||||
|
|
||||||
# Parallel work scheduler
|
|
||||||
[profile.dev.package.rayon]
|
|
||||||
opt-level = 3
|
|
||||||
[profile.dev.package.rayon-core]
|
|
||||||
opt-level = 3
|
|
||||||
|
|
||||||
# Tokenisation (embedding model pre-processing)
|
|
||||||
[profile.dev.package.tokenizers]
|
|
||||||
opt-level = 3
|
|
||||||
|
|
||||||
# Hashing (duplicate finder)
|
|
||||||
[profile.dev.package.xxhash-rust]
|
|
||||||
opt-level = 3
|
|
||||||
|
|
||||||
# SQLite (frequent db calls in workers)
|
|
||||||
[profile.dev.package.rusqlite]
|
|
||||||
opt-level = 3
|
|
||||||
[profile.dev.package.libsqlite3-sys]
|
|
||||||
opt-level = 3
|
|
||||||
|
|||||||
@@ -15,17 +15,13 @@ fn main() {
|
|||||||
if let Ok(path) = &cuda_path {
|
if let Ok(path) = &cuda_path {
|
||||||
println!("cargo:warning=CUDA Toolkit found at {path} — GPU acceleration enabled.");
|
println!("cargo:warning=CUDA Toolkit found at {path} — GPU acceleration enabled.");
|
||||||
} else {
|
} else {
|
||||||
println!(
|
println!("cargo:warning=CUDA Toolkit (nvcc) found in PATH — GPU acceleration enabled.");
|
||||||
"cargo:warning=CUDA Toolkit (nvcc) found in PATH — GPU acceleration enabled."
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
println!("cargo:warning=━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
println!("cargo:warning=━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
||||||
println!("cargo:warning= candle-cuda feature is enabled but no CUDA Toolkit found.");
|
println!("cargo:warning= candle-cuda feature is enabled but no CUDA Toolkit found.");
|
||||||
println!("cargo:warning= Install CUDA Toolkit from https://developer.nvidia.com/cuda-downloads");
|
println!("cargo:warning= Install CUDA Toolkit from https://developer.nvidia.com/cuda-downloads");
|
||||||
println!(
|
println!("cargo:warning= Or build without GPU support: cargo build --no-default-features");
|
||||||
"cargo:warning= Or build without GPU support: cargo build --no-default-features"
|
|
||||||
);
|
|
||||||
println!("cargo:warning=━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
println!("cargo:warning=━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,9 +2,7 @@
|
|||||||
"$schema": "../gen/schemas/desktop-schema.json",
|
"$schema": "../gen/schemas/desktop-schema.json",
|
||||||
"identifier": "default",
|
"identifier": "default",
|
||||||
"description": "Capability for the main window",
|
"description": "Capability for the main window",
|
||||||
"windows": [
|
"windows": ["main"],
|
||||||
"main"
|
|
||||||
],
|
|
||||||
"permissions": [
|
"permissions": [
|
||||||
"core:default",
|
"core:default",
|
||||||
"opener:default",
|
"opener:default",
|
||||||
@@ -15,14 +13,9 @@
|
|||||||
"fs:allow-read-dir",
|
"fs:allow-read-dir",
|
||||||
"fs:read-files",
|
"fs:read-files",
|
||||||
"fs:read-dirs",
|
"fs:read-dirs",
|
||||||
"notification:default",
|
|
||||||
"updater:default",
|
|
||||||
"process:allow-restart",
|
|
||||||
"core:window:allow-minimize",
|
"core:window:allow-minimize",
|
||||||
"core:window:allow-close",
|
"core:window:allow-close",
|
||||||
"core:window:allow-toggle-maximize",
|
"core:window:allow-toggle-maximize",
|
||||||
"core:window:allow-is-maximized",
|
"core:window:allow-is-maximized"
|
||||||
"core:window:allow-start-dragging",
|
|
||||||
"core:window:allow-start-resize-dragging"
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
Before Width: | Height: | Size: 5.6 KiB After Width: | Height: | Size: 3.4 KiB |
|
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 6.8 KiB |
|
Before Width: | Height: | Size: 1.4 KiB After Width: | Height: | Size: 974 B |
|
Before Width: | Height: | Size: 4.7 KiB After Width: | Height: | Size: 2.8 KiB |
|
Before Width: | Height: | Size: 6.1 KiB After Width: | Height: | Size: 3.8 KiB |
|
Before Width: | Height: | Size: 6.4 KiB After Width: | Height: | Size: 3.9 KiB |
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 7.6 KiB |
|
Before Width: | Height: | Size: 1.4 KiB After Width: | Height: | Size: 903 B |
|
Before Width: | Height: | Size: 14 KiB After Width: | Height: | Size: 8.4 KiB |
|
Before Width: | Height: | Size: 1.9 KiB After Width: | Height: | Size: 1.3 KiB |
|
Before Width: | Height: | Size: 3.2 KiB After Width: | Height: | Size: 2.0 KiB |
|
Before Width: | Height: | Size: 3.9 KiB After Width: | Height: | Size: 2.4 KiB |
|
Before Width: | Height: | Size: 2.2 KiB After Width: | Height: | Size: 1.5 KiB |
|
Before Width: | Height: | Size: 24 KiB After Width: | Height: | Size: 85 KiB |
|
Before Width: | Height: | Size: 23 KiB After Width: | Height: | Size: 14 KiB |
@@ -1,54 +0,0 @@
|
|||||||
/// AI-generated tags that are too broad/noisy to be useful in this gallery.
|
|
||||||
/// Edit this list to change what the tagger removes. Manual user tags are not
|
|
||||||
/// affected.
|
|
||||||
pub const AI_TAG_REMOVAL_LIST: &[&str] = &["1girl", "1boy", "no humans", "2girls", "2boys"];
|
|
||||||
|
|
||||||
fn normalize_tag_for_removal(tag: &str) -> String {
|
|
||||||
tag.trim()
|
|
||||||
.chars()
|
|
||||||
.filter(|c| !matches!(c, ' ' | '_' | '-'))
|
|
||||||
.flat_map(char::to_lowercase)
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn is_removed_ai_tag(tag: &str) -> bool {
|
|
||||||
let normalized = normalize_tag_for_removal(tag);
|
|
||||||
AI_TAG_REMOVAL_LIST
|
|
||||||
.iter()
|
|
||||||
.any(|blocked| normalize_tag_for_removal(blocked) == normalized)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn removed_ai_tags_match_common_spellings() {
|
|
||||||
for tag in [
|
|
||||||
"1girl",
|
|
||||||
"1 girl",
|
|
||||||
"1_girl",
|
|
||||||
"1-girl",
|
|
||||||
"NO_HUMANS",
|
|
||||||
"no humans",
|
|
||||||
] {
|
|
||||||
assert!(is_removed_ai_tag(tag), "{tag} should be removed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn removed_ai_tags_do_not_match_unrelated_tags() {
|
|
||||||
for tag in ["girl", "boy", "humans", "solo", "landscape"] {
|
|
||||||
assert!(!is_removed_ai_tag(tag), "{tag} should be kept");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn removed_ai_tags_tolerate_padding_and_mixed_separators() {
|
|
||||||
assert!(is_removed_ai_tag(" 1girl "));
|
|
||||||
assert!(is_removed_ai_tag("1_-_girl"));
|
|
||||||
assert!(is_removed_ai_tag("No Humans"));
|
|
||||||
assert!(!is_removed_ai_tag(""));
|
|
||||||
assert!(!is_removed_ai_tag(" "));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,30 +1,18 @@
|
|||||||
use crate::onnx_runtime::{
|
|
||||||
self, DIRECTML_DLL_FILE, ONNX_RUNTIME_DLL_FILE, ONNX_RUNTIME_PROVIDERS_DLL_FILE,
|
|
||||||
};
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use hf_hub::{api::sync::Api, Repo, RepoType};
|
use hf_hub::{api::sync::Api, Repo, RepoType};
|
||||||
use image::{imageops::FilterType, ImageReader};
|
use image::{imageops::FilterType, ImageReader};
|
||||||
use ort::ep;
|
|
||||||
use ort::session::SessionInputValue;
|
use ort::session::SessionInputValue;
|
||||||
use ort::session::SessionOutputs;
|
|
||||||
use ort::session::{builder::GraphOptimizationLevel, Session};
|
use ort::session::{builder::GraphOptimizationLevel, Session};
|
||||||
use ort::value::{Shape, Tensor};
|
use ort::value::{Shape, Tensor};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::Serialize;
|
||||||
use std::borrow::Cow;
|
use std::borrow::Cow;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
|
||||||
use std::time::Instant;
|
|
||||||
use tokenizers::Tokenizer;
|
use tokenizers::Tokenizer;
|
||||||
|
|
||||||
pub const FLORENCE_MODEL_ID: &str = "onnx-community/Florence-2-base-ft";
|
pub const FLORENCE_MODEL_ID: &str = "onnx-community/Florence-2-base-ft";
|
||||||
pub const FLORENCE_CAPTION_MODEL_NAME: &str = "florence-2-base-ft-onnx-q4";
|
pub const FLORENCE_CAPTION_MODEL_NAME: &str = "florence-2-base-ft-onnx-q4";
|
||||||
const CAPTION_ACCELERATION_FILE: &str = "settings/caption_acceleration.txt";
|
|
||||||
const CAPTION_DETAIL_FILE: &str = "settings/caption_detail.txt";
|
|
||||||
|
|
||||||
const REQUIRED_FILES: &[&str] = &[
|
const REQUIRED_FILES: &[&str] = &[
|
||||||
ONNX_RUNTIME_DLL_FILE,
|
|
||||||
ONNX_RUNTIME_PROVIDERS_DLL_FILE,
|
|
||||||
DIRECTML_DLL_FILE,
|
|
||||||
"config.json",
|
"config.json",
|
||||||
"generation_config.json",
|
"generation_config.json",
|
||||||
"preprocessor_config.json",
|
"preprocessor_config.json",
|
||||||
@@ -33,69 +21,10 @@ const REQUIRED_FILES: &[&str] = &[
|
|||||||
"special_tokens_map.json",
|
"special_tokens_map.json",
|
||||||
"onnx/vision_encoder_fp16.onnx",
|
"onnx/vision_encoder_fp16.onnx",
|
||||||
"onnx/encoder_model_q4.onnx",
|
"onnx/encoder_model_q4.onnx",
|
||||||
"onnx/decoder_model_q4.onnx",
|
|
||||||
"onnx/decoder_model_merged_q4.onnx",
|
"onnx/decoder_model_merged_q4.onnx",
|
||||||
"onnx/embed_tokens_fp16.onnx",
|
"onnx/embed_tokens_fp16.onnx",
|
||||||
];
|
];
|
||||||
|
|
||||||
/// Set to `true` by `set_caption_acceleration` so the caption worker loop
|
|
||||||
/// knows to drop its cached `FlorenceCaptioner` and reload with the new EP.
|
|
||||||
pub static CAPTION_SESSION_DIRTY: AtomicBool = AtomicBool::new(false);
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
|
|
||||||
#[serde(rename_all = "lowercase")]
|
|
||||||
pub enum CaptionAcceleration {
|
|
||||||
#[default]
|
|
||||||
Auto,
|
|
||||||
Cpu,
|
|
||||||
Directml,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl CaptionAcceleration {
|
|
||||||
fn as_str(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::Auto => "auto",
|
|
||||||
Self::Cpu => "cpu",
|
|
||||||
Self::Directml => "directml",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
|
|
||||||
#[serde(rename_all = "lowercase")]
|
|
||||||
pub enum CaptionDetail {
|
|
||||||
Short,
|
|
||||||
Detailed,
|
|
||||||
#[default]
|
|
||||||
Paragraph,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl CaptionDetail {
|
|
||||||
fn as_str(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::Short => "short",
|
|
||||||
Self::Detailed => "detailed",
|
|
||||||
Self::Paragraph => "paragraph",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn prompt(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::Short => "What does the image describe?",
|
|
||||||
Self::Detailed => "Describe in detail what is shown in the image.",
|
|
||||||
Self::Paragraph => "Describe with a paragraph what is shown in the image.",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn max_new_tokens(self) -> usize {
|
|
||||||
match self {
|
|
||||||
Self::Short => 32,
|
|
||||||
Self::Detailed => 56,
|
|
||||||
Self::Paragraph => 96,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub struct CaptionModelStatus {
|
pub struct CaptionModelStatus {
|
||||||
pub model_id: &'static str,
|
pub model_id: &'static str,
|
||||||
@@ -116,8 +45,6 @@ pub struct CaptionModelProgress {
|
|||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub struct CaptionRuntimeProbe {
|
pub struct CaptionRuntimeProbe {
|
||||||
pub ready: bool,
|
pub ready: bool,
|
||||||
pub acceleration: CaptionAcceleration,
|
|
||||||
pub detail: CaptionDetail,
|
|
||||||
pub tokenizer_vocab_size: usize,
|
pub tokenizer_vocab_size: usize,
|
||||||
pub sessions: Vec<CaptionRuntimeSessionProbe>,
|
pub sessions: Vec<CaptionRuntimeSessionProbe>,
|
||||||
}
|
}
|
||||||
@@ -134,7 +61,6 @@ pub struct CaptionVisionProbe {
|
|||||||
pub input_shape: Vec<i64>,
|
pub input_shape: Vec<i64>,
|
||||||
pub output_shape: Vec<i64>,
|
pub output_shape: Vec<i64>,
|
||||||
pub output_values: usize,
|
pub output_values: usize,
|
||||||
pub acceleration: CaptionAcceleration,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -145,11 +71,9 @@ struct TensorData {
|
|||||||
|
|
||||||
pub struct FlorenceCaptioner {
|
pub struct FlorenceCaptioner {
|
||||||
tokenizer: Tokenizer,
|
tokenizer: Tokenizer,
|
||||||
caption_detail: CaptionDetail,
|
|
||||||
vision_session: Session,
|
vision_session: Session,
|
||||||
embed_session: Session,
|
embed_session: Session,
|
||||||
encoder_session: Session,
|
encoder_session: Session,
|
||||||
decoder_prefill_session: Session,
|
|
||||||
decoder_session: Session,
|
decoder_session: Session,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -157,55 +81,6 @@ pub fn model_dir(app_data_dir: &Path) -> PathBuf {
|
|||||||
app_data_dir.join("models").join("florence-2-base-ft")
|
app_data_dir.join("models").join("florence-2-base-ft")
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn caption_acceleration(app_data_dir: &Path) -> CaptionAcceleration {
|
|
||||||
let path = app_data_dir.join(CAPTION_ACCELERATION_FILE);
|
|
||||||
let Ok(value) = std::fs::read_to_string(path) else {
|
|
||||||
return CaptionAcceleration::default();
|
|
||||||
};
|
|
||||||
|
|
||||||
match value.trim().to_ascii_lowercase().as_str() {
|
|
||||||
"cpu" => CaptionAcceleration::Cpu,
|
|
||||||
"directml" => CaptionAcceleration::Directml,
|
|
||||||
_ => CaptionAcceleration::Auto,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn set_caption_acceleration(
|
|
||||||
app_data_dir: &Path,
|
|
||||||
acceleration: CaptionAcceleration,
|
|
||||||
) -> Result<CaptionAcceleration> {
|
|
||||||
let path = app_data_dir.join(CAPTION_ACCELERATION_FILE);
|
|
||||||
if let Some(parent) = path.parent() {
|
|
||||||
std::fs::create_dir_all(parent)?;
|
|
||||||
}
|
|
||||||
std::fs::write(path, acceleration.as_str())?;
|
|
||||||
CAPTION_SESSION_DIRTY.store(true, Ordering::Relaxed);
|
|
||||||
Ok(acceleration)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn caption_detail(app_data_dir: &Path) -> CaptionDetail {
|
|
||||||
let path = app_data_dir.join(CAPTION_DETAIL_FILE);
|
|
||||||
let Ok(value) = std::fs::read_to_string(path) else {
|
|
||||||
return CaptionDetail::default();
|
|
||||||
};
|
|
||||||
|
|
||||||
match value.trim().to_ascii_lowercase().as_str() {
|
|
||||||
"short" => CaptionDetail::Short,
|
|
||||||
"paragraph" => CaptionDetail::Paragraph,
|
|
||||||
_ => CaptionDetail::Detailed,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn set_caption_detail(app_data_dir: &Path, detail: CaptionDetail) -> Result<CaptionDetail> {
|
|
||||||
let path = app_data_dir.join(CAPTION_DETAIL_FILE);
|
|
||||||
if let Some(parent) = path.parent() {
|
|
||||||
std::fs::create_dir_all(parent)?;
|
|
||||||
}
|
|
||||||
std::fs::write(path, detail.as_str())?;
|
|
||||||
CAPTION_SESSION_DIRTY.store(true, Ordering::Relaxed);
|
|
||||||
Ok(detail)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn caption_model_status(app_data_dir: &Path) -> CaptionModelStatus {
|
pub fn caption_model_status(app_data_dir: &Path) -> CaptionModelStatus {
|
||||||
let local_dir = model_dir(app_data_dir);
|
let local_dir = model_dir(app_data_dir);
|
||||||
let missing_files = REQUIRED_FILES
|
let missing_files = REQUIRED_FILES
|
||||||
@@ -258,20 +133,9 @@ pub fn prepare_caption_model_with_progress(
|
|||||||
if let Some(parent) = destination.parent() {
|
if let Some(parent) = destination.parent() {
|
||||||
std::fs::create_dir_all(parent)?;
|
std::fs::create_dir_all(parent)?;
|
||||||
}
|
}
|
||||||
if onnx_runtime::ONNX_RUNTIME_FILES
|
let cached = repo.get(file)?;
|
||||||
.iter()
|
std::fs::copy(cached, destination)?;
|
||||||
.any(|(runtime_file, _, _)| runtime_file == file)
|
completed_files += 1;
|
||||||
{
|
|
||||||
onnx_runtime::download_onnx_runtime_files(&local_dir)?;
|
|
||||||
completed_files = REQUIRED_FILES
|
|
||||||
.iter()
|
|
||||||
.filter(|file| local_dir.join(file).exists())
|
|
||||||
.count();
|
|
||||||
} else {
|
|
||||||
let cached = repo.get(file)?;
|
|
||||||
std::fs::copy(cached, destination)?;
|
|
||||||
completed_files += 1;
|
|
||||||
}
|
|
||||||
emit_progress(CaptionModelProgress {
|
emit_progress(CaptionModelProgress {
|
||||||
total_files: REQUIRED_FILES.len(),
|
total_files: REQUIRED_FILES.len(),
|
||||||
completed_files,
|
completed_files,
|
||||||
@@ -308,40 +172,21 @@ pub fn probe_caption_runtime(app_data_dir: &Path) -> Result<CaptionRuntimeProbe>
|
|||||||
}
|
}
|
||||||
|
|
||||||
let local_dir = model_dir(app_data_dir);
|
let local_dir = model_dir(app_data_dir);
|
||||||
onnx_runtime::ensure_onnx_runtime(&local_dir)?;
|
|
||||||
let tokenizer =
|
let tokenizer =
|
||||||
Tokenizer::from_file(local_dir.join("tokenizer.json")).map_err(anyhow::Error::msg)?;
|
Tokenizer::from_file(local_dir.join("tokenizer.json")).map_err(anyhow::Error::msg)?;
|
||||||
|
|
||||||
let acceleration = caption_acceleration(app_data_dir);
|
let sessions = [
|
||||||
|
"onnx/vision_encoder_fp16.onnx",
|
||||||
// For the vision encoder (the only session that runs on the GPU EP) we
|
|
||||||
// create an actual ORT session so we can verify which EP was actually
|
|
||||||
// loaded, rather than just echoing the settings file.
|
|
||||||
let vision_session = probe_vision_session(
|
|
||||||
&local_dir.join("onnx/vision_encoder_fp16.onnx"),
|
|
||||||
acceleration,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
// The remaining entries are DLLs and CPU-only text/decoder models — probe
|
|
||||||
// them with the lightweight file-size check as before.
|
|
||||||
let other_files = [
|
|
||||||
"onnxruntime/onnxruntime.dll",
|
|
||||||
"onnxruntime/onnxruntime_providers_shared.dll",
|
|
||||||
"onnxruntime/DirectML.dll",
|
|
||||||
"onnx/embed_tokens_fp16.onnx",
|
"onnx/embed_tokens_fp16.onnx",
|
||||||
"onnx/encoder_model_q4.onnx",
|
"onnx/encoder_model_q4.onnx",
|
||||||
"onnx/decoder_model_q4.onnx",
|
|
||||||
"onnx/decoder_model_merged_q4.onnx",
|
"onnx/decoder_model_merged_q4.onnx",
|
||||||
];
|
]
|
||||||
let mut sessions = vec![vision_session];
|
.into_iter()
|
||||||
for file in other_files {
|
.map(|file| probe_session(file, &local_dir.join(file)))
|
||||||
sessions.push(probe_session(file, &local_dir.join(file))?);
|
.collect::<Result<Vec<_>>>()?;
|
||||||
}
|
|
||||||
|
|
||||||
Ok(CaptionRuntimeProbe {
|
Ok(CaptionRuntimeProbe {
|
||||||
ready: true,
|
ready: true,
|
||||||
acceleration,
|
|
||||||
detail: caption_detail(app_data_dir),
|
|
||||||
tokenizer_vocab_size: tokenizer.get_vocab_size(false),
|
tokenizer_vocab_size: tokenizer.get_vocab_size(false),
|
||||||
sessions,
|
sessions,
|
||||||
})
|
})
|
||||||
@@ -357,17 +202,11 @@ pub fn probe_caption_vision(app_data_dir: &Path, image_path: &Path) -> Result<Ca
|
|||||||
}
|
}
|
||||||
|
|
||||||
let local_dir = model_dir(app_data_dir);
|
let local_dir = model_dir(app_data_dir);
|
||||||
onnx_runtime::ensure_onnx_runtime(&local_dir)?;
|
|
||||||
let pixels = preprocess_image(image_path)?;
|
let pixels = preprocess_image(image_path)?;
|
||||||
let input_shape = vec![1, 3, 768, 768];
|
let input_shape = vec![1, 3, 768, 768];
|
||||||
let input = Tensor::from_array(([1usize, 3, 768, 768], pixels.into_boxed_slice()))
|
let input = Tensor::from_array(([1usize, 3, 768, 768], pixels.into_boxed_slice()))
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
let acceleration = caption_acceleration(app_data_dir);
|
let mut session = create_session(&local_dir.join("onnx/vision_encoder_fp16.onnx"))?;
|
||||||
let mut session = create_session(
|
|
||||||
&local_dir.join("onnx/vision_encoder_fp16.onnx"),
|
|
||||||
acceleration,
|
|
||||||
true,
|
|
||||||
)?;
|
|
||||||
let outputs = session
|
let outputs = session
|
||||||
.run(ort::inputs! {
|
.run(ort::inputs! {
|
||||||
"pixel_values" => input
|
"pixel_values" => input
|
||||||
@@ -381,7 +220,6 @@ pub fn probe_caption_vision(app_data_dir: &Path, image_path: &Path) -> Result<Ca
|
|||||||
input_shape,
|
input_shape,
|
||||||
output_shape: output_shape.to_vec(),
|
output_shape: output_shape.to_vec(),
|
||||||
output_values: output_values.len(),
|
output_values: output_values.len(),
|
||||||
acceleration,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -392,7 +230,6 @@ pub fn generate_caption(app_data_dir: &Path, image_path: &Path) -> Result<String
|
|||||||
|
|
||||||
impl FlorenceCaptioner {
|
impl FlorenceCaptioner {
|
||||||
pub fn new(app_data_dir: &Path) -> Result<Self> {
|
pub fn new(app_data_dir: &Path) -> Result<Self> {
|
||||||
let started_at = Instant::now();
|
|
||||||
let status = caption_model_status(app_data_dir);
|
let status = caption_model_status(app_data_dir);
|
||||||
if !status.ready {
|
if !status.ready {
|
||||||
anyhow::bail!(
|
anyhow::bail!(
|
||||||
@@ -402,92 +239,48 @@ impl FlorenceCaptioner {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let local_dir = model_dir(app_data_dir);
|
let local_dir = model_dir(app_data_dir);
|
||||||
onnx_runtime::ensure_onnx_runtime(&local_dir)?;
|
|
||||||
let tokenizer =
|
let tokenizer =
|
||||||
Tokenizer::from_file(local_dir.join("tokenizer.json")).map_err(anyhow::Error::msg)?;
|
Tokenizer::from_file(local_dir.join("tokenizer.json")).map_err(anyhow::Error::msg)?;
|
||||||
let caption_detail = caption_detail(app_data_dir);
|
|
||||||
|
|
||||||
let sessions_started_at = Instant::now();
|
let vision_session = create_session(&local_dir.join("onnx/vision_encoder_fp16.onnx"))?;
|
||||||
let acceleration = caption_acceleration(app_data_dir);
|
let embed_session = create_session(&local_dir.join("onnx/embed_tokens_fp16.onnx"))?;
|
||||||
let vision_session = create_session(
|
let encoder_session = create_session(&local_dir.join("onnx/encoder_model_q4.onnx"))?;
|
||||||
&local_dir.join("onnx/vision_encoder_fp16.onnx"),
|
let decoder_session = create_session(&local_dir.join("onnx/decoder_model_merged_q4.onnx"))?;
|
||||||
acceleration,
|
|
||||||
true,
|
|
||||||
)?;
|
|
||||||
let embed_session = create_session(
|
|
||||||
&local_dir.join("onnx/embed_tokens_fp16.onnx"),
|
|
||||||
acceleration,
|
|
||||||
false,
|
|
||||||
)?;
|
|
||||||
let encoder_session = create_session(
|
|
||||||
&local_dir.join("onnx/encoder_model_q4.onnx"),
|
|
||||||
acceleration,
|
|
||||||
false,
|
|
||||||
)?;
|
|
||||||
let decoder_prefill_session = create_session(
|
|
||||||
&local_dir.join("onnx/decoder_model_q4.onnx"),
|
|
||||||
acceleration,
|
|
||||||
false,
|
|
||||||
)?;
|
|
||||||
let decoder_session = create_session(
|
|
||||||
&local_dir.join("onnx/decoder_model_merged_q4.onnx"),
|
|
||||||
acceleration,
|
|
||||||
false,
|
|
||||||
)?;
|
|
||||||
log::info!(
|
|
||||||
"Florence sessions loaded in {:?} with {:?} acceleration (total init {:?})",
|
|
||||||
sessions_started_at.elapsed(),
|
|
||||||
acceleration,
|
|
||||||
started_at.elapsed()
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
tokenizer,
|
tokenizer,
|
||||||
caption_detail,
|
|
||||||
vision_session,
|
vision_session,
|
||||||
embed_session,
|
embed_session,
|
||||||
encoder_session,
|
encoder_session,
|
||||||
decoder_prefill_session,
|
|
||||||
decoder_session,
|
decoder_session,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn generate(&mut self, image_path: &Path) -> Result<String> {
|
pub fn generate(&mut self, image_path: &Path) -> Result<String> {
|
||||||
let started_at = Instant::now();
|
|
||||||
log::info!("Florence caption started: {}", image_path.display());
|
|
||||||
let image_features = run_vision_encoder(&mut self.vision_session, image_path)?;
|
let image_features = run_vision_encoder(&mut self.vision_session, image_path)?;
|
||||||
log::info!("Florence vision encoder done in {:?}", started_at.elapsed());
|
|
||||||
let prompt_ids = self
|
let prompt_ids = self
|
||||||
.tokenizer
|
.tokenizer
|
||||||
.encode(self.caption_detail.prompt(), false)
|
.encode("What does the image describe?", false)
|
||||||
.map_err(anyhow::Error::msg)?
|
.map_err(anyhow::Error::msg)?
|
||||||
.get_ids()
|
.get_ids()
|
||||||
.iter()
|
.iter()
|
||||||
.map(|id| i64::from(*id))
|
.map(|id| i64::from(*id))
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let prompt_embeds = run_token_embedder(&mut self.embed_session, &prompt_ids)?;
|
let prompt_embeds = run_token_embedder(&mut self.embed_session, &prompt_ids)?;
|
||||||
log::info!(
|
let encoder_embeds = concatenate_sequence_embeddings(&prompt_embeds, &image_features)?;
|
||||||
"Florence token embeddings done in {:?}",
|
|
||||||
started_at.elapsed()
|
|
||||||
);
|
|
||||||
let encoder_embeds = concatenate_sequence_embeddings(&image_features, &prompt_embeds)?;
|
|
||||||
let encoder_attention_mask = vec![1_i64; encoder_embeds.shape[1] as usize];
|
let encoder_attention_mask = vec![1_i64; encoder_embeds.shape[1] as usize];
|
||||||
let encoder_hidden_states = run_encoder(
|
let encoder_hidden_states = run_encoder(
|
||||||
&mut self.encoder_session,
|
&mut self.encoder_session,
|
||||||
&encoder_embeds,
|
&encoder_embeds,
|
||||||
&encoder_attention_mask,
|
&encoder_attention_mask,
|
||||||
)?;
|
)?;
|
||||||
log::info!("Florence encoder done in {:?}", started_at.elapsed());
|
|
||||||
|
|
||||||
let generated_ids = run_decoder(
|
let generated_ids = run_decoder(
|
||||||
&mut self.decoder_prefill_session,
|
|
||||||
&mut self.decoder_session,
|
&mut self.decoder_session,
|
||||||
&mut self.embed_session,
|
&mut self.embed_session,
|
||||||
&encoder_hidden_states,
|
&encoder_hidden_states,
|
||||||
&encoder_attention_mask,
|
&encoder_attention_mask,
|
||||||
self.caption_detail.max_new_tokens(),
|
|
||||||
)?;
|
)?;
|
||||||
log::info!("Florence decoder done in {:?}", started_at.elapsed());
|
|
||||||
|
|
||||||
let generated_u32 = generated_ids
|
let generated_u32 = generated_ids
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@@ -537,26 +330,6 @@ fn probe_session(file: &'static str, path: &Path) -> Result<CaptionRuntimeSessio
|
|||||||
],
|
],
|
||||||
vec!["logits".to_string(), "present_key_values".to_string()],
|
vec!["logits".to_string(), "present_key_values".to_string()],
|
||||||
),
|
),
|
||||||
"onnxruntime/onnxruntime.dll" => (
|
|
||||||
vec!["OrtGetApiBase".to_string()],
|
|
||||||
vec!["ORT DirectML 1.24.2".to_string()],
|
|
||||||
),
|
|
||||||
"onnxruntime/onnxruntime_providers_shared.dll" => (
|
|
||||||
vec!["providers".to_string()],
|
|
||||||
vec!["shared provider bridge".to_string()],
|
|
||||||
),
|
|
||||||
"onnxruntime/DirectML.dll" => (
|
|
||||||
vec!["DirectX 12".to_string()],
|
|
||||||
vec!["DirectML 1.15.4".to_string()],
|
|
||||||
),
|
|
||||||
"onnx/decoder_model_q4.onnx" => (
|
|
||||||
vec![
|
|
||||||
"encoder_attention_mask".to_string(),
|
|
||||||
"encoder_hidden_states".to_string(),
|
|
||||||
"inputs_embeds".to_string(),
|
|
||||||
],
|
|
||||||
vec!["logits".to_string(), "present_key_values".to_string()],
|
|
||||||
),
|
|
||||||
_ => (Vec::new(), Vec::new()),
|
_ => (Vec::new(), Vec::new()),
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -567,50 +340,6 @@ fn probe_session(file: &'static str, path: &Path) -> Result<CaptionRuntimeSessio
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create an actual ORT session for the vision encoder and return a probe
|
|
||||||
/// entry that reflects which execution provider was successfully loaded.
|
|
||||||
/// This is the only session that can legitimately run on DirectML; all
|
|
||||||
/// text/decoder sessions intentionally use CPU only.
|
|
||||||
fn probe_vision_session(
|
|
||||||
path: &Path,
|
|
||||||
acceleration: CaptionAcceleration,
|
|
||||||
) -> Result<CaptionRuntimeSessionProbe> {
|
|
||||||
let metadata = std::fs::metadata(path)?;
|
|
||||||
if metadata.len() == 0 {
|
|
||||||
anyhow::bail!("{} is empty", path.display());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to create the session with the requested acceleration. If DirectML
|
|
||||||
// was requested but fails we report what actually loaded.
|
|
||||||
let loaded_acceleration = match acceleration {
|
|
||||||
CaptionAcceleration::Cpu => {
|
|
||||||
create_session(path, CaptionAcceleration::Cpu, false)?;
|
|
||||||
CaptionAcceleration::Cpu
|
|
||||||
}
|
|
||||||
CaptionAcceleration::Auto => {
|
|
||||||
// `fail_silently` — session will fall back to CPU if DirectML
|
|
||||||
// is unavailable; we detect this by trying Directml explicitly.
|
|
||||||
let directml_ok = create_session(path, CaptionAcceleration::Directml, true).is_ok();
|
|
||||||
if directml_ok {
|
|
||||||
CaptionAcceleration::Directml
|
|
||||||
} else {
|
|
||||||
create_session(path, CaptionAcceleration::Cpu, false)?;
|
|
||||||
CaptionAcceleration::Cpu
|
|
||||||
}
|
|
||||||
}
|
|
||||||
CaptionAcceleration::Directml => {
|
|
||||||
create_session(path, CaptionAcceleration::Directml, true)?;
|
|
||||||
CaptionAcceleration::Directml
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(CaptionRuntimeSessionProbe {
|
|
||||||
file: "onnx/vision_encoder_fp16.onnx",
|
|
||||||
inputs: vec!["pixel_values".to_string()],
|
|
||||||
outputs: vec![format!("image_features [EP: {:?}]", loaded_acceleration)],
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn run_vision_encoder(session: &mut Session, image_path: &Path) -> Result<TensorData> {
|
fn run_vision_encoder(session: &mut Session, image_path: &Path) -> Result<TensorData> {
|
||||||
let pixels = preprocess_image(image_path)?;
|
let pixels = preprocess_image(image_path)?;
|
||||||
let input = Tensor::from_array(([1usize, 3, 768, 768], pixels.into_boxed_slice()))
|
let input = Tensor::from_array(([1usize, 3, 768, 768], pixels.into_boxed_slice()))
|
||||||
@@ -658,50 +387,24 @@ fn run_encoder(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn run_decoder(
|
fn run_decoder(
|
||||||
decoder_prefill_session: &mut Session,
|
|
||||||
decoder_session: &mut Session,
|
decoder_session: &mut Session,
|
||||||
embed_session: &mut Session,
|
embed_session: &mut Session,
|
||||||
encoder_hidden_states: &TensorData,
|
encoder_hidden_states: &TensorData,
|
||||||
encoder_attention_mask: &[i64],
|
encoder_attention_mask: &[i64],
|
||||||
max_new_tokens: usize,
|
|
||||||
) -> Result<Vec<i64>> {
|
) -> Result<Vec<i64>> {
|
||||||
const DECODER_LAYERS: usize = 6;
|
const DECODER_LAYERS: usize = 6;
|
||||||
|
const DECODER_HEADS: usize = 12;
|
||||||
|
const HEAD_DIM: usize = 64;
|
||||||
const DECODER_START_TOKEN_ID: i64 = 2;
|
const DECODER_START_TOKEN_ID: i64 = 2;
|
||||||
const FORCED_BOS_TOKEN_ID: i64 = 0;
|
|
||||||
const EOS_TOKEN_ID: i64 = 2;
|
const EOS_TOKEN_ID: i64 = 2;
|
||||||
|
const MAX_NEW_TOKENS: usize = 32;
|
||||||
|
|
||||||
let mut generated = Vec::new();
|
let mut generated = Vec::new();
|
||||||
let encoder_attention_mask_tensor = Tensor::from_array((
|
let mut next_input_id = DECODER_START_TOKEN_ID;
|
||||||
[1usize, encoder_attention_mask.len()],
|
let mut past: Vec<TensorData> = Vec::new();
|
||||||
encoder_attention_mask.to_vec().into_boxed_slice(),
|
let mut use_cache_branch = false;
|
||||||
))
|
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
|
||||||
let encoder_hidden_states_tensor = tensor_from_data(encoder_hidden_states)?;
|
|
||||||
let prefill_inputs_embeds = run_token_embedder(embed_session, &[DECODER_START_TOKEN_ID])?;
|
|
||||||
let prefill_inputs_embeds_tensor = tensor_from_data(&prefill_inputs_embeds)?;
|
|
||||||
let prefill_started_at = Instant::now();
|
|
||||||
let prefill_outputs = decoder_prefill_session
|
|
||||||
.run(ort::inputs! {
|
|
||||||
"encoder_attention_mask" => encoder_attention_mask_tensor,
|
|
||||||
"encoder_hidden_states" => encoder_hidden_states_tensor,
|
|
||||||
"inputs_embeds" => prefill_inputs_embeds_tensor
|
|
||||||
})
|
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
|
||||||
log::info!(
|
|
||||||
"Florence decoder prefill done in {:?}",
|
|
||||||
prefill_started_at.elapsed()
|
|
||||||
);
|
|
||||||
let _prefill_logits = tensor_data(&prefill_outputs[0])?;
|
|
||||||
let mut next_input_id = FORCED_BOS_TOKEN_ID;
|
|
||||||
let encoder_kv = collect_present_key_values(&prefill_outputs, DECODER_LAYERS)?;
|
|
||||||
let mut decoder_kv = encoder_kv.clone();
|
|
||||||
|
|
||||||
for _ in 0..max_new_tokens {
|
|
||||||
if next_input_id == EOS_TOKEN_ID {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
generated.push(next_input_id);
|
|
||||||
|
|
||||||
|
for _ in 0..MAX_NEW_TOKENS {
|
||||||
let inputs_embeds = run_token_embedder(embed_session, &[next_input_id])?;
|
let inputs_embeds = run_token_embedder(embed_session, &[next_input_id])?;
|
||||||
let encoder_attention_mask_tensor = Tensor::from_array((
|
let encoder_attention_mask_tensor = Tensor::from_array((
|
||||||
[1usize, encoder_attention_mask.len()],
|
[1usize, encoder_attention_mask.len()],
|
||||||
@@ -710,8 +413,9 @@ fn run_decoder(
|
|||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
let encoder_hidden_states_tensor = tensor_from_data(encoder_hidden_states)?;
|
let encoder_hidden_states_tensor = tensor_from_data(encoder_hidden_states)?;
|
||||||
let inputs_embeds_tensor = tensor_from_data(&inputs_embeds)?;
|
let inputs_embeds_tensor = tensor_from_data(&inputs_embeds)?;
|
||||||
let use_cache_branch_tensor = Tensor::from_array(([1usize], vec![true].into_boxed_slice()))
|
let use_cache_branch_tensor =
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
Tensor::from_array(([1usize], vec![use_cache_branch].into_boxed_slice()))
|
||||||
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
|
|
||||||
let mut inputs = ort::inputs! {
|
let mut inputs = ort::inputs! {
|
||||||
"encoder_attention_mask" => encoder_attention_mask_tensor,
|
"encoder_attention_mask" => encoder_attention_mask_tensor,
|
||||||
@@ -720,82 +424,92 @@ fn run_decoder(
|
|||||||
"use_cache_branch" => use_cache_branch_tensor
|
"use_cache_branch" => use_cache_branch_tensor
|
||||||
};
|
};
|
||||||
|
|
||||||
for layer in 0..DECODER_LAYERS {
|
if past.is_empty() {
|
||||||
push_tensor_input(
|
for layer in 0..DECODER_LAYERS {
|
||||||
&mut inputs,
|
push_tensor_input(
|
||||||
format!("past_key_values.{layer}.decoder.key"),
|
&mut inputs,
|
||||||
decoder_kv[layer * 4].clone(),
|
format!("past_key_values.{layer}.decoder.key"),
|
||||||
)?;
|
TensorData::zeros(vec![1, DECODER_HEADS as i64, 0, HEAD_DIM as i64]),
|
||||||
push_tensor_input(
|
)?;
|
||||||
&mut inputs,
|
push_tensor_input(
|
||||||
format!("past_key_values.{layer}.decoder.value"),
|
&mut inputs,
|
||||||
decoder_kv[layer * 4 + 1].clone(),
|
format!("past_key_values.{layer}.decoder.value"),
|
||||||
)?;
|
TensorData::zeros(vec![1, DECODER_HEADS as i64, 0, HEAD_DIM as i64]),
|
||||||
push_tensor_input(
|
)?;
|
||||||
&mut inputs,
|
push_tensor_input(
|
||||||
format!("past_key_values.{layer}.encoder.key"),
|
&mut inputs,
|
||||||
encoder_kv[layer * 4 + 2].clone(),
|
format!("past_key_values.{layer}.encoder.key"),
|
||||||
)?;
|
TensorData::zeros(vec![1, DECODER_HEADS as i64, 0, HEAD_DIM as i64]),
|
||||||
push_tensor_input(
|
)?;
|
||||||
&mut inputs,
|
push_tensor_input(
|
||||||
format!("past_key_values.{layer}.encoder.value"),
|
&mut inputs,
|
||||||
encoder_kv[layer * 4 + 3].clone(),
|
format!("past_key_values.{layer}.encoder.value"),
|
||||||
)?;
|
TensorData::zeros(vec![1, DECODER_HEADS as i64, 0, HEAD_DIM as i64]),
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for layer in 0..DECODER_LAYERS {
|
||||||
|
for cache_name in [
|
||||||
|
"decoder.key",
|
||||||
|
"decoder.value",
|
||||||
|
"encoder.key",
|
||||||
|
"encoder.value",
|
||||||
|
] {
|
||||||
|
let past_index = layer * 4
|
||||||
|
+ match cache_name {
|
||||||
|
"decoder.key" => 0,
|
||||||
|
"decoder.value" => 1,
|
||||||
|
"encoder.key" => 2,
|
||||||
|
"encoder.value" => 3,
|
||||||
|
_ => unreachable!(),
|
||||||
|
};
|
||||||
|
push_tensor_input(
|
||||||
|
&mut inputs,
|
||||||
|
format!("past_key_values.{layer}.{cache_name}"),
|
||||||
|
past[past_index].clone(),
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let outputs = decoder_session
|
let outputs = decoder_session
|
||||||
.run(inputs)
|
.run(inputs)
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
let logits = tensor_data(&outputs["logits"])?;
|
let logits = tensor_data(&outputs["logits"])?;
|
||||||
next_input_id = argmax_last_token(&logits)?;
|
let token_id = argmax_last_token(&logits)?;
|
||||||
decoder_kv = collect_present_key_values(&outputs, DECODER_LAYERS)?;
|
if token_id == EOS_TOKEN_ID {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
generated.push(token_id);
|
||||||
|
next_input_id = token_id;
|
||||||
|
|
||||||
|
past.clear();
|
||||||
|
for layer in 0..DECODER_LAYERS {
|
||||||
|
for cache_name in [
|
||||||
|
"decoder.key",
|
||||||
|
"decoder.value",
|
||||||
|
"encoder.key",
|
||||||
|
"encoder.value",
|
||||||
|
] {
|
||||||
|
past.push(tensor_data(
|
||||||
|
&outputs[format!("present.{layer}.{cache_name}").as_str()],
|
||||||
|
)?);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
use_cache_branch = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
log::info!("Florence decoder produced {} token(s)", generated.len());
|
|
||||||
Ok(generated)
|
Ok(generated)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn create_session(
|
fn create_session(path: &Path) -> Result<Session> {
|
||||||
path: &Path,
|
|
||||||
acceleration: CaptionAcceleration,
|
|
||||||
allow_directml: bool,
|
|
||||||
) -> Result<Session> {
|
|
||||||
let builder = Session::builder().map_err(|error| anyhow::anyhow!("{error}"))?;
|
let builder = Session::builder().map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
let builder = builder
|
let builder = builder
|
||||||
.with_optimization_level(GraphOptimizationLevel::Level3)
|
.with_optimization_level(GraphOptimizationLevel::Level3)
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
let use_directml = allow_directml
|
let mut builder = builder
|
||||||
&& matches!(
|
|
||||||
acceleration,
|
|
||||||
CaptionAcceleration::Auto | CaptionAcceleration::Directml
|
|
||||||
);
|
|
||||||
let builder = builder
|
|
||||||
.with_memory_pattern(!use_directml)
|
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
|
||||||
let builder = builder
|
|
||||||
.with_parallel_execution(false)
|
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
|
||||||
let builder = builder
|
|
||||||
.with_intra_threads(1)
|
.with_intra_threads(1)
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
let mut builder = match acceleration {
|
|
||||||
CaptionAcceleration::Cpu => builder,
|
|
||||||
CaptionAcceleration::Auto if use_directml => builder
|
|
||||||
.with_execution_providers([ep::DirectML::default().build().fail_silently()])
|
|
||||||
.unwrap_or_else(|error| error.recover()),
|
|
||||||
CaptionAcceleration::Directml if use_directml => builder
|
|
||||||
.with_execution_providers([ep::DirectML::default().build().error_on_failure()])
|
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?,
|
|
||||||
CaptionAcceleration::Auto | CaptionAcceleration::Directml => {
|
|
||||||
// `allow_directml` is false for the 4 text/decoder sessions —
|
|
||||||
// they intentionally run on CPU regardless of the setting.
|
|
||||||
log::info!(
|
|
||||||
"Florence: using CPU for {} (DirectML disabled for this session type)",
|
|
||||||
path.display()
|
|
||||||
);
|
|
||||||
builder
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let session = builder
|
let session = builder
|
||||||
.commit_from_file(path)
|
.commit_from_file(path)
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
@@ -823,24 +537,6 @@ fn concatenate_sequence_embeddings(text: &TensorData, image: &TensorData) -> Res
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn collect_present_key_values(
|
|
||||||
outputs: &SessionOutputs<'_>,
|
|
||||||
layers: usize,
|
|
||||||
) -> Result<Vec<TensorData>> {
|
|
||||||
let expected = layers * 4;
|
|
||||||
if outputs.len() < expected + 1 {
|
|
||||||
anyhow::bail!(
|
|
||||||
"Decoder returned {} output(s), expected at least {}",
|
|
||||||
outputs.len(),
|
|
||||||
expected + 1
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
(1..=expected)
|
|
||||||
.map(|index| tensor_data(&outputs[index]))
|
|
||||||
.collect::<Result<Vec<_>>>()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn tensor_data(value: &ort::value::DynValue) -> Result<TensorData> {
|
fn tensor_data(value: &ort::value::DynValue) -> Result<TensorData> {
|
||||||
let (shape, values) = value
|
let (shape, values) = value
|
||||||
.try_extract_tensor::<f32>()
|
.try_extract_tensor::<f32>()
|
||||||
@@ -912,3 +608,13 @@ fn preprocess_image(image_path: &Path) -> Result<Vec<f32>> {
|
|||||||
|
|
||||||
Ok(pixel_values)
|
Ok(pixel_values)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl TensorData {
|
||||||
|
fn zeros(shape: Vec<i64>) -> Self {
|
||||||
|
let values_len = shape.iter().map(|value| (*value).max(0) as usize).product();
|
||||||
|
Self {
|
||||||
|
shape,
|
||||||
|
values: vec![0.0; values_len],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,91 +0,0 @@
|
|||||||
//! Dominant-color palette extraction for color search.
|
|
||||||
//!
|
|
||||||
//! Colors are sampled from the already-generated thumbnail (small, fast) rather
|
|
||||||
//! than the full image. We coarse-quantize pixels into an RGB histogram, then
|
|
||||||
//! return the most populated bins as representative colors with their weight
|
|
||||||
//! (fraction of sampled pixels). Search then filters images whose palette has a
|
|
||||||
//! color within a distance threshold of the query color.
|
|
||||||
|
|
||||||
use image::RgbImage;
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy)]
|
|
||||||
pub struct PaletteColor {
|
|
||||||
pub r: u8,
|
|
||||||
pub g: u8,
|
|
||||||
pub b: u8,
|
|
||||||
/// Fraction of sampled pixels (0.0–1.0) that fell in this color's bin.
|
|
||||||
pub weight: f32,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Bits kept per channel when binning. 4 bits → 16 levels/channel → 4096 bins:
|
|
||||||
/// coarse enough to group near-identical shades, fine enough to separate hues.
|
|
||||||
const QUANT_BITS: u32 = 4;
|
|
||||||
/// Cap on sampled pixels so very large frames stay cheap; thumbnails are tiny so
|
|
||||||
/// this rarely bites, but the backfill may read arbitrary thumbnail sizes.
|
|
||||||
const MAX_SAMPLES: usize = 50_000;
|
|
||||||
|
|
||||||
/// Extract up to `k` dominant colors from an RGB image, most-common first.
|
|
||||||
pub fn extract_palette(img: &RgbImage, k: usize) -> Vec<PaletteColor> {
|
|
||||||
let pixels = img.as_raw();
|
|
||||||
let pixel_count = pixels.len() / 3;
|
|
||||||
if pixel_count == 0 {
|
|
||||||
return Vec::new();
|
|
||||||
}
|
|
||||||
let step = (pixel_count / MAX_SAMPLES).max(1);
|
|
||||||
let shift = 8 - QUANT_BITS;
|
|
||||||
|
|
||||||
// bin key → (sum_r, sum_g, sum_b, count); summing lets us return the bin's
|
|
||||||
// average color rather than the quantized corner.
|
|
||||||
let mut bins: HashMap<u16, (u64, u64, u64, u64)> = HashMap::new();
|
|
||||||
let mut total: u64 = 0;
|
|
||||||
for pixel in pixels.chunks_exact(3).step_by(step) {
|
|
||||||
let (r, g, b) = (pixel[0], pixel[1], pixel[2]);
|
|
||||||
let key = (((r as u16) >> shift) << (QUANT_BITS * 2))
|
|
||||||
| (((g as u16) >> shift) << QUANT_BITS)
|
|
||||||
| ((b as u16) >> shift);
|
|
||||||
let entry = bins.entry(key).or_insert((0, 0, 0, 0));
|
|
||||||
entry.0 += r as u64;
|
|
||||||
entry.1 += g as u64;
|
|
||||||
entry.2 += b as u64;
|
|
||||||
entry.3 += 1;
|
|
||||||
total += 1;
|
|
||||||
}
|
|
||||||
if total == 0 {
|
|
||||||
return Vec::new();
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut entries: Vec<(u64, u64, u64, u64)> = bins.into_values().collect();
|
|
||||||
entries.sort_unstable_by_key(|entry| std::cmp::Reverse(entry.3));
|
|
||||||
entries
|
|
||||||
.into_iter()
|
|
||||||
.take(k)
|
|
||||||
.map(|(sum_r, sum_g, sum_b, count)| PaletteColor {
|
|
||||||
r: (sum_r / count) as u8,
|
|
||||||
g: (sum_g / count) as u8,
|
|
||||||
b: (sum_b / count) as u8,
|
|
||||||
weight: count as f32 / total as f32,
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Decode a thumbnail file and extract its palette. Used by the backfill pass.
|
|
||||||
pub fn extract_palette_from_file(thumbnail_path: &Path, k: usize) -> Option<Vec<PaletteColor>> {
|
|
||||||
let img = image::ImageReader::open(thumbnail_path)
|
|
||||||
.ok()?
|
|
||||||
.decode()
|
|
||||||
.ok()?;
|
|
||||||
Some(extract_palette(&img.into_rgb8(), k))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Number of palette colors stored per image.
|
|
||||||
pub const PALETTE_SIZE: usize = 5;
|
|
||||||
|
|
||||||
/// Max squared RGB distance for a palette color to count as matching a query
|
|
||||||
/// color (~70 units in RGB space). Tunable feel/precision of color search.
|
|
||||||
pub const MATCH_DISTANCE_SQ: i64 = 4900;
|
|
||||||
|
|
||||||
/// Minimum weight (fraction of pixels) a palette color must have to match, so
|
|
||||||
/// trivial specks of a color don't trigger a match.
|
|
||||||
pub const MATCH_MIN_WEIGHT: f64 = 0.05;
|
|
||||||
@@ -1,240 +0,0 @@
|
|||||||
//! Resilient file downloads via the system `curl` binary.
|
|
||||||
//!
|
|
||||||
//! Used for all large model/runtime downloads (tagger models, ONNX Runtime
|
|
||||||
//! DLLs, caption model). ureq's read timeout does not fire on a stalled large
|
|
||||||
//! transfer on Windows (schannel doesn't honor the socket read timeout), so a
|
|
||||||
//! stall there hangs forever. curl detects an inactivity stall
|
|
||||||
//! (`--speed-time`), resumes from the partial file (`-C -`), and retries
|
|
||||||
//! internally — the same behavior a browser gets.
|
|
||||||
|
|
||||||
use anyhow::Result;
|
|
||||||
use std::io::Read;
|
|
||||||
use std::path::Path;
|
|
||||||
use std::process::Command;
|
|
||||||
|
|
||||||
// Suppress the console window when spawning curl from the GUI app.
|
|
||||||
#[cfg(target_os = "windows")]
|
|
||||||
use std::os::windows::process::CommandExt;
|
|
||||||
#[cfg(target_os = "windows")]
|
|
||||||
const CREATE_NO_WINDOW: u32 = 0x08000000;
|
|
||||||
|
|
||||||
/// Discard target for curl output we only need the headers of.
|
|
||||||
#[cfg(target_os = "windows")]
|
|
||||||
const NULL_DEVICE: &str = "NUL";
|
|
||||||
#[cfg(not(target_os = "windows"))]
|
|
||||||
const NULL_DEVICE: &str = "/dev/null";
|
|
||||||
|
|
||||||
/// Build a `curl` command with platform quirks applied. Windows resolves the
|
|
||||||
/// bare name to `curl.exe` (bundled since Windows 10 1803) and needs the
|
|
||||||
/// no-window flag; macOS ships curl; Linux is expected to have it installed.
|
|
||||||
fn curl_command() -> Command {
|
|
||||||
#[allow(unused_mut)]
|
|
||||||
let mut command = Command::new("curl");
|
|
||||||
#[cfg(target_os = "windows")]
|
|
||||||
command.creation_flags(CREATE_NO_WINDOW);
|
|
||||||
command
|
|
||||||
}
|
|
||||||
|
|
||||||
// Give up only after this many *consecutive* curl runs that download nothing;
|
|
||||||
// a run that makes any progress resets the counter, so a large file completes
|
|
||||||
// across however many resumes it takes. Kept low so a hard stall (e.g. a
|
|
||||||
// broken VM NIC) fails in a couple of minutes — surfacing a retryable error —
|
|
||||||
// rather than locking the UI on "preparing" for many minutes.
|
|
||||||
const MAX_STALL_RETRIES: usize = 3;
|
|
||||||
|
|
||||||
/// Resiliently download `url` to `destination` using the system `curl`.
|
|
||||||
///
|
|
||||||
/// We monitor the `.part` file's size for the progress bar and wrap curl in an
|
|
||||||
/// outer progress-aware retry as a backstop. The partial file survives an app
|
|
||||||
/// restart, so a later retry continues from disk.
|
|
||||||
pub fn download_file_resilient(
|
|
||||||
url: &str,
|
|
||||||
destination: &Path,
|
|
||||||
mut on_progress: impl FnMut(u64, Option<u64>),
|
|
||||||
) -> Result<()> {
|
|
||||||
if let Some(parent) = destination.parent() {
|
|
||||||
std::fs::create_dir_all(parent)?;
|
|
||||||
}
|
|
||||||
let part = match destination.extension() {
|
|
||||||
Some(ext) => destination.with_extension(format!("{}.part", ext.to_string_lossy())),
|
|
||||||
None => destination.with_extension("part"),
|
|
||||||
};
|
|
||||||
let name = destination
|
|
||||||
.file_name()
|
|
||||||
.map(|n| n.to_string_lossy().into_owned())
|
|
||||||
.unwrap_or_else(|| url.to_string());
|
|
||||||
|
|
||||||
log::info!("{name}: resolving download size");
|
|
||||||
let total = remote_content_length(url);
|
|
||||||
|
|
||||||
// Reconcile any existing `.part` against the real size: exactly complete →
|
|
||||||
// finish; oversized (stale/corrupt) → discard so curl restarts cleanly
|
|
||||||
// (otherwise `curl -C -` would 416 forever).
|
|
||||||
if let Some(total) = total {
|
|
||||||
let size = std::fs::metadata(&part).map(|m| m.len()).unwrap_or(0);
|
|
||||||
if size == total {
|
|
||||||
std::fs::rename(&part, destination)?;
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
if size > total {
|
|
||||||
let _ = std::fs::remove_file(&part);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log::info!(
|
|
||||||
"{name}: downloading via curl ({} bytes)",
|
|
||||||
total
|
|
||||||
.map(|t| t.to_string())
|
|
||||||
.unwrap_or_else(|| "unknown size".into())
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut stalls = 0usize;
|
|
||||||
loop {
|
|
||||||
let before = std::fs::metadata(&part).map(|m| m.len()).unwrap_or(0);
|
|
||||||
match run_curl_download(url, &part, total, &mut on_progress) {
|
|
||||||
Ok(()) => break,
|
|
||||||
Err(error) => {
|
|
||||||
let after = std::fs::metadata(&part).map(|m| m.len()).unwrap_or(0);
|
|
||||||
if after > before {
|
|
||||||
log::warn!("{name}: curl interrupted at {after} bytes, resuming: {error}");
|
|
||||||
stalls = 0;
|
|
||||||
} else {
|
|
||||||
stalls += 1;
|
|
||||||
log::warn!(
|
|
||||||
"{name}: curl made no progress ({stalls}/{MAX_STALL_RETRIES}): {error}"
|
|
||||||
);
|
|
||||||
if stalls >= MAX_STALL_RETRIES {
|
|
||||||
// Discard the partial so a future attempt restarts clean
|
|
||||||
// rather than getting stuck re-resuming a bad file.
|
|
||||||
let _ = std::fs::remove_file(&part);
|
|
||||||
return Err(error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
std::thread::sleep(std::time::Duration::from_secs(2));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(total) = total {
|
|
||||||
let got = std::fs::metadata(&part).map(|m| m.len()).unwrap_or(0);
|
|
||||||
if got < total {
|
|
||||||
anyhow::bail!("{name}: incomplete after curl ({got}/{total} bytes)");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
std::fs::rename(&part, destination)?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Size probe via `curl -r 0-0` (a 1-byte Range request), parsing the total
|
|
||||||
/// from the `Content-Range: bytes 0-0/<total>` header. Uses curl rather than
|
|
||||||
/// ureq so no part of the download path depends on ureq (which hangs on this
|
|
||||||
/// VM's TLS stack). Returns None if the server doesn't report a size.
|
|
||||||
fn remote_content_length(url: &str) -> Option<u64> {
|
|
||||||
let mut command = curl_command();
|
|
||||||
command.args([
|
|
||||||
"-sL",
|
|
||||||
"-r",
|
|
||||||
"0-0",
|
|
||||||
"-D",
|
|
||||||
"-",
|
|
||||||
"-o",
|
|
||||||
NULL_DEVICE,
|
|
||||||
"--connect-timeout",
|
|
||||||
"30",
|
|
||||||
"--max-time",
|
|
||||||
"30",
|
|
||||||
"--",
|
|
||||||
url,
|
|
||||||
]);
|
|
||||||
let output = command.output().ok()?;
|
|
||||||
let headers = String::from_utf8_lossy(&output.stdout);
|
|
||||||
for line in headers.lines() {
|
|
||||||
if let Some(rest) = line.to_ascii_lowercase().strip_prefix("content-range:") {
|
|
||||||
if let Some(total) = rest.rsplit('/').next().map(str::trim) {
|
|
||||||
if let Ok(n) = total.parse::<u64>() {
|
|
||||||
return Some(n);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Run one `curl` download to `dest`, resuming from any partial file, while
|
|
||||||
/// reporting progress from the growing file size. Returns an error (leaving the
|
|
||||||
/// partial in place) if curl exits non-zero.
|
|
||||||
fn run_curl_download(
|
|
||||||
url: &str,
|
|
||||||
dest: &Path,
|
|
||||||
total: Option<u64>,
|
|
||||||
on_progress: &mut impl FnMut(u64, Option<u64>),
|
|
||||||
) -> Result<()> {
|
|
||||||
let mut command = curl_command();
|
|
||||||
command
|
|
||||||
.arg("-fSL") // fail on HTTP errors, follow redirects, show errors
|
|
||||||
.args(["-C", "-"]) // resume from the existing output file
|
|
||||||
.args(["--retry", "3", "--retry-delay", "1", "--retry-connrefused"])
|
|
||||||
.args(["--connect-timeout", "30"])
|
|
||||||
// Abort (then --retry resumes) if under 1 KB/s for 30s — a real
|
|
||||||
// inactivity timeout, which is what ureq couldn't deliver here.
|
|
||||||
.args(["--speed-limit", "1024", "--speed-time", "30"])
|
|
||||||
.arg("-s") // no progress meter (we watch the file instead)
|
|
||||||
.arg("-o")
|
|
||||||
.arg(dest)
|
|
||||||
.arg("--")
|
|
||||||
.arg(url)
|
|
||||||
.stdout(std::process::Stdio::null())
|
|
||||||
.stderr(std::process::Stdio::piped());
|
|
||||||
|
|
||||||
let mut child = command
|
|
||||||
.spawn()
|
|
||||||
.map_err(|e| anyhow::anyhow!("failed to launch curl (required for downloads): {e}"))?;
|
|
||||||
|
|
||||||
loop {
|
|
||||||
if let Some(status) = child.try_wait()? {
|
|
||||||
if status.success() {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
let mut stderr = String::new();
|
|
||||||
if let Some(mut pipe) = child.stderr.take() {
|
|
||||||
let _ = pipe.read_to_string(&mut stderr);
|
|
||||||
}
|
|
||||||
anyhow::bail!(
|
|
||||||
"curl exited with {}: {}",
|
|
||||||
status
|
|
||||||
.code()
|
|
||||||
.map(|c| c.to_string())
|
|
||||||
.unwrap_or_else(|| "signal".into()),
|
|
||||||
stderr.trim()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
let downloaded = std::fs::metadata(dest).map(|m| m.len()).unwrap_or(0);
|
|
||||||
on_progress(downloaded, total);
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(300));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Download a NuGet package (a zip) resiliently, then extract the single file
|
|
||||||
/// at `archive_path` into `destination`.
|
|
||||||
pub fn download_nuget_file(
|
|
||||||
source_url: &str,
|
|
||||||
archive_path: &str,
|
|
||||||
destination: &Path,
|
|
||||||
on_progress: impl FnMut(u64, Option<u64>),
|
|
||||||
) -> Result<()> {
|
|
||||||
let package = destination.with_extension("nupkg");
|
|
||||||
download_file_resilient(source_url, &package, on_progress)?;
|
|
||||||
|
|
||||||
log::info!("extracting {archive_path} from package");
|
|
||||||
let file = std::fs::File::open(&package)?;
|
|
||||||
let mut archive = zip::ZipArchive::new(file)?;
|
|
||||||
let mut dll = archive.by_name(archive_path)?;
|
|
||||||
let temp_destination = destination.with_extension("tmp");
|
|
||||||
{
|
|
||||||
let mut out = std::fs::File::create(&temp_destination)?;
|
|
||||||
std::io::copy(&mut dll, &mut out)?;
|
|
||||||
}
|
|
||||||
std::fs::rename(&temp_destination, destination)?;
|
|
||||||
let _ = std::fs::remove_file(&package);
|
|
||||||
log::info!("extracted {archive_path}");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
@@ -20,7 +20,7 @@ fn with_text_embedder<T>(f: impl FnOnce(&ClipImageEmbedder) -> Result<T>) -> Res
|
|||||||
.lock()
|
.lock()
|
||||||
.map_err(|_| anyhow::anyhow!("Text embedder lock poisoned"))?;
|
.map_err(|_| anyhow::anyhow!("Text embedder lock poisoned"))?;
|
||||||
if guard.is_none() {
|
if guard.is_none() {
|
||||||
log::info!("Initializing CLIP text embedder...");
|
println!("Initializing CLIP text embedder...");
|
||||||
*guard = Some(ClipImageEmbedder::new()?);
|
*guard = Some(ClipImageEmbedder::new()?);
|
||||||
}
|
}
|
||||||
f(guard.as_ref().unwrap())
|
f(guard.as_ref().unwrap())
|
||||||
@@ -35,13 +35,13 @@ pub struct ClipImageEmbedder {
|
|||||||
|
|
||||||
impl ClipImageEmbedder {
|
impl ClipImageEmbedder {
|
||||||
pub fn new() -> Result<Self> {
|
pub fn new() -> Result<Self> {
|
||||||
log::info!("Initializing CLIP image embedder...");
|
println!("Initializing CLIP image embedder...");
|
||||||
let api = Api::new()?;
|
let api = Api::new()?;
|
||||||
let repo = api.repo(Repo::new(
|
let repo = api.repo(Repo::new(
|
||||||
"laion/CLIP-ViT-B-32-laion2B-s34B-b79K".to_string(),
|
"laion/CLIP-ViT-B-32-laion2B-s34B-b79K".to_string(),
|
||||||
RepoType::Model,
|
RepoType::Model,
|
||||||
));
|
));
|
||||||
log::info!("Resolving CLIP model weights from Hugging Face cache...");
|
println!("Resolving CLIP model weights from Hugging Face cache...");
|
||||||
let model_path = repo.get("model.safetensors")?;
|
let model_path = repo.get("model.safetensors")?;
|
||||||
let tokenizer_repo = api.repo(Repo::new(
|
let tokenizer_repo = api.repo(Repo::new(
|
||||||
"openai/clip-vit-base-patch32".to_string(),
|
"openai/clip-vit-base-patch32".to_string(),
|
||||||
@@ -60,7 +60,7 @@ impl ClipImageEmbedder {
|
|||||||
};
|
};
|
||||||
let model = ClipModel::new(vb, &config)?;
|
let model = ClipModel::new(vb, &config)?;
|
||||||
let tokenizer = Tokenizer::from_file(tokenizer_path).map_err(anyhow::Error::msg)?;
|
let tokenizer = Tokenizer::from_file(tokenizer_path).map_err(anyhow::Error::msg)?;
|
||||||
log::info!("CLIP image embedder ready.");
|
println!("CLIP image embedder ready.");
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
model,
|
model,
|
||||||
@@ -74,51 +74,6 @@ impl ClipImageEmbedder {
|
|||||||
Ok(self.embed_images(&[path.to_path_buf()])?.remove(0))
|
Ok(self.embed_images(&[path.to_path_buf()])?.remove(0))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Embed a cropped region of an image without writing a temp file to disk.
|
|
||||||
/// `crop_x`, `crop_y`, `crop_w`, `crop_h` are normalized 0.0–1.0 coordinates.
|
|
||||||
pub fn embed_image_crop(
|
|
||||||
&self,
|
|
||||||
path: &Path,
|
|
||||||
crop_x: f32,
|
|
||||||
crop_y: f32,
|
|
||||||
crop_w: f32,
|
|
||||||
crop_h: f32,
|
|
||||||
) -> Result<Vec<f32>> {
|
|
||||||
let img = image::ImageReader::open(path)?
|
|
||||||
.with_guessed_format()?
|
|
||||||
.decode()?;
|
|
||||||
|
|
||||||
let img_w = img.width() as f32;
|
|
||||||
let img_h = img.height() as f32;
|
|
||||||
|
|
||||||
let x = ((crop_x * img_w) as u32).min(img.width().saturating_sub(1));
|
|
||||||
let y = ((crop_y * img_h) as u32).min(img.height().saturating_sub(1));
|
|
||||||
let w = ((crop_w * img_w) as u32).max(1).min(img.width() - x);
|
|
||||||
let h = ((crop_h * img_h) as u32).max(1).min(img.height() - y);
|
|
||||||
|
|
||||||
let cropped = img.crop_imm(x, y, w, h);
|
|
||||||
let resized = cropped.resize_to_fill(
|
|
||||||
self.image_size as u32,
|
|
||||||
self.image_size as u32,
|
|
||||||
image::imageops::FilterType::Triangle,
|
|
||||||
);
|
|
||||||
|
|
||||||
let raw = resized.to_rgb8().into_raw();
|
|
||||||
let tensor = candle_core::Tensor::from_vec(
|
|
||||||
raw,
|
|
||||||
(self.image_size, self.image_size, 3),
|
|
||||||
&candle_core::Device::Cpu,
|
|
||||||
)?
|
|
||||||
.permute((2, 0, 1))?
|
|
||||||
.to_dtype(candle_core::DType::F32)?
|
|
||||||
.affine(2.0 / 255.0, -1.0)?;
|
|
||||||
|
|
||||||
let batch = tensor.unsqueeze(0)?.to_device(&self.device)?;
|
|
||||||
let features = self.model.get_image_features(&batch)?;
|
|
||||||
let normalized = candle_transformers::models::clip::div_l2_norm(&features)?;
|
|
||||||
Ok(normalized.get(0)?.flatten_all()?.to_vec1::<f32>()?)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn embed_images(&self, paths: &[PathBuf]) -> Result<Vec<Vec<f32>>> {
|
pub fn embed_images(&self, paths: &[PathBuf]) -> Result<Vec<Vec<f32>>> {
|
||||||
let images = load_images(paths, self.image_size)?.to_device(&self.device)?;
|
let images = load_images(paths, self.image_size)?.to_device(&self.device)?;
|
||||||
let features = self.model.get_image_features(&images)?;
|
let features = self.model.get_image_features(&images)?;
|
||||||
@@ -160,7 +115,7 @@ impl ClipImageEmbedder {
|
|||||||
let ids = enc.get_ids();
|
let ids = enc.get_ids();
|
||||||
let len = ids.len().min(max_len);
|
let len = ids.len().min(max_len);
|
||||||
for j in 0..len {
|
for j in 0..len {
|
||||||
flat[i * max_len + j] = ids[j];
|
flat[i * max_len + j] = ids[j] as u32;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -177,25 +132,23 @@ impl ClipImageEmbedder {
|
|||||||
fn resolve_device() -> Result<Device> {
|
fn resolve_device() -> Result<Device> {
|
||||||
match Device::cuda_if_available(0) {
|
match Device::cuda_if_available(0) {
|
||||||
Ok(device) if device.is_cuda() => {
|
Ok(device) if device.is_cuda() => {
|
||||||
log::info!("CLIP embedder: using CUDA GPU (device 0).");
|
println!("CLIP embedder: using CUDA GPU (device 0).");
|
||||||
return Ok(device);
|
return Ok(device);
|
||||||
}
|
}
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
log::info!("CLIP embedder: no compatible CUDA GPU found — using CPU.");
|
println!("CLIP embedder: no compatible CUDA GPU found — using CPU.");
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::info!("CLIP embedder: CUDA init failed ({e}) — using CPU.");
|
println!("CLIP embedder: CUDA init failed ({e}) — using CPU.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(Device::Cpu)
|
Ok(Device::Cpu)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn load_image(path: &Path, image_size: usize) -> Result<Tensor> {
|
fn load_image(path: &Path, image_size: usize) -> Result<Tensor> {
|
||||||
// Scaled decode: CLIP only needs image_size² pixels, so decoding a large
|
let image = image::ImageReader::open(path)?
|
||||||
// JPEG at full resolution is wasted work. Cover mode keeps the shortest
|
.with_guessed_format()?
|
||||||
// side at or above image_size for the fill-crop below. Also applies EXIF
|
.decode()?;
|
||||||
// orientation, so rotated photos embed the way they are displayed.
|
|
||||||
let image = crate::thumbnail::decode_image_scaled(path, image_size as u32, true)?;
|
|
||||||
let image = image.resize_to_fill(
|
let image = image.resize_to_fill(
|
||||||
image_size as u32,
|
image_size as u32,
|
||||||
image_size as u32,
|
image_size as u32,
|
||||||
@@ -210,29 +163,28 @@ fn load_image(path: &Path, image_size: usize) -> Result<Tensor> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn load_images(paths: &[PathBuf], image_size: usize) -> Result<Tensor> {
|
fn load_images(paths: &[PathBuf], image_size: usize) -> Result<Tensor> {
|
||||||
use rayon::prelude::*;
|
let mut images = Vec::with_capacity(paths.len());
|
||||||
let images = paths
|
for path in paths {
|
||||||
.par_iter()
|
images.push(load_image(path, image_size)?);
|
||||||
.map(|path| load_image(path, image_size))
|
}
|
||||||
.collect::<Result<Vec<_>>>()?;
|
|
||||||
Ok(Tensor::stack(&images, 0)?)
|
Ok(Tensor::stack(&images, 0)?)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns the path that should be fed to the CLIP image embedder for a given media file.
|
/// Returns the path that should be fed to the CLIP image embedder for a given media file.
|
||||||
///
|
///
|
||||||
/// For videos and AVIFs the thumbnail image is used because CLIP preprocessing
|
/// For videos the thumbnail image is used (because CLIP only understands still images).
|
||||||
/// only uses decoders from the `image` crate, while AVIF is decoded through
|
/// If a video has no thumbnail yet, an error is returned — the caller should mark the
|
||||||
/// FFmpeg into a JPEG thumbnail.
|
/// embedding job as failed rather than trying to decode the raw video file.
|
||||||
pub fn embedding_source_path(
|
pub fn embedding_source_path(
|
||||||
path: &str,
|
path: &str,
|
||||||
thumbnail_path: Option<&str>,
|
thumbnail_path: Option<&str>,
|
||||||
media_kind: &str,
|
media_kind: &str,
|
||||||
) -> Result<PathBuf> {
|
) -> Result<PathBuf> {
|
||||||
if media_kind == "video" || is_avif_path(path) {
|
if media_kind == "video" {
|
||||||
match thumbnail_path {
|
match thumbnail_path {
|
||||||
Some(thumb) => Ok(PathBuf::from(thumb)),
|
Some(thumb) => Ok(PathBuf::from(thumb)),
|
||||||
None => Err(anyhow::anyhow!(
|
None => Err(anyhow::anyhow!(
|
||||||
"No thumbnail available yet for '{}' — embedding deferred until thumbnail is generated",
|
"No thumbnail available yet for video '{}' — embedding deferred until thumbnail is generated",
|
||||||
std::path::Path::new(path)
|
std::path::Path::new(path)
|
||||||
.file_name()
|
.file_name()
|
||||||
.map(|n| n.to_string_lossy())
|
.map(|n| n.to_string_lossy())
|
||||||
@@ -243,10 +195,3 @@ pub fn embedding_source_path(
|
|||||||
Ok(PathBuf::from(path))
|
Ok(PathBuf::from(path))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_avif_path(path: &str) -> bool {
|
|
||||||
std::path::Path::new(path)
|
|
||||||
.extension()
|
|
||||||
.and_then(|ext| ext.to_str())
|
|
||||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("avif"))
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,162 +0,0 @@
|
|||||||
use crate::vector;
|
|
||||||
use anyhow::Result;
|
|
||||||
use hnsw_rs::prelude::{DistCosine, Hnsw, Neighbour};
|
|
||||||
use rusqlite::Connection;
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::{OnceLock, RwLock};
|
|
||||||
|
|
||||||
const HNSW_MAX_CONNECTIONS: usize = 24;
|
|
||||||
const HNSW_EF_CONSTRUCTION: usize = 300;
|
|
||||||
const HNSW_EF_SEARCH: usize = 96;
|
|
||||||
|
|
||||||
struct CachedHnswIndex {
|
|
||||||
revision: String,
|
|
||||||
image_ids_by_external: Vec<i64>,
|
|
||||||
external_by_image_id: HashMap<i64, usize>,
|
|
||||||
hnsw: Hnsw<'static, f32, DistCosine>,
|
|
||||||
}
|
|
||||||
|
|
||||||
static IMAGE_HNSW_INDEX: OnceLock<RwLock<Option<CachedHnswIndex>>> = OnceLock::new();
|
|
||||||
|
|
||||||
fn cache() -> &'static RwLock<Option<CachedHnswIndex>> {
|
|
||||||
IMAGE_HNSW_INDEX.get_or_init(|| RwLock::new(None))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn build_index(conn: &Connection) -> Result<CachedHnswIndex> {
|
|
||||||
// Read the revision *before* fetching embeddings so we can detect any write
|
|
||||||
// that races with the build. If the revision advances while we are building,
|
|
||||||
// the resulting index would be stale — retry until it is stable.
|
|
||||||
loop {
|
|
||||||
let revision_before = vector::get_embedding_revision(conn)?;
|
|
||||||
|
|
||||||
let embeddings = vector::get_all_image_embeddings_with_ids(conn, None)?;
|
|
||||||
let max_elements = embeddings.len().max(1);
|
|
||||||
let max_layer = 16.min((max_elements as f32).ln().trunc() as usize).max(1);
|
|
||||||
let mut hnsw = Hnsw::<f32, DistCosine>::new(
|
|
||||||
HNSW_MAX_CONNECTIONS,
|
|
||||||
max_elements,
|
|
||||||
max_layer,
|
|
||||||
HNSW_EF_CONSTRUCTION,
|
|
||||||
DistCosine {},
|
|
||||||
);
|
|
||||||
|
|
||||||
let image_ids_by_external = embeddings
|
|
||||||
.iter()
|
|
||||||
.map(|(image_id, _)| *image_id)
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let external_by_image_id = image_ids_by_external
|
|
||||||
.iter()
|
|
||||||
.enumerate()
|
|
||||||
.map(|(external_id, image_id)| (*image_id, external_id))
|
|
||||||
.collect::<HashMap<_, _>>();
|
|
||||||
let data_with_id = embeddings
|
|
||||||
.iter()
|
|
||||||
.enumerate()
|
|
||||||
.map(|(external_id, (_, embedding))| (embedding, external_id))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
hnsw.parallel_insert(&data_with_id);
|
|
||||||
hnsw.set_searching_mode(true);
|
|
||||||
|
|
||||||
// If the revision is unchanged the index reflects a consistent snapshot.
|
|
||||||
let revision_after = vector::get_embedding_revision(conn)?;
|
|
||||||
if revision_before == revision_after {
|
|
||||||
return Ok(CachedHnswIndex {
|
|
||||||
revision: revision_after,
|
|
||||||
image_ids_by_external,
|
|
||||||
external_by_image_id,
|
|
||||||
hnsw,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
// A concurrent write advanced the revision — discard this build and retry.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn ensure_index(conn: &Connection) -> Result<()> {
|
|
||||||
let revision = vector::get_embedding_revision(conn)?;
|
|
||||||
|
|
||||||
{
|
|
||||||
let guard = cache().read().expect("hnsw cache poisoned");
|
|
||||||
if guard.as_ref().map(|cached| cached.revision.as_str()) == Some(revision.as_str()) {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let next = build_index(conn)?;
|
|
||||||
let mut guard = cache().write().expect("hnsw cache poisoned");
|
|
||||||
*guard = Some(next);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn find_similar_image_matches(
|
|
||||||
conn: &Connection,
|
|
||||||
image_id: i64,
|
|
||||||
folder_id: Option<i64>,
|
|
||||||
album_id: Option<i64>,
|
|
||||||
threshold: f32,
|
|
||||||
offset: usize,
|
|
||||||
limit: usize,
|
|
||||||
) -> Result<Vec<(i64, f32)>> {
|
|
||||||
ensure_index(conn)?;
|
|
||||||
|
|
||||||
let query_embedding = match vector::get_image_embedding(conn, image_id)? {
|
|
||||||
Some(embedding) => embedding,
|
|
||||||
None => return Ok(Vec::new()),
|
|
||||||
};
|
|
||||||
|
|
||||||
// Build the allowed-id set *before* acquiring the read lock so we don't hold
|
|
||||||
// the lock across a potentially slow SQLite query, which would delay any
|
|
||||||
// concurrent ensure_index call waiting for a write lock. Album scope takes
|
|
||||||
// precedence over folder scope; both reuse the HNSW filtered search.
|
|
||||||
let allowed_image_ids: Option<Vec<i64>> = if let Some(album_id) = album_id {
|
|
||||||
let mut stmt = conn.prepare("SELECT image_id FROM album_images WHERE album_id = ?1")?;
|
|
||||||
let ids = stmt
|
|
||||||
.query_map([album_id], |row| row.get::<_, i64>(0))?
|
|
||||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
|
||||||
Some(ids)
|
|
||||||
} else if let Some(folder_id) = folder_id {
|
|
||||||
let ids = vector::get_all_image_embeddings_with_ids(conn, Some(folder_id))?
|
|
||||||
.into_iter()
|
|
||||||
.map(|(id, _)| id)
|
|
||||||
.collect();
|
|
||||||
Some(ids)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
let guard = cache().read().expect("hnsw cache poisoned");
|
|
||||||
let Some(cached) = guard.as_ref() else {
|
|
||||||
return Ok(Vec::new());
|
|
||||||
};
|
|
||||||
|
|
||||||
let knbn = (offset + limit).max(limit).saturating_add(32);
|
|
||||||
let neighbours: Vec<Neighbour> = if let Some(image_ids) = allowed_image_ids {
|
|
||||||
let mut allowed_ids = image_ids
|
|
||||||
.into_iter()
|
|
||||||
.filter_map(|allowed_image_id| {
|
|
||||||
cached.external_by_image_id.get(&allowed_image_id).copied()
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
allowed_ids.sort_unstable();
|
|
||||||
cached
|
|
||||||
.hnsw
|
|
||||||
.search_filter(&query_embedding, knbn, HNSW_EF_SEARCH, Some(&allowed_ids))
|
|
||||||
} else {
|
|
||||||
cached.hnsw.search(&query_embedding, knbn, HNSW_EF_SEARCH)
|
|
||||||
};
|
|
||||||
|
|
||||||
let matches = neighbours
|
|
||||||
.into_iter()
|
|
||||||
.filter_map(|neighbour| {
|
|
||||||
let image_id_match = cached.image_ids_by_external.get(neighbour.d_id).copied()?;
|
|
||||||
if image_id_match == image_id || neighbour.distance > threshold {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
Some((image_id_match, neighbour.distance))
|
|
||||||
})
|
|
||||||
.skip(offset)
|
|
||||||
.take(limit)
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
Ok(matches)
|
|
||||||
}
|
|
||||||