feat(tagger): real byte progress for model download instead of a spinner
The 1.3 GB model.onnx downloaded via hf-hub's repo.get() and the ONNX runtime DLLs via a read_to_end — neither reported bytes, so the user saw only an indeterminate spinner on a multi-minute download. - tagger model files now download via repo.download_with_progress with a HubProgress adapter over hf-hub's Progress trait (throttled 200ms events carrying downloaded/total bytes) - captioner's nuget DLL download streams in 64 KB chunks, parsing content-length, and reports per-chunk byte progress - TaggerModelProgress carries downloaded_bytes/total_bytes; onboarding and Settings show a determinate bar plus 'X MB / Y MB' for the current file
This commit is contained in:
+57
-11
@@ -669,20 +669,45 @@ pub fn ensure_onnx_runtime(local_dir: &Path) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Download any ONNX Runtime DLLs that are missing from `local_dir`. Unlike
|
/// Download any ONNX Runtime DLLs missing from `local_dir`, reporting per-file
|
||||||
|
/// byte progress as `(short_label, downloaded_bytes, total_bytes)`.
|
||||||
|
/// `total_bytes` is `None` when the server omits Content-Length. Unlike
|
||||||
/// `ensure_onnx_runtime` (init only), this actually provisions the files —
|
/// `ensure_onnx_runtime` (init only), this actually provisions the files —
|
||||||
/// callers that can run on a clean install must call this first.
|
/// callers that can run on a clean install must call this first. The callback
|
||||||
pub fn provision_onnx_runtime(local_dir: &Path) -> Result<()> {
|
/// fires per chunk; callers should throttle.
|
||||||
|
pub fn provision_onnx_runtime_with_progress(
|
||||||
|
local_dir: &Path,
|
||||||
|
mut on_progress: impl FnMut(&str, u64, Option<u64>),
|
||||||
|
) -> Result<()> {
|
||||||
for (destination_file, source_url, archive_path) in ONNX_RUNTIME_FILES {
|
for (destination_file, source_url, archive_path) in ONNX_RUNTIME_FILES {
|
||||||
let destination = local_dir.join(destination_file);
|
let destination = local_dir.join(destination_file);
|
||||||
if destination.exists() {
|
if destination.exists() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
download_nuget_file(source_url, archive_path, &destination)?;
|
// Strip the "onnxruntime/" prefix for a clean label.
|
||||||
|
let label = destination_file
|
||||||
|
.rsplit('/')
|
||||||
|
.next()
|
||||||
|
.unwrap_or(destination_file);
|
||||||
|
download_nuget_file(
|
||||||
|
source_url,
|
||||||
|
archive_path,
|
||||||
|
&destination,
|
||||||
|
|downloaded, total| on_progress(label, downloaded, total),
|
||||||
|
)?;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Number of ONNX Runtime DLLs still missing from `local_dir` (for progress
|
||||||
|
/// step counts before downloading).
|
||||||
|
pub fn missing_onnx_runtime_count(local_dir: &Path) -> usize {
|
||||||
|
ONNX_RUNTIME_FILES
|
||||||
|
.iter()
|
||||||
|
.filter(|(destination_file, _, _)| !local_dir.join(destination_file).exists())
|
||||||
|
.count()
|
||||||
|
}
|
||||||
|
|
||||||
fn download_onnx_runtime_files(local_dir: &Path) -> Result<()> {
|
fn download_onnx_runtime_files(local_dir: &Path) -> Result<()> {
|
||||||
if !cfg!(target_os = "windows") {
|
if !cfg!(target_os = "windows") {
|
||||||
anyhow::bail!(
|
anyhow::bail!(
|
||||||
@@ -692,22 +717,43 @@ fn download_onnx_runtime_files(local_dir: &Path) -> Result<()> {
|
|||||||
|
|
||||||
for (destination_file, source_url, archive_path) in ONNX_RUNTIME_FILES {
|
for (destination_file, source_url, archive_path) in ONNX_RUNTIME_FILES {
|
||||||
let destination = local_dir.join(destination_file);
|
let destination = local_dir.join(destination_file);
|
||||||
download_nuget_file(source_url, archive_path, &destination)?;
|
download_nuget_file(source_url, archive_path, &destination, |_, _| {})?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn download_nuget_file(source_url: &str, archive_path: &str, destination: &Path) -> Result<()> {
|
fn download_nuget_file(
|
||||||
|
source_url: &str,
|
||||||
|
archive_path: &str,
|
||||||
|
destination: &Path,
|
||||||
|
mut on_progress: impl FnMut(u64, Option<u64>),
|
||||||
|
) -> Result<()> {
|
||||||
let mut response = ureq::get(source_url)
|
let mut response = ureq::get(source_url)
|
||||||
.call()
|
.call()
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
let mut bytes = Vec::new();
|
let total_bytes = response
|
||||||
response
|
.headers()
|
||||||
.body_mut()
|
.get("content-length")
|
||||||
.as_reader()
|
.and_then(|value| value.to_str().ok())
|
||||||
.read_to_end(&mut bytes)
|
.and_then(|value| value.parse::<u64>().ok());
|
||||||
|
|
||||||
|
// Stream into memory (the zip reader needs Seek) while reporting bytes.
|
||||||
|
let mut bytes = Vec::with_capacity(total_bytes.unwrap_or(0) as usize);
|
||||||
|
let mut reader = response.body_mut().as_reader();
|
||||||
|
let mut buffer = [0u8; 64 * 1024];
|
||||||
|
let mut downloaded = 0u64;
|
||||||
|
loop {
|
||||||
|
let read = reader
|
||||||
|
.read(&mut buffer)
|
||||||
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
.map_err(|error| anyhow::anyhow!("{error}"))?;
|
||||||
|
if read == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
bytes.extend_from_slice(&buffer[..read]);
|
||||||
|
downloaded += read as u64;
|
||||||
|
on_progress(downloaded, total_bytes);
|
||||||
|
}
|
||||||
|
|
||||||
let mut archive = zip::ZipArchive::new(Cursor::new(bytes))?;
|
let mut archive = zip::ZipArchive::new(Cursor::new(bytes))?;
|
||||||
let mut dll = archive.by_name(archive_path)?;
|
let mut dll = archive.by_name(archive_path)?;
|
||||||
|
|||||||
+107
-29
@@ -87,9 +87,63 @@ pub struct TaggerModelProgress {
|
|||||||
pub total_files: usize,
|
pub total_files: usize,
|
||||||
pub completed_files: usize,
|
pub completed_files: usize,
|
||||||
pub current_file: Option<String>,
|
pub current_file: Option<String>,
|
||||||
|
// Byte progress for the file currently downloading. None for files whose
|
||||||
|
// size isn't known up front (or between files).
|
||||||
|
pub downloaded_bytes: Option<u64>,
|
||||||
|
pub total_bytes: Option<u64>,
|
||||||
pub done: bool,
|
pub done: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Adapts hf-hub's `Progress` trait to throttled `tagger-model-progress`
|
||||||
|
/// events so the 1.3 GB model download reports real bytes, not a spinner.
|
||||||
|
struct HubProgress<'a, F: Fn(TaggerModelProgress)> {
|
||||||
|
emit: &'a F,
|
||||||
|
total_files: usize,
|
||||||
|
completed_files: usize,
|
||||||
|
current_file: String,
|
||||||
|
downloaded: u64,
|
||||||
|
total: u64,
|
||||||
|
last_emit: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<F: Fn(TaggerModelProgress)> hf_hub::api::Progress for HubProgress<'_, F> {
|
||||||
|
fn init(&mut self, size: usize, _filename: &str) {
|
||||||
|
self.total = size as u64;
|
||||||
|
self.downloaded = 0;
|
||||||
|
self.last_emit = Instant::now();
|
||||||
|
(self.emit)(self.snapshot(false));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update(&mut self, size: usize) {
|
||||||
|
self.downloaded = self.downloaded.saturating_add(size as u64);
|
||||||
|
if self.last_emit.elapsed() >= std::time::Duration::from_millis(200) {
|
||||||
|
self.last_emit = Instant::now();
|
||||||
|
(self.emit)(self.snapshot(false));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finish(&mut self) {
|
||||||
|
(self.emit)(self.snapshot(false));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<F: Fn(TaggerModelProgress)> HubProgress<'_, F> {
|
||||||
|
fn snapshot(&self, done: bool) -> TaggerModelProgress {
|
||||||
|
TaggerModelProgress {
|
||||||
|
total_files: self.total_files,
|
||||||
|
completed_files: self.completed_files,
|
||||||
|
current_file: Some(self.current_file.clone()),
|
||||||
|
downloaded_bytes: Some(self.downloaded),
|
||||||
|
total_bytes: if self.total > 0 {
|
||||||
|
Some(self.total)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
},
|
||||||
|
done,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Runtime probe types exposed to the frontend
|
// Runtime probe types exposed to the frontend
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -250,59 +304,83 @@ pub fn prepare_tagger_model_with_progress(
|
|||||||
let local_dir = model_dir(app_data_dir);
|
let local_dir = model_dir(app_data_dir);
|
||||||
std::fs::create_dir_all(&local_dir)?;
|
std::fs::create_dir_all(&local_dir)?;
|
||||||
|
|
||||||
// Ensure the shared ONNX runtime DLLs are present. The tagger shares
|
// The tagger shares the ONNX runtime DLLs with the captioner; download
|
||||||
// them with the captioner; download them here so the tagger is fully
|
// them here so the tagger works even on a clean install where the caption
|
||||||
// functional even on a clean install where the caption model has never
|
// model has never been fetched (ensure_onnx_runtime only initializes).
|
||||||
// been downloaded (ensure_onnx_runtime only initializes, never downloads).
|
const DOWNLOAD_FILES: &[&str] = &["model.onnx", "selected_tags.csv"];
|
||||||
let caption_model_dir = crate::captioner::model_dir(app_data_dir);
|
let caption_model_dir = crate::captioner::model_dir(app_data_dir);
|
||||||
std::fs::create_dir_all(&caption_model_dir)?;
|
std::fs::create_dir_all(&caption_model_dir)?;
|
||||||
crate::captioner::provision_onnx_runtime(&caption_model_dir)?;
|
|
||||||
crate::captioner::ensure_onnx_runtime(&caption_model_dir)?;
|
|
||||||
|
|
||||||
// Download the two tagger-specific files.
|
// Unified step count across DLLs and tagger files so the bar is coherent.
|
||||||
const DOWNLOAD_FILES: &[&str] = &["model.onnx", "selected_tags.csv"];
|
let dll_count = crate::captioner::missing_onnx_runtime_count(&caption_model_dir);
|
||||||
|
let model_pending = DOWNLOAD_FILES
|
||||||
let api = Api::new()?;
|
|
||||||
let repo = api.repo(Repo::new(WD_TAGGER_MODEL_ID.to_string(), RepoType::Model));
|
|
||||||
|
|
||||||
let mut completed_files = DOWNLOAD_FILES
|
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|file| local_dir.join(file).exists())
|
.filter(|file| !local_dir.join(file).exists())
|
||||||
.count();
|
.count();
|
||||||
|
let total_files = dll_count + model_pending;
|
||||||
|
let mut completed_files = 0usize;
|
||||||
|
|
||||||
emit_progress(TaggerModelProgress {
|
emit_progress(TaggerModelProgress {
|
||||||
total_files: DOWNLOAD_FILES.len(),
|
total_files,
|
||||||
completed_files,
|
completed_files,
|
||||||
current_file: None,
|
current_file: None,
|
||||||
done: completed_files == DOWNLOAD_FILES.len(),
|
downloaded_bytes: None,
|
||||||
|
total_bytes: None,
|
||||||
|
done: total_files == 0,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── ONNX runtime DLLs (small, but non-trivial on a slow link) ──
|
||||||
|
{
|
||||||
|
let mut last_emit = Instant::now() - std::time::Duration::from_secs(1);
|
||||||
|
crate::captioner::provision_onnx_runtime_with_progress(
|
||||||
|
&caption_model_dir,
|
||||||
|
|label, downloaded, total| {
|
||||||
|
if last_emit.elapsed() >= std::time::Duration::from_millis(200) {
|
||||||
|
last_emit = Instant::now();
|
||||||
|
emit_progress(TaggerModelProgress {
|
||||||
|
total_files,
|
||||||
|
completed_files,
|
||||||
|
current_file: Some(format!("ONNX Runtime: {label}")),
|
||||||
|
downloaded_bytes: Some(downloaded),
|
||||||
|
total_bytes: total,
|
||||||
|
done: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
completed_files += dll_count;
|
||||||
|
}
|
||||||
|
crate::captioner::ensure_onnx_runtime(&caption_model_dir)?;
|
||||||
|
|
||||||
|
// ── Tagger model files (model.onnx is ~1.3 GB) ──
|
||||||
|
let api = Api::new()?;
|
||||||
|
let repo = api.repo(Repo::new(WD_TAGGER_MODEL_ID.to_string(), RepoType::Model));
|
||||||
|
|
||||||
for file in DOWNLOAD_FILES {
|
for file in DOWNLOAD_FILES {
|
||||||
let destination = local_dir.join(file);
|
let destination = local_dir.join(file);
|
||||||
if destination.exists() {
|
if destination.exists() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
emit_progress(TaggerModelProgress {
|
let progress = HubProgress {
|
||||||
total_files: DOWNLOAD_FILES.len(),
|
emit: &emit_progress,
|
||||||
|
total_files,
|
||||||
completed_files,
|
completed_files,
|
||||||
current_file: Some((*file).to_string()),
|
current_file: (*file).to_string(),
|
||||||
done: false,
|
downloaded: 0,
|
||||||
});
|
total: 0,
|
||||||
let cached = repo.get(file)?;
|
last_emit: Instant::now(),
|
||||||
|
};
|
||||||
|
let cached = repo.download_with_progress(file, progress)?;
|
||||||
std::fs::copy(cached, destination)?;
|
std::fs::copy(cached, destination)?;
|
||||||
completed_files += 1;
|
completed_files += 1;
|
||||||
emit_progress(TaggerModelProgress {
|
|
||||||
total_files: DOWNLOAD_FILES.len(),
|
|
||||||
completed_files,
|
|
||||||
current_file: Some((*file).to_string()),
|
|
||||||
done: completed_files == DOWNLOAD_FILES.len(),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
emit_progress(TaggerModelProgress {
|
emit_progress(TaggerModelProgress {
|
||||||
total_files: DOWNLOAD_FILES.len(),
|
total_files,
|
||||||
completed_files,
|
completed_files,
|
||||||
current_file: None,
|
current_file: None,
|
||||||
|
downloaded_bytes: None,
|
||||||
|
total_bytes: None,
|
||||||
done: true,
|
done: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,12 @@ const SECTIONS: { id: SettingsSection; label: string; detail: string }[] = [
|
|||||||
{ id: "general", label: "General", detail: "App data and diagnostics" },
|
{ id: "general", label: "General", detail: "App data and diagnostics" },
|
||||||
];
|
];
|
||||||
|
|
||||||
|
function formatBytesShort(bytes: number): string {
|
||||||
|
if (bytes >= 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
|
||||||
|
if (bytes >= 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)} MB`;
|
||||||
|
return `${(bytes / 1024).toFixed(0)} KB`;
|
||||||
|
}
|
||||||
|
|
||||||
function StatusPill({ children, tone }: { children: React.ReactNode; tone: "ready" | "muted" | "busy" }) {
|
function StatusPill({ children, tone }: { children: React.ReactNode; tone: "ready" | "muted" | "busy" }) {
|
||||||
const className =
|
const className =
|
||||||
tone === "ready"
|
tone === "ready"
|
||||||
@@ -235,14 +241,22 @@ export function SettingsModal() {
|
|||||||
: "no folders selected";
|
: "no folders selected";
|
||||||
const thresholdDisplay = taggerThresholdDraft ?? String(taggerThreshold);
|
const thresholdDisplay = taggerThresholdDraft ?? String(taggerThreshold);
|
||||||
const batchSizeDisplay = taggerBatchSizeDraft ?? String(taggerBatchSize);
|
const batchSizeDisplay = taggerBatchSizeDraft ?? String(taggerBatchSize);
|
||||||
const taggerDownloadLabel = taggerModelProgress
|
const taggerBytesKnown =
|
||||||
|
taggerModelProgress?.downloaded_bytes != null &&
|
||||||
|
taggerModelProgress.total_bytes != null &&
|
||||||
|
taggerModelProgress.total_bytes > 0;
|
||||||
|
const taggerDownloadLabel = taggerBytesKnown
|
||||||
|
? `Downloading ${formatBytesShort(taggerModelProgress!.downloaded_bytes!)} / ${formatBytesShort(taggerModelProgress!.total_bytes!)}`
|
||||||
|
: taggerModelProgress
|
||||||
? `Downloading ${taggerModelProgress.completed_files}/${taggerModelProgress.total_files}`
|
? `Downloading ${taggerModelProgress.completed_files}/${taggerModelProgress.total_files}`
|
||||||
: taggerModelPreparing
|
: taggerModelPreparing
|
||||||
? "Preparing WD Tagger..."
|
? "Preparing WD Tagger..."
|
||||||
: taggerReady
|
: taggerReady
|
||||||
? "Installed"
|
? "Installed"
|
||||||
: "Install model";
|
: "Install model";
|
||||||
const taggerDownloadPercent = taggerModelProgress
|
const taggerDownloadPercent = taggerBytesKnown
|
||||||
|
? Math.round((taggerModelProgress!.downloaded_bytes! / taggerModelProgress!.total_bytes!) * 100)
|
||||||
|
: taggerModelProgress
|
||||||
? Math.round((taggerModelProgress.completed_files / Math.max(taggerModelProgress.total_files, 1)) * 100)
|
? Math.round((taggerModelProgress.completed_files / Math.max(taggerModelProgress.total_files, 1)) * 100)
|
||||||
: 0;
|
: 0;
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,20 @@
|
|||||||
import { useEffect } from "react";
|
import { useEffect } from "react";
|
||||||
import { useGalleryStore } from "../../store";
|
import { TaggerModelProgress, useGalleryStore } from "../../store";
|
||||||
import { FakeProgressBar } from "./fakes";
|
import { FakeProgressBar, formatBytes } from "./fakes";
|
||||||
|
|
||||||
const FAKE_TAGS = ["landscape", "sunset", "outdoors", "no_humans", "ocean", "cloudy_sky"];
|
const FAKE_TAGS = ["landscape", "sunset", "outdoors", "no_humans", "ocean", "cloudy_sky"];
|
||||||
|
|
||||||
|
// Prefer the current file's byte fraction (the 1.3 GB model dominates); fall
|
||||||
|
// back to the coarse step count; null renders an indeterminate bar.
|
||||||
|
function taggerDownloadFraction(progress: TaggerModelProgress | null): number | null {
|
||||||
|
if (!progress) return null;
|
||||||
|
if (progress.downloaded_bytes != null && progress.total_bytes != null && progress.total_bytes > 0) {
|
||||||
|
return progress.downloaded_bytes / progress.total_bytes;
|
||||||
|
}
|
||||||
|
if (progress.total_files > 0) return progress.completed_files / progress.total_files;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
export function StepAiFeatures() {
|
export function StepAiFeatures() {
|
||||||
const taggerModelStatus = useGalleryStore((s) => s.taggerModelStatus);
|
const taggerModelStatus = useGalleryStore((s) => s.taggerModelStatus);
|
||||||
const taggerModelPreparing = useGalleryStore((s) => s.taggerModelPreparing);
|
const taggerModelPreparing = useGalleryStore((s) => s.taggerModelPreparing);
|
||||||
@@ -61,17 +72,16 @@ export function StepAiFeatures() {
|
|||||||
</div>
|
</div>
|
||||||
{taggerModelPreparing ? (
|
{taggerModelPreparing ? (
|
||||||
<div className="mt-3">
|
<div className="mt-3">
|
||||||
<FakeProgressBar
|
<FakeProgressBar fraction={taggerDownloadFraction(taggerModelProgress)} />
|
||||||
fraction={
|
<div className="mt-1.5 flex items-center justify-between gap-3 text-[11px] text-gray-600">
|
||||||
taggerModelProgress && taggerModelProgress.total_files > 0
|
<span className="truncate">{taggerModelProgress?.current_file ?? "Preparing..."}</span>
|
||||||
? taggerModelProgress.completed_files / taggerModelProgress.total_files
|
{taggerModelProgress?.downloaded_bytes != null && taggerModelProgress.total_bytes != null ? (
|
||||||
: null
|
<span className="shrink-0 tabular-nums">
|
||||||
}
|
{formatBytes(taggerModelProgress.downloaded_bytes)} / {formatBytes(taggerModelProgress.total_bytes)}
|
||||||
/>
|
</span>
|
||||||
{taggerModelProgress?.current_file ? (
|
|
||||||
<p className="mt-1.5 truncate text-[11px] text-gray-600">{taggerModelProgress.current_file}</p>
|
|
||||||
) : null}
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
{!taggerModelPreparing && taggerModelError ? (
|
{!taggerModelPreparing && taggerModelError ? (
|
||||||
<p className="mt-2 text-xs leading-relaxed text-amber-300/90">
|
<p className="mt-2 text-xs leading-relaxed text-amber-300/90">
|
||||||
|
|||||||
@@ -112,6 +112,8 @@ export interface TaggerModelProgress {
|
|||||||
total_files: number;
|
total_files: number;
|
||||||
completed_files: number;
|
completed_files: number;
|
||||||
current_file: string | null;
|
current_file: string | null;
|
||||||
|
downloaded_bytes: number | null;
|
||||||
|
total_bytes: number | null;
|
||||||
done: boolean;
|
done: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user