make pipeline responsive at scale: cap review query, debounce sse reload, indexable done count
All checks were successful
Build and Push Docker Image / build (push) Successful in 37s

The pipeline endpoint returned every pending plan (no LIMIT) while the audio
list capped at 500 — that alone was the main lag. SSE compounded it: every
job_update (which fires per-line of running ffmpeg output) re-ran the entire
endpoint and re-rendered every card.

- review query: LIMIT 500 + a separate COUNT for reviewTotal; column header
  shows 'X of Y' and a footer 'Showing first X of Y. Approve some to see
  the rest' when truncated
- doneCount: split the OR-form into two indexable counts (is_noop + done&!noop),
  added together — uses idx_review_plans_is_noop and idx_review_plans_status
  instead of full scan
- pipeline page: 1s debounce on SSE-triggered reload so a burst of
  job_update events collapses into one refetch
This commit is contained in:
2026-04-13 10:00:08 +02:00
parent 9ee0dd445f
commit ec28e43484
3 changed files with 46 additions and 19 deletions

View File

@@ -229,6 +229,10 @@ app.get("/pipeline", (c) => {
const db = getDb();
const jellyfinUrl = getConfig("jellyfin_url") ?? "";
// Cap the review column to keep the page snappy at scale; pipelines
// with thousands of pending items would otherwise ship 10k+ rows on
// every refresh and re-render every card.
const REVIEW_LIMIT = 500;
const review = db
.prepare(`
SELECT rp.*, mi.name, mi.series_name, mi.series_jellyfin_id,
@@ -242,8 +246,12 @@ app.get("/pipeline", (c) => {
CASE rp.confidence WHEN 'high' THEN 0 ELSE 1 END,
COALESCE(mi.series_name, mi.name),
mi.season_number, mi.episode_number
LIMIT ${REVIEW_LIMIT}
`)
.all();
const reviewTotal = (
db.prepare("SELECT COUNT(*) as n FROM review_plans WHERE status = 'pending' AND is_noop = 0").get() as { n: number }
).n;
const queued = db
.prepare(`
@@ -281,17 +289,15 @@ app.get("/pipeline", (c) => {
`)
.all();
// "Done" = files that are already in the desired end state. Two ways
// to get there: (a) the analyzer says nothing to do (is_noop=1), or
// (b) we ran a job that finished. Both count toward the same total.
const doneCount = (
db
.prepare(`
SELECT COUNT(*) as count FROM review_plans
WHERE is_noop = 1 OR status = 'done'
`)
.get() as { count: number }
).count;
// "Done" = files already in the desired end state. Either the analyzer
// says nothing to do (is_noop=1) or a job finished. Use two indexable
// counts and add — the OR form (is_noop=1 OR status='done') can't use
// our single-column indexes and gets slow on large libraries.
const noopRow = db.prepare("SELECT COUNT(*) as n FROM review_plans WHERE is_noop = 1").get() as { n: number };
const doneRow = db.prepare("SELECT COUNT(*) as n FROM review_plans WHERE status = 'done' AND is_noop = 0").get() as {
n: number;
};
const doneCount = noopRow.n + doneRow.n;
// Batch transcode reasons for all review plans in one query (avoids N+1)
const planIds = (review as { id: number }[]).map((r) => r.id);
@@ -315,7 +321,7 @@ app.get("/pipeline", (c) => {
item.transcode_reasons = reasonsByPlan.get(item.id) ?? [];
}
return c.json({ review, queued, processing, done, doneCount, jellyfinUrl });
return c.json({ review, reviewTotal, queued, processing, done, doneCount, jellyfinUrl });
});
// ─── List ─────────────────────────────────────────────────────────────────────