make pipeline responsive at scale: cap review query, debounce sse reload, indexable done count
All checks were successful
Build and Push Docker Image / build (push) Successful in 37s
All checks were successful
Build and Push Docker Image / build (push) Successful in 37s
The pipeline endpoint returned every pending plan (no LIMIT) while the audio list capped at 500 — that alone was the main lag. SSE compounded it: every job_update (which fires per-line of running ffmpeg output) re-ran the entire endpoint and re-rendered every card. - review query: LIMIT 500 + a separate COUNT for reviewTotal; column header shows 'X of Y' and a footer 'Showing first X of Y. Approve some to see the rest' when truncated - doneCount: split the OR-form into two indexable counts (is_noop + done&!noop), added together — uses idx_review_plans_is_noop and idx_review_plans_status instead of full scan - pipeline page: 1s debounce on SSE-triggered reload so a burst of job_update events collapses into one refetch
This commit is contained in:
@@ -229,6 +229,10 @@ app.get("/pipeline", (c) => {
|
||||
const db = getDb();
|
||||
const jellyfinUrl = getConfig("jellyfin_url") ?? "";
|
||||
|
||||
// Cap the review column to keep the page snappy at scale; pipelines
|
||||
// with thousands of pending items would otherwise ship 10k+ rows on
|
||||
// every refresh and re-render every card.
|
||||
const REVIEW_LIMIT = 500;
|
||||
const review = db
|
||||
.prepare(`
|
||||
SELECT rp.*, mi.name, mi.series_name, mi.series_jellyfin_id,
|
||||
@@ -242,8 +246,12 @@ app.get("/pipeline", (c) => {
|
||||
CASE rp.confidence WHEN 'high' THEN 0 ELSE 1 END,
|
||||
COALESCE(mi.series_name, mi.name),
|
||||
mi.season_number, mi.episode_number
|
||||
LIMIT ${REVIEW_LIMIT}
|
||||
`)
|
||||
.all();
|
||||
const reviewTotal = (
|
||||
db.prepare("SELECT COUNT(*) as n FROM review_plans WHERE status = 'pending' AND is_noop = 0").get() as { n: number }
|
||||
).n;
|
||||
|
||||
const queued = db
|
||||
.prepare(`
|
||||
@@ -281,17 +289,15 @@ app.get("/pipeline", (c) => {
|
||||
`)
|
||||
.all();
|
||||
|
||||
// "Done" = files that are already in the desired end state. Two ways
|
||||
// to get there: (a) the analyzer says nothing to do (is_noop=1), or
|
||||
// (b) we ran a job that finished. Both count toward the same total.
|
||||
const doneCount = (
|
||||
db
|
||||
.prepare(`
|
||||
SELECT COUNT(*) as count FROM review_plans
|
||||
WHERE is_noop = 1 OR status = 'done'
|
||||
`)
|
||||
.get() as { count: number }
|
||||
).count;
|
||||
// "Done" = files already in the desired end state. Either the analyzer
|
||||
// says nothing to do (is_noop=1) or a job finished. Use two indexable
|
||||
// counts and add — the OR form (is_noop=1 OR status='done') can't use
|
||||
// our single-column indexes and gets slow on large libraries.
|
||||
const noopRow = db.prepare("SELECT COUNT(*) as n FROM review_plans WHERE is_noop = 1").get() as { n: number };
|
||||
const doneRow = db.prepare("SELECT COUNT(*) as n FROM review_plans WHERE status = 'done' AND is_noop = 0").get() as {
|
||||
n: number;
|
||||
};
|
||||
const doneCount = noopRow.n + doneRow.n;
|
||||
|
||||
// Batch transcode reasons for all review plans in one query (avoids N+1)
|
||||
const planIds = (review as { id: number }[]).map((r) => r.id);
|
||||
@@ -315,7 +321,7 @@ app.get("/pipeline", (c) => {
|
||||
item.transcode_reasons = reasonsByPlan.get(item.id) ?? [];
|
||||
}
|
||||
|
||||
return c.json({ review, queued, processing, done, doneCount, jellyfinUrl });
|
||||
return c.json({ review, reviewTotal, queued, processing, done, doneCount, jellyfinUrl });
|
||||
});
|
||||
|
||||
// ─── List ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user