diff --git a/frontend/packages/dashboard/src/builds.js b/frontend/packages/dashboard/src/builds.js index 37f4a1be..bb881809 100644 --- a/frontend/packages/dashboard/src/builds.js +++ b/frontend/packages/dashboard/src/builds.js @@ -131,6 +131,7 @@ const QUEUE_KIND_GLYPH = { graceful_stop: '⏹', start: '▶', stop: '■', + reconcile: '⇄', }; const QUEUE_STATE_GLYPH = { queued: '⏸', @@ -139,11 +140,33 @@ const QUEUE_STATE_GLYPH = { failed: '✖', cancelled: '⊘', }; +// Short display labels for the per-DAG node kinds (the primitive ops). +const NODE_KIND_LABEL = { + prebuild: 'prebuild', + stop_for_update: 'stop', + swap: 'swap', + create: 'create', + meta_lock: 'meta lock', + reconcile: 'reconcile', + signal: 'signal', + drain: 'drain', + write_dropin: 'dropin', + write_perm_file: 'perm file', + approval_deploy: 'deploy', +}; + +// The currently-running node of a DAG (per-node `step` labels + build +// logs live on nodes now; the DAG's `state` is a roll-up). +function runningNode(entry) { + return (entry.nodes || []).find((n) => n.state === 'running') || null; +} +function firstFailedNode(entry) { + return (entry.nodes || []).find((n) => n.state === 'failed') || null; +} function rebuildQueueEntryFingerprint(entry, isChild) { return JSON.stringify({ state: entry.state, - step: entry.step, kind: entry.kind, agent: entry.agent, source: entry.source, @@ -151,8 +174,7 @@ function rebuildQueueEntryFingerprint(entry, isChild) { enqueued_at: entry.enqueued_at, finished_at: entry.finished_at, reason: entry.reason, - error: entry.error, - build_log_id: entry.build_log_id, + nodes: (entry.nodes || []).map((n) => [n.kind, n.state, n.step, n.build_log_id, n.error]), isChild, }); } @@ -257,22 +279,39 @@ function renderQueueEntry(entry, _byId, isChild) { const r = entry.reason.split('\n')[0]; li.append(' ', el('span', { class: 'rqe-reason', title: entry.reason }, '— ' + truncate(r, 60))); } - if (entry.step) { - li.append(el('div', { class: 'rqe-step' }, '↳ ' + entry.step)); + // Per-node chain: every DAG node in dependency order with its own + // state glyph, live step label, and build-log link. This is the + // node-aware render that makes queue jumps / partial progress + // visible (e.g. reconcile running while swap failed). + const nodes = entry.nodes || []; + if (nodes.length) { + const chain = el('div', { class: 'rqe-nodes' }); + nodes.forEach((n, i) => { + if (i > 0) chain.append(el('span', { class: 'rqe-node-arrow' }, ' → ')); + const chip = el('span', { + class: 'rqe-node rqe-node-' + n.state, + title: n.kind + ' · ' + n.state + (n.error ? ' — ' + n.error : ''), + }, + (QUEUE_STATE_GLYPH[n.state] || '?') + ' ' + (NODE_KIND_LABEL[n.kind] || n.kind)); + chain.append(chip); + if (n.build_log_id != null) { + chain.append(el('a', { + class: 'rqe-log-link rqe-node-log', + href: '/builds.html?id=' + n.build_log_id + '#buildlogs', + target: '_blank', + title: 'view build log #' + n.build_log_id, + }, '⎙')); + } + }); + li.append(chain); } - if (entry.build_log_id != null) { - li.append( - ' ', - el('a', { - class: 'rqe-log-link', - href: '/builds.html?id=' + entry.build_log_id + '#buildlogs', - target: '_blank', - title: 'view build log #' + entry.build_log_id, - }, 'logs →'), - ); + const running = runningNode(entry); + if (running && running.step) { + li.append(el('div', { class: 'rqe-step' }, '↳ ' + running.step)); } - if (entry.error) { - li.append(el('pre', { class: 'rqe-error', title: entry.error }, truncate(entry.error, 200))); + const failed = firstFailedNode(entry); + if (failed && failed.error) { + li.append(el('pre', { class: 'rqe-error', title: failed.error }, truncate(failed.error, 200))); } if (entry.state === 'queued') { const cancelForm = el('form', { @@ -296,11 +335,12 @@ function renderQueueEntry(entry, _byId, isChild) { } // ─── running-rebuild live log ───────────────────────────────────────────────── -// One persistent live-log panel for the currently-running rebuild (the queue -// runs one build at a time). Keyed to the running entry's build_log_id and -// kept in its own container (#rebuild-live-log) so the queue's row re-render -// — which rebuilds rows as the build step advances — never tears down the open -// SSE stream. +// One persistent live-log panel for the first currently-building node +// (build logs are per-node now; with buildSlots = 1 at most one nix +// build runs at a time). Keyed to that node's build_log_id and kept in +// its own container (#rebuild-live-log) so the queue's row re-render — +// which rebuilds rows as the build step advances — never tears down the +// open SSE stream. let liveLogEs = null; let liveLogId = null; let liveLogDone = false; @@ -310,23 +350,36 @@ function closeLiveLogStream() { if (liveLogEs) { liveLogEs.close(); liveLogEs = null; } } +// First (dag, node) pair with a running node that opened a build log. +function findLiveBuild(queue) { + for (const e of queue || []) { + if (e.state !== 'running') continue; + for (const n of e.nodes || []) { + if (n.state === 'running' && n.build_log_id != null) { + return { entry: e, node: n }; + } + } + } + return null; +} + function renderRebuildLiveLog(queue) { const root = $('rebuild-live-log'); if (!root) return; - const running = (queue || []).find( - (e) => e.state === 'running' && e.build_log_id != null); + const live = findLiveBuild(queue); - if (!running) { + if (!live) { closeLiveLogStream(); liveLogId = null; liveLogDone = false; if (!root.hidden) { root.hidden = true; root.replaceChildren(); } return; } - if (running.build_log_id === liveLogId && (liveLogEs || liveLogDone)) return; + const { entry: running, node: liveNode } = live; + if (liveNode.build_log_id === liveLogId && (liveLogEs || liveLogDone)) return; closeLiveLogStream(); - liveLogId = running.build_log_id; + liveLogId = liveNode.build_log_id; liveLogDone = false; root.hidden = false; root.replaceChildren(); @@ -351,17 +404,18 @@ function renderRebuildLiveLog(queue) { toggle, ' ', el('span', { class: 'rebuild-live-log-title' }, 'live build log — '), el('code', { class: 'rqe-agent' }, running.agent), - ' ', el('span', { class: 'rqe-kind' }, running.kind || 'rebuild'), + ' ', el('span', { class: 'rqe-kind' }, + (running.kind || 'rebuild') + ' · ' + (NODE_KIND_LABEL[liveNode.kind] || liveNode.kind)), ' ', badge, ' ', el('a', { class: 'rebuild-live-log-raw', - href: '/api/build-logs/id/' + running.build_log_id + '/raw', - download: 'build-log-' + running.build_log_id + '.txt', + href: '/api/build-logs/id/' + liveNode.build_log_id + '/raw', + download: 'build-log-' + liveNode.build_log_id + '.txt', }, '↓ raw'), ); root.append(header, pre); - liveLogEs = openBuildLogStream(running.build_log_id, pre, { + liveLogEs = openBuildLogStream(liveNode.build_log_id, pre, { onDone: (status) => { liveLogDone = true; badge.className = 'rebuild-live-log-badge ' + (status === 'ok' ? 'rll-ok' : 'rll-fail'); diff --git a/frontend/packages/dashboard/src/system-sections.css b/frontend/packages/dashboard/src/system-sections.css index 480eb0c9..019b9216 100644 --- a/frontend/packages/dashboard/src/system-sections.css +++ b/frontend/packages/dashboard/src/system-sections.css @@ -134,6 +134,33 @@ .rqe-source-approval { color: var(--green); border-color: var(--green); } .rqe-when { color: var(--muted); font-size: 0.85em; } .rqe-reason { color: var(--muted); font-size: 0.85em; flex: 1 1 auto; } +/* Per-node DAG chain: one chip per primitive node, dependency order. */ +.rqe-nodes { + flex-basis: 100%; + margin: 0.15em 0 0 1.8em; + font-size: 0.85em; + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 0.15em; +} +.rqe-node { + padding: 0.05em 0.45em; + border: 1px solid var(--border); + border-radius: 0.7em; + color: var(--muted); + white-space: nowrap; +} +.rqe-node-running { + color: var(--purple); + border-color: var(--purple); + animation: badge-pulse 1.6s ease-in-out infinite; +} +.rqe-node-done { color: var(--green); border-color: color-mix(in srgb, var(--green) 45%, transparent); } +.rqe-node-failed { color: var(--red); border-color: var(--red); } +.rqe-node-cancelled { opacity: 0.55; text-decoration: line-through; } +.rqe-node-arrow { color: var(--muted); } +.rqe-node-log { margin-left: 0.1em; text-decoration: none; } .rqe-step { flex-basis: 100%; margin: 0.1em 0 0 1.8em; diff --git a/nix/modules/hive-c0re.nix b/nix/modules/hive-c0re.nix index 451a5a46..8d748f8f 100644 --- a/nix/modules/hive-c0re.nix +++ b/nix/modules/hive-c0re.nix @@ -52,6 +52,7 @@ let agent_cpu_quota = cfg.agentCpuQuota; agent_memory_max = cfg.agentMemoryMax; model_prices = cfg.modelPrices; + build_slots = cfg.buildSlots; }; # Stylix theme integration (zero-op auto-detect). When the operator's @@ -771,6 +772,22 @@ in `"2G"`. ''; }; + + buildSlots = lib.mkOption { + type = lib.types.ints.positive; + default = 1; + example = 2; + description = '' + Number of nix-heavy job-queue nodes (container prebuilds, + profile swaps, first-spawn creates, meta lock bumps) hive-c0re + runs concurrently. The default of 1 serializes all heavy nix + work like the pre-DAG rebuild queue did; raise it on hosts with + the cores/RAM to build several agent toplevels at once. + Per-agent correctness is independent of this count — each + agent's container-affecting operations are serialized by its + lifecycle lease regardless. + ''; + }; }; config = lib.mkIf cfg.enable {