vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-builder: build each agent config PR head and post a commit status

Browse source 
A new `swarm-builder` service, in its own container on the host that sets
`deploy.builder.enable` (defaulted on by `allSwarmServices` where the host
has the builder's store leaf, like the review classifier).

When an `agent-configs/<agent>` PR is opened, reopened or synchronized,
swarm-controller queues a `build_agent` jobq node. The node sets a
`swarm-builder` commit status to pending on the PR head, sends
`{agent, rev, pr}` to the builder over NATS request-reply
(`$SWARM.builder.build`), and writes success (store path), failure (last
line nix printed) or error (no builder / no reply) on the head. The node
fails when the build does, so the Jobs page matches the PR.

The builder builds nix/builder/check-flake's `check` with every input
overridden: hyperhive and nixpkgs to the host's own
`c0re.hyperhiveFlake`/`c0re.nixpkgsFlake` (what a hive there renders into
its meta flake), and the agent to the PR head on the forge. Evaluation is
pure and runs in the container; builds go to the host nix daemon as an
untrusted client. The private config repo is fetched with the token of a
new read-only `builder` forge account (in the `hives` team), which the
controller mints into `swarm/services/builder/forge/token`; git gets it
from a credential helper reading the child's environment, so it is in no
argument or URL.

Credentials: a `builder` store leaf minted by glue-bao-tls on the store
host, a `swarm-builder` reader policy (forge token + its own OIDC client
secret), an authelia machine client, and a `--builder-client` grant in the
queue's auth-callout responder. The controller gains a grant on the token
path and the secret publisher a deny on it.

This is the agent-config build check atlas scoped on #5081 (slice 3):
nothing is pushed to a cache, nothing is written to KV, and a red status
gates no deploy. The check composes agent-base + the config without the
values a hive adds (CA, docs, claude-code path, ports, input follows), so
it can disagree with a hive's own build.
This commit is contained in:
atlas 2026-10-12 22:04:51 +02:00
commit b68b74d687
43 changed files with 2452 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

231
nix/host-modules/swarm-builder.nix Normal file
View file

@ -0,0 +1,231 @@
# The swarm builder, in a `swarm-builder` container on the host that sets
# `deploy.builder.enable`, which `deploy.allSwarmServices` defaults on where
# the host has the builder's store identity. It builds agent configs for
# swarm-controller: the controller sends a config PR's head over the queue,
# and the daemon builds that config on this host's agent base and replies.
# docs/swarm/builder.md has the design.
#
# ⚠️ Its own network namespace on the bridge, as ./swarm-ci-runner.nix has: a
# config PR is unreviewed input to a nix evaluation, so the container reaches
# the swarm services by name and nothing on the host's loopback.
#
# Builds go through the HOST nix-daemon, as the CI runner's do. The daemon
# runs as a dynamic user, an untrusted daemon client: a config can set no
# daemon option, and its builds run in the host's sandbox.
#
# The container holds one credential, its own store leaf, which nspawn hands in
# as a system credential with no copy on disk. The daemon reads everything else
# from the store with it: its queue client secret at start and its forge token
# on every build.
{
pkgs,
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
cfg = hyperhiveCfg.deploy.builder;
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = hyperhiveCfg.deploy.bao;
natsCfg = hyperhiveCfg.swarm.nats;
autheliaCfg = hyperhiveCfg.swarm.authelia;
forgeCfg = hyperhiveCfg.swarm.forge;
gatewayCfg = hyperhiveCfg.gateway;
networkCfg = hyperhiveCfg.network;
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
machine = "swarm-builder";
unit = "swarm-builder";
clientId = hyperhiveCfg.swarm.builder.clientId;
# The daemon reaches authelia, the store and the queue by name over TLS, and
# `git` reaches the forge; in self-signed mode their leaves chain to the hive
# CA.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = machine;
consumers = [ unit ];
};
haveClientIdentity =
baoDeploy.builderClientCertFile != null && baoDeploy.builderClientKeyFile != null;
# Credential id → host source, as ./swarm-review-classifier.nix hands its
# leaf in.
storeFiles = lib.filterAttrs (_: source: source != null) {
builder-bao-cert = baoDeploy.builderClientCertFile;
builder-bao-key = baoDeploy.builderClientKeyFile;
builder-bao-server-ca = baoDeploy.serverCaFile;
};
# nixos-container expands `extraFlags` unquoted, so a source path must be
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
credential = id: "/run/credentials/${unit}.service/${id}";
# The `nix` the host daemon runs, so client and daemon speak one protocol.
nix = config.nix.package;
# Strings, not paths: both are already store paths, and the flake reference
# the daemon passes must be the same one a hive's meta flake names.
hyperhiveFlake = hyperhiveCfg.c0re.hyperhiveFlake;
nixpkgsFlake = hyperhiveCfg.c0re.nixpkgsFlake;
privateNetwork = true;
in
{
options.services.hyperhive.swarm.builder.clientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-builder";
description = ''
The OAuth2 client id the swarm builder presents to the swarm queue.
Read-only: the host running authelia registers it, the queue's
auth-callout responder grants it the build subject, and the secret
publisher carries its secret to the store, all from this one value.
'';
};
options.services.hyperhive.deploy.builder.package = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-builder";
description = "The swarm builder daemon package.";
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = haveClientIdentity;
message = ''
services.hyperhive.deploy.builder.enable requires the builder's
swarm-secret-store client identity: set both
services.hyperhive.deploy.bao.builderClientCertFile
services.hyperhive.deploy.bao.builderClientKeyFile
The builder reads its queue client secret and its forge token with it.
Both default to the pair glue-bao-tls.nix mints when
services.hyperhive.deploy.bao.enable is true (the host that runs the
store). Otherwise copy builder.pem and builder-key.pem from the store
host's /var/lib/swarm-bao-pki/ and name them in those two options.
'';
}
{
assertion = lib.all oneWord (lib.attrValues storeFiles);
message = ''
The swarm builder's store identity paths must not contain whitespace:
nspawn's --load-credential receives them through nixos-container's
unquoted extra flags.
'';
}
];
# The container hangs off the bridge, takes its address from the gateway's
# DHCP and resolves through its dnsmasq.
services.hyperhive.network.enable = lib.mkDefault true;
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# nspawn fails to start the container when a credential source is
# missing, so where this host mints the leaf the container waits for it.
systemd.services."container@${machine}" = lib.mkMerge [
caTrust.containerOrdering
{ after = [ "swarm-bao-pki.service" ]; }
];
containers.${machine} = {
autoStart = true;
# The daemon keeps nothing but `git`'s fetch cache, which a restart may
# drop.
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [
"--link-journal=host"
]
++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
bindMounts = {
# The directory rather than the socket file, so the mount survives a
# daemon restart replacing the inode.
"/nix/var/nix/daemon-socket" = {
hostPath = "/nix/var/nix/daemon-socket";
isReadOnly = false;
};
}
// caTrust.bindMount;
config =
{ lib, ... }:
{
imports = [
../container-modules/swarm-container.nix
caBundleModule
];
services.hyperhive.swarmContainer = {
inherit privateNetwork;
writesOwnResolvConf = false;
};
networking.nameservers = [ networkCfg.bridgeIp ];
networking.interfaces.eth0.useDHCP = true;
# Both the service and its socket are off: the socket unit owns
# `/nix/var/nix/daemon-socket/socket` and would race the bind mount.
systemd.globalEnvironment.NIX_REMOTE = "daemon";
systemd.services.nix-daemon.enable = false;
systemd.sockets.nix-daemon.enable = false;
systemd.services.${unit} = {
description = "build agent configs for swarm-controller";
wantedBy = [ "multi-user.target" ];
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
# nix fetches the config repo with `git`.
path = [ pkgs.git ];
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = credential "builder-bao-cert";
BAO_CLIENT_KEY = credential "builder-bao-key";
NIX_CONFIG = "experimental-features = nix-command flakes";
# nix's fetch cache, under `CacheDirectory`; the dynamic user has
# no home of its own.
HOME = "/var/cache/${unit}";
XDG_CACHE_HOME = "/var/cache/${unit}";
SWARM_BUILDER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
SWARM_BUILDER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
SWARM_BUILDER_OIDC_CLIENT_ID = clientId;
SWARM_BUILDER_NIX = "${nix}/bin/nix";
SWARM_BUILDER_CHECK_FLAKE = "${../builder/check-flake}";
SWARM_BUILDER_HYPERHIVE_FLAKE = hyperhiveFlake;
SWARM_BUILDER_NIXPKGS_FLAKE = nixpkgsFlake;
SWARM_BUILDER_FORGE_URL = "https://${forgeCfg.domain}";
SWARM_BUILDER_CONFIG_ORG = "agent-configs";
}
// lib.optionalAttrs (storeFiles ? builder-bao-server-ca) {
BAO_CACERT = credential "builder-bao-server-ca";
}
// lib.optionalAttrs caTrust.useSelfSigned {
SWARM_BUILDER_OIDC_CA_FILE = caTrust.bundlePathFor machine;
# `git`'s trust store, which the bundle's system CAs keep whole.
GIT_SSL_CAINFO = caTrust.bundlePathFor machine;
};
serviceConfig = {
ExecStart = "${cfg.package}/bin/swarm-builder";
DynamicUser = true;
CacheDirectory = unit;
# Bare ids: the system credentials nspawn handed in.
LoadCredential = lib.attrNames storeFiles;
Restart = "on-failure";
RestartSec = "10s";
SyslogIdentifier = unit;
};
};
};
};
};
}