vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-builder: build each agent config PR head and post a commit status

Browse source 
A new `swarm-builder` service, in its own container on the host that sets
`deploy.builder.enable` (defaulted on by `allSwarmServices` where the host
has the builder's store leaf, like the review classifier).

When an `agent-configs/<agent>` PR is opened, reopened or synchronized,
swarm-controller queues a `build_agent` jobq node. The node sets a
`swarm-builder` commit status to pending on the PR head, sends
`{agent, rev, pr}` to the builder over NATS request-reply
(`$SWARM.builder.build`), and writes success (store path), failure (last
line nix printed) or error (no builder / no reply) on the head. The node
fails when the build does, so the Jobs page matches the PR.

The builder builds nix/builder/check-flake's `check` with every input
overridden: hyperhive and nixpkgs to the host's own
`c0re.hyperhiveFlake`/`c0re.nixpkgsFlake` (what a hive there renders into
its meta flake), and the agent to the PR head on the forge. Evaluation is
pure and runs in the container; builds go to the host nix daemon as an
untrusted client. The private config repo is fetched with the token of a
new read-only `builder` forge account (in the `hives` team), which the
controller mints into `swarm/services/builder/forge/token`; git gets it
from a credential helper reading the child's environment, so it is in no
argument or URL.

Credentials: a `builder` store leaf minted by glue-bao-tls on the store
host, a `swarm-builder` reader policy (forge token + its own OIDC client
secret), an authelia machine client, and a `--builder-client` grant in the
queue's auth-callout responder. The controller gains a grant on the token
path and the secret publisher a deny on it.

This is the agent-config build check atlas scoped on #5081 (slice 3):
nothing is pushed to a cache, nothing is written to KV, and a red status
gates no deploy. The check composes agent-base + the config without the
values a hive adds (CA, docs, claude-code path, ports, input follows), so
it can disagree with a hive's own build.
This commit is contained in:
atlas 2026-10-12 22:04:51 +02:00
commit b68b74d687
43 changed files with 2452 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

26
nix/builder/check-flake/flake.nix Normal file
View file

@ -0,0 +1,26 @@
# The swarm builder's build check: one agent config on hyperhive's agent base,
# composed the way a hive's meta flake composes it, minus the values a hive
# adds (its CA, docs source, claude-code path and ports).
#
# Every input is replaced at build time with `--override-input`
# (swarm-builder/src/nix_build.rs), so the URLs below are never fetched and
# the flake has no lock.
{
description = "swarm-builder: an agent config built on agent-base";
inputs = {
nixpkgs.url = "flake:nixpkgs";
hyperhive.url = "flake:hyperhive";
hyperhive.inputs.nixpkgs.follows = "nixpkgs";
agent.url = "flake:agent";
};
outputs =
{ hyperhive, agent, ... }:
{
packages.x86_64-linux.check =
(hyperhive.nixosConfigurations.agent-base.extendModules {
modules = [ agent.nixosModules.default ];
}).config.system.build.toplevel;
};
}

4
nix/checks.nix
View file

@ -44,6 +44,10 @@ in
inherit (pkgs) lib;
shared = moduleEvalShared;
};
module-eval-builder = import ./module-eval/builder.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;
};
module-eval-nats-authelia = import ./module-eval/nats-authelia.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;

2
nix/host-modules/default.nix
View file

@ -44,6 +44,7 @@
./glue-nats-oidc-client.nix
./glue-otel-scrape-targets.nix
./glue-queue-agent-credential.nix
./glue-builder-oidc-client.nix
./glue-review-classifier-oidc-client.nix
./glue-secret-publisher-authelia.nix
./glue-secret-publisher-bao-identity.nix
@ -53,6 +54,7 @@
./swarm-authelia.nix
./swarm-bao-service.nix
./swarm-bao.nix
./swarm-builder.nix
./swarm-ci-runner.nix
./swarm-review-classifier.nix
./swarm-secret-publisher.nix

18
nix/host-modules/deploy.nix
View file

@ -679,6 +679,24 @@ in
'';
};
builder.enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = ''
Run the swarm builder in a `swarm-builder` container on this host. It
builds each agent config PR's head on this host's agent base, through
this host's nix daemon, and swarm-controller writes the result as a
`swarm-builder` commit status on the PR.
On by default where
{option}`services.hyperhive.deploy.allSwarmServices` is set and this
host has the builder's store identity (the pair
{option}`services.hyperhive.deploy.bao.builderClientCertFile` and
`builderClientKeyFile`, minted on the store's host).
'';
};
hive-controller.enable = lib.mkOption {
type = lib.types.bool;
default = false;

9
nix/host-modules/glue-bao-tls.nix
View file

@ -115,6 +115,8 @@ in
ciRunnerClientKeyFile = lib.mkDefault "${pkiDir}/ci-runner-key.pem";
reviewClassifierClientCertFile = lib.mkDefault "${pkiDir}/review-classifier.pem";
reviewClassifierClientKeyFile = lib.mkDefault "${pkiDir}/review-classifier-key.pem";
builderClientCertFile = lib.mkDefault "${pkiDir}/builder.pem";
builderClientKeyFile = lib.mkDefault "${pkiDir}/builder-key.pem";
forgePublisherClientCertFile = lib.mkDefault "${pkiDir}/forge-publisher.pem";
forgePublisherClientKeyFile = lib.mkDefault "${pkiDir}/forge-publisher-key.pem";
servicesIssuerClientCertFile = lib.mkDefault "${pkiDir}/services-issuer.pem";
@ -290,6 +292,13 @@ in
[ -s ${pkiDir}/review-classifier.pem ] || ${signLeaf} ${pkiDir} review-classifier \
${lib.escapeShellArg deployCfg.bao.reviewClassifierCommonName} "" clientAuth
# The swarm builder's, which reads its forge token and queue client
# secret. Minted here for the matrix-ctl leaf's reason: the builder is a
# swarm service, so on a builder host without the store this is the
# file an operator copies.
[ -s ${pkiDir}/builder.pem ] || ${signLeaf} ${pkiDir} builder \
${lib.escapeShellArg deployCfg.bao.builderCommonName} "" clientAuth
# The forge's token publisher. Minted here for the matrix-ctl leaf's
# reason: the forge is a swarm singleton, so on a forge host without the
# store this is the file an operator copies.

34
nix/host-modules/glue-builder-oidc-client.nix Normal file
View file

@ -0,0 +1,34 @@
# Glue: register the swarm builder as an OIDC client wherever authelia runs.
#
# ONE PAIRING PER FILE — swarm-builder ← authelia, and nothing else. Deleting
# this leaves a builder whose token requests authelia answers with a 401, and
# no minted secret for swarm-secret-publish to put in the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the builder,
# for ./glue-controller-oidc-client.nix's reason: a client is a row in THIS
# host's provider config. Registered whether or not any host enables the
# builder: nothing in `swarm.*` records that, so a swarm without one carries
# one unused client and one unused minted secret.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
deployCfg = hyperhiveCfg.deploy;
clientId = hyperhiveCfg.swarm.builder.clientId;
in
{
config = lib.mkIf deployCfg.authelia.enable {
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = clientId;
description = "HyperHive swarm builder";
# `client_credentials`: a daemon authenticating as itself.
kind = "machine";
audience = [ clientId ];
}
];
};
}

68
nix/host-modules/swarm-bao.nix
View file

@ -441,6 +441,10 @@ let
capabilities = ["create", "read", "update"]
}
path "${credentialMountPath}/data/${builderForgeTokenLeaf}" {
capabilities = ["create", "read", "update"]
}
path "${credentialMountPath}/data/${swarmAppserviceTokenLeaf}" {
capabilities = ["read"]
}
@ -492,6 +496,8 @@ let
# The review classifier's forge token is denied too: the controller posts
# the classifier's reviews with whatever token is there, so a write to it
# picks the account those reviews come from.
# So is the builder's: a write to it picks the account the builder fetches
# config repos as.
secretPublisherPolicyText = ''
path "${credentialMountPath}/data/swarm/hives/*" {
capabilities = ["create", "update"]
@ -521,6 +527,10 @@ let
capabilities = ["deny"]
}
path "${credentialMountPath}/data/${builderForgeTokenLeaf}" {
capabilities = ["deny"]
}
path "${credentialMountPath}/data/${controllerClientLeaf}" {
capabilities = ["create", "update"]
}
@ -588,6 +598,12 @@ let
# the UI's OIDC login; the classifier reads it on every run.
classifierApiKeyLeaf = "swarm/services/review-classifier/anthropic-api-key";
# The swarm builder's forge token, the nix half of
# `swarm_secret_client::builder::forge_token_path`. The controller mints it
# and reads it back before deciding whether to re-mint; the builder reads it
# to fetch config repos.
builderForgeTokenLeaf = "swarm/services/builder/forge/token";
# The UI's OIDC client secret. The publisher writes it; the granter reads it
# into the `oidc` auth method's config, which is the only copy bao uses.
baoUiClientLeaf = "swarm/services/${cfg.ui.oidc.clientId}/oidc/client";
@ -998,6 +1014,18 @@ let
}
];
# The swarm builder: its forge token, read once per build, and its own queue
# client secret, read at start. Both by the daemon inside its container.
builderReaders = [
{
name = "swarm-builder";
cn = baoDeploy.builderCommonName;
policyText =
readStanza "${credentialMountPath}/data/${builderForgeTokenLeaf}"
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.builder.clientId}/oidc/client";
}
];
# The forge host's one store identity. `swarm-bao-forge-publisher` writes
# the controller's forge token under it, the one writer of that path and
# write-only: it copies a file in and never reads one back.
@ -1848,6 +1876,19 @@ in
'';
};
builderCommonName = lib.mkOption {
type = lib.types.str;
default = "swarm-builder";
description = ''
Subject the store's `swarm-builder` cert-auth role accepts: the
identity the swarm builder presents from inside its container. Its
grant is two paths, read only: `swarm/services/builder/forge/token`
and its own queue client secret.
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
'';
};
forgePublisherCommonName = lib.mkOption {
type = lib.types.str;
default = "swarm-forge-publisher";
@ -2123,6 +2164,31 @@ in
'';
};
builderClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/builder.pem";
description = ''
Certificate the swarm builder presents to the store. Its subject must
be {option}`services.hyperhive.deploy.bao.builderCommonName`.
./glue-bao-tls.nix points it at the leaf it mints on the store's host.
On a builder host without the store, it is a file the operator copies
from the store's host. A path, never a value.
'';
};
builderClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/builder-key.pem";
description = ''
Private key for
{option}`services.hyperhive.deploy.bao.builderClientCertFile`.
A path, never a value.
'';
};
forgePublisherClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -3201,6 +3267,8 @@ in
systemd.services.swarm-bao-review-classifier-policy = readerPolicyUnit "write the review classifier's API-key-reader bao policy and cert-auth role" reviewClassifierReaders;
systemd.services.swarm-bao-builder-policy = readerPolicyUnit "write the swarm builder's forge-token-reader bao policy and cert-auth role" builderReaders;
systemd.services.swarm-bao-forge-publisher-policy = readerPolicyUnit "write the forge's controller-token-publisher bao policy and cert-auth role" forgePublisherWriters;
# A FOURTH sibling, same shape and same reasons as the two above. This

231
nix/host-modules/swarm-builder.nix Normal file
View file

@ -0,0 +1,231 @@
# The swarm builder, in a `swarm-builder` container on the host that sets
# `deploy.builder.enable`, which `deploy.allSwarmServices` defaults on where
# the host has the builder's store identity. It builds agent configs for
# swarm-controller: the controller sends a config PR's head over the queue,
# and the daemon builds that config on this host's agent base and replies.
# docs/swarm/builder.md has the design.
#
# ⚠️ Its own network namespace on the bridge, as ./swarm-ci-runner.nix has: a
# config PR is unreviewed input to a nix evaluation, so the container reaches
# the swarm services by name and nothing on the host's loopback.
#
# Builds go through the HOST nix-daemon, as the CI runner's do. The daemon
# runs as a dynamic user, an untrusted daemon client: a config can set no
# daemon option, and its builds run in the host's sandbox.
#
# The container holds one credential, its own store leaf, which nspawn hands in
# as a system credential with no copy on disk. The daemon reads everything else
# from the store with it: its queue client secret at start and its forge token
# on every build.
{
pkgs,
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
cfg = hyperhiveCfg.deploy.builder;
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = hyperhiveCfg.deploy.bao;
natsCfg = hyperhiveCfg.swarm.nats;
autheliaCfg = hyperhiveCfg.swarm.authelia;
forgeCfg = hyperhiveCfg.swarm.forge;
gatewayCfg = hyperhiveCfg.gateway;
networkCfg = hyperhiveCfg.network;
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
machine = "swarm-builder";
unit = "swarm-builder";
clientId = hyperhiveCfg.swarm.builder.clientId;
# The daemon reaches authelia, the store and the queue by name over TLS, and
# `git` reaches the forge; in self-signed mode their leaves chain to the hive
# CA.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = machine;
consumers = [ unit ];
};
haveClientIdentity =
baoDeploy.builderClientCertFile != null && baoDeploy.builderClientKeyFile != null;
# Credential id → host source, as ./swarm-review-classifier.nix hands its
# leaf in.
storeFiles = lib.filterAttrs (_: source: source != null) {
builder-bao-cert = baoDeploy.builderClientCertFile;
builder-bao-key = baoDeploy.builderClientKeyFile;
builder-bao-server-ca = baoDeploy.serverCaFile;
};
# nixos-container expands `extraFlags` unquoted, so a source path must be
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
credential = id: "/run/credentials/${unit}.service/${id}";
# The `nix` the host daemon runs, so client and daemon speak one protocol.
nix = config.nix.package;
# Strings, not paths: both are already store paths, and the flake reference
# the daemon passes must be the same one a hive's meta flake names.
hyperhiveFlake = hyperhiveCfg.c0re.hyperhiveFlake;
nixpkgsFlake = hyperhiveCfg.c0re.nixpkgsFlake;
privateNetwork = true;
in
{
options.services.hyperhive.swarm.builder.clientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-builder";
description = ''
The OAuth2 client id the swarm builder presents to the swarm queue.
Read-only: the host running authelia registers it, the queue's
auth-callout responder grants it the build subject, and the secret
publisher carries its secret to the store, all from this one value.
'';
};
options.services.hyperhive.deploy.builder.package = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-builder";
description = "The swarm builder daemon package.";
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = haveClientIdentity;
message = ''
services.hyperhive.deploy.builder.enable requires the builder's
swarm-secret-store client identity: set both
services.hyperhive.deploy.bao.builderClientCertFile
services.hyperhive.deploy.bao.builderClientKeyFile
The builder reads its queue client secret and its forge token with it.
Both default to the pair glue-bao-tls.nix mints when
services.hyperhive.deploy.bao.enable is true (the host that runs the
store). Otherwise copy builder.pem and builder-key.pem from the store
host's /var/lib/swarm-bao-pki/ and name them in those two options.
'';
}
{
assertion = lib.all oneWord (lib.attrValues storeFiles);
message = ''
The swarm builder's store identity paths must not contain whitespace:
nspawn's --load-credential receives them through nixos-container's
unquoted extra flags.
'';
}
];
# The container hangs off the bridge, takes its address from the gateway's
# DHCP and resolves through its dnsmasq.
services.hyperhive.network.enable = lib.mkDefault true;
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# nspawn fails to start the container when a credential source is
# missing, so where this host mints the leaf the container waits for it.
systemd.services."container@${machine}" = lib.mkMerge [
caTrust.containerOrdering
{ after = [ "swarm-bao-pki.service" ]; }
];
containers.${machine} = {
autoStart = true;
# The daemon keeps nothing but `git`'s fetch cache, which a restart may
# drop.
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [
"--link-journal=host"
]
++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
bindMounts = {
# The directory rather than the socket file, so the mount survives a
# daemon restart replacing the inode.
"/nix/var/nix/daemon-socket" = {
hostPath = "/nix/var/nix/daemon-socket";
isReadOnly = false;
};
}
// caTrust.bindMount;
config =
{ lib, ... }:
{
imports = [
../container-modules/swarm-container.nix
caBundleModule
];
services.hyperhive.swarmContainer = {
inherit privateNetwork;
writesOwnResolvConf = false;
};
networking.nameservers = [ networkCfg.bridgeIp ];
networking.interfaces.eth0.useDHCP = true;
# Both the service and its socket are off: the socket unit owns
# `/nix/var/nix/daemon-socket/socket` and would race the bind mount.
systemd.globalEnvironment.NIX_REMOTE = "daemon";
systemd.services.nix-daemon.enable = false;
systemd.sockets.nix-daemon.enable = false;
systemd.services.${unit} = {
description = "build agent configs for swarm-controller";
wantedBy = [ "multi-user.target" ];
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
# nix fetches the config repo with `git`.
path = [ pkgs.git ];
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = credential "builder-bao-cert";
BAO_CLIENT_KEY = credential "builder-bao-key";
NIX_CONFIG = "experimental-features = nix-command flakes";
# nix's fetch cache, under `CacheDirectory`; the dynamic user has
# no home of its own.
HOME = "/var/cache/${unit}";
XDG_CACHE_HOME = "/var/cache/${unit}";
SWARM_BUILDER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
SWARM_BUILDER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
SWARM_BUILDER_OIDC_CLIENT_ID = clientId;
SWARM_BUILDER_NIX = "${nix}/bin/nix";
SWARM_BUILDER_CHECK_FLAKE = "${../builder/check-flake}";
SWARM_BUILDER_HYPERHIVE_FLAKE = hyperhiveFlake;
SWARM_BUILDER_NIXPKGS_FLAKE = nixpkgsFlake;
SWARM_BUILDER_FORGE_URL = "https://${forgeCfg.domain}";
SWARM_BUILDER_CONFIG_ORG = "agent-configs";
}
// lib.optionalAttrs (storeFiles ? builder-bao-server-ca) {
BAO_CACERT = credential "builder-bao-server-ca";
}
// lib.optionalAttrs caTrust.useSelfSigned {
SWARM_BUILDER_OIDC_CA_FILE = caTrust.bundlePathFor machine;
# `git`'s trust store, which the bundle's system CAs keep whole.
GIT_SSL_CAINFO = caTrust.bundlePathFor machine;
};
serviceConfig = {
ExecStart = "${cfg.package}/bin/swarm-builder";
DynamicUser = true;
CacheDirectory = unit;
# Bare ids: the system credentials nspawn handed in.
LoadCredential = lib.attrNames storeFiles;
Restart = "on-failure";
RestartSec = "10s";
SyslogIdentifier = unit;
};
};
};
};
};
}

2
nix/host-modules/swarm-nats.nix
View file

@ -683,6 +683,8 @@ in
"--reader-client ${lib.escapeShellArg controllerCfg.queueClientId}"
# Hears try-outs and answers each by reply, and nothing else.
"--review-classifier-client ${lib.escapeShellArg config.services.hyperhive.swarm.review-classifier.clientId}"
# Hears builds and answers each by reply, and nothing else.
"--builder-client ${lib.escapeShellArg config.services.hyperhive.swarm.builder.clientId}"
# What an agent may publish to, keyed on its own name: its
# terminal rows and its turn-state header
# (`hive-agent::swarm_agent_state`), two subject families so a

7
nix/host-modules/swarm-required-services.nix
View file

@ -117,4 +117,11 @@ in
&& deployCfg.bao.reviewClassifierClientCertFile != null
&& deployCfg.bao.reviewClassifierClientKeyFile != null
);
# The swarm builder, defaulted on the same terms as the review classifier.
config.services.hyperhive.deploy.builder.enable = lib.mkDefault (
deployCfg.allSwarmServices
&& deployCfg.bao.builderClientCertFile != null
&& deployCfg.bao.builderClientKeyFile != null
);
}

2
nix/host-modules/swarm-secret-publisher.nix
View file

@ -94,6 +94,8 @@ let
hyperhiveCfg.swarm.forge.sso.clientId
# The review classifier, which reads it itself inside its container.
hyperhiveCfg.swarm.review-classifier.clientId
# The swarm builder, which reads it itself inside its container.
hyperhiveCfg.swarm.builder.clientId
];
# The swarm controller's OIDC client, which it reads back from the store

1
nix/host-modules/swarm.nix
View file

@ -57,6 +57,7 @@ let
deployCfg.bao.ciRunnerCommonName
deployCfg.bao.logsAuthCommonName
deployCfg.bao.reviewClassifierCommonName
deployCfg.bao.builderCommonName
deployCfg.bao.forgePublisherCommonName
]
# The per-hive subjects, spelled out per hive rather than as the prefix.

53
nix/module-eval/bao-grants.nix
View file

@ -189,6 +189,7 @@ let
"swarm-bao-logs-auth-policy"
"swarm-bao-forge-publisher-policy"
"swarm-bao-review-classifier-policy"
"swarm-bao-builder-policy"
];
# Comment lines dropped first: both the HCL and the scripts explain
@ -1491,6 +1492,58 @@ let
"swarm-bao-controller-policy"
];
}
{
# The builder reads its forge token and its own queue client secret,
# each read-only and spelled to the leaf: `services/*` would reach every
# swarm service's secret.
name = "the swarm builder's grant is its forge token and its queue client secret, and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-builder-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/services/builder/forge/token\" {\n capabilities = [\"read\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/services/swarm-builder/oidc/client\" {\n capabilities = [\"read\"]\n}" s
&& lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 2
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
&& lib.hasInfix "auth/cert/certs/swarm-builder" s
&& lib.hasInfix "allowed_common_names=swarm-builder" s
&& lib.hasInfix "token_policies=swarm-builder" s;
}
{
# `forge::builder_account` mints the token and reads it back to decide
# on a re-mint. Pinned as the whole stanza, and spelled to the leaf.
name = "the controller's grant on the swarm builder is its forge token alone";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/services/builder/forge/token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
&& !(lib.hasInfix "secret/data/swarm/services/builder/*" s);
}
{
# The token sits under the publisher's `services/*` write, and the
# builder fetches config repos with whatever is there. Its OIDC leaf is
# the control: still the prefix's.
name = "the publisher may not write the swarm builder's forge token";
ok =
let
grants = grantsIn baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script;
in
(grantFor grants "secret/data/swarm/services/builder/forge/token").caps == [ "deny" ]
&&
(grantFor grants "secret/data/swarm/services/swarm-builder/oidc/client").caps == [
"create"
"update"
];
}
{
name = "the PKI unit signs the swarm builder's leaf under its own subject";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
in
lib.hasInfix "/builder.pem ]" s && lib.hasInfix "swarm-builder \"\" clientAuth" s;
}
{
name = "the PKI unit signs the CI runner's leaf under its own subject";
ok =

131
nix/module-eval/builder.nix Normal file
View file

@ -0,0 +1,131 @@
# `checks.module-eval-builder` — see ./lib.nix for the shared rationale (why
# this suite exists, naming convention, "evaluates not executes"). The swarm
# builder's container, its default and its queue and IdP registrations; its
# store grants are in ./bao-grants.nix beside every other principal's.
{
pkgs,
lib,
self,
nixosSystem,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
hive
hiveWithController
runGroup
;
bare = hiveWithController { };
# On the store's host, which mints the builder's store identity.
withBuilder = hive {
deploy.bao.enable = true;
deploy.builder.enable = true;
};
servicesHere = hiveWithController { deploy.allSwarmServices = true; };
servicesBaoElsewhere = hive {
deploy.allSwarmServices = true;
deploy.bao.enable = false;
};
natsHere = hive {
swarm.nats.enable = true;
swarm.nats.authPackage = pkgs.emptyDirectory;
};
autheliaHere = hive {
deploy.bao.enable = true;
deploy.authelia.enable = true;
};
container = withBuilder.containers.swarm-builder;
inner = container.config;
svc = inner.systemd.services.swarm-builder;
env = svc.environment;
cases = [
{
name = "the swarm builder's container exists only where its host enables it";
ok = !(bare.containers ? swarm-builder) && withBuilder.containers ? swarm-builder;
}
{
# Its module asserts a store identity that only the store's host mints,
# so with the store elsewhere the default stays off rather than failing
# that host's eval. `nats` is the control that the switch is on at all.
name = "the shared services turn the builder on with the store's leaf and leave it off without";
ok =
servicesHere.services.hyperhive.deploy.builder.enable
&& servicesHere.containers ? swarm-builder
&& !servicesBaoElsewhere.services.hyperhive.deploy.builder.enable
&& !(servicesBaoElsewhere.containers ? swarm-builder)
&& servicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
}
{
# A config PR is unreviewed input to an evaluation: the container keeps
# its own netns, and reaches the host's nix daemon through its socket
# alone, with no daemon of its own racing the bind.
name = "the builder's container has its own netns and builds only through the host's daemon socket";
ok =
container.privateNetwork
&& inner.services.hyperhive.swarmContainer.privateNetwork
&& container.bindMounts."/nix/var/nix/daemon-socket".hostPath == "/nix/var/nix/daemon-socket"
&& !inner.systemd.services.nix-daemon.enable
&& !inner.systemd.sockets.nix-daemon.enable
&& inner.systemd.globalEnvironment.NIX_REMOTE == "daemon";
}
{
# The daemon holds its store leaf, handed in by nspawn without a copy on
# disk; the forge token is read per build and reaches only `git`.
name = "the builder loads only its store leaf, runs as a dynamic user and carries no token in its unit";
ok =
lib.elem "builder-bao-key" svc.serviceConfig.LoadCredential
&& env.BAO_CLIENT_KEY == "/run/credentials/swarm-builder.service/builder-bao-key"
&& env.SWARM_BUILDER_OIDC_CLIENT_ID == "swarm-builder"
&& lib.elem "--load-credential=builder-bao-key:/var/lib/swarm-bao-pki/builder-key.pem" container.extraFlags
&& svc.serviceConfig.DynamicUser
&& !(lib.any (lib.hasSuffix "TOKEN") (lib.attrNames env));
}
{
# The check builds on the sources a hive on this host would render into
# its meta flake, so a green check means the same base.
name = "the builder builds on the hyperhive and nixpkgs this host's hives use";
ok =
env.SWARM_BUILDER_HYPERHIVE_FLAKE == withBuilder.services.hyperhive.c0re.hyperhiveFlake
&& env.SWARM_BUILDER_NIXPKGS_FLAKE == withBuilder.services.hyperhive.c0re.nixpkgsFlake
&& lib.hasSuffix "-check-flake" env.SWARM_BUILDER_CHECK_FLAKE
&& env.SWARM_BUILDER_CONFIG_ORG == "agent-configs"
&& env.SWARM_BUILDER_FORGE_URL == "https://${withBuilder.services.hyperhive.swarm.forge.domain}";
}
{
# The responder denies a client id it does not know, and a NATS denial
# arrives as a timeout.
name = "the queue's responder grants the builder's client its subject";
ok = lib.hasInfix "--builder-client swarm-builder" natsHere.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart;
}
{
name = "the builder's client is registered as a machine, and the publisher carries its secret";
ok =
let
c = lib.findFirst (
c: c.id == "swarm-builder"
) null autheliaHere.services.hyperhive.swarm.authelia.oidc.clients;
in
c != null
&& c.kind == "machine"
&& lib.hasInfix "secret/swarm/services/swarm-builder/oidc/client" (
autheliaHere.systemd.services.swarm-secret-publish.script
);
}
];
in
runGroup "builder" cases

2
nix/module-eval/swarm-services-switch.nix
View file

@ -472,7 +472,7 @@ let
# that lost a container can't pass by comparing fewer of them.
name = "every service container restates its host-side privateNetwork";
ok =
lib.length (lib.attrNames serviceContainers) == 11
lib.length (lib.attrNames serviceContainers) == 12
&& lib.all (
c: c.privateNetwork == (c.config.services.hyperhive.swarmContainer.privateNetwork or null)
) (lib.attrValues serviceContainers);

1
nix/packages/default.nix
View file

@ -177,6 +177,7 @@ in
# The review classifier's daemon. Out of `daemonBins` for the responder's
# reason: it runs inside its own container, not on the host.
swarm-review-classifier = mkBinPackage "swarm-review-classifier" "hyperhive review classifier try-out daemon";
swarm-builder = mkBinPackage "swarm-builder" "hyperhive swarm builder: agent config build checks";
# The only process allowed to write swarm-authelia's users database —
# same "runs *inside* a container, not on the host" placement as