swarm-builder: build each agent config PR head and post a commit status
A new `swarm-builder` service, in its own container on the host that sets
`deploy.builder.enable` (defaulted on by `allSwarmServices` where the host
has the builder's store leaf, like the review classifier).
When an `agent-configs/<agent>` PR is opened, reopened or synchronized,
swarm-controller queues a `build_agent` jobq node. The node sets a
`swarm-builder` commit status to pending on the PR head, sends
`{agent, rev, pr}` to the builder over NATS request-reply
(`$SWARM.builder.build`), and writes success (store path), failure (last
line nix printed) or error (no builder / no reply) on the head. The node
fails when the build does, so the Jobs page matches the PR.
The builder builds nix/builder/check-flake's `check` with every input
overridden: hyperhive and nixpkgs to the host's own
`c0re.hyperhiveFlake`/`c0re.nixpkgsFlake` (what a hive there renders into
its meta flake), and the agent to the PR head on the forge. Evaluation is
pure and runs in the container; builds go to the host nix daemon as an
untrusted client. The private config repo is fetched with the token of a
new read-only `builder` forge account (in the `hives` team), which the
controller mints into `swarm/services/builder/forge/token`; git gets it
from a credential helper reading the child's environment, so it is in no
argument or URL.
Credentials: a `builder` store leaf minted by glue-bao-tls on the store
host, a `swarm-builder` reader policy (forge token + its own OIDC client
secret), an authelia machine client, and a `--builder-client` grant in the
queue's auth-callout responder. The controller gains a grant on the token
path and the secret publisher a deny on it.
This is the agent-config build check atlas scoped on #5081 (slice 3):
nothing is pushed to a cache, nothing is written to KV, and a red status
gates no deploy. The check composes agent-base + the config without the
values a hive adds (CA, docs, claude-code path, ports, input follows), so
it can disagree with a hive's own build.
This commit is contained in:
atlas
parent
aece1c26d0
commit
b68b74d687
43 changed files with 2452 additions and 42 deletions
|
|
@ -189,6 +189,7 @@ let
|
|||
"swarm-bao-logs-auth-policy"
|
||||
"swarm-bao-forge-publisher-policy"
|
||||
"swarm-bao-review-classifier-policy"
|
||||
"swarm-bao-builder-policy"
|
||||
];
|
||||
|
||||
# Comment lines dropped first: both the HCL and the scripts explain
|
||||
|
|
@ -1491,6 +1492,58 @@ let
|
|||
"swarm-bao-controller-policy"
|
||||
];
|
||||
}
|
||||
{
|
||||
# The builder reads its forge token and its own queue client secret,
|
||||
# each read-only and spelled to the leaf: `services/*` would reach every
|
||||
# swarm service's secret.
|
||||
name = "the swarm builder's grant is its forge token and its queue client secret, and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-builder-policy.script;
|
||||
in
|
||||
lib.hasInfix "path \"secret/data/swarm/services/builder/forge/token\" {\n capabilities = [\"read\"]\n}" s
|
||||
&& lib.hasInfix "path \"secret/data/swarm/services/swarm-builder/oidc/client\" {\n capabilities = [\"read\"]\n}" s
|
||||
&& lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 2
|
||||
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
|
||||
&& lib.hasInfix "auth/cert/certs/swarm-builder" s
|
||||
&& lib.hasInfix "allowed_common_names=swarm-builder" s
|
||||
&& lib.hasInfix "token_policies=swarm-builder" s;
|
||||
}
|
||||
{
|
||||
# `forge::builder_account` mints the token and reads it back to decide
|
||||
# on a re-mint. Pinned as the whole stanza, and spelled to the leaf.
|
||||
name = "the controller's grant on the swarm builder is its forge token alone";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
lib.hasInfix "path \"secret/data/swarm/services/builder/forge/token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
|
||||
&& !(lib.hasInfix "secret/data/swarm/services/builder/*" s);
|
||||
}
|
||||
{
|
||||
# The token sits under the publisher's `services/*` write, and the
|
||||
# builder fetches config repos with whatever is there. Its OIDC leaf is
|
||||
# the control: still the prefix's.
|
||||
name = "the publisher may not write the swarm builder's forge token";
|
||||
ok =
|
||||
let
|
||||
grants = grantsIn baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script;
|
||||
in
|
||||
(grantFor grants "secret/data/swarm/services/builder/forge/token").caps == [ "deny" ]
|
||||
&&
|
||||
(grantFor grants "secret/data/swarm/services/swarm-builder/oidc/client").caps == [
|
||||
"create"
|
||||
"update"
|
||||
];
|
||||
}
|
||||
{
|
||||
name = "the PKI unit signs the swarm builder's leaf under its own subject";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
|
||||
in
|
||||
lib.hasInfix "/builder.pem ]" s && lib.hasInfix "swarm-builder \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
name = "the PKI unit signs the CI runner's leaf under its own subject";
|
||||
ok =
|
||||
|
|
|
|||
131
nix/module-eval/builder.nix
Normal file
131
nix/module-eval/builder.nix
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
# `checks.module-eval-builder` — see ./lib.nix for the shared rationale (why
|
||||
# this suite exists, naming convention, "evaluates not executes"). The swarm
|
||||
# builder's container, its default and its queue and IdP registrations; its
|
||||
# store grants are in ./bao-grants.nix beside every other principal's.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
self,
|
||||
nixosSystem,
|
||||
}:
|
||||
let
|
||||
inherit
|
||||
(import ./lib.nix {
|
||||
inherit
|
||||
pkgs
|
||||
lib
|
||||
self
|
||||
nixosSystem
|
||||
;
|
||||
})
|
||||
hive
|
||||
hiveWithController
|
||||
runGroup
|
||||
;
|
||||
|
||||
bare = hiveWithController { };
|
||||
|
||||
# On the store's host, which mints the builder's store identity.
|
||||
withBuilder = hive {
|
||||
deploy.bao.enable = true;
|
||||
deploy.builder.enable = true;
|
||||
};
|
||||
|
||||
servicesHere = hiveWithController { deploy.allSwarmServices = true; };
|
||||
|
||||
servicesBaoElsewhere = hive {
|
||||
deploy.allSwarmServices = true;
|
||||
deploy.bao.enable = false;
|
||||
};
|
||||
|
||||
natsHere = hive {
|
||||
swarm.nats.enable = true;
|
||||
swarm.nats.authPackage = pkgs.emptyDirectory;
|
||||
};
|
||||
|
||||
autheliaHere = hive {
|
||||
deploy.bao.enable = true;
|
||||
deploy.authelia.enable = true;
|
||||
};
|
||||
|
||||
container = withBuilder.containers.swarm-builder;
|
||||
inner = container.config;
|
||||
svc = inner.systemd.services.swarm-builder;
|
||||
env = svc.environment;
|
||||
|
||||
cases = [
|
||||
{
|
||||
name = "the swarm builder's container exists only where its host enables it";
|
||||
ok = !(bare.containers ? swarm-builder) && withBuilder.containers ? swarm-builder;
|
||||
}
|
||||
{
|
||||
# Its module asserts a store identity that only the store's host mints,
|
||||
# so with the store elsewhere the default stays off rather than failing
|
||||
# that host's eval. `nats` is the control that the switch is on at all.
|
||||
name = "the shared services turn the builder on with the store's leaf and leave it off without";
|
||||
ok =
|
||||
servicesHere.services.hyperhive.deploy.builder.enable
|
||||
&& servicesHere.containers ? swarm-builder
|
||||
&& !servicesBaoElsewhere.services.hyperhive.deploy.builder.enable
|
||||
&& !(servicesBaoElsewhere.containers ? swarm-builder)
|
||||
&& servicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
|
||||
}
|
||||
{
|
||||
# A config PR is unreviewed input to an evaluation: the container keeps
|
||||
# its own netns, and reaches the host's nix daemon through its socket
|
||||
# alone, with no daemon of its own racing the bind.
|
||||
name = "the builder's container has its own netns and builds only through the host's daemon socket";
|
||||
ok =
|
||||
container.privateNetwork
|
||||
&& inner.services.hyperhive.swarmContainer.privateNetwork
|
||||
&& container.bindMounts."/nix/var/nix/daemon-socket".hostPath == "/nix/var/nix/daemon-socket"
|
||||
&& !inner.systemd.services.nix-daemon.enable
|
||||
&& !inner.systemd.sockets.nix-daemon.enable
|
||||
&& inner.systemd.globalEnvironment.NIX_REMOTE == "daemon";
|
||||
}
|
||||
{
|
||||
# The daemon holds its store leaf, handed in by nspawn without a copy on
|
||||
# disk; the forge token is read per build and reaches only `git`.
|
||||
name = "the builder loads only its store leaf, runs as a dynamic user and carries no token in its unit";
|
||||
ok =
|
||||
lib.elem "builder-bao-key" svc.serviceConfig.LoadCredential
|
||||
&& env.BAO_CLIENT_KEY == "/run/credentials/swarm-builder.service/builder-bao-key"
|
||||
&& env.SWARM_BUILDER_OIDC_CLIENT_ID == "swarm-builder"
|
||||
&& lib.elem "--load-credential=builder-bao-key:/var/lib/swarm-bao-pki/builder-key.pem" container.extraFlags
|
||||
&& svc.serviceConfig.DynamicUser
|
||||
&& !(lib.any (lib.hasSuffix "TOKEN") (lib.attrNames env));
|
||||
}
|
||||
{
|
||||
# The check builds on the sources a hive on this host would render into
|
||||
# its meta flake, so a green check means the same base.
|
||||
name = "the builder builds on the hyperhive and nixpkgs this host's hives use";
|
||||
ok =
|
||||
env.SWARM_BUILDER_HYPERHIVE_FLAKE == withBuilder.services.hyperhive.c0re.hyperhiveFlake
|
||||
&& env.SWARM_BUILDER_NIXPKGS_FLAKE == withBuilder.services.hyperhive.c0re.nixpkgsFlake
|
||||
&& lib.hasSuffix "-check-flake" env.SWARM_BUILDER_CHECK_FLAKE
|
||||
&& env.SWARM_BUILDER_CONFIG_ORG == "agent-configs"
|
||||
&& env.SWARM_BUILDER_FORGE_URL == "https://${withBuilder.services.hyperhive.swarm.forge.domain}";
|
||||
}
|
||||
{
|
||||
# The responder denies a client id it does not know, and a NATS denial
|
||||
# arrives as a timeout.
|
||||
name = "the queue's responder grants the builder's client its subject";
|
||||
ok = lib.hasInfix "--builder-client swarm-builder" natsHere.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart;
|
||||
}
|
||||
{
|
||||
name = "the builder's client is registered as a machine, and the publisher carries its secret";
|
||||
ok =
|
||||
let
|
||||
c = lib.findFirst (
|
||||
c: c.id == "swarm-builder"
|
||||
) null autheliaHere.services.hyperhive.swarm.authelia.oidc.clients;
|
||||
in
|
||||
c != null
|
||||
&& c.kind == "machine"
|
||||
&& lib.hasInfix "secret/swarm/services/swarm-builder/oidc/client" (
|
||||
autheliaHere.systemd.services.swarm-secret-publish.script
|
||||
);
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "builder" cases
|
||||
|
|
@ -472,7 +472,7 @@ let
|
|||
# that lost a container can't pass by comparing fewer of them.
|
||||
name = "every service container restates its host-side privateNetwork";
|
||||
ok =
|
||||
lib.length (lib.attrNames serviceContainers) == 11
|
||||
lib.length (lib.attrNames serviceContainers) == 12
|
||||
&& lib.all (
|
||||
c: c.privateNetwork == (c.config.services.hyperhive.swarmContainer.privateNetwork or null)
|
||||
) (lib.attrValues serviceContainers);
|
||||
|
|
|
|||
Loading…
Reference in a new issue