cafa120d77
argus's review on #5177: the git credential helper answered every `get`, so a config PR's own flake inputs could have git send the builder's forge token to any host. The helper is now configured under `credential.<forge-url>.helper` and itself answers only an https request whose host is the forge's; a test runs it as git does and checks that other hosts, http and non-get operations get nothing. Builds are bounded: every `build_agent` node needs the one `builder` jobq resource, so one runs at a time and the rest wait pending, and a newer head cancels its PR's pending build (a running one finishes and writes on its own commit). The builder also holds a one-permit semaphore around each build. `builder` and `review-classifier`, the forge logins swarm-controller owns, join nix/reserved-names.nix, with a test tying the list to both constants. The builder is now opt-in like the CI runner: `allSwarmServices` no longer turns it on, since it runs unreviewed config PRs through the host's nix daemon. mara had not answered atlas's question on the PR.
232 lines
9.2 KiB
Nix
232 lines
9.2 KiB
Nix
# The swarm builder, in a `swarm-builder` container on the host that sets
|
|
# `deploy.builder.enable`, which nothing turns on for the operator: it runs
|
|
# unreviewed config PRs through the host's nix daemon. It builds agent configs
|
|
# for swarm-controller: the controller sends a config PR's head over the
|
|
# queue, and the daemon builds that config on this host's agent base and
|
|
# replies.
|
|
# docs/swarm/builder.md has the design.
|
|
#
|
|
# ⚠️ Its own network namespace on the bridge, as ./swarm-ci-runner.nix has: a
|
|
# config PR is unreviewed input to a nix evaluation, so the container reaches
|
|
# the swarm services by name and nothing on the host's loopback.
|
|
#
|
|
# Builds go through the HOST nix-daemon, as the CI runner's do. The daemon
|
|
# runs as a dynamic user, an untrusted daemon client: a config can set no
|
|
# daemon option, and its builds run in the host's sandbox.
|
|
#
|
|
# The container holds one credential, its own store leaf, which nspawn hands in
|
|
# as a system credential with no copy on disk. The daemon reads everything else
|
|
# from the store with it: its queue client secret at start and its forge token
|
|
# on every build.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
cfg = hyperhiveCfg.deploy.builder;
|
|
baoCfg = hyperhiveCfg.swarm.bao;
|
|
baoDeploy = hyperhiveCfg.deploy.bao;
|
|
natsCfg = hyperhiveCfg.swarm.nats;
|
|
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
|
forgeCfg = hyperhiveCfg.swarm.forge;
|
|
gatewayCfg = hyperhiveCfg.gateway;
|
|
networkCfg = hyperhiveCfg.network;
|
|
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
|
|
|
|
machine = "swarm-builder";
|
|
unit = "swarm-builder";
|
|
clientId = hyperhiveCfg.swarm.builder.clientId;
|
|
|
|
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
|
# `git` reaches the forge; in self-signed mode their leaves chain to the hive
|
|
# CA.
|
|
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
caBundleModule = caTrust.trustBundle {
|
|
inherit pkgs;
|
|
name = machine;
|
|
consumers = [ unit ];
|
|
};
|
|
|
|
haveClientIdentity =
|
|
baoDeploy.builderClientCertFile != null && baoDeploy.builderClientKeyFile != null;
|
|
|
|
# Credential id → host source, as ./swarm-review-classifier.nix hands its
|
|
# leaf in.
|
|
storeFiles = lib.filterAttrs (_: source: source != null) {
|
|
builder-bao-cert = baoDeploy.builderClientCertFile;
|
|
builder-bao-key = baoDeploy.builderClientKeyFile;
|
|
builder-bao-server-ca = baoDeploy.serverCaFile;
|
|
};
|
|
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
|
# one word.
|
|
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
|
credential = id: "/run/credentials/${unit}.service/${id}";
|
|
|
|
# The `nix` the host daemon runs, so client and daemon speak one protocol.
|
|
nix = config.nix.package;
|
|
|
|
# Strings, not paths: both are already store paths, and the flake reference
|
|
# the daemon passes must be the same one a hive's meta flake names.
|
|
hyperhiveFlake = hyperhiveCfg.c0re.hyperhiveFlake;
|
|
nixpkgsFlake = hyperhiveCfg.c0re.nixpkgsFlake;
|
|
|
|
privateNetwork = true;
|
|
in
|
|
{
|
|
options.services.hyperhive.swarm.builder.clientId = lib.mkOption {
|
|
type = lib.types.str;
|
|
readOnly = true;
|
|
default = "swarm-builder";
|
|
description = ''
|
|
The OAuth2 client id the swarm builder presents to the swarm queue.
|
|
Read-only: the host running authelia registers it, the queue's
|
|
auth-callout responder grants it the build subject, and the secret
|
|
publisher carries its secret to the store, all from this one value.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.builder.package = lib.mkOption {
|
|
type = lib.types.package;
|
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-builder";
|
|
description = "The swarm builder daemon package.";
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = haveClientIdentity;
|
|
message = ''
|
|
services.hyperhive.deploy.builder.enable requires the builder's
|
|
swarm-secret-store client identity: set both
|
|
|
|
services.hyperhive.deploy.bao.builderClientCertFile
|
|
services.hyperhive.deploy.bao.builderClientKeyFile
|
|
|
|
The builder reads its queue client secret and its forge token with it.
|
|
|
|
Both default to the pair glue-bao-tls.nix mints when
|
|
services.hyperhive.deploy.bao.enable is true (the host that runs the
|
|
store). Otherwise copy builder.pem and builder-key.pem from the store
|
|
host's /var/lib/swarm-bao-pki/ and name them in those two options.
|
|
'';
|
|
}
|
|
{
|
|
assertion = lib.all oneWord (lib.attrValues storeFiles);
|
|
message = ''
|
|
The swarm builder's store identity paths must not contain whitespace:
|
|
nspawn's --load-credential receives them through nixos-container's
|
|
unquoted extra flags.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The container hangs off the bridge, takes its address from the gateway's
|
|
# DHCP and resolves through its dnsmasq.
|
|
services.hyperhive.network.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# nspawn fails to start the container when a credential source is
|
|
# missing, so where this host mints the leaf the container waits for it.
|
|
systemd.services."container@${machine}" = lib.mkMerge [
|
|
caTrust.containerOrdering
|
|
{ after = [ "swarm-bao-pki.service" ]; }
|
|
];
|
|
|
|
containers.${machine} = {
|
|
autoStart = true;
|
|
# The daemon keeps nothing but `git`'s fetch cache, which a restart may
|
|
# drop.
|
|
ephemeral = false;
|
|
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
|
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
|
extraFlags = [
|
|
"--link-journal=host"
|
|
]
|
|
++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
|
|
inherit privateNetwork;
|
|
hostBridge = networkCfg.bridgeName;
|
|
|
|
bindMounts = {
|
|
# The directory rather than the socket file, so the mount survives a
|
|
# daemon restart replacing the inode.
|
|
"/nix/var/nix/daemon-socket" = {
|
|
hostPath = "/nix/var/nix/daemon-socket";
|
|
isReadOnly = false;
|
|
};
|
|
}
|
|
// caTrust.bindMount;
|
|
|
|
config =
|
|
{ lib, ... }:
|
|
{
|
|
imports = [
|
|
../container-modules/swarm-container.nix
|
|
caBundleModule
|
|
];
|
|
|
|
services.hyperhive.swarmContainer = {
|
|
inherit privateNetwork;
|
|
writesOwnResolvConf = false;
|
|
};
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
networking.interfaces.eth0.useDHCP = true;
|
|
|
|
# Both the service and its socket are off: the socket unit owns
|
|
# `/nix/var/nix/daemon-socket/socket` and would race the bind mount.
|
|
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
|
systemd.services.nix-daemon.enable = false;
|
|
systemd.sockets.nix-daemon.enable = false;
|
|
|
|
systemd.services.${unit} = {
|
|
description = "build agent configs for swarm-controller";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "network-online.target" ];
|
|
after = [ "network-online.target" ];
|
|
# nix fetches the config repo with `git`.
|
|
path = [ pkgs.git ];
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = credential "builder-bao-cert";
|
|
BAO_CLIENT_KEY = credential "builder-bao-key";
|
|
NIX_CONFIG = "experimental-features = nix-command flakes";
|
|
# nix's fetch cache, under `CacheDirectory`; the dynamic user has
|
|
# no home of its own.
|
|
HOME = "/var/cache/${unit}";
|
|
XDG_CACHE_HOME = "/var/cache/${unit}";
|
|
SWARM_BUILDER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
|
|
SWARM_BUILDER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
|
|
SWARM_BUILDER_OIDC_CLIENT_ID = clientId;
|
|
SWARM_BUILDER_NIX = "${nix}/bin/nix";
|
|
SWARM_BUILDER_CHECK_FLAKE = "${../builder/check-flake}";
|
|
SWARM_BUILDER_HYPERHIVE_FLAKE = hyperhiveFlake;
|
|
SWARM_BUILDER_NIXPKGS_FLAKE = nixpkgsFlake;
|
|
SWARM_BUILDER_FORGE_URL = "https://${forgeCfg.domain}";
|
|
SWARM_BUILDER_CONFIG_ORG = "agent-configs";
|
|
}
|
|
// lib.optionalAttrs (storeFiles ? builder-bao-server-ca) {
|
|
BAO_CACERT = credential "builder-bao-server-ca";
|
|
}
|
|
// lib.optionalAttrs caTrust.useSelfSigned {
|
|
SWARM_BUILDER_OIDC_CA_FILE = caTrust.bundlePathFor machine;
|
|
# `git`'s trust store, which the bundle's system CAs keep whole.
|
|
GIT_SSL_CAINFO = caTrust.bundlePathFor machine;
|
|
};
|
|
serviceConfig = {
|
|
ExecStart = "${cfg.package}/bin/swarm-builder";
|
|
DynamicUser = true;
|
|
CacheDirectory = unit;
|
|
# Bare ids: the system credentials nspawn handed in.
|
|
LoadCredential = lib.attrNames storeFiles;
|
|
Restart = "on-failure";
|
|
RestartSec = "10s";
|
|
SyslogIdentifier = unit;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|