a2a94f63c8
Each container now runs the journal forwarder the queue's container runs, pushing to a receiver of its own on the swarm collector as the service's existing OIDC client, with the client's secret read inside the container by the fetch unit. Their store identities are system credentials, so the fetch unit loads them too. The three containers drop --link-journal=host, which hid their journals from the forwarder, and the host collector stops reading them.
281 lines
12 KiB
Nix
281 lines
12 KiB
Nix
# The swarm builder, in a `swarm-builder` container on the host that sets
|
|
# `deploy.builder.enable`, which `deploy.allSwarmServices` defaults on where the
|
|
# host has the builder's store identity (./swarm-required-services.nix). It runs
|
|
# unreviewed config PRs through the host's nix daemon. It builds agent configs
|
|
# for swarm-controller: the controller sends a config PR's head over the
|
|
# queue, and the daemon builds that config on this host's agent base and
|
|
# replies.
|
|
# docs/swarm/builder.md has the design.
|
|
#
|
|
# ⚠️ Its own network namespace on the bridge, as ./swarm-ci-runner.nix has: a
|
|
# config PR is unreviewed input to a nix evaluation, so the container reaches
|
|
# the swarm services by name and nothing on the host's loopback.
|
|
#
|
|
# Builds go through the HOST nix-daemon, as the CI runner's do. The daemon
|
|
# runs as a dynamic user, an untrusted daemon client: a config can set no
|
|
# daemon option, and its builds run in the host's sandbox.
|
|
#
|
|
# The container holds one credential, its own store leaf, which nspawn hands in
|
|
# as a system credential with no copy on disk. The daemon reads everything else
|
|
# from the store with it: its queue client secret at start and its forge token
|
|
# on every build.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
cfg = hyperhiveCfg.deploy.builder;
|
|
baoCfg = hyperhiveCfg.swarm.bao;
|
|
baoDeploy = hyperhiveCfg.deploy.bao;
|
|
natsCfg = hyperhiveCfg.swarm.nats;
|
|
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
|
forgeCfg = hyperhiveCfg.swarm.forge;
|
|
gatewayCfg = hyperhiveCfg.gateway;
|
|
networkCfg = hyperhiveCfg.network;
|
|
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
|
|
|
|
machine = "swarm-builder";
|
|
unit = "swarm-builder";
|
|
clientId = hyperhiveCfg.swarm.builder.clientId;
|
|
|
|
# The container's journal forwarder pushes to its own receiver on the swarm
|
|
# collector, the route `serviceProducers.swarm-builder` in
|
|
# ./swarm-otel-service.nix declares, by the collector's gateway name. The
|
|
# port suffix is elided on 443, as in ./swarm-bao.nix.
|
|
swarmOtelCfg = hyperhiveCfg.swarm.otel;
|
|
gatewayPortSuffix = lib.optionalString (
|
|
gatewayCfg.httpsPort != 443
|
|
) ":${toString gatewayCfg.httpsPort}";
|
|
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
|
|
forwarderUnit = "swarm-journal-forwarder-oidc";
|
|
# Loopback in the container's own netns, and distinct from the other service
|
|
# containers' forwarders all the same.
|
|
forwarderTelemetryPort = 8892;
|
|
|
|
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
|
# `git` reaches the forge; in self-signed mode their leaves chain to the hive
|
|
# CA.
|
|
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
caBundleModule = caTrust.trustBundle {
|
|
inherit pkgs;
|
|
name = machine;
|
|
consumers = [
|
|
unit
|
|
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
|
|
# collector's and authelia's names.
|
|
"opentelemetry-collector"
|
|
];
|
|
};
|
|
|
|
haveClientIdentity =
|
|
baoDeploy.builderClientCertFile != null && baoDeploy.builderClientKeyFile != null;
|
|
|
|
# Credential id → host source, as ./swarm-review-classifier.nix hands its
|
|
# leaf in.
|
|
storeFiles = lib.filterAttrs (_: source: source != null) {
|
|
builder-bao-cert = baoDeploy.builderClientCertFile;
|
|
builder-bao-key = baoDeploy.builderClientKeyFile;
|
|
builder-bao-server-ca = baoDeploy.serverCaFile;
|
|
};
|
|
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
|
# one word.
|
|
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
|
credential = id: "/run/credentials/${unit}.service/${id}";
|
|
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
|
|
|
# The `nix` the host daemon runs, so client and daemon speak one protocol.
|
|
nix = config.nix.package;
|
|
|
|
# Strings, not paths: both are already store paths, and the flake reference
|
|
# the daemon passes must be the same one a hive's meta flake names.
|
|
hyperhiveFlake = hyperhiveCfg.c0re.hyperhiveFlake;
|
|
nixpkgsFlake = hyperhiveCfg.c0re.nixpkgsFlake;
|
|
|
|
privateNetwork = true;
|
|
in
|
|
{
|
|
options.services.hyperhive.swarm.builder.clientId = lib.mkOption {
|
|
type = lib.types.str;
|
|
readOnly = true;
|
|
default = "swarm-builder";
|
|
description = ''
|
|
The OAuth2 client id the swarm builder presents to the swarm queue.
|
|
Read-only: the host running authelia registers it, the queue's
|
|
auth-callout responder grants it the build subject, and the secret
|
|
publisher carries its secret to the store, all from this one value.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.builder.package = lib.mkOption {
|
|
type = lib.types.package;
|
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-builder";
|
|
description = "The swarm builder daemon package.";
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = haveClientIdentity;
|
|
message = ''
|
|
services.hyperhive.deploy.builder.enable requires the builder's
|
|
swarm-secret-store client identity: set both
|
|
|
|
services.hyperhive.deploy.bao.builderClientCertFile
|
|
services.hyperhive.deploy.bao.builderClientKeyFile
|
|
|
|
The builder reads its queue client secret and its forge token with it.
|
|
|
|
Both default to the pair glue-bao-tls.nix mints when
|
|
services.hyperhive.deploy.bao.enable is true (the host that runs the
|
|
store). Otherwise copy builder.pem and builder-key.pem from the store
|
|
host's /var/lib/swarm-bao-pki/ and name them in those two options.
|
|
'';
|
|
}
|
|
{
|
|
assertion = lib.all oneWord (lib.attrValues storeFiles);
|
|
message = ''
|
|
The swarm builder's store identity paths must not contain whitespace:
|
|
nspawn's --load-credential receives them through nixos-container's
|
|
unquoted extra flags.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The container hangs off the bridge, takes its address from the gateway's
|
|
# DHCP and resolves through its dnsmasq.
|
|
services.hyperhive.network.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# nspawn fails to start the container when a credential source is
|
|
# missing, so where this host mints the leaf the container pulls the
|
|
# minting unit in and waits for it. Ordering alone is not enough: the
|
|
# container starts from `machines.target`, outside the transaction that
|
|
# runs the minting unit from `multi-user.target`. Ignored elsewhere, where
|
|
# the unit does not exist.
|
|
systemd.services."container@${machine}" = lib.mkMerge [
|
|
caTrust.containerOrdering
|
|
{
|
|
after = [ "swarm-bao-pki.service" ];
|
|
wants = [ "swarm-bao-pki.service" ];
|
|
}
|
|
];
|
|
|
|
containers.${machine} = {
|
|
autoStart = true;
|
|
# The daemon keeps nothing but `git`'s fetch cache, which a restart may
|
|
# drop.
|
|
ephemeral = false;
|
|
# No `--link-journal=host`: the journal stays in the guest, where the
|
|
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
|
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
|
|
inherit privateNetwork;
|
|
hostBridge = networkCfg.bridgeName;
|
|
|
|
bindMounts = {
|
|
# The directory rather than the socket file, so the mount survives a
|
|
# daemon restart replacing the inode.
|
|
"/nix/var/nix/daemon-socket" = {
|
|
hostPath = "/nix/var/nix/daemon-socket";
|
|
isReadOnly = false;
|
|
};
|
|
}
|
|
// caTrust.bindMount;
|
|
|
|
config =
|
|
{ lib, ... }:
|
|
{
|
|
imports = [
|
|
../container-modules/swarm-container.nix
|
|
../container-modules/swarm-journal-forwarder.nix
|
|
caBundleModule
|
|
];
|
|
|
|
services.hyperhive.swarmContainer = {
|
|
inherit privateNetwork;
|
|
writesOwnResolvConf = false;
|
|
};
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
networking.interfaces.eth0.useDHCP = true;
|
|
|
|
# Both the service and its socket are off: the socket unit owns
|
|
# `/nix/var/nix/daemon-socket/socket` and would race the bind mount.
|
|
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
|
systemd.services.nix-daemon.enable = false;
|
|
systemd.sockets.nix-daemon.enable = false;
|
|
|
|
# Authenticated as the daemon's own OIDC client, whose secret its
|
|
# store identity already reads: one identity for the whole container.
|
|
# That identity is a system credential, so the fetch unit loads it
|
|
# too; the daemon's `LoadCredential` below is that unit's alone.
|
|
services.hyperhive.swarmJournalForwarder = {
|
|
enable = true;
|
|
serviceName = "builder";
|
|
endpoint = forwarderEndpoint;
|
|
inherit clientId;
|
|
tokenUrl = "${autheliaCfg.url}/api/oidc/token";
|
|
telemetryPort = forwarderTelemetryPort;
|
|
store = {
|
|
package = baoDeploy.package;
|
|
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
secretPath = "secret/swarm/services/${clientId}/oidc/client";
|
|
clientCertFile = forwarderCredential "builder-bao-cert";
|
|
clientKeyFile = forwarderCredential "builder-bao-key";
|
|
caFile =
|
|
if storeFiles ? builder-bao-server-ca then forwarderCredential "builder-bao-server-ca" else null;
|
|
};
|
|
};
|
|
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
|
|
|
|
systemd.services.${unit} = {
|
|
description = "build agent configs for swarm-controller";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "network-online.target" ];
|
|
after = [ "network-online.target" ];
|
|
# nix fetches the config repo with `git`.
|
|
path = [ pkgs.git ];
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = credential "builder-bao-cert";
|
|
BAO_CLIENT_KEY = credential "builder-bao-key";
|
|
NIX_CONFIG = "experimental-features = nix-command flakes";
|
|
# nix's fetch cache, under `CacheDirectory`; the dynamic user has
|
|
# no home of its own.
|
|
HOME = "/var/cache/${unit}";
|
|
XDG_CACHE_HOME = "/var/cache/${unit}";
|
|
SWARM_BUILDER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
|
|
SWARM_BUILDER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
|
|
SWARM_BUILDER_OIDC_CLIENT_ID = clientId;
|
|
SWARM_BUILDER_NIX = "${nix}/bin/nix";
|
|
SWARM_BUILDER_CHECK_FLAKE = "${../builder/check-flake}";
|
|
SWARM_BUILDER_HYPERHIVE_FLAKE = hyperhiveFlake;
|
|
SWARM_BUILDER_NIXPKGS_FLAKE = nixpkgsFlake;
|
|
SWARM_BUILDER_FORGE_URL = "https://${forgeCfg.domain}";
|
|
SWARM_BUILDER_CONFIG_ORG = "agent-configs";
|
|
}
|
|
// lib.optionalAttrs (storeFiles ? builder-bao-server-ca) {
|
|
BAO_CACERT = credential "builder-bao-server-ca";
|
|
}
|
|
// lib.optionalAttrs caTrust.useSelfSigned {
|
|
SWARM_BUILDER_OIDC_CA_FILE = caTrust.bundlePathFor machine;
|
|
# `git`'s trust store, which the bundle's system CAs keep whole.
|
|
GIT_SSL_CAINFO = caTrust.bundlePathFor machine;
|
|
};
|
|
serviceConfig = {
|
|
ExecStart = "${cfg.package}/bin/swarm-builder";
|
|
DynamicUser = true;
|
|
CacheDirectory = unit;
|
|
# Bare ids: the system credentials nspawn handed in.
|
|
LoadCredential = lib.attrNames storeFiles;
|
|
Restart = "on-failure";
|
|
RestartSec = "10s";
|
|
SyslogIdentifier = unit;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|