Watch
0
0
Fork
You've already forked hyperhive
0

nix: hand the review classifier its store leaf as an nspawn credential

The leaf's key was copied by a host unit into the container's root on
host disk. nspawn now reads the cert, key and store CA as root at
container start (--load-credential) and passes them in as system
credentials; the daemon's unit loads them by bare id, as an agent's
container does with its own leaf (nix/agent-modules/bao.nix). The copy
unit is gone. Where this host mints the leaf, the container starts after
swarm-bao-pki, since nspawn refuses a missing credential source. An
assertion refuses identity paths with whitespace, which nixos-container's
unquoted extra flags would split.
This commit is contained in:
atlas 2026-10-12 04:09:03 +02:00 • committed by mara
commit 8e33de6fb7
3 changed files with 57 additions and 56 deletions

View file

@ -8,9 +8,10 @@
# PR's diff and commit messages are model input, so the container reaches the # PR's diff and commit messages are model input, so the container reaches the
# swarm services by name and nothing on the host's loopback. # swarm services by name and nothing on the host's loopback.
# #
# The container holds one credential, its own store leaf, which the host unit # The container holds one credential, its own store leaf, which nspawn hands in
# below copies in. The daemon reads everything else from the store with it: # as a system credential with no copy on disk. The daemon reads everything else
# its queue client secret at start and the API key on every run. # from the store with it: its queue client secret at start and the API key on
# every run.
{ {
pkgs, pkgs,
lib, lib,
@ -45,18 +46,19 @@ let
haveClientIdentity = haveClientIdentity =
baoDeploy.reviewClassifierClientCertFile != null && baoDeploy.reviewClassifierClientKeyFile != null; baoDeploy.reviewClassifierClientCertFile != null && baoDeploy.reviewClassifierClientKeyFile != null;
# Credential id → host source, copied into the container and loaded by the # Credential id → host source. nspawn reads each source as root when the
# daemon's unit alone. # container starts and hands it in as a system credential, held in memory;
storeFiles = { # the daemon's unit alone loads them. No copy lands on disk, as for an
"bao-client.pem" = baoDeploy.reviewClassifierClientCertFile; # agent's own leaf (`nix/agent-modules/bao.nix`). Ids carry no dot, the
"bao-client-key.pem" = baoDeploy.reviewClassifierClientKeyFile; # spelling `--load-credential` is safe with.
} storeFiles = lib.filterAttrs (_: source: source != null) {
// lib.optionalAttrs (baoDeploy.serverCaFile != null) { review-classifier-bao-cert = baoDeploy.reviewClassifierClientCertFile;
"bao-ca.pem" = baoDeploy.serverCaFile; review-classifier-bao-key = baoDeploy.reviewClassifierClientKeyFile;
review-classifier-bao-server-ca = baoDeploy.serverCaFile;
}; };
secretDirInContainer = "/var/lib/swarm-review-classifier"; # nixos-container expands `extraFlags` unquoted, so a source path must be
inContainer = id: "${secretDirInContainer}/${id}"; # one word.
secretDir = "/var/lib/nixos-containers/${machine}${secretDirInContainer}"; oneWord = path: builtins.match ".*[[:space:]].*" path == null;
credential = id: "/run/credentials/${unit}.service/${id}"; credential = id: "/run/credentials/${unit}.service/${id}";
# What `claude` exports its metrics with, where this host runs a hive-tier # What `claude` exports its metrics with, where this host runs a hive-tier
@ -115,6 +117,14 @@ in
and name them in those two options. and name them in those two options.
''; '';
} }
{
assertion = lib.all oneWord (lib.attrValues storeFiles);
message = ''
The review classifier's store identity paths must not contain
whitespace: nspawn's --load-credential receives them through
nixos-container's unquoted extra flags.
'';
}
]; ];
# The container hangs off the bridge, takes its address from the gateway's # The container hangs off the bridge, takes its address from the gateway's
@ -123,42 +133,23 @@ in
services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# A copy rather than a bind mount: nixos-container refuses to start on a # nspawn fails to start the container when a credential source is
# missing bind source, and the leaf may not be minted yet. # missing, so where this host mints the leaf the container waits for it.
systemd.services.swarm-review-classifier-identity = { systemd.services."container@${machine}" = lib.mkMerge [
description = "deliver the review classifier's store identity into its container"; caTrust.containerOrdering
before = [ "container@${machine}.service" ]; { after = [ "swarm-bao-pki.service" ]; }
wantedBy = [ "container@${machine}.service" ]; ];
# Ordering only: where the store's PKI is minted on this host, the leaf
# is one of its files.
after = [ "swarm-bao-pki.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-review-classifier-identity";
};
path = [ pkgs.coreutils ];
script = ''
set -euo pipefail
install -d -m 0700 ${lib.escapeShellArg secretDir}
''
+ lib.concatStrings (
lib.mapAttrsToList (id: source: ''
install -m 0400 ${lib.escapeShellArg source} ${lib.escapeShellArg "${secretDir}/${id}"}
'') storeFiles
);
};
systemd.services."container@${machine}" = caTrust.containerOrdering;
containers.${machine} = { containers.${machine} = {
autoStart = true; autoStart = true;
# Not ephemeral: an ephemeral root lives under /run, where the identity # The daemon keeps no state; its scratch directories are under PrivateTmp.
# copy above does not write. The daemon itself keeps no state.
ephemeral = false; ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes # Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ]; extraFlags = [
"--link-journal=host"
]
++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork; inherit privateNetwork;
hostBridge = networkCfg.bridgeName; hostBridge = networkCfg.bridgeName;
bindMounts = caTrust.bindMount; bindMounts = caTrust.bindMount;
@ -189,15 +180,15 @@ in
after = [ "network-online.target" ]; after = [ "network-online.target" ];
environment = { environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = credential "bao-client.pem"; BAO_CLIENT_CERT = credential "review-classifier-bao-cert";
BAO_CLIENT_KEY = credential "bao-client-key.pem"; BAO_CLIENT_KEY = credential "review-classifier-bao-key";
SWARM_REVIEW_CLASSIFIER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}"; SWARM_REVIEW_CLASSIFIER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
SWARM_REVIEW_CLASSIFIER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token"; SWARM_REVIEW_CLASSIFIER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID = clientId; SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID = clientId;
SWARM_REVIEW_CLASSIFIER_CLAUDE = "${pkgs.claude-code}/bin/claude"; SWARM_REVIEW_CLASSIFIER_CLAUDE = "${pkgs.claude-code}/bin/claude";
} }
// lib.optionalAttrs (storeFiles ? "bao-ca.pem") { // lib.optionalAttrs (storeFiles ? review-classifier-bao-server-ca) {
BAO_CACERT = credential "bao-ca.pem"; BAO_CACERT = credential "review-classifier-bao-server-ca";
} }
// lib.optionalAttrs caTrust.useSelfSigned { // lib.optionalAttrs caTrust.useSelfSigned {
SWARM_REVIEW_CLASSIFIER_OIDC_CA_FILE = caTrust.bundlePathFor machine; SWARM_REVIEW_CLASSIFIER_OIDC_CA_FILE = caTrust.bundlePathFor machine;
@ -208,7 +199,8 @@ in
DynamicUser = true; DynamicUser = true;
# Each classification's scratch directory, gone with the unit. # Each classification's scratch directory, gone with the unit.
PrivateTmp = true; PrivateTmp = true;
LoadCredential = lib.mapAttrsToList (id: _: "${id}:${inContainer id}") storeFiles; # Bare ids: the system credentials nspawn handed in.
LoadCredential = lib.attrNames storeFiles;
Restart = "on-failure"; Restart = "on-failure";
RestartSec = "10s"; RestartSec = "10s";
SyslogIdentifier = unit; SyslogIdentifier = unit;

View file

@ -311,9 +311,10 @@ let
&& withClassifier.containers ? swarm-review-classifier; && withClassifier.containers ? swarm-review-classifier;
} }
{ {
# The daemon holds its store leaf and nothing else from the host; the # The daemon holds its store leaf, handed in by nspawn without a copy on
# key is read per run and reaches only `claude`'s environment. Reading # disk, and nothing else from the host; the key is read per run and
# the claude path also evaluates the container's unfree allowance. # reaches only `claude`'s environment. Reading the claude path also
# evaluates the container's unfree allowance.
name = "the review classifier loads only its store leaf and carries no API key in its unit"; name = "the review classifier loads only its store leaf and carries no API key in its unit";
ok = ok =
let let
@ -321,12 +322,19 @@ let
withClassifier.containers.swarm-review-classifier.config.systemd.services.swarm-review-classifier; withClassifier.containers.swarm-review-classifier.config.systemd.services.swarm-review-classifier;
env = svc.environment; env = svc.environment;
in in
lib.elem "bao-client.pem:/var/lib/swarm-review-classifier/bao-client.pem" svc.serviceConfig.LoadCredential lib.elem "review-classifier-bao-key" svc.serviceConfig.LoadCredential
&& env.BAO_CLIENT_CERT == "/run/credentials/swarm-review-classifier.service/bao-client.pem" &&
env.BAO_CLIENT_KEY == "/run/credentials/swarm-review-classifier.service/review-classifier-bao-key"
&& env.SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID == "swarm-review-classifier" && env.SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID == "swarm-review-classifier"
&& lib.hasSuffix "/bin/claude" env.SWARM_REVIEW_CLASSIFIER_CLAUDE && lib.hasSuffix "/bin/claude" env.SWARM_REVIEW_CLASSIFIER_CLAUDE
&& !(env ? ANTHROPIC_API_KEY) && !(env ? ANTHROPIC_API_KEY)
&& withClassifier.systemd.services ? swarm-review-classifier-identity; && lib.elem "--load-credential=review-classifier-bao-key:/var/lib/swarm-bao-pki/review-classifier-key.pem" (
withClassifier.containers.swarm-review-classifier.extraFlags
)
# No host unit copies the leaf into the container's root.
&& !(lib.any (u: lib.hasInfix "nixos-containers/swarm-review-classifier" (u.script or "")) (
lib.attrValues withClassifier.systemd.services
));
} }
{ {
# The collector reaches these routes through the gateway now, so each # The collector reaches these routes through the gateway now, so each

View file

@ -26,7 +26,8 @@ forge and holds no forge credential.
## Credentials ## Credentials
The container holds one file: its own store certificate. With it the service The container holds one credential: its own store certificate, which nspawn
hands in at start without a copy on disk. With it the service
reads its queue client secret at start and the API key at every run, so a key reads its queue client secret at start and the API key at every run, so a key
written or removed in the store takes effect on the next run. Its store role written or removed in the store takes effect on the next run. Its store role
reads exactly those two paths. reads exactly those two paths.