nix: hand the review classifier its store leaf as an nspawn credential
The leaf's key was copied by a host unit into the container's root on host disk. nspawn now reads the cert, key and store CA as root at container start (--load-credential) and passes them in as system credentials; the daemon's unit loads them by bare id, as an agent's container does with its own leaf (nix/agent-modules/bao.nix). The copy unit is gone. Where this host mints the leaf, the container starts after swarm-bao-pki, since nspawn refuses a missing credential source. An assertion refuses identity paths with whitespace, which nixos-container's unquoted extra flags would split.
This commit is contained in:
parent
c0fd4e9848
commit
8e33de6fb7
3 changed files with 57 additions and 56 deletions
|
|
@ -8,9 +8,10 @@
|
||||||
# PR's diff and commit messages are model input, so the container reaches the
|
# PR's diff and commit messages are model input, so the container reaches the
|
||||||
# swarm services by name and nothing on the host's loopback.
|
# swarm services by name and nothing on the host's loopback.
|
||||||
#
|
#
|
||||||
# The container holds one credential, its own store leaf, which the host unit
|
# The container holds one credential, its own store leaf, which nspawn hands in
|
||||||
# below copies in. The daemon reads everything else from the store with it:
|
# as a system credential with no copy on disk. The daemon reads everything else
|
||||||
# its queue client secret at start and the API key on every run.
|
# from the store with it: its queue client secret at start and the API key on
|
||||||
|
# every run.
|
||||||
{
|
{
|
||||||
pkgs,
|
pkgs,
|
||||||
lib,
|
lib,
|
||||||
|
|
@ -45,18 +46,19 @@ let
|
||||||
haveClientIdentity =
|
haveClientIdentity =
|
||||||
baoDeploy.reviewClassifierClientCertFile != null && baoDeploy.reviewClassifierClientKeyFile != null;
|
baoDeploy.reviewClassifierClientCertFile != null && baoDeploy.reviewClassifierClientKeyFile != null;
|
||||||
|
|
||||||
# Credential id → host source, copied into the container and loaded by the
|
# Credential id → host source. nspawn reads each source as root when the
|
||||||
# daemon's unit alone.
|
# container starts and hands it in as a system credential, held in memory;
|
||||||
storeFiles = {
|
# the daemon's unit alone loads them. No copy lands on disk, as for an
|
||||||
"bao-client.pem" = baoDeploy.reviewClassifierClientCertFile;
|
# agent's own leaf (`nix/agent-modules/bao.nix`). Ids carry no dot, the
|
||||||
"bao-client-key.pem" = baoDeploy.reviewClassifierClientKeyFile;
|
# spelling `--load-credential` is safe with.
|
||||||
}
|
storeFiles = lib.filterAttrs (_: source: source != null) {
|
||||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
review-classifier-bao-cert = baoDeploy.reviewClassifierClientCertFile;
|
||||||
"bao-ca.pem" = baoDeploy.serverCaFile;
|
review-classifier-bao-key = baoDeploy.reviewClassifierClientKeyFile;
|
||||||
|
review-classifier-bao-server-ca = baoDeploy.serverCaFile;
|
||||||
};
|
};
|
||||||
secretDirInContainer = "/var/lib/swarm-review-classifier";
|
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
||||||
inContainer = id: "${secretDirInContainer}/${id}";
|
# one word.
|
||||||
secretDir = "/var/lib/nixos-containers/${machine}${secretDirInContainer}";
|
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
||||||
credential = id: "/run/credentials/${unit}.service/${id}";
|
credential = id: "/run/credentials/${unit}.service/${id}";
|
||||||
|
|
||||||
# What `claude` exports its metrics with, where this host runs a hive-tier
|
# What `claude` exports its metrics with, where this host runs a hive-tier
|
||||||
|
|
@ -115,6 +117,14 @@ in
|
||||||
and name them in those two options.
|
and name them in those two options.
|
||||||
'';
|
'';
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
assertion = lib.all oneWord (lib.attrValues storeFiles);
|
||||||
|
message = ''
|
||||||
|
The review classifier's store identity paths must not contain
|
||||||
|
whitespace: nspawn's --load-credential receives them through
|
||||||
|
nixos-container's unquoted extra flags.
|
||||||
|
'';
|
||||||
|
}
|
||||||
];
|
];
|
||||||
|
|
||||||
# The container hangs off the bridge, takes its address from the gateway's
|
# The container hangs off the bridge, takes its address from the gateway's
|
||||||
|
|
@ -123,42 +133,23 @@ in
|
||||||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||||||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||||||
|
|
||||||
# A copy rather than a bind mount: nixos-container refuses to start on a
|
# nspawn fails to start the container when a credential source is
|
||||||
# missing bind source, and the leaf may not be minted yet.
|
# missing, so where this host mints the leaf the container waits for it.
|
||||||
systemd.services.swarm-review-classifier-identity = {
|
systemd.services."container@${machine}" = lib.mkMerge [
|
||||||
description = "deliver the review classifier's store identity into its container";
|
caTrust.containerOrdering
|
||||||
before = [ "container@${machine}.service" ];
|
{ after = [ "swarm-bao-pki.service" ]; }
|
||||||
wantedBy = [ "container@${machine}.service" ];
|
];
|
||||||
# Ordering only: where the store's PKI is minted on this host, the leaf
|
|
||||||
# is one of its files.
|
|
||||||
after = [ "swarm-bao-pki.service" ];
|
|
||||||
serviceConfig = {
|
|
||||||
Type = "oneshot";
|
|
||||||
RemainAfterExit = true;
|
|
||||||
SyslogIdentifier = "swarm-review-classifier-identity";
|
|
||||||
};
|
|
||||||
path = [ pkgs.coreutils ];
|
|
||||||
script = ''
|
|
||||||
set -euo pipefail
|
|
||||||
install -d -m 0700 ${lib.escapeShellArg secretDir}
|
|
||||||
''
|
|
||||||
+ lib.concatStrings (
|
|
||||||
lib.mapAttrsToList (id: source: ''
|
|
||||||
install -m 0400 ${lib.escapeShellArg source} ${lib.escapeShellArg "${secretDir}/${id}"}
|
|
||||||
'') storeFiles
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
|
||||||
|
|
||||||
containers.${machine} = {
|
containers.${machine} = {
|
||||||
autoStart = true;
|
autoStart = true;
|
||||||
# Not ephemeral: an ephemeral root lives under /run, where the identity
|
# The daemon keeps no state; its scratch directories are under PrivateTmp.
|
||||||
# copy above does not write. The daemon itself keeps no state.
|
|
||||||
ephemeral = false;
|
ephemeral = false;
|
||||||
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
||||||
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
||||||
extraFlags = [ "--link-journal=host" ];
|
extraFlags = [
|
||||||
|
"--link-journal=host"
|
||||||
|
]
|
||||||
|
++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
|
||||||
inherit privateNetwork;
|
inherit privateNetwork;
|
||||||
hostBridge = networkCfg.bridgeName;
|
hostBridge = networkCfg.bridgeName;
|
||||||
bindMounts = caTrust.bindMount;
|
bindMounts = caTrust.bindMount;
|
||||||
|
|
@ -189,15 +180,15 @@ in
|
||||||
after = [ "network-online.target" ];
|
after = [ "network-online.target" ];
|
||||||
environment = {
|
environment = {
|
||||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||||
BAO_CLIENT_CERT = credential "bao-client.pem";
|
BAO_CLIENT_CERT = credential "review-classifier-bao-cert";
|
||||||
BAO_CLIENT_KEY = credential "bao-client-key.pem";
|
BAO_CLIENT_KEY = credential "review-classifier-bao-key";
|
||||||
SWARM_REVIEW_CLASSIFIER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
|
SWARM_REVIEW_CLASSIFIER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
|
||||||
SWARM_REVIEW_CLASSIFIER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
|
SWARM_REVIEW_CLASSIFIER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
|
||||||
SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID = clientId;
|
SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID = clientId;
|
||||||
SWARM_REVIEW_CLASSIFIER_CLAUDE = "${pkgs.claude-code}/bin/claude";
|
SWARM_REVIEW_CLASSIFIER_CLAUDE = "${pkgs.claude-code}/bin/claude";
|
||||||
}
|
}
|
||||||
// lib.optionalAttrs (storeFiles ? "bao-ca.pem") {
|
// lib.optionalAttrs (storeFiles ? review-classifier-bao-server-ca) {
|
||||||
BAO_CACERT = credential "bao-ca.pem";
|
BAO_CACERT = credential "review-classifier-bao-server-ca";
|
||||||
}
|
}
|
||||||
// lib.optionalAttrs caTrust.useSelfSigned {
|
// lib.optionalAttrs caTrust.useSelfSigned {
|
||||||
SWARM_REVIEW_CLASSIFIER_OIDC_CA_FILE = caTrust.bundlePathFor machine;
|
SWARM_REVIEW_CLASSIFIER_OIDC_CA_FILE = caTrust.bundlePathFor machine;
|
||||||
|
|
@ -208,7 +199,8 @@ in
|
||||||
DynamicUser = true;
|
DynamicUser = true;
|
||||||
# Each classification's scratch directory, gone with the unit.
|
# Each classification's scratch directory, gone with the unit.
|
||||||
PrivateTmp = true;
|
PrivateTmp = true;
|
||||||
LoadCredential = lib.mapAttrsToList (id: _: "${id}:${inContainer id}") storeFiles;
|
# Bare ids: the system credentials nspawn handed in.
|
||||||
|
LoadCredential = lib.attrNames storeFiles;
|
||||||
Restart = "on-failure";
|
Restart = "on-failure";
|
||||||
RestartSec = "10s";
|
RestartSec = "10s";
|
||||||
SyslogIdentifier = unit;
|
SyslogIdentifier = unit;
|
||||||
|
|
|
||||||
|
|
@ -311,9 +311,10 @@ let
|
||||||
&& withClassifier.containers ? swarm-review-classifier;
|
&& withClassifier.containers ? swarm-review-classifier;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# The daemon holds its store leaf and nothing else from the host; the
|
# The daemon holds its store leaf, handed in by nspawn without a copy on
|
||||||
# key is read per run and reaches only `claude`'s environment. Reading
|
# disk, and nothing else from the host; the key is read per run and
|
||||||
# the claude path also evaluates the container's unfree allowance.
|
# reaches only `claude`'s environment. Reading the claude path also
|
||||||
|
# evaluates the container's unfree allowance.
|
||||||
name = "the review classifier loads only its store leaf and carries no API key in its unit";
|
name = "the review classifier loads only its store leaf and carries no API key in its unit";
|
||||||
ok =
|
ok =
|
||||||
let
|
let
|
||||||
|
|
@ -321,12 +322,19 @@ let
|
||||||
withClassifier.containers.swarm-review-classifier.config.systemd.services.swarm-review-classifier;
|
withClassifier.containers.swarm-review-classifier.config.systemd.services.swarm-review-classifier;
|
||||||
env = svc.environment;
|
env = svc.environment;
|
||||||
in
|
in
|
||||||
lib.elem "bao-client.pem:/var/lib/swarm-review-classifier/bao-client.pem" svc.serviceConfig.LoadCredential
|
lib.elem "review-classifier-bao-key" svc.serviceConfig.LoadCredential
|
||||||
&& env.BAO_CLIENT_CERT == "/run/credentials/swarm-review-classifier.service/bao-client.pem"
|
&&
|
||||||
|
env.BAO_CLIENT_KEY == "/run/credentials/swarm-review-classifier.service/review-classifier-bao-key"
|
||||||
&& env.SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID == "swarm-review-classifier"
|
&& env.SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID == "swarm-review-classifier"
|
||||||
&& lib.hasSuffix "/bin/claude" env.SWARM_REVIEW_CLASSIFIER_CLAUDE
|
&& lib.hasSuffix "/bin/claude" env.SWARM_REVIEW_CLASSIFIER_CLAUDE
|
||||||
&& !(env ? ANTHROPIC_API_KEY)
|
&& !(env ? ANTHROPIC_API_KEY)
|
||||||
&& withClassifier.systemd.services ? swarm-review-classifier-identity;
|
&& lib.elem "--load-credential=review-classifier-bao-key:/var/lib/swarm-bao-pki/review-classifier-key.pem" (
|
||||||
|
withClassifier.containers.swarm-review-classifier.extraFlags
|
||||||
|
)
|
||||||
|
# No host unit copies the leaf into the container's root.
|
||||||
|
&& !(lib.any (u: lib.hasInfix "nixos-containers/swarm-review-classifier" (u.script or "")) (
|
||||||
|
lib.attrValues withClassifier.systemd.services
|
||||||
|
));
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# The collector reaches these routes through the gateway now, so each
|
# The collector reaches these routes through the gateway now, so each
|
||||||
|
|
|
||||||
|
|
@ -26,7 +26,8 @@ forge and holds no forge credential.
|
||||||
|
|
||||||
## Credentials
|
## Credentials
|
||||||
|
|
||||||
The container holds one file: its own store certificate. With it the service
|
The container holds one credential: its own store certificate, which nspawn
|
||||||
|
hands in at start without a copy on disk. With it the service
|
||||||
reads its queue client secret at start and the API key at every run, so a key
|
reads its queue client secret at start and the API key at every run, so a key
|
||||||
written or removed in the store takes effect on the next run. Its store role
|
written or removed in the store takes effect on the next run. Its store role
|
||||||
reads exactly those two paths.
|
reads exactly those two paths.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue