nix: run the review classifier in a container of its own when a host enables it
deploy.review-classifier.enable (off by default, not derived from allSwarmServices) puts swarm-review-classifier in its own network namespace on the bridge, with claude-code and the daemon. A host unit copies the classifier's store leaf into the container; the daemon reads its queue client secret and the API key from the store with it. - glue-bao-tls mints the review-classifier leaf; swarm-bao writes the swarm-review-classifier role, reading exactly the API key path and the classifier's queue client secret; swarm.nix reserves its CN. - glue-review-classifier-oidc-client registers the machine client where authelia runs, and the publisher carries its secret to the store. - swarm-nats passes the client id to the responder as --review-classifier-client. - Where the host runs a hive-tier collector, claude exports its metrics there; the daemon adds the repo and the PR to the resource attributes. - module-eval: the new grant and leaf (bao-grants, now sixteen granting units), the container's presence and its unit (core-toggle), and the client registration and publishing (secret-publisher).
This commit is contained in:
parent
d70ed41537
commit
5d8ab675cd
14 changed files with 435 additions and 8 deletions
|
|
@ -39,6 +39,7 @@
|
|||
./glue-nats-bao-identity.nix
|
||||
./glue-nats-oidc-client.nix
|
||||
./glue-queue-agent-credential.nix
|
||||
./glue-review-classifier-oidc-client.nix
|
||||
./glue-secret-publisher-bao-identity.nix
|
||||
./glue-swarm-bao-otel-oidc-client.nix
|
||||
./glue-swarm-otel-oidc-client.nix
|
||||
|
|
@ -47,6 +48,7 @@
|
|||
./swarm-bao-service.nix
|
||||
./swarm-bao.nix
|
||||
./swarm-ci-runner.nix
|
||||
./swarm-review-classifier.nix
|
||||
./swarm-secret-publisher.nix
|
||||
./swarm-nats-service.nix
|
||||
./swarm-nats.nix
|
||||
|
|
|
|||
|
|
@ -642,6 +642,23 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
review-classifier.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
example = true;
|
||||
description = ''
|
||||
Run the review classifier in a `swarm-review-classifier` container on
|
||||
this host. It answers the swarm UI's review try-outs with `claude` on
|
||||
the Anthropic API key the operator writes to the secret store at
|
||||
`secret/swarm/services/review-classifier/anthropic-api-key`, field
|
||||
`api_key`.
|
||||
|
||||
Not derived from
|
||||
{option}`services.hyperhive.deploy.allSwarmServices`: every run spends
|
||||
money on that key, so running one is a decision of its own.
|
||||
'';
|
||||
};
|
||||
|
||||
hive-controller.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
|
|
|
|||
|
|
@ -107,6 +107,8 @@ in
|
|||
forwarderOidcClientKeyFile = lib.mkDefault "${pkiDir}/forwarder-oidc-key.pem";
|
||||
ciRunnerClientCertFile = lib.mkDefault "${pkiDir}/ci-runner.pem";
|
||||
ciRunnerClientKeyFile = lib.mkDefault "${pkiDir}/ci-runner-key.pem";
|
||||
reviewClassifierClientCertFile = lib.mkDefault "${pkiDir}/review-classifier.pem";
|
||||
reviewClassifierClientKeyFile = lib.mkDefault "${pkiDir}/review-classifier-key.pem";
|
||||
forgePublisherClientCertFile = lib.mkDefault "${pkiDir}/forge-publisher.pem";
|
||||
forgePublisherClientKeyFile = lib.mkDefault "${pkiDir}/forge-publisher-key.pem";
|
||||
servicesIssuerClientCertFile = lib.mkDefault "${pkiDir}/services-issuer.pem";
|
||||
|
|
@ -268,6 +270,13 @@ in
|
|||
[ -s ${pkiDir}/ci-runner.pem ] || ${signLeaf} ${pkiDir} ci-runner \
|
||||
${lib.escapeShellArg deployCfg.bao.ciRunnerCommonName} "" clientAuth
|
||||
|
||||
# The review classifier's, which reads its API key and queue client
|
||||
# secret. Minted here for the matrix-ctl leaf's reason: the classifier
|
||||
# is a swarm service, so on a classifier host without the store this is
|
||||
# the file an operator copies.
|
||||
[ -s ${pkiDir}/review-classifier.pem ] || ${signLeaf} ${pkiDir} review-classifier \
|
||||
${lib.escapeShellArg deployCfg.bao.reviewClassifierCommonName} "" clientAuth
|
||||
|
||||
# The forge's token publisher. Minted here for the matrix-ctl leaf's
|
||||
# reason: the forge is a swarm singleton, so on a forge host without the
|
||||
# store this is the file an operator copies.
|
||||
|
|
|
|||
35
nix/host-modules/glue-review-classifier-oidc-client.nix
Normal file
35
nix/host-modules/glue-review-classifier-oidc-client.nix
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
# Glue: register the review classifier as an OIDC client wherever authelia
|
||||
# runs.
|
||||
#
|
||||
# ONE PAIRING PER FILE — swarm-review-classifier ← authelia, and nothing else.
|
||||
# Deleting this leaves a classifier whose token requests authelia answers with
|
||||
# a 401, and no minted secret for swarm-secret-publish to put in the store.
|
||||
#
|
||||
# ⚠️ Gated on authelia being HERE, and NOT on this host running the
|
||||
# classifier, for ./glue-controller-oidc-client.nix's reason: a client is a row
|
||||
# in THIS host's provider config. Registered whether or not any host enables
|
||||
# the classifier: nothing in `swarm.*` records that, so a swarm without one
|
||||
# carries one unused client and one unused minted secret.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
deployCfg = hyperhiveCfg.deploy;
|
||||
clientId = hyperhiveCfg.swarm.review-classifier.clientId;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf deployCfg.authelia.enable {
|
||||
services.hyperhive.swarm.authelia.oidc.clients = [
|
||||
{
|
||||
id = clientId;
|
||||
description = "HyperHive review classifier";
|
||||
# `client_credentials`: a daemon authenticating as itself.
|
||||
kind = "machine";
|
||||
audience = [ clientId ];
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
@ -932,6 +932,19 @@ let
|
|||
}
|
||||
];
|
||||
|
||||
# The review classifier: the API key the operator writes, read once per run,
|
||||
# and its own queue client secret, read at start. Both by the daemon inside
|
||||
# its container.
|
||||
reviewClassifierReaders = [
|
||||
{
|
||||
name = "swarm-review-classifier";
|
||||
cn = baoDeploy.reviewClassifierCommonName;
|
||||
policyText =
|
||||
readStanza "${credentialMountPath}/data/swarm/services/review-classifier/anthropic-api-key"
|
||||
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.review-classifier.clientId}/oidc/client";
|
||||
}
|
||||
];
|
||||
|
||||
# The forge host's one store identity. `swarm-bao-forge-publisher` writes
|
||||
# the controller's forge token under it, the one writer of that path and
|
||||
# write-only: it copies a file in and never reads one back.
|
||||
|
|
@ -1754,6 +1767,20 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
reviewClassifierCommonName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "swarm-review-classifier";
|
||||
description = ''
|
||||
Subject the store's `swarm-review-classifier` cert-auth role accepts:
|
||||
the identity the review classifier presents from inside its
|
||||
container. Its grant is two paths, read only:
|
||||
`swarm/services/review-classifier/anthropic-api-key` and its own
|
||||
queue client secret.
|
||||
|
||||
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
|
||||
'';
|
||||
};
|
||||
|
||||
forgePublisherCommonName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "swarm-forge-publisher";
|
||||
|
|
@ -2003,6 +2030,32 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
reviewClassifierClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-bao-pki/review-classifier.pem";
|
||||
description = ''
|
||||
Certificate the review classifier presents to the store. Its subject
|
||||
must be
|
||||
{option}`services.hyperhive.deploy.bao.reviewClassifierCommonName`.
|
||||
|
||||
./glue-bao-tls.nix points it at the leaf it mints on the store's host.
|
||||
On a classifier host without the store, it is a file the operator
|
||||
copies from the store's host. A path, never a value.
|
||||
'';
|
||||
};
|
||||
|
||||
reviewClassifierClientKeyFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-bao-pki/review-classifier-key.pem";
|
||||
description = ''
|
||||
Private key for
|
||||
{option}`services.hyperhive.deploy.bao.reviewClassifierClientCertFile`.
|
||||
A path, never a value.
|
||||
'';
|
||||
};
|
||||
|
||||
forgePublisherClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
|
|
@ -3074,6 +3127,8 @@ in
|
|||
|
||||
systemd.services.swarm-bao-ci-runner-policy = readerPolicyUnit "write the CI runner's registration-token-reader bao policy and cert-auth role" ciRunnerReaders;
|
||||
|
||||
systemd.services.swarm-bao-review-classifier-policy = readerPolicyUnit "write the review classifier's API-key-reader bao policy and cert-auth role" reviewClassifierReaders;
|
||||
|
||||
systemd.services.swarm-bao-forge-publisher-policy = readerPolicyUnit "write the forge's controller-token-publisher bao policy and cert-auth role" forgePublisherWriters;
|
||||
|
||||
# A FOURTH sibling, same shape and same reasons as the two above. This
|
||||
|
|
|
|||
|
|
@ -708,6 +708,8 @@ in
|
|||
"--hive-client-prefix ${lib.escapeShellArg autheliaCfg.hiveClientPrefix}"
|
||||
"--agent-client-suffix ${lib.escapeShellArg autheliaCfg.agentClientSuffix}"
|
||||
"--reader-client ${lib.escapeShellArg controllerCfg.queueClientId}"
|
||||
# Hears try-outs and answers each by reply, and nothing else.
|
||||
"--review-classifier-client ${lib.escapeShellArg config.services.hyperhive.swarm.review-classifier.clientId}"
|
||||
# What an agent may publish to, keyed on its own name: its
|
||||
# terminal rows and its turn-state header
|
||||
# (`hive-agent::swarm_agent_state`), two subject families so a
|
||||
|
|
|
|||
220
nix/host-modules/swarm-review-classifier.nix
Normal file
220
nix/host-modules/swarm-review-classifier.nix
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
# The review classifier, in a `swarm-review-classifier` container on the host
|
||||
# that sets `deploy.review-classifier.enable`. It answers the swarm UI's review
|
||||
# try-outs: swarm-controller sends a PR and the operator's classifications over
|
||||
# the queue, and the daemon runs `claude` once per classification and replies.
|
||||
# docs/swarm/review-classifier.md has the design.
|
||||
#
|
||||
# ⚠️ Its own network namespace on the bridge, as ./swarm-ci-runner.nix has: a
|
||||
# PR's diff and commit messages are model input, so the container reaches the
|
||||
# swarm services by name and nothing on the host's loopback.
|
||||
#
|
||||
# The container holds one credential, its own store leaf, which the host unit
|
||||
# below copies in. The daemon reads everything else from the store with it:
|
||||
# its queue client secret at start and the API key on every run.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
cfg = hyperhiveCfg.deploy.review-classifier;
|
||||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
baoDeploy = hyperhiveCfg.deploy.bao;
|
||||
natsCfg = hyperhiveCfg.swarm.nats;
|
||||
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
||||
gatewayCfg = hyperhiveCfg.gateway;
|
||||
networkCfg = hyperhiveCfg.network;
|
||||
otelCfg = hyperhiveCfg.otel;
|
||||
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
|
||||
|
||||
machine = "swarm-review-classifier";
|
||||
unit = "swarm-review-classifier";
|
||||
clientId = hyperhiveCfg.swarm.review-classifier.clientId;
|
||||
|
||||
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
||||
# in self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
caBundleModule = caTrust.trustBundle {
|
||||
inherit pkgs;
|
||||
name = machine;
|
||||
consumers = [ unit ];
|
||||
};
|
||||
|
||||
haveClientIdentity =
|
||||
baoDeploy.reviewClassifierClientCertFile != null && baoDeploy.reviewClassifierClientKeyFile != null;
|
||||
|
||||
# Credential id → host source, copied into the container and loaded by the
|
||||
# daemon's unit alone.
|
||||
storeFiles = {
|
||||
"bao-client.pem" = baoDeploy.reviewClassifierClientCertFile;
|
||||
"bao-client-key.pem" = baoDeploy.reviewClassifierClientKeyFile;
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
"bao-ca.pem" = baoDeploy.serverCaFile;
|
||||
};
|
||||
secretDirInContainer = "/var/lib/swarm-review-classifier";
|
||||
inContainer = id: "${secretDirInContainer}/${id}";
|
||||
secretDir = "/var/lib/nixos-containers/${machine}${secretDirInContainer}";
|
||||
credential = id: "/run/credentials/${unit}.service/${id}";
|
||||
|
||||
# What `claude` exports its metrics with, where this host runs a hive-tier
|
||||
# collector on the bridge. The daemon adds the repo and the PR to the
|
||||
# resource attributes for each run.
|
||||
otelEnv = lib.optionalAttrs otelCfg.enable {
|
||||
CLAUDE_CODE_ENABLE_TELEMETRY = "1";
|
||||
OTEL_METRICS_EXPORTER = "otlp";
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = "http://${networkCfg.bridgeIp}:${toString otelCfg.collector.port}";
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf";
|
||||
# Claude Code defaults to delta, which the metrics store drops.
|
||||
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
|
||||
OTEL_RESOURCE_ATTRIBUTES = "service.name=hyperhive-review-classifier,swarm=${
|
||||
if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name
|
||||
}";
|
||||
};
|
||||
|
||||
privateNetwork = true;
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.swarm.review-classifier.clientId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "swarm-review-classifier";
|
||||
description = ''
|
||||
The OAuth2 client id the review classifier presents to the swarm queue.
|
||||
Read-only: the host running authelia registers it, the queue's
|
||||
auth-callout responder grants it the try-out subject, and the secret
|
||||
publisher carries its secret to the store, all from this one value.
|
||||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.review-classifier.package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-review-classifier";
|
||||
description = "The review classifier daemon package.";
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = haveClientIdentity;
|
||||
message = ''
|
||||
services.hyperhive.deploy.review-classifier.enable requires the
|
||||
classifier's swarm-secret-store client identity: set both
|
||||
|
||||
services.hyperhive.deploy.bao.reviewClassifierClientCertFile
|
||||
services.hyperhive.deploy.bao.reviewClassifierClientKeyFile
|
||||
|
||||
The classifier reads its queue client secret and the API key with it.
|
||||
|
||||
Both default to the pair glue-bao-tls.nix mints when
|
||||
services.hyperhive.deploy.bao.enable is true (the host that runs the
|
||||
store). Otherwise copy review-classifier.pem and
|
||||
review-classifier-key.pem from the store host's /var/lib/swarm-bao-pki/
|
||||
and name them in those two options.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
# The container hangs off the bridge, takes its address from the gateway's
|
||||
# DHCP and resolves through its dnsmasq.
|
||||
services.hyperhive.network.enable = lib.mkDefault true;
|
||||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||||
|
||||
# A copy rather than a bind mount: nixos-container refuses to start on a
|
||||
# missing bind source, and the leaf may not be minted yet.
|
||||
systemd.services.swarm-review-classifier-identity = {
|
||||
description = "deliver the review classifier's store identity into its container";
|
||||
before = [ "container@${machine}.service" ];
|
||||
wantedBy = [ "container@${machine}.service" ];
|
||||
# Ordering only: where the store's PKI is minted on this host, the leaf
|
||||
# is one of its files.
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
SyslogIdentifier = "swarm-review-classifier-identity";
|
||||
};
|
||||
path = [ pkgs.coreutils ];
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
install -d -m 0700 ${lib.escapeShellArg secretDir}
|
||||
''
|
||||
+ lib.concatStrings (
|
||||
lib.mapAttrsToList (id: source: ''
|
||||
install -m 0400 ${lib.escapeShellArg source} ${lib.escapeShellArg "${secretDir}/${id}"}
|
||||
'') storeFiles
|
||||
);
|
||||
};
|
||||
|
||||
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
||||
|
||||
containers.${machine} = {
|
||||
autoStart = true;
|
||||
# Not ephemeral: an ephemeral root lives under /run, where the identity
|
||||
# copy above does not write. The daemon itself keeps no state.
|
||||
ephemeral = false;
|
||||
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
||||
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
||||
extraFlags = [ "--link-journal=host" ];
|
||||
inherit privateNetwork;
|
||||
hostBridge = networkCfg.bridgeName;
|
||||
bindMounts = caTrust.bindMount;
|
||||
|
||||
config =
|
||||
{ pkgs, lib, ... }:
|
||||
{
|
||||
imports = [
|
||||
../container-modules/swarm-container.nix
|
||||
caBundleModule
|
||||
];
|
||||
|
||||
services.hyperhive.swarmContainer = {
|
||||
inherit privateNetwork;
|
||||
writesOwnResolvConf = false;
|
||||
};
|
||||
networking.nameservers = [ networkCfg.bridgeIp ];
|
||||
networking.interfaces.eth0.useDHCP = true;
|
||||
|
||||
# `claude-code` is unfree, and this container evaluates its own
|
||||
# nixpkgs, so the host's allowance does not reach here.
|
||||
nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "claude-code" ];
|
||||
|
||||
systemd.services.${unit} = {
|
||||
description = "answer the swarm UI's review try-outs";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = credential "bao-client.pem";
|
||||
BAO_CLIENT_KEY = credential "bao-client-key.pem";
|
||||
SWARM_REVIEW_CLASSIFIER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
|
||||
SWARM_REVIEW_CLASSIFIER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
|
||||
SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID = clientId;
|
||||
SWARM_REVIEW_CLASSIFIER_CLAUDE = "${pkgs.claude-code}/bin/claude";
|
||||
}
|
||||
// lib.optionalAttrs (storeFiles ? "bao-ca.pem") {
|
||||
BAO_CACERT = credential "bao-ca.pem";
|
||||
}
|
||||
// lib.optionalAttrs caTrust.useSelfSigned {
|
||||
SWARM_REVIEW_CLASSIFIER_OIDC_CA_FILE = caTrust.bundlePathFor machine;
|
||||
}
|
||||
// otelEnv;
|
||||
serviceConfig = {
|
||||
ExecStart = "${cfg.package}/bin/swarm-review-classifier";
|
||||
DynamicUser = true;
|
||||
# Each classification's scratch directory, gone with the unit.
|
||||
PrivateTmp = true;
|
||||
LoadCredential = lib.mapAttrsToList (id: _: "${id}:${inContainer id}") storeFiles;
|
||||
Restart = "on-failure";
|
||||
RestartSec = "10s";
|
||||
SyslogIdentifier = unit;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -90,6 +90,8 @@ let
|
|||
hyperhiveCfg.swarm.matrix.sso.clientId
|
||||
# The forge's login source, read by `swarm-bao-forge-oidc` on its host.
|
||||
hyperhiveCfg.swarm.forge.sso.clientId
|
||||
# The review classifier, which reads it itself inside its container.
|
||||
hyperhiveCfg.swarm.review-classifier.clientId
|
||||
];
|
||||
|
||||
# The swarm controller's OIDC client, which it reads back from the store
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ let
|
|||
deployCfg.bao.granterCommonName
|
||||
deployCfg.bao.natsAuthCommonName
|
||||
deployCfg.bao.ciRunnerCommonName
|
||||
deployCfg.bao.reviewClassifierCommonName
|
||||
deployCfg.bao.forgePublisherCommonName
|
||||
]
|
||||
# The per-hive subjects, spelled out per hive rather than as the prefix.
|
||||
|
|
|
|||
Loading…
Reference in a new issue