a2a94f63c8
Each container now runs the journal forwarder the queue's container runs, pushing to a receiver of its own on the swarm collector as the service's existing OIDC client, with the client's secret read inside the container by the fetch unit. Their store identities are system credentials, so the fetch unit loads them too. The three containers drop --link-journal=host, which hid their journals from the forwarder, and the host collector stops reading them.
277 lines
12 KiB
Nix
277 lines
12 KiB
Nix
# The review classifier, in a `swarm-review-classifier` container on the host
|
|
# that sets `deploy.review-classifier.enable`, which `deploy.allSwarmServices`
|
|
# defaults on where the host has the classifier's store identity. It answers the
|
|
# swarm UI's review try-outs: swarm-controller sends a PR and the operator's classifications over
|
|
# the queue, and the daemon runs `claude`, or `opencode` on the ACP runtime,
|
|
# once per classification and replies.
|
|
# docs/swarm/review-classifier.md has the design.
|
|
#
|
|
# ⚠️ Its own network namespace on the bridge, as ./swarm-ci-runner.nix has: a
|
|
# PR's diff and commit messages are model input, so the container reaches the
|
|
# swarm services by name and nothing on the host's loopback.
|
|
#
|
|
# The container holds one credential, its own store leaf, which nspawn hands in
|
|
# as a system credential with no copy on disk. The daemon reads everything else
|
|
# from the store with it: its queue client secret at start and the run's
|
|
# runtime's key on every run.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
|
|
# The `claude-code` the agents run, so this container does not trail them on
|
|
# the CLI; `fallback`, the container's own nixpkgs', only where the hive pins
|
|
# none.
|
|
claudeCode =
|
|
fallback:
|
|
if hyperhiveCfg.c0re.claudeCodePackage != null then
|
|
hyperhiveCfg.c0re.claudeCodePackage
|
|
else
|
|
fallback;
|
|
cfg = hyperhiveCfg.deploy.review-classifier;
|
|
baoCfg = hyperhiveCfg.swarm.bao;
|
|
baoDeploy = hyperhiveCfg.deploy.bao;
|
|
natsCfg = hyperhiveCfg.swarm.nats;
|
|
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
|
gatewayCfg = hyperhiveCfg.gateway;
|
|
networkCfg = hyperhiveCfg.network;
|
|
otelCfg = hyperhiveCfg.otel;
|
|
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
|
|
|
|
machine = "swarm-review-classifier";
|
|
unit = "swarm-review-classifier";
|
|
clientId = hyperhiveCfg.swarm.review-classifier.clientId;
|
|
|
|
# The container's journal forwarder pushes to its own receiver on the swarm
|
|
# collector, the route `serviceProducers.swarm-review-classifier` in
|
|
# ./swarm-otel-service.nix declares, by the collector's gateway name. The
|
|
# port suffix is elided on 443, as in ./swarm-bao.nix.
|
|
swarmOtelCfg = hyperhiveCfg.swarm.otel;
|
|
gatewayPortSuffix = lib.optionalString (
|
|
gatewayCfg.httpsPort != 443
|
|
) ":${toString gatewayCfg.httpsPort}";
|
|
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
|
|
forwarderUnit = "swarm-journal-forwarder-oidc";
|
|
# Loopback in the container's own netns, and distinct from the other service
|
|
# containers' forwarders all the same.
|
|
forwarderTelemetryPort = 8894;
|
|
|
|
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
|
# in self-signed mode their leaves chain to the hive CA.
|
|
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
caBundleModule = caTrust.trustBundle {
|
|
inherit pkgs;
|
|
name = machine;
|
|
consumers = [
|
|
unit
|
|
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
|
|
# collector's and authelia's names.
|
|
"opentelemetry-collector"
|
|
];
|
|
};
|
|
|
|
haveClientIdentity =
|
|
baoDeploy.reviewClassifierClientCertFile != null && baoDeploy.reviewClassifierClientKeyFile != null;
|
|
|
|
# Credential id → host source. nspawn reads each source as root when the
|
|
# container starts and hands it in as a system credential, held in memory;
|
|
# the daemon's unit alone loads them. No copy lands on disk, as for an
|
|
# agent's own leaf (`nix/agent-modules/bao.nix`). Ids carry no dot, the
|
|
# spelling `--load-credential` is safe with.
|
|
storeFiles = lib.filterAttrs (_: source: source != null) {
|
|
review-classifier-bao-cert = baoDeploy.reviewClassifierClientCertFile;
|
|
review-classifier-bao-key = baoDeploy.reviewClassifierClientKeyFile;
|
|
review-classifier-bao-server-ca = baoDeploy.serverCaFile;
|
|
};
|
|
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
|
# one word.
|
|
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
|
credential = id: "/run/credentials/${unit}.service/${id}";
|
|
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
|
|
|
# What `claude` exports its metrics with, and the daemon its ACP runs' cost,
|
|
# where this host runs a hive-tier collector on the bridge. The daemon adds
|
|
# the repo and the PR to each `claude` run's resource attributes, and to
|
|
# each cost record's attributes.
|
|
otelEnv = lib.optionalAttrs otelCfg.enable {
|
|
CLAUDE_CODE_ENABLE_TELEMETRY = "1";
|
|
OTEL_METRICS_EXPORTER = "otlp";
|
|
OTEL_EXPORTER_OTLP_ENDPOINT = "http://${networkCfg.bridgeIp}:${toString otelCfg.collector.port}";
|
|
OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf";
|
|
# Claude Code defaults to delta, which the metrics store drops.
|
|
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
|
|
OTEL_RESOURCE_ATTRIBUTES = "service.name=hyperhive-review-classifier,swarm=${
|
|
if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name
|
|
}";
|
|
};
|
|
|
|
privateNetwork = true;
|
|
in
|
|
{
|
|
options.services.hyperhive.swarm.review-classifier.clientId = lib.mkOption {
|
|
type = lib.types.str;
|
|
readOnly = true;
|
|
default = "swarm-review-classifier";
|
|
description = ''
|
|
The OAuth2 client id the review classifier presents to the swarm queue.
|
|
Read-only: the host running authelia registers it, the queue's
|
|
auth-callout responder grants it the try-out subject, and the secret
|
|
publisher carries its secret to the store, all from this one value.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.review-classifier.package = lib.mkOption {
|
|
type = lib.types.package;
|
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-review-classifier";
|
|
description = "The review classifier daemon package.";
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = haveClientIdentity;
|
|
message = ''
|
|
services.hyperhive.deploy.review-classifier.enable requires the
|
|
classifier's swarm-secret-store client identity: set both
|
|
|
|
services.hyperhive.deploy.bao.reviewClassifierClientCertFile
|
|
services.hyperhive.deploy.bao.reviewClassifierClientKeyFile
|
|
|
|
The classifier reads its queue client secret and its runtime keys with it.
|
|
|
|
Both default to the pair glue-bao-tls.nix mints when
|
|
services.hyperhive.deploy.bao.enable is true (the host that runs the
|
|
store). Otherwise copy review-classifier.pem and
|
|
review-classifier-key.pem from the store host's /var/lib/swarm-bao-pki/
|
|
and name them in those two options.
|
|
'';
|
|
}
|
|
{
|
|
assertion = lib.all oneWord (lib.attrValues storeFiles);
|
|
message = ''
|
|
The review classifier's store identity paths must not contain
|
|
whitespace: nspawn's --load-credential receives them through
|
|
nixos-container's unquoted extra flags.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The container hangs off the bridge, takes its address from the gateway's
|
|
# DHCP and resolves through its dnsmasq.
|
|
services.hyperhive.network.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# nspawn fails to start the container when a credential source is
|
|
# missing, so where this host mints the leaf the container pulls the
|
|
# minting unit in and waits for it. Ordering alone is not enough: the
|
|
# container starts from `machines.target`, outside the transaction that
|
|
# runs the minting unit from `multi-user.target`. Ignored elsewhere, where
|
|
# the unit does not exist.
|
|
systemd.services."container@${machine}" = lib.mkMerge [
|
|
caTrust.containerOrdering
|
|
{
|
|
after = [ "swarm-bao-pki.service" ];
|
|
wants = [ "swarm-bao-pki.service" ];
|
|
}
|
|
];
|
|
|
|
containers.${machine} = {
|
|
autoStart = true;
|
|
# The daemon keeps no state; its scratch directories are under PrivateTmp.
|
|
ephemeral = false;
|
|
# No `--link-journal=host`: the journal stays in the guest, where the
|
|
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
|
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
|
|
inherit privateNetwork;
|
|
hostBridge = networkCfg.bridgeName;
|
|
bindMounts = caTrust.bindMount;
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
imports = [
|
|
../container-modules/swarm-container.nix
|
|
../container-modules/swarm-journal-forwarder.nix
|
|
caBundleModule
|
|
];
|
|
|
|
services.hyperhive.swarmContainer = {
|
|
inherit privateNetwork;
|
|
writesOwnResolvConf = false;
|
|
};
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
networking.interfaces.eth0.useDHCP = true;
|
|
|
|
# `claude-code` is unfree, and this container evaluates its own
|
|
# nixpkgs, so the host's allowance does not reach here.
|
|
nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "claude-code" ];
|
|
|
|
# Authenticated as the daemon's own OIDC client, whose secret its
|
|
# store identity already reads: one identity for the whole container.
|
|
# That identity is a system credential, so the fetch unit loads it
|
|
# too; the daemon's `LoadCredential` below is that unit's alone.
|
|
services.hyperhive.swarmJournalForwarder = {
|
|
enable = true;
|
|
serviceName = "review-classifier";
|
|
endpoint = forwarderEndpoint;
|
|
inherit clientId;
|
|
tokenUrl = "${autheliaCfg.url}/api/oidc/token";
|
|
telemetryPort = forwarderTelemetryPort;
|
|
store = {
|
|
package = baoDeploy.package;
|
|
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
secretPath = "secret/swarm/services/${clientId}/oidc/client";
|
|
clientCertFile = forwarderCredential "review-classifier-bao-cert";
|
|
clientKeyFile = forwarderCredential "review-classifier-bao-key";
|
|
caFile =
|
|
if storeFiles ? review-classifier-bao-server-ca then
|
|
forwarderCredential "review-classifier-bao-server-ca"
|
|
else
|
|
null;
|
|
};
|
|
};
|
|
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
|
|
|
|
systemd.services.${unit} = {
|
|
description = "answer the swarm UI's review try-outs";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "network-online.target" ];
|
|
after = [ "network-online.target" ];
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = credential "review-classifier-bao-cert";
|
|
BAO_CLIENT_KEY = credential "review-classifier-bao-key";
|
|
SWARM_REVIEW_CLASSIFIER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
|
|
SWARM_REVIEW_CLASSIFIER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
|
|
SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID = clientId;
|
|
SWARM_REVIEW_CLASSIFIER_CLAUDE = "${claudeCode pkgs.claude-code}/bin/claude";
|
|
SWARM_REVIEW_CLASSIFIER_OPENCODE = "${pkgs.opencode}/bin/opencode";
|
|
}
|
|
// lib.optionalAttrs (storeFiles ? review-classifier-bao-server-ca) {
|
|
BAO_CACERT = credential "review-classifier-bao-server-ca";
|
|
}
|
|
// lib.optionalAttrs caTrust.useSelfSigned {
|
|
SWARM_REVIEW_CLASSIFIER_OIDC_CA_FILE = caTrust.bundlePathFor machine;
|
|
}
|
|
// otelEnv;
|
|
serviceConfig = {
|
|
ExecStart = "${cfg.package}/bin/swarm-review-classifier";
|
|
DynamicUser = true;
|
|
# Each classification's scratch directory, gone with the unit.
|
|
PrivateTmp = true;
|
|
# Bare ids: the system credentials nspawn handed in.
|
|
LoadCredential = lib.attrNames storeFiles;
|
|
Restart = "on-failure";
|
|
RestartSec = "10s";
|
|
SyslogIdentifier = unit;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|