fc32f57481
Five host options whose values hive-c0re renders into every agent's build move under services.hyperhive.swarm, so the swarm states each of them once instead of every hive carrying its own copy: c0re.dashboardPort -> swarm.dashboardPort c0re.claudeCodePackage -> swarm.claudeCodePackage github.enable -> swarm.github.enable otel.collector.port -> swarm.hiveOtel.collectorPort otel.extraResourceAttributes -> swarm.hiveOtel.extraResourceAttributes Every old path is a mkRenamedOptionModule alias, so a host that sets one keeps its value and gets an eval warning naming the new path. Types and defaults are unchanged, so the rendered meta flake, serve.json and hive-c0re's environment are the same as before on a host that sets none of them. The readers move with them: serve.json and its gc-root assertion, the hive-c0re environment, the hive collector's listen address and bridge port, the swarm collector's port-collision assertion, and the claude-login and review-classifier containers' claude pin. The swarm-services-switch fixture sets the swarm path and now also checks that serve.json carries the pinned claude. Part of #4857 (slice 2).
277 lines
12 KiB
Nix
277 lines
12 KiB
Nix
# The review classifier, in a `swarm-review-classifier` container on the host
|
|
# that sets `deploy.review-classifier.enable`, which `deploy.allSwarmServices`
|
|
# defaults on where the host has the classifier's store identity. It answers the
|
|
# swarm UI's review try-outs: swarm-controller sends a PR and the operator's classifications over
|
|
# the queue, and the daemon runs `claude`, or `opencode` on the ACP runtime,
|
|
# once per classification and replies.
|
|
# docs/swarm/review-classifier.md has the design.
|
|
#
|
|
# ⚠️ Its own network namespace on the bridge, as ./swarm-ci-runner.nix has: a
|
|
# PR's diff and commit messages are model input, so the container reaches the
|
|
# swarm services by name and nothing on the host's loopback.
|
|
#
|
|
# The container holds one credential, its own store leaf, which nspawn hands in
|
|
# as a system credential with no copy on disk. The daemon reads everything else
|
|
# from the store with it: its queue client secret at start and the run's
|
|
# runtime's key on every run.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
|
|
# The `claude-code` the agents run, so this container does not trail them on
|
|
# the CLI; `fallback`, the container's own nixpkgs', only where the swarm pins
|
|
# none.
|
|
claudeCode =
|
|
fallback:
|
|
if hyperhiveCfg.swarm.claudeCodePackage != null then
|
|
hyperhiveCfg.swarm.claudeCodePackage
|
|
else
|
|
fallback;
|
|
cfg = hyperhiveCfg.deploy.review-classifier;
|
|
baoCfg = hyperhiveCfg.swarm.bao;
|
|
baoDeploy = hyperhiveCfg.deploy.bao;
|
|
natsCfg = hyperhiveCfg.swarm.nats;
|
|
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
|
gatewayCfg = hyperhiveCfg.gateway;
|
|
networkCfg = hyperhiveCfg.network;
|
|
otelCfg = hyperhiveCfg.otel;
|
|
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
|
|
|
|
machine = "swarm-review-classifier";
|
|
unit = "swarm-review-classifier";
|
|
clientId = hyperhiveCfg.swarm.review-classifier.clientId;
|
|
|
|
# The container's journal forwarder pushes to its own receiver on the swarm
|
|
# collector, the route `serviceProducers.swarm-review-classifier` in
|
|
# ./swarm-otel-service.nix declares, by the collector's gateway name. The
|
|
# port suffix is elided on 443, as in ./swarm-bao.nix.
|
|
swarmOtelCfg = hyperhiveCfg.swarm.otel;
|
|
gatewayPortSuffix = lib.optionalString (
|
|
gatewayCfg.httpsPort != 443
|
|
) ":${toString gatewayCfg.httpsPort}";
|
|
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
|
|
forwarderUnit = "swarm-journal-forwarder-oidc";
|
|
# Loopback in the container's own netns, and distinct from the other service
|
|
# containers' forwarders all the same.
|
|
forwarderTelemetryPort = 8894;
|
|
|
|
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
|
# in self-signed mode their leaves chain to the hive CA.
|
|
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
caBundleModule = caTrust.trustBundle {
|
|
inherit pkgs;
|
|
name = machine;
|
|
consumers = [
|
|
unit
|
|
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
|
|
# collector's and authelia's names.
|
|
"opentelemetry-collector"
|
|
];
|
|
};
|
|
|
|
haveClientIdentity =
|
|
baoDeploy.reviewClassifierClientCertFile != null && baoDeploy.reviewClassifierClientKeyFile != null;
|
|
|
|
# Credential id → host source. nspawn reads each source as root when the
|
|
# container starts and hands it in as a system credential, held in memory;
|
|
# the daemon's unit alone loads them. No copy lands on disk, as for an
|
|
# agent's own leaf (`nix/agent-modules/bao.nix`). Ids carry no dot, the
|
|
# spelling `--load-credential` is safe with.
|
|
storeFiles = lib.filterAttrs (_: source: source != null) {
|
|
review-classifier-bao-cert = baoDeploy.reviewClassifierClientCertFile;
|
|
review-classifier-bao-key = baoDeploy.reviewClassifierClientKeyFile;
|
|
review-classifier-bao-server-ca = baoDeploy.serverCaFile;
|
|
};
|
|
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
|
# one word.
|
|
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
|
credential = id: "/run/credentials/${unit}.service/${id}";
|
|
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
|
|
|
# What `claude` exports its metrics with, and the daemon its ACP runs' cost,
|
|
# where this host runs a hive-tier collector on the bridge. The daemon adds
|
|
# the repo and the PR to each `claude` run's resource attributes, and to
|
|
# each cost record's attributes.
|
|
otelEnv = lib.optionalAttrs otelCfg.enable {
|
|
CLAUDE_CODE_ENABLE_TELEMETRY = "1";
|
|
OTEL_METRICS_EXPORTER = "otlp";
|
|
OTEL_EXPORTER_OTLP_ENDPOINT = "http://${networkCfg.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}";
|
|
OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf";
|
|
# Claude Code defaults to delta, which the metrics store drops.
|
|
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
|
|
OTEL_RESOURCE_ATTRIBUTES = "service.name=hyperhive-review-classifier,swarm=${
|
|
if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name
|
|
}";
|
|
};
|
|
|
|
privateNetwork = true;
|
|
in
|
|
{
|
|
options.services.hyperhive.swarm.review-classifier.clientId = lib.mkOption {
|
|
type = lib.types.str;
|
|
readOnly = true;
|
|
default = "swarm-review-classifier";
|
|
description = ''
|
|
The OAuth2 client id the review classifier presents to the swarm queue.
|
|
Read-only: the host running authelia registers it, the queue's
|
|
auth-callout responder grants it the try-out subject, and the secret
|
|
publisher carries its secret to the store, all from this one value.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.review-classifier.package = lib.mkOption {
|
|
type = lib.types.package;
|
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-review-classifier";
|
|
description = "The review classifier daemon package.";
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = haveClientIdentity;
|
|
message = ''
|
|
services.hyperhive.deploy.review-classifier.enable requires the
|
|
classifier's swarm-secret-store client identity: set both
|
|
|
|
services.hyperhive.deploy.bao.reviewClassifierClientCertFile
|
|
services.hyperhive.deploy.bao.reviewClassifierClientKeyFile
|
|
|
|
The classifier reads its queue client secret and its runtime keys with it.
|
|
|
|
Both default to the pair glue-bao-tls.nix mints when
|
|
services.hyperhive.deploy.bao.enable is true (the host that runs the
|
|
store). Otherwise copy review-classifier.pem and
|
|
review-classifier-key.pem from the store host's /var/lib/swarm-bao-pki/
|
|
and name them in those two options.
|
|
'';
|
|
}
|
|
{
|
|
assertion = lib.all oneWord (lib.attrValues storeFiles);
|
|
message = ''
|
|
The review classifier's store identity paths must not contain
|
|
whitespace: nspawn's --load-credential receives them through
|
|
nixos-container's unquoted extra flags.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The container hangs off the bridge, takes its address from the gateway's
|
|
# DHCP and resolves through its dnsmasq.
|
|
services.hyperhive.network.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# nspawn fails to start the container when a credential source is
|
|
# missing, so where this host mints the leaf the container pulls the
|
|
# minting unit in and waits for it. Ordering alone is not enough: the
|
|
# container starts from `machines.target`, outside the transaction that
|
|
# runs the minting unit from `multi-user.target`. Ignored elsewhere, where
|
|
# the unit does not exist.
|
|
systemd.services."container@${machine}" = lib.mkMerge [
|
|
caTrust.containerOrdering
|
|
{
|
|
after = [ "swarm-bao-pki.service" ];
|
|
wants = [ "swarm-bao-pki.service" ];
|
|
}
|
|
];
|
|
|
|
containers.${machine} = {
|
|
autoStart = true;
|
|
# The daemon keeps no state; its scratch directories are under PrivateTmp.
|
|
ephemeral = false;
|
|
# No `--link-journal=host`: the journal stays in the guest, where the
|
|
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
|
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
|
|
inherit privateNetwork;
|
|
hostBridge = networkCfg.bridgeName;
|
|
bindMounts = caTrust.bindMount;
|
|
|
|
config =
|
|
{ pkgs, lib, ... }:
|
|
{
|
|
imports = [
|
|
../container-modules/swarm-container.nix
|
|
../container-modules/swarm-journal-forwarder.nix
|
|
caBundleModule
|
|
];
|
|
|
|
services.hyperhive.swarmContainer = {
|
|
inherit privateNetwork;
|
|
writesOwnResolvConf = false;
|
|
};
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
networking.interfaces.eth0.useDHCP = true;
|
|
|
|
# `claude-code` is unfree, and this container evaluates its own
|
|
# nixpkgs, so the host's allowance does not reach here.
|
|
nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "claude-code" ];
|
|
|
|
# Authenticated as the daemon's own OIDC client, whose secret its
|
|
# store identity already reads: one identity for the whole container.
|
|
# That identity is a system credential, so the fetch unit loads it
|
|
# too; the daemon's `LoadCredential` below is that unit's alone.
|
|
services.hyperhive.swarmJournalForwarder = {
|
|
enable = true;
|
|
serviceName = "review-classifier";
|
|
endpoint = forwarderEndpoint;
|
|
inherit clientId;
|
|
tokenUrl = "${autheliaCfg.url}/api/oidc/token";
|
|
telemetryPort = forwarderTelemetryPort;
|
|
store = {
|
|
package = baoDeploy.package;
|
|
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
secretPath = "secret/swarm/services/${clientId}/oidc/client";
|
|
clientCertFile = forwarderCredential "review-classifier-bao-cert";
|
|
clientKeyFile = forwarderCredential "review-classifier-bao-key";
|
|
caFile =
|
|
if storeFiles ? review-classifier-bao-server-ca then
|
|
forwarderCredential "review-classifier-bao-server-ca"
|
|
else
|
|
null;
|
|
};
|
|
};
|
|
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
|
|
|
|
systemd.services.${unit} = {
|
|
description = "answer the swarm UI's review try-outs";
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = [ "network-online.target" ];
|
|
after = [ "network-online.target" ];
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = credential "review-classifier-bao-cert";
|
|
BAO_CLIENT_KEY = credential "review-classifier-bao-key";
|
|
SWARM_REVIEW_CLASSIFIER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
|
|
SWARM_REVIEW_CLASSIFIER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
|
|
SWARM_REVIEW_CLASSIFIER_OIDC_CLIENT_ID = clientId;
|
|
SWARM_REVIEW_CLASSIFIER_CLAUDE = "${claudeCode pkgs.claude-code}/bin/claude";
|
|
SWARM_REVIEW_CLASSIFIER_OPENCODE = "${pkgs.opencode}/bin/opencode";
|
|
}
|
|
// lib.optionalAttrs (storeFiles ? review-classifier-bao-server-ca) {
|
|
BAO_CACERT = credential "review-classifier-bao-server-ca";
|
|
}
|
|
// lib.optionalAttrs caTrust.useSelfSigned {
|
|
SWARM_REVIEW_CLASSIFIER_OIDC_CA_FILE = caTrust.bundlePathFor machine;
|
|
}
|
|
// otelEnv;
|
|
serviceConfig = {
|
|
ExecStart = "${cfg.package}/bin/swarm-review-classifier";
|
|
DynamicUser = true;
|
|
# Each classification's scratch directory, gone with the unit.
|
|
PrivateTmp = true;
|
|
# Bare ids: the system credentials nspawn handed in.
|
|
LoadCredential = lib.attrNames storeFiles;
|
|
Restart = "on-failure";
|
|
RestartSec = "10s";
|
|
SyslogIdentifier = unit;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|