vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-controller reads its forge token from the store, not a host file

Browse source 
The controller's forge token never entered bao: the forge minted it in its
container, hive-forge-swarm-controller-token copied it to a host file, and the
controller LoadCredential'ed that file. forgeTokenFile defaulted to that path
only where the forge ran, so a controller on any other host had no forge
access unless an operator copied the file by hand.

Now the forge's host publishes it and the controller reads it:

- swarm-bao-forge-publisher (replaces hive-forge-swarm-controller-token)
  waits for the in-container token, logs in with its own forge-publisher leaf
  and writes it to swarm/controller/swarm-controller/forge/token. It runs on
  every forge container start, the old unit's trigger.
- New principal swarm-forge-publisher: leaf minted by glue-bao-tls.nix,
  policy create/update on that one path, written by
  swarm-bao-forge-publisher-policy, CN reserved as a hive name.
- The controller policy gains read on that path. swarm-controller reads it
  at start under its own certificate with the OIDC secret's retry, and runs
  without forge access if it still cannot.
- forgeTokenFile, its LoadCredential and the forgeEnv gate are gone;
  SWARM_CONTROLLER_FORGE_URL is always set. hostSwarmControllerTokenFile is
  removed, and a tmpfiles rule deletes the old host copy on the forge host.

The in-container file stays: the mint unit's only-once guard reads it, and
it is the publisher's source.

Refs #4998
This commit is contained in:
atlas 2026-10-08 13:23:17 +02:00
commit 121c4678aa
18 changed files with 580 additions and 277 deletions
Download patch file Download diff file Expand all files Collapse all files

20
nix/host-modules/deploy.nix
View file

@ -70,10 +70,12 @@ in
[ "services" "hyperhive" "swarm" "controller" "enable" ]
[ "services" "hyperhive" "deploy" "swarm-controller" "enable" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "controller" "forgeTokenFile" ]
[ "services" "hyperhive" "deploy" "swarm-controller" "forgeTokenFile" ]
)
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "controller" "forgeTokenFile" ] ''
The controller reads its forge token from the swarm's secret store, under
its own certificate, and swarm-bao-forge-publisher on the forge's host
puts it there. Remove this definition; the file it named is read by
nothing now and can be deleted.
'')
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "controller" "authBridgeUrl" ]
[ "services" "hyperhive" "deploy" "swarm-controller" "authBridgeUrl" ]
@ -231,9 +233,13 @@ in
[ "services" "hyperhive" "swarm" "forge" "sso" "clientSecretFile" ]
[ "services" "hyperhive" "deploy" "forgejo" "sso" "clientSecretFile" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "forge" "hostSwarmControllerTokenFile" ]
[ "services" "hyperhive" "deploy" "forgejo" "hostSwarmControllerTokenFile" ]
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "forge" "hostSwarmControllerTokenFile" ]
''
The forge publishes the swarm-controller's forge token to the swarm
secret store (swarm-bao-forge-publisher), and the controller reads it
there. Remove this definition; the file it named is read by nothing now
and can be deleted.
''
)
# The homeserver, split the same way. `swarm.matrix` keeps what it IS to

2
nix/host-modules/glue-bao-readers-policy-order.nix
View file

@ -48,6 +48,8 @@ let
swarm-bao-nats-tls = deployCfg.nats.enable;
# ./swarm-ci-runner.nix
swarm-bao-ci-runner = deployCfg.ci-runner.enable;
# ./hive-forge/default.nix
swarm-bao-forge-publisher = deployCfg.forgejo.enable;
};
# Readers that log in under another unit's principal: reader unit → the

8
nix/host-modules/glue-bao-tls.nix
View file

@ -107,6 +107,8 @@ in
forwarderOidcClientKeyFile = lib.mkDefault "${pkiDir}/forwarder-oidc-key.pem";
ciRunnerClientCertFile = lib.mkDefault "${pkiDir}/ci-runner.pem";
ciRunnerClientKeyFile = lib.mkDefault "${pkiDir}/ci-runner-key.pem";
forgePublisherClientCertFile = lib.mkDefault "${pkiDir}/forge-publisher.pem";
forgePublisherClientKeyFile = lib.mkDefault "${pkiDir}/forge-publisher-key.pem";
servicesIssuerClientCertFile = lib.mkDefault "${pkiDir}/services-issuer.pem";
servicesIssuerClientKeyFile = lib.mkDefault "${pkiDir}/services-issuer-key.pem";
@ -266,6 +268,12 @@ in
[ -s ${pkiDir}/ci-runner.pem ] || ${signLeaf} ${pkiDir} ci-runner \
${lib.escapeShellArg deployCfg.bao.ciRunnerCommonName} "" clientAuth
# The forge's token publisher. Minted here for the matrix-ctl leaf's
# reason: the forge is a swarm singleton, so on a forge host without the
# store this is the file an operator copies.
[ -s ${pkiDir}/forge-publisher.pem ] || ${signLeaf} ${pkiDir} forge-publisher \
${lib.escapeShellArg deployCfg.bao.forgePublisherCommonName} "" clientAuth
${lib.optionalString deployCfg.hive-controller.enable ''
# This hive's gateway-leaf identity, which opens
# `pki/issue/<hiveLeafPkiRolePrefix>-<hive>` and nothing else. Its

177
nix/host-modules/hive-forge/default.nix
View file

@ -56,6 +56,16 @@ let
# `forgeSecretPath` above.
swarmControllerTokenPath = "/var/lib/forgejo/swarm-controller-token";
# Where `swarm-bao-forge-publisher` puts it: `controllerForgeTokenLeaf` in
# ../swarm-bao.nix, under the KV mount.
controllerTokenStorePath = "secret/swarm/controller/swarm-controller/forge/token";
baoCfg = config.services.hyperhive.swarm.bao;
storeRetry = import ../lib/store-retry.nix { };
havePublisherIdentity =
deployCfg.bao.forgePublisherClientCertFile != null
&& deployCfg.bao.forgePublisherClientKeyFile != null;
caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
# ROOT_URL forgejo advertises in clone links + outbound URLs.
@ -101,7 +111,18 @@ in
# `/var/lib/nixos-containers/hive-forge/var/lib/forgejo/` survives
# restart. See `docs/networking/gateway.md::hive-forge container shape`.
imports = [ ./service.nix ];
imports = [
./service.nix
(lib.mkRemovedOptionModule
[ "services" "hyperhive" "deploy" "forgejo" "hostSwarmControllerTokenFile" ]
''
The forge publishes the swarm-controller's forge token to the swarm
secret store (swarm-bao-forge-publisher), and the controller reads it
there. Remove this definition; the file it named is read by nothing now
and can be deleted.
''
)
];
# What ./service.nix declares is what the forge IS from any hive's point of
# view: the names and ports it answers on, the URLs it advertises, and the
@ -226,28 +247,6 @@ in
eval error.
'';
};
hostSwarmControllerTokenFile = lib.mkOption {
type = lib.types.str;
default = "/var/lib/hyperhive-forge/swarm-controller.token";
description = ''
Host path where this forge deposits the freshly-minted forge
access token for the swarm's `swarm-controller` account (see
`systemd.services.forgejo-swarm-controller-account` inside the
forge container, and `hive-forge-swarm-controller-token` on the
host, which copies the token out).
Same role for this token as
`services.hyperhive.deploy.authelia.hostClientSecretDir` plays
for the OIDC secret: a **host**-local path (not inside any
container), read directly by `swarm-controller.nix`'s
`LoadCredential` when the controller runs on this same host.
On any other host the token has to get there somehow — copy it
out of this path with whatever secret management this
deployment already uses, the same shape `swarm.nix`'s own
`clientSecretFile` documents for the analogous cross-host case.
'';
};
};
config = lib.mkIf deployCfg.forgejo.enable {
@ -1049,53 +1048,119 @@ in
'';
};
# The reverse direction of the unit above: collect the
# swarm-controller's freshly-minted forge token OUT of the container
# onto a host path other units/hosts can read. Same "only the host can
# see both trees" reasoning, same ordering constraint — this can only
# depend on the container being up (`container@hive-forge.service`),
# not on the specific in-container oneshot that mints the token
# (containers run their own systemd instance, invisible to this one by
# unit name) — so it polls the same bounded way
# `hive-forge-oidc-secret` does while waiting on authelia above.
# The reverse direction of the unit above: publish the swarm-controller's
# freshly-minted forge token OUT of the container into the swarm secret
# store, where the controller reads it wherever it runs. On the host for the
# reason `hive-forge-oidc-secret` gives, and under the forge's own store
# identity, so nothing here asks where the store or the controller is.
#
# Unconditional, like the in-container unit it collects from: forge is
# a swarm-wide singleton, so the token always gets minted and always
# gets collected here, regardless of whether swarm-controller runs on
# this host, another host, or nowhere in this swarm at all.
systemd.services.hive-forge-swarm-controller-token = {
description = "collect the swarm-controller's forge token onto the host";
after = [ "container@hive-forge.service" ];
# It can only depend on the container being up, not on the in-container
# oneshot that mints the token (containers run their own systemd instance,
# invisible to this one by unit name), so it polls the same bounded way
# `hive-forge-oidc-secret` does. `wantedBy` the container, so every
# container start re-publishes whatever the mint unit left there.
#
# Unconditional, like the in-container unit it publishes from: forge is a
# swarm-wide singleton, so the token always gets minted and published,
# regardless of whether swarm-controller runs on this host, another host,
# or nowhere in this swarm at all. A host with no publisher leaf fails
# here, naming the options, rather than rendering no unit.
systemd.services.swarm-bao-forge-publisher = {
description = "publish the swarm-controller's forge token to the swarm secret store";
# The store's units exist only where the store runs; `Requires=` on an
# absent unit fails the job.
after = [
"container@hive-forge.service"
]
++ lib.optionals deployCfg.bao.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
wantedBy = [ "container@hive-forge.service" ];
serviceConfig = {
path = [
deployCfg.bao.package
pkgs.coreutils
];
# ../lib/store-retry.nix: a sealed or unreachable store is retried, and
# so is a token the forge has not minted yet.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "hive-forge-swarm-controller-token";
SyslogIdentifier = "swarm-bao-forge-publisher";
# Longer than the 120s bounded wait below. `DefaultTimeoutStartSec`
# is 90s, so without this systemd kills the unit at 90 — before it
# can emit the message naming the file it was waiting for.
TimeoutStartSec = "180s";
};
path = [ pkgs.coreutils ];
script = ''
set -euo pipefail
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs havePublisherIdentity {
BAO_CLIENT_CERT = deployCfg.bao.forgePublisherClientCertFile;
BAO_CLIENT_KEY = deployCfg.bao.forgePublisherClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
script =
if !havePublisherIdentity then
''
echo "the forge's host has no forge-publisher store identity, so it cannot publish the swarm-controller's forge token: set services.hyperhive.deploy.bao.forgePublisherClientCertFile and forgePublisherClientKeyFile (they default to the leaf the store mints only where services.hyperhive.deploy.bao.enable is set on this host)" >&2
exit 1
''
else
''
set -euo pipefail
src=${lib.escapeShellArg "/var/lib/nixos-containers/hive-forge${swarmControllerTokenPath}"}
dst=${lib.escapeShellArg deployCfg.forgejo.hostSwarmControllerTokenFile}
src=${lib.escapeShellArg "/var/lib/nixos-containers/hive-forge${swarmControllerTokenPath}"}
for _ in $(seq 1 60); do
[ -s "$src" ] && break
sleep 2
done
if [ ! -s "$src" ]; then
echo "forge has not minted $src after 120s" >&2
exit 1
fi
for _ in $(seq 1 60); do
[ -s "$src" ] && break
sleep 2
done
if [ ! -s "$src" ]; then
echo "forge has not minted $src after 120s" >&2
exit 1
fi
install -D -m 0400 "$src" "$dst"
'';
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# `-token-only` answers on stdout and skips the token helper, a
# `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with the forge publisher's certificate; the swarm-controller's forge token is not published." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
# `value=@$src` hands bao the PATH: bao opens the file itself, so
# the token is never an argument of this process.
if ! bao kv put ${lib.escapeShellArg controllerTokenStorePath} value=@"$src" >/dev/null 2>"$err"; then
echo "swarm-bao refused the write to ${controllerTokenStorePath}." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
echo "published the swarm-controller's forge token to ${controllerTokenStorePath}"
'';
};
# Unused: nothing writes or reads this host path, and it may hold a copy of
# the controller's forge token.
systemd.tmpfiles.rules = [ "r /var/lib/hyperhive-forge/swarm-controller.token" ];
networking.firewall = lib.mkIf deployCfg.forgejo.openFirewall {
allowedTCPPorts = [
cfg.httpPort

73
nix/host-modules/swarm-bao.nix
View file

@ -369,9 +369,9 @@ let
# Each hive's matrix sender token (`matrix_account::hive_sender`) grants `read`
# for the same keep-if-live reason, and `+` for the same glob-only-as-last-segment reason.
#
# The swarm appservice token and its own OIDC client secret, read-only: it
# uses both and writes neither. matrix-ctl publishes the token
# (`matrixCtlPolicyText` below), the publisher the secret.
# Three leaves it reads and never writes: the swarm appservice token
# (matrix-ctl, `matrixCtlPolicyText` below), its own OIDC client secret (the
# publisher) and its own forge token (the forge's host, `forgePublisherWriters`).
#
# The agent PKI grant is its only path on that mount: it can ask the one role
# for a certificate, not write that role or reach the issuer.
@ -445,6 +445,10 @@ let
capabilities = ["read"]
}
path "${credentialMountPath}/data/${controllerForgeTokenLeaf}" {
capabilities = ["read"]
}
path "${agentPkiMountPath}/issue/${agentPkiRoleName}" {
capabilities = ["update"]
}
@ -545,6 +549,11 @@ let
# it; the controller reads it and holds no other copy.
controllerClientLeaf = "swarm/controller/swarm-controller/oidc/client";
# The controller's own forge token, the nix half of
# `swarm_secret_client::forge::controller_token_path`. The forge's host writes
# it; the controller reads it and holds no other copy.
controllerForgeTokenLeaf = "swarm/controller/swarm-controller/forge/token";
# The forge's runner registration token, the nix half of
# `swarm_secret_client::ci::registration_token_path`. The controller writes
# it and cannot read it back (`forge::runner_token` keeps the value it last
@ -922,6 +931,21 @@ let
}
];
# The forge's host, publishing the controller's forge token. The one writer
# of the path, and write-only like the secret publisher: it copies a file in
# and never reads one back.
forgePublisherWriters = [
{
name = "swarm-forge-publisher";
cn = baoDeploy.forgePublisherCommonName;
policyText = ''
path "${credentialMountPath}/data/${controllerForgeTokenLeaf}" {
capabilities = ["create", "update"]
}
'';
}
];
# The role name IS the policy name, as for the three service principals
# above: the role attaches the policy by spelling it identically, and one
# string for both objects removes the way they drift apart.
@ -1721,6 +1745,20 @@ in
'';
};
forgePublisherCommonName = lib.mkOption {
type = lib.types.str;
default = "swarm-forge-publisher";
description = ''
Subject the store's `swarm-forge-publisher` cert-auth role accepts: the
identity `swarm-bao-forge-publisher`, the unit on the forge's host that
publishes the swarm-controller's forge token, presents. Its grant is
one path, `swarm/controller/swarm-controller/forge/token`, and write
only.
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
'';
};
queueAgentCommonNamePrefix = lib.mkOption {
type = lib.types.str;
default = "swarm-bao-queue-agent";
@ -1954,6 +1992,32 @@ in
'';
};
forgePublisherClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/forge-publisher.pem";
description = ''
Certificate the unit that publishes the swarm-controller's forge token
presents to the store. Its subject must be
{option}`services.hyperhive.deploy.bao.forgePublisherCommonName`.
./glue-bao-tls.nix points it at the leaf it mints on the store's host.
On a forge host without the store, it is a file the operator copies
from the store's host. A path, never a value.
'';
};
forgePublisherClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/forge-publisher-key.pem";
description = ''
Private key for
{option}`services.hyperhive.deploy.bao.forgePublisherClientCertFile`.
A path, never a value.
'';
};
servicesIssuerClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -3053,6 +3117,8 @@ in
systemd.services.swarm-bao-ci-runner-policy = readerPolicyUnit "write the CI runner's registration-token-reader bao policy and cert-auth role" ciRunnerReaders;
systemd.services.swarm-bao-forge-publisher-policy = readerPolicyUnit "write the forge's controller-token-publisher bao policy and cert-auth role" forgePublisherWriters;
# A FOURTH sibling, same shape and same reasons as the two above. This
# one is what turns `swarm-services-issuer` from a declaration into a
# grant: a bao policy reaches nothing until a login role hands it to a
@ -3258,6 +3324,7 @@ in
"swarm-bao-nats-tls-policy.service"
"swarm-bao-hive-leaf-policy.service"
"swarm-bao-ci-runner-policy.service"
"swarm-bao-forge-publisher-policy.service"
]
++ granterAfter;
requires = [ "swarm-bao-pki.service" ];

96
nix/host-modules/swarm-controller.nix
View file

@ -189,24 +189,11 @@ let
SWARM_CONTROLLER_OIDC_CLIENT_ID = queueClientId;
};
# Independent of `queueEnv` on purpose, and now for a simpler reason
# than when this was written: the queue's own co-location guard
# (`queueLocal`) is gone — its coordinates are options with assertions,
# so a split-host controller fails at eval instead of silently losing
# its identity. Forge access still has nothing to do with where authelia
# and NATS run, only with whether `forgeTokenFile` resolves to a real
# file — which that option's own default already handles
# (forge-local vs. operator-copied). Gating a second time here would
# just repeat that option's own logic under a different name.
#
# ⚠️ The asymmetry is deliberate: the queue is REQUIRED for a controller
# and the forge is not, so absence is an error for one and a supported
# shape for the other.
forgeEnv = lib.optionalAttrs (deployCfg.swarm-controller.forgeTokenFile != null) {
# Unconditional: every swarm has a forge, reached by name wherever it runs.
# The token is not here: the daemon reads it from the swarm's secret store at
# start (`forge.rs`), and runs without forge access when it cannot.
forgeEnv = {
SWARM_CONTROLLER_FORGE_URL = "https://${forgeCfg.domain}";
# `%d` is systemd's credentials directory: root reads the plaintext at
# unit start, the daemon's own user sees a 0400 copy.
SWARM_CONTROLLER_FORGE_TOKEN_FILE = "%d/forge-token";
# `agent-configs` org avatar for the forge-objects pass
# (`forge/objects.rs`). Only meaningful with forge access, hence here.
SWARM_CONTROLLER_CONFIG_ORG_AVATAR_PNG =
@ -268,8 +255,8 @@ let
# `vcs_metrics`/`hive_jobq_metrics` (this daemon's OTLP push exporters)
# read the standard `OTEL_EXPORTER_OTLP_ENDPOINT` var and are silent,
# graceful no-ops without it — same shape as `forgeEnv`/`authBridgeEnv`
# above, genuinely optional and gated on the option resolving rather
# graceful no-ops without it — same shape as `authBridgeEnv` above,
# genuinely optional and gated on the option resolving rather
# than assumed.
#
# By NAME through the gateway, never a loopback literal or a bare
@ -294,8 +281,7 @@ let
# HERE"): the collector is reachable by name wherever it runs, so a
# controller not co-located with it now exports over https instead of
# exporting nothing — the graceful-absence case left is "no swarm-otel
# anywhere in this swarm at all," which `forgeEnv` already models the
# same way.
# anywhere in this swarm at all."
otelSwarmCfg = config.services.hyperhive.swarm.otel;
otelEnv = lib.optionalAttrs deployCfg.swarm-otel.enable {
OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}";
@ -352,6 +338,14 @@ in
be deleted.
''
)
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "swarm-controller" "forgeTokenFile" ]
''
The controller reads its forge token from the swarm's secret store, under
its own certificate (deploy.swarm-controller.baoClientCertFile), and
swarm-bao-forge-publisher on the forge's host puts it there. Remove this
definition; the file it named is read by nothing now and can be deleted.
''
)
];
options.services.hyperhive.swarm.controller = {
@ -554,40 +548,6 @@ in
'';
};
forgeTokenFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
# Forge's own delivery path when the forge runs here, since nothing
# writes that path anywhere else. `null` otherwise, so a controller
# away from the forge logs that it has no forge access instead of
# waiting on a file that never appears. The controller's units are
# the thing that decides whether the file is ever read: they only
# exist under `deploy.swarm-controller.enable`.
default = if deployCfg.forgejo.enable then deployCfg.forgejo.hostSwarmControllerTokenFile else null;
defaultText = lib.literalExpression ''
if config.services.hyperhive.deploy.forgejo.enable
then config.services.hyperhive.deploy.forgejo.hostSwarmControllerTokenFile
else null
'';
example = "/var/lib/secrets/swarm-controller-forge.token";
description = ''
Path to this host's copy of the swarm-controller's forge access
token (see `hive-forge/default.nix`'s
`forgejo-swarm-controller-account` + `hive-forge-swarm-controller-token`
units, which mint and collect it onto forge's own host).
Defaults to forge's own delivery path when
{option}`services.hyperhive.deploy.forgejo.enable` is set on this
host, and to `null` otherwise. Set it explicitly when the forge
runs on another host — copy the token out of forge's
{option}`services.hyperhive.deploy.forgejo.hostSwarmControllerTokenFile`
with whatever secret management this deployment already uses,
the same shape `swarm.nix`'s `clientSecretFile` documents for
its own cross-host case. `null` means no forge access — the
daemon logs that and continues without it, the same
graceful-absence shape the queue coordinates already use.
'';
};
authBridgeUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "${autheliaCfg.url}/bridge";
@ -606,7 +566,7 @@ in
already covers it. `null` means no agent-identity support:
`CreateIdentity` jobs fail with a clear "no auth bridge
configured here" error rather than the daemon refusing to
start, the same graceful-absence shape `forgeTokenFile` uses.
start, the same graceful-absence shape forge access has.
'';
};
@ -758,21 +718,21 @@ in
systemd.services.swarm-controller = {
description = "hyperhive swarm-level controller daemon";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
# Where the forge runs here too, the first read waits out one publish
# attempt. `After=` on a unit this host does not define is a no-op.
after = [
"network.target"
"swarm-bao-forge-publisher.service"
];
serviceConfig = {
ExecStart = "${deployCfg.swarm-controller.package}/bin/swarm-controller";
# The forge token is optional: a controller with no forge access
# still serves its HTTP surface, and that IS a supported shape.
# The store identity, same shape and same reason as hive-c0re's: the
# key is root-owned `0600` and this daemon runs as `swarm-controller`,
# so it never gets read access to the original file.
LoadCredential =
lib.optional (
deployCfg.swarm-controller.forgeTokenFile != null
) "forge-token:${deployCfg.swarm-controller.forgeTokenFile}"
# The store identity, same shape and same reason as hive-c0re's: the
# key is root-owned `0600` and this daemon runs as `swarm-controller`,
# so it never gets read access to the original file.
++ lib.optionals haveBaoIdentity [
lib.optionals haveBaoIdentity [
"bao-client.pem:${deployCfg.swarm-controller.baoClientCertFile}"
"bao-client-key.pem:${deployCfg.swarm-controller.baoClientKeyFile}"
]
@ -846,8 +806,8 @@ in
#
# They differ in how absence is prevented: the queue's is checked by
# the assertions above, because a controller without a queue is
# broken rather than lighter; forge's and the auth bridge's are
# genuinely optional and stay gated on their own option resolving.
# broken rather than lighter; the auth bridge's is genuinely optional
# and stays gated on its own option resolving. Forge's is always set.
environment = {
SWARM_CONTROLLER_SOCKET = deployCfg.swarm-controller.socketPath;
# The swarm's hive directory, JSON-encoded — the full directory

1
nix/host-modules/swarm.nix
View file

@ -55,6 +55,7 @@ let
deployCfg.bao.granterCommonName
deployCfg.bao.natsAuthCommonName
deployCfg.bao.ciRunnerCommonName
deployCfg.bao.forgePublisherCommonName
]
# The per-hive subjects, spelled out per hive rather than as the prefix.
# The prefix alone would reserve the wrong string: the role for hive

21
nix/module-eval/bao-controller.nix
View file

@ -197,6 +197,27 @@ let
&& builtins.elem "bao-client.pem:/var/lib/swarm-bao-pki/controller.pem" s.swarm-controller.serviceConfig.LoadCredential
&& builtins.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/controller-key.pem" s.swarm-controller.serviceConfig.LoadCredential;
}
{
# The forge token comes from the store, so the unit is handed no file for
# it, and neither fixture runs the forge: a controller reaches it by name
# wherever it runs.
name = "the controller is handed no forge token file, and is pointed at the forge away from it";
ok =
lib.all
(
m:
let
u = m.systemd.services.swarm-controller;
in
!(lib.any (c: lib.hasPrefix "forge-token:" c) u.serviceConfig.LoadCredential)
&& !(u.environment ? SWARM_CONTROLLER_FORGE_TOKEN_FILE)
&& u.environment.SWARM_CONTROLLER_FORGE_URL == "https://forge.t.local"
)
[
baoControllerHere
controllerNoStore
];
}
{
# A hive's cert-auth role carries the authority by value, so the daemon
# has to be handed the file rather than a path into the store's own

66
nix/module-eval/bao-grants.nix
View file

@ -170,7 +170,7 @@ let
_: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile
) baoGrantWithConsumers.systemd.services;
# The fourteen units that write a `swarm-*` grant, by name, for the discovery
# The fifteen units that write a `swarm-*` grant, by name, for the discovery
# control below.
grantingUnitNames = [
"swarm-bao-controller-policy"
@ -187,6 +187,7 @@ let
"swarm-bao-operator-viewer-policy"
"swarm-bao-hive-leaf-policy"
"swarm-bao-ci-runner-policy"
"swarm-bao-forge-publisher-policy"
];
# Comment lines dropped first: both the HCL and the scripts explain
@ -851,24 +852,24 @@ let
}
{
# A store host without the granter's pair writes its grants some other
# way, so none of the fourteen units may exist. Without this arm
# way, so none of the fifteen units may exist. Without this arm
# `lib.mkIf haveGranter` could be dropped from any of them and every other
# case here would still pass.
name = "without the granter's pair none of the fourteen granting units render";
name = "without the granter's pair none of the fifteen granting units render";
ok =
let
s = baoGranterOptOut.systemd.services;
in
lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ])
# The control: the same store with the pair renders all fourteen.
# The control: the same store with the pair renders all fifteen.
&& lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames;
}
{
# 🩸 What replaced the silent skip. With no bootstrap token the fourteen still
# 🩸 What replaced the silent skip. With no bootstrap token the fifteen still
# render, and a refused granter fails them with the step that fixes it.
# A store host that never named a token is told to name one, since the
# unit that sets the granter up renders only where it has.
name = "a store host without a bootstrap token renders the fourteen, each failing loudly with the one-time step";
name = "a store host without a bootstrap token renders the fifteen, each failing loudly with the one-time step";
ok =
let
s = baoGranterNoToken.systemd.services;
@ -1371,6 +1372,55 @@ let
in
lib.hasInfix "/ci-runner.pem ]" s && lib.hasInfix "swarm-ci-runner \"\" clientAuth" s;
}
{
# The forge's host is the one writer of the controller's forge token.
# Pinned as the whole stanza, so an added `read` fails: it copies a file
# in and never reads one back.
name = "the forge publisher's grant is writing the controller's forge token and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-forge-publisher-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/forge/token\" {\n capabilities = [\"create\", \"update\"]\n}" s
&& lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 1
&& lib.hasInfix "auth/cert/certs/swarm-forge-publisher" s
&& lib.hasInfix "allowed_common_names=swarm-forge-publisher" s
&& lib.hasInfix "token_policies=swarm-forge-publisher" s;
}
{
# The controller's side: read on the one leaf, pinned as the whole
# stanza so an added write fails.
name = "the controller reads its own forge token and cannot write it";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/forge/token\" {\n capabilities = [\"read\"]\n}" s
&& !(lib.hasInfix "secret/data/swarm/controller/*" s);
}
{
# The policy units that name the path are the whole list of who reaches
# it: the publisher, which writes it, and the controller, which reads it.
name = "no granting unit but the controller's and the forge publisher's names the controller's forge token";
ok =
let
naming = lib.filter (
u: lib.hasInfix "swarm-controller/forge/token" baoGrantHere.systemd.services.${u}.script
) grantingUnitNames;
in
lib.sort lib.lessThan naming == [
"swarm-bao-controller-policy"
"swarm-bao-forge-publisher-policy"
];
}
{
name = "the PKI unit signs the forge publisher's leaf under its own subject";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
in
lib.hasInfix "/forge-publisher.pem ]" s && lib.hasInfix "swarm-forge-publisher \"\" clientAuth" s;
}
{
# Revocation, and the reason it is a stanza of its own: `delete` on the
# `data/` path soft-deletes the newest version and leaves earlier ones
@ -1503,8 +1553,8 @@ let
}
{
# What makes the case above mean something: discovery by the granter's
# certificate reaches all fourteen units, and each yields calls.
name = "the granter-policy check sees all fourteen granting units, and parses calls from each";
# certificate reaches all fifteen units, and each yields calls.
name = "the granter-policy check sees all fifteen granting units, and parses calls from each";
ok =
lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames
&& lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)

10
nix/module-eval/core-toggle.nix
View file

@ -76,16 +76,16 @@ let
&& centralToggleOff.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local";
}
{
# The controller's token path follows where the forge runs
# The controller's token publisher follows where the forge runs
# (./forge-placement.nix), never the central toggle: a forge host with
# the toggle off still renders the path.
name = "the swarm controller's forgeTokenFile default does not consult the central toggle";
# the toggle off still publishes it.
name = "the forge's controller-token publisher does not consult the central toggle";
ok =
(hive {
deploy.hive-controller.enable = false;
deploy.forgejo.enable = true;
}).services.hyperhive.deploy.swarm-controller.forgeTokenFile
== "/var/lib/hyperhive-forge/swarm-controller.token";
}).systemd.services
? swarm-bao-forge-publisher;
}
{
name = "a hive that does not host the swarm's shared services runs none of them";

44
nix/module-eval/forge-placement.nix
View file

@ -104,8 +104,7 @@ let
# ./grafana.nix's `grafanaRefusedFor`.
refusedOver = m: needle: lib.any (a: !a.assertion && lib.hasInfix needle a.message) m.assertions;
tokenFile = m: m.services.hyperhive.deploy.swarm-controller.forgeTokenFile;
forgePath = "/var/lib/hyperhive-forge/swarm-controller.token";
publisher = m: m.systemd.services.swarm-bao-forge-publisher;
forgeSettings = forgeHere.containers.hive-forge.config.services.forgejo.settings;
@ -122,7 +121,7 @@ let
ok =
!bare.services.hyperhive.deploy.forgejo.enable
&& !(bare.containers ? hive-forge)
&& !(bare.systemd.services ? hive-forge-swarm-controller-token)
&& !(bare.systemd.services ? swarm-bao-forge-publisher)
&& !(lib.elem "forge.t.local" bare.services.hyperhive.gateway.localNames)
&& !(refusedOver bare "deploy.forgejo.sso.clientSecretFile");
}
@ -280,14 +279,39 @@ let
&& refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.enable";
}
{
# Nothing writes the delivery path away from the forge, and a
# `LoadCredential=` naming a missing path is fatal to the unit.
name = "the controller's forge token defaults to the delivery path only where the forge runs";
# The forge's host publishes the controller's token under its own leaf,
# which the store mints beside it here. `value=@` is what keeps the token
# out of the process's arguments.
name = "the forge's host publishes the controller's forge token to the store under its own leaf";
ok =
tokenFile bare == null
&& tokenFile servicesForgeElsewhere == null
&& tokenFile forgeHere == forgePath
&& tokenFile allLocal == forgePath;
let
u = publisher allLocal;
in
u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/forge-publisher.pem"
&& u.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/forge-publisher-key.pem"
&& lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/forge/token value=@\"$src\"" u.script
&& lib.hasInfix "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token" u.script
&& lib.elem "container@hive-forge.service" u.wantedBy
&& !(allLocal.systemd.services ? hive-forge-swarm-controller-token);
}
{
# The absence arm: a forge host without the store has no leaf unless one
# is placed, and the unit says which options to set.
name = "a forge host with no publisher leaf renders a publisher that fails naming the options";
ok =
let
s = (publisher forgeHere).script;
in
lib.hasInfix "forgePublisherClientCertFile" s
&& lib.hasInfix "exit 1" s
&& !(lib.hasInfix "bao kv put" s);
}
{
# The host copy is removed where the forge runs: nothing reads it.
name = "the forge's host removes its old copy of the controller's forge token";
ok =
lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" forgeHere.systemd.tmpfiles.rules
&& !(lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" bare.systemd.tmpfiles.rules);
}
{
# The only thing that makes a human's forge account: nothing else

17
nix/module-eval/name-guards.nix
View file

@ -83,6 +83,13 @@ let
swarm.hives.fwctl.domain = "f.t.local";
};
# The forge publisher's, the one writer of the controller's forge token.
hiveNamedAfterForgePublisherSubject = hive {
deploy.swarm-otel.enable = false;
deploy.bao.forgePublisherCommonName = "fpctl";
swarm.hives.fpctl.domain = "fp.t.local";
};
# The granter's, the one subject whose role may write every `swarm-*` grant.
hiveNamedAfterGranterSubject = hive {
deploy.swarm-otel.enable = false;
@ -188,6 +195,16 @@ let
a: !a.assertion && lib.hasInfix "'fwctl'" a.message
) hiveNamedAfterForwarderOidcSubject.assertions;
}
{
# And the forge publisher's: a hive holding a leaf it accepts could
# replace the token the controller provisions every forge object with.
name = "a hive named after the forge publisher's subject is refused too";
ok =
equalityGuardFired hiveNamedAfterForgePublisherSubject
&& lib.any (
a: !a.assertion && lib.hasInfix "'fpctl'" a.message
) hiveNamedAfterForgePublisherSubject.assertions;
}
{
# And the granter's, whose role is root-equivalent: a hive holding a leaf
# it accepts could grant itself anything.