swarm-controller reads its forge token from the store, not a host file
The controller's forge token never entered bao: the forge minted it in its container, hive-forge-swarm-controller-token copied it to a host file, and the controller LoadCredential'ed that file. forgeTokenFile defaulted to that path only where the forge ran, so a controller on any other host had no forge access unless an operator copied the file by hand. Now the forge's host publishes it and the controller reads it: - swarm-bao-forge-publisher (replaces hive-forge-swarm-controller-token) waits for the in-container token, logs in with its own forge-publisher leaf and writes it to swarm/controller/swarm-controller/forge/token. It runs on every forge container start, the old unit's trigger. - New principal swarm-forge-publisher: leaf minted by glue-bao-tls.nix, policy create/update on that one path, written by swarm-bao-forge-publisher-policy, CN reserved as a hive name. - The controller policy gains read on that path. swarm-controller reads it at start under its own certificate with the OIDC secret's retry, and runs without forge access if it still cannot. - forgeTokenFile, its LoadCredential and the forgeEnv gate are gone; SWARM_CONTROLLER_FORGE_URL is always set. hostSwarmControllerTokenFile is removed, and a tmpfiles rule deletes the old host copy on the forge host. The in-container file stays: the mint unit's only-once guard reads it, and it is the publisher's source. Refs #4998
This commit is contained in:
atlas
parent
c433bdef92
commit
121c4678aa
18 changed files with 580 additions and 277 deletions
|
|
@ -197,6 +197,27 @@ let
|
|||
&& builtins.elem "bao-client.pem:/var/lib/swarm-bao-pki/controller.pem" s.swarm-controller.serviceConfig.LoadCredential
|
||||
&& builtins.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/controller-key.pem" s.swarm-controller.serviceConfig.LoadCredential;
|
||||
}
|
||||
{
|
||||
# The forge token comes from the store, so the unit is handed no file for
|
||||
# it, and neither fixture runs the forge: a controller reaches it by name
|
||||
# wherever it runs.
|
||||
name = "the controller is handed no forge token file, and is pointed at the forge away from it";
|
||||
ok =
|
||||
lib.all
|
||||
(
|
||||
m:
|
||||
let
|
||||
u = m.systemd.services.swarm-controller;
|
||||
in
|
||||
!(lib.any (c: lib.hasPrefix "forge-token:" c) u.serviceConfig.LoadCredential)
|
||||
&& !(u.environment ? SWARM_CONTROLLER_FORGE_TOKEN_FILE)
|
||||
&& u.environment.SWARM_CONTROLLER_FORGE_URL == "https://forge.t.local"
|
||||
)
|
||||
[
|
||||
baoControllerHere
|
||||
controllerNoStore
|
||||
];
|
||||
}
|
||||
{
|
||||
# A hive's cert-auth role carries the authority by value, so the daemon
|
||||
# has to be handed the file rather than a path into the store's own
|
||||
|
|
|
|||
|
|
@ -170,7 +170,7 @@ let
|
|||
_: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile
|
||||
) baoGrantWithConsumers.systemd.services;
|
||||
|
||||
# The fourteen units that write a `swarm-*` grant, by name, for the discovery
|
||||
# The fifteen units that write a `swarm-*` grant, by name, for the discovery
|
||||
# control below.
|
||||
grantingUnitNames = [
|
||||
"swarm-bao-controller-policy"
|
||||
|
|
@ -187,6 +187,7 @@ let
|
|||
"swarm-bao-operator-viewer-policy"
|
||||
"swarm-bao-hive-leaf-policy"
|
||||
"swarm-bao-ci-runner-policy"
|
||||
"swarm-bao-forge-publisher-policy"
|
||||
];
|
||||
|
||||
# Comment lines dropped first: both the HCL and the scripts explain
|
||||
|
|
@ -851,24 +852,24 @@ let
|
|||
}
|
||||
{
|
||||
# A store host without the granter's pair writes its grants some other
|
||||
# way, so none of the fourteen units may exist. Without this arm
|
||||
# way, so none of the fifteen units may exist. Without this arm
|
||||
# `lib.mkIf haveGranter` could be dropped from any of them and every other
|
||||
# case here would still pass.
|
||||
name = "without the granter's pair none of the fourteen granting units render";
|
||||
name = "without the granter's pair none of the fifteen granting units render";
|
||||
ok =
|
||||
let
|
||||
s = baoGranterOptOut.systemd.services;
|
||||
in
|
||||
lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ])
|
||||
# The control: the same store with the pair renders all fourteen.
|
||||
# The control: the same store with the pair renders all fifteen.
|
||||
&& lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames;
|
||||
}
|
||||
{
|
||||
# 🩸 What replaced the silent skip. With no bootstrap token the fourteen still
|
||||
# 🩸 What replaced the silent skip. With no bootstrap token the fifteen still
|
||||
# render, and a refused granter fails them with the step that fixes it.
|
||||
# A store host that never named a token is told to name one, since the
|
||||
# unit that sets the granter up renders only where it has.
|
||||
name = "a store host without a bootstrap token renders the fourteen, each failing loudly with the one-time step";
|
||||
name = "a store host without a bootstrap token renders the fifteen, each failing loudly with the one-time step";
|
||||
ok =
|
||||
let
|
||||
s = baoGranterNoToken.systemd.services;
|
||||
|
|
@ -1371,6 +1372,55 @@ let
|
|||
in
|
||||
lib.hasInfix "/ci-runner.pem ]" s && lib.hasInfix "swarm-ci-runner \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
# The forge's host is the one writer of the controller's forge token.
|
||||
# Pinned as the whole stanza, so an added `read` fails: it copies a file
|
||||
# in and never reads one back.
|
||||
name = "the forge publisher's grant is writing the controller's forge token and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-forge-publisher-policy.script;
|
||||
in
|
||||
lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/forge/token\" {\n capabilities = [\"create\", \"update\"]\n}" s
|
||||
&& lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 1
|
||||
&& lib.hasInfix "auth/cert/certs/swarm-forge-publisher" s
|
||||
&& lib.hasInfix "allowed_common_names=swarm-forge-publisher" s
|
||||
&& lib.hasInfix "token_policies=swarm-forge-publisher" s;
|
||||
}
|
||||
{
|
||||
# The controller's side: read on the one leaf, pinned as the whole
|
||||
# stanza so an added write fails.
|
||||
name = "the controller reads its own forge token and cannot write it";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/forge/token\" {\n capabilities = [\"read\"]\n}" s
|
||||
&& !(lib.hasInfix "secret/data/swarm/controller/*" s);
|
||||
}
|
||||
{
|
||||
# The policy units that name the path are the whole list of who reaches
|
||||
# it: the publisher, which writes it, and the controller, which reads it.
|
||||
name = "no granting unit but the controller's and the forge publisher's names the controller's forge token";
|
||||
ok =
|
||||
let
|
||||
naming = lib.filter (
|
||||
u: lib.hasInfix "swarm-controller/forge/token" baoGrantHere.systemd.services.${u}.script
|
||||
) grantingUnitNames;
|
||||
in
|
||||
lib.sort lib.lessThan naming == [
|
||||
"swarm-bao-controller-policy"
|
||||
"swarm-bao-forge-publisher-policy"
|
||||
];
|
||||
}
|
||||
{
|
||||
name = "the PKI unit signs the forge publisher's leaf under its own subject";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
|
||||
in
|
||||
lib.hasInfix "/forge-publisher.pem ]" s && lib.hasInfix "swarm-forge-publisher \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
# Revocation, and the reason it is a stanza of its own: `delete` on the
|
||||
# `data/` path soft-deletes the newest version and leaves earlier ones
|
||||
|
|
@ -1503,8 +1553,8 @@ let
|
|||
}
|
||||
{
|
||||
# What makes the case above mean something: discovery by the granter's
|
||||
# certificate reaches all fourteen units, and each yields calls.
|
||||
name = "the granter-policy check sees all fourteen granting units, and parses calls from each";
|
||||
# certificate reaches all fifteen units, and each yields calls.
|
||||
name = "the granter-policy check sees all fifteen granting units, and parses calls from each";
|
||||
ok =
|
||||
lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames
|
||||
&& lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)
|
||||
|
|
|
|||
|
|
@ -76,16 +76,16 @@ let
|
|||
&& centralToggleOff.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local";
|
||||
}
|
||||
{
|
||||
# The controller's token path follows where the forge runs
|
||||
# The controller's token publisher follows where the forge runs
|
||||
# (./forge-placement.nix), never the central toggle: a forge host with
|
||||
# the toggle off still renders the path.
|
||||
name = "the swarm controller's forgeTokenFile default does not consult the central toggle";
|
||||
# the toggle off still publishes it.
|
||||
name = "the forge's controller-token publisher does not consult the central toggle";
|
||||
ok =
|
||||
(hive {
|
||||
deploy.hive-controller.enable = false;
|
||||
deploy.forgejo.enable = true;
|
||||
}).services.hyperhive.deploy.swarm-controller.forgeTokenFile
|
||||
== "/var/lib/hyperhive-forge/swarm-controller.token";
|
||||
}).systemd.services
|
||||
? swarm-bao-forge-publisher;
|
||||
}
|
||||
{
|
||||
name = "a hive that does not host the swarm's shared services runs none of them";
|
||||
|
|
|
|||
|
|
@ -104,8 +104,7 @@ let
|
|||
# ./grafana.nix's `grafanaRefusedFor`.
|
||||
refusedOver = m: needle: lib.any (a: !a.assertion && lib.hasInfix needle a.message) m.assertions;
|
||||
|
||||
tokenFile = m: m.services.hyperhive.deploy.swarm-controller.forgeTokenFile;
|
||||
forgePath = "/var/lib/hyperhive-forge/swarm-controller.token";
|
||||
publisher = m: m.systemd.services.swarm-bao-forge-publisher;
|
||||
|
||||
forgeSettings = forgeHere.containers.hive-forge.config.services.forgejo.settings;
|
||||
|
||||
|
|
@ -122,7 +121,7 @@ let
|
|||
ok =
|
||||
!bare.services.hyperhive.deploy.forgejo.enable
|
||||
&& !(bare.containers ? hive-forge)
|
||||
&& !(bare.systemd.services ? hive-forge-swarm-controller-token)
|
||||
&& !(bare.systemd.services ? swarm-bao-forge-publisher)
|
||||
&& !(lib.elem "forge.t.local" bare.services.hyperhive.gateway.localNames)
|
||||
&& !(refusedOver bare "deploy.forgejo.sso.clientSecretFile");
|
||||
}
|
||||
|
|
@ -280,14 +279,39 @@ let
|
|||
&& refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.enable";
|
||||
}
|
||||
{
|
||||
# Nothing writes the delivery path away from the forge, and a
|
||||
# `LoadCredential=` naming a missing path is fatal to the unit.
|
||||
name = "the controller's forge token defaults to the delivery path only where the forge runs";
|
||||
# The forge's host publishes the controller's token under its own leaf,
|
||||
# which the store mints beside it here. `value=@` is what keeps the token
|
||||
# out of the process's arguments.
|
||||
name = "the forge's host publishes the controller's forge token to the store under its own leaf";
|
||||
ok =
|
||||
tokenFile bare == null
|
||||
&& tokenFile servicesForgeElsewhere == null
|
||||
&& tokenFile forgeHere == forgePath
|
||||
&& tokenFile allLocal == forgePath;
|
||||
let
|
||||
u = publisher allLocal;
|
||||
in
|
||||
u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/forge-publisher.pem"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/forge-publisher-key.pem"
|
||||
&& lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/forge/token value=@\"$src\"" u.script
|
||||
&& lib.hasInfix "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token" u.script
|
||||
&& lib.elem "container@hive-forge.service" u.wantedBy
|
||||
&& !(allLocal.systemd.services ? hive-forge-swarm-controller-token);
|
||||
}
|
||||
{
|
||||
# The absence arm: a forge host without the store has no leaf unless one
|
||||
# is placed, and the unit says which options to set.
|
||||
name = "a forge host with no publisher leaf renders a publisher that fails naming the options";
|
||||
ok =
|
||||
let
|
||||
s = (publisher forgeHere).script;
|
||||
in
|
||||
lib.hasInfix "forgePublisherClientCertFile" s
|
||||
&& lib.hasInfix "exit 1" s
|
||||
&& !(lib.hasInfix "bao kv put" s);
|
||||
}
|
||||
{
|
||||
# The host copy is removed where the forge runs: nothing reads it.
|
||||
name = "the forge's host removes its old copy of the controller's forge token";
|
||||
ok =
|
||||
lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" forgeHere.systemd.tmpfiles.rules
|
||||
&& !(lib.elem "r /var/lib/hyperhive-forge/swarm-controller.token" bare.systemd.tmpfiles.rules);
|
||||
}
|
||||
{
|
||||
# The only thing that makes a human's forge account: nothing else
|
||||
|
|
|
|||
|
|
@ -83,6 +83,13 @@ let
|
|||
swarm.hives.fwctl.domain = "f.t.local";
|
||||
};
|
||||
|
||||
# The forge publisher's, the one writer of the controller's forge token.
|
||||
hiveNamedAfterForgePublisherSubject = hive {
|
||||
deploy.swarm-otel.enable = false;
|
||||
deploy.bao.forgePublisherCommonName = "fpctl";
|
||||
swarm.hives.fpctl.domain = "fp.t.local";
|
||||
};
|
||||
|
||||
# The granter's, the one subject whose role may write every `swarm-*` grant.
|
||||
hiveNamedAfterGranterSubject = hive {
|
||||
deploy.swarm-otel.enable = false;
|
||||
|
|
@ -188,6 +195,16 @@ let
|
|||
a: !a.assertion && lib.hasInfix "'fwctl'" a.message
|
||||
) hiveNamedAfterForwarderOidcSubject.assertions;
|
||||
}
|
||||
{
|
||||
# And the forge publisher's: a hive holding a leaf it accepts could
|
||||
# replace the token the controller provisions every forge object with.
|
||||
name = "a hive named after the forge publisher's subject is refused too";
|
||||
ok =
|
||||
equalityGuardFired hiveNamedAfterForgePublisherSubject
|
||||
&& lib.any (
|
||||
a: !a.assertion && lib.hasInfix "'fpctl'" a.message
|
||||
) hiveNamedAfterForgePublisherSubject.assertions;
|
||||
}
|
||||
{
|
||||
# And the granter's, whose role is root-equivalent: a hive holding a leaf
|
||||
# it accepts could grant itself anything.
|
||||
|
|
|
|||
Loading…
Reference in a new issue