hive-forge: the controller-token mint writes straight to the store, and the on-disk token goes
`forgejo-swarm-controller-account`, in the forge's container, now puts the swarm-controller's forge token into the swarm secret store itself, under the forge publisher's leaf. The container already receives that leaf as nspawn system credentials for its journal forwarder; the mint loads the same three. The token reaches bao on stdin (`value=-`) and the forge's API through a header file, never as an argument, and never touches disk. The mint's guard is a marker holding the published token's name, at /var/lib/forgejo/swarm-controller-token-name, written last: after the put, and after every other `swarm-controller-boot-*` token of the account is revoked through forgejo's admin API, authenticated as the new token. So a put that failed, or a marker lost while the store kept an older token, leaves no live token behind once a mint completes. The mint logs in to the store and checks that the account is a site admin before it mints, so a store that is down or an account the admin API would refuse mints nothing on each retry. Failures retry under store-retry; forgejo does not wait for the mint. No bao read and no grant change: the forge publisher keeps create/update on the token. Removed: the host's `swarm-bao-forge-publisher` service and path unit, which read the token file through the container's root, and their entries in the store and policy ordering glue. The forge publisher's policy unit stays; `swarm-bao-forge-oidc` still uses the leaf. On deploy, an existing forge host has no marker, so the mint runs once: it mints a new token, publishes it, revokes the old one, and deletes the old /var/lib/forgejo/swarm-controller-token. The controller's next forge call gets a 401, re-reads the store and carries on.
This commit is contained in:
atlas
mara
parent
fadcac8479
commit
141be16c6d
14 changed files with 294 additions and 261 deletions
|
|
@ -107,7 +107,7 @@ let
|
|||
# ./grafana.nix's `grafanaRefusedFor`.
|
||||
refusedOver = m: needle: lib.any (a: !a.assertion && lib.hasInfix needle a.message) m.assertions;
|
||||
|
||||
publisher = m: m.systemd.services.swarm-bao-forge-publisher;
|
||||
mint = m: m.containers.hive-forge.config.systemd.services.forgejo-swarm-controller-account;
|
||||
oidcFetch = m: m.systemd.services.swarm-bao-forge-oidc;
|
||||
|
||||
forgeSettings = forgeHere.containers.hive-forge.config.services.forgejo.settings;
|
||||
|
|
@ -125,7 +125,6 @@ let
|
|||
ok =
|
||||
!bare.services.hyperhive.deploy.forgejo.enable
|
||||
&& !(bare.containers ? hive-forge)
|
||||
&& !(bare.systemd.services ? swarm-bao-forge-publisher)
|
||||
&& !(lib.elem "forge.t.local" bare.services.hyperhive.gateway.localNames);
|
||||
}
|
||||
{
|
||||
|
|
@ -327,55 +326,93 @@ let
|
|||
&& refusedOver ciUnderForgeNoIdentity "services.hyperhive.deploy.bao.enable";
|
||||
}
|
||||
{
|
||||
# The forge's host publishes the controller's token under its own leaf,
|
||||
# which the store mints beside it here. `value=@` is what keeps the token
|
||||
# out of the process's arguments.
|
||||
name = "the forge's host publishes the controller's forge token to the store under its own leaf";
|
||||
# The mint, in the forge's container, writes the controller's token to
|
||||
# the store under the forge host's leaf, loaded as a credential. On
|
||||
# stdin, so the token is no process's argument; the copy an earlier mint
|
||||
# kept on the container's disk goes once a token is published, and no
|
||||
# host unit reads into the container.
|
||||
name = "the forge's mint writes the controller's forge token straight to the store and keeps no copy";
|
||||
ok =
|
||||
let
|
||||
u = publisher allLocal;
|
||||
u = mint allLocal;
|
||||
cred = id: "/run/credentials/forgejo-swarm-controller-account.service/${id}";
|
||||
in
|
||||
u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/forge-publisher.pem"
|
||||
&& u.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/forge-publisher-key.pem"
|
||||
&& lib.hasInfix "bao kv put secret/swarm/controller/swarm-controller/forge/token value=@\"$src\"" u.script
|
||||
&& lib.hasInfix "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token" u.script
|
||||
&& lib.elem "container@hive-forge.service" u.wantedBy
|
||||
# The deploy that adds the unit starts it only through a target.
|
||||
&& lib.elem "multi-user.target" u.wantedBy
|
||||
lib.elem "forge-bao-cert" u.serviceConfig.LoadCredential
|
||||
&& lib.elem "forge-bao-key" u.serviceConfig.LoadCredential
|
||||
&& u.environment.BAO_CLIENT_CERT == cred "forge-bao-cert"
|
||||
&& u.environment.BAO_CLIENT_KEY == cred "forge-bao-key"
|
||||
&& lib.elem "--load-credential=forge-bao-cert:/var/lib/swarm-bao-pki/forge-publisher.pem" allLocal.containers.hive-forge.extraFlags
|
||||
&& lib.elem "--load-credential=forge-bao-cert:/etc/pki/forge-publisher.pem" forgeAlone.containers.hive-forge.extraFlags
|
||||
&& lib.hasInfix "printf '%s' \"$token\" | bao kv put secret/swarm/controller/swarm-controller/forge/token value=-" u.script
|
||||
&& !(lib.hasInfix "value=@" u.script)
|
||||
&& !(lib.hasInfix "atomic_write_secret" u.script)
|
||||
&& lib.hasInfix "rm -f /var/lib/forgejo/swarm-controller-token\n" u.script
|
||||
&& !(allLocal.systemd.services ? swarm-bao-forge-publisher)
|
||||
&& !(allLocal.systemd.paths ? swarm-bao-forge-publisher)
|
||||
&& !(forgeAlone.systemd.services ? swarm-bao-forge-publisher)
|
||||
&& !(allLocal.systemd.services ? hive-forge-swarm-controller-token);
|
||||
}
|
||||
{
|
||||
# What makes it re-run. A oneshot left `active (exited)` makes every later
|
||||
# start a no-op, so `RemainAfterExit` would silence every trigger: the
|
||||
# targets and the container's start (`wantedBy`, above) and a write of
|
||||
# the token file. A missing file exits 0, so a switch with the container
|
||||
# down does not fail.
|
||||
name = "the forge publisher re-runs on each container start and each write of the token file";
|
||||
# The order that keeps a token live only if the store has it: log in
|
||||
# before the mint, put, revoke every other `swarm-controller-boot-*`
|
||||
# token, and only then write the marker, which holds the name. A marker
|
||||
# present skips all of it.
|
||||
name = "the forge's mint writes its marker only after the put and the revocation of every earlier token";
|
||||
ok =
|
||||
let
|
||||
p = allLocal.systemd.paths.swarm-bao-forge-publisher;
|
||||
s = (mint allLocal).script;
|
||||
at = needle: lib.stringLength (lib.head (lib.splitString needle s));
|
||||
order = l: lib.length l < 2 || (at (lib.elemAt l 0) < at (lib.elemAt l 1) && order (lib.tail l));
|
||||
steps = [
|
||||
"if [ -s \"$name_path\" ]; then"
|
||||
"forgejo admin user list --admin"
|
||||
"bao login -method=cert"
|
||||
"forgejo admin user generate-access-token"
|
||||
"bao kv put"
|
||||
"forge GET \"/admin/users/swarm-controller/tokens?limit=50&page=$page\""
|
||||
"forge DELETE \"/admin/users/swarm-controller/tokens/$n\""
|
||||
"rm -f /var/lib/forgejo/swarm-controller-token\n"
|
||||
"mv -f \"$name_path.tmp\" \"$name_path\""
|
||||
];
|
||||
in
|
||||
!((publisher allLocal).serviceConfig.RemainAfterExit or false)
|
||||
&&
|
||||
p.pathConfig.PathChanged
|
||||
== "/var/lib/nixos-containers/hive-forge/var/lib/forgejo/swarm-controller-token"
|
||||
&& p.pathConfig.Unit == "swarm-bao-forge-publisher.service"
|
||||
&& lib.hasInfix "the path unit publishes it when it does" (publisher allLocal).script
|
||||
&& lib.hasInfix "exit 0" (publisher allLocal).script
|
||||
&& lib.elem "multi-user.target" p.wantedBy
|
||||
&& !(bare.systemd.paths ? swarm-bao-forge-publisher);
|
||||
lib.all (step: lib.hasInfix step s) steps
|
||||
&& order steps
|
||||
&& lib.hasInfix "name_path=/var/lib/forgejo/swarm-controller-token-name" s
|
||||
&& lib.hasInfix "printf '%s\\n' \"$name\" >\"$name_path.tmp\"" s
|
||||
&& lib.hasInfix "swarm-controller-boot | swarm-controller-boot-*) stale+=(\"$n\") ;;" s
|
||||
&& lib.hasInfix "-H @<(printf 'Authorization: token %s\\n' \"$token\")" s;
|
||||
}
|
||||
{
|
||||
# A store that is down retries the mint (../host-modules/lib/store-retry.nix),
|
||||
# and forgejo never waits for it.
|
||||
name = "the forge's mint retries on failure, and forgejo does not wait for it";
|
||||
ok =
|
||||
let
|
||||
c = allLocal.containers.hive-forge.config.systemd.services;
|
||||
u = mint allLocal;
|
||||
in
|
||||
u.serviceConfig.Restart == "on-failure"
|
||||
&& u.startLimitBurst == 2880
|
||||
&& u.serviceConfig.RemainAfterExit
|
||||
&& lib.elem "forgejo.service" u.after
|
||||
&& !(lib.elem "forgejo-swarm-controller-account.service" (
|
||||
c.forgejo.after ++ c.forgejo.requires ++ c.forgejo.wants
|
||||
))
|
||||
&& lib.elem "forgejo-swarm-controller-account.service" c.swarm-bridge-dns.before;
|
||||
}
|
||||
{
|
||||
# The absence arm: a forge host without the store has no leaf unless one
|
||||
# is placed, and the unit says which options to set.
|
||||
name = "a forge host with no publisher leaf renders a publisher that fails naming the options";
|
||||
# is placed, and the mint says which options to set before it mints.
|
||||
name = "a forge host with no publisher leaf renders a mint that fails naming the options, minting nothing";
|
||||
ok =
|
||||
let
|
||||
s = (publisher forgeHere).script;
|
||||
u = mint forgeHere;
|
||||
in
|
||||
lib.hasInfix "forgePublisherClientCertFile" s
|
||||
&& lib.hasInfix "exit 1" s
|
||||
&& !(lib.hasInfix "bao kv put" s);
|
||||
!(u.environment ? BAO_CLIENT_CERT)
|
||||
&& lib.hasInfix "forgePublisherClientCertFile" u.script
|
||||
&& lib.hasInfix "exit 1" u.script
|
||||
&& !(lib.hasInfix "forgejo admin user generate-access-token" u.script)
|
||||
&& !(lib.hasInfix "bao kv put" u.script);
|
||||
}
|
||||
{
|
||||
# The forge's OIDC secret has one route, the store, under the forge
|
||||
|
|
|
|||
Loading…
Reference in a new issue