diff --git a/docs/scheduler/ci.md b/docs/scheduler/ci.md index 2009da2b..d2762c66 100644 --- a/docs/scheduler/ci.md +++ b/docs/scheduler/ci.md @@ -160,8 +160,8 @@ needs `deploy.forgejo.enable` or a hive on its host. with the `ci-runner` leaf, whose `swarm-ci-runner` role reads that one path and nothing else, retrying every 30 seconds until the token is there. It writes the token to a root-only file inside the container, and restarts - the runner when the value changes. It runs at boot and on deploy, not on a - timer. + the runner when the value changes. It runs at boot, on deploy, and every + ten minutes on a timer. 3. **`swarm-ci-runner-register.service`** in the container is the only unit that loads that file, with `LoadCredential=`. systemd hands a unit's credentials to all of its processes, and for a `host` label act_runner diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 8ca6160a..a45bd2e7 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -134,8 +134,9 @@ reads its secret from the swarm's secret store under the hive's own leaf (`deploy.bao.clientCertFile`): `swarm-bao-hive-otel-oidc.service` fetches `swarm/hives//oidc/client`, which the publisher on authelia's host writes. It runs at boot and retries every 30s until the store answers; when a -run lands a changed value it restarts the collector. It doesn't run on a timer, -so a secret rotated later arrives the next time the unit runs. On a hive that +run lands a changed value it restarts the collector. A timer runs it again +every ten minutes, so a secret rotated later reaches the collector within +about ten minutes. On a hive that runs the store the leaf is already there; elsewhere it's the credential you place — see [secret placement](../swarm/secrets.md). diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index 8e4db43e..f9c24a2f 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -72,7 +72,7 @@ of the cell says how. | `swarm/hives//queue/agent` | authelia | `swarm-bao-queue-agent` on the hive's host, under its own per-hive certificate; no agent's policy reaches it | must be stated | must be stated | | `swarm/hives//oidc/client` | authelia, at its first boot; `swarm-secret-publish` copies it in | hive-c0re, under the hive's own certificate, when it first connects to the queue | ❌ authelia mints it once. A re-mint is republished by `swarm-secret-publish`'s path unit | ❌ read until the first connect succeeds, retried with a backoff of up to five minutes while it's missing; then held until hive-c0re restarts | | `swarm/services//oidc/client` | authelia | that service's own reader, under its own certificate — the queue responder's under `swarm-nats-auth`'s, the homeserver's under `swarm-matrix-ctl`'s, the forge's under `swarm-forge-publisher`'s, the review classifier's under `swarm-review-classifier`'s; the bao UI's by the store's granter | must be stated | must be stated | -| `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge's admin API, at start and in a five-minute pass; its grant on the path is write-only | `swarm-bao-ci-runner` on the runner's host, under the runner's own certificate, into a file only `swarm-ci-runner-register` in the runner's container loads; no other principal's policy reads it | ❌ the forge hands out one token until an admin resets it; the controller writes a reset token within five minutes | ❌ `swarm-bao-ci-runner` reads it when it starts, at boot or deploy, and restarts the runner when the value changed | +| `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge's admin API, at start and in a five-minute pass; its grant on the path is write-only | `swarm-bao-ci-runner` on the runner's host, under the runner's own certificate, into a file only `swarm-ci-runner-register` in the runner's container loads; no other principal's policy reads it | ❌ the forge hands out one token until an admin resets it; the controller writes a reset token within five minutes | ✅ `swarm-bao-ci-runner` reads it at boot, on deploy and every ten minutes, and restarts the runner when the value changed | | `swarm/services/review-classifier/anthropic-api-key` | the operator, by hand in the store's UI, field `api_key` | `swarm-review-classifier` in its container, under its own certificate, at the start of every run; held for that run and handed to `claude`'s environment only; no other principal's policy reads it | ❌ an operator's key; replaced by writing a new one | ✅ read on every run, so a written or removed key takes effect on the next run | | `swarm/services/review-classifier/forge/token` | `swarm-controller`, with the forge's admin API, for its `review-classifier` account, at start and in a five-minute pass; the secret publisher's policy denies it | `swarm-controller` under its own certificate, before it decides whether to mint, and on every review it posts as the account; the classifier service's policy doesn't read it | ✅ the controller's pass re-mints when the stored token is missing or no longer matches the forge (last eight characters and scopes) | ✅ the controller reads the store on every review it posts | | _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | must be stated | diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index a85feddf..206093a6 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -68,6 +68,7 @@ let baoCfg = config.services.hyperhive.swarm.bao; storeRetry = import ../lib/store-retry.nix { }; + storeRefreshTimer = import ../lib/store-refresh-timer.nix { }; atomicWriteSecret = import ../lib/atomic-write-secret.nix { }; refreshConsumer = import ../lib/refresh-consumer.nix { }; havePublisherIdentity = @@ -1033,8 +1034,9 @@ in # later attempt lands re-runs `forgejo-sso-source` (below). inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { + # No `RemainAfterExit`: the timer below starts this again, and + # starting an active unit is a no-op. Type = "oneshot"; - RemainAfterExit = true; SyslogIdentifier = "swarm-bao-forge-oidc"; TimeoutStartSec = 30; }; @@ -1113,6 +1115,8 @@ in ''; }; + systemd.timers.swarm-bao-forge-oidc = storeRefreshTimer "re-read the forge's OIDC client secret from the swarm secret store"; + # The reverse direction of the unit above: publish the swarm-controller's # freshly-minted forge token OUT of the container into the swarm secret # store, where the controller reads it wherever it runs. On the host and diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index 65d63e16..ee260bbf 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -51,6 +51,7 @@ let atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; refreshConsumer = import ./lib/refresh-consumer.nix { }; storeRetry = import ./lib/store-retry.nix { }; + storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; # ⚠️ tuwunel does NOT read the host path directly, and this indirection # is not ceremony. Upstream's own words: "under systemd the path must be @@ -617,8 +618,9 @@ in # later attempt lands restarts tuwunel (below). inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { + # No `RemainAfterExit`: the timer below starts this again, and + # starting an active unit is a no-op. Type = "oneshot"; - RemainAfterExit = true; SyslogIdentifier = "swarm-bao-matrix-oidc"; TimeoutStartSec = 30; }; @@ -689,6 +691,8 @@ in ''; }; + systemd.timers.swarm-bao-matrix-oidc = storeRefreshTimer "re-read the homeserver's OIDC client secret from the swarm secret store"; + # ⚠️ Deliberately NO `networking.hosts` entry for authelia's name. This # container resolves through the hive's dnsmasq at `bridgeIp` (see the # static resolv.conf below), and every `gateway.localNames` entry — diff --git a/nix/host-modules/lib/store-refresh-timer.nix b/nix/host-modules/lib/store-refresh-timer.nix new file mode 100644 index 00000000..8361d2e7 --- /dev/null +++ b/nix/host-modules/lib/store-refresh-timer.nix @@ -0,0 +1,27 @@ +# Timer for a host oneshot that fetches a credential from the swarm secret +# store: it starts the oneshot again 10 minutes after each run, so a value +# re-minted in the store lands on the host and restarts its consumer with no +# operator step. Ten minutes matches the agent's own store re-read +# (../../agent-modules/bao.nix). +# +# ⚠️ The oneshot must not be `RemainAfterExit`: starting a unit that stays +# `active (exited)` is a no-op, so the timer would fire and fetch nothing. +# A run that reads an unchanged value writes nothing and restarts nothing — +# its `secret_differs` gate and ./refresh-consumer.nix see to that. +# +# Gate the timer exactly as its service is gated: a timer whose unit does +# not exist fails to load. +# +# Pure function — NOT a NixOS module. Call it from a module's `let`: +# +# storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; +# systemd.timers.swarm-bao-foo = storeRefreshTimer "re-read foo's secret from the swarm secret store"; +{ }: +description: { + inherit description; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnUnitInactiveSec = "10min"; + RandomizedDelaySec = "1min"; + }; +} diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 76dfc680..bb0c2586 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -348,6 +348,7 @@ in atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; refreshConsumer = import ./lib/refresh-consumer.nix { }; storeRetry = import ./lib/store-retry.nix { }; + storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in { # Reachable from agent containers and nowhere else: this opens @@ -766,8 +767,9 @@ in # a later attempt lands restarts it (below). inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { + # No `RemainAfterExit`: the timer below starts this again, and + # starting an active unit is a no-op. Type = "oneshot"; - RemainAfterExit = true; SyslogIdentifier = fetchUnit; TimeoutStartSec = 30; }; @@ -828,6 +830,10 @@ in ''; }; + systemd.timers.${fetchUnit} = lib.mkIf fetches ( + storeRefreshTimer "re-read this hive's telemetry OIDC client secret from the swarm secret store" + ); + assertions = [ { # A hive authenticates to the swarm's collector as itself — there diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 8f05a1b6..2392aec7 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -1367,6 +1367,7 @@ let # rather than a convenience for nginx. privateNetwork = false; storeRetry = import ./lib/store-retry.nix { }; + storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in { imports = [ @@ -2566,8 +2567,9 @@ in # which this boot waits on. inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { + # No `RemainAfterExit`: the timer below starts this again, and + # starting an active unit is a no-op. Type = "oneshot"; - RemainAfterExit = true; SyslogIdentifier = "swarm-bao-forwarder-oidc"; TimeoutStartSec = 30; }; @@ -2639,6 +2641,8 @@ in refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderHostSecretPath} ''; }; + + systemd.timers.swarm-bao-forwarder-oidc = storeRefreshTimer "re-read the secret store forwarder's OIDC client secret from the store"; }) (lib.mkIf (deployCfg.bao.enable && haveServerTls) { diff --git a/nix/host-modules/swarm-ci-runner.nix b/nix/host-modules/swarm-ci-runner.nix index e890e80c..d8b0d747 100644 --- a/nix/host-modules/swarm-ci-runner.nix +++ b/nix/host-modules/swarm-ci-runner.nix @@ -91,6 +91,7 @@ let atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; refreshConsumer = import ./lib/refresh-consumer.nix { }; storeRetry = import ./lib/store-retry.nix { }; + storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; privateNetwork = true; in @@ -206,8 +207,9 @@ in ]; inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { + # No `RemainAfterExit`: the timer below starts this again, and + # starting an active unit is a no-op. Type = "oneshot"; - RemainAfterExit = true; SyslogIdentifier = "swarm-bao-ci-runner"; TimeoutStartSec = 30; }; @@ -268,6 +270,8 @@ in ''; }; + systemd.timers.swarm-bao-ci-runner = storeRefreshTimer "re-read the CI runner's registration token from the swarm secret store"; + systemd.services."container@${machine}" = caTrust.containerOrdering; containers.${machine} = { diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index ad0bd74f..2286f9a2 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -278,6 +278,7 @@ let # reaches this at 127.0.0.1:. privateNetwork = false; storeRetry = import ./lib/store-retry.nix { }; + storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in { # What the service IS to every hive is `swarm.grafana` in @@ -576,8 +577,9 @@ in # Grafana (below). inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { + # No `RemainAfterExit`: the timer below starts this again, and + # starting an active unit is a no-op. Type = "oneshot"; - RemainAfterExit = true; SyslogIdentifier = "swarm-bao-grafana-oidc"; # What actually bounds each attempt below. Stated here rather than left # to systemd's default, so the number a boot waits on is in the file @@ -665,6 +667,8 @@ in ''; }; + systemd.timers.swarm-bao-grafana-oidc = storeRefreshTimer "re-read Grafana's OIDC client secret from the swarm secret store"; + # The admin password: created in the store if it holds none, then copied # into the container, under Grafana's own leaf. Same shape as # `swarm-bao-nats-callout-xkey` in ./swarm-nats.nix. The grant is diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 44b12646..ce07b7e9 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -303,6 +303,7 @@ let # its own trust material. privateNetwork = false; storeRetry = import ./lib/store-retry.nix { }; + storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in { # What the collector IS to the swarm — the client it is registered as, where @@ -453,8 +454,9 @@ in # the collector (below). inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { + # No `RemainAfterExit`: the timer below starts this again, and + # starting an active unit is a no-op. Type = "oneshot"; - RemainAfterExit = true; SyslogIdentifier = "swarm-bao-otel-oidc"; # What actually bounds each attempt below. Stated here rather than left # to systemd's default, so the number a boot waits on is in the file @@ -532,6 +534,10 @@ in ''; }; + systemd.timers.swarm-bao-otel-oidc = lib.mkIf haveClientIdentity ( + storeRefreshTimer "re-read the swarm collector's OIDC client secret from the swarm secret store" + ); + # Where the delivery unit above lands the secret. `mkDefault`, so a # deployment delivering it some other way just sets the option — and # `mkIf haveClientIdentity` so a host with no store identity is left with diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 48326b2f..986bce99 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -73,6 +73,43 @@ let vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs); in builtins.foldl' (acc: v: builtins.seq v acc) true vals; + + # The host fetches that restart their consumer when the store's value + # changes (../host-modules/lib/refresh-consumer.nix), each on a fixture + # that renders it. + storeRefreshed = [ + { + unit = "swarm-bao-grafana-oidc"; + machine = allLocal; + } + { + unit = "swarm-bao-otel-oidc"; + machine = allLocal; + } + { + unit = "swarm-bao-matrix-oidc"; + machine = allLocal; + } + { + unit = "swarm-bao-forge-oidc"; + machine = allLocal; + } + { + unit = "swarm-bao-hive-otel-oidc"; + machine = allLocal; + } + { + unit = "swarm-bao-forwarder-oidc"; + machine = allLocal; + } + { + unit = "swarm-bao-ci-runner"; + machine = withCi; + } + ]; + + remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false; + cases = [ { # The domain is the stub's swarm domain, which both fixtures share. @@ -621,6 +658,42 @@ let } && lib.all (a: a.assertion) (noDomain { deploy.hive-controller.enable = false; }).assertions; } - ]; + { + # Control: a store fetch with no timer still renders `RemainAfterExit`, + # so the cases below read a real value rather than a missing one. + name = "Grafana's admin-password fetch stays active after it exits and has no timer"; + ok = + allLocal.systemd.services ? swarm-bao-grafana-admin + && remainsAfterExit allLocal.systemd.services.swarm-bao-grafana-admin + && !(allLocal.systemd.timers ? swarm-bao-grafana-admin); + } + { + # A timer whose unit does not exist fails to load, so each timer is + # gated exactly as its fetch is. + name = "a host running none of the store-refreshed fetches renders none of their timers"; + ok = lib.all ( + { unit, ... }: !(bare.systemd.services ? ${unit}) && !(bare.systemd.timers ? ${unit}) + ) storeRefreshed; + } + ] + # Starting a unit that stays `active (exited)` is a no-op, so a timer aimed + # at one fires and fetches nothing. + ++ map ( + { unit, machine }: + { + name = "${unit} re-runs on a ten-minute timer and does not stay active after it exits"; + ok = + let + s = machine.systemd.services; + t = machine.systemd.timers; + in + s ? ${unit} + && t ? ${unit} + && !(remainsAfterExit s.${unit}) + && lib.elem "timers.target" t.${unit}.wantedBy + && t.${unit}.timerConfig.OnUnitInactiveSec == "10min" + && (t.${unit}.timerConfig.Unit or "${unit}.service") == "${unit}.service"; + } + ) storeRefreshed; in runGroup "core-toggle" cases