diff --git a/docs/scheduler/ci.md b/docs/scheduler/ci.md index 5719856f..c8692d1c 100644 --- a/docs/scheduler/ci.md +++ b/docs/scheduler/ci.md @@ -159,8 +159,11 @@ needs `deploy.forgejo.enable` or a hive on its host. 2. **`swarm-bao-ci-runner.service`** on the runner's host logs in to the store with the `ci-runner` leaf, whose `swarm-ci-runner` role reads that one path and nothing else, retrying every 30 seconds until the token is there. It - writes the token to a root-only file inside the container, and restarts - the runner when the value changes. It runs at boot, on deploy, and every + writes the token to a root-only file in `/run/swarm-ci-runner` on the host, + which the container sees read-only, and restarts the runner when the value + changes. `/run` is tmpfs, so no copy reaches persistent disk; after a reboot + the unit fetches the token again, and until the store answers only a + runner that has no `.runner` yet waits. It runs at boot, on deploy, and every ten minutes on a timer. 3. **`swarm-ci-runner-register.service`** in the container is the only unit that loads that file, with `LoadCredential=`. systemd hands a unit's diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index d175db98..7fb61edf 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -75,7 +75,7 @@ of the cell says how. | `swarm/hives//queue/agent` | authelia | `swarm-bao-queue-agent` on the hive's host, under its own per-hive certificate; no agent's policy reaches it | must be stated | must be stated | | `swarm/hives//oidc/client` | authelia, at its first boot; `swarm-secret-publish` copies it in | hive-c0re, under the hive's own certificate, when it first connects to the queue | ❌ authelia mints it once. A re-mint is republished by `swarm-secret-publish`'s path unit | ✅ read until the first connect succeeds, retried with a backoff of up to five minutes while it's missing; then held in memory and read again when authelia refuses it | | `swarm/services//oidc/client` | authelia | that service's own reader, under its own certificate — the queue responder's under `swarm-nats-auth`'s, the homeserver's under `swarm-matrix-ctl`'s, the forge's under `swarm-forge-publisher`'s, the review classifier's under `swarm-review-classifier`'s; the bao UI's by the store's granter | must be stated | must be stated | -| `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge's admin API, at start and in a five-minute pass; its grant on the path is write-only | `swarm-bao-ci-runner` on the runner's host, under the runner's own certificate, into a file only `swarm-ci-runner-register` in the runner's container loads; no other principal's policy reads it | ❌ the forge hands out one token until an admin resets it; the controller writes a reset token within five minutes | ✅ `swarm-bao-ci-runner` reads it at boot, on deploy and every ten minutes, and restarts the runner when the value changed | +| `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge's admin API, at start and in a five-minute pass; its grant on the path is write-only | `swarm-bao-ci-runner` on the runner's host, under the runner's own certificate, into a file under the host's `/run` that only `swarm-ci-runner-register` in the runner's container loads; no other principal's policy reads it | ❌ the forge hands out one token until an admin resets it; the controller writes a reset token within five minutes | ✅ `swarm-bao-ci-runner` reads it at boot, on deploy and every ten minutes, and restarts the runner when the value changed | | `swarm/services//metrics/bearer` | `swarm-bao-grafana-metrics-bearer` on Grafana's host, under Grafana's certificate, when the store holds none | `swarm-metrics-auth` on the metrics store's host, under its own certificate, at most once a minute, to check the bearer Grafana's datasource presents at the store's read API | ❌ nothing re-mints it; an operator deletes it and restarts `swarm-bao-grafana-metrics-bearer` | ✅ `swarm-metrics-auth` reads it again once its copy is a minute old. ❌ Grafana reads its file copy only as it starts; the bearer unit restarts Grafana when the copy changes | | `swarm/services/review-classifier/anthropic-api-key` | the operator, by hand in the store's UI, field `api_key` | `swarm-review-classifier` in its container, under its own certificate, at the start of every API key run; held for that run and handed to `claude`'s environment only; no other principal's policy reads it | ❌ an operator's key; replaced by writing a new one | ✅ read on every run, so a written or removed key takes effect on the next run | | `swarm/services/review-classifier/acp-provider` | the operator, by hand in the store's UI, field `api_key` | `swarm-review-classifier` in its container, under its own certificate, at the start of every ACP run; held for that run and handed to `opencode`'s environment only; no other principal's policy reads it | ❌ an operator's key; replaced by writing a new one | ✅ read on every run, so a written or removed key takes effect on the next run | diff --git a/nix/host-modules/swarm-ci-runner.nix b/nix/host-modules/swarm-ci-runner.nix index 4c19f19c..127e5265 100644 --- a/nix/host-modules/swarm-ci-runner.nix +++ b/nix/host-modules/swarm-ci-runner.nix @@ -63,13 +63,16 @@ let # it, which is the spelling a step run as root (`+`) can follow. runnerFile = "/var/lib/gitea-runner/${instance}/.runner"; - # Inside the container, root-only. The token file holds the bare token and - # is a `LoadCredential=` of the register unit alone; the host unit below - # writes it through the container's root, at `hostTokenFile`. stateDir = "/var/lib/swarm-ci-runner"; - tokenFile = "${stateDir}/registration-token"; + # Root-only, on tmpfs: the token never reaches persistent disk, and after a + # reboot the host unit below fetches it again. The host unit writes it at + # `hostTokenFile`; the container sees the directory read-only at `tokenDir`, + # where the token file is a `LoadCredential=` of the register unit alone. + hostTokenDir = "/run/${machine}"; + hostTokenFile = "${hostTokenDir}/registration-token"; + tokenDir = "/run/${machine}"; + tokenFile = "${tokenDir}/registration-token"; tokenCredential = "registration-token"; - hostTokenFile = "/var/lib/nixos-containers/${machine}${tokenFile}"; # The epoch second the daemon last started. The precond reads the daemon's # journal from there. daemonStartedFile = "${stateDir}/daemon-started"; @@ -223,6 +226,12 @@ in ${atomicWriteSecret} ${refreshConsumer} + # The container's bind source, so it starts even when the store does not + # answer; tmpfiles creates it too, but unordered with the container on + # a switch. Root-owned: systemd reads a `LoadCredential=` source as + # root, before the runner drops to its dynamic user. + install -d -m 0700 ${lib.escapeShellArg hostTokenDir} + err="$(mktemp)" trap 'rm -f "$err"' EXIT @@ -253,9 +262,6 @@ in exit 1 fi - # Root-owned: systemd reads a `LoadCredential=` source as root, before - # the runner drops to its dynamic user. - install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostTokenFile)} if secret_differs ${lib.escapeShellArg hostTokenFile} "$token"; then atomic_write_secret 0600 "" ${lib.escapeShellArg hostTokenFile} "$token" fi @@ -270,6 +276,14 @@ in systemd.services."container@${machine}" = caTrust.containerOrdering; + systemd.tmpfiles.rules = [ + # The bind source must exist when the container starts. + "d ${hostTokenDir} 0700 root root -" + # `LoadCredential=` fails the unit when the file is missing; an empty + # one lets the register unit report the missing token. + "f ${hostTokenFile} 0600 root root -" + ]; + containers.${machine} = { autoStart = true; ephemeral = false; @@ -287,6 +301,10 @@ in hostPath = "/nix/var/nix/daemon-socket"; isReadOnly = false; }; + ${tokenDir} = { + hostPath = hostTokenDir; + isReadOnly = true; + }; } // caTrust.bindMount; @@ -355,11 +373,10 @@ in systemd.tmpfiles.rules = [ "d ${stateDir} 0700 root root -" - # `LoadCredential=` fails the unit when the file is missing; an - # empty one lets the register unit report the missing token. - "f ${tokenFile} 0600 root root -" # Unused: nothing in this module writes or reads them. The first - # holds a token; the others are upstream's re-registration markers. + # two hold a token; the others are upstream's re-registration + # markers. + "r ${stateDir}/registration-token" "r ${stateDir}/registration-token.env" "r /var/lib/private/gitea-runner/${instance}/.token-hash" "r /var/lib/private/gitea-runner/${instance}/.labels" diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index c3ab67cf..5ff17294 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -233,7 +233,7 @@ let let sc = registerService.serviceConfig; in - sc.LoadCredential == [ "registration-token:/var/lib/swarm-ci-runner/registration-token" ] + sc.LoadCredential == [ "registration-token:/run/swarm-ci-runner/registration-token" ] && sc.Type == "oneshot" && sc.User == runnerService.serviceConfig.User && sc.DynamicUser @@ -245,6 +245,24 @@ let s: lib.hasInfix "gitea-register-runner" (toString s) ) runnerService.serviceConfig.ExecStartPre); } + { + # A copy on persistent disk outlives the token's rotation and lands in + # backups of the container's root. + name = "the CI runner's registration token is written to the host's /run only, and the old copy removed"; + ok = + let + script = ciAlone.systemd.services.swarm-bao-ci-runner.script; + mount = ciAlone.containers.swarm-ci-runner.bindMounts."/run/swarm-ci-runner"; + guestRules = ciAlone.containers.swarm-ci-runner.config.systemd.tmpfiles.rules; + in + lib.hasInfix "atomic_write_secret 0600 \"\" /run/swarm-ci-runner/registration-token" script + && !(lib.hasInfix "/var/lib/nixos-containers" script) + && mount.hostPath == "/run/swarm-ci-runner" + && mount.isReadOnly + && lib.elem "d /run/swarm-ci-runner 0700 root root -" ciAlone.systemd.tmpfiles.rules + && lib.elem "r /var/lib/swarm-ci-runner/registration-token" guestRules + && !(lib.any (lib.hasPrefix "f /var/lib/swarm-ci-runner/registration-token") guestRules); + } { name = "the CI register unit registers from its credential, and only without a .runner"; ok =