diff --git a/.forgejo/workflows/coverage.yml b/.forgejo/workflows/coverage.yml index 2d38e361..2b61ecd0 100644 --- a/.forgejo/workflows/coverage.yml +++ b/.forgejo/workflows/coverage.yml @@ -4,7 +4,7 @@ name: coverage # than printed into the run log, so a scheduled run leaves something to fetch. # # ⚠️ The hour is deliberate: at the default job capacity of 1 -# (`services.hyperhive.deploy.forgejo.ci.concurrency`) this job holds the +# (`services.hyperhive.deploy.ci-runner.concurrency`) this job holds the # runner for its whole timeout and everything else queues behind it. # # Separate from ci.yml because a `workflow_dispatch` trigger there fires every diff --git a/.forgejo/workflows/flake-update.yml b/.forgejo/workflows/flake-update.yml index 6d366f4a..d67bd5f6 100644 --- a/.forgejo/workflows/flake-update.yml +++ b/.forgejo/workflows/flake-update.yml @@ -11,7 +11,7 @@ name: flake update # apply. Stale PRs just stack; newest lock wins. # # `PR_BOT_TOKEN` (repo secret) is the `flake-bot` account's write token. -# Self-signed forge cert: not worked around with `curl -k` - hive-ci.nix +# Self-signed forge cert: not worked around with `curl -k` - swarm-ci-runner.nix # already sets SSL_CERT_FILE to a bundle with the hive CA (wiring # confirmed; anchor coverage for the swarm forge specifically isn't). on: diff --git a/README.md b/README.md index e8a8cded..8f4a319d 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ the swarm — one of each, each on whichever host you choose ├── swarm-grafana dashboards over both stores │ └── opt-in, not part of singleHostSwarm - ├── hive-ci Forgejo Actions runner (deploy.forgejo.ci) + ├── swarm-ci-runner Forgejo Actions runner (deploy.ci-runner) ├── swarm-snapshot-store btrfs-receive endpoint for agent snapshots └── wg-hive WireGuard mesh between hives (deploy.wireguard) diff --git a/docs/README.md b/docs/README.md index dd8217e4..e39e8b79 100644 --- a/docs/README.md +++ b/docs/README.md @@ -91,7 +91,7 @@ declarations. [`scheduler/jobq.md`](scheduler/jobq.md) — operator-facing, no implementation detail. - **How does the rebuild queue work? What are the concrete step kinds, queue sources, scheduler internals?** → [`scheduler/coordinator.md`](scheduler/coordinator.md). -- **How does the CI runner work? What's the autoregistration flow?** → +- **How does the CI runner work? How does it register?** → [`scheduler/ci.md`](scheduler/ci.md). - **How do I export Claude Code metrics (tokens, cost, tool calls) to Prometheus/Grafana?** → [`scheduler/observability.md`](scheduler/observability.md). diff --git a/docs/integrations/forge.md b/docs/integrations/forge.md index 143f8f37..9cf22b40 100644 --- a/docs/integrations/forge.md +++ b/docs/integrations/forge.md @@ -31,12 +31,14 @@ by a test). swarm-controller mints every agent token with it: | `read:notification` | Poll `GET /notifications` for unread events. | | `write:notification` | Mark notifications read via `PATCH /notifications/threads/{id}`. | -**`CORE_TOKEN_SCOPES`** (`hive-c0re::forge`, for its own `core` user): -everything in `AGENT_TOKEN_SCOPES` plus `read:admin` and `write:admin`. -Site-admin membership alone isn't sufficient — Forgejo's token scope gate runs -before the user-permission check, so `/api/v1/admin/*` returns -`403 Forbidden` for any token without the admin scope bits, even when -the bearer is a site admin. +**`CORE_TOKEN_SCOPES`** (`hive-c0re::forge`, for its own `core` user): the +same set as `AGENT_TOKEN_SCOPES`, with no admin scope. Forgejo's token scope +gate runs before the user-permission check, so this token gets +`403 Forbidden` on `/api/v1/admin/*` even though `core` is a site admin. +hive-c0re only mints the token when the file at +`/var/lib/hyperhive/forge-core-token` is missing or the forge rejects it, so +a token minted with admin scopes keeps them until an operator deletes that +file and restarts hive-c0re. --- diff --git a/docs/networking/network.md b/docs/networking/network.md index d399fe85..194a5c44 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -34,21 +34,21 @@ that touches it. │ hive-br0 10.42.0.1/24 │ │ ┌──────────┼──────────────┐ │ └──────────────┼──────────┼──────────────┼───────────────────────────┘ - vb-h- vb-h- vb-hive-ci veth pairs + vb-h- vb-h- vb- veth pairs │ │ │ ┌────┴────┐ ┌───┴─────┐ ┌──────┴──┐ one private netns - │ agent a │ │ agent b │ │ hive-ci │ each; eth0 leases + │ agent a │ │ agent b │ │ runner │ each; eth0 leases │ eth0 │ │ eth0 │ │ eth0 │ from the DHCP pool └─────────┘ └─────────┘ └─────────┘ ``` -| container | netns | IPv4 | listens / reached via | -| -------------- | ----------------------- | -------------- | ----------------------------------------------------------------------------------------------- | -| gateway (host) | host | host addresses | nginx `:80`/`:443` (every vhost); dnsmasq `bridgeIp:53` + DHCP `:67` on the bridge | -| `hive-forge` | host (shared) | host addresses | forgejo `:3000` http, `:2222` git-ssh; fronted by the `forge.` vhost | -| `hive-matrix` | host (shared) | host addresses | tuwunel `:8008` (+ optional federation port); fronted by the matrix vhost | -| `hive-ci` | private, veth on bridge | DHCP pool | outbound only (runner → forge); no inbound surface | -| `h-` | private, veth on bridge | DHCP pool | web UI via UDS `/run/hive-agent/` → nginx sub-path; in-container UI port hashed 8100–8999 | +| container | netns | IPv4 | listens / reached via | +| ----------------- | ----------------------- | -------------- | ----------------------------------------------------------------------------------------------- | +| gateway (host) | host | host addresses | nginx `:80`/`:443` (every vhost); dnsmasq `bridgeIp:53` + DHCP `:67` on the bridge | +| `hive-forge` | host (shared) | host addresses | forgejo `:3000` http, `:2222` git-ssh; fronted by the `forge.` vhost | +| `hive-matrix` | host (shared) | host addresses | tuwunel `:8008` (+ optional federation port); fronted by the matrix vhost | +| `swarm-ci-runner` | private, veth on bridge | DHCP pool | outbound only (runner → forge); no inbound surface | +| `h-` | private, veth on bridge | DHCP pool | web UI via UDS `/run/hive-agent/` → nginx sub-path; in-container UI port hashed 8100–8999 | The flows, end to end: @@ -172,7 +172,7 @@ boot its `firewall-start` flushes the `nixos-fw` chains, rebuilds them from the container's (empty) port list, and deletes the host's `nixos-nat-*` chains without recreating them, silently wiping the bridge holes above plus the agents' NAT. Private-netns containers -(agents, hive-ci) may keep their own firewall — it's scoped to their +(agents, swarm-ci-runner) may keep their own firewall — it's scoped to their namespace. ### Reaching host services (`exposeHostPorts`) diff --git a/docs/scheduler/ci.md b/docs/scheduler/ci.md index 56dd7e1b..50df5869 100644 --- a/docs/scheduler/ci.md +++ b/docs/scheduler/ci.md @@ -1,13 +1,21 @@ -# hive-ci: Forgejo Actions Runner +# swarm-ci-runner: Forgejo Actions Runner -The `hive-ci` module runs a Forgejo Actions runner in a `hive-ci` nixos-container, -executing CI jobs from `.forgejo/workflows/ci.yml` on every PR. +The `swarm-ci-runner` module runs the swarm's Forgejo Actions runner in a +`swarm-ci-runner` nixos-container, executing CI jobs from +`.forgejo/workflows/ci.yml` on every PR. It's a swarm service: it runs on +whichever host sets it, and needs neither the forge nor a hive on that host. ## For operators -**Enabling it's one line**: `services.hyperhive.deploy.forgejo.ci.enable = true` -in the host NixOS config. No manual token provisioning — hive-c0re -registers the runner with the forge automatically. +**Enabling it's one line**: `services.hyperhive.deploy.ci-runner.enable = true` +in the NixOS config of the host that should run it. No manual token +provisioning: swarm-controller publishes the forge's runner registration token +to the secret store, and the runner reads it with a store identity of its own. +On the store's host `glue-bao-tls.nix` mints that identity; on any other host, copy +`ci-runner.pem` and `ci-runner-key.pem` from the store host's +`/var/lib/swarm-bao-pki/` and name them in +`services.hyperhive.deploy.bao.ciRunnerClientCertFile` and +`ciRunnerClientKeyFile`. Two things worth knowing before you rely on it: @@ -25,9 +33,8 @@ Two things worth knowing before you rely on it: disk-pressure GC config from _Host store maintenance_ below to your host's NixOS config (not optional if you plan to actually use this). -Everything below this point is implementation detail: exact -autoregistration internals, container design, and the full security -writeup. +Everything below this point is implementation detail: registration +internals, container design, and the full security writeup. ## CI checks @@ -105,62 +112,68 @@ slow); run those manually before pushing Rust changes. ## Configuration reference -The runner registers against the forge on its own host, so enable it on the -swarm's forge host (`services.hyperhive.deploy.forgejo.enable`, see -[`../swarm/services.md`](../swarm/services.md)). Evaluation fails on any other -host. There, nothing extra to enable beyond -`services.hyperhive.deploy.forgejo.ci.enable = true` (see _For operators_ above). +`services.hyperhive.deploy.ci-runner.enable = true` on the host that runs it +(see _For operators_ above). The runner reaches the forge at +`https://`, so that name has to resolve from the host: the +gateway's dnsmasq answers it locally where the forge runs, and forwards it to +the operator's DNS everywhere else. -Optional tuning: `services.hyperhive.deploy.forgejo.ci.name` (runner name in forge -admin panel), `concurrency` (parallel job capacity), `labels` (workflow -targeting), `jobTimeout` (per-job wall-clock cap, default `"1h"`, Go duration -string for example `"3h"` — a job that exceeds it's killed so a hung or runaway -build can't hold the runner's single slot indefinitely). +Optional tuning under `services.hyperhive.deploy.ci-runner`: `name` (runner +name in the forge admin panel), `concurrency` (parallel job capacity), `labels` +(workflow targeting; the default `hive-ci:host` is what this repo's workflows +name), `jobTimeout` (per-job wall-clock cap, default `"1h"`, Go duration string +for example `"3h"` — a job that exceeds it's killed so a hung or runaway build +can't hold the runner's single slot indefinitely). ## Container design -- **Private netns, bridge-attached**: the container runs in its own network namespace (`privateNetwork = true`, `hostBridge`) and reaches hive-forge through the gateway at `http://` (resolved to the bridge IP via `networking.extraHosts`). It can't reach host-loopback services — the core dashboard at `127.0.0.1:7000` and the raw forge port are unreachable from CI. -- **Non-ephemeral**: runner credentials persist across restarts (written to container's stateDir on first registration, reused thereafter). +- **Private netns, bridge-attached**: the container runs in its own network namespace (`privateNetwork = true`, `hostBridge`), takes its address from the gateway's DHCP, and resolves through the gateway's dnsmasq on the bridge IP. It can't reach host-loopback services — the core dashboard at `127.0.0.1:7000` and the raw forge port are unreachable from CI. +- **Non-ephemeral**: runner credentials (`.runner`) persist across restarts, written to the container's state dir on registration and reused after. - **Sandbox fallback**: nspawn containers can't create user-namespaces, so nix's sandboxing would always fail. Module sets `nix.settings.sandbox-fallback = true` in the container — nix builds run unsandboxed (safe because the container is already isolated). See `docs/process/gotchas.md`. -- **Credential isolation**: the forge admin token (`forge-core-token`) never enters the container. hive-c0re holds it and performs all forge API calls (runner validation + registration-token mint, in `forge/ci_runner.rs`); via hive-priv it writes only the runner registration token to the host env-file `/run/hive-ci/runner-token`, which the container bind-mounts read-only. +- **Credential isolation**: no forge admin credential enters the container. swarm-controller holds the forge admin token; the runner holds only the registration token and its own `.runner`, and the host unit that fetches the token holds a store identity that reads that one path. -## Autoregistration flow +## Registration -Registration is **off the container's boot-critical path** — hive-c0re owns -it and runs it out of band, so a slow forge or core-token never delays the -container's start. Gotcha: don't gate the container's own start on a forge -round-trip (a host-side unit that did this could exceed the nspawn start -timeout and trip a restart loop) — registration must stay something -hive-c0re drives after the container is already up. hive-c0re holds the -core admin token only on the host; only the runner registration token -reaches the container. +1. **swarm-controller** (`swarm-controller/src/forge/runner_token.rs`) reads + the forge's instance-wide token from + `GET /api/v1/admin/runners/registration-token` at start and every five + minutes, and writes it to the secret store at + `swarm/services/ci-runner/registration-token` when it differs from the + value it last wrote. Its store grant on that path is write-only. +2. **`swarm-bao-ci-runner.service`** on the runner's host logs in to the store + with the `ci-runner` leaf, whose `swarm-ci-runner` role reads that one path + and nothing else, retrying every 30 seconds until the token is there. It + writes `TOKEN=` to the runner's `EnvironmentFile` inside the + container, and restarts the runner when the value changes. It runs at boot + and on deploy, not on a timer. +3. **The runner unit** (`gitea-runner-swarm` in the container) registers in + the upstream pre-start step whenever `.runner` is missing or the token or + labels changed. A pre-start step of this module's, ahead of it, removes + `.runner` in two cases upstream doesn't check: + - The forge has no record of the runner. Forgejo answers every call from an + unknown runner with `unregistered runner`, and the daemon exits 1 when its + startup `Declare` gets that answer. On the next start the step finds the + message in the daemon's journal since its last start, and registers again. + A daemon that's already running when an admin deletes its runner keeps polling + and logging the error without exiting; the next restart of the unit or + container registers it again. + - `.runner` names another forge address than the configured one. -### hive-c0re side (`forge/ci_runner.rs`, run during the startup sweep) + Until a token arrives the step fails, and `Restart=on-failure` retries every + 15 seconds with no start limit. -Gated on `HYPERHIVE_FORGE_CI_ENABLED` (the nix module sets it on `hive-c0re.service` when `deploy.forgejo.ci.enable`). Best-effort — hive-c0re logs failures and never aborts the sweep; a healthy runner is never restarted. - -1. If `.runner` exists at `/var/lib/nixos-containers/hive-ci/var/lib/gitea-runner/hive/.runner`, validate its id against `GET /api/v1/admin/runners/{id}` with the core admin token: - - **200**: still registered — done, no restart. - - **404 / other non-200 / malformed**: stale — re-register (below). - - **transport error** (forge unreachable): keep the existing creds; a network blip must not wipe a valid runner. -2. If absent or stale: mint a fresh token from `GET /api/v1/admin/runners/registration-token`, then hand it to hive-priv's `RegisterCiRunner`, which (as root) writes `TOKEN=` **in place** to the host env-file `/run/hive-ci/runner-token` (preserving the inode nspawn pinned into the container at start) and restarts `gitea-runner-hive.service` inside the container so it picks up the credential and registers. - -### Container side - -- The container boots immediately — nothing gates its start on registration. -- tmpfiles seeds `/run/hive-ci/runner-token` with `TOKEN=placeholder` so the runner's `EnvironmentFile` always exists. -- `gitea-runner-hive.service` has an `ExecStartPre` precond (ahead of the nix-daemon wait) that **fails fast** unless it's already registered (`.runner` present) or a real, non-placeholder token is in place. `Restart=on-failure` (no start-limit cap) self-heals it: a runner that precond-fails at boot keeps retrying until hive-c0re writes the token (c0re's explicit restart is the primary path; the retry is the safety net). -- **Convergence**: because the token write targets the _host_ file, even if c0re's restart races the container being down, the container later starts, reads the now-real token, passes the precond, and registers on its own. +A new registration is a new runner on the forge: the previous entry stays in +the forge admin panel as offline until an admin deletes it. ## Actions checkout mirror -When `deploy.forgejo.ci.enable` is set, the swarm controller autoseeds an -`actions/checkout` pull-mirror on the local forge and sets Forgejo's -`DEFAULT_ACTIONS_URL` to point at the local instance. This means CI -`uses: actions/checkout@vN` steps resolve entirely on loopback — no -external DNS on the CI critical path. +With `services.hyperhive.deploy.forgejo.actionsFromSelf` set on the forge's +host, the swarm controller seeds an `actions/checkout` pull-mirror on the forge +and Forgejo's `DEFAULT_ACTIONS_URL` points at the forge itself. CI +`uses: actions/checkout@vN` steps then resolve on the forge, with no external +DNS on the CI critical path. @@ -192,7 +205,7 @@ to avoid provisioning collisions. ## Security: unsandboxed builds and trusted contributors -**hive-ci should only run CI for trusted contributors.** The security boundary is weaker than it looks: +**The runner should only run CI for trusted contributors.** The security boundary is weaker than it looks: ### What unsandboxed builds mean @@ -200,10 +213,10 @@ nspawn containers can't create user-namespaces, so the module sets `nix.settings A malicious `default.nix` or build script in a PR can therefore: -- **Make arbitrary network requests** to any address reachable from the container. The container runs in its own netns behind the hive bridge, so it reaches the forge only through the gateway (`http://`, public/read endpoints — no admin credentials) and **can't** reach host-loopback services: the unauthenticated core dashboard at `127.0.0.1:7000` and the raw forge port are off-limits (a firewall rule drops bridge→127.0.0.0/8 traffic). +- **Make arbitrary network requests** to any address reachable from the container. The container runs in its own netns behind the hive bridge, so it reaches the forge by name at `https://` (no admin credentials) and **can't** reach host-loopback services: the unauthenticated core dashboard at `127.0.0.1:7000` and the raw forge port are off-limits (a firewall rule drops bridge→127.0.0.0/8 traffic). - **Write to the container filesystem**, including corrupting the runner's state dir or `.runner` credentials. -The core admin token (`forge-core-token`) **isn't** bind-mounted into the container. it's held and used only by hive-c0re on the host (`forge/ci_runner.rs`), which mints per-runner registration tokens; only that registration token reaches the container's env-file. A build process can still reach forge over the network, but can't use the admin token to issue privileged API calls. +No forge admin token enters the container: swarm-controller holds it and publishes only the registration token, which reaches the container's env-file. A build process can still reach the forge over the network, and it runs as the runner's own user, so it can read `.runner` and act as this runner; it can't issue admin API calls. Note: `nix flake check --no-build` (eval-only) reduces the attack surface but doesn't eliminate it — `builtins.fetchGit`, `builtins.fetchurl`, and import-from-derivation can reach the network and filesystem during evaluation. The default CI workflow runs full `nix flake check` (builds derivations), which is the higher-risk path. @@ -221,7 +234,7 @@ The current design is appropriate for a trusted-team hive where all contributors ## Host store maintenance (recommended) The CI runner builds derivations through the **host** nix-daemon — the -hive-ci container shares the host store and has no daemon of its own. Build +`swarm-ci-runner` container shares the host store and has no daemon of its own. Build outputs accumulate in `/nix/store` with no automatic collection, and a busy CI day can fill the disk until every job fails fast with `ENOSPC`. @@ -278,6 +291,7 @@ builder — disk-pressure collection at least runs while nothing is mid-copy. ## References -- `nix/host-modules/hive-ci.nix`: runner configuration, autoregistration script, container setup. +- `nix/host-modules/swarm-ci-runner.nix`: runner configuration, token fetch, re-registration step, container setup. +- `swarm-controller/src/forge/runner_token.rs`: publishes the registration token. - `.forgejo/workflows/ci.yml`: workflow definition. - `docs/process/gotchas.md`: nix sandboxing limitations in containers. diff --git a/docs/scheduler/coordinator.md b/docs/scheduler/coordinator.md index eed67688..a98f1f46 100644 --- a/docs/scheduler/coordinator.md +++ b/docs/scheduler/coordinator.md @@ -474,8 +474,8 @@ Two things to know about the weights: same value and the weight does _not_ rank agents against each other. What `80` buys is that agents yield to everything **not** on this drop-in path: host services (nginx and dnsmasq among them) and the - infra containers (`hive-ci`, `hive-forge`, `hive-gateway`, - `hive-matrix`), which stay at the kernel default of `100`. + infra containers (`hive-forge`, `hive-gateway`, `hive-matrix`), which + stay at the kernel default of `100`. - `IOWeight=` is only honoured when the backing device runs the BFQ scheduler or has blk-iocost QoS enabled. On a host using `none`/`mq-deadline`/`kyber` without iocost, systemd writes the value diff --git a/docs/swarm/README.md b/docs/swarm/README.md index d3c3982d..b0a76b62 100644 --- a/docs/swarm/README.md +++ b/docs/swarm/README.md @@ -265,7 +265,7 @@ one per hive. It ensures them at start and every five minutes after whitelists = the `operators` team, and no user. The controller leaves a repo with no `main` rule alone; - the pull-mirrors from `deploy.forgejo.mirrors` on the controller's - host (with the `actions/checkout` one `deploy.forgejo.ci.enable` adds); + host (with the `actions/checkout` one `deploy.forgejo.actionsFromSelf` adds); - `internal/docs` (private) and `internal/knowledge` (public, with a seed `README.md` while empty); - the `agent-configs` org avatar diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index 21b62274..89bff47c 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -213,7 +213,7 @@ Two consumers, and only one of them is fine: next to every agent's flake. The bundle is the runtime-to-build-time bridge. - **The Matrix container crosses the same bridge**, via the shared - `lib/hive-ca-trust.nix` helper that `hive-ci` and `hive-forge` already + `lib/hive-ca-trust.nix` helper that `hive-forge` and `swarm-ci-runner` use: the bundle is bind-mounted read-only into the container, and a oneshot concatenates it with the system CAs before tuwunel starts. diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index 186721a8..d2aadec2 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -69,6 +69,7 @@ of the cell says how. | `swarm/hives//matrix/sender-token` | `swarm-controller`, with the swarm's appservice token, for every hive in its directory in a five-minute pass | `swarm-controller` under its own certificate, before it decides whether to mint, and hive-c0re's `stored_sender_token()`, under the hive's own certificate | ✅ the controller's pass re-mints when the stored token is missing, unknown to the homeserver, or someone else's | ✅ hive-c0re's matrix sweep reads the store every run and overwrites its token file when the store's token differs | | `swarm/hives//queue/agent` | authelia | `swarm-bao-queue-agent` on the hive's host, under its own per-hive certificate; no agent's policy reaches it | must be stated | must be stated | | `swarm/services//oidc/client` | authelia | that service's own reader, under its own certificate — the queue responder's under `swarm-nats-auth`'s, the homeserver's under `swarm-matrix-ctl`'s; the bao UI's by the store's granter | must be stated | must be stated | +| `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge's admin API, at start and in a five-minute pass; its grant on the path is write-only | `swarm-bao-ci-runner` on the runner's host, under the runner's own certificate, into the runner's `EnvironmentFile`; no other principal's policy reads it | ❌ the forge hands out one token until an admin resets it; the controller writes a reset token within five minutes | ❌ `swarm-bao-ci-runner` reads it when it starts, at boot or deploy, and restarts the runner when the value changed | | _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | must be stated | diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 313e4051..93789f78 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -302,7 +302,7 @@ next controller restart. ### Per-principal identities Bao matches a cert-auth role on the certificate's subject, so a certificate is an -identity and sharing one merges the identities. These four units read one path +identity and sharing one merges the identities. These five units read one path each and each holds a leaf, a role and a policy of its own: | unit | option pair under `deploy.bao.` | reads | @@ -311,20 +311,23 @@ each and each holds a leaf, a role and a policy of its own: | `swarm-bao-grafana-oidc` | `grafanaOidcClientCertFile` / `grafanaOidcClientKeyFile` | `swarm/services//oidc/client` | | `swarm-bao-otel-oidc` | `otelOidcClientCertFile` / `otelOidcClientKeyFile` | `swarm/services//oidc/client` | | `swarm-bao-forwarder-oidc` | `forwarderOidcClientCertFile` / `forwarderOidcClientKeyFile` | `swarm/services//oidc/client` | +| `swarm-bao-ci-runner` | `ciRunnerClientCertFile` / `ciRunnerClientKeyFile` | `swarm/services/ci-runner/registration-token` | The first exists **per hive**, because the path it reads carries a hive name and every hive runs its own reader. Its subject is `-`; `swarm.nix` reserves that composed spelling as a hive name, so nobody can name a hive into another hive's -role. The other three read a path that names a swarm service rather than a hive, +role. The other four read a path that names a swarm service rather than a hive, so one role each is enough and their subjects are the flat -`deploy.bao.grafanaOidcCommonName`, `deploy.bao.otelOidcCommonName` and -`deploy.bao.forwarderOidcCommonName`. +`deploy.bao.grafanaOidcCommonName`, `deploy.bao.otelOidcCommonName`, +`deploy.bao.forwarderOidcCommonName` and `deploy.bao.ciRunnerCommonName`. -On a host that mints its own PKI, `glue-bao-tls.nix` signs all four and defaults -all eight options, and there is nothing to do. Elsewhere you issue each leaf from +On a host that mints its own PKI, `glue-bao-tls.nix` signs all five and defaults +all ten options, and there is nothing to do. Elsewhere you issue each leaf from that CA out of band and name it here — one file per principal rather than one file -shared by four, which is the whole of what this buys. +shared by five, which is the whole of what this buys. `swarm-ci-runner.nix` +refuses the build wherever the runner runs without its pair, as +`swarm-grafana.nix` does for Grafana. On a host without the store, where the forwarder never runs, forgetting one of the other six isn't a quiet degrade. A hand-configured remote-store hive that diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 40181a0d..423f4bd7 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -221,3 +221,16 @@ has one either way, reached by its swarm name through the gateway. Agent-side configuration, and what a hive's own collector does, are in [`../scheduler/observability.md`](../scheduler/observability.md). + +### CI runner + +One Forgejo Actions runner, in a `swarm-ci-runner` container on the host that +sets `deploy.ci-runner.enable`. It doesn't derive from `allSwarmServices`: a +runner executes pull-request code, so running one is a decision of its own. + +It needs neither the forge nor a hive on its host. It reaches the forge at +`https://`, and registers with the token swarm-controller +publishes to the secret store, which it reads under its own `ci-runner` leaf. +On a host without the store, that leaf is the one file to copy. Registration, +the container and the threat model are in +[`../scheduler/ci.md`](../scheduler/ci.md). diff --git a/docs/tools/hivectl-cli.md b/docs/tools/hivectl-cli.md index 5bf96253..99bb2b1e 100644 --- a/docs/tools/hivectl-cli.md +++ b/docs/tools/hivectl-cli.md @@ -600,7 +600,6 @@ Bare `hivectl stop` stops everything; scope flags narrow it to specific sub-agen * `--agents` — All sub-agent containers * `--agent ` — A specific sub-agent by name. Repeatable: `--agent a --agent b` -* `--ci` — The CI runner container (`hive-ci`) * `--forge` — The forge container (`hive-forge`) * `--gateway` — The gateway (`hive-gateway`) — nginx on the host, not a container * `--matrix` — The matrix container (`hive-matrix`) @@ -621,7 +620,6 @@ Bare `hivectl start` restores the agents stopped by the last broad-scope `stop` * `--agents` — All sub-agent containers * `--agent ` — A specific sub-agent by name. Repeatable: `--agent a --agent b` -* `--ci` — The CI runner container (`hive-ci`) * `--forge` — The forge container (`hive-forge`) * `--gateway` — The gateway (`hive-gateway`) — nginx on the host, not a container * `--matrix` — The matrix container (`hive-matrix`) @@ -641,7 +639,6 @@ Bare `hivectl restart` restarts everything; scope flags narrow it. * `--agents` — All sub-agent containers * `--agent ` — A specific sub-agent by name. Repeatable: `--agent a --agent b` -* `--ci` — The CI runner container (`hive-ci`) * `--forge` — The forge container (`hive-forge`) * `--gateway` — The gateway (`hive-gateway`) — nginx on the host, not a container * `--matrix` — The matrix container (`hive-matrix`) diff --git a/docs/trust-boundary/boundary.md b/docs/trust-boundary/boundary.md index dfc81a67..eda24e83 100644 --- a/docs/trust-boundary/boundary.md +++ b/docs/trust-boundary/boundary.md @@ -56,13 +56,13 @@ Concretely, the core daemon's dashboard `/api` carries **no application-layer authentication** — the core daemon serves operator-authority routes unauthenticated at the HTTP layer. Their protection is entirely (a) the gateway, which fronts all operator traffic and is where operator auth lives, -and (b) network isolation, which keeps agents — and `hive-ci`'s untrusted PR -builds — off host-loopback so nothing can reach `127.0.0.1:` +and (b) network isolation, which keeps agents — and the CI runner's untrusted +PR builds — off host-loopback so nothing can reach `127.0.0.1:` directly. This is deliberate given the load-bearing role of network isolation above, but it's a standing invariant: the `/api` must never bind to a non-loopback address or get exposed outside the gateway, and every new -operator-authority route inherits that assumption. hyperhive treats `hive-ci` -like an agent for this purpose — it runs untrusted PR code and is +operator-authority route inherits that assumption. hyperhive treats +`swarm-ci-runner` like an agent for this purpose — it runs untrusted PR code and is netns-isolated for the same reason. The boundary rests on three layers: diff --git a/docs/trust-boundary/security.md b/docs/trust-boundary/security.md index da648a5a..91578cf3 100644 --- a/docs/trust-boundary/security.md +++ b/docs/trust-boundary/security.md @@ -205,9 +205,9 @@ Unsandboxed nix builds run as `nixbld` users (non-root, typically UIDs 30001-300 Without sandbox isolation, a build derivation's builder script has read access to any file in the container that the nixbld user can read. -The blast radius also has a **network** dimension. hive-ci runs its unsandboxed -builds of untrusted PR code in its own private netns behind the hive bridge: a -build reaches the forge only through the gateway and can't reach host-loopback +The blast radius also has a **network** dimension. `swarm-ci-runner` runs its +unsandboxed builds of untrusted PR code in its own private netns behind the +bridge: a build reaches the forge by name and can't reach host-loopback services — including the core dashboard at `127.0.0.1:`, which has no application-layer auth of its own (see [`docs/scheduler/ci.md`](../scheduler/ci.md)). The `0600` token policy bounds file reads; network isolation bounds network reach. @@ -286,7 +286,6 @@ known operations; there is no arbitrary command pass-through: | `ControlInfraContainer` | `systemctl container@.service` — the `InfraContainer` enum is the allowlist, and serde rejects unknown names at the wire boundary (`hive-c0re` has no variant, so no request can name it) | | `SyncAgentTmpfiles` | unlink `/etc/tmpfiles.d/hyperhive-agents.conf` and return `Ok` | | `SetAgentPaused` | create / remove the `//harness/paused` marker that parks an agent's turn loop | -| `RegisterCiRunner` | write `/run/hive-ci/runner-token` (host path, root-owned) then `systemctl --machine=hive-ci restart gitea-runner-hive.service`. Only the registration token crosses; the forge admin token never enters the container | | `EnsureAgentSubvolume` | `btrfs subvolume create /` for a new agent — no-op when the path exists or the filesystem isn't btrfs | | `UpgradeAgentSubvolume` | migrate an existing plain state dir into a subvolume: create, `cp -a --reflink=auto`, atomic swap. Operator opt-in, and the caller stops the agent first | | `DeleteAgentSubvolume` | `btrfs subvolume delete /` — purge path only, never a plain destroy | @@ -301,7 +300,7 @@ known operations; there is no arbitrary command pass-through: **Container allowlist** — `hive-priv` validates every request against an allowlist before any operation: the allowlist accepts only names matching the agent-name convention (char-validated) or the known -sibling service containers (`hive-forge`, `hive-matrix`, `hive-ci`), +sibling service containers (`hive-forge`, `hive-matrix`), and rejects arbitrary container names. `hive-gateway` is a host unit, not a container, so it's not in this list — see `ReloadGatewayNginx` above for how the hard-coded unit name scopes its access instead. diff --git a/docs/web-ui/dashboard.md b/docs/web-ui/dashboard.md index 654ce9b1..d316ec74 100644 --- a/docs/web-ui/dashboard.md +++ b/docs/web-ui/dashboard.md @@ -515,15 +515,15 @@ deep-link directly to a specific agent's journal. A "fetched N ago" chip appears after the `↻ refresh` button following each successful fetch and ticks every 30 s. -**INFRA sub-tab** — journald viewer for the four hive infrastructure -containers (`hive-ci`, `hive-forge`, `hive-gateway`, `hive-matrix`), a +**INFRA sub-tab** — journald viewer for the three hive infrastructure +containers (`hive-forge`, `hive-gateway`, `hive-matrix`), a fixed client-side list (`INFRA_NAMES` in `logs.js` — no dashboard API exposes just the name list). No unit filter (infra containers don't run the per-agent hive daemons) — always the full machine journal (or, for the gateway, the host journal filtered to its own unit). Fetches `GET /api/journal/{name}?lines=500`, same "fetched N ago" ticker as AGENT. A `?agent=` deep-link routes here instead of AGENT when -the name is one of the four infra containers. +the name is one of the three infra containers. **SYSTEM sub-tab** — host-side service logs. Unit selector (`hive-c0re.service` / `hive-priv.service`). Fetches diff --git a/flake.nix b/flake.nix index a0f82a2e..69807366 100644 --- a/flake.nix +++ b/flake.nix @@ -167,7 +167,6 @@ lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.swagger-ui-theme; }; - hive-ci = ./nix/host-modules/hive-ci.nix; hive-forge = ./nix/host-modules/hive-forge; }; diff --git a/frontend/packages/dashboard/src/core.html b/frontend/packages/dashboard/src/core.html index 1bdcaae8..b74cc42c 100644 --- a/frontend/packages/dashboard/src/core.html +++ b/frontend/packages/dashboard/src/core.html @@ -20,7 +20,7 @@ /builds.html (the build lifecycle hub). Default tab: K3PT ST4T3. The section
ids (tombstones-section, container-load-section) match what core.js's renderers target. Hive infra containers - (`hive-ci`, `hive-forge`, `hive-gateway`, `hive-matrix`) have no + (`hive-forge`, `hive-gateway`, `hive-matrix`) have no dashboard panel — `hivectl stop`/`start`/`restart` is the only control surface. -->
diff --git a/frontend/packages/dashboard/src/logs.html b/frontend/packages/dashboard/src/logs.html index 00b854de..f8ad20ac 100644 --- a/frontend/packages/dashboard/src/logs.html +++ b/frontend/packages/dashboard/src/logs.html @@ -60,11 +60,11 @@ select an agent above - + Deep-link: ?agent=hive-forge routes here instead of the AGENT tab. -->
bool { - std::env::var("HYPERHIVE_FORGE_CI_ENABLED") - .is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("true")) -} - -/// Ensure the hive-ci runner is registered against the forge. Best-effort: -/// every failure is logged and swallowed so it never aborts the startup -/// sweep ([`super::ensure_all`]). No-op when CI is disabled or the runner -/// already holds valid credentials (so a healthy runner is never restarted). -pub(super) async fn ensure_ci_runner_registered(core_token: &str) { - if !ci_enabled() { - return; - } - // Existing creds still valid on the forge, AND pointed at the forge - // address we are configured for → nothing to do. - if let Some(id) = existing_runner_id() - && runner_address_current() - && runner_valid(core_token, id).await - { - return; - } - // Absent or stale → mint a fresh registration token and hand it to - // hive-priv (root) to write the env-file + restart the runner. - match fetch_registration_token(core_token).await { - Ok(token) => { - if let Err(e) = crate::priv_client::register_ci_runner(&token).await { - tracing::warn!(error = ?e, "ci runner: hive-priv register_ci_runner failed"); - crate::warnings::set_boot_warning( - "ci_runner_register", - "crit", - format!("ci runner: hive-priv register_ci_runner failed: {e}"), - ); - } else { - tracing::info!("ci runner: registered hive-ci runner with a fresh token"); - } - } - Err(e) => { - tracing::warn!(error = ?e, "ci runner: fetch registration token failed"); - crate::warnings::set_boot_warning( - "ci_runner_fetch_token", - "crit", - format!("ci runner: fetch registration token failed: {e}"), - ); - } - } -} - -/// Parse the runner id from the persisted `.runner` JSON, if present and sane -/// (a `0` id is the gitea-runner "unregistered" sentinel). -fn existing_runner_id() -> Option { - let raw = std::fs::read_to_string(RUNNER_FILE).ok()?; - let json: Value = serde_json::from_str(&raw).ok()?; - let id = json.get("id")?.as_u64()?; - (id != 0).then_some(id) -} - -/// Whether the persisted `.runner` still names the forge host we are -/// configured for. -/// -/// `act_runner` records the `--instance` URL it was registered with and the -/// daemon reads the address from there and nowhere else — the nix option only -/// reaches `register`, and upstream re-registers on a changed token or labels, -/// never on a changed URL. So a forge rename leaves a runner dialling a name -/// its container's `extraHosts` no longer resolves, with every existing check -/// still green: the runner id is untouched, so `runner_valid` says yes. -/// -/// Fail-safe in the same direction as [`runner_valid`]: anything we cannot -/// read or parse returns `true` (keep the existing creds). Only a *positive* -/// disagreement between two parseable hosts is treated as stale, because the -/// cost of a false stale is re-registering on every boot. -fn runner_address_current() -> bool { - let Ok(raw) = std::fs::read_to_string(RUNNER_FILE) else { - return true; - }; - runner_address_matches(&raw, forge_http_base()) -} - -/// Host-only comparison of `.runner`'s `address` against the configured base. -/// -/// Split from [`runner_address_current`] so it is testable without the -/// filesystem. Compares **hosts**, not whole URLs: both strings come from the -/// same nix expression (`http://${swarm.forge.domain}`, reaching the runner as -/// `instances.hive.url` and this daemon as `HIVE_FORGE_URL`), but scheme, -/// port and trailing slash are exactly the kind of cosmetic difference that -/// would otherwise re-register the runner on every boot. -fn runner_address_matches(raw: &str, configured: &str) -> bool { - let Some(address) = serde_json::from_str::(raw) - .ok() - .and_then(|j| j.get("address")?.as_str().map(str::to_owned)) - else { - return true; - }; - match (Url::parse(&address), Url::parse(configured)) { - (Ok(a), Ok(b)) => match (a.host_str(), b.host_str()) { - (Some(a), Some(b)) => a.eq_ignore_ascii_case(b), - _ => true, - }, - _ => true, - } -} - -/// Bound on reaching the forge. -const HTTP_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); -/// Bound on one whole forge admin call, body included. -const HTTP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); - -/// A client with both forge bounds applied. -fn http_client() -> Result { - reqwest::Client::builder() - .connect_timeout(HTTP_CONNECT_TIMEOUT) - .timeout(HTTP_TIMEOUT) - .build() - .context("build HTTP client") -} - -/// Context for `what` failing with `e`; a timeout names the bound that fired. -fn http_context(what: &str, e: &reqwest::Error) -> String { - if e.is_connect() && e.is_timeout() { - format!("{what}: connect timed out after {HTTP_CONNECT_TIMEOUT:?}") - } else if e.is_timeout() { - format!("{what}: timed out after {HTTP_TIMEOUT:?}") - } else { - what.to_owned() - } -} - -/// `GET /admin/runners/{id}` — `true` iff the runner still exists on the forge -/// (HTTP 200). A 404 (deleted from the admin panel) or any other status means -/// re-registration is needed. A transport error (forge unreachable) is treated -/// as "keep the existing creds" so a network blip never wipes a valid runner. -async fn runner_valid(core_token: &str, id: u64) -> bool { - let url = format!("{}/api/v1/admin/runners/{id}", forge_http_base()); - let client = match http_client() { - Ok(c) => c, - Err(e) => { - tracing::warn!(error = ?e, "ci runner: build HTTP client failed; keeping existing creds"); - return true; - } - }; - match client - .get(&url) - .header("Authorization", format!("token {core_token}")) - .send() - .await - { - Ok(resp) => resp.status().is_success(), - Err(e) => { - let what = http_context("validation request failed", &e); - tracing::warn!(error = ?e, "ci runner: {what}; keeping existing creds"); - true - } - } -} - -/// `GET /admin/runners/registration-token` — mint a fresh registration token. -/// forgejo-api (0.11) doesn't wrap this endpoint, so this is a raw request -/// against the local HTTP forge ([`forge_http_base`]), mirroring the prefetch's -/// former curl; Forgejo accepts `Authorization: token `. -async fn fetch_registration_token(core_token: &str) -> Result { - let url = format!( - "{}/api/v1/admin/runners/registration-token", - forge_http_base() - ); - let resp = http_client()? - .get(&url) - .header("Authorization", format!("token {core_token}")) - .send() - .await - .map_err(|e| { - let what = http_context("GET admin/runners/registration-token", &e); - anyhow::Error::new(e).context(what) - })?; - let status = resp.status(); - let json: Value = resp.json().await.map_err(|e| { - let what = http_context("parse registration-token response", &e); - anyhow::Error::new(e).context(what) - })?; - if !status.is_success() { - anyhow::bail!("registration-token HTTP {status}: {json}"); - } - json.get("token") - .and_then(Value::as_str) - .map(str::to_owned) - .context("registration-token response missing 'token' field") -} - -#[cfg(test)] -mod tests { - use super::runner_address_matches; - - fn runner(address: &str) -> String { - format!(r#"{{"id":7,"uuid":"u","name":"hive","token":"t","address":"{address}"}}"#) - } - - /// The case the bug is about: a forge rename leaves the old host behind. - #[test] - fn differing_host_is_stale() { - assert!(!runner_address_matches( - &runner("http://forge.pr1ma.darkest.space"), - "http://forge.constellation.darkest.space", - )); - } - - /// Control for the arm above — without it, a helper that always returned - /// `false` would pass `differing_host_is_stale` and re-register forever. - #[test] - fn same_host_is_current() { - assert!(runner_address_matches( - &runner("http://forge.example"), - "http://forge.example", - )); - } - - /// Cosmetic differences must NOT count: both sides are rendered from one - /// nix expression, but a port or trailing slash appearing on one side is - /// not a rename. - #[test] - fn port_scheme_and_trailing_slash_do_not_count() { - assert!(runner_address_matches( - &runner("http://forge.example:80/"), - "https://FORGE.example", - )); - } - - /// Fail-safe: anything unreadable keeps the existing credentials, so a - /// malformed or older `.runner` never costs a working runner. - #[test] - fn unparseable_input_keeps_creds() { - assert!(runner_address_matches("not json", "http://forge.example")); - assert!(runner_address_matches( - r#"{"id":7}"#, - "http://forge.example" - )); - assert!(runner_address_matches( - &runner("://nonsense"), - "http://forge.example", - )); - } -} diff --git a/hive-c0re/src/forge/mod.rs b/hive-c0re/src/forge/mod.rs index 34f10b0a..50e90e7e 100644 --- a/hive-c0re/src/forge/mod.rs +++ b/hive-c0re/src/forge/mod.rs @@ -4,7 +4,6 @@ //! teams and repos). //! No-op when `hive-forge` isn't running. Full design: `docs/integrations/forge.md`. -mod ci_runner; mod pr_comment; mod reconcile; mod repos; @@ -233,8 +232,8 @@ pub async fn sync_agent(name: &str, core_token: Option<&str>) -> bool { } /// The `core_token.is_some()` half of [`ensure_all`]: the meta repo, the -/// local knowledge clone, the core avatar, CI runner registration, and -/// removing this hive's own config-PR hook — +/// local knowledge clone, the core avatar, and removing this hive's own +/// config-PR hook — /// every step that needs an authenticated forge client. The swarm-wide /// objects (orgs, the `operators` team, mirrors, `internal/docs`, /// `internal/knowledge`, the `agent-configs` avatar) are the @@ -271,9 +270,6 @@ async fn ensure_all_orgs_and_repos(token: &str) { format!("forge: core avatar upload failed: {e}"), ); } - // Register the hive-ci Actions runner (off the container's boot path; - // no-op when CI is disabled or the runner already holds valid creds). - ci_runner::ensure_ci_runner_registered(token).await; if let Err(e) = remove_config_pr_webhook(token).await { tracing::warn!(error = ?e, "forge: remove_config_pr_webhook failed"); } diff --git a/hive-c0re/src/forge/users.rs b/hive-c0re/src/forge/users.rs index 4b313aa1..4bbdc059 100644 --- a/hive-c0re/src/forge/users.rs +++ b/hive-c0re/src/forge/users.rs @@ -38,13 +38,10 @@ fn core_avatar_png_path() -> std::path::PathBuf { std::path::PathBuf::from(dir).join("branding/hyperhive.png") } -/// Bootstrap `core` token scopes — adds `read:admin,write:admin` on -/// top of the agent scopes (swarm-controller's `AGENT_TOKEN_SCOPES`) -/// so the host daemon can drive `/api/v1/admin/*`. Site-admin -/// membership alone isn't enough: the token's own scope gate runs -/// before the user-permission check. -/// See `docs/integrations/forge.md::Token scopes`. -const CORE_TOKEN_SCOPES: &str = "read:admin,write:admin,read:user,write:user,read:notification,write:notification,write:repository,write:issue,write:organization,write:misc"; +/// Bootstrap `core` token scopes — the same set as swarm-controller's +/// `AGENT_TOKEN_SCOPES`, with no admin scope: nothing in hive-c0re calls +/// `/api/v1/admin/*`. See `docs/integrations/forge.md::Token scopes`. +const CORE_TOKEN_SCOPES: &str = "read:user,write:user,read:notification,write:notification,write:repository,write:issue,write:organization,write:misc"; /// Pull the access token out of forgejo's success message. Format /// has shifted across versions (table form vs. "Access token was @@ -197,9 +194,7 @@ pub(super) async fn ensure_core_avatar(token: &str) -> Result<()> { /// Outcome of probing whether the persisted core token still works /// against the *current* forge. Existence on disk is not validity: a /// token minted before a forge rebuild / re-provision is unknown to the -/// new forge's DB and 401s on every call — which silently breaks the -/// hive-ci runner-registration prefetch (it reads this same token to -/// fetch a runner registration token). +/// new forge's DB and 401s on every call. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum CoreTokenCheck { /// Token authenticated successfully — keep using it. @@ -260,8 +255,8 @@ async fn check_core_token(token: &str) -> CoreTokenCheck { /// Ensure the bootstrap `core` admin user + a token at /// `CORE_TOKEN_PATH`. The token is what hive-c0re uses for forgejo -/// API calls (org creation, meta-repo push, and the hive-ci -/// runner-registration prefetch). Returns the token. +/// API calls (meta-repo push, collaborator grants, PR comments). +/// Returns the token. /// /// Idempotent, but validity-aware: when a token file is already present /// it is **probed against the current forge** before being trusted. A diff --git a/hive-c0re/src/paths.rs b/hive-c0re/src/paths.rs index b281d9d7..c72e2a88 100644 --- a/hive-c0re/src/paths.rs +++ b/hive-c0re/src/paths.rs @@ -242,7 +242,7 @@ pub fn gateway_agents_conf() -> PathBuf { /// (used in `Path::new` + user-facing `format!` messages), so it stays a /// const rather than a `PathBuf` fn. // nix: read at provisioning; the same literal is in -// `nix/host-modules/hive-c0re/default.nix` and `nix/host-modules/hive-ci.nix` — must match. +// `nix/host-modules/hive-c0re/default.nix` — must match. pub const FORGE_CORE_TOKEN: &str = "/var/lib/hyperhive/forge-core-token"; /// `/run/hyperhive` — the runtime root (host admin socket + per-agent dirs). diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index 9550204f..287a5c2e 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -369,25 +369,8 @@ pub async fn set_agent_paused(agent_name: &str, paused: bool) -> Result<()> { .await?) } -/// Register the hive-ci Forgejo Actions runner: hand the freshly-minted -/// registration token to hive-priv, which writes it to the host-side -/// `/run/hive-ci/runner-token` env-file and restarts the in-container runner. -/// The forge admin token stays in hive-c0re; only the registration token -/// crosses to the (host-path) env-file the container bind-mounts read-only. -/// -/// # Errors -/// Propagates the hive-priv call failure: the socket / IPC error, or the -/// root-side error when the token is rejected (empty or control characters), -/// the env-file write fails, or the runner restart exits non-zero. -pub async fn register_ci_runner(token: &str) -> Result<()> { - ok(call(&PrivRequest::RegisterCiRunner { - token: token.to_owned(), - }) - .await?) -} - -/// Start / stop a hive infrastructure service (`hive-ci`, `hive-gateway`, -/// `hive-forge`, `hive-matrix`) on the host via `systemctl `, +/// Start / stop a hive infrastructure service (`hive-gateway`, `hive-forge`, +/// `hive-matrix`) on the host via `systemctl `, /// where the unit is derived root-side from the variant /// (`container@.service`, or `nginx.service` for the gateway). The /// [`InfraContainer`] enum is the allowlist — hive-priv needs no name diff --git a/hive-c0re/src/server.rs b/hive-c0re/src/server.rs index 7a63dc35..de085d53 100644 --- a/hive-c0re/src/server.rs +++ b/hive-c0re/src/server.rs @@ -883,9 +883,6 @@ async fn scoped_agents(scope: &LifecycleScope) -> Result> { fn scoped_infra(scope: &LifecycleScope) -> Vec { let everything = scope.is_everything(); let mut out = Vec::new(); - if everything || scope.ci { - out.push(InfraContainer::Ci); - } if everything || scope.forge { out.push(InfraContainer::Forge); } diff --git a/hive-host-sock/src/lib.rs b/hive-host-sock/src/lib.rs index d670854d..e2bc0c78 100644 --- a/hive-host-sock/src/lib.rs +++ b/hive-host-sock/src/lib.rs @@ -387,12 +387,12 @@ pub struct QuotaRow { /// (the bare `hivectl stop` / `start`); set individual fields to restrict /// (e.g. only `agents` → just the sub-agent containers). `agents` covers /// every managed sub-agent container; the rest are the named infra -/// services (`hive-ci`, `hive-forge`, `hive-gateway`, `hive-matrix`) — +/// services (`hive-forge`, `hive-gateway`, `hive-matrix`) — /// containers, bar the gateway, which is the host's nginx unit. // // A flat bag of independent flag toggles — one bool per selectable // container class — is exactly the right shape here: each maps 1:1 to a -// `hivectl` `--ci` / `--forge` / `--gateway` / `--matrix` flag, and they're +// `hivectl` `--forge` / `--gateway` / `--matrix` flag, and they're // orthogonal (any subset is valid), so a state machine or two-variant enums // would only obscure the mapping. Hence the `struct_excessive_bools` allow. #[allow(clippy::struct_excessive_bools)] @@ -407,8 +407,6 @@ pub struct LifecycleScope { #[serde(default)] pub agent_names: Vec, #[serde(default)] - pub ci: bool, - #[serde(default)] pub forge: bool, #[serde(default)] pub gateway: bool, @@ -421,8 +419,7 @@ impl LifecycleScope { /// classes" (the bare `hivectl stop` / `start` with no scope flags). #[must_use] pub fn is_everything(&self) -> bool { - !(self.agents || self.ci || self.forge || self.gateway || self.matrix) - && self.agent_names.is_empty() + !(self.agents || self.forge || self.gateway || self.matrix) && self.agent_names.is_empty() } } diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index 7fcbe007..ad2b5418 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -45,7 +45,7 @@ pub const AGENT_PREFIX: &str = "h-"; /// [`PrivRequest::ControlInfraContainer`], and it has one variant more than /// this list (the gateway). Keep the distinction — a name that belongs to /// no container has no business reaching a `-M` / `nixos-container` call. -pub const SIBLING_CONTAINERS: &[&str] = &["hive-forge", "hive-matrix", "hive-ci"]; +pub const SIBLING_CONTAINERS: &[&str] = &["hive-forge", "hive-matrix"]; /// Lifecycle verb for [`PrivRequest::ControlInfraContainer`]. Maps directly /// to `systemctl `, where the unit comes from @@ -75,14 +75,13 @@ impl InfraAction { /// unknown or unsafe target — notably `hive-c0re`, which has no variant and /// would sever the daemon socket — is *unrepresentable* rather than caught /// by a runtime check. The c0re↔hive-priv wire form uses serde's default -/// variant naming (`"Ci"`, `"Forge"`, …); it's an internal protocol (both +/// variant naming (`"Forge"`, `"Gateway"`, …); it's an internal protocol (both /// ends rebuild together) so it needn't match the container name. /// [`name`](Self::name) is the separate stable identity string -/// (`hive-ci`), and [`service_unit`](Self::service_unit) the systemd unit +/// (`hive-forge`), and [`service_unit`](Self::service_unit) the systemd unit /// it actually resolves to. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum InfraContainer { - Ci, Forge, Gateway, Matrix, @@ -108,23 +107,21 @@ impl InfraContainer { /// Every controllable infra target. A superset of /// [`SIBLING_CONTAINERS`] — not all of these are containers (see the /// test). - pub const ALL: [InfraContainer; 4] = [ - InfraContainer::Ci, + pub const ALL: [InfraContainer; 3] = [ InfraContainer::Forge, InfraContainer::Gateway, InfraContainer::Matrix, ]; - /// Stable identity string, e.g. `hive-ci`. This is what the operator + /// Stable identity string, e.g. `hive-forge`. This is what the operator /// types, what the dashboard displays, and what [`FromStr`](std::str::FromStr) /// parses — it stays `hive-gateway` even though the gateway is no /// longer a container, because it names the *service*, not its /// implementation. (Distinct from the serde wire form, which is the - /// default variant name `"Ci"`.) + /// default variant name `"Forge"`.) #[must_use] pub fn name(self) -> &'static str { match self { - InfraContainer::Ci => "hive-ci", InfraContainer::Forge => "hive-forge", InfraContainer::Gateway => "hive-gateway", InfraContainer::Matrix => "hive-matrix", @@ -153,7 +150,7 @@ impl InfraContainer { impl std::str::FromStr for InfraContainer { type Err = (); - /// Parse an infra name (`hive-ci`, …) into a variant. `Err(())` for + /// Parse an infra name (`hive-forge`, …) into a variant. `Err(())` for /// anything that isn't one. fn from_str(s: &str) -> Result { Self::ALL.into_iter().find(|c| c.name() == s).ok_or(()) @@ -478,19 +475,6 @@ pub enum PrivRequest { paused: bool, }, - /// Register the hive-ci Forgejo Actions runner: write the registration - /// token to the host-side `/run/hive-ci/runner-token` env-file (root-owned, - /// bind-mounted read-only into the container) as `TOKEN=`, then - /// `systemctl --machine=hive-ci restart gitea-runner-hive.service` so the - /// runner picks up the credential. hive-c0re holds the forge admin token - /// and mints the registration token; only the registration token is written - /// here, and only to a host path — the admin token never enters the - /// container. The token is validated single-line + non-empty root-side. - RegisterCiRunner { - /// Forge runner registration token minted by hive-c0re. - token: String, - }, - /// Start / stop a hive infrastructure container on the host via /// `systemctl container@.service`. The /// [`InfraContainer`] enum is the allowlist — serde rejects unknown / @@ -810,8 +794,8 @@ mod tests { #[test] fn service_unit_wraps_containers_but_not_host_units() { assert_eq!( - InfraContainer::Ci.service_unit(), - "container@hive-ci.service" + InfraContainer::Forge.service_unit(), + "container@hive-forge.service" ); assert_eq!(InfraContainer::Gateway.service_unit(), "nginx.service"); } diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index a886be3f..f8202673 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -407,8 +407,6 @@ async fn exec( paused, } => exec_set_agent_paused(agent_name, paused), - PrivRequest::RegisterCiRunner { ref token } => register_ci_runner(token).await, - PrivRequest::ControlInfraContainer { container, action } => { control_infra_container(container, action).await } @@ -1167,107 +1165,6 @@ async fn daemon_reload() -> Result<(String, String)> { Ok((String::new(), String::new())) } -/// Host path to the hive-ci runner's persisted registration credentials. -/// -/// Paired with `hive-c0re`'s `forge::ci_runner::RUNNER_FILE`, which reads the -/// same file to decide whether a runner is registered and whether it still -/// names the configured forge host. Deliberately duplicated rather than shared: -/// `hive-priv` is the minimal root helper and does not depend on `hive-c0re`. -const RUNNER_CREDENTIALS: &str = - "/var/lib/nixos-containers/hive-ci/var/lib/gitea-runner/hive/.runner"; - -/// Delete the runner's persisted credentials so upstream's `ExecStartPre` takes -/// its **absence** branch on the next start. -/// -/// Absence is the state we want, so `NotFound` is success. Anything else — a -/// permission error above all — is NOT swallowed: it means the file is still -/// there, the restart will take upstream's already-registered branch, and the -/// caller would return `Ok` for a registration that never happened. That is the -/// same shape as a precondition that "passes" because it could not read the file -/// it was checking, and it is worth failing loudly to avoid. -/// -/// Split from [`register_ci_runner`] purely so this rule is testable without a -/// container or a `systemctl`. -fn clear_runner_credentials(path: &str) -> Result<()> { - match std::fs::remove_file(path) { - Ok(()) => Ok(()), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(e) => { - Err(anyhow::Error::new(e).context(format!("remove stale runner credentials {path}"))) - } - } -} - -/// `RegisterCiRunner` — write the runner registration token to the host-side -/// `/run/hive-ci/runner-token` env-file, then restart the in-container runner -/// so it re-registers. The forge admin token never enters the container; only -/// the registration token c0re passes here is written, and it lands on a host -/// path bind-mounted read-only into hive-ci. -async fn register_ci_runner(token: &str) -> Result<(String, String)> { - use std::io::Write as _; - use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _}; - // Reject anything that could corrupt the `KEY=VALUE` env-file or smuggle a - // second line — a forge registration token is an opaque single-line string. - if token.is_empty() || token.contains(['\n', '\r', '\0']) { - bail!("ci runner registration token empty or contains control characters"); - } - let token_path = "/run/hive-ci/runner-token"; - // In-place truncate+write of the existing inode (mirrors the prefetch's - // `echo > $FILE`), NOT a temp+rename: nspawn pins this file's inode into - // hive-ci at container start, so a rename would leave the running runner - // reading the old content. Format + perms match the tmpfiles seed and the - // prefetch: `TOKEN=`, mode 0600, root-owned. Created 0600 when the - // seed is missing, so the token is never on disk at a wider mode. - std::fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .mode(0o600) - .open(token_path) - .and_then(|mut f| f.write_all(format!("TOKEN={token}\n").as_bytes())) - .with_context(|| format!("write {token_path}"))?; - std::fs::set_permissions(token_path, std::fs::Permissions::from_mode(0o600)) - .with_context(|| format!("chmod {token_path}"))?; - // Remove the persisted credentials BEFORE restarting, or the restart is a - // no-op as far as registration goes. - // - // Upstream's `ExecStartPre` only re-registers when `.runner` is absent, the - // labels changed, or the *registration token hash* changed — never when the - // instance URL changed. c0re only calls this helper once it has already - // decided the existing credentials are absent or stale - // (`forge::ci_runner::ensure_ci_runner_registered` returns early otherwise), - // so by the time we are here a re-registration is exactly what is wanted and - // deleting the file is the narrow way to guarantee it happens. - // - // Writing a fresh token is NOT sufficient on its own: whether the hash - // changes depends on whether the forge mints a new registration token per - // request or hands back a stable one, which is Forgejo's behaviour to - // choose and change. Gating our remediation on the absence branch — the one - // upstream evaluates unconditionally — makes that question moot instead of - // load-bearing. - // - // See [`clear_runner_credentials`] for why absence is the branch we aim at - // and why only `NotFound` counts as success. - clear_runner_credentials(RUNNER_CREDENTIALS)?; - // Restart the in-container runner so it reads the new token and registers. - let out = Command::new("systemctl") - .args(["--machine=hive-ci", "restart", "gitea-runner-hive.service"]) - .output() - .await - .context("systemctl restart gitea-runner-hive.service in hive-ci")?; - if !out.status.success() { - bail!( - "systemctl restart gitea-runner-hive.service in hive-ci exited {}: {}", - out.status, - String::from_utf8_lossy(&out.stderr).trim() - ); - } - Ok(( - String::from_utf8_lossy(&out.stdout).into_owned(), - String::from_utf8_lossy(&out.stderr).into_owned(), - )) -} - /// `ControlInfraContainer` — start/stop a hive infrastructure service via /// `systemctl `. The [`InfraContainer`] enum is the allowlist: /// serde already rejected any unknown / unsafe name (hive-c0re has no @@ -3196,12 +3093,11 @@ fn ensure_socket_dir_in(root: &Path, name: &str) -> Result<()> { mod tests { use super::{ BindMount, BoundedRun, OwnedFd, PAUSED_MARKER_FILE, PrivRequest, StagedFile, - check_fd_agreement, clear_runner_credentials, contains_secret_shaped_run, - describe_forge_admin, ensure_plain_filename, ensure_socket_dir_in, git_overlay_flags, - limits_dropin_body, open_dir, open_export_dest, partial_name, publish_file, - redact_secret_line, remove_marker_in, run_bounded, single_output_path, toplevel_attr, - validate_credential_name, validate_snapshot_name, write_agent_dir_file, - write_bridge_dns_marker_in, + check_fd_agreement, contains_secret_shaped_run, describe_forge_admin, + ensure_plain_filename, ensure_socket_dir_in, git_overlay_flags, limits_dropin_body, + open_dir, open_export_dest, partial_name, publish_file, redact_secret_line, + remove_marker_in, run_bounded, single_output_path, toplevel_attr, validate_credential_name, + validate_snapshot_name, write_agent_dir_file, write_bridge_dns_marker_in, }; use std::path::PathBuf; use std::sync::atomic::{AtomicU32, Ordering}; @@ -4035,60 +3931,6 @@ mod tests { assert!(serde_json::from_str::(r#"{"op":"sync_agent_tmpfilez"}"#).is_err()); } - /// The runner-credential clear, in all three states that matter. The - /// PRESENCE arm is the load-bearing one: an implementation that did nothing - /// at all would pass the "absent is fine" arm perfectly, and the whole point - /// of the call is that the file is *gone* afterwards — upstream re-registers - /// on absence and on nothing else we control. - #[test] - fn clearing_runner_credentials_removes_it_and_tolerates_absence() { - let dir = scratch(); - let path = dir.join(".runner"); - let path_str = path.to_str().unwrap(); - - // Absent → success (this is the state we are aiming for). - assert!(!path.exists()); - clear_runner_credentials(path_str).unwrap(); - - // Present → success AND actually gone. Without this arm a no-op passes. - std::fs::write(&path, r#"{"id":7,"address":"http://old.invalid"}"#).unwrap(); - assert!( - path.exists(), - "control: the file must exist before the clear" - ); - clear_runner_credentials(path_str).unwrap(); - assert!( - !path.exists(), - "stale credentials must be GONE, or the restart takes upstream's \ - already-registered branch and registration silently never happens" - ); - - std::fs::remove_dir_all(&dir).ok(); - } - - /// A failure that is not `NotFound` must propagate, never read as success. - /// A directory in the file's place makes `remove_file` fail with a non- - /// `NotFound` error without needing to drop privileges in a test. - #[test] - fn clearing_runner_credentials_propagates_a_real_failure() { - let dir = scratch(); - let path = dir.join(".runner"); - std::fs::create_dir(&path).unwrap(); - - let err = clear_runner_credentials(path.to_str().unwrap()) - .expect_err("a non-NotFound failure must NOT be reported as success"); - assert!( - format!("{err:#}").contains("remove stale runner credentials"), - "error must name what it failed to do, got: {err:#}" - ); - assert!( - path.exists(), - "nothing was removed, and the caller must know" - ); - - std::fs::remove_dir_all(&dir).ok(); - } - /// The pause marker round-trips through the same root-only path the /// credential writes use, and BOTH directions are idempotent — the /// dashboard toggle and `hivectl pause|resume` fire blind, without diff --git a/hivectl/src/cli.rs b/hivectl/src/cli.rs index 93499ddb..89348e78 100644 --- a/hivectl/src/cli.rs +++ b/hivectl/src/cli.rs @@ -202,9 +202,6 @@ pub struct ScopeArgs { /// A specific sub-agent by name. Repeatable: `--agent a --agent b`. #[arg(long = "agent", value_name = "NAME")] agent: Vec, - /// The CI runner container (`hive-ci`). - #[arg(long)] - ci: bool, /// The forge container (`hive-forge`). #[arg(long)] forge: bool, @@ -221,7 +218,6 @@ impl ScopeArgs { hive_host_sock::LifecycleScope { agents: self.agents, agent_names: self.agent.clone(), - ci: self.ci, forge: self.forge, gateway: self.gateway, matrix: self.matrix, diff --git a/hivectl/src/subvol.rs b/hivectl/src/subvol.rs index f6740fb8..5e901fb2 100644 --- a/hivectl/src/subvol.rs +++ b/hivectl/src/subvol.rs @@ -16,7 +16,6 @@ fn single_agent_scope(name: &str) -> hive_host_sock::LifecycleScope { hive_host_sock::LifecycleScope { agents: false, agent_names: vec![name.to_owned()], - ci: false, forge: false, gateway: false, matrix: false, diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index fa9aa670..95351c5c 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -17,7 +17,6 @@ ./deploy.nix ./local-defaults.nix ./hive-c0re - ./hive-ci.nix ./hive-forge ./hive-gateway ./hive-matrix-service.nix @@ -49,6 +48,7 @@ ./swarm-bao-service.nix ./swarm-bao.nix ./swarm-ca.nix + ./swarm-ci-runner.nix ./swarm-secret-publisher.nix ./swarm-nats-service.nix ./swarm-nats.nix diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index f2ba81a1..9fcebbc1 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -20,9 +20,9 @@ # # ⚠️ Each entry is an attrset with an `enable`, not a bare bool, so a # service that grows a second *deployment* decision has somewhere to put -# it — `deploy.forgejo = { enable; ci; }` is then an ordinary addition -# rather than a migration. `ci` ("does this host run the runner too") is -# exactly that shape, and a bare bool leaves it unrepresentable. +# it — `deploy.nats = { enable; autoGenerateCallout; }` is then an ordinary +# addition rather than a migration, where a bare bool leaves it +# unrepresentable. # # ⚠️ The renames below are deliberately in this one file rather than # spread across the service modules, so all of them have a single home @@ -31,6 +31,32 @@ { lib, config, ... }: let deployCfg = config.services.hyperhive.deploy; + + # A removed CI-runner option. Its replacement has the same name under + # `deploy.ci-runner`. + ciRunnerRemoved = + from: opt: + lib.mkRemovedOptionModule + ( + [ + "services" + "hyperhive" + ] + ++ from + ++ [ opt ] + ) + '' + The CI runner is the swarm service in swarm-ci-runner.nix. It runs on + any host, needs neither the forge nor a hive there, and registers with + the token swarm-controller publishes to the secret store. Set + services.hyperhive.deploy.ci-runner.${opt} instead. + + The runner registers as a new runner on the forge, so delete the old + one in the forge's admin panel (Site administration, Actions, Runners). + To keep resolving `uses: actions/checkout` from the forge, set + services.hyperhive.deploy.forgejo.actionsFromSelf = true on its host. + See docs/scheduler/ci.md. + ''; in { imports = [ @@ -146,33 +172,20 @@ in [ "services" "hyperhive" "deploy" "swarm-otel" "clientSecretFile" ] ) - # The CI runner, and the only entry here that renames more than an - # `enable`: every knob under it describes the runner THIS host would run, - # so leaving `name`/`concurrency`/`labels`/`package` in the namespace that - # must be identical swarm-wide would keep the original defect for four - # more options. Renamed one by one because `ci` is a plain attrset of - # options rather than a submodule type, so there is no parent path to - # rename in a single entry. - (lib.mkRenamedOptionModule - [ "services" "hyperhive" "swarm" "forge" "ci" "enable" ] - [ "services" "hyperhive" "deploy" "forgejo" "ci" "enable" ] - ) - (lib.mkRenamedOptionModule - [ "services" "hyperhive" "swarm" "forge" "ci" "name" ] - [ "services" "hyperhive" "deploy" "forgejo" "ci" "name" ] - ) - (lib.mkRenamedOptionModule - [ "services" "hyperhive" "swarm" "forge" "ci" "concurrency" ] - [ "services" "hyperhive" "deploy" "forgejo" "ci" "concurrency" ] - ) - (lib.mkRenamedOptionModule - [ "services" "hyperhive" "swarm" "forge" "ci" "labels" ] - [ "services" "hyperhive" "deploy" "forgejo" "ci" "labels" ] - ) - (lib.mkRenamedOptionModule - [ "services" "hyperhive" "swarm" "forge" "ci" "package" ] - [ "services" "hyperhive" "deploy" "forgejo" "ci" "package" ] - ) + # Removed rather than renamed: the runner is a different container with a + # registration of its own, so an old definition is a deployment to re-read + # rather than a path to forward. + (ciRunnerRemoved [ "swarm" "forge" "ci" ] "enable") + (ciRunnerRemoved [ "swarm" "forge" "ci" ] "name") + (ciRunnerRemoved [ "swarm" "forge" "ci" ] "concurrency") + (ciRunnerRemoved [ "swarm" "forge" "ci" ] "labels") + (ciRunnerRemoved [ "swarm" "forge" "ci" ] "package") + (ciRunnerRemoved [ "deploy" "forgejo" "ci" ] "enable") + (ciRunnerRemoved [ "deploy" "forgejo" "ci" ] "name") + (ciRunnerRemoved [ "deploy" "forgejo" "ci" ] "concurrency") + (ciRunnerRemoved [ "deploy" "forgejo" "ci" ] "labels") + (ciRunnerRemoved [ "deploy" "forgejo" "ci" ] "package") + (ciRunnerRemoved [ "deploy" "forgejo" "ci" ] "jobTimeout") # The queue's callout identity. What stays under `swarm.nats` is what the # queue IS to every hive — the ports it answers on and the client id it is @@ -216,8 +229,7 @@ in # # `mirrors` renames in one entry rather than one per field: it is a single # option of a list-of-submodule type, so the rename carries its whole - # value. The `ci` block above needs five because it is a plain attrset of - # separate options, which is the case with no parent path to rename. + # value. (lib.mkRenamedOptionModule [ "services" "hyperhive" "swarm" "forge" "openFirewall" ] [ "services" "hyperhive" "deploy" "forgejo" "openFirewall" ] @@ -451,9 +463,8 @@ in ]; # ⚠️ Only `deploy.forgejo.enable` is declared here. The rest of - # `deploy.forgejo` is in ./hive-forge/default.nix and ./hive-ci.nix: the - # knobs live with the module that reads them; this file stays the registry - # of toggles. + # `deploy.forgejo` is in ./hive-forge/default.nix: the knobs live with the + # module that reads them; this file stays the registry of toggles. options.services.hyperhive.deploy = { forgejo.enable = lib.mkOption { type = lib.types.bool; @@ -602,6 +613,23 @@ in ''; }; + ci-runner.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = '' + Run the swarm's Forgejo Actions runner in a `swarm-ci-runner` + container on this host. It reaches the forge at + {option}`services.hyperhive.swarm.forge.domain` and registers with + the token swarm-controller publishes to the secret store, so it + needs neither the forge nor a hive on this host. + + Not derived from + {option}`services.hyperhive.deploy.allSwarmServices`: a runner + executes pull-request code, so running one is a decision of its own. + ''; + }; + hive-controller.enable = lib.mkOption { type = lib.types.bool; default = false; diff --git a/nix/host-modules/glue-bao-readers-policy-order.nix b/nix/host-modules/glue-bao-readers-policy-order.nix index 6bd3c318..f65c90cb 100644 --- a/nix/host-modules/glue-bao-readers-policy-order.nix +++ b/nix/host-modules/glue-bao-readers-policy-order.nix @@ -45,6 +45,8 @@ let swarm-bao-forwarder-oidc = true; # ./swarm-nats.nix: the queue's TLS leaf, not a secret, but the same wait. swarm-bao-nats-tls = deployCfg.nats.enable; + # ./swarm-ci-runner.nix + swarm-bao-ci-runner = deployCfg.ci-runner.enable; }; # Readers that log in under another unit's principal: reader unit → the diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index 835d376c..8771a808 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -105,6 +105,8 @@ in # The store forwarder's. forwarderOidcClientCertFile = lib.mkDefault "${pkiDir}/forwarder-oidc.pem"; forwarderOidcClientKeyFile = lib.mkDefault "${pkiDir}/forwarder-oidc-key.pem"; + ciRunnerClientCertFile = lib.mkDefault "${pkiDir}/ci-runner.pem"; + ciRunnerClientKeyFile = lib.mkDefault "${pkiDir}/ci-runner-key.pem"; # The granter: every `swarm-bao-*-policy` unit on this host logs in with # it. ⚠️ Unlike every other leaf here, never the file an operator copies: @@ -256,6 +258,12 @@ in [ -s ${pkiDir}/nats-auth.pem ] || ${signLeaf} ${pkiDir} nats-auth \ ${lib.escapeShellArg deployCfg.bao.natsAuthCommonName} "" clientAuth + # The CI runner's token fetch. Minted here for the matrix-ctl leaf's + # reason: the runner is a swarm singleton, so on a runner host without + # the store this is the file an operator copies. + [ -s ${pkiDir}/ci-runner.pem ] || ${signLeaf} ${pkiDir} ci-runner \ + ${lib.escapeShellArg deployCfg.bao.ciRunnerCommonName} "" clientAuth + ${lib.optionalString deployCfg.hive-controller.enable '' # This hive's gateway-leaf identity, which opens # `pki/issue/-` and nothing else. Its diff --git a/nix/host-modules/hive-c0re/options.nix b/nix/host-modules/hive-c0re/options.nix index e074bb6f..a8153c42 100644 --- a/nix/host-modules/hive-c0re/options.nix +++ b/nix/host-modules/hive-c0re/options.nix @@ -349,8 +349,8 @@ The kernel default is `100`. The hyperhive default of `80` means agent containers yield slightly to everything that is *not* on this drop-in path — host services and the infrastructure - containers (`hive-ci`, `hive-forge`, `hive-gateway`, - `hive-matrix`), which stay at `100`. Note this is a hive-wide + containers (`hive-forge`, `hive-gateway`, `hive-matrix`), which + stay at `100`. Note this is a hive-wide value, so it does not rank agents against *each other*: they all share one weight. diff --git a/nix/host-modules/hive-ci.nix b/nix/host-modules/hive-ci.nix deleted file mode 100644 index f2424ae9..00000000 --- a/nix/host-modules/hive-ci.nix +++ /dev/null @@ -1,470 +0,0 @@ -{ - pkgs, - lib, - config, - ... -}: -let - cfg = config.services.hyperhive.deploy.forgejo.ci; - forgeCfg = config.services.hyperhive.swarm.forge; - forgeDeployCfg = config.services.hyperhive.deploy.forgejo; - gatewayCfg = config.services.hyperhive.gateway; - networkCfg = config.services.hyperhive.network; - tlsCfg = config.services.hyperhive.deploy.hive-controller.tls; - - # Self-signed TLS is the gateway default (no operator cert / ACME). When - # active, forgejo's ROOT_URL is `https://forge.` and the leaf is - # signed by the host hive CA — so the runner's Node-based actions (e.g. - # `upload-artifact`, which POSTs to the ROOT_URL-derived artifact endpoint) - # reject the chain, since Node trusts only its bundled CA bundle, not the - # system store. Trust the hive CA explicitly via NODE_EXTRA_CA_CERTS below. - # The bind-mount + `container@` ordering that makes the CA reachable are - # shared with hive-forge via the `hive-ca-trust` helper; only the Node - # `NODE_EXTRA_CA_CERTS` consumption is hive-ci-specific. - caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; - useSelfSigned = caTrust.useSelfSigned; - caContainerPath = caTrust.caContainerPath; - - # The runner daemon needs the hive CA too, and `NODE_EXTRA_CA_CERTS` above - # does not give it to them: that variable is Node's, and it is ADDITIVE. - # `gitea-runner` is Go, whose trust store is REPLACING -- it reads - # `SSL_CERT_FILE` and then trusts only what that file contains. So the two - # consumers need different treatment from the same CA, which is exactly the - # split the helper documents at its head. - # - # Without this the daemon fails every startup call with - # `x509: certificate signed by unknown authority` the moment it reaches a - # TLS endpoint -- and it reaches one whenever the address it is registered - # at redirects to the gateway's https vhost. The unit then crash-loops on - # `Restart=on-failure`, no job is ever picked up, and nothing reports it as - # broken: every pull request simply sits at "Waiting to run". - # - # ⚠️ This is deliberately NOT claimed as the whole story of that outage -- - # what address the daemon holds is a separate question. It is here because - # the trust gap is real under every explanation of it, and adding a CA to - # one unit cannot break a path that already works. - caBundleModule = caTrust.trustBundle { - inherit pkgs; - name = "hive-ci"; - consumers = [ "gitea-runner-hive" ]; - }; - - # Private network namespace, attached to the hive bridge so the - # runner reaches the forge via the gateway — and cannot reach - # host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.). - # See docs/networking/network.md. - privateNetwork = true; -in -{ - # Forgejo Actions runner in a `hive-ci` nixos-container. - # - # ⚠️ Two properties here are load-bearing and neither is visible from the - # config that implements it: - # - # - The container gets its OWN network namespace, not the host's, so a - # PR's nix build cannot reach host-loopback services. That is a - # prompt-injection boundary; moving this to host netns for convenience - # removes it. Reachability of the forge from inside: - # `docs/networking/network.md::Container isolation`. - # - The forge ADMIN token never enters this container — hive-c0re holds - # it and hands in only a registration token. Full flow: - # `docs/scheduler/ci.md::Container design`. - # - # Non-ephemeral on purpose: gitea-actions-runner writes its registered - # credentials to its stateDir on first registration and reuses them, so an - # ephemeral container would re-register on every restart. - # - # Nix builds here use the shared /nix/store with `sandbox-fallback = true` - # — nspawn containers cannot create the user namespaces nix sandboxing - # wants: `docs/process/gotchas.md`. - - options.services.hyperhive.deploy.forgejo.ci = { - enable = lib.mkOption { - type = lib.types.bool; - default = false; - example = true; - description = '' - Run a Forgejo Actions runner in a `hive-ci` nixos-container. - Grouped under `services.hyperhive.deploy.forgejo` because the runner - is tightly coupled to the forge instance it registers against. - Disabled by default. It registers against the forge on this same - host, so it requires - {option}`services.hyperhive.deploy.forgejo.enable` here. - - On first start the container auto-registers against hive-forge using - hive-c0re's admin token — no manual token provisioning needed. - Runner credentials are persisted in the container's state dir and - reused on every subsequent boot. - ''; - }; - - name = lib.mkOption { - type = lib.types.str; - default = "hive-ci"; - example = "prod-hive"; - description = '' - Runner name as shown in the Forgejo admin panel. Defaults to - "hive-ci"; override when multiple hives share a Forgejo instance. - ''; - }; - - concurrency = lib.mkOption { - type = lib.types.ints.positive; - default = 1; - example = 4; - description = '' - Maximum number of workflow jobs the runner executes in parallel. - Each job gets its own temporary working directory; multiple parallel - jobs share the container's nix store and cargo registry cache. - Higher values trade memory + CPU headroom for throughput. - ''; - }; - - labels = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ "hive-ci:host" ]; - example = [ - "hive-ci:host" - "nix:host" - ]; - description = '' - Runner labels in `:` format. The `host` scheme runs - commands directly in the container (no docker/podman). Workflow - files target this runner with `runs-on: [hive-ci]`. - ''; - }; - - package = lib.mkOption { - type = lib.types.package; - default = pkgs.gitea-actions-runner; - defaultText = lib.literalExpression "pkgs.gitea-actions-runner"; - description = "gitea-actions-runner package."; - }; - - jobTimeout = lib.mkOption { - type = lib.types.str; - default = "1h"; - example = "3h"; - description = '' - Per-job wall-clock timeout the runner enforces (act_runner's - `runner.timeout`). A job that exceeds it is killed, so a hung or - runaway build is bounded instead of holding the runner's single - slot indefinitely. Default `1h` comfortably covers a cold-cache - nix build while still bounding a stuck job; raise it (e.g. - `"3h"`) if you legitimately run jobs longer than that. Accepts a - Go duration string (`30m`, `1h`, `2h30m`). Note: this is - enforced by the runner process, so it only fires while that - process is itself healthy. - ''; - }; - }; - - config = lib.mkIf cfg.enable { - assertions = [ - { - # The runner reaches the forge through THIS host's gateway, and - # hive-c0re registers it through the local forge container; neither - # exists on a host that does not run the forge. A runner on another - # host, registered by the swarm instead, is a separate design. - assertion = forgeDeployCfg.enable; - message = '' - services.hyperhive.deploy.forgejo.ci.enable requires - services.hyperhive.deploy.forgejo.enable on the same host. - The CI runner reaches the forge through this host's gateway and - is registered through the local forge container, so it can only - run where the forge does. Enable CI on the swarm's forge host. - ''; - } - ]; - - # The runner container has a private netns of its own, so it is - # attached to the bridge and reaches the forge by name through the - # gateway vhost the assertion above already insists on. - services.hyperhive.network.enable = lib.mkDefault true; - services.hyperhive.gateway.enable = lib.mkDefault true; - services.hyperhive.gateway.dns.enable = lib.mkDefault true; - - # Create /run/hive-ci/ on the host and seed runner-token with a - # placeholder. The container bind-mounts this file read-only; hive-c0re - # (via hive-priv's RegisterCiRunner) overwrites it with the real - # registration token when it registers the runner out of band. The - # placeholder keeps the runner's EnvironmentFile present from first boot, - # before c0re has registered — the runner's ExecStartPre precond (below) - # distinguishes the placeholder from a real token. - systemd.tmpfiles.rules = [ - "d /run/hive-ci 0700 root root -" - "f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder" - ]; - - # Tell hive-c0re that CI is enabled so its startup sweep registers the - # runner. Registration is OFF the container's boot-critical path: - # hive-c0re (`forge/ci_runner.rs`) validates the persisted `.runner` - # against the forge and, when absent/stale, mints a registration token and - # hands it to hive-priv to write `/run/hive-ci/runner-token` + restart the - # runner. The forge admin token stays in hive-c0re; only the registration - # token reaches the host env-file the container mounts read-only. - systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_CI_ENABLED = "1"; - - # `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service` - # in self-signed mode (see the hive-ca-trust helper). Runner registration - # is not on the boot-critical path — hive-c0re owns it out of band — so - # the container boots immediately and needs no start-timeout bump. - systemd.services."container@hive-ci" = caTrust.containerOrdering; - - containers.hive-ci = { - autoStart = true; - ephemeral = false; - # Journal files on the host, not inside the container: nixpkgs hardcodes - # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. - extraFlags = [ "--link-journal=host" ]; - inherit privateNetwork; - hostBridge = networkCfg.bridgeName; - - bindMounts = { - # Pre-filled by hive-c0re (via hive-priv) with the runner - # registration token; tmpfiles seeds a `TOKEN=placeholder` before - # that. Read-only: the container reads TOKEN= from here; the core - # admin token never enters the container. - "/run/hive-ci/runner-token" = { - hostPath = "/run/hive-ci/runner-token"; - isReadOnly = true; - }; - # Route the container's nix through the HOST nix-daemon (as the agent - # containers do) instead of an in-container daemon: with - # NIX_REMOTE=daemon below, builds run on the host daemon and inherit - # its buildMachines + max-jobs. Note that `fallback` is NOT inherited - # that way — it's a client-side option transmitted per connection, so - # the container sets it itself (see nix.settings.fallback below). - # Bind the *directory* (not the socket file) so the mount stays live - # across a host nix-daemon restart, which recreates the socket inode. - "/nix/var/nix/daemon-socket" = { - hostPath = "/nix/var/nix/daemon-socket"; - isReadOnly = false; - }; - } - # Self-signed mode: bind the public hive CA cert read-only so the - # runner's Node actions trust the gateway/forge leaf (consumed via - # NODE_EXTRA_CA_CERTS in the container config). Shared bind-mount + - # ordering come from the hive-ca-trust helper. - // caTrust.bindMount; - - config = - { pkgs, lib, ... }: - { - # Assembles system CAs + the hive CA into one bundle and sets - # `SSL_CERT_FILE` on the runner unit. See `caBundleModule` above for - # why the Node variable beside it is not enough. - imports = [ - ../container-modules/swarm-container.nix - caBundleModule - ]; - - # Its own netns, so it keeps its own firewall; resolvconf stays on - # and serves `networking.nameservers` below. - services.hyperhive.swarmContainer = { - inherit privateNetwork; - writesOwnResolvConf = false; - }; - - # Point the forge domain at the bridge IP so the runner can - # reach the forge through the gateway — both for registration / - # polling (runner URL below) and for artifact uploads (the - # Forgejo Actions artifact API uses ROOT_URL, i.e. the public - # forge domain, not a localhost URL). The gateway vhost for - # `forgeCfg.domain` proxies all `/` → forge; `addSSL = true` - # means HTTP:80 is served without redirect alongside HTTPS:443. - # Ports 80 and 443 are always open on the bridge firewall (see - # hive-network.nix). No DNS lookup needed — /etc/hosts wins. - networking.extraHosts = "${networkCfg.bridgeIp} ${forgeCfg.domain}"; - # DNS: use the hive resolver on the bridge IP (dnsmasq in - # hive-gateway) for external lookups (git checkout, crate - # registries, etc.). The bridge→loopback DROP rule does not - # affect traffic destined for the bridge IP itself. - networking.nameservers = [ networkCfg.bridgeIp ]; - # Bridge-attached via privateNetwork=true + hostBridge. The - # gateway's dnsmasq serves a DHCP pool covering all usable bridge - # addresses (see dhcp-range in hive-gateway.nix) — agents and - # service containers alike receive IPs dynamically. - networking.interfaces.eth0.useDHCP = true; - - # nspawn containers can't create user-namespaces, so nix - # sandboxing always fails. Fall back to unsandboxed builds. - # Moot once every nix invocation in the container routes - # through the host daemon (the daemon governs sandboxing). - # See docs/process/gotchas.md and nix/agent-modules/default.nix. - nix.settings.sandbox-fallback = lib.mkForce true; - # Degrade to a local build when a remote builder is unreachable - # rather than failing the check. `fallback` is a client-side - # option (transmitted per daemon connection), so routing through - # the host daemon does NOT inherit the host's value — the - # container has to set it. Same reasoning as the agent - # containers; see nix/agent-modules/default.nix. - nix.settings.fallback = true; - nix.settings.experimental-features = [ - "nix-command" - "flakes" - ]; - - # Build through the HOST nix-daemon (bind-mounted socket above), not - # an in-container daemon. NIX_REMOTE=daemon makes root services (the - # runner) use the daemon socket rather than store=auto → local store. - # Disable BOTH the service and its .socket unit: the .socket owns the - # listen path /nix/var/nix/daemon-socket/socket and would otherwise - # race the bind mount over the same path. - systemd.globalEnvironment.NIX_REMOTE = "daemon"; - systemd.services.nix-daemon.enable = false; - systemd.sockets.nix-daemon.enable = false; - - # package is top-level on gitea-actions-runner, not per-instance. - services.gitea-actions-runner.package = cfg.package; - - services.gitea-actions-runner.instances.hive = { - enable = true; - name = cfg.name; - # Route through the gateway (bridge IP, port 80) so the - # runner never touches host-loopback. The forge domain - # resolves to the bridge IP via networking.extraHosts above; - # the gateway vhost `forgeCfg.domain` proxies to the forge - # on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect). - url = "http://${forgeCfg.domain}"; - # EnvironmentFile providing TOKEN= — pre-filled by hive-c0re - # (via hive-priv) with the runner registration token; - # bind-mounted read-only from /run/hive-ci/runner-token on the - # host. The runner's ExecStartPre precond gates on this being a - # real (non-placeholder) token or an already-registered .runner. - tokenFile = "/run/hive-ci/runner-token"; - labels = cfg.labels; - settings = { - runner.capacity = cfg.concurrency; - # Per-job wall-clock cap — see the `jobTimeout` option. - runner.timeout = cfg.jobTimeout; - }; - }; - - # No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted - # read-only from the host (pre-filled before container start). - # nspawn creates the /run/hive-ci/ mount-point directory - # automatically before launching the container's init. - - # No hive-ci-register.service inside the container: all forge - # API calls (runner validation, token fetch) live in hive-c0re - # (forge/ci_runner.rs), out of band. The core admin token never - # enters this container. - - # git is already in the gitea-actions-runner service PATH (the - # nixpkgs module builds it from the package's runtime deps). - # nix is required for `nix flake check` / `nix build` workflow - # steps — it's not included by the upstream module. - # Use the `path` service attribute (generates ExecSearchPath=) - # to prepend nix's bin dir to PATH without touching the - # environment.PATH the nixpkgs module sets — overriding that - # would lose git, curl, nodejs, and other runner deps. - environment.systemPackages = [ - pkgs.git - pkgs.nix - ]; - - systemd.services."gitea-runner-hive" = { - path = [ pkgs.nix ]; - # A CI runner must be able to EXECUTE what it builds, and by - # default it cannot: the upstream unit uses `DynamicUser`, and - # systemd mounts a dynamic unit's state directory `noexec`. - # Measured on the live runner rather than inferred — from a job's - # own /proc/self/mountinfo: - # - # /var/lib/private ro,nosuid,nodev,noexec (tmpfs) - # /var/lib/private/gitea-runner rw,nosuid,nodev,noexec,idmapped - # - # with /tmp as the control at rw,nosuid,nodev — no noexec, and a - # script there executes fine. Nothing noticed for as long as every - # workflow built inside the nix sandbox and executed nothing in the - # workspace; the first job that compiled a build script into the - # runner's own target dir died with EACCES on execve, reported as - # "could not execute process ... (never executed)". - # - # Both spellings are listed because `ExecPaths=` resolves against - # the HOST root, where the dynamic-user layout makes - # /var/lib/gitea-runner a symlink into private/. The `-` prefix - # means "ignore if absent", so whichever layout the unit ends up - # with, the other is a no-op rather than a startup failure. - serviceConfig.ExecPaths = [ - "-/var/lib/gitea-runner" - "-/var/lib/private/gitea-runner" - ]; - # Trust the hive CA in Node-based actions. With self-signed TLS, - # forgejo's ROOT_URL is `https://forge.` (CA-signed leaf), - # so actions like `upload-artifact` — whose Node HTTP client uses - # Node's *bundled* CA bundle, not the system store — reject the - # chain with "unable to verify the first certificate". Pointing - # NODE_EXTRA_CA_CERTS at the bind-mounted CA adds it to Node's - # roots for every action, hive-wide. Inherited by the job - # processes the runner spawns (host execution mode). Only set in - # self-signed mode; with an operator cert / ACME the public CA - # already validates and the bind-mount is absent. - environment = lib.mkIf useSelfSigned { - NODE_EXTRA_CA_CERTS = caContainerPath; - }; - # Gate runner start on credentials being present. Registration is - # done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints a - # token and, via hive-priv, writes /run/hive-ci/runner-token + - # restarts this unit. Pass iff already registered (.runner present) - # OR a real (non-placeholder) token is in place; else exit 1 so the - # runner stays down (Restart=on-failure retries) until c0re - # registers it, rather than blocking the container's start. - # - # No nix-daemon wait: builds route through the HOST daemon - # (bound socket above), which is up before the container — nix's - # own connect retries cover any brief window. `mkBefore` runs this - # ahead of any upstream pre-steps. - # - # The `+` prefix runs this step with FULL PRIVILEGES instead of as - # the unit's (dynamic) `gitea-runner` user. The token file is 0600 - # root-owned, so an unprivileged check cannot read it — and it did - # not fail closed: `[ -s ]` succeeds on a stat alone, `grep` then - # died with EACCES, and `! grep` turned that error into `true`, so - # the gate exited 0 on an unreadable file. It reported "a real token - # is present" without ever having looked, for its entire existence. - # - # This grants the runner nothing new: `tokenFile` above becomes - # `EnvironmentFile=`, which systemd already reads AS ROOT before - # dropping privileges, so the payload never passes through an - # unprivileged reader either way. The `+` only lets the gate observe - # what systemd observes. - serviceConfig.ExecStartPre = lib.mkBefore [ - "+${pkgs.writeShellScript "hive-ci-runner-precond" '' - RUNNER=/var/lib/gitea-runner/hive/.runner - TOKEN_FILE=/run/hive-ci/runner-token - if [ -f "$RUNNER" ]; then exit 0; fi - # Readability is asserted SEPARATELY and fails loudly. Folding - # it into the test below is what hid the bug: an unreadable file - # and a file holding a real token produced the same verdict, so - # "I could not look" was indistinguishable from "I looked and it - # is fine". - if [ -e "$TOKEN_FILE" ] && [ ! -r "$TOKEN_FILE" ]; then - echo "hive-ci runner: $TOKEN_FILE exists but is unreadable by this step; the credential gate cannot run" >&2 - exit 1 - fi - if [ -s "$TOKEN_FILE" ] && ! ${pkgs.gnugrep}/bin/grep -q '^TOKEN=placeholder$' "$TOKEN_FILE"; then - exit 0 - fi - echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2 - exit 1 - ''}" - ]; - # Registration is out of band: hive-c0re explicitly restarts this - # unit once it writes the real token, which is the primary path. As - # a safety net, if the precond above fires first (no creds yet at - # boot) retry until c0re registers, rather than staying down — with - # no start-limit rate cap so it keeps retrying however long forge / - # the core token take to settle. - serviceConfig.Restart = lib.mkForce "on-failure"; - # `mkForce`: the upstream gitea-actions-runner module also sets - # RestartSec (2), so ours needs the higher priority or the two - # collide into an eval error — same as the sibling `Restart`. - serviceConfig.RestartSec = lib.mkForce 15; - startLimitIntervalSec = 0; - }; - }; - }; - }; -} diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index b4ee8458..aebc5449 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -74,23 +74,18 @@ let "https://${cfg.domain}${portSuffix}/"; effectiveRootUrl = if cfg.rootUrl != null then cfg.rootUrl else defaultRootUrl; - # When CI is enabled, the runner needs `actions/checkout` resolvable - # without external DNS (hive-ci shares the host netns, so a host-resolver - # blip otherwise reds every `actions/checkout@vN` fetch from - # data.forgejo.org). Auto-append a pull-mirror of it and point - # forgejo's DEFAULT_ACTIONS_URL at this instance so `uses:` resolves local. - ciEnabled = config.services.hyperhive.deploy.forgejo.ci.enable; + # `actionsFromSelf` resolves `uses:` against this forge, which then has to + # hold the action. + actionsFromSelf = deployCfg.forgejo.actionsFromSelf; actionCheckoutMirror = { upstream = "https://github.com/actions/checkout"; dest = "actions/checkout"; }; - # Auto-append the actions/checkout mirror only when CI is on AND the - # operator hasn't already declared that dest themselves (else CI-on + - # an explicit `actions/checkout` entry would duplicate it). + # Unless the operator already declared that dest, which would duplicate it. effectiveMirrors = deployCfg.forgejo.mirrors ++ lib.optional ( - ciEnabled && !(lib.any (m: m.dest == actionCheckoutMirror.dest) deployCfg.forgejo.mirrors) + actionsFromSelf && !(lib.any (m: m.dest == actionCheckoutMirror.dest) deployCfg.forgejo.mirrors) ) actionCheckoutMirror; # Share host netns — forgejo's HTTP / SSH listeners then look @@ -179,12 +174,24 @@ in so a host-resolver blip leaves a *stale* mirror, never a hard failure on whatever reads it. - When `services.hyperhive.deploy.forgejo.ci.enable` is set, an - `actions/checkout` mirror is auto-appended to this list and - forgejo's `DEFAULT_ACTIONS_URL` is pointed at this instance, so CI - `uses: actions/checkout@vN` steps resolve entirely on loopback with - no external DNS on the critical path (the seed/re-sync needs - external DNS, but that's off the CI path). + With {option}`services.hyperhive.deploy.forgejo.actionsFromSelf` + set, an `actions/checkout` mirror is appended to this list. + ''; + }; + + actionsFromSelf = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = '' + Resolve a workflow's `uses: /@vN` against this forge + instead of `data.forgejo.org`, and add an `actions/checkout` + pull-mirror to {option}`services.hyperhive.deploy.forgejo.mirrors` + so the checkout step has something to resolve to. A CI job's + checkout then needs no external DNS; only the mirror's re-sync + does. + + Any other action a workflow names has to be mirrored here too. ''; }; @@ -641,16 +648,16 @@ in # Enable Forgejo Actions so the runner registration token # API endpoint is available. Without this the endpoint # returns "runner registration token not found" regardless - # of token scopes. Required by `hive-ci-register.service` - # in the hive-ci container. + # of token scopes. swarm-controller's `forge::runner_token` + # reads it. actions.ENABLED = true; - # When CI is enabled, resolve `uses: /@vN` from + # With `actionsFromSelf`, resolve `uses: /@vN` from # THIS instance (the seeded `actions/checkout` pull-mirror) # instead of the upstream default `data.forgejo.org` — keeps # the checkout step on loopback, immune to a host-resolver # blip. `self` = forgejo expands actions against its # own ROOT_URL. - actions.DEFAULT_ACTIONS_URL = lib.mkIf ciEnabled "self"; + actions.DEFAULT_ACTIONS_URL = lib.mkIf actionsFromSelf "self"; # F3 (federation) computes its data dir relative to the # forgejo binary, which lands in the read-only nix # store and crashes anything that touches the F3 @@ -1099,7 +1106,7 @@ in lib.mkIf deployCfg.swarm-controller.enable (builtins.toJSON effectiveMirrors); warnings = lib.optional (effectiveMirrors != [ ] && !deployCfg.swarm-controller.enable) '' services.hyperhive.deploy.forgejo.mirrors (or the actions/checkout mirror - deploy.forgejo.ci.enable adds) is set on a host that does not run the + deploy.forgejo.actionsFromSelf adds) is set on a host that does not run the swarm-controller. The controller seeds mirrors from its own host's list, so these are not seeded: declare them on the swarm-controller's host. ''; diff --git a/nix/host-modules/hive-gateway/dnsmasq.nix b/nix/host-modules/hive-gateway/dnsmasq.nix index e5de27f5..e93b5498 100644 --- a/nix/host-modules/hive-gateway/dnsmasq.nix +++ b/nix/host-modules/hive-gateway/dnsmasq.nix @@ -68,7 +68,7 @@ # DHCP pool covering all usable host addresses on the bridge # subnet — bounds computed by hive-network.nix from # bridgeIp/bridgePrefixLength. All containers (agents and service - # containers such as hive-ci) receive their IPs dynamically. + # containers such as swarm-ci-runner) receive their IPs dynamically. dhcp-range = "${networkCfg.dhcpRangeStart},${networkCfg.dhcpRangeEnd},1h"; dhcp-leasefile = "/var/lib/dnsmasq/dnsmasq.leases"; # No explicit upstream: non-hive queries follow dnsmasq's diff --git a/nix/host-modules/hive-gateway/options.nix b/nix/host-modules/hive-gateway/options.nix index 57750821..15c37ae6 100644 --- a/nix/host-modules/hive-gateway/options.nix +++ b/nix/host-modules/hive-gateway/options.nix @@ -229,7 +229,7 @@ in Read-only derived flag: `true` when the gateway serves the self-signed leaf the secret store issues — i.e. neither `tls.certDir` nor `tls.acme.enable` is configured. Single source of truth for the - self-signed condition; consumed by the `hive-tls` and `hive-ci` + self-signed condition; consumed by the `hive-tls` and `hive-ca-trust` modules so the derivation isn't duplicated. Internal — not meant to be set by operators (use `tls.certDir` / `tls.acme` to override the self-signed default). diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index f19234be..756134ad 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -11,7 +11,7 @@ let gatewayCfg = config.services.hyperhive.gateway; hyperhiveDomain = config.services.hyperhive.domain; - # Same runtime→build-time bridge hive-ci and hive-forge already cross: + # Same runtime→build-time bridge hive-forge and swarm-ci-runner cross: # binds the hive trust bundle (which folds in the swarm root) into the # container and orders the container after `hive-tls-ca.service`. The # assembled bundle itself comes from `caTrust.trustBundle`, imported in diff --git a/nix/host-modules/hive-network.nix b/nix/host-modules/hive-network.nix index 73f7a86e..f5f42072 100644 --- a/nix/host-modules/hive-network.nix +++ b/nix/host-modules/hive-network.nix @@ -42,7 +42,7 @@ let d.swarm-controller.enable d.swarm-ui.enable d.forgejo.enable - d.forgejo.ci.enable + d.ci-runner.enable d.matrix.enable d.authelia.enable d.bao.enable diff --git a/nix/host-modules/hive-priv.nix b/nix/host-modules/hive-priv.nix index f98573f9..69b31f63 100644 --- a/nix/host-modules/hive-priv.nix +++ b/nix/host-modules/hive-priv.nix @@ -118,9 +118,6 @@ in # /run/hive-agent — creates per-agent socket dirs # /run/systemd — container@ drop-ins + machined state # /run/lock — nixos-container's create/destroy lock file - # /run/hive-ci — register_ci_runner's runner-token write - # (else token stays placeholder → runner - # crash-loops → no CI hive-wide) # /var/lib/nixos-containers — container rootfs # /var/lib/hyperhive — agent state (forge/matrix token files) # /nix — nix store + profile updates on create/update @@ -131,7 +128,6 @@ in "/run/hive-agent" "/run/systemd" "/run/lock" - "/run/hive-ci" "/var/lib/nixos-containers" "/var/lib/hyperhive" "/nix" diff --git a/nix/host-modules/lib/hive-ca-trust.nix b/nix/host-modules/lib/hive-ca-trust.nix index 63f9b972..c955fa45 100644 --- a/nix/host-modules/lib/hive-ca-trust.nix +++ b/nix/host-modules/lib/hive-ca-trust.nix @@ -9,7 +9,7 @@ # # `bindMount` + `containerOrdering` are the language-agnostic half. The # *consumption* differs per runtime: an additive variable (Node's -# `NODE_EXTRA_CA_CERTS`, hive-ci) points straight at `caContainerPath` from +# `NODE_EXTRA_CA_CERTS`, swarm-ci-runner) points straight at `caContainerPath` from # the call site, while a *replacing* one (Go's `SSL_CERT_FILE`, rustls) needs # the system-CAs+hive-CA concat that `trustBundle` below does for it. # @@ -19,7 +19,7 @@ # caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; # # then, in the container: # # bindMounts = { … } // caTrust.bindMount; -# # systemd.services."container@hive-ci" = lib.mkMerge [ caTrust.containerOrdering … ]; +# # systemd.services."container@swarm-ci-runner" = lib.mkMerge [ caTrust.containerOrdering … ]; # # environment.NODE_EXTRA_CA_CERTS = caTrust.caContainerPath; # consumption, per-caller # # `tlsCfg` = config.services.hyperhive.deploy.hive-controller.tls @@ -87,7 +87,7 @@ in }; # Fold into the caller's `container@` unit (via `lib.mkMerge` if the - # caller adds its own keys, e.g. hive-ci's `TimeoutStartSec`). Orders the + # caller adds its own keys). Orders the # container after the host `hive-tls-ca.service` so the bind source exists # before nspawn sets the mount up — a condition-skipped/late CA would # otherwise fail the container start. @@ -99,7 +99,7 @@ in # System CAs + hive CA in one bundle, with `SSL_CERT_FILE` set on each # consumer — for runtimes whose trust variable *replaces* the store (Go, # rustls-native-certs). An additive one (Node's `NODE_EXTRA_CA_CERTS`, - # hive-ci) needs no bundle and should not use this. + # swarm-ci-runner) needs no bundle and should not use this. # # imports = [ (caTrust.trustBundle { inherit pkgs; name = "swarm-nats"; # consumers = [ "swarm-nats-auth" ]; }) ]; diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 8e6de00e..b5ceb34b 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -429,6 +429,10 @@ let capabilities = ["create", "read", "update"] } + path "${credentialMountPath}/data/${ciRunnerTokenLeaf}" { + capabilities = ["create", "update"] + } + path "${credentialMountPath}/data/${swarmAppserviceTokenLeaf}" { capabilities = ["read"] } @@ -519,6 +523,12 @@ let # it; the controller reads it and holds no other copy. controllerClientLeaf = "swarm/controller/swarm-controller/oidc/client"; + # The forge's runner registration token, the nix half of + # `swarm_secret_client::ci::registration_token_path`. The controller writes + # it and cannot read it back (`forge::runner_token` keeps the value it last + # wrote in memory); the CI runner's fetch unit is its one reader. + ciRunnerTokenLeaf = "swarm/services/ci-runner/registration-token"; + # The UI's OIDC client secret. The publisher writes it; the granter reads it # into the `oidc` auth method's config, which is the only copy bao uses. baoUiClientLeaf = "swarm/services/${cfg.ui.oidc.clientId}/oidc/client"; @@ -853,6 +863,16 @@ let } ]; + # The CI runner's token fetch. The one reader of the path, which the + # controller writes and cannot read back. + ciRunnerReaders = [ + { + name = "swarm-ci-runner"; + cn = baoDeploy.ciRunnerCommonName; + policyText = readStanza "${credentialMountPath}/data/${ciRunnerTokenLeaf}"; + } + ]; + # The role name IS the policy name, as for the three service principals # above: the role attaches the policy by spelling it identically, and one # string for both objects removes the way they drift apart. @@ -1640,6 +1660,19 @@ in ''; }; + ciRunnerCommonName = lib.mkOption { + type = lib.types.str; + default = "swarm-ci-runner"; + description = '' + Subject the store's `swarm-ci-runner` cert-auth role accepts: the + identity `swarm-bao-ci-runner`, the unit that fetches the forge's + runner registration token for the CI runner, presents. Its grant is one + path, `swarm/services/ci-runner/registration-token`, and read only. + + ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings. + ''; + }; + queueAgentCommonNamePrefix = lib.mkOption { type = lib.types.str; default = "swarm-bao-queue-agent"; @@ -1847,6 +1880,32 @@ in ''; }; + ciRunnerClientCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/ci-runner.pem"; + description = '' + Certificate the unit that fetches the CI runner's registration token + presents to the store. Its subject must be + {option}`services.hyperhive.deploy.bao.ciRunnerCommonName`. + + ./glue-bao-tls.nix points it at the leaf it mints on the store's host. + On a runner host without the store, it is a file the operator copies + from the store's host. A path, never a value. + ''; + }; + + ciRunnerClientKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/ci-runner-key.pem"; + description = '' + Private key for + {option}`services.hyperhive.deploy.bao.ciRunnerClientCertFile`. + A path, never a value. + ''; + }; + granterCommonName = lib.mkOption { type = lib.types.str; default = "bao-granter"; @@ -2907,6 +2966,8 @@ in systemd.services.swarm-bao-nats-auth-policy = readerPolicyUnit "write the queue responder's agent-credential-reader bao policy and cert-auth role" natsAuthReaders; + systemd.services.swarm-bao-ci-runner-policy = readerPolicyUnit "write the CI runner's registration-token-reader bao policy and cert-auth role" ciRunnerReaders; + # A FOURTH sibling, same shape and same reasons as the two above. This # one is what turns `swarm-services-issuer` from a declaration into a # grant: a bao policy reaches nothing until a login role hands it to a @@ -3111,6 +3172,7 @@ in "swarm-bao-services-issuer-policy.service" "swarm-bao-nats-tls-policy.service" "swarm-bao-hive-leaf-policy.service" + "swarm-bao-ci-runner-policy.service" ] ++ granterAfter; requires = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-ci-runner.nix b/nix/host-modules/swarm-ci-runner.nix new file mode 100644 index 00000000..9739b80e --- /dev/null +++ b/nix/host-modules/swarm-ci-runner.nix @@ -0,0 +1,421 @@ +# The swarm's Forgejo Actions runner, in a `swarm-ci-runner` container on the +# host that sets `deploy.ci-runner.enable`. It reaches the forge by name and +# reads its registration token from the secret store under a certificate of +# its own, so nothing here asks whether this host runs the forge or a hive. +# +# ⚠️ Two properties here are load-bearing and neither is visible from the +# config that implements it: +# +# - The container gets its OWN network namespace, not the host's, so a PR's +# nix build cannot reach host-loopback services. That is a +# prompt-injection boundary; moving this to host netns for convenience +# removes it. `docs/networking/network.md::Container isolation`. +# - The forge ADMIN token never enters this container. swarm-controller holds +# it and publishes only the registration token, which the runner's own store +# identity reads. `docs/scheduler/ci.md::Registration`. +# +# Non-ephemeral on purpose: the runner writes its registered credentials +# (`.runner`) to its state dir and reuses them, so an ephemeral container +# would register a new runner on every restart. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.deploy.ci-runner; + forgeCfg = config.services.hyperhive.swarm.forge; + baoCfg = config.services.hyperhive.swarm.bao; + baoDeploy = config.services.hyperhive.deploy.bao; + gatewayCfg = config.services.hyperhive.gateway; + networkCfg = config.services.hyperhive.network; + tlsCfg = config.services.hyperhive.deploy.hive-controller.tls; + + machine = "swarm-ci-runner"; + # The upstream module names its unit `gitea-runner-`. + instance = "swarm"; + runnerUnit = "gitea-runner-${instance}"; + + # What the runner registers against and records in `.runner`. The precond + # below compares the two, so both come from here. + forgeUrl = "https://${forgeCfg.domain}"; + + # The forge's gateway serves a leaf from the swarm CA in self-signed mode. + # The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store, + # so it gets the assembled bundle; the Node actions it runs get + # `NODE_EXTRA_CA_CERTS`, which is additive, further down. + caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; + caBundleModule = caTrust.trustBundle { + inherit pkgs; + name = machine; + consumers = [ runnerUnit ]; + }; + + # The upstream unit is `DynamicUser`, so its state dir is + # `/var/lib/private/gitea-runner`; `/var/lib/gitea-runner` is a symlink to + # it, which is the spelling a step run as root (`+`) can follow. + runnerFile = "/var/lib/gitea-runner/${instance}/.runner"; + + # Inside the container, root-only. The token file is the runner's + # `EnvironmentFile` (`TOKEN=`); the host unit below writes it through + # the container's root, at `hostTokenFile`. + stateDir = "/var/lib/swarm-ci-runner"; + tokenFile = "${stateDir}/registration-token.env"; + hostTokenFile = "/var/lib/nixos-containers/${machine}${tokenFile}"; + # The epoch second the daemon last started. The precond reads the daemon's + # journal from there. + daemonStartedFile = "${stateDir}/daemon-started"; + + # The nix half of `swarm_secret_client::ci::registration_token_path`, in the + # form `bao kv get` takes (no KV v2 `data/`). + storePath = "secret/swarm/services/ci-runner/registration-token"; + + # What Forgejo answers a runner it has no record of — deleted in the admin + # panel, or a forge whose database was replaced — on every call but + # `Register` (`routers/api/actions/runner/interceptor.go`). The daemon logs + # it and exits 1 when its startup `Declare` gets it. ⚠️ Nothing this module + # prints may contain it: the precond would read its own line as a rejection. + rejectedMessage = "unregistered runner"; + + haveClientIdentity = + baoDeploy.ciRunnerClientCertFile != null && baoDeploy.ciRunnerClientKeyFile != null; + + atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; + refreshConsumer = import ./lib/refresh-consumer.nix { }; + storeRetry = import ./lib/store-retry.nix { }; + + privateNetwork = true; +in +{ + options.services.hyperhive.deploy.ci-runner = { + # `enable` is in ./deploy.nix. + + name = lib.mkOption { + type = lib.types.str; + default = "swarm-ci-runner"; + description = '' + Runner name as shown in the Forgejo admin panel. + ''; + }; + + concurrency = lib.mkOption { + type = lib.types.ints.positive; + default = 1; + example = 4; + description = '' + Maximum number of workflow jobs the runner executes in parallel. + Each job gets its own temporary working directory; parallel jobs + share the host's nix store and the container's cargo registry + cache. Higher values trade memory and CPU headroom for throughput. + ''; + }; + + labels = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ "hive-ci:host" ]; + example = [ + "hive-ci:host" + "nix:host" + ]; + description = '' + Runner labels in `:` format. The `host` scheme runs + commands directly in the container (no docker/podman). This repo's + workflows target the default with `runs-on: hive-ci`. + ''; + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.gitea-actions-runner; + defaultText = lib.literalExpression "pkgs.gitea-actions-runner"; + description = "gitea-actions-runner package."; + }; + + jobTimeout = lib.mkOption { + type = lib.types.str; + default = "1h"; + example = "3h"; + description = '' + Per-job wall-clock timeout the runner enforces (act_runner's + `runner.timeout`). A job that exceeds it is killed, so a hung build + does not hold a runner slot indefinitely. Accepts a Go duration + string (`30m`, `1h`, `2h30m`). Enforced by the runner process, so + it only fires while that process is healthy. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = haveClientIdentity; + message = '' + services.hyperhive.deploy.ci-runner.enable requires the runner's + swarm-secret-store client identity: set both + + services.hyperhive.deploy.bao.ciRunnerClientCertFile + services.hyperhive.deploy.bao.ciRunnerClientKeyFile + + swarm-bao-ci-runner.service reads the forge's runner registration + token with it, and the runner cannot register without that token. + + On the host that runs the store, glue-bao-tls.nix supplies both as + defaults. Elsewhere, copy ci-runner.pem and ci-runner-key.pem from + the store host's /var/lib/swarm-bao-pki/ and name them here. + ''; + } + ]; + + # The container hangs off the bridge, takes its address from the + # gateway's DHCP and resolves through its dnsmasq. The gateway also runs + # `hive-tls-ca`, which writes the trust bundle bound in below. + services.hyperhive.network.enable = lib.mkDefault true; + services.hyperhive.gateway.enable = lib.mkDefault true; + services.hyperhive.gateway.dns.enable = lib.mkDefault true; + + # Shaped after ./swarm-grafana.nix's `swarm-bao-grafana-oidc`, except that + # a missing token fails too: swarm-controller publishes it at its start and + # every five minutes, so a retry is what turns "no value there" into one. + systemd.services.swarm-bao-ci-runner = { + description = "fetch the CI runner's registration token from the swarm secret store"; + after = lib.optionals baoDeploy.enable [ + "swarm-bao-pki.service" + "container@${baoCfg.machine}.service" + ]; + wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; + requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; + before = [ "container@${machine}.service" ]; + wantedBy = [ + "multi-user.target" + "container@${machine}.service" + ]; + path = [ + baoDeploy.package + pkgs.coreutils + pkgs.systemd + ]; + inherit (storeRetry) startLimitBurst startLimitIntervalSec; + serviceConfig = storeRetry.serviceConfig // { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-bao-ci-runner"; + TimeoutStartSec = 30; + }; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = baoDeploy.ciRunnerClientCertFile; + BAO_CLIENT_KEY = baoDeploy.ciRunnerClientKeyFile; + } + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; + script = '' + set -euo pipefail + + ${atomicWriteSecret} + ${refreshConsumer} + + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + + # `-token-only` answers on stdout and needs no token helper. + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + echo "could not log in to swarm-bao with the CI runner's certificate; leaving its registration token as it is." >&2 + if [ -s "$err" ]; then + cat "$err" >&2 + else + echo "bao failed without writing a diagnostic." >&2 + fi + exit 1 + fi + export BAO_TOKEN + + if ! token="$(bao kv get -field=value ${lib.escapeShellArg storePath} 2>"$err")"; then + echo "swarm-bao did not return ${storePath}; retrying until swarm-controller publishes it." >&2 + if [ -s "$err" ]; then + cat "$err" >&2 + else + echo "bao failed without writing a diagnostic." >&2 + fi + exit 1 + fi + + if [ -z "$token" ]; then + echo "swarm-bao returned an empty ${storePath}; retrying." >&2 + exit 1 + fi + + # Root-owned: systemd reads an `EnvironmentFile` as root, before the + # runner drops to its dynamic user. + install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostTokenFile)} + if secret_differs ${lib.escapeShellArg hostTokenFile} "TOKEN=$token"; then + atomic_write_secret 0600 "" ${lib.escapeShellArg hostTokenFile} "TOKEN=$token" + fi + + # `EnvironmentFile=` is read once, at start. A changed token makes the + # upstream pre-start step register again. + refresh_consumer ${lib.escapeShellArg machine} ${runnerUnit}.service ${lib.escapeShellArg hostTokenFile} + ''; + }; + + systemd.services."container@${machine}" = caTrust.containerOrdering; + + containers.${machine} = { + autoStart = true; + ephemeral = false; + # Journal files on the host, not inside the container: nixpkgs hardcodes + # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. + extraFlags = [ "--link-journal=host" ]; + inherit privateNetwork; + hostBridge = networkCfg.bridgeName; + + bindMounts = { + # Builds go through the HOST nix-daemon, so they inherit its + # buildMachines and max-jobs. The directory rather than the socket + # file, so the mount survives a daemon restart replacing the inode. + "/nix/var/nix/daemon-socket" = { + hostPath = "/nix/var/nix/daemon-socket"; + isReadOnly = false; + }; + } + // caTrust.bindMount; + + config = + { pkgs, lib, ... }: + { + imports = [ + ../container-modules/swarm-container.nix + caBundleModule + ]; + + # Its own netns, so it keeps its own firewall; resolvconf stays on + # and serves `networking.nameservers` below. + services.hyperhive.swarmContainer = { + inherit privateNetwork; + writesOwnResolvConf = false; + }; + + # The gateway's dnsmasq on the bridge answers the forge's name with + # the bridge IP where this host runs the forge, and forwards it + # upstream everywhere else. + networking.nameservers = [ networkCfg.bridgeIp ]; + networking.interfaces.eth0.useDHCP = true; + + # nspawn containers can't create user namespaces, so nix sandboxing + # fails; see docs/process/gotchas.md. `fallback` is a client-side + # option sent per daemon connection, so the host's value does not + # reach builds started here. + nix.settings.sandbox-fallback = lib.mkForce true; + nix.settings.fallback = true; + nix.settings.experimental-features = [ + "nix-command" + "flakes" + ]; + + # Both the service and its socket are off: the socket unit owns + # `/nix/var/nix/daemon-socket/socket` and would race the bind mount. + systemd.globalEnvironment.NIX_REMOTE = "daemon"; + systemd.services.nix-daemon.enable = false; + systemd.sockets.nix-daemon.enable = false; + + systemd.tmpfiles.rules = [ + "d ${stateDir} 0700 root root -" + # `EnvironmentFile=` fails the unit when the file is missing; an + # empty one leaves `TOKEN` unset for the precond to report. + "f ${tokenFile} 0600 root root -" + ]; + + services.gitea-actions-runner.package = cfg.package; + + services.gitea-actions-runner.instances.${instance} = { + enable = true; + inherit (cfg) name labels; + url = forgeUrl; + inherit tokenFile; + settings = { + runner.capacity = cfg.concurrency; + runner.timeout = cfg.jobTimeout; + }; + }; + + environment.systemPackages = [ + pkgs.git + pkgs.nix + ]; + + systemd.services.${runnerUnit} = { + # `path` adds to the PATH the upstream module builds instead of + # replacing it. + path = [ pkgs.nix ]; + # The upstream unit is `DynamicUser`, whose state dir systemd mounts + # `noexec`, so a job could not execute what it builds in its + # workspace. `ExecPaths=` resolves against the host root, where + # `/var/lib/gitea-runner` is a symlink into `private/`; `-` makes + # whichever spelling is absent a no-op. + serviceConfig.ExecPaths = [ + "-/var/lib/gitea-runner" + "-/var/lib/private/gitea-runner" + ]; + environment = lib.mkIf caTrust.useSelfSigned { + NODE_EXTRA_CA_CERTS = caTrust.caContainerPath; + }; + # Runs before upstream's pre-start step, which registers whenever + # `.runner` is absent or the token or labels changed. This one + # removes `.runner` in the two cases upstream cannot see, and holds + # the unit down until there is a token. + # + # `+`: root, because `.runner`, the token and the system journal + # are all out of the dynamic user's reach. + serviceConfig.ExecStartPre = lib.mkMerge [ + (lib.mkBefore [ + "+${pkgs.writeShellScript "swarm-ci-runner-precond" '' + set -eu + runner=${runnerFile} + started=${daemonStartedFile} + + # The forge refused the registration the daemon last started + # with. Read on the next start rather than when the daemon + # exits, so journald has its last lines by then. + if [ -f "$started" ]; then + since="$(< "$started")" + ${pkgs.coreutils}/bin/rm -f "$started" + if ${pkgs.systemd}/bin/journalctl -u ${runnerUnit}.service --since "@$since" -o cat --no-pager \ + | ${pkgs.gnugrep}/bin/grep -qF ${lib.escapeShellArg rejectedMessage}; then + echo "the forge has no record of this runner's registration; registering again" >&2 + ${pkgs.coreutils}/bin/rm -f "$runner" + fi + fi + + # `.runner` records the address it registered at, and the + # daemon dials that one, not the configured one. + if [ -f "$runner" ] \ + && [ "$(${pkgs.jq}/bin/jq -r .address "$runner")" != ${lib.escapeShellArg forgeUrl} ]; then + echo "$runner names another forge address; registering again at ${forgeUrl}" >&2 + ${pkgs.coreutils}/bin/rm -f "$runner" + fi + + # Upstream registers on a changed token hash, an empty token + # included, and a registration with no token fails. + if [ -z "''${TOKEN:-}" ]; then + echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2 + exit 1 + fi + ''}" + ]) + (lib.mkAfter [ + "+${pkgs.writeShellScript "swarm-ci-runner-mark-start" '' + ${pkgs.coreutils}/bin/date +%s > ${daemonStartedFile} + ''}" + ]) + ]; + # Restarted until the token arrives, however long the store and + # the controller take. `mkForce`: upstream sets both. + serviceConfig.Restart = lib.mkForce "on-failure"; + serviceConfig.RestartSec = lib.mkForce 15; + startLimitIntervalSec = 0; + }; + }; + }; + }; +} diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index 04e5b864..2051f116 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -54,6 +54,7 @@ let deployCfg.bao.natsCommonName deployCfg.bao.granterCommonName deployCfg.bao.natsAuthCommonName + deployCfg.bao.ciRunnerCommonName ] # The per-hive subjects, spelled out per hive rather than as the prefix. # The prefix alone would reserve the wrong string: the role for hive diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 7dff3929..e4079a24 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -170,7 +170,7 @@ let _: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile ) baoGrantWithConsumers.systemd.services; - # The thirteen units that write a `swarm-*` grant, by name, for the discovery + # The fourteen units that write a `swarm-*` grant, by name, for the discovery # control below. grantingUnitNames = [ "swarm-bao-controller-policy" @@ -186,6 +186,7 @@ let "swarm-bao-nats-auth-policy" "swarm-bao-operator-viewer-policy" "swarm-bao-hive-leaf-policy" + "swarm-bao-ci-runner-policy" ]; # Comment lines dropped first: both the HCL and the scripts explain @@ -794,24 +795,24 @@ let } { # A store host without the granter's pair writes its grants some other - # way, so none of the thirteen units may exist. Without this arm + # way, so none of the fourteen units may exist. Without this arm # `lib.mkIf haveGranter` could be dropped from any of them and every other # case here would still pass. - name = "without the granter's pair none of the thirteen granting units render"; + name = "without the granter's pair none of the fourteen granting units render"; ok = let s = baoGranterOptOut.systemd.services; in lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ]) - # The control: the same store with the pair renders all thirteen. + # The control: the same store with the pair renders all fourteen. && lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames; } { - # 🩸 What replaced the silent skip. With no bootstrap token the thirteen still + # 🩸 What replaced the silent skip. With no bootstrap token the fourteen still # render, and a refused granter fails them with the step that fixes it. # A store host that never named a token is told to name one, since the # unit that sets the granter up renders only where it has. - name = "a store host without a bootstrap token renders the thirteen, each failing loudly with the one-time step"; + name = "a store host without a bootstrap token renders the fourteen, each failing loudly with the one-time step"; ok = let s = baoGranterNoToken.systemd.services; @@ -1254,6 +1255,62 @@ let && !(lib.hasInfix "secret/data/swarm/hives/*" s) && !(lib.hasInfix "secret/metadata/swarm/hives" s); } + { + # `forge::runner_token` writes the runner's registration token and keeps + # the last value in memory, so its stanza has no `read`. Pinned as the + # whole stanza, so an added capability fails, and spelled to the leaf: + # `services/*` would reach every swarm service's OIDC client secret. + name = "the controller writes the CI runner's registration token and cannot read it back"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "path \"secret/data/swarm/services/ci-runner/registration-token\" {\n capabilities = [\"create\", \"update\"]\n}" s + && !(lib.hasInfix "secret/data/swarm/services/*" s); + } + { + # The runner's role is the token's only reader. Its policy is the one + # leaf, read-only, under the runner's own subject. + name = "the CI runner's grant is its registration token and nothing else"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-ci-runner-policy.script; + in + lib.hasInfix "path \"secret/data/swarm/services/ci-runner/registration-token\" {" s + && lib.hasInfix "capabilities = [\"read\"]" s + && lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 1 + && !(lib.hasInfix "secret/data/swarm/services/*" s) + && !(lib.hasInfix "secret/data/swarm/agents" s) + && !(lib.hasInfix "secret/data/swarm/hives" s) + && lib.hasInfix "auth/cert/certs/swarm-ci-runner" s + && lib.hasInfix "allowed_common_names=swarm-ci-runner" s + && lib.hasInfix "token_policies=swarm-ci-runner" s; + } + { + # Read access to the token is a grant on its path, so the policy units + # that name it are the whole list of who reaches it: the controller, + # which writes it, and the runner. The publisher's `services/*` write + # also covers it and holds no read. + name = "no granting unit but the controller's and the CI runner's names the runner's token"; + ok = + let + naming = lib.filter ( + u: lib.hasInfix "ci-runner/registration-token" baoGrantHere.systemd.services.${u}.script + ) grantingUnitNames; + in + lib.sort lib.lessThan naming == [ + "swarm-bao-ci-runner-policy" + "swarm-bao-controller-policy" + ]; + } + { + name = "the PKI unit signs the CI runner's leaf under its own subject"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-pki.script; + in + lib.hasInfix "/ci-runner.pem ]" s && lib.hasInfix "swarm-ci-runner \"\" clientAuth" s; + } { # Revocation, and the reason it is a stanza of its own: `delete` on the # `data/` path soft-deletes the newest version and leaves earlier ones @@ -1386,8 +1443,8 @@ let } { # What makes the case above mean something: discovery by the granter's - # certificate reaches all thirteen units, and each yields calls. - name = "the granter-policy check sees all thirteen granting units, and parses calls from each"; + # certificate reaches all fourteen units, and each yields calls. + name = "the granter-policy check sees all fourteen granting units, and parses calls from each"; ok = lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits) diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index e7e57456..bba749be 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -47,10 +47,10 @@ let # only hive without one is a hive told to have none. noAgentQueue = hiveWithController { deploy.hive-controller.queue.agentNatsUrl = null; }; - # On the forge's host: the runner is refused anywhere else. + # On the store's host, which mints the runner's store identity. withCi = hive { - deploy.forgejo.enable = true; - deploy.forgejo.ci.enable = true; + deploy.bao.enable = true; + deploy.ci-runner.enable = true; }; # A priority collision is a property of the *option*, not @@ -63,7 +63,7 @@ let # stack (measured — this is not a hypothetical). forceCiServiceConfigs = let - svcs = withCi.containers.hive-ci.config.systemd.services; + svcs = withCi.containers.swarm-ci-runner.config.systemd.services; vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs); in builtins.foldl' (acc: v: builtins.seq v acc) true vals; @@ -245,7 +245,7 @@ let # `gitea-actions-runner.nix` adds a plain `serviceConfig.*` # definition that collides with one of ours, and nix refuses to # merge two plain definitions at *host* eval. No other check - # instantiates a host with `containers.hive-ci` actually enabled, so + # instantiates a host with `containers.swarm-ci-runner` actually enabled, so # the collision only surfaces on operator deploy, not in CI. name = "the CI container's unit definitions merge without a priority collision"; ok = forceCiServiceConfigs; diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index 02ea4f97..d82a7da3 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -46,11 +46,21 @@ let deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret"; }; - ciNoForge = hive { deploy.forgejo.ci.enable = true; }; - ciWithForge = hive { - deploy.forgejo.enable = true; - deploy.forgejo.ci.enable = true; + # The runner alone: no hive, no forge, no store, its store identity placed + # by hand. + ciAlone = hive { + deploy.hive-controller.enable = false; + deploy.ci-runner.enable = true; + deploy.bao.ciRunnerClientCertFile = "/etc/pki/bao-ci-runner.pem"; + deploy.bao.ciRunnerClientKeyFile = "/etc/pki/bao-ci-runner-key.pem"; }; + ciNoIdentity = hive { + deploy.hive-controller.enable = false; + deploy.ci-runner.enable = true; + }; + + # A host config that still names the runner under the forge. + ciUnderForge = hive { deploy.forgejo.ci.enable = true; }; runsForge = m: m.services.hyperhive.deploy.forgejo.enable && m.containers ? hive-forge; @@ -122,13 +132,28 @@ let == "https://forge.t.local/user/oauth2/authelia/callback"; } { - # The runner reaches the forge through this host's gateway and is - # registered through the local container. The second arm is the - # control: a refusal that fires everywhere is not a check. - name = "CI is refused on a host that does not run the forge"; + # The runner reaches the forge by name and reads its token with its own + # certificate, so nothing about the host it runs on is asked. + name = "the CI runner runs on a host with neither the forge nor a hive"; ok = - refusedOver ciNoForge "deploy.forgejo.enable on the same host" - && !(refusedOver ciWithForge "deploy.forgejo.enable on the same host"); + ciAlone.containers ? swarm-ci-runner + && !(ciAlone.containers ? hive-forge) + && !ciAlone.services.hyperhive.deploy.hive-controller.enable + && !(refusedOver ciAlone "ci-runner") + && + ciAlone.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT + == "/etc/pki/bao-ci-runner.pem"; + } + { + # Without the identity the token never arrives; the control is the + # placed pair above. + name = "the CI runner is refused without its store identity"; + ok = refusedOver ciNoIdentity "deploy.bao.ciRunnerClientCertFile"; + } + { + # A removed option evaluates to an assertion naming the replacement. + name = "a CI option under deploy.forgejo names its replacement under deploy.ci-runner"; + ok = refusedOver ciUnderForge "services.hyperhive.deploy.ci-runner.enable instead"; } { # Nothing writes the delivery path away from the forge, and a diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 17f590ff..1e65171f 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -116,6 +116,12 @@ let deploy.forgejo.enable = true; deploy.forgejo.mirrors = [ aMirror ]; }; + controllerWithSelfActions = hive { + deploy.swarm-controller.enable = true; + deploy.forgejo.enable = true; + deploy.forgejo.mirrors = [ aMirror ]; + deploy.forgejo.actionsFromSelf = true; + }; # The controller with every shared service on another host, and the same # controller hosting them all: the swarm UI's links must not tell the two @@ -150,7 +156,7 @@ let serviceContainersWithCi = hive { deploy.hive-controller.enable = false; deploy.allSwarmServices = true; - deploy.forgejo.ci.enable = true; + deploy.ci-runner.enable = true; }; serviceContainers = serviceContainersWithCi.containers; cases = [ @@ -412,7 +418,7 @@ let name = "a service container on a netns of its own keeps its firewall and resolvconf"; ok = let - c = serviceContainers.hive-ci.config; + c = serviceContainers.swarm-ci-runner.config; in c.networking.firewall.enable && c.networking.resolvconf.enable; } @@ -446,6 +452,22 @@ let in warns mirrorsNoController && !(warns controllerWithMirrors); } + { + # The mirror is what `self` resolves `actions/checkout` to, so the two + # come together. The control is the same forge without the option. + name = "actionsFromSelf adds the checkout mirror and resolves actions from the forge"; + ok = + let + mirrorsOf = + m: builtins.fromJSON m.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS; + actionsUrlOf = + m: m.containers.hive-forge.config.services.forgejo.settings.actions.DEFAULT_ACTIONS_URL or null; + in + lib.elem "actions/checkout" (map (m: m.dest) (mirrorsOf controllerWithSelfActions)) + && actionsUrlOf controllerWithSelfActions == "self" + && mirrorsOf controllerWithMirrors == [ aMirror ] + && actionsUrlOf controllerWithMirrors == null; + } { name = "the old c0re org-avatar option lands on the controller's unit"; ok = diff --git a/swarm-controller/src/forge.rs b/swarm-controller/src/forge.rs index d00dd7ff..738b29b6 100644 --- a/swarm-controller/src/forge.rs +++ b/swarm-controller/src/forge.rs @@ -36,6 +36,7 @@ use crate::webhook::DeliveryKind; pub mod agent_token; pub mod legacy_tokens; pub mod objects; +pub mod runner_token; pub mod site_admin; /// An agent's open config-PR, as [`Client::list_open_config_prs`] reports it diff --git a/swarm-controller/src/forge/objects.rs b/swarm-controller/src/forge/objects.rs index 0a055447..1b10ec8b 100644 --- a/swarm-controller/src/forge/objects.rs +++ b/swarm-controller/src/forge/objects.rs @@ -84,9 +84,8 @@ const OPERATORS_TEAM_UNITS: [&str; 7] = [ ]; /// Periodic sync interval for pull-mirrors. Forgejo syncs mirrors on-access by -/// default, which re-introduces external DNS latency on every `git clone` (the -/// hive-ci runner shares the host netns and is therefore affected by host -/// resolver blips). A fixed periodic interval isolates CI from transient DNS +/// default, which re-introduces external DNS latency on every `git clone`. A +/// fixed periodic interval isolates CI from transient DNS /// failures — a stale mirror is acceptable; a broken clone because of a /// momentary DNS blip is not. Written in the form Forgejo echoes back, so an /// unchanged mirror compares equal and is not re-patched. @@ -118,7 +117,7 @@ Do **not** push directly to `main` — agents have read-only access. "; /// The operator-declared pull-mirrors (nix `deploy.forgejo.mirrors`, plus the -/// CI-auto `actions/checkout` entry), JSON-encoded by `hive-forge/default.nix`. +/// `actions/checkout` entry `deploy.forgejo.actionsFromSelf` adds), JSON-encoded by `hive-forge/default.nix`. /// Absent or empty means no mirrors. const MIRRORS_ENV: &str = "SWARM_CONTROLLER_FORGE_MIRRORS"; diff --git a/swarm-controller/src/forge/runner_token.rs b/swarm-controller/src/forge/runner_token.rs new file mode 100644 index 00000000..ff04dbb0 --- /dev/null +++ b/swarm-controller/src/forge/runner_token.rs @@ -0,0 +1,116 @@ +//! The forge's runner registration token, published to the swarm secret store +//! at [`ci::registration_token_path`], where the CI runner's fetch unit +//! (`nix/host-modules/swarm-ci-runner.nix`) reads it under the runner's own +//! certificate. +//! +//! Forgejo answers `GET /admin/runners/registration-token` with the same +//! instance-wide token until someone resets it. The runner re-registers on +//! every new value it reads, which leaves its previous entry on the forge, so +//! a pass writes only when the forge's token differs from the one this process +//! last wrote. The controller's store grant on the path is write-only, so that +//! last write is held in memory: a restarted controller writes the current +//! token once. + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use swarm_secret_client::ci; + +use super::Client; + +/// How often [`spawn`] re-reads the forge's token. +const RECONCILE_INTERVAL: std::time::Duration = std::time::Duration::from_mins(5); + +/// Whether `current`, the forge's token now, has to be written, given the one +/// this process last wrote (`None` before its first write). +fn needs_write(last_written: Option<&str>, current: &str) -> bool { + last_written != Some(current) +} + +impl Client { + /// The forge's instance-wide runner registration token. + /// + /// # Errors + /// When the forge refuses the call, or answers without a token. + pub async fn runner_registration_token(&self) -> Result { + let answer = self + .api + .admin_get_registration_token() + .await + .context("GET /admin/runners/registration-token")?; + answer + .token + .filter(|t| !t.is_empty()) + .context("the forge answered without a runner registration token") + } +} + +/// One pass: read the forge's token and write it when it is not the one +/// `last_written` holds. Returns the token it wrote, if it wrote one. +async fn pass(client: &Client, last_written: Option<&str>) -> Result> { + let token = client.runner_registration_token().await?; + if !needs_write(last_written, &token) { + return Ok(None); + } + let path = ci::registration_token_path()?; + let store = crate::store::connect() + .await + .context("logging in to the swarm secret store")?; + store + .write( + &path, + &ci::Credential { + value: token.clone(), + }, + ) + .await + .with_context(|| format!("writing {path}"))?; + Ok(Some(token)) +} + +/// Publish the forge's runner registration token now and every +/// [`RECONCILE_INTERVAL`] after. A pass that fails is logged and retried on +/// the next tick; it never stops the daemon. +pub fn spawn(client: Arc) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); + let mut last_written: Option = None; + loop { + ticker.tick().await; + match pass(&client, last_written.as_deref()).await { + Ok(Some(written)) => { + tracing::info!("runner registration token: written to the store"); + last_written = Some(written); + } + Ok(None) => tracing::debug!("runner registration token: unchanged"), + Err(e) => tracing::warn!( + error = %format!("{e:#}"), + retry_in_s = RECONCILE_INTERVAL.as_secs(), + "runner registration token: pass failed; retrying next tick" + ), + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::needs_write; + + #[test] + fn the_first_pass_writes() { + assert!(needs_write(None, "t1")); + } + + #[test] + fn an_unchanged_token_is_not_rewritten() { + // A rewrite of the same value is harmless to the runner, but it is a + // new store version every five minutes for nothing. + assert!(!needs_write(Some("t1"), "t1")); + } + + #[test] + fn a_reset_token_is_written() { + assert!(needs_write(Some("t1"), "t2")); + } +} diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index ac9c5591..daf55ded 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -2314,8 +2314,9 @@ fn spawn_matrix_account_backfill( }); } -/// Start the forge's periodic passes — the swarm-wide objects, and agents' -/// tokens (which also sweeps legacy tokens; see `forge::legacy_tokens`) — +/// Start the forge's periodic passes — the swarm-wide objects, the runner +/// registration token, and agents' tokens (which also sweeps legacy tokens; +/// see `forge::legacy_tokens`) — /// when a forge is configured. Lifted out of `main` for /// `clippy::too_many_lines`. fn spawn_forge_workers( @@ -2326,6 +2327,7 @@ fn spawn_forge_workers( return; }; forge::objects::spawn(Arc::clone(&client)); + forge::runner_token::spawn(Arc::clone(&client)); let sched = Arc::clone(jobq); forge::agent_token::spawn(client, move |agents| { if let Err(e) = queue_forge_token_mints(&sched, agents) { diff --git a/swarm-secret-client/src/ci.rs b/swarm-secret-client/src/ci.rs new file mode 100644 index 00000000..6c063af0 --- /dev/null +++ b/swarm-secret-client/src/ci.rs @@ -0,0 +1,54 @@ +//! The CI runner agreement: where the forge's runner registration token lives, +//! and what the object at that path holds. +//! +//! `swarm-controller` fetches the token with the forge's admin API and writes +//! it here; `nix/host-modules/swarm-ci-runner.nix` reads it under the runner's +//! own certificate. That runner's read stanza is the only one in the store that +//! covers this path. + +use serde::{Deserialize, Serialize}; + +use crate::{ + Error, + path::{Kind, principal_prefix}, +}; + +/// The runner's name segment under [`Kind::Service`]. +const SERVICE: &str = "ci-runner"; + +/// The path holding the forge's runner registration token. +/// +/// # Errors +/// Never in practice: the name segment is a constant. The `Result` is +/// [`principal_prefix`]'s. +pub fn registration_token_path() -> Result { + let prefix = principal_prefix(Kind::Service, SERVICE)?; + Ok(format!("{prefix}/registration-token")) +} + +/// What the registration-token path holds. +/// +/// No `Debug` derive: `value` registers a runner that receives jobs from every +/// repo on the forge, and a derived `Debug` is one `{:?}` away from a log line. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Credential { + /// The token itself. `swarm-ci-runner.nix` reads the store with + /// `bao kv get -field=value`, so this name is load-bearing for a reader + /// this crate does not control, the same as [`crate::matrix::Credential`]'s. + pub value: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_registration_token_is_under_the_runners_service_prefix() { + // The literal is what `swarm-bao.nix` grants the runner's role, and + // what the controller's write stanza names. + assert_eq!( + registration_token_path().expect("a constant segment is legal"), + "swarm/services/ci-runner/registration-token" + ); + } +} diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index cf7fbd77..1d9a6b5e 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -5,7 +5,7 @@ //! the rules every path obeys ([`path`]), the translation from this //! deployment's environment into a logged-in client ([`client`]), and, per kind //! of secret, the path it lives at together with the fields it holds -//! ([`matrix`], [`queue`], [`mtls`], [`forge`], [`github`], [`acp`]). Each of those is a thing the controller +//! ([`matrix`], [`queue`], [`mtls`], [`forge`], [`github`], [`acp`], [`ci`]). Each of those is a thing the controller //! and a hive must say identically, so it is said once here. //! //! [`policy`] is the same kind of agreement seen from the other side: which of @@ -23,6 +23,7 @@ //! value inside it, and its doc explains why that is not circular. pub mod acp; +pub mod ci; pub mod client; pub mod forge; pub mod github; diff --git a/swarm-secret-client/src/path.rs b/swarm-secret-client/src/path.rs index 7aaa5244..3e85c04d 100644 --- a/swarm-secret-client/src/path.rs +++ b/swarm-secret-client/src/path.rs @@ -40,6 +40,9 @@ pub enum Kind { /// is what uses it. #[strum(serialize = "hives")] Hive, + /// One swarm service — the things a swarm runs beside the controller. + #[strum(serialize = "services")] + Service, /// The controller itself. There is one per swarm, so the name segment /// below it does not vary; the shape stays uniform anyway, because one /// grant pattern over `/` is cheaper than a special case. @@ -50,7 +53,7 @@ pub enum Kind { impl Kind { /// Every kind, so callers that must cover the whole set can iterate rather /// than restate it — a second list is a list that drifts. - pub const ALL: [Kind; 3] = [Kind::Agent, Kind::Hive, Kind::Controller]; + pub const ALL: [Kind; 4] = [Kind::Agent, Kind::Hive, Kind::Service, Kind::Controller]; /// What to call the name in an error — singular, because the message reads /// "hive name ... is not a single path segment". @@ -59,6 +62,7 @@ impl Kind { match self { Kind::Agent => "agent", Kind::Hive => "hive", + Kind::Service => "service", Kind::Controller => "controller", } } @@ -171,7 +175,7 @@ mod tests { // decision. Spelling each one out is what makes adding a kind a // deliberate edit. let segments: Vec<&str> = Kind::ALL.iter().map(|k| (*k).into()).collect(); - assert_eq!(segments, ["agents", "hives", "controller"]); + assert_eq!(segments, ["agents", "hives", "services", "controller"]); } #[test]