diff --git a/Cargo.lock b/Cargo.lock index 3c5a9fad3..e7d202fed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1774,6 +1774,7 @@ dependencies = [ "clap", "clap-markdown", "clap_complete", + "ed25519-dalek", "forgejo-api", "futures-util", "hive-core-agent-sock", diff --git a/Cargo.toml b/Cargo.toml index cbea8d235..5ef450f61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -248,6 +248,9 @@ matrix-sdk = { version = "0.18", default-features = false, features = [ futures-util = "0.3" hmac = "0.13" sha2 = "0.11" +# Verifies the cache signature over a narinfo (hive-c0re). Already in the lock +# through nkeys. +ed25519-dalek = "2" # Constant-time comparison of a presented secret against the stored one # (`swarm-agent-verify`). Already in the tree through the TLS stack. subtle = "2.6" diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index c9e5b4a15..272d15587 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -334,15 +334,16 @@ exists, so off-the-shelf OTEL/Grafana container dashboards work. Resource `service.name = hyperhive-c0re`; hive-c0re tags each data point `container.name` (= the `h-` machine) and the hive `agent` label: -| Metric | Unit | Kind | Source | -| --------------------------------------- | ---- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds | -| `container.memory.usage` | `By` | gauge | `memory.current` | -| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) | -| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) | -| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + writable rootfs, measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) | -| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) | -| `hyperhive.agent.unrestricted_internet` | `1` | gauge | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable | +| Metric | Unit | Kind | Source | +| --------------------------------------- | ---- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds | +| `container.memory.usage` | `By` | gauge | `memory.current` | +| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) | +| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) | +| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + writable rootfs, measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) | +| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) | +| `hyperhive.agent.unrestricted_internet` | `1` | gauge | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable | +| `hyperhive.agent.closure_pullable` | `1` | gauge | `1` when this hive can fetch the agent's latest closure from the swarm's binary cache and verify its signature, else `0` with a `reason` label (`no-record`, `rev-mismatch`, `not-cached`, `no-credentials`, `fetch-failed`, `bad-signature`; see [the builder docs](../swarm/builder.md#what-hives-do-with-the-cache)). Omitted before the first probe pass, every 5 minutes | The `hyperhive.`-prefixed metrics have no semconv equivalent (memory limit + peak, on-disk footprint, and an instantaneous cpu percent kept diff --git a/docs/swarm/builder.md b/docs/swarm/builder.md index e90780d2b..f1a0f542f 100644 --- a/docs/swarm/builder.md +++ b/docs/swarm/builder.md @@ -249,8 +249,9 @@ KV bucket: the `internal/meta` commit, the closure's store path, the builder's platform and the time. Only the controller writes the bucket, and the builder holds no grant on it. A failed build writes nothing and leaves the previous key. The record's `cached` field says whether the builder pushed the closure to -[the binary cache](#binary-cache). Nothing reads the bucket yet. The closure stays -unrooted in the builder host's store. +[the binary cache](#binary-cache). Each hive reads its own agents' keys to +[report whether it can pull them](#what-hives-do-with-the-cache); no deploy +uses a record yet. The closure stays unrooted in the builder host's store. The controller gives the `builder` forge account read on `internal/meta` when it updates the flake, and again before each build, since the repo is private. A @@ -311,12 +312,21 @@ principal and create what's missing. - `swarm-bao-attic-tokens` creates the cache and mints the push token, which the builder reads, at `secret/swarm/services/attic/push-token`, and a read-only pull token for the hives at `secret/swarm/services/attic/pull-token`. + It also stores the facts a hive pulls with: the cache's address at + `secret/swarm/services/attic/url` (`https://`), its name at + `secret/swarm/services/attic/cache`, and the public key it signs with at + `secret/swarm/services/attic/public-key` (the key `attic cache info` prints). A unit creates each secret only while the store holds none, so a second start mints nothing. Both tokens last ten years and nothing renews them. To replace one, write the new token to its path in the store: nothing mints over a value that exists. A new signing key invalidates both tokens. +The unit creates the address, name and key once and never updates them. After +you change `swarm.attic.domain`, or after the server's database goes and the +cache gets a new key, delete the stale leaf and restart +`swarm-bao-attic-tokens`. + `swarm.attic.domain` and `swarm.attic.cache` default to `attic.` and `swarm` on every host, whether or not it runs the cache, so a builder on any host pushes to it. @@ -345,6 +355,14 @@ the swarm at an existing tenant: or `bao kv put` with `value=-` and the token on stdin, so the token is in no command line. The token needs push rights on that cache. Until it exists, every push fails and builds stay green with `cached` false. + Seed the hives' pull token the same way at + `secret/swarm/services/attic/pull-token`, with read rights on that cache. +3. Seed the three facts hives pull with, each as field `value`: + `secret/swarm/services/attic/url` (`https://` and the host, no trailing + slash; hives refuse `http://`), `secret/swarm/services/attic/cache` (the cache's name) and + `secret/swarm/services/attic/public-key` (`:`, from the + cache's page or `attic cache info`). Until all four exist, hives report + `no-credentials`. The builder checks the instance's certificate against the system roots and pins nothing, so a Let's Encrypt certificate needs no setup. The builder @@ -358,6 +376,58 @@ The swarm then renders no container, vhost or unit of its own cache, and creates no secret in the store. The store's host still mints the cache's leaf and writes its role, which nothing logs in to. +## What hives do with the cache + +Every five minutes `swarm-controller` copies the four `swarm/services/attic/` +leaves (`pull-token`, `url`, `cache`, `public-key`) to +`swarm/hives//attic/` for each hive, because a hive reads only its +own prefix. It writes a copy only when the value differs, skips a leaf the +store doesn't hold, and leaves a copy whose source disappeared. + +`hive-c0re` reads its four leaves every five minutes under the hive's own store +identity and writes `/run/hyperhive/attic/netrc`, `substituter` and +`public-key` (tmpfs, each `0600`). It accepts only an `https` URL, since the +token goes out in a header and in the netrc; a hive that finds another scheme +writes nothing. No deploy reads the files yet: deploys still build locally. + +The same pass checks each of the hive's agents. It reads the agent's record in +`agent-closures`, then fetches the closure's narinfo from +`//.narinfo` with the pull token as the basic-auth password. +It checks that a signature by the public key covers the narinfo, in Rust, with +no help from the host's nix daemon. It downloads nothing. The result is a log +line (at info when it changes) and the gauge +`hyperhive.agent.closure_pullable{agent}` (the `hive` label comes from the +collector, as for every `hyperhive.*` metric): `1` when the hive can pull the +latest closure, else `0` with a `reason` label. The first check that fails +decides the reason: + +| `reason` | Meaning | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `no-credentials` | One of the four leaves is missing or unusable (a URL that isn't `https` is unusable). The swarm has no cache, the controller is too old to copy them, or you haven't seeded them yet. | +| `no-record` | `agent-closures` holds no build for the agent. | +| `rev-mismatch` | The record's build came from another `internal/meta` commit than its head, or the hive can't read the head. | +| `not-cached` | The record says the builder didn't push the closure, or the cache answers 404 for the narinfo. | +| `fetch-failed` | The request failed, or the cache answered with a status other than 200 or 404 (a refused token is a 401 or 403). | +| `bad-signature` | The narinfo describes another path, or no signature by the public key covers it. | + +A failed read of the store, the queue or the forge leaves the last result in +place and raises a warning after three failed passes. A hive reports nothing +before its first pass. + +Mixed versions: + +| controller | hive | what happens | +| ---------- | ---- | ----------------------------------------------------------------------------------- | +| new | new | the copies exist and the hive reports per agent | +| old | new | the controller copies nothing, so the hive reports `no-credentials` for every agent | +| new | old | the controller writes copies nobody reads | +| any | any | a swarm with no cache seeds no leaves, and every hive reports `no-credentials` | + +The signature check follows Nix's fingerprint format but hasn't run against a +live attic. A `bad-signature` on every agent in the first pass after a deploy +means check attic's fingerprint format; it says nothing about an attacker. A +`fetch-failed` on every agent points at the URL, the token or the network. + ## Credentials | What | Where | Who reads it | diff --git a/docs/swarm/credentials-reference.md b/docs/swarm/credentials-reference.md index f73686b72..42f85c799 100644 --- a/docs/swarm/credentials-reference.md +++ b/docs/swarm/credentials-reference.md @@ -1,3 +1,5 @@ +| this hive's cache pull facts | `swarm-controller`, into the secret store at `swarm/hives//attic/{pull-token,url,cache,public-key}`, copied every five minutes from `swarm/services/attic/` | `hive-c0re` reads them under the hive's own store identity and writes `/run/hyperhive/attic/netrc` (the pull token), `substituter` and `public-key` (tmpfs, each `0600` `hive-core`). Nothing uses the files yet. | + # Credentials: the per-secret reference Every credential in the swarm's secret store, in one table: who mints it, who @@ -14,40 +16,44 @@ the rest of the cell says how. -| store path | minter | reader | automatic re-mint | automatic re-pull | -| ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `swarm/agents//matrix/main` | `swarm-controller`, at agent creation and every 5 minutes | the agent | ✅ when missing or rejected by the homeserver | ✅ the daemon restarts within 5 minutes of a rejection | -| `swarm/agents//matrix/` | `swarm-controller`, from a credential you hand it | the agent | must be stated | ✅ retried within 2 to 16 minutes after a rejection | -| `swarm/agents//matrix-recovery/` | `swarm-controller`, a random passphrase per account | the agent | ❌ never replaced: the account's key backup opens only with this passphrase | ❌ read when the account comes up | -| `swarm/controller/swarm-controller/matrix/appservice-token` | `swarm-matrix-ctl appservice sync` in `hive-matrix`, from the registration's `as_token` | `swarm-controller` | ❌ follows the registration below | ✅ read every 5 minutes | -| `swarm/controller/swarm-controller/matrix/appservice-registration` | `swarm-matrix-ctl appservice sync` in `hive-matrix`: seeded from the container's own tokens when the store holds none, a new pair only when the container has none either | `swarm-matrix-ctl` in `hive-matrix`; the controller has no grant to it | ❌ minted once; an operator who replaces it makes `sync` write the new pair into the container and restart the homeserver | ✅ `sync` re-reads it every 10 minutes | -| `swarm/controller/swarm-controller/oidc/client` | authelia, first boot | `swarm-controller` | ❌ minted once; `swarm-secret-publish` republishes a re-mint | ✅ re-read when authelia refuses it | -| `swarm/controller/swarm-controller/forge/token` | the forge, once | `swarm-controller` | ❌ minted once; delete `/var/lib/forgejo/swarm-controller-token-name` in the forge container to mint a new one | ✅ re-read on a 401 | -| `swarm/controller/swarm-controller/grafana/token` | `swarm-bao-grafana-controller-token` on Grafana's host | `swarm-controller` | ✅ every 10 minutes, when Grafana refuses it or none is stored | ✅ re-read on a 401 | -| `swarm/controller/swarm-controller/forge/webhook-secret` | `swarm-controller`, at start, from its old `webhook-secret` file; a new one only when there is no file | `swarm-controller` | ❌ minted once; a rotated secret fails every delivery until the forge hooks are re-registered | ❌ read at start | -| `swarm/agents//bao-mtls` | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), at agent creation | `hive-c0re`, then the agent | ✅ re-issued every 5 minutes once past half its 7-day validity | ✅ the agent re-reads every 10 minutes; it renews only while its current leaf is valid | -| `swarm/agents//queue` | `swarm-controller`, at agent creation | the agent; the queue, log and metrics auth services | ✅ re-minted at 45 days; skipped for `Destroyed` agents, whose path is deleted | ✅ read on every reconnect; an open connection keeps the old secret for up to an hour; the log and metrics stores honour a replaced secret for up to `deploy.victorialogs.authVerdictTtl` and `deploy.victoriametrics.agentAuthVerdictTtl` (60 s by default) | -| `swarm/agents//forge-token` | `swarm-controller`, at agent creation and every 5 minutes | the agent | ✅ when missing or no longer matching the forge | ✅ every 10 minutes | -| `swarm/agents//forge/