From ecd4445680013726d77e0f8ed8eddaab113085be Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 7 Nov 2026 13:20:55 +0100 Subject: [PATCH] hive-c0re pulls closures through the nix daemon; hive-priv verifies the swarm signature before applying --- docs/2-handbook/agents/config-changes.md | 10 +- .../2-handbook/swarm/credentials-reference.md | 20 +- .../4-implementation/scheduler/coordinator.md | 10 +- docs/4-implementation/swarm/builder.md | 59 +- .../trust-boundary/hive-c0re-privilege.md | 135 ++-- hive-c0re/src/cache_pull.rs | 484 ++++++++++++ hive-c0re/src/job_queue/exec.rs | 8 +- hive-c0re/src/main.rs | 1 + hive-c0re/src/priv_client.rs | 106 ++- hive-priv-sock/src/lib.rs | 55 +- hive-priv/README.md | 17 +- hive-priv/src/closure.rs | 719 ++++++++---------- hive-priv/src/container.rs | 24 +- hive-priv/src/main.rs | 6 +- nix/host-modules/hive-c0re/default.nix | 8 +- nix/host-modules/hive-priv-cache.nix | 43 +- nix/host-modules/hive-priv.nix | 4 +- nix/host-modules/lib/attic-root-facts.nix | 47 +- nix/module-eval/hive-tls.nix | 19 +- nix/script-tests/attic-root-facts.nix | 2 + nix/script-tests/attic-root-facts.sh | 74 +- 21 files changed, 1262 insertions(+), 589 deletions(-) create mode 100644 hive-c0re/src/cache_pull.rs diff --git a/docs/2-handbook/agents/config-changes.md b/docs/2-handbook/agents/config-changes.md index 4abfc448..c73e2b8e 100644 --- a/docs/2-handbook/agents/config-changes.md +++ b/docs/2-handbook/agents/config-changes.md @@ -154,11 +154,11 @@ the swarm builder built the agent from the updated repo (the `MetaSync`; its `Create` reads the record once. - **The closure.** The hive builds nothing. `Prebuild`, `Swap` and `Create` read the agent's record in the swarm's `agent-closures` bucket (the controller - writes it from the builder's reply) and have `hive-priv` download the closure - it names from the swarm cache (`RealiseClosure`). `Swap` and `Create` then - give `hive-priv` the store path (`toplevel` on `UpdateContainer` and - `CreateContainer`), which accepts only the path its last `RealiseClosure` for - that agent returned. On the wire the + writes it from the builder's reply) and pull the closure it names from the + swarm cache through the host's nix daemon. `Swap` and `Create` then give + `hive-priv` the store path (`toplevel` on `UpdateContainer` and + `CreateContainer`), which accepts only a path in the store that carries the + swarm cache's signature. On the wire the `hive_priv_sock::PrivRequest` variants carry a required field: `{"op":"update_container","name":"atlas","stream":true,"toplevel":"/nix/store/-nixos-system-atlas"}`. A rebuild with no record, or none the builder pushed, fails with an error diff --git a/docs/2-handbook/swarm/credentials-reference.md b/docs/2-handbook/swarm/credentials-reference.md index 0e6126d8..14e3c795 100644 --- a/docs/2-handbook/swarm/credentials-reference.md +++ b/docs/2-handbook/swarm/credentials-reference.md @@ -151,16 +151,16 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](credentials. ### Hive-level — one of each per hive -| secret | generated by | lives at | -| ---------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| hive leaf certs | `hive-tls.nix`, issued by the secret store's `pki` mount | `/.pem` | -| the hive's own store identity | `glue-bao-tls.nix` on the store's host; placed by hand on every other host ([`ca.md`](../../4-implementation/swarm/ca.md#the-gateway-leaf-comes-from-the-secret-store)) | `deploy.hive-controller.tls.baoHiveIdentityCertFile` / `baoHiveIdentityKeyFile` | -| the cache's public key and pull token, for hive-priv | `swarm-controller` copies them to `swarm/hives//attic/{public-key,pull-token}` (see the cache-facts row above) | `hive-priv-cache-facts` (root) reads those two under the hive's own store identity every ten minutes and writes `/run/hyperhive-priv/attic/public-key` and `netrc` (tmpfs, root, each `0600`, directory `0700`); the cache's address comes from `services.hyperhive.swarm.attic.domain` and `cache`, not from the store. A store that holds no facts, or that doesn't answer, leaves the files as they stand, and a hive that never got them has `hive-priv` answer `not available`. | -| the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | -| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | -| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity for each forge read, and hands it to the `git` commands it runs. It keeps no copy on disk | -| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.deploy.hive-controller.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `unrestricted` beside it lists the users whose agent holds the swarm's `unrestricted_internet` flag, names only; it lists every agent the hive never read as off `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; an agent without one sends none, so it goes direct | -| the squid self-check's password | `swarm-controller` mints `swarm/hives//squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Both self-check users (`selfcheck.squid` and the always-unrestricted `selfcheck-unrestricted.squid`) use it, so whoever holds it has unrestricted egress through this squid Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks | +| secret | generated by | lives at | +| ------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| hive leaf certs | `hive-tls.nix`, issued by the secret store's `pki` mount | `/.pem` | +| the hive's own store identity | `glue-bao-tls.nix` on the store's host; placed by hand on every other host ([`ca.md`](../../4-implementation/swarm/ca.md#the-gateway-leaf-comes-from-the-secret-store)) | `deploy.hive-controller.tls.baoHiveIdentityCertFile` / `baoHiveIdentityKeyFile` | +| the cache's public key and pull token, for the host's nix daemon and hive-priv | `swarm-controller` copies them to `swarm/hives//attic/{public-key,pull-token}` (see the cache-facts row above) | `hive-priv-cache-facts` (root) reads those two under the hive's own store identity every ten minutes and writes `/run/hyperhive-priv/attic/public-key` and `netrc` (tmpfs, root, each `0600`, directory `0700`); it also writes `/run/hyperhive-nix/cache.conf` (tmpfs, root, `0644`, no secret in it): the cache's address, the key and the netrc's path as nix.conf settings that the host's nix.conf includes. The cache's address comes from `services.hyperhive.swarm.attic.domain` and `cache`, not from the store. A store that holds no facts, or that doesn't answer, leaves the files as they stand; a hive that never got them has `hive-c0re` ask for a refresh and then fail the pull with `this hive has no cache facts yet`. `hive-core` reads only the fragment, for the address. | +| the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | +| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | +| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity for each forge read, and hands it to the `git` commands it runs. It keeps no copy on disk | +| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.deploy.hive-controller.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `unrestricted` beside it lists the users whose agent holds the swarm's `unrestricted_internet` flag, names only; it lists every agent the hive never read as off `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; an agent without one sends none, so it goes direct | +| the squid self-check's password | `swarm-controller` mints `swarm/hives//squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Both self-check users (`selfcheck.squid` and the always-unrestricted `selfcheck-unrestricted.squid`) use it, so whoever holds it has unrestricted egress through this squid Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks | ### Store identities, one per principal diff --git a/docs/4-implementation/scheduler/coordinator.md b/docs/4-implementation/scheduler/coordinator.md index d32f4087..9ac6c8d8 100644 --- a/docs/4-implementation/scheduler/coordinator.md +++ b/docs/4-implementation/scheduler/coordinator.md @@ -42,11 +42,11 @@ because there is no malformed spec to reject. Nix-heavy — hold one of the `buildSlots` permits for the node's duration: -| Node | Wraps | -| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Prebuild` | download the swarm's closure for the agent into the store while the container keeps serving (its meta preamble is the upstream `MetaSync` node). With no closure to deploy it fails here, before the stop. Skipped when the container is already down; `Swap` downloads it instead | -| `Swap` | drop-in rewrite + profile-swap onto the swarm's closure (requires the container stopped); the post-swap bookkeeping tail lives in the sibling `RebuildBookkeeping` node | -| `Create` | first-spawn `nixos-container create` proper; assumes the upstream `Provision` node already registered the agent; downloads the closure the swarm built for the agent and creates the container from it (with none to deploy it fails and creates nothing), then marks the closure applied | +| Node | Wraps | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `Prebuild` | pull the swarm's closure for the agent into the store through the host's nix daemon while the container keeps serving (its meta preamble is the upstream `MetaSync` node). With no closure to deploy it fails here, before the stop. Skipped when the container is already down; `Swap` downloads it instead | +| `Swap` | drop-in rewrite + profile-swap onto the swarm's closure (requires the container stopped); the post-swap bookkeeping tail lives in the sibling `RebuildBookkeeping` node | +| `Create` | first-spawn `nixos-container create` proper; assumes the upstream `Provision` node already registered the agent; pulls the closure the swarm built for the agent and creates the container from it (with none to deploy it fails and creates nothing), then marks the closure applied | Cheap — no build slot: diff --git a/docs/4-implementation/swarm/builder.md b/docs/4-implementation/swarm/builder.md index 045008d0..17d44eed 100644 --- a/docs/4-implementation/swarm/builder.md +++ b/docs/4-implementation/swarm/builder.md @@ -452,7 +452,8 @@ you change `swarm.attic.domain`, delete the stale `url` leaf and restart replaces the stored one with the key `attic cache info` prints, so a cache that loses its database and gets a new key needs no step: the next run of the unit stores the new key, the controller copies it to the hives within five -minutes, and `hive-priv-cache-facts` reads it within ten. +minutes, and `hive-priv-cache-facts` reads it within ten (or at once, when a +rebuild finds the host without facts). `swarm.attic.domain` and `swarm.attic.cache` default to `attic.` and `swarm` on every host, whether or not it runs the cache, so a builder on any @@ -532,14 +533,17 @@ identity and writes `/run/hyperhive/attic/netrc`, `substituter` and token goes out in a header and in the netrc; a hive that finds another scheme writes nothing. -`hive-priv` doesn't read those files. The key that must have signed a download and -the pull token come from `hive-priv-cache-facts`, a root unit that reads the -hive's own `attic/public-key` and `attic/pull-token` from the store under the -hive's own store identity every ten minutes and writes them to -`/run/hyperhive-priv/attic/` (`0700`, root; each file `0600`). The cache's -address is the host's `services.hyperhive.swarm.attic.domain` and `cache`, -handed to `hive-priv` in its unit's environment. A file `hive-c0re` writes -decides nothing `hive-priv` trusts. +`hive-priv` and the nix daemon don't read those files. The key that must have +signed a download and the pull token come from `hive-priv-cache-facts`, a root +unit that reads the hive's own `attic/public-key` and `attic/pull-token` from +the store under the hive's own store identity every ten minutes and writes them +to `/run/hyperhive-priv/attic/` (`0700`, root; each file `0600`). It also writes +the nix.conf fragment `/run/hyperhive-nix/cache.conf`, which the host's nix.conf +includes: the cache's address (the host's `services.hyperhive.swarm.attic.domain` +and `cache`) as a trusted substituter, the key as a trusted key and the netrc's +path. It restarts `nix-daemon` only when the fragment changed after the daemon +started, which matters for CppNix; Lix re-reads nix.conf on every connection. A +file `hive-c0re` writes decides nothing the daemon or `hive-priv` trusts. The same pass checks each of the hive's agents. It reads the agent's record in `agent-closures`, then fetches the closure's narinfo from @@ -581,17 +585,20 @@ means check attic's fingerprint format; it says nothing about an attacker. A ### Rebuilds download the closure A hive builds nothing. A rebuild's `Prebuild` node, which runs while the agent is -still up, reads the agent's record in `agent-closures` and asks `hive-priv` to -realise the record's store path (`RealiseClosure`). `hive-priv` runs -`nix-store --realise` with `--max-jobs 0`, so nix can't build the path, and with -the host's configured cache address, and the public key and netrc from the -root-owned `/run/hyperhive-priv/attic/`, as per-call settings. The settings replace the host's `substituters` and -`trusted-public-keys` and set `require-sigs true`: the swarm cache is the only -source, and nix stores the path only when the cache's key signed it. The host's -nix configuration stays as it was. +still up, reads the agent's record in `agent-closures` and has `hive-c0re` pull +the record's store path. `hive-c0re` isn't root and needs no root: it runs +`nix-store --realise --max-jobs 0 --option substituters ` +through the host's nix daemon. The address is the one in the fragment above, the +only substituter the daemon lets an unprivileged client name. The daemon +ignores a client's keys, `require-sigs` and netrc, so it stores the path only +when a key it trusts signed it. When the fragment is missing, `hive-c0re` first +asks `hive-priv` to run `hive-priv-cache-facts` (up to 60 s) instead of waiting for +its timer. A host that still has no facts, a signature refusal and a path the +cache lacks each fail with their own message. -`Swap` then does the same lookup and download, a no-op once the path is in the -store, and gives `hive-priv` the store path to apply. A stopped agent skips +`Swap` then does the same lookup and pull, a no-op once the path is in the +store, and gives `hive-priv` the store path to apply. `hive-priv` applies it only +when `nix store verify --sigs-needed 1` finds the swarm key's signature on it. A stopped agent skips `Prebuild`, so `Swap` downloads it. A first deploy's `Create` node does the same and creates the container from that path. @@ -608,13 +615,13 @@ an unreachable cache costs is one function. Mixed versions: -| hive | controller or record | what happens | -| ---- | -------------------- | ------------------------------------------------------------------------------------------ | -| old | any | builds locally, as before | -| new | old, or no record | refuses the create; a running agent keeps running, and the rebuild logs an error naming it | -| new | cache down | the download fails; same as no record, with the download's error in the log | -| new | signature mismatch | nix refuses the path; same as no record, and `hive-priv` logs the refusal as well | -| new | old `hive-priv` | it refuses the request, which has no `meta_rev`, so the swap or create fails | +| hive | controller or record | what happens | +| ---- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| old | any | builds locally, as before | +| new | old, or no record | refuses the create; a running agent keeps running, and the rebuild logs an error naming it | +| new | cache down | the download fails; same as no record, with the download's error in the log | +| new | signature mismatch | nix refuses the path; same as no record, and `hive-priv` logs the refusal as well | +| new | old `hive-priv` | it refuses the apply of a path it didn't download; `hive-c0re` has it download the path (already in the store) and applies again | ## Credentials diff --git a/docs/4-implementation/trust-boundary/hive-c0re-privilege.md b/docs/4-implementation/trust-boundary/hive-c0re-privilege.md index 191ca589..465d2dec 100644 --- a/docs/4-implementation/trust-boundary/hive-c0re-privilege.md +++ b/docs/4-implementation/trust-boundary/hive-c0re-privilege.md @@ -28,35 +28,36 @@ for every operation that genuinely requires root. **Narrow interface** — `PrivRequest` variants map 1:1 to specific known operations; there is no arbitrary command pass-through: -| Operation | What it runs | -| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `StartContainer` / `StopContainer` | `nixos-container start/stop ` | -| `KillContainer` | `machinectl kill --signal=SIGKILL` (`nixos-container` has no kill verb) | -| `StartFailureLog` | `journalctl _SYSTEMD_UNIT=container@h-.service --since @ -n 30 -o cat --no-pager`, fixed argv, 5 s timeout, output capped at 8 KiB; hive-priv clamps `` to the last hour. hive-c0re appends the lines to the failed start's build log, which only the dashboard serves. | -| `CreateContainer` / `UpdateContainer` | `nixos-container create --system-path `; for an update, `nix-env -p --set ` and a reload of the unit. `` must be the store path the last successful `RealiseClosure` returned for that agent (in memory, per agent), so hive-c0re can't name a path never realised for the agent; hive-priv builds nothing. The binding doesn't mean hive-c0re can't choose the closure: it can still ask hive-priv to download any swarm-signed path for any agent and then apply it. The swarm key and the cache bound that to a cross-assignment risk (one agent gets another's closure), not an arbitrary-path one | -| `RealiseClosure` | `nix-store --realise --max-jobs 0` against the swarm cache, with the host's configured cache address, and the public key and netrc from root-owned `/run/hyperhive-priv/attic/`, as per-call settings that replace the host's substituters and trusted keys and require a signature. The path must be `/nix/store/<32 chars>-` and the agent name valid. It downloads and never builds | -| `DestroyContainer` | `nixos-container destroy ` | -| `ListContainers` | `nixos-container list` | -| `ReloadGatewayNginx` | `systemctl reload/start/reset-failed nginx` (host unit; the unit name is hard-coded, not a parameter) | -| `WriteNspawnFlags` | write `/etc/nixos-containers/.conf` (bind-mount list, network isolation vars, and `--ephemeral` after the root's salvage). hive-priv first copies a stopped container's root to the agent's `work/`, then renames it aside (it deletes nothing), following no link the container could have planted; for an agent whose closure record sets `private_users` it also adds `--private-users=pick`, mounts every bind `owneridmap`, and once chowns the agent's `state`, `harness`, `work` and `claude` dirs (never `config`, following no link) to the data owner | -| `EnsureAgentSocketDir` | `mkdirat` `/run/hive-agent/` `0751 root:root` under an `O_NOFOLLOW` parent fd; leaves an existing directory alone, refuses anything else; for an agent whose closure record sets `private_users` it then chowns the dir to the data owner | -| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=`/`CPUWeight=`/`IOWeight=` systemd drop-in for agent container, with an `ExecStartPre=` that runs `hive-priv check-rootfs ` (no-follow check of the root paths the start script touches; see persistence.md), and a GC root on the build it names | -| `RemoveServiceDropin` | remove `container@.service.d/` drop-in on destroy, and the GC root of the hive-priv build it named | -| `DaemonReload` | `systemctl daemon-reload` | -| `ControlInfraContainer` | `systemctl nginx.service` for the gateway — the `InfraContainer` enum is the allowlist, and serde rejects unknown names at the wire boundary (`hive-c0re`, the forge and the matrix have no variant, so no request can name them) | -| `SyncAgentTmpfiles` | unlink `/etc/tmpfiles.d/hyperhive-agents.conf` and return `Ok` | -| `SetAgentPaused` | create / remove the `//harness/paused` marker that parks an agent's turn loop | -| `EnsureAgentSubvolume` | `btrfs subvolume create /` for a new agent — no-op when the path exists or the filesystem isn't btrfs | -| `UpgradeAgentSubvolume` | migrate an existing plain state dir into a subvolume: create, `cp -a --reflink=auto`, atomic swap. Operator opt-in, and the caller stops the agent first | -| `DeleteAgentSubvolume` | `btrfs subvolume delete /` — purge path only, never a plain destroy | -| `EnsureBtrfsQuota` | `btrfs quota enable `. Operator opt-in — enabling forces a full rescan | -| `ReadSubvolumeUsage` | `btrfs qgroup show -f --raw /` | -| `SetSubvolumeQuota` | `btrfs qgroup limit /` | -| `MeasureAgentDisk` | `du -s -x -b -c -P` over `//state`, `//work` and `/var/lib/nixos-containers/h-`, killed after 10 min. hive-priv derives the paths from the name; the request carries none | -| `SnapshotAgentSubvolume` | `btrfs subvolume snapshot -r ` — the frozen point-in-time copy a migration streams from | -| `DeleteAgentSnapshot` | `btrfs subvolume delete ` | -| `SendAgentSnapshotToFile` | `btrfs send [-p ] ` into a bare filename under `MIGRATE_STAGING_ROOT`; refuses to overwrite an existing export | -| `SendAgentSnapshotToFd` | `btrfs send [-p ] ` into a **file descriptor passed with the request** (`SCM_RIGHTS`). hive-c0re connects to the peer and hands over the connected socket, so hive-priv never learns an address or protocol. hive-priv requires exactly one descriptor here, and refuses one that arrives alongside any other operation | +| Operation | What it runs | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `StartContainer` / `StopContainer` | `nixos-container start/stop ` | +| `KillContainer` | `machinectl kill --signal=SIGKILL` (`nixos-container` has no kill verb) | +| `StartFailureLog` | `journalctl _SYSTEMD_UNIT=container@h-.service --since @ -n 30 -o cat --no-pager`, fixed argv, 5 s timeout, output capped at 8 KiB; hive-priv clamps `` to the last hour. hive-c0re appends the lines to the failed start's build log, which only the dashboard serves. | +| `CreateContainer` / `UpdateContainer` | `nixos-container create --system-path `; for an update, `nix-env -p --set ` and a reload of the unit. `` must already be in the store with a signature by the swarm cache's key: hive-priv runs `nix store verify --no-contents --sigs-needed 1` with `trusted-public-keys` set to the key in root-owned `/run/hyperhive-priv/attic/public-key`, and refuses a path that's absent or that the key didn't sign. hive-priv builds nothing. The check doesn't bind the path to the agent whose pull produced it: hive-c0re could already pull any swarm-signed path and apply it to any agent, so the check bounds the same risk (one agent gets another's closure), not an arbitrary-path one | +| `RealiseClosure` | Kept for one release, for an older hive-c0re that still asks hive-priv to download. Downloads nothing: answers `Ok` when the path is in the store and signed by the swarm key (what `CreateContainer` needs), else `not available: `. The path must be `/nix/store/<32 chars>-` and the agent name valid | +| `RefreshCacheFacts` | `systemctl start hive-priv-cache-facts.service`, waited for up to 60 s. The request names no unit and has no field. hive-c0re asks for it when its cache fragment is missing | +| `DestroyContainer` | `nixos-container destroy ` | +| `ListContainers` | `nixos-container list` | +| `ReloadGatewayNginx` | `systemctl reload/start/reset-failed nginx` (host unit; the unit name is hard-coded, not a parameter) | +| `WriteNspawnFlags` | write `/etc/nixos-containers/.conf` (bind-mount list, network isolation vars, and `--ephemeral` after the root's salvage). hive-priv first copies a stopped container's root to the agent's `work/`, then renames it aside (it deletes nothing), following no link the container could have planted; for an agent whose closure record sets `private_users` it also adds `--private-users=pick`, mounts every bind `owneridmap`, and once chowns the agent's `state`, `harness`, `work` and `claude` dirs (never `config`, following no link) to the data owner | +| `EnsureAgentSocketDir` | `mkdirat` `/run/hive-agent/` `0751 root:root` under an `O_NOFOLLOW` parent fd; leaves an existing directory alone, refuses anything else; for an agent whose closure record sets `private_users` it then chowns the dir to the data owner | +| `WriteResourceLimits` | write `CPUQuota=`/`MemoryMax=`/`CPUWeight=`/`IOWeight=` systemd drop-in for agent container, with an `ExecStartPre=` that runs `hive-priv check-rootfs ` (no-follow check of the root paths the start script touches; see persistence.md), and a GC root on the build it names | +| `RemoveServiceDropin` | remove `container@.service.d/` drop-in on destroy, and the GC root of the hive-priv build it named | +| `DaemonReload` | `systemctl daemon-reload` | +| `ControlInfraContainer` | `systemctl nginx.service` for the gateway — the `InfraContainer` enum is the allowlist, and serde rejects unknown names at the wire boundary (`hive-c0re`, the forge and the matrix have no variant, so no request can name them) | +| `SyncAgentTmpfiles` | unlink `/etc/tmpfiles.d/hyperhive-agents.conf` and return `Ok` | +| `SetAgentPaused` | create / remove the `//harness/paused` marker that parks an agent's turn loop | +| `EnsureAgentSubvolume` | `btrfs subvolume create /` for a new agent — no-op when the path exists or the filesystem isn't btrfs | +| `UpgradeAgentSubvolume` | migrate an existing plain state dir into a subvolume: create, `cp -a --reflink=auto`, atomic swap. Operator opt-in, and the caller stops the agent first | +| `DeleteAgentSubvolume` | `btrfs subvolume delete /` — purge path only, never a plain destroy | +| `EnsureBtrfsQuota` | `btrfs quota enable `. Operator opt-in — enabling forces a full rescan | +| `ReadSubvolumeUsage` | `btrfs qgroup show -f --raw /` | +| `SetSubvolumeQuota` | `btrfs qgroup limit /` | +| `MeasureAgentDisk` | `du -s -x -b -c -P` over `//state`, `//work` and `/var/lib/nixos-containers/h-`, killed after 10 min. hive-priv derives the paths from the name; the request carries none | +| `SnapshotAgentSubvolume` | `btrfs subvolume snapshot -r ` — the frozen point-in-time copy a migration streams from | +| `DeleteAgentSnapshot` | `btrfs subvolume delete ` | +| `SendAgentSnapshotToFile` | `btrfs send [-p ] ` into a bare filename under `MIGRATE_STAGING_ROOT`; refuses to overwrite an existing export | +| `SendAgentSnapshotToFd` | `btrfs send [-p ] ` into a **file descriptor passed with the request** (`SCM_RIGHTS`). hive-c0re connects to the peer and hands over the connected socket, so hive-priv never learns an address or protocol. hive-priv requires exactly one descriptor here, and refuses one that arrives alongside any other operation | **Strings that reach root-loaded config** — `WriteResourceLimits` and `WriteNspawnFlags` copy request strings into a systemd drop-in and an env @@ -68,39 +69,61 @@ refuses, never escapes, any value outside its shape. `memory_max` is bind or credential path is absolute with no whitespace, control byte, `\`, `$`, backtick, `"` or `:`. The size limits of a value stay with systemd. -**Closures from the swarm cache** — `RealiseClosure` is how a hive's rebuild -takes the closure the swarm built instead of building it. Root-owned sources -alone decide what root accepts: +**Closures from the swarm cache** — `hive-c0re` pulls the closure the swarm +built through the host's nix daemon, and `hive-priv` applies it only when the +swarm's key signed it. Pulling needs no root: `nix-store --realise` of a store +path is a request to the daemon. Root-owned sources alone decide what the +daemon accepts: - The cache signs. The swarm's cache (attic) holds the signing key and signs each narinfo it serves; the private key never leaves it. The builder pushes with a push token and holds no signing key. - The address is the host's configuration: - `services.hyperhive.swarm.attic.domain` and `cache` become - `HIVE_PRIV_CACHE_URL` in the `hive-priv` unit. No request and no file chooses - it, and `hive-priv` refuses an address that isn't `https`. -- The key and the pull token come from the secret store. The root unit - `hive-priv-cache-facts` reads the hive's own copies under the hive's own store - identity and writes them to `/run/hyperhive-priv/attic/` (directory `0700`, - files `0600`, root). `hive-priv` reads nothing from `/run/hyperhive/attic/`, - which `hive-c0re` writes for the probe, and it refuses a file in the root - directory that isn't a plain file owned by root or that others can read, and - the directory under the same rule. -- Nix verifies. For that one call `hive-priv` sets `substituters` and - `trusted-public-keys` to the swarm cache and its key, and `require-sigs true`. - A path without a signature by that key isn't stored; `hive-priv` answers - `not available` and logs the refusal at error level. `--max-jobs 0` means no - answer, whatever the cache sends, starts a build as root. -- A hive realises only paths whose signature checks. The request carries an - agent name and a store path, both validated, and no address, key or file; any - other field never reaches the verb. + `services.hyperhive.swarm.attic.domain` and `cache`. The root unit + `hive-priv-cache-facts` lists it as `extra-trusted-substituters` in a nix.conf + fragment, `/run/hyperhive-nix/cache.conf`, that the host's nix.conf includes + (`!include`, so a host whose unit hasn't run yet still starts). +- The key and the pull token come from the secret store. The unit reads the + hive's own copies under the hive's own store identity and writes the key to + `/run/hyperhive-priv/attic/public-key` and the token to `netrc` there + (directory `0700`, files `0600`, root), and the key into the fragment as + `extra-trusted-public-keys` and the netrc's path as `netrc-file`. The fragment + holds no secret. `hive-c0re` reads its cache address, and nothing else, from + it; `hive-priv` reads nothing from `/run/hyperhive/attic/`, which `hive-c0re` + writes for the probe. +- The daemon enforces. `hive-core` is an unprivileged client: the daemon takes + its `--option substituters` only for an address in `trusted-substituters`, + ignores its `trusted-public-keys`, `require-sigs` and `netrc-file`, and + refuses a path no trusted key signed. `--max-jobs 0` means no answer, whatever + the cache sends, starts a build. +- `hive-priv` verifies. Before `CreateContainer` or `UpdateContainer` applies a + toplevel, it runs `nix store verify --no-contents --sigs-needed 1` with + `trusted-public-keys` set to the key in the root file, and refuses a path that + is absent or that the key didn't sign. The daemon's trusted keys also include + the host's other caches, so this check is the one that counts only the swarm's. +- When the fragment is missing, `hive-c0re` asks `hive-priv` for `RefreshCacheFacts`, + which starts the facts unit and waits up to 60 s, instead of waiting for the + unit's timer. A host the swarm hasn't given facts to answers + `this hive has no cache facts yet`. -What a compromised `hive-c0re` can do with this verb: ask for any store path -that has the shape of one and that the swarm cache signed, which fills the -store with paths the swarm vouches for. It can't make root accept a path from -a cache or under a key of its choosing, because it writes none of the places -those come from. The trust anchor is the cache's signing key and the store that -hands out its public key, as for every other credential. +What a compromised `hive-c0re` can do: pull any store path that the swarm cache +signed, which fills the store with paths the swarm vouches for, and then ask +`hive-priv` to apply any such path to any agent. Before this change it could +already do both through `RealiseClosure` and an apply; the former check bound an +apply to the path `hive-priv` itself downloaded for that agent, and not to the +agent the swarm built it for. It can't make root accept a path from a cache or +under a key of its choosing, because it writes none of the places those come +from. The trust anchor is the cache's signing key and the store that hands out +its public key, as for every other credential. + +**Mixed versions on one host.** `hive-priv` and `hive-c0re` restart separately. +A new `hive-c0re` with an older `hive-priv`: the older one refuses an apply of a +path it didn't download, so `hive-c0re` answers that refusal by sending the +older `RealiseClosure`, which downloads and pins the path (already in the +store), and applies again. A new `hive-priv` with an older `hive-c0re`: it still +accepts `RealiseClosure`, answers `Ok` for a path already in the store with the +swarm's signature and `not available` for one that isn't, and applies. Both +shims go in the release after this one. **Container allowlist** — `hive-priv` validates every request against an allowlist before any operation: the allowlist accepts only names diff --git a/hive-c0re/src/cache_pull.rs b/hive-c0re/src/cache_pull.rs new file mode 100644 index 00000000..173b98a5 --- /dev/null +++ b/hive-c0re/src/cache_pull.rs @@ -0,0 +1,484 @@ +//! Pull the swarm's closure into the store through the host's nix daemon. +//! +//! hive-c0re is not root and does not need to be: `nix-store --realise` of a +//! store path is a request to the daemon, which substitutes it. What the daemon +//! accepts is root's: the root unit `hive-priv-cache-facts` writes a nix.conf +//! fragment ([`CONF_FRAGMENT`]) that lists the swarm cache as a +//! `trusted-substituter`, adds its key to `trusted-public-keys` and names the +//! netrc. An unprivileged client may name a substituter only from that list, +//! cannot lower `require-sigs` or replace the keys, and a path without a +//! signature by a trusted key is refused. hive-priv then applies the toplevel +//! only if the swarm's key signed it. +//! +//! The cache's address is read from the fragment, so the address named is +//! byte for byte the one the daemon trusts. A missing fragment means this host +//! has not been given its cache facts; [`pull`] first asks hive-priv to run the +//! facts unit, which is how the swarm's copy of the facts reaches the host +//! without waiting for the unit's timer. + +use std::time::Duration; + +use anyhow::Result; +use hive_priv_sock::valid_closure_path; + +/// The nix.conf fragment `hive-priv-cache-facts` writes (root-owned, readable). +pub const CONF_FRAGMENT: &str = "/run/hyperhive-nix/cache.conf"; + +/// The fragment's line that lists the cache the daemon lets a client name. +const URL_SETTING: &str = "extra-trusted-substituters"; + +/// Wall-clock bound on one pull. A stalled transfer is bounded by nix's own +/// `stalled-download-timeout`; this covers the rest. +const PULL_TIMEOUT: Duration = Duration::from_mins(30); + +/// Why a closure was not pulled. +#[derive(Debug, PartialEq, Eq)] +pub enum PullError { + /// This host has no cache facts: no fragment, no address in it, or a + /// daemon that has not loaded it. Waiting for the facts unit fixes it. + FactsMissing(String), + /// Nix refused the path: no signature by a key this host trusts. + SignatureRefused(String), + /// The cache does not have the path. + NotFound(String), + /// Anything else nix said, or could not do. + Failed(String), + /// Nix was still running at [`PULL_TIMEOUT`]. + TimedOut, +} + +impl std::fmt::Display for PullError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::FactsMissing(why) => write!( + f, + "this hive has no cache facts yet ({why}); a later deploy retries" + ), + Self::SignatureRefused(last) => write!( + f, + "nix refused the closure, which carries no signature by a key this host trusts: {last}" + ), + Self::NotFound(last) => write!(f, "the swarm cache does not have the closure: {last}"), + Self::Failed(last) => f.write_str(last), + Self::TimedOut => write!(f, "nix-store timed out after {}s", PULL_TIMEOUT.as_secs()), + } + } +} + +impl std::error::Error for PullError {} + +/// What a `nix-store` run came to. +pub enum Run { + Exited { success: bool, stderr: String }, + TimedOut, +} + +/// What [`pull`] does on the host, so a test can stand in for it. +pub trait Effects: Sync { + /// The fragment's text; `None` when it cannot be read. + async fn fragment(&self) -> Option; + /// Run the facts unit through hive-priv and wait for it. + async fn refresh(&self) -> Result<()>; + /// Run `nix-store` with `args`, bounded by [`PULL_TIMEOUT`]. + async fn nix_store(&self, args: Vec) -> std::io::Result; +} + +/// The cache's address in `fragment`: the value of its +/// `extra-trusted-substituters` line, when that is one https address. +#[must_use] +pub fn cache_url(fragment: &str) -> Option<&str> { + fragment.lines().find_map(|line| { + let (name, value) = line.split_once('=')?; + let value = value.trim(); + (name.trim() == URL_SETTING + && value.starts_with("https://") + && !value.contains(char::is_whitespace)) + .then_some(value) + }) +} + +/// The `nix-store` arguments that fetch `store_path` and nothing else. +/// +/// - `--option substituters ` names the one cache the daemon lists as +/// trusted. An unprivileged client's `substituters` that is not on that list +/// empties the list and fails the pull, so the address must be the fragment's. +/// - `--max-jobs 0` forbids a local build, so a path the cache lacks fails. +/// - No `trusted-public-keys`, `require-sigs` or `netrc-file`: the daemon +/// ignores them from an unprivileged client and applies its own. +#[must_use] +pub fn pull_args(store_path: &str, url: &str) -> Vec { + [ + "--realise", + store_path, + "--max-jobs", + "0", + "--option", + "substituters", + url, + ] + .map(str::to_owned) + .into() +} + +/// Name nix's refusal from its stderr. A refusal for want of a signature +/// or a daemon that does not trust the address is told apart from a path the +/// cache lacks and from a cache that is down. +fn classify(stderr: &str) -> PullError { + let last = stderr + .lines() + .rev() + .find(|l| !l.trim().is_empty()) + .unwrap_or("nix-store failed with no message") + .trim() + .to_owned(); + let lower = stderr.to_lowercase(); + if lower.contains("untrusted substituter") { + PullError::FactsMissing( + "the nix daemon does not list the swarm cache as a trusted substituter".to_owned(), + ) + } else if lower.contains("not signed by any of the keys") || lower.contains("signature") { + PullError::SignatureRefused(last) + } else if lower.contains("don't know how to build") + || lower.contains("no substituter that can build") + { + PullError::NotFound(last) + } else { + PullError::Failed(last) + } +} + +/// Pull `store_path` into the store. +/// +/// # Errors +/// [`PullError`]: no cache facts on this host (after one refresh), nix +/// refusing the signature, the cache lacking the path, nix failing otherwise, +/// or a timeout. +pub async fn pull(store_path: &str) -> Result<(), PullError> { + pull_with(&Real, store_path).await +} + +async fn pull_with(fx: &impl Effects, store_path: &str) -> Result<(), PullError> { + if !valid_closure_path(store_path) { + return Err(PullError::Failed(format!( + "{store_path:?} is not a store path" + ))); + } + let url = address_after_refresh(fx).await?; + match fx + .nix_store(pull_args(store_path, &url)) + .await + .map_err(|e| PullError::Failed(format!("running nix-store: {e}")))? + { + Run::Exited { success: true, .. } => Ok(()), + Run::Exited { stderr, .. } => Err(classify(&stderr)), + Run::TimedOut => Err(PullError::TimedOut), + } +} + +/// The cache's address, after one facts refresh when the host has none. +async fn address_after_refresh(fx: &impl Effects) -> Result { + if let Some(url) = address(fx).await { + return Ok(url); + } + let refreshed = fx.refresh().await; + address(fx).await.ok_or_else(|| { + PullError::FactsMissing(match refreshed { + Ok(()) => "the facts unit ran and the swarm has given this hive none".to_owned(), + Err(e) => format!("refreshing them failed: {e:#}"), + }) + }) +} + +async fn address(fx: &impl Effects) -> Option { + fx.fragment() + .await + .as_deref() + .and_then(cache_url) + .map(str::to_owned) +} + +/// The host: the fragment on disk, hive-priv for the refresh, `nix-store` from +/// the system profile through the daemon. +struct Real; + +impl Effects for Real { + async fn fragment(&self) -> Option { + tokio::fs::read_to_string(CONF_FRAGMENT).await.ok() + } + + async fn refresh(&self) -> Result<()> { + crate::priv_client::refresh_cache_facts().await + } + + async fn nix_store(&self, args: Vec) -> std::io::Result { + let mut cmd = tokio::process::Command::new("nix-store"); + // Explicit, so the store is the daemon's whatever the environment. + cmd.args(args) + .env("NIX_REMOTE", "daemon") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .kill_on_drop(true); + match tokio::time::timeout(PULL_TIMEOUT, cmd.output()).await { + Ok(out) => { + let out = out?; + Ok(Run::Exited { + success: out.status.success(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + }) + } + Err(_) => Ok(Run::TimedOut), + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::*; + + const PATH: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; + const URL: &str = "https://cache.example/swarm"; + + fn fragment(url: &str) -> String { + format!( + "extra-trusted-substituters = {url}\nextra-trusted-public-keys = swarm:AAAA\nnetrc-file = /run/hyperhive-priv/attic/netrc" + ) + } + + /// A host that answers from a script: the fragment as it is now (and the + /// one a refresh leaves), what `nix-store` comes to, and a log of calls. + struct Fake { + fragment: Mutex>, + after_refresh: Option, + refresh_fails: bool, + nix: Mutex>, + calls: Mutex>, + } + + impl Fake { + fn new(fragment: Option, nix: Run) -> Self { + Self { + fragment: Mutex::new(fragment), + after_refresh: None, + refresh_fails: false, + nix: Mutex::new(Some(nix)), + calls: Mutex::new(Vec::new()), + } + } + + fn calls(&self) -> Vec { + self.calls.lock().unwrap().clone() + } + } + + impl Effects for Fake { + async fn fragment(&self) -> Option { + self.calls.lock().unwrap().push("fragment".to_owned()); + self.fragment.lock().unwrap().clone() + } + + async fn refresh(&self) -> Result<()> { + self.calls.lock().unwrap().push("refresh".to_owned()); + if self.refresh_fails { + anyhow::bail!("hive-priv unreachable"); + } + *self.fragment.lock().unwrap() = self.after_refresh.clone(); + Ok(()) + } + + async fn nix_store(&self, args: Vec) -> std::io::Result { + self.calls + .lock() + .unwrap() + .push(format!("nix-store {}", args.join(" "))); + Ok(self.nix.lock().unwrap().take().expect("one run")) + } + } + + fn exited(success: bool, stderr: &str) -> Run { + Run::Exited { + success, + stderr: stderr.to_owned(), + } + } + + fn option<'a>(args: &'a [String], name: &str) -> Option<&'a str> { + args.windows(3) + .find(|w| w[0] == "--option" && w[1] == name) + .map(|w| w[2].as_str()) + } + + /// An unprivileged client can name only the trusted address and cannot set + /// keys, signatures or the netrc, so the command carries exactly that, and + /// never lets nix build. + #[test] + fn the_command_names_only_the_swarm_cache_and_never_builds() { + let args = pull_args(PATH, URL); + assert_eq!(args[..2], ["--realise", PATH]); + let jobs = args.iter().position(|a| a == "--max-jobs").expect("flag"); + assert_eq!(args[jobs + 1], "0"); + assert_eq!(option(&args, "substituters"), Some(URL)); + let options: Vec<_> = args.iter().filter(|a| *a == "--option").collect(); + assert_eq!(options.len(), 1, "{args:?}"); + for forbidden in ["trusted-public-keys", "require-sigs", "netrc-file"] { + assert_eq!(option(&args, forbidden), None, "{forbidden}"); + } + assert!(!args.iter().any(|a| a.starts_with("extra-")), "{args:?}"); + } + + #[test] + fn the_address_is_the_fragments_https_line() { + assert_eq!(cache_url(&fragment(URL)), Some(URL)); + for bad in [ + String::new(), + "netrc-file = /x".to_owned(), + fragment("http://cache.example/swarm"), + fragment("https://cache.example/a b"), + fragment(""), + // The key's line is not the address's. + "extra-trusted-public-keys = https://cache.example/swarm".to_owned(), + ] { + assert_eq!(cache_url(&bad), None, "{bad:?}"); + } + } + + #[tokio::test] + async fn with_the_fragment_in_place_it_pulls_without_a_refresh() { + let fx = Fake::new(Some(fragment(URL)), exited(true, "")); + pull_with(&fx, PATH).await.expect("pulled"); + assert_eq!( + fx.calls(), + [ + "fragment".to_owned(), + format!("nix-store {}", pull_args(PATH, URL).join(" ")) + ] + ); + } + + /// Missing facts are refreshed once, and the pull then goes ahead with the + /// address the refresh wrote. + #[tokio::test] + async fn a_missing_fragment_is_refreshed_before_the_pull() { + let mut fx = Fake::new(None, exited(true, "")); + fx.after_refresh = Some(fragment(URL)); + pull_with(&fx, PATH).await.expect("pulled"); + assert_eq!( + fx.calls(), + [ + "fragment".to_owned(), + "refresh".to_owned(), + "fragment".to_owned(), + format!("nix-store {}", pull_args(PATH, URL).join(" ")) + ] + ); + } + + /// Facts that are still absent after the refresh, whether the unit ran or + /// hive-priv could not start it, stop the pull before nix is asked. + #[tokio::test] + async fn facts_still_missing_after_the_refresh_are_reported_and_nix_is_not_run() { + for (case, refresh_fails, want) in [ + ("unit ran", false, "none"), + ("refresh failed", true, "hive-priv unreachable"), + ] { + let mut fx = Fake::new(None, exited(true, "")); + fx.refresh_fails = refresh_fails; + let err = pull_with(&fx, PATH).await.expect_err(case); + let PullError::FactsMissing(why) = &err else { + panic!("{case}: {err:?}"); + }; + assert!(why.contains(want), "{case}: {why}"); + assert!( + format!("{err}").contains("a later deploy retries"), + "{case}" + ); + assert!( + !fx.calls().iter().any(|c| c.starts_with("nix-store")), + "{case}" + ); + } + } + + #[tokio::test] + async fn a_fragment_without_an_address_is_refreshed_like_a_missing_one() { + let mut fx = Fake::new(Some("netrc-file = /x".to_owned()), exited(true, "")); + fx.after_refresh = Some(fragment(URL)); + pull_with(&fx, PATH).await.expect("pulled"); + assert!(fx.calls().contains(&"refresh".to_owned())); + } + + #[test] + fn nixs_refusals_are_told_apart() { + for (stderr, want) in [ + ( + "warning: ignoring substitute for '/nix/store/x' from 'https://c', as it's not signed by any of the keys in 'trusted-public-keys'\nerror: build of '/nix/store/x' failed", + "signature", + ), + ( + "error: cannot add path 'x' because it lacks a signature by a trusted key", + "signature", + ), + ( + "warning: ignoring untrusted substituter 'https://c', you are not a trusted user.\ndon't know how to build these paths:\nerror: build of 'x' failed", + "facts", + ), + ( + "don't know how to build these paths:\n /nix/store/x\nerror: build of '/nix/store/x' failed", + "not found", + ), + ( + "error: path '/nix/store/x' is required, but there is no substituter that can build it", + "not found", + ), + ( + "error: unable to download 'https://c/x.narinfo': Couldn't connect to server", + "other", + ), + ] { + let kind = match classify(stderr) { + PullError::SignatureRefused(_) => "signature", + PullError::FactsMissing(_) => "facts", + PullError::NotFound(_) => "not found", + PullError::Failed(_) => "other", + PullError::TimedOut => "timeout", + }; + assert_eq!(kind, want, "{stderr}"); + } + // The last line is what the operator reads. + let PullError::Failed(last) = classify("warning: a\n\nerror: Couldn't connect\n\n") else { + panic!("not a plain failure"); + }; + assert_eq!(last, "error: Couldn't connect"); + } + + #[tokio::test] + async fn a_failed_run_is_mapped_and_a_timeout_is_a_timeout() { + let fx = Fake::new( + Some(fragment(URL)), + exited( + false, + "warning: ignoring substitute from 'https://c', as it's not signed by any of the keys", + ), + ); + assert!(matches!( + pull_with(&fx, PATH).await, + Err(PullError::SignatureRefused(_)) + )); + let fx = Fake::new(Some(fragment(URL)), Run::TimedOut); + assert_eq!(pull_with(&fx, PATH).await, Err(PullError::TimedOut)); + } + + /// Nothing but a store path is ever handed to nix. + #[tokio::test] + async fn only_a_store_path_is_pulled() { + for bad in ["", "main", "/etc/passwd", "--help", "/nix/store/x"] { + let fx = Fake::new(Some(fragment(URL)), exited(true, "")); + assert!( + matches!(pull_with(&fx, bad).await, Err(PullError::Failed(_))), + "{bad}" + ); + assert!(fx.calls().is_empty(), "{bad}"); + } + } +} diff --git a/hive-c0re/src/job_queue/exec.rs b/hive-c0re/src/job_queue/exec.rs index 95cd594b..21bbf23f 100644 --- a/hive-c0re/src/job_queue/exec.rs +++ b/hive-c0re/src/job_queue/exec.rs @@ -51,8 +51,8 @@ pub(super) trait Host: Sync { fn pinned_closure(&self, name: &str) -> Option; /// Drop the agent's pin. fn unpin_closure(&self, name: &str); - /// Download `store_path` from the swarm's cache into the store, never - /// building it. + /// Download `store_path` from the swarm's cache into the store through the + /// host's nix daemon, never building it. async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()>; /// Re-apply drop-ins, then point the container at the closure `toplevel`. async fn swap_update( @@ -128,7 +128,9 @@ impl Host for RealHost { } async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()> { - crate::priv_client::realise_closure(name, store_path).await + crate::cache_pull::pull(store_path).await?; + tracing::debug!(%name, %store_path, "pulled the swarm's closure through the nix daemon"); + Ok(()) } async fn swap_update( diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index a525806d..8b9c4c87 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -17,6 +17,7 @@ mod actions; mod agent_config; mod agent_set; mod attic; +mod cache_pull; mod container_view; mod coordinator; mod dashboard; diff --git a/hive-c0re/src/priv_client.rs b/hive-c0re/src/priv_client.rs index a742b5de..97d02372 100644 --- a/hive-c0re/src/priv_client.rs +++ b/hive-c0re/src/priv_client.rs @@ -267,9 +267,15 @@ pub async fn kill_container(name: &str) -> Result<()> { pub async fn update_container_streaming( name: &str, toplevel: &str, - on_line: impl FnMut(PrivStream, &str), + mut on_line: impl FnMut(PrivStream, &str), ) -> Result<()> { - ok(call_streaming(&update_container_request(name, toplevel), on_line).await?) + let req = update_container_request(name, toplevel); + let first = ok(call_streaming(&req, &mut on_line).await?); + if !older_hive_priv_refused(&first) { + return first; + } + ask_older_hive_priv_to_download(name, toplevel).await?; + ok(call_streaming(&req, &mut on_line).await?) } fn update_container_request(name: &str, toplevel: &str) -> PrivRequest { @@ -285,9 +291,15 @@ fn update_container_request(name: &str, toplevel: &str) -> PrivRequest { pub async fn create_container_streaming( name: &str, toplevel: &str, - on_line: impl FnMut(PrivStream, &str), + mut on_line: impl FnMut(PrivStream, &str), ) -> Result<()> { - ok(call_streaming(&create_container_request(name, toplevel), on_line).await?) + let req = create_container_request(name, toplevel); + let first = ok(call_streaming(&req, &mut on_line).await?); + if !older_hive_priv_refused(&first) { + return first; + } + ask_older_hive_priv_to_download(name, toplevel).await?; + ok(call_streaming(&req, &mut on_line).await?) } fn create_container_request(name: &str, toplevel: &str) -> PrivRequest { @@ -298,17 +310,49 @@ fn create_container_request(name: &str, toplevel: &str) -> PrivRequest { } } -/// See [`PrivRequest::RealiseClosure`]. A path hive-priv could not download -/// reads as an error whose message starts `not available: `. -/// -/// # Errors -/// The call failed, or hive-priv answered `not available`. -pub async fn realise_closure(agent: &str, store_path: &str) -> Result<()> { +/// What an older hive-priv, which applies only a path it downloaded itself, +/// says to an apply of any other. +const OLDER_PIN_REFUSAL: &str = "is not the closure realised for this agent"; + +/// Whether `applied` is an older hive-priv refusing an apply because it did not +/// download the path itself (hive-c0re pulled it through the nix daemon). A +/// hive-priv of the same release never refuses that way, so a retry through +/// [`ask_older_hive_priv_to_download`] happens only while the two restart out of +/// step. +fn older_hive_priv_refused(applied: &Result<()>) -> bool { + applied + .as_ref() + .is_err_and(|e| format!("{e:#}").contains(OLDER_PIN_REFUSAL)) +} + +/// Ask an older hive-priv to download `toplevel`, as it did before, which also +/// pins the path for the apply that follows. The path is already in the store, +/// so this is a lookup. +async fn ask_older_hive_priv_to_download(name: &str, toplevel: &str) -> Result<()> { + tracing::warn!(%name, %toplevel, "an older hive-priv refused a closure it did not download; asking it to"); ok(call(&PrivRequest::RealiseClosure { - agent: agent.to_owned(), - store_path: store_path.to_owned(), + agent: name.to_owned(), + store_path: toplevel.to_owned(), }) .await?) + .context("asking the older hive-priv to download the closure") +} + +/// How long hive-c0re waits for hive-priv's facts refresh: hive-priv bounds +/// the unit at 60 s. +const REFRESH_WAIT: std::time::Duration = std::time::Duration::from_secs(90); + +/// Run the root unit that reads this hive's cache facts from the store and +/// writes the nix.conf fragment, and wait for it. An older hive-priv does not +/// know the op and answers an error. +/// +/// # Errors +/// The call failed or did not answer in time, or the unit failed. +pub async fn refresh_cache_facts() -> Result<()> { + within(REFRESH_WAIT, async { + ok(call(&PrivRequest::RefreshCacheFacts).await?) + }) + .await } pub async fn destroy_container(name: &str) -> Result<()> { @@ -684,9 +728,11 @@ mod tests { use super::{ BindMount, CredentialMount, HiveFacts, NetworkIsolation, NixBuilds, PrivRequest, PrivResponse, PrivStream, call_at, call_streaming_at, call_with_fd_at, check, - create_container_request, parse_qgroup_usage, resource_limits_request, - send_snapshot_request, update_container_request, within, write_nspawn_flags_request, + create_container_request, older_hive_priv_refused, parse_qgroup_usage, + resource_limits_request, send_snapshot_request, update_container_request, within, + write_nspawn_flags_request, }; + use anyhow::Context as _; use serde_json::{Value, json}; use std::io::{Read as _, Write as _}; use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd, RawFd}; @@ -697,6 +743,38 @@ mod tests { serde_json::to_value(req).expect("a request serialises") } + const REFUSAL: &str = "atlas: /nix/store/x is not the closure realised for this agent (none)"; + + /// An older hive-priv's refusal of a path it did not download is the + /// signal to ask it to download; a success and any other failure are not. + #[test] + fn only_an_older_hive_privs_pin_refusal_asks_for_a_download() { + assert!(older_hive_priv_refused(&Err(anyhow::anyhow!(REFUSAL)))); + // As the streaming call words it: the error wraps hive-priv's text. + assert!(older_hive_priv_refused( + &Err(anyhow::anyhow!(REFUSAL)).context("nixos-container create") + )); + assert!(!older_hive_priv_refused(&Ok(()))); + for other in [ + "nixos-container create failed", + "atlas: refusing /nix/store/x: has no signature by the swarm cache's key", + "connect to hive-priv socket", + ] { + assert!( + !older_hive_priv_refused(&Err(anyhow::anyhow!(other))), + "{other}" + ); + } + } + + #[test] + fn the_cache_facts_refresh_names_no_unit() { + assert_eq!( + wire(&PrivRequest::RefreshCacheFacts), + json!({"op": "refresh_cache_facts"}) + ); + } + #[test] fn create_and_update_keep_the_name_and_toplevel_apart_and_stream() { assert_eq!( diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index dc0aa62f..d36172bf 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -110,9 +110,10 @@ pub fn valid_meta_rev(rev: &str) -> bool { /// Whether `path` is a store path in the shape nix prints one: `/nix/store/`, /// 32 lowercase letters or digits, `-`, then a name of the characters store -/// names allow. The path crosses the wire in [`PrivRequest::RealiseClosure`], -/// [`PrivRequest::UpdateContainer`] and [`PrivRequest::CreateContainer`], so -/// `hive-priv` checks it before it hands it to nix. +/// names allow. The path crosses the wire in [`PrivRequest::UpdateContainer`] +/// and [`PrivRequest::CreateContainer`] (and the older +/// [`PrivRequest::RealiseClosure`]), so `hive-priv` checks it before it hands +/// it to nix, and hive-c0re before it names it to nix. #[must_use] pub fn valid_closure_path(path: &str) -> bool { let Some(rest) = path.strip_prefix("/nix/store/") else { @@ -131,11 +132,6 @@ pub fn valid_closure_path(path: &str) -> bool { }) } -/// What a failed [`PrivRequest::RealiseClosure`] says when nix refused a path -/// because no trusted key signed it, so a caller can tell that apart from a -/// cache that is down. -pub const SIGNATURE_REFUSED: &str = "no signature by the swarm cache's key"; - /// Root of per-agent state directories on the host. /// Subdirectory layout: `//state/`. /// Used by `WriteAgentStateFile` to derive the write path so the @@ -269,8 +265,8 @@ pub enum PrivRequest { KillContainer { name: String }, /// Point `name`'s container profile at `toplevel`, then reload the unit if - /// it runs. hive-priv never builds: `toplevel` must be the path the last - /// successful `RealiseClosure` for `name` returned, and it refuses any other. + /// it runs. hive-priv never builds: `toplevel` must already be in the store + /// and signed by the swarm cache's key, and it refuses any other. /// When `stream` is `true`, hive-priv sends `PrivEvent::Line` messages /// as the process runs, then a terminal `PrivEvent::Done`. UpdateContainer { @@ -292,18 +288,24 @@ pub enum PrivRequest { toplevel: String, }, - /// `nix-store --realise ` from the swarm's binary cache, never - /// building: the closure the swarm built for `agent`, downloaded into the - /// store ahead of the update that applies it. + /// Kept for an older hive-c0re that still asks hive-priv to download: + /// hive-c0re now pulls through the nix daemon and hive-priv downloads + /// nothing. The answer is `Ok` when `store_path` is already in the store + /// and signed by the swarm cache's key (what `CreateContainer` and + /// `UpdateContainer` require), else an error that starts `not available: `. + /// `store_path` is checked by [`valid_closure_path`]. /// - /// The request names no cache and no key: hive-priv takes the cache's - /// address from its own unit's environment and the key and the pull token - /// from files only root can write, and nix accepts the path only with a - /// signature by that key. `store_path` is checked by - /// [`valid_closure_path`]. A path nix could not substitute answers with an - /// error that starts `not available: `. + /// An older hive-priv answers the same request by downloading, and pins the + /// path for the apply that follows; hive-c0re sends it to one only when + /// that older hive-priv refused an apply. RealiseClosure { agent: String, store_path: String }, + /// `systemctl start hive-priv-cache-facts.service`, bounded at 60 s: the + /// root unit that reads this hive's cache facts from the store and writes + /// the nix.conf fragment the daemon trusts. hive-c0re asks for it when the + /// fragment is missing. The request names no unit. + RefreshCacheFacts, + /// `nixos-container destroy ` DestroyContainer { name: String }, @@ -880,6 +882,21 @@ mod tests { assert!(serde_json::from_str::(without).is_err()); } + /// The facts refresh names no unit and carries no field: a stray field is + /// dropped when it is read, so a request cannot choose what to start. + #[test] + fn the_cache_facts_refresh_carries_nothing() { + let line = r#"{"op":"refresh_cache_facts","unit":"sshd.service"}"#; + assert!(matches!( + serde_json::from_str::(line), + Ok(PrivRequest::RefreshCacheFacts) + )); + assert_eq!( + serde_json::to_string(&PrivRequest::RefreshCacheFacts).unwrap(), + r#"{"op":"refresh_cache_facts"}"# + ); + } + /// `toplevel` is required: hive-priv has no flake to build one from. #[test] fn a_request_without_a_toplevel_is_refused() { diff --git a/hive-priv/README.md b/hive-priv/README.md index bfd47563..970f7099 100644 --- a/hive-priv/README.md +++ b/hive-priv/README.md @@ -46,15 +46,14 @@ Each module's `//!` header lists its ops. `container_toplevel_action` (in `src/container.rs`) applies a closure that is already in the store; hive-priv builds nothing. hive-c0re names the closure -(`toplevel` on `CreateContainer` / `UpdateContainer`), downloaded beforehand -with `RealiseClosure`. hive-priv remembers, per agent, the path its last -successful `RealiseClosure` returned (in memory), and applies only that path to -that agent. It refuses a path never realised for the agent, another agent's -and the host's own, and logs an error naming the agent and both paths. A -restart between the download and the apply empties the record, so the apply -fails and the job retries. hive-priv does not re-check the signature at apply -time: `RealiseClosure` enforces it, and the binding limits an apply to what that -call stored. +(`toplevel` on `CreateContainer` / `UpdateContainer`), pulled beforehand by +hive-c0re through the nix daemon. hive-priv applies a path only when +`nix store verify --no-contents --sigs-needed 1`, with the key in root-owned +`/run/hyperhive-priv/attic/public-key`, finds the swarm cache's signature on it. +It refuses a path that is absent, unsigned, signed by another key or not a store +path, and logs an error naming the agent and the path. hive-priv keeps no record +of what was pulled, so a restart between the pull and the apply loses nothing. +`RefreshCacheFacts` starts the root unit that refreshes the cache facts. `create` passes the path to `nixos-container create --system-path`, so `nixos-container`'s own `buildFlake()` never runs. Without the flag, it builds diff --git a/hive-priv/src/closure.rs b/hive-priv/src/closure.rs index 49f759b5..6ded92cc 100644 --- a/hive-priv/src/closure.rs +++ b/hive-priv/src/closure.rs @@ -1,42 +1,34 @@ -//! `RealiseClosure`: download the closure the swarm built for an agent from -//! the swarm's binary cache into the store, without ever building. +//! The check on a closure before it is applied, and the two small verbs around +//! the swarm cache. //! -//! What this accepts is decided by root-owned sources alone: the cache's -//! address is the host's configuration, handed to the unit in -//! [`CACHE_URL_ENV`], and the cache's public key and the pull token's netrc are -//! files a root unit keeps in [`ROOT_FACTS_DIR`]. Nothing in the request, and -//! nothing hive-c0re (unprivileged) writes, chooses a cache or a key. They go to -//! nix's daemon as per-call settings, which the daemon applies for a trusted -//! client such as this one (root), so the host's nix configuration is untouched. +//! hive-c0re (unprivileged) downloads the swarm's closure through the host's +//! nix daemon. `CreateContainer` and `UpdateContainer` then apply a toplevel +//! only when it is in the store and carries a signature by the swarm cache's +//! key, which `nix store verify` checks against the key in [`ROOT_FACTS_DIR`]. +//! The key is a file a root unit keeps; nothing in the request, and nothing +//! hive-c0re writes, chooses it. use std::os::unix::fs::MetadataExt as _; -use std::path::{Path, PathBuf}; +use std::path::Path; use anyhow::{Context as _, Result, bail}; -use hive_priv_sock::{SIGNATURE_REFUSED, valid_closure_path}; +use hive_priv_sock::valid_closure_path; use crate::container::{BoundedRun, run_bounded}; -use crate::shared::validate_agent_name; - -/// The unit's environment variable naming the cache: `https:///` -/// from the host's `services.hyperhive.swarm.attic` options. -pub(crate) const CACHE_URL_ENV: &str = "HIVE_PRIV_CACHE_URL"; +use crate::shared::{RealSpawn, Spawn, validate_agent_name}; /// Where the root unit `hive-priv-cache-facts` keeps `public-key` and `netrc`. /// Not `/run/hyperhive/attic`, which hive-c0re writes. const ROOT_FACTS_DIR: &str = "/run/hyperhive-priv/attic"; -/// Wall-clock bound on one download. A stalled transfer is bounded by nix's -/// own `stalled-download-timeout`; this covers the rest. -const REALISE_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(30); +/// The root unit that reads this hive's cache facts from the store. +const FACTS_UNIT: &str = "hive-priv-cache-facts.service"; -/// The cache this host trusts. -#[derive(Debug, PartialEq, Eq)] -struct Cache { - substituter: String, - public_key: String, - netrc: PathBuf, -} +/// Wall-clock bound on one `nix store verify`, which reads a path's metadata. +const VERIFY_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(1); + +/// Wall-clock bound on the facts unit: a store login and three small writes. +const REFRESH_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(1); /// Whether `path` is a regular file or directory owned by `owner` that nobody /// else can read or write. @@ -64,305 +56,180 @@ fn private_to(path: &Path, owner: u32, directory: bool) -> Result<()> { Ok(()) } -impl Cache { - /// The cache at `url`, trusting the key in `dir`. - /// - /// # Errors - /// No `url`, one that is not a plain `https` address (the token would go - /// out in clear), a `dir` or file that is missing, not owned by `owner` or - /// open to others, or a key that is not `:`. - fn read(url: Option<&str>, dir: &Path, owner: u32) -> Result { - let url = url - .map(str::trim) - .filter(|u| !u.is_empty()) - .context("this host names no cache")?; - if !url.starts_with("https://") || url.contains(char::is_whitespace) { - bail!("the cache address is not an https URL"); - } - private_to(dir, owner, true)?; - let key_file = dir.join("public-key"); - private_to(&key_file, owner, false)?; - let netrc = dir.join("netrc"); - private_to(&netrc, owner, false)?; - let key = std::fs::read_to_string(&key_file) - .with_context(|| format!("reading {}", key_file.display()))?; - let key = key.trim(); - if key.contains(char::is_whitespace) - || key - .split_once(':') - .is_none_or(|(name, key)| name.is_empty() || key.is_empty()) - { - bail!("the cache's public key is not :"); - } - Ok(Self { - substituter: url.to_owned(), - public_key: key.to_owned(), - netrc, - }) - } -} - -/// The `nix-store` arguments that fetch `store_path` and nothing else. -/// -/// - `--max-jobs 0` forbids a local build, so a path the cache lacks fails. -/// - `substituters` and `trusted-public-keys` are replaced, not extended: the -/// swarm cache is the only source and its key the only one that counts. -/// - `require-sigs true` makes nix refuse a path no trusted key signed. -fn realise_args(store_path: &str, cache: &Cache) -> Vec { - let mut args: Vec = ["--realise", store_path, "--max-jobs", "0"] - .map(str::to_owned) - .into(); - let netrc = cache.netrc.display().to_string(); - for (name, value) in [ - ("substituters", cache.substituter.as_str()), - ("trusted-public-keys", cache.public_key.as_str()), - ("require-sigs", "true"), - ("netrc-file", netrc.as_str()), - ] { - args.extend(["--option".to_owned(), name.to_owned(), value.to_owned()]); - } - args -} - -/// Why nix did not realise the path, from its last stderr line. A refusal for -/// want of a signature is named, so the caller can tell it from a cache that -/// is down. -fn reason(stderr: &str) -> String { - let last = stderr.lines().last().unwrap_or("").trim(); - if last.to_lowercase().contains("signature") { - format!("{SIGNATURE_REFUSED}: {last}") - } else { - last.to_owned() - } -} - -/// `RealiseClosure`. +/// The swarm cache's public key in `dir`, as `:`. /// /// # Errors -/// Bad arguments, or `not available: ` for anything that left the path -/// out of the store: no cache or key on this host, nix failing or timing out. -pub(crate) async fn realise_closure(agent: &str, store_path: &str) -> Result<(String, String)> { - let url = std::env::var(CACHE_URL_ENV).ok(); - let done = realise_with( - url.as_deref(), - Path::new(ROOT_FACTS_DIR), - 0, - agent, - store_path, - ) - .await?; - if let Ok(mut realised) = REALISED.lock() { - remember(&mut realised, agent, store_path); +/// A `dir` or key file that is missing, not owned by `owner` or open to +/// others, or a key that is not `:`. +fn swarm_key(dir: &Path, owner: u32) -> Result { + private_to(dir, owner, true)?; + let key_file = dir.join("public-key"); + private_to(&key_file, owner, false)?; + let key = std::fs::read_to_string(&key_file) + .with_context(|| format!("reading {}", key_file.display()))?; + let key = key.trim(); + if key.contains(char::is_whitespace) + || key + .split_once(':') + .is_none_or(|(name, key)| name.is_empty() || key.is_empty()) + { + bail!("the cache's public key is not :"); } - Ok(done) + Ok(key.to_owned()) } -/// The store path each agent's last successful `RealiseClosure` put in the -/// store. `CreateContainer` / `UpdateContainer` apply only that path for that -/// agent. hive-priv is a long-running process, so the entry outlives the -/// calls between the download and the apply; a restart in between empties it -/// and the apply is refused. -static REALISED: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); - -fn remember(realised: &mut Vec<(String, String)>, agent: &str, store_path: &str) { - realised.retain(|(a, _)| a != agent); - realised.push((agent.to_owned(), store_path.to_owned())); -} - -/// The path recorded for `agent`. -fn lookup<'a>(realised: &'a [(String, String)], agent: &str) -> Option<&'a str> { - realised - .iter() - .find(|(a, _)| a == agent) - .map(|(_, p)| p.as_str()) -} - -/// `toplevel` when it is a store path and the one `agent`'s last -/// `RealiseClosure` returned (`realised`); else the refusal, which is also -/// logged at error level with the agent and both paths. +/// The `nix` arguments that check `toplevel` is in the store and signed by +/// `key`, and by no other key. /// -/// The request comes from hive-c0re, which is not root. A signed path is not -/// enough: it could be another agent's closure or the host's own. -/// -/// # Errors -/// `toplevel` is not a store path, or is not `realised`. -pub(crate) fn applicable<'a>( - agent: &str, - toplevel: &'a str, - realised: Option<&str>, -) -> Result<&'a str> { +/// - `--no-contents` checks signatures and validity, not the NAR hash. +/// - `--sigs-needed 1` with `trusted-public-keys` replaced by `key` means one +/// signature by the swarm cache's key; the host's other keys do not count. +fn verify_args(toplevel: &str, key: &str) -> Vec { + [ + "--extra-experimental-features", + "nix-command", + "store", + "verify", + "--no-contents", + "--sigs-needed", + "1", + "--option", + "trusted-public-keys", + key, + toplevel, + ] + .map(str::to_owned) + .into() +} + +/// What `nix store verify` reports through its exit status, as a bit set: 2 is +/// a path no trusted key signed, 4 a path that is not in the store. +fn verdict(code: Option, toplevel: &str, stderr: &str) -> Result<()> { + match code { + Some(0) => Ok(()), + Some(c) if c & 4 != 0 => bail!("{toplevel} is not in the store"), + Some(c) if c & 2 != 0 => bail!("{toplevel} has no signature by the swarm cache's key"), + _ => bail!( + "nix store verify failed: {}", + stderr.lines().last().unwrap_or("").trim() + ), + } +} + +/// Whether `toplevel` is in the store with a signature by the key in `dir`. +async fn check_signed(spawn: &impl Spawn, dir: &Path, owner: u32, toplevel: &str) -> Result<()> { if !valid_closure_path(toplevel) { bail!("{toplevel:?} is not a store path"); } - if realised == Some(toplevel) { - return Ok(toplevel); + let key = swarm_key(dir, owner)?; + let mut cmd = spawn.command("nix"); + cmd.args(verify_args(toplevel, &key)); + match run_bounded(cmd, VERIFY_TIMEOUT) + .await + .context("running nix store verify")? + { + BoundedRun::Exited { status, stderr } => verdict(status.code(), toplevel, &stderr), + BoundedRun::TimedOut => bail!( + "nix store verify timed out after {}s; killed it", + VERIFY_TIMEOUT.as_secs() + ), } - let realised = realised.unwrap_or("none"); - tracing::error!( - %agent, requested = %toplevel, %realised, - "refusing to apply a closure that is not the one realised for the agent" - ); - bail!("{agent}: {toplevel} is not the closure realised for this agent ({realised})") } -/// [`applicable`] against what `RealiseClosure` recorded for `agent`. +/// `toplevel` when it is a store path that is in the store with a signature by +/// the swarm cache's key; else the refusal, which is also logged at error +/// level with the agent and the path. +/// +/// The request comes from hive-c0re, which is not root and can ask for any +/// path of that shape. A path in the store is not enough: a build of the host's +/// own, or a path from another cache, carries no signature by this key. /// /// # Errors -/// As [`applicable`]. -pub(crate) fn applicable_for<'a>(agent: &str, toplevel: &'a str) -> Result<&'a str> { - let realised = REALISED - .lock() - .ok() - .and_then(|r| lookup(&r, agent).map(str::to_owned)); - applicable(agent, toplevel, realised.as_deref()) +/// `toplevel` is not a store path, is not in the store, or carries no +/// signature by the swarm cache's key, or the key cannot be read. +pub(crate) async fn applicable_for<'a>(agent: &str, toplevel: &'a str) -> Result<&'a str> { + applicable_with(&RealSpawn, Path::new(ROOT_FACTS_DIR), 0, agent, toplevel).await } -/// [`realise_closure`] with the cache's address and the key's directory and -/// owner named, so a test can stand in for the host. -async fn realise_with( - url: Option<&str>, +async fn applicable_with<'a>( + spawn: &impl Spawn, dir: &Path, owner: u32, agent: &str, - store_path: &str, -) -> Result<(String, String)> { + toplevel: &'a str, +) -> Result<&'a str> { + match check_signed(spawn, dir, owner, toplevel).await { + Ok(()) => Ok(toplevel), + Err(e) => { + tracing::error!( + %agent, requested = %toplevel, error = %format!("{e:#}"), + "refusing to apply a closure the swarm cache did not sign" + ); + bail!("{agent}: refusing {toplevel}: {e:#}") + } + } +} + +/// `RealiseClosure`, from an older hive-c0re: nothing is downloaded. `Ok` when +/// the path is already what an apply requires, else `not available: `. +/// +/// # Errors +/// Bad arguments, or `not available: `. +pub(crate) async fn realise_closure(agent: &str, store_path: &str) -> Result<(String, String)> { validate_agent_name(agent)?; if !valid_closure_path(store_path) { bail!("{store_path:?} is not a store path"); } - let outcome = async { - let cache = Cache::read(url, dir, owner)?; - let mut cmd = tokio::process::Command::new("nix-store"); - cmd.args(realise_args(store_path, &cache)); - match run_bounded(cmd, REALISE_TIMEOUT) - .await - .context("running nix-store")? - { - BoundedRun::Exited { status, .. } if status.success() => Ok(()), - BoundedRun::Exited { stderr, .. } => bail!("{}", reason(&stderr)), - BoundedRun::TimedOut => bail!( - "nix-store timed out after {}s; killed it", - REALISE_TIMEOUT.as_secs() - ), - } + match check_signed(&RealSpawn, Path::new(ROOT_FACTS_DIR), 0, store_path).await { + Ok(()) => Ok(("present".to_owned(), String::new())), + Err(e) => bail!( + "not available: {e:#}; hive-priv no longer downloads, hive-c0re pulls through the nix daemon" + ), } - .await; - match outcome { - Ok(()) => { - tracing::info!(%agent, %store_path, "substituted the agent's closure"); - Ok(("substituted".to_owned(), String::new())) - } - Err(e) => { - let reason = format!("{e:#}"); - if reason.contains(SIGNATURE_REFUSED) { - tracing::error!(%agent, %store_path, %reason, "the cache's closure was refused"); - } - bail!("not available: {reason}") +} + +/// `RefreshCacheFacts`: start the root unit that reads this hive's cache facts +/// from the store and writes the nix.conf fragment, and wait for it. The unit +/// is fixed here, not named by the request. +/// +/// # Errors +/// The unit failed to start or did not finish within [`REFRESH_TIMEOUT`]. +pub(crate) async fn refresh_cache_facts() -> Result<(String, String)> { + refresh_with(&RealSpawn, REFRESH_TIMEOUT).await +} + +async fn refresh_with(spawn: &impl Spawn, limit: std::time::Duration) -> Result<(String, String)> { + let mut cmd = spawn.command("systemctl"); + cmd.args(["start", FACTS_UNIT]); + match run_bounded(cmd, limit) + .await + .with_context(|| format!("running systemctl start {FACTS_UNIT}"))? + { + BoundedRun::Exited { status, .. } if status.success() => { + tracing::info!("refreshed the cache facts"); + Ok((String::new(), String::new())) } + BoundedRun::Exited { status, stderr } => bail!( + "systemctl start {FACTS_UNIT} exited {status}: {}", + stderr.lines().last().unwrap_or("").trim() + ), + BoundedRun::TimedOut => bail!( + "{FACTS_UNIT} did not finish within {}s; killed the wait", + limit.as_secs() + ), } } #[cfg(test)] mod tests { use std::os::unix::fs::PermissionsExt as _; + use std::path::PathBuf; use super::*; - use crate::shared::test_support::scratch; + use crate::shared::test_support::{ScriptSpawn, scratch}; const PATH: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; - const URL: &str = "https://cache.example/swarm"; - - fn cache() -> Cache { - Cache { - substituter: URL.to_owned(), - public_key: "swarm:AAAA".to_owned(), - netrc: PathBuf::from("/run/hyperhive-priv/attic/netrc"), - } - } - - const OTHER: &str = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-iris"; - - #[test] - fn the_path_realised_for_the_agent_is_applied() { - assert_eq!(applicable("atlas", PATH, Some(PATH)).unwrap(), PATH); - } - - /// Another agent's realised path, a valid-shaped path never realised, and - /// a restart that emptied the record are all refused, naming the agent - /// and both paths. - #[test] - fn any_other_path_is_refused() { - for (case, realised) in [ - ("another path was realised for the agent", Some(OTHER)), - ("nothing was realised for the agent", None), - ] { - let err = applicable("atlas", PATH, realised).unwrap_err(); - let msg = format!("{err:#}"); - assert!( - msg.contains("atlas") - && msg.contains(PATH) - && msg.contains(realised.unwrap_or("none")), - "{case}: {msg}" - ); - } - } - - #[test] - fn only_a_store_path_is_applied_even_when_it_was_recorded() { - for bad in [ - "", - "main", - "/etc/passwd", - "/nix/store/x", - &format!("{PATH}/bin"), - ] { - let err = applicable("atlas", bad, Some(bad)).unwrap_err(); - assert!(format!("{err:#}").contains("not a store path"), "{bad}"); - } - } - - /// Each agent keeps its own entry, and a newer download replaces the - /// older one. - #[test] - fn a_record_is_per_agent_and_the_latest_wins() { - let mut realised = Vec::new(); - remember(&mut realised, "atlas", PATH); - remember(&mut realised, "iris", OTHER); - assert_eq!(lookup(&realised, "atlas"), Some(PATH)); - assert_eq!(lookup(&realised, "iris"), Some(OTHER)); - remember(&mut realised, "atlas", OTHER); - assert_eq!(lookup(&realised, "atlas"), Some(OTHER)); - assert_eq!(lookup(&realised, "nobody"), None); - assert_eq!(realised.len(), 2); - } - - fn option<'a>(args: &'a [String], name: &str) -> Option<&'a str> { - args.windows(3) - .find(|w| w[0] == "--option" && w[1] == name) - .map(|w| w[2].as_str()) - } - - #[test] - fn the_command_never_builds() { - let args = realise_args(PATH, &cache()); - assert_eq!(args[..2], ["--realise", PATH]); - let jobs = args.iter().position(|a| a == "--max-jobs").expect("flag"); - assert_eq!(args[jobs + 1], "0"); - } - - #[test] - fn only_the_swarm_cache_and_its_key_count_and_signatures_are_required() { - let args = realise_args(PATH, &cache()); - assert_eq!(option(&args, "substituters"), Some(URL)); - assert_eq!(option(&args, "trusted-public-keys"), Some("swarm:AAAA")); - assert_eq!(option(&args, "require-sigs"), Some("true")); - assert_eq!( - option(&args, "netrc-file"), - Some("/run/hyperhive-priv/attic/netrc") - ); - // The settings that would add to the defaults are not used. - assert!(!args.iter().any(|a| a.starts_with("extra-")), "{args:?}"); - } + const KEY: &str = "swarm:AAAA"; /// The uid the test runs as, which stands in for root. fn me() -> u32 { @@ -387,126 +254,169 @@ mod tests { dir } - #[test] - fn the_cache_is_the_hosts_address_and_the_root_files_key() { - let dir = facts("swarm:AAAA"); - let cache = Cache::read(Some(URL), &dir, me()).expect("read"); - assert_eq!(cache.substituter, URL); - assert_eq!(cache.public_key, "swarm:AAAA"); - assert_eq!(cache.netrc, dir.join("netrc")); + /// A `nix` that behaves like `nix store verify --sigs-needed 1` over a store + /// holding `PATH`, signed by `KEY` only: exit 0 for that path under that key, + /// 2 for that path under any other key, 4 for any other path. It exits 99 + /// for anything but the verify invocation, and logs its arguments. + fn nix(log: &Path) -> ScriptSpawn { + ScriptSpawn(format!( + r#"echo "$@" >>{log} +[ "$1 $2 $3 $4 $5" = "--extra-experimental-features nix-command store verify --no-contents" ] || exit 99 +[ "$6 $7 $8 $9" = "--sigs-needed 1 --option trusted-public-keys" ] || exit 99 +[ "${{11}}" = {PATH} ] || exit 4 +[ "${{10}}" = {KEY} ] || exit 2 +exit 0"#, + log = log.display() + )) } - /// A `substituter` or `public-key` file beside the facts, as hive-c0re - /// writes them, decides nothing: the address is the argument and the key is - /// the root file's. - #[test] - fn the_address_is_never_read_from_a_file_and_the_key_only_from_the_root_files() { - let dir = facts("swarm:ROOT"); - put( - &dir.join("substituter"), - "https://evil.example/cache\n", - 0o600, + #[tokio::test] + async fn a_toplevel_signed_by_the_swarm_key_is_applied() { + let log = scratch().join("calls"); + let got = applicable_with(&nix(&log), &facts(KEY), me(), "atlas", PATH) + .await + .expect("signed"); + assert_eq!(got, PATH); + // The key checked against is the one in the root file. + let calls = std::fs::read_to_string(&log).expect("log"); + assert!( + calls.contains(&format!("trusted-public-keys {KEY} {PATH}")), + "{calls}" ); - let cache = Cache::read(Some(URL), &dir, me()).expect("read"); - assert_eq!(cache.substituter, URL); - assert_eq!(cache.public_key, "swarm:ROOT"); - // The directory hive-c0re writes is not the one this reads. - assert!(!ROOT_FACTS_DIR.starts_with("/run/hyperhive/")); - assert_eq!(ROOT_FACTS_DIR, "/run/hyperhive-priv/attic"); - // And without an address from the host there is no cache, whatever the - // directory holds. - for none in [None, Some(""), Some(" ")] { - assert!(Cache::read(none, &dir, me()).is_err(), "{none:?}"); - } } - #[test] - fn a_file_that_is_not_the_roots_or_is_open_to_others_is_refused() { - let me = me(); - // Control: the facts as the unit leaves them are accepted. - assert!(Cache::read(Some(URL), &facts("swarm:AAAA"), me).is_ok()); - // Another owner than the one trusted. - assert!(Cache::read(Some(URL), &facts("swarm:AAAA"), me + 1).is_err()); - // The key or the netrc readable by others, or the directory open. - for (name, mode) in [("public-key", 0o644), ("netrc", 0o644), ("netrc", 0o660)] { - let dir = facts("swarm:AAAA"); - std::fs::set_permissions(dir.join(name), std::fs::Permissions::from_mode(mode)) - .expect("chmod"); - assert!(Cache::read(Some(URL), &dir, me).is_err(), "{name} {mode:o}"); - } - let dir = facts("swarm:AAAA"); - std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).expect("chmod"); - assert!(Cache::read(Some(URL), &dir, me).is_err()); - // A symlink in place of the key. - let dir = facts("swarm:AAAA"); - std::fs::remove_file(dir.join("public-key")).expect("rm"); - std::os::unix::fs::symlink(dir.join("netrc"), dir.join("public-key")).expect("link"); - assert!(Cache::read(Some(URL), &dir, me).is_err()); - } - - #[test] - fn missing_files_and_unusable_values_are_refused() { - let me = me(); - for url in ["http://cache.example/swarm", "https://cache.example/a b"] { + /// A path no swarm key signed, one the store lacks, and a key file that + /// names another key are each refused, naming the agent and the path. + #[tokio::test] + async fn an_unsigned_or_absent_toplevel_is_refused() { + let log = scratch().join("calls"); + let other = "/nix/store/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-nixos-system-iris"; + for (case, dir, path, want) in [ + ("absent", facts(KEY), other, "is not in the store"), + ( + "unsigned by the key on file", + facts("other:BBBB"), + PATH, + "has no signature by the swarm cache's key", + ), + ] { + let err = applicable_with(&nix(&log), &dir, me(), "atlas", path) + .await + .expect_err(case); + let msg = format!("{err:#}"); assert!( - Cache::read(Some(url), &facts("swarm:AAAA"), me).is_err(), - "{url}" + msg.contains("atlas") && msg.contains(path) && msg.contains(want), + "{case}: {msg}" ); } - for key in ["nokeyname", "swarm: AAAA", ":AAAA", "swarm:"] { - assert!(Cache::read(Some(URL), &facts(key), me).is_err(), "{key:?}"); - } - let dir = facts("swarm:AAAA"); - std::fs::remove_file(dir.join("netrc")).expect("rm"); - assert!(Cache::read(Some(URL), &dir, me).is_err()); - assert!(Cache::read(Some(URL), &dir.join("nowhere"), me).is_err()); } - /// Missing facts reach the caller as `not available`, which a rebuild - /// reports as a closure that cannot be pulled. + /// A tool failure other than the two verdicts is a refusal too, with the + /// tool's last line. #[tokio::test] - async fn without_the_root_files_the_answer_is_not_available() { - let nowhere = scratch().join("nowhere"); - let err = realise_with(Some(URL), &nowhere, 0, "atlas", PATH) + async fn a_failing_verify_refuses() { + let spawn = ScriptSpawn("echo 'error: cannot connect' >&2; exit 1".to_owned()); + let err = applicable_with(&spawn, &facts(KEY), me(), "atlas", PATH) .await - .expect_err("no facts"); - assert!(format!("{err:#}").starts_with("not available: "), "{err:#}"); - let err = realise_with(None, &facts("swarm:AAAA"), me(), "atlas", PATH) - .await - .expect_err("no address"); - assert!(format!("{err:#}").starts_with("not available: "), "{err:#}"); + .expect_err("refused"); + assert!(format!("{err:#}").contains("cannot connect"), "{err:#}"); } - /// The request carries an agent and a path and nothing that chooses a - /// cache: fields beyond those are dropped when it is read. - #[test] - fn the_request_cannot_carry_an_address_or_a_key() { - let line = r#"{"op":"realise_closure","agent":"atlas","store_path":"/nix/store/x", - "substituter":"https://evil.example/c","public_key":"evil:AAAA","netrc":"/tmp/x"}"#; - let hive_priv_sock::PrivRequest::RealiseClosure { agent, store_path } = - serde_json::from_str(line).expect("reads") - else { - panic!("not a RealiseClosure"); - }; - assert_eq!( - (agent.as_str(), store_path.as_str()), - ("atlas", "/nix/store/x") + #[tokio::test] + async fn only_a_store_path_is_verified() { + // A stand-in that would accept anything, so only the shape check refuses. + let yes = ScriptSpawn("exit 0".to_owned()); + for bad in [ + "", + "main", + "/etc/passwd", + "/nix/store/x", + &format!("{PATH}/bin"), + ] { + let err = applicable_with(&yes, &facts(KEY), me(), "atlas", bad) + .await + .expect_err(bad); + assert!(format!("{err:#}").contains("not a store path"), "{bad}"); + } + // Control: the same stand-in accepts a store path. + assert!( + applicable_with(&yes, &facts(KEY), me(), "atlas", PATH) + .await + .is_ok() ); } - #[test] - fn a_signature_refusal_is_named_and_another_failure_is_not() { - let refused = - reason("error: cannot add path 'x' because it lacks a signature by a trusted key"); - assert!(refused.starts_with(SIGNATURE_REFUSED), "{refused}"); - let down = - reason("warning: x\nerror: unable to download 'https://c/x.narinfo': Couldn't connect"); - assert!(!down.contains(SIGNATURE_REFUSED), "{down}"); - assert!(down.contains("Couldn't connect"), "{down}"); + /// Without the root key there is nothing to verify against: refused, and + /// nix is not asked. + #[tokio::test] + async fn without_the_root_key_nothing_is_applied() { + let log = scratch().join("calls"); + let nowhere = scratch().join("nowhere"); + let err = applicable_with(&nix(&log), &nowhere, 0, "atlas", PATH) + .await + .expect_err("no key"); + assert!(format!("{err:#}").contains("missing"), "{err:#}"); + assert!(!log.exists(), "nix ran without a key"); } + #[test] + fn the_key_is_the_root_files_and_only_a_private_one() { + let me = me(); + assert_eq!(swarm_key(&facts(KEY), me).unwrap(), KEY); + // Another owner than the one trusted. + assert!(swarm_key(&facts(KEY), me + 1).is_err()); + // The key open to others, or the directory open. + let dir = facts(KEY); + std::fs::set_permissions( + dir.join("public-key"), + std::fs::Permissions::from_mode(0o644), + ) + .expect("chmod"); + assert!(swarm_key(&dir, me).is_err()); + let dir = facts(KEY); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + assert!(swarm_key(&dir, me).is_err()); + // A symlink in place of the key. + let dir = facts(KEY); + std::fs::remove_file(dir.join("public-key")).expect("rm"); + std::os::unix::fs::symlink(dir.join("netrc"), dir.join("public-key")).expect("link"); + assert!(swarm_key(&dir, me).is_err()); + for key in ["nokeyname", "swarm: AAAA", ":AAAA", "swarm:"] { + assert!(swarm_key(&facts(key), me).is_err(), "{key:?}"); + } + // The directory hive-c0re writes is not the one this reads. + assert_eq!(ROOT_FACTS_DIR, "/run/hyperhive-priv/attic"); + } + + #[test] + fn the_command_counts_only_the_swarm_key() { + let args = verify_args(PATH, KEY); + assert_eq!(args.last().map(String::as_str), Some(PATH)); + let at = args + .iter() + .position(|a| a == "--sigs-needed") + .expect("flag"); + assert_eq!(args[at + 1], "1"); + let at = args + .iter() + .position(|a| a == "trusted-public-keys") + .expect("option"); + assert_eq!(args[at - 1], "--option"); + assert_eq!(args[at + 1], KEY); + // The setting that would add to the host's keys is not used. + assert!( + !args.iter().any(|a| a.starts_with("extra-trusted")), + "{args:?}" + ); + } + + /// An older hive-c0re's download request downloads nothing: it succeeds + /// only for a path an apply would accept. #[tokio::test] - async fn bad_arguments_are_refused_before_anything_runs() { + async fn the_older_download_request_answers_not_available_for_a_path_not_in_the_store() { + // The real `nix` is not on the path of the test, so this is refused for + // want of the root key before anything runs; the shape and name checks + // come first. for (agent, path) in [ ("Bad Agent", PATH), ("", PATH), @@ -517,5 +427,34 @@ mod tests { let err = realise_closure(agent, path).await.expect_err("refused"); assert!(!format!("{err:#}").contains("not available"), "{err:#}"); } + let err = realise_closure("atlas", PATH).await.expect_err("no key"); + let msg = format!("{err:#}"); + assert!( + msg.starts_with("not available: ") && msg.contains("no longer downloads"), + "{msg}" + ); + } + + /// The refresh starts the fixed unit and nothing else, and says why it + /// failed or timed out. + #[tokio::test] + async fn the_refresh_starts_the_facts_unit_and_reports_a_failure() { + let log = scratch().join("calls"); + let ok = ScriptSpawn(format!("echo \"$0 $@\" >>{}", log.display())); + refresh_with(&ok, REFRESH_TIMEOUT).await.expect("started"); + assert_eq!( + std::fs::read_to_string(&log).expect("log").trim(), + "systemctl start hive-priv-cache-facts.service" + ); + let failing = ScriptSpawn("echo 'Unit not found.' >&2; exit 5".to_owned()); + let err = refresh_with(&failing, REFRESH_TIMEOUT) + .await + .expect_err("failed"); + assert!(format!("{err:#}").contains("Unit not found."), "{err:#}"); + let slow = ScriptSpawn("sleep 30".to_owned()); + let err = refresh_with(&slow, std::time::Duration::from_millis(100)) + .await + .expect_err("timed out"); + assert!(format!("{err:#}").contains("did not finish"), "{err:#}"); } } diff --git a/hive-priv/src/container.rs b/hive-priv/src/container.rs index 5fc7f1a1..89542630 100644 --- a/hive-priv/src/container.rs +++ b/hive-priv/src/container.rs @@ -207,9 +207,10 @@ async fn destroy_container_with(host: &Host, name: &str) -> Result<( } /// Shared body for `CreateContainer` / `UpdateContainer`: validate the name -/// and that `toplevel` is the closure realised for the agent, and apply it (streaming line events to `writer` when -/// `stream` is set). Nothing is built here: `toplevel` is a closure already in -/// the store, which `RealiseClosure` put there. +/// and that `toplevel` is in the store with a signature by the swarm cache's +/// key, and apply it (streaming line events to `writer` when `stream` is set). +/// Nothing is built here: `toplevel` is a closure already in the store, which +/// hive-c0re pulled through the nix daemon. /// /// `create` passes `--system-path` so `nixos-container`'s own `buildFlake()` /// never runs; it builds to a relative `.tmp` out-link that concurrent calls @@ -228,12 +229,12 @@ pub(crate) async fn container_toplevel_action( writer: &mut OwnedWriteHalf, ) -> Result<(String, String)> { validate_agent_name(name)?; - let toplevel = crate::closure::applicable_for(name, toplevel)?; + let toplevel = crate::closure::applicable_for(name, toplevel).await?; apply_toplevel(&Host::REAL, verb, name, toplevel, stream, writer).await } -/// [`container_toplevel_action`] past the check that `toplevel` is the -/// closure realised for `name`. +/// [`container_toplevel_action`] past the check that `toplevel` is signed by +/// the swarm cache's key. async fn apply_toplevel( host: &Host, verb: &str, @@ -987,11 +988,12 @@ mod host_tests { // ---- the apply gate ---- - /// Only the closure realised for the agent is applied. Nothing is - /// realised for these names, so each request is refused at the gate: the - /// message is the gate's, not a `nixos-container` failure. + /// Only a closure the swarm key signed is applied. A test has no swarm key + /// at the root path, so each request is refused at the gate: the message + /// is the gate's, not a `nixos-container` failure. The signed and unsigned + /// verdicts are tested in `closure`. #[tokio::test] - async fn an_unrealised_or_foreign_closure_is_refused_at_the_gate() { + async fn a_closure_without_the_swarm_key_to_check_it_against_is_refused_at_the_gate() { for verb in ["create", "update"] { let (writer, _peer) = &mut pipe(); let err = container_toplevel_action(verb, "never-realised", TOPLEVEL, false, writer) @@ -999,7 +1001,7 @@ mod host_tests { .unwrap_err(); let msg = format!("{err:#}"); assert!( - msg.contains("is not the closure realised for this agent (none)"), + msg.contains(&format!("never-realised: refusing {TOPLEVEL}")), "{verb}: {msg}" ); } diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index b80c8e29..7d72e46c 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -24,8 +24,8 @@ //! [`prestart`] (the check of a container's root that runs before each start, //! as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk //! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles), -//! [`infra`] (gateway nginx, infra containers) and [`closure`] (downloads from -//! the swarm cache). Helpers more than one family uses are in [`shared`]. +//! [`infra`] (gateway nginx, infra containers) and [`closure`] (the swarm-signature +//! check before an apply, and the cache-facts refresh). Helpers more than one family uses are in [`shared`]. mod closure; mod container; @@ -403,6 +403,8 @@ async fn exec( ref store_path, } => closure::realise_closure(agent, store_path).await, + PrivRequest::RefreshCacheFacts => closure::refresh_cache_facts().await, + PrivRequest::DestroyContainer { ref name } => container::destroy_container(name).await, PrivRequest::ListContainers => container::container_run(&["list"]).await, diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index e116ab01..61ba6ac2 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -315,9 +315,11 @@ in # StateDirectory (/var/lib/hyperhive), RuntimeDirectory # (/run/hyperhive), and ReadWritePaths below, which systemd keeps # writable. Beyond the managed directories: - # - nix is invoked directly (lifecycle, meta, flake_check), but - # NIX_REMOTE=daemon routes all store writes through the host - # daemon — hive-c0re never writes to /nix itself. + # - the one nix call is the closure pull (src/cache_pull.rs): + # `nix-store --realise` with NIX_REMOTE=daemon, so the host daemon + # writes the store — hive-c0re never writes to /nix itself. It + # connects to the daemon's socket, which RestrictAddressFamilies + # (deferred above) must keep allowing. # - nix build worktrees live in PrivateTmp, not /tmp. # - /etc writes (bind-mount edits) go through hive-priv via the diff --git a/nix/host-modules/hive-priv-cache.nix b/nix/host-modules/hive-priv-cache.nix index be38c926..3cdb8318 100644 --- a/nix/host-modules/hive-priv-cache.nix +++ b/nix/host-modules/hive-priv-cache.nix @@ -1,18 +1,21 @@ -# The binary-cache facts hive-priv (root) trusts, kept where only root can -# change them. +# The binary-cache facts the host's nix daemon and hive-priv (root) trust, kept +# where only root can change them. # -# hive-priv downloads an agent's closure from the swarm cache when a rebuild -# asks it to. What decides whether a download is accepted — which cache, and -# which key must have signed it — must not come from a file the unprivileged -# hive-c0re writes, or a compromised hive-c0re could have root accept a -# closure from a cache of its own. So: +# hive-c0re downloads an agent's closure from the swarm cache through the +# host's nix daemon, and hive-priv applies it only if the swarm's key signed +# it. What decides whether a download is accepted — which cache the daemon +# lets an unprivileged client name, and which key must have signed it — must +# not come from a file the unprivileged hive-c0re writes, or a compromised +# hive-c0re could have root accept a closure from a cache of its own. So: # # - the cache's address is this host's configuration -# ({option}`services.hyperhive.swarm.attic.domain` and `cache`), handed to -# hive-priv in its unit's environment; +# ({option}`services.hyperhive.swarm.attic.domain` and `cache`); # - the cache's public key and the pull token are read from the store by a root -# oneshot, `hive-priv-cache-facts`, under this hive's own store identity, and -# land in `/run/hyperhive-priv/attic/` (root, `0700`; files `0600`). +# oneshot, `hive-priv-cache-facts`, under this hive's own store identity. The +# key and the token land in `/run/hyperhive-priv/attic/` (root, `0700`; files +# `0600`), and the address, the key and the netrc's path in a nix.conf +# fragment, `/run/hyperhive-nix/cache.conf`, which the host's nix.conf +# includes (`!include`, so a host whose unit has not run yet still starts). # # hive-c0re keeps its own copies under `/run/hyperhive/attic/` for the probe, # which only reports. @@ -42,6 +45,9 @@ let && tls.baoHiveIdentityKeyFile != null; dir = "/run/hyperhive-priv/attic"; + confDir = "/run/hyperhive-nix"; + confFile = "${confDir}/cache.conf"; + url = "https://${atticCfg.domain}/${atticCfg.cache}"; # Where the store runs on this host, the first login waits for its container. # No `wants`: a pull would start the container mid-switch on the old closure. @@ -52,15 +58,16 @@ let in { config = lib.mkIf enabled { - systemd.tmpfiles.rules = [ "d /run/hyperhive-priv 0700 root root - -" ]; + systemd.tmpfiles.rules = [ + "d /run/hyperhive-priv 0700 root root - -" + "d ${confDir} 0755 root root - -" + ]; - # The URL nix is told is the only substituter for the download; no request - # chooses it. - systemd.services.hive-priv.environment.HIVE_PRIV_CACHE_URL = - "https://${atticCfg.domain}/${atticCfg.cache}"; + # `!include` tolerates a fragment that does not exist yet. + nix.extraOptions = "!include ${confFile}"; systemd.services.hive-priv-cache-facts = { - description = "Read the swarm cache's public key and pull token for hive-priv"; + description = "Read the swarm cache's public key and pull token for the host's nix daemon and hive-priv"; wantedBy = [ "multi-user.target" ]; wants = [ "network-online.target" ]; after = [ @@ -94,7 +101,7 @@ in script = import ./lib/attic-root-facts.nix { inherit lib; } { hive = hiveController.hiveName; host = atticCfg.domain; - inherit dir; + inherit dir url confFile; }; }; diff --git a/nix/host-modules/hive-priv.nix b/nix/host-modules/hive-priv.nix index 64bcce8a..f4d9e5e8 100644 --- a/nix/host-modules/hive-priv.nix +++ b/nix/host-modules/hive-priv.nix @@ -124,8 +124,8 @@ in # hive-priv runs as root. Root nix defaults to store=auto which # resolves to the LOCAL store — bypassing the host daemon, its # remote builders, and prebuilt derivation outputs. Force daemon - # routing so the closure `RealiseClosure` downloads is in the same - # store the container is then pointed at. + # routing so the closure hive-c0re pulls through the daemon is in + # the same store the container is then pointed at. NIX_REMOTE = "daemon"; # The owner a namespaced agent's data moves to. Which agents are # namespaced is not set here: hive-c0re sends it with each start. diff --git a/nix/host-modules/lib/attic-root-facts.nix b/nix/host-modules/lib/attic-root-facts.nix index 8e0a8652..8dcaa0c3 100644 --- a/nix/host-modules/lib/attic-root-facts.nix +++ b/nix/host-modules/lib/attic-root-facts.nix @@ -1,22 +1,26 @@ -# The root-side copy of the two cache facts hive-priv trusts: the key a -# downloaded closure must be signed by and the netrc the pull token goes out -# in. The script runs as root on the hive host under the hive's own store -# identity and writes both into `dir`, which only root can read. +# The root-side copy of the cache facts the host's nix daemon and hive-priv +# trust: the key a downloaded closure must be signed by and the netrc the pull +# token goes out in. The script runs as root on the hive host under the hive's +# own store identity and writes both into `dir`, which only root can read, and +# a nix.conf fragment at `confFile` that the host's nix.conf includes. # -# Everything hive-priv decides on is here or in the host's configuration, and -# nothing is read from a file hive-c0re (unprivileged) writes: the key and the -# token come from the store, the cache's address from -# `services.hyperhive.swarm.attic.{domain,cache}`. +# What the daemon and hive-priv decide on is here or in the host's +# configuration, and nothing is read from a file hive-c0re (unprivileged) +# writes: the key and the token come from the store, the cache's address from +# `services.hyperhive.swarm.attic.{domain,cache}`. The fragment holds no secret +# (the key is public, the netrc stays in `dir`) and hive-c0re can read it: it +# takes the cache's address from there. # # A store that holds no facts yet, or cannot be reached, leaves what is in `dir` -# as it is. Nothing here removes a file. +# and `confFile` as they are. Nothing here removes a file. # # Pure function — NOT a NixOS module. Call it from a unit's `script`, with -# `bao`, `coreutils` and `gnugrep` on its `path` and the `BAO_*` client -# environment set: +# `bao`, `coreutils`, `gnugrep` and `systemd` on its `path` and the `BAO_*` +# client environment set: # # script = import ./lib/attic-root-facts.nix { inherit lib; } { # hive = "alpha"; host = "attic.example.org"; dir = "/run/hyperhive-priv/attic"; +# url = "https://attic.example.org/swarm"; confFile = "/run/hyperhive-nix/cache.conf"; # }; { lib }: { @@ -25,8 +29,12 @@ hive, # The cache server's host name, which the netrc line names. host, - # Where the two files go. + # Where the key and the netrc go. dir, + # The cache's address as the daemon lists it in `trusted-substituters`. + url, + # Where the nix.conf fragment goes; its directory is root's and readable. + confFile, }: '' set -euo pipefail @@ -35,7 +43,9 @@ ${import ./refresh-consumer.nix { }} dir=${lib.escapeShellArg dir} + conf=${lib.escapeShellArg confFile} install -d -m 0700 "$dir" + install -d -m 0755 "$(dirname -- "$conf")" store_init store_login @@ -84,4 +94,17 @@ if secret_differs "$dir/netrc" "$netrc"; then atomic_write_secret 0600 "" "$dir/netrc" "$netrc" fi + + # Last, so a daemon that reads the fragment finds the netrc in place. The + # address is the one hive-c0re may name as the substituter. The mtime moves + # only with the content. Lix re-reads nix.conf for every connection, CppNix + # only at start, so a daemon that started before this write is restarted. + printf -v fragment '%s\n%s\n%s' \ + ${lib.escapeShellArg "extra-trusted-substituters = ${url}"} \ + "extra-trusted-public-keys = $key" \ + "netrc-file = $dir/netrc" + if secret_differs "$conf" "$fragment"; then + atomic_write_secret 0644 "" "$conf" "$fragment" + fi + refresh_unit nix-daemon.service "$conf" '' diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index a66c34d6..9ebf386d 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -878,10 +878,23 @@ let inner.ProcSubset == "all" && inner.ProtectProc == "invisible"; } { - name = "hive-priv is told the cache's address by the host's configuration"; + # The daemon only learns the cache from the fragment the root unit + # writes, and lists as trusted the one address from the host's + # configuration. A host with no unit has no include. + name = "the host's nix.conf includes the cache fragment the root unit writes, with the cache's address"; ok = - allLocal.systemd.services.hive-priv.environment.HIVE_PRIV_CACHE_URL == "https://attic.t.local/swarm" - && !(noIdentity.systemd.services.hive-priv.environment ? HIVE_PRIV_CACHE_URL); + let + u = units.hive-priv-cache-facts; + in + lib.hasInfix "!include /run/hyperhive-nix/cache.conf" allLocal.nix.extraOptions + && !(lib.hasInfix "hyperhive-nix" noIdentity.nix.extraOptions) + && lib.hasInfix "conf=/run/hyperhive-nix/cache.conf" u.script + && lib.hasInfix "extra-trusted-substituters = https://attic.t.local/swarm" u.script + && lib.hasInfix ''atomic_write_secret 0644 "" "$conf"'' u.script + && lib.elem "d /run/hyperhive-nix 0755 root root - -" allLocal.systemd.tmpfiles.rules + # Control for the absence: the lookup finds the unit's other variable. + && allLocal.systemd.services.hive-priv.environment ? NIX_REMOTE + && !(allLocal.systemd.services.hive-priv.environment ? HIVE_PRIV_CACHE_URL); } ]; in diff --git a/nix/script-tests/attic-root-facts.nix b/nix/script-tests/attic-root-facts.nix index 405aea8a..ee5d2ef9 100644 --- a/nix/script-tests/attic-root-facts.nix +++ b/nix/script-tests/attic-root-facts.nix @@ -15,6 +15,8 @@ let hive = "alpha"; host = "cache.example"; dir = "@DIR@/attic"; + url = "https://cache.example/swarm"; + confFile = "@DIR@/nix/cache.conf"; } ); in diff --git a/nix/script-tests/attic-root-facts.sh b/nix/script-tests/attic-root-facts.sh index 74faf2bf..e051d62c 100644 --- a/nix/script-tests/attic-root-facts.sh +++ b/nix/script-tests/attic-root-facts.sh @@ -45,15 +45,30 @@ case "\$1 \$2" in esac STUB chmod +x "$dir/bin/bao" + +# `systemctl` answers for nix-daemon.service: DAEMON_STARTED is when it last +# became active (empty: never) and every restart request is logged. +cat >"$dir/bin/systemctl" <>"$dir/systemctl" +case "\$1" in + show) printf '%s\n' "\${DAEMON_STARTED-}" ;; + is-failed) exit 1 ;; + try-restart) ;; + *) echo "unexpected call: \$*" >&2; exit 98 ;; +esac +STUB +chmod +x "$dir/bin/systemctl" export PATH="$dir/bin:$PATH" KEYLEAF=secret_swarm_hives_alpha_attic_public-key TOKENLEAF=secret_swarm_hives_alpha_attic_pull-token reset() { - rm -rf "$dir/store" "$dir/attic" "$calls" + rm -rf "$dir/store" "$dir/attic" "$dir/nix" "$calls" "$dir/systemctl" mkdir -p "$dir/store" : >"$calls" + : >"$dir/systemctl" } seed() { # @@ -66,6 +81,7 @@ run() { env "$@" bash "$dir/script.sh" >"$dir/out" 2>"$dir/err" } +restarts() { grep -c '^systemctl try-restart' "$dir/systemctl"; } mode() { stat -c %a "$1" 2>/dev/null || echo missing; } ino() { stat -c %i "$1" 2>/dev/null || echo missing; } body() { cat "$1" 2>/dev/null || echo missing; } @@ -80,14 +96,22 @@ expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok- expect "key mode" "$(mode "$dir/attic/public-key")" 600 expect "netrc mode" "$(mode "$dir/attic/netrc")" 600 expect "dir mode" "$(mode "$dir/attic")" 700 +expect "fragment" "$(body "$dir/nix/cache.conf")" "extra-trusted-substituters = https://cache.example/swarm +extra-trusted-public-keys = swarm:S0VZ +netrc-file = $dir/attic/netrc" +expect "fragment mode" "$(mode "$dir/nix/cache.conf")" 644 +expect "fragment dir mode" "$(mode "$dir/nix")" 755 +expect "no secret in the fragment" "$(grep -c tok-1 "$dir/nix/cache.conf")" 0 case="a second run with the same facts rewrites nothing" k=$(ino "$dir/attic/public-key") n=$(ino "$dir/attic/netrc") +c=$(ino "$dir/nix/cache.conf") run expect "exit" $? 0 expect "key untouched" "$(ino "$dir/attic/public-key")" "$k" expect "netrc untouched" "$(ino "$dir/attic/netrc")" "$n" +expect "fragment untouched" "$(ino "$dir/nix/cache.conf")" "$c" case="a rotated token replaces the netrc and leaves the key" seed "swarm:S0VZ" "tok-2" @@ -170,6 +194,54 @@ run expect "exit" $? 1 intact +# A restart of nix-daemon is asked for only when the fragment was written after +# the daemon last started; `now` is when the daemon "started", read between runs. +now() { date -u '+%a %Y-%m-%d %H:%M:%S.%6N UTC'; } + +case="a new fragment restarts a daemon that started before it" +reset +seed "swarm:S0VZ" "tok-1" +run DAEMON_STARTED="Sat 2000-01-01 00:00:00.000000 UTC" +expect "exit" $? 0 +expect "restart asked" "$(grep -c '^systemctl try-restart --no-block nix-daemon.service' "$dir/systemctl")" 1 + +case="the same facts leave the fragment and the daemon alone" +started=$(now) +: >"$dir/systemctl" +run DAEMON_STARTED="$started" +expect "exit" $? 0 +expect "no restart" "$(restarts)" 0 + +case="a rotated token changes the netrc but not the fragment, so no restart" +started=$(now) +seed "swarm:S0VZ" "tok-2" +run DAEMON_STARTED="$started" +expect "exit" $? 0 +expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok-2" +expect "no restart" "$(restarts)" 0 + +case="a rotated key rewrites the fragment and restarts the daemon" +started=$(now) +seed "swarm:Tk9XCg" "tok-2" +run DAEMON_STARTED="$started" +expect "exit" $? 0 +expect "key in the fragment" "$(grep -c 'extra-trusted-public-keys = swarm:Tk9XCg' "$dir/nix/cache.conf")" 1 +expect "restart asked" "$(restarts)" 1 + +case="a store with no facts writes no fragment and restarts nothing" +reset +run DAEMON_STARTED="Sat 2000-01-01 00:00:00.000000 UTC" +expect "exit" $? 0 +expect "no fragment" "$(body "$dir/nix/cache.conf")" missing +expect "no restart" "$(restarts)" 0 + +case="a key that is refused writes no fragment" +reset +seed "swarm:S0 VZ" "tok-1" +run +expect "exit" $? 1 +expect "no fragment" "$(body "$dir/nix/cache.conf")" missing + if [ "$failures" -ne 0 ]; then echo "$failures of $count checks failed" >&2 exit 1