From ef68baa00ff07eb92d695fb207fcfbe332757c25 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 8 Nov 2026 10:29:46 +0100 Subject: [PATCH] pull swarm closures through the nix daemon: substituters in the fragment, merged root-only netrc --- .../2-handbook/swarm/credentials-reference.md | 20 +- docs/4-implementation/swarm/builder.md | 44 ++- .../trust-boundary/hive-c0re-privilege.md | 72 ++-- .../trust-boundary/nix-builds.md | 12 +- hive-c0re/src/cache_pull.rs | 343 ++++++++---------- hive-priv-sock/src/lib.rs | 6 +- hive-priv/src/closure.rs | 4 +- nix/checks.nix | 11 +- nix/host-modules/hive-c0re/default.nix | 2 +- nix/host-modules/hive-priv-cache.nix | 59 +-- nix/host-modules/lib/attic-root-facts.nix | 91 +++-- nix/host-modules/lib/refresh-consumer.nix | 8 +- nix/module-eval/hive-tls.nix | 33 +- nix/script-tests/attic-root-facts.nix | 5 +- nix/script-tests/attic-root-facts.sh | 118 ++++-- nix/script-tests/cache-pull-substitute.nix | 18 + nix/script-tests/cache-pull-substitute.sh | 95 +++++ nix/script-tests/refresh-consumer.sh | 23 +- 18 files changed, 582 insertions(+), 382 deletions(-) create mode 100644 nix/script-tests/cache-pull-substitute.nix create mode 100644 nix/script-tests/cache-pull-substitute.sh diff --git a/docs/2-handbook/swarm/credentials-reference.md b/docs/2-handbook/swarm/credentials-reference.md index e72e1b2a..ea1b536d 100644 --- a/docs/2-handbook/swarm/credentials-reference.md +++ b/docs/2-handbook/swarm/credentials-reference.md @@ -151,16 +151,16 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](credentials. ### Hive-level — one of each per hive -| secret | generated by | lives at | -| ------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| hive leaf certs | `hive-tls.nix`, issued by the secret store's `pki` mount | `/.pem` | -| the hive's own store identity | `glue-bao-tls.nix` on the store's host; placed by hand on every other host ([`ca.md`](../../4-implementation/swarm/ca.md#the-gateway-leaf-comes-from-the-secret-store)) | `deploy.hive-controller.tls.baoHiveIdentityCertFile` / `baoHiveIdentityKeyFile` | -| the cache's public key and pull token, for the host's nix daemon and hive-priv | `swarm-controller` copies them to `swarm/hives//attic/{public-key,pull-token}` (see the cache-facts row above) | `hive-priv-cache-facts` (root) reads those two under the hive's own store identity every ten minutes and writes `/run/hyperhive-priv/attic/public-key` and `netrc` (tmpfs, root, each `0600`, directory `0700`); it also writes `/run/hyperhive-nix/cache.conf` (tmpfs, root, `0644`, no secret in it): the key as `extra-trusted-public-keys`, which the host's nix.conf includes, with no substituter and no netrc; and `/run/hyperhive-nix/netrc` (`root:hive-core 0640`), the token as a netrc that `hive-c0re` passes to `nix copy --from` as the client's own setting. `hive-core` is the one unprivileged reader of the token, and no other client of the host's nix daemon gets it. The cache's address comes from `services.hyperhive.swarm.attic.domain` and `cache`, not from the store. A store that holds no facts, or that doesn't answer, leaves the files as they stand; a hive that never got them has `hive-c0re` ask for a refresh and then fail the pull with `this hive has no cache facts yet`. | -| the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | -| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | -| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity for each forge read, and hands it to the `git` commands it runs. It keeps no copy on disk | -| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.deploy.hive-controller.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `unrestricted` beside it lists the users whose agent holds the swarm's `unrestricted_internet` flag, names only; it lists every agent the hive never read as off `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; an agent without one sends none, so it goes direct | -| the squid self-check's password | `swarm-controller` mints `swarm/hives//squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Both self-check users (`selfcheck.squid` and the always-unrestricted `selfcheck-unrestricted.squid`) use it, so whoever holds it has unrestricted egress through this squid Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks | +| secret | generated by | lives at | +| ------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| hive leaf certs | `hive-tls.nix`, issued by the secret store's `pki` mount | `/.pem` | +| the hive's own store identity | `glue-bao-tls.nix` on the store's host; placed by hand on every other host ([`ca.md`](../../4-implementation/swarm/ca.md#the-gateway-leaf-comes-from-the-secret-store)) | `deploy.hive-controller.tls.baoHiveIdentityCertFile` / `baoHiveIdentityKeyFile` | +| the cache's public key and pull token, for the host's nix daemon and hive-priv | `swarm-controller` copies them to `swarm/hives//attic/{public-key,pull-token}` (see the cache-facts row above) | `hive-priv-cache-facts` (root) reads those two under the hive's own store identity every ten minutes and writes `/run/hyperhive-priv/attic/public-key` (tmpfs, root, `0600`, directory `0700`); it also writes `/run/hyperhive-nix/cache.conf` (tmpfs, root, `0644`, no secret in it): the cache's address as `extra-substituters`, the key as `extra-trusted-public-keys` and `netrc-file`, which the host's nix.conf includes; and `/run/hyperhive-nix/netrc` (`root:root 0600`), the token's entry followed by the entries of the host's own netrc (`nix.settings.netrc-file`, else `/etc/nix/netrc`), merged again on every run. The host's nix daemon reads it, so every client of the daemon can have a swarm closure substituted, and nothing unprivileged reads the token for the pull. The cache's address comes from `services.hyperhive.swarm.attic.domain` and `cache`, not from the store. A store that holds no facts, or that doesn't answer, leaves the files as they stand; a hive that never got them has `hive-c0re` ask for a refresh and then fail the pull with `this hive has no cache facts yet`. | +| the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | +| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | +| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity for each forge read, and hands it to the `git` commands it runs. It keeps no copy on disk | +| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.deploy.hive-controller.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `unrestricted` beside it lists the users whose agent holds the swarm's `unrestricted_internet` flag, names only; it lists every agent the hive never read as off `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; an agent without one sends none, so it goes direct | +| the squid self-check's password | `swarm-controller` mints `swarm/hives//squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Both self-check users (`selfcheck.squid` and the always-unrestricted `selfcheck-unrestricted.squid`) use it, so whoever holds it has unrestricted egress through this squid Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks | ### Store identities, one per principal diff --git a/docs/4-implementation/swarm/builder.md b/docs/4-implementation/swarm/builder.md index 9a7e9551..b3761c82 100644 --- a/docs/4-implementation/swarm/builder.md +++ b/docs/4-implementation/swarm/builder.md @@ -536,18 +536,24 @@ writes nothing. `hive-priv` and the nix daemon don't read those files. The key that must have signed a download and the pull token come from `hive-priv-cache-facts`, a root unit that reads the hive's own `attic/public-key` and `attic/pull-token` from -the store under the hive's own store identity every ten minutes and writes them -to `/run/hyperhive-priv/attic/` (`0700`, root; each file `0600`). It also writes -the nix.conf fragment `/run/hyperhive-nix/cache.conf`, which the host's nix.conf -includes, with the key as `extra-trusted-public-keys` and nothing else: no -substituter and no netrc, so the daemon never contacts the cache and no other -client of the daemon, such as an agent on `nixBuilds = host`, can fetch from it. -A second copy of the netrc, `/run/hyperhive-nix/netrc` (`root:hive-core 0640`), -is the pull token `hive-c0re` uses. The cache's address is the host's -`services.hyperhive.swarm.attic.domain` and `cache`, handed to `hive-c0re`. The -unit restarts `nix-daemon` only when the fragment changed after the daemon -started, which matters for CppNix; Lix re-reads nix.conf on every connection. A -file `hive-c0re` writes decides nothing the daemon or `hive-priv` trusts. +the store under the hive's own store identity every ten minutes. It writes the +key to `/run/hyperhive-priv/attic/public-key` (`0700` directory, `0600` file) +and the nix.conf fragment `/run/hyperhive-nix/cache.conf`, which the host's +nix.conf includes, with the cache's address as `extra-substituters`, the key as +`extra-trusted-public-keys` and `netrc-file = /run/hyperhive-nix/netrc`. The +netrc (`root:root 0600`) holds the pull token's entry for the cache's host and +then the entries of the host's own netrc, which is `nix.settings.netrc-file` +when the host sets it and nix's default `/etc/nix/netrc` otherwise, because +`netrc-file` takes one path and would replace the host's. The unit merges again +on every run, so an edit to the host's netrc reaches the daemon within ten +minutes. An entry for the cache's own host in the host's netrc gives way to the +pull token's. The unit doesn't see a netrc that the host names only in +`nix.extraOptions`. The cache's address is the host's +`services.hyperhive.swarm.attic.domain` and `cache`. The unit restarts +`nix-daemon` only when the fragment changed after the daemon started, which +matters for CppNix; Lix re-reads nix.conf on every connection and runs no +`nix-daemon.service`, so the unit logs and restarts nothing there. A file +`hive-c0re` writes decides nothing the daemon or `hive-priv` trusts. The same pass checks each of the hive's agents. It reads the agent's record in `agent-closures`, then fetches the closure's narinfo from @@ -591,13 +597,15 @@ means check attic's fingerprint format; it says nothing about an attacker. A A hive builds nothing. A rebuild's `Prebuild` node, which runs while the agent is still up, reads the agent's record in `agent-closures` and has `hive-c0re` pull the record's store path. `hive-c0re` isn't root and needs no root: it runs -`nix copy --from --option netrc-file /run/hyperhive-nix/netrc ` -as `hive-core`. The nix client fetches from the cache with the pull token and -hands the NARs to the host's nix daemon, which stores each path only when a key -it trusts signed it; nothing builds. When the netrc or the key is missing, +`nix-store --realise --max-jobs 0` as `hive-core`, which asks the host's +nix daemon to substitute the path. The daemon takes each path of the closure +from the swarm cache or from its other substituters (`cache.nixos.org`), +whichever has it, which matters because the swarm cache leaves out the paths +`cache.nixos.org` serves. It stores a path only when a key it trusts signed it, +and nothing builds. When the fragment is missing or lacks a setting, `hive-c0re` first asks `hive-priv` to run `hive-priv-cache-facts` (up to 60 s) -instead of waiting for its timer. A host that still has no facts, a signature refusal and a path the -cache lacks each fail with their own message. +instead of waiting for its timer. A host that still has no facts, a signature +refusal and a path no substituter has each fail with their own message. `Swap` then does the same lookup and pull, a no-op once the path is in the store, and gives `hive-priv` the store path to apply. `hive-priv` applies it only diff --git a/docs/4-implementation/trust-boundary/hive-c0re-privilege.md b/docs/4-implementation/trust-boundary/hive-c0re-privilege.md index 08720306..964eb0ba 100644 --- a/docs/4-implementation/trust-boundary/hive-c0re-privilege.md +++ b/docs/4-implementation/trust-boundary/hive-c0re-privilege.md @@ -35,7 +35,7 @@ known operations; there is no arbitrary command pass-through: | `StartFailureLog` | `journalctl _SYSTEMD_UNIT=container@h-.service --since @ -n 30 -o cat --no-pager`, fixed argv, 5 s timeout, output capped at 8 KiB; hive-priv clamps `` to the last hour. hive-c0re appends the lines to the failed start's build log, which only the dashboard serves. | | `CreateContainer` / `UpdateContainer` | `nixos-container create --system-path `; for an update, `nix-env -p --set ` and a reload of the unit. `` must already be in the store with a signature by the swarm cache's key: hive-priv runs `nix store verify --no-contents --sigs-needed 1` with `trusted-public-keys` set to the key in root-owned `/run/hyperhive-priv/attic/public-key`, and refuses a path that's absent or that the key didn't sign. hive-priv builds nothing. The check doesn't bind the path to the agent whose pull produced it: hive-c0re could already pull any swarm-signed path and apply it to any agent, so the check bounds the same risk (one agent gets another's closure), not an arbitrary-path one | | `RealiseClosure` | Kept for one release, for an older hive-c0re that still asks hive-priv to download. Downloads nothing: answers `Ok` when the path is in the store and signed by the swarm key (what `CreateContainer` needs), else `not available: `. The path must be `/nix/store/<32 chars>-` and the agent name valid | -| `RefreshCacheFacts` | `systemctl start hive-priv-cache-facts.service`, waited for up to 60 s. The request names no unit and has no field. hive-c0re asks for it when its netrc or the swarm key in the fragment is missing | +| `RefreshCacheFacts` | `systemctl start hive-priv-cache-facts.service`, waited for up to 60 s. The request names no unit and has no field. hive-c0re asks for it when the fragment is missing or lacks a setting | | `DestroyContainer` | `nixos-container destroy ` | | `ListContainers` | `nixos-container list` | | `ReloadGatewayNginx` | `systemctl reload/start/reset-failed nginx` (host unit; the unit name is hard-coded, not a parameter) | @@ -70,58 +70,62 @@ bind or credential path is absolute with no whitespace, control byte, `\`, `$`, backtick, `"` or `:`. The size limits of a value stay with systemd. **Closures from the swarm cache** — `hive-c0re` pulls the closure the swarm -built with `nix copy --from `, and `hive-priv` applies it only -when the swarm's key signed it. Pulling needs no root. The nix client does the -HTTP fetch itself, as `hive-core`, with a netrc only `hive-core` can read, and -hands the NARs to the host's nix daemon, which stores a path from an -unprivileged client only if a key in its `trusted-public-keys` signed it. -Root-owned sources alone decide what the daemon accepts: +built with `nix-store --realise --max-jobs 0`, and `hive-priv` applies it +only when the swarm's key signed it. Pulling needs no root. The command is a +request to the host's nix daemon, which substitutes each path of the closure +from the swarm cache or from its other substituters (`cache.nixos.org`), +whichever has it, and stores a path from an unprivileged client only if a key +in its `trusted-public-keys` signed it. Root-owned sources alone decide what +the daemon accepts: - The cache signs. The swarm's cache (attic) holds the signing key and signs each narinfo it serves; the private key never leaves it. The builder pushes with a push token and holds no signing key. - The address is the host's configuration: - `services.hyperhive.swarm.attic.domain` and `cache`, handed to `hive-c0re` in - `HIVE_C0RE_CACHE_URL`. The daemon has no address: it never contacts the - cache. + `services.hyperhive.swarm.attic.domain` and `cache`. The daemon holds it as a + substituter; `hive-c0re` has no address and names no source. - The key and the pull token come from the secret store. The root unit `hive-priv-cache-facts` reads the hive's own copies under the hive's own store identity and writes the key to `/run/hyperhive-priv/attic/public-key` - and the token to `netrc` there (directory `0700`, files `0600`, root). It - also writes the key into a nix.conf fragment, `/run/hyperhive-nix/cache.conf` - (`0644`), as `extra-trusted-public-keys`, which the host's nix.conf includes - (`!include`, so a host whose unit hasn't run yet still starts). The fragment - holds no secret, names no substituter and no netrc. `hive-priv` reads nothing - from `/run/hyperhive/attic/`, which `hive-c0re` writes for the probe. -- `hive-core` holds the pull token. The unit writes a second copy of the netrc, - `/run/hyperhive-nix/netrc`, `root:hive-core 0640`, and `hive-c0re` passes it - to `nix copy` as the client's `--option netrc-file`. This is a deliberate - grant, and no new capability: `hive-c0re` could already have `hive-priv` - download any swarm-signed path for any agent and apply it (the - `CreateContainer` row above), and the token reads only the cache the swarm - signs. The daemon ignores a client's `netrc-file`. + (directory `0700`, file `0600`, root). It also writes a nix.conf fragment, + `/run/hyperhive-nix/cache.conf` (`0644`), with `extra-substituters` (the + address), `extra-trusted-public-keys` (the key) and `netrc-file`, which the + host's nix.conf includes (`!include`, so a host whose unit hasn't run yet + still starts). The fragment holds no secret. `hive-priv` reads nothing from + `/run/hyperhive/attic/`, which `hive-c0re` writes for the probe. +- Only root reads the token. The netrc the fragment names, + `/run/hyperhive-nix/netrc` (`root:root 0600`), holds the token's entry plus + the entries of the host's own netrc (`nix.settings.netrc-file`, else nix's + default `/etc/nix/netrc`), because `netrc-file` takes one path and would + replace the host's. The unit merges again on every run, every ten minutes. + The daemon is root and reads the file itself, so `hive-core` has no copy of + the token for the pull. The probe's copy under `/run/hyperhive/attic/` is + separate. - The daemon enforces. It refuses a path from an unprivileged client unless a - trusted key signed it; a client can't lower `require-sigs` or add keys. - `nix copy` imports and never builds. + trusted key signed it; a client can't lower `require-sigs` or add keys or + substituters. `--max-jobs 0` forbids a build, so a path no substituter has + fails. - `hive-priv` verifies. Before `CreateContainer` or `UpdateContainer` applies a toplevel, it runs `nix store verify --no-contents --sigs-needed 1` with `trusted-public-keys` set to the key in the root file, and refuses a path that is absent or that the key didn't sign. The daemon's trusted keys also include the host's other caches, so this check is the one that counts only the swarm's. -- When the netrc or the key is missing, `hive-c0re` asks `hive-priv` for +- When the fragment is missing or lacks a setting, `hive-c0re` asks `hive-priv` for `RefreshCacheFacts`, which starts the facts unit and waits up to 60 s, instead of waiting for the unit's timer. A host the swarm hasn't given facts to answers `this hive has no cache facts yet`. **What an agent with daemon access can fetch.** Agent containers on -`nixBuilds = host` reach the same daemon ([nix-builds.md](nix-builds.md)). They -get nothing from the private cache: the daemon has no trusted substituter and no -netrc for it, an agent can't read `/run/hyperhive-nix/netrc` (`hive-core`'s -group) or `/run/hyperhive-priv/attic/`, and an unprivileged client can't make -the daemon use a substituter or a netrc it didn't configure. The one new thing -every client of the daemon shares is the swarm's public key in -`trusted-public-keys`: the daemon takes a path the swarm key signed from any source, -and only the swarm can produce one. +`nixBuilds = host` reach the same daemon ([nix-builds.md](nix-builds.md)). The +daemon holds the swarm cache as a substituter with the pull token, so any client +of it can have a swarm closure substituted, as it can have a path from +`cache.nixos.org`. That gives an agent nothing it can't read already: a closure +is world-readable in `/nix/store` once it's on the host, it holds no secret +(every secret comes from the store at run time), and the agent never sees the +token, which stays in root-only `/run/hyperhive-nix/netrc`. What the agent gains +is the chance to pull another hive's closure that isn't on this host yet. The +daemon takes a path the swarm key signed from any source, and only the swarm can +produce one. What a compromised `hive-c0re` can do: pull any store path that the swarm cache signed, which fills the store with paths the swarm vouches for, and then ask diff --git a/docs/4-implementation/trust-boundary/nix-builds.md b/docs/4-implementation/trust-boundary/nix-builds.md index d98fb530..df29d028 100644 --- a/docs/4-implementation/trust-boundary/nix-builds.md +++ b/docs/4-implementation/trust-boundary/nix-builds.md @@ -56,12 +56,12 @@ next start. `none` cuts the daemon only. It leaves the agent's egress alone. -Under `host` an agent shares the daemon with the hive's own closure pull, and -gains nothing from it. The daemon trusts the swarm cache's public key, but it -has no cache address and no pull token (hive-c0re fetches with a netrc readable -only by `hive-core`), so an agent can't fetch from the private cache. It can -still ask the daemon to substitute from the public caches the host configures, as -before. See +Under `host` an agent shares the daemon with the hive's own closure pull. The +daemon holds the swarm cache as a substituter, with the pull token in a netrc +only root reads, so an agent can ask it to substitute a swarm closure as well as +a path from the public caches the host configures. The agent never sees the +token, and a closure is world-readable once it's on the host and holds no +secret. See [hive-c0re-privilege.md](hive-c0re-privilege.md#closures-from-the-swarm-cache). ## Threat model diff --git a/hive-c0re/src/cache_pull.rs b/hive-c0re/src/cache_pull.rs index 09fc44df..8c8ecc65 100644 --- a/hive-c0re/src/cache_pull.rs +++ b/hive-c0re/src/cache_pull.rs @@ -1,36 +1,35 @@ -//! Pull the swarm's closure into the store with `nix copy --from`. +//! Pull the swarm's closure into the store through the host's nix daemon. //! -//! hive-c0re is not root and does not need to be. The nix client does the HTTP -//! fetch itself, with a netrc only hive-core can read ([`NETRC`]), and hands -//! the NARs to the host's nix daemon, which stores a path from an unprivileged -//! client only if a key in its `trusted-public-keys` signed it. The root unit -//! `hive-priv-cache-facts` puts the swarm's key there through a nix.conf -//! fragment ([`CONF_FRAGMENT`]) that names no substituter and no netrc, so the -//! daemon itself never contacts the cache and no other client of it can fetch -//! from the cache or send the token. hive-priv then applies the toplevel only -//! if the swarm's key signed it. +//! hive-c0re is not root and does not need to be: `nix-store --realise` of a +//! store path is a request to the daemon, which substitutes each path of the +//! closure from the swarm cache or from its other substituters (cache.nixos.org), +//! whichever has it. The swarm cache leaves out the paths cache.nixos.org +//! serves, so a client that fetched from the cache alone could not complete a +//! closure. The root unit `hive-priv-cache-facts` writes a nix.conf fragment +//! ([`CONF_FRAGMENT`]) that gives the daemon the cache's address, the swarm's +//! key and the netrc, which only root reads. The daemon stores a path only if a +//! key in its `trusted-public-keys` signed it, and hive-priv then applies the +//! toplevel only if the swarm's key signed it. //! -//! The cache's address is the host's configuration, handed to the unit in -//! [`URL_ENV`]. [`pull`] first asks hive-priv to run the facts unit when the -//! netrc or the key is not on the host yet, which is how the swarm's copy of the -//! facts reaches the host without waiting for the unit's timer. +//! [`pull`] first asks hive-priv to run the facts unit when the fragment is not +//! on the host yet, which is how the swarm's copy of the facts reaches the host +//! without waiting for the unit's timer. use std::time::Duration; use anyhow::Result; use hive_priv_sock::valid_closure_path; -/// The pull token as a netrc, `root:hive-core 0640`. -pub const NETRC: &str = "/run/hyperhive-nix/netrc"; - /// The nix.conf fragment `hive-priv-cache-facts` writes (root-owned, readable). pub const CONF_FRAGMENT: &str = "/run/hyperhive-nix/cache.conf"; -/// The unit environment variable naming the cache: `https:///`. -const URL_ENV: &str = "HIVE_C0RE_CACHE_URL"; - -/// The fragment's line that gives the daemon the swarm's key. -const KEY_SETTING: &str = "extra-trusted-public-keys"; +/// The fragment's lines the daemon needs to substitute from the swarm cache and +/// to trust what it gets. +const REQUIRED_SETTINGS: [&str; 3] = [ + "extra-substituters", + "extra-trusted-public-keys", + "netrc-file", +]; /// Wall-clock bound on one pull. A stalled transfer is bounded by nix's own /// `stalled-download-timeout`; this covers the rest. @@ -39,12 +38,12 @@ const PULL_TIMEOUT: Duration = Duration::from_mins(30); /// Why a closure was not pulled. #[derive(Debug, PartialEq, Eq)] pub enum PullError { - /// This host has no cache facts: no address, no netrc for hive-core, or no - /// key in the fragment. Waiting for the facts unit fixes the last two. + /// This host has no cache facts: no fragment, or one that lacks a setting + /// the daemon needs. Waiting for the facts unit fixes it. FactsMissing(String), /// The daemon refused the path: no signature by a key it trusts. SignatureRefused(String), - /// The cache does not have the path. + /// No substituter has the path. NotFound(String), /// Anything else nix said, or could not do. Failed(String), @@ -63,16 +62,19 @@ impl std::fmt::Display for PullError { f, "the nix daemon refused the closure, which carries no signature by a key this host trusts: {last}" ), - Self::NotFound(last) => write!(f, "the swarm cache does not have the closure: {last}"), + Self::NotFound(last) => write!( + f, + "no substituter of the nix daemon has the closure: {last}" + ), Self::Failed(last) => f.write_str(last), - Self::TimedOut => write!(f, "nix copy timed out after {}s", PULL_TIMEOUT.as_secs()), + Self::TimedOut => write!(f, "nix-store timed out after {}s", PULL_TIMEOUT.as_secs()), } } } impl std::error::Error for PullError {} -/// What a `nix copy` run came to. +/// What a `nix-store` run came to. pub enum Run { Exited { success: bool, stderr: String }, TimedOut, @@ -80,63 +82,53 @@ pub enum Run { /// What [`pull`] does on the host, so a test can stand in for it. pub trait Effects: Sync { - /// The cache's address, when the host names one. - fn cache_url(&self) -> Option; - /// Whether hive-core can read the netrc and the fragment gives the daemon - /// the swarm's key. - async fn facts_present(&self) -> bool; + /// The fragment's text; `None` when it cannot be read. + async fn fragment(&self) -> Option; /// Run the facts unit through hive-priv and wait for it. async fn refresh(&self) -> Result<()>; - /// Run `nix` with `args`, bounded by [`PULL_TIMEOUT`]. - async fn nix(&self, args: Vec) -> std::io::Result; + /// Run `nix-store` with `args`, bounded by [`PULL_TIMEOUT`]. + async fn nix_store(&self, args: Vec) -> std::io::Result; } -/// Whether `fragment` has a non-empty `extra-trusted-public-keys` line. +/// The name of the first setting in [`REQUIRED_SETTINGS`] that `fragment` gives +/// no value, or `None` when it gives all of them. #[must_use] -pub fn gives_key(fragment: &str) -> bool { - fragment.lines().any(|line| { - line.split_once('=') - .is_some_and(|(name, value)| name.trim() == KEY_SETTING && !value.trim().is_empty()) +pub fn missing_setting(fragment: &str) -> Option<&'static str> { + REQUIRED_SETTINGS.into_iter().find(|wanted| { + !fragment.lines().any(|line| { + line.split_once('=') + .is_some_and(|(name, value)| name.trim() == *wanted && !value.trim().is_empty()) + }) }) } -/// The `nix` arguments that fetch `store_path` from the cache at `url`. +/// The `nix-store` arguments that fetch `store_path` and nothing else. /// -/// - `copy --from ` makes the client fetch. The daemon is never asked to -/// contact the cache, so it needs no trust in the address. -/// - `--option netrc-file` is the client's own setting: it reads the file as -/// hive-core. The daemon ignores the setting from an unprivileged client. -/// - No `--no-check-sigs`: the daemon checks every path's signature against its -/// trusted keys. Nothing builds: `copy` only imports. +/// - `--realise ` asks the daemon to substitute the path and every path +/// it references that is not yet valid, from the substituters the daemon is +/// configured with. +/// - `--max-jobs 0` forbids a local build, so a path no substituter has fails. +/// - No `substituters`, `trusted-public-keys`, `require-sigs` or `netrc-file`: +/// the daemon ignores them from an unprivileged client and applies its own. #[must_use] -pub fn pull_args(store_path: &str, url: &str, netrc: &str) -> Vec { - [ - "--extra-experimental-features", - "nix-command", - "copy", - "--from", - url, - "--option", - "netrc-file", - netrc, - store_path, - ] - .map(str::to_owned) - .into() +pub fn pull_args(store_path: &str) -> Vec { + ["--realise", store_path, "--max-jobs", "0"] + .map(str::to_owned) + .into() } /// Name nix's refusal from its stderr. A refusal for want of a signature is -/// told apart from a path the cache lacks and from a cache that is down. +/// told apart from a path no substituter has and from a cache that is down. fn classify(stderr: &str) -> PullError { let last = stderr .lines() .rev() .find(|l| !l.trim().is_empty()) - .unwrap_or("nix copy failed with no message") + .unwrap_or("nix-store failed with no message") .trim() .to_owned(); let lower = stderr.to_lowercase(); - if lower.contains("signature") { + if lower.contains("signature") || lower.contains("not signed by any of the keys") { PullError::SignatureRefused(last) } else if lower.contains("don't know how to build") || lower.contains("no substituter that can build") @@ -150,9 +142,9 @@ fn classify(stderr: &str) -> PullError { /// Pull `store_path` into the store. /// /// # Errors -/// [`PullError`]: no cache facts on this host (after one refresh), the daemon -/// refusing the signature, the cache lacking the path, nix failing otherwise, -/// or a timeout. +/// [`PullError`]: no cache facts on this host (after one refresh), nix +/// refusing the signature, no substituter having the path, nix failing +/// otherwise, or a timeout. pub async fn pull(store_path: &str) -> Result<(), PullError> { pull_with(&Real, store_path).await } @@ -163,14 +155,11 @@ async fn pull_with(fx: &impl Effects, store_path: &str) -> Result<(), PullError> "{store_path:?} is not a store path" ))); } - let url = fx - .cache_url() - .ok_or_else(|| PullError::FactsMissing("this host names no cache".to_owned()))?; ensure_facts(fx).await?; match fx - .nix(pull_args(store_path, &url, NETRC)) + .nix_store(pull_args(store_path)) .await - .map_err(|e| PullError::Failed(format!("running nix: {e}")))? + .map_err(|e| PullError::Failed(format!("running nix-store: {e}")))? { Run::Exited { success: true, .. } => Ok(()), Run::Exited { stderr, .. } => Err(classify(&stderr)), @@ -178,44 +167,44 @@ async fn pull_with(fx: &impl Effects, store_path: &str) -> Result<(), PullError> } } +/// What the fragment lacks, or `None` when the daemon has what it needs. +async fn facts_lacking(fx: &impl Effects) -> Option { + match fx.fragment().await { + None => Some("no fragment".to_owned()), + Some(text) => missing_setting(&text).map(|name| format!("the fragment has no {name}")), + } +} + /// The facts are on the host, after one refresh when they are not. async fn ensure_facts(fx: &impl Effects) -> Result<(), PullError> { - if fx.facts_present().await { + if facts_lacking(fx).await.is_none() { return Ok(()); } let refreshed = fx.refresh().await; - if fx.facts_present().await { - return Ok(()); + match facts_lacking(fx).await { + None => Ok(()), + Some(lacking) => Err(PullError::FactsMissing(match refreshed { + Ok(()) => format!("the facts unit ran and {lacking}"), + Err(e) => format!("refreshing them failed: {e:#}"), + })), } - Err(PullError::FactsMissing(match refreshed { - Ok(()) => "the facts unit ran and the swarm has given this hive none".to_owned(), - Err(e) => format!("refreshing them failed: {e:#}"), - })) } -/// The host: the environment and files on disk, hive-priv for the refresh, -/// `nix` from the system profile. +/// The host: the fragment on disk, hive-priv for the refresh, `nix-store` from +/// the system profile. struct Real; impl Effects for Real { - fn cache_url(&self) -> Option { - std::env::var(URL_ENV) - .ok() - .map(|u| u.trim().to_owned()) - .filter(|u| u.starts_with("https://") && !u.contains(char::is_whitespace)) - } - - async fn facts_present(&self) -> bool { - let fragment = tokio::fs::read_to_string(CONF_FRAGMENT).await; - fragment.is_ok_and(|f| gives_key(&f)) && tokio::fs::File::open(NETRC).await.is_ok() + async fn fragment(&self) -> Option { + tokio::fs::read_to_string(CONF_FRAGMENT).await.ok() } async fn refresh(&self) -> Result<()> { crate::priv_client::refresh_cache_facts().await } - async fn nix(&self, args: Vec) -> std::io::Result { - let mut cmd = tokio::process::Command::new("nix"); + async fn nix_store(&self, args: Vec) -> std::io::Result { + let mut cmd = tokio::process::Command::new("nix-store"); // Explicit, so the store is the daemon's whatever the environment. cmd.args(args) .env("NIX_REMOTE", "daemon") @@ -242,25 +231,23 @@ mod tests { use super::*; const PATH: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; - const URL: &str = "https://cache.example/swarm"; + const FRAGMENT: &str = "extra-substituters = https://cache.example/swarm\nextra-trusted-public-keys = swarm:AAAA\nnetrc-file = /run/hyperhive-priv/attic/netrc"; - /// A host that answers from a script: whether the facts are present now - /// (and after a refresh), what `nix` comes to, and a log of calls. + /// A host that answers from a script: the fragment now (and after a + /// refresh), what `nix-store` comes to, and a log of calls. struct Fake { - url: Option<&'static str>, - present: Mutex, - present_after_refresh: bool, + fragment: Mutex>, + fragment_after_refresh: Option<&'static str>, refresh_fails: bool, nix: Mutex>, calls: Mutex>, } impl Fake { - fn new(present: bool, nix: Run) -> Self { + fn new(fragment: Option<&'static str>, nix: Run) -> Self { Self { - url: Some(URL), - present: Mutex::new(present), - present_after_refresh: true, + fragment: Mutex::new(fragment), + fragment_after_refresh: Some(FRAGMENT), refresh_fails: false, nix: Mutex::new(Some(nix)), calls: Mutex::new(Vec::new()), @@ -273,13 +260,9 @@ mod tests { } impl Effects for Fake { - fn cache_url(&self) -> Option { - self.url.map(str::to_owned) - } - - async fn facts_present(&self) -> bool { - self.calls.lock().unwrap().push("facts".to_owned()); - *self.present.lock().unwrap() + async fn fragment(&self) -> Option { + self.calls.lock().unwrap().push("fragment".to_owned()); + self.fragment.lock().unwrap().map(str::to_owned) } async fn refresh(&self) -> Result<()> { @@ -287,15 +270,15 @@ mod tests { if self.refresh_fails { anyhow::bail!("hive-priv unreachable"); } - *self.present.lock().unwrap() = self.present_after_refresh; + *self.fragment.lock().unwrap() = self.fragment_after_refresh; Ok(()) } - async fn nix(&self, args: Vec) -> std::io::Result { + async fn nix_store(&self, args: Vec) -> std::io::Result { self.calls .lock() .unwrap() - .push(format!("nix {}", args.join(" "))); + .push(format!("nix-store {}", args.join(" "))); Ok(self.nix.lock().unwrap().take().expect("one run")) } } @@ -307,75 +290,55 @@ mod tests { } } - fn after(args: &[String], flag: &str) -> Option { - args.iter() - .position(|a| a == flag) - .and_then(|i| args.get(i + 1)) - .cloned() - } - - /// The client fetches: `copy --from `, with its own netrc as a client - /// setting. Nothing asks the daemon to contact the cache or to skip the - /// signature check, and nothing can build. + /// The daemon substitutes: `--realise `, no source named, no client + /// setting, nothing that widens trust, and nothing can build. #[test] - fn the_command_copies_from_the_cache_with_the_clients_netrc_and_checks_signatures() { - let args = pull_args(PATH, URL, NETRC); - assert_eq!(args.last().map(String::as_str), Some(PATH)); - let at = args.iter().position(|a| a == "copy").expect("verb"); - assert_eq!(args[at + 1], "--from"); - assert_eq!(args[at + 2], URL); - assert_eq!(after(&args, "--option").as_deref(), Some("netrc-file")); + fn the_command_realises_the_path_through_the_daemon_without_naming_a_source() { assert_eq!( - args.windows(3) - .find(|w| w[0] == "--option" && w[1] == "netrc-file") - .map(|w| w[2].as_str()), - Some(NETRC) + pull_args(PATH), + ["--realise", PATH, "--max-jobs", "0"].map(str::to_owned) ); - // Exactly one setting, and none that widens trust. - assert_eq!( - args.iter().filter(|a| *a == "--option").count(), - 1, - "{args:?}" - ); - for forbidden in [ - "--no-check-sigs", - "--substitute-on-destination", - "--to", - "substituters", - "trusted-public-keys", - "require-sigs", - ] { - assert!( - !args.iter().any(|a| a == forbidden), - "{forbidden}: {args:?}" - ); - } - assert!(!args.iter().any(|a| a.starts_with("extra-")), "{args:?}"); } #[test] - fn the_fragment_gives_the_key_when_it_has_a_value() { - assert!(gives_key("extra-trusted-public-keys = swarm:AAAA")); - for bad in [ - "", - "extra-trusted-public-keys =", - "extra-trusted-public-keys", - "trusted-public-keys = swarm:AAAA", - "extra-trusted-substituters = https://cache.example/swarm", + fn the_fragment_is_complete_when_every_setting_has_a_value() { + assert_eq!(missing_setting(FRAGMENT), None); + for (bad, want) in [ + ("", "extra-substituters"), + ( + "extra-substituters =\nextra-trusted-public-keys = k\nnetrc-file = n", + "extra-substituters", + ), + ( + "extra-substituters = u\nextra-trusted-public-keys\nnetrc-file = n", + "extra-trusted-public-keys", + ), + ( + "extra-substituters = u\nextra-trusted-public-keys = k", + "netrc-file", + ), + ( + "extra-trusted-substituters = u\nextra-trusted-public-keys = k\nnetrc-file = n", + "extra-substituters", + ), + ( + "extra-substituters = u\ntrusted-public-keys = k\nnetrc-file = n", + "extra-trusted-public-keys", + ), ] { - assert!(!gives_key(bad), "{bad:?}"); + assert_eq!(missing_setting(bad), Some(want), "{bad:?}"); } } #[tokio::test] async fn with_the_facts_in_place_it_pulls_without_a_refresh() { - let fx = Fake::new(true, exited(true, "")); + let fx = Fake::new(Some(FRAGMENT), exited(true, "")); pull_with(&fx, PATH).await.expect("pulled"); assert_eq!( fx.calls(), [ - "facts".to_owned(), - format!("nix {}", pull_args(PATH, URL, NETRC).join(" ")) + "fragment".to_owned(), + format!("nix-store {}", pull_args(PATH).join(" ")) ] ); } @@ -383,30 +346,37 @@ mod tests { /// Missing facts are refreshed once, and the pull then goes ahead. #[tokio::test] async fn missing_facts_are_refreshed_before_the_pull() { - let fx = Fake::new(false, exited(true, "")); + let fx = Fake::new(None, exited(true, "")); pull_with(&fx, PATH).await.expect("pulled"); assert_eq!( fx.calls(), [ - "facts".to_owned(), + "fragment".to_owned(), "refresh".to_owned(), - "facts".to_owned(), - format!("nix {}", pull_args(PATH, URL, NETRC).join(" ")) + "fragment".to_owned(), + format!("nix-store {}", pull_args(PATH).join(" ")) ] ); } /// Facts still absent after the refresh, whether the unit ran or hive-priv - /// could not start it, stop the pull before nix is asked. + /// could not start it, stop the pull before nix is asked. A fragment that + /// lacks a setting counts as absent. #[tokio::test] async fn facts_still_missing_after_the_refresh_are_reported_and_nix_is_not_run() { - for (case, refresh_fails, want) in [ - ("unit ran", false, "none"), - ("refresh failed", true, "hive-priv unreachable"), + for (case, fragment, refresh_fails, want) in [ + ("no fragment", None, false, "no fragment"), + ( + "no substituter", + Some("extra-trusted-public-keys = k\nnetrc-file = n"), + false, + "has no extra-substituters", + ), + ("refresh failed", None, true, "hive-priv unreachable"), ] { - let mut fx = Fake::new(false, exited(true, "")); + let mut fx = Fake::new(None, exited(true, "")); fx.refresh_fails = refresh_fails; - fx.present_after_refresh = false; + fx.fragment_after_refresh = fragment; let err = pull_with(&fx, PATH).await.expect_err(case); let PullError::FactsMissing(why) = &err else { panic!("{case}: {err:?}"); @@ -416,20 +386,13 @@ mod tests { format!("{err}").contains("a later deploy retries"), "{case}" ); - assert!(!fx.calls().iter().any(|c| c.starts_with("nix")), "{case}"); + assert!( + !fx.calls().iter().any(|c| c.starts_with("nix-store")), + "{case}" + ); } } - /// A host that names no cache has nothing to refresh toward. - #[tokio::test] - async fn a_host_that_names_no_cache_does_not_pull_or_refresh() { - let mut fx = Fake::new(true, exited(true, "")); - fx.url = None; - let err = pull_with(&fx, PATH).await.expect_err("no cache"); - assert!(matches!(err, PullError::FactsMissing(_)), "{err:?}"); - assert!(fx.calls().is_empty(), "{:?}", fx.calls()); - } - #[test] fn nixs_refusals_are_told_apart() { for (stderr, want) in [ @@ -472,7 +435,7 @@ mod tests { #[tokio::test] async fn a_failed_run_is_mapped_and_a_timeout_is_a_timeout() { let fx = Fake::new( - true, + Some(FRAGMENT), exited( false, "error: cannot add path 'x' because it lacks a signature by a trusted key", @@ -482,7 +445,7 @@ mod tests { pull_with(&fx, PATH).await, Err(PullError::SignatureRefused(_)) )); - let fx = Fake::new(true, Run::TimedOut); + let fx = Fake::new(Some(FRAGMENT), Run::TimedOut); assert_eq!(pull_with(&fx, PATH).await, Err(PullError::TimedOut)); } @@ -490,7 +453,7 @@ mod tests { #[tokio::test] async fn only_a_store_path_is_pulled() { for bad in ["", "main", "/etc/passwd", "--help", "/nix/store/x"] { - let fx = Fake::new(true, exited(true, "")); + let fx = Fake::new(Some(FRAGMENT), exited(true, "")); assert!( matches!(pull_with(&fx, bad).await, Err(PullError::Failed(_))), "{bad}" diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index 609d8adb..977917fc 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -289,7 +289,7 @@ pub enum PrivRequest { }, /// Kept for an older hive-c0re that still asks hive-priv to download: - /// hive-c0re now pulls with `nix copy --from` and hive-priv downloads + /// hive-c0re now pulls through the nix daemon and hive-priv downloads /// nothing. The answer is `Ok` when `store_path` is already in the store /// and signed by the swarm cache's key (what `CreateContainer` and /// `UpdateContainer` require), else an error that starts `not available: `. @@ -302,8 +302,8 @@ pub enum PrivRequest { /// `systemctl start hive-priv-cache-facts.service`, bounded at 60 s: the /// root unit that reads this hive's cache facts from the store and writes - /// the swarm's key into the nix.conf fragment the daemon trusts and the pull - /// token into the netrc hive-c0re reads. hive-c0re asks for it when either is + /// the cache's address, the swarm's key and the netrc's path into the nix.conf + /// fragment the daemon reads. hive-c0re asks for it when the fragment is /// missing. The request names no unit. RefreshCacheFacts, diff --git a/hive-priv/src/closure.rs b/hive-priv/src/closure.rs index 7d4cdaf4..7fcb8a7c 100644 --- a/hive-priv/src/closure.rs +++ b/hive-priv/src/closure.rs @@ -17,7 +17,7 @@ use hive_priv_sock::valid_closure_path; use crate::container::{BoundedRun, run_bounded}; use crate::shared::{RealSpawn, Spawn, validate_agent_name}; -/// Where the root unit `hive-priv-cache-facts` keeps `public-key` and `netrc`. +/// Where the root unit `hive-priv-cache-facts` keeps `public-key`. /// Not `/run/hyperhive/attic`, which hive-c0re writes. const ROOT_FACTS_DIR: &str = "/run/hyperhive-priv/attic"; @@ -183,7 +183,7 @@ pub(crate) async fn realise_closure(agent: &str, store_path: &str) -> Result<(St match check_signed(&RealSpawn, Path::new(ROOT_FACTS_DIR), 0, store_path).await { Ok(()) => Ok(("present".to_owned(), String::new())), Err(e) => bail!( - "not available: {e:#}; hive-priv no longer downloads, hive-c0re pulls with nix copy" + "not available: {e:#}; hive-priv no longer downloads, hive-c0re pulls through the nix daemon" ), } } diff --git a/nix/checks.nix b/nix/checks.nix index a0b0cc48..b3c08877 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -89,9 +89,16 @@ in # run must mint nothing. script-test-attic-init = import ./script-tests/attic-init.nix { inherit pkgs; }; # Runs ./host-modules/lib/attic-root-facts.nix, the root unit that keeps the - # cache's key and pull token for hive-priv, against a stub `bao`: file modes, - # no rewrite when nothing changed, and no file removed when the store fails. + # cache's key and the merged netrc for the host's nix daemon, against a stub + # `bao`: the fragment, file modes, the host's own netrc entries kept, no rewrite + # when nothing changed, and no file removed when the store fails. script-test-attic-root-facts = import ./script-tests/attic-root-facts.nix { inherit pkgs; }; + # Runs the closure pull, `nix-store --realise --max-jobs 0`, against a + # swarm cache that lacks a dependency and a second cache that has it: the daemon + # takes each path from whichever substituter has it. + script-test-cache-pull-substitute = import ./script-tests/cache-pull-substitute.nix { + inherit pkgs; + }; # Runs ./host-modules/lib/store-seeded-secret.nix's seed, store-wins and mint # decisions against a stub `bao` that keeps its leaves in files. script-test-store-seeded-secret = import ./script-tests/store-seeded-secret.nix { inherit pkgs; }; diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 3132021d..61ba6ac2 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -316,7 +316,7 @@ in # (/run/hyperhive), and ReadWritePaths below, which systemd keeps # writable. Beyond the managed directories: # - the one nix call is the closure pull (src/cache_pull.rs): - # `nix copy --from` with NIX_REMOTE=daemon, so the host daemon + # `nix-store --realise` with NIX_REMOTE=daemon, so the host daemon # writes the store — hive-c0re never writes to /nix itself. It # connects to the daemon's socket, which RestrictAddressFamilies # (deferred above) must keep allowing. diff --git a/nix/host-modules/hive-priv-cache.nix b/nix/host-modules/hive-priv-cache.nix index bbc2401f..df1ef3d8 100644 --- a/nix/host-modules/hive-priv-cache.nix +++ b/nix/host-modules/hive-priv-cache.nix @@ -1,30 +1,33 @@ # The binary-cache facts the host's nix daemon, hive-priv (root) and hive-c0re # use, kept where only root can change them. # -# hive-c0re pulls an agent's closure from the swarm cache with its own -# `nix copy --from`: the client does the HTTP fetch with the pull token, and -# hands the NAR to the host's nix daemon, which stores it only if a key in its +# hive-c0re pulls an agent's closure with `nix-store --realise` through the +# host's nix daemon, which substitutes each path from the swarm cache or from +# its other substituters (cache.nixos.org) and stores it only if a key in its # `trusted-public-keys` signed it. hive-priv applies a closure only if the -# swarm's key signed it. What decides that must not come from a file the -# unprivileged hive-c0re writes. So: +# swarm's key signed it. Neither decision may come from a file hive-c0re writes: # -# - the cache's address is this host's configuration -# ({option}`services.hyperhive.swarm.attic.domain` and `cache`), handed to -# hive-c0re in its unit's environment; +# - the cache's address is the host's +# {option}`services.hyperhive.swarm.attic.domain` and `cache`; # - the cache's public key and the pull token are read from the store by a root # oneshot, `hive-priv-cache-facts`, under this hive's own store identity. The -# key and the token land in `/run/hyperhive-priv/attic/` (root, `0700`; files -# `0600`). The key also lands in a nix.conf fragment, -# `/run/hyperhive-nix/cache.conf`, as `extra-trusted-public-keys`, which the -# host's nix.conf includes (`!include`, so a host whose unit has not run yet -# still starts). The fragment names no substituter and no netrc, so a client -# of the daemon that is not hive-c0re (an agent container on `nixBuilds = -# host`) cannot fetch from the cache or send the token; -# - the token also lands in `/run/hyperhive-nix/netrc`, `root:hive-core 0640`: -# the one place hive-core, and nothing else unprivileged, can read it. +# key lands in `/run/hyperhive-priv/attic/public-key` (directory `0700`, file +# `0600`). +# The cache's address, the key and a netrc also land in a nix.conf fragment, +# `/run/hyperhive-nix/cache.conf`, as `extra-substituters`, +# `extra-trusted-public-keys` and `netrc-file`, which the host's nix.conf +# includes (`!include`, so a host whose unit has not run yet still starts). +# Every client of the daemon (hive-c0re, agent containers on `nixBuilds = +# host`) can have a swarm closure substituted; closures hold no secret; +# - that netrc, `/run/hyperhive-nix/netrc` (`root:root 0600`), is the pull +# token's entry plus the entries of the host's own netrc, which +# `netrc-file` would otherwise replace. The host's netrc is +# `nix.settings.netrc-file` when the host sets it, else nix's default +# `/etc/nix/netrc`. The unit merges again on every run, so an edit reaches the +# daemon within the timer's ten minutes. A netrc named only in +# `nix.extraOptions` is not seen. # -# hive-c0re keeps its own copies under `/run/hyperhive/attic/` for the probe, -# which only reports. +# hive-c0re keeps its own copies under `/run/hyperhive/attic/` for the probe. { pkgs, lib, @@ -53,7 +56,10 @@ let dir = "/run/hyperhive-priv/attic"; confDir = "/run/hyperhive-nix"; confFile = "${confDir}/cache.conf"; - clientNetrc = "${confDir}/netrc"; + cacheUrl = "https://${atticCfg.domain}/${atticCfg.cache}"; + netrcFile = "${confDir}/netrc"; + # `netrc-file` is not additive: the fragment's setting replaces the host's. + hostNetrc = toString (config.nix.settings.netrc-file or "/etc/nix/netrc"); # Where the store runs on this host, the first login waits for its container. # No `wants`: a pull would start the container mid-switch on the old closure. @@ -72,10 +78,6 @@ in # `!include` tolerates a fragment that does not exist yet. nix.extraOptions = "!include ${confFile}"; - # The address hive-c0re names to `nix copy --from`; no request chooses it. - systemd.services.hive-c0re.environment.HIVE_C0RE_CACHE_URL = - "https://${atticCfg.domain}/${atticCfg.cache}"; - systemd.services.hive-priv-cache-facts = { description = "Read the swarm cache's public key and pull token for the host's nix daemon and hive-priv"; wantedBy = [ "multi-user.target" ]; @@ -111,8 +113,13 @@ in script = import ./lib/attic-root-facts.nix { inherit lib; } { hive = hiveController.hiveName; host = atticCfg.domain; - inherit dir confFile clientNetrc; - clientGroup = config.users.users.hive-core.group; + inherit + dir + confFile + netrcFile + hostNetrc + ; + url = cacheUrl; }; }; diff --git a/nix/host-modules/lib/attic-root-facts.nix b/nix/host-modules/lib/attic-root-facts.nix index a1a07ae0..4734e2e8 100644 --- a/nix/host-modules/lib/attic-root-facts.nix +++ b/nix/host-modules/lib/attic-root-facts.nix @@ -1,22 +1,23 @@ -# The root-side copy of the cache facts the host's nix daemon, hive-priv and -# hive-c0re use: the key a downloaded closure must be signed by and the netrc -# the pull token goes out in. The script runs as root on the hive host under -# the hive's own store identity and writes: +# The root-side copy of the cache facts the host's nix daemon and hive-priv +# use: the key a downloaded closure must be signed by and the netrc the pull +# token goes out in. The script runs as root on the hive host under the hive's +# own store identity and writes: # -# - the key and the netrc into `dir`, which only root can read (hive-priv's -# check of the key); +# - the key into `dir`, which only root can read; +# - the netrc at `netrcFile`, root 0600: the pull token's entry for `host` +# followed by the entries of the host's own netrc (`hostNetrc`) for other +# machines. `netrc-file` is one path, so a fragment that names this file must +# carry what the host already keeps there; # - a nix.conf fragment at `confFile` that the host's nix.conf includes, with -# `extra-trusted-public-keys` and nothing else: no trusted substituter and no -# netrc, so no other client of the daemon can fetch from the cache or send -# the token; -# - a second copy of the netrc at `clientNetrc`, `root: 0640`, -# that hive-c0re passes to its own `nix copy --from` as a client setting. +# `extra-substituters` (the cache's address), `extra-trusted-public-keys` and +# `netrc-file` (that merged netrc), so the daemon +# substitutes from the cache and its other substituters. It holds no secret. # # Nothing is read from a file hive-c0re (unprivileged) writes: the key and the -# token come from the store. The fragment holds no secret. +# token come from the store. # # A store that holds no facts yet, or cannot be reached, leaves what is in -# `dir`, `confFile` and `clientNetrc` as they are. Nothing here removes a file. +# `dir`, `netrcFile` and `confFile` as they are. Nothing here removes a file. # # Pure function — NOT a NixOS module. Call it from a unit's `script`, with # `bao`, `coreutils`, `gnugrep` and `systemd` on its `path` and the `BAO_*` @@ -24,8 +25,8 @@ # # script = import ./lib/attic-root-facts.nix { inherit lib; } { # hive = "alpha"; host = "attic.example.org"; dir = "/run/hyperhive-priv/attic"; -# confFile = "/run/hyperhive-nix/cache.conf"; clientNetrc = "/run/hyperhive-nix/netrc"; -# clientGroup = "hive-core"; +# url = "https://attic.example.org/swarm"; confFile = "/run/hyperhive-nix/cache.conf"; +# netrcFile = "/run/hyperhive-nix/netrc"; hostNetrc = "/etc/nix/netrc"; # }; { lib }: { @@ -34,13 +35,16 @@ hive, # The cache server's host name, which the netrc line names. host, - # Where the key and the netrc go. + # Where the key goes. dir, + # The cache's address as the daemon lists it in `extra-substituters`. + url, # Where the nix.conf fragment goes; its directory is root's and readable. confFile, - # Where the netrc copy for the client goes, and the group that may read it. - clientNetrc, - clientGroup, + # Where the merged netrc goes, and the netrc the host uses without this + # module, which is merged in and never written. + netrcFile, + hostNetrc, }: '' set -euo pipefail @@ -50,9 +54,10 @@ dir=${lib.escapeShellArg dir} conf=${lib.escapeShellArg confFile} - client_netrc=${lib.escapeShellArg clientNetrc} + netrc_file=${lib.escapeShellArg netrcFile} + host_netrc=${lib.escapeShellArg hostNetrc} install -d -m 0700 "$dir" - install -d -m 0755 "$(dirname -- "$conf")" "$(dirname -- "$client_netrc")" + install -d -m 0755 "$(dirname -- "$conf")" "$(dirname -- "$netrc_file")" store_init store_login @@ -97,22 +102,40 @@ if secret_differs "$dir/public-key" "$key"; then atomic_write_secret 0600 "" "$dir/public-key" "$key" fi + + # The host's netrc without its lines for this cache's host: an entry runs from + # a `machine` or `default` line to the next one. A missing file is no entries. + # Our entry goes first, which is the one curl takes when an entry the filter + # does not recognise names the cache's host too. + other_netrc_entries() { + local line skip=0 tok kept="" + [ -f "$host_netrc" ] || return 0 + while IFS= read -r line || [ -n "$line" ]; do + read -ra tok <<<"$line" + case "''${tok[0]-}" in + machine) if [ "''${tok[1]-}" = ${lib.escapeShellArg host} ]; then skip=1; else skip=0; fi ;; + default) skip=0 ;; + esac + [ "$skip" = 1 ] || kept+="$line"$'\n' + done <"$host_netrc" + printf '%s' "$kept" + } netrc="machine ${host} password $token" - if secret_differs "$dir/netrc" "$netrc"; then - atomic_write_secret 0600 "" "$dir/netrc" "$netrc" + others="$(other_netrc_entries)" + if [ -n "$others" ]; then netrc="$netrc"$'\n'"$others"; fi + # The mode too: an earlier deploy left this path 0640 for a group. + if secret_differs "$netrc_file" "$netrc" || [ "$(stat -c %a "$netrc_file")" != 600 ]; then + atomic_write_secret 0600 "" "$netrc_file" "$netrc" fi - # The client's copy: readable by the group, which is a deliberate grant of the - # pull token to hive-c0re's user. Written before the fragment, so a host with - # the key has the netrc. - if secret_differs "$client_netrc" "$netrc"; then - atomic_write_secret 0640 ":${clientGroup}" "$client_netrc" "$netrc" - fi - - # Last. The mtime moves only with the content. Lix re-reads nix.conf for every - # connection, CppNix only at start, so a daemon that started before this write - # is restarted. - fragment="extra-trusted-public-keys = $key" + # Last, so a daemon that reads the fragment finds the netrc in place. The + # mtime moves only with the content. Lix re-reads nix.conf for every + # connection, CppNix only at start, so a CppNix daemon that started before + # this write is restarted. + printf -v fragment '%s\n%s\n%s' \ + ${lib.escapeShellArg "extra-substituters = ${url}"} \ + "extra-trusted-public-keys = $key" \ + "netrc-file = $netrc_file" if secret_differs "$conf" "$fragment"; then atomic_write_secret 0644 "" "$conf" "$fragment" fi diff --git a/nix/host-modules/lib/refresh-consumer.nix b/nix/host-modules/lib/refresh-consumer.nix index 8f74567e..b703cc56 100644 --- a/nix/host-modules/lib/refresh-consumer.nix +++ b/nix/host-modules/lib/refresh-consumer.nix @@ -61,7 +61,9 @@ # [systemctl option…]. Nothing unless `written_after_start`. # Otherwise a running is restarted and a failed one — it may have # hit its start limit without the credential — is reset and started; one - # stopped on purpose stays stopped. `--no-block` because a caller ordered + # that is neither running nor failed (stopped on purpose, never started, or + # a per-connection daemon with no unit of its own) is left alone, and + # nothing is logged for it. `--no-block` because a caller ordered # `Before=` its consumer must not wait on a job that waits on the caller # (see `swarm-services-cert`'s propagation in ../hive-tls.nix). refresh_unit() { @@ -72,8 +74,8 @@ echo "$path changed after $unit last started, and $unit had failed — starting it" systemctl "$@" reset-failed "$unit" systemctl "$@" start --no-block "$unit" - else - echo "$path changed after $unit last started — restarting it if it runs" + elif systemctl "$@" is-active --quiet "$unit"; then + echo "$path changed after $unit last started — restarting it" systemctl "$@" try-restart --no-block "$unit" fi } diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index a05e39ba..4659b928 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -773,7 +773,6 @@ let && lib.hasInfix "dir=/run/hyperhive-priv/attic" u.script && lib.hasInfix "install -d -m 0700 \"$dir\"" u.script && lib.hasInfix ''atomic_write_secret 0600 "" "$dir/public-key"'' u.script - && lib.hasInfix ''atomic_write_secret 0600 "" "$dir/netrc"'' u.script && lib.hasInfix "machine attic.t.local password" u.script && u.environment.BAO_CLIENT_CERT == leafUnit.environment.BAO_CLIENT_CERT && u.environment.BAO_CLIENT_KEY == leafUnit.environment.BAO_CLIENT_KEY @@ -879,9 +878,9 @@ let } { # The daemon only learns the cache from the fragment the root unit - # writes, and lists as trusted the one address from the host's - # configuration. A host with no unit has no include. - name = "the host's nix.conf includes a fragment that trusts only the swarm key, and only hive-core gets the token"; + # writes, from the address in the host's configuration. A host with no + # unit has no include. + name = "the host's nix.conf includes a fragment that gives the daemon the cache, the key and a root-only netrc that keeps the host's own entries"; ok = let u = units.hive-priv-cache-facts; @@ -890,23 +889,21 @@ let lib.hasInfix "!include /run/hyperhive-nix/cache.conf" allLocal.nix.extraOptions && !(lib.hasInfix "hyperhive-nix" noIdentity.nix.extraOptions) && lib.hasInfix "conf=/run/hyperhive-nix/cache.conf" u.script - && lib.hasInfix ''fragment="extra-trusted-public-keys = $key"'' u.script + # The fragment names the cache, the key and the merged netrc. + && lib.hasInfix "extra-substituters = https://attic.t.local/swarm" u.script + && lib.hasInfix "extra-trusted-public-keys = $key" u.script + && lib.hasInfix "netrc-file = $netrc_file" u.script + && lib.hasInfix "netrc_file=/run/hyperhive-nix/netrc" u.script + && lib.hasInfix ''atomic_write_secret 0600 "" "$netrc_file"'' u.script && lib.hasInfix ''atomic_write_secret 0644 "" "$conf"'' u.script - # The daemon is told no substituter and no netrc: no other client of it - # can fetch from the cache or send the token. - && !(lib.hasInfix "extra-trusted-substituters" u.script) - && !(lib.hasInfix "trusted-substituters" allLocal.nix.extraOptions) - && !(lib.hasInfix "netrc-file" u.script) - # The token's one unprivileged reader is the hive-core group. - && lib.hasInfix "client_netrc=/run/hyperhive-nix/netrc" u.script - && lib.hasInfix ''atomic_write_secret 0640 ":hive-core" "$client_netrc"'' u.script + # The host's own netrc is merged in: nix's default path when the host sets none. + && lib.hasInfix "host_netrc=/etc/nix/netrc" u.script + # No client copy of the token for hive-core, and hive-c0re is not told the cache. + && !(lib.hasInfix "client_netrc" u.script) + && !(c0re ? HIVE_C0RE_CACHE_URL) && lib.elem "d /run/hyperhive-nix 0755 root root - -" allLocal.systemd.tmpfiles.rules - # hive-c0re is told the address; hive-priv no longer is. - && c0re.HIVE_C0RE_CACHE_URL == "https://attic.t.local/swarm" - && !(noIdentity.systemd.services.hive-c0re.environment ? HIVE_C0RE_CACHE_URL) - # Control for the absences: the lookups find the units' other variables. + # Control for the absence: the lookup finds the unit's other variables. && c0re ? HOME - && noIdentity.systemd.services.hive-c0re.environment ? HOME && allLocal.systemd.services.hive-priv.environment ? NIX_REMOTE && !(allLocal.systemd.services.hive-priv.environment ? HIVE_PRIV_CACHE_URL); } diff --git a/nix/script-tests/attic-root-facts.nix b/nix/script-tests/attic-root-facts.nix index 9f4556a7..dea81c44 100644 --- a/nix/script-tests/attic-root-facts.nix +++ b/nix/script-tests/attic-root-facts.nix @@ -15,9 +15,10 @@ let hive = "alpha"; host = "cache.example"; dir = "@DIR@/attic"; + url = "https://cache.example/swarm"; confFile = "@DIR@/nix/cache.conf"; - clientNetrc = "@DIR@/nix/netrc"; - clientGroup = "@GROUP@"; + netrcFile = "@DIR@/nix/netrc"; + hostNetrc = "@DIR@/host-netrc"; } ); in diff --git a/nix/script-tests/attic-root-facts.sh b/nix/script-tests/attic-root-facts.sh index 263ce950..9179c12b 100644 --- a/nix/script-tests/attic-root-facts.sh +++ b/nix/script-tests/attic-root-facts.sh @@ -19,11 +19,7 @@ expect() { # dir=$(mktemp -d) mkdir -p "$dir/bin" -# A group other than the primary one when the runner has one, so the group the script sets -# differs from the one a new file gets. -group=$(id -Gn | tr " " "\n" | grep -vx "$(id -gn)" | head -1 || true) -group=${group:-$(id -gn)} -sed -e "s|@DIR@|$dir|g" -e "s|@GROUP@|$group|g" "$SCRIPT" >"$dir/script.sh" +sed -e "s|@DIR@|$dir|g" "$SCRIPT" >"$dir/script.sh" calls=$dir/calls export DIR=$dir @@ -51,13 +47,14 @@ STUB chmod +x "$dir/bin/bao" # `systemctl` answers for nix-daemon.service: DAEMON_STARTED is when it last -# became active (empty: never) and every restart request is logged. +# became active (empty: never, so it is not running) and every call is logged. cat >"$dir/bin/systemctl" <>"$dir/systemctl" case "\$1" in show) printf '%s\n' "\${DAEMON_STARTED-}" ;; is-failed) exit 1 ;; + is-active) [ -n "\${DAEMON_STARTED-}" ] ;; try-restart) ;; *) echo "unexpected call: \$*" >&2; exit 98 ;; esac @@ -69,7 +66,7 @@ KEYLEAF=secret_swarm_hives_alpha_attic_public-key TOKENLEAF=secret_swarm_hives_alpha_attic_pull-token reset() { - rm -rf "$dir/store" "$dir/attic" "$dir/nix" "$calls" "$dir/systemctl" + rm -rf "$dir/store" "$dir/attic" "$dir/nix" "$dir/host-netrc" "$calls" "$dir/systemctl" mkdir -p "$dir/store" : >"$calls" : >"$dir/systemctl" @@ -90,43 +87,41 @@ mode() { stat -c %a "$1" 2>/dev/null || echo missing; } ino() { stat -c %i "$1" 2>/dev/null || echo missing; } body() { cat "$1" 2>/dev/null || echo missing; } -case="first run writes the key and the netrc, each 0600, in a 0700 directory" +case="first run writes the key (0600, in a 0700 directory) and the netrc (0600)" reset seed "swarm:S0VZ" "tok-1" run expect "exit" $? 0 expect "key" "$(body "$dir/attic/public-key")" "swarm:S0VZ" -expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok-1" +expect "netrc" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1" expect "key mode" "$(mode "$dir/attic/public-key")" 600 -expect "netrc mode" "$(mode "$dir/attic/netrc")" 600 +expect "netrc mode" "$(mode "$dir/nix/netrc")" 600 expect "dir mode" "$(mode "$dir/attic")" 700 -expect "fragment is the key and nothing else" "$(body "$dir/nix/cache.conf")" "extra-trusted-public-keys = swarm:S0VZ" +expect "fragment names the cache, the key and the root-only netrc" "$(body "$dir/nix/cache.conf")" "extra-substituters = https://cache.example/swarm +extra-trusted-public-keys = swarm:S0VZ +netrc-file = $dir/nix/netrc" expect "fragment mode" "$(mode "$dir/nix/cache.conf")" 644 expect "fragment dir mode" "$(mode "$dir/nix")" 755 expect "no secret in the fragment" "$(grep -c tok-1 "$dir/nix/cache.conf")" 0 -expect "no substituter or netrc in the fragment" "$(grep -c -E "substituter|netrc" "$dir/nix/cache.conf")" 0 -expect "client netrc" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1" -expect "client netrc mode" "$(mode "$dir/nix/netrc")" 640 -expect "client netrc group" "$(stat -c %G "$dir/nix/netrc")" "$group" +expect "control: the token is in the netrc" "$(grep -c tok-1 "$dir/nix/netrc")" 1 +expect "the netrc the fragment names is root-only" "$(mode "$dir/nix/netrc")" 600 case="a second run with the same facts rewrites nothing" k=$(ino "$dir/attic/public-key") -n=$(ino "$dir/attic/netrc") -cn=$(ino "$dir/nix/netrc") +n=$(ino "$dir/nix/netrc") c=$(ino "$dir/nix/cache.conf") run expect "exit" $? 0 expect "key untouched" "$(ino "$dir/attic/public-key")" "$k" -expect "netrc untouched" "$(ino "$dir/attic/netrc")" "$n" +expect "netrc untouched" "$(ino "$dir/nix/netrc")" "$n" expect "fragment untouched" "$(ino "$dir/nix/cache.conf")" "$c" -expect "client netrc untouched" "$(ino "$dir/nix/netrc")" "$cn" case="a rotated token replaces the netrc and leaves the key" seed "swarm:S0VZ" "tok-2" run expect "exit" $? 0 -expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok-2" -expect "netrc mode" "$(mode "$dir/attic/netrc")" 600 +expect "netrc" "$(body "$dir/nix/netrc")" "machine cache.example password tok-2" +expect "netrc mode" "$(mode "$dir/nix/netrc")" 600 expect "key untouched" "$(ino "$dir/attic/public-key")" "$k" case="a rotated key replaces the key file" @@ -140,12 +135,71 @@ expect "reads" "$(grep -c '^bao kv get' "$calls")" "$(grep -c '^bao kv get -fiel expect "login first" "$(head -1 "$calls")" "bao login -method=cert -token-only" expect "something was read" "$([ "$(grep -c '^bao kv get' "$calls")" -ge 2 ] && echo yes || echo no)" yes +# The merged netrc: the cache's entry first, then the host's other entries as +# they are. A missing host file is no entries. +HOSTNETRC='machine git.example login bob password hunter2 +machine other.example + login alice + password s3cret' + +case="a host netrc with two other machines keeps both, after the cache's entry" +reset +seed "swarm:S0VZ" "tok-1" +printf '%s\n' "$HOSTNETRC" >"$dir/host-netrc" +run +expect "exit" $? 0 +expect "merged" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1 +$HOSTNETRC" +expect "mode" "$(mode "$dir/nix/netrc")" 600 +expect "the host's file is not written" "$(body "$dir/host-netrc")" "$HOSTNETRC" + +case="a host entry for the cache's own machine is replaced and the rest is kept" +reset +seed "swarm:S0VZ" "tok-1" +printf '%s\n%s\n' "$HOSTNETRC" "machine cache.example login old password stale" >"$dir/host-netrc" +run +expect "exit" $? 0 +expect "merged" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1 +$HOSTNETRC" +expect "control: the stale entry was in the host file" "$(grep -c stale "$dir/host-netrc")" 1 + +case="a multi-line host entry for the cache's machine goes whole" +reset +seed "swarm:S0VZ" "tok-1" +printf '%s\n%s\n' "machine cache.example + login old + password stale" "$HOSTNETRC" >"$dir/host-netrc" +run +expect "exit" $? 0 +expect "merged" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1 +$HOSTNETRC" + +case="an edit to the host netrc reaches the merged file on the next run" +reset +seed "swarm:S0VZ" "tok-1" +printf '%s\n' "$HOSTNETRC" >"$dir/host-netrc" +run +printf '%s\n' "machine new.example login n password p" >>"$dir/host-netrc" +run +expect "exit" $? 0 +expect "new entry merged" "$(grep -c new.example "$dir/nix/netrc")" 1 +expect "old entries kept" "$(grep -c git.example "$dir/nix/netrc")" 1 + +case="a netrc left 0640 by an earlier deploy is rewritten 0600 though its content is current" +reset +seed "swarm:S0VZ" "tok-1" +run +chmod 640 "$dir/nix/netrc" +run +expect "exit" $? 0 +expect "mode" "$(mode "$dir/nix/netrc")" 600 + case="a store with no facts exits cleanly and creates nothing" reset run expect "exit" $? 0 expect "no key" "$(body "$dir/attic/public-key")" missing -expect "no netrc" "$(body "$dir/attic/netrc")" missing +expect "no netrc" "$(body "$dir/nix/netrc")" missing case="one fact without the other creates nothing" reset @@ -153,7 +207,7 @@ printf '%s' "swarm:S0VZ" >"$dir/store/$KEYLEAF" run expect "exit" $? 0 expect "no key" "$(body "$dir/attic/public-key")" missing -expect "no netrc" "$(body "$dir/attic/netrc")" missing +expect "no netrc" "$(body "$dir/nix/netrc")" missing # The failure cases start from a good state and an extra file in the directory. good() { @@ -164,7 +218,7 @@ good() { } intact() { expect "key kept" "$(body "$dir/attic/public-key")" "swarm:S0VZ" - expect "netrc kept" "$(body "$dir/attic/netrc")" "machine cache.example password tok-1" + expect "netrc kept" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1" expect "other file kept" "$(body "$dir/attic/other")" "" } @@ -212,6 +266,16 @@ seed "swarm:S0VZ" "tok-1" run DAEMON_STARTED="Sat 2000-01-01 00:00:00.000000 UTC" expect "exit" $? 0 expect "restart asked" "$(grep -c '^systemctl try-restart --no-block nix-daemon.service' "$dir/systemctl")" 1 +expect "restart claimed" "$(grep -c "cache.conf changed after nix-daemon.service last started — restarting it\$" "$dir/out")" 1 + +case="a daemon that never ran is not restarted, and nothing claims it was" +reset +seed "swarm:S0VZ" "tok-1" +run DAEMON_STARTED= +expect "exit" $? 0 +expect "fragment written" "$(grep -c "^extra-substituters" "$dir/nix/cache.conf")" 1 +expect "no restart" "$(restarts)" 0 +expect "no change-vs-start claim" "$(grep -c "changed after" "$dir/out")" 0 case="the same facts leave the fragment and the daemon alone" started=$(now) @@ -225,9 +289,7 @@ started=$(now) seed "swarm:S0VZ" "tok-2" run DAEMON_STARTED="$started" expect "exit" $? 0 -expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok-2" -expect "client netrc follows" "$(body "$dir/nix/netrc")" "machine cache.example password tok-2" -expect "client netrc mode" "$(mode "$dir/nix/netrc")" 640 +expect "netrc" "$(body "$dir/nix/netrc")" "machine cache.example password tok-2" expect "no restart" "$(restarts)" 0 case="a rotated key rewrites the fragment and restarts the daemon" @@ -243,7 +305,6 @@ reset run DAEMON_STARTED="Sat 2000-01-01 00:00:00.000000 UTC" expect "exit" $? 0 expect "no fragment" "$(body "$dir/nix/cache.conf")" missing -expect "no client netrc" "$(body "$dir/nix/netrc")" missing expect "no restart" "$(restarts)" 0 case="a key that is refused writes no fragment" @@ -252,7 +313,6 @@ seed "swarm:S0 VZ" "tok-1" run expect "exit" $? 1 expect "no fragment" "$(body "$dir/nix/cache.conf")" missing -expect "no client netrc" "$(body "$dir/nix/netrc")" missing if [ "$failures" -ne 0 ]; then echo "$failures of $count checks failed" >&2 diff --git a/nix/script-tests/cache-pull-substitute.nix b/nix/script-tests/cache-pull-substitute.nix new file mode 100644 index 00000000..cb8d7bd5 --- /dev/null +++ b/nix/script-tests/cache-pull-substitute.nix @@ -0,0 +1,18 @@ +# `checks.script-test-cache-pull-substitute` — runs the closure pull +# (`nix-store --realise --max-jobs 0`, hive-c0re/src/cache_pull.rs) +# against two file caches: a swarm cache that has the top path and lacks a +# dependency, and the host's own substituter that has the dependency. The cases +# are in ./cache-pull-substitute.sh. +{ pkgs, ... }: +pkgs.runCommand "hyperhive-script-test-cache-pull-substitute" + { + nativeBuildInputs = [ + pkgs.nix + pkgs.coreutils + pkgs.gnugrep + ]; + } + '' + ${pkgs.bash}/bin/bash ${./cache-pull-substitute.sh} + touch "$out" + '' diff --git a/nix/script-tests/cache-pull-substitute.sh b/nix/script-tests/cache-pull-substitute.sh new file mode 100644 index 00000000..dab372d4 --- /dev/null +++ b/nix/script-tests/cache-pull-substitute.sh @@ -0,0 +1,95 @@ +# Cases for the closure pull (hive-c0re/src/cache_pull.rs): `nix-store --realise +# --max-jobs 0` against a store whose configuration is what the daemon +# has on a hive host, the host's own substituter plus the swarm cache from the +# nix.conf fragment. +# +# The swarm cache holds the top path and NOT its dependency, as attic does for +# a path cache.nixos.org serves; a second cache, standing for cache.nixos.org, +# holds the dependency. The stores are chroot stores, so the realise runs in +# this process where a host runs it in the daemon; the substitution logic is +# the same. +set -uo pipefail + +failures=0 +count=0 +case="setup" + +fail() { + echo "FAILED: $case: $*" >&2 + failures=$((failures + 1)) +} + +expect() { # + count=$((count + 1)) + if [ "$2" != "$3" ]; then fail "$1: got '$2', wanted '$3'"; fi +} + +T=$(mktemp -d) +export HOME=$T/home XDG_CONFIG_HOME=$T/home NIX_USER_CONF_FILES= +mkdir -p "$HOME" "$T/empty" +export NIX_CONF_DIR=$T/empty +printf 'build-users-group =\nsandbox = false\nexperimental-features = nix-command\n' >"$T/empty/nix.conf" + +src="local?root=$T/src" +dep=$(nix eval --raw --store "$src" --expr 'builtins.toFile "dep" "dependency"') +top=$(nix eval --raw --store "$src" --expr 'let dep = builtins.toFile "dep" "dependency"; in builtins.toFile "top" "needs ${dep}"') +nix-store --generate-binary-cache-key swarm-test-1 "$T/swarm.sec" "$T/swarm.pub" >/dev/null +nix-store --generate-binary-cache-key upstream-test-1 "$T/up.sec" "$T/up.pub" >/dev/null +attic=file://$T/attic +upstream=file://$T/upstream +nix copy --from "$src" --to "$attic?secret-key=$T/swarm.sec" "$top" +nix copy --from "$src" --to "$upstream?secret-key=$T/up.sec" "$dep" +# What attic leaves out because the upstream serves it. +rm -f "$T/attic/$(basename "$dep" | cut -c1-32).narinfo" + +# The daemon's nix.conf: its own substituter and key, then the include. +# conf : a NIX_CONF_DIR with that fragment. +conf() { + mkdir -p "$T/$1" + printf 'build-users-group =\nsandbox = false\nsubstituters = %s\ntrusted-public-keys = %s\n!include %s/cache.conf\n' \ + "$upstream" "$(cat "$T/up.pub")" "$T/$1" >"$T/$1/nix.conf" + printf '%s\n' "$2" >"$T/$1/cache.conf" +} +fragment="extra-substituters = $attic +extra-trusted-public-keys = $(cat "$T/swarm.pub")" +# The paths are content-addressed (`builtins.toFile`), which `require-sigs` does +# not apply to, so this test cannot show the signature gate; hive-priv's +# `nix store verify` step and the fragment test carry the key. + +# pull : the exact pull of cache_pull.rs. +pull() { + NIX_CONF_DIR=$T/$1 nix-store --store "local?root=$T/$2" --realise "$top" --max-jobs 0 >/dev/null 2>"$T/err" +} +valid() { # + nix-store --store "local?root=$T/$1" --check-validity "$2" 2>/dev/null && echo yes || echo no +} + +narinfo() { # : whether the cache has its narinfo + if [ -e "$T/$1/$(basename "$2" | cut -c1-32).narinfo" ]; then echo yes; else echo no; fi +} + +case="control: the swarm cache has the top path and lacks the dependency" +expect "attic has the top path" "$(narinfo attic "$top")" yes +expect "attic lacks the dependency" "$(narinfo attic "$dep")" no +expect "upstream has the dependency" "$(narinfo upstream "$dep")" yes + +case="the client-side copy that failed on a host fails here too" +if nix copy --from "$attic" --to "local?root=$T/d0" "$top" 2>"$T/err"; then + fail "nix copy --from the swarm cache alone succeeded" +fi +expect "no top path" "$(valid d0 "$top")" no + +case="the daemon with both substituters realises the top path and its dependency" +conf both "$fragment" +pull both d1 +expect "exit" $? 0 +expect "top path valid" "$(valid d1 "$top")" yes +expect "dependency valid" "$(valid d1 "$dep")" yes + +case="without the fragment the daemon has only the upstream and cannot realise the top path" +conf plain "" +if pull plain d2; then fail "realised without the swarm cache"; fi +expect "no top path" "$(valid d2 "$top")" no + +echo "$count checks, $failures failed" +[ "$failures" = 0 ] diff --git a/nix/script-tests/refresh-consumer.sh b/nix/script-tests/refresh-consumer.sh index 55b55293..1c2d7df9 100644 --- a/nix/script-tests/refresh-consumer.sh +++ b/nix/script-tests/refresh-consumer.sh @@ -26,13 +26,17 @@ cat >"$dir/bin/systemctl" <>"$calls" ;; esac STUB chmod +x "$dir/bin/systemctl" -export PATH="$dir/bin:$PATH" ACTIVE=1 FAILED=0 STARTED= +export PATH="$dir/bin:$PATH" ACTIVE=1 UNIT_ACTIVE=1 FAILED=0 STARTED= export LC_ALL=de_DE.UTF-8 @@ -106,10 +110,21 @@ check "a failed unit is reset and started" \ refresh_unit gitea-runner-swarm.service "$path" FAILED=0 +STARTED=$(systemd_ts 1791232888.193158) +got=$(refresh_unit gitea-runner-swarm.service "$path" 2>&1) +case="control: a running unit is said to be restarted" +count=$((count + 1)) +if [ "$got" != "$path changed after gitea-runner-swarm.service last started — restarting it" ]; then fail "logged: $got"; fi + STARTED= -check "a unit never active is restarted" \ - "try-restart --no-block gitea-runner-swarm.service" "" \ +UNIT_ACTIVE=0 +check "a unit never active is not asked to restart" "" "" \ refresh_unit gitea-runner-swarm.service "$path" +got=$(refresh_unit gitea-runner-swarm.service "$path" 2>&1) +case="a unit never active is not said to be restarted" +count=$((count + 1)) +if [ -n "$got" ]; then fail "logged: $got"; fi +UNIT_ACTIVE=1 ACTIVE=0 check "nothing while the container is down" "" "" \