diff --git a/docs/2-handbook/swarm/credentials-reference.md b/docs/2-handbook/swarm/credentials-reference.md index 14e3c795..e72e1b2a 100644 --- a/docs/2-handbook/swarm/credentials-reference.md +++ b/docs/2-handbook/swarm/credentials-reference.md @@ -151,16 +151,16 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](credentials. ### Hive-level — one of each per hive -| secret | generated by | lives at | -| ------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| hive leaf certs | `hive-tls.nix`, issued by the secret store's `pki` mount | `/.pem` | -| the hive's own store identity | `glue-bao-tls.nix` on the store's host; placed by hand on every other host ([`ca.md`](../../4-implementation/swarm/ca.md#the-gateway-leaf-comes-from-the-secret-store)) | `deploy.hive-controller.tls.baoHiveIdentityCertFile` / `baoHiveIdentityKeyFile` | -| the cache's public key and pull token, for the host's nix daemon and hive-priv | `swarm-controller` copies them to `swarm/hives//attic/{public-key,pull-token}` (see the cache-facts row above) | `hive-priv-cache-facts` (root) reads those two under the hive's own store identity every ten minutes and writes `/run/hyperhive-priv/attic/public-key` and `netrc` (tmpfs, root, each `0600`, directory `0700`); it also writes `/run/hyperhive-nix/cache.conf` (tmpfs, root, `0644`, no secret in it): the cache's address, the key and the netrc's path as nix.conf settings that the host's nix.conf includes. The cache's address comes from `services.hyperhive.swarm.attic.domain` and `cache`, not from the store. A store that holds no facts, or that doesn't answer, leaves the files as they stand; a hive that never got them has `hive-c0re` ask for a refresh and then fail the pull with `this hive has no cache facts yet`. `hive-core` reads only the fragment, for the address. | -| the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | -| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | -| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity for each forge read, and hands it to the `git` commands it runs. It keeps no copy on disk | -| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.deploy.hive-controller.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `unrestricted` beside it lists the users whose agent holds the swarm's `unrestricted_internet` flag, names only; it lists every agent the hive never read as off `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; an agent without one sends none, so it goes direct | -| the squid self-check's password | `swarm-controller` mints `swarm/hives//squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Both self-check users (`selfcheck.squid` and the always-unrestricted `selfcheck-unrestricted.squid`) use it, so whoever holds it has unrestricted egress through this squid Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks | +| secret | generated by | lives at | +| ------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| hive leaf certs | `hive-tls.nix`, issued by the secret store's `pki` mount | `/.pem` | +| the hive's own store identity | `glue-bao-tls.nix` on the store's host; placed by hand on every other host ([`ca.md`](../../4-implementation/swarm/ca.md#the-gateway-leaf-comes-from-the-secret-store)) | `deploy.hive-controller.tls.baoHiveIdentityCertFile` / `baoHiveIdentityKeyFile` | +| the cache's public key and pull token, for the host's nix daemon and hive-priv | `swarm-controller` copies them to `swarm/hives//attic/{public-key,pull-token}` (see the cache-facts row above) | `hive-priv-cache-facts` (root) reads those two under the hive's own store identity every ten minutes and writes `/run/hyperhive-priv/attic/public-key` and `netrc` (tmpfs, root, each `0600`, directory `0700`); it also writes `/run/hyperhive-nix/cache.conf` (tmpfs, root, `0644`, no secret in it): the key as `extra-trusted-public-keys`, which the host's nix.conf includes, with no substituter and no netrc; and `/run/hyperhive-nix/netrc` (`root:hive-core 0640`), the token as a netrc that `hive-c0re` passes to `nix copy --from` as the client's own setting. `hive-core` is the one unprivileged reader of the token, and no other client of the host's nix daemon gets it. The cache's address comes from `services.hyperhive.swarm.attic.domain` and `cache`, not from the store. A store that holds no facts, or that doesn't answer, leaves the files as they stand; a hive that never got them has `hive-c0re` ask for a refresh and then fail the pull with `this hive has no cache facts yet`. | +| the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | +| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | +| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity for each forge read, and hands it to the `git` commands it runs. It keeps no copy on disk | +| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.deploy.hive-controller.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `unrestricted` beside it lists the users whose agent holds the swarm's `unrestricted_internet` flag, names only; it lists every agent the hive never read as off `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; an agent without one sends none, so it goes direct | +| the squid self-check's password | `swarm-controller` mints `swarm/hives//squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Both self-check users (`selfcheck.squid` and the always-unrestricted `selfcheck-unrestricted.squid`) use it, so whoever holds it has unrestricted egress through this squid Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks | ### Store identities, one per principal diff --git a/docs/4-implementation/swarm/builder.md b/docs/4-implementation/swarm/builder.md index 17d44eed..9a7e9551 100644 --- a/docs/4-implementation/swarm/builder.md +++ b/docs/4-implementation/swarm/builder.md @@ -539,9 +539,13 @@ unit that reads the hive's own `attic/public-key` and `attic/pull-token` from the store under the hive's own store identity every ten minutes and writes them to `/run/hyperhive-priv/attic/` (`0700`, root; each file `0600`). It also writes the nix.conf fragment `/run/hyperhive-nix/cache.conf`, which the host's nix.conf -includes: the cache's address (the host's `services.hyperhive.swarm.attic.domain` -and `cache`) as a trusted substituter, the key as a trusted key and the netrc's -path. It restarts `nix-daemon` only when the fragment changed after the daemon +includes, with the key as `extra-trusted-public-keys` and nothing else: no +substituter and no netrc, so the daemon never contacts the cache and no other +client of the daemon, such as an agent on `nixBuilds = host`, can fetch from it. +A second copy of the netrc, `/run/hyperhive-nix/netrc` (`root:hive-core 0640`), +is the pull token `hive-c0re` uses. The cache's address is the host's +`services.hyperhive.swarm.attic.domain` and `cache`, handed to `hive-c0re`. The +unit restarts `nix-daemon` only when the fragment changed after the daemon started, which matters for CppNix; Lix re-reads nix.conf on every connection. A file `hive-c0re` writes decides nothing the daemon or `hive-priv` trusts. @@ -587,13 +591,12 @@ means check attic's fingerprint format; it says nothing about an attacker. A A hive builds nothing. A rebuild's `Prebuild` node, which runs while the agent is still up, reads the agent's record in `agent-closures` and has `hive-c0re` pull the record's store path. `hive-c0re` isn't root and needs no root: it runs -`nix-store --realise --max-jobs 0 --option substituters ` -through the host's nix daemon. The address is the one in the fragment above, the -only substituter the daemon lets an unprivileged client name. The daemon -ignores a client's keys, `require-sigs` and netrc, so it stores the path only -when a key it trusts signed it. When the fragment is missing, `hive-c0re` first -asks `hive-priv` to run `hive-priv-cache-facts` (up to 60 s) instead of waiting for -its timer. A host that still has no facts, a signature refusal and a path the +`nix copy --from --option netrc-file /run/hyperhive-nix/netrc ` +as `hive-core`. The nix client fetches from the cache with the pull token and +hands the NARs to the host's nix daemon, which stores each path only when a key +it trusts signed it; nothing builds. When the netrc or the key is missing, +`hive-c0re` first asks `hive-priv` to run `hive-priv-cache-facts` (up to 60 s) +instead of waiting for its timer. A host that still has no facts, a signature refusal and a path the cache lacks each fail with their own message. `Swap` then does the same lookup and pull, a no-op once the path is in the diff --git a/docs/4-implementation/trust-boundary/hive-c0re-privilege.md b/docs/4-implementation/trust-boundary/hive-c0re-privilege.md index 465d2dec..08720306 100644 --- a/docs/4-implementation/trust-boundary/hive-c0re-privilege.md +++ b/docs/4-implementation/trust-boundary/hive-c0re-privilege.md @@ -35,7 +35,7 @@ known operations; there is no arbitrary command pass-through: | `StartFailureLog` | `journalctl _SYSTEMD_UNIT=container@h-.service --since @ -n 30 -o cat --no-pager`, fixed argv, 5 s timeout, output capped at 8 KiB; hive-priv clamps `` to the last hour. hive-c0re appends the lines to the failed start's build log, which only the dashboard serves. | | `CreateContainer` / `UpdateContainer` | `nixos-container create --system-path `; for an update, `nix-env -p --set ` and a reload of the unit. `` must already be in the store with a signature by the swarm cache's key: hive-priv runs `nix store verify --no-contents --sigs-needed 1` with `trusted-public-keys` set to the key in root-owned `/run/hyperhive-priv/attic/public-key`, and refuses a path that's absent or that the key didn't sign. hive-priv builds nothing. The check doesn't bind the path to the agent whose pull produced it: hive-c0re could already pull any swarm-signed path and apply it to any agent, so the check bounds the same risk (one agent gets another's closure), not an arbitrary-path one | | `RealiseClosure` | Kept for one release, for an older hive-c0re that still asks hive-priv to download. Downloads nothing: answers `Ok` when the path is in the store and signed by the swarm key (what `CreateContainer` needs), else `not available: `. The path must be `/nix/store/<32 chars>-` and the agent name valid | -| `RefreshCacheFacts` | `systemctl start hive-priv-cache-facts.service`, waited for up to 60 s. The request names no unit and has no field. hive-c0re asks for it when its cache fragment is missing | +| `RefreshCacheFacts` | `systemctl start hive-priv-cache-facts.service`, waited for up to 60 s. The request names no unit and has no field. hive-c0re asks for it when its netrc or the swarm key in the fragment is missing | | `DestroyContainer` | `nixos-container destroy ` | | `ListContainers` | `nixos-container list` | | `ReloadGatewayNginx` | `systemctl reload/start/reset-failed nginx` (host unit; the unit name is hard-coded, not a parameter) | @@ -70,42 +70,59 @@ bind or credential path is absolute with no whitespace, control byte, `\`, `$`, backtick, `"` or `:`. The size limits of a value stay with systemd. **Closures from the swarm cache** — `hive-c0re` pulls the closure the swarm -built through the host's nix daemon, and `hive-priv` applies it only when the -swarm's key signed it. Pulling needs no root: `nix-store --realise` of a store -path is a request to the daemon. Root-owned sources alone decide what the -daemon accepts: +built with `nix copy --from `, and `hive-priv` applies it only +when the swarm's key signed it. Pulling needs no root. The nix client does the +HTTP fetch itself, as `hive-core`, with a netrc only `hive-core` can read, and +hands the NARs to the host's nix daemon, which stores a path from an +unprivileged client only if a key in its `trusted-public-keys` signed it. +Root-owned sources alone decide what the daemon accepts: - The cache signs. The swarm's cache (attic) holds the signing key and signs each narinfo it serves; the private key never leaves it. The builder pushes with a push token and holds no signing key. - The address is the host's configuration: - `services.hyperhive.swarm.attic.domain` and `cache`. The root unit - `hive-priv-cache-facts` lists it as `extra-trusted-substituters` in a nix.conf - fragment, `/run/hyperhive-nix/cache.conf`, that the host's nix.conf includes - (`!include`, so a host whose unit hasn't run yet still starts). -- The key and the pull token come from the secret store. The unit reads the - hive's own copies under the hive's own store identity and writes the key to - `/run/hyperhive-priv/attic/public-key` and the token to `netrc` there - (directory `0700`, files `0600`, root), and the key into the fragment as - `extra-trusted-public-keys` and the netrc's path as `netrc-file`. The fragment - holds no secret. `hive-c0re` reads its cache address, and nothing else, from - it; `hive-priv` reads nothing from `/run/hyperhive/attic/`, which `hive-c0re` - writes for the probe. -- The daemon enforces. `hive-core` is an unprivileged client: the daemon takes - its `--option substituters` only for an address in `trusted-substituters`, - ignores its `trusted-public-keys`, `require-sigs` and `netrc-file`, and - refuses a path no trusted key signed. `--max-jobs 0` means no answer, whatever - the cache sends, starts a build. + `services.hyperhive.swarm.attic.domain` and `cache`, handed to `hive-c0re` in + `HIVE_C0RE_CACHE_URL`. The daemon has no address: it never contacts the + cache. +- The key and the pull token come from the secret store. The root unit + `hive-priv-cache-facts` reads the hive's own copies under the hive's own + store identity and writes the key to `/run/hyperhive-priv/attic/public-key` + and the token to `netrc` there (directory `0700`, files `0600`, root). It + also writes the key into a nix.conf fragment, `/run/hyperhive-nix/cache.conf` + (`0644`), as `extra-trusted-public-keys`, which the host's nix.conf includes + (`!include`, so a host whose unit hasn't run yet still starts). The fragment + holds no secret, names no substituter and no netrc. `hive-priv` reads nothing + from `/run/hyperhive/attic/`, which `hive-c0re` writes for the probe. +- `hive-core` holds the pull token. The unit writes a second copy of the netrc, + `/run/hyperhive-nix/netrc`, `root:hive-core 0640`, and `hive-c0re` passes it + to `nix copy` as the client's `--option netrc-file`. This is a deliberate + grant, and no new capability: `hive-c0re` could already have `hive-priv` + download any swarm-signed path for any agent and apply it (the + `CreateContainer` row above), and the token reads only the cache the swarm + signs. The daemon ignores a client's `netrc-file`. +- The daemon enforces. It refuses a path from an unprivileged client unless a + trusted key signed it; a client can't lower `require-sigs` or add keys. + `nix copy` imports and never builds. - `hive-priv` verifies. Before `CreateContainer` or `UpdateContainer` applies a toplevel, it runs `nix store verify --no-contents --sigs-needed 1` with `trusted-public-keys` set to the key in the root file, and refuses a path that is absent or that the key didn't sign. The daemon's trusted keys also include the host's other caches, so this check is the one that counts only the swarm's. -- When the fragment is missing, `hive-c0re` asks `hive-priv` for `RefreshCacheFacts`, - which starts the facts unit and waits up to 60 s, instead of waiting for the - unit's timer. A host the swarm hasn't given facts to answers +- When the netrc or the key is missing, `hive-c0re` asks `hive-priv` for + `RefreshCacheFacts`, which starts the facts unit and waits up to 60 s, instead + of waiting for the unit's timer. A host the swarm hasn't given facts to answers `this hive has no cache facts yet`. +**What an agent with daemon access can fetch.** Agent containers on +`nixBuilds = host` reach the same daemon ([nix-builds.md](nix-builds.md)). They +get nothing from the private cache: the daemon has no trusted substituter and no +netrc for it, an agent can't read `/run/hyperhive-nix/netrc` (`hive-core`'s +group) or `/run/hyperhive-priv/attic/`, and an unprivileged client can't make +the daemon use a substituter or a netrc it didn't configure. The one new thing +every client of the daemon shares is the swarm's public key in +`trusted-public-keys`: the daemon takes a path the swarm key signed from any source, +and only the swarm can produce one. + What a compromised `hive-c0re` can do: pull any store path that the swarm cache signed, which fills the store with paths the swarm vouches for, and then ask `hive-priv` to apply any such path to any agent. Before this change it could diff --git a/docs/4-implementation/trust-boundary/nix-builds.md b/docs/4-implementation/trust-boundary/nix-builds.md index 82fec20f..d98fb530 100644 --- a/docs/4-implementation/trust-boundary/nix-builds.md +++ b/docs/4-implementation/trust-boundary/nix-builds.md @@ -56,6 +56,14 @@ next start. `none` cuts the daemon only. It leaves the agent's egress alone. +Under `host` an agent shares the daemon with the hive's own closure pull, and +gains nothing from it. The daemon trusts the swarm cache's public key, but it +has no cache address and no pull token (hive-c0re fetches with a netrc readable +only by `hive-core`), so an agent can't fetch from the private cache. It can +still ask the daemon to substitute from the public caches the host configures, as +before. See +[hive-c0re-privilege.md](hive-c0re-privilege.md#closures-from-the-swarm-cache). + ## Threat model Unsandboxed nix builds run as `nixbld` users (non-root; `nrBuildUsers` is at least 32, so UIDs 30001 and up). diff --git a/hive-c0re/src/cache_pull.rs b/hive-c0re/src/cache_pull.rs index 173b98a5..09fc44df 100644 --- a/hive-c0re/src/cache_pull.rs +++ b/hive-c0re/src/cache_pull.rs @@ -1,31 +1,36 @@ -//! Pull the swarm's closure into the store through the host's nix daemon. +//! Pull the swarm's closure into the store with `nix copy --from`. //! -//! hive-c0re is not root and does not need to be: `nix-store --realise` of a -//! store path is a request to the daemon, which substitutes it. What the daemon -//! accepts is root's: the root unit `hive-priv-cache-facts` writes a nix.conf -//! fragment ([`CONF_FRAGMENT`]) that lists the swarm cache as a -//! `trusted-substituter`, adds its key to `trusted-public-keys` and names the -//! netrc. An unprivileged client may name a substituter only from that list, -//! cannot lower `require-sigs` or replace the keys, and a path without a -//! signature by a trusted key is refused. hive-priv then applies the toplevel -//! only if the swarm's key signed it. +//! hive-c0re is not root and does not need to be. The nix client does the HTTP +//! fetch itself, with a netrc only hive-core can read ([`NETRC`]), and hands +//! the NARs to the host's nix daemon, which stores a path from an unprivileged +//! client only if a key in its `trusted-public-keys` signed it. The root unit +//! `hive-priv-cache-facts` puts the swarm's key there through a nix.conf +//! fragment ([`CONF_FRAGMENT`]) that names no substituter and no netrc, so the +//! daemon itself never contacts the cache and no other client of it can fetch +//! from the cache or send the token. hive-priv then applies the toplevel only +//! if the swarm's key signed it. //! -//! The cache's address is read from the fragment, so the address named is -//! byte for byte the one the daemon trusts. A missing fragment means this host -//! has not been given its cache facts; [`pull`] first asks hive-priv to run the -//! facts unit, which is how the swarm's copy of the facts reaches the host -//! without waiting for the unit's timer. +//! The cache's address is the host's configuration, handed to the unit in +//! [`URL_ENV`]. [`pull`] first asks hive-priv to run the facts unit when the +//! netrc or the key is not on the host yet, which is how the swarm's copy of the +//! facts reaches the host without waiting for the unit's timer. use std::time::Duration; use anyhow::Result; use hive_priv_sock::valid_closure_path; +/// The pull token as a netrc, `root:hive-core 0640`. +pub const NETRC: &str = "/run/hyperhive-nix/netrc"; + /// The nix.conf fragment `hive-priv-cache-facts` writes (root-owned, readable). pub const CONF_FRAGMENT: &str = "/run/hyperhive-nix/cache.conf"; -/// The fragment's line that lists the cache the daemon lets a client name. -const URL_SETTING: &str = "extra-trusted-substituters"; +/// The unit environment variable naming the cache: `https:///`. +const URL_ENV: &str = "HIVE_C0RE_CACHE_URL"; + +/// The fragment's line that gives the daemon the swarm's key. +const KEY_SETTING: &str = "extra-trusted-public-keys"; /// Wall-clock bound on one pull. A stalled transfer is bounded by nix's own /// `stalled-download-timeout`; this covers the rest. @@ -34,10 +39,10 @@ const PULL_TIMEOUT: Duration = Duration::from_mins(30); /// Why a closure was not pulled. #[derive(Debug, PartialEq, Eq)] pub enum PullError { - /// This host has no cache facts: no fragment, no address in it, or a - /// daemon that has not loaded it. Waiting for the facts unit fixes it. + /// This host has no cache facts: no address, no netrc for hive-core, or no + /// key in the fragment. Waiting for the facts unit fixes the last two. FactsMissing(String), - /// Nix refused the path: no signature by a key this host trusts. + /// The daemon refused the path: no signature by a key it trusts. SignatureRefused(String), /// The cache does not have the path. NotFound(String), @@ -56,18 +61,18 @@ impl std::fmt::Display for PullError { ), Self::SignatureRefused(last) => write!( f, - "nix refused the closure, which carries no signature by a key this host trusts: {last}" + "the nix daemon refused the closure, which carries no signature by a key this host trusts: {last}" ), Self::NotFound(last) => write!(f, "the swarm cache does not have the closure: {last}"), Self::Failed(last) => f.write_str(last), - Self::TimedOut => write!(f, "nix-store timed out after {}s", PULL_TIMEOUT.as_secs()), + Self::TimedOut => write!(f, "nix copy timed out after {}s", PULL_TIMEOUT.as_secs()), } } } impl std::error::Error for PullError {} -/// What a `nix-store` run came to. +/// What a `nix copy` run came to. pub enum Run { Exited { success: bool, stderr: String }, TimedOut, @@ -75,68 +80,63 @@ pub enum Run { /// What [`pull`] does on the host, so a test can stand in for it. pub trait Effects: Sync { - /// The fragment's text; `None` when it cannot be read. - async fn fragment(&self) -> Option; + /// The cache's address, when the host names one. + fn cache_url(&self) -> Option; + /// Whether hive-core can read the netrc and the fragment gives the daemon + /// the swarm's key. + async fn facts_present(&self) -> bool; /// Run the facts unit through hive-priv and wait for it. async fn refresh(&self) -> Result<()>; - /// Run `nix-store` with `args`, bounded by [`PULL_TIMEOUT`]. - async fn nix_store(&self, args: Vec) -> std::io::Result; + /// Run `nix` with `args`, bounded by [`PULL_TIMEOUT`]. + async fn nix(&self, args: Vec) -> std::io::Result; } -/// The cache's address in `fragment`: the value of its -/// `extra-trusted-substituters` line, when that is one https address. +/// Whether `fragment` has a non-empty `extra-trusted-public-keys` line. #[must_use] -pub fn cache_url(fragment: &str) -> Option<&str> { - fragment.lines().find_map(|line| { - let (name, value) = line.split_once('=')?; - let value = value.trim(); - (name.trim() == URL_SETTING - && value.starts_with("https://") - && !value.contains(char::is_whitespace)) - .then_some(value) +pub fn gives_key(fragment: &str) -> bool { + fragment.lines().any(|line| { + line.split_once('=') + .is_some_and(|(name, value)| name.trim() == KEY_SETTING && !value.trim().is_empty()) }) } -/// The `nix-store` arguments that fetch `store_path` and nothing else. +/// The `nix` arguments that fetch `store_path` from the cache at `url`. /// -/// - `--option substituters ` names the one cache the daemon lists as -/// trusted. An unprivileged client's `substituters` that is not on that list -/// empties the list and fails the pull, so the address must be the fragment's. -/// - `--max-jobs 0` forbids a local build, so a path the cache lacks fails. -/// - No `trusted-public-keys`, `require-sigs` or `netrc-file`: the daemon -/// ignores them from an unprivileged client and applies its own. +/// - `copy --from ` makes the client fetch. The daemon is never asked to +/// contact the cache, so it needs no trust in the address. +/// - `--option netrc-file` is the client's own setting: it reads the file as +/// hive-core. The daemon ignores the setting from an unprivileged client. +/// - No `--no-check-sigs`: the daemon checks every path's signature against its +/// trusted keys. Nothing builds: `copy` only imports. #[must_use] -pub fn pull_args(store_path: &str, url: &str) -> Vec { +pub fn pull_args(store_path: &str, url: &str, netrc: &str) -> Vec { [ - "--realise", - store_path, - "--max-jobs", - "0", - "--option", - "substituters", + "--extra-experimental-features", + "nix-command", + "copy", + "--from", url, + "--option", + "netrc-file", + netrc, + store_path, ] .map(str::to_owned) .into() } -/// Name nix's refusal from its stderr. A refusal for want of a signature -/// or a daemon that does not trust the address is told apart from a path the -/// cache lacks and from a cache that is down. +/// Name nix's refusal from its stderr. A refusal for want of a signature is +/// told apart from a path the cache lacks and from a cache that is down. fn classify(stderr: &str) -> PullError { let last = stderr .lines() .rev() .find(|l| !l.trim().is_empty()) - .unwrap_or("nix-store failed with no message") + .unwrap_or("nix copy failed with no message") .trim() .to_owned(); let lower = stderr.to_lowercase(); - if lower.contains("untrusted substituter") { - PullError::FactsMissing( - "the nix daemon does not list the swarm cache as a trusted substituter".to_owned(), - ) - } else if lower.contains("not signed by any of the keys") || lower.contains("signature") { + if lower.contains("signature") { PullError::SignatureRefused(last) } else if lower.contains("don't know how to build") || lower.contains("no substituter that can build") @@ -150,7 +150,7 @@ fn classify(stderr: &str) -> PullError { /// Pull `store_path` into the store. /// /// # Errors -/// [`PullError`]: no cache facts on this host (after one refresh), nix +/// [`PullError`]: no cache facts on this host (after one refresh), the daemon /// refusing the signature, the cache lacking the path, nix failing otherwise, /// or a timeout. pub async fn pull(store_path: &str) -> Result<(), PullError> { @@ -163,11 +163,14 @@ async fn pull_with(fx: &impl Effects, store_path: &str) -> Result<(), PullError> "{store_path:?} is not a store path" ))); } - let url = address_after_refresh(fx).await?; + let url = fx + .cache_url() + .ok_or_else(|| PullError::FactsMissing("this host names no cache".to_owned()))?; + ensure_facts(fx).await?; match fx - .nix_store(pull_args(store_path, &url)) + .nix(pull_args(store_path, &url, NETRC)) .await - .map_err(|e| PullError::Failed(format!("running nix-store: {e}")))? + .map_err(|e| PullError::Failed(format!("running nix: {e}")))? { Run::Exited { success: true, .. } => Ok(()), Run::Exited { stderr, .. } => Err(classify(&stderr)), @@ -175,43 +178,44 @@ async fn pull_with(fx: &impl Effects, store_path: &str) -> Result<(), PullError> } } -/// The cache's address, after one facts refresh when the host has none. -async fn address_after_refresh(fx: &impl Effects) -> Result { - if let Some(url) = address(fx).await { - return Ok(url); +/// The facts are on the host, after one refresh when they are not. +async fn ensure_facts(fx: &impl Effects) -> Result<(), PullError> { + if fx.facts_present().await { + return Ok(()); } let refreshed = fx.refresh().await; - address(fx).await.ok_or_else(|| { - PullError::FactsMissing(match refreshed { - Ok(()) => "the facts unit ran and the swarm has given this hive none".to_owned(), - Err(e) => format!("refreshing them failed: {e:#}"), - }) - }) + if fx.facts_present().await { + return Ok(()); + } + Err(PullError::FactsMissing(match refreshed { + Ok(()) => "the facts unit ran and the swarm has given this hive none".to_owned(), + Err(e) => format!("refreshing them failed: {e:#}"), + })) } -async fn address(fx: &impl Effects) -> Option { - fx.fragment() - .await - .as_deref() - .and_then(cache_url) - .map(str::to_owned) -} - -/// The host: the fragment on disk, hive-priv for the refresh, `nix-store` from -/// the system profile through the daemon. +/// The host: the environment and files on disk, hive-priv for the refresh, +/// `nix` from the system profile. struct Real; impl Effects for Real { - async fn fragment(&self) -> Option { - tokio::fs::read_to_string(CONF_FRAGMENT).await.ok() + fn cache_url(&self) -> Option { + std::env::var(URL_ENV) + .ok() + .map(|u| u.trim().to_owned()) + .filter(|u| u.starts_with("https://") && !u.contains(char::is_whitespace)) + } + + async fn facts_present(&self) -> bool { + let fragment = tokio::fs::read_to_string(CONF_FRAGMENT).await; + fragment.is_ok_and(|f| gives_key(&f)) && tokio::fs::File::open(NETRC).await.is_ok() } async fn refresh(&self) -> Result<()> { crate::priv_client::refresh_cache_facts().await } - async fn nix_store(&self, args: Vec) -> std::io::Result { - let mut cmd = tokio::process::Command::new("nix-store"); + async fn nix(&self, args: Vec) -> std::io::Result { + let mut cmd = tokio::process::Command::new("nix"); // Explicit, so the store is the daemon's whatever the environment. cmd.args(args) .env("NIX_REMOTE", "daemon") @@ -240,27 +244,23 @@ mod tests { const PATH: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas"; const URL: &str = "https://cache.example/swarm"; - fn fragment(url: &str) -> String { - format!( - "extra-trusted-substituters = {url}\nextra-trusted-public-keys = swarm:AAAA\nnetrc-file = /run/hyperhive-priv/attic/netrc" - ) - } - - /// A host that answers from a script: the fragment as it is now (and the - /// one a refresh leaves), what `nix-store` comes to, and a log of calls. + /// A host that answers from a script: whether the facts are present now + /// (and after a refresh), what `nix` comes to, and a log of calls. struct Fake { - fragment: Mutex>, - after_refresh: Option, + url: Option<&'static str>, + present: Mutex, + present_after_refresh: bool, refresh_fails: bool, nix: Mutex>, calls: Mutex>, } impl Fake { - fn new(fragment: Option, nix: Run) -> Self { + fn new(present: bool, nix: Run) -> Self { Self { - fragment: Mutex::new(fragment), - after_refresh: None, + url: Some(URL), + present: Mutex::new(present), + present_after_refresh: true, refresh_fails: false, nix: Mutex::new(Some(nix)), calls: Mutex::new(Vec::new()), @@ -273,9 +273,13 @@ mod tests { } impl Effects for Fake { - async fn fragment(&self) -> Option { - self.calls.lock().unwrap().push("fragment".to_owned()); - self.fragment.lock().unwrap().clone() + fn cache_url(&self) -> Option { + self.url.map(str::to_owned) + } + + async fn facts_present(&self) -> bool { + self.calls.lock().unwrap().push("facts".to_owned()); + *self.present.lock().unwrap() } async fn refresh(&self) -> Result<()> { @@ -283,15 +287,15 @@ mod tests { if self.refresh_fails { anyhow::bail!("hive-priv unreachable"); } - *self.fragment.lock().unwrap() = self.after_refresh.clone(); + *self.present.lock().unwrap() = self.present_after_refresh; Ok(()) } - async fn nix_store(&self, args: Vec) -> std::io::Result { + async fn nix(&self, args: Vec) -> std::io::Result { self.calls .lock() .unwrap() - .push(format!("nix-store {}", args.join(" "))); + .push(format!("nix {}", args.join(" "))); Ok(self.nix.lock().unwrap().take().expect("one run")) } } @@ -303,87 +307,106 @@ mod tests { } } - fn option<'a>(args: &'a [String], name: &str) -> Option<&'a str> { - args.windows(3) - .find(|w| w[0] == "--option" && w[1] == name) - .map(|w| w[2].as_str()) + fn after(args: &[String], flag: &str) -> Option { + args.iter() + .position(|a| a == flag) + .and_then(|i| args.get(i + 1)) + .cloned() } - /// An unprivileged client can name only the trusted address and cannot set - /// keys, signatures or the netrc, so the command carries exactly that, and - /// never lets nix build. + /// The client fetches: `copy --from `, with its own netrc as a client + /// setting. Nothing asks the daemon to contact the cache or to skip the + /// signature check, and nothing can build. #[test] - fn the_command_names_only_the_swarm_cache_and_never_builds() { - let args = pull_args(PATH, URL); - assert_eq!(args[..2], ["--realise", PATH]); - let jobs = args.iter().position(|a| a == "--max-jobs").expect("flag"); - assert_eq!(args[jobs + 1], "0"); - assert_eq!(option(&args, "substituters"), Some(URL)); - let options: Vec<_> = args.iter().filter(|a| *a == "--option").collect(); - assert_eq!(options.len(), 1, "{args:?}"); - for forbidden in ["trusted-public-keys", "require-sigs", "netrc-file"] { - assert_eq!(option(&args, forbidden), None, "{forbidden}"); + fn the_command_copies_from_the_cache_with_the_clients_netrc_and_checks_signatures() { + let args = pull_args(PATH, URL, NETRC); + assert_eq!(args.last().map(String::as_str), Some(PATH)); + let at = args.iter().position(|a| a == "copy").expect("verb"); + assert_eq!(args[at + 1], "--from"); + assert_eq!(args[at + 2], URL); + assert_eq!(after(&args, "--option").as_deref(), Some("netrc-file")); + assert_eq!( + args.windows(3) + .find(|w| w[0] == "--option" && w[1] == "netrc-file") + .map(|w| w[2].as_str()), + Some(NETRC) + ); + // Exactly one setting, and none that widens trust. + assert_eq!( + args.iter().filter(|a| *a == "--option").count(), + 1, + "{args:?}" + ); + for forbidden in [ + "--no-check-sigs", + "--substitute-on-destination", + "--to", + "substituters", + "trusted-public-keys", + "require-sigs", + ] { + assert!( + !args.iter().any(|a| a == forbidden), + "{forbidden}: {args:?}" + ); } assert!(!args.iter().any(|a| a.starts_with("extra-")), "{args:?}"); } #[test] - fn the_address_is_the_fragments_https_line() { - assert_eq!(cache_url(&fragment(URL)), Some(URL)); + fn the_fragment_gives_the_key_when_it_has_a_value() { + assert!(gives_key("extra-trusted-public-keys = swarm:AAAA")); for bad in [ - String::new(), - "netrc-file = /x".to_owned(), - fragment("http://cache.example/swarm"), - fragment("https://cache.example/a b"), - fragment(""), - // The key's line is not the address's. - "extra-trusted-public-keys = https://cache.example/swarm".to_owned(), + "", + "extra-trusted-public-keys =", + "extra-trusted-public-keys", + "trusted-public-keys = swarm:AAAA", + "extra-trusted-substituters = https://cache.example/swarm", ] { - assert_eq!(cache_url(&bad), None, "{bad:?}"); + assert!(!gives_key(bad), "{bad:?}"); } } #[tokio::test] - async fn with_the_fragment_in_place_it_pulls_without_a_refresh() { - let fx = Fake::new(Some(fragment(URL)), exited(true, "")); + async fn with_the_facts_in_place_it_pulls_without_a_refresh() { + let fx = Fake::new(true, exited(true, "")); pull_with(&fx, PATH).await.expect("pulled"); assert_eq!( fx.calls(), [ - "fragment".to_owned(), - format!("nix-store {}", pull_args(PATH, URL).join(" ")) + "facts".to_owned(), + format!("nix {}", pull_args(PATH, URL, NETRC).join(" ")) ] ); } - /// Missing facts are refreshed once, and the pull then goes ahead with the - /// address the refresh wrote. + /// Missing facts are refreshed once, and the pull then goes ahead. #[tokio::test] - async fn a_missing_fragment_is_refreshed_before_the_pull() { - let mut fx = Fake::new(None, exited(true, "")); - fx.after_refresh = Some(fragment(URL)); + async fn missing_facts_are_refreshed_before_the_pull() { + let fx = Fake::new(false, exited(true, "")); pull_with(&fx, PATH).await.expect("pulled"); assert_eq!( fx.calls(), [ - "fragment".to_owned(), + "facts".to_owned(), "refresh".to_owned(), - "fragment".to_owned(), - format!("nix-store {}", pull_args(PATH, URL).join(" ")) + "facts".to_owned(), + format!("nix {}", pull_args(PATH, URL, NETRC).join(" ")) ] ); } - /// Facts that are still absent after the refresh, whether the unit ran or - /// hive-priv could not start it, stop the pull before nix is asked. + /// Facts still absent after the refresh, whether the unit ran or hive-priv + /// could not start it, stop the pull before nix is asked. #[tokio::test] async fn facts_still_missing_after_the_refresh_are_reported_and_nix_is_not_run() { for (case, refresh_fails, want) in [ ("unit ran", false, "none"), ("refresh failed", true, "hive-priv unreachable"), ] { - let mut fx = Fake::new(None, exited(true, "")); + let mut fx = Fake::new(false, exited(true, "")); fx.refresh_fails = refresh_fails; + fx.present_after_refresh = false; let err = pull_with(&fx, PATH).await.expect_err(case); let PullError::FactsMissing(why) = &err else { panic!("{case}: {err:?}"); @@ -393,44 +416,39 @@ mod tests { format!("{err}").contains("a later deploy retries"), "{case}" ); - assert!( - !fx.calls().iter().any(|c| c.starts_with("nix-store")), - "{case}" - ); + assert!(!fx.calls().iter().any(|c| c.starts_with("nix")), "{case}"); } } + /// A host that names no cache has nothing to refresh toward. #[tokio::test] - async fn a_fragment_without_an_address_is_refreshed_like_a_missing_one() { - let mut fx = Fake::new(Some("netrc-file = /x".to_owned()), exited(true, "")); - fx.after_refresh = Some(fragment(URL)); - pull_with(&fx, PATH).await.expect("pulled"); - assert!(fx.calls().contains(&"refresh".to_owned())); + async fn a_host_that_names_no_cache_does_not_pull_or_refresh() { + let mut fx = Fake::new(true, exited(true, "")); + fx.url = None; + let err = pull_with(&fx, PATH).await.expect_err("no cache"); + assert!(matches!(err, PullError::FactsMissing(_)), "{err:?}"); + assert!(fx.calls().is_empty(), "{:?}", fx.calls()); } #[test] fn nixs_refusals_are_told_apart() { for (stderr, want) in [ ( - "warning: ignoring substitute for '/nix/store/x' from 'https://c', as it's not signed by any of the keys in 'trusted-public-keys'\nerror: build of '/nix/store/x' failed", + "error: cannot add path '/nix/store/x' because it lacks a signature by a trusted key", "signature", ), ( - "error: cannot add path 'x' because it lacks a signature by a trusted key", - "signature", - ), - ( - "warning: ignoring untrusted substituter 'https://c', you are not a trusted user.\ndon't know how to build these paths:\nerror: build of 'x' failed", - "facts", - ), - ( - "don't know how to build these paths:\n /nix/store/x\nerror: build of '/nix/store/x' failed", + "don't know how to build these paths:\n /nix/store/x\nerror: path '/nix/store/x' is required, but there is no substituter that can build it", "not found", ), ( "error: path '/nix/store/x' is required, but there is no substituter that can build it", "not found", ), + ( + "error: unable to download 'https://c/nix-cache-info': HTTP error 401", + "other", + ), ( "error: unable to download 'https://c/x.narinfo': Couldn't connect to server", "other", @@ -438,10 +456,9 @@ mod tests { ] { let kind = match classify(stderr) { PullError::SignatureRefused(_) => "signature", - PullError::FactsMissing(_) => "facts", PullError::NotFound(_) => "not found", PullError::Failed(_) => "other", - PullError::TimedOut => "timeout", + PullError::FactsMissing(_) | PullError::TimedOut => "unexpected", }; assert_eq!(kind, want, "{stderr}"); } @@ -455,17 +472,17 @@ mod tests { #[tokio::test] async fn a_failed_run_is_mapped_and_a_timeout_is_a_timeout() { let fx = Fake::new( - Some(fragment(URL)), + true, exited( false, - "warning: ignoring substitute from 'https://c', as it's not signed by any of the keys", + "error: cannot add path 'x' because it lacks a signature by a trusted key", ), ); assert!(matches!( pull_with(&fx, PATH).await, Err(PullError::SignatureRefused(_)) )); - let fx = Fake::new(Some(fragment(URL)), Run::TimedOut); + let fx = Fake::new(true, Run::TimedOut); assert_eq!(pull_with(&fx, PATH).await, Err(PullError::TimedOut)); } @@ -473,7 +490,7 @@ mod tests { #[tokio::test] async fn only_a_store_path_is_pulled() { for bad in ["", "main", "/etc/passwd", "--help", "/nix/store/x"] { - let fx = Fake::new(Some(fragment(URL)), exited(true, "")); + let fx = Fake::new(true, exited(true, "")); assert!( matches!(pull_with(&fx, bad).await, Err(PullError::Failed(_))), "{bad}" diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index d36172bf..609d8adb 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -289,7 +289,7 @@ pub enum PrivRequest { }, /// Kept for an older hive-c0re that still asks hive-priv to download: - /// hive-c0re now pulls through the nix daemon and hive-priv downloads + /// hive-c0re now pulls with `nix copy --from` and hive-priv downloads /// nothing. The answer is `Ok` when `store_path` is already in the store /// and signed by the swarm cache's key (what `CreateContainer` and /// `UpdateContainer` require), else an error that starts `not available: `. @@ -302,8 +302,9 @@ pub enum PrivRequest { /// `systemctl start hive-priv-cache-facts.service`, bounded at 60 s: the /// root unit that reads this hive's cache facts from the store and writes - /// the nix.conf fragment the daemon trusts. hive-c0re asks for it when the - /// fragment is missing. The request names no unit. + /// the swarm's key into the nix.conf fragment the daemon trusts and the pull + /// token into the netrc hive-c0re reads. hive-c0re asks for it when either is + /// missing. The request names no unit. RefreshCacheFacts, /// `nixos-container destroy ` diff --git a/hive-priv/src/closure.rs b/hive-priv/src/closure.rs index 6ded92cc..7d4cdaf4 100644 --- a/hive-priv/src/closure.rs +++ b/hive-priv/src/closure.rs @@ -183,7 +183,7 @@ pub(crate) async fn realise_closure(agent: &str, store_path: &str) -> Result<(St match check_signed(&RealSpawn, Path::new(ROOT_FACTS_DIR), 0, store_path).await { Ok(()) => Ok(("present".to_owned(), String::new())), Err(e) => bail!( - "not available: {e:#}; hive-priv no longer downloads, hive-c0re pulls through the nix daemon" + "not available: {e:#}; hive-priv no longer downloads, hive-c0re pulls with nix copy" ), } } diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 61ba6ac2..3132021d 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -316,7 +316,7 @@ in # (/run/hyperhive), and ReadWritePaths below, which systemd keeps # writable. Beyond the managed directories: # - the one nix call is the closure pull (src/cache_pull.rs): - # `nix-store --realise` with NIX_REMOTE=daemon, so the host daemon + # `nix copy --from` with NIX_REMOTE=daemon, so the host daemon # writes the store — hive-c0re never writes to /nix itself. It # connects to the daemon's socket, which RestrictAddressFamilies # (deferred above) must keep allowing. diff --git a/nix/host-modules/hive-priv-cache.nix b/nix/host-modules/hive-priv-cache.nix index 3cdb8318..bbc2401f 100644 --- a/nix/host-modules/hive-priv-cache.nix +++ b/nix/host-modules/hive-priv-cache.nix @@ -1,21 +1,27 @@ -# The binary-cache facts the host's nix daemon and hive-priv (root) trust, kept -# where only root can change them. +# The binary-cache facts the host's nix daemon, hive-priv (root) and hive-c0re +# use, kept where only root can change them. # -# hive-c0re downloads an agent's closure from the swarm cache through the -# host's nix daemon, and hive-priv applies it only if the swarm's key signed -# it. What decides whether a download is accepted — which cache the daemon -# lets an unprivileged client name, and which key must have signed it — must -# not come from a file the unprivileged hive-c0re writes, or a compromised -# hive-c0re could have root accept a closure from a cache of its own. So: +# hive-c0re pulls an agent's closure from the swarm cache with its own +# `nix copy --from`: the client does the HTTP fetch with the pull token, and +# hands the NAR to the host's nix daemon, which stores it only if a key in its +# `trusted-public-keys` signed it. hive-priv applies a closure only if the +# swarm's key signed it. What decides that must not come from a file the +# unprivileged hive-c0re writes. So: # # - the cache's address is this host's configuration -# ({option}`services.hyperhive.swarm.attic.domain` and `cache`); +# ({option}`services.hyperhive.swarm.attic.domain` and `cache`), handed to +# hive-c0re in its unit's environment; # - the cache's public key and the pull token are read from the store by a root # oneshot, `hive-priv-cache-facts`, under this hive's own store identity. The # key and the token land in `/run/hyperhive-priv/attic/` (root, `0700`; files -# `0600`), and the address, the key and the netrc's path in a nix.conf -# fragment, `/run/hyperhive-nix/cache.conf`, which the host's nix.conf -# includes (`!include`, so a host whose unit has not run yet still starts). +# `0600`). The key also lands in a nix.conf fragment, +# `/run/hyperhive-nix/cache.conf`, as `extra-trusted-public-keys`, which the +# host's nix.conf includes (`!include`, so a host whose unit has not run yet +# still starts). The fragment names no substituter and no netrc, so a client +# of the daemon that is not hive-c0re (an agent container on `nixBuilds = +# host`) cannot fetch from the cache or send the token; +# - the token also lands in `/run/hyperhive-nix/netrc`, `root:hive-core 0640`: +# the one place hive-core, and nothing else unprivileged, can read it. # # hive-c0re keeps its own copies under `/run/hyperhive/attic/` for the probe, # which only reports. @@ -47,7 +53,7 @@ let dir = "/run/hyperhive-priv/attic"; confDir = "/run/hyperhive-nix"; confFile = "${confDir}/cache.conf"; - url = "https://${atticCfg.domain}/${atticCfg.cache}"; + clientNetrc = "${confDir}/netrc"; # Where the store runs on this host, the first login waits for its container. # No `wants`: a pull would start the container mid-switch on the old closure. @@ -66,6 +72,10 @@ in # `!include` tolerates a fragment that does not exist yet. nix.extraOptions = "!include ${confFile}"; + # The address hive-c0re names to `nix copy --from`; no request chooses it. + systemd.services.hive-c0re.environment.HIVE_C0RE_CACHE_URL = + "https://${atticCfg.domain}/${atticCfg.cache}"; + systemd.services.hive-priv-cache-facts = { description = "Read the swarm cache's public key and pull token for the host's nix daemon and hive-priv"; wantedBy = [ "multi-user.target" ]; @@ -101,7 +111,8 @@ in script = import ./lib/attic-root-facts.nix { inherit lib; } { hive = hiveController.hiveName; host = atticCfg.domain; - inherit dir url confFile; + inherit dir confFile clientNetrc; + clientGroup = config.users.users.hive-core.group; }; }; diff --git a/nix/host-modules/lib/attic-root-facts.nix b/nix/host-modules/lib/attic-root-facts.nix index 8dcaa0c3..a1a07ae0 100644 --- a/nix/host-modules/lib/attic-root-facts.nix +++ b/nix/host-modules/lib/attic-root-facts.nix @@ -1,18 +1,22 @@ -# The root-side copy of the cache facts the host's nix daemon and hive-priv -# trust: the key a downloaded closure must be signed by and the netrc the pull -# token goes out in. The script runs as root on the hive host under the hive's -# own store identity and writes both into `dir`, which only root can read, and -# a nix.conf fragment at `confFile` that the host's nix.conf includes. +# The root-side copy of the cache facts the host's nix daemon, hive-priv and +# hive-c0re use: the key a downloaded closure must be signed by and the netrc +# the pull token goes out in. The script runs as root on the hive host under +# the hive's own store identity and writes: # -# What the daemon and hive-priv decide on is here or in the host's -# configuration, and nothing is read from a file hive-c0re (unprivileged) -# writes: the key and the token come from the store, the cache's address from -# `services.hyperhive.swarm.attic.{domain,cache}`. The fragment holds no secret -# (the key is public, the netrc stays in `dir`) and hive-c0re can read it: it -# takes the cache's address from there. +# - the key and the netrc into `dir`, which only root can read (hive-priv's +# check of the key); +# - a nix.conf fragment at `confFile` that the host's nix.conf includes, with +# `extra-trusted-public-keys` and nothing else: no trusted substituter and no +# netrc, so no other client of the daemon can fetch from the cache or send +# the token; +# - a second copy of the netrc at `clientNetrc`, `root: 0640`, +# that hive-c0re passes to its own `nix copy --from` as a client setting. # -# A store that holds no facts yet, or cannot be reached, leaves what is in `dir` -# and `confFile` as they are. Nothing here removes a file. +# Nothing is read from a file hive-c0re (unprivileged) writes: the key and the +# token come from the store. The fragment holds no secret. +# +# A store that holds no facts yet, or cannot be reached, leaves what is in +# `dir`, `confFile` and `clientNetrc` as they are. Nothing here removes a file. # # Pure function — NOT a NixOS module. Call it from a unit's `script`, with # `bao`, `coreutils`, `gnugrep` and `systemd` on its `path` and the `BAO_*` @@ -20,7 +24,8 @@ # # script = import ./lib/attic-root-facts.nix { inherit lib; } { # hive = "alpha"; host = "attic.example.org"; dir = "/run/hyperhive-priv/attic"; -# url = "https://attic.example.org/swarm"; confFile = "/run/hyperhive-nix/cache.conf"; +# confFile = "/run/hyperhive-nix/cache.conf"; clientNetrc = "/run/hyperhive-nix/netrc"; +# clientGroup = "hive-core"; # }; { lib }: { @@ -31,10 +36,11 @@ host, # Where the key and the netrc go. dir, - # The cache's address as the daemon lists it in `trusted-substituters`. - url, # Where the nix.conf fragment goes; its directory is root's and readable. confFile, + # Where the netrc copy for the client goes, and the group that may read it. + clientNetrc, + clientGroup, }: '' set -euo pipefail @@ -44,8 +50,9 @@ dir=${lib.escapeShellArg dir} conf=${lib.escapeShellArg confFile} + client_netrc=${lib.escapeShellArg clientNetrc} install -d -m 0700 "$dir" - install -d -m 0755 "$(dirname -- "$conf")" + install -d -m 0755 "$(dirname -- "$conf")" "$(dirname -- "$client_netrc")" store_init store_login @@ -95,14 +102,17 @@ atomic_write_secret 0600 "" "$dir/netrc" "$netrc" fi - # Last, so a daemon that reads the fragment finds the netrc in place. The - # address is the one hive-c0re may name as the substituter. The mtime moves - # only with the content. Lix re-reads nix.conf for every connection, CppNix - # only at start, so a daemon that started before this write is restarted. - printf -v fragment '%s\n%s\n%s' \ - ${lib.escapeShellArg "extra-trusted-substituters = ${url}"} \ - "extra-trusted-public-keys = $key" \ - "netrc-file = $dir/netrc" + # The client's copy: readable by the group, which is a deliberate grant of the + # pull token to hive-c0re's user. Written before the fragment, so a host with + # the key has the netrc. + if secret_differs "$client_netrc" "$netrc"; then + atomic_write_secret 0640 ":${clientGroup}" "$client_netrc" "$netrc" + fi + + # Last. The mtime moves only with the content. Lix re-reads nix.conf for every + # connection, CppNix only at start, so a daemon that started before this write + # is restarted. + fragment="extra-trusted-public-keys = $key" if secret_differs "$conf" "$fragment"; then atomic_write_secret 0644 "" "$conf" "$fragment" fi diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index 9ebf386d..a05e39ba 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -881,18 +881,32 @@ let # The daemon only learns the cache from the fragment the root unit # writes, and lists as trusted the one address from the host's # configuration. A host with no unit has no include. - name = "the host's nix.conf includes the cache fragment the root unit writes, with the cache's address"; + name = "the host's nix.conf includes a fragment that trusts only the swarm key, and only hive-core gets the token"; ok = let u = units.hive-priv-cache-facts; + c0re = allLocal.systemd.services.hive-c0re.environment; in lib.hasInfix "!include /run/hyperhive-nix/cache.conf" allLocal.nix.extraOptions && !(lib.hasInfix "hyperhive-nix" noIdentity.nix.extraOptions) && lib.hasInfix "conf=/run/hyperhive-nix/cache.conf" u.script - && lib.hasInfix "extra-trusted-substituters = https://attic.t.local/swarm" u.script + && lib.hasInfix ''fragment="extra-trusted-public-keys = $key"'' u.script && lib.hasInfix ''atomic_write_secret 0644 "" "$conf"'' u.script + # The daemon is told no substituter and no netrc: no other client of it + # can fetch from the cache or send the token. + && !(lib.hasInfix "extra-trusted-substituters" u.script) + && !(lib.hasInfix "trusted-substituters" allLocal.nix.extraOptions) + && !(lib.hasInfix "netrc-file" u.script) + # The token's one unprivileged reader is the hive-core group. + && lib.hasInfix "client_netrc=/run/hyperhive-nix/netrc" u.script + && lib.hasInfix ''atomic_write_secret 0640 ":hive-core" "$client_netrc"'' u.script && lib.elem "d /run/hyperhive-nix 0755 root root - -" allLocal.systemd.tmpfiles.rules - # Control for the absence: the lookup finds the unit's other variable. + # hive-c0re is told the address; hive-priv no longer is. + && c0re.HIVE_C0RE_CACHE_URL == "https://attic.t.local/swarm" + && !(noIdentity.systemd.services.hive-c0re.environment ? HIVE_C0RE_CACHE_URL) + # Control for the absences: the lookups find the units' other variables. + && c0re ? HOME + && noIdentity.systemd.services.hive-c0re.environment ? HOME && allLocal.systemd.services.hive-priv.environment ? NIX_REMOTE && !(allLocal.systemd.services.hive-priv.environment ? HIVE_PRIV_CACHE_URL); } diff --git a/nix/script-tests/attic-root-facts.nix b/nix/script-tests/attic-root-facts.nix index ee5d2ef9..9f4556a7 100644 --- a/nix/script-tests/attic-root-facts.nix +++ b/nix/script-tests/attic-root-facts.nix @@ -15,8 +15,9 @@ let hive = "alpha"; host = "cache.example"; dir = "@DIR@/attic"; - url = "https://cache.example/swarm"; confFile = "@DIR@/nix/cache.conf"; + clientNetrc = "@DIR@/nix/netrc"; + clientGroup = "@GROUP@"; } ); in diff --git a/nix/script-tests/attic-root-facts.sh b/nix/script-tests/attic-root-facts.sh index e051d62c..263ce950 100644 --- a/nix/script-tests/attic-root-facts.sh +++ b/nix/script-tests/attic-root-facts.sh @@ -19,7 +19,11 @@ expect() { # dir=$(mktemp -d) mkdir -p "$dir/bin" -sed "s|@DIR@|$dir|g" "$SCRIPT" >"$dir/script.sh" +# A group other than the primary one when the runner has one, so the group the script sets +# differs from the one a new file gets. +group=$(id -Gn | tr " " "\n" | grep -vx "$(id -gn)" | head -1 || true) +group=${group:-$(id -gn)} +sed -e "s|@DIR@|$dir|g" -e "s|@GROUP@|$group|g" "$SCRIPT" >"$dir/script.sh" calls=$dir/calls export DIR=$dir @@ -96,22 +100,26 @@ expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok- expect "key mode" "$(mode "$dir/attic/public-key")" 600 expect "netrc mode" "$(mode "$dir/attic/netrc")" 600 expect "dir mode" "$(mode "$dir/attic")" 700 -expect "fragment" "$(body "$dir/nix/cache.conf")" "extra-trusted-substituters = https://cache.example/swarm -extra-trusted-public-keys = swarm:S0VZ -netrc-file = $dir/attic/netrc" +expect "fragment is the key and nothing else" "$(body "$dir/nix/cache.conf")" "extra-trusted-public-keys = swarm:S0VZ" expect "fragment mode" "$(mode "$dir/nix/cache.conf")" 644 expect "fragment dir mode" "$(mode "$dir/nix")" 755 expect "no secret in the fragment" "$(grep -c tok-1 "$dir/nix/cache.conf")" 0 +expect "no substituter or netrc in the fragment" "$(grep -c -E "substituter|netrc" "$dir/nix/cache.conf")" 0 +expect "client netrc" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1" +expect "client netrc mode" "$(mode "$dir/nix/netrc")" 640 +expect "client netrc group" "$(stat -c %G "$dir/nix/netrc")" "$group" case="a second run with the same facts rewrites nothing" k=$(ino "$dir/attic/public-key") n=$(ino "$dir/attic/netrc") +cn=$(ino "$dir/nix/netrc") c=$(ino "$dir/nix/cache.conf") run expect "exit" $? 0 expect "key untouched" "$(ino "$dir/attic/public-key")" "$k" expect "netrc untouched" "$(ino "$dir/attic/netrc")" "$n" expect "fragment untouched" "$(ino "$dir/nix/cache.conf")" "$c" +expect "client netrc untouched" "$(ino "$dir/nix/netrc")" "$cn" case="a rotated token replaces the netrc and leaves the key" seed "swarm:S0VZ" "tok-2" @@ -218,6 +226,8 @@ seed "swarm:S0VZ" "tok-2" run DAEMON_STARTED="$started" expect "exit" $? 0 expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok-2" +expect "client netrc follows" "$(body "$dir/nix/netrc")" "machine cache.example password tok-2" +expect "client netrc mode" "$(mode "$dir/nix/netrc")" 640 expect "no restart" "$(restarts)" 0 case="a rotated key rewrites the fragment and restarts the daemon" @@ -233,6 +243,7 @@ reset run DAEMON_STARTED="Sat 2000-01-01 00:00:00.000000 UTC" expect "exit" $? 0 expect "no fragment" "$(body "$dir/nix/cache.conf")" missing +expect "no client netrc" "$(body "$dir/nix/netrc")" missing expect "no restart" "$(restarts)" 0 case="a key that is refused writes no fragment" @@ -241,6 +252,7 @@ seed "swarm:S0 VZ" "tok-1" run expect "exit" $? 1 expect "no fragment" "$(body "$dir/nix/cache.conf")" missing +expect "no client netrc" "$(body "$dir/nix/netrc")" missing if [ "$failures" -ne 0 ]; then echo "$failures of $count checks failed" >&2