From 0e1a975f9f88dd5eb87b05bb8f8254e26993f331 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 12 Aug 2026 00:05:53 +0200 Subject: [PATCH] fix(3179): the gateway's config files get their own state dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `agents.conf` and `gateway.htpasswd` move from /var/lib/hyperhive/gateway to /var/lib/hive-gateway/conf, alongside the `tls/` the gateway already kept there. nginx reads both as an unprivileged user. Under c0re's state dir it could only reach them by traversing a directory systemd re-declares `0750 hive-core` on every c0re start — so nginx was given `SupplementaryGroups = [ "hive-core" ]`, which also handed it read access to everything else group-readable in that tree. The tokens are individually 0600, but the broker sqlite carries no explicit mode: every message between every agent was readable by the process whose job is parsing untrusted network input. Moving the files removes the need and the exposure together. The group is gone, and its absence is now commented as load-bearing so it doesn't come back as a fix for a symptom it would recreate. Also drops this module's `/var/lib/hyperhive` tmpfiles rule. It declared `0755 root root` and could never win against `StateDirectoryMode`, and a losing declaration still reads as a guarantee — that is what sent the first diagnosis of the outage looking for who had changed the mode. Ordering is unchanged and still the thing that makes a fresh boot work: tmpfiles runs before services and seeds both files empty-but-valid, nginx names them (an `include` of a missing file is fatal, not empty), and content arrives when c0re writes and reloads — which it does on every topology change, so a boot against the empty seed resolves itself. Folds in the mode fix: `write` now sets 0644 on the tmp file before the rename, because a rename carries the source's mode and discards the destination's, and the tmpfiles rule that declares 0644 is create-if-absent so it never re-applies. --- docs/gateway.md | 6 +- docs/tools/hivectl.md | 2 +- hive-c0re/src/gateway_nginx.rs | 14 +++- hive-c0re/src/meta.rs | 2 +- hive-c0re/src/paths.rs | 24 ++++--- hive-host-sock/src/lib.rs | 9 ++- nix/host-modules/hive-gateway/default.nix | 81 +++++++++++++++-------- nix/host-modules/hive-gateway/options.nix | 2 +- nix/host-modules/hive-gateway/vhosts.nix | 8 +-- 9 files changed, 98 insertions(+), 50 deletions(-) diff --git a/docs/gateway.md b/docs/gateway.md index b02b9752..6b65fe85 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -113,12 +113,12 @@ now set unconditionally for every agent. The mechanism: only agents whose harness has actually bound the socket appear there. (Legacy name `.bound` also accepted during the transition window.) 4. **Gateway side**. `gateway_nginx::write` generates - `/var/lib/hyperhive/gateway/agents.conf` — a plain nginx include + `/var/lib/hive-gateway/conf/agents.conf` — a plain nginx include file with one `location /agent//` block per agent. Always a UDS upstream (`http://unix:/run/hive-agent//web.sock:/`); if the socket is not yet bound, nginx returns 502 caught by the `error_page 502 503 504 = /__hive_agent_unreachable` directive. - nginx includes `/var/lib/hyperhive/gateway/agents.conf` — the same + nginx includes `/var/lib/hive-gateway/conf/agents.conf` — the same path c0re writes, since both run on the host. After each write, c0re triggers the appropriate nginx action via `hive-priv` (which is root; hive-c0re runs as the unprivileged @@ -505,7 +505,7 @@ services.hyperhive.gateway.auth = { ``` The credential store lives at the fixed path -`/var/lib/hyperhive/gateway/gateway.htpasswd` on the host. A tmpfiles +`/var/lib/hive-gateway/conf/gateway.htpasswd` on the host. A tmpfiles rule pre-creates the file on first boot; no manual path configuration is required. nginx reads it at that path directly. diff --git a/docs/tools/hivectl.md b/docs/tools/hivectl.md index 302052ab..a8d502c2 100644 --- a/docs/tools/hivectl.md +++ b/docs/tools/hivectl.md @@ -115,7 +115,7 @@ hivectl github set-token damocles --token # inline (visible in shell hi Manage users in the gateway's HTTP Basic auth htpasswd file (`services.hyperhive.gateway.auth`). `hivectl` sends the request over the host admin socket; the `hive-c0re` daemon owns the htpasswd file at its -canonical path (`/var/lib/hyperhive/gateway/gateway.htpasswd`) and +canonical path (`/var/lib/hive-gateway/conf/gateway.htpasswd`) and performs the write. ```bash diff --git a/hive-c0re/src/gateway_nginx.rs b/hive-c0re/src/gateway_nginx.rs index 7a9fda1d..7f94e802 100644 --- a/hive-c0re/src/gateway_nginx.rs +++ b/hive-c0re/src/gateway_nginx.rs @@ -1,11 +1,12 @@ //! Runtime nginx include-file generator for the gateway's per-agent //! `/agent//` location blocks. Writes -//! `/var/lib/hyperhive/gateway/agents.conf` on every topology change. +//! `/var/lib/hive-gateway/conf/agents.conf` on every topology change. //! UDS upstream selection, the reload trigger, and idempotency: //! `docs/gateway.md::Per-agent unix-socket upstream`. use anyhow::{Context, Result}; use std::fmt::Write as _; +use std::os::unix::fs::PermissionsExt as _; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Mutex, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -192,6 +193,17 @@ pub async fn write(names: &[String]) -> Result<()> { } let tmp = path.with_extension("conf.tmp"); std::fs::write(&tmp, &body).with_context(|| format!("write tmp {}", tmp.display()))?; + // Mode set on the TMP file, before the rename, because a rename + // carries the source's mode and owner and discards the destination's. + // The tmpfiles rule declaring 0644 is `f` (create-if-absent), so it + // never re-applies: from the first republish on, the published mode is + // whatever this process's umask happened to produce. That is fine + // today and becomes a dead gateway the day anything tightens c0re's + // umask, since nginx reads this file as a different, unprivileged + // user. Making it explicit means the mode is a property of the + // publisher rather than of an unrelated unit's settings. + std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o644)) + .with_context(|| format!("chmod tmp {}", tmp.display()))?; std::fs::rename(&tmp, &path).with_context(|| { format!( "rename {} -> {} (atomic publish)", diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 54d3e6ef..8ea3dbe5 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -167,7 +167,7 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { tracing::warn!(error = ?e, "agent_sockets::write failed (non-fatal)"); } - // Refresh /var/lib/hyperhive/gateway/agents.conf — the nginx include + // Refresh /var/lib/hive-gateway/conf/agents.conf — the nginx include // file the gateway reads at runtime. c0re then triggers a reload (or // start) of the host's nginx via hive-priv, since c0re is unprivileged. // Same best-effort + non-fatal shape. diff --git a/hive-c0re/src/paths.rs b/hive-c0re/src/paths.rs index 5e8a89e8..c45d761b 100644 --- a/hive-c0re/src/paths.rs +++ b/hive-c0re/src/paths.rs @@ -218,16 +218,24 @@ pub fn shared_root() -> PathBuf { // nix: bind-mounted read-only into agent containers as `/knowledge` (the harness nix modules) — must match. pub const KNOWLEDGE_DIR: &str = "/var/lib/hyperhive/knowledge"; -/// `gateway/` — generated nginx include fragments for the gateway vhost. -/// nginx runs on the host and reads this path directly. ⚠️ Nothing about -/// this path confines it: what keeps nginx away from the rest of -/// `/var/lib/hyperhive/` (forge/matrix tokens, etc.) is the unit's own -/// sandbox, so widening that sandbox widens what a gateway compromise -/// reaches. -// nix: named by the gateway's nginx config (hive-gateway/vhosts.nix) — must match. +/// `/var/lib/hive-gateway/conf` — generated nginx include fragments, +/// deliberately **outside** `STATE_ROOT`. +/// +/// nginx runs on the host as an unprivileged user and reads these +/// directly. Keeping them here rather than under `/var/lib/hyperhive` +/// means nginx needs no access to c0re's state dir at all — no shared +/// parent to traverse, so no group membership handing it everything else +/// that lives there (the broker db above all). The separation IS the +/// confinement; nothing else is doing that job. +/// +/// Sibling of the gateway's `tls/`, which already lived here. +// nix: named by the gateway's nginx config (hive-gateway/vhosts.nix) and created by +// its tmpfiles rules (hive-gateway/default.nix) — must match. +pub const GATEWAY_CONF_DIR: &str = "/var/lib/hive-gateway/conf"; + #[must_use] pub fn gateway_dir() -> PathBuf { - state_root().join("gateway") + PathBuf::from(GATEWAY_CONF_DIR) } /// `gateway/agents.conf` — per-agent nginx `location` blocks (UDS upstreams). diff --git a/hive-host-sock/src/lib.rs b/hive-host-sock/src/lib.rs index 7781d145..8fef8d95 100644 --- a/hive-host-sock/src/lib.rs +++ b/hive-host-sock/src/lib.rs @@ -48,10 +48,13 @@ pub fn agent_state_dir(name: &Ident) -> PathBuf { PathBuf::from(AGENTS_ROOT).join(name.as_str()) } -/// `gateway/gateway.htpasswd` — nginx basic-auth credential store for the -/// operator dashboard vhost. `hivectl`'s `--htpasswd-file` clap default. +/// nginx basic-auth credential store for the operator dashboard vhost. +/// `hivectl`'s `--htpasswd-file` clap default. +/// +/// Under the gateway's own state dir rather than c0re's, so that nginx +/// can read it without being given access to everything else c0re keeps. // nix: read by the gateway's nginx on the host (hive-gateway/) — must match. -pub const GATEWAY_HTPASSWD: &str = "/var/lib/hyperhive/gateway/gateway.htpasswd"; +pub const GATEWAY_HTPASSWD: &str = "/var/lib/hive-gateway/conf/gateway.htpasswd"; /// `/run/hive-agent` — per-agent runtime socket dir root (web UI unix /// socket + bound marker), one subdir per agent. The gateway's nginx diff --git a/nix/host-modules/hive-gateway/default.nix b/nix/host-modules/hive-gateway/default.nix index 92d5f7c9..0b76cef0 100644 --- a/nix/host-modules/hive-gateway/default.nix +++ b/nix/host-modules/hive-gateway/default.nix @@ -117,23 +117,44 @@ in # `create_dir_all(/run/hive-agent/)` itself, so a root-owned # parent would EACCES on the very first agent create on a fresh host # (hive-priv only chowns the subdir afterwards, it doesn't make it). - # /var/lib/hyperhive — hyperhive state dir, created by c0re on - # first run. Also pre-seed agents.conf with an empty-but-valid - # header so nginx can start + include the file before c0re writes - # its first real content (f = create-if-absent, no overwrite). + # + # ⚠️ There is deliberately NO rule for /var/lib/hyperhive here. One + # used to declare it `0755 root root` and could never win: + # `hive-c0re.service` sets `StateDirectory = "hyperhive"` with + # `StateDirectoryMode = "0750"`, which systemd re-applies on every + # start. Two mechanisms owning one path, and the loser still read as + # a guarantee — it is why the resulting outage was first misdiagnosed + # as someone having changed the mode. c0re's own unit owns that dir; + # this module no longer has an opinion about it. systemd.tmpfiles.rules = [ # Must stay in step with the identical rule hive-priv generates into # /etc/tmpfiles.d/hyperhive-agents.conf — the two used to declare # different owners for this path. "d /run/hive-agent 0755 hive-core hive-core - -" - "d /var/lib/hyperhive 0755 root root - -" - "d /var/lib/hyperhive/gateway 0755 root root - -" - "f /var/lib/hyperhive/gateway/agents.conf 0644 root root - # Generated by hive-c0re — do not edit.\n" - # Pre-create the htpasswd file so nginx can open it even before any - # users have been added. An empty file causes all auth checks to - # return 401 (no valid credentials), which is the correct no-users - # behaviour. `f` = create-if-absent, never overwrite. - "f /var/lib/hyperhive/gateway/gateway.htpasswd 0644 root root - -" + # The gateway's own config dir — NOT under /var/lib/hyperhive. c0re + # writes here, nginx reads here, and neither needs any access to the + # other's tree: no shared parent to traverse means no group + # membership handing nginx c0re's broker db and everything else + # beside it. Sibling of `tls/`, which already lived under this root. + # + # Owned by hive-core because c0re is the writer; 0755 so the + # unprivileged nginx user can traverse and read. This is now the + # ONLY declaration of these paths' modes — nothing re-applies a + # different owner on top the way `StateDirectory=` does for + # /var/lib/hyperhive. + "d /var/lib/hive-gateway 0755 root root - -" + "d /var/lib/hive-gateway/conf 0755 hive-core hive-core - -" + "f /var/lib/hive-gateway/conf/agents.conf 0644 hive-core hive-core - # Generated by hive-c0re — do not edit.\n" + # Pre-create both files so nginx's config test can open them before + # c0re has ever written: nginx names them, and an `include` of a + # missing file is a fatal config error, not an empty one. tmpfiles + # runs before services, which is the whole ordering guarantee — + # content arrives when c0re writes and reloads, which it does on + # every topology change. `f` = create-if-absent, never overwrite. + # + # An empty htpasswd causes all auth checks to return 401 (no valid + # credentials), which is the correct no-users behaviour. + "f /var/lib/hive-gateway/conf/gateway.htpasswd 0644 hive-core hive-core - -" ]; # ⚠️ REMOVED WITH THE CONTAINER, and each one was a workaround for the @@ -285,22 +306,26 @@ in inherit (nginxTree) appendHttpConfig virtualHosts; }; - # nginx now reads `/var/lib/hyperhive/gateway/agents.conf` (+ - # gateway.htpasswd) directly off the host filesystem instead of - # through the old container's dedicated `/run/hive-state` - # bind-mount. `hive-c0re.service` declares `StateDirectory = - # "hyperhive"` with `StateDirectoryMode = "0750"` owned by - # `hive-core`, and systemd re-applies that owner/mode to the - # top-level `/var/lib/hyperhive` dir on every c0re start — - # overriding this module's own `0755 root:root` tmpfiles rule - # above. Without group membership, the `nginx` user can't even - # traverse into the directory, so nginx fails its config test and - # never starts (`nginx: [emerg] open() ".../agents.conf" failed - # (13: Permission denied)`) — the whole gateway, and every hive - # domain behind it, goes down. `gateway/` and `agents.conf` are - # already declared world-readable (0755 / 0644), so group - # traversal on the parent is the only thing missing. - systemd.services.nginx.serviceConfig.SupplementaryGroups = [ "hive-core" ]; + # ⚠️ NO `SupplementaryGroups = [ "hive-core" ]` on nginx, and its + # absence is load-bearing rather than an omission. + # + # It used to be here, to let nginx traverse `/var/lib/hyperhive` and + # reach the config fragments that lived inside: `hive-c0re.service` + # declares `StateDirectory = "hyperhive"` with `StateDirectoryMode = + # "0750"` owned by `hive-core`, and systemd re-applies that on every + # c0re start — beating this module's own tmpfiles rule for the same + # path. Without the group, nginx failed its config test and never + # started (`nginx: [emerg] open() ".../agents.conf" failed (13: + # Permission denied)`), taking the gateway and every hive domain + # behind it down. + # + # The group fixed the symptom and paid for it: it also gave nginx + # read access to everything ELSE group-readable under that dir, + # including the broker sqlite — i.e. every message between every + # agent, reachable by the process whose entire job is parsing + # untrusted network input. Moving the fragments to the gateway's own + # dir removes the need and the exposure together. Re-adding this line + # would restore both. # dnsmasq is a host service alongside nginx, so it reads the host's # /etc/resolv.conf directly and picks up network changes as they diff --git a/nix/host-modules/hive-gateway/options.nix b/nix/host-modules/hive-gateway/options.nix index 772ab8d8..30c38227 100644 --- a/nix/host-modules/hive-gateway/options.nix +++ b/nix/host-modules/hive-gateway/options.nix @@ -232,7 +232,7 @@ in enabled, every request to the gateway's main vhost requires a valid username and password. nginx's built-in `auth_basic` module validates credentials against - `/var/lib/hyperhive/gateway/gateway.htpasswd`. Off by default. + `/var/lib/hive-gateway/conf/gateway.htpasswd`. Off by default. Manage users with `hivectl gateway create-user`, `delete-user`, and `list-users` — see `hivectl gateway --help` for usage. diff --git a/nix/host-modules/hive-gateway/vhosts.nix b/nix/host-modules/hive-gateway/vhosts.nix index 06eb4482..969d8c3a 100644 --- a/nix/host-modules/hive-gateway/vhosts.nix +++ b/nix/host-modules/hive-gateway/vhosts.nix @@ -289,7 +289,7 @@ let # `/agent/` catch-all 404 + the two internal error-page targets it # points at. Per-agent `location /agent//` blocks live in the - # runtime-generated `/var/lib/hyperhive/gateway/agents.conf` (included via + # runtime-generated `/var/lib/hive-gateway/conf/agents.conf` (included via # `extraConfig` on the vhost); nginx longest-prefix-match makes a # real `/agent//` beat this catch-all. `internal` keeps the # error pages reachable only through nginx's error handling. @@ -323,7 +323,7 @@ let # secret (`X-Hub-Signature-256`) protects those endpoints instead. dashboardAuth = lib.optionalString cfg.auth.enable '' auth_basic "${cfg.auth.realm}"; - auth_basic_user_file /var/lib/hyperhive/gateway/gateway.htpasswd; + auth_basic_user_file /var/lib/hive-gateway/conf/gateway.htpasswd; # `=401` keeps the status 401 so the login dialog shows; the # internal page explains `hivectl gateway create-user`. error_page 401 =401 /__hive_auth_unauthorized; @@ -444,7 +444,7 @@ in }; }; # Per-agent location blocks, generated at runtime by - # hive-c0re and written to /var/lib/hyperhive/gateway/agents.conf + # hive-c0re and written to /var/lib/hive-gateway/conf/agents.conf # on the host — the same machine nginx runs on. nginx parses # `include` at config-load time so a reload (triggered by c0re # after each agents.conf write) picks up new or removed @@ -452,7 +452,7 @@ in # match rule ensures `/agent//` from this file beats # the `/agent/` catch-all above. extraConfig = securityHeaders + '' - include /var/lib/hyperhive/gateway/agents.conf; + include /var/lib/hive-gateway/conf/agents.conf; ''; }; }