From 712dcf2b25970260e2ceea88f50994a38444bb2d Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 13 Nov 2026 05:17:01 +0100 Subject: [PATCH] squid intercepts restricted and log agents' web traffic, named from the bound source address --- docs/4-implementation/networking/network.md | 121 ++++++-- .../swarm/credentials-reference.md | 80 ++--- hive-c0re/src/workers/squid_allow.rs | 49 ++- hive-c0re/src/workers/squid_deny.rs | 14 +- hive-priv/README.md | 8 +- hive-priv/src/egress.rs | 282 ++++++++++++++++-- nix/host-modules/hive-egress.nix | 91 +++++- nix/host-modules/hive-squid.nix | 113 +++++-- nix/module-eval/core-toggle.nix | 100 ++++++- nix/module-eval/hive-otel.nix | 35 ++- nix/vm-tests.nix | 2 +- nix/vm-tests/hive-squid.nix | 262 ++++++++++++++-- 12 files changed, 1004 insertions(+), 153 deletions(-) diff --git a/docs/4-implementation/networking/network.md b/docs/4-implementation/networking/network.md index 22121ca27..99e9878c6 100644 --- a/docs/4-implementation/networking/network.md +++ b/docs/4-implementation/networking/network.md @@ -229,13 +229,18 @@ Every hive host runs squid in the host netns, listening on `:3128` (`nix/host-modules/hive-squid.nix`). The port is open on the bridge interface only. An agent sends its outbound HTTP and HTTPS through it once its hive says the proxy is ready and it has fetched its proxy credential (see _Agents and the -proxy_ below). Any other agent goes straight out through the NAT, as before, -and so does any program that ignores the proxy variables: nothing blocks direct -egress yet. The host logs those direct connections (see _Agent bridge egress log_ -below). +proxy_ below). A program that ignores the proxy variables still reaches squid +when it speaks HTTP on port 80 or TLS on port 443 and its agent is `restricted` +or in `log` mode: the host redirects that traffic into squid's intercept ports +(see _Intercepted web traffic_ below). Everything else goes straight out through +the NAT, as before: the `unrestricted` agents, the service containers and every +other port and protocol. Nothing blocks that yet. The host logs those direct +connections (see _Agent bridge egress log_ below). Squid filters on the destination hostname: a `CONNECT` target or a plain-HTTP -host. It never decrypts TLS, so it holds no signing key. It caches nothing. +host. It never decrypts TLS. The one TLS key it loads is a throwaway that the +unit generates at each start, that signs nothing and that no agent trusts. It +caches nothing. - **Credentials.** A request for a listed name must carry a per-agent proxy credential (HTTP Basic, the agent's name and its secret). `swarm-controller` @@ -310,10 +315,10 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. `internet_log: true`, so a `hive-c0re` that doesn't know the field still enforces the list. For such an agent `hive-c0re` adds to `agent-allow.conf`, after the allow rules for every agent's own names, - `http_access allow !allowed_dst log_mark`, where `log_mark` is an - `annotate_transaction` ACL that sets `would_deny=1`. The rule comes last in - the include, so every destination deny and `deny !authed` before it still - applies, and a name the agent may reach never carries the mark. A request the + `http_access allow !icpt !allowed_dst log_mark` and the same rule for + the intercept ports, where `log_mark` is an `annotate_transaction` ACL that + sets `would_deny=1`. The rules come last in the include, so every destination + deny and `deny !icpt !authed` before them still applies, and a name the agent may reach never carries the mark. A request the rule lets through gets a second log line, `would_deny user= method= host= port=

result=`, from the `would_denied` `access_log` in `hive-squid.nix`; the usual request line keeps @@ -344,7 +349,7 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. (`/var/lib/hive-squid/deny.conf`, `hive-core:squid`, `0640`) for the agents it runs, and `squid-unrestricted-reload.path` reloads squid when it changes. - **Precedence.** The configuration includes the file after the destination - denies and before `deny !authed` and every allow, so a denied name is + denies and before `deny !icpt !authed` and every allow, so a denied name is refused even when the base list, the hive's `allowedDomains`, the swarm's allow-list, the unrestricted flag or `log` mode would allow it. A swarm-wide entry names no user, so every client gets a 403 for it, one without a @@ -406,10 +411,10 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing. logs `squid allow list: 2 swarm, 3 agent entries` when they change. The swarm-wide names join the allow ACL, so a restricted user reaches them. A per-agent entry is a rule for that agent's squid user, `http_access allow - `, in the generated `agent-allow.conf` that the configuration - includes at the point where the base list gets its allow: after every - destination deny and - `deny !authed`, so the rules under _Denied before any allow_ still bind it, +!icpt `, and a second one for the user the intercept helper names + (`icpt `), in the generated `agent-allow.conf` that the + configuration includes at the point where the base list gets its allow: after + every destination deny and `deny !icpt !authed`, so the rules under _Denied before any allow_ still bind it, and before the unrestricted rule. Only this hive's agents get a rule. `squid-unrestricted-reload.path` reloads squid when either file changes, so there is no deploy or restart. @@ -507,6 +512,84 @@ credential=present|absent unrestricted_canary=… unrestricted_status=…` line host's resolver looks up the canary, so a host whose resolver maps that name to a loopback, bridge or private address fails the check. +### Intercepted web traffic + +A program that ignores the proxy variables still gets squid's policy for its +web traffic. In the `inet hive_egress` table (`nix/host-modules/hive-egress.nix`) +the `intercept` chain, at `prerouting`, redirects IPv4 TCP port 80 to +`:3129` and port 443 to `:3130`, the two intercept listeners +that `hive-squid.nix` opens. Squid reads the original destination from +conntrack. The chain exists where the host runs squid +(`services.hyperhive.network.egress.intercept.enable`). + +- **Who gets redirected.** Every flow from the bridge to port 80 or 443, except + a flow to an address that stays on the bridge, to one of the host's own + addresses or to the `wg-hive` interface, and a flow from a MAC in the set + `direct_mac`. That set holds the MACs of the ports of every container that's + not an agent (the swarm's service containers) and of the agents listed in + `services.hyperhive.deploy.hive-controller.squid.unrestrictedFile`. So + `restricted` and `log` agents get redirected, and `unrestricted` agents and + service containers keep the direct path. Every pass of `hive-priv egress-pin` + and every `egress-load` refills the set from the bindings it also uses for the + bridge table, and `hive-egress-direct.path` runs a pass when `hive-c0re` + rewrites the list; the timer covers the rest. A MAC in no binding is in no + exemption: the flow gets redirected, and squid refuses it. A new agent that + the list doesn't name yet counts as restricted until the next pass, and a flag + change reaches the redirect after the hive's next read of the permissions and + one pass. IPv6 and the other ports aren't redirected. +- **Who the flow belongs to.** An intercepted request carries no proxy + credential, so squid asks its external ACL `hive_agent`, which runs + `hive-priv egress-whois`. The helper looks the source address up in dnsmasq's + lease file for a MAC, the MAC in the bindings (`/run/hive-egress-pins`) for a + machine, and takes the agent from the machine name `h-`. While + `antiSpoof` is `drop`, an address on the bridge is the one leased to the MAC + of the port it came from (see _Agent bridge source check_), so the source says + which container sent the flow. Squid keeps an answer for 10 seconds and a + refusal for 2. An address with no lease or an expired one, two MACs with the + same address, a MAC no port holds, a machine that's no agent: none gets an + answer, and squid refuses the flow with a `TCP_DENIED/403` and user `-`. A + host that has no bindings or leases yet refuses every intercepted flow rather + than guess. With `antiSpoof = log` a container can forge the source, so use that mode + to diagnose and not to run. The proxy port never takes this identity: a + request without a credential through port 3128 still gets a 407, whatever its + address. +- **Policy.** The same lists as for the proxy port, matched on the agent the + helper names: the destination denies, the deny list, the base list, the + allow-list and the agent's own names, and `log` mode with its `would_deny` + line. `hive-c0re` renders each agent rule twice, once for the credential + (`!icpt`) and once for the helper's user (`icpt`). +- **HTTPS.** Squid peeks at the ClientHello for the SNI and splices the + connection. It never decrypts, and no agent trusts a CA of the host. Squid + builds a `CONNECT` with the SNI as its host and applies every rule to it. A + refused name gets a 403 that the client reads as a TLS error. Squid also + refuses: + - an SNI that doesn't resolve to the address the agent dialed + (`host_verify_strict`; squid answers 409 and the client reads a TLS error). + Squid resolves on the host's resolver, which is the same dnsmasq as the + agent's, so a name with changing answers can fail now and then; + - a ClientHello with no SNI, and a URL with an IP literal, which sends none: + the host of the `CONNECT` stays an address, and the IP-literal denies refuse + it; + - bytes on port 443 that aren't a ClientHello, and on port 80 that aren't an + HTTP request. + + Squid 7.6 can't see the `encrypted_client_hello` extension, only the public + name in the outer hello. An agent that holds an ECH configuration can send a + hello with an allowed outer name, a destination address that name resolves + to, and a hidden inner name that the same address serves. Squid lets that + flow through, so ECH isn't refused here. + +- **Reading the log.** Each intercepted HTTPS connection logs twice: first + `NONE_NONE/000 … CONNECT

:443 `, before the ClientHello, then + the `CONNECT :443 ` line with `TCP_TUNNEL/200` or `TCP_DENIED/403` + (`NONE_NONE/409` when the SNI and the address disagree). An intercepted HTTP + request logs `GET http:/// `. A tunnel to an intercepted + destination ends in `ORIGINAL_DST/
`. `cache.log` notes an ACL that met + a flow squid refused before its request. +- **Rollout.** The redirect, squid's configuration and `hive-priv` ship in one + build. A hive that still runs an older build has no redirect, and its agents go + direct as before. The agents that use the proxy variables see no change. + ## Container isolation Each agent container runs in a private network namespace with a dedicated @@ -590,14 +673,18 @@ loopback — the hive-c0re admin socket is a UDS, not TCP. the `inet hive_egress` table does with traffic the host routes from the agent bridge to anywhere but the bridge and the `wg-hive` mesh interface. It takes `accept`, `log` (the default) or `drop`. The evaluation refuses `drop` for now: -the interception that would carry an agent's web traffic through squid instead -hasn't landed. In `log` mode the chain blocks nothing. It logs each new flow +the traffic that squid doesn't handle (the unrestricted agents, the service +containers, every port but 80 and 443) has no rule that lets it through yet. In +`log` mode the chain blocks nothing. It logs each new flow and counts every packet. - **Which traffic.** IPv4 and IPv6 alike (the table is `inet`). The chain never sees traffic to the host's own addresses, because the host delivers it locally: a connection to squid on `:3128`, to dnsmasq or to the - gateway leaves no line. Neither does anything squid or the host opens itself. + gateway leaves no line. Nor does anything squid or the host opens itself. The + web traffic of a `restricted` or `log` agent that the `intercept` chain + redirects into squid reaches the host too, so squid's log shows it instead + (see _Intercepted web traffic_). - **The log line.** The kernel log carries one line per new flow, with the prefix `hive-egress-would-drop ` followed by the usual netfilter fields (`IN`, `OUT`, `SRC`, `DST`, `PROTO` and the ports). Read them with diff --git a/docs/4-implementation/swarm/credentials-reference.md b/docs/4-implementation/swarm/credentials-reference.md index 012732d1d..12d2913b2 100644 --- a/docs/4-implementation/swarm/credentials-reference.md +++ b/docs/4-implementation/swarm/credentials-reference.md @@ -115,45 +115,45 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](../../2-hand ### Swarm-level — one of each per swarm -| secret | generated by | lives at | hive elsewhere | -| ----------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name and needs a store identity | -| authelia session, JWT, storage-encryption and OIDC HMAC keys | `authelia-swarm-secrets` in authelia's container. It stores each in the swarm secret store at `swarm/services/swarm-authelia/keys/` when the store holds none: the value of the key an earlier release generated, or a new one when there is no such file. The files stay on disk. While the store is down, refuses the login, or fails the read of a leaf (a refusal, a timeout, an error), the unit runs on the files, logs the store's error, and writes nothing to the store; the unit seeds or mints only a leaf the store reports as absent. A container with no secret publisher identity generates the files, as before | `/var/lib/authelia-swarm/{session,jwt,storage-encryption,oidc-hmac}.key` | nothing to place: the publisher's store leaf is the one in the row below. A store that stays unreachable on a start with no key file keeps authelia from starting | -| authelia OIDC issuer key (RSA) | same unit, same store path (`oidc-issuer`) | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | -| swarm appservice registration (`as_token`, `hs_token`) | `swarm-matrix-ctl appservice render` in `hive-matrix`. The store's pair wins; a store holding none takes the tokens an earlier version kept in the container, so the value doesn't change by moving. Only a store that answers, holds none, and a container with no such files gets a new pair | `/run/swarm-matrix-appservice/swarm.yaml` in `hive-matrix` (tmpfs), which tuwunel loads through `LoadCredential`. The earlier `/var/lib/swarm-matrix-appservice/{as-token,hs-token,swarm.yaml}` stay, unused, and bao wins if they differ; the store path is `swarm/controller/swarm-controller/matrix/appservice-registration` | nothing to place: matrix-ctl's store leaf is the one the container binds. A store that's down keeps the homeserver down and mints nothing, and `render` retries every 30 seconds for 24 hours | -| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | the swarm secret store, at the path its reader fetches: `swarm/hives//oidc/client`, `swarm/controller/swarm-controller/oidc/client` or `swarm/services//oidc/client`. The bridge reads its own into `/run/swarm-authelia-bridge/client-secret` before each start. A container with no secret publisher identity keeps it in `/var/lib/authelia-swarm/oidc-clients/.secret`; a file an earlier release left seeds the store once when it holds no value, and stays unused | nothing to place: readers fetch it from the store ([how](../../2-handbook/swarm/credentials.md#hive-level-secrets)) | -| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | -| the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/run/swarm-otel-oidc/.secret` on the host's tmpfs, bind-mounted read-only into the `swarm-otel` container — never on persistent disk, so a reboot fetches it again | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see [swarm-level secrets](../../2-handbook/swarm/credentials.md#swarm-level-secrets) | -| Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/run/swarm-grafana-secrets/.secret` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container — never on persistent disk, so a reboot fetches it again | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see [hive-level secrets](../../2-handbook/swarm/credentials.md#hive-level-secrets) | -| Grafana's admin password | `swarm-bao-grafana-admin.service` creates it in the swarm secret store at `swarm/services//admin/password` when the store holds none, then copies it out, **on every host that runs Grafana**. Grafana's leaf may create that leaf but not update it, so the store keeps the first value, and the secret publisher can't write it. To rotate, run `bao kv metadata delete secret/swarm/services//admin/password` with a token that may, such as the root token (Grafana's, the publisher's and the controller's can't), then `systemctl restart swarm-bao-grafana-admin`; a plain `bao kv delete` isn't enough, see [the callout xkey's section](credentials.md#the-queues-callout-xkey) | `/run/swarm-grafana-secrets/admin-password` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container, never on persistent disk: `security.admin_password` for a new database, and `grafana cli admin reset-admin-password` in `grafana.service`'s `preStart` for an existing one, on every start | same unit, same path. After a reboot the unit fetches the file again, and Grafana doesn't start until it exists, so it stays down rather than falling back to the upstream default `admin` | -| Grafana's metrics bearer | `swarm-bao-grafana-metrics-bearer.service` creates it in the swarm secret store at `swarm/services//metrics/bearer` when the store holds none, then copies it out, **on every host that runs Grafana**. Same grant as the admin password, create without update. To rotate, run `bao kv metadata delete secret/swarm/services//metrics/bearer` with a token that may, then `systemctl restart swarm-bao-grafana-metrics-bearer` | `/run/swarm-grafana-secrets/metrics-bearer` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container, holding `Bearer `: the metrics datasource's `Authorization` header. The logs datasource sends the same header. `swarm-metrics-auth` on the metrics store's host, and a second instance on the log store's host (`swarm-logs-grafana-auth`), read the stored value themselves, each under its own leaf | same unit, same path. The unit restarts Grafana when the file changes | -| Grafana's datasource encryption key | `swarm-bao-grafana-secret-key.service` stores it in the swarm secret store at `swarm/services//secret-key` when the store holds none, then copies it out, **on every host that runs Grafana**. It stores the key an earlier release generated at `/var/lib/grafana-secret/secret_key` in the container, so the key doesn't change, and mints a new one only on a host with no such file. Same grant as the admin password, create without update. The earlier file stays on disk, unused once the store holds the key. A store that's down or doesn't grant the leaf fails the unit, and Grafana doesn't start | `/run/swarm-grafana-secrets/secret-key` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container: `security.secret_key` | same unit, same path. Grafana doesn't start until the file exists. Changing the key leaves Grafana unable to read the datasource secrets it stored, so never delete the store path to rotate it | -| the swarm controller's webhook secret | `swarm-controller` stores it in the swarm secret store at `swarm/controller/swarm-controller/forge/webhook-secret` when the store holds none: the value of `webhook-secret` in its state directory, or a new one when there is no such file. The state directory file stays on disk and is the fallback while the store is unreachable or its policy doesn't grant the path yet | the swarm secret store; the controller holds it in memory and registers the same value with the forge's hooks | nothing to place: the controller's store leaf already exists. A store that stays unreachable on a start with no state directory file leaves the webhook endpoint answering 503 until the store answers: the controller re-reads it in the background, at most five minutes apart, and registers the forge hooks once it has the secret | -| the metrics store responder's own store leaf (cert + key) | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on the host that runs the secret store, once, subject `deploy.bao.metricsAuthCommonName`. `swarm-bao-metrics-auth-policy.service` writes the `swarm-metrics-auth` role and its read grants on the bearer leaf and on the metrics store forwarder's client secret, nothing else | `/var/lib/swarm-bao-pki/metrics-auth.pem` and `metrics-auth-key.pem` on the store's host, handed to `swarm-metrics-auth` through `LoadCredential` (paths in argv, never values) and read by path by `swarm-victoriametrics-forwarder-oidc` | **the operator** places a leaf with that subject on the metrics store's host and names it in `deploy.victoriametrics.authBaoClientCertFile` / `authBaoClientKeyFile`; evaluation fails naming both until then. If the store is on another host, deploy that host first: the role must exist before the responder's login succeeds, and until it does the gateway refuses every Grafana query (fail closed) | -| the secret publisher's store leaf (cert + key) | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on the host that runs the secret store, once, subject `deploy.bao.secretPublisherCommonName`. `swarm-bao-secret-publisher-policy.service` writes the `swarm-secret-publisher` role: create and update under `swarm/hives/*`, `swarm/services/*` and the controller's client, and read on the `oidc/client` leaves it mints, the controller's client and authelia's five keys alone | `/var/lib/swarm-bao-pki/secret-publisher.pem` and `secret-publisher-key.pem` on the host, handed into authelia's container as nspawn system credentials, which `authelia-swarm-secrets`, `swarm-authelia-bridge` and the forwarder's `swarm-journal-forwarder-oidc` load with `LoadCredential`. nspawn won't start authelia's container while a named file is missing | **the operator** copies the leaf to authelia's host, beside the hive's own store leaf, where `glue-secret-publisher-bao-identity.nix` looks, or names it in `deploy.swarm-secret-publisher.baoClientCertFile` / `baoClientKeyFile` | -| the operator's store leaf (cert + key) | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on every host that runs the secret store, once, subject `deploy.bao.operatorCommonName`. No cert-auth role accepts that subject, so the leaf opens the TLS handshake and logs in to nothing | `/var/lib/swarm-bao-pki/operator.pem` and `operator-key.pem` on the host, `0600` root; the host's `bao` wrapper presents it when the host runs no hive, via `deploy.bao.operatorClientCertFile` / `operatorClientKeyFile`. A store host from before this leaf has no file until `swarm-bao-pki` next runs, and its wrapper sets no client certificate | nothing to place: the leaf is for the shell on the store's host and is never copied to another host | -| the swarm controller's copy of its OIDC secret | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`) | -| a hive's copy of its own OIDC secret | hive-c0re reads it out of the swarm secret store when it first connects to the queue, under the hive's own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | -| the hive collector's copy of its hive's OIDC secret | `swarm-bao-hive-otel-oidc.service` reads it out of the swarm secret store under the hive's own leaf, **on every hive that runs the collector and holds that leaf**; the leaf is the source | `/run/hive-otel-oidc/hive-.secret` on the hive's host tmpfs, 0400 root, loaded by `LoadCredential` — never on persistent disk, so a reboot fetches it again | same unit, same path, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`). Naming a secret file with `otel.clientSecretFile` instead is legacy for a host without a leaf: the build warns, and the fix is to place the leaf from `/var/lib/swarm-bao-pki/hives//` and drop the setting | -| the swarm controller's forge token | the forge's `forgejo-swarm-controller-account`, in-container, once. An attempt that fails or runs past 30 seconds retries after 5 seconds, then at growing intervals up to 5 minutes | the swarm secret store at `swarm/controller/swarm-controller/forge/token`, written by the mint under the forge publisher's leaf, which the forge container loads as a credential; the controller holds it in memory only. `/var/lib/forgejo/swarm-controller-token-name` in the forge container holds its name, the minter's record that it has minted | nothing to place beyond the controller's store leaf. A forge host without the store needs the publisher's leaf placed (`deploy.bao.forgePublisherClientCertFile`) | -| the swarm controller's Grafana token | `swarm-bao-grafana-controller-token.service`, **on the host that runs Grafana**, as Grafana's built-in admin over its socket: it creates the `swarm-controller` service account as a Viewer (and sets it back to Viewer on every run), mints a token for it and writes it to the swarm secret store at `swarm/controller/swarm-controller/grafana/token` under Grafana's leaf. It waits up to 60 seconds for Grafana's socket to answer before its first call. Re-run every ten minutes: it keeps a stored token Grafana accepts, and replaces one Grafana refuses with 401 (or a missing one), deleting every other token of that account first | the swarm secret store only; the controller reads it for `GET /api/alerts` and holds it in memory, re-reading the store once when Grafana answers 401 | nothing to place: Grafana's leaf is already required to run Grafana. Until the unit has run, `/api/alerts` answers 503 naming the empty path | -| an agent's API-key environment (`ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, an ACP provider key) | `hive-agent-backend-env.service`, in the agent's container, on an agent with a store that uses `useApiKey` or the `acp` runtime. When the store holds nothing at `swarm/agents//backend-env` and `backendEnvironmentFile` is a non-empty file, it creates the leaf with the file's exact bytes and `-cas=0`, under the agent's own leaf, which may create that path and not update it. It never rotates a value and never rewrites or deletes the file. To set or replace a key, use **set the API-key environment** in the swarm UI (`PUT /api/hives/{hive}/agents/{agent}/backend-env`): `swarm-controller` overwrites the leaf with its own grant, and the store's value then wins over the file. Its **remove** (`DELETE` on the same path) deletes the leaf, and an agent whose file has content then creates it from that file again | the swarm secret store at that path; the unit writes it to `/run/hive-agent-backend-env/env` (tmpfs, `0600`, root), and `hive-agent` and `hive-subagent-daemon` load that after `backendEnvironmentFile` as `EnvironmentFile=` | the same unit, same path. A variable in the store wins over the same one in the file. With the store down at a start, the file alone carries the agent, as it did before the store held a copy; with an older hive build the agent ignores the leaf and keeps reading the file | -| an agent's proxy settings (`HTTPS_PROXY`, `HTTP_PROXY`, `NO_PROXY`, the credential in the proxy URL) | `hive-agent-proxy-credential.service`, in the agent's container, as the agent user, at start and every 10 minutes: it reads `swarm/agents//proxy-credential` under the agent's own leaf and builds the file from it. It writes nothing while the path holds nothing or the hive's facts name no squid | `/run/hive-agent-proxy-credential/proxy.env` (tmpfs, `0600`, the agent user); the harness, its bash tool, the subagents, the MCP servers and the pollers load it as `EnvironmentFile=-` and read it when they start | With no credential, or on a hive that writes no `HIVE_SQUID_PROXY` and `HIVE_EGRESS_NO_PROXY` fact, nothing writes the file and a unit has no proxy variable and reaches the internet directly. A failed read keeps the file already in place. A unit started before the file appeared goes direct until its next start. The credential is also in the environment of every process those units start, so `env` in the agent's shell prints it | -| an agent's claude login (`.credentials.json`) | `swarm-claude-login.service`, in its container, when an operator completes `claude auth login` from the agent's card in the swarm UI; it stores the file as a new version at `swarm/agents//claude-credentials` under its own leaf, which may create and update that path and read nothing under an agent | the swarm secret store at that path; the agent's `hive-agent` reads it under the agent's own leaf every 30 seconds while it waits for a login and writes it to `~/.claude/.credentials.json` | the same read, co-located or not; nothing for an operator to place beyond the agent's store leaf. When the store's copy and the agent's differ the KV version decides: the agent installs only a version newer than the last it installed (`claude-credentials-version` under the harness dir), because claude rotates the refresh token in its own copy and the stored one goes stale. The agent ignores a stale or equal version, and a relogin writes a newer one — [claude-login.md](../../2-handbook/swarm/claude-login.md) | -| the queue responder's copy of its OIDC secret | the responder reads it out of the swarm secret store itself as it starts, under its own leaf, and again when authelia refuses it | the daemon's memory only; never on disk | same read, wherever authelia runs. Nothing for an operator to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| the queue container's journal forwarder's copy of the queue's OIDC secret | `swarm-journal-forwarder-oidc.service` inside the `swarm-nats` container reads it out of the swarm secret store under the responder's store leaf, and again every 10 minutes | `/run/swarm-journal-forwarder-oidc/client-secret` on the container's tmpfs, root `0400`, never on persistent disk; `LoadCredential` hands it to the collector | same unit, same path. After a reboot the fetch fails fast while the store is down and retries with a delay that grows to 5 minutes; the attempt that succeeds starts the collector, which isn't part of the container's boot, and it resumes from its saved journal cursor. Nothing for an operator to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| the CI runner, Grafana, log store, cache, forge and homeserver containers' journal forwarders' OIDC secrets | `swarm-journal-forwarder-oidc.service` inside each container reads its forwarder client's secret (`swarm..otel.clientId`) out of the swarm secret store under the store leaf the host already holds for that service, handed in as a system credential (bound in read-only, for the homeserver's), and again every 10 minutes | `/run/swarm-journal-forwarder-oidc/client-secret` on the container's tmpfs, root `0400`, never on persistent disk; `LoadCredential` hands it to the collector | same unit, same path. Nothing for an operator to place beyond those leaves (`deploy.bao.ciRunnerClientCertFile`, `deploy.bao.grafanaOidcClientCertFile`, `deploy.victorialogs.authBaoClientCertFile`, `deploy.bao.forgePublisherClientCertFile`, `deploy.bao.atticClientCertFile`, `deploy.matrix.ctlBaoClientCertFile`) | -| the metrics store container's journal forwarder's OIDC secret | `swarm-victoriametrics-forwarder-oidc.service` on the store's host reads its forwarder client's secret (`swarm.victoriametrics.otel.clientId`) out of the swarm secret store under the metrics store responder's leaf, and again every 10 minutes: the container has no network to reach the store | `/run/swarm-victoriametrics-forwarder-oidc/.secret` on the host (tmpfs), root `0400`, never on persistent disk; bound into the container read-only, where `LoadCredential` hands it to the collector | same unit, same path. Nothing for an operator to place beyond that leaf (`deploy.victoriametrics.authBaoClientCertFile`) | -| the CI run poller's forge token | `swarm-controller`, with the forge's admin API, for its `ci-metrics` account, at start and in a five-minute pass | the swarm secret store at `swarm/services/ci-metrics/forge/token`; `swarm-ci-metrics` reads it once per pass under its own leaf and holds it in memory for that pass. When the store's token no longer matches the forge's, the controller's next pass mints a new one; until then the poller's forge calls answer 401, the unit fails and no cursor moves | nothing to place beyond the poller's leaf (`deploy.bao.ciMetricsClientCertFile`), on a host without the store | -| the CI run poller's copy of its OIDC secret | authelia's mint, which creates it in the swarm secret store | `swarm-ci-metrics` reads `swarm/services//oidc/client` once per pass under its own leaf, and again when authelia refuses it; memory only. While it differs from authelia's, the token request fails, nothing gets pushed and no cursor moves | same leaf, nothing else | -| authelia's container's journal forwarder's OIDC secret | authelia's mint, in its container, like every client's secret (`swarm.authelia.otel.clientId`). It creates it at `swarm/services//oidc/client`, and `swarm-journal-forwarder-oidc.service` in the same container reads it back under the secret publisher's leaf, at start and every 10 minutes | `/run/swarm-journal-forwarder-oidc/client-secret` on the container's tmpfs, root `0400`; `LoadCredential` hands it to the collector. | none: the forwarder runs only in authelia's container, and is off there without the publisher's leaf | -| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | -| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | -| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed; the verb stores a seed it finds at `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed` as is, then removes the file | the swarm secret store: the seeds at `swarm/services/swarm-nats/callout/{user,issuer}-seed`, read by the `swarm-nats-auth` role alone; the public halves at `swarm/public/swarm-nats/callout-{user,issuer}` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| queue auth-callout xkey (curve seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed | the swarm secret store: the seed at `swarm/services/swarm-nats/callout/xkey-seed`, read by the `swarm-nats-auth` role alone; the public half at `swarm/public/swarm-nats/callout-xkey` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| the queue's TLS key, copied for the gateway's `/metrics` vhost | `swarm-nats-gateway-cert.service` on the queue's host, where the gateway uses self-signed TLS: a copy of the queue's bao-PKI TLS leaf (`/var/lib/swarm-nats-tls/`) once `swarm-bao-nats-tls` has issued it, else a throwaway 30-day self-signed pair for the queue's name until then | `/var/lib/swarm-nats-gateway-tls/{cert,key}.pem`; the key is `0640` group `nginx`, read by nginx for the queue's `/metrics` vhost. The queue's own key stays `0600` root | the copy is redone on each leaf renewal (`swarm-bao-nats-tls` starts the unit, which queues a non-blocking nginx reload when nginx is running); a restart before any leaf exists keeps the pair already there. Until `swarm-bao-nats-tls` issues the leaf the vhost serves the throwaway cert, so the collector's scrape fails TLS. Nothing for an operator to place | -| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | -| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | +| secret | generated by | lives at | hive elsewhere | +| ----------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name and needs a store identity | +| authelia session, JWT, storage-encryption and OIDC HMAC keys | `authelia-swarm-secrets` in authelia's container. It stores each in the swarm secret store at `swarm/services/swarm-authelia/keys/` when the store holds none: the value of the key an earlier release generated, or a new one when there is no such file. The files stay on disk. While the store is down, refuses the login, or fails the read of a leaf (a refusal, a timeout, an error), the unit runs on the files, logs the store's error, and writes nothing to the store; the unit seeds or mints only a leaf the store reports as absent. A container with no secret publisher identity generates the files, as before | `/var/lib/authelia-swarm/{session,jwt,storage-encryption,oidc-hmac}.key` | nothing to place: the publisher's store leaf is the one in the row below. A store that stays unreachable on a start with no key file keeps authelia from starting | +| authelia OIDC issuer key (RSA) | same unit, same store path (`oidc-issuer`) | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | +| swarm appservice registration (`as_token`, `hs_token`) | `swarm-matrix-ctl appservice render` in `hive-matrix`. The store's pair wins; a store holding none takes the tokens an earlier version kept in the container, so the value doesn't change by moving. Only a store that answers, holds none, and a container with no such files gets a new pair | `/run/swarm-matrix-appservice/swarm.yaml` in `hive-matrix` (tmpfs), which tuwunel loads through `LoadCredential`. The earlier `/var/lib/swarm-matrix-appservice/{as-token,hs-token,swarm.yaml}` stay, unused, and bao wins if they differ; the store path is `swarm/controller/swarm-controller/matrix/appservice-registration` | nothing to place: matrix-ctl's store leaf is the one the container binds. A store that's down keeps the homeserver down and mints nothing, and `render` retries every 30 seconds for 24 hours | +| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | the swarm secret store, at the path its reader fetches: `swarm/hives//oidc/client`, `swarm/controller/swarm-controller/oidc/client` or `swarm/services//oidc/client`. The bridge reads its own into `/run/swarm-authelia-bridge/client-secret` before each start. A container with no secret publisher identity keeps it in `/var/lib/authelia-swarm/oidc-clients/.secret`; a file an earlier release left seeds the store once when it holds no value, and stays unused | nothing to place: readers fetch it from the store ([how](../../2-handbook/swarm/credentials.md#hive-level-secrets)) | +| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | +| the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/run/swarm-otel-oidc/.secret` on the host's tmpfs, bind-mounted read-only into the `swarm-otel` container — never on persistent disk, so a reboot fetches it again | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see [swarm-level secrets](../../2-handbook/swarm/credentials.md#swarm-level-secrets) | +| Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/run/swarm-grafana-secrets/.secret` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container — never on persistent disk, so a reboot fetches it again | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see [hive-level secrets](../../2-handbook/swarm/credentials.md#hive-level-secrets) | +| Grafana's admin password | `swarm-bao-grafana-admin.service` creates it in the swarm secret store at `swarm/services//admin/password` when the store holds none, then copies it out, **on every host that runs Grafana**. Grafana's leaf may create that leaf but not update it, so the store keeps the first value, and the secret publisher can't write it. To rotate, run `bao kv metadata delete secret/swarm/services//admin/password` with a token that may, such as the root token (Grafana's, the publisher's and the controller's can't), then `systemctl restart swarm-bao-grafana-admin`; a plain `bao kv delete` isn't enough, see [the callout xkey's section](credentials.md#the-queues-callout-xkey) | `/run/swarm-grafana-secrets/admin-password` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container, never on persistent disk: `security.admin_password` for a new database, and `grafana cli admin reset-admin-password` in `grafana.service`'s `preStart` for an existing one, on every start | same unit, same path. After a reboot the unit fetches the file again, and Grafana doesn't start until it exists, so it stays down rather than falling back to the upstream default `admin` | +| Grafana's metrics bearer | `swarm-bao-grafana-metrics-bearer.service` creates it in the swarm secret store at `swarm/services//metrics/bearer` when the store holds none, then copies it out, **on every host that runs Grafana**. Same grant as the admin password, create without update. To rotate, run `bao kv metadata delete secret/swarm/services//metrics/bearer` with a token that may, then `systemctl restart swarm-bao-grafana-metrics-bearer` | `/run/swarm-grafana-secrets/metrics-bearer` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container, holding `Bearer `: the metrics datasource's `Authorization` header. The logs datasource sends the same header. `swarm-metrics-auth` on the metrics store's host, and a second instance on the log store's host (`swarm-logs-grafana-auth`), read the stored value themselves, each under its own leaf | same unit, same path. The unit restarts Grafana when the file changes | +| Grafana's datasource encryption key | `swarm-bao-grafana-secret-key.service` stores it in the swarm secret store at `swarm/services//secret-key` when the store holds none, then copies it out, **on every host that runs Grafana**. It stores the key an earlier release generated at `/var/lib/grafana-secret/secret_key` in the container, so the key doesn't change, and mints a new one only on a host with no such file. Same grant as the admin password, create without update. The earlier file stays on disk, unused once the store holds the key. A store that's down or doesn't grant the leaf fails the unit, and Grafana doesn't start | `/run/swarm-grafana-secrets/secret-key` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container: `security.secret_key` | same unit, same path. Grafana doesn't start until the file exists. Changing the key leaves Grafana unable to read the datasource secrets it stored, so never delete the store path to rotate it | +| the swarm controller's webhook secret | `swarm-controller` stores it in the swarm secret store at `swarm/controller/swarm-controller/forge/webhook-secret` when the store holds none: the value of `webhook-secret` in its state directory, or a new one when there is no such file. The state directory file stays on disk and is the fallback while the store is unreachable or its policy doesn't grant the path yet | the swarm secret store; the controller holds it in memory and registers the same value with the forge's hooks | nothing to place: the controller's store leaf already exists. A store that stays unreachable on a start with no state directory file leaves the webhook endpoint answering 503 until the store answers: the controller re-reads it in the background, at most five minutes apart, and registers the forge hooks once it has the secret | +| the metrics store responder's own store leaf (cert + key) | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on the host that runs the secret store, once, subject `deploy.bao.metricsAuthCommonName`. `swarm-bao-metrics-auth-policy.service` writes the `swarm-metrics-auth` role and its read grants on the bearer leaf and on the metrics store forwarder's client secret, nothing else | `/var/lib/swarm-bao-pki/metrics-auth.pem` and `metrics-auth-key.pem` on the store's host, handed to `swarm-metrics-auth` through `LoadCredential` (paths in argv, never values) and read by path by `swarm-victoriametrics-forwarder-oidc` | **the operator** places a leaf with that subject on the metrics store's host and names it in `deploy.victoriametrics.authBaoClientCertFile` / `authBaoClientKeyFile`; evaluation fails naming both until then. If the store is on another host, deploy that host first: the role must exist before the responder's login succeeds, and until it does the gateway refuses every Grafana query (fail closed) | +| the secret publisher's store leaf (cert + key) | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on the host that runs the secret store, once, subject `deploy.bao.secretPublisherCommonName`. `swarm-bao-secret-publisher-policy.service` writes the `swarm-secret-publisher` role: create and update under `swarm/hives/*`, `swarm/services/*` and the controller's client, and read on the `oidc/client` leaves it mints, the controller's client and authelia's five keys alone | `/var/lib/swarm-bao-pki/secret-publisher.pem` and `secret-publisher-key.pem` on the host, handed into authelia's container as nspawn system credentials, which `authelia-swarm-secrets`, `swarm-authelia-bridge` and the forwarder's `swarm-journal-forwarder-oidc` load with `LoadCredential`. nspawn won't start authelia's container while a named file is missing | **the operator** copies the leaf to authelia's host, beside the hive's own store leaf, where `glue-secret-publisher-bao-identity.nix` looks, or names it in `deploy.swarm-secret-publisher.baoClientCertFile` / `baoClientKeyFile` | +| the operator's store leaf (cert + key) | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on every host that runs the secret store, once, subject `deploy.bao.operatorCommonName`. No cert-auth role accepts that subject, so the leaf opens the TLS handshake and logs in to nothing | `/var/lib/swarm-bao-pki/operator.pem` and `operator-key.pem` on the host, `0600` root; the host's `bao` wrapper presents it when the host runs no hive, via `deploy.bao.operatorClientCertFile` / `operatorClientKeyFile`. A store host from before this leaf has no file until `swarm-bao-pki` next runs, and its wrapper sets no client certificate | nothing to place: the leaf is for the shell on the store's host and is never copied to another host | +| the swarm controller's copy of its OIDC secret | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`) | +| a hive's copy of its own OIDC secret | hive-c0re reads it out of the swarm secret store when it first connects to the queue, under the hive's own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | +| the hive collector's copy of its hive's OIDC secret | `swarm-bao-hive-otel-oidc.service` reads it out of the swarm secret store under the hive's own leaf, **on every hive that runs the collector and holds that leaf**; the leaf is the source | `/run/hive-otel-oidc/hive-.secret` on the hive's host tmpfs, 0400 root, loaded by `LoadCredential` — never on persistent disk, so a reboot fetches it again | same unit, same path, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`). Naming a secret file with `otel.clientSecretFile` instead is legacy for a host without a leaf: the build warns, and the fix is to place the leaf from `/var/lib/swarm-bao-pki/hives//` and drop the setting | +| the swarm controller's forge token | the forge's `forgejo-swarm-controller-account`, in-container, once. An attempt that fails or runs past 30 seconds retries after 5 seconds, then at growing intervals up to 5 minutes | the swarm secret store at `swarm/controller/swarm-controller/forge/token`, written by the mint under the forge publisher's leaf, which the forge container loads as a credential; the controller holds it in memory only. `/var/lib/forgejo/swarm-controller-token-name` in the forge container holds its name, the minter's record that it has minted | nothing to place beyond the controller's store leaf. A forge host without the store needs the publisher's leaf placed (`deploy.bao.forgePublisherClientCertFile`) | +| the swarm controller's Grafana token | `swarm-bao-grafana-controller-token.service`, **on the host that runs Grafana**, as Grafana's built-in admin over its socket: it creates the `swarm-controller` service account as a Viewer (and sets it back to Viewer on every run), mints a token for it and writes it to the swarm secret store at `swarm/controller/swarm-controller/grafana/token` under Grafana's leaf. It waits up to 60 seconds for Grafana's socket to answer before its first call. Re-run every ten minutes: it keeps a stored token Grafana accepts, and replaces one Grafana refuses with 401 (or a missing one), deleting every other token of that account first | the swarm secret store only; the controller reads it for `GET /api/alerts` and holds it in memory, re-reading the store once when Grafana answers 401 | nothing to place: Grafana's leaf is already required to run Grafana. Until the unit has run, `/api/alerts` answers 503 naming the empty path | +| an agent's API-key environment (`ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, an ACP provider key) | `hive-agent-backend-env.service`, in the agent's container, on an agent with a store that uses `useApiKey` or the `acp` runtime. When the store holds nothing at `swarm/agents//backend-env` and `backendEnvironmentFile` is a non-empty file, it creates the leaf with the file's exact bytes and `-cas=0`, under the agent's own leaf, which may create that path and not update it. It never rotates a value and never rewrites or deletes the file. To set or replace a key, use **set the API-key environment** in the swarm UI (`PUT /api/hives/{hive}/agents/{agent}/backend-env`): `swarm-controller` overwrites the leaf with its own grant, and the store's value then wins over the file. Its **remove** (`DELETE` on the same path) deletes the leaf, and an agent whose file has content then creates it from that file again | the swarm secret store at that path; the unit writes it to `/run/hive-agent-backend-env/env` (tmpfs, `0600`, root), and `hive-agent` and `hive-subagent-daemon` load that after `backendEnvironmentFile` as `EnvironmentFile=` | the same unit, same path. A variable in the store wins over the same one in the file. With the store down at a start, the file alone carries the agent, as it did before the store held a copy; with an older hive build the agent ignores the leaf and keeps reading the file | +| an agent's proxy settings (`HTTPS_PROXY`, `HTTP_PROXY`, `NO_PROXY`, the credential in the proxy URL) | `hive-agent-proxy-credential.service`, in the agent's container, as the agent user, at start and every 10 minutes: it reads `swarm/agents//proxy-credential` under the agent's own leaf and builds the file from it. It writes nothing while the path holds nothing or the hive's facts name no squid | `/run/hive-agent-proxy-credential/proxy.env` (tmpfs, `0600`, the agent user); the harness, its bash tool, the subagents, the MCP servers and the pollers load it as `EnvironmentFile=-` and read it when they start | With no credential, or on a hive that writes no `HIVE_SQUID_PROXY` and `HIVE_EGRESS_NO_PROXY` fact, nothing writes the file and a unit has no proxy variable and reaches the internet without the proxy; the host still redirects the web traffic of a `restricted` or `log` agent into squid (see [the network reference](../networking/network.md#intercepted-web-traffic)). A failed read keeps the file already in place. A unit started before the file appeared goes direct until its next start. The credential is also in the environment of every process those units start, so `env` in the agent's shell prints it | +| an agent's claude login (`.credentials.json`) | `swarm-claude-login.service`, in its container, when an operator completes `claude auth login` from the agent's card in the swarm UI; it stores the file as a new version at `swarm/agents//claude-credentials` under its own leaf, which may create and update that path and read nothing under an agent | the swarm secret store at that path; the agent's `hive-agent` reads it under the agent's own leaf every 30 seconds while it waits for a login and writes it to `~/.claude/.credentials.json` | the same read, co-located or not; nothing for an operator to place beyond the agent's store leaf. When the store's copy and the agent's differ the KV version decides: the agent installs only a version newer than the last it installed (`claude-credentials-version` under the harness dir), because claude rotates the refresh token in its own copy and the stored one goes stale. The agent ignores a stale or equal version, and a relogin writes a newer one — [claude-login.md](../../2-handbook/swarm/claude-login.md) | +| the queue responder's copy of its OIDC secret | the responder reads it out of the swarm secret store itself as it starts, under its own leaf, and again when authelia refuses it | the daemon's memory only; never on disk | same read, wherever authelia runs. Nothing for an operator to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| the queue container's journal forwarder's copy of the queue's OIDC secret | `swarm-journal-forwarder-oidc.service` inside the `swarm-nats` container reads it out of the swarm secret store under the responder's store leaf, and again every 10 minutes | `/run/swarm-journal-forwarder-oidc/client-secret` on the container's tmpfs, root `0400`, never on persistent disk; `LoadCredential` hands it to the collector | same unit, same path. After a reboot the fetch fails fast while the store is down and retries with a delay that grows to 5 minutes; the attempt that succeeds starts the collector, which isn't part of the container's boot, and it resumes from its saved journal cursor. Nothing for an operator to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| the CI runner, Grafana, log store, cache, forge and homeserver containers' journal forwarders' OIDC secrets | `swarm-journal-forwarder-oidc.service` inside each container reads its forwarder client's secret (`swarm..otel.clientId`) out of the swarm secret store under the store leaf the host already holds for that service, handed in as a system credential (bound in read-only, for the homeserver's), and again every 10 minutes | `/run/swarm-journal-forwarder-oidc/client-secret` on the container's tmpfs, root `0400`, never on persistent disk; `LoadCredential` hands it to the collector | same unit, same path. Nothing for an operator to place beyond those leaves (`deploy.bao.ciRunnerClientCertFile`, `deploy.bao.grafanaOidcClientCertFile`, `deploy.victorialogs.authBaoClientCertFile`, `deploy.bao.forgePublisherClientCertFile`, `deploy.bao.atticClientCertFile`, `deploy.matrix.ctlBaoClientCertFile`) | +| the metrics store container's journal forwarder's OIDC secret | `swarm-victoriametrics-forwarder-oidc.service` on the store's host reads its forwarder client's secret (`swarm.victoriametrics.otel.clientId`) out of the swarm secret store under the metrics store responder's leaf, and again every 10 minutes: the container has no network to reach the store | `/run/swarm-victoriametrics-forwarder-oidc/.secret` on the host (tmpfs), root `0400`, never on persistent disk; bound into the container read-only, where `LoadCredential` hands it to the collector | same unit, same path. Nothing for an operator to place beyond that leaf (`deploy.victoriametrics.authBaoClientCertFile`) | +| the CI run poller's forge token | `swarm-controller`, with the forge's admin API, for its `ci-metrics` account, at start and in a five-minute pass | the swarm secret store at `swarm/services/ci-metrics/forge/token`; `swarm-ci-metrics` reads it once per pass under its own leaf and holds it in memory for that pass. When the store's token no longer matches the forge's, the controller's next pass mints a new one; until then the poller's forge calls answer 401, the unit fails and no cursor moves | nothing to place beyond the poller's leaf (`deploy.bao.ciMetricsClientCertFile`), on a host without the store | +| the CI run poller's copy of its OIDC secret | authelia's mint, which creates it in the swarm secret store | `swarm-ci-metrics` reads `swarm/services//oidc/client` once per pass under its own leaf, and again when authelia refuses it; memory only. While it differs from authelia's, the token request fails, nothing gets pushed and no cursor moves | same leaf, nothing else | +| authelia's container's journal forwarder's OIDC secret | authelia's mint, in its container, like every client's secret (`swarm.authelia.otel.clientId`). It creates it at `swarm/services//oidc/client`, and `swarm-journal-forwarder-oidc.service` in the same container reads it back under the secret publisher's leaf, at start and every 10 minutes | `/run/swarm-journal-forwarder-oidc/client-secret` on the container's tmpfs, root `0400`; `LoadCredential` hands it to the collector. | none: the forwarder runs only in authelia's container, and is off there without the publisher's leaf | +| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | +| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | +| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed; the verb stores a seed it finds at `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed` as is, then removes the file | the swarm secret store: the seeds at `swarm/services/swarm-nats/callout/{user,issuer}-seed`, read by the `swarm-nats-auth` role alone; the public halves at `swarm/public/swarm-nats/callout-{user,issuer}` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| queue auth-callout xkey (curve seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed | the swarm secret store: the seed at `swarm/services/swarm-nats/callout/xkey-seed`, read by the `swarm-nats-auth` role alone; the public half at `swarm/public/swarm-nats/callout-xkey` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| the queue's TLS key, copied for the gateway's `/metrics` vhost | `swarm-nats-gateway-cert.service` on the queue's host, where the gateway uses self-signed TLS: a copy of the queue's bao-PKI TLS leaf (`/var/lib/swarm-nats-tls/`) once `swarm-bao-nats-tls` has issued it, else a throwaway 30-day self-signed pair for the queue's name until then | `/var/lib/swarm-nats-gateway-tls/{cert,key}.pem`; the key is `0640` group `nginx`, read by nginx for the queue's `/metrics` vhost. The queue's own key stays `0600` root | the copy is redone on each leaf renewal (`swarm-bao-nats-tls` starts the unit, which queues a non-blocking nginx reload when nginx is running); a restart before any leaf exists keeps the pair already there. Until `swarm-bao-nats-tls` issues the leaf the vhost serves the throwaway cert, so the collector's scrape fails TLS. Nothing for an operator to place | +| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | +| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | ### Hive-level — one of each per hive @@ -165,7 +165,7 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](../../2-hand | the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` | | the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` | | this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives//forge/token`, for the `hive-` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity for each forge read, and hands it to the `git` commands it runs. It keeps no copy on disk | -| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.deploy.hive-controller.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `unrestricted` beside it lists the users whose agent holds the swarm's `unrestricted_internet` flag, names only; it lists every agent the hive never read as off `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; an agent without one sends none, so it goes direct | +| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.deploy.hive-controller.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `:` lines, SHA-512 crypt hashes only, renamed into place when it changed. `unrestricted` beside it lists the users whose agent holds the swarm's `unrestricted_internet` flag, names only; it lists every agent the hive never read as off `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; an agent without one sends none, so its web traffic skips the proxy and, for a `restricted` or `log` agent, squid intercepts it | | the squid self-check's password | `swarm-controller` mints `swarm/hives//squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Both self-check users (`selfcheck.squid` and the always-unrestricted `selfcheck-unrestricted.squid`) use it, so whoever holds it has unrestricted egress through this squid Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks | ### Store identities, one per principal diff --git a/hive-c0re/src/workers/squid_allow.rs b/hive-c0re/src/workers/squid_allow.rs index 0b12a0ff2..8f7ce4c03 100644 --- a/hive-c0re/src/workers/squid_allow.rs +++ b/hive-c0re/src/workers/squid_allow.rs @@ -8,9 +8,9 @@ //! - [`ENV_ALLOWED_FILE`]: the swarm-wide names, one per line, which //! `nix/host-modules/hive-squid.nix` reads as part of `allowed_dst`. //! - [`ENV_AGENT_ALLOW_FILE`]: squid configuration `include`d where the base -//! list is allowed: for each of this hive's agents with entries, an `acl` -//! for its squid user, an `acl` for its names and the `http_access allow` -//! joining them, then the rules of the agents in `log` mode ([`log_rules`]). +//! list is allowed: per agent with entries, `acl`s for its users and names +//! and an `http_access allow` for the proxy port and one for the intercept +//! ports, then the rules of the agents in `log` mode ([`log_rules`]). //! //! Squid reads both when it parses its configuration, so the path unit in //! `nix/host-modules/hive-squid.nix` reloads it when either changes. @@ -328,7 +328,8 @@ fn log_agents(agents: &[String], perms: &permissions::Stored) -> Vec { /// the agent is allowed never carries the mark. The mark is the last ACL of its /// rule: squid applies `annotate_transaction` when it is reached, whatever the /// rule decides after it. Everything in squid's configuration before the -/// include is a deny that applies to these agents as to any other. +/// include is a deny that applies to these agents as to any other. Each agent +/// has a rule for the proxy port and one for the intercept ports (`icpt`). fn log_rules(logging: &[String]) -> String { if logging.is_empty() { return String::new(); @@ -338,7 +339,9 @@ fn log_rules(logging: &[String]) -> String { let _ = write!( rules, "acl agent_log_{agent} proxy_auth {agent}\n\ - http_access allow agent_log_{agent} !allowed_dst {MARK_ACL}\n" + acl agent_log_{agent}_ix ext_user {agent}\n\ + http_access allow !icpt agent_log_{agent} !allowed_dst {MARK_ACL}\n\ + http_access allow icpt agent_log_{agent}_ix !allowed_dst {MARK_ACL}\n" ); } rules @@ -393,8 +396,10 @@ fn render( agent_count += names.len(); agent_rules.push(format!( "acl agent_allow_{agent} proxy_auth {agent}\n\ + acl agent_allow_{agent}_ix ext_user {agent}\n\ acl agent_allow_{agent}_dst dstdomain {}\n\ - http_access allow agent_allow_{agent} agent_allow_{agent}_dst\n", + http_access allow !icpt agent_allow_{agent} agent_allow_{agent}_dst\n\ + http_access allow icpt agent_allow_{agent}_ix agent_allow_{agent}_dst\n", names.join(" ") )); } @@ -482,8 +487,10 @@ mod tests { r.agent_allow, "# source: swarm\n\ acl agent_allow_atlas proxy_auth atlas\n\ + acl agent_allow_atlas_ix ext_user atlas\n\ acl agent_allow_atlas_dst dstdomain crates.io pypi.org\n\ - http_access allow agent_allow_atlas agent_allow_atlas_dst\n" + http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n\ + http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n" ); let counts = r.counts.expect("a swarm list"); assert_eq!((counts.swarm, counts.agent), (0, 2)); @@ -530,8 +537,10 @@ mod tests { r.agent_allow, "# source: swarm\n\ acl agent_allow_atlas proxy_auth atlas\n\ + acl agent_allow_atlas_ix ext_user atlas\n\ acl agent_allow_atlas_dst dstdomain crates.io\n\ - http_access allow agent_allow_atlas agent_allow_atlas_dst\n" + http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n\ + http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n" ); // Control: the same entries survive when nothing is always allowed. let bare = render( @@ -586,19 +595,23 @@ mod tests { text, "# source: swarm\n\ acl agent_allow_atlas proxy_auth atlas\n\ + acl agent_allow_atlas_ix ext_user atlas\n\ acl agent_allow_atlas_dst dstdomain pypi.org\n\ - http_access allow agent_allow_atlas agent_allow_atlas_dst\n\ + http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n\ + http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n\ acl log_mark annotate_transaction would_deny=1\n\ acl agent_log_uma proxy_auth uma\n\ - http_access allow agent_log_uma !allowed_dst log_mark\n" + acl agent_log_uma_ix ext_user uma\n\ + http_access allow !icpt agent_log_uma !allowed_dst log_mark\n\ + http_access allow icpt agent_log_uma_ix !allowed_dst log_mark\n" ); // The rule of an agent with its own names also follows them. let both = log_list(&["atlas"]); let own = both - .find("http_access allow agent_allow_atlas ") + .find("http_access allow !icpt agent_allow_atlas ") .expect("own rule"); let log = both - .find("http_access allow agent_log_atlas ") + .find("http_access allow !icpt agent_log_atlas ") .expect("log rule"); assert!(own < log, "{both}"); // Control: with no log agent there is no rule and no mark. @@ -616,9 +629,10 @@ mod tests { let text = log_list(&["atlas", "iris", "uma"]); let rules: Vec<&str> = text .lines() - .filter(|l| l.starts_with("http_access allow agent_log_")) + .filter(|l| l.starts_with("http_access allow ") && l.contains(" agent_log_")) .collect(); - assert_eq!(rules.len(), 3, "{text}"); + // One rule for the proxy port and one for the intercept ports per agent. + assert_eq!(rules.len(), 6, "{text}"); for rule in rules { assert!(rule.ends_with(" !allowed_dst log_mark"), "{rule}"); } @@ -698,7 +712,8 @@ mod tests { .expect("renders"); let text = text(&d.agent_allow()); assert!(text.starts_with(SEED_MARKER), "{text}"); - assert!(text.contains("http_access allow agent_log_uma !allowed_dst log_mark")); + assert!(text.contains("http_access allow !icpt agent_log_uma !allowed_dst log_mark")); + assert!(text.contains("http_access allow icpt agent_log_uma_ix !allowed_dst log_mark")); } struct Dir(tempfile::TempDir); @@ -786,8 +801,10 @@ mod tests { text(&d.agent_allow()), "# source: swarm\n\ acl agent_allow_atlas proxy_auth atlas\n\ + acl agent_allow_atlas_ix ext_user atlas\n\ acl agent_allow_atlas_dst dstdomain pypi.org\n\ - http_access allow agent_allow_atlas agent_allow_atlas_dst\n" + http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n\ + http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n" ); } diff --git a/hive-c0re/src/workers/squid_deny.rs b/hive-c0re/src/workers/squid_deny.rs index a13e026d4..7322dd31d 100644 --- a/hive-c0re/src/workers/squid_deny.rs +++ b/hive-c0re/src/workers/squid_deny.rs @@ -169,6 +169,12 @@ struct Rendered { /// The destination ACL comes first in each rule: squid checks ACLs left to /// right, so a swarm-wide rule never reaches a `proxy_auth` ACL, which would /// answer an anonymous client 407 instead of the 403. +/// +/// A per-agent name gets two rules. For a request through the proxy port the +/// agent is the one whose credential came with it; `!icpt` comes first, since a +/// `proxy_auth` ACL evaluated on an intercepted request only logs a warning. +/// For an intercepted request the agent is the user the `hive_agent` helper +/// named (`ext_user`). fn render(list: &DenyList, agents: &[String]) -> Rendered { let mut skipped = Vec::new(); let mut swarm_wide = Vec::new(); @@ -217,7 +223,9 @@ fn render(list: &DenyList, agents: &[String]) -> Rendered { rules, "acl deny_agent_{agent}_dst dstdomain {}\n\ acl deny_agent_{agent} proxy_auth {agent}\n\ - http_access deny deny_agent_{agent}_dst deny_agent_{agent} {MARK_ACL}\n", + acl deny_agent_{agent}_ix ext_user {agent}\n\ + http_access deny deny_agent_{agent}_dst !icpt deny_agent_{agent} {MARK_ACL}\n\ + http_access deny deny_agent_{agent}_dst icpt deny_agent_{agent}_ix {MARK_ACL}\n", names.join(" ") ); } @@ -292,7 +300,9 @@ mod tests { format!( "{MARK_LINE}acl deny_agent_atlas_dst dstdomain crates.io pypi.org\n\ acl deny_agent_atlas proxy_auth atlas\n\ - http_access deny deny_agent_atlas_dst deny_agent_atlas deny_hit_mark\n" + acl deny_agent_atlas_ix ext_user atlas\n\ + http_access deny deny_agent_atlas_dst !icpt deny_agent_atlas deny_hit_mark\n\ + http_access deny deny_agent_atlas_dst icpt deny_agent_atlas_ix deny_hit_mark\n" ) ); assert_eq!((r.swarm, r.agent), (0, 2)); diff --git a/hive-priv/README.md b/hive-priv/README.md index dde12b67d..0b96b24ab 100644 --- a/hive-priv/README.md +++ b/hive-priv/README.md @@ -37,9 +37,11 @@ not the whole daemon-shared crate. `container` (lifecycle), `container_config` (nspawn flags, systemd drop-ins), `storage` (btrfs subvolumes, quotas, snapshots, disk measurement), `files` (pause marker, socket dir, legacy tmpfiles) and `infra` (gateway nginx, forge -admin, infra containers). `egress` holds three sub-commands that never -touch the socket, `egress-load`, `egress-pin` and `egress-lease`; the `hive-egress` -unit, udev, a timer and dnsmasq run them to fill the sets of the anti-spoofing table (see +admin, infra containers). `egress` holds four sub-commands that never +touch the socket, `egress-load`, `egress-pin`, `egress-lease` and `egress-whois`; the +`hive-egress` unit, udev, a timer and dnsmasq run the first three to fill the sets of +the anti-spoofing table and the interception exemptions, and squid runs the last to +name the agent of an intercepted flow (see `docs/4-implementation/networking/network.md`). Helpers more than one family calls live in `shared`. Each module's `//!` header lists its ops. diff --git a/hive-priv/src/egress.rs b/hive-priv/src/egress.rs index 37d5b4232..6adfc5971 100644 --- a/hive-priv/src/egress.rs +++ b/hive-priv/src/egress.rs @@ -1,6 +1,6 @@ //! The per-port bindings behind the `bridge hive_egress` anti-spoofing table -//! (`nix/host-modules/hive-egress.nix`), as three sub-commands run by systemd -//! units and by dnsmasq, never through the socket. +//! (`nix/host-modules/hive-egress.nix`), as four sub-commands run by systemd +//! units, by dnsmasq and by squid, never through the socket. //! //! - `egress-load ` replaces the tables with the ruleset, the //! current bindings and the leases of dnsmasq's lease file, in one nft @@ -14,6 +14,8 @@ //! still unbound. //! - `egress-lease [ []]` is dnsmasq's `dhcp-script`: //! it keeps the table's set of leased MAC and IPv4 pairs in step. +//! - `egress-whois` is squid's external ACL helper: it answers a source address +//! with the agent that holds it ([`agent_of`]). //! //! A port's owner is found from the host side: the port's peer index is the //! index of one link in a registered container's network namespace (the leader @@ -36,6 +38,9 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use anyhow::{Context as _, Result, bail}; +/// The sub-command squid runs to learn which agent holds a source address. +pub(crate) const WHOIS: &str = "egress-whois"; + /// The sub-command that binds ports. pub(crate) const PIN: &str = "egress-pin"; @@ -72,9 +77,19 @@ const STATE: &str = "/run/hive-egress-pins"; /// `HIVE_EGRESS_LOCK` names no other file. const LOCK: &str = "/run/lock/hive-egress-pin.lock"; +/// The file of the unrestricted agents' names, one per line, that +/// `HIVE_EGRESS_UNRESTRICTED` names. Unset or unreadable lists no agent. +const UNRESTRICTED_ENV: &str = "HIVE_EGRESS_UNRESTRICTED"; + +/// The prefix of an agent container's machine name. +const AGENT_PREFIX: &str = "h-"; + /// The nft table the commands write into. const TABLE: &str = "bridge hive_egress"; +/// The table whose `direct_mac` set the commands fill. +const INET_TABLE: &str = "inet hive_egress"; + /// An Ethernet address. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Mac([u8; 6]); @@ -264,8 +279,27 @@ fn is_plain_name(name: &str) -> bool { .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) } -/// The nft input that adds the bindings `pins` to the two sets. -pub(crate) fn pin_elements(pins: &[Pin]) -> String { +/// The MACs of the ports squid does not intercept: those of containers that are +/// not agents (`h-`), and of the agents `unrestricted` names. A MAC in no +/// binding is in none of them, so a flow from an unknown source is intercepted +/// and squid refuses it. +pub(crate) fn direct_macs(pins: &[Pin], unrestricted: &[String]) -> Vec { + let mut macs: Vec = pins + .iter() + .filter(|p| match p.machine.strip_prefix(AGENT_PREFIX) { + Some(agent) => unrestricted.iter().any(|u| u == agent), + None => true, + }) + .map(|p| p.mac) + .collect(); + macs.sort_by_key(|m| m.0); + macs.dedup(); + macs +} + +/// The nft input that adds the bindings `pins` to the two sets of the bridge +/// table and the [`direct_macs`] to the exemption set of the inet table. +pub(crate) fn pin_elements(pins: &[Pin], unrestricted: &[String]) -> String { let mut script = String::new(); for Pin { port, mac, .. } in pins { let _ = writeln!( @@ -275,18 +309,93 @@ pub(crate) fn pin_elements(pins: &[Pin]) -> String { mac.as_integer() ); } + for mac in direct_macs(pins, unrestricted) { + let _ = writeln!(script, "add element {INET_TABLE} direct_mac {{ {mac} }}"); + } script } -/// The nft input that makes the table's bindings exactly `pins`, in one -/// transaction: a reader never sees the sets empty in between. -pub(crate) fn pin_script(pins: &[Pin]) -> String { +/// The nft input that makes the bindings and the exemptions exactly those of +/// `pins` and `unrestricted`, in one transaction: a reader never sees a set +/// empty in between. +pub(crate) fn pin_script(pins: &[Pin], unrestricted: &[String]) -> String { format!( - "flush set {TABLE} port_mac\nflush set {TABLE} port_chaddr\n{}", - pin_elements(pins) + "flush set {TABLE} port_mac\nflush set {TABLE} port_chaddr\n\ + flush set {INET_TABLE} direct_mac\n{}", + pin_elements(pins, unrestricted) ) } +/// The names in the unrestricted file: trimmed, without blank lines and +/// comments. A file that cannot be read lists no one, so every agent is then +/// intercepted. +fn unrestricted_names() -> Vec { + let Some(path) = std::env::var_os(UNRESTRICTED_ENV).filter(|p| !p.is_empty()) else { + return Vec::new(); + }; + std::fs::read_to_string(path) + .unwrap_or_default() + .lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .map(str::to_owned) + .collect() +} + +/// The agent whose container holds the IPv4 address `ip`, as dnsmasq's lease +/// file and the bindings of the ports say: the lease names a MAC, the MAC is +/// bound to one port, and that port's machine is `h-`. `None` when any +/// step is missing or ambiguous (two leases for the address under different +/// MACs, a MAC bound twice, a machine that is no agent, a name that is no +/// identifier): a flow whose agent cannot be told is refused, never guessed. +pub(crate) fn agent_of(ip: &str, leases: &str, pins: &[Pin], now: u64) -> Option { + let ip: std::net::Ipv4Addr = ip.parse().ok()?; + let mut macs = leases.lines().filter_map(|line| { + let mut cols = line.split_whitespace(); + let (expiry, mac, addr) = (cols.next()?, cols.next()?, cols.next()?); + let live = match expiry.parse::().ok()? { + 0 => true, + t => t > now, + }; + (live && addr.parse::().ok()? == ip).then(|| Mac::parse(mac)) + }); + let mac = macs.next()??; + if macs.any(|other| other != Some(mac)) { + return None; + } + let mut machines = pins.iter().filter(|p| p.mac == mac).map(|p| &p.machine); + let machine = machines.next()?; + if machines.any(|other| other != machine) { + return None; + } + let agent = machine.strip_prefix(AGENT_PREFIX)?; + hive_types::Ident::parse(agent).ok()?; + Some(agent.to_owned()) +} + +/// `egress-whois`: squid's external ACL helper. Each line on stdin starts with +/// a source address; the answer is `OK user=` or `ERR`. The two files +/// are read for every line, since squid caches the answers itself. +fn whois() -> Result<()> { + use std::io::{BufRead as _, Write as _}; + let leasefile = std::env::var("HIVE_EGRESS_LEASEFILE").unwrap_or_else(|_| LEASEFILE.into()); + let stdin = std::io::stdin(); + let mut stdout = std::io::stdout().lock(); + for line in stdin.lock().lines() { + let line = line.context("read stdin")?; + let ip = line.split_whitespace().next().unwrap_or_default(); + let pins = parse_state(&std::fs::read_to_string(state_path()).unwrap_or_default()); + let leases = std::fs::read_to_string(&leasefile).unwrap_or_default(); + match agent_of(ip, &leases, &pins, now_secs()) { + Some(agent) => writeln!(stdout, "OK user={agent}"), + None => writeln!(stdout, "ERR"), + } + .context("write stdout")?; + stdout.flush().context("flush stdout")?; + } + Ok(()) +} + /// The nft input for one lease event, or `None` for an event that changes /// nothing: `init`, an IPv6 address, an address or MAC that does not parse. /// @@ -588,7 +697,7 @@ fn pin(args: &[String]) -> Result<()> { let survey = { let _lock = lock()?; let survey = survey(&bridge)?; - nft(&pin_script(&survey.resolution.pins))?; + nft(&pin_script(&survey.resolution.pins, &unrestricted_names()))?; commit(&survey)?; survey }; @@ -654,7 +763,7 @@ fn load(args: &[String]) -> Result<()> { let _lock = lock()?; let found = survey(&bridge); let pins = match &found { - Ok(found) => pin_elements(&found.resolution.pins), + Ok(found) => pin_elements(&found.resolution.pins, &unrestricted_names()), Err(e) => { eprintln!("{LOAD}: binding the ports failed, loading without bindings: {e:#}"); String::new() @@ -690,6 +799,7 @@ pub(crate) fn run(args: &[String]) -> Option { let (name, rest) = args.split_first()?; let result = match name.as_str() { PIN => pin(rest), + WHOIS => whois(), LOAD => load(rest), LEASE => lease(rest), _ => return None, @@ -902,7 +1012,7 @@ mod tests { #[test] fn the_ruleset_loads_with_the_bindings_and_leases_in_one_script() { let calls = std::cell::RefCell::new(Vec::new()); - let pins = pin_elements(&[pin("vb-a", 10, 10, "a")]); + let pins = pin_elements(&[pin("vb-a", 10, 10, "a")], &[]); let leases = lease_script("add", "02:00:00:00:00:0a", "10.42.0.10", 90).unwrap(); load_ruleset("table bridge hive_egress { }", &pins, &leases, |s| { calls.borrow_mut().push(s.to_string()); @@ -921,7 +1031,7 @@ mod tests { #[test] fn a_lease_load_that_fails_is_a_warning_and_the_bindings_still_load() { let calls = std::cell::RefCell::new(Vec::new()); - let pins = pin_elements(&[pin("vb-a", 10, 10, "a")]); + let pins = pin_elements(&[pin("vb-a", 10, 10, "a")], &[]); let leases = lease_script("add", "02:00:00:00:00:0a", "10.42.0.10", 90).unwrap(); let result = load_ruleset("RULESET", &pins, &leases, |s| { calls.borrow_mut().push(s.to_string()); @@ -944,16 +1054,152 @@ mod tests { } #[test] - fn pin_script_replaces_both_sets_in_one_script() { - let script = pin_script(&[pin("vb-a", 10, 10, "a")]); + fn pin_script_replaces_all_three_sets_in_one_script() { + let script = pin_script(&[pin("vb-h-a", 10, 10, "h-a")], &[]); assert_eq!( script, "flush set bridge hive_egress port_mac\n\ flush set bridge hive_egress port_chaddr\n\ - add element bridge hive_egress port_mac { \"vb-a\" . 02:00:00:00:00:0a }\n\ - add element bridge hive_egress port_chaddr { \"vb-a\" . 0x02000000000a }\n" + flush set inet hive_egress direct_mac\n\ + add element bridge hive_egress port_mac { \"vb-h-a\" . 02:00:00:00:00:0a }\n\ + add element bridge hive_egress port_chaddr { \"vb-h-a\" . 0x02000000000a }\n" ); - assert!(!pin_script(&[]).contains("add element")); + assert!(!pin_script(&[], &[]).contains("add element")); + } + + /// Service containers and the agents the file lists are exempt; an agent the + /// file does not list, and one it lists under another name, are not. + #[test] + fn only_service_containers_and_unrestricted_agents_are_exempt() { + let pins = [ + pin("vb-h-atlas", 10, 1, "h-atlas"), + pin("vb-h-uma", 11, 2, "h-uma"), + pin("vb-matrix", 12, 3, "hive-matrix"), + pin("vb-h-umax", 13, 4, "h-umax"), + ]; + let listed = ["uma".to_string()]; + assert_eq!(direct_macs(&pins, &listed), vec![mac(2), mac(3)]); + // Control: with nothing listed, only the service container stays. + assert_eq!(direct_macs(&pins, &[]), vec![mac(3)]); + let script = pin_elements(&pins, &listed); + assert!(script.contains("add element inet hive_egress direct_mac { 02:00:00:00:00:02 }")); + assert!(script.contains("add element inet hive_egress direct_mac { 02:00:00:00:00:03 }")); + assert!(!script.contains("direct_mac { 02:00:00:00:00:01 }")); + assert!(!script.contains("direct_mac { 02:00:00:00:00:04 }")); + } + + const NOW: u64 = 1_000; + + fn lease_line(expiry: u64, last: u8, ip: &str) -> String { + format!("{expiry} {} {ip} * *\n", mac(last)) + } + + #[test] + fn an_address_belongs_to_the_agent_of_the_leased_mac() { + let pins = [ + pin("vb-h-atlas", 10, 1, "h-atlas"), + pin("vb-h-uma", 11, 2, "h-uma"), + ]; + let leases = lease_line(NOW + 60, 1, "10.42.0.5") + &lease_line(NOW + 60, 2, "10.42.0.6"); + assert_eq!( + agent_of("10.42.0.5", &leases, &pins, NOW).as_deref(), + Some("atlas") + ); + assert_eq!( + agent_of("10.42.0.6", &leases, &pins, NOW).as_deref(), + Some("uma") + ); + // A lease with no end counts. + let forever = lease_line(0, 1, "10.42.0.7"); + assert_eq!( + agent_of("10.42.0.7", &forever, &pins, NOW).as_deref(), + Some("atlas") + ); + } + + #[test] + fn an_address_nobody_can_be_told_to_hold_has_no_agent() { + let pins = [ + pin("vb-h-atlas", 10, 1, "h-atlas"), + pin("vb-matrix", 11, 2, "hive-matrix"), + pin("vb-h-x", 12, 3, "h-x y"), + ]; + let leases = lease_line(NOW + 60, 1, "10.42.0.5"); + // Control for every refusal below: the same lease and pins name an agent. + assert!(agent_of("10.42.0.5", &leases, &pins, NOW).is_some()); + // No lease for the address, an expired one, and text that is no address. + assert_eq!(agent_of("10.42.0.9", &leases, &pins, NOW), None); + assert_eq!( + agent_of("10.42.0.5", &lease_line(NOW, 1, "10.42.0.5"), &pins, NOW), + None + ); + assert_eq!( + agent_of( + "10.42.0.5", + &lease_line(NOW - 1, 1, "10.42.0.5"), + &pins, + NOW + ), + None + ); + for bad in ["", "-", "10.42.0.5.1", "fe80::1", "10.42.0.5/32"] { + assert_eq!(agent_of(bad, &leases, &pins, NOW), None, "{bad:?}"); + } + // A leased MAC no port is bound to. + assert_eq!( + agent_of( + "10.42.0.8", + &lease_line(NOW + 60, 7, "10.42.0.8"), + &pins, + NOW + ), + None + ); + // A service container is no agent, and a machine name that is no identifier names none. + assert_eq!( + agent_of( + "10.42.0.8", + &lease_line(NOW + 60, 2, "10.42.0.8"), + &pins, + NOW + ), + None + ); + assert_eq!( + agent_of( + "10.42.0.8", + &lease_line(NOW + 60, 3, "10.42.0.8"), + &pins, + NOW + ), + None + ); + } + + #[test] + fn two_macs_leased_the_same_address_name_no_agent() { + let pins = [ + pin("vb-h-atlas", 10, 1, "h-atlas"), + pin("vb-h-uma", 11, 2, "h-uma"), + ]; + let leases = lease_line(NOW + 60, 1, "10.42.0.5") + &lease_line(NOW + 60, 2, "10.42.0.5"); + assert_eq!(agent_of("10.42.0.5", &leases, &pins, NOW), None); + // Control: an expired second lease is no rival. + let leases = lease_line(NOW + 60, 1, "10.42.0.5") + &lease_line(NOW - 5, 2, "10.42.0.5"); + assert_eq!( + agent_of("10.42.0.5", &leases, &pins, NOW).as_deref(), + Some("atlas") + ); + } + + #[test] + fn a_mac_bound_to_two_machines_names_no_agent() { + let pins = [ + pin("vb-h-atlas", 10, 1, "h-atlas"), + pin("vb-h-uma", 11, 1, "h-uma"), + ]; + let leases = lease_line(NOW + 60, 1, "10.42.0.5"); + assert_eq!(agent_of("10.42.0.5", &leases, &pins, NOW), None); } #[test] diff --git a/nix/host-modules/hive-egress.nix b/nix/host-modules/hive-egress.nix index 8ddb57795..096fb07ad 100644 --- a/nix/host-modules/hive-egress.nix +++ b/nix/host-modules/hive-egress.nix @@ -5,13 +5,18 @@ # to hold on every host lives here, in tables no firewall backend writes. Only # `networking.nftables.flushRuleset` removes them; the unit re-applies them then. # -# The `inet` table holds two chains. `input`: traffic from the agent bridge to host +# The `inet` table holds three chains. `input`: traffic from the agent bridge to host # loopback is dropped. The kernel does the same by default (a packet for # 127.0.0.0/8 arriving on a non-loopback interface is a martian while # `route_localnet` is 0); this rule is the second layer, and the assertion # below keeps the first one on. `forward` (`egress.forward = "log"`): traffic # the host routes from the agent bridge to anywhere else is logged and -# counted, never blocked. +# counted, never blocked. `intercept` (`egress.intercept.enable`, set by +# hive-squid.nix): the web traffic of the bridge's agents is redirected into +# squid's intercept ports. The traffic of the ports in the set `direct_mac` is +# left alone: the service containers and the unrestricted agents. `hive-priv +# egress-pin` and `egress-load` fill that set from the bindings the bridge +# table uses, so a MAC in no binding is intercepted, and squid refuses the flow. # # The `bridge` table checks the source address of every frame a port of the # agent bridge sends (`egress.antiSpoof`). It is a `bridge`-family table because @@ -48,7 +53,12 @@ let pkgs.iproute2 pkgs.util-linux ]; - pinEnvironment.HIVE_EGRESS_BRIDGE = br; + pinEnvironment = { + HIVE_EGRESS_BRIDGE = br; + } + // lib.optionalAttrs (cfg.egress.intercept.unrestrictedFile != null) { + HIVE_EGRESS_UNRESTRICTED = toString cfg.egress.intercept.unrestrictedFile; + }; leaseScript = pkgs.writeShellScript "hive-egress-lease" '' export PATH=${lib.makeBinPath [ pkgs.nftables ]} exec ${hivePriv} egress-lease "$@" @@ -71,6 +81,24 @@ let family: addr: ''${fromBridge} ct state new update @log_${family} { ${addr} saddr limit rate ${logRate} } log prefix "hive-egress-would-drop " accept''; + # IPv4 TCP from the agent bridge to ports 80 and 443. `fib daddr oifname` is + # the interface the route to the destination leaves by: the bridge (another + # container), `lo` (an address of the host: `fib daddr type local`) and the + # mesh interface are not egress. The redirect rewrites the destination to + # the bridge address and the port of squid's intercept listener, which reads + # the original destination from conntrack. IPv6 is left alone: the agent + # bridge has no IPv6 addressing. + interceptChain = '' + chain intercept { + type nat hook prerouting priority dstnat; policy accept; + iifname "${br}" meta nfproto ipv4 ether saddr @direct_mac return + iifname "${br}" meta nfproto ipv4 fib daddr type local return + iifname "${br}" meta nfproto ipv4 fib daddr oifname { "${br}", "${meshInterface}" } return + iifname "${br}" meta nfproto ipv4 tcp dport 80 counter redirect to :${toString cfg.egress.intercept.httpPort} + iifname "${br}" meta nfproto ipv4 tcp dport 443 counter redirect to :${toString cfg.egress.intercept.httpsPort} + } + ''; + # Priority `filter - 10` runs before the NixOS firewall's forward handling, # which sits at `filter` on both backends (`FORWARD` of the iptables backend, # the `forward` chain of `nixos-fw`). An `accept` here ends this chain only, @@ -179,6 +207,8 @@ let iifname "${br}" ip daddr 127.0.0.0/8 counter drop iifname "${br}" ip6 daddr ::1 counter drop } + set direct_mac { type ether_addr; } + ${lib.optionalString cfg.egress.intercept.enable interceptChain} ${lib.optionalString (cfg.egress.forward == "log") forwardChain} } @@ -243,9 +273,48 @@ in address is never affected, so nothing legitimate breaks. - `log`: the same lines and counters, and the frame goes through. For finding out why a legitimate container is refused on a live hive. + + With `log`, the identity squid takes from the source of an intercepted + flow (`egress.intercept`) can be spoofed: use it to diagnose, not to run. ''; }; + options.services.hyperhive.network.egress.intercept = { + enable = lib.mkEnableOption "the redirect of the agents' web traffic into squid's intercept ports"; + + httpPort = lib.mkOption { + type = lib.types.port; + default = 3129; + description = '' + The port on the bridge address where squid listens for the redirected + TCP port 80 traffic. + ''; + }; + + httpsPort = lib.mkOption { + type = lib.types.port; + default = 3130; + description = '' + The port on the bridge address where squid listens for the redirected + TCP port 443 traffic. + ''; + }; + + unrestrictedFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + The file hive-c0re renders with the names of the agents that hold the + `unrestricted_internet` flag, one per line. The `direct_mac` set of the + `inet hive_egress` table holds the MACs of the ports of the agents it + lists and of every container that is not an agent, and the redirect + skips them. `null` or an unreadable file lists no agent, so every agent's + web traffic is redirected. The file is read when a port is bound, on + each pass of the 30 second timer and when the file changes. + ''; + }; + }; + options.services.hyperhive.network.egress.leaseScript = lib.mkOption { type = lib.types.path; internal = true; @@ -274,8 +343,9 @@ in assertion = cfg.egress.forward != "drop"; message = '' services.hyperhive.network.egress.forward = "drop" is not available yet: - the interception that lets an agent's web traffic through squid has not - landed, so a drop would cut every direct connection with no replacement. + the traffic squid does not handle (the unrestricted agents, the service + containers, every port but 80 and 443) has no rule that lets it through, so a + drop would cut it. Use "log" or "accept". ''; } @@ -361,6 +431,17 @@ in }; }; + # A pass also refills `direct_mac`, so a change of the unrestricted list + # takes effect at once instead of at the next tick of the timer. + systemd.paths.hive-egress-direct = lib.mkIf (cfg.egress.intercept.unrestrictedFile != null) { + description = "Watch the unrestricted agents list for the egress intercept exemptions"; + wantedBy = [ "paths.target" ]; + pathConfig = { + PathChanged = toString cfg.egress.intercept.unrestrictedFile; + Unit = "hive-egress-pin.service"; + }; + }; + # nspawn names the host end of a container's veth `vb-` when the # container joins a bridge. services.udev.extraRules = '' diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index 6120b8bb8..d88210195 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -1,8 +1,9 @@ # Egress proxy for this hive's agents: squid on the host, bound to the bridge # IP, default-deny on the destination hostname. It filters CONNECT targets and -# plain-HTTP hosts and never decrypts, so it holds no signing key. A request -# must carry a per-agent proxy credential, checked against the users file -# hive-c0re renders. A credential's agent either has the swarm's +# plain-HTTP hosts and never decrypts: the one TLS key it loads is generated at +# each start, signs nothing and is trusted by no agent. A request through the +# proxy port must carry a per-agent proxy credential, checked against the users +# file hive-c0re renders. A credential's agent either has the swarm's # `unrestricted_internet` flag, and may then reach any destination the deny # rules leave, or reaches only the listed names: the base list, the allow-list # the swarm keeps plus this host's `allowedDomains` (rendered by hive-c0re @@ -15,6 +16,15 @@ # An agent sends its credential # once it has fetched it (../agent-modules/egress-proxy.nix); the exporter, # journal forwarding and the self-check below show the proxy works. +# +# Web traffic that ignores the proxy variables reaches squid anyway: hive-egress.nix +# redirects the TCP 80 and 443 flows of every agent that does not hold the flag +# into two intercept ports. Such a flow carries no credential, so the agent is +# the one whose container holds the flow's source address, which +# `hive-priv egress-whois` reads from the bindings and leases (the external ACL +# `hive_agent`); a source it cannot name is refused. The listed-names and deny +# rules are the same ones, matched on the identity from that helper. HTTPS is +# peeked at for its SNI and spliced: squid checks the name and never decrypts. { pkgs, lib, @@ -30,6 +40,14 @@ let listen = "${netCfg.bridgeIp}:${toString port}"; proxyUrl = "http://${netCfg.bridgeIp}:${toString port}"; exporterListen = "127.0.0.1:9301"; + interceptCfg = netCfg.egress.intercept; + # The path every platform container's unit runs hive-priv by (hive-priv.nix + # points it at the current build). Squid runs `egress-whois` through it. + hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv; + # Written at each start of the unit by `interceptCert`: `https_port` requires + # a certificate and key, and nothing uses them, since every flow is spliced. + interceptCertFile = "/run/squid/intercept.crt"; + interceptKeyFile = "/run/squid/intercept.key"; denyCanary = "deny-canary.invalid"; allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]); @@ -92,11 +110,41 @@ let # refusal into an allow. The `dst` denies cost an unlisted name a lookup on # the host's resolver before it is refused. # The swarm's deny list (`denyFile`) sits right after those destination - # denies and before `deny !authed`, so a denied name is refused before any + # denies and before `deny !icpt !authed`, so a denied name is refused before any # credential check and before every allow, in every mode: its rules name the # destination first, and a swarm-wide one names no user at all. + + # A flow on an intercept port has no credential: `icpt` is those ports and + # `icpt_agent` the helper's answer for the flow's source, which also sets the + # user that `ext_user` ACLs and the log read. A source the helper cannot name + # is refused first. The first request of an intercepted HTTPS connection is + # the CONNECT squid builds from the destination address before it has read + # the ClientHello (`first_hop`); it passes once the address denies have, so + # squid can peek. The CONNECT it builds next has the SNI as its host and meets + # every rule; with no SNI the host stays an address, which the IP-literal + # denies refuse, and `host_verify_strict` answers 409 for an SNI that does not + # resolve to the destination. `!icpt` guards each rule that reads the proxy + # credential, since squid logs a warning when such an ACL meets an + # intercepted request; the rules for those read `ext_user`. squidConf = '' http_port ${listen} + http_port ${netCfg.bridgeIp}:${toString interceptCfg.httpPort} intercept name=icpt_http + https_port ${netCfg.bridgeIp}:${toString interceptCfg.httpsPort} intercept ssl-bump name=icpt_tls tls-cert=${interceptCertFile} tls-key=${interceptKeyFile} generate-host-certificates=off + # Never decrypt: peek at the ClientHello for the SNI, then splice every + # connection. The decision on a name is made by `http_access` on the CONNECT + # that carries it. A connection whose first bytes are not a ClientHello + # (or not an HTTP request, on the other port) is refused. + acl icpt myportname icpt_http icpt_tls + acl first_hop at_step SslBump1 + ssl_bump peek first_hop + ssl_bump splice all + host_verify_strict on + + # The agent that holds the source address of a flow on an intercept port. + # An answer is kept for 10 seconds, a refusal for 2. + external_acl_type hive_agent ttl=10 negative_ttl=2 children-startup=1 children-max=4 %SRC ${hivePriv} egress-whois + acl icpt_agent external hive_agent + pid_filename /run/squid/squid.pid cache_log stdio:/var/log/squid/cache.log coredump_dir /var/cache/squid @@ -161,6 +209,8 @@ let http_access deny !Safe_ports http_access deny CONNECT !SSL_ports + http_access deny icpt !icpt_agent + http_access allow icpt first_hop CONNECT !to_localhost !to_linklocal !to_bridge !to_private http_access deny ip_v4_host http_access deny ip_v6_host http_access deny to_localhost @@ -168,15 +218,27 @@ let http_access deny to_bridge http_access deny to_private include ${cfg.denyFile} - http_access deny !authed + http_access deny !icpt !authed ${lib.optionalString (base != [ ]) "http_access allow base_dst"} include ${cfg.agentAllowFile} - http_access allow unrestricted + http_access allow !icpt unrestricted http_access deny !allowed_dst http_access allow allowed_dst http_access deny all ''; + # A throwaway self-signed key and certificate for the intercept HTTPS port, + # written to the two paths given. Squid refuses to start without them and + # loads them to build its TLS context; it never presents them, since every + # connection is spliced. + interceptCert = pkgs.writeShellScript "squid-intercept-cert" '' + set -eu + umask 077 + ${lib.getExe pkgs.openssl} req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 \ + -nodes -subj /CN=hive-squid-intercept -days 3650 \ + -keyout "$1" -out "$2" + ''; + # `squid -k parse` on the configuration with the includes pointing at an empty # file. The nixpkgs module runs the same parse on the real text, which # cannot open the includes in the build sandbox. @@ -186,11 +248,13 @@ let nativeBuildInputs = [ config.services.squid.package ]; } '' - squid -k parse -f ${ + ${interceptCert} key.pem cert.pem + sed -e "s|${interceptKeyFile}|$PWD/key.pem|; s|${interceptCertFile}|$PWD/cert.pem|" ${ pkgs.writeText "squid.conf" ( builtins.replaceStrings [ cfg.agentAllowFile cfg.denyFile ] [ "/dev/null" "/dev/null" ] squidConf ) - } + } > squid.conf + squid -k parse -f squid.conf touch $out ''; @@ -385,12 +449,13 @@ in defaultText = lib.literalMD "`agent-allow.conf`, in the directory of `usersFile`"; description = '' Squid configuration that hive-c0re generates and the main configuration - includes after every destination deny and `deny !authed`, where the + includes after every destination deny and `deny !icpt !authed`, where the base list is allowed: for each agent of this hive the swarm lists - names for, an ACL for its user, an ACL for its names and the - `http_access allow` joining the two. After those, for each agent in - `log` mode, an `http_access allow` for names `allowed_dst` lacks that - annotates the request `would_deny=1`. Empty until the controller has a + names for, an ACL for its proxy user, one for the user the intercept + helper names, an ACL for its names and two `http_access allow` rules + joining them, one for the proxy port and one for the intercept ports. + After those, for each agent in `log` mode, the same pair of rules for + names `allowed_dst` lacks, which annotate the request `would_deny=1`. Empty until the controller has a list or an agent is in `log` mode. Squid is reloaded when it changes. Owned by `hive-core` and readable by the `squid` group. ''; @@ -404,9 +469,10 @@ in description = '' Squid configuration that hive-c0re generates from the swarm's deny list (`/api/egress/denied-domains`) and the main configuration includes after - the destination denies and before `deny !authed` and every allow: a - `http_access deny` for the swarm-wide names, and one per agent of this - hive the swarm lists names for. A denied name is refused for every user + the destination denies and before `deny !icpt !authed` and every allow: a + `http_access deny` for the swarm-wide names, and two per agent of this + hive the swarm lists names for, one for the proxy port and one for the + intercept ports. A denied name is refused for every user in every egress mode, whether or not an allow rule covers it. The file is empty, and squid configured as without it, until the swarm lists a name. Squid is reloaded when it changes. Owned by `hive-core` and readable @@ -478,7 +544,15 @@ in config = lib.mkIf hyperhiveCfg.deploy.hive-controller.enable { # Reachable from agent containers and nowhere else: this opens the port on # the bridge interface only. - services.hyperhive.network.exposeHostPorts = [ port ]; + services.hyperhive.network.exposeHostPorts = [ + port + interceptCfg.httpPort + interceptCfg.httpsPort + ]; + services.hyperhive.network.egress.intercept = { + enable = lib.mkDefault true; + inherit (cfg) unrestrictedFile; + }; services.hyperhive.network.enable = lib.mkDefault true; services.hyperhive.otel.scrapeTargets.squid = exporterListen; @@ -555,6 +629,11 @@ in ]; systemd.services.squid = { + # Before the module's own pre-start, which parses the configuration and + # so loads the certificate. + preStart = lib.mkBefore '' + ${interceptCert} ${interceptKeyFile} ${interceptCertFile} + ''; serviceConfig = sandbox // { User = "squid"; Group = "squid"; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index af041f356..105ca2b13 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -1686,6 +1686,104 @@ let in refused egressForwardDrop && !(refused bare) && !(refused egressForwardAccept); } + { + # The redirect of the agents' web traffic: IPv4 TCP 80 and 443 from the + # bridge go to squid's two intercept ports, after the rules that leave + # alone the exempt MACs (service containers, unrestricted agents), the + # host's own addresses, the bridge and the mesh. The chain is read as the + # lines between its header and the log sets that follow it. The host + # without a controller (`egressForwardAccept`) has the set but no chain. + name = "the egress table redirects the bridge's web traffic into squid after its exemptions"; + ok = + let + inet = lib.head ( + lib.splitString "table bridge hive_egress {" bare.services.hyperhive.network.egress.ruleset + ); + chain = lib.head ( + lib.splitString "set log_v4" (lib.last (lib.splitString "chain intercept {" inet)) + ); + rules = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" chain)); + idx = pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules; + redirects = lib.filter (lib.hasInfix " redirect to ") rules; + without = egressForwardAccept.services.hyperhive.network.egress.ruleset; + in + lib.hasInfix "type nat hook prerouting priority dstnat; policy accept;" chain + && lib.hasInfix "set direct_mac { type ether_addr; }" inet + && idx "ether saddr @direct_mac return" < idx "tcp dport 80" + && idx "fib daddr type local return" < idx "tcp dport 80" + && idx ''fib daddr oifname { "hive-br0", "wg-hive" } return'' < idx "tcp dport 80" + && lib.length redirects == 2 + && lib.hasInfix "tcp dport 80 counter redirect to :3129" (lib.head redirects) + && lib.hasInfix "tcp dport 443 counter redirect to :3130" (lib.last redirects) + # Only IPv4 that comes in on the bridge is looked at. + && lib.all (l: lib.hasPrefix ''iifname "hive-br0" meta nfproto ipv4 '' l) (lib.drop 1 rules) + && lib.hasInfix "set direct_mac { type ether_addr; }" without + && !(lib.hasInfix "chain intercept" without); + } + { + # Squid listens on the two intercept ports, peeks and splices and never + # decrypts, names the agent of a flow with the helper, refuses a source + # it cannot name before any other rule, lets the CONNECT it builds + # before the ClientHello through only after the address denies, and + # leaves the proxy credential rules to the proxy port. + name = "squid's intercept ports splice, name the agent from the bindings and keep the credential rules for the proxy port"; + ok = + let + conf = bare.services.squid.configText; + lines = map lib.trim (lib.splitString "\n" conf); + idx = line: lib.lists.findFirstIndex (l: l == line) 1000 lines; + squidCfg = bare.services.hyperhive.deploy.hive-controller.squid; + in + lib.hasInfix "http_port 10.42.0.1:3129 intercept name=icpt_http" conf + && lib.hasInfix "https_port 10.42.0.1:3130 intercept ssl-bump name=icpt_tls " conf + && lib.hasInfix "generate-host-certificates=off" conf + && lib.elem "ssl_bump peek first_hop" lines + && lib.elem "ssl_bump splice all" lines + && !(lib.hasInfix "ssl_bump bump" conf) + && !(lib.hasInfix "ssl_bump stare" conf) + && !(lib.hasInfix "sslcrtd_program" conf) + && lib.elem "host_verify_strict on" lines + && lib.hasInfix "%SRC /run/hive-rootfs-check egress-whois" conf + && idx "http_access deny icpt !icpt_agent" < idx "http_access deny ip_v4_host" + && idx "http_access deny icpt !icpt_agent" < idx "include ${squidCfg.denyFile}" + && + idx "http_access deny icpt !icpt_agent" + < idx "http_access allow icpt first_hop CONNECT !to_localhost !to_linklocal !to_bridge !to_private" + && + idx "http_access allow icpt first_hop CONNECT !to_localhost !to_linklocal !to_bridge !to_private" + < idx "http_access deny ip_v4_host" + && lib.elem "http_access deny !icpt !authed" lines + && !(lib.elem "http_access deny !authed" lines) + && lib.elem "http_access allow !icpt unrestricted" lines + && !(lib.elem "http_access allow unrestricted" lines) + && lib.all (p: lib.elem p bare.services.hyperhive.network.exposeHostPorts) [ + 3128 + 3129 + 3130 + ]; + } + { + # `hive-priv` fills `direct_mac` from the same bindings as the bridge + # table's, so the units that run it are told the list of unrestricted + # agents, and a change of the list runs a pass. A host without squid has + # neither. + name = "the units that fill the egress exemptions read squid's unrestricted list and a change of it runs a pass"; + ok = + let + list = bare.services.hyperhive.deploy.hive-controller.squid.unrestrictedFile; + envOf = m: name: m.systemd.services.${name}.environment; + in + list == "/var/lib/hive-squid/unrestricted" + && lib.all (name: ((envOf bare name).HIVE_EGRESS_UNRESTRICTED or null) == list) [ + "hive-egress" + "hive-egress-pin@" + "hive-egress-pin" + ] + && bare.systemd.paths.hive-egress-direct.pathConfig.PathChanged == list + && bare.systemd.paths.hive-egress-direct.pathConfig.Unit == "hive-egress-pin.service" + && !((envOf egressForwardAccept "hive-egress") ? HIVE_EGRESS_UNRESTRICTED) + && !(egressForwardAccept.systemd.paths ? hive-egress-direct); + } { # The host that turns `route_localnet` on is refused by an assertion # naming it; the ordinary host, which leaves it alone, is not. @@ -2189,7 +2287,7 @@ let "http_access deny to_linklocal" "http_access deny to_bridge" "http_access deny to_private" - "http_access deny !authed" + "http_access deny !icpt !authed" ]; in e.HIVE_SQUID_PROXY == "${withIdentity.services.hyperhive.network.bridgeIp}:3128" diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 2a275ce1b..3d0c36825 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -513,17 +513,24 @@ let } { # Control for the absence checks: the bridge listener is there, so the - # absences below are not a misspelled lookup. - name = "squid listens on the bridge IP only, without a second listener or ssl_bump"; + # absences below are not a misspelled lookup. The two intercept ports are + # on the bridge IP too, and the only TLS directives are the ones that + # peek and splice. + name = "squid listens on the bridge IP only, with one proxy port, two intercept ports and no decryption"; ok = let conf = hiveOtel.services.squid.configText; lines = lib.splitString "\n" conf; + ip = hiveOtel.services.hyperhive.network.bridgeIp; + listeners = lib.filter (l: lib.hasPrefix "http_port " l || lib.hasPrefix "https_port " l) lines; in - builtins.elem "http_port ${hiveOtel.services.hyperhive.network.bridgeIp}:3128" lines - && lib.length (lib.filter (l: lib.hasPrefix "http_port " l) lines) == 1 - && !(lib.hasInfix "ssl_bump" conf) - && !(lib.hasInfix "ssl-bump" conf) + builtins.elem "http_port ${ip}:3128" lines + && lib.length listeners == 3 + && lib.all (l: lib.hasPrefix "${lib.head (lib.splitString " " l)} ${ip}:" l) listeners + && lib.length (lib.filter (lib.hasInfix " intercept ") listeners) == 2 + && lib.elem "ssl_bump peek first_hop" lines + && lib.elem "ssl_bump splice all" lines + && lib.length (lib.filter (lib.hasPrefix "ssl_bump ") lines) == 2 && !(lib.hasInfix "localnet" conf); } { @@ -540,8 +547,8 @@ let at = l: lib.lists.findFirstIndex (x: x == l) null lines; allow = at "http_access allow allowed_dst"; base = at "http_access allow base_dst"; - unrestricted = at "http_access allow unrestricted"; - authed = at "http_access deny !authed"; + unrestricted = at "http_access allow !icpt unrestricted"; + authed = at "http_access deny !icpt !authed"; # The generated per-agent `http_access allow` rules sit in this file. agentRules = at "include ${hiveOtel.services.hyperhive.deploy.hive-controller.squid.agentAllowFile}"; firstAllow = lib.min agentRules (lib.min base (lib.min unrestricted allow)); @@ -631,11 +638,11 @@ let at = l: lib.lists.findFirstIndex (x: x == l) null lines; list = hiveOtel.services.hyperhive.deploy.hive-controller.squid.unrestrictedFile; in - at "http_access allow unrestricted" != null + at "http_access allow !icpt unrestricted" != null && at "http_access deny !allowed_dst" != null - && at "http_access deny !authed" < at "http_access allow base_dst" - && at "http_access allow base_dst" < at "http_access allow unrestricted" - && at "http_access allow unrestricted" < at "http_access deny !allowed_dst" + && at "http_access deny !icpt !authed" < at "http_access allow base_dst" + && at "http_access allow base_dst" < at "http_access allow !icpt unrestricted" + && at "http_access allow !icpt unrestricted" < at "http_access deny !allowed_dst" && at "http_access deny !allowed_dst" < at "http_access allow allowed_dst" && builtins.elem ''acl unrestricted proxy_auth "${list}"'' lines && list == "/var/lib/hive-squid/unrestricted"; @@ -655,7 +662,7 @@ let allows = map at [ "http_access allow base_dst" "include ${sq.agentAllowFile}" - "http_access allow unrestricted" + "http_access allow !icpt unrestricted" "http_access allow allowed_dst" ]; in @@ -665,7 +672,7 @@ let && lib.all (a: deny < a) allows && at "http_access deny to_private" < deny && at "http_access deny ip_v4_host" < deny - && deny < at "http_access deny !authed" + && deny < at "http_access deny !icpt !authed" # Two includes: this one and the per-agent allow rules. && lib.length (lib.filter (l: lib.hasPrefix "include " l) lines) == 2; } diff --git a/nix/vm-tests.nix b/nix/vm-tests.nix index 67790c44c..e246095df 100644 --- a/nix/vm-tests.nix +++ b/nix/vm-tests.nix @@ -9,7 +9,7 @@ { agent-ephemeral-rootfs = import ./vm-tests/agent-ephemeral-rootfs.nix { inherit pkgs hivePriv; }; wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; }; - hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; }; + hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs hivePriv; }; hive-egress-antispoof = import ./vm-tests/hive-egress-antispoof.nix { inherit pkgs hivePriv; }; swarm-archive = import ./vm-tests/swarm-archive.nix { inherit pkgs; }; swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; }; diff --git a/nix/vm-tests/hive-squid.nix b/nix/vm-tests/hive-squid.nix index 27802a234..c591020d6 100644 --- a/nix/vm-tests/hive-squid.nix +++ b/nix/vm-tests/hive-squid.nix @@ -12,8 +12,10 @@ # It also covers ../host-modules/hive-egress.nix: `proxy` loads the real module, so the # agent's dials to the host's loopback, and its direct connects to the uplink # (`proxy` forwards and masquerades like the hive host), run against the table -# the module installs. -{ pkgs }: +# the module installs. The same table redirects the agents' web traffic into +# squid's intercept ports; the test writes the bindings and leases that +# `hive-priv egress-whois` reads, standing in for `egress-pin` and dnsmasq. +{ pkgs, hivePriv }: let bridgeIp = "192.168.1.1"; @@ -105,6 +107,14 @@ let prefixLength = 64; }; + # The certificate `web` answers TLS with. Nothing trusts it: squid splices, + # so the agent's curl sees the origin's own and runs with `-k`. + webCert = pkgs.runCommand "hive-squid-web-cert" { nativeBuildInputs = [ pkgs.openssl ]; } '' + mkdir $out + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ + -subj /CN=web.t.local -days 3650 -keyout $out/key.pem -out $out/cert.pem + ''; + # Plain TCP on 443: a CONNECT is answered before any TLS, which is all the # test reads. listen443 = addrs: { @@ -199,10 +209,13 @@ pkgs.testers.runNixOSTest { networking.firewall.interfaces.eth1.allowedTCPPorts = [ 443 ]; # hive-priv.nix links the binary the hive-egress unit loads its tables with; # `eth1` is no bridge here, so a stub that only loads the ruleset (no - # bindings, no leases) stands in. + # bindings, no leases) stands in, and hands `egress-whois` to the real + # binary. systemd.tmpfiles.rules = [ + "d /var/lib/dnsmasq 0755 root root - -" "L+ /run/hive-rootfs-check - - - - ${pkgs.writeShellScript "hive-priv-stub" '' [ "$1" = egress-load ] && exec ${pkgs.nftables}/bin/nft -f "$2" + [ "$1" = egress-whois ] && exec ${hivePriv}/bin/hive-priv "$@" exit 0 ''}" ]; @@ -231,12 +244,37 @@ pkgs.testers.runNixOSTest { web = { lib, ... }: { - imports = [ - (listen443 [ - "0.0.0.0" - "[::]" - ]) - ]; + # TLS on 443 for the intercepted flows (a CONNECT through the proxy is + # answered before any TLS), plain HTTP on 80, and a plain listener on + # 8443 that the proxy never intercepts. + services.nginx = { + enable = true; + virtualHosts.default = { + listen = [ + { + addr = "0.0.0.0"; + port = 443; + ssl = true; + } + { + addr = "[::]"; + port = 443; + ssl = true; + } + { + addr = "0.0.0.0"; + port = 80; + } + { + addr = "0.0.0.0"; + port = 8443; + } + ]; + sslCertificate = "${webCert}/cert.pem"; + sslCertificateKey = "${webCert}/key.pem"; + locations."/".return = "200 web-ok"; + }; + }; virtualisation.vlans = [ 2 ]; networking.interfaces.eth1.ipv4.addresses = lib.mkForce ( [ @@ -244,6 +282,12 @@ pkgs.testers.runNixOSTest { address = "203.0.113.2"; prefixLength = 24; } + # Another address of the same host: where a flow goes whose SNI + # names the first. + { + address = "203.0.113.9"; + prefixLength = 24; + } ] ++ privateAddrs 2 ); @@ -255,15 +299,27 @@ pkgs.testers.runNixOSTest { via = "fd00:99::1"; } ]; - networking.firewall.allowedTCPPorts = [ 443 ]; + networking.firewall.allowedTCPPorts = [ + 80 + 443 + 8443 + ]; }; agent = - { lib, ... }: + { lib, pkgs, ... }: { virtualisation.vlans = [ 1 ]; # The bridge listener, by the name the proxy resolves to the same address. networking.hosts.${bridgeIp} = [ "bridge.t.local" ]; + # The names the proxy resolves, for the flows that skip the proxy variables. + networking.hosts."203.0.113.2" = [ + "forge.t.local" + "other.t.local" + "agentonly.t.local" + "api.t.local" + ]; + environment.systemPackages = [ pkgs.openssl ]; networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ { address = "192.168.1.3"; @@ -637,7 +693,9 @@ pkgs.testers.runNixOSTest { "# source: swarm\n" "acl agent_allow_atlas proxy_auth atlas\n" "acl agent_allow_atlas_dst dstdomain agentonly.t.local priv10.t.local\n" - "http_access allow agent_allow_atlas agent_allow_atlas_dst\n", + "acl agent_allow_atlas_ix ext_user atlas\n" + "http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n" + "http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n", ) wait_status("swarm.t.local", RUTH, "200") wait_status("agentonly.t.local", ATLAS, "200") @@ -659,7 +717,9 @@ pkgs.testers.runNixOSTest { "# source: swarm\n" "acl log_mark annotate_transaction would_deny=1\n" "acl agent_log_ruth proxy_auth ruth\n" - "http_access allow agent_log_ruth !allowed_dst log_mark\n", + "acl agent_log_ruth_ix ext_user ruth\n" + "http_access allow !icpt agent_log_ruth !allowed_dst log_mark\n" + "http_access allow icpt agent_log_ruth_ix !allowed_dst log_mark\n", ) wait_status("other.t.local", RUTH, "200") proxy.wait_until_succeeds( @@ -694,7 +754,9 @@ pkgs.testers.runNixOSTest { "# source: swarm\n" "acl log_mark annotate_transaction would_deny=1\n" "acl agent_log_ruth proxy_auth ruth\n" - "http_access allow agent_log_ruth !allowed_dst log_mark\n", + "acl agent_log_ruth_ix ext_user ruth\n" + "http_access allow !icpt agent_log_ruth !allowed_dst log_mark\n" + "http_access allow icpt agent_log_ruth_ix !allowed_dst log_mark\n", ) wait_status("other.t.local", RUTH, "200") write_deny( @@ -703,7 +765,9 @@ pkgs.testers.runNixOSTest { "http_access deny deny_swarm_dst deny_hit_mark\n" "acl deny_agent_atlas_dst dstdomain api.t.local\n" "acl deny_agent_atlas proxy_auth atlas\n" - "http_access deny deny_agent_atlas_dst deny_agent_atlas deny_hit_mark\n" + "acl deny_agent_atlas_ix ext_user atlas\n" + "http_access deny deny_agent_atlas_dst !icpt deny_agent_atlas deny_hit_mark\n" + "http_access deny deny_agent_atlas_dst icpt deny_agent_atlas_ix deny_hit_mark\n" ) wait_status("forge.t.local", ATLAS, "403") assert connect("forge.t.local", RUTH) == "403", "a listed name, log mode" @@ -748,8 +812,8 @@ pkgs.testers.runNixOSTest { assert connect("forge.t.local", ATLAS) == "200", "the file appeared" assert connect("other.t.local", UMA) == "200", "listed, and now logged in" - def dial(addr): - return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/443'")[0] == 0 + def dial(addr, port=443): + return agent.execute(f"timeout 3 bash -c 'echo > /dev/tcp/{addr}/{port}'")[0] == 0 def dropped(): # Packets the table's v4 rule has dropped so far. @@ -796,10 +860,11 @@ pkgs.testers.runNixOSTest { # first: none of them is forwarded traffic, so none may be logged. assert connect("forge.t.local", ATLAS) == "200", "through squid" assert dial("${bridgeIp}"), "the gateway on the bridge IP" - assert dial("203.0.113.2"), "a direct connect to an outside IP works" + # Port 8443: a flow to 80 or 443 is redirected into squid and never forwarded. + assert dial("203.0.113.2", 8443), "a direct connect to an outside IP works" proxy.wait_until_succeeds( "journalctl -k -o cat --no-pager | grep -E " - "'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=443 '" + "'hive-egress-would-drop IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=8443 '" ) agent.succeed("ping -c1 -W2 203.0.113.2") proxy.wait_until_succeeds( @@ -850,5 +915,164 @@ pkgs.testers.runNixOSTest { out = proxy.succeed("nft list chain inet hive_egress forward") found = re.search(r'oifname != "eth1" counter packets (\d+) bytes \d+$', out, re.M) assert found and int(found.group(1)) >= 100, out + + # ---- The agents' web traffic that skips the proxy variables ---- + # The test stands in for `egress-pin` and dnsmasq's lease script: it writes + # the pin file and the lease file that `hive-priv egress-whois` reads, and + # the exemption set that `egress-pin` fills on a hive. The agent node holds + # three addresses, one per agent, and the exemption is by its MAC. + AGENT_MAC = agent.succeed("cat /sys/class/net/eth1/address").strip() + ATLAS_IP, LOGAN_IP, NOBODY_IP = "192.168.1.3", "192.168.1.4", "192.168.1.7" + BOUND = {"atlas": (ATLAS_IP, AGENT_MAC), "logan": (LOGAN_IP, "02:00:00:00:00:04")} + agent.succeed(f"ip addr add {LOGAN_IP}/24 dev eth1; ip addr add {NOBODY_IP}/24 dev eth1") + + def bind(*names): + pins = "".join(f"vb-{n} {i + 2} {BOUND[n][1]} h-{n}\n" for i, n in enumerate(names)) + leases = "".join(f"0 {BOUND[n][1]} {BOUND[n][0]} {n} *\n" for n in names) + proxy.succeed( + f"printf '%s' {shlex.quote(pins)} > /run/hive-egress-pins " + f"&& printf '%s' {shlex.quote(leases)} > /var/lib/dnsmasq/dnsmasq.leases" + ) + + def icpt(url, src=ATLAS_IP, extra=""): + # The status of the answer to a request that carries no proxy variables; + # 000 when the connection was closed or the TLS handshake failed. + return agent.succeed( + f"curl -sS -k -m 15 --interface {src} {extra} -o /dev/null -w '%{{http_code}}' {url} 2>/dev/null || true" + ).strip() + + def wait_icpt(url, want, src=ATLAS_IP, extra=""): + # The reload that follows a list write is asynchronous. + got = None + for _ in range(30): + got = icpt(url, src, extra) + if got == want: + return + time.sleep(1) + raise AssertionError(f"{url} from {src}: wanted {want}, last got {got}") + + def journal(): + return proxy.succeed("journalctl -u squid.service -o cat --no-pager") + + def agent_lines(): + # Log lines of requests from the agent node's addresses. The self-check + # connects from the bridge address, so its lines are not counted. + return len(re.findall(r" 192\.168\.1\.[347] [A-Z_]+/\d+ ", journal())) + + seeded = proxy.succeed(f"cat {ALLOWED}") + bind("atlas", "logan") + # What hive-c0re renders: `atlas` has a name of its own, `logan` is in log mode. + write_allow( + seeded, + "# source: swarm\n" + "acl agent_allow_atlas proxy_auth atlas\n" + "acl agent_allow_atlas_ix ext_user atlas\n" + "acl agent_allow_atlas_dst dstdomain agentonly.t.local\n" + "http_access allow !icpt agent_allow_atlas agent_allow_atlas_dst\n" + "http_access allow icpt agent_allow_atlas_ix agent_allow_atlas_dst\n" + "acl log_mark annotate_transaction would_deny=1\n" + "acl agent_log_logan proxy_auth logan\n" + "acl agent_log_logan_ix ext_user logan\n" + "http_access allow !icpt agent_log_logan !allowed_dst log_mark\n" + "http_access allow icpt agent_log_logan_ix !allowed_dst log_mark\n", + ) + + with checked("a restricted agent's allowed name is tunnelled without the proxy variables, and the log names the agent"): + wait_icpt("https://forge.t.local/", "200") + assert re.search( + r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ORIGINAL_DST/203\.0\.113\.2 ", journal() + ), "the HTTPS flow, as atlas" + assert icpt("http://forge.t.local/") == "200", "plain HTTP" + assert re.search(r"192\.168\.1\.3 TCP_\w+/200 .* GET http://forge\.t\.local/ atlas ", journal()), "the HTTP flow, as atlas" + assert icpt("https://agentonly.t.local/") == "200", "a name of atlas alone" + assert re.search(r"CONNECT agentonly\.t\.local:443 atlas ", journal()), "as atlas" + # The proxy-aware path is the same as before, and is not given the address's identity. + assert connect("forge.t.local", ATLAS) == "200", "a credential through the proxy port" + assert connect("forge.t.local") == "407", "no credential: the address does not stand in for one" + + with checked("a restricted agent's unlisted name is refused, intercepted, and logged as the agent's"): + assert icpt("https://other.t.local/") == "000", "HTTPS" + assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT other\.t\.local:443 atlas ", journal()), "logged as atlas" + assert icpt("http://other.t.local/") == "403", "plain HTTP" + assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* GET http://other\.t\.local/ atlas ", journal()), "logged as atlas" + # Control: the same address reaches the listed name, so the refusal is the policy's. + assert icpt("https://forge.t.local/") == "200" + + with checked("the swarm's deny list refuses an intercepted name that is otherwise allowed, and logs egress_denied"): + assert icpt("https://api.t.local/") == "200", "control: a base-list name" + write_deny( + "acl deny_hit_mark annotate_transaction egress_deny=1\n" + "acl deny_swarm_dst dstdomain api.t.local\n" + "http_access deny deny_swarm_dst deny_hit_mark\n" + ) + wait_icpt("https://api.t.local/", "000") + proxy.wait_until_succeeds( + "journalctl -u squid.service -o cat | grep -E " + "'egress_denied user=atlas method=CONNECT host=api\\.t\\.local port=443 result=TCP_DENIED/403'" + ) + assert icpt("https://forge.t.local/") == "200", "control: another name is unaffected" + + with checked("a log-mode agent's unlisted name is allowed and marked would_deny, and the deny list still wins"): + assert icpt("https://other.t.local/", LOGAN_IP) == "200" + proxy.wait_until_succeeds( + "journalctl -u squid.service -o cat | grep -E " + "'would_deny user=logan method=CONNECT host=other\\.t\\.local port=443 result=TCP_TUNNEL/200'" + ) + assert icpt("https://forge.t.local/", LOGAN_IP) == "200", "a listed name" + assert not re.search(r"would_deny user=logan .* host=forge\.t\.local", journal()), "a listed name is not marked" + assert icpt("https://api.t.local/", LOGAN_IP) == "000", "the deny list refuses it in log mode too" + assert icpt("https://other.t.local/") == "000", "control: atlas, unmarked and refused" + write_deny("") + wait_icpt("https://api.t.local/", "200") + + with checked("a source the bindings do not name is refused, and so is every source when they are gone"): + assert icpt("https://forge.t.local/", NOBODY_IP) == "000", "no lease for the address" + assert re.search(r"192\.168\.1\.7 TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 - ", journal()), "logged without a user" + assert icpt("http://forge.t.local/", NOBODY_IP) == "403" + assert icpt("https://forge.t.local/") == "200", "control: the bound address, the same name" + # A hive whose pin file is missing: the lease names a MAC, no port holds it. + proxy.succeed("rm /run/hive-egress-pins") + time.sleep(12) # the helper's answers are kept for 10 seconds + assert icpt("https://forge.t.local/") == "000", "no pin file" + bind("atlas", "logan") + wait_icpt("https://forge.t.local/", "200") + + with checked("an SNI that the destination does not serve, a missing SNI and an IP literal are refused"): + # Control: the right SNI at the right address is the 200 above. The + # same name sent to another address of the origin is not. + assert icpt("https://forge.t.local/", extra="--resolve forge.t.local:443:203.0.113.9") == "000" + assert re.search(r"NONE_NONE/409 .* CONNECT forge\.t\.local:443 ", journal()), "squid answered 409" + assert icpt("http://forge.t.local/", extra="-H 'Host: forge.t.local' --resolve forge.t.local:80:203.0.113.9") == "409", "the Host header too" + hello = "echo | timeout 10 openssl s_client -bind 192.168.1.3 -connect 203.0.113.2:443" + out = agent.succeed(f"{hello} -servername forge.t.local 2>&1 || true") + assert "BEGIN CERTIFICATE" in out, "control: with an SNI the origin's certificate comes back" + out = agent.succeed(f"{hello} -noservername 2>&1 || true") + assert "BEGIN CERTIFICATE" not in out, "no SNI, no certificate" + assert icpt("https://203.0.113.2/") == "000", "an IP literal sends no SNI" + assert re.search(r"TCP_DENIED/403 .* CONNECT 203\.0\.113\.2:443 atlas ", journal()), "logged as atlas" + assert icpt("http://203.0.113.2/") == "403", "an IP literal as the Host" + out = agent.succeed( + "timeout 5 bash -c 'exec 3<>/dev/tcp/203.0.113.2/443; echo hello >&3; cat <&3' 2>/dev/null || true" + ) + assert out.strip() == "", "bytes that are no ClientHello get no answer" + + with checked("the traffic that stays on the bridge or is the unrestricted agents' is not intercepted"): + seen = agent_lines() + # The gateway on the bridge address, in the stand-in's port 443. + assert icpt("http://${bridgeIp}:443/") != "000", "the host's own listener answers" + assert agent_lines() == seen, "squid saw nothing of it" + # Unrestricted agents and service containers are the MACs in `direct_mac`. + assert icpt("https://other.t.local/") == "000", "control: an unlisted name is refused" + proxy.succeed(f"nft add element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'") + seen = agent_lines() + assert icpt("https://other.t.local/") == "200", "an unlisted name, directly" + assert icpt("https://203.0.113.2/") == "200", "an IP literal, directly" + assert agent_lines() == seen, "squid saw nothing of it" + proxy.wait_until_succeeds( + "journalctl -k -o cat --no-pager | grep -E " + "'hive-egress-would-drop(-unmetered)? IN=eth1 OUT=eth2 .*SRC=192\\.168\\.1\\.3 DST=203\\.0\\.113\\.2 .*PROTO=TCP .*DPT=443 '" + ) + proxy.succeed(f"nft delete element inet hive_egress direct_mac '{{ {AGENT_MAC} }}'") + assert icpt("https://other.t.local/") == "000", "intercepted again" ''; }