From 9073c9b3f7f68eedaa3a426b3d823c7a74c75940 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 17 Aug 2026 20:29:22 +0200 Subject: [PATCH 1/3] fix(#3396): move grafana and matrix onto the shared trust-bundle helper Both hand-rolled the same concat with wantedBy + before and no requires, so a failed assembly let the consumer start against a missing file and trust nothing at all -- which fails every outbound TLS call while the unit looks healthy. The helper puts requires on the consumer and verifies the assembled bundle before moving it into place. Removes each module's now-dead useSelfSigned binding: its only use was the mkIf on the deleted block, and nix does not warn about an unused let binding. Also corrects two comments the deletion orphaned -- one described the removed path binding, the other pointed at a bundle service that no longer exists. --- nix/host-modules/hive-matrix.nix | 62 ++++++++++-------------------- nix/host-modules/swarm-grafana.nix | 43 +++++---------------- 2 files changed, 31 insertions(+), 74 deletions(-) diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index c3d6034b..248695f6 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -14,13 +14,10 @@ let # Same runtime→build-time bridge hive-ci and hive-forge already cross: # binds the hive trust bundle (which folds in the swarm root) into the # container and orders the container after `hive-tls-ca.service`. The - # *consumption* is per-runtime and stays here — see the bundle service - # in the container config below. + # assembled bundle itself comes from `caTrust.trustBundle`, imported in + # the container config below. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; - useSelfSigned = caTrust.useSelfSigned; - # tuwunel's own combined bundle, assembled at start. /run is tmpfs, so - # it is rebuilt from the current CA every boot rather than going stale. - matrixCaBundle = "/run/hive-matrix-ca/ca-bundle.crt"; + swarmDomain = config.services.hyperhive.swarm.domain; # `url` is the half of the authelia module that exists on EVERY hive — @@ -790,6 +787,24 @@ in config = { ... }: { + imports = [ + # tuwunel's rustls verifier resolves through `rustls-native-certs` + # → `openssl-probe`, which reads `SSL_CERT_FILE` — so the + # openssl-shaped variable is the lever despite tuwunel linking no + # openssl. + # + # ⚠️ The helper CONCATENATES, and that is load-bearing here beyond + # the usual reason: `SSL_CERT_FILE` replaces the default location, + # so naming the hive anchor alone would drop every public CA and + # break federation with the wider matrix network — trading a small + # outage for a much larger one. + (caTrust.trustBundle { + inherit pkgs; + name = cfg.machine; + consumers = [ "tuwunel" ]; + }) + ]; + system.stateVersion = "26.05"; # Shared host netns: this container's own firewall.service @@ -933,41 +948,6 @@ in # the unit's start — a credentials path does. ++ lib.optional cfg.sso.enable "oidc_client_secret:${toString cfg.sso.clientSecretFile}"; - # Federation TLS against a peer whose cert chains to the swarm - # root: tuwunel's outbound client is reqwest with the `rustls` - # feature, which builds a `rustls_platform_verifier::Verifier` - # and — because tuwunel calls `tls_certs_merge` rather than - # `tls_certs_only` — keeps the platform roots alongside its - # compiled-in webpki set. On Linux that verifier resolves through - # `rustls-native-certs` → `openssl-probe`, which reads - # `SSL_CERT_FILE`. So the openssl-shaped variable IS the lever - # here, despite tuwunel linking no openssl. - # - # ⚠️ CONCATENATE, never point at the anchor alone. `openssl-probe` - # uses `SSL_CERT_FILE` *instead of* the default location, so - # naming just the hive bundle would drop every public CA and - # break federation with the wider matrix network — trading a - # small outage for a much larger one. - systemd.services.hive-matrix-ca-bundle = lib.mkIf useSelfSigned { - description = "assemble tuwunel TLS trust bundle (system CAs + hive CA)"; - wantedBy = [ "tuwunel.service" ]; - before = [ "tuwunel.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - SyslogIdentifier = "hive-matrix-ca-bundle"; - }; - path = [ pkgs.coreutils ]; - script = '' - set -euo pipefail - install -d -m 0755 /run/hive-matrix-ca - cat /etc/ssl/certs/ca-certificates.crt ${caTrust.caContainerPath} \ - > ${matrixCaBundle} - chmod 0644 ${matrixCaBundle} - ''; - }; - systemd.services.tuwunel.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned matrixCaBundle; - environment.systemPackages = [ cfg.package ]; }; }; diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 42a0f2ce..878f4b1f 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -23,7 +23,6 @@ let swarmDomain = hyperhiveCfg.swarm.domain; caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; - useSelfSigned = caTrust.useSelfSigned; # Total on a null swarm domain for the same reason every sibling module is: # the required-domain assertion in hive-network.nix should be what an @@ -61,7 +60,6 @@ let # the authelia client and Grafana itself. redirectUri = "https://${cfg.domain}/login/generic_oauth"; - grafanaCaBundle = "/run/swarm-grafana-ca/ca-bundle.crt"; in { options.services.hyperhive.swarm.grafana = { @@ -357,6 +355,16 @@ in inherit (networkCfg) bridgeIp; dnsConsumers = [ "grafana.service" ]; }) + # Grafana is Go, and `SSL_CERT_FILE` *replaces* the default store + # rather than adding to it — so it needs the system CAs and the + # hive CA concatenated, not the CA alone. Without it the browser + # half of the login succeeds and the server-to-server token + # exchange fails `x509: unknown authority`. + (caTrust.trustBundle { + inherit pkgs; + name = cfg.machine; + consumers = [ "grafana" ]; + }) ]; system.stateVersion = "26.05"; @@ -371,37 +379,6 @@ in # the boundary after start. networking.resolvconf.enable = lib.mkForce false; - # Self-signed mode: Grafana is Go, and Go's `SSL_CERT_FILE` - # *replaces* the default bundle rather than adding to it — so - # concatenate the system CAs with the bind-mounted hive CA instead - # of pointing at the CA alone, which would lose every public - # anchor. /run is tmpfs, so this is rebuilt from the current CA - # each boot rather than going stale. - # - # Without it the browser half of the login succeeds and the - # server-to-server token exchange fails with an x509 "unknown - # authority" — the same shape of failure the swarm queue hit. - systemd.services.swarm-grafana-ca-bundle = lib.mkIf useSelfSigned { - description = "assemble Grafana TLS trust bundle (system CAs + hive CA)"; - wantedBy = [ "grafana.service" ]; - before = [ "grafana.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - SyslogIdentifier = "swarm-grafana-ca-bundle"; - }; - path = [ pkgs.coreutils ]; - script = '' - set -euo pipefail - install -d -m 0755 /run/swarm-grafana-ca - cat /etc/ssl/certs/ca-certificates.crt ${caTrust.caContainerPath} \ - > ${grafanaCaBundle} - chmod 0644 ${grafanaCaBundle} - ''; - }; - - systemd.services.grafana.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned grafanaCaBundle; - # Grafana's `secret_key` has **no default in nixpkgs** and an # assertion refuses the build without one — which is how the first # version of this module broke a deploy. It signs the datasource From 2eb74f3a74aa7ba6a026ed035f0a07d9d854329d Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 17 Aug 2026 20:35:56 +0200 Subject: [PATCH 2/3] fix(#3396): move the forge onto the shared trust-bundle helper Last of the three modules that hand-rolled the same concat with wantedBy + before and no requires, so a failed assembly left the consumer running against a missing file and trusting nothing -- every outbound TLS call fails while the unit looks healthy. The forge is the one with two consumers: forgejo-sso-source fetches the issuer's discovery document over the swarm CA and once shipped without the trust its sibling had. It only exists when SSO is on, so the consumer list is conditional -- naming an absent unit would define a serviceless one and order nothing. Removes the now-dead useSelfSigned and caContainerPath bindings (nix does not warn) and retargets three comments the deletion orphaned, including the helper header that still named this module as the per-call-site concat. --- nix/host-modules/hive-forge/default.nix | 96 +++++++++---------------- nix/host-modules/lib/hive-ca-trust.nix | 9 +-- 2 files changed, 38 insertions(+), 67 deletions(-) diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index 039aaf0a..ec3b54ee 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -11,18 +11,6 @@ let swarmDomain = config.services.hyperhive.swarm.domain; tlsCfg = config.services.hyperhive.tls; - # Self-signed gateway TLS: forgejo (Go) validates outbound webhook - # deliveries (e.g. the config-PR webhook to https:///webhook/...) - # against its system cert store, which lacks the runtime-generated hive - # CA — so delivery fails with an x509 "unknown authority". Go has no - # additive trust env var (SSL_CERT_FILE *replaces* the default bundle), - # so bind the public CA in and hand forgejo a combined bundle (system - # CAs + hive CA) via SSL_CERT_FILE. Only active in self-signed mode; - # with an operator cert / ACME the public chain already validates and - # this whole block drops out. The bind-mount + `container@` ordering - # that make the CA reachable are shared with hive-ci via the - # `hive-ca-trust` helper; only the Go SSL_CERT_FILE concat below is - # hive-forge-specific. # Forgejo's name for the login source. A constant, not an option: it # is the key this module's own idempotency check looks up, so making # it configurable would buy nothing and add a way for the lookup and @@ -72,9 +60,6 @@ let ssoRedirectUri = "${effectiveRootUrl}user/oauth2/${ssoSourceName}/callback"; caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; - useSelfSigned = caTrust.useSelfSigned; - caContainerPath = caTrust.caContainerPath; - forgeCaBundle = "/run/hive-forge-ca/ca-bundle.crt"; # ROOT_URL forgejo advertises in clone links + outbound URLs. When # served behind the gateway, `cfg.domain` doubles as both the @@ -579,9 +564,9 @@ in privateNetwork = false; # Self-signed mode: bind the public hive CA cert read-only so forgejo # can trust the gateway's self-signed leaf for outbound webhook - # delivery (combined bundle assembled at container start by - # hive-forge-ca-bundle below). Shared bind-mount + ordering come from - # the hive-ca-trust helper. + # delivery. The bind-mount, the `container@` ordering and the bundle + # the container's consumers read all come from the hive-ca-trust + # helper — see the container's `imports` for the consumption half. bindMounts = caTrust.bindMount; config = { pkgs, ... }: @@ -607,6 +592,26 @@ in ''; in { + imports = [ + # Forgejo is Go, and `SSL_CERT_FILE` *replaces* the default store + # rather than adding to it — so it needs the system CAs and the + # hive CA concatenated, not the CA alone, or public mirror fetches + # lose every anchor they had. + # + # BOTH units that make an outbound HTTPS call are consumers, not + # just the obvious one: `forgejo-sso-source` fetches the issuer's + # `.well-known/openid-configuration` over the swarm CA, and it + # once shipped without the trust its sibling had. `optional` + # rather than a flat list because that unit only exists when SSO + # is on — naming an absent unit would order nothing and quietly + # define a serviceless one. + (caTrust.trustBundle { + inherit pkgs; + name = "hive-forge"; + consumers = [ "forgejo" ] ++ lib.optional cfg.sso.enable "forgejo-sso-source"; + }) + ]; + system.stateVersion = "25.11"; # Shared host netns: this container's own firewall.service @@ -764,36 +769,6 @@ in "d /var/lib/forgejo/data/actions_artifacts 0750 forgejo forgejo - -" ]; - # Self-signed mode: assemble the combined TLS trust bundle - # (system CAs + the bind-mounted hive CA) forgejo's Go HTTP - # client validates outbound webhook deliveries against. Go's - # SSL_CERT_FILE *replaces* the default bundle, so we concatenate - # rather than point at the CA alone — otherwise mirror fetches - # from public hosts would lose their trust anchors. Runs before - # forgejo each boot; /run is tmpfs so the bundle is rebuilt from - # the current CA every start. - systemd.services.hive-forge-ca-bundle = lib.mkIf useSelfSigned { - description = "assemble forgejo TLS trust bundle (system CAs + hive CA)"; - wantedBy = [ "forgejo.service" ]; - before = [ "forgejo.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - SyslogIdentifier = "hive-forge-ca-bundle"; - }; - path = [ pkgs.coreutils ]; - script = '' - set -euo pipefail - install -d -m 0755 /run/hive-forge-ca - cat /etc/ssl/certs/ca-certificates.crt ${caContainerPath} \ - > ${forgeCaBundle} - chmod 0644 ${forgeCaBundle} - ''; - }; - # Point forgejo's Go TLS stack at the combined bundle so webhook - # delivery to the self-signed gateway validates. - systemd.services.forgejo.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned forgeCaBundle; - # Ensure Forgejo has a usable GPG signing key so UI merges / CRUD # commits are signed instead of erroring "does not have a signing # key". This service (a) generates a key in forgejo's persistent @@ -888,23 +863,18 @@ in # `FORGEJO_CUSTOM` (not `GITEA_CUSTOM` — forgejo renamed it) # is how the CLI finds the app.ini upstream's module wrote. # - # `SSL_CERT_FILE` for the same reason `forgejo.service` has it, - # and it was missing here: registering the login source makes an - # **outbound HTTPS call** — the CLI fetches - # `/.well-known/openid-configuration` to validate the - # provider before writing the row. That URL is a swarm service - # name served under the swarm CA, which the default system store - # has never heard of, so without this the unit fails every single - # time with `x509: certificate signed by unknown authority` and no - # restart can help it. - # - # The trust belongs to every process that makes the call, not to - # the service that happens to be the obvious consumer. Same - # binary, same host, different unit — and only one of them had it. + # This unit is also a trust-bundle consumer (declared in the + # container's `imports` above, which is where `SSL_CERT_FILE` + # comes from): registering the login source makes an **outbound + # HTTPS call** to `/.well-known/openid-configuration`, + # served under the swarm CA the default store has never heard of. + # It once shipped without the trust `forgejo.service` had, and + # failed every single time with `x509: certificate signed by + # unknown authority`. The trust belongs to every process that + # makes the call, not to the obvious consumer. environment = { FORGEJO_CUSTOM = "/var/lib/forgejo/custom"; - } - // lib.optionalAttrs useSelfSigned { SSL_CERT_FILE = forgeCaBundle; }; + }; path = [ cfg.package pkgs.coreutils diff --git a/nix/host-modules/lib/hive-ca-trust.nix b/nix/host-modules/lib/hive-ca-trust.nix index 5e46e878..e1959f93 100644 --- a/nix/host-modules/lib/hive-ca-trust.nix +++ b/nix/host-modules/lib/hive-ca-trust.nix @@ -6,10 +6,11 @@ # and orders its `container@` unit after `hive-tls-ca.service` so the # bind source exists before nspawn sets the mount up. # -# This is the language-agnostic half (bind-mount + systemd ordering). The -# *consumption* differs per runtime and stays at each call site: Node's -# `NODE_EXTRA_CA_CERTS` is additive (hive-ci), Go's `SSL_CERT_FILE` replaces -# the bundle so it needs a system-CAs+hive-CA concat step (hive-forge). +# `bindMount` + `containerOrdering` are the language-agnostic half. The +# *consumption* differs per runtime: an additive variable (Node's +# `NODE_EXTRA_CA_CERTS`, hive-ci) points straight at `caContainerPath` from +# the call site, while a *replacing* one (Go's `SSL_CERT_FILE`, rustls) needs +# the system-CAs+hive-CA concat that `trustBundle` below does for it. # # Pure function — NOT a NixOS module (don't add it to the host-modules # aggregator). Call it from a module's `let`: From 3fdc3bb15b64ea40ab9d4a90af5d88430a6c5d6d Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 17 Aug 2026 20:37:52 +0200 Subject: [PATCH 3/3] fix(#3396): name the matrix bundle from a literal, not a missing option hive-matrix.nix declares no `machine` option -- the idiom was copied from swarm-grafana, which does. Any hive with matrix enabled failed to evaluate. --- nix/host-modules/hive-matrix.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index 248695f6..0661aa6b 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -798,9 +798,12 @@ in # so naming the hive anchor alone would drop every public CA and # break federation with the wider matrix network — trading a small # outage for a much larger one. + # A literal, not an option: this module names its container + # `containers.hive-matrix` directly and declares no `machine` + # option to derive it from. (caTrust.trustBundle { inherit pkgs; - name = cfg.machine; + name = "hive-matrix"; consumers = [ "tuwunel" ]; }) ];