nix/host-modules/hive-forge/default.nix

@@ -11,6 +11,18 @@ let
   swarmDomain = config.services.hyperhive.swarm.domain;
   tlsCfg = config.services.hyperhive.tls;
 
+  # Self-signed gateway TLS: forgejo (Go) validates outbound webhook
+  # deliveries (e.g. the config-PR webhook to https://<domain>/webhook/...)
+  # against its system cert store, which lacks the runtime-generated hive
+  # CA — so delivery fails with an x509 "unknown authority". Go has no
+  # additive trust env var (SSL_CERT_FILE *replaces* the default bundle),
+  # so bind the public CA in and hand forgejo a combined bundle (system
+  # CAs + hive CA) via SSL_CERT_FILE. Only active in self-signed mode;
+  # with an operator cert / ACME the public chain already validates and
+  # this whole block drops out. The bind-mount + `container@` ordering
+  # that make the CA reachable are shared with hive-ci via the
+  # `hive-ca-trust` helper; only the Go SSL_CERT_FILE concat below is
+  # hive-forge-specific.
   # Forgejo's name for the login source. A constant, not an option: it
   # is the key this module's own idempotency check looks up, so making
   # it configurable would buy nothing and add a way for the lookup and
@@ -60,6 +72,9 @@ let
   ssoRedirectUri = "${effectiveRootUrl}user/oauth2/${ssoSourceName}/callback";
 
   caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
+  useSelfSigned = caTrust.useSelfSigned;
+  caContainerPath = caTrust.caContainerPath;
+  forgeCaBundle = "/run/hive-forge-ca/ca-bundle.crt";
 
   # ROOT_URL forgejo advertises in clone links + outbound URLs. When
   # served behind the gateway, `cfg.domain` doubles as both the
@@ -564,9 +579,9 @@ in
       privateNetwork = false;
       # Self-signed mode: bind the public hive CA cert read-only so forgejo
       # can trust the gateway's self-signed leaf for outbound webhook
-      # delivery. The bind-mount, the `container@` ordering and the bundle
-      # the container's consumers read all come from the hive-ca-trust
-      # helper — see the container's `imports` for the consumption half.
+      # delivery (combined bundle assembled at container start by
+      # hive-forge-ca-bundle below). Shared bind-mount + ordering come from
+      # the hive-ca-trust helper.
       bindMounts = caTrust.bindMount;
       config =
         { pkgs, ... }:
@@ -592,26 +607,6 @@ in
           '';
         in
         {
-          imports = [
-            # Forgejo is Go, and `SSL_CERT_FILE` *replaces* the default store
-            # rather than adding to it — so it needs the system CAs and the
-            # hive CA concatenated, not the CA alone, or public mirror fetches
-            # lose every anchor they had.
-            #
-            # BOTH units that make an outbound HTTPS call are consumers, not
-            # just the obvious one: `forgejo-sso-source` fetches the issuer's
-            # `.well-known/openid-configuration` over the swarm CA, and it
-            # once shipped without the trust its sibling had. `optional`
-            # rather than a flat list because that unit only exists when SSO
-            # is on — naming an absent unit would order nothing and quietly
-            # define a serviceless one.
-            (caTrust.trustBundle {
-              inherit pkgs;
-              name = "hive-forge";
-              consumers = [ "forgejo" ] ++ lib.optional cfg.sso.enable "forgejo-sso-source";
-            })
-          ];
-
           system.stateVersion = "25.11";
 
           # Shared host netns: this container's own firewall.service
@@ -769,6 +764,36 @@ in
             "d /var/lib/forgejo/data/actions_artifacts 0750 forgejo forgejo - -"
           ];
 
+          # Self-signed mode: assemble the combined TLS trust bundle
+          # (system CAs + the bind-mounted hive CA) forgejo's Go HTTP
+          # client validates outbound webhook deliveries against. Go's
+          # SSL_CERT_FILE *replaces* the default bundle, so we concatenate
+          # rather than point at the CA alone — otherwise mirror fetches
+          # from public hosts would lose their trust anchors. Runs before
+          # forgejo each boot; /run is tmpfs so the bundle is rebuilt from
+          # the current CA every start.
+          systemd.services.hive-forge-ca-bundle = lib.mkIf useSelfSigned {
+            description = "assemble forgejo TLS trust bundle (system CAs + hive CA)";
+            wantedBy = [ "forgejo.service" ];
+            before = [ "forgejo.service" ];
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+              SyslogIdentifier = "hive-forge-ca-bundle";
+            };
+            path = [ pkgs.coreutils ];
+            script = ''
+              set -euo pipefail
+              install -d -m 0755 /run/hive-forge-ca
+              cat /etc/ssl/certs/ca-certificates.crt ${caContainerPath} \
+                > ${forgeCaBundle}
+              chmod 0644 ${forgeCaBundle}
+            '';
+          };
+          # Point forgejo's Go TLS stack at the combined bundle so webhook
+          # delivery to the self-signed gateway validates.
+          systemd.services.forgejo.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned forgeCaBundle;
+
           # Ensure Forgejo has a usable GPG signing key so UI merges / CRUD
           # commits are signed instead of erroring "does not have a signing
           # key". This service (a) generates a key in forgejo's persistent
@@ -863,18 +888,23 @@ in
             # `FORGEJO_CUSTOM` (not `GITEA_CUSTOM` — forgejo renamed it)
             # is how the CLI finds the app.ini upstream's module wrote.
             #
-            # This unit is also a trust-bundle consumer (declared in the
-            # container's `imports` above, which is where `SSL_CERT_FILE`
-            # comes from): registering the login source makes an **outbound
-            # HTTPS call** to `<issuer>/.well-known/openid-configuration`,
-            # served under the swarm CA the default store has never heard of.
-            # It once shipped without the trust `forgejo.service` had, and
-            # failed every single time with `x509: certificate signed by
-            # unknown authority`. The trust belongs to every process that
-            # makes the call, not to the obvious consumer.
+            # `SSL_CERT_FILE` for the same reason `forgejo.service` has it,
+            # and it was missing here: registering the login source makes an
+            # **outbound HTTPS call** — the CLI fetches
+            # `<issuer>/.well-known/openid-configuration` to validate the
+            # provider before writing the row. That URL is a swarm service
+            # name served under the swarm CA, which the default system store
+            # has never heard of, so without this the unit fails every single
+            # time with `x509: certificate signed by unknown authority` and no
+            # restart can help it.
+            #
+            # The trust belongs to every process that makes the call, not to
+            # the service that happens to be the obvious consumer. Same
+            # binary, same host, different unit — and only one of them had it.
             environment = {
               FORGEJO_CUSTOM = "/var/lib/forgejo/custom";
-            };
+            }
+            // lib.optionalAttrs useSelfSigned { SSL_CERT_FILE = forgeCaBundle; };
             path = [
               cfg.package
               pkgs.coreutils

nix/host-modules/hive-matrix.nix

@@ -14,10 +14,13 @@ let
   # Same runtime→build-time bridge hive-ci and hive-forge already cross:
   # binds the hive trust bundle (which folds in the swarm root) into the
   # container and orders the container after `hive-tls-ca.service`. The
-  # assembled bundle itself comes from `caTrust.trustBundle`, imported in
-  # the container config below.
+  # *consumption* is per-runtime and stays here — see the bundle service
+  # in the container config below.
   caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
-
+  useSelfSigned = caTrust.useSelfSigned;
+  # tuwunel's own combined bundle, assembled at start. /run is tmpfs, so
+  # it is rebuilt from the current CA every boot rather than going stale.
+  matrixCaBundle = "/run/hive-matrix-ca/ca-bundle.crt";
   swarmDomain = config.services.hyperhive.swarm.domain;
 
   # `url` is the half of the authelia module that exists on EVERY hive —
@@ -787,27 +790,6 @@ in
       config =
         { ... }:
         {
-          imports = [
-            # tuwunel's rustls verifier resolves through `rustls-native-certs`
-            # → `openssl-probe`, which reads `SSL_CERT_FILE` — so the
-            # openssl-shaped variable is the lever despite tuwunel linking no
-            # openssl.
-            #
-            # ⚠️ The helper CONCATENATES, and that is load-bearing here beyond
-            # the usual reason: `SSL_CERT_FILE` replaces the default location,
-            # so naming the hive anchor alone would drop every public CA and
-            # break federation with the wider matrix network — trading a small
-            # outage for a much larger one.
-            # A literal, not an option: this module names its container
-            # `containers.hive-matrix` directly and declares no `machine`
-            # option to derive it from.
-            (caTrust.trustBundle {
-              inherit pkgs;
-              name = "hive-matrix";
-              consumers = [ "tuwunel" ];
-            })
-          ];
-
           system.stateVersion = "26.05";
 
           # Shared host netns: this container's own firewall.service
@@ -951,6 +933,41 @@ in
           # the unit's start — a credentials path does.
           ++ lib.optional cfg.sso.enable "oidc_client_secret:${toString cfg.sso.clientSecretFile}";
 
+          # Federation TLS against a peer whose cert chains to the swarm
+          # root: tuwunel's outbound client is reqwest with the `rustls`
+          # feature, which builds a `rustls_platform_verifier::Verifier`
+          # and — because tuwunel calls `tls_certs_merge` rather than
+          # `tls_certs_only` — keeps the platform roots alongside its
+          # compiled-in webpki set. On Linux that verifier resolves through
+          # `rustls-native-certs` → `openssl-probe`, which reads
+          # `SSL_CERT_FILE`. So the openssl-shaped variable IS the lever
+          # here, despite tuwunel linking no openssl.
+          #
+          # ⚠️ CONCATENATE, never point at the anchor alone. `openssl-probe`
+          # uses `SSL_CERT_FILE` *instead of* the default location, so
+          # naming just the hive bundle would drop every public CA and
+          # break federation with the wider matrix network — trading a
+          # small outage for a much larger one.
+          systemd.services.hive-matrix-ca-bundle = lib.mkIf useSelfSigned {
+            description = "assemble tuwunel TLS trust bundle (system CAs + hive CA)";
+            wantedBy = [ "tuwunel.service" ];
+            before = [ "tuwunel.service" ];
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+              SyslogIdentifier = "hive-matrix-ca-bundle";
+            };
+            path = [ pkgs.coreutils ];
+            script = ''
+              set -euo pipefail
+              install -d -m 0755 /run/hive-matrix-ca
+              cat /etc/ssl/certs/ca-certificates.crt ${caTrust.caContainerPath} \
+                > ${matrixCaBundle}
+              chmod 0644 ${matrixCaBundle}
+            '';
+          };
+          systemd.services.tuwunel.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned matrixCaBundle;
+
           environment.systemPackages = [ cfg.package ];
         };
     };

nix/host-modules/lib/hive-ca-trust.nix

@@ -6,11 +6,10 @@
 # and orders its `container@<name>` unit after `hive-tls-ca.service` so the
 # bind source exists before nspawn sets the mount up.
 #
-# `bindMount` + `containerOrdering` are the language-agnostic half. The
-# *consumption* differs per runtime: an additive variable (Node's
-# `NODE_EXTRA_CA_CERTS`, hive-ci) points straight at `caContainerPath` from
-# the call site, while a *replacing* one (Go's `SSL_CERT_FILE`, rustls) needs
-# the system-CAs+hive-CA concat that `trustBundle` below does for it.
+# This is the language-agnostic half (bind-mount + systemd ordering). The
+# *consumption* differs per runtime and stays at each call site: Node's
+# `NODE_EXTRA_CA_CERTS` is additive (hive-ci), Go's `SSL_CERT_FILE` replaces
+# the bundle so it needs a system-CAs+hive-CA concat step (hive-forge).
 #
 # Pure function — NOT a NixOS module (don't add it to the host-modules
 # aggregator). Call it from a module's `let`:

nix/host-modules/swarm-grafana.nix

@@ -23,6 +23,7 @@ let
   swarmDomain = hyperhiveCfg.swarm.domain;
 
   caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
+  useSelfSigned = caTrust.useSelfSigned;
 
   # Total on a null swarm domain for the same reason every sibling module is:
   # the required-domain assertion in hive-network.nix should be what an
@@ -60,6 +61,7 @@ let
   # the authelia client and Grafana itself.
   redirectUri = "https://${cfg.domain}/login/generic_oauth";
 
+  grafanaCaBundle = "/run/swarm-grafana-ca/ca-bundle.crt";
 in
 {
   options.services.hyperhive.swarm.grafana = {
@@ -355,16 +357,6 @@ in
               inherit (networkCfg) bridgeIp;
               dnsConsumers = [ "grafana.service" ];
             })
-            # Grafana is Go, and `SSL_CERT_FILE` *replaces* the default store
-            # rather than adding to it — so it needs the system CAs and the
-            # hive CA concatenated, not the CA alone. Without it the browser
-            # half of the login succeeds and the server-to-server token
-            # exchange fails `x509: unknown authority`.
-            (caTrust.trustBundle {
-              inherit pkgs;
-              name = cfg.machine;
-              consumers = [ "grafana" ];
-            })
           ];
 
           system.stateVersion = "26.05";
@@ -379,6 +371,37 @@ in
           # the boundary after start.
           networking.resolvconf.enable = lib.mkForce false;
 
+          # Self-signed mode: Grafana is Go, and Go's `SSL_CERT_FILE`
+          # *replaces* the default bundle rather than adding to it — so
+          # concatenate the system CAs with the bind-mounted hive CA instead
+          # of pointing at the CA alone, which would lose every public
+          # anchor. /run is tmpfs, so this is rebuilt from the current CA
+          # each boot rather than going stale.
+          #
+          # Without it the browser half of the login succeeds and the
+          # server-to-server token exchange fails with an x509 "unknown
+          # authority" — the same shape of failure the swarm queue hit.
+          systemd.services.swarm-grafana-ca-bundle = lib.mkIf useSelfSigned {
+            description = "assemble Grafana TLS trust bundle (system CAs + hive CA)";
+            wantedBy = [ "grafana.service" ];
+            before = [ "grafana.service" ];
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+              SyslogIdentifier = "swarm-grafana-ca-bundle";
+            };
+            path = [ pkgs.coreutils ];
+            script = ''
+              set -euo pipefail
+              install -d -m 0755 /run/swarm-grafana-ca
+              cat /etc/ssl/certs/ca-certificates.crt ${caTrust.caContainerPath} \
+                > ${grafanaCaBundle}
+              chmod 0644 ${grafanaCaBundle}
+            '';
+          };
+
+          systemd.services.grafana.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned grafanaCaBundle;
+
           # Grafana's `secret_key` has **no default in nixpkgs** and an
           # assertion refuses the build without one — which is how the first
           # version of this module broke a deploy. It signs the datasource