diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index ec3b54ee..039aaf0a 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -11,6 +11,18 @@ let swarmDomain = config.services.hyperhive.swarm.domain; tlsCfg = config.services.hyperhive.tls; + # Self-signed gateway TLS: forgejo (Go) validates outbound webhook + # deliveries (e.g. the config-PR webhook to https:///webhook/...) + # against its system cert store, which lacks the runtime-generated hive + # CA — so delivery fails with an x509 "unknown authority". Go has no + # additive trust env var (SSL_CERT_FILE *replaces* the default bundle), + # so bind the public CA in and hand forgejo a combined bundle (system + # CAs + hive CA) via SSL_CERT_FILE. Only active in self-signed mode; + # with an operator cert / ACME the public chain already validates and + # this whole block drops out. The bind-mount + `container@` ordering + # that make the CA reachable are shared with hive-ci via the + # `hive-ca-trust` helper; only the Go SSL_CERT_FILE concat below is + # hive-forge-specific. # Forgejo's name for the login source. A constant, not an option: it # is the key this module's own idempotency check looks up, so making # it configurable would buy nothing and add a way for the lookup and @@ -60,6 +72,9 @@ let ssoRedirectUri = "${effectiveRootUrl}user/oauth2/${ssoSourceName}/callback"; caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; + useSelfSigned = caTrust.useSelfSigned; + caContainerPath = caTrust.caContainerPath; + forgeCaBundle = "/run/hive-forge-ca/ca-bundle.crt"; # ROOT_URL forgejo advertises in clone links + outbound URLs. When # served behind the gateway, `cfg.domain` doubles as both the @@ -564,9 +579,9 @@ in privateNetwork = false; # Self-signed mode: bind the public hive CA cert read-only so forgejo # can trust the gateway's self-signed leaf for outbound webhook - # delivery. The bind-mount, the `container@` ordering and the bundle - # the container's consumers read all come from the hive-ca-trust - # helper — see the container's `imports` for the consumption half. + # delivery (combined bundle assembled at container start by + # hive-forge-ca-bundle below). Shared bind-mount + ordering come from + # the hive-ca-trust helper. bindMounts = caTrust.bindMount; config = { pkgs, ... }: @@ -592,26 +607,6 @@ in ''; in { - imports = [ - # Forgejo is Go, and `SSL_CERT_FILE` *replaces* the default store - # rather than adding to it — so it needs the system CAs and the - # hive CA concatenated, not the CA alone, or public mirror fetches - # lose every anchor they had. - # - # BOTH units that make an outbound HTTPS call are consumers, not - # just the obvious one: `forgejo-sso-source` fetches the issuer's - # `.well-known/openid-configuration` over the swarm CA, and it - # once shipped without the trust its sibling had. `optional` - # rather than a flat list because that unit only exists when SSO - # is on — naming an absent unit would order nothing and quietly - # define a serviceless one. - (caTrust.trustBundle { - inherit pkgs; - name = "hive-forge"; - consumers = [ "forgejo" ] ++ lib.optional cfg.sso.enable "forgejo-sso-source"; - }) - ]; - system.stateVersion = "25.11"; # Shared host netns: this container's own firewall.service @@ -769,6 +764,36 @@ in "d /var/lib/forgejo/data/actions_artifacts 0750 forgejo forgejo - -" ]; + # Self-signed mode: assemble the combined TLS trust bundle + # (system CAs + the bind-mounted hive CA) forgejo's Go HTTP + # client validates outbound webhook deliveries against. Go's + # SSL_CERT_FILE *replaces* the default bundle, so we concatenate + # rather than point at the CA alone — otherwise mirror fetches + # from public hosts would lose their trust anchors. Runs before + # forgejo each boot; /run is tmpfs so the bundle is rebuilt from + # the current CA every start. + systemd.services.hive-forge-ca-bundle = lib.mkIf useSelfSigned { + description = "assemble forgejo TLS trust bundle (system CAs + hive CA)"; + wantedBy = [ "forgejo.service" ]; + before = [ "forgejo.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "hive-forge-ca-bundle"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -euo pipefail + install -d -m 0755 /run/hive-forge-ca + cat /etc/ssl/certs/ca-certificates.crt ${caContainerPath} \ + > ${forgeCaBundle} + chmod 0644 ${forgeCaBundle} + ''; + }; + # Point forgejo's Go TLS stack at the combined bundle so webhook + # delivery to the self-signed gateway validates. + systemd.services.forgejo.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned forgeCaBundle; + # Ensure Forgejo has a usable GPG signing key so UI merges / CRUD # commits are signed instead of erroring "does not have a signing # key". This service (a) generates a key in forgejo's persistent @@ -863,18 +888,23 @@ in # `FORGEJO_CUSTOM` (not `GITEA_CUSTOM` — forgejo renamed it) # is how the CLI finds the app.ini upstream's module wrote. # - # This unit is also a trust-bundle consumer (declared in the - # container's `imports` above, which is where `SSL_CERT_FILE` - # comes from): registering the login source makes an **outbound - # HTTPS call** to `/.well-known/openid-configuration`, - # served under the swarm CA the default store has never heard of. - # It once shipped without the trust `forgejo.service` had, and - # failed every single time with `x509: certificate signed by - # unknown authority`. The trust belongs to every process that - # makes the call, not to the obvious consumer. + # `SSL_CERT_FILE` for the same reason `forgejo.service` has it, + # and it was missing here: registering the login source makes an + # **outbound HTTPS call** — the CLI fetches + # `/.well-known/openid-configuration` to validate the + # provider before writing the row. That URL is a swarm service + # name served under the swarm CA, which the default system store + # has never heard of, so without this the unit fails every single + # time with `x509: certificate signed by unknown authority` and no + # restart can help it. + # + # The trust belongs to every process that makes the call, not to + # the service that happens to be the obvious consumer. Same + # binary, same host, different unit — and only one of them had it. environment = { FORGEJO_CUSTOM = "/var/lib/forgejo/custom"; - }; + } + // lib.optionalAttrs useSelfSigned { SSL_CERT_FILE = forgeCaBundle; }; path = [ cfg.package pkgs.coreutils diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index 0661aa6b..c3d6034b 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -14,10 +14,13 @@ let # Same runtime→build-time bridge hive-ci and hive-forge already cross: # binds the hive trust bundle (which folds in the swarm root) into the # container and orders the container after `hive-tls-ca.service`. The - # assembled bundle itself comes from `caTrust.trustBundle`, imported in - # the container config below. + # *consumption* is per-runtime and stays here — see the bundle service + # in the container config below. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; - + useSelfSigned = caTrust.useSelfSigned; + # tuwunel's own combined bundle, assembled at start. /run is tmpfs, so + # it is rebuilt from the current CA every boot rather than going stale. + matrixCaBundle = "/run/hive-matrix-ca/ca-bundle.crt"; swarmDomain = config.services.hyperhive.swarm.domain; # `url` is the half of the authelia module that exists on EVERY hive — @@ -787,27 +790,6 @@ in config = { ... }: { - imports = [ - # tuwunel's rustls verifier resolves through `rustls-native-certs` - # → `openssl-probe`, which reads `SSL_CERT_FILE` — so the - # openssl-shaped variable is the lever despite tuwunel linking no - # openssl. - # - # ⚠️ The helper CONCATENATES, and that is load-bearing here beyond - # the usual reason: `SSL_CERT_FILE` replaces the default location, - # so naming the hive anchor alone would drop every public CA and - # break federation with the wider matrix network — trading a small - # outage for a much larger one. - # A literal, not an option: this module names its container - # `containers.hive-matrix` directly and declares no `machine` - # option to derive it from. - (caTrust.trustBundle { - inherit pkgs; - name = "hive-matrix"; - consumers = [ "tuwunel" ]; - }) - ]; - system.stateVersion = "26.05"; # Shared host netns: this container's own firewall.service @@ -951,6 +933,41 @@ in # the unit's start — a credentials path does. ++ lib.optional cfg.sso.enable "oidc_client_secret:${toString cfg.sso.clientSecretFile}"; + # Federation TLS against a peer whose cert chains to the swarm + # root: tuwunel's outbound client is reqwest with the `rustls` + # feature, which builds a `rustls_platform_verifier::Verifier` + # and — because tuwunel calls `tls_certs_merge` rather than + # `tls_certs_only` — keeps the platform roots alongside its + # compiled-in webpki set. On Linux that verifier resolves through + # `rustls-native-certs` → `openssl-probe`, which reads + # `SSL_CERT_FILE`. So the openssl-shaped variable IS the lever + # here, despite tuwunel linking no openssl. + # + # ⚠️ CONCATENATE, never point at the anchor alone. `openssl-probe` + # uses `SSL_CERT_FILE` *instead of* the default location, so + # naming just the hive bundle would drop every public CA and + # break federation with the wider matrix network — trading a + # small outage for a much larger one. + systemd.services.hive-matrix-ca-bundle = lib.mkIf useSelfSigned { + description = "assemble tuwunel TLS trust bundle (system CAs + hive CA)"; + wantedBy = [ "tuwunel.service" ]; + before = [ "tuwunel.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "hive-matrix-ca-bundle"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -euo pipefail + install -d -m 0755 /run/hive-matrix-ca + cat /etc/ssl/certs/ca-certificates.crt ${caTrust.caContainerPath} \ + > ${matrixCaBundle} + chmod 0644 ${matrixCaBundle} + ''; + }; + systemd.services.tuwunel.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned matrixCaBundle; + environment.systemPackages = [ cfg.package ]; }; }; diff --git a/nix/host-modules/lib/hive-ca-trust.nix b/nix/host-modules/lib/hive-ca-trust.nix index e1959f93..5e46e878 100644 --- a/nix/host-modules/lib/hive-ca-trust.nix +++ b/nix/host-modules/lib/hive-ca-trust.nix @@ -6,11 +6,10 @@ # and orders its `container@` unit after `hive-tls-ca.service` so the # bind source exists before nspawn sets the mount up. # -# `bindMount` + `containerOrdering` are the language-agnostic half. The -# *consumption* differs per runtime: an additive variable (Node's -# `NODE_EXTRA_CA_CERTS`, hive-ci) points straight at `caContainerPath` from -# the call site, while a *replacing* one (Go's `SSL_CERT_FILE`, rustls) needs -# the system-CAs+hive-CA concat that `trustBundle` below does for it. +# This is the language-agnostic half (bind-mount + systemd ordering). The +# *consumption* differs per runtime and stays at each call site: Node's +# `NODE_EXTRA_CA_CERTS` is additive (hive-ci), Go's `SSL_CERT_FILE` replaces +# the bundle so it needs a system-CAs+hive-CA concat step (hive-forge). # # Pure function — NOT a NixOS module (don't add it to the host-modules # aggregator). Call it from a module's `let`: diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 878f4b1f..42a0f2ce 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -23,6 +23,7 @@ let swarmDomain = hyperhiveCfg.swarm.domain; caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; + useSelfSigned = caTrust.useSelfSigned; # Total on a null swarm domain for the same reason every sibling module is: # the required-domain assertion in hive-network.nix should be what an @@ -60,6 +61,7 @@ let # the authelia client and Grafana itself. redirectUri = "https://${cfg.domain}/login/generic_oauth"; + grafanaCaBundle = "/run/swarm-grafana-ca/ca-bundle.crt"; in { options.services.hyperhive.swarm.grafana = { @@ -355,16 +357,6 @@ in inherit (networkCfg) bridgeIp; dnsConsumers = [ "grafana.service" ]; }) - # Grafana is Go, and `SSL_CERT_FILE` *replaces* the default store - # rather than adding to it — so it needs the system CAs and the - # hive CA concatenated, not the CA alone. Without it the browser - # half of the login succeeds and the server-to-server token - # exchange fails `x509: unknown authority`. - (caTrust.trustBundle { - inherit pkgs; - name = cfg.machine; - consumers = [ "grafana" ]; - }) ]; system.stateVersion = "26.05"; @@ -379,6 +371,37 @@ in # the boundary after start. networking.resolvconf.enable = lib.mkForce false; + # Self-signed mode: Grafana is Go, and Go's `SSL_CERT_FILE` + # *replaces* the default bundle rather than adding to it — so + # concatenate the system CAs with the bind-mounted hive CA instead + # of pointing at the CA alone, which would lose every public + # anchor. /run is tmpfs, so this is rebuilt from the current CA + # each boot rather than going stale. + # + # Without it the browser half of the login succeeds and the + # server-to-server token exchange fails with an x509 "unknown + # authority" — the same shape of failure the swarm queue hit. + systemd.services.swarm-grafana-ca-bundle = lib.mkIf useSelfSigned { + description = "assemble Grafana TLS trust bundle (system CAs + hive CA)"; + wantedBy = [ "grafana.service" ]; + before = [ "grafana.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-grafana-ca-bundle"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -euo pipefail + install -d -m 0755 /run/swarm-grafana-ca + cat /etc/ssl/certs/ca-certificates.crt ${caTrust.caContainerPath} \ + > ${grafanaCaBundle} + chmod 0644 ${grafanaCaBundle} + ''; + }; + + systemd.services.grafana.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned grafanaCaBundle; + # Grafana's `secret_key` has **no default in nixpkgs** and an # assertion refuses the build without one — which is how the first # version of this module broke a deploy. It signs the datasource