b8b571fea0
The swarm collector could never verify its OIDC issuer, so it exited at startup on every boot and the hive tier dropped every metric. `issuer_ca_path` loads only the FIRST certificate in the file it names. The bundle assembled for this container is `system CAs ++ hive anchors`, so the swarm CA sits ~123rd and was never in the pool: the extension got whichever public CA sorts first, could verify nothing of ours, and failed `x509: certificate signed by unknown authority` — with 125 valid certificates in the file. Leaving the option unset makes the extension use the process trust store, which `trustBundle` already populates via `SSL_CERT_FILE`, and that consumer reads every certificate regardless of order. One file, two consumers, opposite parsing; the fix is to stop naming it twice rather than to reorder the bundle. Measured with the deployed binary against the deployed config, varying only the CA source: root-only OK, root-first OK, root-last FAILS, root-second FAILS, and unset-with-SSL_CERT_FILE OK against a control that correctly fails when the anchor is absent.
192 lines
8.9 KiB
Nix
192 lines
8.9 KiB
Nix
# Shared hive-CA trust plumbing for containers that must trust the
|
|
# self-signed gateway/forge leaf for *outbound* TLS (webhook delivery,
|
|
# CI artifact upload, …). The hive CA is generated at runtime by the host
|
|
# `hive-tls-ca.service` (see `hive-tls.nix`) — it can't be baked into a
|
|
# derivation — so each such container binds the public `ca.pem` read-only
|
|
# and orders its `container@<name>` unit after `hive-tls-ca.service` so the
|
|
# bind source exists before nspawn sets the mount up.
|
|
#
|
|
# `bindMount` + `containerOrdering` are the language-agnostic half. The
|
|
# *consumption* differs per runtime: an additive variable (Node's
|
|
# `NODE_EXTRA_CA_CERTS`, hive-ci) points straight at `caContainerPath` from
|
|
# the call site, while a *replacing* one (Go's `SSL_CERT_FILE`, rustls) needs
|
|
# the system-CAs+hive-CA concat that `trustBundle` below does for it.
|
|
#
|
|
# Pure function — NOT a NixOS module (don't add it to the host-modules
|
|
# aggregator). Call it from a module's `let`:
|
|
#
|
|
# caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
# # then, in the container:
|
|
# # bindMounts = { … } // caTrust.bindMount;
|
|
# # systemd.services."container@hive-ci" = lib.mkMerge [ caTrust.containerOrdering … ];
|
|
# # environment.NODE_EXTRA_CA_CERTS = caTrust.caContainerPath; # consumption, per-caller
|
|
#
|
|
# `tlsCfg` = config.services.hyperhive.tls
|
|
# `gatewayCfg` = config.services.hyperhive.gateway
|
|
{
|
|
lib,
|
|
tlsCfg,
|
|
gatewayCfg,
|
|
}:
|
|
let
|
|
# `gateway.useSelfSigned` is the single source of truth for the
|
|
# self-signed condition — no duplicated derivation.
|
|
useSelfSigned = gatewayCfg.useSelfSigned;
|
|
# The bundle, not `ca.pem`: the hive CA is an intermediate under the
|
|
# swarm root, and openssl (which is what both consumers below sit on —
|
|
# Node's `NODE_EXTRA_CA_CERTS`, Go's `SSL_CERT_FILE`) will not end a
|
|
# chain at a trusted cert that isn't self-signed. `hive-tls.nix` writes
|
|
# the bundle next to the CA and explains the split.
|
|
caHostPath = "${tlsCfg.stateDir}/trust-bundle.pem";
|
|
caContainerPath = "/run/hive-ca/trust-bundle.pem";
|
|
|
|
# One definition, used by `trustBundle` to WRITE the bundle and published
|
|
# below so a caller can NAME it. Two copies of this path would be two
|
|
# things to keep in step, and the one that drifts is the reader.
|
|
bundleDirFor = name: "/run/${name}-ca";
|
|
bundlePathFor = name: "${bundleDirFor name}/trust-bundle.pem";
|
|
in
|
|
{
|
|
inherit useSelfSigned caContainerPath;
|
|
|
|
# Where `trustBundle` below puts the assembled bundle, for the callers
|
|
# that must NAME it rather than just have it exported. `SSL_CERT_FILE` is
|
|
# set for you and needs no path here; a consumer that takes its own CA
|
|
# argument (a client's `--cacert`) does, and the alternative is copying
|
|
# `/run/<name>-ca/…` to the call site. That copy breaks silently: the
|
|
# bundle keeps being written, the consumer keeps reading a path that no
|
|
# longer exists, and the failure surfaces as a TLS error naming the peer
|
|
# rather than the file.
|
|
#
|
|
# ⛔ **Do not hand this path to a consumer that reads only ONE
|
|
# certificate from it.** This comment used to offer an OIDC verifier's
|
|
# `issuer_ca_path` as the motivating example; that is exactly what took
|
|
# the swarm collector down for forty minutes once. The bundle is
|
|
# `system CAs ++ hive anchors`, so the anchor is ~123rd, and a
|
|
# first-certificate-only reader gets whichever public CA sorts first and
|
|
# can verify nothing of ours — with a full, valid, 125-certificate file
|
|
# on disk and no check able to see it.
|
|
#
|
|
# 🔑 The general trust store (`SSL_CERT_FILE`, set by `trustBundle`)
|
|
# reads every certificate and is order-independent, which is why a
|
|
# consumer that can use the process trust store should simply be left to
|
|
# do so. Before naming this path, check how that consumer parses it —
|
|
# the requirement is a property of the *reader*, and nothing here can
|
|
# enforce it.
|
|
inherit bundlePathFor;
|
|
|
|
# Fold into the container's `bindMounts` via `//`. Binds ONLY the public
|
|
# CA cert (never the `hive-tls` state dir — it holds the CA + leaf private
|
|
# keys), read-only. Empty when not self-signed, so the whole trust path
|
|
# drops out cleanly.
|
|
bindMount = lib.optionalAttrs useSelfSigned {
|
|
${caContainerPath} = {
|
|
hostPath = caHostPath;
|
|
isReadOnly = true;
|
|
};
|
|
};
|
|
|
|
# Fold into the caller's `container@<name>` unit (via `lib.mkMerge` if the
|
|
# caller adds its own keys, e.g. hive-ci's `TimeoutStartSec`). Orders the
|
|
# container after the host `hive-tls-ca.service` so the bind source exists
|
|
# before nspawn sets the mount up — a condition-skipped/late CA would
|
|
# otherwise fail the container start.
|
|
containerOrdering = lib.mkIf useSelfSigned {
|
|
after = [ "hive-tls-ca.service" ];
|
|
requires = [ "hive-tls-ca.service" ];
|
|
};
|
|
|
|
# System CAs + hive CA in one bundle, with `SSL_CERT_FILE` set on each
|
|
# consumer — for runtimes whose trust variable *replaces* the store (Go,
|
|
# rustls-native-certs). An additive one (Node's `NODE_EXTRA_CA_CERTS`,
|
|
# hive-ci) needs no bundle and should not use this.
|
|
#
|
|
# imports = [ (caTrust.trustBundle { inherit pkgs; name = "swarm-nats";
|
|
# consumers = [ "swarm-nats-auth" ]; }) ];
|
|
#
|
|
# Three constraints, each earned:
|
|
# - `requires` on the CONSUMER: `before` orders but does not gate, so a
|
|
# failed assembly otherwise leaves it running and trusting *nothing*.
|
|
# - assemble to a temp path, verify, then move: `cat` of an empty bind
|
|
# exits 0, and a partial bundle must never appear under the final name.
|
|
# - `consumers` are BARE unit names — they are `systemd.services` keys
|
|
# (no suffix) *and* go in `before`/`requires` (suffixed). Reversed, the
|
|
# edge names a unit that does not exist and systemd orders nothing.
|
|
#
|
|
# Returns a module, not bare services: a caller already writing
|
|
# `systemd.services.<consumer>` cannot also write `systemd.services`.
|
|
#
|
|
# The two host-consumer flags are documented at their use sites below.
|
|
trustBundle =
|
|
{
|
|
name,
|
|
consumers,
|
|
pkgs,
|
|
# A HOST systemd service rather than something inside a container.
|
|
# `caContainerPath` is a bind mount that only exists in a container, so
|
|
# a host consumer reads the host copy — which means waiting for the
|
|
# unit that writes it. In a container that wait is the container's own
|
|
# (`containerOrdering`); here nothing carries it. One flag, because a
|
|
# host source without the ordering is a race.
|
|
hostUnit ? false,
|
|
# Whether the consumer exists at all. A container caller imports this
|
|
# into the CONTAINER's module set, so it vanishes with the container. A
|
|
# host caller imports it at the host's top level, where `imports` is
|
|
# unconditional — without this, a hive with the consumer off still gets
|
|
# a bundle oneshot *and* a `systemd.services.<consumer>` conjured by
|
|
# `genAttrs`, holding an `SSL_CERT_FILE` and no `ExecStart`.
|
|
enable ? true,
|
|
}:
|
|
let
|
|
dir = bundleDirFor name;
|
|
bundlePath = bundlePathFor name;
|
|
unit = "${name}-ca-bundle";
|
|
source = if hostUnit then caHostPath else caContainerPath;
|
|
in
|
|
{
|
|
_file = "hive-ca-trust.nix#trustBundle:${name}";
|
|
config.systemd.services = lib.optionalAttrs (useSelfSigned && enable) (
|
|
{
|
|
${unit} = {
|
|
description = "assemble ${name} TLS trust bundle (system CAs + hive CA)";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = map (c: "${c}.service") consumers;
|
|
# Only for a host consumer: the CA file is written at runtime by
|
|
# `hive-tls-ca.service`, and reading it directly means waiting
|
|
# for it. A container consumer reads a bind mount instead, and
|
|
# its `container@` unit carries the equivalent wait.
|
|
after = lib.optionals hostUnit [ "hive-tls-ca.service" ];
|
|
requires = lib.optionals hostUnit [ "hive-tls-ca.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
SyslogIdentifier = unit;
|
|
};
|
|
path = [
|
|
pkgs.coreutils
|
|
pkgs.gnugrep
|
|
];
|
|
script = ''
|
|
set -euo pipefail
|
|
install -d -m 0755 ${dir}
|
|
tmp=${bundlePath}.tmp
|
|
cat /etc/ssl/certs/ca-certificates.crt ${source} > "$tmp"
|
|
# `cat` of an empty or missing-but-mounted source exits 0, so the
|
|
# result has to be inspected rather than the command trusted.
|
|
if ! grep -q 'BEGIN CERTIFICATE' "$tmp"; then
|
|
echo "${unit}: assembled bundle contains no certificate" >&2
|
|
exit 1
|
|
fi
|
|
chmod 0644 "$tmp"
|
|
mv "$tmp" ${bundlePath}
|
|
'';
|
|
};
|
|
}
|
|
// lib.genAttrs consumers (_: {
|
|
requires = [ "${unit}.service" ];
|
|
after = [ "${unit}.service" ];
|
|
environment.SSL_CERT_FILE = bundlePath;
|
|
})
|
|
);
|
|
};
|
|
}
|