40557ffb74
The guard inspected the assembled file for any certificate. The system store always holds certificates, so it passed unconditionally — including in the one case it was written to catch, where the hive CA half contributed nothing. That half is the only one that matters here: every name these consumers verify is issued by our own CA, so a bundle of nothing but public CAs is, for this purpose, an empty bundle that measures as full. The failure is silent and total — the unit reports success and every egress TLS call to a swarm service then fails. Counts the source on its own before assembling, and checks the result carries what both halves brought, so a source truncated between the count and the copy is caught too. Scope is stated at the guard: it proves the anchor was contributed, not that it is usable. A consumer reading only the first certificate ignores it regardless, which is what took the swarm collector down, and no check on this file can see that. Only a handshake can.
219 lines
11 KiB
Nix
219 lines
11 KiB
Nix
# Shared hive-CA trust plumbing for containers that must trust the
|
||
# self-signed gateway/forge leaf for *outbound* TLS (webhook delivery,
|
||
# CI artifact upload, …). The hive CA is generated at runtime by the host
|
||
# `hive-tls-ca.service` (see `hive-tls.nix`) — it can't be baked into a
|
||
# derivation — so each such container binds the public `ca.pem` read-only
|
||
# and orders its `container@<name>` unit after `hive-tls-ca.service` so the
|
||
# bind source exists before nspawn sets the mount up.
|
||
#
|
||
# `bindMount` + `containerOrdering` are the language-agnostic half. The
|
||
# *consumption* differs per runtime: an additive variable (Node's
|
||
# `NODE_EXTRA_CA_CERTS`, hive-ci) points straight at `caContainerPath` from
|
||
# the call site, while a *replacing* one (Go's `SSL_CERT_FILE`, rustls) needs
|
||
# the system-CAs+hive-CA concat that `trustBundle` below does for it.
|
||
#
|
||
# Pure function — NOT a NixOS module (don't add it to the host-modules
|
||
# aggregator). Call it from a module's `let`:
|
||
#
|
||
# caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||
# # then, in the container:
|
||
# # bindMounts = { … } // caTrust.bindMount;
|
||
# # systemd.services."container@hive-ci" = lib.mkMerge [ caTrust.containerOrdering … ];
|
||
# # environment.NODE_EXTRA_CA_CERTS = caTrust.caContainerPath; # consumption, per-caller
|
||
#
|
||
# `tlsCfg` = config.services.hyperhive.tls
|
||
# `gatewayCfg` = config.services.hyperhive.gateway
|
||
{
|
||
lib,
|
||
tlsCfg,
|
||
gatewayCfg,
|
||
}:
|
||
let
|
||
# `gateway.useSelfSigned` is the single source of truth for the
|
||
# self-signed condition — no duplicated derivation.
|
||
useSelfSigned = gatewayCfg.useSelfSigned;
|
||
# The bundle, not `ca.pem`: the hive CA is an intermediate under the
|
||
# swarm root, and openssl (which is what both consumers below sit on —
|
||
# Node's `NODE_EXTRA_CA_CERTS`, Go's `SSL_CERT_FILE`) will not end a
|
||
# chain at a trusted cert that isn't self-signed. `hive-tls.nix` writes
|
||
# the bundle next to the CA and explains the split.
|
||
caHostPath = "${tlsCfg.stateDir}/trust-bundle.pem";
|
||
caContainerPath = "/run/hive-ca/trust-bundle.pem";
|
||
|
||
# One definition, used by `trustBundle` to WRITE the bundle and published
|
||
# below so a caller can NAME it. Two copies of this path would be two
|
||
# things to keep in step, and the one that drifts is the reader.
|
||
bundleDirFor = name: "/run/${name}-ca";
|
||
bundlePathFor = name: "${bundleDirFor name}/trust-bundle.pem";
|
||
in
|
||
{
|
||
inherit useSelfSigned caContainerPath;
|
||
|
||
# Where `trustBundle` below puts the assembled bundle, for the callers
|
||
# that must NAME it rather than just have it exported. `SSL_CERT_FILE` is
|
||
# set for you and needs no path here; a consumer that takes its own CA
|
||
# argument (a client's `--cacert`) does, and the alternative is copying
|
||
# `/run/<name>-ca/…` to the call site. That copy breaks silently: the
|
||
# bundle keeps being written, the consumer keeps reading a path that no
|
||
# longer exists, and the failure surfaces as a TLS error naming the peer
|
||
# rather than the file.
|
||
#
|
||
# ⛔ **Do not hand this path to a consumer that reads only ONE
|
||
# certificate from it.** This comment used to offer an OIDC verifier's
|
||
# `issuer_ca_path` as the motivating example; that is exactly what took
|
||
# the swarm collector down for forty minutes once. The bundle is
|
||
# `system CAs ++ hive anchors`, so the anchor is ~123rd, and a
|
||
# first-certificate-only reader gets whichever public CA sorts first and
|
||
# can verify nothing of ours — with a full, valid, 125-certificate file
|
||
# on disk and no check able to see it.
|
||
#
|
||
# 🔑 The general trust store (`SSL_CERT_FILE`, set by `trustBundle`)
|
||
# reads every certificate and is order-independent, which is why a
|
||
# consumer that can use the process trust store should simply be left to
|
||
# do so. Before naming this path, check how that consumer parses it —
|
||
# the requirement is a property of the *reader*, and nothing here can
|
||
# enforce it.
|
||
inherit bundlePathFor;
|
||
|
||
# Fold into the container's `bindMounts` via `//`. Binds ONLY the public
|
||
# CA cert (never the `hive-tls` state dir — it holds the CA + leaf private
|
||
# keys), read-only. Empty when not self-signed, so the whole trust path
|
||
# drops out cleanly.
|
||
bindMount = lib.optionalAttrs useSelfSigned {
|
||
${caContainerPath} = {
|
||
hostPath = caHostPath;
|
||
isReadOnly = true;
|
||
};
|
||
};
|
||
|
||
# Fold into the caller's `container@<name>` unit (via `lib.mkMerge` if the
|
||
# caller adds its own keys, e.g. hive-ci's `TimeoutStartSec`). Orders the
|
||
# container after the host `hive-tls-ca.service` so the bind source exists
|
||
# before nspawn sets the mount up — a condition-skipped/late CA would
|
||
# otherwise fail the container start.
|
||
containerOrdering = lib.mkIf useSelfSigned {
|
||
after = [ "hive-tls-ca.service" ];
|
||
requires = [ "hive-tls-ca.service" ];
|
||
};
|
||
|
||
# System CAs + hive CA in one bundle, with `SSL_CERT_FILE` set on each
|
||
# consumer — for runtimes whose trust variable *replaces* the store (Go,
|
||
# rustls-native-certs). An additive one (Node's `NODE_EXTRA_CA_CERTS`,
|
||
# hive-ci) needs no bundle and should not use this.
|
||
#
|
||
# imports = [ (caTrust.trustBundle { inherit pkgs; name = "swarm-nats";
|
||
# consumers = [ "swarm-nats-auth" ]; }) ];
|
||
#
|
||
# Three constraints, each earned:
|
||
# - `requires` on the CONSUMER: `before` orders but does not gate, so a
|
||
# failed assembly otherwise leaves it running and trusting *nothing*.
|
||
# - assemble to a temp path, verify, then move: `cat` of an empty bind
|
||
# exits 0, and a partial bundle must never appear under the final name.
|
||
# - `consumers` are BARE unit names — they are `systemd.services` keys
|
||
# (no suffix) *and* go in `before`/`requires` (suffixed). Reversed, the
|
||
# edge names a unit that does not exist and systemd orders nothing.
|
||
#
|
||
# Returns a module, not bare services: a caller already writing
|
||
# `systemd.services.<consumer>` cannot also write `systemd.services`.
|
||
#
|
||
# The two host-consumer flags are documented at their use sites below.
|
||
trustBundle =
|
||
{
|
||
name,
|
||
consumers,
|
||
pkgs,
|
||
# A HOST systemd service rather than something inside a container.
|
||
# `caContainerPath` is a bind mount that only exists in a container, so
|
||
# a host consumer reads the host copy — which means waiting for the
|
||
# unit that writes it. In a container that wait is the container's own
|
||
# (`containerOrdering`); here nothing carries it. One flag, because a
|
||
# host source without the ordering is a race.
|
||
hostUnit ? false,
|
||
# Whether the consumer exists at all. A container caller imports this
|
||
# into the CONTAINER's module set, so it vanishes with the container. A
|
||
# host caller imports it at the host's top level, where `imports` is
|
||
# unconditional — without this, a hive with the consumer off still gets
|
||
# a bundle oneshot *and* a `systemd.services.<consumer>` conjured by
|
||
# `genAttrs`, holding an `SSL_CERT_FILE` and no `ExecStart`.
|
||
enable ? true,
|
||
}:
|
||
let
|
||
dir = bundleDirFor name;
|
||
bundlePath = bundlePathFor name;
|
||
unit = "${name}-ca-bundle";
|
||
source = if hostUnit then caHostPath else caContainerPath;
|
||
in
|
||
{
|
||
_file = "hive-ca-trust.nix#trustBundle:${name}";
|
||
config.systemd.services = lib.optionalAttrs (useSelfSigned && enable) (
|
||
{
|
||
${unit} = {
|
||
description = "assemble ${name} TLS trust bundle (system CAs + hive CA)";
|
||
wantedBy = [ "multi-user.target" ];
|
||
before = map (c: "${c}.service") consumers;
|
||
# Only for a host consumer: the CA file is written at runtime by
|
||
# `hive-tls-ca.service`, and reading it directly means waiting
|
||
# for it. A container consumer reads a bind mount instead, and
|
||
# its `container@` unit carries the equivalent wait.
|
||
after = lib.optionals hostUnit [ "hive-tls-ca.service" ];
|
||
requires = lib.optionals hostUnit [ "hive-tls-ca.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = unit;
|
||
};
|
||
path = [
|
||
pkgs.coreutils
|
||
pkgs.gnugrep
|
||
];
|
||
script = ''
|
||
set -euo pipefail
|
||
install -d -m 0755 ${dir}
|
||
tmp=${bundlePath}.tmp
|
||
# Count the HIVE half on its own, before assembling.
|
||
#
|
||
# Inspecting the assembled file cannot work: the system store
|
||
# always holds certificates, so "does the result contain a
|
||
# certificate" passes unconditionally — including in the one case
|
||
# worth catching, where this source contributed nothing. And the
|
||
# public CAs are irrelevant to what this bundle is for: every name
|
||
# the consumer verifies is issued by our own CA, so a bundle of
|
||
# nothing but public CAs is, for this purpose, an empty one that
|
||
# measures as full.
|
||
#
|
||
# `grep -c` exits 1 on zero matches and `set -e` is on, hence the
|
||
# `|| true` — without it the guard would die instead of reporting.
|
||
contributed=$(grep -c 'BEGIN CERTIFICATE' ${source} || true)
|
||
if [ "$contributed" -eq 0 ]; then
|
||
echo "${unit}: ${source} contributed no certificate to the bundle" >&2
|
||
exit 1
|
||
fi
|
||
system=$(grep -c 'BEGIN CERTIFICATE' /etc/ssl/certs/ca-certificates.crt || true)
|
||
cat /etc/ssl/certs/ca-certificates.crt ${source} > "$tmp"
|
||
# `cat` of a source truncated between the count and the copy still
|
||
# exits 0, so the result is checked against what both halves
|
||
# brought rather than merely for being non-empty.
|
||
total=$(grep -c 'BEGIN CERTIFICATE' "$tmp" || true)
|
||
if [ "$total" -ne $((system + contributed)) ]; then
|
||
echo "${unit}: bundle has $total certificates, expected $system + $contributed" >&2
|
||
exit 1
|
||
fi
|
||
# ⚠️ Scope: this proves the anchor was CONTRIBUTED, not that it is
|
||
# USABLE. A consumer that reads only the first certificate of the
|
||
# file ignores it anyway — that is what took the swarm collector
|
||
# down — and no check on this file can see it. Only a real
|
||
# handshake can. Do not read a green assembly as a working trust
|
||
# store.
|
||
chmod 0644 "$tmp"
|
||
mv "$tmp" ${bundlePath}
|
||
'';
|
||
};
|
||
}
|
||
// lib.genAttrs consumers (_: {
|
||
requires = [ "${unit}.service" ];
|
||
after = [ "${unit}.service" ];
|
||
environment.SSL_CERT_FILE = bundlePath;
|
||
})
|
||
);
|
||
};
|
||
}
|