08faa0970e
The swarm collector accepted OTLP from anyone who could reach it, and took the `hive` resource attribute from the payload. So any writer on the swarm network could attribute metrics to any hive, and nothing downstream could tell. The label now comes from which receiver accepted the sample: one receiver per hive, each behind an `oidc` extension verifying a token minted for that hive's audience, each feeding a pipeline whose `resource` processor upserts a constant. A sender cannot influence it, because the only input is which authenticated port the bytes arrived on. That multiplicity is forced rather than preferred. A processor cannot read the token's claims — `from_context` reads request metadata, and asking it for an auth claim yields nothing, silently, with a healthy startup — and one receiver holding several credentials never reveals which one matched. The per-hive ports are internal: a hive reaches its receiver as a path under this collector's existing gateway name, so nginx (rendered from this same evaluation) is the only thing that names a port. Fronting each hive with its own vhost would need a certificate, a DNS name and a gateway entry per hive to express routing the gateway already does. Turning this on removes the unauthenticated receiver. While an open port still accepts samples the per-hive receivers are decoration, so this is the switch itself rather than a hardening layer beside it; a swarm that wants the open receiver says so. `hive-ca-trust.nix` grows `bundlePathFor`, because a consumer taking its own CA argument has to name the bundle rather than just have `SSL_CERT_FILE` exported at it.
176 lines
8.1 KiB
Nix
176 lines
8.1 KiB
Nix
# Shared hive-CA trust plumbing for containers that must trust the
|
|
# self-signed gateway/forge leaf for *outbound* TLS (webhook delivery,
|
|
# CI artifact upload, …). The hive CA is generated at runtime by the host
|
|
# `hive-tls-ca.service` (see `hive-tls.nix`) — it can't be baked into a
|
|
# derivation — so each such container binds the public `ca.pem` read-only
|
|
# and orders its `container@<name>` unit after `hive-tls-ca.service` so the
|
|
# bind source exists before nspawn sets the mount up.
|
|
#
|
|
# `bindMount` + `containerOrdering` are the language-agnostic half. The
|
|
# *consumption* differs per runtime: an additive variable (Node's
|
|
# `NODE_EXTRA_CA_CERTS`, hive-ci) points straight at `caContainerPath` from
|
|
# the call site, while a *replacing* one (Go's `SSL_CERT_FILE`, rustls) needs
|
|
# the system-CAs+hive-CA concat that `trustBundle` below does for it.
|
|
#
|
|
# Pure function — NOT a NixOS module (don't add it to the host-modules
|
|
# aggregator). Call it from a module's `let`:
|
|
#
|
|
# caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
# # then, in the container:
|
|
# # bindMounts = { … } // caTrust.bindMount;
|
|
# # systemd.services."container@hive-ci" = lib.mkMerge [ caTrust.containerOrdering … ];
|
|
# # environment.NODE_EXTRA_CA_CERTS = caTrust.caContainerPath; # consumption, per-caller
|
|
#
|
|
# `tlsCfg` = config.services.hyperhive.tls
|
|
# `gatewayCfg` = config.services.hyperhive.gateway
|
|
{
|
|
lib,
|
|
tlsCfg,
|
|
gatewayCfg,
|
|
}:
|
|
let
|
|
# `gateway.useSelfSigned` is the single source of truth for the
|
|
# self-signed condition — no duplicated derivation.
|
|
useSelfSigned = gatewayCfg.useSelfSigned;
|
|
# The bundle, not `ca.pem`: the hive CA is an intermediate under the
|
|
# swarm root, and openssl (which is what both consumers below sit on —
|
|
# Node's `NODE_EXTRA_CA_CERTS`, Go's `SSL_CERT_FILE`) will not end a
|
|
# chain at a trusted cert that isn't self-signed. `hive-tls.nix` writes
|
|
# the bundle next to the CA and explains the split.
|
|
caHostPath = "${tlsCfg.stateDir}/trust-bundle.pem";
|
|
caContainerPath = "/run/hive-ca/trust-bundle.pem";
|
|
|
|
# One definition, used by `trustBundle` to WRITE the bundle and published
|
|
# below so a caller can NAME it. Two copies of this path would be two
|
|
# things to keep in step, and the one that drifts is the reader.
|
|
bundleDirFor = name: "/run/${name}-ca";
|
|
bundlePathFor = name: "${bundleDirFor name}/trust-bundle.pem";
|
|
in
|
|
{
|
|
inherit useSelfSigned caContainerPath;
|
|
|
|
# Where `trustBundle` below puts the assembled bundle, for the callers
|
|
# that must NAME it rather than just have it exported. `SSL_CERT_FILE` is
|
|
# set for you and needs no path here; a consumer that takes its own CA
|
|
# argument (an OIDC verifier's `issuer_ca_path`, a client's `--cacert`)
|
|
# does, and the alternative is copying `/run/<name>-ca/…` to the call
|
|
# site. That copy breaks silently: the bundle keeps being written, the
|
|
# consumer keeps reading a path that no longer exists, and the failure
|
|
# surfaces as a TLS error naming the peer rather than the file.
|
|
inherit bundlePathFor;
|
|
|
|
# Fold into the container's `bindMounts` via `//`. Binds ONLY the public
|
|
# CA cert (never the `hive-tls` state dir — it holds the CA + leaf private
|
|
# keys), read-only. Empty when not self-signed, so the whole trust path
|
|
# drops out cleanly.
|
|
bindMount = lib.optionalAttrs useSelfSigned {
|
|
${caContainerPath} = {
|
|
hostPath = caHostPath;
|
|
isReadOnly = true;
|
|
};
|
|
};
|
|
|
|
# Fold into the caller's `container@<name>` unit (via `lib.mkMerge` if the
|
|
# caller adds its own keys, e.g. hive-ci's `TimeoutStartSec`). Orders the
|
|
# container after the host `hive-tls-ca.service` so the bind source exists
|
|
# before nspawn sets the mount up — a condition-skipped/late CA would
|
|
# otherwise fail the container start.
|
|
containerOrdering = lib.mkIf useSelfSigned {
|
|
after = [ "hive-tls-ca.service" ];
|
|
requires = [ "hive-tls-ca.service" ];
|
|
};
|
|
|
|
# System CAs + hive CA in one bundle, with `SSL_CERT_FILE` set on each
|
|
# consumer — for runtimes whose trust variable *replaces* the store (Go,
|
|
# rustls-native-certs). An additive one (Node's `NODE_EXTRA_CA_CERTS`,
|
|
# hive-ci) needs no bundle and should not use this.
|
|
#
|
|
# imports = [ (caTrust.trustBundle { inherit pkgs; name = "swarm-nats";
|
|
# consumers = [ "swarm-nats-auth" ]; }) ];
|
|
#
|
|
# Three constraints, each earned:
|
|
# - `requires` on the CONSUMER: `before` orders but does not gate, so a
|
|
# failed assembly otherwise leaves it running and trusting *nothing*.
|
|
# - assemble to a temp path, verify, then move: `cat` of an empty bind
|
|
# exits 0, and a partial bundle must never appear under the final name.
|
|
# - `consumers` are BARE unit names — they are `systemd.services` keys
|
|
# (no suffix) *and* go in `before`/`requires` (suffixed). Reversed, the
|
|
# edge names a unit that does not exist and systemd orders nothing.
|
|
#
|
|
# Returns a module, not bare services: a caller already writing
|
|
# `systemd.services.<consumer>` cannot also write `systemd.services`.
|
|
#
|
|
# The two host-consumer flags are documented at their use sites below.
|
|
trustBundle =
|
|
{
|
|
name,
|
|
consumers,
|
|
pkgs,
|
|
# A HOST systemd service rather than something inside a container.
|
|
# `caContainerPath` is a bind mount that only exists in a container, so
|
|
# a host consumer reads the host copy — which means waiting for the
|
|
# unit that writes it. In a container that wait is the container's own
|
|
# (`containerOrdering`); here nothing carries it. One flag, because a
|
|
# host source without the ordering is a race.
|
|
hostUnit ? false,
|
|
# Whether the consumer exists at all. A container caller imports this
|
|
# into the CONTAINER's module set, so it vanishes with the container. A
|
|
# host caller imports it at the host's top level, where `imports` is
|
|
# unconditional — without this, a hive with the consumer off still gets
|
|
# a bundle oneshot *and* a `systemd.services.<consumer>` conjured by
|
|
# `genAttrs`, holding an `SSL_CERT_FILE` and no `ExecStart`.
|
|
enable ? true,
|
|
}:
|
|
let
|
|
dir = bundleDirFor name;
|
|
bundlePath = bundlePathFor name;
|
|
unit = "${name}-ca-bundle";
|
|
source = if hostUnit then caHostPath else caContainerPath;
|
|
in
|
|
{
|
|
_file = "hive-ca-trust.nix#trustBundle:${name}";
|
|
config.systemd.services = lib.optionalAttrs (useSelfSigned && enable) (
|
|
{
|
|
${unit} = {
|
|
description = "assemble ${name} TLS trust bundle (system CAs + hive CA)";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = map (c: "${c}.service") consumers;
|
|
# Only for a host consumer: the CA file is written at runtime by
|
|
# `hive-tls-ca.service`, and reading it directly means waiting
|
|
# for it. A container consumer reads a bind mount instead, and
|
|
# its `container@` unit carries the equivalent wait.
|
|
after = lib.optionals hostUnit [ "hive-tls-ca.service" ];
|
|
requires = lib.optionals hostUnit [ "hive-tls-ca.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
SyslogIdentifier = unit;
|
|
};
|
|
path = [
|
|
pkgs.coreutils
|
|
pkgs.gnugrep
|
|
];
|
|
script = ''
|
|
set -euo pipefail
|
|
install -d -m 0755 ${dir}
|
|
tmp=${bundlePath}.tmp
|
|
cat /etc/ssl/certs/ca-certificates.crt ${source} > "$tmp"
|
|
# `cat` of an empty or missing-but-mounted source exits 0, so the
|
|
# result has to be inspected rather than the command trusted.
|
|
if ! grep -q 'BEGIN CERTIFICATE' "$tmp"; then
|
|
echo "${unit}: assembled bundle contains no certificate" >&2
|
|
exit 1
|
|
fi
|
|
chmod 0644 "$tmp"
|
|
mv "$tmp" ${bundlePath}
|
|
'';
|
|
};
|
|
}
|
|
// lib.genAttrs consumers (_: {
|
|
requires = [ "${unit}.service" ];
|
|
after = [ "${unit}.service" ];
|
|
environment.SSL_CERT_FILE = bundlePath;
|
|
})
|
|
);
|
|
};
|
|
}
|