fix(#3391): give the swarm controller hive-CA trust
The controller's forge client speaks TLS to `https://<forge domain>`, which the gateway serves with a leaf signed by the hive CA. That CA is generated at runtime, so nothing build-time can name it and it is not in the system store -- and `reqwest`/`rustls` resolves roots through `rustls-native-certs`, whose `SSL_CERT_FILE` *replaces* the store rather than adding to it. With no bundle wired, every forge call failed `invalid peer certificate: UnknownIssuer` and agent creation died at its first step. `lib/hive-ca-trust.nix` already solved this, but only for containers: it sources the CA through `/run/hive-ca/trust-bundle.pem`, a bind mount that does not exist on the host. `hostUnit` makes it read the host copy and wait on `hive-tls-ca.service` itself -- one flag driving both, because a host source without that ordering is a race. `enable` is the other half, and it is the sharp edge: a container caller imports this into the container's module set, so it disappears with the container. A host caller imports it at the host's top level, where `imports` is unconditional -- without the flag, a hive with the controller turned off would get a bundle oneshot and a `swarm-controller` service conjured by `genAttrs`, holding an `SSL_CERT_FILE` and no `ExecStart`.
This commit is contained in:
parent
8b83eca0c1
commit
d14963ad2b
2 changed files with 55 additions and 2 deletions
|
|
@ -83,25 +83,48 @@ in
|
||||||
#
|
#
|
||||||
# Returns a module, not bare services: a caller already writing
|
# Returns a module, not bare services: a caller already writing
|
||||||
# `systemd.services.<consumer>` cannot also write `systemd.services`.
|
# `systemd.services.<consumer>` cannot also write `systemd.services`.
|
||||||
|
#
|
||||||
|
# The two host-consumer flags are documented at their use sites below.
|
||||||
trustBundle =
|
trustBundle =
|
||||||
{
|
{
|
||||||
name,
|
name,
|
||||||
consumers,
|
consumers,
|
||||||
pkgs,
|
pkgs,
|
||||||
|
# A HOST systemd service rather than something inside a container.
|
||||||
|
# `caContainerPath` is a bind mount that only exists in a container, so
|
||||||
|
# a host consumer reads the host copy — which means waiting for the
|
||||||
|
# unit that writes it. In a container that wait is the container's own
|
||||||
|
# (`containerOrdering`); here nothing carries it. One flag, because a
|
||||||
|
# host source without the ordering is a race.
|
||||||
|
hostUnit ? false,
|
||||||
|
# Whether the consumer exists at all. A container caller imports this
|
||||||
|
# into the CONTAINER's module set, so it vanishes with the container. A
|
||||||
|
# host caller imports it at the host's top level, where `imports` is
|
||||||
|
# unconditional — without this, a hive with the consumer off still gets
|
||||||
|
# a bundle oneshot *and* a `systemd.services.<consumer>` conjured by
|
||||||
|
# `genAttrs`, holding an `SSL_CERT_FILE` and no `ExecStart`.
|
||||||
|
enable ? true,
|
||||||
}:
|
}:
|
||||||
let
|
let
|
||||||
dir = "/run/${name}-ca";
|
dir = "/run/${name}-ca";
|
||||||
bundlePath = "${dir}/trust-bundle.pem";
|
bundlePath = "${dir}/trust-bundle.pem";
|
||||||
unit = "${name}-ca-bundle";
|
unit = "${name}-ca-bundle";
|
||||||
|
source = if hostUnit then caHostPath else caContainerPath;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
_file = "hive-ca-trust.nix#trustBundle:${name}";
|
_file = "hive-ca-trust.nix#trustBundle:${name}";
|
||||||
config.systemd.services = lib.optionalAttrs useSelfSigned (
|
config.systemd.services = lib.optionalAttrs (useSelfSigned && enable) (
|
||||||
{
|
{
|
||||||
${unit} = {
|
${unit} = {
|
||||||
description = "assemble ${name} TLS trust bundle (system CAs + hive CA)";
|
description = "assemble ${name} TLS trust bundle (system CAs + hive CA)";
|
||||||
wantedBy = [ "multi-user.target" ];
|
wantedBy = [ "multi-user.target" ];
|
||||||
before = map (c: "${c}.service") consumers;
|
before = map (c: "${c}.service") consumers;
|
||||||
|
# Only for a host consumer: the CA file is written at runtime by
|
||||||
|
# `hive-tls-ca.service`, and reading it directly means waiting
|
||||||
|
# for it. A container consumer reads a bind mount instead, and
|
||||||
|
# its `container@` unit carries the equivalent wait.
|
||||||
|
after = lib.optionals hostUnit [ "hive-tls-ca.service" ];
|
||||||
|
requires = lib.optionals hostUnit [ "hive-tls-ca.service" ];
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
Type = "oneshot";
|
Type = "oneshot";
|
||||||
RemainAfterExit = true;
|
RemainAfterExit = true;
|
||||||
|
|
@ -115,7 +138,7 @@ in
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
install -d -m 0755 ${dir}
|
install -d -m 0755 ${dir}
|
||||||
tmp=${bundlePath}.tmp
|
tmp=${bundlePath}.tmp
|
||||||
cat /etc/ssl/certs/ca-certificates.crt ${caContainerPath} > "$tmp"
|
cat /etc/ssl/certs/ca-certificates.crt ${source} > "$tmp"
|
||||||
# `cat` of an empty or missing-but-mounted source exits 0, so the
|
# `cat` of an empty or missing-but-mounted source exits 0, so the
|
||||||
# result has to be inspected rather than the command trusted.
|
# result has to be inspected rather than the command trusted.
|
||||||
if ! grep -q 'BEGIN CERTIFICATE' "$tmp"; then
|
if ! grep -q 'BEGIN CERTIFICATE' "$tmp"; then
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,26 @@ let
|
||||||
forgeCfg = config.services.hyperhive.swarm.forge;
|
forgeCfg = config.services.hyperhive.swarm.forge;
|
||||||
uiCfg = config.services.hyperhive.swarm.ui;
|
uiCfg = config.services.hyperhive.swarm.ui;
|
||||||
|
|
||||||
|
# The controller's forge client speaks TLS to `https://${forgeCfg.domain}`,
|
||||||
|
# which the gateway serves with a leaf signed by the hive CA — a CA
|
||||||
|
# generated at runtime, so nothing build-time can name it and it is not in
|
||||||
|
# the system store. `reqwest`/`rustls` resolves roots through
|
||||||
|
# `rustls-native-certs`, whose `SSL_CERT_FILE` *replaces* that store rather
|
||||||
|
# than adding to it, so without the assembled bundle every forge call fails
|
||||||
|
# `invalid peer certificate: UnknownIssuer` and agent creation dies at its
|
||||||
|
# first step.
|
||||||
|
#
|
||||||
|
# `hostUnit`: this consumer is a host service, not a container, so it reads
|
||||||
|
# the CA from the host path and the oneshot waits on `hive-tls-ca.service`
|
||||||
|
# itself. `enable`: `imports` is unconditional at the host's top level, so
|
||||||
|
# without it a hive with the controller off would get a bundle unit and a
|
||||||
|
# phantom `swarm-controller` service holding an `SSL_CERT_FILE`.
|
||||||
|
caTrust = import ./lib/hive-ca-trust.nix {
|
||||||
|
inherit lib;
|
||||||
|
tlsCfg = config.services.hyperhive.tls;
|
||||||
|
gatewayCfg = config.services.hyperhive.gateway;
|
||||||
|
};
|
||||||
|
|
||||||
# The controller's own OAuth2 client. It is NOT a hive: the per-hive
|
# The controller's own OAuth2 client. It is NOT a hive: the per-hive
|
||||||
# clients the roster issues belong to hives, and the responder's client
|
# clients the roster issues belong to hives, and the responder's client
|
||||||
# belongs to the responder. One identity per principal — the rule is that
|
# belongs to the responder. One identity per principal — the rule is that
|
||||||
|
|
@ -139,6 +159,16 @@ let
|
||||||
};
|
};
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
|
imports = [
|
||||||
|
(caTrust.trustBundle {
|
||||||
|
inherit pkgs;
|
||||||
|
name = "swarm-controller";
|
||||||
|
consumers = [ "swarm-controller" ];
|
||||||
|
hostUnit = true;
|
||||||
|
enable = cfg.enable;
|
||||||
|
})
|
||||||
|
];
|
||||||
|
|
||||||
options.services.hyperhive.swarm.controller = {
|
options.services.hyperhive.swarm.controller = {
|
||||||
queueClientId = lib.mkOption {
|
queueClientId = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue