7d24feaf25
`swarmctl login` (and any `--swarm` verb) on a controller host failed TLS with `UnknownIssuer` against the swarm's own gateway: its leaf chains to the store's `pki` root, which no system store holds, and the only way through was `--ca-file`/`SWARMCTL_CA_FILE` by hand. The host's swarmctl wrapper now sets `SSL_CERT_FILE` to the bundle the swarm-controller daemon already trusts (`/run/swarm-controller-ca/ trust-bundle.pem`: system CAs plus the swarm's root), when the gateway is self-signed. swarmctl's reqwest/rustls resolves roots the same way the daemon's does, so this is the path the daemon uses to reach the same gateway. `--set` rather than a default, since a nix shell exports its own `SSL_CERT_FILE`. Pointing `SWARMCTL_CA_FILE` at the bundle instead would not work (#5243): `--ca-file` adds each certificate as a root and fails the whole client on one it can't parse. Hosts without the controller (an operator laptop) still need `--ca-file`. Refs #5239 Fixes #5243
387 lines
17 KiB
Nix
387 lines
17 KiB
Nix
# `checks.module-eval-bao-controller` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
# Store and controller on one machine, with a CN no default could supply.
|
|
# The odd value is what lets the case below tell "both ends read the same
|
|
# option" from "both ends happen to say swarm-controller".
|
|
baoControllerHere = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
|
|
deploy.bao.controllerCommonName = "cn-marker-not-a-default";
|
|
deploy.swarm-controller.enable = true;
|
|
};
|
|
|
|
# Store and controller with an agent PKI mount and role no default could
|
|
# supply.
|
|
controllerAgentPkiMarker = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.agentPkiMountPath = "pki-agents-marker";
|
|
deploy.bao.agentPkiRoleName = "swarm-agent-marker";
|
|
deploy.swarm-controller.enable = true;
|
|
};
|
|
|
|
# The controller with no store, which is every spread deployment. Nothing
|
|
# mints here, so the pairing must leave the paths unset rather than name
|
|
# files this host will never have.
|
|
controllerNoStore = shared.controllerAlone;
|
|
|
|
# A deployment that pinned `gatewayHost` away from its own swarm-domain
|
|
# default (`hive-matrix-service.nix`'s own doc: "To use the per-hive name
|
|
# ... instead, pin it here").
|
|
# Only this fixture can tell "the controller reads gatewayHost" apart from
|
|
# "the controller re-derives gatewayHost's default itself" — both render
|
|
# the same URL on every other fixture in this file.
|
|
controllerPinnedGateway = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
swarm.matrix.gatewayHost = "matrix.hive.t.local";
|
|
};
|
|
|
|
# No matrix vhost for this swarm at all: tuwunel stays direct on
|
|
# `httpPort`, so there is no homeserver URL to hand the controller.
|
|
controllerNoGatewayHost = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
swarm.matrix.gatewayHost = null;
|
|
};
|
|
|
|
# An ignore list no default could supply, with two entries so the case below
|
|
# also pins the separator the daemon splits on.
|
|
controllerIgnoresAdmins = hive {
|
|
deploy.swarm-controller.enable = true;
|
|
swarm.matrix.adminSync.ignoreUsers = [
|
|
"@keep-a:t.local"
|
|
"@keep-b:t.local"
|
|
];
|
|
};
|
|
|
|
# The two authorities told apart. A deployment that self-signs both ends
|
|
# points `clientCaFile` and `serverCaFile` at one file, so on the fixture
|
|
# above the CA a hive is issued from and the CA the store is verified by are
|
|
# the same string — and a case wiring either into the other's slot passes.
|
|
# This is the deployment where they differ, which is what makes the arm
|
|
# below able to fail at all.
|
|
controllerTwoCas = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.bao.clientCaFile = lib.mkForce "/etc/pki/hive-clients-ca.pem";
|
|
deploy.bao.serverCaFile = lib.mkForce "/etc/pki/store-server-ca.pem";
|
|
};
|
|
|
|
# The host's `bao` wrapper, pulled apart once so each case below names one
|
|
# property instead of a conjunction — a failing conjunction says only that
|
|
# something is wrong.
|
|
baoHostPackages = controllerTwoCas.environment.systemPackages;
|
|
|
|
baoWrapper = lib.findFirst (p: (p.name or "") == "bao-hive") null baoHostPackages;
|
|
|
|
baoWrapperCmd = if baoWrapper == null then "" else (baoWrapper.buildCommand or "");
|
|
|
|
swarmctlWrapper = lib.findFirst (
|
|
p: (p.name or "") == "swarmctl-configured"
|
|
) null controllerNoStore.environment.systemPackages;
|
|
|
|
swarmctlWrapperCmd = if swarmctlWrapper == null then "" else (swarmctlWrapper.buildCommand or "");
|
|
|
|
refusedWithoutIdentity =
|
|
m:
|
|
lib.any (
|
|
a: !a.assertion && lib.hasInfix "swarm-controller.baoClientCertFile and" a.message
|
|
) m.assertions;
|
|
cases = [
|
|
{
|
|
# The queue client secret lives only in the store, so a controller that
|
|
# cannot log in there cannot reach the queue. Refused at eval rather than
|
|
# left to start and never connect. The control is the co-located
|
|
# controller, which is handed the minted leaf and passes.
|
|
name = "a controller with no store identity is refused at eval, one given the minted leaf is not";
|
|
ok = refusedWithoutIdentity controllerNoStore && !(refusedWithoutIdentity baoControllerHere);
|
|
}
|
|
{
|
|
# No hive mints an agent's matrix account, so a controller that
|
|
# did not know the homeserver would create every agent without one. The
|
|
# default is the swarm's own `chat.` vhost, with no operator setting.
|
|
name = "the controller is told the swarm's homeserver by default";
|
|
ok =
|
|
controllerNoStore.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_MATRIX_HOMESERVER_URL
|
|
or null == "https://chat.t.local";
|
|
}
|
|
{
|
|
# The case above alone can't tell "reads gatewayHost" from "re-derives
|
|
# gatewayHost's own default": both land on chat.t.local when nothing is
|
|
# pinned. This fixture pins gatewayHost away from that default, so only
|
|
# the former passes.
|
|
name = "the controller's homeserver follows a pinned gatewayHost, not its own chat.<domain> default";
|
|
ok =
|
|
controllerPinnedGateway.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_MATRIX_HOMESERVER_URL
|
|
or null == "https://matrix.hive.t.local";
|
|
}
|
|
{
|
|
# Login-only: `SWARMCTL_SWARM` would move every socket verb onto the
|
|
# gateway. The socket variable is the control that the wrapper text was
|
|
# found at all.
|
|
name = "the host's swarmctl logs in to its own swarm by default, and its other verbs stay on the socket";
|
|
ok =
|
|
lib.hasInfix "--set SWARM_CONTROLLER_SOCKET " swarmctlWrapperCmd
|
|
&& lib.hasInfix "--set-default SWARMCTL_LOGIN_DEFAULT_SWARM https://t.local" swarmctlWrapperCmd
|
|
&& !(lib.hasInfix "SWARMCTL_SWARM" swarmctlWrapperCmd);
|
|
}
|
|
{
|
|
# The gateway's leaf chains to the store's root, which no system store
|
|
# holds; without the bundle `login` fails `UnknownIssuer`.
|
|
name = "the host's swarmctl trusts the swarm's root through the daemon's bundle";
|
|
ok = lib.hasInfix "--set SSL_CERT_FILE /run/swarm-controller-ca/trust-bundle.pem" swarmctlWrapperCmd;
|
|
}
|
|
{
|
|
name = "a swarm with no matrix gateway vhost gives the controller no homeserver variable";
|
|
ok =
|
|
!(
|
|
controllerNoGatewayHost.systemd.services.swarm-controller.environment
|
|
? SWARM_CONTROLLER_MATRIX_HOMESERVER_URL
|
|
);
|
|
}
|
|
{
|
|
# operator_admin.rs splits this on whitespace; an entry missing here is
|
|
# an account the sync would revoke.
|
|
name = "the controller is told the operator admin sync's ignore list";
|
|
ok =
|
|
controllerIgnoresAdmins.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS
|
|
or null == "@keep-a:t.local @keep-b:t.local";
|
|
}
|
|
{
|
|
# Nothing asserted the PKI script before this, so a third leaf could be
|
|
# added to it and every case still passed — measured, not assumed: the
|
|
# commit that added one left `module-eval`'s derivation unchanged.
|
|
name = "the store mints a leaf for the controller, and the controller is pointed at it";
|
|
ok =
|
|
let
|
|
m = baoControllerHere;
|
|
pki = m.systemd.services.swarm-bao-pki.script;
|
|
in
|
|
lib.hasInfix "controller.pem" pki
|
|
&&
|
|
m.services.hyperhive.deploy.swarm-controller.baoClientCertFile
|
|
== "/var/lib/swarm-bao-pki/controller.pem"
|
|
&&
|
|
m.services.hyperhive.deploy.swarm-controller.baoClientKeyFile
|
|
== "/var/lib/swarm-bao-pki/controller-key.pem";
|
|
}
|
|
{
|
|
# What makes the one above mean something: a controller with no store
|
|
# has nothing to be pointed at. Naming a path here would be a file this
|
|
# host never gets, which fails at a TLS handshake rather than at eval.
|
|
name = "a controller on a host with no store is left without certificate paths";
|
|
ok =
|
|
let
|
|
c = controllerNoStore.services.hyperhive.deploy.swarm-controller;
|
|
in
|
|
c.baoClientCertFile == null && c.baoClientKeyFile == null;
|
|
}
|
|
{
|
|
# Being *pointed at* a leaf and *being handed* one are different claims,
|
|
# and the options above were the first without the second — declared,
|
|
# defaulted, and read by nothing. This is the arm that makes them reach
|
|
# the process.
|
|
#
|
|
# ⚠️ The LoadCredential source is asserted, not just the `%d` name: the
|
|
# controller's leaf and the hive reader's are two identities with two
|
|
# policies, and wiring `deploy.bao.clientCertFile` here would satisfy
|
|
# every `%d`-only check while giving the daemon a policy that cannot
|
|
# write an agent's credential.
|
|
name = "the controller is handed its own store leaf, not the hive reader's";
|
|
ok =
|
|
let
|
|
s = baoControllerHere.systemd.services;
|
|
in
|
|
s ? swarm-controller
|
|
&& (s.swarm-controller.environment ? BAO_ADDR)
|
|
&& (s.swarm-controller.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem"
|
|
&& (s.swarm-controller.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem"
|
|
&& builtins.elem "bao-client.pem:/var/lib/swarm-bao-pki/controller.pem" s.swarm-controller.serviceConfig.LoadCredential
|
|
&& builtins.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/controller-key.pem" s.swarm-controller.serviceConfig.LoadCredential;
|
|
}
|
|
{
|
|
# The forge token comes from the store, so the unit is handed no file for
|
|
# it, and neither fixture runs the forge: a controller reaches it by name
|
|
# wherever it runs.
|
|
name = "the controller is handed no forge token file, and is pointed at the forge away from it";
|
|
ok =
|
|
lib.all
|
|
(
|
|
m:
|
|
let
|
|
u = m.systemd.services.swarm-controller;
|
|
in
|
|
!(lib.any (c: lib.hasPrefix "forge-token:" c) u.serviceConfig.LoadCredential)
|
|
&& !(u.environment ? SWARM_CONTROLLER_FORGE_TOKEN_FILE)
|
|
&& u.environment.SWARM_CONTROLLER_FORGE_URL == "https://forge.t.local"
|
|
)
|
|
[
|
|
baoControllerHere
|
|
controllerNoStore
|
|
];
|
|
}
|
|
{
|
|
# A hive's cert-auth role carries the authority by value, so the daemon
|
|
# has to be handed the file rather than a path into the store's own
|
|
# directory it cannot read.
|
|
#
|
|
# ⚠️ The LoadCredential source is asserted, not just the `%d` name, for
|
|
# the reason the arm above gives — and here the wrong file is a
|
|
# *plausible* one: `deploy.bao.serverCaFile` is the CA a reader checks
|
|
# the store's certificate with, evaluates fine in this slot, and would
|
|
# make every hive role trust the wrong authority.
|
|
name = "the controller is handed the CA hives are issued from";
|
|
ok =
|
|
let
|
|
s = controllerTwoCas.systemd.services;
|
|
m = controllerTwoCas.services.hyperhive;
|
|
in
|
|
(s.swarm-controller.environment.SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE or null)
|
|
== "%d/hive-client-ca.pem"
|
|
&& builtins.elem "hive-client-ca.pem:/etc/pki/hive-clients-ca.pem" s.swarm-controller.serviceConfig.LoadCredential
|
|
&& !builtins.elem "hive-client-ca.pem:/etc/pki/store-server-ca.pem" s.swarm-controller.serviceConfig.LoadCredential
|
|
&& m.deploy.swarm-controller.hiveClientCaFile == m.deploy.bao.clientCaFile;
|
|
}
|
|
{
|
|
# Same two-CA fixture, for the same reason: the wrapper verifies the
|
|
# STORE, so it takes `serverCaFile`. On a self-signing deployment both
|
|
# options name one file and either would pass; here the client CA in that
|
|
# slot is a case this arm fails.
|
|
#
|
|
# ⚠️ The package itself stays off `PATH` — `wrapProgram` renames the real
|
|
# binary, so an unwrapped `bao` is unreachable rather than merely
|
|
# discouraged. Operator's instruction, and the last assertion is what
|
|
# keeps a later "install the package too" from quietly undoing it.
|
|
name = "the host gets a wrapped bao CLI";
|
|
ok = baoWrapper != null;
|
|
}
|
|
{
|
|
name = "the wrapped bao CLI carries this store's address";
|
|
ok = lib.hasInfix "--set-default BAO_ADDR" baoWrapperCmd;
|
|
}
|
|
{
|
|
# `serverCaFile` and not `clientCaFile`: the wrapper verifies the STORE.
|
|
# On a self-signing deployment both options name one file and either
|
|
# would pass, which is why this uses the two-CA fixture.
|
|
#
|
|
# ⚠️ The flag and its VALUE together, escaped the same way the module
|
|
# escapes it: `BAO_CACERT` present and `store-server-ca.pem` present
|
|
# somewhere are two facts that do not add up to "the CA is set to that
|
|
# file", and a weaker pair of `hasInfix`es passes on a wrapper that sets
|
|
# neither to the other.
|
|
name = "the wrapped bao CLI verifies the store with the server CA";
|
|
ok =
|
|
lib.hasInfix "--set-default BAO_CACERT ${lib.escapeShellArg "/etc/pki/store-server-ca.pem"}" baoWrapperCmd
|
|
&& !lib.hasInfix "hive-clients-ca.pem" baoWrapperCmd;
|
|
}
|
|
{
|
|
# `wrapProgram` renames the real binary, so an unwrapped `bao` is
|
|
# unreachable rather than merely discouraged — operator's instruction.
|
|
# This is what keeps a later "install the package too" from undoing it.
|
|
name = "the unwrapped bao package stays off the host PATH";
|
|
ok = !builtins.elem controllerTwoCas.services.hyperhive.deploy.bao.package baoHostPackages;
|
|
}
|
|
{
|
|
# Absence arm for the one above: without a store identity there is
|
|
# nothing to write a role with, so handing over the authority would be
|
|
# giving a file to a daemon that cannot act on it.
|
|
name = "a controller with no store leaf is given no hive CA either";
|
|
ok =
|
|
let
|
|
s = controllerNoStore.systemd.services;
|
|
in
|
|
!(s.swarm-controller.environment ? SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE)
|
|
&& !(lib.any (c: lib.hasPrefix "hive-client-ca" c) s.swarm-controller.serviceConfig.LoadCredential);
|
|
}
|
|
{
|
|
# Absence arm for the one above, and what makes it mean anything: a
|
|
# controller with no leaf gets no store environment at all rather than
|
|
# variables naming files this host never receives.
|
|
name = "a controller with no store leaf is given no store environment";
|
|
ok =
|
|
let
|
|
s = controllerNoStore.systemd.services;
|
|
in
|
|
s ? swarm-controller
|
|
&& !(s.swarm-controller.environment ? BAO_ADDR)
|
|
&& !(lib.any (c: lib.hasPrefix "bao-" c) s.swarm-controller.serviceConfig.LoadCredential);
|
|
}
|
|
{
|
|
# The CN is an interface between two files: the store writes a role that
|
|
# matches it, the PKI mints a leaf that carries it. They read one option,
|
|
# and this is what says so — the fixture's value cannot come from a
|
|
# default, so matching it in both places is not a coincidence.
|
|
name = "the cert-auth role and the minted leaf take their subject from one option";
|
|
ok =
|
|
let
|
|
m = baoControllerHere;
|
|
role = m.systemd.services.swarm-bao-controller-policy.script;
|
|
pki = m.systemd.services.swarm-bao-pki.script;
|
|
in
|
|
lib.hasInfix "cn-marker-not-a-default" role && lib.hasInfix "cn-marker-not-a-default" pki;
|
|
}
|
|
{
|
|
# Agent leaves come from the store's own agent PKI, so the controller
|
|
# holds no authority of its own: no minting unit, no key, no variable
|
|
# naming one.
|
|
name = "the controller holds no agent CA and is told the store's agent PKI mount and role";
|
|
ok =
|
|
let
|
|
s = baoControllerHere.systemd.services;
|
|
e = s.swarm-controller.environment;
|
|
in
|
|
s ? swarm-controller
|
|
&& !(s ? swarm-controller-agent-ca)
|
|
&& !(e ? SWARM_CONTROLLER_AGENT_CA_FILE)
|
|
&& !(e ? SWARM_CONTROLLER_AGENT_CA_KEY_FILE)
|
|
&& !(lib.any (c: lib.hasPrefix "agent-ca" c) s.swarm-controller.serviceConfig.LoadCredential)
|
|
&& (e.SWARM_CONTROLLER_AGENT_PKI_MOUNT or null) == "pki-agents"
|
|
&& (e.SWARM_CONTROLLER_AGENT_PKI_ROLE or null) == "swarm-agent";
|
|
}
|
|
{
|
|
# The controller issues through whatever mount and role it is told, and
|
|
# its grant names a path. Both read the store's options, which the
|
|
# marker values prove: no default could supply them.
|
|
name = "the controller's issue grant and its environment name one mount and role";
|
|
ok =
|
|
let
|
|
s = controllerAgentPkiMarker.systemd.services;
|
|
in
|
|
s.swarm-controller.environment.SWARM_CONTROLLER_AGENT_PKI_MOUNT == "pki-agents-marker"
|
|
&& s.swarm-controller.environment.SWARM_CONTROLLER_AGENT_PKI_ROLE == "swarm-agent-marker"
|
|
&& lib.hasInfix ''path "pki-agents-marker/issue/swarm-agent-marker"'' s.swarm-bao-controller-policy.script;
|
|
}
|
|
{
|
|
# Absence arm: without a store identity nothing can be issued, so the
|
|
# variables are not set.
|
|
name = "a controller with no store leaf is not told an agent PKI";
|
|
ok =
|
|
let
|
|
e = controllerNoStore.systemd.services.swarm-controller.environment;
|
|
in
|
|
!(e ? SWARM_CONTROLLER_AGENT_PKI_MOUNT) && !(e ? SWARM_CONTROLLER_AGENT_PKI_ROLE);
|
|
}
|
|
];
|
|
in
|
|
runGroup "bao-controller" cases
|