dc418a5223
The single module-eval derivation forced ~62 full nixosSystem fixtures live at once to compute its cases list: 10.6GB peak RSS / 5m25s to evaluate, by far the dominant cost in nix flake check. Splits it into 21 independent checks.module-eval-* derivations (1-7 fixtures each) sharing builders/helpers via module-eval/lib.nix, so no single derivation needs more than a handful of fixtures live at once. A few cases spanning two clusters carry a small duplicated fixture rather than threading shared state through lib.nix.
216 lines
9.8 KiB
Nix
216 lines
9.8 KiB
Nix
# `checks.module-eval-bao-controller` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
# Store and controller on one machine, with a CN no default could supply.
|
|
# The odd value is what lets the case below tell "both ends read the same
|
|
# option" from "both ends happen to say swarm-controller".
|
|
baoControllerHere = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
|
|
deploy.bao.controllerCommonName = "cn-marker-not-a-default";
|
|
deploy.swarm-controller.enable = true;
|
|
};
|
|
|
|
# The controller with no store, which is every spread deployment. Nothing
|
|
# mints here, so the pairing must leave the paths unset rather than name
|
|
# files this host will never have.
|
|
controllerNoStore = hive { deploy.swarm-controller.enable = true; };
|
|
|
|
# The two authorities told apart. A deployment that self-signs both ends
|
|
# points `clientCaFile` and `serverCaFile` at one file, so on the fixture
|
|
# above the CA a hive is issued from and the CA the store is verified by are
|
|
# the same string — and a case wiring either into the other's slot passes.
|
|
# This is the deployment where they differ, which is what makes the arm
|
|
# below able to fail at all.
|
|
controllerTwoCas = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
|
|
deploy.swarm-controller.enable = true;
|
|
deploy.bao.clientCaFile = lib.mkForce "/etc/pki/hive-clients-ca.pem";
|
|
deploy.bao.serverCaFile = lib.mkForce "/etc/pki/store-server-ca.pem";
|
|
};
|
|
|
|
# The host's `bao` wrapper, pulled apart once so each case below names one
|
|
# property instead of a conjunction — a failing conjunction says only that
|
|
# something is wrong.
|
|
baoHostPackages = controllerTwoCas.environment.systemPackages;
|
|
|
|
baoWrapper = lib.findFirst (p: (p.name or "") == "bao-hive") null baoHostPackages;
|
|
|
|
baoWrapperCmd = if baoWrapper == null then "" else (baoWrapper.buildCommand or "");
|
|
cases = [
|
|
{
|
|
# Nothing asserted the PKI script before this, so a third leaf could be
|
|
# added to it and every case still passed — measured, not assumed: the
|
|
# commit that added one left `module-eval`'s derivation unchanged.
|
|
name = "the store mints a leaf for the controller, and the controller is pointed at it";
|
|
ok =
|
|
let
|
|
m = baoControllerHere;
|
|
pki = m.systemd.services.swarm-bao-pki.script;
|
|
in
|
|
lib.hasInfix "controller.pem" pki
|
|
&&
|
|
m.services.hyperhive.deploy.swarm-controller.baoClientCertFile
|
|
== "/var/lib/swarm-bao-pki/controller.pem"
|
|
&&
|
|
m.services.hyperhive.deploy.swarm-controller.baoClientKeyFile
|
|
== "/var/lib/swarm-bao-pki/controller-key.pem";
|
|
}
|
|
{
|
|
# What makes the one above mean something: a controller with no store
|
|
# has nothing to be pointed at. Naming a path here would be a file this
|
|
# host never gets, which fails at a TLS handshake rather than at eval.
|
|
name = "a controller on a host with no store is left without certificate paths";
|
|
ok =
|
|
let
|
|
c = controllerNoStore.services.hyperhive.deploy.swarm-controller;
|
|
in
|
|
c.baoClientCertFile == null && c.baoClientKeyFile == null;
|
|
}
|
|
{
|
|
# Being *pointed at* a leaf and *being handed* one are different claims,
|
|
# and the options above were the first without the second — declared,
|
|
# defaulted, and read by nothing. This is the arm that makes them reach
|
|
# the process.
|
|
#
|
|
# ⚠️ The LoadCredential source is asserted, not just the `%d` name: the
|
|
# controller's leaf and the hive reader's are two identities with two
|
|
# policies, and wiring `deploy.bao.clientCertFile` here would satisfy
|
|
# every `%d`-only check while giving the daemon a policy that cannot
|
|
# write an agent's credential.
|
|
name = "the controller is handed its own store leaf, not the hive reader's";
|
|
ok =
|
|
let
|
|
s = baoControllerHere.systemd.services;
|
|
in
|
|
s ? swarm-controller
|
|
&& (s.swarm-controller.environment ? BAO_ADDR)
|
|
&& (s.swarm-controller.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem"
|
|
&& (s.swarm-controller.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem"
|
|
&& builtins.elem "bao-client.pem:/var/lib/swarm-bao-pki/controller.pem" s.swarm-controller.serviceConfig.LoadCredential
|
|
&& builtins.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/controller-key.pem" s.swarm-controller.serviceConfig.LoadCredential;
|
|
}
|
|
{
|
|
# A hive's cert-auth role carries the authority by value, so the daemon
|
|
# has to be handed the file rather than a path into the store's own
|
|
# directory it cannot read.
|
|
#
|
|
# ⚠️ The LoadCredential source is asserted, not just the `%d` name, for
|
|
# the reason the arm above gives — and here the wrong file is a
|
|
# *plausible* one: `deploy.bao.serverCaFile` is the CA a reader checks
|
|
# the store's certificate with, evaluates fine in this slot, and would
|
|
# make every hive role trust the wrong authority.
|
|
name = "the controller is handed the CA hives are issued from";
|
|
ok =
|
|
let
|
|
s = controllerTwoCas.systemd.services;
|
|
m = controllerTwoCas.services.hyperhive;
|
|
in
|
|
(s.swarm-controller.environment.SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE or null)
|
|
== "%d/hive-client-ca.pem"
|
|
&& builtins.elem "hive-client-ca.pem:/etc/pki/hive-clients-ca.pem" s.swarm-controller.serviceConfig.LoadCredential
|
|
&& !builtins.elem "hive-client-ca.pem:/etc/pki/store-server-ca.pem" s.swarm-controller.serviceConfig.LoadCredential
|
|
&& m.deploy.swarm-controller.hiveClientCaFile == m.deploy.bao.clientCaFile;
|
|
}
|
|
{
|
|
# Same two-CA fixture, for the same reason: the wrapper verifies the
|
|
# STORE, so it takes `serverCaFile`. On a self-signing deployment both
|
|
# options name one file and either would pass; here the client CA in that
|
|
# slot is a case this arm fails.
|
|
#
|
|
# ⚠️ The package itself stays off `PATH` — `wrapProgram` renames the real
|
|
# binary, so an unwrapped `bao` is unreachable rather than merely
|
|
# discouraged. Operator's instruction, and the last assertion is what
|
|
# keeps a later "install the package too" from quietly undoing it.
|
|
name = "the host gets a wrapped bao CLI";
|
|
ok = baoWrapper != null;
|
|
}
|
|
{
|
|
name = "the wrapped bao CLI carries this store's address";
|
|
ok = lib.hasInfix "--set-default BAO_ADDR" baoWrapperCmd;
|
|
}
|
|
{
|
|
# `serverCaFile` and not `clientCaFile`: the wrapper verifies the STORE.
|
|
# On a self-signing deployment both options name one file and either
|
|
# would pass, which is why this uses the two-CA fixture.
|
|
#
|
|
# ⚠️ The flag and its VALUE together, escaped the same way the module
|
|
# escapes it: `BAO_CACERT` present and `store-server-ca.pem` present
|
|
# somewhere are two facts that do not add up to "the CA is set to that
|
|
# file", and a weaker pair of `hasInfix`es passes on a wrapper that sets
|
|
# neither to the other.
|
|
name = "the wrapped bao CLI verifies the store with the server CA";
|
|
ok =
|
|
lib.hasInfix "--set-default BAO_CACERT ${lib.escapeShellArg "/etc/pki/store-server-ca.pem"}" baoWrapperCmd
|
|
&& !lib.hasInfix "hive-clients-ca.pem" baoWrapperCmd;
|
|
}
|
|
{
|
|
# `wrapProgram` renames the real binary, so an unwrapped `bao` is
|
|
# unreachable rather than merely discouraged — operator's instruction.
|
|
# This is what keeps a later "install the package too" from undoing it.
|
|
name = "the unwrapped bao package stays off the host PATH";
|
|
ok = !builtins.elem controllerTwoCas.services.hyperhive.deploy.bao.package baoHostPackages;
|
|
}
|
|
{
|
|
# Absence arm for the one above: without a store identity there is
|
|
# nothing to write a role with, so handing over the authority would be
|
|
# giving a file to a daemon that cannot act on it.
|
|
name = "a controller with no store leaf is given no hive CA either";
|
|
ok =
|
|
let
|
|
s = controllerNoStore.systemd.services;
|
|
in
|
|
!(s.swarm-controller.environment ? SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE)
|
|
&& !(lib.any (c: lib.hasPrefix "hive-client-ca" c) s.swarm-controller.serviceConfig.LoadCredential);
|
|
}
|
|
{
|
|
# Absence arm for the one above, and what makes it mean anything: a
|
|
# controller with no leaf gets no store environment at all rather than
|
|
# variables naming files this host never receives.
|
|
name = "a controller with no store leaf is given no store environment";
|
|
ok =
|
|
let
|
|
s = controllerNoStore.systemd.services;
|
|
in
|
|
s ? swarm-controller
|
|
&& !(s.swarm-controller.environment ? BAO_ADDR)
|
|
&& !(lib.any (c: lib.hasPrefix "bao-" c) s.swarm-controller.serviceConfig.LoadCredential);
|
|
}
|
|
{
|
|
# The CN is an interface between two files: the store writes a role that
|
|
# matches it, the PKI mints a leaf that carries it. They read one option,
|
|
# and this is what says so — the fixture's value cannot come from a
|
|
# default, so matching it in both places is not a coincidence.
|
|
name = "the cert-auth role and the minted leaf take their subject from one option";
|
|
ok =
|
|
let
|
|
m = baoControllerHere;
|
|
role = m.systemd.services.swarm-bao-controller-policy.script;
|
|
pki = m.systemd.services.swarm-bao-pki.script;
|
|
in
|
|
lib.hasInfix "cn-marker-not-a-default" role && lib.hasInfix "cn-marker-not-a-default" pki;
|
|
}
|
|
];
|
|
in
|
|
runGroup "bao-controller" cases
|