From 7f9e65e92366bf198ac345dbcd79890cefc8c428 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 9 Sep 2026 16:32:29 +0200 Subject: [PATCH] swarm-controller: hand the daemon the authority hives are issued from Creating a hive's cert-auth role means writing the authority into the role by value -- the store matches a presented certificate against the role's own copy -- and nothing gave this daemon that file. Named separately from deploy.bao.clientCaFile rather than read off it: that option is the store's, saying which readers the store trusts on the host that runs it, while a controller runs anywhere. The glue module supplies it where the two are co-located, which is the same split baoClientCertFile already makes against the hive reader's leaf. Gated on the identity as well as the CA. Without a leaf there is nothing to write a role with, so the file would reach a daemon that cannot act on it. The module-eval arm needed a fixture of its own: a deployment that self-signs both ends points clientCaFile and serverCaFile at one file, so on the existing fixture the two authorities are the same string and wiring either into the other's slot passes. controllerTwoCas is where they differ. --- .../glue-controller-bao-identity.nix | 6 +++ nix/host-modules/swarm-controller.nix | 40 +++++++++++++++- nix/module-eval.nix | 48 +++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/nix/host-modules/glue-controller-bao-identity.nix b/nix/host-modules/glue-controller-bao-identity.nix index 049c3746..555102f3 100644 --- a/nix/host-modules/glue-controller-bao-identity.nix +++ b/nix/host-modules/glue-controller-bao-identity.nix @@ -36,6 +36,12 @@ in services.hyperhive.deploy.swarm-controller = { baoClientCertFile = lib.mkDefault "${pkiDir}/controller.pem"; baoClientKeyFile = lib.mkDefault "${pkiDir}/controller-key.pem"; + } + // lib.optionalAttrs (baoDeploy.clientCaFile != null) { + # The authority the store already trusts hives by, handed to the daemon + # that has to write it into each hive's role. One file, two readers — + # a second copy would authenticate hives the store does not. + hiveClientCaFile = lib.mkDefault baoDeploy.clientCaFile; }; }; } diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 509cb7fd..e2168bcf 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -32,6 +32,11 @@ let deployCfg.swarm-controller.baoClientCertFile != null && deployCfg.swarm-controller.baoClientKeyFile != null; + # Creating a hive's role means writing to the store, so the CA alone is not + # enough — without an identity there is nothing to write it with, and the + # file would be handed to a daemon that cannot use it. + haveHiveClientCa = haveBaoIdentity && deployCfg.swarm-controller.hiveClientCaFile != null; + # `swarm_secret_client` reads these spellings explicitly rather than # vaultrs's `VAULT_*` defaults — falling through to those builds a client # with no identity and fails at the TLS handshake, naming neither. `%d` and @@ -47,6 +52,13 @@ let # the self-signed one ./glue-bao-tls.nix mints, which is why that file # names this path rather than leaving it to a default. BAO_CACERT = "%d/bao-ca.pem"; + } + // lib.optionalAttrs haveHiveClientCa { + # Not a `BAO_` name: that family is `swarm_secret_client`'s, and + # `BAO_CACERT` above is the *server* CA. This is the authority the + # store should trust hives by, which the controller reads as a value + # to put in each hive's cert-auth role. + SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE = "%d/hive-client-ca.pem"; }; # What `swarmctl` needs in order to act on authelia from the host. @@ -529,6 +541,31 @@ in ''; }; + hiveClientCaFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-ca/root.pem"; + description = '' + Authority whose leaves are hive client certificates, so that this + daemon can create each hive's cert-auth role. The store matches a + presented certificate against the role's copy of this, which is why the + role carries the authority itself rather than a path to it. + + The same material as {option}`services.hyperhive.deploy.bao.clientCaFile`, + named separately because that option is the **store's**: it says which + readers the store trusts, on the host that runs the store. A controller + runs anywhere, so it names its own copy, and a glue module supplies this + as a `mkDefault` where the two are co-located. + + Public material — a certificate authority, not a key — so unlike every + other credential here it leaks nothing. It stays a path anyway, because + the file it names is the one the store already installs. + + `null` leaves hive roles uncreated, which is the state a swarm is in + before anyone has onboarded a hive. + ''; + }; + queue = { clientSecretFile = lib.mkOption { type = lib.types.str; @@ -686,7 +723,8 @@ in ] ++ lib.optional ( haveBaoIdentity && deployCfg.bao.serverCaFile != null - ) "bao-ca.pem:${deployCfg.bao.serverCaFile}"; + ) "bao-ca.pem:${deployCfg.bao.serverCaFile}" + ++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}"; # The placeholder default that makes the above non-fatal. # `LoadCredential=` takes priority over `SetCredential=`, so this is diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 5b19eb96..a229fe44 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -268,6 +268,20 @@ let # files this host will never have. controllerNoStore = hive { deploy.swarm-controller.enable = true; }; + # The two authorities told apart. A deployment that self-signs both ends + # points `clientCaFile` and `serverCaFile` at one file, so on the fixture + # above the CA a hive is issued from and the CA the store is verified by are + # the same string — and a case wiring either into the other's slot passes. + # This is the deployment where they differ, which is what makes the arm + # below able to fail at all. + controllerTwoCas = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + deploy.swarm-controller.enable = true; + deploy.bao.clientCaFile = lib.mkForce "/etc/pki/hive-clients-ca.pem"; + deploy.bao.serverCaFile = lib.mkForce "/etc/pki/store-server-ca.pem"; + }; + # The store and the token, with no CA to trust. `mkForce` because the PKI # glue supplies one by default here — this is the deployment that brings its # own certificates and has not named the authority yet, and it separates @@ -621,6 +635,40 @@ let && builtins.elem "bao-client.pem:/var/lib/swarm-bao-pki/controller.pem" s.swarm-controller.serviceConfig.LoadCredential && builtins.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/controller-key.pem" s.swarm-controller.serviceConfig.LoadCredential; } + { + # A hive's cert-auth role carries the authority by value, so the daemon + # has to be handed the file rather than a path into the store's own + # directory it cannot read. + # + # ⚠️ The LoadCredential source is asserted, not just the `%d` name, for + # the reason the arm above gives — and here the wrong file is a + # *plausible* one: `deploy.bao.serverCaFile` is the CA a reader checks + # the store's certificate with, evaluates fine in this slot, and would + # make every hive role trust the wrong authority. + name = "the controller is handed the CA hives are issued from"; + ok = + let + s = controllerTwoCas.systemd.services; + m = controllerTwoCas.services.hyperhive; + in + (s.swarm-controller.environment.SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE or null) + == "%d/hive-client-ca.pem" + && builtins.elem "hive-client-ca.pem:/etc/pki/hive-clients-ca.pem" s.swarm-controller.serviceConfig.LoadCredential + && !builtins.elem "hive-client-ca.pem:/etc/pki/store-server-ca.pem" s.swarm-controller.serviceConfig.LoadCredential + && m.deploy.swarm-controller.hiveClientCaFile == m.deploy.bao.clientCaFile; + } + { + # Absence arm for the one above: without a store identity there is + # nothing to write a role with, so handing over the authority would be + # giving a file to a daemon that cannot act on it. + name = "a controller with no store leaf is given no hive CA either"; + ok = + let + s = controllerNoStore.systemd.services; + in + !(s.swarm-controller.environment ? SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE) + && !(lib.any (c: lib.hasPrefix "hive-client-ca" c) s.swarm-controller.serviceConfig.LoadCredential); + } { # Absence arm for the one above, and what makes it mean anything: a # controller with no leaf gets no store environment at all rather than