From eaa52ef20076553de9ef3ddcb46c8ad7baf211f9 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 11 Sep 2026 23:11:05 +0200 Subject: [PATCH] swarm: publish minted OIDC client secrets into the swarm store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hive that does not host authelia has no path to its own agent queue client secret. The mint writes the plaintext to a host directory whose other reader lives in a different container, so the host that mints is the only place both trees are addressable — which is where this unit runs. Four pieces, in the order they depend on each other: the leaf (glue-bao-tls.nix signs it, because the thing that owns a private key owns issuing from it), the module declaring its own cert/key options, the one-pairing glue file pointing them at that leaf, and the imports. The unit is gated on holding a client identity, never on deploy.bao.enable — that option is the co-location assumption itself, and the publisher is the case that assumption excludes. The secret is passed to bao as `value=@`, never as an argv element: bao is an external binary, so an argument is world-readable in /proc for the life of the call. Refs #3853 --- nix/host-modules/default.nix | 2 + nix/host-modules/glue-bao-tls.nix | 7 + .../glue-secret-publisher-bao-identity.nix | 42 +++++ nix/host-modules/swarm-secret-publisher.nix | 172 ++++++++++++++++++ 4 files changed, 223 insertions(+) create mode 100644 nix/host-modules/glue-secret-publisher-bao-identity.nix create mode 100644 nix/host-modules/swarm-secret-publisher.nix diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index fd80803f..da8d748d 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -26,9 +26,11 @@ ./glue-bao-tls.nix ./glue-controller-bao-identity.nix ./glue-matrix-bao-token.nix + ./glue-secret-publisher-bao-identity.nix ./swarm-authelia.nix ./swarm-bao.nix ./swarm-ca.nix + ./swarm-secret-publisher.nix ./swarm-nats.nix ./swarm-controller.nix ./swarm-grafana.nix diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index 18549577..419f9c63 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -140,6 +140,13 @@ in # this hive's name. [ -s ${pkiDir}/controller.pem ] || ${signLeaf} ${pkiDir} controller \ ${lib.escapeShellArg deployCfg.bao.controllerCommonName} "" clientAuth + + # The secret publisher's, minted here for the reason the controller's + # line above gives — and it serves that case more often, not less: the + # publisher runs beside AUTHELIA, which is the one host guaranteed not + # to be this one whenever the store has a host of its own. + [ -s ${pkiDir}/secret-publisher.pem ] || ${signLeaf} ${pkiDir} secret-publisher \ + ${lib.escapeShellArg deployCfg.bao.secretPublisherCommonName} "" clientAuth ''; }; }; diff --git a/nix/host-modules/glue-secret-publisher-bao-identity.nix b/nix/host-modules/glue-secret-publisher-bao-identity.nix new file mode 100644 index 00000000..88fc92b9 --- /dev/null +++ b/nix/host-modules/glue-secret-publisher-bao-identity.nix @@ -0,0 +1,42 @@ +# Glue: point the secret publisher at the bao leaf minted for it. +# +# ONE PAIRING PER FILE — publisher ← bao, and nothing else. Deleting this +# leaves a publisher that takes operator-provided certificate paths, which is +# what any deployment not minting its own already does. +# +# ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the +# leaf, because the thing that owns a private key owns issuing from it. What +# belongs here is the pairing: which paths this host's publisher reads. +# +# ⚠️ Gated on the leaf existing, not on the store being enabled — the same rule +# ./glue-controller-bao-identity.nix states. A publisher on the store's own +# host is one deployment; a publisher beside a remote authelia holding a leaf +# issued out of band is another, and both want this wiring. +# +# Everything is `mkDefault`. An operator naming their own paths wins. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + baoDeploy = deployCfg.bao; + + # Where ./glue-bao-tls.nix puts the leaves, derived from the reader's own + # path rather than repeating that file's directory literal: an operator who + # moves the PKI moves both, and the two cannot drift apart. + haveMintedPki = baoDeploy.clientCertFile != null; + pkiDir = if haveMintedPki then builtins.dirOf baoDeploy.clientCertFile else null; +in +{ + config = + lib.mkIf (hyperhiveCfg.enable && deployCfg.swarm-secret-publisher.enable && haveMintedPki) + { + services.hyperhive.deploy.swarm-secret-publisher = { + baoClientCertFile = lib.mkDefault "${pkiDir}/secret-publisher.pem"; + baoClientKeyFile = lib.mkDefault "${pkiDir}/secret-publisher-key.pem"; + }; + }; +} diff --git a/nix/host-modules/swarm-secret-publisher.nix b/nix/host-modules/swarm-secret-publisher.nix new file mode 100644 index 00000000..27b68adb --- /dev/null +++ b/nix/host-modules/swarm-secret-publisher.nix @@ -0,0 +1,172 @@ +# The unit that copies authelia's minted OIDC client secrets into the swarm's +# secret store, so a hive that does not host authelia can read its own. +# +# ⚠️ IT RUNS WHERE AUTHELIA DOES, and that is the whole reason it exists as a +# separate thing. `deploy.authelia.hostClientSecretDir`'s own description says +# why: the plaintext's other reader lives in a **different container**, and +# containers that share this host's network namespace still have separate +# filesystem roots, so "the host is the only place both trees are addressable". +# A delivery step therefore runs on the host that mints — not on the store's +# host, and not on the reading hive's. +# +# ⚠️ ITS OWN STORE IDENTITY, not swarm-controller's. That principal may rewrite +# every hive's policy and login role; a unit whose entire job is copying one +# file has no business holding it. ./swarm-bao.nix grants this one +# `create`/`update` under the hive prefix and nothing else. +# +# ⚠️ THE SECRET NEVER REACHES `argv`. `bao` is an external binary, so every +# argument is world-readable in /proc for the life of the call — the value is +# passed as `@` and read by bao itself. (`/knowledge/secret-hygiene.md`: +# "a path keeps the secret out of the store, and reading it into a shell +# variable puts it straight into argv".) +{ + pkgs, + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + baoDeploy = deployCfg.bao; + autheliaCfg = hyperhiveCfg.swarm.authelia; + cfg = deployCfg.swarm-secret-publisher; + + # A reader is defined by holding a certificate the store accepts, never by + # standing next to it — the rule ./glue-matrix-bao-token.nix states in full. + # `deploy.bao.enable` here would be the co-location assumption itself. + haveClientIdentity = cfg.baoClientCertFile != null && cfg.baoClientKeyFile != null; + + # Both halves have to be here: the mint (authelia, for the plaintext) and an + # identity (for the store). Neither implies the other. + active = hyperhiveCfg.enable && cfg.enable && deployCfg.authelia.enable && haveClientIdentity; + + hiveNames = lib.attrNames hyperhiveCfg.swarm.hives; + + # The client id agent containers present, per hive — composed exactly as + # ./swarm-authelia.nix composes it, from the same two read-only options, so a + # rename there cannot leave this spelling behind. + agentClientId = hive: "${autheliaCfg.hiveClientPrefix}${hive}${autheliaCfg.agentClientSuffix}"; +in +{ + options.services.hyperhive.deploy.swarm-secret-publisher = { + enable = lib.mkOption { + type = lib.types.bool; + default = deployCfg.authelia.enable; + defaultText = lib.literalExpression "deploy.authelia.enable"; + description = '' + Publish the OIDC client secrets this host mints into the swarm's + secret store, so hives that do not run authelia can read their own. + + Defaults to whether this host mints them, which is the only half of + the question that is a property of *this* host. + + ⚠️ Deliberately **not** `deploy.bao.enable`. That asks whether the + store stands here, and a publisher beside a remote store, holding a + leaf issued out of band, is a deployment this exists to serve — + defaulting on co-location would leave exactly that shape silently + publishing nothing. Whether the wiring is complete is the client + identity's job (see `baoClientCertFile`), not this option's. + + Turning it off leaves every non-co-located hive without a delivery + path, which is the state this exists to end — so the honest reason + to set it false is a deployment delivering those secrets by some + other mechanism it owns. + ''; + }; + + baoClientCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + Client certificate this publisher presents to the store. Its subject + must be {option}`services.hyperhive.deploy.bao.secretPublisherCommonName` + — cert auth matches on the CN, and the role accepts nothing else. + + No default: a module that guessed would be holding the CA opinion + ./swarm-bao.nix deliberately does not hold. + ./glue-secret-publisher-bao-identity.nix points it at the leaf + ./glue-bao-tls.nix mints, where this host mints one. + ''; + }; + + baoClientKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + Private key for {option}`services.hyperhive.deploy.swarm-secret-publisher.baoClientCertFile`. + Both or neither — the unit does not exist unless each is set. + ''; + }; + }; + + config = lib.mkIf active { + services.hyperhive.swarm.otel.journaldUnits = [ "swarm-secret-publish" ]; + + # Re-publish when authelia rotates a secret. The mint writes the file, so + # the file is the event — there is no signal from authelia to subscribe to. + systemd.paths.swarm-secret-publish = { + description = "watch for minted OIDC client secrets to publish"; + wantedBy = [ "multi-user.target" ]; + pathConfig = { + PathChanged = deployCfg.authelia.hostClientSecretDir; + Unit = "swarm-secret-publish.service"; + }; + }; + + systemd.services.swarm-secret-publish = { + description = "publish minted OIDC client secrets to the swarm secret store"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ + baoDeploy.package + pkgs.coreutils + ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + # Bounded here rather than left to systemd's default, so the number a + # boot waits on is in the file that waits. A sealed store answers on + # the port and never answers the write. + TimeoutStartSec = 60; + Restart = "on-failure"; + RestartSec = 30; + }; + environment = { + BAO_ADDR = "https://${hyperhiveCfg.swarm.bao.domain}:${toString hyperhiveCfg.swarm.bao.port}"; + BAO_CLIENT_CERT = cfg.baoClientCertFile; + BAO_CLIENT_KEY = cfg.baoClientKeyFile; + } + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; + script = '' + set -euo pipefail + + published=0 + skipped=0 + + ${lib.concatMapStringsSep "\n" (hive: '' + src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${agentClientId hive}.secret"} + if [ -s "$src" ]; then + # `value=@$src` hands bao the PATH: bao opens the file itself, so + # the plaintext is never an argument of this process. Writing it as + # `value="$(cat "$src")"` would publish it to /proc for anyone on + # the host to read. + bao kv put ${lib.escapeShellArg "secret/swarm/hives/${hive}/queue/agent"} \ + value=@"$src" \ + client_id=${lib.escapeShellArg (agentClientId hive)} + published=$((published + 1)) + else + # Not an error: authelia mints on its FIRST BOOT, so an absent file + # is "not yet", and the path unit above re-runs this when it lands. + echo "no minted secret at $src yet; the path unit will re-run this" >&2 + skipped=$((skipped + 1)) + fi + '') hiveNames} + + echo "published $published client secret(s), skipped $skipped" + ''; + }; + }; +}