hyperhive/nix/host-modules/swarm-secret-publisher.nix

# The unit that copies authelia's minted OIDC client secrets into the swarm's
# secret store, so whoever needs one can read it there: a hive its agents'
# credential, a swarm service its own.
#
# ⚠️ "Whoever", including a reader on THIS host. A swarm service's secret goes
# into the store even when the service runs beside authelia, because its
# reader fetches it from the store in every deployment — ./swarm-grafana.nix
# states the ruling that made that the only route. Publishing "only when the
# reader is elsewhere" would be a second shape of this unit, gated on a fact
# about another host, to save a round trip on the one host that can afford it.
#
# ⚠️ IT RUNS WHERE AUTHELIA DOES, and that is the whole reason it exists as a
# separate thing. `deploy.authelia.hostClientSecretDir`'s own description says
# why: the plaintext's other reader lives in a **different container**, and
# containers that share this host's network namespace still have separate
# filesystem roots, so "the host is the only place both trees are addressable".
# A delivery step therefore runs on the host that mints — not on the store's
# host, and not on the reading hive's.
#
# ⚠️ ITS OWN STORE IDENTITY, not swarm-controller's. That principal may rewrite
# every hive's policy and login role; a unit whose entire job is copying one
# file has no business holding it. ./swarm-bao.nix grants this one
# `create`/`update` under the hive prefix and nothing else.
#
# ⚠️ THE SECRET NEVER REACHES `argv`. `bao` is an external binary, so every
# argument is world-readable in /proc for the life of the call — the value is
# passed as `@<path>` and read by bao itself. (`/knowledge/secret-hygiene.md`:
# "a path keeps the secret out of the store, and reading it into a shell
# variable puts it straight into argv".)
{
  pkgs,
  lib,
  config,
  ...
}:
let
  hyperhiveCfg = config.services.hyperhive;
  deployCfg = hyperhiveCfg.deploy;
  baoDeploy = deployCfg.bao;
  cfg = deployCfg.swarm-secret-publisher;

  # A reader is defined by holding a certificate the store accepts, never by
  # standing next to it — the rule ./glue-matrix-bao-token.nix states in full.
  # `deploy.bao.enable` here would be the co-location assumption itself.
  haveClientIdentity = cfg.baoClientCertFile != null && cfg.baoClientKeyFile != null;

  # Both halves have to be here: the mint (authelia, for the plaintext) and an
  # identity (for the store). Neither implies the other.
  active = hyperhiveCfg.enable && cfg.enable && deployCfg.authelia.enable && haveClientIdentity;

  hiveNames = lib.attrNames hyperhiveCfg.swarm.hives;

  # The client id agent containers present, per hive — composed exactly as
  # ./swarm-authelia.nix composes it, from the same two read-only options, so a
  # rename there cannot leave this spelling behind.
  agentClientId =
    hive:
    "${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${hive}${hyperhiveCfg.swarm.authelia.agentClientSuffix}";

  # The swarm's own services, as opposed to its hives. One client for the whole
  # swarm rather than one per hive, so one value in the store rather than a copy
  # each: `swarm/services/<id>/oidc/client`, under the `services` kind
  # `swarm-secret-client`'s `path::Kind` declares.
  #
  # The ids come from `swarm.*`, which is identical on every host — that is what
  # lets this host name a service's client while running none of them, and it is
  # the same read each service's own module registers its client with. A swarm
  # that runs no Grafana, or no collector, mints no secret for the one it lacks,
  # so its entry skips below rather than needing a condition here.
  serviceClientIds = [
    hyperhiveCfg.swarm.grafana.oidc.clientId
    hyperhiveCfg.swarm.otel.clientId
  ];
in
{
  options.services.hyperhive.deploy.swarm-secret-publisher = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = deployCfg.authelia.enable;
      defaultText = lib.literalExpression "deploy.authelia.enable";
      description = ''
        Publish the OIDC client secrets this host mints into the swarm's
        secret store, so a hive that does not run authelia can read its
        agents' credential and a swarm service can read its own — from
        wherever it runs, this host included.

        Defaults to whether this host mints them, which is the only half of
        the question that is a property of *this* host.

        ⚠️ Deliberately **not** `deploy.bao.enable`. That asks whether the
        store stands here, and a publisher beside a remote store, holding a
        leaf issued out of band, is a deployment this exists to serve —
        defaulting on co-location would leave exactly that shape silently
        publishing nothing. Whether the wiring is complete is the client
        identity's job (see `baoClientCertFile`), not this option's.

        Turning it off leaves every hive but this one without its agents'
        credential, the swarm's Grafana without any login at all, and its
        collector pushing unauthenticated — each secret has exactly one route
        and this is the producer's end of it. So the honest reason to set it
        false is a deployment delivering those secrets by some other
        mechanism it owns.
      '';
    };

    baoClientCertFile = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = ''
        Client certificate this publisher presents to the store. Its subject
        must be {option}`services.hyperhive.deploy.bao.secretPublisherCommonName`
        — cert auth matches on the CN, and the role accepts nothing else.

        No default: a module that guessed would be holding the CA opinion
        ./swarm-bao.nix deliberately does not hold.
        ./glue-secret-publisher-bao-identity.nix points it at the leaf
        ./glue-bao-tls.nix mints, where this host mints one.
      '';
    };

    baoClientKeyFile = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = ''
        Private key for {option}`services.hyperhive.deploy.swarm-secret-publisher.baoClientCertFile`.
        Both or neither — the unit does not exist unless each is set.
      '';
    };
  };

  config = lib.mkIf active {
    services.hyperhive.swarm.otel.journaldUnits = [ "swarm-secret-publish" ];

    # Re-publish when authelia rotates a secret. The mint writes the file, so
    # the file is the event — there is no signal from authelia to subscribe to.
    systemd.paths.swarm-secret-publish = {
      description = "watch for minted OIDC client secrets to publish";
      wantedBy = [ "multi-user.target" ];
      pathConfig = {
        PathChanged = deployCfg.authelia.hostClientSecretDir;
        Unit = "swarm-secret-publish.service";
      };
    };

    systemd.services.swarm-secret-publish = {
      description = "publish minted OIDC client secrets to the swarm secret store";
      wantedBy = [ "multi-user.target" ];
      after = [ "network.target" ];
      path = [
        baoDeploy.package
        pkgs.coreutils
      ];
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
        # Bounded here rather than left to systemd's default, so the number a
        # boot waits on is in the file that waits. A sealed store answers on
        # the port and never answers the write.
        TimeoutStartSec = 60;
        Restart = "on-failure";
        RestartSec = 30;
      };
      environment = {
        BAO_ADDR = "https://${hyperhiveCfg.swarm.bao.domain}:${toString hyperhiveCfg.swarm.bao.port}";
        BAO_CLIENT_CERT = cfg.baoClientCertFile;
        BAO_CLIENT_KEY = cfg.baoClientKeyFile;
      }
      // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
        BAO_CACERT = baoDeploy.serverCaFile;
      };
      script = ''
        set -euo pipefail

        # Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
        # variables above only decide which certificate the TLS handshake
        # presents; without a token `bao` asks its token helper instead, and
        # that is a `sh` this unit's `path` does not carry. `-token-only`
        # answers on stdout and skips the helper on both sides.
        #
        # Unhandled on purpose: `Restart=on-failure` above is what a store that
        # cannot authenticate this host should get. Degrading here would report
        # "published 0" as an ordinary quiet day.
        BAO_TOKEN="$(bao login -method=cert -token-only)"
        export BAO_TOKEN

        published=0
        skipped=0

        ${lib.concatMapStringsSep "\n" (hive: ''
          src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${agentClientId hive}.secret"}
          if [ -s "$src" ]; then
            # `value=@$src` hands bao the PATH: bao opens the file itself, so
            # the plaintext is never an argument of this process. Writing it as
            # `value="$(cat "$src")"` would publish it to /proc for anyone on
            # the host to read.
            bao kv put ${lib.escapeShellArg "secret/swarm/hives/${hive}/queue/agent"} \
              value=@"$src" \
              client_id=${lib.escapeShellArg (agentClientId hive)}
            published=$((published + 1))
          else
            # Not an error: authelia mints on its FIRST BOOT, so an absent file
            # is "not yet", and the path unit above re-runs this when it lands.
            echo "no minted secret at $src yet; the path unit will re-run this" >&2
            skipped=$((skipped + 1))
          fi
        '') hiveNames}

        ${lib.concatMapStringsSep "\n" (id: ''
          src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${id}.secret"}
          if [ -s "$src" ]; then
            # `value=@$src` for the same reason as the hive loop above: bao
            # opens the file itself, so the plaintext is never an argument of
            # this process.
            #
            # No `client_id` field beside it, unlike a hive's credential: that
            # one is derived per hive and has to be reconstructable from the
            # store alone, whereas a service's client id is the swarm-wide
            # option both ends already read.
            bao kv put ${lib.escapeShellArg "secret/swarm/services/${id}/oidc/client"} \
              value=@"$src"
            published=$((published + 1))
          else
            # Not an error, and the ordinary state of a swarm that runs this
            # service nowhere: nothing registered the client, so authelia minted
            # nothing to publish.
            echo "no minted secret at $src yet; the path unit will re-run this" >&2
            skipped=$((skipped + 1))
          fi
        '') serviceClientIds}

        echo "published $published client secret(s), skipped $skipped"
      '';
    };
  };
}