hyperhive/nix/host-modules/swarm-secret-publisher.nix

# The unit that puts swarm-level secrets into the swarm's secret store, so
# whoever needs one can read it there: a hive its own and its agents'
# credentials, a service or the controller its own. It mints nothing: every
# value is an OIDC client secret authelia minted, and this unit copies it.
#
# ⚠️ "Whoever", including a reader on THIS host. A swarm service's secret goes
# into the store even when the service runs beside authelia, because its
# reader fetches it from the store in every deployment — ./swarm-grafana.nix
# states the ruling that made that the only route. Publishing "only when the
# reader is elsewhere" would be a second shape of this unit, gated on a fact
# about another host, to save a round trip on the one host that can afford it.
#
# ⚠️ IT RUNS WHERE AUTHELIA DOES, and that is the whole reason it exists as a
# separate thing. `deploy.authelia.hostClientSecretDir`'s own description says
# why: the plaintext's other reader lives in a **different container**, and
# containers that share this host's network namespace still have separate
# filesystem roots, so "the host is the only place both trees are addressable".
# A delivery step therefore runs on the minting host, not the store's or the
# reading hive's.
#
# ⚠️ ITS OWN STORE IDENTITY, not swarm-controller's. That principal may rewrite
# every hive's policy and login role; a unit that writes a handful of secrets
# has no business holding it. ./swarm-bao.nix grants this one `create`/`update`
# under the hive prefix and nothing else — no `read`.
#
# ⚠️ THE SECRET NEVER REACHES `argv`. `bao` is an external binary, so every
# argument is world-readable in /proc for the life of the call — the value is
# passed as `@<path>` and read by bao itself.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  hyperhiveCfg = config.services.hyperhive;
  deployCfg = hyperhiveCfg.deploy;
  baoDeploy = deployCfg.bao;
  cfg = deployCfg.swarm-secret-publisher;

  # A reader is defined by holding a certificate the store accepts, never by
  # standing next to it — the rule ./glue-queue-agent-credential.nix states in
  # full.
  # `deploy.bao.enable` here would be the co-location assumption itself.
  haveClientIdentity = cfg.baoClientCertFile != null && cfg.baoClientKeyFile != null;

  # Both halves have to be here: the mint (authelia, for the plaintext) and an
  # identity (for the store). Neither implies the other. Authelia here is a
  # requirement, not a co-location toggle: every file the unit copies is under
  # its `hostClientSecretDir`.
  active = cfg.enable && deployCfg.authelia.enable && haveClientIdentity;

  hiveNames = lib.attrNames hyperhiveCfg.swarm.hives;

  # The client id agent containers present, per hive — composed exactly as
  # ./swarm-authelia.nix composes it, from the same two read-only options, so a
  # rename there cannot leave this spelling behind.
  agentClientId =
    hive:
    "${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${hive}${hyperhiveCfg.swarm.authelia.agentClientSuffix}";

  # The hive's own client id, which hive-c0re publishes the hive's status with.
  # Composed as ./swarm-authelia.nix composes it, for the reason above.
  hiveClientId = hive: "${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${hive}";

  # The swarm's own services, as opposed to its hives. One client for the whole
  # swarm rather than one per hive, so one value in the store rather than a copy
  # each: `swarm/services/<id>/oidc/client`.
  #
  # The ids come from `swarm.*`, which is identical on every host — that is what
  # lets this host name a service's client while running none of them, and it is
  # the same read each service's own module registers its client with. A swarm
  # that runs no Grafana, or no collector, mints no secret for the one it lacks,
  # so its entry skips below rather than needing a condition here.
  serviceClientIds = [
    hyperhiveCfg.swarm.grafana.oidc.clientId
    hyperhiveCfg.swarm.otel.clientId
    # The secret store's own journal forwarder. A third swarm service by this
    # list's own rule — a client registered once for the whole swarm, whose
    # secret one host mints and another reads — even though the principal
    # behind it runs inside the store's container rather than beside it.
    # Missing from here, authelia mints the value and nothing carries it, so
    # the reader waits on a path that is never written.
    hyperhiveCfg.swarm.bao.otel.clientId
    # The store's browser UI, whose `oidc` auth method the store's host
    # configures with this secret.
    hyperhiveCfg.swarm.bao.ui.oidc.clientId
    # The queue's auth-callout responder, which reads it itself on the
    # queue's host.
    hyperhiveCfg.swarm.nats.clientId
    # The homeserver, read by `swarm-bao-matrix-oidc` on its host.
    hyperhiveCfg.swarm.matrix.sso.clientId
    # The forge's login source, read by `swarm-bao-forge-oidc` on its host.
    hyperhiveCfg.swarm.forge.sso.clientId
    # The review classifier, which reads it itself inside its container.
    hyperhiveCfg.swarm.review-classifier.clientId
    # The swarm builder, which reads it itself inside its container.
    hyperhiveCfg.swarm.builder.clientId
    # The claude login minter, which reads it itself inside its container.
    hyperhiveCfg.swarm.claude-login.clientId
  ];

  # The swarm controller's OIDC client, which it reads back from the store
  # under its own certificate.
  controllerClientId = hyperhiveCfg.swarm.controller.queueClientId;
  controllerSecretPath = "secret/swarm/controller/swarm-controller/oidc/client";
in
{
  options.services.hyperhive.deploy.swarm-secret-publisher = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Publish the OIDC client secrets this host mints into the swarm's
        secret store, so a hive that does not run authelia can read its
        own and its agents' credentials and a swarm service or the
        controller can read its own — from wherever it runs, this host
        included.

        Defaults to whether this host mints the OIDC secrets (set by
        `glue-secret-publisher-authelia.nix`), which is the only half of the
        question that is a property of *this* host.

        ⚠️ Deliberately **not** `deploy.bao.enable`. That asks whether the
        store stands here, and a publisher beside a remote store, holding a
        leaf issued out of band, is a deployment this exists to serve —
        defaulting on co-location would leave exactly that shape silently
        publishing nothing. Whether the wiring is complete is the client
        identity's job (see `baoClientCertFile`), not this option's.

        Turning it off leaves every hive but this one without its agents'
        credential, every hive unable to publish its status, the swarm
        controller unable to start, the swarm's Grafana without any login at
        all, its collector pushing unauthenticated, the queue's responder
        unable to start, the homeserver unable to start, and the forge's login
        source without its secret.
        Each secret has exactly one route and this is the producer's
        end of it, so the honest reason to set it false is a deployment
        delivering those secrets by some other mechanism it owns.
      '';
    };

    baoClientCertFile = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = ''
        Client certificate this publisher presents to the store. Its subject
        must be {option}`services.hyperhive.deploy.bao.secretPublisherCommonName`
        — cert auth matches on the CN, and the role accepts nothing else.

        No default: a module that guessed would be holding the CA opinion
        ./swarm-bao.nix deliberately does not hold.
        ./glue-secret-publisher-bao-identity.nix points it at the leaf
        ./glue-bao-tls.nix mints, where this host mints one.
      '';
    };

    baoClientKeyFile = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = ''
        Private key for {option}`services.hyperhive.deploy.swarm-secret-publisher.baoClientCertFile`.
        Both or neither — the unit does not exist unless each is set.
      '';
    };
  };

  config = lib.mkIf active {
    # Re-publish when authelia rotates a secret. The mint writes the file, so
    # the file is the event — there is no signal from authelia to subscribe to.
    systemd.paths.swarm-secret-publish = {
      description = "watch for minted OIDC client secrets to publish";
      wantedBy = [ "multi-user.target" ];
      pathConfig = {
        PathChanged = deployCfg.authelia.hostClientSecretDir;
        Unit = "swarm-secret-publish.service";
      };
    };

    systemd.services.swarm-secret-publish = {
      description = "publish the swarm's minted secrets to the swarm secret store";
      wantedBy = [ "multi-user.target" ];
      after = [ "network.target" ];
      path = [
        baoDeploy.package
        pkgs.coreutils
      ];
      # ⚠️ No `RemainAfterExit`. A oneshot that stays `active (exited)` turns
      # every later start from the path unit above into a no-op.
      serviceConfig = {
        Type = "oneshot";
        # Bounded here rather than left to systemd's default, so the number a
        # boot waits on is in the file that waits. A sealed store answers on
        # the port and never answers the write.
        TimeoutStartSec = 60;
        Restart = "on-failure";
        RestartSec = 30;
      };
      environment = {
        BAO_ADDR = "https://${hyperhiveCfg.swarm.bao.domain}:${toString hyperhiveCfg.swarm.bao.port}";
        BAO_CLIENT_CERT = cfg.baoClientCertFile;
        BAO_CLIENT_KEY = cfg.baoClientKeyFile;
      }
      // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
        BAO_CACERT = baoDeploy.serverCaFile;
      };
      script = ''
        set -euo pipefail

        # Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
        # variables above only decide which certificate the TLS handshake
        # presents; without a token `bao` asks its token helper instead, and
        # that is a `sh` this unit's `path` does not carry. `-token-only`
        # answers on stdout and skips the helper on both sides.
        #
        # Unhandled on purpose: `Restart=on-failure` above is what a store that
        # cannot authenticate this host should get. Degrading here would report
        # "published 0" as an ordinary quiet day.
        BAO_TOKEN="$(bao login -method=cert -token-only)"
        export BAO_TOKEN

        published=0
        skipped=0

        ${lib.concatMapStringsSep "\n" (hive: ''
          src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${agentClientId hive}.secret"}
          if [ -s "$src" ]; then
            # `value=@$src` hands bao the PATH: bao opens the file itself, so
            # the plaintext is never an argument of this process. Writing it as
            # `value="$(cat "$src")"` would publish it to /proc for anyone on
            # the host to read.
            bao kv put ${lib.escapeShellArg "secret/swarm/hives/${hive}/queue/agent"} \
              value=@"$src" \
              client_id=${lib.escapeShellArg (agentClientId hive)}
            published=$((published + 1))
          else
            # Not an error: authelia mints on its FIRST BOOT, so an absent file
            # is "not yet", and the path unit above re-runs this when it lands.
            echo "no minted secret at $src yet; the path unit will re-run this" >&2
            skipped=$((skipped + 1))
          fi
        '') hiveNames}

        # Each hive's own client, which that hive's hive-c0re reads under its
        # own certificate. The nix half of
        # `swarm_secret_client::queue::hive_client_path`.
        ${lib.concatMapStringsSep "\n" (hive: ''
          src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${hiveClientId hive}.secret"}
          if [ -s "$src" ]; then
            bao kv put ${lib.escapeShellArg "secret/swarm/hives/${hive}/oidc/client"} \
              value=@"$src"
            published=$((published + 1))
          else
            echo "no minted secret at $src yet; the path unit will re-run this" >&2
            skipped=$((skipped + 1))
          fi
        '') hiveNames}

        ${lib.concatMapStringsSep "\n" (id: ''
          src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${id}.secret"}
          if [ -s "$src" ]; then
            # `value=@$src` for the same reason as the hive loop above: bao
            # opens the file itself, so the plaintext is never an argument of
            # this process.
            #
            # No `client_id` field beside it, unlike a hive's credential: that
            # one is derived per hive and has to be reconstructable from the
            # store alone, whereas a service's client id is the swarm-wide
            # option both ends already read.
            bao kv put ${lib.escapeShellArg "secret/swarm/services/${id}/oidc/client"} \
              value=@"$src"
            published=$((published + 1))
          else
            # Not an error, and the ordinary state of a swarm that runs this
            # service nowhere: nothing registered the client, so authelia minted
            # nothing to publish.
            echo "no minted secret at $src yet; the path unit will re-run this" >&2
            skipped=$((skipped + 1))
          fi
        '') serviceClientIds}

        # The controller's own client, under `controller/`, which no hive
        # reads: this client may read every hive's status. The nix half of
        # `swarm_secret_client::queue::controller_client_path`.
        src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${controllerClientId}.secret"}
        if [ -s "$src" ]; then
          bao kv put ${lib.escapeShellArg controllerSecretPath} \
            value=@"$src"
          published=$((published + 1))
        else
          # Not an error: authelia mints on its first boot, and the path unit
          # above re-runs this when it lands.
          echo "no minted secret at $src yet; the path unit will re-run this" >&2
          skipped=$((skipped + 1))
        fi

        echo "published $published client secret(s), skipped $skipped"
      '';
    };
  };
}