hyperhive/nix/agent-modules/queue-identity.nix

# This agent's own credential for the swarm queue, fetched from the store by
# the agent itself.
#
# The credential beside it in ./queue.nix is keyed per **hive**: one OIDC
# client minted at deploy time and handed to every agent container on the
# hive, so at the queue's auth callout one agent is indistinguishable from its
# co-hived neighbours. This one is keyed per **agent** — `swarm-controller`
# mints it at agent creation into `swarm/agents/<agent>/queue`
# (`swarm_secret_client::queue::agent_queue_path`), at swarm level, with no
# hive anywhere in the chain.
#
# That is also why this unit *fetches* rather than being handed a credential:
# a hive courier in the path would be the hive vouching for which agent this
# is, which is the property the per-agent credential exists to remove. The
# agent authenticates to the store as itself, with the certificate ./bao.nix
# already proves it can log in with, and reads its own path.
#
# ⛔ The store certificate is for reaching the store and nothing else. It is
# never presented to the queue: what goes to the queue is the secret read
# back from this path.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.agent.bao;

  # This container's agent name — the same string `swarm-controller` minted
  # the credential under, because the agent's unix user is named for the
  # agent (see ./user.nix).
  agentName = config.services.hyperhive.agent.user.name;

  # The same three ids ./bao.nix loads. Both units present the same
  # certificate because there is one store identity per agent; the ids are
  # `hive_c0re::lifecycle::agent_identity`'s and a rename is a rename there
  # too.
  certCredential = "hive-agent-bao-cert";
  keyCredential = "hive-agent-bao-key";
  serverCaCredential = "hive-agent-bao-server-ca";

  unitName = "hive-agent-queue-credential";

  # The nix half of `swarm_secret_client::queue::agent_queue_path` plus
  # `path::MOUNT`, spelled exactly as ./bao.nix spells its own sibling path.
  queuePath = "secret/swarm/agents/${agentName}/queue";

  # `RuntimeDirectory=` under the unit's own `User=`, so the file is owned by
  # the agent and readable by the harness without a mode change. /run and not
  # the state dir on purpose: a secret fetched at boot has no business
  # surviving one.
  runtimeDir = "${unitName}";
  secretFile = "/run/${runtimeDir}/secret";

  # The store's address is the whole switch, exactly as in ./bao.nix — and
  # deliberately *not* the queue coordinates in ./queue.nix. Those are the
  # hive's, and gating a swarm-minted per-agent credential on a hive-level
  # option would put the hive back in a path whose entire purpose is not
  # having one.
  configured = cfg.addr != null;
in
{
  options.services.hyperhive.agent.queue.agentSecretFile = lib.mkOption {
    type = lib.types.str;
    readOnly = true;
    default = secretFile;
    description = ''
      Path this agent's own swarm-queue secret is fetched to, for a consumer
      outside the harness unit. Read-only for the same reason
      {option}`services.hyperhive.agent.queue.clientSecretFile` is: it is a
      fact about where the fetch writes, not a knob.

      🩸 A PATH and never a value. The file is `0400` to the agent user and
      is read at the moment it is needed; nothing in this tree puts its
      contents in an environment variable, where `/proc/<pid>/environ` would
      publish them to every process in the container.

      The file exists only once the swarm has minted a credential for this
      agent. An agent created before its swarm did so has none, and
      `${unitName}.service` says so in the journal rather than failing —
      see that unit.
    '';
  };

  config = lib.mkIf configured {
    systemd.services.${unitName} = {
      description = "fetch this agent's own swarm-queue credential from the secret store";
      after = [
        "network.target"
        # Ordering only, not a requirement: that unit is the one that reports
        # a store this agent cannot reach as itself, and it should get to say
        # so before this one reports a path it could not read.
        "hive-agent-bao-identity.service"
      ];
      before = [ "hive-agent.service" ];
      wantedBy = [ "multi-user.target" ];
      path = [
        pkgs.openbao
        pkgs.coreutils
      ];
      # Same sizing and the same `[Unit]`-not-`[Service]` placement as
      # ./bao.nix's check: a few short attempts cover a store that comes up
      # alongside this container, and a longer window only delays the report.
      startLimitBurst = 4;
      startLimitIntervalSec = 300;
      serviceConfig = {
        Type = "oneshot";
        # Keeps the unit active, which is what keeps `RuntimeDirectory=`
        # from being removed out from under the harness.
        RemainAfterExit = true;
        TimeoutStartSec = 30;
        Restart = "on-failure";
        RestartSec = 15;
        User = agentName;
        Group = agentName;
        RuntimeDirectory = runtimeDir;
        RuntimeDirectoryMode = "0700";
        UMask = "0377";
        # Bare ids, no paths: the terse `LoadCredential=` form that inherits
        # a credential the service *manager* received, which is what the
        # container manager passed in. ./bao.nix and ./queue.nix state the
        # same shape.
        LoadCredential = [
          certCredential
          keyCredential
          serverCaCredential
        ];
      };
      environment = {
        BAO_ADDR = cfg.addr;
        # `%d` is `$CREDENTIALS_DIRECTORY`, per-unit and owned by `User=`.
        BAO_CLIENT_CERT = "%d/${certCredential}";
        BAO_CLIENT_KEY = "%d/${keyCredential}";
      };
      script = ''
        set -euo pipefail

        # No identity delivered at all. ./bao.nix's check reports this as the
        # failure it is; there is nothing for this unit to add, and failing
        # here too would be the same cause stated twice.
        for id in ${lib.escapeShellArg certCredential} ${lib.escapeShellArg keyCredential}; do
          if [ ! -s "$CREDENTIALS_DIRECTORY/$id" ]; then
            echo "this agent has no store identity, so it cannot fetch its own queue credential." >&2
            exit 0
          fi
        done

        # Only when one was delivered — absent means verify the store's
        # listener against the container's own trust store, which is what a
        # deployment with a real CA wants. ./bao.nix says the same.
        if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
          export BAO_CACERT="$CREDENTIALS_DIRECTORY/${serverCaCredential}"
        fi

        err="$(mktemp)"
        trap 'rm -f "$err"' EXIT

        # Cert auth is a login, not a transport setting: the `BAO_CLIENT_*`
        # variables above only pick the certificate the handshake presents.
        # `-token-only` answers on stdout and skips the token helper, which
        # is a `sh` this unit's `path` does not carry.
        if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
          echo "this agent's certificate was refused by the swarm secret store at $BAO_ADDR." >&2
          if [ -s "$err" ]; then cat "$err" >&2; fi
          exit 1
        fi
        export BAO_TOKEN

        # 🩸 Degrades where ./bao.nix's check fails, and the reason is that
        # the two reads are governed by the *same* policy stanza:
        # `swarm_secret_client::policy::render_agent` grants read on
        # `secret/data/swarm/agents/<agent>/*`, which covers this path and
        # the `bao-mtls` one beside it alike. So a refusal this unit sees and
        # that check did not cannot be a policy that drifted — it is an
        # object that has not been minted, which is the ordinary state of
        # every agent created before its swarm knew to mint one. A unit that
        # failed at every boot over that would be loud about a deployment
        # doing nothing wrong.
        #
        # ⚠️ Written by redirect into the runtime directory, never echoed:
        # the field is the secret itself.
        if ! bao kv get -field=value ${lib.escapeShellArg queuePath} > ${lib.escapeShellArg secretFile} 2>"$err"; then
          rm -f ${lib.escapeShellArg secretFile}
          echo "no per-agent queue credential at ${queuePath} yet; this agent falls back to its hive's shared one." >&2
          if [ -s "$err" ]; then cat "$err" >&2; fi
          exit 0
        fi

        echo "fetched this agent's own queue credential from ${queuePath}."
      '';
    };

    # The harness reads a path and never a value, the same shape ./queue.nix
    # hands it the hive-scoped secret in. Not `%d` here: this credential is
    # not a systemd credential at all — it is a file this container fetched
    # for itself, which is the whole point.
    systemd.services.hive-agent = {
      after = [ "${unitName}.service" ];
      environment.HIVE_AGENT_QUEUE_AGENT_SECRET_FILE = secretFile;
    };
  };
}