hyperhive/nix/module-eval/agent-queue-bao.nix

# `checks.module-eval-agent-queue-bao` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
  shared,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    agent
    agentWith
    runGroup
    agentHarness
    ;

  # An `agent.nix` still setting the queue coordinates the agent module no
  # longer declares.
  agentQueue = agent {
    queue.natsUrl = "nats://10.42.0.1:4222";
    queue.tokenEndpoint = "https://auth.t.local/api/oidc/token";
  };

  # The agent side of the swarm secret store. The address is the whole switch —
  # it is both what generates the login check and what that check points at —
  # so it and the empty fixture beside it are the two arms worth having.
  agentBao = shared.agentWithBao;

  agentNoBao = shared.agentBare;

  # `swarm-logs` and `swarm-metrics` authenticate with the agent's queue secret
  # from the store, so the store address gates each beside its store URL.
  logsUrl = "https://logs.t.local/select/logsql/query";
  metricsUrl = "https://metrics.t.local";
  agentLogs = agentWith {
    services.hyperhive.agent.bao.addr = "https://bao.t.local:8200";
    services.hyperhive.agent.logs.queryUrl = logsUrl;
    services.hyperhive.agent.metrics.url = metricsUrl;
  };
  agentLogsNoBao = agentWith {
    services.hyperhive.agent.logs.queryUrl = logsUrl;
    services.hyperhive.agent.metrics.url = metricsUrl;
  };
  configuredIn =
    name: machine: lib.findFirst (p: (p.name or "") == name) null machine.environment.systemPackages;
  swarmLogsIn = configuredIn "swarm-logs-configured";
  swarmMetricsIn = configuredIn "swarm-metrics-configured";

  agentBaoIdentity = machine: machine.systemd.services.hive-agent-bao-identity;
  cases = [
    {
      # The wrapper names the store identity ./queue-identity.nix gives the
      # harness, by path, and nothing of the hive's shared OIDC client.
      name = "swarm-logs is wrapped with the agent's own store identity";
      ok =
        let
          w = (swarmLogsIn agentLogs).buildCommand;
          a = agentLogs.services.hyperhive.agent;
          set = k: v: lib.hasInfix "--set ${k} ${lib.escapeShellArg v}" w;
        in
        set "HIVE_AGENT_LOGS_QUERY_URL" logsUrl
        && set "HIVE_AGENT_NAME" a.user.name
        && set "BAO_ADDR" "https://bao.t.local:8200"
        && set "BAO_CLIENT_CERT" a.bao.clientCertFile
        && set "BAO_CLIENT_KEY" a.bao.clientKeyFile
        && set "BAO_CACERT" "/run/credentials/hive-agent.service/hive-agent-bao-server-ca"
        && !(lib.hasInfix "OIDC" w);
    }
    {
      # A query URL with nothing to authenticate as installs nothing.
      name = "an agent with a log store but no secret store gets no swarm-logs";
      ok = swarmLogsIn agentLogsNoBao == null && swarmLogsIn agentLogs != null;
    }
    {
      # The same store identity as `swarm-logs`, and the store's origin.
      name = "swarm-metrics is wrapped with the agent's own store identity";
      ok =
        let
          w = (swarmMetricsIn agentLogs).buildCommand;
          a = agentLogs.services.hyperhive.agent;
          set = k: v: lib.hasInfix "--set ${k} ${lib.escapeShellArg v}" w;
        in
        set "HIVE_AGENT_METRICS_URL" metricsUrl
        && set "HIVE_AGENT_NAME" a.user.name
        && set "BAO_ADDR" "https://bao.t.local:8200"
        && set "BAO_CLIENT_CERT" a.bao.clientCertFile
        && set "BAO_CLIENT_KEY" a.bao.clientKeyFile
        && set "BAO_CACERT" "/run/credentials/hive-agent.service/hive-agent-bao-server-ca";
    }
    {
      # A metrics URL with nothing to authenticate as installs nothing.
      name = "an agent with a metrics store but no secret store gets no swarm-metrics";
      ok = swarmMetricsIn agentLogsNoBao == null && swarmMetricsIn agentLogs != null;
    }
    {
      # The hive forwards no shared queue credential, so the harness unit
      # names none, whatever the agent's config says.
      name = "an agent imports no hive-shared queue credential";
      ok =
        let
          credentials = m: (agentHarness m).serviceConfig.LoadCredential or [ ];
        in
        !lib.any (lib.hasPrefix "hive-queue-agent-") (credentials agentQueue ++ credentials agentBao)
        # Control: the lookup finds the store identity's credential.
        && lib.elem "hive-agent-bao-server-ca" (credentials agentBao);
    }
    {
      # Both options are refused, naming where the address
      # comes from now, rather than dropped silently.
      name = "an agent setting the removed queue options is refused";
      ok =
        lib.count (a: !a.assertion && lib.hasInfix "HIVE_AGENT_NATS_URL" a.message) agentQueue.assertions
        == 2;
    }
    {
      # The harness exits when it cannot reach the queue. It has to come back
      # however long the outage lasts, and slowly enough that every agent of a
      # swarm restarting at once does not hammer the store and the queue.
      name = "the harness restarts every 30s with no start limit";
      ok =
        let
          u = agentHarness agentQueue;
        in
        u.serviceConfig.Restart == "on-failure"
        && u.serviceConfig.RestartSec == 30
        && u.startLimitIntervalSec == 0;
    }
    {
      # The three ids `hive_c0re::lifecycle::agent_identity` forwards under.
      # Neither end can discover the other's spelling, and a mismatch is a
      # credential that is simply not there — which this unit then reports as
      # a hive that delivered nothing.
      name = "an agent with the store enabled imports every half of its identity";
      ok =
        let
          c = (agentBaoIdentity agentBao).serviceConfig.LoadCredential;
        in
        builtins.elem "hive-agent-bao-cert" c
        && builtins.elem "hive-agent-bao-key" c
        && builtins.elem "hive-agent-bao-server-ca" c;
    }
    {
      # The live pair sits in the unit's own runtime directory, which outlives
      # each run so the timer's next run logs in with what the last one
      # wrote. The address is the option's value rather than a literal that
      # agrees with it today.
      name = "the identity check presents the live pair it keeps in its runtime directory";
      ok =
        let
          u = agentBaoIdentity agentBao;
          e = u.environment;
        in
        e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
        && e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
        && u.serviceConfig.RuntimeDirectory == "hive-agent-bao-identity"
        && u.serviceConfig.RuntimeDirectoryPreserve == "yes"
        && e.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr;
    }
    {
      # A leaf the controller re-issues reaches a running container only
      # through this re-run, and an active unit cannot be started again.
      name = "the identity check re-runs on a timer";
      ok =
        (agentBaoIdentity agentBao).serviceConfig.RemainAfterExit == false
        && agentBao.systemd.timers.hive-agent-bao-identity.timerConfig.OnUnitInactiveSec == "10min";
    }
    {
      # Same 403-not-a-miss reason as the hive-side readers: the path
      # `swarm_secret_client::mtls::identity_path` builds is the one this
      # agent's own policy stanza covers, and a path outside it is refused
      # however correct it looks. Built from the agent's own name rather than
      # from a literal, because the name is what makes it this agent's path
      # and not some other agent's.
      name = "the identity check reads the agent's own path";
      ok =
        let
          m = agentBao;
          name = m.services.hyperhive.agent.user.name;
        in
        lib.hasInfix "secret/swarm/agents/${name}/bao-mtls" (agentBaoIdentity m).script;
    }
    {
      # The whole point of the unit, and the thing a quieter default would
      # undo: every arm of the check ends the unit non-zero, so an agent that
      # cannot authenticate as itself says so at boot instead of at whichever
      # pull needed the store first.
      name = "the identity check fails the unit rather than degrading";
      ok =
        let
          u = agentBaoIdentity agentBao;
        in
        lib.hasInfix "exit 1" u.script
        && !(lib.hasInfix "exit 0" u.script)
        && u.serviceConfig.Restart == "on-failure";
    }
    {
      # Nothing about the identity may be printed, and the read-back is where
      # that could slip: `bao kv get` on this path answers with certificate
      # material and a private key. The answer goes to a file in the unit's
      # own `0700` directory.
      #
      # The path goes through `lib.escapeShellArg` here for the same reason the
      # module passes it through one — that helper decides whether an argument
      # needs quotes at all, and this one (only `[a-z0-9/-]`) comes back bare.
      # Spelling the quotes in by hand asserts a rendering nixpkgs chooses
      # rather than the redirect this property is about.
      name = "the identity check writes what it reads back to a file";
      ok =
        let
          m = agentBao;
          name = m.services.hyperhive.agent.user.name;
          arg = lib.escapeShellArg "secret/swarm/agents/${name}/bao-mtls";
        in
        lib.hasInfix "bao kv get -format=json ${arg} >\"$raw\"" (agentBaoIdentity m).script;
    }
    {
      # The absence arm, and what makes the four above able to fail. An agent
      # whose swarm never minted an identity has nothing to log in with, and a
      # failed unit at every boot would be the loudest possible statement
      # about a deployment that never asked for one.
      name = "an agent told no store address runs no identity check";
      ok = !(agentNoBao.systemd.services ? hive-agent-bao-identity);
    }
    {
      # The harness reads its per-agent queue secret from the store itself,
      # under the same store identity the check above keeps current, because
      # there is one identity per agent.
      name = "the harness presents the agent's own store identity";
      ok =
        let
          u = agentHarness agentBao;
        in
        u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
        && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
        && builtins.elem "hive-agent-bao-identity.service" u.after
        && u.environment.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr;
    }
    {
      # `swarm_secret_client` builds both the store path and the cert-auth role
      # from this name, so it has to be the name the credential was minted
      # under.
      name = "the harness is told the agent name its queue secret is minted under";
      ok =
        (agentHarness agentBao).environment.HIVE_AGENT_NAME == agentBao.services.hyperhive.agent.user.name;
    }
    {
      # The per-agent queue secret lives in the harness's memory only: no unit
      # fetches it to a file, and the harness is handed no file to read it
      # from.
      name = "no unit writes the per-agent queue secret to disk";
      ok =
        !(agentBao.systemd.services ? hive-agent-queue-credential)
        && !((agentHarness agentBao).environment ? HIVE_AGENT_QUEUE_AGENT_SECRET_FILE);
    }
    {
      # The absence arm. An agent whose swarm gave it no store has nothing to
      # log in with, so the harness is handed no store coordinates at all.
      name = "an agent told no store address hands the harness no store identity";
      ok =
        let
          u = agentHarness agentNoBao;
        in
        !(u.environment ? BAO_ADDR)
        && !(u.environment ? HIVE_AGENT_NAME)
        && !(builtins.elem "hive-agent-bao-server-ca" (u.serviceConfig.LoadCredential or [ ]));
    }
  ];
in
runGroup "agent-queue-bao" cases