# `checks.module-eval-agent-queue-bao` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, shared, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) agent agentWith runGroup agentHarness ; # The agent side of the swarm queue. Both coordinates set is the only state # in which the harness unit declares a credential at all, so the pair and # the empty fixture beside it are the two arms worth having. agentQueue = agent { queue.natsUrl = "nats://10.42.0.1:4222"; queue.tokenEndpoint = "https://auth.t.local/api/oidc/token"; }; agentNoQueue = shared.agentBare; # The agent side of the swarm secret store. The address is the whole switch — # it is both what generates the login check and what that check points at — # so it and the empty fixture beside it are the two arms worth having. agentBao = shared.agentWithBao; agentNoBao = shared.agentBare; # `swarm-logs` authenticates with the agent's queue secret from the store, so # the store address gates it beside the query URL. logsUrl = "https://logs.t.local/select/logsql/query"; agentLogs = agentWith { services.hyperhive.agent.bao.addr = "https://bao.t.local:8200"; services.hyperhive.agent.logs.queryUrl = logsUrl; }; agentLogsNoBao = agentWith { services.hyperhive.agent.logs.queryUrl = logsUrl; }; swarmLogsIn = machine: lib.findFirst ( p: (p.name or "") == "swarm-logs-configured" ) null machine.environment.systemPackages; agentBaoIdentity = machine: machine.systemd.services.hive-agent-bao-identity; cases = [ { # The wrapper names the store identity ./queue-identity.nix gives the # harness, by path, and nothing of the hive's shared OIDC client. name = "swarm-logs is wrapped with the agent's own store identity"; ok = let w = (swarmLogsIn agentLogs).buildCommand; a = agentLogs.services.hyperhive.agent; set = k: v: lib.hasInfix "--set ${k} ${lib.escapeShellArg v}" w; in set "HIVE_AGENT_LOGS_QUERY_URL" logsUrl && set "HIVE_AGENT_NAME" a.user.name && set "BAO_ADDR" "https://bao.t.local:8200" && set "BAO_CLIENT_CERT" a.bao.clientCertFile && set "BAO_CLIENT_KEY" a.bao.clientKeyFile && set "BAO_CACERT" "/run/credentials/hive-agent.service/hive-agent-bao-server-ca" && !(lib.hasInfix "OIDC" w); } { # A query URL with nothing to authenticate as installs nothing. name = "an agent with a log store but no secret store gets no swarm-logs"; ok = swarmLogsIn agentLogsNoBao == null && swarmLogsIn agentLogs != null; } { # Both ids or neither: the secret authenticates nobody without the id it # belongs to, and `swarm-logs` reads both. name = "an agent with queue coordinates imports both halves of its credential"; ok = let c = (agentHarness agentQueue).serviceConfig.LoadCredential; in builtins.elem "hive-queue-agent-secret" c && builtins.elem "hive-queue-agent-client-id" c; } { # An agent built before its hive was handed the queue's address. It # must declare nothing rather than name a credential that never # arrives. name = "an agent with no queue coordinates declares no credential"; ok = !((agentHarness agentNoQueue).serviceConfig ? LoadCredential); } { # The harness exits when it cannot reach the queue. It has to come back # however long the outage lasts, and slowly enough that every agent of a # swarm restarting at once does not hammer the store and the queue. name = "the harness restarts every 30s with no start limit"; ok = let u = agentHarness agentQueue; in u.serviceConfig.Restart == "on-failure" && u.serviceConfig.RestartSec == 30 && u.startLimitIntervalSec == 0; } { # The three ids `hive_c0re::lifecycle::agent_identity` forwards under. # Neither end can discover the other's spelling, and a mismatch is a # credential that is simply not there — which this unit then reports as # a hive that delivered nothing. name = "an agent with the store enabled imports every half of its identity"; ok = let c = (agentBaoIdentity agentBao).serviceConfig.LoadCredential; in builtins.elem "hive-agent-bao-cert" c && builtins.elem "hive-agent-bao-key" c && builtins.elem "hive-agent-bao-server-ca" c; } { # The live pair sits in the unit's own runtime directory, which outlives # each run so the timer's next run logs in with what the last one # wrote. The address is the option's value rather than a literal that # agrees with it today. name = "the identity check presents the live pair it keeps in its runtime directory"; ok = let u = agentBaoIdentity agentBao; e = u.environment; in e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert" && e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key" && u.serviceConfig.RuntimeDirectory == "hive-agent-bao-identity" && u.serviceConfig.RuntimeDirectoryPreserve == "yes" && e.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr; } { # A leaf the controller re-issues reaches a running container only # through this re-run, and an active unit cannot be started again. name = "the identity check re-runs on a timer"; ok = (agentBaoIdentity agentBao).serviceConfig.RemainAfterExit == false && agentBao.systemd.timers.hive-agent-bao-identity.timerConfig.OnUnitInactiveSec == "10min"; } { # Same 403-not-a-miss reason as the hive-side readers: the path # `swarm_secret_client::mtls::identity_path` builds is the one this # agent's own policy stanza covers, and a path outside it is refused # however correct it looks. Built from the agent's own name rather than # from a literal, because the name is what makes it this agent's path # and not some other agent's. name = "the identity check reads the agent's own path"; ok = let m = agentBao; name = m.services.hyperhive.agent.user.name; in lib.hasInfix "secret/swarm/agents/${name}/bao-mtls" (agentBaoIdentity m).script; } { # The whole point of the unit, and the thing a quieter default would # undo: every arm of the check ends the unit non-zero, so an agent that # cannot authenticate as itself says so at boot instead of at whichever # pull needed the store first. name = "the identity check fails the unit rather than degrading"; ok = let u = agentBaoIdentity agentBao; in lib.hasInfix "exit 1" u.script && !(lib.hasInfix "exit 0" u.script) && u.serviceConfig.Restart == "on-failure"; } { # Nothing about the identity may be printed, and the read-back is where # that could slip: `bao kv get` on this path answers with certificate # material and a private key. The answer goes to a file in the unit's # own `0700` directory. # # The path goes through `lib.escapeShellArg` here for the same reason the # module passes it through one — that helper decides whether an argument # needs quotes at all, and this one (only `[a-z0-9/-]`) comes back bare. # Spelling the quotes in by hand asserts a rendering nixpkgs chooses # rather than the redirect this property is about. name = "the identity check writes what it reads back to a file"; ok = let m = agentBao; name = m.services.hyperhive.agent.user.name; arg = lib.escapeShellArg "secret/swarm/agents/${name}/bao-mtls"; in lib.hasInfix "bao kv get -format=json ${arg} >\"$raw\"" (agentBaoIdentity m).script; } { # The absence arm, and what makes the four above able to fail. An agent # whose swarm never minted an identity has nothing to log in with, and a # failed unit at every boot would be the loudest possible statement # about a deployment that never asked for one. name = "an agent told no store address runs no identity check"; ok = !(agentNoBao.systemd.services ? hive-agent-bao-identity); } { # The harness reads its per-agent queue secret from the store itself, # under the same store identity the check above keeps current, because # there is one identity per agent. name = "the harness presents the agent's own store identity"; ok = let u = agentHarness agentBao; in u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert" && u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key" && builtins.elem "hive-agent-bao-identity.service" u.after && u.environment.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr; } { # `swarm_secret_client` builds both the store path and the cert-auth role # from this name, so it has to be the name the credential was minted # under. name = "the harness is told the agent name its queue secret is minted under"; ok = (agentHarness agentBao).environment.HIVE_AGENT_NAME == agentBao.services.hyperhive.agent.user.name; } { # The per-agent queue secret lives in the harness's memory only: no unit # fetches it to a file, and the harness is handed no file to read it # from. name = "no unit writes the per-agent queue secret to disk"; ok = !(agentBao.systemd.services ? hive-agent-queue-credential) && !((agentHarness agentBao).environment ? HIVE_AGENT_QUEUE_AGENT_SECRET_FILE); } { # The absence arm. An agent whose swarm gave it no store has nothing to # log in with, so the harness is handed no store coordinates at all. name = "an agent told no store address hands the harness no store identity"; ok = let u = agentHarness agentNoBao; in !(u.environment ? BAO_ADDR) && !(u.environment ? HIVE_AGENT_NAME) && !(builtins.elem "hive-agent-bao-server-ca" (u.serviceConfig.LoadCredential or [ ])); } ]; in runGroup "agent-queue-bao" cases