swarm-logs-auth: check agent tokens on the log store's query route

A new `auth_request` responder on the log store's host. nginx sends it
every `/select/logsql/` request whose `Authorization` is
`Bearer swarm-agent.<agent>.<secret>`; it answers 200 when the secret is
the one stored at `swarm/agents/<agent>/queue` (`swarm-agent-verify`),
401 otherwise. Every other bearer still goes to authelia, so agents
still presenting the hive's shared OIDC client keep working.

`auth_request` takes a fixed URI, so the split is a `map` on
`$http_authorization` plus an internal location that rewrites to the
map's pick. Checked against nginx 1.30.5 with stub upstreams: an agent
token only the stub validator accepts gets 200, a token only the stub
authelia accepts gets 200, every wrong one 401, and the internal
locations 404 from outside.

The responder serves a unix socket in its own traverse-only runtime
directory and reads the store under its own leaf
(`swarm-logs-auth`), whose policy is `swarm/agents/+/queue`, read-only.
glue-bao-tls mints the leaf where the store runs; elsewhere the
operator places it, and eval refuses a log store host without one.

Part of #4459.

nix/host-modules/default.nix

@@ -33,6 +33,7 @@
     ./glue-forge-oidc-client.nix
     ./glue-grafana-oidc-client.nix
     ./glue-hive-tls-bao-identity.nix
+    ./glue-logs-auth-bao-identity.nix
     ./glue-matrix-ctl-bao-identity.nix
     ./glue-matrix-oidc-client.nix
     ./glue-nats-auth-bao-identity.nix

nix/host-modules/glue-bao-tls.nix

@@ -270,6 +270,13 @@ in
         [ -s ${pkiDir}/nats-auth.pem ] || ${signLeaf} ${pkiDir} nats-auth \
           ${lib.escapeShellArg deployCfg.bao.natsAuthCommonName} "" clientAuth
 
+        # The log store's agent auth responder, which reads agent queue
+        # credentials. Minted here for the matrix-ctl leaf's reason: the log
+        # store is a swarm singleton, so elsewhere this is the file an operator
+        # copies.
+        [ -s ${pkiDir}/logs-auth.pem ] || ${signLeaf} ${pkiDir} logs-auth \
+          ${lib.escapeShellArg deployCfg.bao.logsAuthCommonName} "" clientAuth
+
         # The CI runner's token fetch. Minted here for the matrix-ctl leaf's
         # reason: the runner is a swarm singleton, so on a runner host without
         # the store this is the file an operator copies.

nix/host-modules/glue-logs-auth-bao-identity.nix

@@ -0,0 +1,31 @@
+# Glue: point the log store's agent auth responder at the bao leaf minted for it.
+#
+# ONE PAIRING PER FILE — the swarm-logs-auth principal ← bao, and nothing else.
+# Deleting this leaves a log store with no responder identity unless the
+# operator names one, which ./swarm-victorialogs.nix refuses at eval.
+#
+# ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the
+# leaf. What belongs here is the pairing: which paths the responder presents.
+#
+# ⚠️ Gated on the store running on this host, the only place the leaf is
+# minted. A log-store host without the store gets no default here; the
+# operator places the responder's leaf and names it.
+#
+# Everything is `mkDefault`. An operator naming their own paths wins.
+{
+  lib,
+  config,
+  ...
+}:
+let
+  deployCfg = config.services.hyperhive.deploy;
+  pkiDir = import ./lib/bao-pki-dir.nix;
+in
+{
+  config = lib.mkIf (deployCfg.victorialogs.enable && deployCfg.bao.enable) {
+    services.hyperhive.deploy.victorialogs = {
+      authBaoClientCertFile = lib.mkDefault "${pkiDir}/logs-auth.pem";
+      authBaoClientKeyFile = lib.mkDefault "${pkiDir}/logs-auth-key.pem";
+    };
+  };
+}

nix/host-modules/swarm-bao.nix

@@ -964,6 +964,17 @@ let
     }
   '';
 
+  # The log store's agent auth responder, which checks the secret an agent
+  # presents against the one stored for it: the same `+` leaf as the queue
+  # responder's, and nothing else.
+  logsAuthReaders = [
+    {
+      name = "swarm-logs-auth";
+      cn = baoDeploy.logsAuthCommonName;
+      policyText = readStanza "${credentialMountPath}/data/swarm/agents/+/queue";
+    }
+  ];
+
   # The CI runner's token fetch. The one reader of the path, which the
   # controller writes and cannot read back.
   ciRunnerReaders = [
@@ -1797,6 +1808,19 @@ in
       '';
     };
 
+    logsAuthCommonName = lib.mkOption {
+      type = lib.types.str;
+      default = "swarm-logs-auth";
+      description = ''
+        Subject the store's `swarm-logs-auth` cert-auth role accepts: the
+        identity the log store's agent auth responder presents to read agent
+        queue credentials, `swarm/agents/<agent>/queue` and nothing else under
+        an agent.
+
+        ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
+      '';
+    };
+
     ciRunnerCommonName = lib.mkOption {
       type = lib.types.str;
       default = "swarm-ci-runner";
@@ -3171,6 +3195,8 @@ in
 
       systemd.services.swarm-bao-nats-auth-policy = readerPolicyUnit "write the queue responder's agent-credential-reader bao policy and cert-auth role" natsAuthReaders;
 
+      systemd.services.swarm-bao-logs-auth-policy = readerPolicyUnit "write the log store responder's agent-credential-reader bao policy and cert-auth role" logsAuthReaders;
+
       systemd.services.swarm-bao-ci-runner-policy = readerPolicyUnit "write the CI runner's registration-token-reader bao policy and cert-auth role" ciRunnerReaders;
 
       systemd.services.swarm-bao-review-classifier-policy = readerPolicyUnit "write the review classifier's API-key-reader bao policy and cert-auth role" reviewClassifierReaders;
@@ -3302,6 +3328,7 @@ in
           "swarm-bao-nats-tls-policy.service"
           "swarm-bao-hive-leaf-policy.service"
           "swarm-bao-ci-runner-policy.service"
+          "swarm-bao-logs-auth-policy.service"
           "swarm-bao-forge-publisher-policy.service"
         ]
         ++ granterAfter;

nix/host-modules/swarm-victorialogs.nix

@@ -52,6 +52,22 @@ let
     error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
   '';
 
+  # The agents' `auth_request` responder (`swarm-logs-auth`). Its socket's
+  # directory holds that socket alone and is traverse-only for others, the
+  # shape `swarm-controller.nix` gives its own.
+  logsAuthUnit = "swarm-logs-auth";
+  logsAuthSocket = "/run/${logsAuthUnit}/auth.sock";
+  logsAuthCredential = id: "/run/credentials/${logsAuthUnit}.service/${id}";
+  # Credential id → host path of the responder's store identity.
+  logsAuthStoreFiles = {
+    "bao-client.pem" = deployCfg.victorialogs.authBaoClientCertFile;
+    "bao-client-key.pem" = deployCfg.victorialogs.authBaoClientKeyFile;
+  }
+  // lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
+    "bao-ca.pem" = deployCfg.bao.serverCaFile;
+  };
+  baoCfg = hyperhiveCfg.swarm.bao;
+
   # Shared host netns, like every sibling swarm container: the collector
   # and Grafana reach this at 127.0.0.1:<port>.
   privateNetwork = false;
@@ -88,7 +104,74 @@ in
     '';
   };
 
+  options.services.hyperhive.deploy.victorialogs.authPackage = lib.mkOption {
+    type = lib.types.package;
+    defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-logs-auth";
+    description = ''
+      The `swarm-logs-auth` package: the gateway's `auth_request` responder
+      that checks an agent's `swarm-agent.<agent>.<secret>` bearer on
+      `/select/logsql/` against the secret store.
+    '';
+  };
+
+  options.services.hyperhive.deploy.victorialogs.authBaoClientCertFile = lib.mkOption {
+    type = lib.types.nullOr lib.types.str;
+    default = null;
+    example = "/var/lib/swarm-bao-pki/logs-auth.pem";
+    description = ''
+      Client certificate `swarm-logs-auth` presents to the swarm's secret store
+      to read agents' queue secrets. Its subject must be
+      {option}`services.hyperhive.deploy.bao.logsAuthCommonName`.
+
+      Required when `deploy.victorialogs.enable` is set; unset fails at eval.
+      ./glue-logs-auth-bao-identity.nix points it at the leaf
+      ./glue-bao-tls.nix mints on a host that runs the store; on any other
+      log-store host it is a file the operator places.
+
+      A path, never a value.
+    '';
+  };
+
+  options.services.hyperhive.deploy.victorialogs.authBaoClientKeyFile = lib.mkOption {
+    type = lib.types.nullOr lib.types.str;
+    default = null;
+    example = "/var/lib/swarm-bao-pki/logs-auth-key.pem";
+    description = ''
+      Private key for {option}`services.hyperhive.deploy.victorialogs.authBaoClientCertFile`.
+      A path, never a value.
+    '';
+  };
+
   config = lib.mkIf deployCfg.victorialogs.enable {
+    assertions = [
+      {
+        assertion =
+          deployCfg.victorialogs.authBaoClientCertFile != null
+          && deployCfg.victorialogs.authBaoClientKeyFile != null;
+        message = ''
+          services.hyperhive.deploy.victorialogs.authBaoClientCertFile and
+          authBaoClientKeyFile must both be set.
+
+          swarm-logs-auth reads agents' queue secrets from the swarm's secret
+          store with this certificate. They default to the leaf the store
+          mints where services.hyperhive.deploy.bao.enable is set on this
+          host; elsewhere, issue a leaf whose CN is
+          services.hyperhive.deploy.bao.logsAuthCommonName and name it in
+          both options.
+        '';
+      }
+    ];
+
+    # Which check a `/select/logsql/` request gets. `auth_request` takes a
+    # fixed URI, so it names `/__swarm_logs_auth`, which rewrites to the
+    # target this map picks from the caller's `Authorization`.
+    services.nginx.appendHttpConfig = ''
+      map $http_authorization $swarm_logs_auth_target {
+        "~^Bearer swarm-agent\." /__swarm_logs_agent_auth;
+        default /__hive_authelia;
+      }
+    '';
+
     # This store publishes its own health as prometheus metrics on the same
     # listener it serves queries on, so the swarm's collector scrapes it with
     # no exporter and no extra port — same arrangement as the metrics store.
@@ -174,12 +257,35 @@ in
           # logs. That is the rule in force, not an omission: read
           # permissions are a later thing, and this location is where one
           # attaches when it exists.
+          #
+          # Two checks: an agent's own `swarm-agent.` bearer goes to
+          # `swarm-logs-auth`, anything else to authelia (`appendHttpConfig`
+          # above picks).
           "^~ /select/logsql/" = {
             # No URI part, so the request path and its query string reach
             # the store as the caller sent them.
             proxyPass = "http://127.0.0.1:${toString cfg.port}";
             extraConfig = ''
-              auth_request /__hive_authelia;
+              auth_request /__swarm_logs_auth;
+            '';
+          };
+
+          # `rewrite … last` re-matches locations, so the subrequest lands in
+          # one of the two internal locations named by the map.
+          "= /__swarm_logs_auth" = {
+            extraConfig = ''
+              internal;
+              rewrite ^ $swarm_logs_auth_target last;
+            '';
+          };
+
+          # The `Authorization` header rides the subrequest unchanged.
+          "= /__swarm_logs_agent_auth" = {
+            proxyPass = "http://unix:${logsAuthSocket}:/";
+            extraConfig = ''
+              internal;
+              proxy_pass_request_body off;
+              proxy_set_header Content-Length "";
             '';
           };
 
@@ -207,6 +313,53 @@ in
         };
       };
 
+    # The agents' check on `/select/logsql/`. On the host rather than in the
+    # store's container: it needs nothing of the store, only nginx's reach to
+    # its socket and its own store identity.
+    systemd.services.${logsAuthUnit} = {
+      description = "swarm log store auth_request responder for agent credentials";
+      wantedBy = [ "multi-user.target" ];
+      # Where the store's PKI is minted on this host, the leaf is one of its
+      # files. Ordering only: elsewhere the unit does not exist.
+      after = [
+        "network.target"
+        "swarm-bao-pki.service"
+      ];
+      serviceConfig = {
+        ExecStart = lib.concatStringsSep " " [
+          "${deployCfg.victorialogs.authPackage}/bin/swarm-logs-auth"
+          "--socket ${logsAuthSocket}"
+          # The role ./swarm-bao.nix writes on the store's `cert` mount.
+          "--store-cert-role swarm-logs-auth"
+        ];
+        # Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
+        LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles;
+        DynamicUser = true;
+        RuntimeDirectory = logsAuthUnit;
+        # Traverse-only for others, so nginx reaches the socket path.
+        RuntimeDirectoryMode = "0751";
+        # The path never vanishes from under a running nginx; the responder
+        # unlinks a stale socket as it starts.
+        RuntimeDirectoryPreserve = "yes";
+        Restart = "on-failure";
+        RestartSec = "5s";
+        SyslogIdentifier = logsAuthUnit;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        PrivateTmp = true;
+        PrivateDevices = true;
+        NoNewPrivileges = true;
+      };
+      environment = {
+        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
+        BAO_CLIENT_CERT = logsAuthCredential "bao-client.pem";
+        BAO_CLIENT_KEY = logsAuthCredential "bao-client-key.pem";
+      }
+      // lib.optionalAttrs (logsAuthStoreFiles ? "bao-ca.pem") {
+        BAO_CACERT = logsAuthCredential "bao-ca.pem";
+      };
+    };
+
     containers.${cfg.machine} = {
       autoStart = true;
       ephemeral = false;

nix/host-modules/swarm.nix

@@ -55,6 +55,7 @@ let
     deployCfg.bao.granterCommonName
     deployCfg.bao.natsAuthCommonName
     deployCfg.bao.ciRunnerCommonName
+    deployCfg.bao.logsAuthCommonName
     deployCfg.bao.reviewClassifierCommonName
     deployCfg.bao.forgePublisherCommonName
   ]