From 1f4736e79b36b8c1a3f80f54df2eb2aba6d20139 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 12 Oct 2026 22:48:19 +0200 Subject: [PATCH] swarm-logs-auth: check agent tokens on the log store's query route A new `auth_request` responder on the log store's host. nginx sends it every `/select/logsql/` request whose `Authorization` is `Bearer swarm-agent..`; it answers 200 when the secret is the one stored at `swarm/agents//queue` (`swarm-agent-verify`), 401 otherwise. Every other bearer still goes to authelia, so agents still presenting the hive's shared OIDC client keep working. `auth_request` takes a fixed URI, so the split is a `map` on `$http_authorization` plus an internal location that rewrites to the map's pick. Checked against nginx 1.30.5 with stub upstreams: an agent token only the stub validator accepts gets 200, a token only the stub authelia accepts gets 200, every wrong one 401, and the internal locations 404 from outside. The responder serves a unix socket in its own traverse-only runtime directory and reads the store under its own leaf (`swarm-logs-auth`), whose policy is `swarm/agents/+/queue`, read-only. glue-bao-tls mints the leaf where the store runs; elsewhere the operator places it, and eval refuses a log store host without one. Part of #4459. --- CLAUDE.md | 9 + Cargo.lock | 15 ++ Cargo.toml | 1 + docs/trust-boundary/security.md | 7 + flake.nix | 3 + nix/host-modules/default.nix | 1 + nix/host-modules/glue-bao-tls.nix | 7 + .../glue-logs-auth-bao-identity.nix | 31 ++++ nix/host-modules/swarm-bao.nix | 27 +++ nix/host-modules/swarm-victorialogs.nix | 155 ++++++++++++++++- nix/host-modules/swarm.nix | 1 + nix/module-eval/bao-grants.nix | 42 ++++- nix/module-eval/core-toggle.nix | 38 +++++ nix/packages/default.nix | 4 + swarm-logs-auth/Cargo.toml | 23 +++ swarm-logs-auth/src/main.rs | 160 ++++++++++++++++++ 16 files changed, 515 insertions(+), 9 deletions(-) create mode 100644 nix/host-modules/glue-logs-auth-bao-identity.nix create mode 100644 swarm-logs-auth/Cargo.toml create mode 100644 swarm-logs-auth/src/main.rs diff --git a/CLAUDE.md b/CLAUDE.md index b91f2f1a..c38666f9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -193,6 +193,15 @@ hand-maintained per-file tree drifts out of sync with the code. `claude --print` per classification (no tools, no session) and replies with the rows and the overall verdict. Holds no forge credential. Design: [`docs/swarm/review-classifier.md`](docs/swarm/review-classifier.md). +- **`swarm-agent-verify/`** — the check that a presented + `swarm-agent..` token holds the secret stored at + `swarm/agents//queue`, read under the verifier's own certificate. + Shared by `swarm-nats-auth` (the queue's auth callout) and + `swarm-logs-auth`. +- **`swarm-logs-auth/`** — the log store gateway's `auth_request` responder + for agents: nginx sends it `/select/logsql/` requests carrying an agent + token, it answers 200 or 401 via `swarm-agent-verify`. Runs on the log + store's host (`swarm-victorialogs.nix`). - **`swarmctl/`** — swarm-level operator CLI, installed by the swarm-controller module on the host that runs the daemon. Runs as **root and acts directly** — it serves no socket, publishes no HTTP diff --git a/Cargo.lock b/Cargo.lock index 1e99c3a7..c1238e6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4884,6 +4884,21 @@ dependencies = [ "swarm-queue-client", ] +[[package]] +name = "swarm-logs-auth" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "clap", + "swarm-agent-verify", + "swarm-queue-client", + "swarm-secret-client", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "swarm-matrix-client" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 049e205c..7d8112bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ members = [ "swarm-queue-client", "swarm-review-classifier", "swarm-logs", + "swarm-logs-auth", "swarm-secret-client", "swarmctl", ] diff --git a/docs/trust-boundary/security.md b/docs/trust-boundary/security.md index 3181a3fe..b2fda6c9 100644 --- a/docs/trust-boundary/security.md +++ b/docs/trust-boundary/security.md @@ -82,6 +82,13 @@ A hive reads no **swarm service's** OIDC client secret. Every reader of collector and the store forwarder each fetch theirs with a leaf of their own, and the store's granter fetches the bao UI's. +Two services read **every agent's queue secret** +(`swarm/agents//queue`), each under a leaf of its own whose policy +reaches that one leaf per agent and nothing else: the queue's auth callout +(`swarm-nats-auth`) and the log store's agent check (`swarm-logs-auth`, on the +log store's host). Either one compromised can present any agent's queue +secret, at the queue and at the log store's query route alike. + ### Per-hive credentials live under the hive kind `swarm_secret_client::policy` scopes the **hive** stanza narrowly (unlike the diff --git a/flake.nix b/flake.nix index 974055dc..36c927d1 100644 --- a/flake.nix +++ b/flake.nix @@ -154,6 +154,9 @@ services.hyperhive.deploy.swarm-ui.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-ui; + services.hyperhive.deploy.victorialogs.authPackage = + lib.mkDefault + self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-logs-auth; services.hyperhive.deploy.nats.authPackage = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-nats-auth; diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 4960a1e7..90a46a5d 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -33,6 +33,7 @@ ./glue-forge-oidc-client.nix ./glue-grafana-oidc-client.nix ./glue-hive-tls-bao-identity.nix + ./glue-logs-auth-bao-identity.nix ./glue-matrix-ctl-bao-identity.nix ./glue-matrix-oidc-client.nix ./glue-nats-auth-bao-identity.nix diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index fc162442..64aee3a9 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -270,6 +270,13 @@ in [ -s ${pkiDir}/nats-auth.pem ] || ${signLeaf} ${pkiDir} nats-auth \ ${lib.escapeShellArg deployCfg.bao.natsAuthCommonName} "" clientAuth + # The log store's agent auth responder, which reads agent queue + # credentials. Minted here for the matrix-ctl leaf's reason: the log + # store is a swarm singleton, so elsewhere this is the file an operator + # copies. + [ -s ${pkiDir}/logs-auth.pem ] || ${signLeaf} ${pkiDir} logs-auth \ + ${lib.escapeShellArg deployCfg.bao.logsAuthCommonName} "" clientAuth + # The CI runner's token fetch. Minted here for the matrix-ctl leaf's # reason: the runner is a swarm singleton, so on a runner host without # the store this is the file an operator copies. diff --git a/nix/host-modules/glue-logs-auth-bao-identity.nix b/nix/host-modules/glue-logs-auth-bao-identity.nix new file mode 100644 index 00000000..9b39e896 --- /dev/null +++ b/nix/host-modules/glue-logs-auth-bao-identity.nix @@ -0,0 +1,31 @@ +# Glue: point the log store's agent auth responder at the bao leaf minted for it. +# +# ONE PAIRING PER FILE — the swarm-logs-auth principal ← bao, and nothing else. +# Deleting this leaves a log store with no responder identity unless the +# operator names one, which ./swarm-victorialogs.nix refuses at eval. +# +# ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the +# leaf. What belongs here is the pairing: which paths the responder presents. +# +# ⚠️ Gated on the store running on this host, the only place the leaf is +# minted. A log-store host without the store gets no default here; the +# operator places the responder's leaf and names it. +# +# Everything is `mkDefault`. An operator naming their own paths wins. +{ + lib, + config, + ... +}: +let + deployCfg = config.services.hyperhive.deploy; + pkiDir = import ./lib/bao-pki-dir.nix; +in +{ + config = lib.mkIf (deployCfg.victorialogs.enable && deployCfg.bao.enable) { + services.hyperhive.deploy.victorialogs = { + authBaoClientCertFile = lib.mkDefault "${pkiDir}/logs-auth.pem"; + authBaoClientKeyFile = lib.mkDefault "${pkiDir}/logs-auth-key.pem"; + }; + }; +} diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 2392aec7..5fd0df35 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -964,6 +964,17 @@ let } ''; + # The log store's agent auth responder, which checks the secret an agent + # presents against the one stored for it: the same `+` leaf as the queue + # responder's, and nothing else. + logsAuthReaders = [ + { + name = "swarm-logs-auth"; + cn = baoDeploy.logsAuthCommonName; + policyText = readStanza "${credentialMountPath}/data/swarm/agents/+/queue"; + } + ]; + # The CI runner's token fetch. The one reader of the path, which the # controller writes and cannot read back. ciRunnerReaders = [ @@ -1797,6 +1808,19 @@ in ''; }; + logsAuthCommonName = lib.mkOption { + type = lib.types.str; + default = "swarm-logs-auth"; + description = '' + Subject the store's `swarm-logs-auth` cert-auth role accepts: the + identity the log store's agent auth responder presents to read agent + queue credentials, `swarm/agents//queue` and nothing else under + an agent. + + ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings. + ''; + }; + ciRunnerCommonName = lib.mkOption { type = lib.types.str; default = "swarm-ci-runner"; @@ -3171,6 +3195,8 @@ in systemd.services.swarm-bao-nats-auth-policy = readerPolicyUnit "write the queue responder's agent-credential-reader bao policy and cert-auth role" natsAuthReaders; + systemd.services.swarm-bao-logs-auth-policy = readerPolicyUnit "write the log store responder's agent-credential-reader bao policy and cert-auth role" logsAuthReaders; + systemd.services.swarm-bao-ci-runner-policy = readerPolicyUnit "write the CI runner's registration-token-reader bao policy and cert-auth role" ciRunnerReaders; systemd.services.swarm-bao-review-classifier-policy = readerPolicyUnit "write the review classifier's API-key-reader bao policy and cert-auth role" reviewClassifierReaders; @@ -3302,6 +3328,7 @@ in "swarm-bao-nats-tls-policy.service" "swarm-bao-hive-leaf-policy.service" "swarm-bao-ci-runner-policy.service" + "swarm-bao-logs-auth-policy.service" "swarm-bao-forge-publisher-policy.service" ] ++ granterAfter; diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index 3f2f0d27..f01d6338 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -52,6 +52,22 @@ let error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url; ''; + # The agents' `auth_request` responder (`swarm-logs-auth`). Its socket's + # directory holds that socket alone and is traverse-only for others, the + # shape `swarm-controller.nix` gives its own. + logsAuthUnit = "swarm-logs-auth"; + logsAuthSocket = "/run/${logsAuthUnit}/auth.sock"; + logsAuthCredential = id: "/run/credentials/${logsAuthUnit}.service/${id}"; + # Credential id → host path of the responder's store identity. + logsAuthStoreFiles = { + "bao-client.pem" = deployCfg.victorialogs.authBaoClientCertFile; + "bao-client-key.pem" = deployCfg.victorialogs.authBaoClientKeyFile; + } + // lib.optionalAttrs (deployCfg.bao.serverCaFile != null) { + "bao-ca.pem" = deployCfg.bao.serverCaFile; + }; + baoCfg = hyperhiveCfg.swarm.bao; + # Shared host netns, like every sibling swarm container: the collector # and Grafana reach this at 127.0.0.1:. privateNetwork = false; @@ -88,7 +104,74 @@ in ''; }; + options.services.hyperhive.deploy.victorialogs.authPackage = lib.mkOption { + type = lib.types.package; + defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-logs-auth"; + description = '' + The `swarm-logs-auth` package: the gateway's `auth_request` responder + that checks an agent's `swarm-agent..` bearer on + `/select/logsql/` against the secret store. + ''; + }; + + options.services.hyperhive.deploy.victorialogs.authBaoClientCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/logs-auth.pem"; + description = '' + Client certificate `swarm-logs-auth` presents to the swarm's secret store + to read agents' queue secrets. Its subject must be + {option}`services.hyperhive.deploy.bao.logsAuthCommonName`. + + Required when `deploy.victorialogs.enable` is set; unset fails at eval. + ./glue-logs-auth-bao-identity.nix points it at the leaf + ./glue-bao-tls.nix mints on a host that runs the store; on any other + log-store host it is a file the operator places. + + A path, never a value. + ''; + }; + + options.services.hyperhive.deploy.victorialogs.authBaoClientKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/logs-auth-key.pem"; + description = '' + Private key for {option}`services.hyperhive.deploy.victorialogs.authBaoClientCertFile`. + A path, never a value. + ''; + }; + config = lib.mkIf deployCfg.victorialogs.enable { + assertions = [ + { + assertion = + deployCfg.victorialogs.authBaoClientCertFile != null + && deployCfg.victorialogs.authBaoClientKeyFile != null; + message = '' + services.hyperhive.deploy.victorialogs.authBaoClientCertFile and + authBaoClientKeyFile must both be set. + + swarm-logs-auth reads agents' queue secrets from the swarm's secret + store with this certificate. They default to the leaf the store + mints where services.hyperhive.deploy.bao.enable is set on this + host; elsewhere, issue a leaf whose CN is + services.hyperhive.deploy.bao.logsAuthCommonName and name it in + both options. + ''; + } + ]; + + # Which check a `/select/logsql/` request gets. `auth_request` takes a + # fixed URI, so it names `/__swarm_logs_auth`, which rewrites to the + # target this map picks from the caller's `Authorization`. + services.nginx.appendHttpConfig = '' + map $http_authorization $swarm_logs_auth_target { + "~^Bearer swarm-agent\." /__swarm_logs_agent_auth; + default /__hive_authelia; + } + ''; + # This store publishes its own health as prometheus metrics on the same # listener it serves queries on, so the swarm's collector scrapes it with # no exporter and no extra port — same arrangement as the metrics store. @@ -174,12 +257,35 @@ in # logs. That is the rule in force, not an omission: read # permissions are a later thing, and this location is where one # attaches when it exists. + # + # Two checks: an agent's own `swarm-agent.` bearer goes to + # `swarm-logs-auth`, anything else to authelia (`appendHttpConfig` + # above picks). "^~ /select/logsql/" = { # No URI part, so the request path and its query string reach # the store as the caller sent them. proxyPass = "http://127.0.0.1:${toString cfg.port}"; extraConfig = '' - auth_request /__hive_authelia; + auth_request /__swarm_logs_auth; + ''; + }; + + # `rewrite … last` re-matches locations, so the subrequest lands in + # one of the two internal locations named by the map. + "= /__swarm_logs_auth" = { + extraConfig = '' + internal; + rewrite ^ $swarm_logs_auth_target last; + ''; + }; + + # The `Authorization` header rides the subrequest unchanged. + "= /__swarm_logs_agent_auth" = { + proxyPass = "http://unix:${logsAuthSocket}:/"; + extraConfig = '' + internal; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; ''; }; @@ -207,6 +313,53 @@ in }; }; + # The agents' check on `/select/logsql/`. On the host rather than in the + # store's container: it needs nothing of the store, only nginx's reach to + # its socket and its own store identity. + systemd.services.${logsAuthUnit} = { + description = "swarm log store auth_request responder for agent credentials"; + wantedBy = [ "multi-user.target" ]; + # Where the store's PKI is minted on this host, the leaf is one of its + # files. Ordering only: elsewhere the unit does not exist. + after = [ + "network.target" + "swarm-bao-pki.service" + ]; + serviceConfig = { + ExecStart = lib.concatStringsSep " " [ + "${deployCfg.victorialogs.authPackage}/bin/swarm-logs-auth" + "--socket ${logsAuthSocket}" + # The role ./swarm-bao.nix writes on the store's `cert` mount. + "--store-cert-role swarm-logs-auth" + ]; + # Paths only: `argv` is world-readable via /proc//cmdline. + LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles; + DynamicUser = true; + RuntimeDirectory = logsAuthUnit; + # Traverse-only for others, so nginx reaches the socket path. + RuntimeDirectoryMode = "0751"; + # The path never vanishes from under a running nginx; the responder + # unlinks a stale socket as it starts. + RuntimeDirectoryPreserve = "yes"; + Restart = "on-failure"; + RestartSec = "5s"; + SyslogIdentifier = logsAuthUnit; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + NoNewPrivileges = true; + }; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = logsAuthCredential "bao-client.pem"; + BAO_CLIENT_KEY = logsAuthCredential "bao-client-key.pem"; + } + // lib.optionalAttrs (logsAuthStoreFiles ? "bao-ca.pem") { + BAO_CACERT = logsAuthCredential "bao-ca.pem"; + }; + }; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index a92479d1..4479321f 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -55,6 +55,7 @@ let deployCfg.bao.granterCommonName deployCfg.bao.natsAuthCommonName deployCfg.bao.ciRunnerCommonName + deployCfg.bao.logsAuthCommonName deployCfg.bao.reviewClassifierCommonName deployCfg.bao.forgePublisherCommonName ] diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 793c8a79..0fe075a1 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -169,7 +169,7 @@ let _: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile ) baoGrantWithConsumers.systemd.services; - # The sixteen units that write a `swarm-*` grant, by name, for the discovery + # The seventeen units that write a `swarm-*` grant, by name, for the discovery # control below. grantingUnitNames = [ "swarm-bao-controller-policy" @@ -186,6 +186,7 @@ let "swarm-bao-operator-viewer-policy" "swarm-bao-hive-leaf-policy" "swarm-bao-ci-runner-policy" + "swarm-bao-logs-auth-policy" "swarm-bao-forge-publisher-policy" "swarm-bao-review-classifier-policy" ]; @@ -704,6 +705,31 @@ let in lib.hasInfix "/nats-auth.pem ]" s && lib.hasInfix "swarm-nats-auth \"\" clientAuth" s; } + { + # The same `+` leaf as the queue responder, read-only, and nothing else: + # this principal checks a presented secret and does nothing with it. + name = "the log store responder's grant is every agent's queue secret, read-only, and nothing else"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-logs-auth-policy.script; + in + lib.hasInfix "path \"secret/data/swarm/agents/+/queue\" {\n capabilities = [\"read\"]\n}" s + && lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 1 + && !(lib.hasInfix "secret/data/swarm/agents/*" s) + && !(lib.hasInfix "secret/data/swarm/hives" s) + && !(lib.hasInfix "secret/data/swarm/services" s) + && lib.hasInfix "auth/cert/certs/swarm-logs-auth" s + && lib.hasInfix "allowed_common_names=swarm-logs-auth" s + && lib.hasInfix "token_policies=swarm-logs-auth" s; + } + { + name = "the PKI unit signs the log store responder's leaf under its own subject"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-pki.script; + in + lib.hasInfix "/logs-auth.pem ]" s && lib.hasInfix "swarm-logs-auth \"\" clientAuth" s; + } { # 🩸 The half that makes the policies above bind: a policy grants only # through a token that carries it, and a token is minted by a cert-auth @@ -915,24 +941,24 @@ let } { # A store host without the granter's pair writes its grants some other - # way, so none of the sixteen units may exist. Without this arm + # way, so none of the seventeen units may exist. Without this arm # `lib.mkIf haveGranter` could be dropped from any of them and every other # case here would still pass. - name = "without the granter's pair none of the sixteen granting units render"; + name = "without the granter's pair none of the seventeen granting units render"; ok = let s = baoGranterOptOut.systemd.services; in lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ]) - # The control: the same store with the pair renders all sixteen. + # The control: the same store with the pair renders all seventeen. && lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames; } { - # 🩸 What replaced the silent skip. With no bootstrap token the sixteen still + # 🩸 What replaced the silent skip. With no bootstrap token the seventeen still # render, and a refused granter fails them with the step that fixes it. # A store host that never named a token is told to name one, since the # unit that sets the granter up renders only where it has. - name = "a store host without a bootstrap token renders the sixteen, each failing loudly with the one-time step"; + name = "a store host without a bootstrap token renders the seventeen, each failing loudly with the one-time step"; ok = let s = baoGranterNoToken.systemd.services; @@ -1695,8 +1721,8 @@ let } { # What makes the case above mean something: discovery by the granter's - # certificate reaches all sixteen units, and each yields calls. - name = "the granter-policy check sees all sixteen granting units, and parses calls from each"; + # certificate reaches all seventeen units, and each yields calls. + name = "the granter-policy check sees all seventeen granting units, and parses calls from each"; ok = lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits) diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 1eb84e82..522e4629 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -504,6 +504,44 @@ let && !(lib.hasInfix "$args" q.extraConfig) && !(lib.hasInfix "?" q.proxyPass); } + { + # `auth_request` takes a fixed URI, so the split between the agents' + # responder and authelia is a map plus a rewrite in an internal + # location. Every link of that chain is pinned: the map keys on the + # agent-token prefix and falls back to authelia, the query location + # names the switch, the switch rewrites to the map's pick, and the + # agent target reaches the responder's socket, which the unit binds. + name = "the machine query location sends agent bearers to swarm-logs-auth and the rest to authelia"; + ok = + let + l = allLocal.services.nginx.virtualHosts."logs.t.local".locations; + http = allLocal.services.nginx.appendHttpConfig; + unit = allLocal.systemd.services.swarm-logs-auth; + in + lib.hasInfix "map \$http_authorization \$swarm_logs_auth_target" http + && lib.hasInfix "\"~^Bearer swarm-agent\\.\" /__swarm_logs_agent_auth;" http + && lib.hasInfix "default /__hive_authelia;" http + && lib.hasInfix "auth_request /__swarm_logs_auth;" l."^~ /select/logsql/".extraConfig + && lib.hasInfix "rewrite ^ \$swarm_logs_auth_target last;" l."= /__swarm_logs_auth".extraConfig + && lib.hasInfix "internal;" l."= /__swarm_logs_agent_auth".extraConfig + && l."= /__swarm_logs_agent_auth".proxyPass == "http://unix:/run/swarm-logs-auth/auth.sock:/" + && lib.hasInfix "--socket /run/swarm-logs-auth/auth.sock" unit.serviceConfig.ExecStart + && l ? "= /__hive_authelia"; + } + { + # The responder reads the store as itself, with the leaf the store's + # host mints for it. + name = "swarm-logs-auth presents its own store leaf"; + ok = + let + v = allLocal.services.hyperhive.deploy.victorialogs; + lc = allLocal.systemd.services.swarm-logs-auth.serviceConfig.LoadCredential; + in + lib.hasSuffix "/logs-auth.pem" v.authBaoClientCertFile + && lib.hasSuffix "/logs-auth-key.pem" v.authBaoClientKeyFile + && builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc + && builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc; + } { # The absence arm for the case above, and the option's own rule — a # service declares its entry under its own `enable` — made checkable. diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 072e2055..ba9b8091 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -163,6 +163,10 @@ in # rather than every hive's. swarm-nats-auth = mkBinPackage "swarm-nats-auth" "hyperhive swarm queue auth-callout responder"; + # The log store gateway's `auth_request` responder for agents. Out of + # `daemonBins` like the responder above: it runs on the log store's host only. + swarm-logs-auth = mkBinPackage "swarm-logs-auth" "hyperhive swarm log store agent auth responder"; + # The matrix container's control binary; `mint` publishes the sender token. Out of `daemonBins` for # "runs *inside* a container, not on the host" reason as the responder # above, and with a second one: putting it in the core bundle would place diff --git a/swarm-logs-auth/Cargo.toml b/swarm-logs-auth/Cargo.toml new file mode 100644 index 00000000..ca6bdcfe --- /dev/null +++ b/swarm-logs-auth/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "swarm-logs-auth" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "swarm-logs-auth" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +axum.workspace = true +clap.workspace = true +swarm-agent-verify.workspace = true +# The agent-token spelling, and the rustls provider the store client needs. +swarm-queue-client.workspace = true +swarm-secret-client.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +[lints] +workspace = true diff --git a/swarm-logs-auth/src/main.rs b/swarm-logs-auth/src/main.rs new file mode 100644 index 00000000..a141492a --- /dev/null +++ b/swarm-logs-auth/src/main.rs @@ -0,0 +1,160 @@ +//! The log store gateway's `auth_request` responder for agents. +//! +//! nginx sends it every `/select/logsql/` request whose `Authorization` is +//! `Bearer swarm-agent..` (`swarm-victorialogs.nix`). It answers +//! 200 when the secret is the one stored at `swarm/agents//queue`, and +//! 401 otherwise, store failures included: `auth_request` turns any other +//! status into a 500 for the caller. +//! +//! It reads the store under its own certificate, whose policy reaches every +//! agent's `queue` secret and nothing else. It serves a unix socket whose +//! directory only the gateway's nginx may enter. + +use std::os::unix::fs::PermissionsExt as _; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Context as _; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode, header}; +use clap::Parser; +use swarm_agent_verify::CredentialSource; +use swarm_queue_client::agent_token::parse_agent_token; + +/// How long one store lookup may take before the request is denied. Well +/// under nginx's 60s `proxy_read_timeout`, so a slow store reads as a 401 +/// rather than a gateway timeout. +const LOOKUP_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Parser)] +#[command(about = "auth_request responder: checks an agent's queue secret against the store")] +struct Args { + /// Unix socket to serve on. + #[arg(long)] + socket: PathBuf, + /// Cert-auth role this responder logs in to the store as. + #[arg(long, default_value = "swarm-logs-auth")] + store_cert_role: String, +} + +/// The answer for one subrequest's `Authorization` header. +async fn decide(source: &impl CredentialSource, authorization: Option<&str>) -> StatusCode { + let Some(token) = authorization.and_then(|v| v.strip_prefix("Bearer ")) else { + tracing::warn!("no bearer token; denying"); + return StatusCode::UNAUTHORIZED; + }; + let token = match parse_agent_token(token) { + Some(Ok(token)) => token, + Some(Err(e)) => { + tracing::warn!(error = %e, "malformed agent token; denying"); + return StatusCode::UNAUTHORIZED; + } + None => { + tracing::warn!("not an agent token; denying"); + return StatusCode::UNAUTHORIZED; + } + }; + if swarm_agent_verify::verify(source, &token, LOOKUP_TIMEOUT).await { + tracing::info!(agent = token.agent, "verified"); + StatusCode::OK + } else { + StatusCode::UNAUTHORIZED + } +} + +async fn handle( + State(store): State>, + headers: HeaderMap, +) -> StatusCode { + let authorization = headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()); + decide(store.as_ref(), authorization).await +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + swarm_queue_client::install_crypto_provider(); + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + // stdout is journald, which keeps ANSI escapes as raw bytes. + .with_ansi(false) + .init(); + let args = Args::parse(); + + let settings = swarm_secret_client::client::Settings::from_env() + .context("reading the secret store's address and this responder's identity")?; + let store = Arc::new(swarm_agent_verify::Store::new( + settings, + args.store_cert_role, + )); + + // `RuntimeDirectoryPreserve=yes` can leave the previous run's socket; the + // directory is this unit's alone, so unlinking it is safe. + if let Err(e) = std::fs::remove_file(&args.socket) + && e.kind() != std::io::ErrorKind::NotFound + { + return Err(e) + .with_context(|| format!("clearing stale socket at {}", args.socket.display())); + } + let listener = tokio::net::UnixListener::bind(&args.socket) + .with_context(|| format!("binding {}", args.socket.display()))?; + // nginx is another user and `connect(2)` needs write. The directory is + // the access control, not this mode. + std::fs::set_permissions(&args.socket, std::fs::Permissions::from_mode(0o666)) + .with_context(|| format!("chmod {}", args.socket.display()))?; + tracing::info!(socket = %args.socket.display(), "swarm-logs-auth listening"); + + let app = axum::Router::new().fallback(handle).with_state(store); + axum::serve(listener, app).await.context("serving") +} + +#[cfg(test)] +mod tests { + use super::*; + use swarm_secret_client::queue::AgentCredential; + + /// A store holding atlas's credential. + struct Atlas; + + impl CredentialSource for Atlas { + async fn lookup(&self, agent: &str) -> anyhow::Result> { + Ok((agent == "atlas").then(|| AgentCredential { + value: "Ab9_-zSECRET".to_owned(), + agent: "atlas".to_owned(), + minted_at: None, + })) + } + } + + #[tokio::test] + async fn the_stored_secret_is_200() { + assert_eq!( + decide(&Atlas, Some("Bearer swarm-agent.atlas.Ab9_-zSECRET")).await, + StatusCode::OK + ); + } + + #[tokio::test] + async fn everything_else_is_401() { + for authorization in [ + None, + Some("swarm-agent.atlas.Ab9_-zSECRET"), + Some("Basic swarm-agent.atlas.Ab9_-zSECRET"), + Some("Bearer swarm-agent.atlas.Ab9_-zSECREX"), + Some("Bearer swarm-agent.argus.Ab9_-zSECRET"), + Some("Bearer swarm-agent.atlas"), + Some("Bearer authelia_at_abc.def"), + ] { + assert_eq!( + decide(&Atlas, authorization).await, + StatusCode::UNAUTHORIZED, + "{authorization:?}" + ); + } + } +}