# `checks.module-eval-secret-publisher` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, shared, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive runGroup ; # The IdP and the store on one machine: the shape where minted plaintext and # a store identity are both present without an operator placing anything. # Two hives in the roster, because the mint walks it — an arm written # against a single-hive fixture passes on a hardcoded name. secretPublisherHere = hive { deploy.bao.enable = true; deploy.authelia.enable = true; swarm.hives.h2.domain = "h2.t.local"; }; # The IdP with no store on the box and a leaf placed by hand, which is the # deployment this unit exists for: authelia is the one host the store is # guaranteed not to share once either has a machine of its own. # # ⚠️ `enable` is deliberately NOT set here: leaving it unset is what makes # this fixture exercise the default rather than mask it. secretPublisherRemote = hive { deploy.authelia.enable = true; deploy.swarm-secret-publisher.baoClientCertFile = "/etc/pki/publisher.pem"; deploy.swarm-secret-publisher.baoClientKeyFile = "/etc/pki/publisher-key.pem"; }; # The same IdP with the identity taken away. Minting the secrets is not being # able to publish them, and this is the arm that separates the two. secretPublisherNoIdentity = shared.autheliaOnly; # Duplicated from grafana.nix — a case here needs it too. # The same UI with the IdP on ANOTHER host and a store leaf placed by hand. # Knowing an IdP is not running one: `swarm.authelia.url` is what says this # swarm has SSO, and nothing about this host does. Identical to the fixture # above in everything the delivery path reads, which is the point. grafanaRemoteAuthelia = hive { deploy.grafana.enable = true; deploy.grafana.plugins = [ ]; deploy.grafana.package = pkgs.emptyDirectory; swarm.authelia.url = "https://auth.example.invalid"; deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; }; # The same collector with the IdP on ANOTHER host and a store leaf placed by # hand. Identical to the fixture above in everything the delivery path # reads, which is the point. otelBaoRemoteAuthelia = shared.otelReaderMissingLeaf; inherit (shared) bare; # The store and a service that reads from it, versus the store alone. The # pair is what makes the reader's absence arm mean anything. inherit (shared) baoWithMatrix; # The mint runs inside authelia's container. inAuthelia = m: m.containers.${m.services.hyperhive.swarm.authelia.machine}.config; cases = [ { # Each hive's hive-c0re reads this exact path # (`swarm_secret_client::queue::hive_client_path`, pinned by that crate's # own test), for every hive in the roster, not only the one this host is. name = "the mint stores every hive's own client under that hive's prefix"; ok = let s = (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script; in lib.all (h: lib.hasInfix "mint_client_secret hive-${h} secret/swarm/hives/${h}/oidc/client" s) [ "h1" "h2" ]; } { # The controller reads this exact path # (`swarm_secret_client::queue::controller_client_path`, pinned by that # crate's own test). Under `controller/`, which no hive reads: this # client may read every hive's status. name = "the mint stores the controller's client under controller/, never services/"; ok = let s = (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script; in lib.hasInfix "secret/swarm/controller/swarm-controller/oidc/client" s && lib.hasInfix "mint_client_secret swarm-controller secret/swarm/controller/swarm-controller/oidc/client" s && !(lib.hasInfix "secret/swarm/services/swarm-controller" s); } { # Both ends of a wire nothing at eval time carries end to end: the # mint in authelia's container writes the path the reader on # Grafana's host reads, and the two files agree only because both compose # it from the same swarm-wide client id. name = "the mint stores the swarm service path grafana reads"; ok = lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" ( (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script ); } { # Registering the client cannot live where the rest of grafana's module # lives: that block is gated on this host RUNNING grafana, so on the split # deployment nothing registered the client, authelia minted no secret, and # every layer below had nothing to carry. The second arm is the control — # a host with no IdP registers nothing. name = "the swarm's grafana client is registered wherever authelia runs"; ok = let clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients; in lib.elem "swarm-grafana" (clients secretPublisherHere) && !(lib.elem "swarm-grafana" (clients grafanaRemoteAuthelia)); } { # The collector's half of the same property: the client is not gated on # `deploy.swarm-otel.enable`, which would let a split deployment register # it nowhere, so authelia would mint nothing to publish. name = "the swarm's collector client is registered wherever authelia runs"; ok = let clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients; in lib.elem "swarm-collector" (clients secretPublisherHere) && !(lib.elem "swarm-collector" (clients otelBaoRemoteAuthelia)); } { # Registration follows authelia, not the service: the IdP host below runs # no controller, queue or homeserver (the first arms), and still lists all # three clients. The last arm is the control — a host with no IdP lists # none of them. name = "an authelia host running no controller, queue or homeserver registers all three clients"; ok = let d = secretPublisherRemote.services.hyperhive.deploy; clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients; byId = id: lib.findFirst ( c: c.id == id ) null secretPublisherRemote.services.hyperhive.swarm.authelia.oidc.clients; controller = byId "swarm-controller"; nats = byId "swarm-nats"; matrix = byId "tuwunel"; in !d.swarm-controller.enable && !d.nats.enable && !d.matrix.enable && controller.kind == "machine" && controller.audience == [ "swarm-controller" ] && nats.kind == "machine" && matrix.redirectUris == [ "https://chat.t.local/_matrix/client/unstable/login/sso/callback/tuwunel" ] && matrix.tokenEndpointAuthMethod == "client_secret_post" && !(lib.any (id: lib.elem id (clients grafanaRemoteAuthelia)) [ "swarm-controller" "swarm-nats" "tuwunel" ]); } { # The other half of the queue's, the homeserver's and the forge's one # route: the mint on an IdP host that runs none of them stores each # secret to the path its reader fetches. name = "the mint on an authelia host stores the queue's, homeserver's and forge's secrets"; ok = let s = (inAuthelia secretPublisherRemote).systemd.services.authelia-swarm-secrets.script; in lib.hasInfix "mint_client_secret swarm-nats secret/swarm/services/swarm-nats/oidc/client" s && lib.hasInfix "mint_client_secret tuwunel secret/swarm/services/tuwunel/oidc/client" s && !secretPublisherRemote.services.hyperhive.deploy.forgejo.enable && lib.hasInfix "mint_client_secret forgejo secret/swarm/services/forgejo/oidc/client" s; } { # Both ends of a wire nothing at eval time carries end to end: the # mint in authelia's container writes the path the reader on the # collector's host reads, and the two files agree only because both # compose it from the same swarm-wide client id. `secretPublisherHere` # already grew this client when `serviceClientIds` did. name = "the mint stores the swarm service path the collector reads"; ok = lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" ( (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script ); } { # The third swarm service on that list, and the one whose absence is # hardest to see: the store's own forwarder holds a client authelia # registers and mints for, so every layer above looks complete while # the one hop that CARRIES the value skips it and the reader on the # store's host waits forever on a path nothing writes. name = "the mint stores the store forwarder's secret too"; ok = lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" ( (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script ); } { # The forwarders in the CI runner's, Grafana's, the log store's, the # forge's, the homeserver's and authelia's containers read their secrets # from these paths, and the metrics store's host reads its container's. name = "the mint stores the CI runner, Grafana, log store, cache, forge, homeserver, metrics store and authelia forwarders' secrets"; ok = lib.all ( id: lib.hasInfix "secret/swarm/services/${id}/oidc/client" ( (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script ) ) [ "swarm-ci-runner-collector" "swarm-grafana-collector" "swarm-victorialogs-collector" "swarm-forge-collector" "swarm-matrix-collector" "swarm-victoriametrics-collector" "swarm-authelia-collector" "swarm-attic-collector" ]; } { # The review classifier reads its queue client secret from the store # inside its container; authelia registers it on every IdP host. name = "the review classifier's client is registered as a machine, and the mint stores its secret"; ok = let c = lib.findFirst ( c: c.id == "swarm-review-classifier" ) null secretPublisherHere.services.hyperhive.swarm.authelia.oidc.clients; in c != null && c.kind == "machine" && lib.hasInfix "secret/swarm/services/swarm-review-classifier/oidc/client" ( (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script ); } { # The store's UI login: authelia registers the client with the callback # the store's `oidc` role allows, and the mint stores its secret # to the path `swarm-bao-operator-viewer-policy` reads. Not registered # where authelia is not. name = "the store UI's client is registered interactive at its callback, and the mint stores its secret"; ok = let ui = lib.findFirst ( c: c.id == "swarm-bao-ui" ) null secretPublisherHere.services.hyperhive.swarm.authelia.oidc.clients; in ui != null && ui.kind == "interactive" && ui.redirectUris == [ "https://bao-ui.t.local/ui/vault/auth/oidc/oidc/callback" ] && lib.hasInfix "secret/swarm/services/swarm-bao-ui/oidc/client" ( (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script ) && !(lib.any ( c: c.id == "swarm-bao-ui" ) grafanaRemoteAuthelia.services.hyperhive.swarm.authelia.oidc.clients); } { # The same hole the controller's case above names, open a second time: the # PKI script grew a third leaf and no case read it. name = "the store mints a leaf for the secret publisher, and the publisher is pointed at it"; ok = let m = secretPublisherHere; p = m.services.hyperhive.deploy.swarm-secret-publisher; in lib.hasInfix "secret-publisher.pem" m.systemd.services.swarm-bao-pki.script && p.baoClientCertFile == "/var/lib/swarm-bao-pki/secret-publisher.pem" && p.baoClientKeyFile == "/var/lib/swarm-bao-pki/secret-publisher-key.pem"; } { # mara caught this by reading, which means no arm existed for it: the # default asked `authelia.enable && bao.enable`, so the split deployment # this unit is FOR defaulted off and published nothing, silently. The # default is set by ../host-modules/glue-secret-publisher-authelia.nix. # # The second clause is the control. Without it this passes on a default # of plain `true`, which would be a different bug with the same symptom # — an IdP-less host claiming it publishes secrets it never mints. name = "the publisher defaults on where secrets are minted, whether or not the store is local"; ok = secretPublisherRemote.services.hyperhive.deploy.swarm-secret-publisher.enable && !bare.services.hyperhive.deploy.swarm-secret-publisher.enable; } { # The one security property of the mint's write: `bao` is an external # binary, so an argument is world-readable in /proc for the life of the # call. `value=-` makes bao read the value from stdin. # # The arms below the real script are what make it mean anything: a # `value=-` elsewhere in the text can sit beside a put that passes the # plaintext as an argument. # # ⚠️ Comments are stripped first, and that is not tidiness. A `script` # renders its own comments into the text, and a comment can name the # hazard verbatim. Without the strip this case would read that warning # and fail — a check the artifact defeats by DESCRIBING the thing it is # checked for. name = "the mint hands bao the secret on stdin, never as an argument"; ok = let code = s: lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s); puts = s: lib.filter (l: lib.hasInfix "kv put" l) (code s); onStdin = s: puts s != [ ] && lib.all (l: lib.hasInfix "value=-" l) (puts s); in onStdin (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script && !(onStdin "printf '%s' \"$secret\" | bao kv put -cas=0 x value=\"$secret\"") && !(onStdin "# kv put x value=-\nbao kv put x value=@/tmp/f") && onStdin "printf '%s' \"$secret\" | bao kv put -cas=0 x value=-"; } { # No hive holds a shared queue credential for its agents, so authelia # mints no agent client and the mint stores no such path. The # control: the same script stores each hive's own client. name = "the mint stores no hive-shared agent queue credential"; ok = let s = (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script; in !(lib.hasInfix "/queue/agent" s) && !(lib.hasInfix "hive-h1-agent" s) && lib.hasInfix "mint_client_secret hive-h1 secret/swarm/hives/h1/oidc/client" s; } { # A property of the SET, not of one unit: both of these authenticate by # certificate, and `BAO_CLIENT_CERT` is transport rather than identity, so # a script that reaches `bao kv` without a token asks a token helper this # host does not carry and fails before the store ever answers. `-token-only` # is what keeps the token off the helper on the way back out. # # Ordering, not presence: the login has to come first, so the check is # that nothing before it is a data command. Comments are stripped because # both units explain this in prose directly above the code. name = "the cert-identity bao units log in before their first read or write, and keep the token out of the helper"; ok = let code = s: lib.concatStringsSep "\n" ( lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s) ); holdsTokenFirst = s: let c = code s; in lib.hasInfix "bao login" c && lib.hasInfix "-token-only" c && !(lib.hasInfix "bao kv" (lib.head (lib.splitString "bao login" c))); in holdsTokenFirst (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script && holdsTokenFirst (inAuthelia secretPublisherHere).systemd.services.swarm-authelia-bridge.preStart && holdsTokenFirst baoWithMatrix.systemd.services.swarm-bao-matrix-oidc.script # Controls, so a clean verdict above means something. In order: a bare # read is refused, a read placed before the login is refused, and a # login that exists only in a comment is refused — that last one is the # arm the comment-stripping exists for. && !(holdsTokenFirst "bao kv get -field=value secret/x") && !(holdsTokenFirst "bao kv get secret/x\nBAO_TOKEN=\"$(bao login -method=cert -token-only)\"") && !(holdsTokenFirst "# bao login -method=cert -token-only goes here\nbao kv get secret/x") && holdsTokenFirst "BAO_TOKEN=\"$(bao login -method=cert -token-only)\"\nbao kv get secret/x"; } { # The doctrine three glue files state, as a property a rewrite has to # keep: a client is defined by holding a certificate the store accepts, # never by standing next to the store. Gating this on `deploy.bao.enable` # would have left the unit rendering only on the one deployment that has # no use for it. It runs beside the mint, in authelia's container, and # never on the host. name = "the mint holding an identity runs in authelia's container on a host with no store"; ok = let m = secretPublisherRemote; c = inAuthelia m; in !m.services.hyperhive.deploy.bao.enable && lib.hasInfix "mint_client_secret" c.systemd.services.authelia-swarm-secrets.script && !(m.systemd.services ? authelia-swarm-secrets) # Control: the host's own unit set is the one read above. && (m.systemd.services ? "container@swarm-authelia"); } { # What makes the arm above able to fail. Without a certificate the # plaintext has no store to go to, so the mint keeps it in a file beside # the digest as before, the bridge reads # that file, and nspawn gets no credential to load. name = "an IdP host with no store identity keeps the client secrets in files"; ok = let m = secretPublisherNoIdentity; c = inAuthelia m; mint = c.systemd.services.authelia-swarm-secrets.script; in m.services.hyperhive.deploy.authelia.enable && lib.hasInfix "mint() {" mint && !(lib.hasInfix "mint_client_secret" mint) && c.systemd.services.swarm-authelia-bridge.environment.SWARM_AUTHELIA_BRIDGE_CLIENT_SECRET_FILE == "/var/lib/authelia-swarm/oidc-clients/swarm-authelia-bridge.secret" && !(c.systemd.services.swarm-authelia-bridge.serviceConfig ? RuntimeDirectory) && !(lib.any (lib.hasPrefix "--load-credential=") m.containers.swarm-authelia.extraFlags) # Controls: with an identity, the same lookups find the other shape. && lib.hasInfix "mint_client_secret" ( (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script ) && lib.any (lib.hasPrefix "--load-credential=") secretPublisherHere.containers.swarm-authelia.extraFlags; } { # With a store identity no unit writes a plaintext client secret under # /var/lib: the mint keeps no `$sec` file, no redirect in the container # writes one, and the bridge reads its secret from a file in /run that it # fetches from the store before each start. The no-identity host is the # control: the same lookups find the write there. name = "no unit writes a plaintext oidc client secret to disk, and the bridge's source is the store"; ok = let writesPlaintext = m: lib.any (u: lib.hasInfix "> \"$sec\"" (u.script or "")) ( lib.attrValues (inAuthelia m).systemd.services ); here = inAuthelia secretPublisherHere; bridge = here.systemd.services.swarm-authelia-bridge; leaf = "secret/swarm/services/swarm-authelia-bridge/oidc/client"; in !(writesPlaintext secretPublisherHere) && writesPlaintext secretPublisherNoIdentity && bridge.environment.SWARM_AUTHELIA_BRIDGE_CLIENT_SECRET_FILE == "/run/swarm-authelia-bridge/client-secret" && bridge.serviceConfig.RuntimeDirectory == "swarm-authelia-bridge" && lib.hasInfix "client_secret_read ${leaf}" bridge.preStart && lib.hasInfix "mint_client_secret swarm-authelia-bridge ${leaf}" here.systemd.services.authelia-swarm-secrets.script && lib.elem "publisher-bao-cert" bridge.serviceConfig.LoadCredential && bridge.serviceConfig.Restart == "on-failure"; } { # Every non-native client is minted at the leaf its reader # fetches, not only the ones with a reader in the publisher's list: the # bridge's own, which only the bridge reads, is the one that list lacks. name = "the mint stores every client at the leaf its reader fetches"; ok = let c = inAuthelia secretPublisherHere; mint = c.systemd.services.authelia-swarm-secrets; pairs = [ "hive-h1 secret/swarm/hives/h1/oidc/client" "hive-h2 secret/swarm/hives/h2/oidc/client" "swarm-grafana secret/swarm/services/swarm-grafana/oidc/client" "swarm-controller secret/swarm/controller/swarm-controller/oidc/client" "swarm-authelia-bridge secret/swarm/services/swarm-authelia-bridge/oidc/client" ]; in lib.all (p: lib.hasInfix "mint_client_secret ${p}" mint.script) pairs # A native client is public: it has no secret to mint. && !(lib.hasInfix "mint_client_secret swarmctl" mint.script) # The mint answers authelia's password prompt through a pty. && lib.any (p: lib.hasPrefix "util-linux" (p.name or "")) mint.path; } { # Every key authelia signs or encrypts with comes from the store, seeded # from the file it was generated into, and lands at the path the config # already names. The storage-encryption key encrypts authelia's database, # so a value that moved would leave it unreadable. With no identity the # unit still generates the files, which is the control for each absence: # the generator and the store sequence are found by the same lookups. name = "authelia's keys are read from the store with the publisher's identity, and generated on disk without one"; ok = let unit = m: (inAuthelia m).systemd.services.authelia-swarm-secrets; viaStore = unit secretPublisherHere; onDisk = unit secretPublisherNoIdentity; in lib.hasInfix "store_seeded secret/swarm/services/swarm-authelia/keys/\"$name\" \"$p\" \"$@\"" viaStore.script && lib.hasInfix "p=/var/lib/authelia-swarm/\"$name\".key" viaStore.script && lib.hasInfix "keep_key oidc-issuer mint_issuer" viaStore.script && lib.hasInfix "for f in jwt session storage-encryption oidc-hmac; do\n keep_key \"$f\" mint_hex" viaStore.script && !(lib.hasInfix "genrsa -out" viaStore.script) && viaStore.environment.BAO_CLIENT_CERT == "/run/credentials/authelia-swarm-secrets.service/publisher-bao-cert" && lib.elem "publisher-bao-cert" viaStore.serviceConfig.LoadCredential && viaStore.serviceConfig.Restart == "on-failure" && !(lib.hasInfix "store_seeded" onDisk.script) && lib.hasInfix "genrsa -out" onDisk.script && lib.hasInfix "if [ ! -s \"$p\" ]; then" onDisk.script && !(onDisk.environment ? BAO_ADDR) && !(onDisk.serviceConfig ? LoadCredential); } { # The leaf crosses into the container as nspawn system credentials, and # the unit loads them and names them in its environment. On the store's # host the container waits for the unit that mints the leaf, since nspawn # refuses a missing credential source; with the leaf placed by hand, # those paths cross. name = "the store identity reaches authelia's container as system credentials"; ok = let dir = "/run/credentials/authelia-swarm-secrets.service"; unit = m: (inAuthelia m).systemd.services.authelia-swarm-secrets; flags = m: m.containers.swarm-authelia.extraFlags; here = unit secretPublisherHere; in lib.elem "--load-credential=publisher-bao-cert:/var/lib/swarm-bao-pki/secret-publisher.pem" ( flags secretPublisherHere ) && lib.elem "--load-credential=publisher-bao-key:/var/lib/swarm-bao-pki/secret-publisher-key.pem" ( flags secretPublisherHere ) && lib.elem "--load-credential=publisher-bao-server-ca:/var/lib/swarm-bao-pki/ca.pem" ( flags secretPublisherHere ) && lib.sort lib.lessThan here.serviceConfig.LoadCredential == [ "publisher-bao-cert" "publisher-bao-key" "publisher-bao-server-ca" ] && here.environment.BAO_CLIENT_CERT == "${dir}/publisher-bao-cert" && here.environment.BAO_CLIENT_KEY == "${dir}/publisher-bao-key" && here.environment.BAO_CACERT == "${dir}/publisher-bao-server-ca" && lib.elem "swarm-bao-pki.service" secretPublisherHere.systemd.services."container@swarm-authelia".after && lib.elem "--load-credential=publisher-bao-cert:/etc/pki/publisher.pem" ( flags secretPublisherRemote ) && lib.elem "--load-credential=publisher-bao-key:/etc/pki/publisher-key.pem" ( flags secretPublisherRemote ) && (unit secretPublisherRemote).environment.BAO_CLIENT_CERT == "${dir}/publisher-bao-cert"; } ]; in runGroup "secret-publisher" cases