From c92bb0dce77997e20985d76bc2c6572ffe16fe8e Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 24 Sep 2026 13:07:18 +0200 Subject: [PATCH 1/3] nix: order each swarm-bao secret reader after the policy unit writing its role The four readers (matrix-token, queue-agent, grafana-oidc, otel-oidc) log in against a cert-auth role that their own swarm-bao-*-policy unit writes. They were ordered after swarm-bao-pki and the store's container but not after that unit, so on an apply a reader could log in before its role existed and be refused by `allowed_common_names` until a retry landed after the role did. After= plus Wants= on the policy unit, never Requires=: the policy unit skips by ConditionPathExists once the bootstrap token is gone, and a skipped unit counts as done for ordering. --- nix/host-modules/glue-matrix-bao-token.nix | 10 +++++++++- nix/host-modules/glue-queue-agent-credential.nix | 10 +++++++++- nix/host-modules/swarm-grafana.nix | 10 +++++++++- nix/host-modules/swarm-otel.nix | 10 +++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/nix/host-modules/glue-matrix-bao-token.nix b/nix/host-modules/glue-matrix-bao-token.nix index 3c0afcb6..07c429ce 100644 --- a/nix/host-modules/glue-matrix-bao-token.nix +++ b/nix/host-modules/glue-matrix-bao-token.nix @@ -136,11 +136,19 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering is # conditional even though the read is not: off-host there is nothing local # to wait for, and the timeout below is what bounds the attempt instead. + # + # The policy unit writes the role this reader logs in with. Ordering + # only: it skips once the bootstrap token is gone, and a skipped unit + # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" + "swarm-bao-matrix-token-policy.service" + ]; + wants = lib.optionals baoDeploy.enable [ + "container@${baoCfg.machine}.service" + "swarm-bao-matrix-token-policy.service" ]; - wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; before = [ "container@${matrixMachine}.service" ]; wantedBy = [ "container@${matrixMachine}.service" ]; diff --git a/nix/host-modules/glue-queue-agent-credential.nix b/nix/host-modules/glue-queue-agent-credential.nix index 8fa2dc06..e2894e35 100644 --- a/nix/host-modules/glue-queue-agent-credential.nix +++ b/nix/host-modules/glue-queue-agent-credential.nix @@ -161,11 +161,19 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering is # conditional even though the read is not: off-host there is nothing local # to wait for, and the timeout below is what bounds the attempt instead. + # + # The policy unit writes the role this reader logs in with. Ordering + # only: it skips once the bootstrap token is gone, and a skipped unit + # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" + "swarm-bao-queue-agent-policy.service" + ]; + wants = lib.optionals baoDeploy.enable [ + "container@${baoCfg.machine}.service" + "swarm-bao-queue-agent-policy.service" ]; - wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; # Ordered before hive-c0re, so no agent container renders ahead of an # attempt at its credential. `Wants=`, not `Requires=`: a store this diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 85e5a2d1..1ebac1f3 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -583,11 +583,19 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering # is conditional even though the read is not: off-host there is nothing # local to wait for, and the timeout below bounds the attempt instead. + # + # The policy unit writes the role this reader logs in with. Ordering + # only: it skips once the bootstrap token is gone, and a skipped unit + # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" + "swarm-bao-grafana-oidc-policy.service" + ]; + wants = lib.optionals baoDeploy.enable [ + "container@${baoCfg.machine}.service" + "swarm-bao-grafana-oidc-policy.service" ]; - wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; before = [ "container@${cfg.machine}.service" ]; wantedBy = [ diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 0b5cc904..51605ba8 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -766,11 +766,19 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering # is conditional even though the read is not: off-host there is nothing # local to wait for, and the timeout below bounds the attempt instead. + # + # The policy unit writes the role this reader logs in with. Ordering + # only: it skips once the bootstrap token is gone, and a skipped unit + # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" + "swarm-bao-otel-oidc-policy.service" + ]; + wants = lib.optionals baoDeploy.enable [ + "container@${baoCfg.machine}.service" + "swarm-bao-otel-oidc-policy.service" ]; - wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; before = [ "container@${cfg.machine}.service" ]; wantedBy = [ From cde3fec956f7e448b8d63a4ad525d5a7fbf015c8 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 24 Sep 2026 13:08:40 +0200 Subject: [PATCH 2/3] module-eval: pin each swarm-bao reader's ordering after its policy unit --- nix/module-eval/bao-grants.nix | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 0d4075dd..1633c2ea 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -438,6 +438,30 @@ let && !(baoGrantHere.containers.swarm-bao.config.systemd.services ? ${unit}) ) units; } + { + # The other end of those units: each reader logs in against the role its + # own policy unit writes, so it has to wait for that unit. Ordering and + # never a requirement, since the policy unit skips once the token is gone. + name = "each of the four readers is ordered after the unit writing its role"; + ok = + let + s = baoGrantWithConsumers.systemd.services; + waitsFor = + reader: + let + policy = "${reader}-policy.service"; + in + lib.elem policy s.${reader}.after + && lib.elem policy s.${reader}.wants + && !(lib.elem policy s.${reader}.requires); + in + lib.all waitsFor [ + "swarm-bao-matrix-token" + "swarm-bao-queue-agent" + "swarm-bao-grafana-oidc" + "swarm-bao-otel-oidc" + ]; + } { # A store host that has not placed a bootstrap token can write no grant at # all, so none of the four units may exist — the same claim From 0bfe354b6ddb037eaf4ec2aac09275f45436ecdf Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 24 Sep 2026 14:04:49 +0200 Subject: [PATCH 3/3] nix: move the reader-after-policy edges into their own colocation glue The four readers' ordering after their policy units only applies where the store and that reader share a host, so it is colocation glue and does not belong in the reader modules (two of which are main modules). glue-bao-readers-policy-order.nix now sets the after+wants edges, gated on deploy.bao.enable AND the reader's own gate, so a store host without a reader gains no stub unit. --- nix/host-modules/default.nix | 1 + .../glue-bao-readers-policy-order.nix | 60 ++++++++++++++++ nix/host-modules/glue-matrix-bao-token.nix | 10 +-- .../glue-queue-agent-credential.nix | 10 +-- nix/host-modules/swarm-grafana.nix | 10 +-- nix/host-modules/swarm-otel.nix | 10 +-- nix/module-eval/bao-grants.nix | 68 +++++++++++++++++-- 7 files changed, 127 insertions(+), 42 deletions(-) create mode 100644 nix/host-modules/glue-bao-readers-policy-order.nix diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 25755fdb..56a3a6ae 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -23,6 +23,7 @@ ./hive-priv.nix ./hive-tls.nix ./otel.nix + ./glue-bao-readers-policy-order.nix ./glue-bao-tls.nix ./glue-controller-bao-identity.nix ./glue-grafana-oidc-client.nix diff --git a/nix/host-modules/glue-bao-readers-policy-order.nix b/nix/host-modules/glue-bao-readers-policy-order.nix new file mode 100644 index 00000000..f0c2d2e7 --- /dev/null +++ b/nix/host-modules/glue-bao-readers-policy-order.nix @@ -0,0 +1,60 @@ +# Glue: where the store and one of its readers share a host, the reader waits +# for the unit that writes the cert-auth role it logs in with. +# +# ONE PAIRING PER FILE — the store's four reader policy units ← the readers +# they grant, and nothing else. Deleting this leaves every reader as it is on a +# host whose store is remote: it may log in before its role exists, and its own +# retries are what carry it past that. +# +# ⚠️ Gated on BOTH the store and that reader being here. Off the store's host +# there is no local policy unit to order against. On the store's host without +# the reader, setting `systemd.services..after` would define a unit +# with no ExecStart, so each gate below restates the one the reader's own +# module puts on it. A reader whose gate changes must change here too. +# +# Ordering, never a requirement: a policy unit skips once the bootstrap token +# is gone, and a skipped unit counts as done. `wants` as well as `after`, so a +# reader started on its own pulls its policy unit into the same transaction. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + baoDeploy = deployCfg.bao; + + havePair = cert: key: cert != null && key != null; + + # Reader unit → the gate its own module defines it under. + readersHere = { + # ./glue-matrix-bao-token.nix + swarm-bao-matrix-token = + hyperhiveCfg.enable + && havePair baoDeploy.matrixTokenClientCertFile baoDeploy.matrixTokenClientKeyFile + && deployCfg.matrix.enable; + # ./glue-queue-agent-credential.nix + swarm-bao-queue-agent = + hyperhiveCfg.enable + && havePair baoDeploy.queueAgentClientCertFile baoDeploy.queueAgentClientKeyFile; + # ./swarm-grafana.nix + swarm-bao-grafana-oidc = hyperhiveCfg.enable && deployCfg.grafana.enable; + # ./swarm-otel.nix + swarm-bao-otel-oidc = + deployCfg.swarm-otel.enable + && havePair baoDeploy.otelOidcClientCertFile baoDeploy.otelOidcClientKeyFile; + }; + + orderAfterPolicy = + reader: here: + lib.mkIf (baoDeploy.enable && here) { + systemd.services.${reader} = { + after = [ "${reader}-policy.service" ]; + wants = [ "${reader}-policy.service" ]; + }; + }; +in +{ + config = lib.mkMerge (lib.mapAttrsToList orderAfterPolicy readersHere); +} diff --git a/nix/host-modules/glue-matrix-bao-token.nix b/nix/host-modules/glue-matrix-bao-token.nix index 07c429ce..3c0afcb6 100644 --- a/nix/host-modules/glue-matrix-bao-token.nix +++ b/nix/host-modules/glue-matrix-bao-token.nix @@ -136,19 +136,11 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering is # conditional even though the read is not: off-host there is nothing local # to wait for, and the timeout below is what bounds the attempt instead. - # - # The policy unit writes the role this reader logs in with. Ordering - # only: it skips once the bootstrap token is gone, and a skipped unit - # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" - "swarm-bao-matrix-token-policy.service" - ]; - wants = lib.optionals baoDeploy.enable [ - "container@${baoCfg.machine}.service" - "swarm-bao-matrix-token-policy.service" ]; + wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; before = [ "container@${matrixMachine}.service" ]; wantedBy = [ "container@${matrixMachine}.service" ]; diff --git a/nix/host-modules/glue-queue-agent-credential.nix b/nix/host-modules/glue-queue-agent-credential.nix index e2894e35..8fa2dc06 100644 --- a/nix/host-modules/glue-queue-agent-credential.nix +++ b/nix/host-modules/glue-queue-agent-credential.nix @@ -161,19 +161,11 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering is # conditional even though the read is not: off-host there is nothing local # to wait for, and the timeout below is what bounds the attempt instead. - # - # The policy unit writes the role this reader logs in with. Ordering - # only: it skips once the bootstrap token is gone, and a skipped unit - # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" - "swarm-bao-queue-agent-policy.service" - ]; - wants = lib.optionals baoDeploy.enable [ - "container@${baoCfg.machine}.service" - "swarm-bao-queue-agent-policy.service" ]; + wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; # Ordered before hive-c0re, so no agent container renders ahead of an # attempt at its credential. `Wants=`, not `Requires=`: a store this diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 1ebac1f3..85e5a2d1 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -583,19 +583,11 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering # is conditional even though the read is not: off-host there is nothing # local to wait for, and the timeout below bounds the attempt instead. - # - # The policy unit writes the role this reader logs in with. Ordering - # only: it skips once the bootstrap token is gone, and a skipped unit - # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" - "swarm-bao-grafana-oidc-policy.service" - ]; - wants = lib.optionals baoDeploy.enable [ - "container@${baoCfg.machine}.service" - "swarm-bao-grafana-oidc-policy.service" ]; + wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; before = [ "container@${cfg.machine}.service" ]; wantedBy = [ diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 51605ba8..0b5cc904 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -766,19 +766,11 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering # is conditional even though the read is not: off-host there is nothing # local to wait for, and the timeout below bounds the attempt instead. - # - # The policy unit writes the role this reader logs in with. Ordering - # only: it skips once the bootstrap token is gone, and a skipped unit - # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" - "swarm-bao-otel-oidc-policy.service" - ]; - wants = lib.optionals baoDeploy.enable [ - "container@${baoCfg.machine}.service" - "swarm-bao-otel-oidc-policy.service" ]; + wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; before = [ "container@${cfg.machine}.service" ]; wantedBy = [ diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 1633c2ea..8e78acb3 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -57,6 +57,43 @@ let deploy.swarm-otel.enable = true; }; + # The store with none of the four readers beside it. Grafana, the collector and + # the homeserver are simply off; the queue reader renders on any host holding + # its leaf, which ./glue-bao-tls.nix mints here, so that leaf is taken away. + baoGrantNoReaders = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + deploy.bao.queueAgentClientCertFile = lib.mkForce null; + deploy.bao.queueAgentClientKeyFile = lib.mkForce null; + }; + + # All four readers against a store they do not run, each with a leaf placed + # by hand. The deployment in which there is no local policy unit to wait for. + baoRemoteReaders = hive { + deploy.matrix.enable = true; + deploy.grafana.enable = true; + deploy.swarm-otel.enable = true; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + deploy.bao.matrixTokenClientCertFile = "/etc/pki/bao-matrix-token.pem"; + deploy.bao.matrixTokenClientKeyFile = "/etc/pki/bao-matrix-token-key.pem"; + deploy.bao.queueAgentClientCertFile = "/etc/pki/bao-queue-agent.pem"; + deploy.bao.queueAgentClientKeyFile = "/etc/pki/bao-queue-agent-key.pem"; + deploy.bao.grafanaOidcClientCertFile = "/etc/pki/bao-grafana-oidc.pem"; + deploy.bao.grafanaOidcClientKeyFile = "/etc/pki/bao-grafana-oidc-key.pem"; + deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem"; + deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem"; + }; + + # The four readers ./glue-bao-readers-policy-order.nix orders after their + # policy units. + policyReaders = [ + "swarm-bao-matrix-token" + "swarm-bao-queue-agent" + "swarm-bao-grafana-oidc" + "swarm-bao-otel-oidc" + ]; + cases = [ { # Reads the rendered unit on the HOST, which is where the write happens: @@ -455,12 +492,31 @@ let && lib.elem policy s.${reader}.wants && !(lib.elem policy s.${reader}.requires); in - lib.all waitsFor [ - "swarm-bao-matrix-token" - "swarm-bao-queue-agent" - "swarm-bao-grafana-oidc" - "swarm-bao-otel-oidc" - ]; + lib.all waitsFor policyReaders; + } + { + # The ordering is set apart from each reader's own definition, so it can + # define a reader by itself: `after` on a unit nothing else declares is a + # unit with no ExecStart. On the store's host without the readers, none + # of the four may exist. + name = "a store host without the readers gains no reader unit from their ordering"; + ok = lib.all (reader: !(baoGrantNoReaders.systemd.services ? ${reader})) policyReaders; + } + { + # Where the store is remote there is no policy unit here to wait for, so + # the readers render as they did before the ordering existed. + name = "a reader whose store is remote is not ordered after a policy unit"; + ok = + let + s = baoRemoteReaders.systemd.services; + unordered = + reader: + let + policy = "${reader}-policy.service"; + in + s ? ${reader} && !(lib.elem policy s.${reader}.after) && !(lib.elem policy s.${reader}.wants); + in + lib.all unordered policyReaders; } { # A store host that has not placed a bootstrap token can write no grant at