From 0bfe354b6ddb037eaf4ec2aac09275f45436ecdf Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 24 Sep 2026 14:04:49 +0200 Subject: [PATCH] nix: move the reader-after-policy edges into their own colocation glue The four readers' ordering after their policy units only applies where the store and that reader share a host, so it is colocation glue and does not belong in the reader modules (two of which are main modules). glue-bao-readers-policy-order.nix now sets the after+wants edges, gated on deploy.bao.enable AND the reader's own gate, so a store host without a reader gains no stub unit. --- nix/host-modules/default.nix | 1 + .../glue-bao-readers-policy-order.nix | 60 ++++++++++++++++ nix/host-modules/glue-matrix-bao-token.nix | 10 +-- .../glue-queue-agent-credential.nix | 10 +-- nix/host-modules/swarm-grafana.nix | 10 +-- nix/host-modules/swarm-otel.nix | 10 +-- nix/module-eval/bao-grants.nix | 68 +++++++++++++++++-- 7 files changed, 127 insertions(+), 42 deletions(-) create mode 100644 nix/host-modules/glue-bao-readers-policy-order.nix diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 25755fdb..56a3a6ae 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -23,6 +23,7 @@ ./hive-priv.nix ./hive-tls.nix ./otel.nix + ./glue-bao-readers-policy-order.nix ./glue-bao-tls.nix ./glue-controller-bao-identity.nix ./glue-grafana-oidc-client.nix diff --git a/nix/host-modules/glue-bao-readers-policy-order.nix b/nix/host-modules/glue-bao-readers-policy-order.nix new file mode 100644 index 00000000..f0c2d2e7 --- /dev/null +++ b/nix/host-modules/glue-bao-readers-policy-order.nix @@ -0,0 +1,60 @@ +# Glue: where the store and one of its readers share a host, the reader waits +# for the unit that writes the cert-auth role it logs in with. +# +# ONE PAIRING PER FILE — the store's four reader policy units ← the readers +# they grant, and nothing else. Deleting this leaves every reader as it is on a +# host whose store is remote: it may log in before its role exists, and its own +# retries are what carry it past that. +# +# ⚠️ Gated on BOTH the store and that reader being here. Off the store's host +# there is no local policy unit to order against. On the store's host without +# the reader, setting `systemd.services..after` would define a unit +# with no ExecStart, so each gate below restates the one the reader's own +# module puts on it. A reader whose gate changes must change here too. +# +# Ordering, never a requirement: a policy unit skips once the bootstrap token +# is gone, and a skipped unit counts as done. `wants` as well as `after`, so a +# reader started on its own pulls its policy unit into the same transaction. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + baoDeploy = deployCfg.bao; + + havePair = cert: key: cert != null && key != null; + + # Reader unit → the gate its own module defines it under. + readersHere = { + # ./glue-matrix-bao-token.nix + swarm-bao-matrix-token = + hyperhiveCfg.enable + && havePair baoDeploy.matrixTokenClientCertFile baoDeploy.matrixTokenClientKeyFile + && deployCfg.matrix.enable; + # ./glue-queue-agent-credential.nix + swarm-bao-queue-agent = + hyperhiveCfg.enable + && havePair baoDeploy.queueAgentClientCertFile baoDeploy.queueAgentClientKeyFile; + # ./swarm-grafana.nix + swarm-bao-grafana-oidc = hyperhiveCfg.enable && deployCfg.grafana.enable; + # ./swarm-otel.nix + swarm-bao-otel-oidc = + deployCfg.swarm-otel.enable + && havePair baoDeploy.otelOidcClientCertFile baoDeploy.otelOidcClientKeyFile; + }; + + orderAfterPolicy = + reader: here: + lib.mkIf (baoDeploy.enable && here) { + systemd.services.${reader} = { + after = [ "${reader}-policy.service" ]; + wants = [ "${reader}-policy.service" ]; + }; + }; +in +{ + config = lib.mkMerge (lib.mapAttrsToList orderAfterPolicy readersHere); +} diff --git a/nix/host-modules/glue-matrix-bao-token.nix b/nix/host-modules/glue-matrix-bao-token.nix index 07c429ce..3c0afcb6 100644 --- a/nix/host-modules/glue-matrix-bao-token.nix +++ b/nix/host-modules/glue-matrix-bao-token.nix @@ -136,19 +136,11 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering is # conditional even though the read is not: off-host there is nothing local # to wait for, and the timeout below is what bounds the attempt instead. - # - # The policy unit writes the role this reader logs in with. Ordering - # only: it skips once the bootstrap token is gone, and a skipped unit - # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" - "swarm-bao-matrix-token-policy.service" - ]; - wants = lib.optionals baoDeploy.enable [ - "container@${baoCfg.machine}.service" - "swarm-bao-matrix-token-policy.service" ]; + wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; before = [ "container@${matrixMachine}.service" ]; wantedBy = [ "container@${matrixMachine}.service" ]; diff --git a/nix/host-modules/glue-queue-agent-credential.nix b/nix/host-modules/glue-queue-agent-credential.nix index e2894e35..8fa2dc06 100644 --- a/nix/host-modules/glue-queue-agent-credential.nix +++ b/nix/host-modules/glue-queue-agent-credential.nix @@ -161,19 +161,11 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering is # conditional even though the read is not: off-host there is nothing local # to wait for, and the timeout below is what bounds the attempt instead. - # - # The policy unit writes the role this reader logs in with. Ordering - # only: it skips once the bootstrap token is gone, and a skipped unit - # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" - "swarm-bao-queue-agent-policy.service" - ]; - wants = lib.optionals baoDeploy.enable [ - "container@${baoCfg.machine}.service" - "swarm-bao-queue-agent-policy.service" ]; + wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; # Ordered before hive-c0re, so no agent container renders ahead of an # attempt at its credential. `Wants=`, not `Requires=`: a store this diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 1ebac1f3..85e5a2d1 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -583,19 +583,11 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering # is conditional even though the read is not: off-host there is nothing # local to wait for, and the timeout below bounds the attempt instead. - # - # The policy unit writes the role this reader logs in with. Ordering - # only: it skips once the bootstrap token is gone, and a skipped unit - # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" - "swarm-bao-grafana-oidc-policy.service" - ]; - wants = lib.optionals baoDeploy.enable [ - "container@${baoCfg.machine}.service" - "swarm-bao-grafana-oidc-policy.service" ]; + wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; before = [ "container@${cfg.machine}.service" ]; wantedBy = [ diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 51605ba8..0b5cc904 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -766,19 +766,11 @@ in # `Requires=` on an absent unit fails the job outright, so the ordering # is conditional even though the read is not: off-host there is nothing # local to wait for, and the timeout below bounds the attempt instead. - # - # The policy unit writes the role this reader logs in with. Ordering - # only: it skips once the bootstrap token is gone, and a skipped unit - # counts as done. after = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" "container@${baoCfg.machine}.service" - "swarm-bao-otel-oidc-policy.service" - ]; - wants = lib.optionals baoDeploy.enable [ - "container@${baoCfg.machine}.service" - "swarm-bao-otel-oidc-policy.service" ]; + wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ]; requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ]; before = [ "container@${cfg.machine}.service" ]; wantedBy = [ diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 1633c2ea..8e78acb3 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -57,6 +57,43 @@ let deploy.swarm-otel.enable = true; }; + # The store with none of the four readers beside it. Grafana, the collector and + # the homeserver are simply off; the queue reader renders on any host holding + # its leaf, which ./glue-bao-tls.nix mints here, so that leaf is taken away. + baoGrantNoReaders = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + deploy.bao.queueAgentClientCertFile = lib.mkForce null; + deploy.bao.queueAgentClientKeyFile = lib.mkForce null; + }; + + # All four readers against a store they do not run, each with a leaf placed + # by hand. The deployment in which there is no local policy unit to wait for. + baoRemoteReaders = hive { + deploy.matrix.enable = true; + deploy.grafana.enable = true; + deploy.swarm-otel.enable = true; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + deploy.bao.matrixTokenClientCertFile = "/etc/pki/bao-matrix-token.pem"; + deploy.bao.matrixTokenClientKeyFile = "/etc/pki/bao-matrix-token-key.pem"; + deploy.bao.queueAgentClientCertFile = "/etc/pki/bao-queue-agent.pem"; + deploy.bao.queueAgentClientKeyFile = "/etc/pki/bao-queue-agent-key.pem"; + deploy.bao.grafanaOidcClientCertFile = "/etc/pki/bao-grafana-oidc.pem"; + deploy.bao.grafanaOidcClientKeyFile = "/etc/pki/bao-grafana-oidc-key.pem"; + deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem"; + deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem"; + }; + + # The four readers ./glue-bao-readers-policy-order.nix orders after their + # policy units. + policyReaders = [ + "swarm-bao-matrix-token" + "swarm-bao-queue-agent" + "swarm-bao-grafana-oidc" + "swarm-bao-otel-oidc" + ]; + cases = [ { # Reads the rendered unit on the HOST, which is where the write happens: @@ -455,12 +492,31 @@ let && lib.elem policy s.${reader}.wants && !(lib.elem policy s.${reader}.requires); in - lib.all waitsFor [ - "swarm-bao-matrix-token" - "swarm-bao-queue-agent" - "swarm-bao-grafana-oidc" - "swarm-bao-otel-oidc" - ]; + lib.all waitsFor policyReaders; + } + { + # The ordering is set apart from each reader's own definition, so it can + # define a reader by itself: `after` on a unit nothing else declares is a + # unit with no ExecStart. On the store's host without the readers, none + # of the four may exist. + name = "a store host without the readers gains no reader unit from their ordering"; + ok = lib.all (reader: !(baoGrantNoReaders.systemd.services ? ${reader})) policyReaders; + } + { + # Where the store is remote there is no policy unit here to wait for, so + # the readers render as they did before the ordering existed. + name = "a reader whose store is remote is not ordered after a policy unit"; + ok = + let + s = baoRemoteReaders.systemd.services; + unordered = + reader: + let + policy = "${reader}-policy.service"; + in + s ? ${reader} && !(lib.elem policy s.${reader}.after) && !(lib.elem policy s.${reader}.wants); + in + lib.all unordered policyReaders; } { # A store host that has not placed a bootstrap token can write no grant at