diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 8c34c1cc..cf88a4f5 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -30,6 +30,7 @@ ./glue-bao-tls.nix ./glue-bao-ui-oidc-client.nix ./glue-controller-bao-identity.nix + ./glue-controller-env.nix ./glue-controller-oidc-client.nix ./glue-forge-controller.nix ./glue-forge-oidc-client.nix diff --git a/nix/host-modules/glue-controller-env.nix b/nix/host-modules/glue-controller-env.nix new file mode 100644 index 00000000..44f30f01 --- /dev/null +++ b/nix/host-modules/glue-controller-env.nix @@ -0,0 +1,55 @@ +# Glue: tell the swarm controller about the authelia and the collector that run +# on this host. +# +# ONE PAIRING PER FILE — controller ← the services whose coordinates it reads +# from the environment. Deleting this leaves a controller that exports no +# metrics, and a `swarmctl` that fails `user add` saying the authelia binary and +# users file are unset. +# +# ⚠️ Two gates, one per service, each ANDed with the controller being here: +# `systemd.services.swarm-controller.environment` on a host without the +# controller would define a unit with no ExecStart. +# +# authelia → `swarmctl`'s wrapper, not the daemon's unit. Gated on authelia +# running HERE: the controller can be enabled on a host that is not the swarm's +# SSO host, and there `swarmctl user add` must fail saying the value is unset — +# a guessed path would resolve cleanly and write a file nothing reads. The +# CONFIGURED authelia, not whatever is on PATH: the argon2 parameters baked into +# a hash have to match the verifier's. +# +# swarm-otel → the daemon's unit. `vcs_metrics`/`hive_jobq_metrics` read the +# standard `OTEL_EXPORTER_OTLP_ENDPOINT` and are silent no-ops without it. The +# endpoint is the collector's swarm-service name through the gateway, never a +# loopback literal, with no trailing slash: the http exporter appends +# `/v1/metrics` and the gateway route strips `/${producerName}`. +# `SWARM_CONTROLLER_OTEL_AUDIENCE` is the audience ./glue-controller-oidc-client.nix +# registers and ./swarm-otel.nix's `oidc/${producerName}` authenticator checks. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + otelSwarmCfg = hyperhiveCfg.swarm.otel; + queueClientId = hyperhiveCfg.swarm.controller.queueClientId; +in +{ + config = lib.mkIf deployCfg.swarm-controller.enable ( + lib.mkMerge [ + (lib.mkIf deployCfg.authelia.enable { + services.hyperhive.deploy.swarm-controller.swarmctlEnvironment = { + SWARMCTL_AUTHELIA_BIN = "${deployCfg.authelia.package}/bin/authelia"; + SWARMCTL_AUTHELIA_USERS_FILE = deployCfg.authelia.hostUsersFile; + }; + }) + (lib.mkIf deployCfg.swarm-otel.enable { + systemd.services.swarm-controller.environment = { + OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}"; + SWARM_CONTROLLER_OTEL_AUDIENCE = queueClientId; + }; + }) + ] + ); +} diff --git a/nix/host-modules/glue-controller-oidc-client.nix b/nix/host-modules/glue-controller-oidc-client.nix index 654d0408..b17868e9 100644 --- a/nix/host-modules/glue-controller-oidc-client.nix +++ b/nix/host-modules/glue-controller-oidc-client.nix @@ -37,7 +37,7 @@ in # omitted `grant_types` means authorization-code alone. kind = "machine"; # Its own id as its own permitted audience. The controller mints a - # token for `SWARM_CONTROLLER_OTEL_AUDIENCE` (./swarm-controller.nix), + # token for `SWARM_CONTROLLER_OTEL_AUDIENCE` (./glue-controller-env.nix), # and ./swarm-otel.nix's `oidc/swarm` authenticator checks for this # same id; either half without the other is a 401 several layers from # this line. diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 2e50c12a..5b6988cd 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -1329,7 +1329,7 @@ let # and over the real network otherwise, with no loopback-vs-remote knob to get # wrong. The path selects WHICH receiver: this container is a swarm-level # producer and not a hive, so it is the swarm tier's own route, the same one - # ./swarm-controller.nix exports to. `/` on that vhost answers 404. + # ./glue-controller-env.nix exports to. `/` on that vhost answers 404. # # The port suffix follows ./hive-forge/default.nix: elided on the canonical # 443 so this renders byte for byte like the tier's other two consumers of diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 5bd8be9f..9efcf00f 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -119,24 +119,10 @@ let SWARM_CONTROLLER_AGENT_PKI_ROLE = deployCfg.bao.agentPkiRoleName; }; - # What `swarmctl` needs in order to act on authelia from the host. - # - # Only set when authelia actually runs **here**: the controller can be - # enabled on a host that is not the swarm's SSO host, and in that case - # the right behaviour is for `swarmctl user add` to fail saying the - # value is unset. A guessed path would resolve cleanly and write a file - # nothing reads, which is the failure mode that costs an afternoon. - autheliaEnv = lib.optionalAttrs deployCfg.authelia.enable { - # The CONFIGURED authelia, not whatever is on PATH: the argon2 - # parameters baked into a hash have to match the verifier's. - SWARMCTL_AUTHELIA_BIN = "${deployCfg.authelia.package}/bin/authelia"; - SWARMCTL_AUTHELIA_USERS_FILE = deployCfg.authelia.hostUsersFile; - }; - - # Where `swarmctl agent create` posts to. Unconditional, unlike - # `autheliaEnv` above: this binary is only installed where the daemon - # runs, so the socket is always the one right below it — and it is the - # same `socketPath` the daemon binds, read once rather than restated. + # Where `swarmctl agent create` posts to. Unconditional: this binary is + # only installed where the daemon runs, so the socket is always the one + # right below it — and it is the same `socketPath` the daemon binds, read + # once rather than restated. controllerSocketEnv = { SWARM_CONTROLLER_SOCKET = deployCfg.swarm-controller.socketPath; }; @@ -259,47 +245,6 @@ let SWARM_CONTROLLER_NAME = config.services.hyperhive.swarm.name; }; - # `vcs_metrics`/`hive_jobq_metrics` (this daemon's OTLP push exporters) - # read the standard `OTEL_EXPORTER_OTLP_ENDPOINT` var and are silent, - # graceful no-ops without it — same shape as `authBridgeEnv` above, - # genuinely optional and gated on the option resolving rather - # than assumed. - # - # By NAME through the gateway, never a loopback literal or a bare - # `producerPort` — mara's ruling on this exact module: "the swarm - # services dont have to run on the same host as the swarm controller." - # A controller and the swarm's collector are two independently-placed - # swarm services, exactly like a hive and the collector already are - # (`otel.nix`'s own `endpoint`), so this reaches it the same way: by - # its swarm-service name, resolved locally by dnsmasq on a co-located - # host and over the real network otherwise — no loopback-vs-remote - # knob to get wrong, and nothing here to gate on co-location because - # there is no co-location requirement left to check. - # - # No trailing slash: `opentelemetry_otlp`'s http/json exporter appends - # `/v1/metrics` to this base itself, and the swarm-otel gateway route - # (`swarm-otel.nix`'s nginx `locations."/${producerName}/"`) strips the - # `/${producerName}` prefix before the request reaches the receiver — - # so the two ends agree on exactly one path, `/v1/metrics`, arrived at - # from opposite directions. - # - # Gated on `deployCfg.swarm-otel.enable` alone (a swarm-wide fact, not "enabled - # HERE"): the collector is reachable by name wherever it runs, so a - # controller not co-located with it now exports over https instead of - # exporting nothing — the graceful-absence case left is "no swarm-otel - # anywhere in this swarm at all." - otelSwarmCfg = config.services.hyperhive.swarm.otel; - otelEnv = lib.optionalAttrs deployCfg.swarm-otel.enable { - OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}"; - # Checked by `swarm-otel.nix`'s `oidc/${producerName}` authenticator - # against exactly this controller's own registered audience (see the - # `audience = [ queueClientId ]` client entry in - # ./glue-controller-oidc-client.nix) — the two have - # to name the same value or the exporter mints a token the receiver - # was never told to accept. - SWARM_CONTROLLER_OTEL_AUDIENCE = queueClientId; - }; - # Wrapped rather than documented: every one of these values is derived # from an option this deployment already set, so making the operator # re-supply them on the command line would be asking them to repeat the @@ -312,7 +257,7 @@ let wrapProgram $out/bin/swarmctl ${ lib.concatStringsSep " " ( lib.mapAttrsToList (name: value: "--set ${name} ${lib.escapeShellArg value}") ( - autheliaEnv // controllerSocketEnv + controllerSocketEnv // deployCfg.swarm-controller.swarmctlEnvironment ) ++ lib.mapAttrsToList ( name: value: "--set-default ${name} ${lib.escapeShellArg value}" @@ -604,6 +549,18 @@ in ''; }; + swarmctlEnvironment = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + internal = true; + description = '' + Variables the installed `swarmctl` wrapper sets, on top of the + controller's socket path. Empty unless another module on this host + supplies some: ./glue-controller-env.nix adds the configured authelia + where authelia runs here. + ''; + }; + baoClientCertFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -852,8 +809,7 @@ in // authBridgeEnv // swarmNameEnv // matrixHomeserverEnv - // baoEnv - // otelEnv; + // baoEnv; }; }; } diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 82e9ef28..7bb04080 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -176,7 +176,7 @@ in (`otlp/''${storeProducerName}`, `oidc/''${storeProducerName}`). Published so `swarm-bao.nix` addresses the receiver by name rather than repeating the string, exactly as `producerName` above is - published for `swarm-controller.nix`. + published for `glue-controller-env.nix`. Read-only and derived from the store's container name: two spellings of a name both ends have to agree on is a 404 on a @@ -193,7 +193,7 @@ in — the same reserved literal this module's collector components (`otlp/swarm`, `oidc/swarm`, `resource/swarm`, `metrics/swarm`) are already named after internally. Published as an option so a - sibling module (`swarm-controller.nix`) can address the receiver + sibling module (`glue-controller-env.nix`) can address the receiver by name — `https://''${domain}/''${producerName}/` — instead of repeating the string. Read-only for the same reason `clientId`/`machine`/`domain` are: two spellings of a name that diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index a81c0075..6867e713 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -36,7 +36,7 @@ let # repeated at each site would let the guard and the config drift apart, which # is the failure this guard exists to prevent. # The read-only `producerName` option in ./swarm-otel-service.nix, not a bare - # literal — `swarm-controller.nix` needs the identical string to build the + # literal — `glue-controller-env.nix` needs the identical string to build the # same audience/endpoint, and a value bound once (rather than copy-pasted at # both sites) is the only way the two can't drift apart. swarmTierName = cfg.producerName; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 1c2a169b..1e123772 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -145,6 +145,38 @@ let name = "the all-local mode turns the swarm controller on"; ok = allLocal.services.hyperhive.deploy.swarm-controller.enable; } + { + # ../host-modules/glue-controller-env.nix: the collector's coordinates + # follow swarm-otel running HERE. `controllerAlone` is the split case — a + # controller whose collector runs elsewhere. + name = "the controller exports to the collector only where the collector runs here"; + ok = + let + env = m: m.systemd.services.swarm-controller.environment; + otel = allLocal.services.hyperhive.swarm.otel; + in + (env allLocal).OTEL_EXPORTER_OTLP_ENDPOINT or null == "https://${otel.domain}/${otel.producerName}" + && + (env allLocal).SWARM_CONTROLLER_OTEL_AUDIENCE or null + == allLocal.services.hyperhive.swarm.controller.queueClientId + && !(env shared.controllerAlone ? OTEL_EXPORTER_OTLP_ENDPOINT) + && !(env shared.controllerAlone ? SWARM_CONTROLLER_OTEL_AUDIENCE); + } + { + # Same file: `swarmctl` is told where the configured authelia is only + # where authelia runs here. + name = "swarmctl is given the authelia binary and users file only where authelia runs here"; + ok = + let + env = m: m.services.hyperhive.deploy.swarm-controller.swarmctlEnvironment; + d = allLocal.services.hyperhive.deploy.authelia; + in + env allLocal == { + SWARMCTL_AUTHELIA_BIN = "${d.package}/bin/authelia"; + SWARMCTL_AUTHELIA_USERS_FILE = d.hostUsersFile; + } + && env shared.controllerAlone == { }; + } { # Where the reader puts the files and where the daemon looks for them is # one agreement spanning two modules. Asserted against the option rather diff --git a/swarmctl/README.md b/swarmctl/README.md index 52e4f2be..f7c0754e 100644 --- a/swarmctl/README.md +++ b/swarmctl/README.md @@ -38,8 +38,10 @@ See `swarm-authelia-bridge/README.md` for what both writers must uphold. ## Configuration -Every path comes from the nix module that installs the binary, because -every one is derived from an option that module owns. They are required +The nix deployment sets every path, each derived from an option it already +has: `swarm-controller.nix` sets the socket, and `glue-controller-env.nix` sets +`SWARMCTL_AUTHELIA_BIN` and `SWARMCTL_AUTHELIA_USERS_FILE`, only where +authelia runs on the same host. They are required rather than defaulted — a default would be an address we _hope_ points at something, and one that resolves cleanly to the wrong place is worse than an error.