glue: the controller's authelia and otel environment moves out of the controller module
The authelia and swarm-otel co-location reads (swarm-controller.nix autheliaEnv, otelEnv) move into glue-controller-env.nix.
This commit is contained in:
atlas
mara
parent
4a36e53982
commit
1219091788
9 changed files with 115 additions and 69 deletions
|
|
@ -30,6 +30,7 @@
|
|||
./glue-bao-tls.nix
|
||||
./glue-bao-ui-oidc-client.nix
|
||||
./glue-controller-bao-identity.nix
|
||||
./glue-controller-env.nix
|
||||
./glue-controller-oidc-client.nix
|
||||
./glue-forge-controller.nix
|
||||
./glue-forge-oidc-client.nix
|
||||
|
|
|
|||
55
nix/host-modules/glue-controller-env.nix
Normal file
55
nix/host-modules/glue-controller-env.nix
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
# Glue: tell the swarm controller about the authelia and the collector that run
|
||||
# on this host.
|
||||
#
|
||||
# ONE PAIRING PER FILE — controller ← the services whose coordinates it reads
|
||||
# from the environment. Deleting this leaves a controller that exports no
|
||||
# metrics, and a `swarmctl` that fails `user add` saying the authelia binary and
|
||||
# users file are unset.
|
||||
#
|
||||
# ⚠️ Two gates, one per service, each ANDed with the controller being here:
|
||||
# `systemd.services.swarm-controller.environment` on a host without the
|
||||
# controller would define a unit with no ExecStart.
|
||||
#
|
||||
# authelia → `swarmctl`'s wrapper, not the daemon's unit. Gated on authelia
|
||||
# running HERE: the controller can be enabled on a host that is not the swarm's
|
||||
# SSO host, and there `swarmctl user add` must fail saying the value is unset —
|
||||
# a guessed path would resolve cleanly and write a file nothing reads. The
|
||||
# CONFIGURED authelia, not whatever is on PATH: the argon2 parameters baked into
|
||||
# a hash have to match the verifier's.
|
||||
#
|
||||
# swarm-otel → the daemon's unit. `vcs_metrics`/`hive_jobq_metrics` read the
|
||||
# standard `OTEL_EXPORTER_OTLP_ENDPOINT` and are silent no-ops without it. The
|
||||
# endpoint is the collector's swarm-service name through the gateway, never a
|
||||
# loopback literal, with no trailing slash: the http exporter appends
|
||||
# `/v1/metrics` and the gateway route strips `/${producerName}`.
|
||||
# `SWARM_CONTROLLER_OTEL_AUDIENCE` is the audience ./glue-controller-oidc-client.nix
|
||||
# registers and ./swarm-otel.nix's `oidc/${producerName}` authenticator checks.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
deployCfg = hyperhiveCfg.deploy;
|
||||
otelSwarmCfg = hyperhiveCfg.swarm.otel;
|
||||
queueClientId = hyperhiveCfg.swarm.controller.queueClientId;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf deployCfg.swarm-controller.enable (
|
||||
lib.mkMerge [
|
||||
(lib.mkIf deployCfg.authelia.enable {
|
||||
services.hyperhive.deploy.swarm-controller.swarmctlEnvironment = {
|
||||
SWARMCTL_AUTHELIA_BIN = "${deployCfg.authelia.package}/bin/authelia";
|
||||
SWARMCTL_AUTHELIA_USERS_FILE = deployCfg.authelia.hostUsersFile;
|
||||
};
|
||||
})
|
||||
(lib.mkIf deployCfg.swarm-otel.enable {
|
||||
systemd.services.swarm-controller.environment = {
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}";
|
||||
SWARM_CONTROLLER_OTEL_AUDIENCE = queueClientId;
|
||||
};
|
||||
})
|
||||
]
|
||||
);
|
||||
}
|
||||
|
|
@ -37,7 +37,7 @@ in
|
|||
# omitted `grant_types` means authorization-code alone.
|
||||
kind = "machine";
|
||||
# Its own id as its own permitted audience. The controller mints a
|
||||
# token for `SWARM_CONTROLLER_OTEL_AUDIENCE` (./swarm-controller.nix),
|
||||
# token for `SWARM_CONTROLLER_OTEL_AUDIENCE` (./glue-controller-env.nix),
|
||||
# and ./swarm-otel.nix's `oidc/swarm` authenticator checks for this
|
||||
# same id; either half without the other is a 401 several layers from
|
||||
# this line.
|
||||
|
|
|
|||
|
|
@ -1329,7 +1329,7 @@ let
|
|||
# and over the real network otherwise, with no loopback-vs-remote knob to get
|
||||
# wrong. The path selects WHICH receiver: this container is a swarm-level
|
||||
# producer and not a hive, so it is the swarm tier's own route, the same one
|
||||
# ./swarm-controller.nix exports to. `/` on that vhost answers 404.
|
||||
# ./glue-controller-env.nix exports to. `/` on that vhost answers 404.
|
||||
#
|
||||
# The port suffix follows ./hive-forge/default.nix: elided on the canonical
|
||||
# 443 so this renders byte for byte like the tier's other two consumers of
|
||||
|
|
|
|||
|
|
@ -119,24 +119,10 @@ let
|
|||
SWARM_CONTROLLER_AGENT_PKI_ROLE = deployCfg.bao.agentPkiRoleName;
|
||||
};
|
||||
|
||||
# What `swarmctl` needs in order to act on authelia from the host.
|
||||
#
|
||||
# Only set when authelia actually runs **here**: the controller can be
|
||||
# enabled on a host that is not the swarm's SSO host, and in that case
|
||||
# the right behaviour is for `swarmctl user add` to fail saying the
|
||||
# value is unset. A guessed path would resolve cleanly and write a file
|
||||
# nothing reads, which is the failure mode that costs an afternoon.
|
||||
autheliaEnv = lib.optionalAttrs deployCfg.authelia.enable {
|
||||
# The CONFIGURED authelia, not whatever is on PATH: the argon2
|
||||
# parameters baked into a hash have to match the verifier's.
|
||||
SWARMCTL_AUTHELIA_BIN = "${deployCfg.authelia.package}/bin/authelia";
|
||||
SWARMCTL_AUTHELIA_USERS_FILE = deployCfg.authelia.hostUsersFile;
|
||||
};
|
||||
|
||||
# Where `swarmctl agent create` posts to. Unconditional, unlike
|
||||
# `autheliaEnv` above: this binary is only installed where the daemon
|
||||
# runs, so the socket is always the one right below it — and it is the
|
||||
# same `socketPath` the daemon binds, read once rather than restated.
|
||||
# Where `swarmctl agent create` posts to. Unconditional: this binary is
|
||||
# only installed where the daemon runs, so the socket is always the one
|
||||
# right below it — and it is the same `socketPath` the daemon binds, read
|
||||
# once rather than restated.
|
||||
controllerSocketEnv = {
|
||||
SWARM_CONTROLLER_SOCKET = deployCfg.swarm-controller.socketPath;
|
||||
};
|
||||
|
|
@ -259,47 +245,6 @@ let
|
|||
SWARM_CONTROLLER_NAME = config.services.hyperhive.swarm.name;
|
||||
};
|
||||
|
||||
# `vcs_metrics`/`hive_jobq_metrics` (this daemon's OTLP push exporters)
|
||||
# read the standard `OTEL_EXPORTER_OTLP_ENDPOINT` var and are silent,
|
||||
# graceful no-ops without it — same shape as `authBridgeEnv` above,
|
||||
# genuinely optional and gated on the option resolving rather
|
||||
# than assumed.
|
||||
#
|
||||
# By NAME through the gateway, never a loopback literal or a bare
|
||||
# `producerPort` — mara's ruling on this exact module: "the swarm
|
||||
# services dont have to run on the same host as the swarm controller."
|
||||
# A controller and the swarm's collector are two independently-placed
|
||||
# swarm services, exactly like a hive and the collector already are
|
||||
# (`otel.nix`'s own `endpoint`), so this reaches it the same way: by
|
||||
# its swarm-service name, resolved locally by dnsmasq on a co-located
|
||||
# host and over the real network otherwise — no loopback-vs-remote
|
||||
# knob to get wrong, and nothing here to gate on co-location because
|
||||
# there is no co-location requirement left to check.
|
||||
#
|
||||
# No trailing slash: `opentelemetry_otlp`'s http/json exporter appends
|
||||
# `/v1/metrics` to this base itself, and the swarm-otel gateway route
|
||||
# (`swarm-otel.nix`'s nginx `locations."/${producerName}/"`) strips the
|
||||
# `/${producerName}` prefix before the request reaches the receiver —
|
||||
# so the two ends agree on exactly one path, `/v1/metrics`, arrived at
|
||||
# from opposite directions.
|
||||
#
|
||||
# Gated on `deployCfg.swarm-otel.enable` alone (a swarm-wide fact, not "enabled
|
||||
# HERE"): the collector is reachable by name wherever it runs, so a
|
||||
# controller not co-located with it now exports over https instead of
|
||||
# exporting nothing — the graceful-absence case left is "no swarm-otel
|
||||
# anywhere in this swarm at all."
|
||||
otelSwarmCfg = config.services.hyperhive.swarm.otel;
|
||||
otelEnv = lib.optionalAttrs deployCfg.swarm-otel.enable {
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}";
|
||||
# Checked by `swarm-otel.nix`'s `oidc/${producerName}` authenticator
|
||||
# against exactly this controller's own registered audience (see the
|
||||
# `audience = [ queueClientId ]` client entry in
|
||||
# ./glue-controller-oidc-client.nix) — the two have
|
||||
# to name the same value or the exporter mints a token the receiver
|
||||
# was never told to accept.
|
||||
SWARM_CONTROLLER_OTEL_AUDIENCE = queueClientId;
|
||||
};
|
||||
|
||||
# Wrapped rather than documented: every one of these values is derived
|
||||
# from an option this deployment already set, so making the operator
|
||||
# re-supply them on the command line would be asking them to repeat the
|
||||
|
|
@ -312,7 +257,7 @@ let
|
|||
wrapProgram $out/bin/swarmctl ${
|
||||
lib.concatStringsSep " " (
|
||||
lib.mapAttrsToList (name: value: "--set ${name} ${lib.escapeShellArg value}") (
|
||||
autheliaEnv // controllerSocketEnv
|
||||
controllerSocketEnv // deployCfg.swarm-controller.swarmctlEnvironment
|
||||
)
|
||||
++ lib.mapAttrsToList (
|
||||
name: value: "--set-default ${name} ${lib.escapeShellArg value}"
|
||||
|
|
@ -604,6 +549,18 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
swarmctlEnvironment = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.str;
|
||||
default = { };
|
||||
internal = true;
|
||||
description = ''
|
||||
Variables the installed `swarmctl` wrapper sets, on top of the
|
||||
controller's socket path. Empty unless another module on this host
|
||||
supplies some: ./glue-controller-env.nix adds the configured authelia
|
||||
where authelia runs here.
|
||||
'';
|
||||
};
|
||||
|
||||
baoClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
|
|
@ -852,8 +809,7 @@ in
|
|||
// authBridgeEnv
|
||||
// swarmNameEnv
|
||||
// matrixHomeserverEnv
|
||||
// baoEnv
|
||||
// otelEnv;
|
||||
// baoEnv;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -176,7 +176,7 @@ in
|
|||
(`otlp/''${storeProducerName}`, `oidc/''${storeProducerName}`).
|
||||
Published so `swarm-bao.nix` addresses the receiver by name rather
|
||||
than repeating the string, exactly as `producerName` above is
|
||||
published for `swarm-controller.nix`.
|
||||
published for `glue-controller-env.nix`.
|
||||
|
||||
Read-only and derived from the store's container name: two
|
||||
spellings of a name both ends have to agree on is a 404 on a
|
||||
|
|
@ -193,7 +193,7 @@ in
|
|||
— the same reserved literal this module's collector components
|
||||
(`otlp/swarm`, `oidc/swarm`, `resource/swarm`, `metrics/swarm`)
|
||||
are already named after internally. Published as an option so a
|
||||
sibling module (`swarm-controller.nix`) can address the receiver
|
||||
sibling module (`glue-controller-env.nix`) can address the receiver
|
||||
by name — `https://''${domain}/''${producerName}/` — instead of
|
||||
repeating the string. Read-only for the same reason
|
||||
`clientId`/`machine`/`domain` are: two spellings of a name that
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ let
|
|||
# repeated at each site would let the guard and the config drift apart, which
|
||||
# is the failure this guard exists to prevent.
|
||||
# The read-only `producerName` option in ./swarm-otel-service.nix, not a bare
|
||||
# literal — `swarm-controller.nix` needs the identical string to build the
|
||||
# literal — `glue-controller-env.nix` needs the identical string to build the
|
||||
# same audience/endpoint, and a value bound once (rather than copy-pasted at
|
||||
# both sites) is the only way the two can't drift apart.
|
||||
swarmTierName = cfg.producerName;
|
||||
|
|
|
|||
|
|
@ -145,6 +145,38 @@ let
|
|||
name = "the all-local mode turns the swarm controller on";
|
||||
ok = allLocal.services.hyperhive.deploy.swarm-controller.enable;
|
||||
}
|
||||
{
|
||||
# ../host-modules/glue-controller-env.nix: the collector's coordinates
|
||||
# follow swarm-otel running HERE. `controllerAlone` is the split case — a
|
||||
# controller whose collector runs elsewhere.
|
||||
name = "the controller exports to the collector only where the collector runs here";
|
||||
ok =
|
||||
let
|
||||
env = m: m.systemd.services.swarm-controller.environment;
|
||||
otel = allLocal.services.hyperhive.swarm.otel;
|
||||
in
|
||||
(env allLocal).OTEL_EXPORTER_OTLP_ENDPOINT or null == "https://${otel.domain}/${otel.producerName}"
|
||||
&&
|
||||
(env allLocal).SWARM_CONTROLLER_OTEL_AUDIENCE or null
|
||||
== allLocal.services.hyperhive.swarm.controller.queueClientId
|
||||
&& !(env shared.controllerAlone ? OTEL_EXPORTER_OTLP_ENDPOINT)
|
||||
&& !(env shared.controllerAlone ? SWARM_CONTROLLER_OTEL_AUDIENCE);
|
||||
}
|
||||
{
|
||||
# Same file: `swarmctl` is told where the configured authelia is only
|
||||
# where authelia runs here.
|
||||
name = "swarmctl is given the authelia binary and users file only where authelia runs here";
|
||||
ok =
|
||||
let
|
||||
env = m: m.services.hyperhive.deploy.swarm-controller.swarmctlEnvironment;
|
||||
d = allLocal.services.hyperhive.deploy.authelia;
|
||||
in
|
||||
env allLocal == {
|
||||
SWARMCTL_AUTHELIA_BIN = "${d.package}/bin/authelia";
|
||||
SWARMCTL_AUTHELIA_USERS_FILE = d.hostUsersFile;
|
||||
}
|
||||
&& env shared.controllerAlone == { };
|
||||
}
|
||||
{
|
||||
# Where the reader puts the files and where the daemon looks for them is
|
||||
# one agreement spanning two modules. Asserted against the option rather
|
||||
|
|
|
|||
Loading…
Reference in a new issue