hive telemetry collector: keep the secret wait for an operator-named file
The previous commit dropped `hive-otel-client-secret` and the collector's `requires`/`after` on it for every hive. Only the store fetch restarts the collector when its file lands; an operator-named `otel.clientSecretFile` has nothing doing that, so a late file failed `LoadCredential` straight into the collector's start limit with no way back. The wait unit and the ordering are back for `senderAuth && !fetches`, and absent only where `swarm-bao-hive-otel-oidc` delivers the file. The module-eval case for a hand-named secret now also pins the wait unit and the collector's `requires`/`after` on it, on both of its fixtures. Refs #5015
This commit is contained in:
atlas
parent
7c50568db1
commit
9ef4f55113
3 changed files with 92 additions and 5 deletions
|
|
@ -139,8 +139,8 @@ so a secret rotated later arrives the next time the unit runs. On a hive that
|
|||
runs the store the leaf is already there; elsewhere it's the credential you
|
||||
place — see [secret placement](../swarm/secrets.md).
|
||||
|
||||
To deliver the secret some other way, name the file, and the fetch unit isn't
|
||||
rendered:
|
||||
To deliver the secret some other way, name the file. The fetch unit isn't
|
||||
rendered, and the collector waits up to 120s for the file to appear:
|
||||
|
||||
```nix
|
||||
services.hyperhive.otel.clientSecretFile = "/run/secrets/hive-telemetry.secret";
|
||||
|
|
|
|||
|
|
@ -338,6 +338,10 @@ in
|
|||
# delivered some other way is the operator's, and a unit restarting the
|
||||
# collector on a file it never loads would be noise.
|
||||
fetches = senderAuth && hiveLeaf && otel.clientSecretFile == fetchedSecretPath;
|
||||
# An operator-delivered file has no unit that restarts the collector
|
||||
# once it lands, so the collector waits for it instead.
|
||||
waits = senderAuth && !fetches;
|
||||
secretWaitUnit = "hive-otel-client-secret";
|
||||
# Written by swarm-secret-publish on authelia's host. The nix half of
|
||||
# `swarm_secret_client::queue::hive_client_path`.
|
||||
storeSecretPath = "secret/swarm/hives/${toString hiveName}/oidc/client";
|
||||
|
|
@ -661,8 +665,76 @@ in
|
|||
(lib.mkIf senderAuth {
|
||||
serviceConfig.LoadCredential = [ "${credName}:${otel.clientSecretFile}" ];
|
||||
})
|
||||
|
||||
# `requires`, not merely `after`: ordering without gating leaves the
|
||||
# collector starting anyway and failing on its own `LoadCredential`,
|
||||
# which is the failure the wait exists to prevent. Same reasoning
|
||||
# `lib/hive-ca-trust.nix` records for its bundle consumers.
|
||||
(lib.mkIf waits {
|
||||
requires = [ "${secretWaitUnit}.service" ];
|
||||
after = [ "${secretWaitUnit}.service" ];
|
||||
})
|
||||
];
|
||||
|
||||
# Wait for an operator-delivered credential rather than racing it.
|
||||
#
|
||||
# `LoadCredential` naming a missing path is FATAL AT UNIT START, and
|
||||
# nothing orders this host unit against whatever delivers the file.
|
||||
# Worse, nixpkgs
|
||||
# sets `Restart = "always"` with no `RestartSec`, so the failure is
|
||||
# instant and the unit burns systemd's 5-starts-in-10s allowance in well
|
||||
# under a second, landing in `start-limit-hit` where it stops retrying
|
||||
# entirely and needs `systemctl reset-failed` by hand.
|
||||
#
|
||||
# 🔑 `Restart = always` reads like it makes this self-healing and does
|
||||
# the opposite: a SLOW-failing unit retries until the secret appears, a
|
||||
# FAST-failing one exhausts its limit before the thing it waits for can
|
||||
# exist. This oneshot converts the fast failure into a slow one, which
|
||||
# is what that restart policy is actually good at.
|
||||
#
|
||||
# Copied from `hive-forge-oidc-secret.service`, which solves exactly
|
||||
# this for the forge — see `docs/swarm/secrets.md`.
|
||||
#
|
||||
# Not rendered beside the store fetch below, which restarts the
|
||||
# collector itself when the file lands.
|
||||
systemd.services.${secretWaitUnit} = lib.mkIf waits {
|
||||
description = "wait for this hive's telemetry client secret";
|
||||
# Deliberately says nothing about the collector: the CONSUMER declares
|
||||
# `requires` + `after` above, which both pulls this into the
|
||||
# transaction and gates on it. Stating the order from both sides too
|
||||
# would be one more thing to keep consistent for no added guarantee.
|
||||
#
|
||||
# When authelia runs on this host the named file may be one it mints.
|
||||
# Elsewhere there is no local unit to order against — naming one that
|
||||
# does not exist orders nothing, silently.
|
||||
after = lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service";
|
||||
requires = lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service";
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
SyslogIdentifier = secretWaitUnit;
|
||||
# ⚠️ Longer than the bounded wait below. `DefaultTimeoutStartSec` is
|
||||
# 90s, so without this systemd kills the unit at 90 — before it can
|
||||
# emit the message naming the file it was waiting for.
|
||||
TimeoutStartSec = "180s";
|
||||
};
|
||||
path = [ pkgs.coreutils ];
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
secret=${lib.escapeShellArg (toString otel.clientSecretFile)}
|
||||
|
||||
# Bounded, then FAIL — never skip. A silent skip produces a
|
||||
# collector that starts and ships nothing, which is the failure
|
||||
# mode this whole path is trying not to have.
|
||||
for _ in $(seq 1 60); do
|
||||
[ -s "$secret" ] && exit 0
|
||||
sleep 2
|
||||
done
|
||||
echo "telemetry client secret $secret has not appeared after 120s" >&2
|
||||
exit 1
|
||||
'';
|
||||
};
|
||||
|
||||
# This hive's OIDC client secret, out of the swarm secret store under the
|
||||
# hive's own leaf. Same shape as ./hive-matrix.nix's
|
||||
# `swarm-bao-matrix-oidc`: the login and the read both fail loudly, and
|
||||
|
|
|
|||
|
|
@ -261,12 +261,27 @@ let
|
|||
}
|
||||
{
|
||||
# A secret delivered some other way is the operator's: no unit fetches
|
||||
# into a file the collector does not load, or restarts it on one.
|
||||
name = "a hand-named collector secret renders no store fetch";
|
||||
# into a file the collector does not load, or restarts it on one. Nothing
|
||||
# restarts the collector when that file lands, so it waits for the file
|
||||
# instead of failing `LoadCredential` into its start limit.
|
||||
name = "a hand-named collector secret renders no store fetch, and the collector waits for the file";
|
||||
ok =
|
||||
let
|
||||
waitsFor =
|
||||
machine:
|
||||
let
|
||||
c = machine.systemd.services.opentelemetry-collector;
|
||||
w = machine.systemd.services.hive-otel-client-secret;
|
||||
in
|
||||
builtins.elem "hive-otel-client-secret.service" c.requires
|
||||
&& builtins.elem "hive-otel-client-secret.service" c.after
|
||||
&& lib.hasInfix "/var/lib/hive-otel-oidc/client.secret" w.script;
|
||||
in
|
||||
!(hiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
|
||||
&& !(baoWithHiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
|
||||
&& builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential;
|
||||
&& builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
||||
&& waitsFor hiveOtel
|
||||
&& waitsFor baoWithHiveOtel;
|
||||
}
|
||||
];
|
||||
in
|
||||
|
|
|
|||
Loading…
Reference in a new issue