vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

start the container journal forwarder's collector from its secret fetch, with backoff

Browse source
This commit is contained in:
atlas 2026-11-01 18:52:06 +01:00 • committed by mara
commit a0deb29e59
2 changed files with 51 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

37
nix/container-modules/swarm-journal-forwarder.nix
View file

@ -275,14 +275,29 @@ in
# `RestartSec` is ours: the 100ms default burns the start-rate limit while
# the secret is still on its way, leaving the unit failed instead of
# retrying.
systemd.services.opentelemetry-collector.serviceConfig = {
LoadCredential = [ "${credentialId}:${credentialSource}" ];
RestartSec = 15;
};
systemd.services.opentelemetry-collector = lib.mkMerge [
{
serviceConfig = {
LoadCredential = [ "${credentialId}:${credentialSource}" ];
RestartSec = 15;
};
}
# With a fetched secret the collector is not part of the boot
# transaction: `LoadCredential` refuses a missing file, and a unit that
# failed on it before the secret landed is not started again by its
# `Restart=`. The fetch starts the collector once it has written the
# secret, so the guest reaches READY without waiting on the store.
(lib.mkIf fetchesHere {
wantedBy = lib.mkForce [ ];
after = [ "${fetchUnit}.service" ];
})
];
# Reads the client secret with the container's own store identity. A store
# or authelia that is not up yet is a failed run that retries
# (../host-modules/lib/store-retry.nix), and a collector that restarts until the file lands.
# that is not up yet fails the attempt in seconds, which keeps the guest's
# boot short, and ../host-modules/lib/store-retry.nix starts it again with
# a growing delay until the store answers. The attempt that succeeds
# starts the collector.
systemd.services.${fetchUnit} = lib.mkIf fetchesHere {
description = "fetch this container's journal forwarder OIDC client secret from the swarm secret store";
after = [ "network-online.target" ];
@ -300,7 +315,13 @@ in
# an active unit is a no-op.
Type = "oneshot";
SyslogIdentifier = fetchUnit;
# Bounds one attempt, and so the guest's boot when the store is down.
TimeoutStartSec = 30;
# Each attempt is a store login: the delay grows from `RestartSec` to
# `RestartMaxDelaySec` over `RestartSteps` restarts (systemd.service(5)).
RestartSec = 5;
RestartSteps = 6;
RestartMaxDelaySec = 300;
};
environment = {
BAO_ADDR = cfg.store.addr;
@ -349,6 +370,10 @@ in
# `LoadCredential` copies the file at start only: a rotated secret
# reaches the collector by restarting it.
refresh_unit opentelemetry-collector.service ${lib.escapeShellArg secretFile}
# Not in the boot transaction (`wantedBy` is forced empty above), so
# nothing else starts it. A running collector is untouched by this.
systemctl start --no-block opentelemetry-collector.service
'';
};

19
nix/module-eval/nats-authelia.nix
View file

@ -147,6 +147,25 @@ let
&& !(lib.elem "--link-journal=host" container.extraFlags)
&& lib.any (lib.hasPrefix "--load-credential=${svc.credential}-cert:") container.extraFlags;
}
{
# The guest is ready when its job queue empties, so a long fetch job
# at boot would hold the container's start past its limit.
name = "the ${svc.name} forwarder's fetch retries with backoff outside the boot wait and starts the collector";
ok =
let
collector = c.systemd.services.opentelemetry-collector;
sc = fetch.serviceConfig;
in
collector.wantedBy == [ ]
&& lib.elem "swarm-journal-forwarder-oidc.service" collector.after
&& lib.elem "multi-user.target" fetch.wantedBy
&& sc.Restart == "on-failure"
&& sc.TimeoutStartSec <= 30
&& sc.RestartSteps > 0
&& sc.RestartMaxDelaySec > sc.RestartSec
&& !(lib.hasInfix "retry_in_job" fetch.script)
&& lib.hasInfix "systemctl start --no-block opentelemetry-collector.service" fetch.script;
}
{
name = "the ${svc.name} forwarder's metrics port is its own and loopback";
ok =