keep each hive's otel oidc secret on /run, not /var/lib
This commit is contained in:
atlas
mara
parent
8b255b43a0
commit
96405621d7
7 changed files with 106 additions and 18 deletions
|
|
@ -137,7 +137,9 @@ the label isn't taken from the payload at all.
|
|||
reads its secret from the swarm's secret store under the hive's own leaf
|
||||
(`deploy.bao.clientCertFile`): `swarm-bao-hive-otel-oidc.service` fetches
|
||||
`swarm/hives/<hive>/oidc/client`, which the publisher in authelia's container
|
||||
writes. It runs at boot and retries every 30s until the store answers; when a
|
||||
writes, into `/run/hive-otel-oidc/` (tmpfs, so never on persistent disk). It
|
||||
runs at boot and retries every 30s until the store answers; until then the
|
||||
collector runs on an empty file and the swarm refuses its exports. When a
|
||||
run lands a changed value it restarts the collector. A timer runs it again
|
||||
every ten minutes, so a secret rotated later reaches the collector within
|
||||
about ten minutes. On a hive that
|
||||
|
|
@ -151,6 +153,10 @@ rendered, and the collector waits up to 120s for the file to appear:
|
|||
services.hyperhive.otel.clientSecretFile = "/run/secrets/hive-telemetry.secret";
|
||||
```
|
||||
|
||||
Setting the option to `/var/lib/hive-otel-oidc/hive-<name>.secret`, the path
|
||||
the fetch used to write, now counts as delivering the file yourself: nothing
|
||||
refreshes it, and the build warns. Remove the setting to fetch from the store.
|
||||
|
||||
**No unauthenticated mode exists.** A hive always presents an identity, so a
|
||||
missing credential is a build error rather than a quieter fallback — the
|
||||
collector has no anonymous route to accept samples on, and every path it serves
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ neither is a renaming of the other.
|
|||
| the secret publisher's store leaf (cert + key) | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on the host that runs the secret store, once, subject `deploy.bao.secretPublisherCommonName`. `swarm-bao-secret-publisher-policy.service` writes the `swarm-secret-publisher` role: create and update under `swarm/hives/*`, `swarm/services/*` and the controller's client, and read on authelia's forwarder's client secret alone | `/var/lib/swarm-bao-pki/secret-publisher.pem` and `secret-publisher-key.pem` on the host, handed into authelia's container as nspawn system credentials, which `swarm-secret-publish` and the forwarder's `swarm-journal-forwarder-oidc` load with `LoadCredential`. nspawn won't start authelia's container while a named file is missing | **the operator** copies the leaf to authelia's host, beside the hive's own store leaf, where `glue-secret-publisher-bao-identity.nix` looks, or names it in `deploy.swarm-secret-publisher.baoClientCertFile` / `baoClientKeyFile` |
|
||||
| the swarm controller's copy of its OIDC secret | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`) |
|
||||
| a hive's copy of its own OIDC secret | hive-c0re reads it out of the swarm secret store when it first connects to the queue, under the hive's own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) |
|
||||
| the hive collector's copy of its hive's OIDC secret | `swarm-bao-hive-otel-oidc.service` reads it out of the swarm secret store under the hive's own leaf, **on every hive that runs the collector and holds that leaf** | `/var/lib/hive-otel-oidc/hive-<name>.secret` on the hive's host, 0400 root, loaded by `LoadCredential` | same unit, same path, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) |
|
||||
| the hive collector's copy of its hive's OIDC secret | `swarm-bao-hive-otel-oidc.service` reads it out of the swarm secret store under the hive's own leaf, **on every hive that runs the collector and holds that leaf** | `/run/hive-otel-oidc/hive-<name>.secret` on the hive's host tmpfs, 0400 root, loaded by `LoadCredential` — never on persistent disk, so a reboot fetches it again | same unit, same path, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) |
|
||||
| the swarm controller's forge token | the forge's `forgejo-swarm-controller-account`, in-container, once | the swarm secret store at `swarm/controller/swarm-controller/forge/token`, written by the mint under the forge publisher's leaf, which the forge container loads as a credential; the controller holds it in memory only. `/var/lib/forgejo/swarm-controller-token-name` in the forge container holds its name, the minter's record that it has minted | nothing to place beyond the controller's store leaf. A forge host without the store needs the publisher's leaf placed (`deploy.bao.forgePublisherClientCertFile`) |
|
||||
| the swarm controller's Grafana token | `swarm-bao-grafana-controller-token.service`, **on the host that runs Grafana**, as Grafana's built-in admin over its socket: it creates the `swarm-controller` service account as a Viewer (and sets it back to Viewer on every run), mints a token for it and writes it to the swarm secret store at `swarm/controller/swarm-controller/grafana/token` under Grafana's leaf. Re-run every ten minutes: it keeps a stored token Grafana accepts, and replaces one Grafana refuses with 401 (or a missing one), deleting every other token of that account first | the swarm secret store only; the controller reads it for `GET /api/alerts` and holds it in memory, re-reading the store once when Grafana answers 401 | nothing to place: Grafana's leaf is already required to run Grafana. Until the unit has run, `/api/alerts` answers 503 naming the empty path |
|
||||
| an agent's claude login (`.credentials.json`) | `swarm-claude-login.service`, in its container, when an operator completes `claude auth login` from the agent's card in the swarm UI; it stores the file as a new version at `swarm/agents/<agent>/claude-credentials` under its own leaf, which may create and update that path and read nothing under an agent | the swarm secret store at that path; the agent's `hive-agent` reads it under the agent's own leaf every 30 seconds while it waits for a login and writes it to `~/.claude/.credentials.json` | the same read, co-located or not; nothing for an operator to place beyond the agent's store leaf. When the store's copy and the agent's differ the KV version decides: the agent installs only a version newer than the last it installed (`claude-credentials-version` under the harness dir), because claude rotates the refresh token in its own copy and the stored one goes stale. The agent ignores a stale or equal version, and a relogin writes a newer one — [claude-login.md](claude-login.md) |
|
||||
|
|
@ -87,7 +87,10 @@ own secret out of the store, under its own identity.
|
|||
The **hive's** collector authenticates to the swarm's collector as its own hive,
|
||||
so it reads its secret as the hive: `swarm-bao-hive-otel-oidc.service` fetches
|
||||
`swarm/hives/<hive>/oidc/client` under the hive's own leaf
|
||||
(`deploy.bao.clientCertFile`), on every topology. A hive with no leaf has no
|
||||
(`deploy.bao.clientCertFile`), on every topology, into `/run/hive-otel-oidc` on
|
||||
the host's tmpfs. The file never reaches persistent disk and a reboot fetches
|
||||
it again; until the store answers, the collector runs on an empty file and the
|
||||
swarm refuses its exports. A hive with no leaf has no
|
||||
default, and the build stops at the collector's identity assertion unless
|
||||
`services.hyperhive.otel.clientSecretFile` names a file delivered some other way.
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,11 @@ let
|
|||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
otel = hyperhiveCfg.otel;
|
||||
|
||||
hiveOtelSecret = import ./lib/hive-otel-secret.nix {
|
||||
inherit (hyperhiveCfg.swarm.authelia) hiveClientPrefix;
|
||||
inherit (hyperhiveCfg) hiveName;
|
||||
};
|
||||
|
||||
havePair = cert: key: cert != null && key != null;
|
||||
|
||||
# Reader unit → the gate its own module defines it under.
|
||||
|
|
@ -44,9 +49,7 @@ let
|
|||
otel.collector.enable
|
||||
&& hyperhiveCfg.hiveName != null
|
||||
&& havePair baoDeploy.clientCertFile baoDeploy.clientKeyFile
|
||||
&&
|
||||
otel.clientSecretFile
|
||||
== "/var/lib/hive-otel-oidc/${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${toString hyperhiveCfg.hiveName}.secret";
|
||||
&& otel.clientSecretFile == hiveOtelSecret.path;
|
||||
# ./swarm-grafana.nix
|
||||
swarm-bao-grafana-oidc = deployCfg.grafana.enable;
|
||||
swarm-bao-grafana-admin = deployCfg.grafana.enable;
|
||||
|
|
|
|||
|
|
@ -26,7 +26,11 @@ let
|
|||
senderAuth = otel.clientSecretFile != null && hyperhiveCfg.hiveName != null;
|
||||
hiveLeaf =
|
||||
hyperhiveCfg.deploy.bao.clientCertFile != null && hyperhiveCfg.deploy.bao.clientKeyFile != null;
|
||||
fetchedSecretPath = "/var/lib/hive-otel-oidc/${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${toString hyperhiveCfg.hiveName}.secret";
|
||||
fetchedSecretPath =
|
||||
(import ./lib/hive-otel-secret.nix {
|
||||
inherit (hyperhiveCfg.swarm.authelia) hiveClientPrefix;
|
||||
inherit (hyperhiveCfg) hiveName;
|
||||
}).path;
|
||||
fetches = senderAuth && hiveLeaf && otel.clientSecretFile == fetchedSecretPath;
|
||||
waits = senderAuth && !fetches;
|
||||
in
|
||||
|
|
|
|||
22
nix/host-modules/lib/hive-otel-secret.nix
Normal file
22
nix/host-modules/lib/hive-otel-secret.nix
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# Where `swarm-bao-hive-otel-oidc` lands this hive's OIDC client secret on
|
||||
# the host, and where it sat before it moved. Spelled once because
|
||||
# ../otel.nix, ../glue-hive-otel-authelia.nix and
|
||||
# ../glue-bao-readers-store-order.nix each compare `otel.clientSecretFile`
|
||||
# against it to decide whether the fetch unit exists; a copy that drifts makes
|
||||
# that comparison false and the secret stops refreshing.
|
||||
#
|
||||
# `path` is on /run (tmpfs): the secret never reaches persistent disk and the
|
||||
# unit fetches it again after a reboot.
|
||||
#
|
||||
# Pure function — NOT a NixOS module. Call it from a module's `let`:
|
||||
#
|
||||
# hiveOtelSecret = import ./lib/hive-otel-secret.nix { inherit (cfg) hiveClientPrefix hiveName; };
|
||||
{ hiveClientPrefix, hiveName }:
|
||||
let
|
||||
file = "${hiveClientPrefix}${toString hiveName}.secret";
|
||||
in
|
||||
{
|
||||
dir = "/run/hive-otel-oidc";
|
||||
path = "/run/hive-otel-oidc/${file}";
|
||||
legacyPath = "/var/lib/hive-otel-oidc/${file}";
|
||||
}
|
||||
|
|
@ -46,7 +46,11 @@ let
|
|||
&& config.services.hyperhive.deploy.bao.clientKeyFile != null;
|
||||
|
||||
# Where `swarm-bao-hive-otel-oidc` lands the secret on this host.
|
||||
fetchedSecretPath = "/var/lib/hive-otel-oidc/${config.services.hyperhive.swarm.authelia.hiveClientPrefix}${toString config.services.hyperhive.hiveName}.secret";
|
||||
hiveOtelSecret = import ./lib/hive-otel-secret.nix {
|
||||
inherit (config.services.hyperhive.swarm.authelia) hiveClientPrefix;
|
||||
inherit (config.services.hyperhive) hiveName;
|
||||
};
|
||||
fetchedSecretPath = hiveOtelSecret.path;
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
|
|
@ -188,7 +192,7 @@ in
|
|||
type = lib.types.nullOr lib.types.str;
|
||||
default =
|
||||
if hiveLeaf && config.services.hyperhive.hiveName != null then fetchedSecretPath else null;
|
||||
defaultText = lib.literalExpression ''"/var/lib/hive-otel-oidc/''${swarm.authelia.hiveClientPrefix}''${hiveName}.secret" when deploy.bao.clientCertFile and clientKeyFile are set, else null'';
|
||||
defaultText = lib.literalExpression ''"/run/hive-otel-oidc/''${swarm.authelia.hiveClientPrefix}''${hiveName}.secret" when deploy.bao.clientCertFile and clientKeyFile are set, else null'';
|
||||
example = "/var/lib/secrets/hive-telemetry.secret";
|
||||
description = ''
|
||||
Absolute path to this hive's OAuth2 client secret, used to
|
||||
|
|
@ -830,7 +834,7 @@ in
|
|||
# Root-owned 0400: the collector runs under `DynamicUser`, so there is
|
||||
# no stable uid to hand the file to, and `LoadCredential` reads it as
|
||||
# root before the sandbox exists.
|
||||
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf fetchedSecretPath)}
|
||||
install -d -m 0700 ${lib.escapeShellArg hiveOtelSecret.dir}
|
||||
if secret_differs ${lib.escapeShellArg fetchedSecretPath} "$secret"; then
|
||||
atomic_write_secret 0400 "" ${lib.escapeShellArg fetchedSecretPath} "$secret"
|
||||
fi
|
||||
|
|
@ -843,6 +847,29 @@ in
|
|||
storeRefreshTimer "re-read this hive's telemetry OIDC client secret from the swarm secret store"
|
||||
);
|
||||
|
||||
systemd.tmpfiles.rules =
|
||||
lib.optionals fetches [
|
||||
"d ${hiveOtelSecret.dir} 0700 root root -"
|
||||
# `LoadCredential=` fails the unit when the file is missing; an empty
|
||||
# one lets the collector start and be refused by the stores until the
|
||||
# fetch lands.
|
||||
"f ${fetchedSecretPath} 0400 root root -"
|
||||
]
|
||||
# Unless a deployment names that very path itself.
|
||||
++ lib.optional (
|
||||
hiveName != null && otel.clientSecretFile != hiveOtelSecret.legacyPath
|
||||
) "r ${hiveOtelSecret.legacyPath}";
|
||||
|
||||
# The fetch unit writes `fetchedSecretPath` now, so a file named at the
|
||||
# old default is delivered by nobody and never refreshed.
|
||||
warnings = lib.optional (hiveName != null && otel.clientSecretFile == hiveOtelSecret.legacyPath) ''
|
||||
services.hyperhive.otel.clientSecretFile is ${hiveOtelSecret.legacyPath}, the path
|
||||
swarm-bao-hive-otel-oidc.service used to write. That unit now writes
|
||||
${fetchedSecretPath} and, with the option set to anything else, does
|
||||
not exist, so nothing refreshes the file. Remove the setting to fetch
|
||||
from the swarm secret store, or deliver the file yourself.
|
||||
'';
|
||||
|
||||
assertions = [
|
||||
{
|
||||
# A hive authenticates to the swarm's collector as itself — there
|
||||
|
|
|
|||
|
|
@ -26,10 +26,11 @@ let
|
|||
|
||||
# This hive's own collector, which is a HOST service — unlike the swarm
|
||||
# tier's, which lives in a container and is read through `otelSettings`.
|
||||
# The IdP is here too, with a hand-named secret file: the collector's wait
|
||||
# for that file exists beside authelia's container.
|
||||
# The IdP is here too, with its secret file named at the path the fetch
|
||||
# unit used to write: the collector's wait for that file exists beside
|
||||
# authelia's container.
|
||||
hiveOtel = hiveWithController {
|
||||
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
|
||||
otel.clientSecretFile = "/var/lib/hive-otel-oidc/hive-h1.secret";
|
||||
deploy.authelia.enable = true;
|
||||
};
|
||||
|
||||
|
|
@ -239,6 +240,28 @@ let
|
|||
&& builtins.elem "opentelemetry-collector.service" u.wantedBy
|
||||
&& builtins.elem "opentelemetry-collector.service" u.before;
|
||||
}
|
||||
{
|
||||
# The secret lives on tmpfs: an empty placeholder gives `LoadCredential`
|
||||
# a source before the first fetch, and the copy the fetch used to leave
|
||||
# on persistent disk is removed. The fixture naming that old path as
|
||||
# its secret keeps it, and is warned that nothing refreshes it.
|
||||
name = "the hive collector's fetched secret is an empty file on /run and the legacy file is removed";
|
||||
ok =
|
||||
let
|
||||
run = "/run/hive-otel-oidc/hive-h1.secret";
|
||||
legacy = "/var/lib/hive-otel-oidc/hive-h1.secret";
|
||||
rules = m: m.systemd.tmpfiles.rules;
|
||||
warned = m: lib.any (w: lib.hasInfix "swarm-bao-hive-otel-oidc.service used to write" w) m.warnings;
|
||||
in
|
||||
hiveOtelLeaf.services.hyperhive.otel.clientSecretFile == run
|
||||
&& builtins.elem "d /run/hive-otel-oidc 0700 root root -" (rules hiveOtelLeaf)
|
||||
&& builtins.elem "f ${run} 0400 root root -" (rules hiveOtelLeaf)
|
||||
&& builtins.elem "r ${legacy}" (rules hiveOtelLeaf)
|
||||
&& !(warned hiveOtelLeaf)
|
||||
&& !(builtins.elem "r ${legacy}" (rules hiveOtel))
|
||||
&& !(builtins.elem "f ${run} 0400 root root -" (rules hiveOtel))
|
||||
&& warned hiveOtel;
|
||||
}
|
||||
{
|
||||
# `LoadCredential` copies the file at the collector's start, so a value
|
||||
# landed later reaches it only through a restart. Retries every 30s
|
||||
|
|
@ -253,7 +276,7 @@ let
|
|||
u.serviceConfig.Restart == "on-failure"
|
||||
&& u.serviceConfig.RestartSec == 30
|
||||
&& toString u.unitConfig.StartLimitBurst == "2880"
|
||||
&& lib.hasInfix "refresh_unit opentelemetry-collector.service /var/lib/hive-otel-oidc/hive-h1.secret" s
|
||||
&& lib.hasInfix "refresh_unit opentelemetry-collector.service /run/hive-otel-oidc/hive-h1.secret" s
|
||||
&& lib.hasInfix "ActiveEnterTimestamp" s
|
||||
&& lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' s;
|
||||
}
|
||||
|
|
@ -337,20 +360,20 @@ let
|
|||
ok =
|
||||
let
|
||||
waitsFor =
|
||||
machine:
|
||||
file: machine:
|
||||
let
|
||||
c = machine.systemd.services.opentelemetry-collector;
|
||||
w = machine.systemd.services.hive-otel-client-secret;
|
||||
in
|
||||
builtins.elem "hive-otel-client-secret.service" c.requires
|
||||
&& builtins.elem "hive-otel-client-secret.service" c.after
|
||||
&& lib.hasInfix "/var/lib/hive-otel-oidc/client.secret" w.script;
|
||||
&& lib.hasInfix file w.script;
|
||||
in
|
||||
!(hiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
|
||||
&& !(baoWithHiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
|
||||
&& builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
||||
&& waitsFor hiveOtel
|
||||
&& waitsFor baoWithHiveOtel;
|
||||
&& waitsFor "/var/lib/hive-otel-oidc/hive-h1.secret" hiveOtel
|
||||
&& waitsFor "/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel;
|
||||
}
|
||||
{
|
||||
# ../host-modules/glue-hive-otel-authelia.nix orders the wait after
|
||||
|
|
|
|||
Loading…
Reference in a new issue