vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive telemetry collector: fetch its OIDC secret from the store under the hive's own leaf

Browse source 
The hive collector's `oauth2client` secret came from a `clientSecretFile`
defaulting to authelia's own minted file on the IdP's host, with a
`hive-otel-client-secret` oneshot that only waited up to 120s for that
file to exist; any other hive had to copy the file across by hand.

`swarm-bao-hive-otel-oidc` now logs in with the hive's leaf
(`deploy.bao.clientCertFile`), reads `swarm/hives/<hive>/oidc/client`
(written by the publisher, granted by the hive leaf's
`swarm/hives/<hive>/*` read), writes it root 0400 when it differs, and
restarts the collector when the file is newer than its last start.
Retries every 30s for 24h (`lib/store-retry.nix`), like the matrix and
forge fetches. `clientSecretFile` defaults to that file wherever the hive
leaf is set and stays as an override; an overridden file renders no fetch.

otelcol-contrib 0.151.0's oauth2client re-reads `client_secret_file` on
every token fetch, but the path is `LoadCredential`'s start-time copy, so a
changed secret still needs a restart. `lib/refresh-consumer.nix` gains
`refresh_unit` for a host unit; `refresh_consumer` now calls it with
`--machine`.

nats-authelia's "consumer of the host client-secret dir" case pointed at
the collector's old default; it now reads the publisher's path unit.

Refs #5015
This commit is contained in:
atlas 2026-10-10 01:09:20 +02:00
commit 7c50568db1
7 changed files with 282 additions and 114 deletions
Download patch file Download diff file Expand all files Collapse all files

95
nix/module-eval/hive-otel.nix
View file

@ -29,6 +29,24 @@ let
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
};
# A hive holding its own store leaf, placed by hand, and naming no secret
# file: the collector's secret comes from the store under that leaf.
hiveOtelLeaf = hive {
otel.enable = true;
deploy.bao.clientCertFile = "/etc/pki/hive.pem";
deploy.bao.clientKeyFile = "/etc/pki/hive-key.pem";
};
# The IdP, the store and the collector on one host: the publisher that
# writes the secret and the unit that reads it, in one config.
hiveOtelAllLocal = hive {
otel.enable = true;
deploy.bao.enable = true;
deploy.authelia.enable = true;
};
hiveOtelFetch = machine: machine.systemd.services.swarm-bao-hive-otel-oidc;
hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines;
hiveOtelSettings = hiveOtel.services.opentelemetry-collector.settings;
@ -173,6 +191,83 @@ let
in
hivePort != swarmPort && hivePort != baoPort && swarmPort != baoPort;
}
{
# The collector authenticates as the hive, so it reads its secret as
# the hive: the leaf `deploy.bao.clientCertFile` names, whose policy
# grants `swarm/hives/<hive>/*`. The last two clauses tie the unit's
# output to what the collector loads.
name = "the hive collector's secret is fetched from the store under the hive's own leaf";
ok =
let
u = hiveOtelFetch hiveOtelLeaf;
b = hiveOtelLeaf.services.hyperhive.deploy.bao;
file = hiveOtelLeaf.services.hyperhive.otel.clientSecretFile;
s = builtins.unsafeDiscardStringContext u.script;
in
hiveOtelLeaf.systemd.services ? swarm-bao-hive-otel-oidc
&& u.environment.BAO_CLIENT_CERT == b.clientCertFile
&& u.environment.BAO_CLIENT_KEY == b.clientKeyFile
&& lib.hasInfix "bao login -method=cert" s
&& lib.hasInfix "bao kv get -field=value secret/swarm/hives/h1/oidc/client" s
&& lib.hasInfix "atomic_write_secret 0400 \"\" ${file} " s
&& builtins.elem "swarm-client.secret:${file}" hiveOtelLeaf.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
&& builtins.elem "opentelemetry-collector.service" u.wantedBy
&& builtins.elem "opentelemetry-collector.service" u.before;
}
{
# `LoadCredential` copies the file at the collector's start, so a value
# landed later reaches it only through a restart. Retries every 30s
# (./lib/store-retry.nix) while the store is down or the secret
# unpublished.
name = "the hive collector's fetch retries and restarts the collector when the file is newer";
ok =
let
u = hiveOtelFetch hiveOtelLeaf;
s = builtins.unsafeDiscardStringContext u.script;
in
u.serviceConfig.Restart == "on-failure"
&& u.serviceConfig.RestartSec == 30
&& toString u.unitConfig.StartLimitBurst == "2880"
&& lib.hasInfix "refresh_unit opentelemetry-collector.service /var/lib/hive-otel-oidc/hive-h1.secret" s
&& lib.hasInfix "ActiveEnterTimestamp" s
&& lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' s;
}
{
# The store is the one route, on the IdP's own host too: no default into
# authelia's tree and no unit waiting for a file to appear there.
name = "the hive collector reads no secret out of authelia's tree and waits on no copied file";
ok =
let
file = hiveOtelAllLocal.services.hyperhive.otel.clientSecretFile;
c = hiveOtelAllLocal.systemd.services.opentelemetry-collector;
in
!(hiveOtelAllLocal.systemd.services ? hive-otel-client-secret)
&& !(hiveOtelLeaf.systemd.services ? hive-otel-client-secret)
&& !(lib.hasInfix "authelia" file)
&& hiveOtelAllLocal.systemd.services ? swarm-bao-hive-otel-oidc
&& !(builtins.elem "hive-otel-client-secret.service" (c.requires or [ ]));
}
{
# The writer and the reader spell the path independently; this is the
# one place that sees both rendered.
name = "the hive collector reads the path the secret publisher writes";
ok =
let
fetch = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelAllLocal).script;
publish = builtins.unsafeDiscardStringContext hiveOtelAllLocal.systemd.services.swarm-secret-publish.script;
path = "secret/swarm/hives/h1/oidc/client";
in
lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish;
}
{
# A secret delivered some other way is the operator's: no unit fetches
# into a file the collector does not load, or restarts it on one.
name = "a hand-named collector secret renders no store fetch";
ok =
!(hiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
&& !(baoWithHiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
&& builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential;
}
];
in
runGroup "hive-otel" cases

7
nix/module-eval/nats-authelia.nix
View file

@ -44,6 +44,9 @@ let
swarm.authelia.usersFile = "/var/lib/test-authelia/users.yml";
swarm.authelia.package = pkgs.emptyDirectory;
swarm.authelia.bridgePackage = pkgs.emptyDirectory;
# Renders the publisher, which reads the host client-secret dir.
deploy.swarm-secret-publisher.baoClientCertFile = "/etc/pki/publisher.pem";
deploy.swarm-secret-publisher.baoClientKeyFile = "/etc/pki/publisher-key.pem";
};
# The queue with a store identity for its responder, placed by hand: the
@ -467,7 +470,9 @@ let
# `swarm.authelia` instead of `deploy.authelia`. Several modules read this
# through an `autheliaCfg` alias, where a path-shaped grep does not see it.
name = "a consumer of authelia's host client-secret dir renders it from the deploy namespace";
ok = lib.hasPrefix "/var/lib/nixos-containers/swarm-authelia/var/lib/authelia-swarm/oidc-clients/" autheliaOldPath.services.hyperhive.otel.clientSecretFile;
ok =
autheliaOldPath.systemd.paths.swarm-secret-publish.pathConfig.PathChanged
== "/var/lib/nixos-containers/swarm-authelia/var/lib/authelia-swarm/oidc-clients";
}
{
# authelia exits at startup on `users: {}`, so an empty database, and a