vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

keep authelia's oidc client secrets in the store, not on disk

Browse source 
The mint creates each client's secret in the swarm secret store and writes only the digest to disk. It mints only when the store answers and holds none. A new unit moves the plaintext files an earlier release left into the store and removes each once the store holds the same value. The bridge reads its secret from the store into /run before each start. The secret publisher unit and its path unit are gone.
This commit is contained in:
atlas 2026-10-30 03:25:57 +01:00 • committed by mara
commit d30cb7d122
36 changed files with 1086 additions and 486 deletions
Download patch file Download diff file Expand all files Collapse all files

5
nix/checks.nix
View file

@ -88,6 +88,11 @@ in
# Runs ./host-modules/lib/store-seeded-secret.nix's seed, store-wins and mint
# decisions against a stub `bao` that keeps its leaves in files.
script-test-store-seeded-secret = import ./script-tests/store-seeded-secret.nix { inherit pkgs; };
# Runs ./host-modules/lib/oidc-client-secrets.nix's mint and migration
# decisions for authelia's client secrets against stub `bao` and `authelia`:
# what is minted, stored and removed, and what is left alone when the store
# cannot answer.
script-test-oidc-client-secrets = import ./script-tests/oidc-client-secrets.nix { inherit pkgs; };
module-eval-nats-tls = import ./module-eval/nats-tls.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;

180
nix/container-modules/swarm-secret-publisher.nix
View file

@ -1,180 +0,0 @@
# The unit that puts swarm-level secrets into the swarm's secret store, so
# whoever needs one can read it there: a hive, a service or the controller its
# own. It mints nothing: every
# value is an OIDC client secret authelia minted, and this unit copies it.
#
# Imported inside authelia's container, beside the mint: it reads each secret
# where authelia writes it. The host module that declares the container sets
# the options below, as for ./swarm-journal-forwarder.nix.
#
# ⚠️ "Whoever", including a reader on authelia's own host. A swarm service's
# secret goes into the store even when the service runs beside authelia,
# because its reader fetches it from the store in every deployment —
# ../host-modules/swarm-grafana.nix states the ruling that made that the only
# route. Publishing "only when the reader is elsewhere" would be a second shape
# of this unit, gated on a fact about another host, to save a round trip on
# the one host that can afford it.
#
# ⚠️ ITS OWN STORE IDENTITY, not swarm-controller's. That principal may rewrite
# every hive's policy and login role; a unit that writes a handful of secrets
# has no business holding it. ../host-modules/swarm-bao.nix grants this one
# `create`/`update` under the hive prefix and nothing else — no `read`.
#
# ⚠️ THE SECRET NEVER REACHES `argv`. `bao` is an external binary, so every
# argument is world-readable in /proc for the life of the call — the value is
# passed as `@<path>` and read by bao itself.
#
# ⚠️ Nothing in the container waits for this unit. Authelia is the swarm's IdP
# and starts whether or not the store answers; a store that is down or sealed
# is a failed run that restarts.
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.hyperhive.swarmSecretPublisher;
in
{
options.services.hyperhive.swarmSecretPublisher = {
enable = lib.mkEnableOption "copying the OIDC client secrets authelia mints into the swarm secret store";
clientSecretDir = lib.mkOption {
type = lib.types.str;
description = ''
Directory authelia mints each client's plaintext into, as
`<client id>.secret`. A path inside the container.
'';
};
secrets = lib.mkOption {
type = lib.types.listOf (
lib.types.submodule {
options = {
clientId = lib.mkOption {
type = lib.types.str;
description = "The minted client, whose `<client id>.secret` is copied.";
};
path = lib.mkOption {
type = lib.types.str;
example = "secret/swarm/services/swarm-grafana/oidc/client";
description = "KV path its reader fetches the value from.";
};
};
}
);
description = "Every secret this unit copies, and where to.";
};
store = {
package = lib.mkOption {
type = lib.types.package;
description = "The `bao` CLI the unit logs in and writes with.";
};
addr = lib.mkOption {
type = lib.types.str;
description = "The swarm secret store's address, by name.";
};
clientCertFile = lib.mkOption {
type = lib.types.str;
description = ''
The publisher's store identity, as a path inside the container. Its
subject must be
{option}`services.hyperhive.deploy.bao.secretPublisherCommonName`.
A path, never a value.
'';
};
clientKeyFile = lib.mkOption {
type = lib.types.str;
description = "Private key for `store.clientCertFile`. A path, never a value.";
};
caFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "CA the store's server certificate chains to; null means the system trust store.";
};
};
};
config = lib.mkIf cfg.enable {
# Re-publish when authelia rotates a secret. The mint writes the file, so
# the file is the event — there is no signal from authelia to subscribe to.
systemd.paths.swarm-secret-publish = {
description = "watch for minted OIDC client secrets to publish";
wantedBy = [ "multi-user.target" ];
pathConfig = {
PathChanged = cfg.clientSecretDir;
Unit = "swarm-secret-publish.service";
};
};
systemd.services.swarm-secret-publish = {
description = "publish the swarm's minted secrets to the swarm secret store";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
path = [
cfg.store.package
pkgs.coreutils
];
# ⚠️ No `RemainAfterExit`. A oneshot that stays `active (exited)` turns
# every later start from the path unit above into a no-op.
serviceConfig = {
Type = "oneshot";
# Bounded here rather than left to systemd's default, so the number a
# boot waits on is in the file that waits. A sealed store answers on
# the port and never answers the write.
TimeoutStartSec = 60;
Restart = "on-failure";
RestartSec = 30;
};
environment = {
BAO_ADDR = cfg.store.addr;
BAO_CLIENT_CERT = cfg.store.clientCertFile;
BAO_CLIENT_KEY = cfg.store.clientKeyFile;
}
// lib.optionalAttrs (cfg.store.caFile != null) {
BAO_CACERT = cfg.store.caFile;
};
script = ''
set -euo pipefail
# Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
# variables above only decide which certificate the TLS handshake
# presents; without a token `bao` asks its token helper instead, and
# that is a `sh` this unit's `path` does not carry. `-token-only`
# answers on stdout and skips the helper on both sides.
#
# Unhandled on purpose: `Restart=on-failure` above is what a store that
# cannot authenticate this container should get. Degrading here would
# report "published 0" as an ordinary quiet day.
BAO_TOKEN="$(bao login -method=cert -token-only)"
export BAO_TOKEN
published=0
skipped=0
${lib.concatMapStringsSep "\n" (s: ''
src=${lib.escapeShellArg "${cfg.clientSecretDir}/${s.clientId}.secret"}
if [ -s "$src" ]; then
# `value=@$src` hands bao the PATH: bao opens the file itself, so
# the plaintext is never an argument of this process. Writing it as
# `value="$(cat "$src")"` would publish it to /proc for anyone in
# the container to read.
bao kv put ${lib.escapeShellArg s.path} \
value=@"$src"
published=$((published + 1))
else
# Not an error: authelia mints on its FIRST BOOT, so an absent file
# is "not yet", and the path unit above re-runs this when it lands.
# A client registered nowhere in this swarm is never minted at all.
echo "no minted secret at $src yet; the path unit will re-run this" >&2
skipped=$((skipped + 1))
fi
'') cfg.secrets}
echo "published $published client secret(s), skipped $skipped"
'';
};
};
}

2
nix/host-modules/deploy.nix
View file

@ -49,7 +49,7 @@ let
statusSecretRemoved = ''
hive-c0re reads this hive's status-publishing client secret from the
swarm's secret store, under the hive's own certificate
(deploy.bao.clientCertFile), and swarm-secret-publish in authelia's container
(deploy.bao.clientCertFile), and authelia's mint, in its container,
puts it there. Remove this definition; the file it named is read by
nothing now and can be deleted.
'';

2
nix/host-modules/glue-builder-oidc-client.nix
View file

@ -2,7 +2,7 @@
#
# ONE PAIRING PER FILE — swarm-builder ← authelia, and nothing else. Deleting
# this leaves a builder whose token requests authelia answers with a 401, and
# no minted secret for swarm-secret-publish to put in the store.
# no minted secret for authelia's mint to put in the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the builder,
# for ./glue-controller-oidc-client.nix's reason: a client is a row in THIS

2
nix/host-modules/glue-claude-login-oidc-client.nix
View file

@ -3,7 +3,7 @@
#
# ONE PAIRING PER FILE — swarm-claude-login ← authelia, and nothing else.
# Deleting this leaves a minter whose token requests authelia answers with a
# 401, and no minted secret for swarm-secret-publish to put in the store.
# 401, and no minted secret for authelia's mint to put in the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the minter,
# for ./glue-controller-oidc-client.nix's reason: a client is a row in THIS

2
nix/host-modules/glue-controller-oidc-client.nix
View file

@ -2,7 +2,7 @@
#
# ONE PAIRING PER FILE — swarm-controller ← authelia, and nothing else. Deleting
# this leaves a controller whose token requests authelia answers with a 401, and
# no minted secret for swarm-secret-publish to put in the store.
# no minted secret for authelia's mint to put in the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the controller.
# A client is a row in THIS host's provider config, so it is declared where that

2
nix/host-modules/glue-matrix-oidc-client.nix
View file

@ -3,7 +3,7 @@
#
# ONE PAIRING PER FILE — matrix ← authelia, and nothing else. Deleting this
# leaves a homeserver whose SSO login authelia has never heard of, and no
# minted secret for swarm-secret-publish to put in the store.
# minted secret for authelia's mint to put in the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the
# homeserver. A client is a row in THIS host's provider config, so it is

2
nix/host-modules/glue-nats-oidc-client.nix
View file

@ -2,7 +2,7 @@
#
# ONE PAIRING PER FILE — swarm-nats ← authelia, and nothing else. Deleting this
# leaves a queue responder whose introspection calls authelia refuses, so it
# denies every client, and no minted secret for swarm-secret-publish to put in
# denies every client, and no minted secret for authelia's mint to put in
# the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the queue. A

2
nix/host-modules/glue-notify-router-oidc-client.nix
View file

@ -3,7 +3,7 @@
#
# ONE PAIRING PER FILE — swarm-notify-router ← authelia, and nothing else.
# Deleting this leaves a router whose token requests authelia answers with a
# 401, and no minted secret for swarm-secret-publish to put in the store.
# 401, and no minted secret for authelia's mint to put in the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the router,
# for ./glue-controller-oidc-client.nix's reason: a client is a row in THIS

2
nix/host-modules/glue-review-classifier-oidc-client.nix
View file

@ -3,7 +3,7 @@
#
# ONE PAIRING PER FILE — swarm-review-classifier ← authelia, and nothing else.
# Deleting this leaves a classifier whose token requests authelia answers with
# a 401, and no minted secret for swarm-secret-publish to put in the store.
# a 401, and no minted secret for authelia's mint to put in the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the
# classifier, for ./glue-controller-oidc-client.nix's reason: a client is a row

2
nix/host-modules/hive-forge/default.nix
View file

@ -40,7 +40,7 @@ let
# root. Removed by the tmpfiles rule below.
oldOidcSecretPath = "/var/lib/nixos-containers/hive-forge/var/lib/forgejo-oidc/${cfg.sso.clientId}.secret";
# Where swarm-secret-publish in authelia's container leaves this client's
# Where authelia's mint, in its container, stores this client's
# secret.
# The forge publisher's store role grants read on this path.
oidcStorePath = "secret/swarm/services/${cfg.sso.clientId}/oidc/client";

4
nix/host-modules/hive-forge/service.nix
View file

@ -164,8 +164,8 @@ in
Read-only because three hosts have to agree on it: authelia
registers the client (`glue-forge-otel-oidc-client.nix`), the
swarm collector admits it on the container's receiver
(`swarm.otel.serviceProducers`), and the secret publisher carries
its secret to the store.
(`swarm.otel.serviceProducers`), and authelia's mint stores
its secret in the store.
'';
};
};

4
nix/host-modules/hive-matrix-service.nix
View file

@ -175,8 +175,8 @@ in
Read-only because three hosts have to agree on it: authelia
registers the client (`glue-matrix-otel-oidc-client.nix`), the
swarm collector admits it on the container's receiver
(`swarm.otel.serviceProducers`), and the secret publisher carries
its secret to the store.
(`swarm.otel.serviceProducers`), and authelia's mint stores
its secret in the store.
'';
};
};

2
nix/host-modules/hive-matrix.nix
View file

@ -52,7 +52,7 @@ let
# Where the secret sat on persistent disk before it moved to /run.
legacyHostSecretPath = "/var/lib/nixos-containers/hive-matrix/var/lib/tuwunel-oidc/${cfg.sso.clientId}.secret";
# Where swarm-secret-publish in authelia's container leaves this client's
# Where authelia's mint, in its container, stores this client's
# secret.
# matrix-ctl's store role grants read on this path.
storeSecretPath = "secret/swarm/services/${cfg.sso.clientId}/oidc/client";

129
nix/host-modules/lib/oidc-client-secrets.nix Normal file
View file

@ -0,0 +1,129 @@
# Shared shell steps for authelia's OIDC client secrets, whose plaintext half
# lives in the swarm secret store and never on disk. authelia keeps only a
# digest of each, in `$clients/<id>.digest`, and starts from it whether or not
# the store answers.
#
# `mint_client_secret` is the only writer of a new secret. It creates the
# store's leaf (`-cas=0`, so an existing value is never replaced) before it
# writes the digest, so a digest on disk always has a secret behind it. It mints
# only after a read that says the store holds none; a store that cannot be read
# never mints, because a second secret would rotate the first.
#
# `migrate_client_secret` moves the plaintext file an earlier release left in
# `$clients/<id>.secret` into the store and removes it once a read-back shows
# the store holds the same value. It never overwrites a store value and never
# removes a file the store does not match.
#
# Return statuses of both: 0 done, 1 retry later (the store could not answer),
# 3 an operator has to look (logged at `err`; retrying changes nothing).
#
# Pure function — NOT a NixOS module. Call it after `./store-seeded-secret.nix`
# and `./atomic-write-secret.nix`, with `store_init` and `store_login` run and
# `$clients` set to the digest directory. Needs `authelia`, `bao`, `coreutils`
# and `sed` on the caller's `path`.
{ }:
''
# <leaf>. Sets CLIENT_SECRET_VALUE. 0: the store holds a value. 1: it holds
# none. 2: it cannot tell (not logged in, down, or refusing the read).
client_secret_read() {
CLIENT_SECRET_VALUE=""
[ "$STORE_UP" = 1 ] || return 2
if CLIENT_SECRET_VALUE="$(store_bao kv get -field=value "$1" 2>"$STORE_ERR")" \
&& [ -n "$CLIENT_SECRET_VALUE" ]; then
return 0
fi
CLIENT_SECRET_VALUE=""
if grep -q '^No value found at ' "$STORE_ERR"; then
return 1
fi
cat "$STORE_ERR" >&2
return 2
}
# <client id> <leaf>. Sets MINTED when it created a secret.
mint_client_secret() {
local id="$1" leaf="$2" dig="$clients/$1.digest" legacy="$clients/$1.secret"
local state=0 out secret digest
if [ -e "$legacy" ]; then
if [ -s "$dig" ]; then
return 0
fi
echo "<3>oidc client $id: $legacy exists but its digest does not; not minting a second secret. Restore the digest, or remove the file and the store's leaf $leaf to mint again." >&2
return 3
fi
client_secret_read "$leaf" || state=$?
if [ "$state" = 0 ]; then
if [ ! -s "$dig" ]; then
echo "<3>oidc client $id: the store holds a secret at $leaf but $dig is missing; not minting a second secret. Restore the digest, or remove the leaf to mint again." >&2
return 3
fi
chmod 0600 "$dig"
return 0
fi
if [ "$state" = 2 ]; then
if [ -s "$dig" ]; then
chmod 0600 "$dig"
echo "oidc client $id: the store cannot be read; keeping its digest" >&2
return 0
fi
echo "oidc client $id: the store cannot be read and there is no digest; not minting" >&2
return 1
fi
echo "oidc client $id: the store holds no secret at $leaf; minting one"
out="$(authelia crypto hash generate pbkdf2 --variant sha512 --random)"
secret="$(printf '%s' "$out" | sed -n 's/^Random Password: *//p')"
digest="$(printf '%s' "$out" | sed -n 's/^Digest: *//p')"
if [ -z "$secret" ] || [ -z "$digest" ]; then
echo "<3>authelia crypto hash generate produced no secret/digest for $id" >&2
return 1
fi
if ! printf '%s' "$secret" | store_bao kv put -cas=0 "$leaf" value=- >/dev/null 2>"$STORE_ERR"; then
cat "$STORE_ERR" >&2
echo "could not create $leaf; reading it back in case another run created it" >&2
fi
client_secret_read "$leaf" || return 1
if [ "$CLIENT_SECRET_VALUE" != "$secret" ]; then
echo "<3>oidc client $id: $leaf holds a different value than the one minted; not writing a digest" >&2
return 3
fi
atomic_write_secret 0600 "" "$dig" "$digest" || return 1
MINTED=1
echo "minted client secret for $id into the store"
}
# <client id> <leaf>
migrate_client_secret() {
local id="$1" leaf="$2" legacy="$clients/$1.secret" value state=0
[ -e "$legacy" ] || return 0
value="$(< "$legacy")"
if [ -z "$value" ]; then
echo "<3>oidc client $id: $legacy is empty; keeping it" >&2
return 3
fi
client_secret_read "$leaf" || state=$?
if [ "$state" = 2 ]; then
echo "oidc client $id: the store cannot be read; keeping $legacy" >&2
return 1
fi
if [ "$state" = 0 ]; then
if [ "$CLIENT_SECRET_VALUE" != "$value" ]; then
echo "<3>oidc client $id: the store at $leaf and $legacy hold different secrets; keeping both" >&2
return 3
fi
rm -f "$legacy" || return 1
echo "oidc client $id: already in the store, removed the file $legacy"
return 0
fi
if ! printf '%s' "$value" | store_bao kv put -cas=0 "$leaf" value=- >/dev/null 2>"$STORE_ERR"; then
cat "$STORE_ERR" >&2
echo "oidc client $id: could not create $leaf; reading it back in case another run created it" >&2
fi
client_secret_read "$leaf" || return 1
if [ "$CLIENT_SECRET_VALUE" != "$value" ]; then
echo "<3>oidc client $id: the store at $leaf and $legacy hold different secrets; keeping both" >&2
return 3
fi
rm -f "$legacy" || return 1
echo "oidc client $id: stored in the store, removed the file $legacy"
}
''

2
nix/host-modules/otel.nix
View file

@ -314,7 +314,7 @@ in
# ./glue-hive-otel-authelia.nix restates this gate.
waits = senderAuth && !fetches;
secretWaitUnit = "hive-otel-client-secret";
# Written by swarm-secret-publish in authelia's container. The nix half of
# Written by authelia's mint, in its container. The nix half of
# `swarm_secret_client::queue::hive_client_path`.
storeSecretPath = "secret/swarm/hives/${toString hiveName}/oidc/client";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };

2
nix/host-modules/swarm-attic-options.nix
View file

@ -24,7 +24,7 @@ in
Read-only because three hosts have to agree on it: authelia registers
the client (`glue-attic-otel-oidc-client.nix`), the swarm collector
admits it on the container's receiver (`swarm.otel.serviceProducers`),
and the secret publisher carries its secret to the store.
and authelia's mint stores its secret in the store.
'';
};

4
nix/host-modules/swarm-authelia-service.nix
View file

@ -373,8 +373,8 @@ in
Read-only because three ends have to agree on it: authelia registers
the client, the swarm collector admits it on the container's receiver
(`swarm.otel.serviceProducers`), and the secret publisher carries its
secret to the store, where the forwarder reads it back under the
(`swarm.otel.serviceProducers`), and authelia's mint stores its
secret in the store, where the forwarder reads it back under the
publisher's identity.
'';
};

270
nix/host-modules/swarm-authelia.nix
View file

@ -237,7 +237,7 @@ let
publisherCfg.enable
&& publisherCfg.baoClientCertFile != null
&& publisherCfg.baoClientKeyFile != null;
publisherUnit = "swarm-secret-publish";
migrateUnit = "${unitName}-client-secrets";
forwarderFetchUnit = "swarm-journal-forwarder-oidc";
# The publisher's store identity, handed into the container as system
# credentials (credential id → host source). The forwarder's fetch presents
@ -465,39 +465,33 @@ let
''
);
# The OIDC half of the first-boot generator, kept out of the script
# body so neither is read through the other's indentation.
oidcGenScript = lib.optionalString oidcEnabled ''
${lib.optionalString (!publishes) ''
# The issuer key is the one secret here that is NOT interchangeable
# with a random blob: it *signs* id tokens, and every relying party
# verifies them against the public half served at `/jwks.json`. A
# symmetric secret cannot do that, so this one is an RSA pair.
#
# Rotating it invalidates every token already issued, which is why
# it is generated once and left alone — same reason as the session
# and storage keys above.
issuer=${lib.escapeShellArg stateDir}/oidc-issuer.key
if [ ! -s "$issuer" ]; then
openssl genrsa -out "$issuer" 4096
echo "generated $issuer"
fi
chmod 0600 "$issuer"
''}
# Each client's secret, minted once and kept as two files: the
# plaintext its relying party authenticates with, and the digest
# authelia compares against. The two live in different containers,
# so neither side can generate it alone — this is the only place
# that sees both.
#
# `--random` is why no plaintext ever reaches an argv: authelia
# generates the password itself and prints it beside its digest,
# so nothing has to be handed to a second process on a command
# line.
clients=${lib.escapeShellArg clientsDir}
mkdir -p "$clients"
chmod 0700 "$clients"
oidcClientSecrets = import ./lib/oidc-client-secrets.nix { };
# Every client with a secret to mint: a native client is public and has none.
secretClients = lib.filter (c: c.kind != "native") cfg.oidc.clients;
# Where a client's plaintext lives in the store: the path its reader fetches
# it from, which is `deploy.swarm-secret-publisher.secrets` for every client
# something reads. The rest, the bridge's own among them, sit with the
# services.
clientLeaf =
id:
let
listed = lib.findFirst (s: s.clientId == id) null publisherCfg.secrets;
in
if listed == null then "secret/swarm/services/${id}/oidc/client" else listed.path;
# Each secret is kept as two halves: the plaintext its relying party
# authenticates with, and the digest authelia compares against. The two live
# in different containers, so neither side can generate it alone, and this is
# the only place that sees both.
#
# `--random` is why no plaintext reaches an argv: authelia generates the
# password itself and prints it beside its digest.
#
# Without a store identity the plaintext has nowhere else to go, so it stays
# in a file beside the digest.
fileMintScript = ''
mint() {
sec="$clients/$1.secret"
dig="$clients/$1.digest"
@ -522,7 +516,46 @@ let
${lib.concatMapStrings (c: ''
mint ${lib.escapeShellArg c.id}
'') (lib.filter (c: c.kind != "native") cfg.oidc.clients)}
'') secretClients}
'';
# The plaintext goes straight into the store and only the digest is written
# to disk. A digest on disk is enough for authelia to start while the store is
# down; a client with no digest cannot be rendered, so that run fails and
# retries (./lib/store-retry.nix). A status of 3 is a state a retry does not
# change, and `RestartPreventExitStatus` leaves the unit failed for an
# operator.
storeMintScript = ''
${oidcClientSecrets}
${lib.concatMapStrings (c: ''
mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || exit $?
'') secretClients}
'';
# The OIDC half of the first-boot generator, kept out of the script
# body so neither is read through the other's indentation.
oidcGenScript = lib.optionalString oidcEnabled ''
${lib.optionalString (!publishes) ''
# The issuer key is the one secret here that is NOT interchangeable
# with a random blob: it *signs* id tokens, and every relying party
# verifies them against the public half served at `/jwks.json`. A
# symmetric secret cannot do that, so this one is an RSA pair.
#
# Rotating it invalidates every token already issued, which is why
# it is generated once and left alone — same reason as the session
# and storage keys above.
issuer=${lib.escapeShellArg stateDir}/oidc-issuer.key
if [ ! -s "$issuer" ]; then
openssl genrsa -out "$issuer" 4096
echo "generated $issuer"
fi
chmod 0600 "$issuer"
''}
clients=${lib.escapeShellArg clientsDir}
mkdir -p "$clients"
chmod 0700 "$clients"
${if publishes then storeMintScript else fileMintScript}
# Re-rendered every boot, deliberately: the secret is minted once,
# but the metadata around it (a new redirect URI, a renamed client)
@ -537,10 +570,42 @@ let
} > ${lib.escapeShellArg "${clientsFile}.tmp"}
chmod 0600 ${lib.escapeShellArg "${clientsFile}.tmp"}
mv ${lib.escapeShellArg "${clientsFile}.tmp"} ${lib.escapeShellArg clientsFile}
${lib.optionalString publishes ''
# authelia reads its clients once, as it starts.
if [ -n "''${MINTED-}" ]; then
refresh_unit ${lib.escapeShellArg cfg.unit} ${lib.escapeShellArg clientsFile}
fi
''}
'';
storeRetry = import ./lib/store-retry.nix { };
# The bridge's own client secret, in the memory-backed runtime directory of
# its unit. While a plaintext file from an earlier release exists the bridge
# keeps using it, since it is the value the digest was made from; the file
# goes once the store holds the same value, and the store's takes over.
bridgeSecretFile = "/run/swarm-authelia-bridge/client-secret";
bridgeSecretFetch = ''
set -euo pipefail
${atomicWriteSecret}
${storeSeededSecret}
${oidcClientSecrets}
legacy=${lib.escapeShellArg "${clientsDir}/${bridgeClientId}.secret"}
store_init
store_login
if [ -s "$legacy" ]; then
value="$(< "$legacy")"
elif client_secret_read ${lib.escapeShellArg (clientLeaf bridgeClientId)}; then
value="$CLIENT_SECRET_VALUE"
else
echo "the store holds no readable secret for ${bridgeClientId}" >&2
exit 1
fi
atomic_write_secret 0600 "" ${lib.escapeShellArg bridgeSecretFile} "$value"
'';
# authelia's keys, one leaf each under its container's name. The nix half of
# `autheliaKeyLeaves` in ./swarm-bao.nix, which grants the publisher
# `create` and `read` on exactly these.
@ -1019,19 +1084,20 @@ in
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
../container-modules/swarm-secret-publisher.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
# The forwarder dials the swarm collector and authelia by name,
# and its fetch, the publisher and the key unit the store.
# and its fetch, the bridge's fetch, the key and secret units the
# store.
dnsConsumers = [
cfg.unit
]
++ lib.optionals publishes [
"opentelemetry-collector.service"
"${forwarderFetchUnit}.service"
"${publisherUnit}.service"
"${migrateUnit}.service"
"${unitName}-secrets.service"
"swarm-authelia-bridge.service"
];
})
caBundleModule
@ -1068,20 +1134,77 @@ in
serviceConfig.LoadCredential = lib.attrNames storeFiles;
};
# Copies what the mint below leaves in `clientsDir` into the store,
# under the publisher's own identity, a system credential. Nothing
# orders authelia after it.
services.hyperhive.swarmSecretPublisher = lib.mkIf publishes {
enable = true;
clientSecretDir = clientsDir;
inherit (publisherCfg) secrets;
store = storeIdentityIn publisherUnit;
};
systemd.services.${publisherUnit} = lib.mkIf publishes {
# A run before the mint would only skip every secret and wait
# for the path unit.
# Moves the plaintext files an earlier release left in `clientsDir`
# into the store and removes each once the store holds the same
# value. With no file left it does nothing, store or no store. Nothing
# orders authelia after it: authelia starts from its digests.
systemd.services.${migrateUnit} = lib.mkIf publishes {
description = "move authelia's plaintext OIDC client secrets into the swarm secret store";
wantedBy = [ "multi-user.target" ];
after = [ "${unitName}-secrets.service" ];
serviceConfig.LoadCredential = lib.attrNames storeFiles;
path = [
pkgs.coreutils
pkgs.gnugrep
deployCfg.bao.package
];
startLimitBurst = storeRetry.startLimitBurst;
startLimitIntervalSec = storeRetry.startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
User = unitName;
Group = socketGroup;
UMask = "0077";
LoadCredential = lib.attrNames storeFiles;
TimeoutStartSec = 120;
RestartPreventExitStatus = 3;
SyslogIdentifier = migrateUnit;
};
environment =
let
store = storeIdentityIn migrateUnit;
in
{
BAO_ADDR = store.addr;
BAO_CLIENT_CERT = store.clientCertFile;
BAO_CLIENT_KEY = store.clientKeyFile;
}
// lib.optionalAttrs (store.caFile != null) { BAO_CACERT = store.caFile; };
script = ''
set -uo pipefail
${storeSeededSecret}
${oidcClientSecrets}
clients=${lib.escapeShellArg clientsDir}
known=${lib.escapeShellArg (lib.concatMapStrings (c: " ${c.id}") secretClients + " ")}
pending=0
for f in "$clients"/*.secret; do
[ -e "$f" ] || continue
id="$(basename "$f" .secret)"
case "$known" in
*" $id "*) pending=1 ;;
*) echo "left $f in place: no client $id is registered here" >&2 ;;
esac
done
if [ "$pending" = 0 ]; then
echo "no plaintext client secret files to move"
exit 0
fi
store_init
store_login
status=0
${lib.concatMapStrings (c: ''
rc=0
migrate_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || rc=$?
if [ "$rc" != 0 ] && [ "$status" != 1 ]; then
status="$rc"
fi
'') secretClients}
exit "$status"
'';
};
# The authelia binary itself, so an operator who gets a shell
@ -1121,6 +1244,7 @@ in
]
++ lib.optionals publishes [
deployCfg.bao.package
pkgs.gnugrep
pkgs.systemd
];
# A store that cannot answer, with a key missing on disk, is a
@ -1142,6 +1266,9 @@ in
// {
LoadCredential = lib.attrNames storeFiles;
TimeoutStartSec = 120;
# A secret that has no digest, or a digest that has no secret,
# is for an operator; minting again would rotate it.
RestartPreventExitStatus = 3;
}
);
environment = lib.optionalAttrs publishes (
@ -1189,23 +1316,40 @@ in
# load-bearing at start, since nothing calls the bridge yet at
# boot, but a clean dependency order beats a would-be-transient
# failure on the first real request).
#
# With the store identity its client secret is read from the store
# into `bridgeSecretFile` before each start (`bridgeSecretFetch`). A
# store that does not answer fails the start, and the unit retries.
systemd.services.swarm-authelia-bridge = {
description = "swarm-authelia-bridge: the only writer of authelia's users database";
wantedBy = [ "multi-user.target" ];
after = [
"${unitName}-secrets.service"
"${unitName}.service"
];
]
++ lib.optional publishes "${migrateUnit}.service";
wants = [
"${unitName}-secrets.service"
"${unitName}.service"
];
path = lib.optionals publishes [
pkgs.coreutils
pkgs.gnugrep
deployCfg.bao.package
];
preStart = lib.mkIf publishes bridgeSecretFetch;
startLimitBurst = lib.mkIf publishes storeRetry.startLimitBurst;
startLimitIntervalSec = lib.mkIf publishes storeRetry.startLimitIntervalSec;
serviceConfig = {
ExecStart = "${deployCfg.authelia.bridgePackage}/bin/swarm-authelia-bridge";
User = unitName;
Group = socketGroup;
Restart = "on-failure";
RestartSec = "5s";
RestartSec = if publishes then storeRetry.serviceConfig.RestartSec else "5s";
}
// lib.optionalAttrs publishes {
RuntimeDirectory = "swarm-authelia-bridge";
LoadCredential = lib.attrNames storeFiles;
};
environment = {
SWARM_AUTHELIA_BRIDGE_SOCKET = bridgeSocket;
@ -1230,17 +1374,27 @@ in
# uses, so there is one idiom rather than two.
SWARM_AUTHELIA_BRIDGE_INTROSPECTION_URL = "${cfg.url}/api/oidc/introspection";
SWARM_AUTHELIA_BRIDGE_CLIENT_ID = bridgeClientId;
# Minted by `${unitName}-secrets`'s `mint` loop (it iterates
# every entry in `cfg.oidc.clients`, which now always
# includes `bridgeClient`) — same file this container's own
# `renderClient` reads the digest half of.
SWARM_AUTHELIA_BRIDGE_CLIENT_SECRET_FILE = "${clientsDir}/${bridgeClientId}.secret";
# Minted by `${unitName}-secrets` (it iterates every entry in
# `cfg.oidc.clients`, which always includes `bridgeClient`).
SWARM_AUTHELIA_BRIDGE_CLIENT_SECRET_FILE =
if publishes then bridgeSecretFile else "${clientsDir}/${bridgeClientId}.secret";
# The one client the bridge answers. Every other active token
# (each hive and its agents hold one) is refused, on every
# request, since `/bridge/` is reachable from any host.
SWARM_AUTHELIA_BRIDGE_CONTROLLER_CLIENT_ID = hyperhiveCfg.swarm.controller.queueClientId;
SWARM_AUTHELIA_BRIDGE_OPERATOR_GROUP = operatorGroup;
};
}
// lib.optionalAttrs publishes (
let
store = storeIdentityIn "swarm-authelia-bridge";
in
{
BAO_ADDR = store.addr;
BAO_CLIENT_CERT = store.clientCertFile;
BAO_CLIENT_KEY = store.clientKeyFile;
}
// lib.optionalAttrs (store.caFile != null) { BAO_CACERT = store.caFile; }
);
};
# Upstream's unit is `ProtectSystem=strict`, so the socket directory

20
nix/host-modules/swarm-bao.nix
View file

@ -546,7 +546,7 @@ let
secretPublisherPolicyName = "swarm-secret-publisher";
secretPublisherCn = baoDeploy.secretPublisherCommonName;
# Four grants and six denials, and every narrowing in each is load-bearing.
# Five grants and six denials, and every narrowing in each is load-bearing.
#
# `secret/data/` is KV v2's ACL prefix, inserted by the engine rather than
# written by the caller — same trap as the controller's grant above.
@ -559,9 +559,9 @@ let
# here too — a grant widens when a path gains a WRITER, not when a kind is
# declared.
#
# Write-only but for its forwarder's client secret and authelia's keys, read at
# exact leaves: a read on a prefix would let a file-copier recover every
# hive's credentials. The forwarder's stanza repeats `create`/`update`.
# Reads only the `oidc/client` leaves it mints, the controller's client and
# authelia's keys: a read on a prefix would recover every hive's credentials.
# Those stanzas repeat `create`/`update`: a narrower path replaces the prefix's.
#
# The queue callout's three seeds sit under `swarm/services/` and are no
# secret this principal copies, so their exact paths, which outrank the
@ -618,10 +618,14 @@ let
}
path "${credentialMountPath}/data/${controllerClientLeaf}" {
capabilities = ["create", "update"]
capabilities = ["create", "update", "read"]
}
path "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.authelia.otel.clientId}/oidc/client" {
path "${credentialMountPath}/data/swarm/hives/+/oidc/client" {
capabilities = ["create", "update", "read"]
}
path "${credentialMountPath}/data/swarm/services/+/oidc/client" {
capabilities = ["create", "update", "read"]
}
''
@ -1184,7 +1188,7 @@ let
grafanaSecretKeyLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/secret-key";
# authelia's jwt, session, storage-encryption, OIDC HMAC and OIDC issuer keys,
# which only the secret publisher in authelia's container creates and reads.
# which only `authelia-swarm-secrets` in authelia's container creates and reads.
# The nix half of `keyLeaf` in ./swarm-authelia.nix. `create` without
# `update`, so a stored key is never replaced: the storage-encryption key
# encrypts authelia's database and the issuer key signs every id token.
@ -4090,7 +4094,7 @@ in
;;
esac
# Published by the secret publisher in authelia's container, which
# Stored by authelia's mint, in its container, which
# this boot does not wait on, so absence is retried.
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg "${credentialMountPath}/${baoUiClientLeaf}"} 2>"$err")"; then
echo ${lib.escapeShellArg "the store did not return ${credentialMountPath}/${baoUiClientLeaf}: the UI's OIDC client secret is not published yet."} >&2

Some files were not shown because too many files have changed in this diff Show more