ce44b6e77e
Each container now runs the journal forwarder the CI runner's, Grafana's and the log store's containers run, pushing to a receiver of its own on the swarm collector (serviceProducers hive-forge and hive-matrix). Each authenticates as a new machine OIDC client of its own (swarm.forge.otel.clientId, swarm.matrix.otel.clientId), registered by a glue file wherever authelia runs, minted by authelia and published to the store by the secret publisher. The forge's and the homeserver's browser login clients are unchanged. The client ids are swarm-forge-collector and swarm-matrix-collector, not hive-<x>-collector, which is the client id a hive named <x>-collector gets. The fetch unit inside each container reads the client secret with the store identity the host already holds for that service. The forge publisher's leaf is handed into the forge container as an nspawn system credential, as the grafana and log store containers do. matrix-ctl's leaf is already bound into the homeserver's container, so its forwarder reads it at those paths, as swarm-matrix-appservice-publish does; with no matrix-ctl identity the homeserver's forwarder is off. Each of the two policies gains one read stanza on its forwarder's client path. The host units that use those identities are unchanged. The two containers drop --link-journal=host, and the host collector stops reading them.
425 lines
19 KiB
Nix
425 lines
19 KiB
Nix
# `checks.module-eval-secret-publisher` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
# The IdP and the store on one machine: the shape where minted plaintext and
|
|
# a store identity are both present without an operator placing anything.
|
|
# Two hives in the roster, because the publisher walks it — an arm written
|
|
# against a single-hive fixture passes on a hardcoded name.
|
|
secretPublisherHere = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.authelia.enable = true;
|
|
swarm.hives.h2.domain = "h2.t.local";
|
|
};
|
|
|
|
# The IdP with no store on the box and a leaf placed by hand, which is the
|
|
# deployment this unit exists for: authelia is the one host the store is
|
|
# guaranteed not to share once either has a machine of its own.
|
|
#
|
|
# ⚠️ `enable` is deliberately NOT set here: leaving it unset is what makes
|
|
# this fixture exercise the default rather than mask it.
|
|
secretPublisherRemote = hive {
|
|
deploy.authelia.enable = true;
|
|
deploy.swarm-secret-publisher.baoClientCertFile = "/etc/pki/publisher.pem";
|
|
deploy.swarm-secret-publisher.baoClientKeyFile = "/etc/pki/publisher-key.pem";
|
|
};
|
|
|
|
# The same IdP with the identity taken away. Minting the secrets is not being
|
|
# able to publish them, and this is the arm that separates the two.
|
|
secretPublisherNoIdentity = shared.autheliaOnly;
|
|
# Duplicated from grafana.nix — a case here needs it too.
|
|
|
|
# The same UI with the IdP on ANOTHER host and a store leaf placed by hand.
|
|
# Knowing an IdP is not running one: `swarm.authelia.url` is what says this
|
|
# swarm has SSO, and nothing about this host does. Identical to the fixture
|
|
# above in everything the delivery path reads, which is the point.
|
|
grafanaRemoteAuthelia = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
};
|
|
|
|
# The same collector with the IdP on ANOTHER host and a store leaf placed by
|
|
# hand. Identical to the fixture above in everything the delivery path
|
|
# reads, which is the point.
|
|
otelBaoRemoteAuthelia = shared.otelReaderMissingLeaf;
|
|
|
|
inherit (shared) bare;
|
|
|
|
# The store and a service that reads from it, versus the store alone. The
|
|
# pair is what makes the reader's absence arm mean anything.
|
|
inherit (shared) baoWithMatrix;
|
|
cases = [
|
|
{
|
|
# Each hive's hive-c0re reads this exact path
|
|
# (`swarm_secret_client::queue::hive_client_path`, pinned by that crate's
|
|
# own test), for every hive in the roster, not only the one this host is.
|
|
# The agents' client beside it stays a separate path.
|
|
name = "the publisher writes every hive's own client under that hive's prefix";
|
|
ok =
|
|
let
|
|
s = secretPublisherHere.systemd.services.swarm-secret-publish.script;
|
|
in
|
|
lib.all
|
|
(h: lib.hasInfix "secret/swarm/hives/${h}/oidc/client" s && lib.hasInfix "/hive-${h}.secret" s)
|
|
[
|
|
"h1"
|
|
"h2"
|
|
]
|
|
&& lib.hasInfix "secret/swarm/hives/h1/queue/agent" s;
|
|
}
|
|
{
|
|
# The controller reads this exact path
|
|
# (`swarm_secret_client::queue::controller_client_path`, pinned by that
|
|
# crate's own test). Under `controller/`, which no hive reads: this
|
|
# client may read every hive's status.
|
|
name = "the publisher writes the controller's client under controller/, never services/";
|
|
ok =
|
|
let
|
|
s = secretPublisherHere.systemd.services.swarm-secret-publish.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/controller/swarm-controller/oidc/client" s
|
|
&& lib.hasInfix "/swarm-controller.secret" s
|
|
&& !(lib.hasInfix "secret/swarm/services/swarm-controller" s);
|
|
}
|
|
{
|
|
# Both ends of a wire nothing at eval time carries end to end: the
|
|
# publisher on authelia's host writes the path the reader on Grafana's host
|
|
# reads, and the two files agree only because both compose it from the same
|
|
# swarm-wide client id.
|
|
name = "the publisher writes the swarm service path grafana reads";
|
|
ok = lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" (
|
|
secretPublisherHere.systemd.services.swarm-secret-publish.script
|
|
);
|
|
}
|
|
{
|
|
# Registering the client cannot live where the rest of grafana's module
|
|
# lives: that block is gated on this host RUNNING grafana, so on the split
|
|
# deployment nothing registered the client, authelia minted no secret, and
|
|
# every layer below had nothing to carry. The second arm is the control —
|
|
# a host with no IdP registers nothing.
|
|
name = "the swarm's grafana client is registered wherever authelia runs";
|
|
ok =
|
|
let
|
|
clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients;
|
|
in
|
|
lib.elem "swarm-grafana" (clients secretPublisherHere)
|
|
&& !(lib.elem "swarm-grafana" (clients grafanaRemoteAuthelia));
|
|
}
|
|
{
|
|
# The collector's half of the same property: the client is not gated on
|
|
# `deploy.swarm-otel.enable`, which would let a split deployment register
|
|
# it nowhere, so authelia would mint nothing to publish.
|
|
name = "the swarm's collector client is registered wherever authelia runs";
|
|
ok =
|
|
let
|
|
clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients;
|
|
in
|
|
lib.elem "swarm-collector" (clients secretPublisherHere)
|
|
&& !(lib.elem "swarm-collector" (clients otelBaoRemoteAuthelia));
|
|
}
|
|
{
|
|
# Registration follows authelia, not the service: the IdP host below runs
|
|
# no controller, queue or homeserver (the first arms), and still lists all
|
|
# three clients. The last arm is the control — a host with no IdP lists
|
|
# none of them.
|
|
name = "an authelia host running no controller, queue or homeserver registers all three clients";
|
|
ok =
|
|
let
|
|
d = secretPublisherRemote.services.hyperhive.deploy;
|
|
clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients;
|
|
byId =
|
|
id:
|
|
lib.findFirst (
|
|
c: c.id == id
|
|
) null secretPublisherRemote.services.hyperhive.swarm.authelia.oidc.clients;
|
|
controller = byId "swarm-controller";
|
|
nats = byId "swarm-nats";
|
|
matrix = byId "tuwunel";
|
|
in
|
|
!d.swarm-controller.enable
|
|
&& !d.nats.enable
|
|
&& !d.matrix.enable
|
|
&& controller.kind == "machine"
|
|
&& controller.audience == [ "swarm-controller" ]
|
|
&& nats.kind == "machine"
|
|
&&
|
|
matrix.redirectUris == [
|
|
"https://chat.t.local/_matrix/client/unstable/login/sso/callback/tuwunel"
|
|
]
|
|
&& matrix.tokenEndpointAuthMethod == "client_secret_post"
|
|
&& !(lib.any (id: lib.elem id (clients grafanaRemoteAuthelia)) [
|
|
"swarm-controller"
|
|
"swarm-nats"
|
|
"tuwunel"
|
|
]);
|
|
}
|
|
{
|
|
# The other half of the queue's, the homeserver's and the forge's one
|
|
# route: the publisher on an IdP host that runs none of them carries each
|
|
# secret to the path its reader fetches.
|
|
name = "the publisher on an authelia host carries the queue's, homeserver's and forge's secrets";
|
|
ok =
|
|
let
|
|
s = secretPublisherRemote.systemd.services.swarm-secret-publish.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/services/swarm-nats/oidc/client" s
|
|
&& lib.hasInfix "/swarm-nats.secret" s
|
|
&& lib.hasInfix "secret/swarm/services/tuwunel/oidc/client" s
|
|
&& lib.hasInfix "/tuwunel.secret" s
|
|
&& !secretPublisherRemote.services.hyperhive.deploy.forgejo.enable
|
|
&& lib.hasInfix "secret/swarm/services/forgejo/oidc/client" s
|
|
&& lib.hasInfix "/forgejo.secret" s;
|
|
}
|
|
{
|
|
# Both ends of a wire nothing at eval time carries end to end: the
|
|
# publisher on authelia's host writes the path the reader on the
|
|
# collector's host reads, and the two files agree only because both
|
|
# compose it from the same swarm-wide client id. `secretPublisherHere`
|
|
# already grew this client when `serviceClientIds` did.
|
|
name = "the publisher writes the swarm service path the collector reads";
|
|
ok = lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" (
|
|
secretPublisherHere.systemd.services.swarm-secret-publish.script
|
|
);
|
|
}
|
|
{
|
|
# The third swarm service on that list, and the one whose absence is
|
|
# hardest to see: the store's own forwarder holds a client authelia
|
|
# registers and mints for, so every layer above looks complete while
|
|
# the one hop that CARRIES the value skips it and the reader on the
|
|
# store's host waits forever on a path nothing writes.
|
|
name = "the publisher carries the store forwarder's secret too";
|
|
ok = lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" (
|
|
secretPublisherHere.systemd.services.swarm-secret-publish.script
|
|
);
|
|
}
|
|
{
|
|
# The forwarders in the CI runner's, Grafana's, the log store's, the
|
|
# forge's and the homeserver's containers read their secrets from these
|
|
# paths.
|
|
name = "the publisher carries the CI runner, Grafana, log store, forge and homeserver forwarders' secrets";
|
|
ok =
|
|
lib.all
|
|
(
|
|
id:
|
|
lib.hasInfix "secret/swarm/services/${id}/oidc/client" (
|
|
secretPublisherHere.systemd.services.swarm-secret-publish.script
|
|
)
|
|
)
|
|
[
|
|
"swarm-ci-runner-collector"
|
|
"swarm-grafana-collector"
|
|
"swarm-victorialogs-collector"
|
|
"swarm-forge-collector"
|
|
"swarm-matrix-collector"
|
|
];
|
|
}
|
|
{
|
|
# The review classifier reads its queue client secret from the store
|
|
# inside its container; authelia registers it on every IdP host.
|
|
name = "the review classifier's client is registered as a machine, and the publisher carries its secret";
|
|
ok =
|
|
let
|
|
c = lib.findFirst (
|
|
c: c.id == "swarm-review-classifier"
|
|
) null secretPublisherHere.services.hyperhive.swarm.authelia.oidc.clients;
|
|
in
|
|
c != null
|
|
&& c.kind == "machine"
|
|
&& lib.hasInfix "secret/swarm/services/swarm-review-classifier/oidc/client" (
|
|
secretPublisherHere.systemd.services.swarm-secret-publish.script
|
|
);
|
|
}
|
|
{
|
|
# The store's UI login: authelia registers the client with the callback
|
|
# the store's `oidc` role allows, and the publisher carries its secret
|
|
# to the path `swarm-bao-operator-viewer-policy` reads. Not registered
|
|
# where authelia is not.
|
|
name = "the store UI's client is registered interactive at its callback, and the publisher carries its secret";
|
|
ok =
|
|
let
|
|
ui = lib.findFirst (
|
|
c: c.id == "swarm-bao-ui"
|
|
) null secretPublisherHere.services.hyperhive.swarm.authelia.oidc.clients;
|
|
in
|
|
ui != null
|
|
&& ui.kind == "interactive"
|
|
&& ui.redirectUris == [ "https://bao-ui.t.local/ui/vault/auth/oidc/oidc/callback" ]
|
|
&& lib.hasInfix "secret/swarm/services/swarm-bao-ui/oidc/client" (
|
|
secretPublisherHere.systemd.services.swarm-secret-publish.script
|
|
)
|
|
&& !(lib.any (
|
|
c: c.id == "swarm-bao-ui"
|
|
) grafanaRemoteAuthelia.services.hyperhive.swarm.authelia.oidc.clients);
|
|
}
|
|
{
|
|
# The same hole the controller's case above names, open a second time: the
|
|
# PKI script grew a third leaf and no case read it.
|
|
name = "the store mints a leaf for the secret publisher, and the publisher is pointed at it";
|
|
ok =
|
|
let
|
|
m = secretPublisherHere;
|
|
p = m.services.hyperhive.deploy.swarm-secret-publisher;
|
|
in
|
|
lib.hasInfix "secret-publisher.pem" m.systemd.services.swarm-bao-pki.script
|
|
&& p.baoClientCertFile == "/var/lib/swarm-bao-pki/secret-publisher.pem"
|
|
&& p.baoClientKeyFile == "/var/lib/swarm-bao-pki/secret-publisher-key.pem";
|
|
}
|
|
{
|
|
# mara caught this by reading, which means no arm existed for it: the
|
|
# default asked `authelia.enable && bao.enable`, so the split deployment
|
|
# this unit is FOR defaulted off and published nothing, silently. The
|
|
# default is set by ../host-modules/glue-secret-publisher-authelia.nix.
|
|
#
|
|
# The second clause is the control. Without it this passes on a default
|
|
# of plain `true`, which would be a different bug with the same symptom
|
|
# — an IdP-less host claiming it publishes secrets it never mints.
|
|
name = "the publisher defaults on where secrets are minted, whether or not the store is local";
|
|
ok =
|
|
secretPublisherRemote.services.hyperhive.deploy.swarm-secret-publisher.enable
|
|
&& !bare.services.hyperhive.deploy.swarm-secret-publisher.enable;
|
|
}
|
|
{
|
|
# The one security property of this unit, and why its push cannot be
|
|
# rewritten into the obvious shape: `bao` is an external binary, so an
|
|
# argument is world-readable in /proc for the life of the call.
|
|
# `value=@<path>` hands it the path and bao opens the file itself.
|
|
#
|
|
# The second arm is what makes the first mean anything — `value=@` can
|
|
# sit one line above a command substitution that put the plaintext in
|
|
# argv anyway.
|
|
#
|
|
# ⚠️ Comments are stripped first, and that is not tidiness. A `script`
|
|
# renders its own comments into the text, and this unit's comments name
|
|
# the hazard verbatim so the next editor does not reintroduce it. Without
|
|
# the strip this case reads that warning and fails — a check the artifact
|
|
# defeats by DESCRIBING the thing it is checked for.
|
|
name = "the publisher hands bao the secret's path, never the secret";
|
|
ok =
|
|
let
|
|
s = secretPublisherHere.systemd.services.swarm-secret-publish.script;
|
|
code = lib.concatStringsSep "\n" (
|
|
lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s)
|
|
);
|
|
in
|
|
lib.hasInfix "value=@" code && !(lib.hasInfix "$(cat" code);
|
|
}
|
|
{
|
|
# Two ends of a wire nothing at eval time carries end to end: this is the
|
|
# path `swarm_secret_client::queue` resolves for the reader. Both hives
|
|
# are asserted, so a publisher that knew one name rather than the roster
|
|
# fails here rather than on the second hive ever added to a swarm.
|
|
name = "the publisher writes every hive in the roster to that hive's own queue path";
|
|
ok =
|
|
let
|
|
s = secretPublisherHere.systemd.services.swarm-secret-publish.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/hives/h1/queue/agent" s
|
|
&& lib.hasInfix "secret/swarm/hives/h2/queue/agent" s;
|
|
}
|
|
{
|
|
# A property of the SET, not of one unit: both of these authenticate by
|
|
# certificate, and `BAO_CLIENT_CERT` is transport rather than identity, so
|
|
# a script that reaches `bao kv` without a token asks a token helper this
|
|
# host does not carry and fails before the store ever answers. `-token-only`
|
|
# is what keeps the token off the helper on the way back out.
|
|
#
|
|
# Ordering, not presence: the login has to come first, so the check is
|
|
# that nothing before it is a data command. Comments are stripped because
|
|
# both units explain this in prose directly above the code.
|
|
name = "the cert-identity bao units log in before their first read or write, and keep the token out of the helper";
|
|
ok =
|
|
let
|
|
code =
|
|
s:
|
|
lib.concatStringsSep "\n" (
|
|
lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s)
|
|
);
|
|
holdsTokenFirst =
|
|
s:
|
|
let
|
|
c = code s;
|
|
in
|
|
lib.hasInfix "bao login" c
|
|
&& lib.hasInfix "-token-only" c
|
|
&& !(lib.hasInfix "bao kv" (lib.head (lib.splitString "bao login" c)));
|
|
in
|
|
holdsTokenFirst secretPublisherHere.systemd.services.swarm-secret-publish.script
|
|
&& holdsTokenFirst baoWithMatrix.systemd.services.swarm-bao-queue-agent.script
|
|
# Controls, so a clean verdict above means something. In order: a bare
|
|
# read is refused, a read placed before the login is refused, and a
|
|
# login that exists only in a comment is refused — that last one is the
|
|
# arm the comment-stripping exists for.
|
|
&& !(holdsTokenFirst "bao kv get -field=value secret/x")
|
|
&& !(holdsTokenFirst "bao kv get secret/x\nBAO_TOKEN=\"$(bao login -method=cert -token-only)\"")
|
|
&& !(holdsTokenFirst "# bao login -method=cert -token-only goes here\nbao kv get secret/x")
|
|
&& holdsTokenFirst "BAO_TOKEN=\"$(bao login -method=cert -token-only)\"\nbao kv get secret/x";
|
|
}
|
|
{
|
|
# The doctrine three glue files state, as a property a rewrite has to
|
|
# keep: a client is defined by holding a certificate the store accepts,
|
|
# never by standing next to the store. Gating this on `deploy.bao.enable`
|
|
# would have left the unit rendering only on the one deployment that has
|
|
# no use for it.
|
|
name = "a publisher holding an identity runs on a host with no store";
|
|
ok =
|
|
let
|
|
m = secretPublisherRemote;
|
|
in
|
|
!m.services.hyperhive.deploy.bao.enable
|
|
&& (m.systemd.services ? swarm-secret-publish)
|
|
&& (m.systemd.paths ? swarm-secret-publish);
|
|
}
|
|
{
|
|
# What makes the arm above able to fail. Minting the secrets is not being
|
|
# able to publish them: with no certificate the unit would fail a TLS
|
|
# handshake on every rotation, so it must not exist at all.
|
|
name = "an IdP host with no store identity renders no publisher";
|
|
ok =
|
|
let
|
|
m = secretPublisherNoIdentity;
|
|
in
|
|
m.services.hyperhive.deploy.authelia.enable
|
|
&& !(m.systemd.services ? swarm-secret-publish)
|
|
&& !(m.systemd.paths ? swarm-secret-publish);
|
|
}
|
|
{
|
|
# The path unit's start is a no-op while its target is `active (exited)`,
|
|
# so a secret minted after the first run would never be published.
|
|
# `Type` is read too, so a missing `serviceConfig` cannot pass the arm.
|
|
name = "the publisher's path unit starts a oneshot that does not remain active";
|
|
ok =
|
|
let
|
|
m = secretPublisherHere;
|
|
sc = m.systemd.services.swarm-secret-publish.serviceConfig;
|
|
in
|
|
m.systemd.paths.swarm-secret-publish.pathConfig.Unit == "swarm-secret-publish.service"
|
|
&& sc.Type == "oneshot"
|
|
&& !(sc.RemainAfterExit or false);
|
|
}
|
|
];
|
|
in
|
|
runGroup "secret-publisher" cases
|