render the matrix appservice registration from the store onto tmpfs
swarm-matrix-ctl appservice render now reads the registration's tokens from the store and renders swarm.yaml under /run/swarm-matrix-appservice, which tuwunel loads through LoadCredential. It mints a pair only when the store answers and holds none; a host that has the old files stores their pair once instead. A store that does not answer fails the unit and keeps tuwunel down; sync starts it once the store answers. Refs #4998 (item 3)
This commit is contained in:
atlas
mara
parent
7813337548
commit
3a4b0ffabf
3 changed files with 594 additions and 378 deletions
|
|
@ -100,6 +100,25 @@ let
|
|||
# environment, so the binary itself spells no role at all.
|
||||
ctlCertRole = "swarm-matrix-ctl";
|
||||
|
||||
# What `swarm-matrix-ctl appservice render` and `sync` are both given: the
|
||||
# store address and matrix-ctl's identity, and the registration's shape. A
|
||||
# secret is a path, never a value, so nothing here is a token.
|
||||
appserviceEnv = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = deployCfg.matrix.ctlBaoClientCertFile;
|
||||
BAO_CLIENT_KEY = deployCfg.matrix.ctlBaoClientKeyFile;
|
||||
MATRIX_APPSERVICE_CERT_ROLE = ctlCertRole;
|
||||
MATRIX_APPSERVICE_RUNTIME_DIR = swarmAppserviceRuntimeDir;
|
||||
MATRIX_APPSERVICE_LEGACY_DIR = swarmAppserviceLegacyDir;
|
||||
MATRIX_APPSERVICE_SENDER = swarmSenderLocalpart;
|
||||
# Agents' localparts are bare `[a-z0-9-]`, so nothing narrower covers
|
||||
# them without also covering people.
|
||||
MATRIX_APPSERVICE_USER_REGEX = appserviceUserRegex;
|
||||
}
|
||||
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
|
||||
BAO_CACERT = deployCfg.bao.serverCaFile;
|
||||
};
|
||||
|
||||
# Every host directory matrix-ctl's bao identity is spread across. Normally
|
||||
# one — ./glue-bao-tls.nix puts all three files in ./glue's PKI dir — but
|
||||
# derived rather than assumed, because an operator naming paths by hand is
|
||||
|
|
@ -140,20 +159,25 @@ let
|
|||
# The homeserver's one appservice registration, and the swarm's identity on
|
||||
# it: `swarm-controller` creates every agent's account and every hive's
|
||||
# `@hive-<hive>` sender with its token. The store holds its tokens;
|
||||
# `swarm-matrix-ctl appservice sync` keeps this container's copy in step with
|
||||
# them, which is why it is gated on matrix-ctl's store identity: with nobody
|
||||
# to sync it, a registration here would be an admin credential nobody reads.
|
||||
# `appservice render` minted a copy here only for a host that has none, and
|
||||
# renders the one the homeserver starts from.
|
||||
# `swarm-matrix-ctl appservice render` fetches them into this container's
|
||||
# tmpfs before each homeserver start, and `appservice sync` keeps that copy
|
||||
# in step with them. Both are gated on matrix-ctl's store identity: with
|
||||
# nobody to fetch it, a registration here would be an admin credential
|
||||
# nobody reads. `render` mints a pair only when the store answers and holds
|
||||
# none, and the homeserver stays down while the store does not answer.
|
||||
#
|
||||
# Its sender is promoted to homeserver admin at boot (`admin_execute`
|
||||
# below), so its token goes only to a store path no hive's policy reaches.
|
||||
# `swarm` is in ../reserved-names.nix, so no agent can be created as this
|
||||
# account.
|
||||
swarmSenderLocalpart = "swarm";
|
||||
# Inside the container: the render unit's `StateDirectory`. `ephemeral =
|
||||
# false` keeps it across restarts, so "mint when absent" is mint once.
|
||||
swarmAppserviceDir = "/var/lib/swarm-matrix-appservice";
|
||||
# Inside the container, on its tmpfs: the registration tuwunel loads, owned
|
||||
# by the two units below through `RuntimeDirectory`.
|
||||
swarmAppserviceRuntimeName = "swarm-matrix-appservice";
|
||||
swarmAppserviceRuntimeDir = "/run/${swarmAppserviceRuntimeName}";
|
||||
# Where earlier versions kept the tokens. Read to seed an empty store, never
|
||||
# written.
|
||||
swarmAppserviceLegacyDir = "/var/lib/swarm-matrix-appservice";
|
||||
swarmAppserviceCredentialId = "swarm-appservice.yaml";
|
||||
|
||||
# Every local user the swarm appservice may create — agents and every
|
||||
|
|
@ -1021,77 +1045,71 @@ in
|
|||
# suffix on purpose: `appservice_dir` above names this very
|
||||
# directory, and tuwunel takes only `.yaml`/`.yml` entries from
|
||||
# it. A missing source fails this unit, which is why the render
|
||||
# below is `requiredBy` it and local only.
|
||||
++ lib.optional ctlActive "${swarmAppserviceCredentialId}:${swarmAppserviceDir}/swarm.yaml";
|
||||
# below is `requiredBy` it.
|
||||
++ lib.optional ctlActive "${swarmAppserviceCredentialId}:${swarmAppserviceRuntimeDir}/swarm.yaml";
|
||||
|
||||
# The swarm registration, rendered before the homeserver loads it from
|
||||
# the tokens in the state dir: the last copy of the store's that
|
||||
# `swarm-matrix-appservice-sync` took, or a minted pair on a host with
|
||||
# none. No network and no store: this is on tuwunel's start path, and
|
||||
# a store outage must not keep the homeserver down.
|
||||
# The swarm registration, fetched from the store and rendered onto
|
||||
# the tmpfs before the homeserver loads it. A store that does not
|
||||
# answer fails this unit, and with it the homeserver's start: the
|
||||
# tokens are on no disk to start from. It retries like the other
|
||||
# store fetches, and `swarm-matrix-appservice-sync` below starts the
|
||||
# homeserver once the registration is there.
|
||||
#
|
||||
# `RuntimeDirectoryPreserve` because the sync unit shares the
|
||||
# directory: neither unit's stop may remove it from under the other.
|
||||
systemd.services.swarm-matrix-appservice-render = lib.mkIf ctlActive {
|
||||
description = "render the swarm's appservice registration";
|
||||
before = [ "tuwunel.service" ];
|
||||
requiredBy = [ "tuwunel.service" ];
|
||||
serviceConfig = {
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
ExecStart = "${deployCfg.matrix.ctlPackage}/bin/swarm-matrix-ctl appservice render";
|
||||
StateDirectory = baseNameOf swarmAppserviceDir;
|
||||
StateDirectoryMode = "0700";
|
||||
# A sealed store answers on the port and never answers the read.
|
||||
TimeoutStartSec = 60;
|
||||
RuntimeDirectory = swarmAppserviceRuntimeName;
|
||||
RuntimeDirectoryMode = "0700";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
UMask = "0077";
|
||||
SyslogIdentifier = "swarm-matrix-appservice";
|
||||
};
|
||||
environment = {
|
||||
MATRIX_APPSERVICE_DIR = swarmAppserviceDir;
|
||||
MATRIX_APPSERVICE_SENDER = swarmSenderLocalpart;
|
||||
# Agents' localparts are bare `[a-z0-9-]`, so nothing narrower
|
||||
# covers them without also covering people.
|
||||
MATRIX_APPSERVICE_USER_REGEX = appserviceUserRegex;
|
||||
};
|
||||
environment = appserviceEnv;
|
||||
};
|
||||
|
||||
# The store holds the swarm registration; this keeps the state dir
|
||||
# the render above reads in step with it, and publishes the
|
||||
# `as_token` to the path `swarm-controller` reads, under matrix-ctl's
|
||||
# identity (the grants are in ./swarm-bao.nix). A store without the
|
||||
# registration is seeded from the state dir, so nothing rotates.
|
||||
# Keeps the rendered registration in step with the store, and
|
||||
# publishes the `as_token` to the path `swarm-controller` reads,
|
||||
# under matrix-ctl's identity (the grants are in ./swarm-bao.nix).
|
||||
# When the registration is newer than tuwunel's last start, this
|
||||
# restarts tuwunel onto it; a tuwunel that failed because the render
|
||||
# did is started the same way.
|
||||
#
|
||||
# ⚠️ Not `before`/`requiredBy` tuwunel, and nothing orders tuwunel
|
||||
# after it: the homeserver starts from the state dir's last copy
|
||||
# whether or not the store answers. When the registration is newer than
|
||||
# tuwunel's last start, this restarts tuwunel onto it.
|
||||
#
|
||||
# `Restart=on-failure` covers a sealed store or one still starting;
|
||||
# the timer below repeats the run, which a unit without `RemainAfterExit` allows.
|
||||
# `wants`, not `requires`, the render: this unit must still run when
|
||||
# the render has failed, since it is what brings the homeserver up
|
||||
# once the store answers. `Restart=on-failure` with the shared budget
|
||||
# covers a sealed store or one still starting; the timer below
|
||||
# repeats the run, which a unit without `RemainAfterExit` allows.
|
||||
# The start timeout is bounded because a sealed store answers on the
|
||||
# port and never answers the read.
|
||||
systemd.services.swarm-matrix-appservice-sync = lib.mkIf ctlActive {
|
||||
description = "keep the swarm's appservice registration in step with the swarm secret store";
|
||||
after = [ "swarm-matrix-appservice-render.service" ];
|
||||
requires = [ "swarm-matrix-appservice-render.service" ];
|
||||
wants = [ "swarm-matrix-appservice-render.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [ pkgs.systemd ];
|
||||
serviceConfig = {
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
ExecStart = "${deployCfg.matrix.ctlPackage}/bin/swarm-matrix-ctl appservice sync";
|
||||
Restart = "on-failure";
|
||||
RestartSec = 30;
|
||||
TimeoutStartSec = 60;
|
||||
RuntimeDirectory = swarmAppserviceRuntimeName;
|
||||
RuntimeDirectoryMode = "0700";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
UMask = "0077";
|
||||
SyslogIdentifier = "swarm-matrix-appservice";
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = deployCfg.matrix.ctlBaoClientCertFile;
|
||||
BAO_CLIENT_KEY = deployCfg.matrix.ctlBaoClientKeyFile;
|
||||
MATRIX_APPSERVICE_CERT_ROLE = ctlCertRole;
|
||||
MATRIX_APPSERVICE_DIR = swarmAppserviceDir;
|
||||
MATRIX_APPSERVICE_SENDER = swarmSenderLocalpart;
|
||||
MATRIX_APPSERVICE_USER_REGEX = appserviceUserRegex;
|
||||
environment = appserviceEnv // {
|
||||
MATRIX_APPSERVICE_RESTART_UNIT = "tuwunel.service";
|
||||
}
|
||||
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
|
||||
BAO_CACERT = deployCfg.bao.serverCaFile;
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -653,7 +653,8 @@ let
|
|||
# write, unlike either sibling, because sync compares before it writes.
|
||||
#
|
||||
# The second is the registration's tokens: the store's copy, which matrix-ctl
|
||||
# seeds from the container's state dir and then follows. `create` and `read`
|
||||
# creates (from the tokens an earlier version kept in the container, or a
|
||||
# minted pair) and then follows. `create` and `read`
|
||||
# only, never `update`: a registration that exists is replaced by an
|
||||
# operator, not by a unit.
|
||||
#
|
||||
|
|
@ -2008,9 +2009,9 @@ in
|
|||
example = "swarm-matrix-ctl.svc";
|
||||
description = ''
|
||||
Subject the store's matrix-ctl cert-auth role accepts — the
|
||||
identity the matrix container's `swarm-matrix-appservice-sync`
|
||||
unit presents when it syncs the swarm appservice's registration and
|
||||
publishes its token.
|
||||
identity the matrix container's `swarm-matrix-appservice-render` and
|
||||
`swarm-matrix-appservice-sync` units present when they fetch the swarm
|
||||
appservice's registration and publish its token.
|
||||
|
||||
A **third** identity rather than reuse of either sibling above. Its
|
||||
grant is that credential and the registration it derives from, plus
|
||||
|
|
|
|||
|
|
@ -258,13 +258,15 @@ let
|
|||
creds =
|
||||
baoWithMatrix.containers.hive-matrix.config.systemd.services.tuwunel.serviceConfig.LoadCredential;
|
||||
in
|
||||
lib.elem "swarm-appservice.yaml:/var/lib/swarm-matrix-appservice/swarm.yaml" creds
|
||||
lib.elem "swarm-appservice.yaml:/run/swarm-matrix-appservice/swarm.yaml" creds
|
||||
&& lib.length (lib.filter (c: lib.hasInfix ".yaml:" c) creds) == 1;
|
||||
}
|
||||
{
|
||||
# A missing `LoadCredential` source fails tuwunel, so the render has to
|
||||
# have run first — and it is local only, so it cannot fail on a store.
|
||||
name = "the swarm registration is rendered before tuwunel and required by it, without the store";
|
||||
# have run first. It fetches from the store, so it carries the store's
|
||||
# address and matrix-ctl's identity, and retries on failure: a store
|
||||
# that does not answer keeps the homeserver down until it does.
|
||||
name = "the swarm registration is rendered before tuwunel and required by it, from the store, with retries";
|
||||
ok =
|
||||
let
|
||||
r = baoWithMatrix.containers.hive-matrix.config.systemd.services.swarm-matrix-appservice-render;
|
||||
|
|
@ -272,17 +274,19 @@ let
|
|||
lib.elem "tuwunel.service" r.before
|
||||
&& lib.elem "tuwunel.service" r.requiredBy
|
||||
&& lib.hasSuffix "swarm-matrix-ctl appservice render" r.serviceConfig.ExecStart
|
||||
&& !(r.environment ? BAO_ADDR);
|
||||
&& r.environment.BAO_ADDR != ""
|
||||
&& r.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/matrix-ctl.pem"
|
||||
&& r.environment.MATRIX_APPSERVICE_CERT_ROLE == "swarm-matrix-ctl"
|
||||
&& r.serviceConfig.Restart == "on-failure";
|
||||
}
|
||||
{
|
||||
# The start path stays free of the store. The render is what tuwunel
|
||||
# waits for and it carries no store address or identity; the sync unit
|
||||
# that does is neither before tuwunel nor required by it, and tuwunel
|
||||
# names it nowhere, so a store outage leaves the homeserver starting on
|
||||
# the state dir's last copy. The presence arms are the render's own
|
||||
# ordering (the same lookup finds `before`/`requiredBy` where they are)
|
||||
# and the sync's order after the render.
|
||||
name = "the sync unit is off tuwunel's start path and the render holds no store identity";
|
||||
# The sync unit is what starts a homeserver the render's failure left
|
||||
# down, so it must run when the render has failed: it is after the
|
||||
# render, and neither requires nor binds to it. It stays off tuwunel's
|
||||
# own start path, and tuwunel names it nowhere. The presence arms are the
|
||||
# render's own ordering (the same lookup finds `before`/`requiredBy`
|
||||
# where they are) and the sync's order after the render.
|
||||
name = "the sync unit runs after the render without requiring it, and is off tuwunel's start path";
|
||||
ok =
|
||||
let
|
||||
c = baoWithMatrix.containers.hive-matrix.config.systemd;
|
||||
|
|
@ -298,15 +302,51 @@ let
|
|||
(u.bindsTo or [ ])
|
||||
];
|
||||
sync = "swarm-matrix-appservice-sync.service";
|
||||
render = "swarm-matrix-appservice-render.service";
|
||||
in
|
||||
lib.elem "tuwunel.service" r.before
|
||||
&& lib.elem "swarm-matrix-appservice-render.service" s.after
|
||||
&& lib.elem render s.after
|
||||
&& !(lib.elem render (s.requires or [ ]))
|
||||
&& !(lib.elem render (s.bindsTo or [ ]))
|
||||
&& !(lib.elem "tuwunel.service" (s.before or [ ]))
|
||||
&& !(lib.elem "tuwunel.service" (s.requiredBy or [ ]))
|
||||
&& !(lib.elem "tuwunel.service" (s.wantedBy or [ ]))
|
||||
&& !(lib.elem sync (names t))
|
||||
&& !(lib.elem sync (r.before or [ ]))
|
||||
&& !(lib.any (lib.hasPrefix "BAO_") (lib.attrNames r.environment));
|
||||
&& !(lib.elem sync (r.before or [ ]));
|
||||
}
|
||||
{
|
||||
# 🩸 The registration holds the tokens that create users, so it is
|
||||
# rendered onto the container's tmpfs and no unit keeps the tokens in
|
||||
# /var/lib. tuwunel's credential source, both units' runtime directory
|
||||
# and the directory they are told to render into are one path under
|
||||
# /run (the presence arms), and neither unit has a state directory or
|
||||
# a writable path under /var/lib. The legacy directory is named in the
|
||||
# environment, to be read, and nowhere else.
|
||||
name = "the swarm registration is rendered under /run and no unit writes it under /var/lib";
|
||||
ok =
|
||||
let
|
||||
c = baoWithMatrix.containers.hive-matrix.config.systemd;
|
||||
units = [
|
||||
c.services.swarm-matrix-appservice-render
|
||||
c.services.swarm-matrix-appservice-sync
|
||||
];
|
||||
creds = c.services.tuwunel.serviceConfig.LoadCredential;
|
||||
runDir = "/run/swarm-matrix-appservice";
|
||||
writesVarLib =
|
||||
u:
|
||||
lib.any (lib.hasPrefix "/var/lib") (
|
||||
lib.toList (u.serviceConfig.ReadWritePaths or [ ]) ++ lib.toList (u.serviceConfig.BindPaths or [ ])
|
||||
)
|
||||
|| (u.serviceConfig ? StateDirectory);
|
||||
in
|
||||
lib.elem "swarm-appservice.yaml:${runDir}/swarm.yaml" creds
|
||||
&& lib.all (u: u.environment.MATRIX_APPSERVICE_RUNTIME_DIR == runDir) units
|
||||
&& lib.all (u: u.serviceConfig.RuntimeDirectory == "swarm-matrix-appservice") units
|
||||
&& !(lib.any writesVarLib units)
|
||||
&& !(lib.any (cr: lib.hasInfix "/var/lib/swarm-matrix-appservice" cr) creds)
|
||||
&& lib.all (
|
||||
u: u.environment.MATRIX_APPSERVICE_LEGACY_DIR == "/var/lib/swarm-matrix-appservice"
|
||||
) units;
|
||||
}
|
||||
{
|
||||
# A timer repeats the sync, which needs a unit that does not stay
|
||||
|
|
|
|||
Loading…
Reference in a new issue