vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

render the matrix appservice registration from the store onto tmpfs

Browse source 
swarm-matrix-ctl appservice render now reads the registration's tokens
from the store and renders swarm.yaml under /run/swarm-matrix-appservice,
which tuwunel loads through LoadCredential. It mints a pair only when the
store answers and holds none; a host that has the old files stores their
pair once instead. A store that does not answer fails the unit and keeps
tuwunel down; sync starts it once the store answers.

Refs #4998 (item 3)
This commit is contained in:
atlas 2026-10-30 05:47:39 +01:00 • committed by mara
commit 3a4b0ffabf
2 changed files with 594 additions and 378 deletions
Download patch file Download diff file Expand all files Collapse all files

122
nix/host-modules/hive-matrix.nix
View file

@ -100,6 +100,25 @@ let
# environment, so the binary itself spells no role at all.
ctlCertRole = "swarm-matrix-ctl";
# What `swarm-matrix-ctl appservice render` and `sync` are both given: the
# store address and matrix-ctl's identity, and the registration's shape. A
# secret is a path, never a value, so nothing here is a token.
appserviceEnv = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = deployCfg.matrix.ctlBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.matrix.ctlBaoClientKeyFile;
MATRIX_APPSERVICE_CERT_ROLE = ctlCertRole;
MATRIX_APPSERVICE_RUNTIME_DIR = swarmAppserviceRuntimeDir;
MATRIX_APPSERVICE_LEGACY_DIR = swarmAppserviceLegacyDir;
MATRIX_APPSERVICE_SENDER = swarmSenderLocalpart;
# Agents' localparts are bare `[a-z0-9-]`, so nothing narrower covers
# them without also covering people.
MATRIX_APPSERVICE_USER_REGEX = appserviceUserRegex;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
# Every host directory matrix-ctl's bao identity is spread across. Normally
# one — ./glue-bao-tls.nix puts all three files in ./glue's PKI dir — but
# derived rather than assumed, because an operator naming paths by hand is
@ -140,20 +159,25 @@ let
# The homeserver's one appservice registration, and the swarm's identity on
# it: `swarm-controller` creates every agent's account and every hive's
# `@hive-<hive>` sender with its token. The store holds its tokens;
# `swarm-matrix-ctl appservice sync` keeps this container's copy in step with
# them, which is why it is gated on matrix-ctl's store identity: with nobody
# to sync it, a registration here would be an admin credential nobody reads.
# `appservice render` minted a copy here only for a host that has none, and
# renders the one the homeserver starts from.
# `swarm-matrix-ctl appservice render` fetches them into this container's
# tmpfs before each homeserver start, and `appservice sync` keeps that copy
# in step with them. Both are gated on matrix-ctl's store identity: with
# nobody to fetch it, a registration here would be an admin credential
# nobody reads. `render` mints a pair only when the store answers and holds
# none, and the homeserver stays down while the store does not answer.
#
# Its sender is promoted to homeserver admin at boot (`admin_execute`
# below), so its token goes only to a store path no hive's policy reaches.
# `swarm` is in ../reserved-names.nix, so no agent can be created as this
# account.
swarmSenderLocalpart = "swarm";
# Inside the container: the render unit's `StateDirectory`. `ephemeral =
# false` keeps it across restarts, so "mint when absent" is mint once.
swarmAppserviceDir = "/var/lib/swarm-matrix-appservice";
# Inside the container, on its tmpfs: the registration tuwunel loads, owned
# by the two units below through `RuntimeDirectory`.
swarmAppserviceRuntimeName = "swarm-matrix-appservice";
swarmAppserviceRuntimeDir = "/run/${swarmAppserviceRuntimeName}";
# Where earlier versions kept the tokens. Read to seed an empty store, never
# written.
swarmAppserviceLegacyDir = "/var/lib/swarm-matrix-appservice";
swarmAppserviceCredentialId = "swarm-appservice.yaml";
# Every local user the swarm appservice may create — agents and every
@ -1021,77 +1045,71 @@ in
# suffix on purpose: `appservice_dir` above names this very
# directory, and tuwunel takes only `.yaml`/`.yml` entries from
# it. A missing source fails this unit, which is why the render
# below is `requiredBy` it and local only.
++ lib.optional ctlActive "${swarmAppserviceCredentialId}:${swarmAppserviceDir}/swarm.yaml";
# below is `requiredBy` it.
++ lib.optional ctlActive "${swarmAppserviceCredentialId}:${swarmAppserviceRuntimeDir}/swarm.yaml";
# The swarm registration, rendered before the homeserver loads it from
# the tokens in the state dir: the last copy of the store's that
# `swarm-matrix-appservice-sync` took, or a minted pair on a host with
# none. No network and no store: this is on tuwunel's start path, and
# a store outage must not keep the homeserver down.
# The swarm registration, fetched from the store and rendered onto
# the tmpfs before the homeserver loads it. A store that does not
# answer fails this unit, and with it the homeserver's start: the
# tokens are on no disk to start from. It retries like the other
# store fetches, and `swarm-matrix-appservice-sync` below starts the
# homeserver once the registration is there.
#
# `RuntimeDirectoryPreserve` because the sync unit shares the
# directory: neither unit's stop may remove it from under the other.
systemd.services.swarm-matrix-appservice-render = lib.mkIf ctlActive {
description = "render the swarm's appservice registration";
before = [ "tuwunel.service" ];
requiredBy = [ "tuwunel.service" ];
serviceConfig = {
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${deployCfg.matrix.ctlPackage}/bin/swarm-matrix-ctl appservice render";
StateDirectory = baseNameOf swarmAppserviceDir;
StateDirectoryMode = "0700";
# A sealed store answers on the port and never answers the read.
TimeoutStartSec = 60;
RuntimeDirectory = swarmAppserviceRuntimeName;
RuntimeDirectoryMode = "0700";
RuntimeDirectoryPreserve = "yes";
UMask = "0077";
SyslogIdentifier = "swarm-matrix-appservice";
};
environment = {
MATRIX_APPSERVICE_DIR = swarmAppserviceDir;
MATRIX_APPSERVICE_SENDER = swarmSenderLocalpart;
# Agents' localparts are bare `[a-z0-9-]`, so nothing narrower
# covers them without also covering people.
MATRIX_APPSERVICE_USER_REGEX = appserviceUserRegex;
};
environment = appserviceEnv;
};
# The store holds the swarm registration; this keeps the state dir
# the render above reads in step with it, and publishes the
# `as_token` to the path `swarm-controller` reads, under matrix-ctl's
# identity (the grants are in ./swarm-bao.nix). A store without the
# registration is seeded from the state dir, so nothing rotates.
# Keeps the rendered registration in step with the store, and
# publishes the `as_token` to the path `swarm-controller` reads,
# under matrix-ctl's identity (the grants are in ./swarm-bao.nix).
# When the registration is newer than tuwunel's last start, this
# restarts tuwunel onto it; a tuwunel that failed because the render
# did is started the same way.
#
# ⚠️ Not `before`/`requiredBy` tuwunel, and nothing orders tuwunel
# after it: the homeserver starts from the state dir's last copy
# whether or not the store answers. When the registration is newer than
# tuwunel's last start, this restarts tuwunel onto it.
#
# `Restart=on-failure` covers a sealed store or one still starting;
# the timer below repeats the run, which a unit without `RemainAfterExit` allows.
# `wants`, not `requires`, the render: this unit must still run when
# the render has failed, since it is what brings the homeserver up
# once the store answers. `Restart=on-failure` with the shared budget
# covers a sealed store or one still starting; the timer below
# repeats the run, which a unit without `RemainAfterExit` allows.
# The start timeout is bounded because a sealed store answers on the
# port and never answers the read.
systemd.services.swarm-matrix-appservice-sync = lib.mkIf ctlActive {
description = "keep the swarm's appservice registration in step with the swarm secret store";
after = [ "swarm-matrix-appservice-render.service" ];
requires = [ "swarm-matrix-appservice-render.service" ];
wants = [ "swarm-matrix-appservice-render.service" ];
wantedBy = [ "multi-user.target" ];
path = [ pkgs.systemd ];
serviceConfig = {
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
ExecStart = "${deployCfg.matrix.ctlPackage}/bin/swarm-matrix-ctl appservice sync";
Restart = "on-failure";
RestartSec = 30;
TimeoutStartSec = 60;
RuntimeDirectory = swarmAppserviceRuntimeName;
RuntimeDirectoryMode = "0700";
RuntimeDirectoryPreserve = "yes";
UMask = "0077";
SyslogIdentifier = "swarm-matrix-appservice";
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = deployCfg.matrix.ctlBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.matrix.ctlBaoClientKeyFile;
MATRIX_APPSERVICE_CERT_ROLE = ctlCertRole;
MATRIX_APPSERVICE_DIR = swarmAppserviceDir;
MATRIX_APPSERVICE_SENDER = swarmSenderLocalpart;
MATRIX_APPSERVICE_USER_REGEX = appserviceUserRegex;
environment = appserviceEnv // {
MATRIX_APPSERVICE_RESTART_UNIT = "tuwunel.service";
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
};

9
nix/host-modules/swarm-bao.nix
View file

@ -653,7 +653,8 @@ let
# write, unlike either sibling, because sync compares before it writes.
#
# The second is the registration's tokens: the store's copy, which matrix-ctl
# seeds from the container's state dir and then follows. `create` and `read`
# creates (from the tokens an earlier version kept in the container, or a
# minted pair) and then follows. `create` and `read`
# only, never `update`: a registration that exists is replaced by an
# operator, not by a unit.
#
@ -2008,9 +2009,9 @@ in
example = "swarm-matrix-ctl.svc";
description = ''
Subject the store's matrix-ctl cert-auth role accepts — the
identity the matrix container's `swarm-matrix-appservice-sync`
unit presents when it syncs the swarm appservice's registration and
publishes its token.
identity the matrix container's `swarm-matrix-appservice-render` and
`swarm-matrix-appservice-sync` units present when they fetch the swarm
appservice's registration and publish its token.
A **third** identity rather than reuse of either sibling above. Its
grant is that credential and the registration it derives from, plus