Watch
0
0
Fork
You've already forked hyperhive
0

swarm-stt: the deploy module (deploy.stt.*, sandboxed container, store leaf, queue grants) and its docs

This commit is contained in:
damocles 2026-11-12 08:36:32 +01:00 • committed by mara
commit 96b5b83fa4
11 changed files with 594 additions and 5 deletions

View file

@ -56,6 +56,7 @@
./glue-ci-runner-otel-oidc-client.nix
./glue-claude-login-oidc-client.nix
./glue-notify-router-oidc-client.nix
./glue-stt-oidc-client.nix
./glue-attic-otel-oidc-client.nix
./glue-forge-otel-oidc-client.nix
./glue-grafana-otel-oidc-client.nix
@ -77,6 +78,7 @@
./swarm-builder.nix
./swarm-claude-login.nix
./swarm-notify-router.nix
./swarm-stt.nix
./swarm-ci-metrics.nix
./swarm-ci-runner.nix
./swarm-review-classifier.nix

View file

@ -691,6 +691,25 @@ in
'';
};
stt.enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = ''
Run the speech-to-text service in a `swarm-stt` container on this
host. It transcribes the voice memos agents send, once per memo, with
whisper.cpp; run it where the inference hardware is. The model is
{option}`services.hyperhive.deploy.stt.settings.modelSize` (default
`small`).
On by default where
{option}`services.hyperhive.deploy.allSwarmServices` is set and this
host has the service's store identity (the pair
{option}`services.hyperhive.deploy.bao.sttClientCertFile`
and `sttClientKeyFile`, minted on the store's host).
'';
};
builder.enable = lib.mkOption {
type = lib.types.bool;
default = false;

View file

@ -124,6 +124,8 @@ in
claudeLoginClientKeyFile = lib.mkDefault "${pkiDir}/claude-login-key.pem";
notifyRouterClientCertFile = lib.mkDefault "${pkiDir}/notify-router.pem";
notifyRouterClientKeyFile = lib.mkDefault "${pkiDir}/notify-router-key.pem";
sttClientCertFile = lib.mkDefault "${pkiDir}/stt.pem";
sttClientKeyFile = lib.mkDefault "${pkiDir}/stt-key.pem";
atticClientCertFile = lib.mkDefault "${pkiDir}/attic.pem";
atticClientKeyFile = lib.mkDefault "${pkiDir}/attic-key.pem";
archiveClientCertFile = lib.mkDefault "${pkiDir}/archive.pem";
@ -337,6 +339,11 @@ in
[ -s ${pkiDir}/notify-router.pem ] || ${signLeaf} ${pkiDir} notify-router \
${lib.escapeShellArg deployCfg.bao.notifyRouterCommonName} "" clientAuth
# The speech-to-text service's, which reads its queue client secret.
# Minted here for the same reason as the router's.
[ -s ${pkiDir}/stt.pem ] || ${signLeaf} ${pkiDir} stt \
${lib.escapeShellArg deployCfg.bao.sttCommonName} "" clientAuth
# The in-swarm cache's, which creates and reads its six secrets.
# Minted here for the matrix-ctl leaf's reason: the cache is a swarm
# service, so on a cache host without the store this is the file an

View file

@ -0,0 +1,39 @@
# Glue: register the speech-to-text service as an OIDC client wherever authelia
# runs.
#
# ONE PAIRING PER FILE — swarm-stt ← authelia, and nothing else.
# Deleting this leaves a router whose token requests authelia answers with a
# 401, and no minted secret for authelia's mint to put in the store.
#
# ⚠️ Gated on authelia being HERE, and NOT on this host running the router,
# for ./glue-controller-oidc-client.nix's reason: a client is a row in THIS
# host's provider config. Registered whether or not any host enables the
# router: nothing in `swarm.*` records that, so a swarm without one carries
# one unused client and one unused minted secret.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
deployCfg = hyperhiveCfg.deploy;
clientId = hyperhiveCfg.swarm.stt.clientId;
in
{
config = lib.mkIf deployCfg.authelia.enable {
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = clientId;
description = "HyperHive speech-to-text service";
# `client_credentials`: a daemon authenticating as itself.
kind = "machine";
audience = [ clientId ];
# The container's journal forwarder mints its token as this client, for
# the swarm collector's receiver, which verifies it offline against
# `/jwks.json`; authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};
}

View file

@ -1472,6 +1472,16 @@ let
}
];
# The speech-to-text service: its own queue client secret, read at start by
# the daemon inside its container, and nothing else.
sttReaders = [
{
name = "swarm-stt";
cn = baoDeploy.sttCommonName;
policyText = readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.stt.clientId}/oidc/client";
}
];
# The forge host's one store identity. The forge's
# `forgejo-swarm-controller-account`, in its container, writes the
# controller's forge token under it, the one writer of that path and
@ -2699,6 +2709,18 @@ in
'';
};
sttCommonName = lib.mkOption {
type = lib.types.str;
default = "swarm-stt";
description = ''
Subject the store's `swarm-stt` cert-auth role accepts: the identity
the speech-to-text service presents from inside its container. Its
grant is read on its own queue client secret.
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
'';
};
atticCommonName = lib.mkOption {
type = lib.types.str;
default = "swarm-attic";
@ -2823,6 +2845,31 @@ in
'';
};
sttClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/stt.pem";
description = ''
Certificate the speech-to-text service presents to the store. Its
subject must be {option}`services.hyperhive.deploy.bao.sttCommonName`.
./glue-bao-tls.nix points it at the leaf it mints on the store's host.
On a service host without the store, it is a file the operator copies
from the store's host. A path, never a value.
'';
};
sttClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/stt-key.pem";
description = ''
Private key for
{option}`services.hyperhive.deploy.bao.sttClientCertFile`.
A path, never a value.
'';
};
forgePublisherClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -4030,6 +4077,8 @@ in
systemd.services.swarm-bao-notify-router-policy = readerPolicyUnit "write the notification router's client-secret-reader bao policy and cert-auth role" notifyRouterReaders;
systemd.services.swarm-bao-stt-policy = readerPolicyUnit "write the speech-to-text service's client-secret-reader bao policy and cert-auth role" sttReaders;
systemd.services.swarm-bao-forge-publisher-policy = readerPolicyUnit "write the forge's controller-token-publisher bao policy and cert-auth role" forgePublisherWriters;
# A FOURTH sibling, same shape and same reasons as the two above. This

View file

@ -958,6 +958,9 @@ in
# Drains its consumer on the forge events stream and queues
# agents' todo ops (`swarm-nats-auth`'s `notify_router_permissions`).
"--notify-router-client ${lib.escapeShellArg config.services.hyperhive.swarm.notify-router.clientId}"
# Hears voice memos on every agent's request subject and
# answers each by reply (`swarm-nats-auth`'s `stt_permissions`).
"--stt-client ${lib.escapeShellArg config.services.hyperhive.swarm.stt.clientId}"
# What an agent may publish to, keyed on its own name: its
# terminal rows and its turn-state header
# (`hive-agent::swarm_agent_state`), two subject families so a
@ -1015,6 +1018,10 @@ in
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$JS.API.CONSUMER.CREATE.ROUTER_TODOS.todo-{agent}.\$\$SWARM.router.todo.{agent}"}"
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$JS.API.CONSUMER.MSG.NEXT.ROUTER_TODOS.todo-{agent}"}"
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$JS.ACK.ROUTER_TODOS.todo-{agent}.>"}"
# An agent sends its voice memo on its own request subject
# (`swarm_wire::stt`), publish only; the answer comes back on
# its inbox.
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.stt.transcribe.{agent}"}"
# Every agent also reads the operator list, the one key of the
# `operator-profiles` bucket, with one direct get. That grant
# names no agent, so the responder adds it itself

View file

@ -224,7 +224,7 @@ in
clientId = hyperhiveCfg.swarm.nats.clientId;
};
# The forwarders in ./swarm-builder.nix, ./swarm-claude-login.nix,
# ./swarm-notify-router.nix and ./swarm-review-classifier.nix, which
# ./swarm-notify-router.nix, ./swarm-stt.nix and ./swarm-review-classifier.nix, which
# address their routes by container name.
swarm-builder = {
clientId = hyperhiveCfg.swarm.builder.clientId;
@ -238,6 +238,9 @@ in
swarm-notify-router = {
clientId = hyperhiveCfg.swarm.notify-router.clientId;
};
swarm-stt = {
clientId = hyperhiveCfg.swarm.stt.clientId;
};
# The forwarders in ./swarm-attic.nix, ./swarm-ci-runner.nix,
# ./swarm-grafana.nix and ./swarm-victorialogs.nix, each authenticated as a
# client of its own.

View file

@ -161,4 +161,11 @@ in
&& deployCfg.bao.notifyRouterClientCertFile != null
&& deployCfg.bao.notifyRouterClientKeyFile != null
);
# The speech-to-text service, defaulted on the same terms.
config.services.hyperhive.deploy.stt.enable = lib.mkDefault (
deployCfg.allSwarmServices
&& deployCfg.bao.sttClientCertFile != null
&& deployCfg.bao.sttClientKeyFile != null
);
}

View file

@ -53,6 +53,8 @@ let
hyperhiveCfg.swarm.claude-login.clientId
# The notification router, which reads it itself inside its container.
hyperhiveCfg.swarm.notify-router.clientId
# The speech-to-text service, which reads it itself inside its container.
hyperhiveCfg.swarm.stt.clientId
# The journal forwarders in the CI runner's, Grafana's, the log store's,
# the cache's, the forge's and the homeserver's containers, each reading it
# inside its container.

View file

@ -0,0 +1,363 @@
# The swarm's speech-to-text service, in a `swarm-stt` container on the host
# that sets `deploy.stt.enable`, which `deploy.allSwarmServices` defaults on
# where the host has the service's store identity. It answers the voice memos
# agents send on `$SWARM.stt.transcribe.<agent>` with whisper.cpp, once per memo
# (the cache is keyed by the audio's hash). Run it where the inference hardware
# is. docs/4-implementation/swarm/stt.md has the design.
#
# The audio comes from rooms with outside people, so the daemon runs as its own
# unprivileged user under a tight systemd sandbox (see `serviceConfig` below),
# with a memory and CPU ceiling, and a state directory as its only writable
# path.
#
# Its own network namespace on the bridge, as ./swarm-claude-login.nix has: it
# reaches the swarm services by name and nothing on the host's loopback. It
# talks only to the queue, the store and authelia, so it can run on any host.
#
# The container holds one credential, its own store leaf, which nspawn hands in
# as a system credential with no copy on disk. The daemon reads its queue
# client secret with it.
{
pkgs,
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
cfg = hyperhiveCfg.deploy.stt;
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = hyperhiveCfg.deploy.bao;
natsCfg = hyperhiveCfg.swarm.nats;
autheliaCfg = hyperhiveCfg.swarm.authelia;
gatewayCfg = hyperhiveCfg.gateway;
networkCfg = hyperhiveCfg.network;
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
machine = "swarm-stt";
unit = "swarm-stt";
clientId = hyperhiveCfg.swarm.stt.clientId;
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-stt` in
# ./swarm-otel-service.nix declares, as ./swarm-claude-login.nix does.
swarmOtelCfg = hyperhiveCfg.swarm.otel;
forwarderEndpoint = "https://${swarmOtelCfg.domain}/${machine}";
forwarderUnit = "swarm-journal-forwarder-oidc";
# Loopback in the container's own netns, and distinct from the other service
# containers' forwarders all the same.
forwarderTelemetryPort = 8904;
# The standard OTLP/HTTP port, on the container's loopback: the forwarder
# takes the service's metrics there and pushes them to the swarm collector.
forwarderOtlpPort = 4318;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# The daemon reaches authelia, the store and the queue by name over TLS, and
# in self-signed mode their leaves chain to the hive CA.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
caBundleModule = caTrust.trustBundle {
inherit pkgs;
name = machine;
consumers = [
unit
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
# collector's and authelia's names.
"opentelemetry-collector"
];
};
haveClientIdentity = baoDeploy.sttClientCertFile != null && baoDeploy.sttClientKeyFile != null;
# Credential id → host source, as ./swarm-claude-login.nix hands its leaf in.
storeFiles = lib.filterAttrs (_: source: source != null) {
stt-bao-cert = baoDeploy.sttClientCertFile;
stt-bao-key = baoDeploy.sttClientKeyFile;
stt-bao-server-ca = baoDeploy.serverCaFile;
};
# nixos-container expands `extraFlags` unquoted, so a source path must be
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
credential = id: "/run/credentials/${unit}.service/${id}";
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
privateNetwork = true;
settings = cfg.settings;
modelHashes = {
base = "sha256-YO1bw90U7qhWST0zQ0m0BXgt3K8AKNS130CINF+6Lv4=";
small = "sha256-G+OpsgY4Z7k35k4ux0gzZKeZF+FX+pjF2UtcH//qmHs=";
};
model =
if settings.modelFile != null then
settings.modelFile
else
pkgs.fetchurl {
url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-${settings.modelSize}.bin";
hash = modelHashes.${settings.modelSize};
};
in
{
options.services.hyperhive.swarm.stt.clientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-stt";
description = ''
The OAuth2 client id the speech-to-text service presents to the swarm
queue. Read-only: the host running authelia registers it, the queue's
auth-callout responder grants it the service's subjects, and
authelia's mint stores its secret in the store, all from this one value.
'';
};
options.services.hyperhive.deploy.stt.package = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-stt";
description = "The speech-to-text service daemon package.";
};
options.services.hyperhive.deploy.stt.settings = {
modelSize = lib.mkOption {
type = lib.types.enum [
"base"
"small"
];
default = "small";
description = ''
The whisper model: `small` (about 470 MB, the default) or `base`
(about 140 MB, faster and less accurate). Another model comes from
{option}`services.hyperhive.deploy.stt.settings.modelFile`.
'';
};
modelFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
A ggml model file to use instead of the one
{option}`services.hyperhive.deploy.stt.settings.modelSize` names, such
as a bigger model on a host with the hardware for it. Copied into the
store.
'';
};
cacheTtlSecs = lib.mkOption {
type = lib.types.ints.positive;
default = 24 * 60 * 60;
description = "How long a transcript stays cached, counted from when it was written.";
};
maxWaiting = lib.mkOption {
type = lib.types.ints.positive;
default = 3;
description = ''
Requests held at once, running or waiting for the engine's one slot. Times
{option}`services.hyperhive.deploy.stt.settings.runTimeoutSecs` it must
stay under the 15-minute reply deadline, or the daemon refuses to start.
'';
};
runTimeoutSecs = lib.mkOption {
type = lib.types.ints.positive;
default = 240;
description = "The longest one run (decode and transcribe) may take.";
};
maxAudioSecs = lib.mkOption {
type = lib.types.ints.positive;
default = 300;
description = "Seconds of audio decoded; ffmpeg drops the rest.";
};
maxBytes = lib.mkOption {
type = lib.types.ints.positive;
default = 6 * 1024 * 1024;
description = "The largest audio file taken, in bytes. The queue carries 8 MiB.";
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = haveClientIdentity;
message = ''
services.hyperhive.deploy.stt.enable requires the service's
swarm-secret-store client identity: set both
services.hyperhive.deploy.bao.sttClientCertFile
services.hyperhive.deploy.bao.sttClientKeyFile
The service reads its queue client secret with it.
Both default to the pair glue-bao-tls.nix mints when
services.hyperhive.deploy.bao.enable is true (the host that runs the
store). Otherwise copy stt.pem and stt-key.pem
from the store host's /var/lib/swarm-bao-pki/ and name them in those
two options.
'';
}
{
assertion = lib.all oneWord (lib.attrValues storeFiles);
message = ''
The speech-to-text service's store identity paths must not contain
whitespace: nspawn's --load-credential receives them through
nixos-container's unquoted extra flags.
'';
}
];
# The container hangs off the bridge, takes its address from the gateway's
# DHCP and resolves through its dnsmasq.
services.hyperhive.network.enable = lib.mkDefault true;
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# nspawn fails to start the container when a credential source is
# missing, so where this host mints the leaf the container waits for it.
systemd.services."container@${machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly machine)
{ after = [ "swarm-bao-pki.service" ]; }
];
containers.${machine} = {
autoStart = true;
# The transcript cache lives in the container's /var/lib/swarm-stt.
ephemeral = false;
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
hostBridge = networkCfg.bridgeName;
bindMounts = caTrust.bindMount;
config =
{ lib, ... }:
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
caBundleModule
];
services.hyperhive.swarmContainer = {
inherit privateNetwork;
writesOwnResolvConf = false;
};
networking.nameservers = [ networkCfg.bridgeIp ];
networking.interfaces.eth0.useDHCP = true;
users.users.${unit} = {
isSystemUser = true;
group = unit;
};
users.groups.${unit} = { };
# Authenticated as the daemon's own OIDC client, whose secret its
# store identity already reads: one identity for the whole container.
services.hyperhive.swarmJournalForwarder = {
enable = true;
serviceName = "stt";
endpoint = forwarderEndpoint;
inherit clientId;
tokenUrl = "${autheliaCfg.url}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
otlpMetricsPort = forwarderOtlpPort;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${clientId}/oidc/client";
clientCertFile = forwarderCredential "stt-bao-cert";
clientKeyFile = forwarderCredential "stt-bao-key";
caFile = if storeFiles ? stt-bao-server-ca then forwarderCredential "stt-bao-server-ca" else null;
};
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
systemd.services.${unit} = {
description = "transcribe voice memos";
wantedBy = [ "multi-user.target" ];
wants = [ "network-online.target" ];
after = [ "network-online.target" ];
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = credential "stt-bao-cert";
BAO_CLIENT_KEY = credential "stt-bao-key";
SWARM_STT_STATE_DIR = "/var/lib/${unit}";
SWARM_STT_FFMPEG = "${pkgs.ffmpeg-headless}/bin/ffmpeg";
SWARM_STT_WHISPER_CLI = "${pkgs.whisper-cpp}/bin/whisper-cli";
SWARM_STT_MODEL = "${model}";
SWARM_STT_CACHE_TTL_SECS = toString settings.cacheTtlSecs;
SWARM_STT_MAX_WAITING = toString settings.maxWaiting;
SWARM_STT_RUN_TIMEOUT_SECS = toString settings.runTimeoutSecs;
SWARM_STT_MAX_AUDIO_SECS = toString settings.maxAudioSecs;
SWARM_STT_MAX_BYTES = toString settings.maxBytes;
SWARM_STT_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
SWARM_STT_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
SWARM_STT_OIDC_CLIENT_ID = clientId;
OTEL_EXPORTER_OTLP_ENDPOINT = "http://127.0.0.1:${toString forwarderOtlpPort}";
OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf";
OTEL_RESOURCE_ATTRIBUTES = "service.name=hyperhive-stt,swarm=${
if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name
}";
}
// lib.optionalAttrs (storeFiles ? stt-bao-server-ca) {
BAO_CACERT = credential "stt-bao-server-ca";
}
// lib.optionalAttrs caTrust.useSelfSigned {
SWARM_STT_OIDC_CA_FILE = caTrust.bundlePathFor machine;
};
serviceConfig = {
ExecStart = "${cfg.package}/bin/swarm-stt";
User = unit;
Group = unit;
UMask = "0077";
# Bare ids: the system credentials nspawn handed in.
LoadCredential = lib.attrNames storeFiles;
Restart = "on-failure";
RestartSec = "10s";
SyslogIdentifier = unit;
# The sandbox for a service that decodes files strangers sent:
# the only writable path is its state directory, it sees no
# other user's processes or devices, cannot gain privileges, and
# may open sockets to the queue, the store and authelia only
# (the container's netns has no other route).
StateDirectory = unit;
StateDirectoryMode = "0700";
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
PrivateUsers = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectKernelLogs = true;
ProtectControlGroups = true;
ProtectClock = true;
ProtectHostname = true;
ProtectProc = "invisible";
ProcSubset = "pid";
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
LockPersonality = true;
MemoryDenyWriteExecute = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
"AF_UNIX"
];
SystemCallArchitectures = "native";
SystemCallFilter = [
"@system-service"
"~@privileged"
"~@resources"
];
CapabilityBoundingSet = "";
# A hostile file must not take the host's memory or every core.
MemoryMax = "3G";
CPUQuota = "400%";
TasksMax = 256;
};
};
};
};
};
}

View file

@ -63,6 +63,7 @@ let
deployCfg.bao.ciMetricsCommonName
deployCfg.bao.claudeLoginCommonName
deployCfg.bao.notifyRouterCommonName
deployCfg.bao.sttCommonName
deployCfg.bao.atticCommonName
deployCfg.bao.archiveCommonName
deployCfg.bao.forgePublisherCommonName