From 96b5b83fa48c1ad941aea5e77f5691a74d973ef1 Mon Sep 17 00:00:00 2001 From: damocles Date: Thu, 12 Nov 2026 08:36:32 +0100 Subject: [PATCH] swarm-stt: the deploy module (deploy.stt.*, sandboxed container, store leaf, queue grants) and its docs --- docs/2-handbook/swarm/README.md | 3 + docs/2-handbook/swarm/services.md | 24 ++ .../swarm/credentials-reference.md | 1 + docs/examples/semi-spread/swarm-services.nix | 1 + flake.nix | 3 + nix/host-modules/default.nix | 2 + nix/host-modules/deploy.nix | 19 + nix/host-modules/glue-bao-tls.nix | 7 + nix/host-modules/glue-stt-oidc-client.nix | 39 ++ nix/host-modules/swarm-bao.nix | 49 +++ nix/host-modules/swarm-nats.nix | 7 + nix/host-modules/swarm-otel-service.nix | 5 +- nix/host-modules/swarm-required-services.nix | 7 + nix/host-modules/swarm-secret-publisher.nix | 2 + nix/host-modules/swarm-stt.nix | 363 ++++++++++++++++++ nix/host-modules/swarm.nix | 1 + nix/module-eval/bao-grants.nix | 22 ++ nix/module-eval/core-toggle.nix | 1 + nix/module-eval/nats-authelia.nix | 7 + nix/module-eval/semi-spread.nix | 1 + nix/module-eval/swarm-services-switch.nix | 34 +- nix/packages/default.nix | 1 + 22 files changed, 594 insertions(+), 5 deletions(-) create mode 100644 nix/host-modules/glue-stt-oidc-client.nix create mode 100644 nix/host-modules/swarm-stt.nix diff --git a/docs/2-handbook/swarm/README.md b/docs/2-handbook/swarm/README.md index 84f8d81ce..800578902 100644 --- a/docs/2-handbook/swarm/README.md +++ b/docs/2-handbook/swarm/README.md @@ -35,6 +35,9 @@ grafana, the controller, authelia, …) → they concern: assigned, unassigned, cleared on close; and the classifier's review requested on a push. → [`notify-router.md`](../../4-implementation/swarm/notify-router.md) +- **speech-to-text** — voice memos agents receive, turned into text once per + memo for the whole swarm. → + [`stt.md`](../../4-implementation/swarm/stt.md) - **swarm UI** — the operator's day-to-day surface, on the swarm apex, `admins` only. → [`ui.md`](ui.md) - **shared services** — one forge, homeserver, SSO, queue and metrics/logs diff --git a/docs/2-handbook/swarm/services.md b/docs/2-handbook/swarm/services.md index e67789a66..70447ce46 100644 --- a/docs/2-handbook/swarm/services.md +++ b/docs/2-handbook/swarm/services.md @@ -128,6 +128,7 @@ look before you switch them on. | Review classifier | Checks PRs against a repo's review rules. | `deploy.review-classifier.enable` | [Review classifier](#review-classifier) | | Claude login minter | Runs the **log in** button on an agent's card. | `deploy.claude-login.enable` | [Claude login minter](#claude-login-minter) | | Notification router | Turns forge events into notifications for agents. | `deploy.notify-router.enable` | [Notification router](#notification-router) | +| Speech to text | Turns voice memos into text for agents. | `deploy.stt.enable` | [Speech to text](#notification-router) | ### SSO (authelia) @@ -416,3 +417,26 @@ It reaches the queue and the secret store by name, under its own `notify-router` leaf. On a host without the store, that leaf is the one file to copy, as for the CI runner. The flows and the delivery path are in [`notify-router.md`](../../4-implementation/swarm/notify-router.md). + +### Speech to text + +The speech-to-text service, in a `swarm-stt` container on the host that sets +`deploy.stt.enable`. `allSwarmServices` defaults it on wherever the host has the +service's store identity, on the router's terms. Run it where the inference +hardware is: it transcribes with whisper.cpp. + +An agent that receives a voice memo sends the audio to the service over the +queue, on its own subject, and gets the text back. The service keeps each +transcript for a day, keyed by a hash of the audio, so the same memo sent to several agents costs +one whisper run. `deploy.stt.settings.modelSize` picks the +model (`small` by default, or `base`), and `modelFile` takes any other ggml +model. The cache time, the limits on file size, audio length and run time, and +how many requests it holds at once are settings next to it. + +The audio comes from rooms with outside people, so the service runs as its own +unprivileged user under a tight systemd sandbox, with a memory ceiling and +a CPU quota, and the container reaches the queue, the store and authelia +and nothing else. The file's first bytes must look like an audio container +before anything decodes it. Its leaf and its queue client work as the router's +do. The design, the settings and the reasons are in +[`stt.md`](../../4-implementation/swarm/stt.md). diff --git a/docs/4-implementation/swarm/credentials-reference.md b/docs/4-implementation/swarm/credentials-reference.md index ef277cb4e..b01101d06 100644 --- a/docs/4-implementation/swarm/credentials-reference.md +++ b/docs/4-implementation/swarm/credentials-reference.md @@ -186,6 +186,7 @@ login minter's, the forge host's and Grafana's, which reach more than one: | `swarm-review-classifier`, in its container | `reviewClassifierClientCertFile` / `reviewClassifierClientKeyFile` | `swarm/services/review-classifier/anthropic-api-key`, `swarm/services/review-classifier/acp-provider` and `swarm/services//oidc/client` | | `swarm-claude-login`, in its container | `claudeLoginClientCertFile` / `claudeLoginClientKeyFile` | creates and updates `swarm/agents/+/claude-credentials`, reads and deletes its metadata; reads `swarm/services//oidc/client` | | `swarm-notify-router`, in its container | `notifyRouterClientCertFile` / `notifyRouterClientKeyFile` | `swarm/services//oidc/client` | +| `swarm-stt`, in its container | `sttClientCertFile` / `sttClientKeyFile` | `swarm/services//oidc/client` | | `swarm-builder`, in its container | `builderClientCertFile` / `builderClientKeyFile` | `swarm/services/builder/forge/token`, `swarm/services/attic/push-token` and `swarm/services//oidc/client` | | `swarm-bao-attic-secret` and `swarm-bao-attic-tokens` | `atticClientCertFile` / `atticClientKeyFile` | creates and reads `swarm/services/attic/jwt-secret`, `swarm/services/attic/push-token` and `swarm/services/attic/pull-token` | | `swarm-archive-users`, `swarm-archive-prune` and, with an S3 bucket, `swarm-archive` | `archiveClientCertFile` / `archiveClientKeyFile` | reads `swarm/services/archive/users/*`, `swarm/services/archive/restic/*` and `swarm/services/archive/s3/*`, lists `swarm/services/archive/users/` | diff --git a/docs/examples/semi-spread/swarm-services.nix b/docs/examples/semi-spread/swarm-services.nix index 83122442f..45e25513d 100644 --- a/docs/examples/semi-spread/swarm-services.nix +++ b/docs/examples/semi-spread/swarm-services.nix @@ -49,6 +49,7 @@ in (pair "forwarderOidc" "forwarder-oidc") (pair "reviewClassifier" "review-classifier") (pair "notifyRouter" "notify-router") + (pair "stt" "stt") (pair "claudeLogin" "claude-login") (pair "ciMetrics" "ci-metrics") ]; diff --git a/flake.nix b/flake.nix index e1e4f0372..8eb4ac5f5 100644 --- a/flake.nix +++ b/flake.nix @@ -186,6 +186,9 @@ services.hyperhive.deploy.notify-router.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-notify-router; + services.hyperhive.deploy.stt.package = + lib.mkDefault + self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-stt; services.hyperhive.deploy.authelia.bridgePackage = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-authelia-bridge; diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 89dd19ce2..955ada379 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -56,6 +56,7 @@ ./glue-ci-runner-otel-oidc-client.nix ./glue-claude-login-oidc-client.nix ./glue-notify-router-oidc-client.nix + ./glue-stt-oidc-client.nix ./glue-attic-otel-oidc-client.nix ./glue-forge-otel-oidc-client.nix ./glue-grafana-otel-oidc-client.nix @@ -77,6 +78,7 @@ ./swarm-builder.nix ./swarm-claude-login.nix ./swarm-notify-router.nix + ./swarm-stt.nix ./swarm-ci-metrics.nix ./swarm-ci-runner.nix ./swarm-review-classifier.nix diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 6c4eb02b2..a6ce135da 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -691,6 +691,25 @@ in ''; }; + stt.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = '' + Run the speech-to-text service in a `swarm-stt` container on this + host. It transcribes the voice memos agents send, once per memo, with + whisper.cpp; run it where the inference hardware is. The model is + {option}`services.hyperhive.deploy.stt.settings.modelSize` (default + `small`). + + On by default where + {option}`services.hyperhive.deploy.allSwarmServices` is set and this + host has the service's store identity (the pair + {option}`services.hyperhive.deploy.bao.sttClientCertFile` + and `sttClientKeyFile`, minted on the store's host). + ''; + }; + builder.enable = lib.mkOption { type = lib.types.bool; default = false; diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index 3d5b19d08..f9aec2f72 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -124,6 +124,8 @@ in claudeLoginClientKeyFile = lib.mkDefault "${pkiDir}/claude-login-key.pem"; notifyRouterClientCertFile = lib.mkDefault "${pkiDir}/notify-router.pem"; notifyRouterClientKeyFile = lib.mkDefault "${pkiDir}/notify-router-key.pem"; + sttClientCertFile = lib.mkDefault "${pkiDir}/stt.pem"; + sttClientKeyFile = lib.mkDefault "${pkiDir}/stt-key.pem"; atticClientCertFile = lib.mkDefault "${pkiDir}/attic.pem"; atticClientKeyFile = lib.mkDefault "${pkiDir}/attic-key.pem"; archiveClientCertFile = lib.mkDefault "${pkiDir}/archive.pem"; @@ -337,6 +339,11 @@ in [ -s ${pkiDir}/notify-router.pem ] || ${signLeaf} ${pkiDir} notify-router \ ${lib.escapeShellArg deployCfg.bao.notifyRouterCommonName} "" clientAuth + # The speech-to-text service's, which reads its queue client secret. + # Minted here for the same reason as the router's. + [ -s ${pkiDir}/stt.pem ] || ${signLeaf} ${pkiDir} stt \ + ${lib.escapeShellArg deployCfg.bao.sttCommonName} "" clientAuth + # The in-swarm cache's, which creates and reads its six secrets. # Minted here for the matrix-ctl leaf's reason: the cache is a swarm # service, so on a cache host without the store this is the file an diff --git a/nix/host-modules/glue-stt-oidc-client.nix b/nix/host-modules/glue-stt-oidc-client.nix new file mode 100644 index 000000000..13742e1ac --- /dev/null +++ b/nix/host-modules/glue-stt-oidc-client.nix @@ -0,0 +1,39 @@ +# Glue: register the speech-to-text service as an OIDC client wherever authelia +# runs. +# +# ONE PAIRING PER FILE — swarm-stt ← authelia, and nothing else. +# Deleting this leaves a router whose token requests authelia answers with a +# 401, and no minted secret for authelia's mint to put in the store. +# +# ⚠️ Gated on authelia being HERE, and NOT on this host running the router, +# for ./glue-controller-oidc-client.nix's reason: a client is a row in THIS +# host's provider config. Registered whether or not any host enables the +# router: nothing in `swarm.*` records that, so a swarm without one carries +# one unused client and one unused minted secret. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + clientId = hyperhiveCfg.swarm.stt.clientId; +in +{ + config = lib.mkIf deployCfg.authelia.enable { + services.hyperhive.swarm.authelia.oidc.clients = [ + { + id = clientId; + description = "HyperHive speech-to-text service"; + # `client_credentials`: a daemon authenticating as itself. + kind = "machine"; + audience = [ clientId ]; + # The container's journal forwarder mints its token as this client, for + # the swarm collector's receiver, which verifies it offline against + # `/jwks.json`; authelia's default opaque handle fails there. + accessTokenSignedResponseAlg = "RS256"; + } + ]; + }; +} diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 41b89091d..d629a115c 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -1472,6 +1472,16 @@ let } ]; + # The speech-to-text service: its own queue client secret, read at start by + # the daemon inside its container, and nothing else. + sttReaders = [ + { + name = "swarm-stt"; + cn = baoDeploy.sttCommonName; + policyText = readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.stt.clientId}/oidc/client"; + } + ]; + # The forge host's one store identity. The forge's # `forgejo-swarm-controller-account`, in its container, writes the # controller's forge token under it, the one writer of that path and @@ -2699,6 +2709,18 @@ in ''; }; + sttCommonName = lib.mkOption { + type = lib.types.str; + default = "swarm-stt"; + description = '' + Subject the store's `swarm-stt` cert-auth role accepts: the identity + the speech-to-text service presents from inside its container. Its + grant is read on its own queue client secret. + + ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings. + ''; + }; + atticCommonName = lib.mkOption { type = lib.types.str; default = "swarm-attic"; @@ -2823,6 +2845,31 @@ in ''; }; + sttClientCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/stt.pem"; + description = '' + Certificate the speech-to-text service presents to the store. Its + subject must be {option}`services.hyperhive.deploy.bao.sttCommonName`. + + ./glue-bao-tls.nix points it at the leaf it mints on the store's host. + On a service host without the store, it is a file the operator copies + from the store's host. A path, never a value. + ''; + }; + + sttClientKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/stt-key.pem"; + description = '' + Private key for + {option}`services.hyperhive.deploy.bao.sttClientCertFile`. + A path, never a value. + ''; + }; + forgePublisherClientCertFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -4030,6 +4077,8 @@ in systemd.services.swarm-bao-notify-router-policy = readerPolicyUnit "write the notification router's client-secret-reader bao policy and cert-auth role" notifyRouterReaders; + systemd.services.swarm-bao-stt-policy = readerPolicyUnit "write the speech-to-text service's client-secret-reader bao policy and cert-auth role" sttReaders; + systemd.services.swarm-bao-forge-publisher-policy = readerPolicyUnit "write the forge's controller-token-publisher bao policy and cert-auth role" forgePublisherWriters; # A FOURTH sibling, same shape and same reasons as the two above. This diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 320f9645f..1c90122a9 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -958,6 +958,9 @@ in # Drains its consumer on the forge events stream and queues # agents' todo ops (`swarm-nats-auth`'s `notify_router_permissions`). "--notify-router-client ${lib.escapeShellArg config.services.hyperhive.swarm.notify-router.clientId}" + # Hears voice memos on every agent's request subject and + # answers each by reply (`swarm-nats-auth`'s `stt_permissions`). + "--stt-client ${lib.escapeShellArg config.services.hyperhive.swarm.stt.clientId}" # What an agent may publish to, keyed on its own name: its # terminal rows and its turn-state header # (`hive-agent::swarm_agent_state`), two subject families so a @@ -1015,6 +1018,10 @@ in "--agent-token-publish-subject ${lib.escapeShellArg "\$\$JS.API.CONSUMER.CREATE.ROUTER_TODOS.todo-{agent}.\$\$SWARM.router.todo.{agent}"}" "--agent-token-publish-subject ${lib.escapeShellArg "\$\$JS.API.CONSUMER.MSG.NEXT.ROUTER_TODOS.todo-{agent}"}" "--agent-token-publish-subject ${lib.escapeShellArg "\$\$JS.ACK.ROUTER_TODOS.todo-{agent}.>"}" + # An agent sends its voice memo on its own request subject + # (`swarm_wire::stt`), publish only; the answer comes back on + # its inbox. + "--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.stt.transcribe.{agent}"}" # Every agent also reads the operator list, the one key of the # `operator-profiles` bucket, with one direct get. That grant # names no agent, so the responder adds it itself diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 1eb0a8b05..2d2b6a7e0 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -224,7 +224,7 @@ in clientId = hyperhiveCfg.swarm.nats.clientId; }; # The forwarders in ./swarm-builder.nix, ./swarm-claude-login.nix, - # ./swarm-notify-router.nix and ./swarm-review-classifier.nix, which + # ./swarm-notify-router.nix, ./swarm-stt.nix and ./swarm-review-classifier.nix, which # address their routes by container name. swarm-builder = { clientId = hyperhiveCfg.swarm.builder.clientId; @@ -238,6 +238,9 @@ in swarm-notify-router = { clientId = hyperhiveCfg.swarm.notify-router.clientId; }; + swarm-stt = { + clientId = hyperhiveCfg.swarm.stt.clientId; + }; # The forwarders in ./swarm-attic.nix, ./swarm-ci-runner.nix, # ./swarm-grafana.nix and ./swarm-victorialogs.nix, each authenticated as a # client of its own. diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix index 93248fafa..e392c36a6 100644 --- a/nix/host-modules/swarm-required-services.nix +++ b/nix/host-modules/swarm-required-services.nix @@ -161,4 +161,11 @@ in && deployCfg.bao.notifyRouterClientCertFile != null && deployCfg.bao.notifyRouterClientKeyFile != null ); + + # The speech-to-text service, defaulted on the same terms. + config.services.hyperhive.deploy.stt.enable = lib.mkDefault ( + deployCfg.allSwarmServices + && deployCfg.bao.sttClientCertFile != null + && deployCfg.bao.sttClientKeyFile != null + ); } diff --git a/nix/host-modules/swarm-secret-publisher.nix b/nix/host-modules/swarm-secret-publisher.nix index b16bbe2d7..3f53e42f3 100644 --- a/nix/host-modules/swarm-secret-publisher.nix +++ b/nix/host-modules/swarm-secret-publisher.nix @@ -53,6 +53,8 @@ let hyperhiveCfg.swarm.claude-login.clientId # The notification router, which reads it itself inside its container. hyperhiveCfg.swarm.notify-router.clientId + # The speech-to-text service, which reads it itself inside its container. + hyperhiveCfg.swarm.stt.clientId # The journal forwarders in the CI runner's, Grafana's, the log store's, # the cache's, the forge's and the homeserver's containers, each reading it # inside its container. diff --git a/nix/host-modules/swarm-stt.nix b/nix/host-modules/swarm-stt.nix new file mode 100644 index 000000000..ec89de144 --- /dev/null +++ b/nix/host-modules/swarm-stt.nix @@ -0,0 +1,363 @@ +# The swarm's speech-to-text service, in a `swarm-stt` container on the host +# that sets `deploy.stt.enable`, which `deploy.allSwarmServices` defaults on +# where the host has the service's store identity. It answers the voice memos +# agents send on `$SWARM.stt.transcribe.` with whisper.cpp, once per memo +# (the cache is keyed by the audio's hash). Run it where the inference hardware +# is. docs/4-implementation/swarm/stt.md has the design. +# +# The audio comes from rooms with outside people, so the daemon runs as its own +# unprivileged user under a tight systemd sandbox (see `serviceConfig` below), +# with a memory and CPU ceiling, and a state directory as its only writable +# path. +# +# Its own network namespace on the bridge, as ./swarm-claude-login.nix has: it +# reaches the swarm services by name and nothing on the host's loopback. It +# talks only to the queue, the store and authelia, so it can run on any host. +# +# The container holds one credential, its own store leaf, which nspawn hands in +# as a system credential with no copy on disk. The daemon reads its queue +# client secret with it. +{ + pkgs, + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + + cfg = hyperhiveCfg.deploy.stt; + baoCfg = hyperhiveCfg.swarm.bao; + baoDeploy = hyperhiveCfg.deploy.bao; + natsCfg = hyperhiveCfg.swarm.nats; + autheliaCfg = hyperhiveCfg.swarm.authelia; + gatewayCfg = hyperhiveCfg.gateway; + networkCfg = hyperhiveCfg.network; + tlsCfg = hyperhiveCfg.deploy.hive-controller.tls; + + machine = "swarm-stt"; + unit = "swarm-stt"; + clientId = hyperhiveCfg.swarm.stt.clientId; + + # The container's journal forwarder pushes to its own receiver on the swarm + # collector, the route `serviceProducers.swarm-stt` in + # ./swarm-otel-service.nix declares, as ./swarm-claude-login.nix does. + swarmOtelCfg = hyperhiveCfg.swarm.otel; + forwarderEndpoint = "https://${swarmOtelCfg.domain}/${machine}"; + forwarderUnit = "swarm-journal-forwarder-oidc"; + # Loopback in the container's own netns, and distinct from the other service + # containers' forwarders all the same. + forwarderTelemetryPort = 8904; + # The standard OTLP/HTTP port, on the container's loopback: the forwarder + # takes the service's metrics there and pushes them to the swarm collector. + forwarderOtlpPort = 4318; + + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + + # The daemon reaches authelia, the store and the queue by name over TLS, and + # in self-signed mode their leaves chain to the hive CA. + caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; + caBundleModule = caTrust.trustBundle { + inherit pkgs; + name = machine; + consumers = [ + unit + # Go: `SSL_CERT_FILE` replaces its trust store, for the swarm + # collector's and authelia's names. + "opentelemetry-collector" + ]; + }; + + haveClientIdentity = baoDeploy.sttClientCertFile != null && baoDeploy.sttClientKeyFile != null; + + # Credential id → host source, as ./swarm-claude-login.nix hands its leaf in. + storeFiles = lib.filterAttrs (_: source: source != null) { + stt-bao-cert = baoDeploy.sttClientCertFile; + stt-bao-key = baoDeploy.sttClientKeyFile; + stt-bao-server-ca = baoDeploy.serverCaFile; + }; + # nixos-container expands `extraFlags` unquoted, so a source path must be + # one word. + oneWord = path: builtins.match ".*[[:space:]].*" path == null; + credential = id: "/run/credentials/${unit}.service/${id}"; + forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}"; + + privateNetwork = true; + + settings = cfg.settings; + modelHashes = { + base = "sha256-YO1bw90U7qhWST0zQ0m0BXgt3K8AKNS130CINF+6Lv4="; + small = "sha256-G+OpsgY4Z7k35k4ux0gzZKeZF+FX+pjF2UtcH//qmHs="; + }; + model = + if settings.modelFile != null then + settings.modelFile + else + pkgs.fetchurl { + url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-${settings.modelSize}.bin"; + hash = modelHashes.${settings.modelSize}; + }; +in +{ + options.services.hyperhive.swarm.stt.clientId = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-stt"; + description = '' + The OAuth2 client id the speech-to-text service presents to the swarm + queue. Read-only: the host running authelia registers it, the queue's + auth-callout responder grants it the service's subjects, and + authelia's mint stores its secret in the store, all from this one value. + ''; + }; + + options.services.hyperhive.deploy.stt.package = lib.mkOption { + type = lib.types.package; + defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-stt"; + description = "The speech-to-text service daemon package."; + }; + + options.services.hyperhive.deploy.stt.settings = { + modelSize = lib.mkOption { + type = lib.types.enum [ + "base" + "small" + ]; + default = "small"; + description = '' + The whisper model: `small` (about 470 MB, the default) or `base` + (about 140 MB, faster and less accurate). Another model comes from + {option}`services.hyperhive.deploy.stt.settings.modelFile`. + ''; + }; + modelFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + A ggml model file to use instead of the one + {option}`services.hyperhive.deploy.stt.settings.modelSize` names, such + as a bigger model on a host with the hardware for it. Copied into the + store. + ''; + }; + cacheTtlSecs = lib.mkOption { + type = lib.types.ints.positive; + default = 24 * 60 * 60; + description = "How long a transcript stays cached, counted from when it was written."; + }; + maxWaiting = lib.mkOption { + type = lib.types.ints.positive; + default = 3; + description = '' + Requests held at once, running or waiting for the engine's one slot. Times + {option}`services.hyperhive.deploy.stt.settings.runTimeoutSecs` it must + stay under the 15-minute reply deadline, or the daemon refuses to start. + ''; + }; + runTimeoutSecs = lib.mkOption { + type = lib.types.ints.positive; + default = 240; + description = "The longest one run (decode and transcribe) may take."; + }; + maxAudioSecs = lib.mkOption { + type = lib.types.ints.positive; + default = 300; + description = "Seconds of audio decoded; ffmpeg drops the rest."; + }; + maxBytes = lib.mkOption { + type = lib.types.ints.positive; + default = 6 * 1024 * 1024; + description = "The largest audio file taken, in bytes. The queue carries 8 MiB."; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = haveClientIdentity; + message = '' + services.hyperhive.deploy.stt.enable requires the service's + swarm-secret-store client identity: set both + + services.hyperhive.deploy.bao.sttClientCertFile + services.hyperhive.deploy.bao.sttClientKeyFile + + The service reads its queue client secret with it. + + Both default to the pair glue-bao-tls.nix mints when + services.hyperhive.deploy.bao.enable is true (the host that runs the + store). Otherwise copy stt.pem and stt-key.pem + from the store host's /var/lib/swarm-bao-pki/ and name them in those + two options. + ''; + } + { + assertion = lib.all oneWord (lib.attrValues storeFiles); + message = '' + The speech-to-text service's store identity paths must not contain + whitespace: nspawn's --load-credential receives them through + nixos-container's unquoted extra flags. + ''; + } + ]; + + # The container hangs off the bridge, takes its address from the gateway's + # DHCP and resolves through its dnsmasq. + services.hyperhive.network.enable = lib.mkDefault true; + services.hyperhive.gateway.enable = lib.mkDefault true; + services.hyperhive.gateway.dns.enable = lib.mkDefault true; + + # nspawn fails to start the container when a credential source is + # missing, so where this host mints the leaf the container waits for it. + systemd.services."container@${machine}" = lib.mkMerge [ + caTrust.containerOrdering + (rootfsCheck.warnOnly machine) + { after = [ "swarm-bao-pki.service" ]; } + ]; + + containers.${machine} = { + autoStart = true; + # The transcript cache lives in the container's /var/lib/swarm-stt. + ephemeral = false; + # No `--link-journal=host`: the journal stays in the guest, where the + # container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix). + extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles; + inherit privateNetwork; + hostBridge = networkCfg.bridgeName; + bindMounts = caTrust.bindMount; + + config = + { lib, ... }: + { + imports = [ + ../container-modules/swarm-container.nix + ../container-modules/swarm-journal-forwarder.nix + caBundleModule + ]; + + services.hyperhive.swarmContainer = { + inherit privateNetwork; + writesOwnResolvConf = false; + }; + networking.nameservers = [ networkCfg.bridgeIp ]; + networking.interfaces.eth0.useDHCP = true; + + users.users.${unit} = { + isSystemUser = true; + group = unit; + }; + users.groups.${unit} = { }; + + # Authenticated as the daemon's own OIDC client, whose secret its + # store identity already reads: one identity for the whole container. + services.hyperhive.swarmJournalForwarder = { + enable = true; + serviceName = "stt"; + endpoint = forwarderEndpoint; + inherit clientId; + tokenUrl = "${autheliaCfg.url}/api/oidc/token"; + telemetryPort = forwarderTelemetryPort; + otlpMetricsPort = forwarderOtlpPort; + store = { + package = baoDeploy.package; + addr = "https://${baoCfg.domain}:${toString baoCfg.port}"; + secretPath = "secret/swarm/services/${clientId}/oidc/client"; + clientCertFile = forwarderCredential "stt-bao-cert"; + clientKeyFile = forwarderCredential "stt-bao-key"; + caFile = if storeFiles ? stt-bao-server-ca then forwarderCredential "stt-bao-server-ca" else null; + }; + }; + systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles; + + systemd.services.${unit} = { + description = "transcribe voice memos"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = credential "stt-bao-cert"; + BAO_CLIENT_KEY = credential "stt-bao-key"; + SWARM_STT_STATE_DIR = "/var/lib/${unit}"; + SWARM_STT_FFMPEG = "${pkgs.ffmpeg-headless}/bin/ffmpeg"; + SWARM_STT_WHISPER_CLI = "${pkgs.whisper-cpp}/bin/whisper-cli"; + SWARM_STT_MODEL = "${model}"; + SWARM_STT_CACHE_TTL_SECS = toString settings.cacheTtlSecs; + SWARM_STT_MAX_WAITING = toString settings.maxWaiting; + SWARM_STT_RUN_TIMEOUT_SECS = toString settings.runTimeoutSecs; + SWARM_STT_MAX_AUDIO_SECS = toString settings.maxAudioSecs; + SWARM_STT_MAX_BYTES = toString settings.maxBytes; + SWARM_STT_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}"; + SWARM_STT_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token"; + SWARM_STT_OIDC_CLIENT_ID = clientId; + OTEL_EXPORTER_OTLP_ENDPOINT = "http://127.0.0.1:${toString forwarderOtlpPort}"; + OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf"; + OTEL_RESOURCE_ATTRIBUTES = "service.name=hyperhive-stt,swarm=${ + if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name + }"; + } + // lib.optionalAttrs (storeFiles ? stt-bao-server-ca) { + BAO_CACERT = credential "stt-bao-server-ca"; + } + // lib.optionalAttrs caTrust.useSelfSigned { + SWARM_STT_OIDC_CA_FILE = caTrust.bundlePathFor machine; + }; + serviceConfig = { + ExecStart = "${cfg.package}/bin/swarm-stt"; + User = unit; + Group = unit; + UMask = "0077"; + # Bare ids: the system credentials nspawn handed in. + LoadCredential = lib.attrNames storeFiles; + Restart = "on-failure"; + RestartSec = "10s"; + SyslogIdentifier = unit; + + # The sandbox for a service that decodes files strangers sent: + # the only writable path is its state directory, it sees no + # other user's processes or devices, cannot gain privileges, and + # may open sockets to the queue, the store and authelia only + # (the container's netns has no other route). + StateDirectory = unit; + StateDirectoryMode = "0700"; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + PrivateUsers = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + ProtectClock = true; + ProtectHostname = true; + ProtectProc = "invisible"; + ProcSubset = "pid"; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + ]; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + CapabilityBoundingSet = ""; + # A hostile file must not take the host's memory or every core. + MemoryMax = "3G"; + CPUQuota = "400%"; + TasksMax = 256; + }; + }; + }; + }; + }; +} diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index 039629056..7240c8640 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -63,6 +63,7 @@ let deployCfg.bao.ciMetricsCommonName deployCfg.bao.claudeLoginCommonName deployCfg.bao.notifyRouterCommonName + deployCfg.bao.sttCommonName deployCfg.bao.atticCommonName deployCfg.bao.archiveCommonName deployCfg.bao.forgePublisherCommonName diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index c137dc064..a909df6c9 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -194,6 +194,7 @@ let "swarm-bao-ci-metrics-policy" "swarm-bao-claude-login-policy" "swarm-bao-notify-router-policy" + "swarm-bao-stt-policy" "swarm-bao-attic-policy" "swarm-bao-archive-policy" ]; @@ -2183,6 +2184,27 @@ let && lib.hasInfix "allowed_common_names=swarm-notify-router" s && lib.hasInfix "token_policies=swarm-notify-router" s; } + { + name = "the speech-to-text service's grant is read on its own queue client secret alone"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-stt-policy.script; + grants = grantsIn s; + in + (grantFor grants "secret/data/swarm/services/swarm-stt/oidc/client").caps == [ "read" ] + && lib.length grants == 1 + && lib.hasInfix "auth/cert/certs/swarm-stt" s + && lib.hasInfix "allowed_common_names=swarm-stt" s + && lib.hasInfix "token_policies=swarm-stt" s; + } + { + name = "the PKI unit signs the speech-to-text service's leaf under its own subject"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-pki.script; + in + lib.hasInfix "/stt.pem ]" s && lib.hasInfix "swarm-stt \"\" clientAuth" s; + } { name = "the PKI unit signs the notification router's leaf under its own subject"; ok = diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 1c360894c..1801da103 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -2286,6 +2286,7 @@ let "swarm-notify-router" "swarm-otel" "swarm-review-classifier" + "swarm-stt" "swarm-victorialogs" "swarm-victoriametrics" ] diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix index c0b069c2b..10ef03aea 100644 --- a/nix/module-eval/nats-authelia.nix +++ b/nix/module-eval/nats-authelia.nix @@ -106,6 +106,12 @@ let credential = "notify-router-bao"; port = 8902; } + { + machine = "swarm-stt"; + name = "stt"; + credential = "stt-bao"; + port = 8904; + } ]; forwarderFetchDir = "/run/credentials/swarm-journal-forwarder-oidc.service"; @@ -545,6 +551,7 @@ let "'$$JS.API.CONSUMER.CREATE.ROUTER_TODOS.todo-{agent}.$$SWARM.router.todo.{agent}'" "'$$JS.API.CONSUMER.MSG.NEXT.ROUTER_TODOS.todo-{agent}'" "'$$JS.ACK.ROUTER_TODOS.todo-{agent}.>'" + "'$$SWARM.stt.transcribe.{agent}'" ]; } { diff --git a/nix/module-eval/semi-spread.nix b/nix/module-eval/semi-spread.nix index 3ed274e4e..84921ee84 100644 --- a/nix/module-eval/semi-spread.nix +++ b/nix/module-eval/semi-spread.nix @@ -219,6 +219,7 @@ let d.victorialogs.enable d.review-classifier.enable d.notify-router.enable + d.stt.enable d.claude-login.enable d.ci-metrics.enable (!d.bao.enable) diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 3d269116d..3cf6330f6 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -177,7 +177,7 @@ let # Every service container at once: the services-only host plus the CI # runner. The CI runner, the review classifier, the notification router, the # builder, the claude login minter, the metrics store, the homeserver, - # authelia, Grafana, the collector and the cache are the eleven containers on + # authelia, Grafana, the collector and the cache are the twelve containers on # a netns of their own. serviceContainersWithCi = hive { deploy.hive-controller.enable = false; @@ -204,6 +204,7 @@ let "swarm-notify-router" "swarm-otel" "swarm-review-classifier" + "swarm-stt" "swarm-victorialogs" "swarm-victoriametrics" ]; @@ -429,6 +430,25 @@ let && !(sc.DynamicUser or false) && swarmServicesHere.services.hyperhive.deploy.nats.enable; } + { + name = "hosting the swarm's shared services runs the speech-to-text service as an unprivileged user under a sandbox"; + ok = + let + c = swarmServicesHere.containers.swarm-stt.config; + sc = c.systemd.services.swarm-stt.serviceConfig; + in + swarmServicesHere.services.hyperhive.deploy.stt.enable + && c.users.users.swarm-stt.isSystemUser + && sc.User == "swarm-stt" + && !(sc.DynamicUser or false) + && sc.NoNewPrivileges + && sc.ProtectSystem == "strict" + && sc.PrivateTmp + && sc.CapabilityBoundingSet == "" + && sc.MemoryMax == "3G" + && sc.StateDirectory == "swarm-stt" + && c.systemd.services.swarm-stt.environment.SWARM_STT_MAX_WAITING == "3"; + } { # dhcpcd without IPv4LL, waiting for IPv4: `network-online` is a lease. # The router sets no `nohook` itself and authelia does; `nohook` is the @@ -454,6 +474,12 @@ let && lib.elem "network-online.target" fetch.after && lib.elem "network-online.target" fetch.wants; } + { + name = "the speech-to-text service stays off where the shared services are hosted without the store"; + ok = + !swarmServicesBaoElsewhere.services.hyperhive.deploy.stt.enable + && !(swarmServicesBaoElsewhere.containers ? swarm-stt); + } { name = "the notification router stays off where the shared services are hosted without the store"; ok = @@ -810,15 +836,15 @@ let # that lost a container can't pass by comparing fewer of them. name = "every service container restates its host-side privateNetwork"; ok = - lib.length (lib.attrNames serviceContainers) == 15 + lib.length (lib.attrNames serviceContainers) == 16 && lib.all ( c: c.privateNetwork == (c.config.services.hyperhive.swarmContainer.privateNetwork or null) ) (lib.attrValues serviceContainers); } { # Both values come from ../container-modules/swarm-container.nix. - # Four of the 15 service containers share the host netns (hive-forge, bao, - # the queue, the log store) and eleven have one of their own. + # Four of the 16 service containers share the host netns (hive-forge, bao, + # the queue, the log store) and twelve have one of their own. name = "a service container on the host netns runs no firewall or resolvconf of its own"; ok = let diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 2eb31485b..c5e38428a 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -192,6 +192,7 @@ in swarm-ci-metrics = mkBinPackage "swarm-ci-metrics" "hyperhive swarm CI run poller"; swarm-claude-login = mkBinPackage "swarm-claude-login" "hyperhive claude login minter"; swarm-notify-router = mkBinPackage "swarm-notify-router" "hyperhive swarm notification router"; + swarm-stt = mkBinPackage "swarm-stt" "hyperhive swarm speech-to-text service"; # The only process allowed to write swarm-authelia's users database — # same "runs *inside* a container, not on the host" placement as