hyperhive/nix/host-modules/swarm-stt.nix

# The swarm's speech-to-text service, in a `swarm-stt` container on the host
# that sets `deploy.stt.enable`, which `deploy.allSwarmServices` defaults on
# where the host has the service's store identity. It answers the voice memos
# agents send on `$SWARM.stt.transcribe.<agent>` with whisper.cpp, once per memo
# (the cache is keyed by the audio's hash). Run it where the inference hardware
# is. docs/4-implementation/swarm/stt.md has the design.
#
# The audio comes from rooms with outside people, so the daemon runs as its own
# unprivileged user under a tight systemd sandbox (see `serviceConfig` below),
# with a memory and CPU ceiling, and a state directory as its only writable
# path.
#
# Its own network namespace on the bridge, as ./swarm-claude-login.nix has: it
# reaches the swarm services by name and nothing on the host's loopback. It
# talks only to the queue, the store and authelia, so it can run on any host.
#
# The container holds one credential, its own store leaf, which nspawn hands in
# as a system credential with no copy on disk. The daemon reads its queue
# client secret with it.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  hyperhiveCfg = config.services.hyperhive;

  cfg = hyperhiveCfg.deploy.stt;
  baoCfg = hyperhiveCfg.swarm.bao;
  baoDeploy = hyperhiveCfg.deploy.bao;
  natsCfg = hyperhiveCfg.swarm.nats;
  autheliaCfg = hyperhiveCfg.swarm.authelia;
  gatewayCfg = hyperhiveCfg.gateway;
  networkCfg = hyperhiveCfg.network;
  tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;

  machine = "swarm-stt";
  unit = "swarm-stt";
  clientId = hyperhiveCfg.swarm.stt.clientId;

  # The container's journal forwarder pushes to its own receiver on the swarm
  # collector, the route `serviceProducers.swarm-stt` in
  # ./swarm-otel-service.nix declares, as ./swarm-claude-login.nix does.
  swarmOtelCfg = hyperhiveCfg.swarm.otel;
  forwarderEndpoint = "https://${swarmOtelCfg.domain}/${machine}";
  forwarderUnit = "swarm-journal-forwarder-oidc";
  # Loopback in the container's own netns, and distinct from the other service
  # containers' forwarders all the same.
  forwarderTelemetryPort = 8904;
  # The standard OTLP/HTTP port, on the container's loopback: the forwarder
  # takes the service's metrics there and pushes them to the swarm collector.
  forwarderOtlpPort = 4318;

  rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };

  # The daemon reaches authelia, the store and the queue by name over TLS, and
  # in self-signed mode their leaves chain to the hive CA.
  caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
  caBundleModule = caTrust.trustBundle {
    inherit pkgs;
    name = machine;
    consumers = [
      unit
      # Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
      # collector's and authelia's names.
      "opentelemetry-collector"
    ];
  };

  haveClientIdentity = baoDeploy.sttClientCertFile != null && baoDeploy.sttClientKeyFile != null;

  # Credential id → host source, as ./swarm-claude-login.nix hands its leaf in.
  storeFiles = lib.filterAttrs (_: source: source != null) {
    stt-bao-cert = baoDeploy.sttClientCertFile;
    stt-bao-key = baoDeploy.sttClientKeyFile;
    stt-bao-server-ca = baoDeploy.serverCaFile;
  };
  # nixos-container expands `extraFlags` unquoted, so a source path must be
  # one word.
  oneWord = path: builtins.match ".*[[:space:]].*" path == null;
  credential = id: "/run/credentials/${unit}.service/${id}";
  forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";

  privateNetwork = true;

  settings = cfg.settings;
  modelHashes = {
    base = "sha256-YO1bw90U7qhWST0zQ0m0BXgt3K8AKNS130CINF+6Lv4=";
    small = "sha256-G+OpsgY4Z7k35k4ux0gzZKeZF+FX+pjF2UtcH//qmHs=";
  };
  model =
    if settings.modelFile != null then
      settings.modelFile
    else
      pkgs.fetchurl {
        url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-${settings.modelSize}.bin";
        hash = modelHashes.${settings.modelSize};
      };
in
{
  options.services.hyperhive.swarm.stt.clientId = lib.mkOption {
    type = lib.types.str;
    readOnly = true;
    default = "swarm-stt";
    description = ''
      The OAuth2 client id the speech-to-text service presents to the swarm
      queue. Read-only: the host running authelia registers it, the queue's
      auth-callout responder grants it the service's subjects, and
      authelia's mint stores its secret in the store, all from this one value.
    '';
  };

  options.services.hyperhive.deploy.stt.package = lib.mkOption {
    type = lib.types.package;
    defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-stt";
    description = "The speech-to-text service daemon package.";
  };

  options.services.hyperhive.deploy.stt.settings = {
    modelSize = lib.mkOption {
      type = lib.types.enum [
        "base"
        "small"
      ];
      default = "small";
      description = ''
        The whisper model: `small` (about 470 MB, the default) or `base`
        (about 140 MB, faster and less accurate). Another model comes from
        {option}`services.hyperhive.deploy.stt.settings.modelFile`.
      '';
    };
    modelFile = lib.mkOption {
      type = lib.types.nullOr lib.types.path;
      default = null;
      description = ''
        A ggml model file to use instead of the one
        {option}`services.hyperhive.deploy.stt.settings.modelSize` names, such
        as a bigger model on a host with the hardware for it. Copied into the
        store.
      '';
    };
    cacheTtlSecs = lib.mkOption {
      type = lib.types.ints.positive;
      default = 24 * 60 * 60;
      description = "How long a transcript stays cached, counted from when it was written.";
    };
    maxWaiting = lib.mkOption {
      type = lib.types.ints.positive;
      default = 3;
      description = ''
        Requests held at once, running or waiting for the engine's one slot. Times
        {option}`services.hyperhive.deploy.stt.settings.runTimeoutSecs` it must
        stay under the 15-minute reply deadline, or the daemon refuses to start.
      '';
    };
    runTimeoutSecs = lib.mkOption {
      type = lib.types.ints.positive;
      default = 240;
      description = "The longest one run (decode and transcribe) may take.";
    };
    maxAudioSecs = lib.mkOption {
      type = lib.types.ints.positive;
      default = 300;
      description = "Seconds of audio decoded; ffmpeg drops the rest.";
    };
    maxBytes = lib.mkOption {
      type = lib.types.ints.positive;
      default = 6 * 1024 * 1024;
      description = "The largest audio file taken, in bytes. The queue carries 8 MiB.";
    };
  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = haveClientIdentity;
        message = ''
          services.hyperhive.deploy.stt.enable requires the service's
          swarm-secret-store client identity: set both

            services.hyperhive.deploy.bao.sttClientCertFile
            services.hyperhive.deploy.bao.sttClientKeyFile

          The service reads its queue client secret with it.

          Both default to the pair glue-bao-tls.nix mints when
          services.hyperhive.deploy.bao.enable is true (the host that runs the
          store). Otherwise copy stt.pem and stt-key.pem
          from the store host's /var/lib/swarm-bao-pki/ and name them in those
          two options.
        '';
      }
      {
        assertion = lib.all oneWord (lib.attrValues storeFiles);
        message = ''
          The speech-to-text service's store identity paths must not contain
          whitespace: nspawn's --load-credential receives them through
          nixos-container's unquoted extra flags.
        '';
      }
    ];

    # The container hangs off the bridge, takes its address from the gateway's
    # DHCP and resolves through its dnsmasq.
    services.hyperhive.network.enable = lib.mkDefault true;
    services.hyperhive.gateway.enable = lib.mkDefault true;
    services.hyperhive.gateway.dns.enable = lib.mkDefault true;

    # nspawn fails to start the container when a credential source is
    # missing, so where this host mints the leaf the container waits for it.
    systemd.services."container@${machine}" = lib.mkMerge [
      caTrust.containerOrdering
      (rootfsCheck.warnOnly machine)
      { after = [ "swarm-bao-pki.service" ]; }
    ];

    containers.${machine} = {
      autoStart = true;
      # The transcript cache lives in the container's /var/lib/swarm-stt.
      ephemeral = false;
      # No `--link-journal=host`: the journal stays in the guest, where the
      # container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
      extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
      inherit privateNetwork;
      hostBridge = networkCfg.bridgeName;
      bindMounts = caTrust.bindMount;

      config =
        { lib, ... }:
        {
          imports = [
            ../container-modules/swarm-container.nix
            ../container-modules/swarm-journal-forwarder.nix
            caBundleModule
          ];

          services.hyperhive.swarmContainer = {
            inherit privateNetwork;
            writesOwnResolvConf = false;
          };
          networking.nameservers = [ networkCfg.bridgeIp ];
          networking.interfaces.eth0.useDHCP = true;

          users.users.${unit} = {
            isSystemUser = true;
            group = unit;
          };
          users.groups.${unit} = { };

          # Authenticated as the daemon's own OIDC client, whose secret its
          # store identity already reads: one identity for the whole container.
          services.hyperhive.swarmJournalForwarder = {
            enable = true;
            serviceName = "stt";
            endpoint = forwarderEndpoint;
            inherit clientId;
            tokenUrl = "${autheliaCfg.url}/api/oidc/token";
            telemetryPort = forwarderTelemetryPort;
            otlpMetricsPort = forwarderOtlpPort;
            store = {
              package = baoDeploy.package;
              addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
              secretPath = "secret/swarm/services/${clientId}/oidc/client";
              clientCertFile = forwarderCredential "stt-bao-cert";
              clientKeyFile = forwarderCredential "stt-bao-key";
              caFile = if storeFiles ? stt-bao-server-ca then forwarderCredential "stt-bao-server-ca" else null;
            };
          };
          systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;

          systemd.services.${unit} = {
            description = "transcribe voice memos";
            wantedBy = [ "multi-user.target" ];
            wants = [ "network-online.target" ];
            after = [ "network-online.target" ];
            environment = {
              BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
              BAO_CLIENT_CERT = credential "stt-bao-cert";
              BAO_CLIENT_KEY = credential "stt-bao-key";
              SWARM_STT_STATE_DIR = "/var/lib/${unit}";
              SWARM_STT_FFMPEG = "${pkgs.ffmpeg-headless}/bin/ffmpeg";
              SWARM_STT_WHISPER_CLI = "${pkgs.whisper-cpp}/bin/whisper-cli";
              SWARM_STT_MODEL = "${model}";
              SWARM_STT_CACHE_TTL_SECS = toString settings.cacheTtlSecs;
              SWARM_STT_MAX_WAITING = toString settings.maxWaiting;
              SWARM_STT_RUN_TIMEOUT_SECS = toString settings.runTimeoutSecs;
              SWARM_STT_MAX_AUDIO_SECS = toString settings.maxAudioSecs;
              SWARM_STT_MAX_BYTES = toString settings.maxBytes;
              SWARM_STT_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
              SWARM_STT_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
              SWARM_STT_OIDC_CLIENT_ID = clientId;
              OTEL_EXPORTER_OTLP_ENDPOINT = "http://127.0.0.1:${toString forwarderOtlpPort}";
              OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf";
              OTEL_RESOURCE_ATTRIBUTES = "service.name=hyperhive-stt,swarm=${
                if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name
              }";
            }
            // lib.optionalAttrs (storeFiles ? stt-bao-server-ca) {
              BAO_CACERT = credential "stt-bao-server-ca";
            }
            // lib.optionalAttrs caTrust.useSelfSigned {
              SWARM_STT_OIDC_CA_FILE = caTrust.bundlePathFor machine;
            };
            serviceConfig = {
              ExecStart = "${cfg.package}/bin/swarm-stt";
              User = unit;
              Group = unit;
              UMask = "0077";
              # Bare ids: the system credentials nspawn handed in.
              LoadCredential = lib.attrNames storeFiles;
              Restart = "on-failure";
              RestartSec = "10s";
              SyslogIdentifier = unit;

              # The sandbox for a service that decodes files strangers sent:
              # the only writable path is its state directory, it sees no
              # other user's processes or devices, cannot gain privileges, and
              # may open sockets to the queue, the store and authelia only
              # (the container's netns has no other route).
              StateDirectory = unit;
              StateDirectoryMode = "0700";
              NoNewPrivileges = true;
              ProtectSystem = "strict";
              ProtectHome = true;
              PrivateTmp = true;
              PrivateDevices = true;
              PrivateUsers = true;
              ProtectKernelTunables = true;
              ProtectKernelModules = true;
              ProtectKernelLogs = true;
              ProtectControlGroups = true;
              ProtectClock = true;
              ProtectHostname = true;
              ProtectProc = "invisible";
              ProcSubset = "pid";
              RestrictNamespaces = true;
              RestrictRealtime = true;
              RestrictSUIDSGID = true;
              LockPersonality = true;
              MemoryDenyWriteExecute = true;
              RestrictAddressFamilies = [
                "AF_INET"
                "AF_INET6"
                "AF_UNIX"
              ];
              SystemCallArchitectures = "native";
              SystemCallFilter = [
                "@system-service"
                "~@privileged"
                "~@resources"
              ];
              CapabilityBoundingSet = "";
              # A hostile file must not take the host's memory or every core.
              MemoryMax = "3G";
              CPUQuota = "400%";
              TasksMax = 256;
            };
          };
        };
    };
  };
}