hyperhive/nix/host-modules/swarm-controller.nix

# The swarm-level controller daemon. Per-host opt-in: a swarm has one
# controller, so most hives leave this off and point at the hive that
# runs it. Distinct from hive-c0re, which every hive runs — c0re owns
# the agents on one host, this owns what is true across hives.
#
# Serves HTTP over a unix socket rather than a TCP port: the gateway's
# nginx is the only intended client, it runs on this same host and so
# reaches the socket by path, and a socket that is never bound to an
# address cannot be reached from off-host by mistake.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.swarm.controller;
  autheliaCfg = config.services.hyperhive.swarm.authelia;

  # What `swarmctl` needs in order to act on authelia from the host.
  #
  # Only set when authelia actually runs **here**: the controller can be
  # enabled on a host that is not the swarm's SSO host, and in that case
  # the right behaviour is for `swarmctl user add` to fail saying the
  # value is unset. A guessed path would resolve cleanly and write a file
  # nothing reads, which is the failure mode that costs an afternoon.
  autheliaEnv = lib.optionalAttrs autheliaCfg.enable {
    # The CONFIGURED authelia, not whatever is on PATH: the argon2
    # parameters baked into a hash have to match the verifier's.
    SWARMCTL_AUTHELIA_BIN = "${autheliaCfg.package}/bin/authelia";
    SWARMCTL_AUTHELIA_USERS_FILE = autheliaCfg.hostUsersFile;
    SWARMCTL_AUTHELIA_MACHINE = autheliaCfg.machine;
    SWARMCTL_AUTHELIA_UNIT = autheliaCfg.unit;
  };

  natsCfg = config.services.hyperhive.swarm.nats;
  forgeCfg = config.services.hyperhive.swarm.forge;

  # The controller's own OAuth2 client. It is NOT a hive: the per-hive
  # clients the roster issues belong to hives, and the responder's client
  # belongs to the responder. One identity per principal — the rule is that
  # a principal's credentials all derive from the same identity, not that
  # the swarm has one.
  queueClientId = "swarm-controller";

  # Both halves have to be here: authelia to have minted the secret, and the
  # queue to connect to. Same guard, and the same reasoning, as `autheliaEnv`
  # above — a value set on a host that runs neither would point at a file
  # that does not exist and produce a daemon that retries forever.
  queueLocal = autheliaCfg.enable && natsCfg.enable;

  # `LoadCredential` and not a copy-oneshot, which is where this deliberately
  # differs from the callout responder: that one delivers INTO a container,
  # so it has to copy across a filesystem boundary. The controller is a plain
  # host unit, so systemd can hand it the file directly — fewer moving parts,
  # and the secret never gains a second on-disk copy to forget about.
  queueEnv = lib.optionalAttrs queueLocal {
    # The queue container shares the host netns, so loopback is correct here
    # and is not the `localhost`-means-the-wrong-thing trap that applies
    # inside agent containers.
    SWARM_CONTROLLER_NATS_URL = "nats://127.0.0.1:${toString natsCfg.port}";
    SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
    SWARM_CONTROLLER_OIDC_CLIENT_ID = queueClientId;
    # `%d` is systemd's credentials directory: root reads the plaintext at
    # unit start and the daemon's own user sees it 0400, without the unit
    # ever being able to read the rest of authelia's state dir.
    SWARM_CONTROLLER_OIDC_CLIENT_SECRET_FILE = "%d/queue-client.secret";
  };

  # Independent of `queueEnv`/`queueLocal` on purpose — the queue
  # coordinates' own co-location guard is narrower than it looks (it
  # silently drops the controller's identity on a host that splits
  # authelia/NATS from swarm-controller; a known gap tracked separately).
  # Forge access has nothing to do with whether authelia+NATS happen to be
  # local, only with whether `cfg.forgeTokenFile` resolves to a real file —
  # which `forgeTokenFile`'s own default already handles (forge-local vs.
  # operator-copied). Gating a second time here would just repeat that
  # option's own logic under a different name.
  forgeEnv = lib.optionalAttrs (cfg.forgeTokenFile != null) {
    SWARM_CONTROLLER_FORGE_URL = "https://${forgeCfg.domain}";
    # Same `%d` shape as the queue secret above — root reads the plaintext
    # at unit start, the daemon's own user sees a 0400 copy.
    SWARM_CONTROLLER_FORGE_TOKEN_FILE = "%d/forge-token";
  };

  # Wrapped rather than documented: every one of these values is derived
  # from an option this deployment already set, so making the operator
  # re-supply them on the command line would be asking them to repeat the
  # config back to it — and to get it wrong the day one of them changes.
  swarmctlConfigured = pkgs.symlinkJoin {
    name = "swarmctl-configured";
    paths = [ cfg.swarmctlPackage ];
    nativeBuildInputs = [ pkgs.makeWrapper ];
    postBuild = ''
      wrapProgram $out/bin/swarmctl ${
        lib.concatStringsSep " " (
          lib.mapAttrsToList (name: value: "--set ${name} ${lib.escapeShellArg value}") autheliaEnv
        )
      }
    '';
  };
in
{
  options.services.hyperhive.swarm.controller = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Run the swarm-controller daemon on this host. Off by default and
        deliberately not derived from `services.hyperhive.enable`: a swarm
        has one controller, so enabling it per hive is a decision about
        swarm topology, not about whether hyperhive is installed.

        `services.hyperhive.enableAllLocalDefaults` does assert it, and
        that is not an exception to the rule above — it is the rule
        applied. That mode says "this box is the whole deployment", which
        answers the topology question outright, where
        `services.hyperhive.enable` alone never can.
      '';
    };

    package = lib.mkOption {
      type = lib.types.package;
      defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller";
      description = ''
        swarm-controller package. Wired by default from this flake's own
        package set (see `flake.nix`); override to run a different build.
      '';
    };

    swarmctlPackage = lib.mkOption {
      type = lib.types.package;
      defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarmctl";
      description = ''
        `swarmctl` package — the swarm operator's CLI, installed on this
        host alongside the daemon and wrapped with the paths it needs.

        A separate option from `package` rather than a second binary in
        the same derivation: the CLI runs as root and acts directly,
        the daemon runs unprivileged and serves a socket, and pinning one
        without the other is a thing an operator may legitimately want.
      '';
    };

    socketPath = lib.mkOption {
      type = lib.types.str;
      default = "/run/swarm-controller/controller.sock";
      description = ''
        Unix socket the daemon serves on, and the path the gateway's nginx
        proxies to.

        The **directory** is the access control here, not the socket mode:
        the socket itself is `0666` (nginx runs as another user, and
        `connect(2)` needs write), exactly as hive-c0re publishes the
        per-agent sockets. What keeps that safe is that the directory holds
        exactly one socket and is traverse-only (`0751`) for everyone else.
        Moving this path under a directory that carries anything else —
        `/run/hyperhive`, which holds the host admin socket, above all —
        exposes whatever else lives there to the same reachability.

        Changing this therefore means re-checking what else lives in the
        new directory, not just the daemon.
      '';
    };

    staleAfterSeconds = lib.mkOption {
      type = lib.types.ints.positive;
      default = 120;
      description = ''
        How old a hive's last status snapshot may be before
        `GET /api/hives/status` reports it as `stale` rather than `fresh`.

        This is a statement about how often hives *publish*, not about how
        patient a reader is — set it above the publishing cadence or every
        hive reads stale between offers. It is an option and not a
        constant precisely because that cadence is a property of the
        deployment.

        Freshness is derived when the endpoint is read, never stored, so
        changing this takes effect for the next request; no hive has to
        re-publish anything.
      '';
    };

    links = lib.mkOption {
      type = lib.types.listOf (
        lib.types.submodule {
          options = {
            label = lib.mkOption {
              type = lib.types.str;
              description = "Display label for the link.";
            };
            icon = lib.mkOption {
              type = lib.types.str;
              default = "";
              description = "Optional icon emoji or short glyph.";
            };
            url = lib.mkOption {
              type = lib.types.str;
              description = "Full URL.";
            };
          };
        }
      );
      default = [ ];
      example = lib.literalExpression ''
        [ { label = "Wiki"; icon = "📖"; url = "https://wiki.example.com/"; } ]
      '';
      description = ''
        Quick links to swarm-wide services, surfaced by the swarm UI's
        links menu (`GET /api/links`). Same shape and same
        zero-code-change-to-extend idea as `hyperhive.dashboardLinks`
        (`nix/agent-modules/dashboard-links.nix`), one level up: rather
        than one central hardcoded list, each service's own module
        contributes its own entry when it is actually enabled on this
        host — `swarm-authelia.nix`, `hive-matrix.nix` and
        `hive-forge/default.nix` all do — the same list-merge idiom
        `services.hyperhive.gateway.localNames` already uses. A future
        service module can push its own entry the same way, and an
        operator can add arbitrary extra entries here directly; neither
        needs a swarm-controller or swarm-ui change.

        Only meaningful on the host that actually runs the controller —
        entries contributed on any other host are computed but never
        read. In a swarm that splits `swarm-authelia`/`hive-matrix`/
        `hive-forge` across hosts other than the controller's, this list
        only reflects what is enabled locally; see each contributing
        module's own activation condition.
      '';
    };

    forgeTokenFile = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = if forgeCfg.enable then forgeCfg.hostSwarmControllerTokenFile else null;
      defaultText = lib.literalExpression ''
        forge's own `hostSwarmControllerTokenFile` when this host runs
        forge, else null
      '';
      example = "/var/lib/secrets/swarm-controller-forge.token";
      description = ''
        Path to this host's copy of the swarm-controller's forge access
        token (see `hive-forge/default.nix`'s
        `forgejo-swarm-controller-account` + `hive-forge-swarm-controller-token`
        units, which mint and collect it onto forge's own host).

        Defaults to forge's own delivery path only when forge runs on
        this same host. On any other host the token has to get here
        somehow — copy it out of forge's
        {option}`services.hyperhive.swarm.forge.hostSwarmControllerTokenFile`
        with whatever secret management this deployment already uses,
        the same shape `swarm.nix`'s `clientSecretFile` documents for
        the analogous cross-host case. `null` (the default when forge
        isn't local and nothing else was set) means no forge access —
        the daemon logs that and continues without it, the same
        graceful-absence shape the queue coordinates already use.
      '';
    };
  };

  config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
    users.users.swarm-controller = {
      isSystemUser = true;
      group = "swarm-controller";
      description = "hyperhive swarm-controller daemon";
    };
    users.groups.swarm-controller = { };

    # Installed host-wide, not into the daemon's unit: `swarmctl` is run
    # by a human on this box and acts as root, so the daemon's sandbox is
    # exactly what it must not inherit.
    environment.systemPackages = [ swarmctlConfigured ];

    # One declaration, two readers — the controller knows which client id it
    # authenticates under, so making the operator restate it in authelia's
    # client list would be a second source of truth for a string whose
    # mismatch is an opaque 401 from the token endpoint. Same shape as the
    # queue's own client declaration.
    services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf autheliaCfg.enable [
      {
        id = queueClientId;
        description = "HyperHive swarm controller";
        # `client_credentials`: a daemon authenticating as itself, with
        # nobody to redirect. Declared rather than inferred from an empty
        # redirect list, because authelia permits only the grants a client
        # names and an omitted `grant_types` means authorization-code alone.
        kind = "machine";
      }
    ];

    systemd.services.swarm-controller = {
      description = "hyperhive swarm-level controller daemon";
      wantedBy = [ "multi-user.target" ];
      after = [ "network.target" ];

      serviceConfig = {
        ExecStart = "${cfg.package}/bin/swarm-controller";

        # Only the credentials that actually resolve on this host. An
        # absent credential is not a failure for either: the daemon logs
        # that no queue / no forge is configured and serves its HTTP
        # surface regardless, which is the correct behaviour on a host
        # that doesn't run one.
        LoadCredential =
          lib.optional queueLocal "queue-client.secret:${autheliaCfg.hostClientSecretDir}/${queueClientId}.secret"
          ++ lib.optional (cfg.forgeTokenFile != null) "forge-token:${cfg.forgeTokenFile}";
        User = "swarm-controller";
        Group = "swarm-controller";
        Restart = "on-failure";
        RestartSec = "5s";

        # `/run/swarm-controller` — its own directory, holding only the
        # socket. See `socketPath`'s description for why that is a security
        # property and not tidiness.
        RuntimeDirectory = "swarm-controller";
        # 0751: traverse-only for others, so the gateway's nginx can reach
        # the socket path without being able to list the directory. Same
        # shape (and same reason) as hive-c0re's runtime dir.
        RuntimeDirectoryMode = "0751";
        # Preserved across restarts so the path never vanishes from under a
        # running nginx. The daemon unlinks a stale socket on start, which
        # is what makes preservation safe.
        RuntimeDirectoryPreserve = "yes";

        StateDirectory = "swarm-controller";
        StateDirectoryMode = "0750";

        # Nothing here needs a writable filesystem, real privileges, or a
        # view of the rest of the machine; the daemon reads its socket path
        # from config and serves.
        PrivateTmp = true;
        ProtectSystem = "strict";
        ProtectHome = true;
        NoNewPrivileges = true;
        PrivateDevices = true;
        ProtectKernelTunables = true;
        ProtectKernelModules = true;
        ProtectControlGroups = true;
        RestrictAddressFamilies = [
          "AF_UNIX"
        ];
      };

      # Queue coordinates (`queueEnv`) and forge coordinates (`forgeEnv`)
      # merge in last, each present only when its own resolution actually
      # succeeded on this host. Both refuse a PARTIAL set rather than
      # treating it as absent, which is why each is built as one attrset
      # and never assigned individually.
      environment = {
        SWARM_CONTROLLER_SOCKET = cfg.socketPath;
        # The swarm's hive directory, JSON-encoded — the full directory
        # (this daemon has no "self" hive to exclude, unlike
        # `swarm.peerHives`, `swarm.hives` minus this hive) rather than
        # peers-minus-self. Consumed by `GET /api/hives`
        # (swarm-controller/src/main.rs::load_hives).
        SWARM_CONTROLLER_HIVES = builtins.toJSON (
          lib.mapAttrsToList (name: h: {
            inherit name;
            inherit (h) domain;
          }) config.services.hyperhive.swarm.hives
        );
        # The merged links list — see `links`' description above for who
        # contributes to it. Consumed by `GET /api/links`
        # (swarm-controller/src/main.rs::load_links).
        SWARM_CONTROLLER_LINKS = builtins.toJSON cfg.links;
        # Staleness threshold for `GET /api/hives/status` — see
        # `staleAfterSeconds`' description. Set unconditionally rather
        # than inside `queueEnv`: it is not a queue coordinate, and
        # nothing about it is unsafe to define on a host whose queue is
        # off (the daemon just has nothing to apply it to).
        SWARM_CONTROLLER_STALE_AFTER_SECS = toString cfg.staleAfterSeconds;
      }
      // queueEnv
      // forgeEnv;
    };
  };
}