hyperhive/nix/host-modules/swarm-controller.nix

# The swarm-level controller daemon. Per-host opt-in: a swarm has one
# controller, so most hives leave this off and point at the hive that
# runs it. Distinct from hive-c0re, which every hive runs — c0re owns
# the agents on one host, this owns what is true across hives.
#
# Serves HTTP over a unix socket rather than a TCP port: the gateway's
# nginx is the only intended client, it runs on this same host and so
# reaches the socket by path, and a socket that is never bound to an
# address cannot be reached from off-host by mistake.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.swarm.controller;
  autheliaCfg = config.services.hyperhive.swarm.authelia;

  # What `swarmctl` needs in order to act on authelia from the host.
  #
  # Only set when authelia actually runs **here**: the controller can be
  # enabled on a host that is not the swarm's SSO host, and in that case
  # the right behaviour is for `swarmctl user add` to fail saying the
  # value is unset. A guessed path would resolve cleanly and write a file
  # nothing reads, which is the failure mode that costs an afternoon.
  autheliaEnv = lib.optionalAttrs autheliaCfg.enable {
    # The CONFIGURED authelia, not whatever is on PATH: the argon2
    # parameters baked into a hash have to match the verifier's.
    SWARMCTL_AUTHELIA_BIN = "${autheliaCfg.package}/bin/authelia";
    SWARMCTL_AUTHELIA_USERS_FILE = autheliaCfg.hostUsersFile;
    SWARMCTL_AUTHELIA_MACHINE = autheliaCfg.machine;
    SWARMCTL_AUTHELIA_UNIT = autheliaCfg.unit;
  };

  natsCfg = config.services.hyperhive.swarm.nats;
  forgeCfg = config.services.hyperhive.swarm.forge;

  # The controller's own OAuth2 client. It is NOT a hive: the per-hive
  # clients the roster issues belong to hives, and the responder's client
  # belongs to the responder. One identity per principal — the rule is that
  # a principal's credentials all derive from the same identity, not that
  # the swarm has one.
  queueClientId = "swarm-controller";

  # Both halves have to be here: authelia to have minted the secret, and the
  # queue to connect to. Same guard, and the same reasoning, as `autheliaEnv`
  # above — a value set on a host that runs neither would point at a file
  # that does not exist and produce a daemon that retries forever.
  queueLocal = autheliaCfg.enable && natsCfg.enable;

  # `LoadCredential` and not a copy-oneshot, which is where this deliberately
  # differs from the callout responder: that one delivers INTO a container,
  # so it has to copy across a filesystem boundary. The controller is a plain
  # host unit, so systemd can hand it the file directly — fewer moving parts,
  # and the secret never gains a second on-disk copy to forget about.
  queueEnv = lib.optionalAttrs queueLocal {
    # The queue container shares the host netns, so loopback is correct here
    # and is not the `localhost`-means-the-wrong-thing trap that applies
    # inside agent containers.
    SWARM_CONTROLLER_NATS_URL = "nats://127.0.0.1:${toString natsCfg.port}";
    SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
    SWARM_CONTROLLER_OIDC_CLIENT_ID = queueClientId;
    # `%d` is systemd's credentials directory: root reads the plaintext at
    # unit start and the daemon's own user sees it 0400, without the unit
    # ever being able to read the rest of authelia's state dir.
    SWARM_CONTROLLER_OIDC_CLIENT_SECRET_FILE = "%d/queue-client.secret";
  };

  # Independent of `queueEnv`/`queueLocal` on purpose — the queue
  # coordinates' own co-location guard is narrower than it looks (it
  # silently drops the controller's identity on a host that splits
  # authelia/NATS from swarm-controller; a known gap tracked separately).
  # Forge access has nothing to do with whether authelia+NATS happen to be
  # local, only with whether `cfg.forgeTokenFile` resolves to a real file —
  # which `forgeTokenFile`'s own default already handles (forge-local vs.
  # operator-copied). Gating a second time here would just repeat that
  # option's own logic under a different name.
  forgeEnv = lib.optionalAttrs (cfg.forgeTokenFile != null) {
    SWARM_CONTROLLER_FORGE_URL = "https://${forgeCfg.domain}";
    # Same `%d` shape as the queue secret above — root reads the plaintext
    # at unit start, the daemon's own user sees a 0400 copy.
    SWARM_CONTROLLER_FORGE_TOKEN_FILE = "%d/forge-token";
  };

  # Wrapped rather than documented: every one of these values is derived
  # from an option this deployment already set, so making the operator
  # re-supply them on the command line would be asking them to repeat the
  # config back to it — and to get it wrong the day one of them changes.
  swarmctlConfigured = pkgs.symlinkJoin {
    name = "swarmctl-configured";
    paths = [ cfg.swarmctlPackage ];
    nativeBuildInputs = [ pkgs.makeWrapper ];
    postBuild = ''
      wrapProgram $out/bin/swarmctl ${
        lib.concatStringsSep " " (
          lib.mapAttrsToList (name: value: "--set ${name} ${lib.escapeShellArg value}") autheliaEnv
        )
      }
    '';
  };
in
{
  options.services.hyperhive.swarm.controller = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Run the swarm-controller daemon on this host. Off by default and
        deliberately not derived from `services.hyperhive.enable`: a swarm
        has one controller, so enabling it per hive is a decision about
        swarm topology, not about whether hyperhive is installed.

        `services.hyperhive.enableAllLocalDefaults` does assert it, and
        that is not an exception to the rule above — it is the rule
        applied. That mode says "this box is the whole deployment", which
        answers the topology question outright, where
        `services.hyperhive.enable` alone never can.
      '';
    };

    package = lib.mkOption {
      type = lib.types.package;
      defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller";
      description = ''
        swarm-controller package. Wired by default from this flake's own
        package set (see `flake.nix`); override to run a different build.
      '';
    };

    swarmctlPackage = lib.mkOption {
      type = lib.types.package;
      defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarmctl";
      description = ''
        `swarmctl` package — the swarm operator's CLI, installed on this
        host alongside the daemon and wrapped with the paths it needs.

        A separate option from `package` rather than a second binary in
        the same derivation: the CLI runs as root and acts directly,
        the daemon runs unprivileged and serves a socket, and pinning one
        without the other is a thing an operator may legitimately want.
      '';
    };

    socketPath = lib.mkOption {
      type = lib.types.str;
      default = "/run/swarm-controller/controller.sock";
      description = ''
        Unix socket the daemon serves on, and the path the gateway's nginx
        proxies to.

        The **directory** is the access control here, not the socket mode:
        the socket itself is `0666` (nginx runs as another user, and
        `connect(2)` needs write), exactly as hive-c0re publishes the
        per-agent sockets. What keeps that safe is that the directory holds
        exactly one socket and is traverse-only (`0751`) for everyone else.
        Moving this path under a directory that carries anything else —
        `/run/hyperhive`, which holds the host admin socket, above all —
        exposes whatever else lives there to the same reachability.

        Changing this therefore means re-checking what else lives in the
        new directory, not just the daemon.
      '';
    };

    staleAfterSeconds = lib.mkOption {
      type = lib.types.ints.positive;
      default = 120;
      description = ''
        How old a hive's last status snapshot may be before
        `GET /api/hives/status` reports it as `stale` rather than `fresh`.

        This is a statement about how often hives *publish*, not about how
        patient a reader is — set it above the publishing cadence or every
        hive reads stale between offers. It is an option and not a
        constant precisely because that cadence is a property of the
        deployment.

        Freshness is derived when the endpoint is read, never stored, so
        changing this takes effect for the next request; no hive has to
        re-publish anything.
      '';
    };

    links = lib.mkOption {
      type = lib.types.listOf (
        lib.types.submodule {
          options = {
            label = lib.mkOption {
              type = lib.types.str;
              description = "Display label for the link.";
            };
            icon = lib.mkOption {
              type = lib.types.str;
              default = "";
              description = "Optional icon emoji or short glyph.";
            };
            url = lib.mkOption {
              type = lib.types.str;
              description = "Full URL.";
            };
          };
        }
      );
      default = [ ];
      example = lib.literalExpression ''
        [ { label = "Wiki"; icon = "📖"; url = "https://wiki.example.com/"; } ]
      '';
      description = ''
        Quick links to swarm-wide services, surfaced by the swarm UI's
        links menu (`GET /api/links`). Same shape and same
        zero-code-change-to-extend idea as `hyperhive.dashboardLinks`
        (`nix/agent-modules/dashboard-links.nix`), one level up: rather
        than one central hardcoded list, each service's own module
        contributes its own entry when it is actually enabled on this
        host — `swarm-authelia.nix`, `hive-matrix.nix` and
        `hive-forge/default.nix` all do — the same list-merge idiom
        `services.hyperhive.gateway.localNames` already uses. A future
        service module can push its own entry the same way, and an
        operator can add arbitrary extra entries here directly; neither
        needs a swarm-controller or swarm-ui change.

        Only meaningful on the host that actually runs the controller —
        entries contributed on any other host are computed but never
        read. In a swarm that splits `swarm-authelia`/`hive-matrix`/
        `hive-forge` across hosts other than the controller's, this list
        only reflects what is enabled locally; see each contributing
        module's own activation condition.
      '';
    };

    forgeTokenFile = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      # `services.hyperhive.swarm.forge` has no `enable` of its own to
      # check — the module activates on the general
      # `config.services.hyperhive.enable` instead (see
      # `hive-forge/default.nix`'s own `config = lib.mkIf
      # config.services.hyperhive.enable { ... }`), so that's the
      # condition to match here too. Referencing a `forge.enable` that
      # doesn't exist threw `attribute 'enable' missing` on every host
      # that turns swarm-controller on — caught in review, not by
      # `nix flake check` (nothing in its checked combinations forced
      # this particular default to actually evaluate).
      default = if config.services.hyperhive.enable then forgeCfg.hostSwarmControllerTokenFile else null;
      defaultText = lib.literalExpression ''
        forge's own `hostSwarmControllerTokenFile` when this host runs
        hyperhive at all (forge has no separate enable), else null
      '';
      example = "/var/lib/secrets/swarm-controller-forge.token";
      description = ''
        Path to this host's copy of the swarm-controller's forge access
        token (see `hive-forge/default.nix`'s
        `forgejo-swarm-controller-account` + `hive-forge-swarm-controller-token`
        units, which mint and collect it onto forge's own host).

        Defaults to forge's own delivery path on every host running
        hyperhive (forge deploys unconditionally alongside it — see
        `hive-forge/default.nix`, it has no `enable` of its own).
        Override explicitly if forge's actual token file ends up
        somewhere else — copy it out of forge's
        {option}`services.hyperhive.swarm.forge.hostSwarmControllerTokenFile`
        with whatever secret management this deployment already uses,
        the same shape `swarm.nix`'s `clientSecretFile` documents for
        its own cross-host case. `null` means no forge access — the
        daemon logs that and continues without it, the same
        graceful-absence shape the queue coordinates already use.
      '';
    };
  };

  config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
    users.users.swarm-controller = {
      isSystemUser = true;
      group = "swarm-controller";
      description = "hyperhive swarm-controller daemon";
    };
    users.groups.swarm-controller = { };

    # Installed host-wide, not into the daemon's unit: `swarmctl` is run
    # by a human on this box and acts as root, so the daemon's sandbox is
    # exactly what it must not inherit.
    environment.systemPackages = [ swarmctlConfigured ];

    # One declaration, two readers — the controller knows which client id it
    # authenticates under, so making the operator restate it in authelia's
    # client list would be a second source of truth for a string whose
    # mismatch is an opaque 401 from the token endpoint. Same shape as the
    # queue's own client declaration.
    services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf autheliaCfg.enable [
      {
        id = queueClientId;
        description = "HyperHive swarm controller";
        # `client_credentials`: a daemon authenticating as itself, with
        # nobody to redirect. Declared rather than inferred from an empty
        # redirect list, because authelia permits only the grants a client
        # names and an omitted `grant_types` means authorization-code alone.
        kind = "machine";
      }
    ];

    systemd.services.swarm-controller = {
      description = "hyperhive swarm-level controller daemon";
      wantedBy = [ "multi-user.target" ];
      after = [ "network.target" ];

      serviceConfig = {
        ExecStart = "${cfg.package}/bin/swarm-controller";

        # Only the credentials that actually resolve on this host. An
        # absent credential is not a failure for either: the daemon logs
        # that no queue / no forge is configured and serves its HTTP
        # surface regardless, which is the correct behaviour on a host
        # that doesn't run one.
        LoadCredential =
          lib.optional queueLocal "queue-client.secret:${autheliaCfg.hostClientSecretDir}/${queueClientId}.secret"
          ++ lib.optional (cfg.forgeTokenFile != null) "forge-token:${cfg.forgeTokenFile}";
        User = "swarm-controller";
        Group = "swarm-controller";
        Restart = "on-failure";
        RestartSec = "5s";

        # `/run/swarm-controller` — its own directory, holding only the
        # socket. See `socketPath`'s description for why that is a security
        # property and not tidiness.
        RuntimeDirectory = "swarm-controller";
        # 0751: traverse-only for others, so the gateway's nginx can reach
        # the socket path without being able to list the directory. Same
        # shape (and same reason) as hive-c0re's runtime dir.
        RuntimeDirectoryMode = "0751";
        # Preserved across restarts so the path never vanishes from under a
        # running nginx. The daemon unlinks a stale socket on start, which
        # is what makes preservation safe.
        RuntimeDirectoryPreserve = "yes";

        StateDirectory = "swarm-controller";
        StateDirectoryMode = "0750";

        # Nothing here needs a writable filesystem, real privileges, or a
        # view of the rest of the machine; the daemon reads its socket path
        # from config and serves.
        PrivateTmp = true;
        ProtectSystem = "strict";
        ProtectHome = true;
        NoNewPrivileges = true;
        PrivateDevices = true;
        ProtectKernelTunables = true;
        ProtectKernelModules = true;
        ProtectControlGroups = true;
        RestrictAddressFamilies = [
          "AF_UNIX"
        ];
      };

      # Queue coordinates (`queueEnv`) and forge coordinates (`forgeEnv`)
      # merge in last, each present only when its own resolution actually
      # succeeded on this host. Both refuse a PARTIAL set rather than
      # treating it as absent, which is why each is built as one attrset
      # and never assigned individually.
      environment = {
        SWARM_CONTROLLER_SOCKET = cfg.socketPath;
        # The swarm's hive directory, JSON-encoded — the full directory
        # (this daemon has no "self" hive to exclude, unlike
        # `swarm.peerHives`, `swarm.hives` minus this hive) rather than
        # peers-minus-self. Consumed by `GET /api/hives`
        # (swarm-controller/src/main.rs::load_hives).
        SWARM_CONTROLLER_HIVES = builtins.toJSON (
          lib.mapAttrsToList (name: h: {
            inherit name;
            inherit (h) domain;
          }) config.services.hyperhive.swarm.hives
        );
        # The merged links list — see `links`' description above for who
        # contributes to it. Consumed by `GET /api/links`
        # (swarm-controller/src/main.rs::load_links).
        SWARM_CONTROLLER_LINKS = builtins.toJSON cfg.links;
        # Staleness threshold for `GET /api/hives/status` — see
        # `staleAfterSeconds`' description. Set unconditionally rather
        # than inside `queueEnv`: it is not a queue coordinate, and
        # nothing about it is unsafe to define on a host whose queue is
        # off (the daemon just has nothing to apply it to).
        SWARM_CONTROLLER_STALE_AFTER_SECS = toString cfg.staleAfterSeconds;
      }
      // queueEnv
      // forgeEnv;
    };
  };
}