hyperhive/nix/host-modules/hive-priv.nix

# hive-priv — the narrow root privileged helper hive-c0re delegates
# to, socket-activated at /run/hive/priv.sock. See docs/trust-boundary/boundary.md
# for the operator/agent trust-boundary design.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.c0re;

  # Same safe.directory gitconfig as the c0re unit (see ./hive-c0re)
  # — hive-priv (root) runs nix, which fetches the hive-core-owned
  # meta/applied repos; libgit2 refuses cross-user reads without it.
  safeDirGitconfig = pkgs.writeText "hyperhive-safe-gitconfig" ''
    [safe]
    directory = *
  '';
in
{
  config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable {
    # The parent of every agent socket dir. hive-priv is its only writer
    # (`EnsureAgentSocketDir`), and its unit lists it in `ReadWritePaths`,
    # which fails the unit when the path is missing.
    systemd.tmpfiles.rules = [ "d /run/hive-agent 0755 root root - -" ];

    # Socket unit for hive-priv — the narrow root helper that executes
    # privileged operations on behalf of hive-c0re. Systemd creates and
    # holds `/run/hive/priv.sock` before the first connection arrives.
    #
    # Mode 0660 hive-core:hive-core: only the hive-c0re service user can
    # connect. hive-priv (server) runs as root and validates every request
    # against a strict allowlist before executing any privileged op.
    systemd.sockets.hive-priv = {
      description = "hive-priv privileged helper socket";
      wantedBy = [ "sockets.target" ];
      socketConfig = {
        ListenStream = "/run/hive/priv.sock";
        SocketMode = "0660";
        SocketGroup = "hive-core";
        # Create /run/hive/ if absent; 0755 so the hive-core user can
        # traverse into it to reach the socket.
        DirectoryMode = "0755";
      };
    };

    # Service unit for hive-priv. Runs as root — it genuinely needs root to
    # invoke `nixos-container`, write `/etc/nixos-containers/`, write
    # systemd drop-ins in `/run/systemd/system/`, and call `chown(2)`.
    # Every request is validated against a strict container-name allowlist
    # inside the binary; the attack surface is narrow by design.
    #
    # Socket-activated: systemd starts hive-priv on the first connection
    # (no earlier). LISTEN_FDS + LISTEN_PID are set by systemd; hive-priv
    # reads them to accept the pre-bound socket fd instead of binding its
    # own.
    systemd.services.hive-priv = {
      description = "hive-priv privileged helper";
      # No wantedBy — socket-activated exclusively. The socket unit is the
      # entry point; systemd starts this service on first connect.
      after = [ "hive-priv.socket" ];
      requires = [ "hive-priv.socket" ];
      # `nixos-container` is a perl script that shells out by bare name to
      # nix / nix-env / nix-instantiate (create + update), machinectl +
      # systemctl (start/stop), and find / rm / umount / chattr (destroy);
      # only nsenter + su are hardcoded. Give the helper exactly those —
      # not the whole system profile — on top of the systemd/coreutils/
      # findutils already in the default unit PATH. Without `nixos-container`
      # on PATH every container op fails ENOENT, which `build_all` silently
      # swallows into an empty list ("no managed containers").
      #
      # `nix` itself shells out by bare name too: `git` whenever it has to
      # fetch/re-resolve a git-source flake input (an agent.nix with a
      # `git+https://…` input, or a stale flake.lock whose node URL no longer
      # matches the flake's declared input → nix re-resolves at eval), and
      # `ssh` to dispatch to remote builders (`nix.buildMachines` /
      # `ssh-ng://`). Without these on PATH `nixos-container update` dies with
      # `executing "git": No such file or directory` / `Could not find
      # executable 'ssh'` — the agent build fails before it starts.
      path = [
        pkgs.nixos-container
        pkgs.nix # nix, nix-env, nix-instantiate — create + update
        pkgs.gitMinimal # git — nix fetches/re-resolves git-source flake inputs
        pkgs.openssh # ssh — nix dispatches builds to remote builders
        pkgs.util-linux # umount (nsenter is hardcoded in the script)
        pkgs.e2fsprogs # chattr
        pkgs.btrfs-progs # btrfs subvolume create/delete — Ensure/DeleteAgentSubvolume
      ];
      environment = {
        # `nixos-container update/create` runs `nix`, which writes its
        # fetcher/eval cache under $HOME/.cache. With ProtectHome and no
        # explicit HOME this lands on the unwritable /var/empty and Lix
        # errors out. Point HOME at the StateDirectory below (persistent,
        # so the cache survives across rebuilds).
        HOME = "/var/lib/hive-priv";
        # hive-priv runs as root. Root nix defaults to store=auto which
        # resolves to the LOCAL store — bypassing the host daemon, its
        # remote builders, and prebuilt derivation outputs. Force daemon
        # routing so nixos-container update and the nix prebuild see the
        # same store and substituters as every other build context.
        NIX_REMOTE = "daemon";
      };
      serviceConfig = {
        ExecStart = "${cfg.package}/bin/hive-priv";
        SyslogIdentifier = "hive-priv";
        Type = "simple";
        User = "root";
        PrivateTmp = true;
        ProtectHome = true;
        # Harden the file system view: strict makes the entire hierarchy
        # read-only by default; ReadWritePaths carves out exactly the paths
        # hive-priv must write to at runtime. Each is a confirmed hive-priv
        # write that EROFSes (os error 30) without its carve-out:
        #   /etc/nixos-containers     — <container>.conf (bind mounts, nspawn flags)
        #   /etc/tmpfiles.d           — unlinks the legacy hyperhive-agents.conf;
        #                               drop once every host has run it
        #   /run/hive-agent           — creates per-agent socket dirs
        #   /run/systemd              — container@ drop-ins + machined state
        #   /run/lock                 — nixos-container's create/destroy lock file
        #   /run/hive-ci              — register_ci_runner's runner-token write
        #                               (else token stays placeholder → runner
        #                               crash-loops → no CI hive-wide)
        #   /var/lib/nixos-containers — container rootfs
        #   /var/lib/hyperhive        — agent state (forge/matrix token files)
        #   /nix                      — nix store + profile updates on create/update
        ProtectSystem = "strict";
        ReadWritePaths = [
          "/etc/nixos-containers"
          "/etc/tmpfiles.d"
          "/run/hive-agent"
          "/run/systemd"
          "/run/lock"
          "/run/hive-ci"
          "/var/lib/nixos-containers"
          "/var/lib/hyperhive"
          "/nix"
        ];
        # Writable HOME for nix's caches (see environment.HOME above).
        StateDirectory = "hive-priv";
        # With ProtectSystem=strict the root filesystem is read-only inside
        # hive-priv. When `nixos-container create/update` invokes nix, nix
        # creates a temporary result symlink in its working directory. Without
        # an explicit WorkingDirectory the cwd is / (inherited from systemd),
        # which is read-only under strict, causing:
        #   error: creating symlink "/.tmp.tmp-..." -> ...: Read-only file system
        # Point the working directory at the writable StateDirectory so nix
        # drops its temp symlink there instead.
        WorkingDirectory = "/var/lib/hive-priv";
        # nix (run here as root for `nixos-container update --flake
        # /var/lib/hyperhive/meta#<agent>`) fetches the hive-core-owned
        # meta/applied repos; libgit2 refuses them without safe.directory.
        # See safeDirGitconfig above.
        ExecStartPre = "+-${pkgs.coreutils}/bin/cp ${safeDirGitconfig} /var/lib/hive-priv/.gitconfig";
      };
    };
  };
}