hyperhive/nix/modules/hive-ci.nix

{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.ci;
  forgeCfg = config.services.hyperhive.forge;

  # hive-c0re writes its own admin token here on first forge startup.
  # The token has read:admin + write:admin scopes — sufficient to call
  # the runner registration-token API endpoint.
  coreTokenPath = "/var/lib/hyperhive/forge-core-token";

  # Script run before the gitea-runner-hive service starts.
  # On first boot (no .runner credentials yet) it fetches a fresh
  # runner registration token from the forge admin API and writes it to
  # /run/hive-ci/runner-token so gitea-actions-runner can register.
  # On subsequent boots the .runner credentials file already exists and
  # the runner ignores the token file entirely, so we write a dummy to
  # satisfy the file-existence check in the NixOS module.
  autoRegisterScript = pkgs.writeShellScript "hive-ci-autoregister" ''
    set -euo pipefail
    TOKEN_FILE=/run/hive-ci/runner-token
    STATE_FILE=/var/lib/gitea-runner/hive/.runner

    mkdir -p /run/hive-ci
    chmod 700 /run/hive-ci

    if [ -f "$STATE_FILE" ]; then
      # Already registered — dummy token satisfies the module's path check.
      echo "already-registered" > "$TOKEN_FILE"
      chmod 600 "$TOKEN_FILE"
      exit 0
    fi

    CORE_TOKEN=$(cat /run/hive-ci/core-token)
    FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"

    # Retry up to 30s for forge to come up (containers autoStart in parallel).
    for i in $(seq 1 30); do
      REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \
        "$FORGE_URL/api/v1/admin/runners/registration-token" \
        -H "Authorization: token $CORE_TOKEN" \
        | ${pkgs.jq}/bin/jq -r .token) && break
      sleep 1
    done

    if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
      echo "hive-ci: failed to fetch runner registration token from forge" >&2
      exit 1
    fi

    echo "$REG_TOKEN" > "$TOKEN_FILE"
    chmod 600 "$TOKEN_FILE"
  '';
in
{
  # Forgejo Actions runner in a `hive-ci` nixos-container.
  # Shares host netns (same as hive-forge), so the runner reaches
  # the forge at `http://127.0.0.1:<httpPort>` without extra plumbing.
  # Container is non-ephemeral: the runner's registered credentials
  # survive restarts (gitea-actions-runner writes them to its stateDir
  # on first registration and reuses them on every subsequent start).
  #
  # Auto-registration: on first boot the container fetches a runner
  # registration token from the forge's admin API using the core token
  # hive-c0re writes to /var/lib/hyperhive/forge-core-token. No manual
  # token handling needed — `ci.enable = true` is the full operator
  # bootstrap. Registration flow: preStart calls the Forgejo admin API,
  # writes the token to /run/hive-ci/runner-token, runner registers and
  # persists credentials to stateDir — token file ignored on next boot.
  #
  # Nix builds inside the container use the shared /nix/store (standard
  # nixos-container behaviour) with sandbox-fallback = true, because
  # nspawn containers can't create the user-namespaces that nix sandboxing
  # requires. See docs/gotchas.md.

  options.services.hyperhive.ci = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      example = true;
      description = ''
        Run a Forgejo Actions runner in a `hive-ci` nixos-container.
        Disabled by default; requires `services.hyperhive.forge.enable = true`.

        On first start the container auto-registers against hive-forge using
        hive-c0re's admin token — no manual token provisioning needed.
        Runner credentials are persisted in the container's state dir and
        reused on every subsequent boot.
      '';
    };

    name = lib.mkOption {
      type = lib.types.str;
      default = "hive-ci";
      example = "prod-hive";
      description = ''
        Runner name as shown in the Forgejo admin panel. Defaults to
        "hive-ci"; override when multiple hives share a Forgejo instance.
      '';
    };

    concurrency = lib.mkOption {
      type = lib.types.ints.positive;
      default = 1;
      example = 4;
      description = ''
        Maximum number of workflow jobs the runner executes in parallel.
        Each job gets its own temporary working directory; multiple parallel
        jobs share the container's nix store and cargo registry cache.
        Higher values trade memory + CPU headroom for throughput.
      '';
    };

    labels = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [ "hive-ci:host" ];
      example = [
        "hive-ci:host"
        "nix:host"
      ];
      description = ''
        Runner labels in `<name>:<scheme>` format. The `host` scheme runs
        commands directly in the container (no docker/podman). Workflow
        files target this runner with `runs-on: [hive-ci]`.
      '';
    };

    package = lib.mkOption {
      type = lib.types.package;
      default = pkgs.gitea-actions-runner;
      defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
      description = "gitea-actions-runner package.";
    };
  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = forgeCfg.enable;
        message = ''
          services.hyperhive.ci.enable = true requires
          services.hyperhive.forge.enable = true — the runner registers
          against the hive-forge Forgejo instance.
        '';
      }
    ];

    containers.hive-ci = {
      autoStart = true;
      ephemeral = false;
      # Shared host netns: runner reaches hive-forge at localhost.
      privateNetwork = false;

      bindMounts = {
        # Core token — used by the auto-register script on first boot.
        # Read-only: the container only reads it, never modifies it.
        "/run/hive-ci/core-token" = {
          hostPath = coreTokenPath;
          isReadOnly = true;
        };
      };

      config =
        { pkgs, lib, ... }:
        {
          system.stateVersion = "26.05";

          # nspawn containers can't create user-namespaces, so nix
          # sandboxing always fails. Fall back to unsandboxed builds.
          # See docs/gotchas.md.
          nix.settings.sandbox-fallback = lib.mkForce true;
          nix.settings.experimental-features = [
            "nix-command"
            "flakes"
          ];

          services.gitea-actions-runner.instances.hive = {
            enable = true;
            name = cfg.name;
            url = "http://127.0.0.1:${toString forgeCfg.httpPort}";
            # Token file is written by the ExecStartPre script below.
            # On first boot: real registration token fetched from forge API.
            # On subsequent boots: dummy value (runner uses .runner creds).
            tokenFile = "/run/hive-ci/runner-token";
            labels = cfg.labels;
            settings = {
              runner.capacity = cfg.concurrency;
              # Generous timeout for cold-cache nix builds.
              runner.timeout = "3h";
            };
            package = cfg.package;
          };

          # Prepend auto-register script before the runner service starts.
          # `+` prefix runs with elevated privileges so it can read the
          # bind-mounted core-token (owned by root on the host).
          systemd.services."gitea-runner-hive".serviceConfig.ExecStartPre = lib.mkBefore [
            "+${autoRegisterScript}"
          ];

          # git is required by the runner's checkout step.
          # Everything else (nix, rust tools) is either part of NixOS
          # by default or pulled in hermetically by nix flake check.
          # curl/jq in the autoregister preStart use absolute store paths.
          environment.systemPackages = [ pkgs.git ];
        };
    };
  };
}