hyperhive/nix/modules/hive-ci.nix

{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.forge.ci;
  forgeCfg = config.services.hyperhive.forge;
  gatewayCfg = config.services.hyperhive.gateway;
  tlsCfg = config.services.hyperhive.tls;

  # Self-signed TLS is the gateway default (no operator cert / ACME). When
  # active, forgejo's ROOT_URL is `https://forge.<domain>` and the leaf is
  # signed by the host hive CA — so the runner's Node-based actions (e.g.
  # `upload-artifact`, which POSTs to the ROOT_URL-derived artifact endpoint)
  # reject the chain, since Node trusts only its bundled CA bundle, not the
  # system store. Trust the hive CA explicitly via NODE_EXTRA_CA_CERTS below.
  # `gateway.useSelfSigned` is the gateway module's single source of truth
  # for the self-signed condition (no duplicated derivation here).
  useSelfSigned = gatewayCfg.useSelfSigned;
  caHostPath = "${tlsCfg.stateDir}/ca.pem";
  caContainerPath = "/run/hive-ca/ca.pem";

  # hive-c0re writes its own admin token here on first forge startup.
  # The token has read:admin + write:admin scopes — sufficient to call
  # the runner registration-token API endpoint.
  # This path is HOST-ONLY. It is never bind-mounted into hive-ci.
  coreTokenPath = "/var/lib/hyperhive/forge-core-token";

  # Container state root on the host. Non-ephemeral containers keep
  # their filesystem here across reboots. The prefetch service accesses
  # the runner's .runner file via this path to validate credentials
  # without entering the container.
  containerRoot = "/var/lib/nixos-containers/hive-ci";

  # Host-side oneshot. Runs before `container@hive-ci.service`.
  #
  # The core token stays on the host. The container only ever sees the
  # TOKEN= env-file populated here, never the core token itself.
  #
  # Flow:
  #   1. If .runner exists: validate the runner ID against forge.
  #      Waits up to 60s for forge-core-token (hive-c0re writes it after
  #      forge container starts and admin is provisioned — this lags
  #      hive-c0re.service becoming active on first boot).
  #      404 → purge .runner (re-registration needed).
  #      000 (forge unreachable) → keep credentials, write placeholder.
  #      other non-200 → purge .runner.
  #      Token absent after 60s → keep credentials (safe — the runner
  #      holds valid creds; next boot will validate properly).
  #   2. If .runner absent or just purged: wait up to 60s for both
  #      forge-core-token to appear AND forge API to respond, then
  #      fetch a fresh registration token and write TOKEN=<real>.
  #      Exits non-zero if both timeout — systemd logs the failure;
  #      the placeholder from tmpfiles means the container still starts
  #      but the runner will error. Operator restarts the service once
  #      forge is healthy.
  prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
    set -euo pipefail
    TOKEN_FILE=/run/hive-ci/runner-token
    FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
    RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"

    # Validate existing .runner credentials against forge. Purge if
    # the runner was deleted (404) or the file is malformed. Keep if
    # forge is unreachable (000) — transient outage shouldn't wipe creds.
    #
    # We wait up to 60s for forge-core-token to appear (hive-c0re writes
    # it after provisioning the forge admin, which requires the forge
    # container to start and become ready — this can lag behind
    # hive-c0re.service becoming "active" on first boot). The same loop
    # doubles as a wait for forge's API to become ready.
    if [ -f "$RUNNER_FILE" ]; then
      RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
      if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
        echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
        rm -f "$RUNNER_FILE"
      else
        # Wait for the core token before validating (same timeout as below).
        CORE_TOKEN=""
        for i in $(seq 1 60); do
          if [ -f "${coreTokenPath}" ]; then
            CORE_TOKEN=$(cat ${coreTokenPath})
            break
          fi
          sleep 1
        done
        if [ -z "''${CORE_TOKEN:-}" ]; then
          echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
        else
          HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
            -H "Authorization: token $CORE_TOKEN" \
            "$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
          if [ "$HTTP" = "404" ]; then
            echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
            rm -f "$RUNNER_FILE"
          elif [ "$HTTP" = "000" ]; then
            echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
          elif [ "$HTTP" != "200" ]; then
            echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
            rm -f "$RUNNER_FILE"
          fi
        fi
      fi
    fi

    # .runner valid → write placeholder; gitea-actions-runner skips
    # re-registration when .runner exists (TOKEN value is irrelevant).
    if [ -f "$RUNNER_FILE" ]; then
      echo "TOKEN=placeholder" > "$TOKEN_FILE"
      chmod 600 "$TOKEN_FILE"
      exit 0
    fi

    # First boot or stale creds purged — wait for forge-core-token, then
    # fetch a fresh registration token. Single retry loop covers both:
    # waiting for hive-c0re to write the token file AND for forge's API
    # to become responsive (they race on first boot).
    REG_TOKEN=""
    for i in $(seq 1 60); do
      if [ ! -f "${coreTokenPath}" ]; then
        echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
        sleep 1
        continue
      fi
      CORE_TOKEN=$(cat ${coreTokenPath})
      # Capture the HTTP status so a stale/invalid core token (401/403) is
      # distinguished from a transient forge hiccup. With the old bare
      # `curl -sf | jq`, a forge-core-token that's stale for the current
      # forge (e.g. after a forge rebuild) 401s and fails silently every
      # attempt for the full 60s loop, then exits with a misleading
      # "core token absent or forge unreachable" — masking the real cause.
      # Fail fast + loudly on 401/403 so the failure mode is legible and
      # the operator/hive-c0re knows to re-mint forge-core-token.
      RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \
        "$FORGE_URL/api/v1/admin/runners/registration-token" \
        -H "Authorization: token $CORE_TOKEN" || printf '\n000')
      HTTP=$(printf '%s' "$RESP" | tail -n1)
      BODY=$(printf '%s' "$RESP" | sed '$d')
      case "$HTTP" in
        2*)
          REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token)
          if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi
          echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2
          ;;
        401 | 403)
          echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s." >&2
          exit 1
          ;;
        *)
          echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2
          ;;
      esac
      sleep 1
    done

    if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
      echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
      exit 1
    fi

    echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
    chmod 600 "$TOKEN_FILE"
  '';
in
{
  # Forgejo Actions runner in a `hive-ci` nixos-container.
  # Shares host netns (same as hive-forge), so the runner reaches
  # the forge at `http://127.0.0.1:<httpPort>` without extra plumbing.
  # Container is non-ephemeral: the runner's registered credentials
  # survive restarts (gitea-actions-runner writes them to its stateDir
  # on first registration and reuses them on every subsequent start).
  #
  # Credential isolation: the forge admin token (`forge-core-token`)
  # never enters the hive-ci container. A host-side oneshot service
  # (`hive-ci-prefetch.service`) performs all forge API calls before
  # the container starts and writes only the runner registration token
  # to `/run/hive-ci/runner-token`. The container bind-mounts this
  # file read-only and never has access to the wider admin token.
  #
  # Nix builds inside the container use the shared /nix/store (standard
  # nixos-container behaviour) with sandbox-fallback = true, because
  # nspawn containers can't create the user-namespaces that nix sandboxing
  # requires. See docs/gotchas.md.

  options.services.hyperhive.forge.ci = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      example = true;
      description = ''
        Run a Forgejo Actions runner in a `hive-ci` nixos-container.
        Grouped under `services.hyperhive.forge` because the runner is
        tightly coupled to the forge instance it registers against.
        Disabled by default; the internal forge it registers against is
        always present (mandatory), so enabling this is all that's needed.

        On first start the container auto-registers against hive-forge using
        hive-c0re's admin token — no manual token provisioning needed.
        Runner credentials are persisted in the container's state dir and
        reused on every subsequent boot.
      '';
    };

    name = lib.mkOption {
      type = lib.types.str;
      default = "hive-ci";
      example = "prod-hive";
      description = ''
        Runner name as shown in the Forgejo admin panel. Defaults to
        "hive-ci"; override when multiple hives share a Forgejo instance.
      '';
    };

    concurrency = lib.mkOption {
      type = lib.types.ints.positive;
      default = 1;
      example = 4;
      description = ''
        Maximum number of workflow jobs the runner executes in parallel.
        Each job gets its own temporary working directory; multiple parallel
        jobs share the container's nix store and cargo registry cache.
        Higher values trade memory + CPU headroom for throughput.
      '';
    };

    labels = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [ "hive-ci:host" ];
      example = [
        "hive-ci:host"
        "nix:host"
      ];
      description = ''
        Runner labels in `<name>:<scheme>` format. The `host` scheme runs
        commands directly in the container (no docker/podman). Workflow
        files target this runner with `runs-on: [hive-ci]`.
      '';
    };

    package = lib.mkOption {
      type = lib.types.package;
      default = pkgs.gitea-actions-runner;
      defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
      description = "gitea-actions-runner package.";
    };

    jobTimeout = lib.mkOption {
      type = lib.types.str;
      default = "1h";
      example = "3h";
      description = ''
        Per-job wall-clock timeout the runner enforces (act_runner's
        `runner.timeout`). A job that exceeds it is killed, so a hung or
        runaway build is bounded instead of holding the runner's single
        slot indefinitely. Default `1h` comfortably covers a cold-cache
        nix build while still bounding a stuck job; raise it (e.g.
        `"3h"`) if you legitimately run jobs longer than that. Accepts a
        Go duration string (`30m`, `1h`, `2h30m`). Note: this is
        enforced by the runner process, so it only fires while that
        process is itself healthy.
      '';
    };
  };

  config = lib.mkIf cfg.enable {
    # No forge-presence assertion needed: the internal forge is mandatory
    # (deploys with hyperhive), so the runner always has an instance to
    # register against.

    # Create /run/hive-ci/ on the host and seed runner-token with a
    # placeholder. hive-ci-prefetch.service overwrites it with the real
    # token (or a fresh placeholder) before the container starts. The
    # placeholder ensures the EnvironmentFile is always present even if
    # the prefetch service hasn't run yet (e.g. tmpfiles-setup timing).
    systemd.tmpfiles.rules = [
      "d /run/hive-ci 0700 root root -"
      "f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
    ];

    # Host-side oneshot: validates/refreshes runner credentials before
    # the container starts. The core admin token stays on the host and
    # is never bind-mounted into the container. Runs on every boot so
    # stale .runner credentials (runner deleted from forge) are detected
    # and the container re-registers on the next start.
    systemd.services.hive-ci-prefetch = {
      description = "Pre-fetch hive-ci runner registration token (host-side)";
      # Run before the container starts but after tmpfiles so the token
      # file directory exists. After hive-c0re so the forge token is
      # likely written (best-effort — the script handles the absent case).
      after = [
        "systemd-tmpfiles-setup.service"
        "hive-c0re.service"
      ];
      before = [ "container@hive-ci.service" ];
      wantedBy = [ "container@hive-ci.service" ];
      # partOf binds this oneshot's lifecycle to the container: when the
      # container is stopped or restarted, systemd propagates that to this
      # unit so it re-runs on the NEXT container start. Without this, the
      # RemainAfterExit=true oneshot stays "active (exited)" forever after
      # its first run — so a container restart skips it and the stale
      # runner-token file (a placeholder from a boot where the forge token
      # wasn't ready yet, or a registration token consumed/rotated since)
      # is never refreshed. The in-container register service then fails
      # with "runner registration token not found". partOf guarantees a
      # fresh token is fetched before every container (re)start.
      #
      # Unit name: a declarative `containers.<name>` is the host unit
      # `container@<name>.service` (the nspawn template), NOT
      # `nixos-container@…`. The earlier `nixos-container@hive-ci.service`
      # matched no real unit, so before/wantedBy/partOf were silent
      # no-ops — the partOf never bound, the oneshot stayed
      # `active (exited)`, and the token was never refreshed on restart.
      # Confirmed against the live `container@hive-matrix.service` unit
      # during the matrix-outage incident.
      partOf = [ "container@hive-ci.service" ];
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
        ExecStart = prefetchScript;
      };
    };

    # Self-signed mode: the CA cert the bind-mount above sources is
    # generated by the host `hive-tls-ca` service. Order the container after
    # it so the bind source exists before nspawn sets the mount up (a
    # condition-skipped/late CA would otherwise fail the container start).
    systemd.services."container@hive-ci" = lib.mkIf useSelfSigned {
      after = [ "hive-tls-ca.service" ];
      requires = [ "hive-tls-ca.service" ];
    };

    containers.hive-ci = {
      autoStart = true;
      ephemeral = false;
      # Shared host netns: runner reaches hive-forge at localhost.
      privateNetwork = false;

      bindMounts = {
        # Pre-filled by hive-ci-prefetch.service (host-side) before the
        # container starts. Read-only: the container reads TOKEN= from
        # here; the core admin token never enters the container.
        "/run/hive-ci/runner-token" = {
          hostPath = "/run/hive-ci/runner-token";
          isReadOnly = true;
        };
      }
      # Self-signed mode: bind ONLY the public hive CA cert (never the
      # `hive-tls` state dir — it holds the CA + leaf private keys) so the
      # runner's Node actions can trust the gateway/forge self-signed leaf
      # (see NODE_EXTRA_CA_CERTS in the container config). Source generated
      # by the host `hive-tls-ca` service; the container@hive-ci ordering
      # below guarantees it exists before this mount is set up.
      // lib.optionalAttrs useSelfSigned {
        ${caContainerPath} = {
          hostPath = caHostPath;
          isReadOnly = true;
        };
      };

      config =
        { pkgs, lib, ... }:
        {
          system.stateVersion = "26.05";

          # Resolve the hive's own forge domain to loopback inside the
          # runner. The Forgejo Actions artifact API hands the runner an
          # upload URL built from forgejo's ROOT_URL — the *public* forge
          # domain (`forge.<domain>`), not the runner's
          # `127.0.0.1:<httpPort>` registration URL. This container shares
          # host netns but carries no resolver entry for the hive's own
          # domains (the gateway's `localHostsEntry` only touches the host),
          # so `actions/upload-artifact` dies with
          #   Error: getaddrinfo ENOTFOUND forge.<domain>
          # while normal CI is unaffected (checkout uses the localhost
          # registration URL). Map the forge domain to loopback so the
          # upload reaches the local forge — via the gateway on :80 when
          # behindGateway, or forge directly on :httpPort otherwise. The
          # gateway proxies all of `/` (artifact endpoints included), and a
          # `localhost` domain (no hive-domain set) is already loopback, so
          # this is a harmless no-op in that case.
          networking.extraHosts = "127.0.0.1 ${forgeCfg.domain}";

          # nspawn containers can't create user-namespaces, so nix
          # sandboxing always fails. Fall back to unsandboxed builds.
          # See docs/gotchas.md.
          nix.settings.sandbox-fallback = lib.mkForce true;
          # Build locally when a configured remote builder is unreachable
          # instead of hard-failing. Without this a single down/unresolvable
          # build machine (e.g. a DNS blip on a `nix.buildMachines` host)
          # turns every cache-miss `nix flake check` red hive-wide — CI must
          # degrade to a slower local build, not fail. (`sandbox-fallback`
          # above is unrelated: it only governs the sandbox, not builders.)
          nix.settings.fallback = lib.mkForce true;
          nix.settings.experimental-features = [
            "nix-command"
            "flakes"
          ];

          # package is top-level on gitea-actions-runner, not per-instance.
          services.gitea-actions-runner.package = cfg.package;

          services.gitea-actions-runner.instances.hive = {
            enable = true;
            name = cfg.name;
            url = "http://127.0.0.1:${toString forgeCfg.httpPort}";
            # EnvironmentFile providing TOKEN= — pre-filled by the
            # host-side hive-ci-prefetch.service before the container
            # starts; bind-mounted read-only from /run/hive-ci/runner-token
            # on the host.
            tokenFile = "/run/hive-ci/runner-token";
            labels = cfg.labels;
            settings = {
              runner.capacity = cfg.concurrency;
              # Per-job wall-clock cap — see the `jobTimeout` option.
              runner.timeout = cfg.jobTimeout;
            };
          };

          # No tmpfiles rule: /run/hive-ci/runner-token is bind-mounted
          # read-only from the host (pre-filled before container start).
          # nspawn creates the /run/hive-ci/ mount-point directory
          # automatically before launching the container's init.

          # No hive-ci-register.service inside the container: all forge
          # API calls (runner validation, token fetch) moved to the
          # host-side hive-ci-prefetch.service. The core admin token
          # never enters this container.

          # git is already in the gitea-actions-runner service PATH (the
          # nixpkgs module builds it from the package's runtime deps).
          # nix is required for `nix flake check` / `nix build` workflow
          # steps — it's not included by the upstream module.
          # Use the `path` service attribute (generates ExecSearchPath=)
          # to prepend nix's bin dir to PATH without touching the
          # environment.PATH the nixpkgs module sets — overriding that
          # would lose git, curl, nodejs, and other runner deps.
          environment.systemPackages = [
            pkgs.git
            pkgs.nix
          ];

          systemd.services."gitea-runner-hive" = {
            path = [ pkgs.nix ];
            # Trust the hive CA in Node-based actions. With self-signed TLS,
            # forgejo's ROOT_URL is `https://forge.<domain>` (CA-signed leaf),
            # so actions like `upload-artifact` — whose Node HTTP client uses
            # Node's *bundled* CA bundle, not the system store — reject the
            # chain with "unable to verify the first certificate". Pointing
            # NODE_EXTRA_CA_CERTS at the bind-mounted CA adds it to Node's
            # roots for every action, hive-wide. Inherited by the job
            # processes the runner spawns (host execution mode). Only set in
            # self-signed mode; with an operator cert / ACME the public CA
            # already validates and the bind-mount is absent.
            environment = lib.mkIf useSelfSigned {
              NODE_EXTRA_CA_CERTS = caContainerPath;
            };
            # Gate runner start (and therefore job registration/claiming) on
            # the in-container nix daemon being reachable. After a hive-ci
            # restart the runner re-registers and immediately claims any
            # queued jobs — which can beat the nix daemon coming up: its
            # `nix-daemon.socket` carries
            # `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket` and is
            # skipped until /nix/var is read-write, so the first
            # nix-dependent build dispatches into a cold daemon and
            # hangs/retries (observed: a 55m48s `nix flake check` vs the
            # normal ~30s, a build-offload stall, not a code failure).
            #
            # Ordering `after`/`wants` the socket unit does NOT fix this — a
            # condition-skipped unit satisfies systemd ordering immediately,
            # so the runner would still start before the daemon is live.
            # Instead block in ExecStartPre by polling the daemon until it
            # actually answers; this is topology-agnostic (works whether the
            # daemon is in-container or a shared host socket). `mkBefore` so
            # this runs ahead of any pre-steps the upstream module adds.
            serviceConfig.ExecStartPre = lib.mkBefore [
              (pkgs.writeShellScript "wait-nix-daemon" ''
                # Up to ~180s; the daemon is normally up within seconds, this
                # only bites in the post-restart cold window. Non-fatal shape:
                # if it never comes up the unit fails cleanly (recoverable)
                # rather than the runner claiming jobs into a dead daemon.
                #
                # `--store daemon` is load-bearing. A bare `nix store ping`
                # uses the `auto` store, which — when run as root with the
                # daemon socket absent — silently resolves to a LOCAL store
                # (root can write /nix/store directly) and pings successfully.
                # systemd service units don't source the profile that sets
                # `NIX_REMOTE=daemon`, so this is the real environment here.
                # At cold boot the daemon socket IS absent: nix-daemon.socket
                # carries `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket`
                # and is condition-skipped until /nix/var goes read-write. So
                # a bare ping would pass against the local store while the
                # daemon is still down — defeating the gate's whole purpose
                # (the runner would start and claim jobs the daemon can't yet
                # service). Pinning `--store daemon` makes the poll verify the
                # actual daemon socket, so the gate honours its contract and
                # waits until the daemon — not a local fallback — answers.
                for _ in $(seq 1 90); do
                  if ${pkgs.nix}/bin/nix store ping --store daemon >/dev/null 2>&1; then
                    exit 0
                  fi
                  sleep 2
                done
                echo "nix daemon not reachable after 180s" >&2
                exit 1
              '')
            ];
          };
        };
    };
  };
}