diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 00000000..9ea39dec --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,17 @@ +name: CI + +on: + pull_request: + branches: ["**"] + +jobs: + check: + name: nix flake check + runs-on: [hive-ci] + steps: + - uses: actions/checkout@v3 + - name: check + # Runs all flake checks: formatting (treefmt+rustfmt), cargo test, + # cargo clippy, and module evaluation. No --no-build: the checks + # derivations are the canonical source of truth. + run: nix flake check diff --git a/flake.nix b/flake.nix index 840653fd..0169702c 100644 --- a/flake.nix +++ b/flake.nix @@ -246,6 +246,7 @@ agentBaseToplevel = self.packages.x86_64-linux.agent-base-toplevel; managerToplevel = self.packages.x86_64-linux.manager-toplevel; }; + hive-ci = ./nix/modules/hive-ci.nix; hive-forge = ./nix/modules/hive-forge.nix; # Convenience alias: one import covers the full hyperhive host # stack (hive-c0re + hive-forge, since hive-c0re already pulls diff --git a/nix/modules/hive-c0re.nix b/nix/modules/hive-c0re.nix index 8de18a11..2565e27e 100644 --- a/nix/modules/hive-c0re.nix +++ b/nix/modules/hive-c0re.nix @@ -22,6 +22,7 @@ in # opt-in (off by default) and asserts that `services.hyperhive.domain` # is set before it can be enabled. imports = [ + ./hive-ci.nix ./hive-forge.nix ./hive-gateway.nix ./hive-matrix.nix @@ -99,34 +100,40 @@ in # `identity.rs::peers()` + the dashboard's `peer_hives` state field # (feeds the P33RS dashboard tab). options.services.hyperhive.peers = lib.mkOption { - type = lib.types.attrsOf (lib.types.submodule { - options = { - domain = lib.mkOption { - type = lib.types.str; - example = "lab.example.com"; - description = '' - DNS domain of the peer hive. Used to construct the peer's - dashboard URL (`http://''${domain}/`) and for Matrix - federation auto-discovery (`matrix.''${domain}`). - Must be reachable from this host. - ''; + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + domain = lib.mkOption { + type = lib.types.str; + example = "lab.example.com"; + description = '' + DNS domain of the peer hive. Used to construct the peer's + dashboard URL (`http://''${domain}/`) and for Matrix + federation auto-discovery (`matrix.''${domain}`). + Must be reachable from this host. + ''; + }; + tlsCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + Optional path to a PEM cert/bundle to trust for this peer's + TLS. Null = system CA bundle (for Let's Encrypt peers). Set + to the peer's self-signed cert for `selfSignedTls = true` + peers. Forward-compat slot; not yet used in v0. + ''; + }; }; - tlsCertFile = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = null; - description = '' - Optional path to a PEM cert/bundle to trust for this peer's - TLS. Null = system CA bundle (for Let's Encrypt peers). Set - to the peer's self-signed cert for `selfSignedTls = true` - peers. Forward-compat slot; not yet used in v0. - ''; - }; - }; - }); + } + ); default = { }; example = { - lab = { domain = "lab.example.com"; }; - edge = { domain = "edge.corp"; }; + lab = { + domain = "lab.example.com"; + }; + edge = { + domain = "edge.corp"; + }; }; description = '' Peer hives in the same swarm. The attrset key is a short label @@ -345,24 +352,29 @@ in # links; when false it falls back to direct `:` TCP. HIVE_GATEWAY_ENABLED = "1"; } - // lib.optionalAttrs (config.services.hyperhive.forge.enable && config.services.hyperhive.forge.behindGateway) { - # Public URL of the forge vhost served by hive-gateway. The - # dashboard uses this to build browser-facing forge links - # instead of hardcoding `:3000`, which breaks when - # the operator accesses the dashboard through the gateway - # (forge sub-domain has no port; direct port URL would be - # wrong). Absent when `behindGateway = false` — dashboard - # falls back to `:3000`. - HIVE_FORGE_PUBLIC_URL = "https://${config.services.hyperhive.forge.domain}"; - } + // + lib.optionalAttrs + (config.services.hyperhive.forge.enable && config.services.hyperhive.forge.behindGateway) + { + # Public URL of the forge vhost served by hive-gateway. The + # dashboard uses this to build browser-facing forge links + # instead of hardcoding `:3000`, which breaks when + # the operator accesses the dashboard through the gateway + # (forge sub-domain has no port; direct port URL would be + # wrong). Absent when `behindGateway = false` — dashboard + # falls back to `:3000`. + HIVE_FORGE_PUBLIC_URL = "https://${config.services.hyperhive.forge.domain}"; + } // lib.optionalAttrs (config.services.hyperhive.peers != { }) { # Peer hives serialised as a JSON array of {label, domain} objects. # Consumed by hive-ag3nt::identity::peers() + the dashboard's # peer_hives StateSnapshot field (P33RS tab). tlsCertFile is # nix-side-only (host nginx/trust config); rust never needs the path. HYPERHIVE_PEERS = builtins.toJSON ( - lib.mapAttrsToList (label: p: { inherit label; inherit (p) domain; }) - config.services.hyperhive.peers + lib.mapAttrsToList (label: p: { + inherit label; + inherit (p) domain; + }) config.services.hyperhive.peers ); }; serviceConfig = { diff --git a/nix/modules/hive-ci.nix b/nix/modules/hive-ci.nix new file mode 100644 index 00000000..4eeabf04 --- /dev/null +++ b/nix/modules/hive-ci.nix @@ -0,0 +1,213 @@ +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.ci; + forgeCfg = config.services.hyperhive.forge; + + # hive-c0re writes its own admin token here on first forge startup. + # The token has read:admin + write:admin scopes — sufficient to call + # the runner registration-token API endpoint. + coreTokenPath = "/var/lib/hyperhive/forge-core-token"; + + # Script run before the gitea-runner-hive service starts. + # On first boot (no .runner credentials yet) it fetches a fresh + # runner registration token from the forge admin API and writes it to + # /run/hive-ci/runner-token so gitea-actions-runner can register. + # On subsequent boots the .runner credentials file already exists and + # the runner ignores the token file entirely, so we write a dummy to + # satisfy the file-existence check in the NixOS module. + autoRegisterScript = pkgs.writeShellScript "hive-ci-autoregister" '' + set -euo pipefail + TOKEN_FILE=/run/hive-ci/runner-token + STATE_FILE=/var/lib/gitea-runner/hive/.runner + + mkdir -p /run/hive-ci + chmod 700 /run/hive-ci + + if [ -f "$STATE_FILE" ]; then + # Already registered — dummy token satisfies the module's path check. + echo "already-registered" > "$TOKEN_FILE" + chmod 600 "$TOKEN_FILE" + exit 0 + fi + + CORE_TOKEN=$(cat /run/hive-ci/core-token) + FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}" + + # Retry up to 30s for forge to come up (containers autoStart in parallel). + for i in $(seq 1 30); do + REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \ + "$FORGE_URL/api/v1/admin/runners/registration-token" \ + -H "Authorization: token $CORE_TOKEN" \ + | ${pkgs.jq}/bin/jq -r .token) && break + sleep 1 + done + + if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then + echo "hive-ci: failed to fetch runner registration token from forge" >&2 + exit 1 + fi + + echo "$REG_TOKEN" > "$TOKEN_FILE" + chmod 600 "$TOKEN_FILE" + ''; +in +{ + # Forgejo Actions runner in a `hive-ci` nixos-container. + # Shares host netns (same as hive-forge), so the runner reaches + # the forge at `http://127.0.0.1:` without extra plumbing. + # Container is non-ephemeral: the runner's registered credentials + # survive restarts (gitea-actions-runner writes them to its stateDir + # on first registration and reuses them on every subsequent start). + # + # Auto-registration: on first boot the container fetches a runner + # registration token from the forge's admin API using the core token + # hive-c0re writes to /var/lib/hyperhive/forge-core-token. No manual + # token handling needed — `ci.enable = true` is the full operator + # bootstrap. Registration flow: preStart calls the Forgejo admin API, + # writes the token to /run/hive-ci/runner-token, runner registers and + # persists credentials to stateDir — token file ignored on next boot. + # + # Nix builds inside the container use the shared /nix/store (standard + # nixos-container behaviour) with sandbox-fallback = true, because + # nspawn containers can't create the user-namespaces that nix sandboxing + # requires. See docs/gotchas.md. + + options.services.hyperhive.ci = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = '' + Run a Forgejo Actions runner in a `hive-ci` nixos-container. + Disabled by default; requires `services.hyperhive.forge.enable = true`. + + On first start the container auto-registers against hive-forge using + hive-c0re's admin token — no manual token provisioning needed. + Runner credentials are persisted in the container's state dir and + reused on every subsequent boot. + ''; + }; + + name = lib.mkOption { + type = lib.types.str; + default = "hive-ci"; + example = "prod-hive"; + description = '' + Runner name as shown in the Forgejo admin panel. Defaults to + "hive-ci"; override when multiple hives share a Forgejo instance. + ''; + }; + + concurrency = lib.mkOption { + type = lib.types.ints.positive; + default = 1; + example = 4; + description = '' + Maximum number of workflow jobs the runner executes in parallel. + Each job gets its own temporary working directory; multiple parallel + jobs share the container's nix store and cargo registry cache. + Higher values trade memory + CPU headroom for throughput. + ''; + }; + + labels = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ "hive-ci:host" ]; + example = [ + "hive-ci:host" + "nix:host" + ]; + description = '' + Runner labels in `:` format. The `host` scheme runs + commands directly in the container (no docker/podman). Workflow + files target this runner with `runs-on: [hive-ci]`. + ''; + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.gitea-actions-runner; + defaultText = lib.literalExpression "pkgs.gitea-actions-runner"; + description = "gitea-actions-runner package."; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = forgeCfg.enable; + message = '' + services.hyperhive.ci.enable = true requires + services.hyperhive.forge.enable = true — the runner registers + against the hive-forge Forgejo instance. + ''; + } + ]; + + containers.hive-ci = { + autoStart = true; + ephemeral = false; + # Shared host netns: runner reaches hive-forge at localhost. + privateNetwork = false; + + bindMounts = { + # Core token — used by the auto-register script on first boot. + # Read-only: the container only reads it, never modifies it. + "/run/hive-ci/core-token" = { + hostPath = coreTokenPath; + isReadOnly = true; + }; + }; + + config = + { pkgs, lib, ... }: + { + system.stateVersion = "26.05"; + + # nspawn containers can't create user-namespaces, so nix + # sandboxing always fails. Fall back to unsandboxed builds. + # See docs/gotchas.md. + nix.settings.sandbox-fallback = lib.mkForce true; + nix.settings.experimental-features = [ + "nix-command" + "flakes" + ]; + + services.gitea-actions-runner.instances.hive = { + enable = true; + name = cfg.name; + url = "http://127.0.0.1:${toString forgeCfg.httpPort}"; + # Token file is written by the ExecStartPre script below. + # On first boot: real registration token fetched from forge API. + # On subsequent boots: dummy value (runner uses .runner creds). + tokenFile = "/run/hive-ci/runner-token"; + labels = cfg.labels; + settings = { + runner.capacity = cfg.concurrency; + # Generous timeout for cold-cache nix builds. + runner.timeout = "3h"; + }; + package = cfg.package; + }; + + # Prepend auto-register script before the runner service starts. + # `+` prefix runs with elevated privileges so it can read the + # bind-mounted core-token (owned by root on the host). + systemd.services."gitea-runner-hive".serviceConfig.ExecStartPre = lib.mkBefore [ + "+${autoRegisterScript}" + ]; + + # git is required by the runner's checkout step. + # Everything else (nix, rust tools) is either part of NixOS + # by default or pulled in hermetically by nix flake check. + # curl/jq in the autoregister preStart use absolute store paths. + environment.systemPackages = [ pkgs.git ]; + }; + }; + }; +}