Compare commits
4 changed files with 37 additions and 280 deletions
|
|
@ -1,17 +0,0 @@
|
||||||
name: CI
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
branches: ["**"]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
check:
|
|
||||||
name: nix flake check
|
|
||||||
runs-on: [hive-ci]
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
- name: check
|
|
||||||
# Runs all flake checks: formatting (treefmt+rustfmt), cargo test,
|
|
||||||
# cargo clippy, and module evaluation. No --no-build: the checks
|
|
||||||
# derivations are the canonical source of truth.
|
|
||||||
run: nix flake check
|
|
||||||
|
|
@ -246,7 +246,6 @@
|
||||||
agentBaseToplevel = self.packages.x86_64-linux.agent-base-toplevel;
|
agentBaseToplevel = self.packages.x86_64-linux.agent-base-toplevel;
|
||||||
managerToplevel = self.packages.x86_64-linux.manager-toplevel;
|
managerToplevel = self.packages.x86_64-linux.manager-toplevel;
|
||||||
};
|
};
|
||||||
hive-ci = ./nix/modules/hive-ci.nix;
|
|
||||||
hive-forge = ./nix/modules/hive-forge.nix;
|
hive-forge = ./nix/modules/hive-forge.nix;
|
||||||
# Convenience alias: one import covers the full hyperhive host
|
# Convenience alias: one import covers the full hyperhive host
|
||||||
# stack (hive-c0re + hive-forge, since hive-c0re already pulls
|
# stack (hive-c0re + hive-forge, since hive-c0re already pulls
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,6 @@ in
|
||||||
# opt-in (off by default) and asserts that `services.hyperhive.domain`
|
# opt-in (off by default) and asserts that `services.hyperhive.domain`
|
||||||
# is set before it can be enabled.
|
# is set before it can be enabled.
|
||||||
imports = [
|
imports = [
|
||||||
./hive-ci.nix
|
|
||||||
./hive-forge.nix
|
./hive-forge.nix
|
||||||
./hive-gateway.nix
|
./hive-gateway.nix
|
||||||
./hive-matrix.nix
|
./hive-matrix.nix
|
||||||
|
|
@ -100,8 +99,7 @@ in
|
||||||
# `identity.rs::peers()` + the dashboard's `peer_hives` state field
|
# `identity.rs::peers()` + the dashboard's `peer_hives` state field
|
||||||
# (feeds the P33RS dashboard tab).
|
# (feeds the P33RS dashboard tab).
|
||||||
options.services.hyperhive.peers = lib.mkOption {
|
options.services.hyperhive.peers = lib.mkOption {
|
||||||
type = lib.types.attrsOf (
|
type = lib.types.attrsOf (lib.types.submodule {
|
||||||
lib.types.submodule {
|
|
||||||
options = {
|
options = {
|
||||||
domain = lib.mkOption {
|
domain = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
|
|
@ -124,16 +122,11 @@ in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
}
|
});
|
||||||
);
|
|
||||||
default = { };
|
default = { };
|
||||||
example = {
|
example = {
|
||||||
lab = {
|
lab = { domain = "lab.example.com"; };
|
||||||
domain = "lab.example.com";
|
edge = { domain = "edge.corp"; };
|
||||||
};
|
|
||||||
edge = {
|
|
||||||
domain = "edge.corp";
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
description = ''
|
description = ''
|
||||||
Peer hives in the same swarm. The attrset key is a short label
|
Peer hives in the same swarm. The attrset key is a short label
|
||||||
|
|
@ -352,10 +345,7 @@ in
|
||||||
# links; when false it falls back to direct `<host>:<port>` TCP.
|
# links; when false it falls back to direct `<host>:<port>` TCP.
|
||||||
HIVE_GATEWAY_ENABLED = "1";
|
HIVE_GATEWAY_ENABLED = "1";
|
||||||
}
|
}
|
||||||
//
|
// lib.optionalAttrs (config.services.hyperhive.forge.enable && config.services.hyperhive.forge.behindGateway) {
|
||||||
lib.optionalAttrs
|
|
||||||
(config.services.hyperhive.forge.enable && config.services.hyperhive.forge.behindGateway)
|
|
||||||
{
|
|
||||||
# Public URL of the forge vhost served by hive-gateway. The
|
# Public URL of the forge vhost served by hive-gateway. The
|
||||||
# dashboard uses this to build browser-facing forge links
|
# dashboard uses this to build browser-facing forge links
|
||||||
# instead of hardcoding `<hostname>:3000`, which breaks when
|
# instead of hardcoding `<hostname>:3000`, which breaks when
|
||||||
|
|
@ -371,10 +361,8 @@ in
|
||||||
# peer_hives StateSnapshot field (P33RS tab). tlsCertFile is
|
# peer_hives StateSnapshot field (P33RS tab). tlsCertFile is
|
||||||
# nix-side-only (host nginx/trust config); rust never needs the path.
|
# nix-side-only (host nginx/trust config); rust never needs the path.
|
||||||
HYPERHIVE_PEERS = builtins.toJSON (
|
HYPERHIVE_PEERS = builtins.toJSON (
|
||||||
lib.mapAttrsToList (label: p: {
|
lib.mapAttrsToList (label: p: { inherit label; inherit (p) domain; })
|
||||||
inherit label;
|
config.services.hyperhive.peers
|
||||||
inherit (p) domain;
|
|
||||||
}) config.services.hyperhive.peers
|
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
|
|
|
||||||
|
|
@ -1,213 +0,0 @@
|
||||||
{
|
|
||||||
pkgs,
|
|
||||||
lib,
|
|
||||||
config,
|
|
||||||
...
|
|
||||||
}:
|
|
||||||
let
|
|
||||||
cfg = config.services.hyperhive.ci;
|
|
||||||
forgeCfg = config.services.hyperhive.forge;
|
|
||||||
|
|
||||||
# hive-c0re writes its own admin token here on first forge startup.
|
|
||||||
# The token has read:admin + write:admin scopes — sufficient to call
|
|
||||||
# the runner registration-token API endpoint.
|
|
||||||
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
|
|
||||||
|
|
||||||
# Script run before the gitea-runner-hive service starts.
|
|
||||||
# On first boot (no .runner credentials yet) it fetches a fresh
|
|
||||||
# runner registration token from the forge admin API and writes it to
|
|
||||||
# /run/hive-ci/runner-token so gitea-actions-runner can register.
|
|
||||||
# On subsequent boots the .runner credentials file already exists and
|
|
||||||
# the runner ignores the token file entirely, so we write a dummy to
|
|
||||||
# satisfy the file-existence check in the NixOS module.
|
|
||||||
autoRegisterScript = pkgs.writeShellScript "hive-ci-autoregister" ''
|
|
||||||
set -euo pipefail
|
|
||||||
TOKEN_FILE=/run/hive-ci/runner-token
|
|
||||||
STATE_FILE=/var/lib/gitea-runner/hive/.runner
|
|
||||||
|
|
||||||
mkdir -p /run/hive-ci
|
|
||||||
chmod 700 /run/hive-ci
|
|
||||||
|
|
||||||
if [ -f "$STATE_FILE" ]; then
|
|
||||||
# Already registered — dummy token satisfies the module's path check.
|
|
||||||
echo "already-registered" > "$TOKEN_FILE"
|
|
||||||
chmod 600 "$TOKEN_FILE"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
CORE_TOKEN=$(cat /run/hive-ci/core-token)
|
|
||||||
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
|
||||||
|
|
||||||
# Retry up to 30s for forge to come up (containers autoStart in parallel).
|
|
||||||
for i in $(seq 1 30); do
|
|
||||||
REG_TOKEN=$(${pkgs.curl}/bin/curl -sf \
|
|
||||||
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
|
||||||
-H "Authorization: token $CORE_TOKEN" \
|
|
||||||
| ${pkgs.jq}/bin/jq -r .token) && break
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
|
|
||||||
echo "hive-ci: failed to fetch runner registration token from forge" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "$REG_TOKEN" > "$TOKEN_FILE"
|
|
||||||
chmod 600 "$TOKEN_FILE"
|
|
||||||
'';
|
|
||||||
in
|
|
||||||
{
|
|
||||||
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
||||||
# Shares host netns (same as hive-forge), so the runner reaches
|
|
||||||
# the forge at `http://127.0.0.1:<httpPort>` without extra plumbing.
|
|
||||||
# Container is non-ephemeral: the runner's registered credentials
|
|
||||||
# survive restarts (gitea-actions-runner writes them to its stateDir
|
|
||||||
# on first registration and reuses them on every subsequent start).
|
|
||||||
#
|
|
||||||
# Auto-registration: on first boot the container fetches a runner
|
|
||||||
# registration token from the forge's admin API using the core token
|
|
||||||
# hive-c0re writes to /var/lib/hyperhive/forge-core-token. No manual
|
|
||||||
# token handling needed — `ci.enable = true` is the full operator
|
|
||||||
# bootstrap. Registration flow: preStart calls the Forgejo admin API,
|
|
||||||
# writes the token to /run/hive-ci/runner-token, runner registers and
|
|
||||||
# persists credentials to stateDir — token file ignored on next boot.
|
|
||||||
#
|
|
||||||
# Nix builds inside the container use the shared /nix/store (standard
|
|
||||||
# nixos-container behaviour) with sandbox-fallback = true, because
|
|
||||||
# nspawn containers can't create the user-namespaces that nix sandboxing
|
|
||||||
# requires. See docs/gotchas.md.
|
|
||||||
|
|
||||||
options.services.hyperhive.ci = {
|
|
||||||
enable = lib.mkOption {
|
|
||||||
type = lib.types.bool;
|
|
||||||
default = false;
|
|
||||||
example = true;
|
|
||||||
description = ''
|
|
||||||
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
|
|
||||||
Disabled by default; requires `services.hyperhive.forge.enable = true`.
|
|
||||||
|
|
||||||
On first start the container auto-registers against hive-forge using
|
|
||||||
hive-c0re's admin token — no manual token provisioning needed.
|
|
||||||
Runner credentials are persisted in the container's state dir and
|
|
||||||
reused on every subsequent boot.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
name = lib.mkOption {
|
|
||||||
type = lib.types.str;
|
|
||||||
default = "hive-ci";
|
|
||||||
example = "prod-hive";
|
|
||||||
description = ''
|
|
||||||
Runner name as shown in the Forgejo admin panel. Defaults to
|
|
||||||
"hive-ci"; override when multiple hives share a Forgejo instance.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
concurrency = lib.mkOption {
|
|
||||||
type = lib.types.ints.positive;
|
|
||||||
default = 1;
|
|
||||||
example = 4;
|
|
||||||
description = ''
|
|
||||||
Maximum number of workflow jobs the runner executes in parallel.
|
|
||||||
Each job gets its own temporary working directory; multiple parallel
|
|
||||||
jobs share the container's nix store and cargo registry cache.
|
|
||||||
Higher values trade memory + CPU headroom for throughput.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
labels = lib.mkOption {
|
|
||||||
type = lib.types.listOf lib.types.str;
|
|
||||||
default = [ "hive-ci:host" ];
|
|
||||||
example = [
|
|
||||||
"hive-ci:host"
|
|
||||||
"nix:host"
|
|
||||||
];
|
|
||||||
description = ''
|
|
||||||
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
||||||
commands directly in the container (no docker/podman). Workflow
|
|
||||||
files target this runner with `runs-on: [hive-ci]`.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
package = lib.mkOption {
|
|
||||||
type = lib.types.package;
|
|
||||||
default = pkgs.gitea-actions-runner;
|
|
||||||
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
||||||
description = "gitea-actions-runner package.";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
config = lib.mkIf cfg.enable {
|
|
||||||
assertions = [
|
|
||||||
{
|
|
||||||
assertion = forgeCfg.enable;
|
|
||||||
message = ''
|
|
||||||
services.hyperhive.ci.enable = true requires
|
|
||||||
services.hyperhive.forge.enable = true — the runner registers
|
|
||||||
against the hive-forge Forgejo instance.
|
|
||||||
'';
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
containers.hive-ci = {
|
|
||||||
autoStart = true;
|
|
||||||
ephemeral = false;
|
|
||||||
# Shared host netns: runner reaches hive-forge at localhost.
|
|
||||||
privateNetwork = false;
|
|
||||||
|
|
||||||
bindMounts = {
|
|
||||||
# Core token — used by the auto-register script on first boot.
|
|
||||||
# Read-only: the container only reads it, never modifies it.
|
|
||||||
"/run/hive-ci/core-token" = {
|
|
||||||
hostPath = coreTokenPath;
|
|
||||||
isReadOnly = true;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
config =
|
|
||||||
{ pkgs, lib, ... }:
|
|
||||||
{
|
|
||||||
system.stateVersion = "26.05";
|
|
||||||
|
|
||||||
# nspawn containers can't create user-namespaces, so nix
|
|
||||||
# sandboxing always fails. Fall back to unsandboxed builds.
|
|
||||||
# See docs/gotchas.md.
|
|
||||||
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
||||||
nix.settings.experimental-features = [
|
|
||||||
"nix-command"
|
|
||||||
"flakes"
|
|
||||||
];
|
|
||||||
|
|
||||||
services.gitea-actions-runner.instances.hive = {
|
|
||||||
enable = true;
|
|
||||||
name = cfg.name;
|
|
||||||
url = "http://127.0.0.1:${toString forgeCfg.httpPort}";
|
|
||||||
# Token file is written by the ExecStartPre script below.
|
|
||||||
# On first boot: real registration token fetched from forge API.
|
|
||||||
# On subsequent boots: dummy value (runner uses .runner creds).
|
|
||||||
tokenFile = "/run/hive-ci/runner-token";
|
|
||||||
labels = cfg.labels;
|
|
||||||
settings = {
|
|
||||||
runner.capacity = cfg.concurrency;
|
|
||||||
# Generous timeout for cold-cache nix builds.
|
|
||||||
runner.timeout = "3h";
|
|
||||||
};
|
|
||||||
package = cfg.package;
|
|
||||||
};
|
|
||||||
|
|
||||||
# Prepend auto-register script before the runner service starts.
|
|
||||||
# `+` prefix runs with elevated privileges so it can read the
|
|
||||||
# bind-mounted core-token (owned by root on the host).
|
|
||||||
systemd.services."gitea-runner-hive".serviceConfig.ExecStartPre = lib.mkBefore [
|
|
||||||
"+${autoRegisterScript}"
|
|
||||||
];
|
|
||||||
|
|
||||||
# git is required by the runner's checkout step.
|
|
||||||
# Everything else (nix, rust tools) is either part of NixOS
|
|
||||||
# by default or pulled in hermetically by nix flake check.
|
|
||||||
# curl/jq in the autoregister preStart use absolute store paths.
|
|
||||||
environment.systemPackages = [ pkgs.git ];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
Loading…
Reference in a new issue