# The swarm's Forgejo Actions runner, in a `swarm-ci-runner` container on the # host that sets `deploy.ci-runner.enable`. It reaches the forge by name and # reads its registration token from the secret store under a certificate of # its own, so nothing here asks whether this host runs the forge or a hive. # # ⚠️ Two properties here are load-bearing and neither is visible from the # config that implements it: # # - The container gets its OWN network namespace, not the host's, so a PR's # nix build cannot reach host-loopback services. That is a # prompt-injection boundary; moving this to host netns for convenience # removes it. `docs/networking/network.md::Container isolation`. # - The forge ADMIN token never enters this container. swarm-controller holds # it and publishes only the registration token, which the runner's own store # identity reads. `docs/scheduler/ci.md::Registration`. # # Non-ephemeral on purpose: the runner writes its registered credentials # (`.runner`) to its state dir and reuses them, so an ephemeral container # would register a new runner on every restart. { pkgs, lib, config, ... }: let cfg = config.services.hyperhive.deploy.ci-runner; forgeCfg = config.services.hyperhive.swarm.forge; baoCfg = config.services.hyperhive.swarm.bao; baoDeploy = config.services.hyperhive.deploy.bao; gatewayCfg = config.services.hyperhive.gateway; networkCfg = config.services.hyperhive.network; tlsCfg = config.services.hyperhive.deploy.hive-controller.tls; machine = "swarm-ci-runner"; # The upstream module names its unit `gitea-runner-`. instance = "swarm"; runnerUnit = "gitea-runner-${instance}"; # Holds the registration token and registers; see its definition. registerUnit = "swarm-ci-runner-register"; # What the runner registers against and records in `.runner`. The precond # below compares the two, so both come from here. forgeUrl = "https://${forgeCfg.domain}"; # The forge's gateway serves a leaf from the store's `pki` mount in # self-signed mode. # The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store, # so it gets the assembled bundle; the Node actions it runs get # `NODE_EXTRA_CA_CERTS`, which is additive, further down. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; caBundleModule = caTrust.trustBundle { inherit pkgs; name = machine; consumers = [ runnerUnit registerUnit # Go: `SSL_CERT_FILE` replaces its trust store, for the swarm # collector's and authelia's names. "opentelemetry-collector" ]; }; # The container's journal forwarder pushes to its own receiver on the swarm # collector, the route `serviceProducers.swarm-ci-runner` in # ./swarm-otel-service.nix declares. The port suffix is elided on 443, as in # ./swarm-bao.nix. swarmOtelCfg = config.services.hyperhive.swarm.otel; forwarderClientId = config.services.hyperhive.swarm.ci-runner.otel.clientId; gatewayPortSuffix = lib.optionalString ( gatewayCfg.httpsPort != 443 ) ":${toString gatewayCfg.httpsPort}"; forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}"; forwarderTokenUrl = "${toString config.services.hyperhive.swarm.authelia.url}/api/oidc/token"; forwarderUnit = "swarm-journal-forwarder-oidc"; # Loopback in the container's own netns, and distinct from the other service # containers' forwarders all the same. forwarderTelemetryPort = 8895; # The runner's store identity, handed into the container as system # credentials (credential id → host source) for the forwarder's fetch unit. storeFiles = lib.filterAttrs (_: source: source != null) { ci-runner-bao-cert = baoDeploy.ciRunnerClientCertFile; ci-runner-bao-key = baoDeploy.ciRunnerClientKeyFile; ci-runner-bao-server-ca = baoDeploy.serverCaFile; }; # nixos-container expands `extraFlags` unquoted, so a source path must be # one word. oneWord = path: builtins.match ".*[[:space:]].*" path == null; forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}"; # The upstream unit is `DynamicUser`, so its state dir is # `/var/lib/private/gitea-runner`; `/var/lib/gitea-runner` is a symlink to # it, which is the spelling a step run as root (`+`) can follow. runnerFile = "/var/lib/gitea-runner/${instance}/.runner"; stateDir = "/var/lib/swarm-ci-runner"; # Root-only, on tmpfs: the token never reaches persistent disk, and after a # reboot the host unit below fetches it again. The host unit writes it at # `hostTokenFile`; the container sees the directory read-only at `tokenDir`, # where the token file is a `LoadCredential=` of the register unit alone. hostTokenDir = "/run/${machine}"; hostTokenFile = "${hostTokenDir}/registration-token"; tokenDir = "/run/${machine}"; tokenFile = "${tokenDir}/registration-token"; tokenCredential = "registration-token"; # The epoch second the daemon last started. The precond reads the daemon's # journal from there. daemonStartedFile = "${stateDir}/daemon-started"; # The nix half of `swarm_secret_client::ci::registration_token_path`, in the # form `bao kv get` takes (no KV v2 `data/`). storePath = "secret/swarm/services/ci-runner/registration-token"; # What Forgejo answers a runner it has no record of — deleted in the admin # panel, or a forge whose database was replaced — on every call but # `Register` (`routers/api/actions/runner/interceptor.go`). The daemon logs # it and exits 1 when its startup `Declare` gets it. ⚠️ Nothing this module # prints may contain it: the precond would read its own line as a rejection. rejectedMessage = "unregistered runner"; haveClientIdentity = baoDeploy.ciRunnerClientCertFile != null && baoDeploy.ciRunnerClientKeyFile != null; atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; refreshConsumer = import ./lib/refresh-consumer.nix { }; storeRetry = import ./lib/store-retry.nix { }; storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; privateNetwork = true; in { options.services.hyperhive.swarm.ci-runner.otel.clientId = lib.mkOption { type = lib.types.str; readOnly = true; default = "swarm-ci-runner-collector"; description = '' OAuth2 client id the journal forwarder inside the CI runner's container authenticates as, and the audience it asks its token for. Read-only because three hosts have to agree on it: authelia registers the client (`glue-ci-runner-otel-oidc-client.nix`), the swarm collector admits it on the container's receiver (`swarm.otel.serviceProducers`), and the secret publisher carries its secret to the store. ''; }; options.services.hyperhive.deploy.ci-runner = { # `enable` is in ./deploy.nix. name = lib.mkOption { type = lib.types.str; default = "swarm-ci-runner"; description = '' Runner name as shown in the Forgejo admin panel. ''; }; concurrency = lib.mkOption { type = lib.types.ints.positive; default = 1; example = 4; description = '' Maximum number of workflow jobs the runner executes in parallel. Each job gets its own temporary working directory; parallel jobs share the host's nix store and the container's cargo registry cache. Higher values trade memory and CPU headroom for throughput. ''; }; labels = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ "hive-ci:host" ]; example = [ "hive-ci:host" "nix:host" ]; description = '' Runner labels in `:` format. The `host` scheme runs commands directly in the container (no docker/podman). This repo's workflows target the default with `runs-on: hive-ci`. ''; }; package = lib.mkOption { type = lib.types.package; default = pkgs.gitea-actions-runner; defaultText = lib.literalExpression "pkgs.gitea-actions-runner"; description = "gitea-actions-runner package."; }; jobTimeout = lib.mkOption { type = lib.types.str; default = "1h"; example = "3h"; description = '' Per-job wall-clock timeout the runner enforces (act_runner's `runner.timeout`). A job that exceeds it is killed, so a hung build does not hold a runner slot indefinitely. Accepts a Go duration string (`30m`, `1h`, `2h30m`). Enforced by the runner process, so it only fires while that process is healthy. ''; }; }; config = lib.mkIf cfg.enable { assertions = [ { assertion = haveClientIdentity; message = '' services.hyperhive.deploy.ci-runner.enable requires the runner's swarm-secret-store client identity: set both services.hyperhive.deploy.bao.ciRunnerClientCertFile services.hyperhive.deploy.bao.ciRunnerClientKeyFile swarm-bao-ci-runner.service reads the forge's runner registration token with it, and the runner cannot register without that token. Both default to the pair glue-bao-tls.nix mints when services.hyperhive.deploy.bao.enable is true (the host that runs the store). Otherwise copy ci-runner.pem and ci-runner-key.pem from the store host's /var/lib/swarm-bao-pki/ and name them in those two options. ''; } { assertion = lib.all oneWord (lib.attrValues storeFiles); message = '' The CI runner's store identity paths must not contain whitespace: nspawn's --load-credential receives them through nixos-container's unquoted extra flags. ''; } ]; # The container hangs off the bridge, takes its address from the # gateway's DHCP and resolves through its dnsmasq. The gateway also runs # `hive-tls-ca`, which writes the trust bundle bound in below. services.hyperhive.network.enable = lib.mkDefault true; services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; # Shaped after ./swarm-grafana.nix's `swarm-bao-grafana-oidc`, except that # a missing token fails too: swarm-controller publishes it at its start and # every five minutes, so a retry is what turns "no value there" into one. systemd.services.swarm-bao-ci-runner = { description = "fetch the CI runner's registration token from the swarm secret store"; # Ordered after the store where it runs here: # ./glue-bao-readers-store-order.nix. before = [ "container@${machine}.service" ]; wantedBy = [ "multi-user.target" "container@${machine}.service" ]; path = [ baoDeploy.package pkgs.coreutils pkgs.systemd ]; inherit (storeRetry) startLimitBurst startLimitIntervalSec; serviceConfig = storeRetry.serviceConfig // { # No `RemainAfterExit`: the timer below starts this again, and # starting an active unit is a no-op. Type = "oneshot"; SyslogIdentifier = "swarm-bao-ci-runner"; TimeoutStartSec = 30; }; environment = { BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; BAO_CLIENT_CERT = baoDeploy.ciRunnerClientCertFile; BAO_CLIENT_KEY = baoDeploy.ciRunnerClientKeyFile; } // lib.optionalAttrs (baoDeploy.serverCaFile != null) { BAO_CACERT = baoDeploy.serverCaFile; }; script = '' set -euo pipefail ${atomicWriteSecret} ${refreshConsumer} # The container's bind source, so it starts even when the store does not # answer; tmpfiles creates it too, but unordered with the container on # a switch. Root-owned: systemd reads a `LoadCredential=` source as # root, before the runner drops to its dynamic user. install -d -m 0700 ${lib.escapeShellArg hostTokenDir} err="$(mktemp)" trap 'rm -f "$err"' EXIT # `-token-only` answers on stdout and needs no token helper. if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then echo "could not log in to swarm-bao with the CI runner's certificate; leaving its registration token as it is." >&2 if [ -s "$err" ]; then cat "$err" >&2 else echo "bao failed without writing a diagnostic." >&2 fi exit 1 fi export BAO_TOKEN if ! token="$(bao kv get -field=value ${lib.escapeShellArg storePath} 2>"$err")"; then echo "swarm-bao did not return ${storePath}; retrying until swarm-controller publishes it." >&2 if [ -s "$err" ]; then cat "$err" >&2 else echo "bao failed without writing a diagnostic." >&2 fi exit 1 fi if [ -z "$token" ]; then echo "swarm-bao returned an empty ${storePath}; retrying." >&2 exit 1 fi if secret_differs ${lib.escapeShellArg hostTokenFile} "$token"; then atomic_write_secret 0600 "" ${lib.escapeShellArg hostTokenFile} "$token" fi # `LoadCredential=` copies the file at start, so a runner still waiting # to register sees a new token only once it restarts. refresh_consumer ${lib.escapeShellArg machine} ${runnerUnit}.service ${lib.escapeShellArg hostTokenFile} ''; }; systemd.timers.swarm-bao-ci-runner = storeRefreshTimer "re-read the CI runner's registration token from the swarm secret store"; # nspawn fails to start the container when a credential source is # missing, so where this host mints the leaf the container pulls the # minting unit in and waits for it, as ./swarm-builder.nix's does. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; } ]; systemd.tmpfiles.rules = [ # The bind source must exist when the container starts. "d ${hostTokenDir} 0700 root root -" # `LoadCredential=` fails the unit when the file is missing; an empty # one lets the register unit report the missing token. "f ${hostTokenFile} 0600 root root -" ]; containers.${machine} = { autoStart = true; ephemeral = false; # No `--link-journal=host`: the journal stays in the guest, where the # container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix). extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles; inherit privateNetwork; hostBridge = networkCfg.bridgeName; bindMounts = { # Builds go through the HOST nix-daemon, so they inherit its # buildMachines and max-jobs. The directory rather than the socket # file, so the mount survives a daemon restart replacing the inode. "/nix/var/nix/daemon-socket" = { hostPath = "/nix/var/nix/daemon-socket"; isReadOnly = false; }; ${tokenDir} = { hostPath = hostTokenDir; isReadOnly = true; }; } // caTrust.bindMount; config = { pkgs, lib, config, ... }: let runnerCfg = config.services.gitea-actions-runner; instanceCfg = runnerCfg.instances.${instance}; # The same file upstream generates and hands the daemon. runnerConfigFile = (pkgs.formats.yaml { }).generate "config.yaml" instanceCfg.settings; # In the form the daemon rewrites `.runner`'s labels to on start # (act_runner's `labels.Parse` then `ToStrings`): `:` # with `host` for a missing scheme, then `:` if there is one. normaliseLabel = label: let parts = lib.splitString ":" label; scheme = if lib.length parts >= 2 then lib.elemAt parts 1 else "host"; arg = lib.concatStringsSep ":" (lib.drop 2 parts); in "${lib.head parts}:${scheme}" + lib.optionalString (arg != "") ":${arg}"; registeredLabels = lib.concatStringsSep "," ( lib.sort lib.lessThan (map normaliseLabel instanceCfg.labels) ); in { imports = [ ../container-modules/swarm-container.nix ../container-modules/swarm-journal-forwarder.nix caBundleModule ]; # Authenticated as the forwarder's own OIDC client, with the runner's # store identity: one identity for the whole container. It is a # system credential, so the fetch unit loads it. services.hyperhive.swarmJournalForwarder = { enable = true; serviceName = "ci-runner"; endpoint = forwarderEndpoint; clientId = forwarderClientId; tokenUrl = forwarderTokenUrl; telemetryPort = forwarderTelemetryPort; store = { package = baoDeploy.package; addr = "https://${baoCfg.domain}:${toString baoCfg.port}"; secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client"; clientCertFile = forwarderCredential "ci-runner-bao-cert"; clientKeyFile = forwarderCredential "ci-runner-bao-key"; caFile = if storeFiles ? ci-runner-bao-server-ca then forwarderCredential "ci-runner-bao-server-ca" else null; }; }; systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles; # Its own netns, so it keeps its own firewall; resolvconf stays on # and serves `networking.nameservers` below. services.hyperhive.swarmContainer = { inherit privateNetwork; writesOwnResolvConf = false; }; # The gateway's dnsmasq on the bridge answers the forge's name with # the bridge IP where this host runs the forge, and forwards it # upstream everywhere else. networking.nameservers = [ networkCfg.bridgeIp ]; networking.interfaces.eth0.useDHCP = true; # nspawn containers can't create user namespaces, so nix sandboxing # fails; see docs/process/gotchas.md. `fallback` is a client-side # option sent per daemon connection, so the host's value does not # reach builds started here. nix.settings.sandbox-fallback = lib.mkForce true; nix.settings.fallback = true; nix.settings.experimental-features = [ "nix-command" "flakes" ]; # Both the service and its socket are off: the socket unit owns # `/nix/var/nix/daemon-socket/socket` and would race the bind mount. systemd.globalEnvironment.NIX_REMOTE = "daemon"; systemd.services.nix-daemon.enable = false; systemd.sockets.nix-daemon.enable = false; systemd.tmpfiles.rules = [ "d ${stateDir} 0700 root root -" # Unused: nothing in this module writes or reads them. The first # two hold a token; the others are upstream's re-registration # markers. "r ${stateDir}/registration-token" "r ${stateDir}/registration-token.env" "r /var/lib/private/gitea-runner/${instance}/.token-hash" "r /var/lib/private/gitea-runner/${instance}/.labels" ]; services.gitea-actions-runner.package = cfg.package; services.gitea-actions-runner.instances.${instance} = { enable = true; inherit (cfg) name labels; url = forgeUrl; # Upstream refuses an instance without a token source, and turns # this one into the `EnvironmentFile=` the daemon unit drops. inherit tokenFile; settings = { runner.capacity = cfg.concurrency; runner.timeout = cfg.jobTimeout; }; }; environment.systemPackages = [ pkgs.git pkgs.nix ]; # The only unit that can read the registration token: systemd hands # a credential to every process of the unit that loads it, and the # daemon's processes include every host job. Leaves a usable # `.runner` behind or fails. # # The daemon's dynamic user and state dir: units naming the same # `DynamicUser` share its uid, so `.runner` is the daemon's own. systemd.services.${registerUnit} = { description = "register the CI runner with the forge when it has no .runner"; wants = [ "network-online.target" ]; after = [ "network-online.target" ]; environment = { HOME = "/var/lib/gitea-runner/${instance}"; # In the form the daemon rewrites `.runner`'s labels to. SWARM_CI_RUNNER_LABELS = registeredLabels; }; serviceConfig = { Type = "oneshot"; DynamicUser = true; User = "gitea-runner"; StateDirectory = "gitea-runner"; LoadCredential = [ "${tokenCredential}:${tokenFile}" ]; # `register` retries an unreachable forge without end; under the # daemon's own start timeout, so the failure is this unit's. TimeoutStartSec = 60; # Removes `.runner` in the cases where reusing it would fail, so # the register step registers again. # # `+`: root, because the system journal is out of the dynamic # user's reach. ExecStartPre = "+${pkgs.writeShellScript "swarm-ci-runner-precond" '' set -eu runner=${runnerFile} started=${daemonStartedFile} # The forge refused the registration the daemon last started # with. Read on the next start rather than when the daemon # exits, so journald has its last lines by then. if [ -f "$started" ]; then since="$(< "$started")" ${pkgs.coreutils}/bin/rm -f "$started" if ${pkgs.systemd}/bin/journalctl -u ${runnerUnit}.service --since "@$since" -o cat --no-pager \ | ${pkgs.gnugrep}/bin/grep -qF ${lib.escapeShellArg rejectedMessage}; then echo "the forge has no record of this runner's registration; registering again" >&2 ${pkgs.coreutils}/bin/rm -f "$runner" fi fi # `.runner` records the address it registered at, and the # daemon dials that one, not the configured one. if [ -f "$runner" ] \ && [ "$(${pkgs.jq}/bin/jq -r .address "$runner")" != ${lib.escapeShellArg forgeUrl} ]; then echo "$runner names another forge address; registering again at ${forgeUrl}" >&2 ${pkgs.coreutils}/bin/rm -f "$runner" fi # The daemon declares the labels `.runner` records, not the # configured ones. if [ -f "$runner" ] \ && [ "$(${pkgs.jq}/bin/jq -r '.labels | sort | join(",")' "$runner")" != "$SWARM_CI_RUNNER_LABELS" ]; then echo "$runner names other labels; registering again with $SWARM_CI_RUNNER_LABELS" >&2 ${pkgs.coreutils}/bin/rm -f "$runner" fi ''}"; }; script = '' set -u dir="$STATE_DIRECTORY/${instance}" mkdir -p "$dir" cd "$dir" if [ -e .runner ]; then exit 0 fi token="$(< "$CREDENTIALS_DIRECTORY/${tokenCredential}")" if [ -z "$token" ]; then echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2 exit 1 fi exec ${lib.getExe runnerCfg.package} register --no-interactive \ --instance ${lib.escapeShellArg instanceCfg.url} \ --token "$token" \ --name ${lib.escapeShellArg instanceCfg.name} \ --labels ${lib.escapeShellArg (lib.concatStringsSep "," instanceCfg.labels)} \ --config ${runnerConfigFile} ''; }; systemd.services.${runnerUnit} = { # `path` adds to the PATH the upstream module builds instead of # replacing it. path = [ pkgs.nix ]; # The upstream unit is `DynamicUser`, whose state dir systemd mounts # `noexec`, so a job could not execute what it builds in its # workspace. `ExecPaths=` resolves against the host root, where # `/var/lib/gitea-runner` is a symlink into `private/`; `-` makes # whichever spelling is absent a no-op. serviceConfig.ExecPaths = [ "-/var/lib/gitea-runner" "-/var/lib/private/gitea-runner" ]; environment = lib.mkIf caTrust.useSelfSigned { NODE_EXTRA_CA_CERTS = caTrust.caContainerPath; }; # A host job inherits the daemon's whole environment and can read # its credentials, so the daemon carries the token in neither. serviceConfig.EnvironmentFile = lib.mkForce [ ]; # Replaces upstream's pre-start step, which registers from # `$TOKEN`, so the register unit is the one registration path. # # Started from here rather than through `Requires=`: a failed # requirement fails the start job, which `Restart=` does not retry, # while a failed pre-start step does. serviceConfig.ExecStartPre = lib.mkForce [ "+${pkgs.systemd}/bin/systemctl start ${registerUnit}.service" "+${pkgs.writeShellScript "swarm-ci-runner-mark-start" '' ${pkgs.coreutils}/bin/date +%s > ${daemonStartedFile} ''}" ]; # Restarted until the token arrives, however long the store and # the controller take. `mkForce`: upstream sets both. serviceConfig.Restart = lib.mkForce "on-failure"; serviceConfig.RestartSec = lib.mkForce 15; startLimitIntervalSec = 0; }; }; }; }; }