# The swarm's Forgejo Actions runner, in a `swarm-ci-runner` container on the
# host that sets `deploy.ci-runner.enable`. It reaches the forge by name and
# reads its registration token from the secret store under a certificate of
# its own, so nothing here asks whether this host runs the forge or a hive.
#
# ⚠️ Two properties here are load-bearing and neither is visible from the
# config that implements it:
#
# - The container gets its OWN network namespace, not the host's, so a PR's
#   nix build cannot reach host-loopback services. That is a
#   prompt-injection boundary; moving this to host netns for convenience
#   removes it. `docs/networking/network.md::Container isolation`.
# - The forge ADMIN token never enters this container. swarm-controller holds
#   it and publishes only the registration token, which the runner's own store
#   identity reads. `docs/scheduler/ci.md::Registration`.
#
# Non-ephemeral on purpose: the runner writes its registered credentials
# (`.runner`) to its state dir and reuses them, so an ephemeral container
# would register a new runner on every restart.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.deploy.ci-runner;
  forgeCfg = config.services.hyperhive.swarm.forge;
  baoCfg = config.services.hyperhive.swarm.bao;
  baoDeploy = config.services.hyperhive.deploy.bao;
  gatewayCfg = config.services.hyperhive.gateway;
  networkCfg = config.services.hyperhive.network;
  tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;

  machine = "swarm-ci-runner";
  # The upstream module names its unit `gitea-runner-<instance>`.
  instance = "swarm";
  runnerUnit = "gitea-runner-${instance}";
  # Holds the registration token and registers; see its definition.
  registerUnit = "swarm-ci-runner-register";

  # What the runner registers against and records in `.runner`. The precond
  # below compares the two, so both come from here.
  forgeUrl = "https://${forgeCfg.domain}";

  # The forge's gateway serves a leaf from the store's `pki` mount in
  # self-signed mode.
  # The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store,
  # so it gets the assembled bundle; the Node actions it runs get
  # `NODE_EXTRA_CA_CERTS`, which is additive, further down.
  caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
  caBundleModule = caTrust.trustBundle {
    inherit pkgs;
    name = machine;
    consumers = [
      runnerUnit
      registerUnit
    ];
  };

  # The upstream unit is `DynamicUser`, so its state dir is
  # `/var/lib/private/gitea-runner`; `/var/lib/gitea-runner` is a symlink to
  # it, which is the spelling a step run as root (`+`) can follow.
  runnerFile = "/var/lib/gitea-runner/${instance}/.runner";

  # Inside the container, root-only. The token file holds the bare token and
  # is a `LoadCredential=` of the register unit alone; the host unit below
  # writes it through the container's root, at `hostTokenFile`.
  stateDir = "/var/lib/swarm-ci-runner";
  tokenFile = "${stateDir}/registration-token";
  tokenCredential = "registration-token";
  hostTokenFile = "/var/lib/nixos-containers/${machine}${tokenFile}";
  # The epoch second the daemon last started. The precond reads the daemon's
  # journal from there.
  daemonStartedFile = "${stateDir}/daemon-started";

  # The nix half of `swarm_secret_client::ci::registration_token_path`, in the
  # form `bao kv get` takes (no KV v2 `data/`).
  storePath = "secret/swarm/services/ci-runner/registration-token";

  # What Forgejo answers a runner it has no record of — deleted in the admin
  # panel, or a forge whose database was replaced — on every call but
  # `Register` (`routers/api/actions/runner/interceptor.go`). The daemon logs
  # it and exits 1 when its startup `Declare` gets it. ⚠️ Nothing this module
  # prints may contain it: the precond would read its own line as a rejection.
  rejectedMessage = "unregistered runner";

  haveClientIdentity =
    baoDeploy.ciRunnerClientCertFile != null && baoDeploy.ciRunnerClientKeyFile != null;

  atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
  refreshConsumer = import ./lib/refresh-consumer.nix { };
  storeRetry = import ./lib/store-retry.nix { };
  storeRefreshTimer = import ./lib/store-refresh-timer.nix { };

  privateNetwork = true;
in
{
  options.services.hyperhive.deploy.ci-runner = {
    # `enable` is in ./deploy.nix.

    name = lib.mkOption {
      type = lib.types.str;
      default = "swarm-ci-runner";
      description = ''
        Runner name as shown in the Forgejo admin panel.
      '';
    };

    concurrency = lib.mkOption {
      type = lib.types.ints.positive;
      default = 1;
      example = 4;
      description = ''
        Maximum number of workflow jobs the runner executes in parallel.
        Each job gets its own temporary working directory; parallel jobs
        share the host's nix store and the container's cargo registry
        cache. Higher values trade memory and CPU headroom for throughput.
      '';
    };

    labels = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [ "hive-ci:host" ];
      example = [
        "hive-ci:host"
        "nix:host"
      ];
      description = ''
        Runner labels in `<name>:<scheme>` format. The `host` scheme runs
        commands directly in the container (no docker/podman). This repo's
        workflows target the default with `runs-on: hive-ci`.
      '';
    };

    package = lib.mkOption {
      type = lib.types.package;
      default = pkgs.gitea-actions-runner;
      defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
      description = "gitea-actions-runner package.";
    };

    jobTimeout = lib.mkOption {
      type = lib.types.str;
      default = "1h";
      example = "3h";
      description = ''
        Per-job wall-clock timeout the runner enforces (act_runner's
        `runner.timeout`). A job that exceeds it is killed, so a hung build
        does not hold a runner slot indefinitely. Accepts a Go duration
        string (`30m`, `1h`, `2h30m`). Enforced by the runner process, so
        it only fires while that process is healthy.
      '';
    };
  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = haveClientIdentity;
        message = ''
          services.hyperhive.deploy.ci-runner.enable requires the runner's
          swarm-secret-store client identity: set both

            services.hyperhive.deploy.bao.ciRunnerClientCertFile
            services.hyperhive.deploy.bao.ciRunnerClientKeyFile

          swarm-bao-ci-runner.service reads the forge's runner registration
          token with it, and the runner cannot register without that token.

          Both default to the pair glue-bao-tls.nix mints when
          services.hyperhive.deploy.bao.enable is true (the host that runs
          the store). Otherwise copy ci-runner.pem and ci-runner-key.pem from
          the store host's /var/lib/swarm-bao-pki/ and name them in those two
          options.
        '';
      }
    ];

    # The container hangs off the bridge, takes its address from the
    # gateway's DHCP and resolves through its dnsmasq. The gateway also runs
    # `hive-tls-ca`, which writes the trust bundle bound in below.
    services.hyperhive.network.enable = lib.mkDefault true;
    services.hyperhive.gateway.enable = lib.mkDefault true;
    services.hyperhive.gateway.dns.enable = lib.mkDefault true;

    # Shaped after ./swarm-grafana.nix's `swarm-bao-grafana-oidc`, except that
    # a missing token fails too: swarm-controller publishes it at its start and
    # every five minutes, so a retry is what turns "no value there" into one.
    systemd.services.swarm-bao-ci-runner = {
      description = "fetch the CI runner's registration token from the swarm secret store";
      # Ordered after the store where it runs here:
      # ./glue-bao-readers-store-order.nix.
      before = [ "container@${machine}.service" ];
      wantedBy = [
        "multi-user.target"
        "container@${machine}.service"
      ];
      path = [
        baoDeploy.package
        pkgs.coreutils
        pkgs.systemd
      ];
      inherit (storeRetry) startLimitBurst startLimitIntervalSec;
      serviceConfig = storeRetry.serviceConfig // {
        # No `RemainAfterExit`: the timer below starts this again, and
        # starting an active unit is a no-op.
        Type = "oneshot";
        SyslogIdentifier = "swarm-bao-ci-runner";
        TimeoutStartSec = 30;
      };
      environment = {
        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
        BAO_CLIENT_CERT = baoDeploy.ciRunnerClientCertFile;
        BAO_CLIENT_KEY = baoDeploy.ciRunnerClientKeyFile;
      }
      // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
        BAO_CACERT = baoDeploy.serverCaFile;
      };
      script = ''
        set -euo pipefail

        ${atomicWriteSecret}
        ${refreshConsumer}

        err="$(mktemp)"
        trap 'rm -f "$err"' EXIT

        # `-token-only` answers on stdout and needs no token helper.
        if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
          echo "could not log in to swarm-bao with the CI runner's certificate; leaving its registration token as it is." >&2
          if [ -s "$err" ]; then
            cat "$err" >&2
          else
            echo "bao failed without writing a diagnostic." >&2
          fi
          exit 1
        fi
        export BAO_TOKEN

        if ! token="$(bao kv get -field=value ${lib.escapeShellArg storePath} 2>"$err")"; then
          echo "swarm-bao did not return ${storePath}; retrying until swarm-controller publishes it." >&2
          if [ -s "$err" ]; then
            cat "$err" >&2
          else
            echo "bao failed without writing a diagnostic." >&2
          fi
          exit 1
        fi

        if [ -z "$token" ]; then
          echo "swarm-bao returned an empty ${storePath}; retrying." >&2
          exit 1
        fi

        # Root-owned: systemd reads a `LoadCredential=` source as root, before
        # the runner drops to its dynamic user.
        install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostTokenFile)}
        if secret_differs ${lib.escapeShellArg hostTokenFile} "$token"; then
          atomic_write_secret 0600 "" ${lib.escapeShellArg hostTokenFile} "$token"
        fi

        # `LoadCredential=` copies the file at start, so a runner still waiting
        # to register sees a new token only once it restarts.
        refresh_consumer ${lib.escapeShellArg machine} ${runnerUnit}.service ${lib.escapeShellArg hostTokenFile}
      '';
    };

    systemd.timers.swarm-bao-ci-runner = storeRefreshTimer "re-read the CI runner's registration token from the swarm secret store";

    systemd.services."container@${machine}" = caTrust.containerOrdering;

    containers.${machine} = {
      autoStart = true;
      ephemeral = false;
      # Journal files on the host, not inside the container: nixpkgs hardcodes
      # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
      extraFlags = [ "--link-journal=host" ];
      inherit privateNetwork;
      hostBridge = networkCfg.bridgeName;

      bindMounts = {
        # Builds go through the HOST nix-daemon, so they inherit its
        # buildMachines and max-jobs. The directory rather than the socket
        # file, so the mount survives a daemon restart replacing the inode.
        "/nix/var/nix/daemon-socket" = {
          hostPath = "/nix/var/nix/daemon-socket";
          isReadOnly = false;
        };
      }
      // caTrust.bindMount;

      config =
        {
          pkgs,
          lib,
          config,
          ...
        }:
        let
          runnerCfg = config.services.gitea-actions-runner;
          instanceCfg = runnerCfg.instances.${instance};
          # The same file upstream generates and hands the daemon.
          runnerConfigFile = (pkgs.formats.yaml { }).generate "config.yaml" instanceCfg.settings;
          # In the form the daemon rewrites `.runner`'s labels to on start
          # (act_runner's `labels.Parse` then `ToStrings`): `<name>:<scheme>`
          # with `host` for a missing scheme, then `:<arg>` if there is one.
          normaliseLabel =
            label:
            let
              parts = lib.splitString ":" label;
              scheme = if lib.length parts >= 2 then lib.elemAt parts 1 else "host";
              arg = lib.concatStringsSep ":" (lib.drop 2 parts);
            in
            "${lib.head parts}:${scheme}" + lib.optionalString (arg != "") ":${arg}";
          registeredLabels = lib.concatStringsSep "," (
            lib.sort lib.lessThan (map normaliseLabel instanceCfg.labels)
          );
        in
        {
          imports = [
            ../container-modules/swarm-container.nix
            caBundleModule
          ];

          # Its own netns, so it keeps its own firewall; resolvconf stays on
          # and serves `networking.nameservers` below.
          services.hyperhive.swarmContainer = {
            inherit privateNetwork;
            writesOwnResolvConf = false;
          };

          # The gateway's dnsmasq on the bridge answers the forge's name with
          # the bridge IP where this host runs the forge, and forwards it
          # upstream everywhere else.
          networking.nameservers = [ networkCfg.bridgeIp ];
          networking.interfaces.eth0.useDHCP = true;

          # nspawn containers can't create user namespaces, so nix sandboxing
          # fails; see docs/process/gotchas.md. `fallback` is a client-side
          # option sent per daemon connection, so the host's value does not
          # reach builds started here.
          nix.settings.sandbox-fallback = lib.mkForce true;
          nix.settings.fallback = true;
          nix.settings.experimental-features = [
            "nix-command"
            "flakes"
          ];

          # Both the service and its socket are off: the socket unit owns
          # `/nix/var/nix/daemon-socket/socket` and would race the bind mount.
          systemd.globalEnvironment.NIX_REMOTE = "daemon";
          systemd.services.nix-daemon.enable = false;
          systemd.sockets.nix-daemon.enable = false;

          systemd.tmpfiles.rules = [
            "d ${stateDir} 0700 root root -"
            # `LoadCredential=` fails the unit when the file is missing; an
            # empty one lets the register unit report the missing token.
            "f ${tokenFile} 0600 root root -"
            # Unused: nothing in this module writes or reads them. The first
            # holds a token; the others are upstream's re-registration markers.
            "r ${stateDir}/registration-token.env"
            "r /var/lib/private/gitea-runner/${instance}/.token-hash"
            "r /var/lib/private/gitea-runner/${instance}/.labels"
          ];

          services.gitea-actions-runner.package = cfg.package;

          services.gitea-actions-runner.instances.${instance} = {
            enable = true;
            inherit (cfg) name labels;
            url = forgeUrl;
            # Upstream refuses an instance without a token source, and turns
            # this one into the `EnvironmentFile=` the daemon unit drops.
            inherit tokenFile;
            settings = {
              runner.capacity = cfg.concurrency;
              runner.timeout = cfg.jobTimeout;
            };
          };

          environment.systemPackages = [
            pkgs.git
            pkgs.nix
          ];

          # The only unit that can read the registration token: systemd hands
          # a credential to every process of the unit that loads it, and the
          # daemon's processes include every host job. Leaves a usable
          # `.runner` behind or fails.
          #
          # The daemon's dynamic user and state dir: units naming the same
          # `DynamicUser` share its uid, so `.runner` is the daemon's own.
          systemd.services.${registerUnit} = {
            description = "register the CI runner with the forge when it has no .runner";
            wants = [ "network-online.target" ];
            after = [ "network-online.target" ];
            environment = {
              HOME = "/var/lib/gitea-runner/${instance}";
              # In the form the daemon rewrites `.runner`'s labels to.
              SWARM_CI_RUNNER_LABELS = registeredLabels;
            };
            serviceConfig = {
              Type = "oneshot";
              DynamicUser = true;
              User = "gitea-runner";
              StateDirectory = "gitea-runner";
              LoadCredential = [ "${tokenCredential}:${tokenFile}" ];
              # `register` retries an unreachable forge without end; under the
              # daemon's own start timeout, so the failure is this unit's.
              TimeoutStartSec = 60;
              # Removes `.runner` in the cases where reusing it would fail, so
              # the register step registers again.
              #
              # `+`: root, because the system journal is out of the dynamic
              # user's reach.
              ExecStartPre = "+${pkgs.writeShellScript "swarm-ci-runner-precond" ''
                set -eu
                runner=${runnerFile}
                started=${daemonStartedFile}

                # The forge refused the registration the daemon last started
                # with. Read on the next start rather than when the daemon
                # exits, so journald has its last lines by then.
                if [ -f "$started" ]; then
                  since="$(< "$started")"
                  ${pkgs.coreutils}/bin/rm -f "$started"
                  if ${pkgs.systemd}/bin/journalctl -u ${runnerUnit}.service --since "@$since" -o cat --no-pager \
                    | ${pkgs.gnugrep}/bin/grep -qF ${lib.escapeShellArg rejectedMessage}; then
                    echo "the forge has no record of this runner's registration; registering again" >&2
                    ${pkgs.coreutils}/bin/rm -f "$runner"
                  fi
                fi

                # `.runner` records the address it registered at, and the
                # daemon dials that one, not the configured one.
                if [ -f "$runner" ] \
                  && [ "$(${pkgs.jq}/bin/jq -r .address "$runner")" != ${lib.escapeShellArg forgeUrl} ]; then
                  echo "$runner names another forge address; registering again at ${forgeUrl}" >&2
                  ${pkgs.coreutils}/bin/rm -f "$runner"
                fi

                # The daemon declares the labels `.runner` records, not the
                # configured ones.
                if [ -f "$runner" ] \
                  && [ "$(${pkgs.jq}/bin/jq -r '.labels | sort | join(",")' "$runner")" != "$SWARM_CI_RUNNER_LABELS" ]; then
                  echo "$runner names other labels; registering again with $SWARM_CI_RUNNER_LABELS" >&2
                  ${pkgs.coreutils}/bin/rm -f "$runner"
                fi
              ''}";
            };
            script = ''
              set -u
              dir="$STATE_DIRECTORY/${instance}"
              mkdir -p "$dir"
              cd "$dir"
              if [ -e .runner ]; then
                exit 0
              fi

              token="$(< "$CREDENTIALS_DIRECTORY/${tokenCredential}")"
              if [ -z "$token" ]; then
                echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2
                exit 1
              fi
              exec ${lib.getExe runnerCfg.package} register --no-interactive \
                --instance ${lib.escapeShellArg instanceCfg.url} \
                --token "$token" \
                --name ${lib.escapeShellArg instanceCfg.name} \
                --labels ${lib.escapeShellArg (lib.concatStringsSep "," instanceCfg.labels)} \
                --config ${runnerConfigFile}
            '';
          };

          systemd.services.${runnerUnit} = {
            # `path` adds to the PATH the upstream module builds instead of
            # replacing it.
            path = [ pkgs.nix ];
            # The upstream unit is `DynamicUser`, whose state dir systemd mounts
            # `noexec`, so a job could not execute what it builds in its
            # workspace. `ExecPaths=` resolves against the host root, where
            # `/var/lib/gitea-runner` is a symlink into `private/`; `-` makes
            # whichever spelling is absent a no-op.
            serviceConfig.ExecPaths = [
              "-/var/lib/gitea-runner"
              "-/var/lib/private/gitea-runner"
            ];
            environment = lib.mkIf caTrust.useSelfSigned {
              NODE_EXTRA_CA_CERTS = caTrust.caContainerPath;
            };
            # A host job inherits the daemon's whole environment and can read
            # its credentials, so the daemon carries the token in neither.
            serviceConfig.EnvironmentFile = lib.mkForce [ ];
            # Replaces upstream's pre-start step, which registers from
            # `$TOKEN`, so the register unit is the one registration path.
            #
            # Started from here rather than through `Requires=`: a failed
            # requirement fails the start job, which `Restart=` does not retry,
            # while a failed pre-start step does.
            serviceConfig.ExecStartPre = lib.mkForce [
              "+${pkgs.systemd}/bin/systemctl start ${registerUnit}.service"
              "+${pkgs.writeShellScript "swarm-ci-runner-mark-start" ''
                ${pkgs.coreutils}/bin/date +%s > ${daemonStartedFile}
              ''}"
            ];
            # Restarted until the token arrives, however long the store and
            # the controller take. `mkForce`: upstream sets both.
            serviceConfig.Restart = lib.mkForce "on-failure";
            serviceConfig.RestartSec = lib.mkForce 15;
            startLimitIntervalSec = 0;
          };
        };
    };
  };
}