hyperhive/nix/host-modules/swarm-ci-metrics.nix

# The swarm's CI run poller: a oneshot `swarm-ci-metrics` on a 5-minute timer,
# on the host that sets `deploy.ci-metrics.enable`, which
# `deploy.allSwarmServices` defaults on where the host has the poller's store
# identity (./swarm-required-services.nix).
#
# Each pass lists the Actions runs of every repo the `ci-metrics` forge account
# sees and pushes each newly finished run's duration and queue wait to the
# swarm collector, stamped at the run's end. The first pass, with no cursor in
# the state directory, walks every run back to the first. docs/swarm/ci-metrics.md
# has the design.
#
# It reaches the forge, authelia, the store and the collector by name, so
# nothing here asks which host runs any of them. Its one credential on disk is
# its store leaf, handed in by `LoadCredential=`; the forge token and its OIDC
# client secret it reads from the store under that leaf on every pass.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  hyperhiveCfg = config.services.hyperhive;
  cfg = hyperhiveCfg.deploy.ci-metrics;
  baoCfg = hyperhiveCfg.swarm.bao;
  baoDeploy = hyperhiveCfg.deploy.bao;
  gatewayCfg = hyperhiveCfg.gateway;
  swarmOtelCfg = hyperhiveCfg.swarm.otel;

  unit = "swarm-ci-metrics";
  clientId = hyperhiveCfg.swarm.ci-metrics.clientId;

  # The route `serviceProducers.swarm-ci-metrics` in ./swarm-otel-service.nix
  # declares, by the collector's gateway name. The port suffix is elided on
  # 443, as in ./swarm-builder.nix.
  gatewayPortSuffix = lib.optionalString (
    gatewayCfg.httpsPort != 443
  ) ":${toString gatewayCfg.httpsPort}";
  otlpEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${unit}";

  haveClientIdentity =
    baoDeploy.ciMetricsClientCertFile != null && baoDeploy.ciMetricsClientKeyFile != null;

  # The forge, authelia and the collector serve leaves issued by the swarm's
  # own CA, which the system store lacks, and the poller's `reqwest`/`rustls`
  # roots come from `SSL_CERT_FILE`, which replaces that store. `trustBundle`
  # below sets it to the system CAs plus the hive anchor; without it every
  # pass fails `invalid peer certificate: UnknownIssuer`.
  caTrust = import ./lib/hive-ca-trust.nix {
    inherit lib gatewayCfg;
    tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
  };

  credential = id: "/run/credentials/${unit}.service/${id}";
  # Credential id → host path of the poller's store identity.
  storeFiles = {
    "bao-client.pem" = baoDeploy.ciMetricsClientCertFile;
    "bao-client-key.pem" = baoDeploy.ciMetricsClientKeyFile;
  }
  // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
    "bao-ca.pem" = baoDeploy.serverCaFile;
  };
in
{
  # `hostUnit`: a host service, so the bundle reads the host CA path and waits
  # on `hive-tls-ca.service`. `enable`: `imports` is unconditional, so without
  # it a host with the poller off gets the bundle unit and an `ExecStart`-less
  # `swarm-ci-metrics` service.
  imports = [
    (caTrust.trustBundle {
      inherit pkgs;
      name = unit;
      consumers = [ unit ];
      hostUnit = true;
      inherit (cfg) enable;
    })
  ];

  options.services.hyperhive.swarm.ci-metrics.clientId = lib.mkOption {
    type = lib.types.str;
    readOnly = true;
    default = "swarm-ci-metrics";
    description = ''
      OAuth2 client id the CI run poller authenticates to the swarm
      collector as, and the audience it asks its token for.

      Read-only because three hosts have to agree on it: authelia registers
      the client (`glue-ci-metrics-otel-oidc-client.nix`), the swarm collector
      admits it on the poller's receiver (`swarm.otel.serviceProducers`), and
      the secret publisher carries its secret to the store.
    '';
  };

  options.services.hyperhive.deploy.ci-metrics = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Run the CI run poller on this host: every 5 minutes it pushes each
        newly finished forge Actions run's duration and queue wait to the
        swarm collector, for the Grafana CI dashboard. Its first run pushes
        every run since the first.

        Requires the poller's store identity,
        {option}`services.hyperhive.deploy.bao.ciMetricsClientCertFile` and
        `ciMetricsClientKeyFile`. Run it on one host per swarm: two pollers
        keep two cursors and push every run twice.
      '';
    };

    package = lib.mkOption {
      type = lib.types.package;
      defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-ci-metrics";
      description = "The CI run poller package.";
    };
  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = haveClientIdentity;
        message = ''
          services.hyperhive.deploy.ci-metrics.enable requires the poller's
          swarm-secret-store client identity: set both

            services.hyperhive.deploy.bao.ciMetricsClientCertFile
            services.hyperhive.deploy.bao.ciMetricsClientKeyFile

          swarm-ci-metrics.service reads its forge token and its OIDC client
          secret with it.

          Both default to the pair glue-bao-tls.nix mints when
          services.hyperhive.deploy.bao.enable is true (the host that runs
          the store). Otherwise copy ci-metrics.pem and ci-metrics-key.pem
          from the store host's /var/lib/swarm-bao-pki/ and name them in
          those two options.
        '';
      }
    ];

    systemd.services.${unit} = {
      description = "push finished forge CI runs to the swarm metrics store";
      after = [ "network-online.target" ];
      wants = [ "network-online.target" ];
      # No `RemainAfterExit`: the timer starts this again, and starting an
      # active unit is a no-op.
      serviceConfig = {
        Type = "oneshot";
        ExecStart = "${cfg.package}/bin/swarm-ci-metrics";
        # The first pass reads every run the forge has, a page a second.
        TimeoutStartSec = "30min";
        # Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
        LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") storeFiles;
        DynamicUser = true;
        # The cursor. Losing it re-pushes every run.
        StateDirectory = unit;
        StateDirectoryMode = "0700";
        SyslogIdentifier = unit;
        ProtectSystem = "strict";
        ProtectHome = true;
        PrivateTmp = true;
        PrivateDevices = true;
        NoNewPrivileges = true;
        RestrictSUIDSGID = true;
        RemoveIPC = true;
      };
      environment = {
        SWARM_CI_METRICS_FORGE_URL = "https://${hyperhiveCfg.swarm.forge.domain}";
        SWARM_CI_METRICS_OTLP_ENDPOINT = otlpEndpoint;
        SWARM_CI_METRICS_OIDC_TOKEN_ENDPOINT = "${hyperhiveCfg.swarm.authelia.url}/api/oidc/token";
        SWARM_CI_METRICS_OIDC_CLIENT_ID = clientId;
        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
        BAO_CLIENT_CERT = credential "bao-client.pem";
        BAO_CLIENT_KEY = credential "bao-client-key.pem";
      }
      // lib.optionalAttrs (storeFiles ? "bao-ca.pem") {
        BAO_CACERT = credential "bao-ca.pem";
      };
    };

    systemd.timers.${unit} = {
      description = "run the CI run poller every 5 minutes";
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnBootSec = "2min";
        OnUnitInactiveSec = "5min";
        RandomizedDelaySec = "30s";
      };
    };
  };
}