hyperhive/nix/host-modules/swarm-builder.nix

# The swarm builder, in a `swarm-builder` container on the host that sets
# `deploy.builder.enable`, which `deploy.allSwarmServices` defaults on where the
# host has the builder's store identity (./swarm-required-services.nix). It runs
# unreviewed config PRs through the host's nix daemon. It builds agent configs
# for swarm-controller: the controller sends a config PR's head over the
# queue, and the daemon builds that config on this host's agent base and
# replies.
# docs/swarm/builder.md has the design.
#
# ⚠️ Its own network namespace on the bridge, as ./swarm-ci-runner.nix has: a
# config PR is unreviewed input to a nix evaluation, so the container reaches
# the swarm services by name and nothing on the host's loopback.
#
# Builds go through the HOST nix-daemon, as the CI runner's do. The daemon
# runs as a dynamic user, an untrusted daemon client: a config can set no
# daemon option, and its builds run in the host's sandbox.
#
# The container holds one credential, its own store leaf, which nspawn hands in
# as a system credential with no copy on disk. The daemon reads everything else
# from the store with it: its queue client secret at start and its forge token
# on every build.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  hyperhiveCfg = config.services.hyperhive;
  cfg = hyperhiveCfg.deploy.builder;
  baoCfg = hyperhiveCfg.swarm.bao;
  baoDeploy = hyperhiveCfg.deploy.bao;
  natsCfg = hyperhiveCfg.swarm.nats;
  autheliaCfg = hyperhiveCfg.swarm.authelia;
  forgeCfg = hyperhiveCfg.swarm.forge;
  gatewayCfg = hyperhiveCfg.gateway;
  networkCfg = hyperhiveCfg.network;
  tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;

  machine = "swarm-builder";
  unit = "swarm-builder";
  clientId = hyperhiveCfg.swarm.builder.clientId;

  # The daemon reaches authelia, the store and the queue by name over TLS, and
  # `git` reaches the forge; in self-signed mode their leaves chain to the hive
  # CA.
  caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
  caBundleModule = caTrust.trustBundle {
    inherit pkgs;
    name = machine;
    consumers = [ unit ];
  };

  haveClientIdentity =
    baoDeploy.builderClientCertFile != null && baoDeploy.builderClientKeyFile != null;

  # Credential id → host source, as ./swarm-review-classifier.nix hands its
  # leaf in.
  storeFiles = lib.filterAttrs (_: source: source != null) {
    builder-bao-cert = baoDeploy.builderClientCertFile;
    builder-bao-key = baoDeploy.builderClientKeyFile;
    builder-bao-server-ca = baoDeploy.serverCaFile;
  };
  # nixos-container expands `extraFlags` unquoted, so a source path must be
  # one word.
  oneWord = path: builtins.match ".*[[:space:]].*" path == null;
  credential = id: "/run/credentials/${unit}.service/${id}";

  # The `nix` the host daemon runs, so client and daemon speak one protocol.
  nix = config.nix.package;

  # Strings, not paths: both are already store paths, and the flake reference
  # the daemon passes must be the same one a hive's meta flake names.
  hyperhiveFlake = hyperhiveCfg.c0re.hyperhiveFlake;
  nixpkgsFlake = hyperhiveCfg.c0re.nixpkgsFlake;

  privateNetwork = true;
in
{
  options.services.hyperhive.swarm.builder.clientId = lib.mkOption {
    type = lib.types.str;
    readOnly = true;
    default = "swarm-builder";
    description = ''
      The OAuth2 client id the swarm builder presents to the swarm queue.
      Read-only: the host running authelia registers it, the queue's
      auth-callout responder grants it the build subject, and the secret
      publisher carries its secret to the store, all from this one value.
    '';
  };

  options.services.hyperhive.deploy.builder.package = lib.mkOption {
    type = lib.types.package;
    defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-builder";
    description = "The swarm builder daemon package.";
  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = haveClientIdentity;
        message = ''
          services.hyperhive.deploy.builder.enable requires the builder's
          swarm-secret-store client identity: set both

            services.hyperhive.deploy.bao.builderClientCertFile
            services.hyperhive.deploy.bao.builderClientKeyFile

          The builder reads its queue client secret and its forge token with it.

          Both default to the pair glue-bao-tls.nix mints when
          services.hyperhive.deploy.bao.enable is true (the host that runs the
          store). Otherwise copy builder.pem and builder-key.pem from the store
          host's /var/lib/swarm-bao-pki/ and name them in those two options.
        '';
      }
      {
        assertion = lib.all oneWord (lib.attrValues storeFiles);
        message = ''
          The swarm builder's store identity paths must not contain whitespace:
          nspawn's --load-credential receives them through nixos-container's
          unquoted extra flags.
        '';
      }
    ];

    # The container hangs off the bridge, takes its address from the gateway's
    # DHCP and resolves through its dnsmasq.
    services.hyperhive.network.enable = lib.mkDefault true;
    services.hyperhive.gateway.enable = lib.mkDefault true;
    services.hyperhive.gateway.dns.enable = lib.mkDefault true;

    # nspawn fails to start the container when a credential source is
    # missing, so where this host mints the leaf the container pulls the
    # minting unit in and waits for it. Ordering alone is not enough: the
    # container starts from `machines.target`, outside the transaction that
    # runs the minting unit from `multi-user.target`. Ignored elsewhere, where
    # the unit does not exist.
    systemd.services."container@${machine}" = lib.mkMerge [
      caTrust.containerOrdering
      {
        after = [ "swarm-bao-pki.service" ];
        wants = [ "swarm-bao-pki.service" ];
      }
    ];

    containers.${machine} = {
      autoStart = true;
      # The daemon keeps nothing but `git`'s fetch cache, which a restart may
      # drop.
      ephemeral = false;
      # Journal files on the host, not inside the container: nixpkgs hardcodes
      # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
      extraFlags = [
        "--link-journal=host"
      ]
      ++ lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
      inherit privateNetwork;
      hostBridge = networkCfg.bridgeName;

      bindMounts = {
        # The directory rather than the socket file, so the mount survives a
        # daemon restart replacing the inode.
        "/nix/var/nix/daemon-socket" = {
          hostPath = "/nix/var/nix/daemon-socket";
          isReadOnly = false;
        };
      }
      // caTrust.bindMount;

      config =
        { lib, ... }:
        {
          imports = [
            ../container-modules/swarm-container.nix
            caBundleModule
          ];

          services.hyperhive.swarmContainer = {
            inherit privateNetwork;
            writesOwnResolvConf = false;
          };
          networking.nameservers = [ networkCfg.bridgeIp ];
          networking.interfaces.eth0.useDHCP = true;

          # Both the service and its socket are off: the socket unit owns
          # `/nix/var/nix/daemon-socket/socket` and would race the bind mount.
          systemd.globalEnvironment.NIX_REMOTE = "daemon";
          systemd.services.nix-daemon.enable = false;
          systemd.sockets.nix-daemon.enable = false;

          systemd.services.${unit} = {
            description = "build agent configs for swarm-controller";
            wantedBy = [ "multi-user.target" ];
            wants = [ "network-online.target" ];
            after = [ "network-online.target" ];
            # nix fetches the config repo with `git`.
            path = [ pkgs.git ];
            environment = {
              BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
              BAO_CLIENT_CERT = credential "builder-bao-cert";
              BAO_CLIENT_KEY = credential "builder-bao-key";
              NIX_CONFIG = "experimental-features = nix-command flakes";
              # nix's fetch cache, under `CacheDirectory`; the dynamic user has
              # no home of its own.
              HOME = "/var/cache/${unit}";
              XDG_CACHE_HOME = "/var/cache/${unit}";
              SWARM_BUILDER_NATS_URL = "tls://${natsCfg.domain}:${toString natsCfg.port}";
              SWARM_BUILDER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
              SWARM_BUILDER_OIDC_CLIENT_ID = clientId;
              SWARM_BUILDER_NIX = "${nix}/bin/nix";
              SWARM_BUILDER_CHECK_FLAKE = "${../builder/check-flake}";
              SWARM_BUILDER_HYPERHIVE_FLAKE = hyperhiveFlake;
              SWARM_BUILDER_NIXPKGS_FLAKE = nixpkgsFlake;
              SWARM_BUILDER_FORGE_URL = "https://${forgeCfg.domain}";
              SWARM_BUILDER_CONFIG_ORG = "agent-configs";
            }
            // lib.optionalAttrs (storeFiles ? builder-bao-server-ca) {
              BAO_CACERT = credential "builder-bao-server-ca";
            }
            // lib.optionalAttrs caTrust.useSelfSigned {
              SWARM_BUILDER_OIDC_CA_FILE = caTrust.bundlePathFor machine;
              # `git`'s trust store, which the bundle's system CAs keep whole.
              GIT_SSL_CAINFO = caTrust.bundlePathFor machine;
            };
            serviceConfig = {
              ExecStart = "${cfg.package}/bin/swarm-builder";
              DynamicUser = true;
              CacheDirectory = unit;
              # Bare ids: the system credentials nspawn handed in.
              LoadCredential = lib.attrNames storeFiles;
              Restart = "on-failure";
              RestartSec = "10s";
              SyslogIdentifier = unit;
            };
          };
        };
    };
  };
}