hyperhive/nix/host-modules/hive-priv-cache.nix

# The binary-cache facts the host's nix daemon, hive-priv (root) and hive-c0re
# use, kept where only root can change them.
#
# hive-c0re pulls an agent's closure with `nix-store --realise` through the
# host's nix daemon, which substitutes each path from the swarm cache or from
# its other substituters (cache.nixos.org) and stores it only if a key in its
# `trusted-public-keys` signed it. hive-priv applies a closure only if the
# swarm's key signed it. Neither decision may come from a file hive-c0re writes:
#
# - the cache's address is the host's
#   {option}`services.hyperhive.swarm.attic.domain` and `cache`;
# - the cache's public key and the pull token are read from the store by a root
#   oneshot, `hive-priv-cache-facts`, under this hive's own store identity. The
#   key lands in `/run/hyperhive-priv/attic/public-key` (directory `0700`, file
#   `0600`). The address, the key, a netrc and a CA bundle also land in a
#   nix.conf fragment, `/run/hyperhive-nix/cache.conf`, as `extra-substituters`,
#   `extra-trusted-public-keys`, `netrc-file` and `ssl-cert-file`, which the
#   host's nix.conf includes (`!include`, so a host whose unit has not run yet
#   still starts). Every nix client of the host can have a swarm closure
#   substituted; the fragment holds no secret;
# - `netrc-file` and `ssl-cert-file` take one path and would replace the host's
#   own, so the files they name merge it in: the pull token's entry plus the
#   host's netrc entries, and the host's CA bundle plus the swarm's root, which
#   the cache's certificate chains to. The host's are `nix.settings.netrc-file`
#   else `/etc/nix/netrc`, and `nix.settings.ssl-cert-file` else
#   `security.pki.caBundle`; one named only in `nix.extraOptions` is not seen.
#
# hive-c0re keeps its own copies under `/run/hyperhive/attic/` for the probe.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  hyperhiveCfg = config.services.hyperhive;
  hiveController = hyperhiveCfg.deploy.hive-controller;
  tls = hiveController.tls;
  baoCfg = hyperhiveCfg.swarm.bao;
  baoDeploy = hyperhiveCfg.deploy.bao;
  atticCfg = hyperhiveCfg.swarm.attic;

  # Without a cache to name, a hive name to read under or an identity to log
  # in with there is nothing to fetch, and hive-priv has no cache to download
  # from.
  enabled =
    hiveController.enable
    && atticCfg.domain != null
    && atticCfg.cache != null
    && hiveController.hiveName != null
    && tls.baoHiveIdentityCertFile != null
    && tls.baoHiveIdentityKeyFile != null;

  dir = "/run/hyperhive-priv/attic";
  confDir = "/run/hyperhive-nix";
  confFile = "${confDir}/cache.conf";
  cacheUrl = "https://${atticCfg.domain}/${atticCfg.cache}";
  netrcFile = "${confDir}/netrc";
  # `netrc-file` is not additive: the fragment's setting replaces the host's.
  hostNetrc = toString (config.nix.settings.netrc-file or "/etc/nix/netrc");
  # `ssl-cert-file` is not additive either. The host's bundle is the one nix
  # would use without the fragment: `nix.settings.ssl-cert-file` when the host
  # sets it, else the system bundle the nix daemon's `CURL_CA_BUNDLE` names.
  caBundleFile = "${confDir}/ca-bundle.crt";
  baseBundle = toString (config.nix.settings.ssl-cert-file or config.security.pki.caBundle);
  # Written by `swarm-services-cert` on a host whose gateway is self-signed.
  swarmCa = "${tls.stateDir}/swarm-services-root.pem";

  # Where the store runs on this host, the first login waits for its container.
  # No `wants`: a pull would start the container mid-switch on the old closure.
  storeContainer = lib.optional baoDeploy.enable "container@${baoCfg.machine}.service";

  storeRetry = import ./lib/store-retry.nix { };
  storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
in
{
  config = lib.mkIf enabled {
    systemd.tmpfiles.rules = [
      "d /run/hyperhive-priv 0700 root root - -"
      "d ${confDir} 0755 root root - -"
    ];

    # `!include` tolerates a fragment that does not exist yet.
    nix.extraOptions = "!include ${confFile}";

    systemd.services.hive-priv-cache-facts = {
      description = "Read the swarm cache's public key and pull token for the host's nix daemon and hive-priv";
      wantedBy = [ "multi-user.target" ];
      wants = [ "network-online.target" ];
      after = [
        "network-online.target"
        "systemd-tmpfiles-setup.service"
      ]
      ++ storeContainer;
      path = [
        baoDeploy.package
        pkgs.coreutils
        pkgs.gnugrep
        pkgs.systemd
      ];
      environment = {
        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
        BAO_CLIENT_CERT = tls.baoHiveIdentityCertFile;
        BAO_CLIENT_KEY = tls.baoHiveIdentityKeyFile;
      }
      // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
        BAO_CACERT = baoDeploy.serverCaFile;
      };
      # ./lib/store-retry.nix: the login fails for as long as the store is
      # sealed or down.
      inherit (storeRetry) startLimitBurst startLimitIntervalSec;
      serviceConfig = storeRetry.serviceConfig // {
        Type = "oneshot";
        UMask = "0077";
        SyslogIdentifier = "hive-priv-cache-facts";
        TimeoutStartSec = "5min";
      };
      script = import ./lib/attic-root-facts.nix { inherit lib; } {
        hive = hiveController.hiveName;
        host = atticCfg.domain;
        inherit
          dir
          confFile
          netrcFile
          hostNetrc
          caBundleFile
          baseBundle
          swarmCa
          ;
        url = cacheUrl;
      };
    };

    systemd.timers.hive-priv-cache-facts = storeRefreshTimer "re-read the swarm cache's facts for hive-priv";
  };
}