hyperhive/nix/host-modules/hive-archive-backup.nix

# The hive's backup of its agents' state to the swarm archive
# (./swarm-archive.nix): one restic repository per agent, at
# `https://archive.<swarm domain>/<agent>/`.
#
# A root oneshot on a timer, not hive-c0re and not a hive-priv verb. Root,
# because the agents' files belong to the agents (and, under user namespaces,
# to remapped ids) and hive-c0re cannot read them; a unit of its own, because
# no caller-supplied argument may reach root, and this one takes none.
#
# Each agent backs up under the credentials swarm-controller wrote for it at
# `swarm/hives/<this hive>/archive/<agent>`
# (../../swarm-controller/src/archive_access.rs), read with the hive's own
# store identity: the agent's login to the archive, and its restic password,
# which restic reads through `RESTIC_PASSWORD_COMMAND` and nothing writes to
# disk. An agent the controller has not granted this hive is skipped; a hive
# with agents and no grant at all fails, which is what a swarm-controller
# without the archive looks like.
#
# A failed agent prints `archive backup failed: <agent>` to the journal,
# which the swarm's `archive backup failed` alert reads, and fails the unit
# after the others have been tried.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  hyperhiveCfg = config.services.hyperhive;
  hiveController = hyperhiveCfg.deploy.hive-controller;
  cfg = hiveController.archive;
  tls = hiveController.tls;
  baoCfg = hyperhiveCfg.swarm.bao;
  baoDeploy = hyperhiveCfg.deploy.bao;

  enabled =
    cfg.enable
    && hiveController.enable
    && hiveController.hiveName != null
    && tls.baoHiveIdentityCertFile != null
    && tls.baoHiveIdentityKeyFile != null;

  # The CA bundle restic trusts: the host's, then the swarm's root, which the
  # archive's certificate chains to when the gateway is self-signed. Written by
  # the unit at each run; `--cacert` replaces restic's own roots.
  runtimeDir = "/run/hive-archive-backup";
  caBundle = "${runtimeDir}/ca.pem";
  swarmCa = "${tls.stateDir}/swarm-services-root.pem";

  storeContainer = lib.optional baoDeploy.enable "container@${baoCfg.machine}.service";

  backup = import ./lib/archive-backup.nix { inherit lib; } {
    inherit (cfg) scope paths excludes;
    restic = cfg.package;
    hive = hiveController.hiveName;
    agentRoot = "/var/lib/hyperhive/agents";
    url = "https://${hyperhiveCfg.swarm.archive.ownDomain}";
    inherit caBundle;
  };
in
{
  options.services.hyperhive.deploy.hive-controller.archive = {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = true;
      description = ''
        Back this hive's agents up to the swarm archive. On wherever the hive
        has its store identity; a swarm that runs no archive
        ({option}`services.hyperhive.deploy.archive.enable` on one host) sees
        every run fail, so turn it off there.
      '';
    };

    schedule = lib.mkOption {
      type = lib.types.str;
      default = "hourly";
      example = "*-*-* 00/6:00:00";
      description = ''
        systemd calendar expression for the backup. Each run sends only what
        changed since the last, so an hourly run on an idle agent is small.
      '';
    };

    scope = lib.mkOption {
      type = lib.types.enum [
        "paths"
        "subvolume"
      ];
      default = "paths";
      description = ''
        What of each agent is backed up. `paths`: the entries of
        {option}`services.hyperhive.deploy.hive-controller.archive.paths`
        inside the agent's directory. `subvolume`: the whole directory, which
        adds `work/` and everything an agent keeps there.
      '';
    };

    paths = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [
        "state"
        "claude"
        "harness"
      ];
      description = ''
        Entries of an agent's directory backed up under `scope = "paths"`: its
        state, its claude login and the harness. Caches and `work/` are left
        out unless listed. An entry the agent does not have is skipped.
      '';
    };

    excludes = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [ ];
      example = [ "*/target" ];
      description = "restic `--exclude` patterns, on top of `--exclude-caches`.";
    };

    package = lib.mkOption {
      type = lib.types.package;
      default = pkgs.restic;
      defaultText = lib.literalExpression "pkgs.restic";
      description = "The restic package.";
    };
  };

  config = lib.mkIf enabled {
    services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [
      "hive-archive-backup.service"
    ];

    systemd.services.hive-archive-backup = {
      description = "back up this hive's agents to the swarm archive";
      wants = [ "network-online.target" ];
      after = [ "network-online.target" ] ++ storeContainer;
      path = [
        baoDeploy.package
        pkgs.btrfs-progs
        pkgs.coreutils
        pkgs.gnugrep
        pkgs.jq
      ];
      environment = {
        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
        BAO_CLIENT_CERT = tls.baoHiveIdentityCertFile;
        BAO_CLIENT_KEY = tls.baoHiveIdentityKeyFile;
        RESTIC_CACHE_DIR = "/var/cache/hive-archive-backup";
      }
      // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
        BAO_CACERT = baoDeploy.serverCaFile;
      };
      serviceConfig = {
        Type = "oneshot";
        UMask = "0077";
        SyslogIdentifier = "hive-archive-backup";
        RuntimeDirectory = "hive-archive-backup";
        CacheDirectory = "hive-archive-backup";
        # A first backup of a large agent sends all of it.
        TimeoutStartSec = "6h";
        Nice = 10;
        IOSchedulingClass = "idle";
      };
      script = ''
        set -euo pipefail
        # A missing file is left out: the bundle is the host's roots alone
        # until the swarm's root is there.
        cat ${lib.escapeShellArg (toString config.security.pki.caBundle)} > ${caBundle}
        if [ -s ${lib.escapeShellArg swarmCa} ]; then
          cat ${lib.escapeShellArg swarmCa} >> ${caBundle}
        fi
        ${backup}
      '';
    };

    systemd.timers.hive-archive-backup = {
      description = "back up this hive's agents to the swarm archive";
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnCalendar = cfg.schedule;
        RandomizedDelaySec = "5min";
        Persistent = true;
      };
    };
  };
}