hyperhive/nix/host-modules/lib/archive-backup.nix

# The hive's backup run: for each agent that has a state directory here and a
# grant from swarm-controller, one `restic backup` into that agent's repository
# on the archive. A pure function, NOT a NixOS module: ../hive-archive-backup.nix
# wraps it in a unit, and the VM test (nix/vm-tests/swarm-archive.nix) runs the
# same script against a stand-in `bao`.
#
# Needs `bao`, `jq`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
# and the `BAO_*` client set. Every failure prints a line starting
# `archive backup failed`, which the swarm's alert reads, and ends the run with
# a non-zero status once the other agents have been tried.
{ lib }:
{
  restic,
  # The hive's name in the store: grants are read from
  # `secret/swarm/hives/<hive>/archive/`.
  hive,
  # The directory holding one directory per agent.
  agentRoot,
  # The repository server's base URL, no trailing slash.
  url,
  # "paths": `paths` under each agent's directory. "subvolume": the whole directory.
  scope,
  paths,
  excludes,
  # A CA file to trust instead of restic's default, or null.
  caBundle,
}:
let
  storeSeededSecret = import ./store-seeded-secret.nix { };

  grants = hive: "secret/swarm/hives/${hive}/archive";
in
''
  set -euo pipefail
  ${storeSeededSecret}

  store_init
  store_login
  if [ "$STORE_UP" != 1 ]; then
    echo "archive backup failed: could not log in to the swarm secret store" >&2
    exit 1
  fi

  ${lib.optionalString (caBundle != null) ''
    export RESTIC_CACERT=${lib.escapeShellArg caBundle}
  ''}
  root=${lib.escapeShellArg agentRoot}
  grants=${lib.escapeShellArg (grants hive)}

  # One agent: a read-only snapshot of its subvolume when it is one, so the
  # databases are copied at one instant, else its directory in place.
  backup_agent() {
    local agent="$1" leaf="$grants/$1" rest base snap="" target targets=()
    rest="$(store_bao kv get -field=rest-password "$leaf")" || return 1
    export RESTIC_REPOSITORY="rest:${url}/$agent/"
    export RESTIC_REST_USERNAME="$agent"
    export RESTIC_REST_PASSWORD="$rest"
    export RESTIC_PASSWORD_COMMAND="bao kv get -field=restic-password $leaf"

    if ! ${restic}/bin/restic cat config >/dev/null 2>&1; then
      ${restic}/bin/restic init || return 1
    fi

    base="$root/$agent"
    if btrfs subvolume show "$base" >/dev/null 2>&1; then
      snap="$root/.$agent.snapshot.archive"
      btrfs subvolume delete "$snap" >/dev/null 2>&1 || true
      btrfs subvolume snapshot -r "$base" "$snap" >/dev/null || return 1
      base="$snap"
    fi

    ${
      if scope == "subvolume" then
        ''targets=("$base")''
      else
        ''
          for target in ${lib.escapeShellArgs paths}; do
                [ -e "$base/$target" ] && targets+=("$base/$target")
              done''
    }
    local rc=0
    if [ "''${#targets[@]}" = 0 ]; then
      echo "$agent has none of its backup paths yet; nothing to back up."
    else
      ${restic}/bin/restic backup --host "$agent" --tag archive --exclude-caches \
        ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \
        -- "''${targets[@]}" || rc=$?
    fi
    if [ -n "$snap" ]; then
      btrfs subvolume delete "$snap" >/dev/null || rc=1
    fi
    return "$rc"
  }

  # The agents the controller granted this hive; none yet is "No value found".
  rc=0
  granted="$(store_bao kv list -format=json "$grants" 2>"$STORE_ERR")" || rc=$?
  if [ "$rc" != 0 ]; then
    if grep -q 'No value found' "$STORE_ERR"; then
      granted='[]'
    else
      cat "$STORE_ERR" >&2
      echo "archive backup failed: could not list $grants" >&2
      exit 1
    fi
  fi
  granted="$(printf '%s' "$granted" | jq -r '.[]')"

  failed=0
  candidates=0
  backed_up=0
  for dir in "$root"/*/; do
    agent="$(basename -- "$dir")"
    case "$agent" in .*) continue ;; esac
    [[ "$agent" =~ ^[A-Za-z0-9_-]+$ ]] || continue
    candidates=$((candidates + 1))
    if ! grep -qxF -- "$agent" <<<"$granted"; then
      echo "$agent has no archive credentials on this hive; skipping."
      continue
    fi
    if (backup_agent "$agent"); then
      backed_up=$((backed_up + 1))
      echo "backed up $agent"
    else
      echo "archive backup failed: $agent" >&2
      failed=1
    fi
  done

  # Agents here and not one of them granted: swarm-controller has not written
  # this hive's grants, which a controller without the archive never does.
  if [ "$candidates" -gt 0 ] && [ -z "$granted" ]; then
    echo "archive backup failed: swarm-controller has granted this hive no archive credentials ($candidates agents here)" >&2
    failed=1
  fi
  exit "$failed"
''