# The hive's backup run: for each agent that has a state directory here and a
# grant from swarm-controller, one `restic backup` into that agent's repository
# on the archive. A pure function, NOT a NixOS module: ../hive-archive-backup.nix
# wraps it in a unit, and the VM test (nix/vm-tests/swarm-archive.nix) runs the
# same script against a stand-in `bao`.
#
# Needs `bao`, `jq`, `curl`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
# and the `BAO_*` client set. A store that cannot be reached, or an agent whose
# backup fails, prints a line starting
# `archive backup failed`, which the swarm's alert reads, and ends the run with
# a non-zero status once the other agents have been tried.
#
# After each agent it pushes `hyperhive.archive.backup.*` gauges (`agent` attribute)
# through ./otlp-gauges.nix: last run and last success time, result, wall time,
# bytes added and files new and changed. A failed agent still reports its result.
{ lib }:
{
  restic,
  # The hive's name in the store: grants are read from
  # `secret/swarm/hives/<hive>/archive/`.
  hive,
  # The directory holding one directory per agent.
  agentRoot,
  # The repository server's base URL, no trailing slash.
  url,
  # "paths": `paths` under each agent's directory. "subvolume": the whole directory.
  scope,
  paths,
  excludes,
  # A CA file to trust instead of restic's default, or null.
  caBundle,
  # The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null.
  otlpEndpoint ? null,
}:
let
  storeSeededSecret = import ./store-seeded-secret.nix { };

  grants = hive: "secret/swarm/hives/${hive}/archive";

  otlpGauges = import ./otlp-gauges.nix { inherit lib; } { endpoint = otlpEndpoint; };
in
''
  set -euo pipefail
  ${storeSeededSecret}

  store_init
  store_login
  if [ "$STORE_UP" != 1 ]; then
    echo "archive backup failed: could not log in to the swarm secret store" >&2
    exit 1
  fi

  ${lib.optionalString (caBundle != null) ''
    export RESTIC_CACERT=${lib.escapeShellArg caBundle}
  ''}
  ${otlpGauges}
  root=${lib.escapeShellArg agentRoot}
  grants=${lib.escapeShellArg (grants hive)}

  # One agent: a read-only snapshot of its subvolume when it is one, so the
  # databases are copied at one instant, else its directory in place.
  backup_agent() {
    local agent="$1" leaf="$grants/$1" rest base snap="" target targets=()
    rest="$(store_bao kv get -field=rest-password "$leaf")" || return 1
    export RESTIC_REPOSITORY="rest:${url}/$agent/"
    export RESTIC_REST_USERNAME="$agent"
    export RESTIC_REST_PASSWORD="$rest"
    export RESTIC_PASSWORD_COMMAND="bao kv get -field=restic-password $leaf"

    if ! ${restic}/bin/restic cat config >/dev/null 2>&1; then
      ${restic}/bin/restic init || return 1
    fi

    base="$root/$agent"
    if btrfs subvolume show "$base" >/dev/null 2>&1; then
      snap="$root/.$agent.snapshot.archive"
      btrfs subvolume delete "$snap" >/dev/null 2>&1 || true
      btrfs subvolume snapshot -r "$base" "$snap" >/dev/null || return 1
      base="$snap"
    fi

    ${
      if scope == "subvolume" then
        ''targets=("$base")''
      else
        ''
          for target in ${lib.escapeShellArgs paths}; do
                [ -e "$base/$target" ] && targets+=("$base/$target")
              done''
    }
    local rc=0
    if [ "''${#targets[@]}" = 0 ]; then
      echo "$agent has none of its backup paths yet; nothing to back up."
    else
      # `--json` for the summary line the metrics read. Only that line and the
      # errors are printed; the progress lines are dropped.
      ${restic}/bin/restic backup --json --host "$agent" --tag archive --exclude-caches \
        ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \
        -- "''${targets[@]}" |
        jq -R -r '. as $line | ((fromjson? | select(type == "object")) // null) as $j
          | if $j == null or $j.message_type == "error" or $j.message_type == "summary"
            then $line else empty end' || rc=$?
    fi
    if [ -n "$snap" ]; then
      btrfs subvolume delete "$snap" >/dev/null || rc=1
    fi
    return "$rc"
  }

  # One agent's run as gauges: `$5` is its output, whose `summary` line has the
  # counts when restic got that far. Never fails the run.
  push_backup_metrics() {
    local agent="$1" ok="$2" started="$3" finished="$4" out="$5" summary
    local metrics=(
      "hyperhive.archive.backup.last_run_timestamp_seconds=$finished"
      "hyperhive.archive.backup.last_run_success=$ok"
      "hyperhive.archive.backup.duration_seconds=$((finished - started))"
    )
    if [ "$ok" = 1 ]; then
      metrics+=("hyperhive.archive.backup.last_success_timestamp_seconds=$finished")
      summary="$(printf '%s\n' "$out" | jq -R -r '
        fromjson? | select(type == "object" and .message_type == "summary")
        | "hyperhive.archive.backup.data_added_bytes=\(.data_added)",
          "hyperhive.archive.backup.files_new=\(.files_new)",
          "hyperhive.archive.backup.files_changed=\(.files_changed)"' 2>/dev/null)" || summary=""
      while IFS= read -r line; do
        [ -z "$line" ] || metrics+=("$line")
      done <<<"$summary"
    fi
    otlp_gauges hive-archive-backup "agent=$agent" "''${metrics[@]}"
  }

  # The agents the controller granted this hive; none yet is "No value found".
  rc=0
  granted="$(store_bao kv list -format=json "$grants" 2>"$STORE_ERR")" || rc=$?
  if [ "$rc" != 0 ]; then
    if grep -q 'No value found' "$STORE_ERR"; then
      granted='[]'
    else
      cat "$STORE_ERR" >&2
      echo "archive backup failed: could not list $grants" >&2
      exit 1
    fi
  fi
  granted="$(printf '%s' "$granted" | jq -r '.[]')"

  failed=0
  candidates=0
  backed_up=0
  for dir in "$root"/*/; do
    agent="$(basename -- "$dir")"
    case "$agent" in .*) continue ;; esac
    [[ "$agent" =~ ^[A-Za-z0-9_-]+$ ]] || continue
    candidates=$((candidates + 1))
    if ! grep -qxF -- "$agent" <<<"$granted"; then
      echo "$agent has no archive credentials on this hive; skipping."
      continue
    fi
    started="$(date +%s)"
    ok=0
    out="$(backup_agent "$agent")" && ok=1
    [ -z "$out" ] || printf '%s\n' "$out"
    finished="$(date +%s)"
    if [ "$ok" = 1 ]; then
      backed_up=$((backed_up + 1))
      echo "backed up $agent"
    else
      echo "archive backup failed: $agent" >&2
      failed=1
    fi
    push_backup_metrics "$agent" "$ok" "$started" "$finished" "$out"
  done

  # No grant for any agent here: the swarm has no archive, or its controller
  # has not granted this hive yet. Neither is a failure of this hive.
  if [ "$candidates" -gt 0 ] && [ -z "$granted" ]; then
    echo "swarm-controller has granted this hive no archive credentials ($candidates agents here); nothing to back up."
  fi
  exit "$failed"
''