hyperhive/nix/host-modules/lib/archive-server.nix

# The archive server's moving parts: the rclone command that serves the
# repositories, the script that renders its users file from the secret store,
# and the script that prunes each repository. A pure function, NOT a NixOS
# module: ../swarm-archive.nix wraps the pieces in units, and the VM test
# (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in
# `bao`.
#
# Both scripts use only `bao login -method=cert -token-only`,
# `bao kv list -format=json` and `bao kv get -field=…`, and need `bao`, `jq`,
# `coreutils`, `gnugrep` on the unit's `path` and the `BAO_*` client set.
{ lib }:
{
  rclone,
  restic,
  # Where the repositories live: `<reposDir>/<agent>/`. The only place that
  # names the backend; `rclone serve restic` takes any remote here.
  reposDir,
  # The address rclone listens on.
  addr,
  # The users file rclone checks logins against, its group and the user the
  # server runs as.
  authFile,
  authGroup,
  # `{ keepWithin, keepLast }`; a null `keepWithin` leaves that flag off.
  retention,
}:
let
  atomicWriteSecret = import ./atomic-write-secret.nix { };
  refreshConsumer = import ./refresh-consumer.nix { };
  storeSeededSecret = import ./store-seeded-secret.nix { };

  usersPrefix = "secret/swarm/services/archive/users";
  resticPrefix = "secret/swarm/services/archive/restic";
  readyPath = "secret/swarm/services/archive/ready";

  keepFlags = lib.concatStringsSep " " (
    [ "--keep-last ${toString retention.keepLast}" ]
    ++ lib.optional (
      retention.keepWithin != null
    ) "--keep-within ${lib.escapeShellArg retention.keepWithin}"
  );
in
{
  inherit usersPrefix resticPrefix;

  # Append-only: a client cannot delete or overwrite a snapshot, a pack or the
  # index, so a hive that is compromised cannot erase history. Private repos:
  # a login reaches `/<its own name>/` and nothing else.
  serverCommand = lib.escapeShellArgs [
    "${rclone}/bin/rclone"
    "serve"
    "restic"
    "--addr"
    addr
    "--append-only"
    "--private-repos"
    "--htpasswd"
    authFile
    reposDir
  ];

  # Renders the users file from `usersPrefix`, one leaf per agent holding that
  # agent's htpasswd line. The file is written only when the content changed,
  # and never from a partial read: any leaf that cannot be read leaves the file
  # as it is.
  usersScript = ''
    set -euo pipefail
    ${storeSeededSecret}
    ${atomicWriteSecret}
    ${refreshConsumer}

    file=${lib.escapeShellArg authFile}
    # The server will not start without the file; an empty one admits nobody.
    if [ ! -e "$file" ]; then
      atomic_write_secret 0640 root:${authGroup} "$file" ""
    fi

    store_init
    store_login
    if [ "$STORE_UP" != 1 ]; then
      echo "could not log in to the swarm secret store; $file stays as it is." >&2
      exit 1
    fi

    # Tells swarm-controller an archive exists; it grants no credentials until
    # then. Created once, never rewritten.
    if ! store_bao kv get -field=ready ${lib.escapeShellArg readyPath} >/dev/null 2>&1; then
      store_bao kv put -cas=0 ${lib.escapeShellArg readyPath} ready=1 >/dev/null
    fi

    # A prefix that holds nothing is "No value found"; it means no agent has a
    # user yet, not a failure.
    rc=0
    names="$(store_bao kv list -format=json ${lib.escapeShellArg usersPrefix} 2>"$STORE_ERR")" || rc=$?
    if [ "$rc" != 0 ]; then
      if grep -q 'No value found' "$STORE_ERR"; then
        names='[]'
      else
        cat "$STORE_ERR" >&2
        echo "could not list ${usersPrefix}; $file stays as it is." >&2
        exit 1
      fi
    fi

    content=""
    while IFS= read -r name; do
      [ -n "$name" ] || continue
      case "$name" in */) continue ;; esac
      if ! line="$(store_bao kv get -field=htpasswd ${lib.escapeShellArg usersPrefix}/"$name" 2>"$STORE_ERR")"; then
        cat "$STORE_ERR" >&2
        echo "could not read the user $name; $file stays as it is." >&2
        exit 1
      fi
      # The line must be this agent's own: a leaf cannot add another user.
      if [[ ! "$name" =~ ^[A-Za-z0-9_-]+$ ]] || [[ "$line" != "$name:{SHA}"* ]] \
        || [[ ! "$line" =~ ^[A-Za-z0-9_-]+:\{SHA\}[A-Za-z0-9+/=]+$ ]]; then
        echo "the user leaf $name does not hold a {SHA} line for $name; leaving it out." >&2
        continue
      fi
      content+="$line"$'\n'
    done < <(printf '%s' "$names" | jq -r '.[]')
    content="''${content%$'\n'}"

    if secret_differs "$file" "$content"; then
      atomic_write_secret 0640 root:${authGroup} "$file" "$content"
      echo "users file updated."
    fi
  '';

  # `restic forget --prune` on every repository, with its password read from
  # the store at run time. The server refuses deletes from clients, so this is
  # the only place history is dropped.
  pruneScript = ''
    set -euo pipefail
    ${storeSeededSecret}

    store_init
    store_login
    if [ "$STORE_UP" != 1 ]; then
      echo "archive prune failed: could not log in to the swarm secret store" >&2
      exit 1
    fi

    failed=0
    for repo in ${lib.escapeShellArg reposDir}/*/; do
      [ -e "$repo/config" ] || continue
      agent="$(basename -- "$repo")"
      if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then
        echo "archive prune failed: $agent is not an agent name" >&2
        failed=1
        continue
      fi
      export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
      if ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
        echo "pruned $agent"
      else
        echo "archive prune failed: $agent" >&2
        failed=1
      fi
    done
    exit "$failed"
  '';
}