swarm-archive: back up every agent's state to an append-only restic repository per agent
swarm-archive serves restic repositories with rclone, append-only, one login and one repository per agent. swarm-controller writes each placed agent's login and restic password to the store, replaces them on a move and removes them when the agent is destroyed. Every hive backs its agents up hourly under its own store identity. The archive host prunes by retention.
This commit is contained in:
atlas
parent
409e7ee5a0
commit
306aae0378
18 changed files with 2214 additions and 5 deletions
|
|
@ -22,6 +22,7 @@
|
|||
./hive-matrix.nix
|
||||
./hive-network.nix
|
||||
./hive-priv.nix
|
||||
./hive-archive-backup.nix
|
||||
./hive-priv-cache.nix
|
||||
./hive-squid.nix
|
||||
./hive-squid-credential.nix
|
||||
|
|
@ -65,6 +66,7 @@
|
|||
./glue-swarm-otel-oidc-client.nix
|
||||
./glue-victorialogs-otel-oidc-client.nix
|
||||
./glue-victoriametrics-otel-oidc-client.nix
|
||||
./swarm-archive.nix
|
||||
./swarm-attic-options.nix
|
||||
./swarm-attic.nix
|
||||
./swarm-authelia-service.nix
|
||||
|
|
|
|||
|
|
@ -734,6 +734,25 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
archive.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
example = true;
|
||||
description = ''
|
||||
Run the swarm's backup archive on this host: `rclone serve restic`
|
||||
behind the gateway at `archive.<swarm domain>`, append-only, with one
|
||||
login and one repository per agent, and a daily prune that applies the
|
||||
retention. Every hive's backup unit finds it by that name, so it runs
|
||||
on one host and the host does not matter. Run it on one host.
|
||||
|
||||
On by default where
|
||||
{option}`services.hyperhive.deploy.allSwarmServices` is set and this
|
||||
host has the archive's store identity (the pair
|
||||
{option}`services.hyperhive.deploy.bao.archiveClientCertFile` and
|
||||
`archiveClientKeyFile`, minted on the store's host).
|
||||
'';
|
||||
};
|
||||
|
||||
hive-controller.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
|
|
|
|||
|
|
@ -90,6 +90,16 @@ let
|
|||
here = deployCfg.attic.enable;
|
||||
policies = [ "swarm-bao-attic-policy" ];
|
||||
};
|
||||
# ./swarm-archive.nix, under the archive's leaf, which reads the users and the
|
||||
# repositories' passwords only once that policy is written.
|
||||
swarm-archive-users = {
|
||||
here = deployCfg.archive.enable;
|
||||
policies = [ "swarm-bao-archive-policy" ];
|
||||
};
|
||||
swarm-archive-prune = {
|
||||
here = deployCfg.archive.enable;
|
||||
policies = [ "swarm-bao-archive-policy" ];
|
||||
};
|
||||
# ./hive-matrix.nix, under matrix-ctl's leaf.
|
||||
swarm-bao-matrix-oidc = {
|
||||
here = deployCfg.matrix.enable;
|
||||
|
|
|
|||
|
|
@ -69,6 +69,9 @@ let
|
|||
# ./swarm-attic.nix
|
||||
swarm-bao-attic-secret = deployCfg.attic.enable;
|
||||
swarm-bao-attic-tokens = deployCfg.attic.enable;
|
||||
# ./swarm-archive.nix
|
||||
swarm-archive-users = deployCfg.archive.enable;
|
||||
swarm-archive-prune = deployCfg.archive.enable;
|
||||
# ./swarm-otel.nix
|
||||
swarm-bao-otel-oidc =
|
||||
deployCfg.swarm-otel.enable
|
||||
|
|
|
|||
|
|
@ -126,6 +126,8 @@ in
|
|||
notifyRouterClientKeyFile = lib.mkDefault "${pkiDir}/notify-router-key.pem";
|
||||
atticClientCertFile = lib.mkDefault "${pkiDir}/attic.pem";
|
||||
atticClientKeyFile = lib.mkDefault "${pkiDir}/attic-key.pem";
|
||||
archiveClientCertFile = lib.mkDefault "${pkiDir}/archive.pem";
|
||||
archiveClientKeyFile = lib.mkDefault "${pkiDir}/archive-key.pem";
|
||||
forgePublisherClientCertFile = lib.mkDefault "${pkiDir}/forge-publisher.pem";
|
||||
forgePublisherClientKeyFile = lib.mkDefault "${pkiDir}/forge-publisher-key.pem";
|
||||
servicesIssuerClientCertFile = lib.mkDefault "${pkiDir}/services-issuer.pem";
|
||||
|
|
@ -337,6 +339,13 @@ in
|
|||
[ -s ${pkiDir}/attic.pem ] || ${signLeaf} ${pkiDir} attic \
|
||||
${lib.escapeShellArg deployCfg.bao.atticCommonName} "" clientAuth
|
||||
|
||||
# The backup archive's, which reads its users and the repositories'
|
||||
# passwords. Minted here for the matrix-ctl leaf's reason: the archive
|
||||
# is a swarm service, so on an archive host without the store this is
|
||||
# the file an operator copies.
|
||||
[ -s ${pkiDir}/archive.pem ] || ${signLeaf} ${pkiDir} archive \
|
||||
${lib.escapeShellArg deployCfg.bao.archiveCommonName} "" clientAuth
|
||||
|
||||
# The forge's token publisher. Minted here for the matrix-ctl leaf's
|
||||
# reason: the forge is a swarm singleton, so on a forge host without the
|
||||
# store this is the file an operator copies.
|
||||
|
|
|
|||
185
nix/host-modules/hive-archive-backup.nix
Normal file
185
nix/host-modules/hive-archive-backup.nix
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
# The hive's backup of its agents' state to the swarm archive
|
||||
# (./swarm-archive.nix): one restic repository per agent, at
|
||||
# `https://archive.<swarm domain>/<agent>/`.
|
||||
#
|
||||
# A root oneshot on a timer, not hive-c0re and not a hive-priv verb. Root,
|
||||
# because the agents' files belong to the agents (and, under user namespaces,
|
||||
# to remapped ids) and hive-c0re cannot read them; a unit of its own, because
|
||||
# no caller-supplied argument may reach root, and this one takes none.
|
||||
#
|
||||
# Each agent backs up under the credentials swarm-controller wrote for it at
|
||||
# `swarm/hives/<this hive>/archive/<agent>`
|
||||
# (../../swarm-controller/src/archive_access.rs), read with the hive's own
|
||||
# store identity: the agent's login to the archive, and its restic password,
|
||||
# which restic reads through `RESTIC_PASSWORD_COMMAND` and nothing writes to
|
||||
# disk. An agent the controller has not granted this hive is skipped; a hive
|
||||
# with agents and no grant at all fails, which is what a swarm-controller
|
||||
# without the archive looks like.
|
||||
#
|
||||
# A failed agent prints `archive backup failed: <agent>` to the journal,
|
||||
# which the swarm's `archive backup failed` alert reads, and fails the unit
|
||||
# after the others have been tried.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
hiveController = hyperhiveCfg.deploy.hive-controller;
|
||||
cfg = hiveController.archive;
|
||||
tls = hiveController.tls;
|
||||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
baoDeploy = hyperhiveCfg.deploy.bao;
|
||||
|
||||
enabled =
|
||||
cfg.enable
|
||||
&& hiveController.enable
|
||||
&& hiveController.hiveName != null
|
||||
&& tls.baoHiveIdentityCertFile != null
|
||||
&& tls.baoHiveIdentityKeyFile != null;
|
||||
|
||||
# The CA bundle restic trusts: the host's, then the swarm's root, which the
|
||||
# archive's certificate chains to when the gateway is self-signed. Written by
|
||||
# the unit at each run; `--cacert` replaces restic's own roots.
|
||||
runtimeDir = "/run/hive-archive-backup";
|
||||
caBundle = "${runtimeDir}/ca.pem";
|
||||
swarmCa = "${tls.stateDir}/swarm-services-root.pem";
|
||||
|
||||
storeContainer = lib.optional baoDeploy.enable "container@${baoCfg.machine}.service";
|
||||
|
||||
backup = import ./lib/archive-backup.nix { inherit lib; } {
|
||||
inherit (cfg) scope paths excludes;
|
||||
restic = cfg.package;
|
||||
hive = hiveController.hiveName;
|
||||
agentRoot = "/var/lib/hyperhive/agents";
|
||||
url = "https://${hyperhiveCfg.swarm.archive.ownDomain}";
|
||||
inherit caBundle;
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.deploy.hive-controller.archive = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Back this hive's agents up to the swarm archive. On wherever the hive
|
||||
has its store identity; a swarm that runs no archive
|
||||
({option}`services.hyperhive.deploy.archive.enable` on one host) sees
|
||||
every run fail, so turn it off there.
|
||||
'';
|
||||
};
|
||||
|
||||
schedule = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "hourly";
|
||||
example = "*-*-* 00/6:00:00";
|
||||
description = ''
|
||||
systemd calendar expression for the backup. Each run sends only what
|
||||
changed since the last, so an hourly run on an idle agent is small.
|
||||
'';
|
||||
};
|
||||
|
||||
scope = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"paths"
|
||||
"subvolume"
|
||||
];
|
||||
default = "paths";
|
||||
description = ''
|
||||
What of each agent is backed up. `paths`: the entries of
|
||||
{option}`services.hyperhive.deploy.hive-controller.archive.paths`
|
||||
inside the agent's directory. `subvolume`: the whole directory, which
|
||||
adds `work/` and everything an agent keeps there.
|
||||
'';
|
||||
};
|
||||
|
||||
paths = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [
|
||||
"state"
|
||||
"claude"
|
||||
"harness"
|
||||
];
|
||||
description = ''
|
||||
Entries of an agent's directory backed up under `scope = "paths"`: its
|
||||
state, its claude login and the harness. Caches and `work/` are left
|
||||
out unless listed. An entry the agent does not have is skipped.
|
||||
'';
|
||||
};
|
||||
|
||||
excludes = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
example = [ "*/target" ];
|
||||
description = "restic `--exclude` patterns, on top of `--exclude-caches`.";
|
||||
};
|
||||
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = pkgs.restic;
|
||||
defaultText = lib.literalExpression "pkgs.restic";
|
||||
description = "The restic package.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf enabled {
|
||||
services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [
|
||||
"hive-archive-backup.service"
|
||||
];
|
||||
|
||||
systemd.services.hive-archive-backup = {
|
||||
description = "back up this hive's agents to the swarm archive";
|
||||
wants = [ "network-online.target" ];
|
||||
after = [ "network-online.target" ] ++ storeContainer;
|
||||
path = [
|
||||
baoDeploy.package
|
||||
pkgs.btrfs-progs
|
||||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
pkgs.jq
|
||||
];
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = tls.baoHiveIdentityCertFile;
|
||||
BAO_CLIENT_KEY = tls.baoHiveIdentityKeyFile;
|
||||
RESTIC_CACHE_DIR = "/var/cache/hive-archive-backup";
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = baoDeploy.serverCaFile;
|
||||
};
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
UMask = "0077";
|
||||
SyslogIdentifier = "hive-archive-backup";
|
||||
RuntimeDirectory = "hive-archive-backup";
|
||||
CacheDirectory = "hive-archive-backup";
|
||||
# A first backup of a large agent sends all of it.
|
||||
TimeoutStartSec = "6h";
|
||||
Nice = 10;
|
||||
IOSchedulingClass = "idle";
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
# A missing file is left out: the bundle is the host's roots alone
|
||||
# until the swarm's root is there.
|
||||
cat ${lib.escapeShellArg (toString config.security.pki.caBundle)} > ${caBundle}
|
||||
if [ -s ${lib.escapeShellArg swarmCa} ]; then
|
||||
cat ${lib.escapeShellArg swarmCa} >> ${caBundle}
|
||||
fi
|
||||
${backup}
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.hive-archive-backup = {
|
||||
description = "back up this hive's agents to the swarm archive";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.schedule;
|
||||
RandomizedDelaySec = "5min";
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
137
nix/host-modules/lib/archive-backup.nix
Normal file
137
nix/host-modules/lib/archive-backup.nix
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
# The hive's backup run: for each agent that has a state directory here and a
|
||||
# grant from swarm-controller, one `restic backup` into that agent's repository
|
||||
# on the archive. A pure function, NOT a NixOS module: ../hive-archive-backup.nix
|
||||
# wraps it in a unit, and the VM test (nix/vm-tests/swarm-archive.nix) runs the
|
||||
# same script against a stand-in `bao`.
|
||||
#
|
||||
# Needs `bao`, `jq`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
|
||||
# and the `BAO_*` client set. Every failure prints a line starting
|
||||
# `archive backup failed`, which the swarm's alert reads, and ends the run with
|
||||
# a non-zero status once the other agents have been tried.
|
||||
{ lib }:
|
||||
{
|
||||
restic,
|
||||
# The hive's name in the store: grants are read from
|
||||
# `secret/swarm/hives/<hive>/archive/`.
|
||||
hive,
|
||||
# The directory holding one directory per agent.
|
||||
agentRoot,
|
||||
# The repository server's base URL, no trailing slash.
|
||||
url,
|
||||
# "paths": `paths` under each agent's directory. "subvolume": the whole directory.
|
||||
scope,
|
||||
paths,
|
||||
excludes,
|
||||
# A CA file to trust instead of restic's default, or null.
|
||||
caBundle,
|
||||
}:
|
||||
let
|
||||
storeSeededSecret = import ./store-seeded-secret.nix { };
|
||||
|
||||
grants = hive: "secret/swarm/hives/${hive}/archive";
|
||||
in
|
||||
''
|
||||
set -euo pipefail
|
||||
${storeSeededSecret}
|
||||
|
||||
store_init
|
||||
store_login
|
||||
if [ "$STORE_UP" != 1 ]; then
|
||||
echo "archive backup failed: could not log in to the swarm secret store" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
${lib.optionalString (caBundle != null) ''
|
||||
export RESTIC_CACERT=${lib.escapeShellArg caBundle}
|
||||
''}
|
||||
root=${lib.escapeShellArg agentRoot}
|
||||
grants=${lib.escapeShellArg (grants hive)}
|
||||
|
||||
# One agent: a read-only snapshot of its subvolume when it is one, so the
|
||||
# databases are copied at one instant, else its directory in place.
|
||||
backup_agent() {
|
||||
local agent="$1" leaf="$grants/$1" rest base snap="" target targets=()
|
||||
rest="$(store_bao kv get -field=rest-password "$leaf")" || return 1
|
||||
export RESTIC_REPOSITORY="rest:${url}/$agent/"
|
||||
export RESTIC_REST_USERNAME="$agent"
|
||||
export RESTIC_REST_PASSWORD="$rest"
|
||||
export RESTIC_PASSWORD_COMMAND="bao kv get -field=restic-password $leaf"
|
||||
|
||||
if ! ${restic}/bin/restic cat config >/dev/null 2>&1; then
|
||||
${restic}/bin/restic init || return 1
|
||||
fi
|
||||
|
||||
base="$root/$agent"
|
||||
if btrfs subvolume show "$base" >/dev/null 2>&1; then
|
||||
snap="$root/.$agent.snapshot.archive"
|
||||
btrfs subvolume delete "$snap" >/dev/null 2>&1 || true
|
||||
btrfs subvolume snapshot -r "$base" "$snap" >/dev/null || return 1
|
||||
base="$snap"
|
||||
fi
|
||||
|
||||
${
|
||||
if scope == "subvolume" then
|
||||
''targets=("$base")''
|
||||
else
|
||||
''
|
||||
for target in ${lib.escapeShellArgs paths}; do
|
||||
[ -e "$base/$target" ] && targets+=("$base/$target")
|
||||
done''
|
||||
}
|
||||
local rc=0
|
||||
if [ "''${#targets[@]}" = 0 ]; then
|
||||
echo "$agent has none of its backup paths yet; nothing to back up."
|
||||
else
|
||||
${restic}/bin/restic backup --host "$agent" --tag archive --exclude-caches \
|
||||
${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \
|
||||
-- "''${targets[@]}" || rc=$?
|
||||
fi
|
||||
if [ -n "$snap" ]; then
|
||||
btrfs subvolume delete "$snap" >/dev/null || rc=1
|
||||
fi
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# The agents the controller granted this hive; none yet is "No value found".
|
||||
rc=0
|
||||
granted="$(store_bao kv list -format=json "$grants" 2>"$STORE_ERR")" || rc=$?
|
||||
if [ "$rc" != 0 ]; then
|
||||
if grep -q 'No value found' "$STORE_ERR"; then
|
||||
granted='[]'
|
||||
else
|
||||
cat "$STORE_ERR" >&2
|
||||
echo "archive backup failed: could not list $grants" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
granted="$(printf '%s' "$granted" | jq -r '.[]')"
|
||||
|
||||
failed=0
|
||||
candidates=0
|
||||
backed_up=0
|
||||
for dir in "$root"/*/; do
|
||||
agent="$(basename -- "$dir")"
|
||||
case "$agent" in .*) continue ;; esac
|
||||
[[ "$agent" =~ ^[A-Za-z0-9_-]+$ ]] || continue
|
||||
candidates=$((candidates + 1))
|
||||
if ! grep -qxF -- "$agent" <<<"$granted"; then
|
||||
echo "$agent has no archive credentials on this hive; skipping."
|
||||
continue
|
||||
fi
|
||||
if (backup_agent "$agent"); then
|
||||
backed_up=$((backed_up + 1))
|
||||
echo "backed up $agent"
|
||||
else
|
||||
echo "archive backup failed: $agent" >&2
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
# Agents here and not one of them granted: swarm-controller has not written
|
||||
# this hive's grants, which a controller without the archive never does.
|
||||
if [ "$candidates" -gt 0 ] && [ -z "$granted" ]; then
|
||||
echo "archive backup failed: swarm-controller has granted this hive no archive credentials ($candidates agents here)" >&2
|
||||
failed=1
|
||||
fi
|
||||
exit "$failed"
|
||||
''
|
||||
156
nix/host-modules/lib/archive-server.nix
Normal file
156
nix/host-modules/lib/archive-server.nix
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
# The archive server's moving parts: the rclone command that serves the
|
||||
# repositories, the script that renders its users file from the secret store,
|
||||
# and the script that prunes each repository. A pure function, NOT a NixOS
|
||||
# module: ../swarm-archive.nix wraps the pieces in units, and the VM test
|
||||
# (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in
|
||||
# `bao`.
|
||||
#
|
||||
# Both scripts use only `bao login -method=cert -token-only`,
|
||||
# `bao kv list -format=json` and `bao kv get -field=…`, and need `bao`, `jq`,
|
||||
# `coreutils`, `gnugrep` on the unit's `path` and the `BAO_*` client set.
|
||||
{ lib }:
|
||||
{
|
||||
rclone,
|
||||
restic,
|
||||
# Where the repositories live: `<reposDir>/<agent>/`. The only place that
|
||||
# names the backend; `rclone serve restic` takes any remote here.
|
||||
reposDir,
|
||||
# The address rclone listens on.
|
||||
addr,
|
||||
# The users file rclone checks logins against, its group and the user the
|
||||
# server runs as.
|
||||
authFile,
|
||||
authGroup,
|
||||
# `{ keepWithin, keepLast }`; a null `keepWithin` leaves that flag off.
|
||||
retention,
|
||||
}:
|
||||
let
|
||||
atomicWriteSecret = import ./atomic-write-secret.nix { };
|
||||
refreshConsumer = import ./refresh-consumer.nix { };
|
||||
storeSeededSecret = import ./store-seeded-secret.nix { };
|
||||
|
||||
usersPrefix = "secret/swarm/services/archive/users";
|
||||
resticPrefix = "secret/swarm/services/archive/restic";
|
||||
|
||||
keepFlags = lib.concatStringsSep " " (
|
||||
[ "--keep-last ${toString retention.keepLast}" ]
|
||||
++ lib.optional (
|
||||
retention.keepWithin != null
|
||||
) "--keep-within ${lib.escapeShellArg retention.keepWithin}"
|
||||
);
|
||||
in
|
||||
{
|
||||
inherit usersPrefix resticPrefix;
|
||||
|
||||
# Append-only: a client cannot delete or overwrite a snapshot, a pack or the
|
||||
# index, so a hive that is compromised cannot erase history. Private repos:
|
||||
# a login reaches `/<its own name>/` and nothing else.
|
||||
serverCommand = lib.escapeShellArgs [
|
||||
"${rclone}/bin/rclone"
|
||||
"serve"
|
||||
"restic"
|
||||
"--addr"
|
||||
addr
|
||||
"--append-only"
|
||||
"--private-repos"
|
||||
"--htpasswd"
|
||||
authFile
|
||||
reposDir
|
||||
];
|
||||
|
||||
# Renders the users file from `usersPrefix`, one leaf per agent holding that
|
||||
# agent's htpasswd line. The file is written only when the content changed,
|
||||
# and never from a partial read: any leaf that cannot be read leaves the file
|
||||
# as it is.
|
||||
usersScript = ''
|
||||
set -euo pipefail
|
||||
${storeSeededSecret}
|
||||
${atomicWriteSecret}
|
||||
${refreshConsumer}
|
||||
|
||||
file=${lib.escapeShellArg authFile}
|
||||
# The server will not start without the file; an empty one admits nobody.
|
||||
if [ ! -e "$file" ]; then
|
||||
atomic_write_secret 0640 root:${authGroup} "$file" ""
|
||||
fi
|
||||
|
||||
store_init
|
||||
store_login
|
||||
if [ "$STORE_UP" != 1 ]; then
|
||||
echo "could not log in to the swarm secret store; $file stays as it is." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A prefix that holds nothing is "No value found"; it means no agent has a
|
||||
# user yet, not a failure.
|
||||
rc=0
|
||||
names="$(store_bao kv list -format=json ${lib.escapeShellArg usersPrefix} 2>"$STORE_ERR")" || rc=$?
|
||||
if [ "$rc" != 0 ]; then
|
||||
if grep -q 'No value found' "$STORE_ERR"; then
|
||||
names='[]'
|
||||
else
|
||||
cat "$STORE_ERR" >&2
|
||||
echo "could not list ${usersPrefix}; $file stays as it is." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
content=""
|
||||
while IFS= read -r name; do
|
||||
[ -n "$name" ] || continue
|
||||
case "$name" in */) continue ;; esac
|
||||
if ! line="$(store_bao kv get -field=htpasswd ${lib.escapeShellArg usersPrefix}/"$name" 2>"$STORE_ERR")"; then
|
||||
cat "$STORE_ERR" >&2
|
||||
echo "could not read the user $name; $file stays as it is." >&2
|
||||
exit 1
|
||||
fi
|
||||
# The line must be this agent's own: a leaf cannot add another user.
|
||||
if [[ ! "$name" =~ ^[A-Za-z0-9_-]+$ ]] || [[ "$line" != "$name:{SHA}"* ]] \
|
||||
|| [[ ! "$line" =~ ^[A-Za-z0-9_-]+:\{SHA\}[A-Za-z0-9+/=]+$ ]]; then
|
||||
echo "the user leaf $name does not hold a {SHA} line for $name; leaving it out." >&2
|
||||
continue
|
||||
fi
|
||||
content+="$line"$'\n'
|
||||
done < <(printf '%s' "$names" | jq -r '.[]')
|
||||
content="''${content%$'\n'}"
|
||||
|
||||
if secret_differs "$file" "$content"; then
|
||||
atomic_write_secret 0640 root:${authGroup} "$file" "$content"
|
||||
echo "users file updated."
|
||||
fi
|
||||
'';
|
||||
|
||||
# `restic forget --prune` on every repository, with its password read from
|
||||
# the store at run time. The server refuses deletes from clients, so this is
|
||||
# the only place history is dropped.
|
||||
pruneScript = ''
|
||||
set -euo pipefail
|
||||
${storeSeededSecret}
|
||||
|
||||
store_init
|
||||
store_login
|
||||
if [ "$STORE_UP" != 1 ]; then
|
||||
echo "archive prune failed: could not log in to the swarm secret store" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
failed=0
|
||||
for repo in ${lib.escapeShellArg reposDir}/*/; do
|
||||
[ -e "$repo/config" ] || continue
|
||||
agent="$(basename -- "$repo")"
|
||||
if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then
|
||||
echo "archive prune failed: $agent is not an agent name" >&2
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
|
||||
if ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
|
||||
echo "pruned $agent"
|
||||
else
|
||||
echo "archive prune failed: $agent" >&2
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
exit "$failed"
|
||||
'';
|
||||
}
|
||||
348
nix/host-modules/swarm-archive.nix
Normal file
348
nix/host-modules/swarm-archive.nix
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
# The swarm's backup archive: `rclone serve restic` on the host that sets
|
||||
# `deploy.archive.enable`, behind the gateway at `archive.<swarm domain>`.
|
||||
# Every hive's backup unit (./hive-archive-backup.nix) writes one restic
|
||||
# repository per agent to it; nothing else uses it.
|
||||
#
|
||||
# - `--append-only`: a client cannot delete or overwrite a snapshot, a pack or
|
||||
# the index, so a compromised hive cannot erase history. Dropping history is
|
||||
# the prune unit's job, on this host, by retention.
|
||||
# - `--private-repos`: a login reaches `/<its own name>/` only. There is one
|
||||
# login per agent, created by swarm-controller (../../swarm-controller/src/archive_access.rs)
|
||||
# when it places the agent, replaced when it moves and removed when it is
|
||||
# destroyed. An agent's hive can write that agent's repository and no other.
|
||||
#
|
||||
# Everything secret lives in the swarm secret store and is read when needed:
|
||||
#
|
||||
# - `swarm-archive-users` (root, every two minutes) renders the htpasswd file
|
||||
# from the users leaves into `/run`, which is tmpfs. rclone re-reads the file
|
||||
# when it changes. The leaves hold password hashes only.
|
||||
# - `swarm-archive-prune` (daily) reads each agent's restic password through
|
||||
# `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can
|
||||
# therefore decrypt every backup, which is the cost of pruning where clients
|
||||
# cannot delete.
|
||||
#
|
||||
# Both units log in as the archive's own store principal (a leaf of the store's
|
||||
# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container:
|
||||
# no secret crosses a mount, and the backend is one string
|
||||
# (`reposDir`, in ./lib/archive-server.nix) that names where the repositories
|
||||
# are.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
cfg = hyperhiveCfg.deploy.archive;
|
||||
archiveCfg = hyperhiveCfg.swarm.archive;
|
||||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
baoDeploy = hyperhiveCfg.deploy.bao;
|
||||
|
||||
swarmDomain = hyperhiveCfg.swarm.domain;
|
||||
# Total on a null swarm domain, as the sibling options modules are: the
|
||||
# required-domain assertion in hive-network.nix is what an operator sees.
|
||||
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
|
||||
|
||||
user = "swarm-archive";
|
||||
authDir = "/run/swarm-archive-auth";
|
||||
authFile = "${authDir}/htpasswd";
|
||||
reposDir = "${cfg.dataDir}/repos";
|
||||
|
||||
server = import ./lib/archive-server.nix { inherit lib; } {
|
||||
inherit (cfg) retention;
|
||||
inherit authFile reposDir;
|
||||
rclone = cfg.package;
|
||||
restic = cfg.resticPackage;
|
||||
addr = "127.0.0.1:${toString cfg.port}";
|
||||
authGroup = user;
|
||||
};
|
||||
|
||||
storeRetry = import ./lib/store-retry.nix { };
|
||||
|
||||
haveClientIdentity =
|
||||
baoDeploy.archiveClientCertFile != null && baoDeploy.archiveClientKeyFile != null;
|
||||
|
||||
storeEnvironment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = baoDeploy.serverCaFile;
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.swarm.archive.ownDomain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "archive.${domainBase}";
|
||||
defaultText = lib.literalExpression ''"archive.''${services.hyperhive.swarm.domain}"'';
|
||||
description = ''
|
||||
Read-only: the name the swarm's backup archive is served under and every
|
||||
hive's backup unit dials. A sibling of the swarm's other service names, so
|
||||
the swarm-services sub-CA can issue for it, and in `swarm.serviceDomains`
|
||||
on every host whether or not one runs the archive.
|
||||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.archive = {
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = pkgs.rclone;
|
||||
defaultText = lib.literalExpression "pkgs.rclone";
|
||||
description = "The rclone package: `rclone serve restic`.";
|
||||
};
|
||||
|
||||
resticPackage = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = pkgs.restic;
|
||||
defaultText = lib.literalExpression "pkgs.restic";
|
||||
description = "The restic package the prune unit runs.";
|
||||
};
|
||||
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8911;
|
||||
description = ''
|
||||
Port rclone listens on, on the host's loopback. The gateway's vhost
|
||||
proxies to it; every login is checked by rclone itself.
|
||||
'';
|
||||
};
|
||||
|
||||
dataDir = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
default = "/var/lib/swarm-archive";
|
||||
description = ''
|
||||
Where the repositories live, under `repos/<agent>/`. Holds restic
|
||||
ciphertext: every backup of every agent, so size it for that.
|
||||
'';
|
||||
};
|
||||
|
||||
retention = {
|
||||
keepWithin = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = "3d";
|
||||
example = "14d";
|
||||
description = ''
|
||||
Every snapshot younger than this is kept (restic's `--keep-within`,
|
||||
written `1y2m3d4h`). It bounds how fast history can be evicted: a
|
||||
hive that appends many snapshots cannot push the last few days out.
|
||||
`null` leaves the flag off.
|
||||
'';
|
||||
};
|
||||
|
||||
keepLast = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 3;
|
||||
description = ''
|
||||
The newest this many snapshots of each agent are kept, however old
|
||||
(restic's `--keep-last`). Together with
|
||||
{option}`services.hyperhive.deploy.archive.retention.keepWithin` the
|
||||
prune keeps a snapshot either rule keeps.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
pruneSchedule = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "daily";
|
||||
example = "*-*-* 04:00:00";
|
||||
description = "systemd calendar expression for the prune: the retention rules are applied then.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = haveClientIdentity;
|
||||
message = ''
|
||||
services.hyperhive.deploy.archive.enable requires the archive's
|
||||
swarm-secret-store client identity: set both
|
||||
|
||||
services.hyperhive.deploy.bao.archiveClientCertFile
|
||||
services.hyperhive.deploy.bao.archiveClientKeyFile
|
||||
|
||||
The archive reads its users and the repositories' passwords in the
|
||||
store with it.
|
||||
|
||||
Both default to the pair glue-bao-tls.nix mints when
|
||||
services.hyperhive.deploy.bao.enable is true (the host that runs the
|
||||
store). Otherwise copy archive.pem and archive-key.pem from the store
|
||||
host's /var/lib/swarm-bao-pki/ and name them in those two options.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
users.users.${user} = {
|
||||
isSystemUser = true;
|
||||
group = user;
|
||||
description = "swarm backup archive";
|
||||
};
|
||||
users.groups.${user} = { };
|
||||
|
||||
# This host serves the vhost, and the host's own resolver finds the name.
|
||||
services.hyperhive.gateway.localNames = [ archiveCfg.ownDomain ];
|
||||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||||
|
||||
# `forceSSL`: the logins ride in a header. No `auth_request`: rclone checks
|
||||
# them, and restic cannot follow a login redirect. The vhost keeps the
|
||||
# gateway's default `public`: `public = false` needs the gateway's
|
||||
# `privateListenAddresses` set, and every request is authenticated either
|
||||
# way. A backup is one request that can run for minutes, so the body is
|
||||
# neither size-limited nor buffered.
|
||||
services.hyperhive.gateway.vhosts."${archiveCfg.ownDomain}" = {
|
||||
forceSSL = true;
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
||||
extraConfig = ''
|
||||
client_max_body_size 0;
|
||||
proxy_request_buffering off;
|
||||
proxy_read_timeout 1h;
|
||||
proxy_send_timeout 1h;
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [
|
||||
"swarm-archive.service"
|
||||
"swarm-archive-users.service"
|
||||
"swarm-archive-prune.service"
|
||||
];
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${cfg.dataDir} 0750 ${user} ${user} -"
|
||||
"d ${reposDir} 0750 ${user} ${user} -"
|
||||
"d ${authDir} 0750 root ${user} -"
|
||||
];
|
||||
|
||||
systemd.services.swarm-archive = {
|
||||
description = "swarm backup archive (rclone serve restic, append-only, one repository per agent)";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
wants = [ "swarm-archive-users.service" ];
|
||||
after = [
|
||||
"network.target"
|
||||
"swarm-archive-users.service"
|
||||
"systemd-tmpfiles-setup.service"
|
||||
];
|
||||
# rclone writes no config, and reads none.
|
||||
environment.RCLONE_CONFIG = "/dev/null";
|
||||
serviceConfig = {
|
||||
ExecStart = server.serverCommand;
|
||||
User = user;
|
||||
Group = user;
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
SyslogIdentifier = "swarm-archive";
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
PrivateDevices = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectControlGroups = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_INET"
|
||||
"AF_INET6"
|
||||
];
|
||||
ReadWritePaths = [ reposDir ];
|
||||
ReadOnlyPaths = [ authDir ];
|
||||
};
|
||||
};
|
||||
|
||||
# The users file, from the store. A failed run leaves the file as it is;
|
||||
# the timer and the retry bring a store that comes back up to date.
|
||||
systemd.services.swarm-archive-users = {
|
||||
description = "render the archive's users file from the swarm secret store";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
after = [
|
||||
"network-online.target"
|
||||
"systemd-tmpfiles-setup.service"
|
||||
];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
pkgs.jq
|
||||
pkgs.systemd
|
||||
];
|
||||
environment = storeEnvironment // {
|
||||
BAO_CLIENT_CERT = baoDeploy.archiveClientCertFile;
|
||||
BAO_CLIENT_KEY = baoDeploy.archiveClientKeyFile;
|
||||
};
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
UMask = "0027";
|
||||
SyslogIdentifier = "swarm-archive-users";
|
||||
TimeoutStartSec = "5min";
|
||||
};
|
||||
script = server.usersScript;
|
||||
};
|
||||
|
||||
# A new grant reaches the server within the controller's pass (five
|
||||
# minutes) plus this interval.
|
||||
systemd.timers.swarm-archive-users = {
|
||||
description = "re-render the archive's users file";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnUnitInactiveSec = "2min";
|
||||
RandomizedDelaySec = "20s";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.swarm-archive-prune = {
|
||||
description = "apply the archive's retention to every agent's repository";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
cfg.resticPackage
|
||||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
pkgs.jq
|
||||
];
|
||||
# The unit runs as the repositories' owner, so the store identity comes
|
||||
# in as credentials, which only it can read.
|
||||
environment = {
|
||||
BAO_ADDR = storeEnvironment.BAO_ADDR;
|
||||
BAO_CLIENT_CERT = "%d/bao-cert";
|
||||
BAO_CLIENT_KEY = "%d/bao-key";
|
||||
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = "%d/bao-ca";
|
||||
};
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = user;
|
||||
Group = user;
|
||||
SyslogIdentifier = "swarm-archive-prune";
|
||||
CacheDirectory = "swarm-archive-prune";
|
||||
LoadCredential = [
|
||||
"bao-cert:${baoDeploy.archiveClientCertFile}"
|
||||
"bao-key:${baoDeploy.archiveClientKeyFile}"
|
||||
]
|
||||
++ lib.optional (baoDeploy.serverCaFile != null) "bao-ca:${baoDeploy.serverCaFile}";
|
||||
NoNewPrivileges = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
PrivateDevices = true;
|
||||
ReadWritePaths = [ reposDir ];
|
||||
TimeoutStartSec = "6h";
|
||||
};
|
||||
script = server.pruneScript;
|
||||
};
|
||||
|
||||
systemd.timers.swarm-archive-prune = {
|
||||
description = "apply the archive's retention";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.pruneSchedule;
|
||||
RandomizedDelaySec = "30min";
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -518,7 +518,39 @@ let
|
|||
capabilities = ["create", "read", "update"]
|
||||
}
|
||||
'') atticHiveFactNames}
|
||||
path "${credentialMountPath}/data/${ciRunnerTokenLeaf}" {
|
||||
# The archive credentials of the agents each hive places
|
||||
# (swarm-controller's archive_access.rs): the hive's copy and the archive's
|
||||
# hash and restic password, listed to find what no placement justifies and
|
||||
# deleted when a move or a destroy revokes it.
|
||||
path "${credentialMountPath}/data/swarm/hives/+/archive/*" {
|
||||
capabilities = ["create", "read", "update"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/metadata/swarm/hives/+/archive/" {
|
||||
capabilities = ["list"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/metadata/swarm/hives/+/archive/+" {
|
||||
capabilities = ["delete"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/data/swarm/services/archive/users/*" {
|
||||
capabilities = ["create", "read", "update"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/data/swarm/services/archive/restic/*" {
|
||||
capabilities = ["create", "read"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/metadata/swarm/services/archive/users/" {
|
||||
capabilities = ["list"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/metadata/swarm/services/archive/users/+" {
|
||||
capabilities = ["delete"]
|
||||
}
|
||||
|
||||
path "${credentialMountPath}/data/${ciRunnerTokenLeaf}" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
|
|
@ -998,7 +1030,11 @@ let
|
|||
"public-key"
|
||||
"pull-token"
|
||||
]
|
||||
+ readStanza "${credentialMountPath}/data/swarm/hives/${obj.hiveName}/squid-selfcheck";
|
||||
+ readStanza "${credentialMountPath}/data/swarm/hives/${obj.hiveName}/squid-selfcheck"
|
||||
# The archive credentials swarm-controller writes for the agents this hive
|
||||
# places: the backup unit (./hive-archive-backup.nix) lists them and reads each.
|
||||
+ readStanza "${credentialMountPath}/data/swarm/hives/${obj.hiveName}/archive/*"
|
||||
+ listStanza "${credentialMountPath}/metadata/swarm/hives/${obj.hiveName}/archive/";
|
||||
|
||||
# The PKI mount agent client certificates are issued from. Its root is
|
||||
# generated inside the store, so the agent CA's key never exists outside it.
|
||||
|
|
@ -1103,6 +1139,12 @@ let
|
|||
}
|
||||
'';
|
||||
|
||||
listStanza = path: ''
|
||||
path "${path}" {
|
||||
capabilities = ["list"]
|
||||
}
|
||||
'';
|
||||
|
||||
# Singletons, so a list of one rather than a second shape: the granting unit
|
||||
# below takes a list either way and the units stay identical apart from
|
||||
# the objects they write.
|
||||
|
|
@ -1339,6 +1381,20 @@ let
|
|||
}
|
||||
];
|
||||
|
||||
# The backup archive: the users file is rendered from one leaf per agent, and
|
||||
# the prune reads each agent's restic password. Read and list only; the
|
||||
# controller writes both.
|
||||
archiveReaders = [
|
||||
{
|
||||
name = "swarm-archive";
|
||||
cn = baoDeploy.archiveCommonName;
|
||||
policyText =
|
||||
readStanza "${credentialMountPath}/data/swarm/services/archive/users/*"
|
||||
+ readStanza "${credentialMountPath}/data/swarm/services/archive/restic/*"
|
||||
+ listStanza "${credentialMountPath}/metadata/swarm/services/archive/users/";
|
||||
}
|
||||
];
|
||||
|
||||
# The CI run poller: its forge token and its own OIDC client secret, both
|
||||
# read once per pass by the host unit.
|
||||
ciMetricsReaders = [
|
||||
|
|
@ -2611,6 +2667,42 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
archiveCommonName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "swarm-archive";
|
||||
description = ''
|
||||
Subject the store's `swarm-archive` cert-auth role accepts: the identity
|
||||
the backup archive's host units present. Its grant is read on
|
||||
`swarm/services/archive/users/*` and `restic/*`, and list on `users`.
|
||||
|
||||
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
|
||||
'';
|
||||
};
|
||||
|
||||
archiveClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-bao-pki/archive.pem";
|
||||
description = ''
|
||||
Certificate the backup archive's host units present to the store. Its
|
||||
subject must be {option}`services.hyperhive.deploy.bao.archiveCommonName`.
|
||||
|
||||
./glue-bao-tls.nix points it at the leaf it mints on the store's host.
|
||||
On an archive host without the store, it is a file the operator copies
|
||||
from the store's host. A path, never a value.
|
||||
'';
|
||||
};
|
||||
|
||||
archiveClientKeyFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-bao-pki/archive-key.pem";
|
||||
description = ''
|
||||
Private key for {option}`services.hyperhive.deploy.bao.archiveClientCertFile`.
|
||||
A path, never a value.
|
||||
'';
|
||||
};
|
||||
|
||||
atticClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
|
|
@ -3847,6 +3939,8 @@ in
|
|||
|
||||
systemd.services.swarm-bao-attic-policy = readerPolicyUnit "write the in-swarm cache's secret-creator bao policy and cert-auth role" atticWriters;
|
||||
|
||||
systemd.services.swarm-bao-archive-policy = readerPolicyUnit "write the backup archive's user and password reader bao policy and cert-auth role" archiveReaders;
|
||||
|
||||
systemd.services.swarm-bao-notify-router-policy = readerPolicyUnit "write the notification router's client-secret-reader bao policy and cert-auth role" notifyRouterReaders;
|
||||
|
||||
systemd.services.swarm-bao-forge-publisher-policy = readerPolicyUnit "write the forge's controller-token-publisher bao policy and cert-auth role" forgePublisherWriters;
|
||||
|
|
|
|||
|
|
@ -1684,6 +1684,29 @@ in
|
|||
for = "0s";
|
||||
summary = "swarm-controller failed to mint agent {{ $labels.F_AGENT }}'s store identity; the reason is the `F_ERROR` field of the swarm-controller log line";
|
||||
})
|
||||
# `hive-archive-backup` (nix/host-modules/hive-archive-backup.nix)
|
||||
# prints one `archive backup failed: <agent>` line per agent
|
||||
# it could not back up, and one for a hive swarm-controller
|
||||
# has granted nothing. The backup runs hourly, so `for`
|
||||
# rides out the minutes a newly placed agent waits for its
|
||||
# grant, and a second failed run keeps it firing.
|
||||
(logCountRule {
|
||||
uid = "hh-archive-backup-failed";
|
||||
title = "archive backup failed";
|
||||
windowSeconds = 7800;
|
||||
expr = ''_time:130m _SYSTEMD_UNIT:="hive-archive-backup.service" _msg:"archive backup failed" | stats by (_HOSTNAME) count() as failures'';
|
||||
for = "30m";
|
||||
summary = "{{ $labels._HOSTNAME }} failed to back up an agent to the archive; the unit's journal names the agent and the cause";
|
||||
})
|
||||
# `swarm-archive-prune` prints `archive prune failed: <agent>`.
|
||||
(logCountRule {
|
||||
uid = "hh-archive-prune-failed";
|
||||
title = "archive prune failed";
|
||||
windowSeconds = 90000;
|
||||
expr = ''_time:25h _SYSTEMD_UNIT:="swarm-archive-prune.service" _msg:"archive prune failed" | stats count() as failures'';
|
||||
for = "0s";
|
||||
summary = "the archive's daily prune failed for an agent's repository, so its retention is not applied; the unit's journal names the agent";
|
||||
})
|
||||
# Logged once per detected crash, by `emit_crash_transitions`
|
||||
# in hive-c0re's crash_watch.rs (10s poll interval). Replaces
|
||||
# the old per-hive dashboard `agents_crashing` banner — see
|
||||
|
|
|
|||
|
|
@ -139,6 +139,14 @@ in
|
|||
&& deployCfg.bao.atticClientKeyFile != null
|
||||
);
|
||||
|
||||
# The backup archive, on the same terms: its units log in to the store as the
|
||||
# archive's own principal.
|
||||
config.services.hyperhive.deploy.archive.enable = lib.mkDefault (
|
||||
deployCfg.allSwarmServices
|
||||
&& deployCfg.bao.archiveClientCertFile != null
|
||||
&& deployCfg.bao.archiveClientKeyFile != null
|
||||
);
|
||||
|
||||
# The claude login minter, defaulted on the same terms as the review
|
||||
# classifier.
|
||||
config.services.hyperhive.deploy.claude-login.enable = lib.mkDefault (
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ let
|
|||
deployCfg.bao.claudeLoginCommonName
|
||||
deployCfg.bao.notifyRouterCommonName
|
||||
deployCfg.bao.atticCommonName
|
||||
deployCfg.bao.archiveCommonName
|
||||
deployCfg.bao.forgePublisherCommonName
|
||||
]
|
||||
# The per-hive subject, spelled out per hive rather than as the prefix.
|
||||
|
|
@ -117,6 +118,8 @@ let
|
|||
# host runs one, as every name here is. `swarm.attic.domain` is not: it is
|
||||
# null where there is no cache and may name an external one.
|
||||
swarmCfg.attic.ownDomain
|
||||
# The backup archive, listed whether or not any host runs one.
|
||||
swarmCfg.archive.ownDomain
|
||||
# The store's browser UI, not the store: `swarm.bao.domain` is mTLS and
|
||||
# never gets a gateway certificate.
|
||||
swarmCfg.bao.ui.domain
|
||||
|
|
|
|||
|
|
@ -195,6 +195,7 @@ let
|
|||
"swarm-bao-claude-login-policy"
|
||||
"swarm-bao-notify-router-policy"
|
||||
"swarm-bao-attic-policy"
|
||||
"swarm-bao-archive-policy"
|
||||
];
|
||||
|
||||
# Comment lines dropped first: both the HCL and the scripts explain
|
||||
|
|
@ -2166,6 +2167,30 @@ let
|
|||
in
|
||||
lib.hasInfix "/notify-router.pem ]" s && lib.hasInfix "swarm-notify-router \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
# The archive reads the users and the repositories' passwords and writes
|
||||
# nothing; the hive-side copies of the REST password are not its to read.
|
||||
# The controller is the only writer: the control is its own grant.
|
||||
name = "the archive's grant is read on its users and restic passwords, list on its users, and nothing else, and only the controller writes them";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-archive-policy.script;
|
||||
grants = grantsIn s;
|
||||
c = grantsIn baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
(grantFor grants "secret/data/swarm/services/archive/users/atlas").caps == [ "read" ]
|
||||
&& (grantFor grants "secret/data/swarm/services/archive/restic/atlas").caps == [ "read" ]
|
||||
&& (grantFor grants "secret/metadata/swarm/services/archive/users/").caps == [ "list" ]
|
||||
&& grantFor grants "secret/data/swarm/hives/h1/archive/atlas" == null
|
||||
&& lib.length grants == 3
|
||||
&& lib.hasInfix "allowed_common_names=swarm-archive" s
|
||||
&& lib.hasInfix "token_policies=swarm-archive" s
|
||||
&& lib.elem "create" (grantFor c "secret/data/swarm/services/archive/users/atlas").caps
|
||||
&& lib.elem "create" (grantFor c "secret/data/swarm/hives/h1/archive/atlas").caps
|
||||
&& (grantFor c "secret/metadata/swarm/hives/h1/archive/atlas").caps == [ "delete" ]
|
||||
&& (grantFor c "secret/metadata/swarm/services/archive/users/atlas").caps == [ "delete" ]
|
||||
&& !(lib.elem "delete" (grantFor c "secret/data/swarm/services/archive/restic/atlas").caps);
|
||||
}
|
||||
{
|
||||
# The cache's host units create each of its six secrets and read them
|
||||
# back, and replace only the public key: `update` there and nowhere else.
|
||||
|
|
|
|||
|
|
@ -561,6 +561,45 @@ let
|
|||
&& names atticExternal == names bare
|
||||
&& !(lib.elem "cache.example.org" (names atticExternal));
|
||||
}
|
||||
{
|
||||
# The archive on the host that hosts the shared services. Append-only and
|
||||
# private repos are what make one hive's compromise and one agent's
|
||||
# login small; the control is the bare host, which runs no archive.
|
||||
name = "the archive serves append-only with private repositories behind a TLS-only vhost, and a host without it renders none";
|
||||
ok =
|
||||
let
|
||||
a = atticHere.systemd.services.swarm-archive;
|
||||
users = atticHere.systemd.services.swarm-archive-users;
|
||||
prune = atticHere.systemd.services.swarm-archive-prune;
|
||||
vhost = atticHere.services.hyperhive.gateway.vhosts."archive.t.local";
|
||||
cfg = atticHere.services.hyperhive.deploy.archive;
|
||||
names = m: m.services.hyperhive.swarm.serviceDomains;
|
||||
in
|
||||
atticHere.services.hyperhive.deploy.archive.enable
|
||||
&& lib.hasInfix "--append-only" a.serviceConfig.ExecStart
|
||||
&& lib.hasInfix "--private-repos" a.serviceConfig.ExecStart
|
||||
&& lib.hasInfix "--htpasswd /run/swarm-archive-auth/htpasswd" a.serviceConfig.ExecStart
|
||||
&& lib.hasInfix "/var/lib/swarm-archive/repos" a.serviceConfig.ExecStart
|
||||
&& a.serviceConfig.User == "swarm-archive"
|
||||
&& vhost.forceSSL
|
||||
&& !(lib.hasInfix "auth_request" vhost.locations."/".extraConfig)
|
||||
&& vhost.locations."/".proxyPass == "http://127.0.0.1:8911"
|
||||
&& lib.elem "archive.t.local" (names atticHere)
|
||||
&& names atticHere == names bare
|
||||
&& cfg.retention.keepWithin == "3d"
|
||||
&& cfg.retention.keepLast == 3
|
||||
&& lib.hasInfix "--keep-last 3 --keep-within 3d --prune" prune.script
|
||||
&& prune.serviceConfig.User == "swarm-archive"
|
||||
&& lib.hasInfix "RESTIC_PASSWORD_COMMAND=" prune.script
|
||||
&& lib.hasInfix "secret/swarm/services/archive/users" users.script
|
||||
&& users.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/archive.pem"
|
||||
&& !(lib.hasInfix "password" (lib.toLower (builtins.toJSON a.environment)))
|
||||
&& atticHere.systemd.timers ? swarm-archive-users
|
||||
&& atticHere.systemd.timers.swarm-archive-prune.timerConfig.OnCalendar == "daily"
|
||||
&& lib.elem "swarm-bao-archive-policy.service" users.after
|
||||
&& !(bare.systemd.services ? swarm-archive)
|
||||
&& !(bare.systemd.services ? swarm-archive-users);
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "builder" cases
|
||||
|
|
|
|||
|
|
@ -802,6 +802,31 @@ let
|
|||
noService.systemd.services.hive-priv-cache-facts.environment.BAO_CLIENT_CERT
|
||||
== "/etc/pki/bao-hive.pem";
|
||||
}
|
||||
{
|
||||
# ./hive-archive-backup.nix. The grants are read under this hive's own
|
||||
# identity and only from its own prefix; the swarm's password lives in no
|
||||
# file and no unit text. The no-identity host is the control that the
|
||||
# lookup is not always empty.
|
||||
name = "the hive backs up its agents to the archive under its own identity, hourly, reading only its own grants";
|
||||
ok =
|
||||
let
|
||||
u = units.hive-archive-backup;
|
||||
t = allLocal.systemd.timers.hive-archive-backup;
|
||||
in
|
||||
lib.hasInfix "secret/swarm/hives/h1/archive" u.script
|
||||
&& !(lib.hasInfix "swarm/services" u.script)
|
||||
&& !(lib.hasInfix "swarm/hives/h2" u.script)
|
||||
&& lib.hasInfix "rest:https://archive.t.local/" u.script
|
||||
&& lib.hasInfix "RESTIC_PASSWORD_COMMAND=" u.script
|
||||
&& lib.hasInfix "--exclude-caches" u.script
|
||||
&& lib.hasInfix "for target in state claude harness" u.script
|
||||
&& u.environment.BAO_CLIENT_CERT == units.hive-priv-cache-facts.environment.BAO_CLIENT_CERT
|
||||
&& u.serviceConfig.Type == "oneshot"
|
||||
&& t.timerConfig.OnCalendar == "hourly"
|
||||
&& lib.elem "timers.target" t.wantedBy
|
||||
&& !(noIdentity.systemd.services ? hive-archive-backup)
|
||||
&& !(noIdentity.systemd.timers ? hive-archive-backup);
|
||||
}
|
||||
{
|
||||
# Control: the lookup finds the sibling root unit of the same shape. The
|
||||
# squid self-check's password is read under this hive's own identity and
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
agent-ephemeral-rootfs = import ./vm-tests/agent-ephemeral-rootfs.nix { inherit pkgs hivePriv; };
|
||||
wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; };
|
||||
hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; };
|
||||
swarm-archive = import ./vm-tests/swarm-archive.nix { inherit pkgs; };
|
||||
swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; };
|
||||
full-setup = import ./vm-tests/full-setup.nix { inherit pkgs self; };
|
||||
}
|
||||
|
|
|
|||
273
nix/vm-tests/swarm-archive.nix
Normal file
273
nix/vm-tests/swarm-archive.nix
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
# `vmTests.<system>.swarm-archive`: runs the archive server's real units
|
||||
# (../host-modules/swarm-archive.nix) and the hive's real backup script
|
||||
# (../host-modules/lib/archive-backup.nix) in a VM, with restic and rclone, and
|
||||
# checks what the module-eval suite can only read:
|
||||
#
|
||||
# - a backup of two agents lands in one repository each, and `work/` is left out;
|
||||
# - one agent's login cannot read another's repository;
|
||||
# - a client cannot delete or overwrite anything, and the server-side prune
|
||||
# applies the retention;
|
||||
# - the users file follows the store: a user removed from it is refused without
|
||||
# a restart of the server;
|
||||
# - a hive that has agents and no grant fails its backup.
|
||||
#
|
||||
# One node, because the gateway hop is outside what is under test: the
|
||||
# backup script dials `http://127.0.0.1:<port>` instead of
|
||||
# `https://archive.<domain>`, and the swarm secret store is a stand-in `bao`
|
||||
# that serves `kv list`, `kv get` and `login` from a directory. The module-eval
|
||||
# suite covers the gateway vhost and the store grants.
|
||||
{ pkgs }:
|
||||
let
|
||||
port = 8911;
|
||||
store = "/var/lib/fake-bao";
|
||||
hive = "h1";
|
||||
agentRoot = "/var/lib/hyperhive/agents";
|
||||
|
||||
# A directory is a leaf, a file in it a field: `kv get -field=<f> <path>`.
|
||||
fakeBao = pkgs.writeShellScriptBin "bao" ''
|
||||
store=${store}
|
||||
case "$1 $2" in
|
||||
"login -method=cert") echo faketoken; exit 0 ;;
|
||||
"kv list")
|
||||
path="$4"
|
||||
[ -d "$store/$path" ] || { echo "No value found at $path" >&2; exit 2; }
|
||||
ls "$store/$path" | ${pkgs.jq}/bin/jq -R . | ${pkgs.jq}/bin/jq -s .
|
||||
exit 0 ;;
|
||||
"kv get")
|
||||
field="''${3#-field=}"
|
||||
[ -f "$store/$4/$field" ] || { echo "No value found at $4" >&2; exit 2; }
|
||||
cat "$store/$4/$field"
|
||||
exit 0 ;;
|
||||
esac
|
||||
echo "fake bao: unsupported: $*" >&2
|
||||
exit 1
|
||||
'';
|
||||
|
||||
# The stubs for the options the archive module reads from the rest of the
|
||||
# hive modules.
|
||||
hiveStubs =
|
||||
{ lib, ... }:
|
||||
{
|
||||
options.services.hyperhive = {
|
||||
swarm.domain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "t.local";
|
||||
};
|
||||
swarm.bao = {
|
||||
domain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "bao.t.local";
|
||||
};
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8200;
|
||||
};
|
||||
};
|
||||
deploy.bao = {
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = fakeBao;
|
||||
};
|
||||
archiveClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = "/etc/fake-archive-cert";
|
||||
};
|
||||
archiveClientKeyFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = "/etc/fake-archive-key";
|
||||
};
|
||||
serverCaFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
};
|
||||
};
|
||||
deploy.swarm-otel.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
};
|
||||
deploy.archive.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
};
|
||||
otel.journalUnits = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
};
|
||||
gateway = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
};
|
||||
dns.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
};
|
||||
localNames = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
};
|
||||
vhosts = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.anything;
|
||||
default = { };
|
||||
};
|
||||
};
|
||||
};
|
||||
config.environment.etc."fake-archive-cert".text = "";
|
||||
config.environment.etc."fake-archive-key".text = "";
|
||||
};
|
||||
|
||||
backupScript = import ../host-modules/lib/archive-backup.nix { inherit (pkgs) lib; } {
|
||||
inherit hive agentRoot;
|
||||
restic = pkgs.restic;
|
||||
url = "http://127.0.0.1:${toString port}";
|
||||
scope = "paths";
|
||||
paths = [
|
||||
"state"
|
||||
"claude"
|
||||
"harness"
|
||||
];
|
||||
excludes = [ ];
|
||||
caBundle = null;
|
||||
};
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "swarm-archive";
|
||||
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
imports = [
|
||||
hiveStubs
|
||||
../host-modules/swarm-archive.nix
|
||||
];
|
||||
|
||||
# Keep only the newest snapshot, so the prune is visible after two backups.
|
||||
services.hyperhive.deploy.archive.retention = {
|
||||
keepWithin = null;
|
||||
keepLast = 1;
|
||||
};
|
||||
|
||||
environment.systemPackages = [
|
||||
pkgs.curl
|
||||
pkgs.openssl
|
||||
pkgs.restic
|
||||
];
|
||||
|
||||
systemd.services.test-archive-backup = {
|
||||
path = [
|
||||
fakeBao
|
||||
pkgs.btrfs-progs
|
||||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
pkgs.jq
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
Environment = "RESTIC_CACHE_DIR=/var/cache/test-archive-backup";
|
||||
};
|
||||
script = backupScript;
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
store = "${store}"
|
||||
url = "http://127.0.0.1:${toString port}"
|
||||
agents = ["alpha", "beta"]
|
||||
|
||||
def sha_line(agent, password):
|
||||
return machine.succeed(
|
||||
f"printf '%s:{{SHA}}%s' {agent} "
|
||||
f"\"$(printf %s {password} | openssl dgst -sha1 -binary | base64)\""
|
||||
)
|
||||
|
||||
def put(path, field, value):
|
||||
machine.succeed(f"mkdir -p {store}/{path} && printf %s '{value}' > {store}/{path}/{field}")
|
||||
|
||||
def status(agent, repo, path="config", method="GET"):
|
||||
return machine.succeed(
|
||||
f"curl -s -o /dev/null -w '%{{http_code}}' -X {method} "
|
||||
f"-u {agent}:restpw-{agent} {url}/{repo}/{path}"
|
||||
)
|
||||
|
||||
def restic(agent, repo, args):
|
||||
return (
|
||||
f"RESTIC_PASSWORD=resticpw-{repo} restic "
|
||||
f"-r rest:http://{agent}:restpw-{agent}@127.0.0.1:${toString port}/{repo}/ {args}"
|
||||
)
|
||||
|
||||
machine.wait_for_unit("multi-user.target")
|
||||
machine.wait_for_open_port(${toString port})
|
||||
|
||||
with subtest("the server starts with no users and admits nobody"):
|
||||
machine.succeed("test -e /run/swarm-archive-auth/htpasswd")
|
||||
assert status("alpha", "alpha") == "401", status("alpha", "alpha")
|
||||
|
||||
# What swarm-controller writes: a user line and a restic password per agent
|
||||
# for the archive, a REST password and the restic password for the hive.
|
||||
for agent in agents:
|
||||
put(f"secret/swarm/services/archive/users/{agent}", "htpasswd", sha_line(agent, f"restpw-{agent}"))
|
||||
put(f"secret/swarm/services/archive/restic/{agent}", "password", f"resticpw-{agent}")
|
||||
put(f"secret/swarm/hives/${hive}/archive/{agent}", "rest-password", f"restpw-{agent}")
|
||||
put(f"secret/swarm/hives/${hive}/archive/{agent}", "restic-password", f"resticpw-{agent}")
|
||||
machine.succeed(
|
||||
f"mkdir -p ${agentRoot}/{agent}/state ${agentRoot}/{agent}/claude "
|
||||
f"${agentRoot}/{agent}/harness ${agentRoot}/{agent}/work"
|
||||
)
|
||||
machine.succeed(f"echo {agent}-state > ${agentRoot}/{agent}/state/data")
|
||||
machine.succeed(f"echo {agent}-work > ${agentRoot}/{agent}/work/scratch")
|
||||
|
||||
with subtest("the users file follows the store without a restart of the server"):
|
||||
pid = machine.succeed("systemctl show -p MainPID --value swarm-archive").strip()
|
||||
machine.succeed("systemctl start swarm-archive-users")
|
||||
assert machine.succeed("systemctl show -p MainPID --value swarm-archive").strip() == pid
|
||||
assert status("alpha", "alpha") in ("200", "404"), status("alpha", "alpha")
|
||||
assert status("beta", "beta") in ("200", "404"), status("beta", "beta")
|
||||
# Wrong password: the control that the 200 above is a login, not an open door.
|
||||
machine.succeed(f"test $(curl -s -o /dev/null -w '%{{http_code}}' -u alpha:wrong {url}/alpha/config) = 401")
|
||||
|
||||
with subtest("the users file holds hashes, readable by the server only"):
|
||||
machine.succeed("grep -q '{SHA}' /run/swarm-archive-auth/htpasswd")
|
||||
machine.fail("grep -q restpw /run/swarm-archive-auth/htpasswd")
|
||||
machine.succeed("test \"$(stat -c '%a %U:%G' /run/swarm-archive-auth/htpasswd)\" = '640 root:swarm-archive'")
|
||||
|
||||
with subtest("the hive's backup puts each agent in its own repository, without work/"):
|
||||
machine.succeed("systemctl start test-archive-backup")
|
||||
for agent in agents:
|
||||
snaps = machine.succeed(restic(agent, agent, "snapshots --json"))
|
||||
assert snaps.count('"id"') == 1, snaps
|
||||
listing = machine.succeed(restic(agent, agent, "ls latest"))
|
||||
assert f"/{agent}/state/data" in listing, listing
|
||||
assert "scratch" not in listing, listing
|
||||
|
||||
with subtest("one agent's login cannot reach another's repository"):
|
||||
assert status("beta", "alpha") == "403", status("beta", "alpha")
|
||||
machine.fail(restic("beta", "alpha", "snapshots"))
|
||||
assert status("alpha", "alpha") == "200", status("alpha", "alpha")
|
||||
assert status("alpha", "beta") == "403", status("alpha", "beta")
|
||||
|
||||
with subtest("a client cannot delete or overwrite"):
|
||||
snap = machine.succeed(restic("alpha", "alpha", "snapshots --json")).split('"id":"')[1].split('"')[0]
|
||||
# The presence control: a lock, which a backup creates and removes, is deletable.
|
||||
machine.succeed(restic("alpha", "alpha", "backup --host alpha ${agentRoot}/alpha/state"))
|
||||
assert status("alpha", "alpha", f"snapshots/{snap}", "DELETE") == "403"
|
||||
assert status("alpha", "alpha", "config", "POST") == "403"
|
||||
machine.fail(restic("alpha", "alpha", "forget --keep-last 1 --prune"))
|
||||
assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2
|
||||
|
||||
with subtest("the server-side prune applies the retention"):
|
||||
machine.succeed("systemctl start swarm-archive-prune")
|
||||
for agent in agents:
|
||||
assert machine.succeed(restic(agent, agent, "snapshots --json")).count('"id"') == 1
|
||||
|
||||
with subtest("a user removed from the store is refused without a restart"):
|
||||
machine.succeed(f"rm -r {store}/secret/swarm/services/archive/users/beta")
|
||||
machine.succeed("systemctl start swarm-archive-users")
|
||||
assert status("beta", "beta") == "401", status("beta", "beta")
|
||||
assert status("alpha", "alpha") == "200", status("alpha", "alpha")
|
||||
|
||||
with subtest("a hive with agents and no grant fails its backup"):
|
||||
machine.succeed(f"rm -r {store}/secret/swarm/hives/${hive}")
|
||||
machine.fail("systemctl start test-archive-backup")
|
||||
machine.succeed("journalctl -u test-archive-backup | grep -q 'archive backup failed: swarm-controller has granted this hive no archive credentials'")
|
||||
'';
|
||||
}
|
||||
Loading…
Reference in a new issue