diff --git a/Cargo.lock b/Cargo.lock index 11e8dc4d8..8013bd1ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5107,6 +5107,7 @@ dependencies = [ "serde", "serde_json", "serde_norway", + "sha1", "sha2 0.11.1", "similar", "strum", diff --git a/Cargo.toml b/Cargo.toml index 0c0d57adf..7c00b1ce7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -252,6 +252,9 @@ matrix-sdk = { version = "0.18", default-features = false, features = [ futures-util = "0.3" hmac = "0.13" sha2 = "0.11" +# SHA-1 for the `{SHA}` htpasswd lines the archive server checks passwords against +# (`swarm-controller`). Already in the lock through the TLS and websocket stacks. +sha1 = "0.10" # SHA-512 crypt (`$6$`) for the users file hive-c0re renders; squid's `basic_ncsa_auth` # hands a `$` hash to the system `crypt()`. sha-crypt = "0.6" diff --git a/docs/2-handbook/swarm/README.md b/docs/2-handbook/swarm/README.md index 88d6eaf36..b3f95b2d0 100644 --- a/docs/2-handbook/swarm/README.md +++ b/docs/2-handbook/swarm/README.md @@ -43,6 +43,8 @@ grafana, the controller, authelia, …) → CLI agents use. → [`forge.md`](forge.md) - **matrix** — the swarm's homeserver, several accounts per agent, and federation. → [`matrix.md`](matrix.md) +- **backup archive** — where the hives back up each agent's state, how long it + stays and how to check it. → [`archive.md`](archive.md) - **knowledge repo** — the reference docs every agent reads, and how to add one. → [`knowledge.md`](knowledge.md) - **`swarmctl`** — the swarm operator's CLI. → [`swarmctl-cli.md`](swarmctl-cli.md) diff --git a/docs/2-handbook/swarm/archive.md b/docs/2-handbook/swarm/archive.md new file mode 100644 index 000000000..c448f1e15 --- /dev/null +++ b/docs/2-handbook/swarm/archive.md @@ -0,0 +1,125 @@ +# Backup archive + +The archive keeps a backup of every agent's state. Each hive backs its agents +up to it every hour, and it keeps the last few days. This page covers what the +hive saves, where it goes, how long it stays and how to check that it works. +Restoring an agent from a backup isn't built yet. + +## What's in the backup + +For each agent a hive places, the hive saves the agent's `state/`, `claude/` and +`harness/` directories. `claude/` holds the agent's login, so a restored agent +keeps its session. Caches and `work/` stay out. + +Two options on the hive change that: + +- `deploy.hive-controller.archive.paths` lists the entries of the agent's + directory to save. The hive skips an entry the agent doesn't have. +- `deploy.hive-controller.archive.scope = "subvolume"` saves the whole agent + directory instead, `work/` included. Add `archive.excludes` patterns to keep + build output out. + +On a btrfs hive the hive saves a read-only snapshot of the agent's subvolume, so +it copies the databases at one instant. Otherwise it saves the directory in +place. + +## Where it goes + +The archive runs as `rclone serve restic` on the host that sets +`deploy.archive.enable`, behind the gateway at `archive.`. Every +hive finds it by that name, so which host runs it makes no difference to them. +`allSwarmServices` turns it on wherever the host has the archive's store identity, +as for the binary cache. + +Each agent has its own [restic](https://restic.net) repository at +`https://archive.//`, under `deploy.archive.dataDir` +(default `/var/lib/swarm-archive`) on that host. restic encrypts it with a +password of its own for that agent. + +Two properties protect the backups from a compromised hive: + +- **One login per agent.** `swarm-controller` creates a login for each agent + when it places the agent. The login reaches that agent's repository and no + other. A hive holds the logins of the agents it places and no others. When an + agent moves, the controller gives the new hive a new password and deletes the + old hive's copy, so the old hive can't reach the repository any more. +- **Append-only.** A client can add snapshots and nothing else: no delete, no + overwrite. The archive host removes old snapshots. + +## How long it stays + +The archive host prunes every repository once a day. A snapshot survives if +either rule keeps it: + +| option | default | keeps | +| ------------------------------------- | ------- | --------------------------------- | +| `deploy.archive.retention.keepWithin` | `3d` | every snapshot younger than this | +| `deploy.archive.retention.keepLast` | `3` | the newest this many, however old | + +The age rule limits what a compromised hive can do by flooding the archive with +fake snapshots: they can't push the last three days out. Set `keepWithin` to +`null` to leave that rule off. `deploy.archive.pruneSchedule` sets when the +prune runs (default `daily`). + +Because the archive host prunes, it reads every agent's restic password from +the secret store and can decrypt every backup. Treat that host as part of the +swarm's trust root. + +## Schedule + +`deploy.hive-controller.archive.schedule` is a systemd calendar expression +(default `hourly`). restic sends only what changed, so a run over an idle agent +is small. The first backup of a large agent sends everything. + +## Check that it works + +On a hive, `systemctl status hive-archive-backup` shows the last run, and +`journalctl -u hive-archive-backup` lists every agent it backed up or failed to +back up. `systemctl list-timers hive-archive-backup` shows the next run. + +To see an agent's snapshots, ask the archive with that agent's password, which +the secret store holds at `swarm/hives//archive/` (fields +`rest-password` and `restic-password`): + +```sh +RESTIC_REPOSITORY="rest:https://archive.//" \ +RESTIC_REST_USERNAME= RESTIC_REST_PASSWORD= \ +restic snapshots +``` + +restic asks for the `restic-password`. On the archive host, `systemctl status +swarm-archive swarm-archive-users swarm-archive-prune` shows the server, the +users file refresh and the last prune. + +## When it fails + +- **The `archive backup failed` alert.** A hive's run failed for an agent or + failed to log in. The journal line `archive backup failed: ` names the + agent, and the lines above it say why. The alert waits 30 minutes, so an + agent placed a few minutes ago doesn't trip it while its login is on its way. + The next hourly run retries. +- **`swarm-controller has granted this hive no archive credentials`.** The + hive has agents and the store holds no login for any of them. Either + `swarm-controller` doesn't have the archive yet (update it), or its archive + pass fails: look for `archive:` lines in its journal. +- **The `archive prune failed` alert.** The daily prune couldn't apply the + retention to one repository, for example because a backup held its lock for + longer than ten minutes. The next prune retries. +- **A swarm without an archive.** If no host sets `deploy.archive.enable`, every + hive's run fails. Set `deploy.hive-controller.archive.enable = false` on the + hives until one does. + +## Mixed versions + +A hive that doesn't have the backup unit yet doesn't back up, and nothing else +changes. A `swarm-controller` without the archive pass writes no logins, so +hives with the unit fail with the message above until it updates. The archive +host admits only the users the controller wrote, so until then it admits nobody. + +## What lives where + +Everything secret lives in the swarm secret store, and the archive keeps none +on disk. The archive host renders its users file (password hashes only) into +`/run/swarm-archive-auth/`, which is tmpfs. Hives read their restic password +when restic asks for it. The paths and who writes and reads them are in the +[credentials reference](credentials-reference.md). diff --git a/docs/2-handbook/swarm/credentials-reference.md b/docs/2-handbook/swarm/credentials-reference.md index a34edaf29..125044a2a 100644 --- a/docs/2-handbook/swarm/credentials-reference.md +++ b/docs/2-handbook/swarm/credentials-reference.md @@ -57,6 +57,9 @@ the rest of the cell says how. | `swarm/services/attic/public-key` | `swarm-bao-attic-tokens` on the cache's host, when none is stored and again whenever the cache reports another key (the key `attic cache info` prints); with an external cache, you, in the store's UI (field `value`), or `bao kv put` with `value=-` and the value on stdin (`:`) | `swarm-controller`, which copies it to each hive | ✅ on the cache's host, the unit's next run replaces any other value, a hand edit included, with the cache's key; with an external cache, replace it yourself | ✅ copied to the hives every 5 minutes | | `swarm/hives//attic/{pull-token,url,cache,public-key}` | `swarm-controller`, from the four `swarm/services/attic/` leaves, every 5 minutes, and before each deploy to the hive; it skips a leaf whose source holds nothing, and a copy whose source is gone stays | the hive (`hive-c0re`) | ✅ a changed source is written over the copy in the next pass | ✅ read every 5 minutes | | `swarm/services/attic/jwt-secret` | `swarm-bao-attic-secret` on the cache's host, when none is stored | `swarm-bao-attic-secret`, `swarm-bao-attic-tokens` | ❌ a new key invalidates both tokens | ❌ atticd restarts when the file changes | +| `swarm/hives//archive/` (fields `rest-password`, `restic-password`) | `swarm-controller`, for every agent a hive places, every 5 minutes: a new random REST password when the agent is new on that hive, the agent's restic password from the leaf below; the copy on a hive that no longer places the agent is deleted | the agent's hive: `hive-archive-backup`, under the hive's own store identity | ✅ a move to another hive writes a new REST password, which locks the old hive out | ✅ read at every backup run | +| `swarm/services/archive/users/` (field `htpasswd`) | `swarm-controller`, with the REST password above: its `{SHA}` hash, never the password; deleted when no hive places the agent | `swarm-archive-users` on the archive's host, which renders the users file into `/run` | ✅ rewritten with the password | ✅ re-read every 2 minutes; the server notices the new file without a restart | +| `swarm/services/archive/restic/` (field `password`) | `swarm-controller`, once per agent, on its first placement; never replaced and never deleted, because it opens every backup of the agent | `swarm-archive-prune` on the archive's host, through `RESTIC_PASSWORD_COMMAND` | ❌ a new password would not open the existing repository | ✅ read at every prune | | `swarm/services/ci-metrics/forge/token` | `swarm-controller`, at start and every 5 minutes | `swarm-ci-metrics`, `swarm-controller` | ✅ when missing or no longer matching the forge | ✅ read on every pass | | _(not in the store)_ a hive's mTLS leaf | the store's PKI, or you by hand | the hive itself, from disk | must be stated | must be stated | @@ -181,6 +184,7 @@ login minter's, the forge host's and Grafana's, which reach more than one: | `swarm-notify-router`, in its container | `notifyRouterClientCertFile` / `notifyRouterClientKeyFile` | `swarm/services//oidc/client` | | `swarm-builder`, in its container | `builderClientCertFile` / `builderClientKeyFile` | `swarm/services/builder/forge/token`, `swarm/services/attic/push-token` and `swarm/services//oidc/client` | | `swarm-bao-attic-secret` and `swarm-bao-attic-tokens` | `atticClientCertFile` / `atticClientKeyFile` | creates and reads `swarm/services/attic/jwt-secret`, `swarm/services/attic/push-token` and `swarm/services/attic/pull-token` | +| `swarm-archive-users` and `swarm-archive-prune` | `archiveClientCertFile` / `archiveClientKeyFile` | reads `swarm/services/archive/users/*` and `swarm/services/archive/restic/*`, lists `swarm/services/archive/users/` | | `swarm-ci-metrics` | `ciMetricsClientCertFile` / `ciMetricsClientKeyFile` | `swarm/services/ci-metrics/forge/token` and `swarm/services//oidc/client` | | `swarm-bao-forge-oidc` and the forge's controller-token mint and its journal forwarder in its container | `forgePublisherClientCertFile` / `forgePublisherClientKeyFile` | writes `swarm/controller/swarm-controller/forge/token`; reads `swarm/services//oidc/client` and `swarm/services//oidc/client` | diff --git a/docs/2-handbook/swarm/credentials.md b/docs/2-handbook/swarm/credentials.md index abdeb246c..02b31dd96 100644 --- a/docs/2-handbook/swarm/credentials.md +++ b/docs/2-handbook/swarm/credentials.md @@ -442,7 +442,7 @@ each is enough and their subjects are the flat `deploy.bao.grafanaOidcCommonName`, `deploy.bao.otelOidcCommonName`, `deploy.bao.forwarderOidcCommonName`, `deploy.bao.ciRunnerCommonName`, `deploy.bao.reviewClassifierCommonName`, `deploy.bao.builderCommonName`, -`deploy.bao.ciMetricsCommonName`, `deploy.bao.atticCommonName`, +`deploy.bao.ciMetricsCommonName`, `deploy.bao.atticCommonName`, `deploy.bao.archiveCommonName`, `deploy.bao.claudeLoginCommonName` and `deploy.bao.forgePublisherCommonName`. The last three are the writers, and the last is the one leaf two units share: the forge's host holds one store identity for the token it publishes and the diff --git a/docs/2-handbook/swarm/services.md b/docs/2-handbook/swarm/services.md index e7f1efddc..b997dfdff 100644 --- a/docs/2-handbook/swarm/services.md +++ b/docs/2-handbook/swarm/services.md @@ -119,6 +119,7 @@ look before you switch them on. | CI runner | Runs Forgejo Actions for pull requests. Heavy. | `deploy.ci-runner.enable` | [CI runner](#ci-runner) | | Builder | Evaluates and builds agent config PRs. Heavy. | `deploy.builder.enable` | [Builder](#builder) | | Binary cache | Serves built closures to the hives. | `deploy.attic.enable` | [Binary cache](#binary-cache) | +| Backup archive | Keeps every agent's state backups, one repository each. | `deploy.archive.enable` | [Backup archive](archive.md) | | CI metrics | Records CI run times in the metrics store. | `deploy.ci-metrics.enable` | [CI metrics](#ci-metrics) | | Review classifier | Checks PRs against a repo's review rules. | `deploy.review-classifier.enable` | [Review classifier](#review-classifier) | | Claude login minter | Runs the **log in** button on an agent's card. | `deploy.claude-login.enable` | [Claude login minter](#claude-login-minter) | diff --git a/docs/4-implementation/scheduler/observability.md b/docs/4-implementation/scheduler/observability.md index ea3cf88f5..a483fa8d8 100644 --- a/docs/4-implementation/scheduler/observability.md +++ b/docs/4-implementation/scheduler/observability.md @@ -390,7 +390,7 @@ in `services.hyperhive.otel.journalUnits`: `hive-c0re.service`, the `container@* units (the start-time rootfs check lines), plus `squid.service` and `squid-selfcheck.service` (see [Egress proxy](../networking/network.md#egress-proxy-squid)). A host that runs the cache without the swarm collector adds `swarm-bao-attic-secret.service` -and `swarm-bao-attic-tokens.service`. +and `swarm-bao-attic-tokens.service`; a host that runs the archive adds `swarm-archive.service`, `swarm-archive-users.service` and `swarm-archive-prune.service`, and a hive adds `hive-archive-backup.service`. A host that also runs the swarm collector names none of these: its whole-host journal receiver ships them already, and the hive collector then has no `journald` receiver. No bind mount is needed. Unlike an agent's journal, this collector already diff --git a/docs/README.md b/docs/README.md index a65f0367b..ae8e351c3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -59,6 +59,8 @@ The services that exist once per swarm. Start at - **How to log an agent in to claude from the swarm UI, and why doesn't a stale stored copy log it out again?** → [`2-handbook/swarm/claude-login.md`](2-handbook/swarm/claude-login.md). +- **Where do the hives back up each agent's state, how long does it stay, and how to + check it?** → [`2-handbook/swarm/archive.md`](2-handbook/swarm/archive.md). ### Hives diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index d0fd09849..63ef2dec2 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -22,6 +22,7 @@ ./hive-matrix.nix ./hive-network.nix ./hive-priv.nix + ./hive-archive-backup.nix ./hive-priv-cache.nix ./hive-squid.nix ./hive-squid-credential.nix @@ -65,6 +66,7 @@ ./glue-swarm-otel-oidc-client.nix ./glue-victorialogs-otel-oidc-client.nix ./glue-victoriametrics-otel-oidc-client.nix + ./swarm-archive.nix ./swarm-attic-options.nix ./swarm-attic.nix ./swarm-authelia-service.nix diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index e686c2506..f7ee82496 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -734,6 +734,25 @@ in ''; }; + archive.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = '' + Run the swarm's backup archive on this host: `rclone serve restic` + behind the gateway at `archive.`, append-only, with one + login and one repository per agent, and a daily prune that applies the + retention. Every hive's backup unit finds it by that name, so it runs + on one host and the host does not matter. Run it on one host. + + On by default where + {option}`services.hyperhive.deploy.allSwarmServices` is set and this + host has the archive's store identity (the pair + {option}`services.hyperhive.deploy.bao.archiveClientCertFile` and + `archiveClientKeyFile`, minted on the store's host). + ''; + }; + hive-controller.enable = lib.mkOption { type = lib.types.bool; default = false; diff --git a/nix/host-modules/glue-bao-readers-policy-order.nix b/nix/host-modules/glue-bao-readers-policy-order.nix index 0631908dc..5ee53a544 100644 --- a/nix/host-modules/glue-bao-readers-policy-order.nix +++ b/nix/host-modules/glue-bao-readers-policy-order.nix @@ -90,6 +90,16 @@ let here = deployCfg.attic.enable; policies = [ "swarm-bao-attic-policy" ]; }; + # ./swarm-archive.nix, under the archive's leaf, which reads the users and the + # repositories' passwords only once that policy is written. + swarm-archive-users = { + here = deployCfg.archive.enable; + policies = [ "swarm-bao-archive-policy" ]; + }; + swarm-archive-prune = { + here = deployCfg.archive.enable; + policies = [ "swarm-bao-archive-policy" ]; + }; # ./hive-matrix.nix, under matrix-ctl's leaf. swarm-bao-matrix-oidc = { here = deployCfg.matrix.enable; diff --git a/nix/host-modules/glue-bao-readers-store-order.nix b/nix/host-modules/glue-bao-readers-store-order.nix index ae9a702db..f29b04fd3 100644 --- a/nix/host-modules/glue-bao-readers-store-order.nix +++ b/nix/host-modules/glue-bao-readers-store-order.nix @@ -69,6 +69,9 @@ let # ./swarm-attic.nix swarm-bao-attic-secret = deployCfg.attic.enable; swarm-bao-attic-tokens = deployCfg.attic.enable; + # ./swarm-archive.nix + swarm-archive-users = deployCfg.archive.enable; + swarm-archive-prune = deployCfg.archive.enable; # ./swarm-otel.nix swarm-bao-otel-oidc = deployCfg.swarm-otel.enable diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index 27d4e2023..4065ec4e0 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -126,6 +126,8 @@ in notifyRouterClientKeyFile = lib.mkDefault "${pkiDir}/notify-router-key.pem"; atticClientCertFile = lib.mkDefault "${pkiDir}/attic.pem"; atticClientKeyFile = lib.mkDefault "${pkiDir}/attic-key.pem"; + archiveClientCertFile = lib.mkDefault "${pkiDir}/archive.pem"; + archiveClientKeyFile = lib.mkDefault "${pkiDir}/archive-key.pem"; forgePublisherClientCertFile = lib.mkDefault "${pkiDir}/forge-publisher.pem"; forgePublisherClientKeyFile = lib.mkDefault "${pkiDir}/forge-publisher-key.pem"; servicesIssuerClientCertFile = lib.mkDefault "${pkiDir}/services-issuer.pem"; @@ -337,6 +339,13 @@ in [ -s ${pkiDir}/attic.pem ] || ${signLeaf} ${pkiDir} attic \ ${lib.escapeShellArg deployCfg.bao.atticCommonName} "" clientAuth + # The backup archive's, which reads its users and the repositories' + # passwords. Minted here for the matrix-ctl leaf's reason: the archive + # is a swarm service, so on an archive host without the store this is + # the file an operator copies. + [ -s ${pkiDir}/archive.pem ] || ${signLeaf} ${pkiDir} archive \ + ${lib.escapeShellArg deployCfg.bao.archiveCommonName} "" clientAuth + # The forge's token publisher. Minted here for the matrix-ctl leaf's # reason: the forge is a swarm singleton, so on a forge host without the # store this is the file an operator copies. diff --git a/nix/host-modules/hive-archive-backup.nix b/nix/host-modules/hive-archive-backup.nix new file mode 100644 index 000000000..240068fbf --- /dev/null +++ b/nix/host-modules/hive-archive-backup.nix @@ -0,0 +1,185 @@ +# The hive's backup of its agents' state to the swarm archive +# (./swarm-archive.nix): one restic repository per agent, at +# `https://archive.//`. +# +# A root oneshot on a timer, not hive-c0re and not a hive-priv verb. Root, +# because the agents' files belong to the agents (and, under user namespaces, +# to remapped ids) and hive-c0re cannot read them; a unit of its own, because +# no caller-supplied argument may reach root, and this one takes none. +# +# Each agent backs up under the credentials swarm-controller wrote for it at +# `swarm/hives//archive/` +# (../../swarm-controller/src/archive_access.rs), read with the hive's own +# store identity: the agent's login to the archive, and its restic password, +# which restic reads through `RESTIC_PASSWORD_COMMAND` and nothing writes to +# disk. An agent the controller has not granted this hive is skipped; a hive +# with agents and no grant at all fails, which is what a swarm-controller +# without the archive looks like. +# +# A failed agent prints `archive backup failed: ` to the journal, +# which the swarm's `archive backup failed` alert reads, and fails the unit +# after the others have been tried. +{ + pkgs, + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + hiveController = hyperhiveCfg.deploy.hive-controller; + cfg = hiveController.archive; + tls = hiveController.tls; + baoCfg = hyperhiveCfg.swarm.bao; + baoDeploy = hyperhiveCfg.deploy.bao; + + enabled = + cfg.enable + && hiveController.enable + && hiveController.hiveName != null + && tls.baoHiveIdentityCertFile != null + && tls.baoHiveIdentityKeyFile != null; + + # The CA bundle restic trusts: the host's, then the swarm's root, which the + # archive's certificate chains to when the gateway is self-signed. Written by + # the unit at each run; `--cacert` replaces restic's own roots. + runtimeDir = "/run/hive-archive-backup"; + caBundle = "${runtimeDir}/ca.pem"; + swarmCa = "${tls.stateDir}/swarm-services-root.pem"; + + storeContainer = lib.optional baoDeploy.enable "container@${baoCfg.machine}.service"; + + backup = import ./lib/archive-backup.nix { inherit lib; } { + inherit (cfg) scope paths excludes; + restic = cfg.package; + hive = hiveController.hiveName; + agentRoot = "/var/lib/hyperhive/agents"; + url = "https://${hyperhiveCfg.swarm.archive.ownDomain}"; + inherit caBundle; + }; +in +{ + options.services.hyperhive.deploy.hive-controller.archive = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Back this hive's agents up to the swarm archive. On wherever the hive + has its store identity; a swarm that runs no archive + ({option}`services.hyperhive.deploy.archive.enable` on one host) sees + every run fail, so turn it off there. + ''; + }; + + schedule = lib.mkOption { + type = lib.types.str; + default = "hourly"; + example = "*-*-* 00/6:00:00"; + description = '' + systemd calendar expression for the backup. Each run sends only what + changed since the last, so an hourly run on an idle agent is small. + ''; + }; + + scope = lib.mkOption { + type = lib.types.enum [ + "paths" + "subvolume" + ]; + default = "paths"; + description = '' + What of each agent is backed up. `paths`: the entries of + {option}`services.hyperhive.deploy.hive-controller.archive.paths` + inside the agent's directory. `subvolume`: the whole directory, which + adds `work/` and everything an agent keeps there. + ''; + }; + + paths = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ + "state" + "claude" + "harness" + ]; + description = '' + Entries of an agent's directory backed up under `scope = "paths"`: its + state, its claude login and the harness. Caches and `work/` are left + out unless listed. An entry the agent does not have is skipped. + ''; + }; + + excludes = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "*/target" ]; + description = "restic `--exclude` patterns, on top of `--exclude-caches`."; + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.restic; + defaultText = lib.literalExpression "pkgs.restic"; + description = "The restic package."; + }; + }; + + config = lib.mkIf enabled { + services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [ + "hive-archive-backup.service" + ]; + + systemd.services.hive-archive-backup = { + description = "back up this hive's agents to the swarm archive"; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ] ++ storeContainer; + path = [ + baoDeploy.package + pkgs.btrfs-progs + pkgs.coreutils + pkgs.gnugrep + pkgs.jq + ]; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = tls.baoHiveIdentityCertFile; + BAO_CLIENT_KEY = tls.baoHiveIdentityKeyFile; + RESTIC_CACHE_DIR = "/var/cache/hive-archive-backup"; + } + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; + serviceConfig = { + Type = "oneshot"; + UMask = "0077"; + SyslogIdentifier = "hive-archive-backup"; + RuntimeDirectory = "hive-archive-backup"; + CacheDirectory = "hive-archive-backup"; + # A first backup of a large agent sends all of it. + TimeoutStartSec = "6h"; + Nice = 10; + IOSchedulingClass = "idle"; + }; + script = '' + set -euo pipefail + # A missing file is left out: the bundle is the host's roots alone + # until the swarm's root is there. + cat ${lib.escapeShellArg (toString config.security.pki.caBundle)} > ${caBundle} + if [ -s ${lib.escapeShellArg swarmCa} ]; then + cat ${lib.escapeShellArg swarmCa} >> ${caBundle} + fi + ${backup} + ''; + }; + + systemd.timers.hive-archive-backup = { + description = "back up this hive's agents to the swarm archive"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = cfg.schedule; + RandomizedDelaySec = "5min"; + Persistent = true; + }; + }; + }; +} diff --git a/nix/host-modules/lib/archive-backup.nix b/nix/host-modules/lib/archive-backup.nix new file mode 100644 index 000000000..22009dd5a --- /dev/null +++ b/nix/host-modules/lib/archive-backup.nix @@ -0,0 +1,137 @@ +# The hive's backup run: for each agent that has a state directory here and a +# grant from swarm-controller, one `restic backup` into that agent's repository +# on the archive. A pure function, NOT a NixOS module: ../hive-archive-backup.nix +# wraps it in a unit, and the VM test (nix/vm-tests/swarm-archive.nix) runs the +# same script against a stand-in `bao`. +# +# Needs `bao`, `jq`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path` +# and the `BAO_*` client set. Every failure prints a line starting +# `archive backup failed`, which the swarm's alert reads, and ends the run with +# a non-zero status once the other agents have been tried. +{ lib }: +{ + restic, + # The hive's name in the store: grants are read from + # `secret/swarm/hives//archive/`. + hive, + # The directory holding one directory per agent. + agentRoot, + # The repository server's base URL, no trailing slash. + url, + # "paths": `paths` under each agent's directory. "subvolume": the whole directory. + scope, + paths, + excludes, + # A CA file to trust instead of restic's default, or null. + caBundle, +}: +let + storeSeededSecret = import ./store-seeded-secret.nix { }; + + grants = hive: "secret/swarm/hives/${hive}/archive"; +in +'' + set -euo pipefail + ${storeSeededSecret} + + store_init + store_login + if [ "$STORE_UP" != 1 ]; then + echo "archive backup failed: could not log in to the swarm secret store" >&2 + exit 1 + fi + + ${lib.optionalString (caBundle != null) '' + export RESTIC_CACERT=${lib.escapeShellArg caBundle} + ''} + root=${lib.escapeShellArg agentRoot} + grants=${lib.escapeShellArg (grants hive)} + + # One agent: a read-only snapshot of its subvolume when it is one, so the + # databases are copied at one instant, else its directory in place. + backup_agent() { + local agent="$1" leaf="$grants/$1" rest base snap="" target targets=() + rest="$(store_bao kv get -field=rest-password "$leaf")" || return 1 + export RESTIC_REPOSITORY="rest:${url}/$agent/" + export RESTIC_REST_USERNAME="$agent" + export RESTIC_REST_PASSWORD="$rest" + export RESTIC_PASSWORD_COMMAND="bao kv get -field=restic-password $leaf" + + if ! ${restic}/bin/restic cat config >/dev/null 2>&1; then + ${restic}/bin/restic init || return 1 + fi + + base="$root/$agent" + if btrfs subvolume show "$base" >/dev/null 2>&1; then + snap="$root/.$agent.snapshot.archive" + btrfs subvolume delete "$snap" >/dev/null 2>&1 || true + btrfs subvolume snapshot -r "$base" "$snap" >/dev/null || return 1 + base="$snap" + fi + + ${ + if scope == "subvolume" then + ''targets=("$base")'' + else + '' + for target in ${lib.escapeShellArgs paths}; do + [ -e "$base/$target" ] && targets+=("$base/$target") + done'' + } + local rc=0 + if [ "''${#targets[@]}" = 0 ]; then + echo "$agent has none of its backup paths yet; nothing to back up." + else + ${restic}/bin/restic backup --host "$agent" --tag archive --exclude-caches \ + ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \ + -- "''${targets[@]}" || rc=$? + fi + if [ -n "$snap" ]; then + btrfs subvolume delete "$snap" >/dev/null || rc=1 + fi + return "$rc" + } + + # The agents the controller granted this hive; none yet is "No value found". + rc=0 + granted="$(store_bao kv list -format=json "$grants" 2>"$STORE_ERR")" || rc=$? + if [ "$rc" != 0 ]; then + if grep -q 'No value found' "$STORE_ERR"; then + granted='[]' + else + cat "$STORE_ERR" >&2 + echo "archive backup failed: could not list $grants" >&2 + exit 1 + fi + fi + granted="$(printf '%s' "$granted" | jq -r '.[]')" + + failed=0 + candidates=0 + backed_up=0 + for dir in "$root"/*/; do + agent="$(basename -- "$dir")" + case "$agent" in .*) continue ;; esac + [[ "$agent" =~ ^[A-Za-z0-9_-]+$ ]] || continue + candidates=$((candidates + 1)) + if ! grep -qxF -- "$agent" <<<"$granted"; then + echo "$agent has no archive credentials on this hive; skipping." + continue + fi + if (backup_agent "$agent"); then + backed_up=$((backed_up + 1)) + echo "backed up $agent" + else + echo "archive backup failed: $agent" >&2 + failed=1 + fi + done + + # Agents here and not one of them granted: swarm-controller has not written + # this hive's grants, which a controller without the archive never does. + if [ "$candidates" -gt 0 ] && [ -z "$granted" ]; then + echo "archive backup failed: swarm-controller has granted this hive no archive credentials ($candidates agents here)" >&2 + failed=1 + fi + exit "$failed" +'' diff --git a/nix/host-modules/lib/archive-server.nix b/nix/host-modules/lib/archive-server.nix new file mode 100644 index 000000000..368693c11 --- /dev/null +++ b/nix/host-modules/lib/archive-server.nix @@ -0,0 +1,156 @@ +# The archive server's moving parts: the rclone command that serves the +# repositories, the script that renders its users file from the secret store, +# and the script that prunes each repository. A pure function, NOT a NixOS +# module: ../swarm-archive.nix wraps the pieces in units, and the VM test +# (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in +# `bao`. +# +# Both scripts use only `bao login -method=cert -token-only`, +# `bao kv list -format=json` and `bao kv get -field=…`, and need `bao`, `jq`, +# `coreutils`, `gnugrep` on the unit's `path` and the `BAO_*` client set. +{ lib }: +{ + rclone, + restic, + # Where the repositories live: `//`. The only place that + # names the backend; `rclone serve restic` takes any remote here. + reposDir, + # The address rclone listens on. + addr, + # The users file rclone checks logins against, its group and the user the + # server runs as. + authFile, + authGroup, + # `{ keepWithin, keepLast }`; a null `keepWithin` leaves that flag off. + retention, +}: +let + atomicWriteSecret = import ./atomic-write-secret.nix { }; + refreshConsumer = import ./refresh-consumer.nix { }; + storeSeededSecret = import ./store-seeded-secret.nix { }; + + usersPrefix = "secret/swarm/services/archive/users"; + resticPrefix = "secret/swarm/services/archive/restic"; + + keepFlags = lib.concatStringsSep " " ( + [ "--keep-last ${toString retention.keepLast}" ] + ++ lib.optional ( + retention.keepWithin != null + ) "--keep-within ${lib.escapeShellArg retention.keepWithin}" + ); +in +{ + inherit usersPrefix resticPrefix; + + # Append-only: a client cannot delete or overwrite a snapshot, a pack or the + # index, so a hive that is compromised cannot erase history. Private repos: + # a login reaches `//` and nothing else. + serverCommand = lib.escapeShellArgs [ + "${rclone}/bin/rclone" + "serve" + "restic" + "--addr" + addr + "--append-only" + "--private-repos" + "--htpasswd" + authFile + reposDir + ]; + + # Renders the users file from `usersPrefix`, one leaf per agent holding that + # agent's htpasswd line. The file is written only when the content changed, + # and never from a partial read: any leaf that cannot be read leaves the file + # as it is. + usersScript = '' + set -euo pipefail + ${storeSeededSecret} + ${atomicWriteSecret} + ${refreshConsumer} + + file=${lib.escapeShellArg authFile} + # The server will not start without the file; an empty one admits nobody. + if [ ! -e "$file" ]; then + atomic_write_secret 0640 root:${authGroup} "$file" "" + fi + + store_init + store_login + if [ "$STORE_UP" != 1 ]; then + echo "could not log in to the swarm secret store; $file stays as it is." >&2 + exit 1 + fi + + # A prefix that holds nothing is "No value found"; it means no agent has a + # user yet, not a failure. + rc=0 + names="$(store_bao kv list -format=json ${lib.escapeShellArg usersPrefix} 2>"$STORE_ERR")" || rc=$? + if [ "$rc" != 0 ]; then + if grep -q 'No value found' "$STORE_ERR"; then + names='[]' + else + cat "$STORE_ERR" >&2 + echo "could not list ${usersPrefix}; $file stays as it is." >&2 + exit 1 + fi + fi + + content="" + while IFS= read -r name; do + [ -n "$name" ] || continue + case "$name" in */) continue ;; esac + if ! line="$(store_bao kv get -field=htpasswd ${lib.escapeShellArg usersPrefix}/"$name" 2>"$STORE_ERR")"; then + cat "$STORE_ERR" >&2 + echo "could not read the user $name; $file stays as it is." >&2 + exit 1 + fi + # The line must be this agent's own: a leaf cannot add another user. + if [[ ! "$name" =~ ^[A-Za-z0-9_-]+$ ]] || [[ "$line" != "$name:{SHA}"* ]] \ + || [[ ! "$line" =~ ^[A-Za-z0-9_-]+:\{SHA\}[A-Za-z0-9+/=]+$ ]]; then + echo "the user leaf $name does not hold a {SHA} line for $name; leaving it out." >&2 + continue + fi + content+="$line"$'\n' + done < <(printf '%s' "$names" | jq -r '.[]') + content="''${content%$'\n'}" + + if secret_differs "$file" "$content"; then + atomic_write_secret 0640 root:${authGroup} "$file" "$content" + echo "users file updated." + fi + ''; + + # `restic forget --prune` on every repository, with its password read from + # the store at run time. The server refuses deletes from clients, so this is + # the only place history is dropped. + pruneScript = '' + set -euo pipefail + ${storeSeededSecret} + + store_init + store_login + if [ "$STORE_UP" != 1 ]; then + echo "archive prune failed: could not log in to the swarm secret store" >&2 + exit 1 + fi + + failed=0 + for repo in ${lib.escapeShellArg reposDir}/*/; do + [ -e "$repo/config" ] || continue + agent="$(basename -- "$repo")" + if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then + echo "archive prune failed: $agent is not an agent name" >&2 + failed=1 + continue + fi + export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent" + if ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then + echo "pruned $agent" + else + echo "archive prune failed: $agent" >&2 + failed=1 + fi + done + exit "$failed" + ''; +} diff --git a/nix/host-modules/swarm-archive.nix b/nix/host-modules/swarm-archive.nix new file mode 100644 index 000000000..777669a59 --- /dev/null +++ b/nix/host-modules/swarm-archive.nix @@ -0,0 +1,348 @@ +# The swarm's backup archive: `rclone serve restic` on the host that sets +# `deploy.archive.enable`, behind the gateway at `archive.`. +# Every hive's backup unit (./hive-archive-backup.nix) writes one restic +# repository per agent to it; nothing else uses it. +# +# - `--append-only`: a client cannot delete or overwrite a snapshot, a pack or +# the index, so a compromised hive cannot erase history. Dropping history is +# the prune unit's job, on this host, by retention. +# - `--private-repos`: a login reaches `//` only. There is one +# login per agent, created by swarm-controller (../../swarm-controller/src/archive_access.rs) +# when it places the agent, replaced when it moves and removed when it is +# destroyed. An agent's hive can write that agent's repository and no other. +# +# Everything secret lives in the swarm secret store and is read when needed: +# +# - `swarm-archive-users` (root, every two minutes) renders the htpasswd file +# from the users leaves into `/run`, which is tmpfs. rclone re-reads the file +# when it changes. The leaves hold password hashes only. +# - `swarm-archive-prune` (daily) reads each agent's restic password through +# `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can +# therefore decrypt every backup, which is the cost of pruning where clients +# cannot delete. +# +# Both units log in as the archive's own store principal (a leaf of the store's +# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container: +# no secret crosses a mount, and the backend is one string +# (`reposDir`, in ./lib/archive-server.nix) that names where the repositories +# are. +{ + pkgs, + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + cfg = hyperhiveCfg.deploy.archive; + archiveCfg = hyperhiveCfg.swarm.archive; + baoCfg = hyperhiveCfg.swarm.bao; + baoDeploy = hyperhiveCfg.deploy.bao; + + swarmDomain = hyperhiveCfg.swarm.domain; + # Total on a null swarm domain, as the sibling options modules are: the + # required-domain assertion in hive-network.nix is what an operator sees. + domainBase = if swarmDomain == null then "invalid" else swarmDomain; + + user = "swarm-archive"; + authDir = "/run/swarm-archive-auth"; + authFile = "${authDir}/htpasswd"; + reposDir = "${cfg.dataDir}/repos"; + + server = import ./lib/archive-server.nix { inherit lib; } { + inherit (cfg) retention; + inherit authFile reposDir; + rclone = cfg.package; + restic = cfg.resticPackage; + addr = "127.0.0.1:${toString cfg.port}"; + authGroup = user; + }; + + storeRetry = import ./lib/store-retry.nix { }; + + haveClientIdentity = + baoDeploy.archiveClientCertFile != null && baoDeploy.archiveClientKeyFile != null; + + storeEnvironment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + } + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; +in +{ + options.services.hyperhive.swarm.archive.ownDomain = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "archive.${domainBase}"; + defaultText = lib.literalExpression ''"archive.''${services.hyperhive.swarm.domain}"''; + description = '' + Read-only: the name the swarm's backup archive is served under and every + hive's backup unit dials. A sibling of the swarm's other service names, so + the swarm-services sub-CA can issue for it, and in `swarm.serviceDomains` + on every host whether or not one runs the archive. + ''; + }; + + options.services.hyperhive.deploy.archive = { + package = lib.mkOption { + type = lib.types.package; + default = pkgs.rclone; + defaultText = lib.literalExpression "pkgs.rclone"; + description = "The rclone package: `rclone serve restic`."; + }; + + resticPackage = lib.mkOption { + type = lib.types.package; + default = pkgs.restic; + defaultText = lib.literalExpression "pkgs.restic"; + description = "The restic package the prune unit runs."; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 8911; + description = '' + Port rclone listens on, on the host's loopback. The gateway's vhost + proxies to it; every login is checked by rclone itself. + ''; + }; + + dataDir = lib.mkOption { + type = lib.types.path; + default = "/var/lib/swarm-archive"; + description = '' + Where the repositories live, under `repos//`. Holds restic + ciphertext: every backup of every agent, so size it for that. + ''; + }; + + retention = { + keepWithin = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = "3d"; + example = "14d"; + description = '' + Every snapshot younger than this is kept (restic's `--keep-within`, + written `1y2m3d4h`). It bounds how fast history can be evicted: a + hive that appends many snapshots cannot push the last few days out. + `null` leaves the flag off. + ''; + }; + + keepLast = lib.mkOption { + type = lib.types.ints.positive; + default = 3; + description = '' + The newest this many snapshots of each agent are kept, however old + (restic's `--keep-last`). Together with + {option}`services.hyperhive.deploy.archive.retention.keepWithin` the + prune keeps a snapshot either rule keeps. + ''; + }; + }; + + pruneSchedule = lib.mkOption { + type = lib.types.str; + default = "daily"; + example = "*-*-* 04:00:00"; + description = "systemd calendar expression for the prune: the retention rules are applied then."; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = haveClientIdentity; + message = '' + services.hyperhive.deploy.archive.enable requires the archive's + swarm-secret-store client identity: set both + + services.hyperhive.deploy.bao.archiveClientCertFile + services.hyperhive.deploy.bao.archiveClientKeyFile + + The archive reads its users and the repositories' passwords in the + store with it. + + Both default to the pair glue-bao-tls.nix mints when + services.hyperhive.deploy.bao.enable is true (the host that runs the + store). Otherwise copy archive.pem and archive-key.pem from the store + host's /var/lib/swarm-bao-pki/ and name them in those two options. + ''; + } + ]; + + users.users.${user} = { + isSystemUser = true; + group = user; + description = "swarm backup archive"; + }; + users.groups.${user} = { }; + + # This host serves the vhost, and the host's own resolver finds the name. + services.hyperhive.gateway.localNames = [ archiveCfg.ownDomain ]; + services.hyperhive.gateway.enable = lib.mkDefault true; + services.hyperhive.gateway.dns.enable = lib.mkDefault true; + + # `forceSSL`: the logins ride in a header. No `auth_request`: rclone checks + # them, and restic cannot follow a login redirect. The vhost keeps the + # gateway's default `public`: `public = false` needs the gateway's + # `privateListenAddresses` set, and every request is authenticated either + # way. A backup is one request that can run for minutes, so the body is + # neither size-limited nor buffered. + services.hyperhive.gateway.vhosts."${archiveCfg.ownDomain}" = { + forceSSL = true; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}"; + extraConfig = '' + client_max_body_size 0; + proxy_request_buffering off; + proxy_read_timeout 1h; + proxy_send_timeout 1h; + ''; + }; + }; + + services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [ + "swarm-archive.service" + "swarm-archive-users.service" + "swarm-archive-prune.service" + ]; + + systemd.tmpfiles.rules = [ + "d ${cfg.dataDir} 0750 ${user} ${user} -" + "d ${reposDir} 0750 ${user} ${user} -" + "d ${authDir} 0750 root ${user} -" + ]; + + systemd.services.swarm-archive = { + description = "swarm backup archive (rclone serve restic, append-only, one repository per agent)"; + wantedBy = [ "multi-user.target" ]; + wants = [ "swarm-archive-users.service" ]; + after = [ + "network.target" + "swarm-archive-users.service" + "systemd-tmpfiles-setup.service" + ]; + # rclone writes no config, and reads none. + environment.RCLONE_CONFIG = "/dev/null"; + serviceConfig = { + ExecStart = server.serverCommand; + User = user; + Group = user; + Restart = "on-failure"; + RestartSec = 5; + SyslogIdentifier = "swarm-archive"; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectControlGroups = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + ]; + ReadWritePaths = [ reposDir ]; + ReadOnlyPaths = [ authDir ]; + }; + }; + + # The users file, from the store. A failed run leaves the file as it is; + # the timer and the retry bring a store that comes back up to date. + systemd.services.swarm-archive-users = { + description = "render the archive's users file from the swarm secret store"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + after = [ + "network-online.target" + "systemd-tmpfiles-setup.service" + ]; + path = [ + baoDeploy.package + pkgs.coreutils + pkgs.gnugrep + pkgs.jq + pkgs.systemd + ]; + environment = storeEnvironment // { + BAO_CLIENT_CERT = baoDeploy.archiveClientCertFile; + BAO_CLIENT_KEY = baoDeploy.archiveClientKeyFile; + }; + inherit (storeRetry) startLimitBurst startLimitIntervalSec; + serviceConfig = storeRetry.serviceConfig // { + Type = "oneshot"; + UMask = "0027"; + SyslogIdentifier = "swarm-archive-users"; + TimeoutStartSec = "5min"; + }; + script = server.usersScript; + }; + + # A new grant reaches the server within the controller's pass (five + # minutes) plus this interval. + systemd.timers.swarm-archive-users = { + description = "re-render the archive's users file"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnUnitInactiveSec = "2min"; + RandomizedDelaySec = "20s"; + }; + }; + + systemd.services.swarm-archive-prune = { + description = "apply the archive's retention to every agent's repository"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + path = [ + baoDeploy.package + cfg.resticPackage + pkgs.coreutils + pkgs.gnugrep + pkgs.jq + ]; + # The unit runs as the repositories' owner, so the store identity comes + # in as credentials, which only it can read. + environment = { + BAO_ADDR = storeEnvironment.BAO_ADDR; + BAO_CLIENT_CERT = "%d/bao-cert"; + BAO_CLIENT_KEY = "%d/bao-key"; + RESTIC_CACHE_DIR = "%C/swarm-archive-prune"; + } + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = "%d/bao-ca"; + }; + serviceConfig = { + Type = "oneshot"; + User = user; + Group = user; + SyslogIdentifier = "swarm-archive-prune"; + CacheDirectory = "swarm-archive-prune"; + LoadCredential = [ + "bao-cert:${baoDeploy.archiveClientCertFile}" + "bao-key:${baoDeploy.archiveClientKeyFile}" + ] + ++ lib.optional (baoDeploy.serverCaFile != null) "bao-ca:${baoDeploy.serverCaFile}"; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ReadWritePaths = [ reposDir ]; + TimeoutStartSec = "6h"; + }; + script = server.pruneScript; + }; + + systemd.timers.swarm-archive-prune = { + description = "apply the archive's retention"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = cfg.pruneSchedule; + RandomizedDelaySec = "30min"; + Persistent = true; + }; + }; + }; +} diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 04a053efc..ce1e54d78 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -518,7 +518,39 @@ let capabilities = ["create", "read", "update"] } '') atticHiveFactNames} - path "${credentialMountPath}/data/${ciRunnerTokenLeaf}" { + # The archive credentials of the agents each hive places + # (swarm-controller's archive_access.rs): the hive's copy and the archive's + # hash and restic password, listed to find what no placement justifies and + # deleted when a move or a destroy revokes it. + path "${credentialMountPath}/data/swarm/hives/+/archive/*" { + capabilities = ["create", "read", "update"] + } + + path "${credentialMountPath}/metadata/swarm/hives/+/archive/" { + capabilities = ["list"] + } + + path "${credentialMountPath}/metadata/swarm/hives/+/archive/+" { + capabilities = ["delete"] + } + + path "${credentialMountPath}/data/swarm/services/archive/users/*" { + capabilities = ["create", "read", "update"] + } + + path "${credentialMountPath}/data/swarm/services/archive/restic/*" { + capabilities = ["create", "read"] + } + + path "${credentialMountPath}/metadata/swarm/services/archive/users/" { + capabilities = ["list"] + } + + path "${credentialMountPath}/metadata/swarm/services/archive/users/+" { + capabilities = ["delete"] + } + + path "${credentialMountPath}/data/${ciRunnerTokenLeaf}" { capabilities = ["create", "update"] } @@ -998,7 +1030,11 @@ let "public-key" "pull-token" ] - + readStanza "${credentialMountPath}/data/swarm/hives/${obj.hiveName}/squid-selfcheck"; + + readStanza "${credentialMountPath}/data/swarm/hives/${obj.hiveName}/squid-selfcheck" + # The archive credentials swarm-controller writes for the agents this hive + # places: the backup unit (./hive-archive-backup.nix) lists them and reads each. + + readStanza "${credentialMountPath}/data/swarm/hives/${obj.hiveName}/archive/*" + + listStanza "${credentialMountPath}/metadata/swarm/hives/${obj.hiveName}/archive/"; # The PKI mount agent client certificates are issued from. Its root is # generated inside the store, so the agent CA's key never exists outside it. @@ -1103,6 +1139,12 @@ let } ''; + listStanza = path: '' + path "${path}" { + capabilities = ["list"] + } + ''; + # Singletons, so a list of one rather than a second shape: the granting unit # below takes a list either way and the units stay identical apart from # the objects they write. @@ -1339,6 +1381,20 @@ let } ]; + # The backup archive: the users file is rendered from one leaf per agent, and + # the prune reads each agent's restic password. Read and list only; the + # controller writes both. + archiveReaders = [ + { + name = "swarm-archive"; + cn = baoDeploy.archiveCommonName; + policyText = + readStanza "${credentialMountPath}/data/swarm/services/archive/users/*" + + readStanza "${credentialMountPath}/data/swarm/services/archive/restic/*" + + listStanza "${credentialMountPath}/metadata/swarm/services/archive/users/"; + } + ]; + # The CI run poller: its forge token and its own OIDC client secret, both # read once per pass by the host unit. ciMetricsReaders = [ @@ -2611,6 +2667,42 @@ in ''; }; + archiveCommonName = lib.mkOption { + type = lib.types.str; + default = "swarm-archive"; + description = '' + Subject the store's `swarm-archive` cert-auth role accepts: the identity + the backup archive's host units present. Its grant is read on + `swarm/services/archive/users/*` and `restic/*`, and list on `users`. + + ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings. + ''; + }; + + archiveClientCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/archive.pem"; + description = '' + Certificate the backup archive's host units present to the store. Its + subject must be {option}`services.hyperhive.deploy.bao.archiveCommonName`. + + ./glue-bao-tls.nix points it at the leaf it mints on the store's host. + On an archive host without the store, it is a file the operator copies + from the store's host. A path, never a value. + ''; + }; + + archiveClientKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/archive-key.pem"; + description = '' + Private key for {option}`services.hyperhive.deploy.bao.archiveClientCertFile`. + A path, never a value. + ''; + }; + atticClientCertFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -3847,6 +3939,8 @@ in systemd.services.swarm-bao-attic-policy = readerPolicyUnit "write the in-swarm cache's secret-creator bao policy and cert-auth role" atticWriters; + systemd.services.swarm-bao-archive-policy = readerPolicyUnit "write the backup archive's user and password reader bao policy and cert-auth role" archiveReaders; + systemd.services.swarm-bao-notify-router-policy = readerPolicyUnit "write the notification router's client-secret-reader bao policy and cert-auth role" notifyRouterReaders; systemd.services.swarm-bao-forge-publisher-policy = readerPolicyUnit "write the forge's controller-token-publisher bao policy and cert-auth role" forgePublisherWriters; diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index a6a18b283..dfb4f6188 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -1684,6 +1684,29 @@ in for = "0s"; summary = "swarm-controller failed to mint agent {{ $labels.F_AGENT }}'s store identity; the reason is the `F_ERROR` field of the swarm-controller log line"; }) + # `hive-archive-backup` (nix/host-modules/hive-archive-backup.nix) + # prints one `archive backup failed: ` line per agent + # it could not back up, and one for a hive swarm-controller + # has granted nothing. The backup runs hourly, so `for` + # rides out the minutes a newly placed agent waits for its + # grant, and a second failed run keeps it firing. + (logCountRule { + uid = "hh-archive-backup-failed"; + title = "archive backup failed"; + windowSeconds = 7800; + expr = ''_time:130m _SYSTEMD_UNIT:="hive-archive-backup.service" _msg:"archive backup failed" | stats by (_HOSTNAME) count() as failures''; + for = "30m"; + summary = "{{ $labels._HOSTNAME }} failed to back up an agent to the archive; the unit's journal names the agent and the cause"; + }) + # `swarm-archive-prune` prints `archive prune failed: `. + (logCountRule { + uid = "hh-archive-prune-failed"; + title = "archive prune failed"; + windowSeconds = 90000; + expr = ''_time:25h _SYSTEMD_UNIT:="swarm-archive-prune.service" _msg:"archive prune failed" | stats count() as failures''; + for = "0s"; + summary = "the archive's daily prune failed for an agent's repository, so its retention is not applied; the unit's journal names the agent"; + }) # Logged once per detected crash, by `emit_crash_transitions` # in hive-c0re's crash_watch.rs (10s poll interval). Replaces # the old per-hive dashboard `agents_crashing` banner — see diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix index de1f7e762..93248fafa 100644 --- a/nix/host-modules/swarm-required-services.nix +++ b/nix/host-modules/swarm-required-services.nix @@ -139,6 +139,14 @@ in && deployCfg.bao.atticClientKeyFile != null ); + # The backup archive, on the same terms: its units log in to the store as the + # archive's own principal. + config.services.hyperhive.deploy.archive.enable = lib.mkDefault ( + deployCfg.allSwarmServices + && deployCfg.bao.archiveClientCertFile != null + && deployCfg.bao.archiveClientKeyFile != null + ); + # The claude login minter, defaulted on the same terms as the review # classifier. config.services.hyperhive.deploy.claude-login.enable = lib.mkDefault ( diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index 9b7602392..24660bab4 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -63,6 +63,7 @@ let deployCfg.bao.claudeLoginCommonName deployCfg.bao.notifyRouterCommonName deployCfg.bao.atticCommonName + deployCfg.bao.archiveCommonName deployCfg.bao.forgePublisherCommonName ] # The per-hive subject, spelled out per hive rather than as the prefix. @@ -117,6 +118,8 @@ let # host runs one, as every name here is. `swarm.attic.domain` is not: it is # null where there is no cache and may name an external one. swarmCfg.attic.ownDomain + # The backup archive, listed whether or not any host runs one. + swarmCfg.archive.ownDomain # The store's browser UI, not the store: `swarm.bao.domain` is mTLS and # never gets a gateway certificate. swarmCfg.bao.ui.domain diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 1bbba9ae4..4223dc26e 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -195,6 +195,7 @@ let "swarm-bao-claude-login-policy" "swarm-bao-notify-router-policy" "swarm-bao-attic-policy" + "swarm-bao-archive-policy" ]; # Comment lines dropped first: both the HCL and the scripts explain @@ -2166,6 +2167,30 @@ let in lib.hasInfix "/notify-router.pem ]" s && lib.hasInfix "swarm-notify-router \"\" clientAuth" s; } + { + # The archive reads the users and the repositories' passwords and writes + # nothing; the hive-side copies of the REST password are not its to read. + # The controller is the only writer: the control is its own grant. + name = "the archive's grant is read on its users and restic passwords, list on its users, and nothing else, and only the controller writes them"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-archive-policy.script; + grants = grantsIn s; + c = grantsIn baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + in + (grantFor grants "secret/data/swarm/services/archive/users/atlas").caps == [ "read" ] + && (grantFor grants "secret/data/swarm/services/archive/restic/atlas").caps == [ "read" ] + && (grantFor grants "secret/metadata/swarm/services/archive/users/").caps == [ "list" ] + && grantFor grants "secret/data/swarm/hives/h1/archive/atlas" == null + && lib.length grants == 3 + && lib.hasInfix "allowed_common_names=swarm-archive" s + && lib.hasInfix "token_policies=swarm-archive" s + && lib.elem "create" (grantFor c "secret/data/swarm/services/archive/users/atlas").caps + && lib.elem "create" (grantFor c "secret/data/swarm/hives/h1/archive/atlas").caps + && (grantFor c "secret/metadata/swarm/hives/h1/archive/atlas").caps == [ "delete" ] + && (grantFor c "secret/metadata/swarm/services/archive/users/atlas").caps == [ "delete" ] + && !(lib.elem "delete" (grantFor c "secret/data/swarm/services/archive/restic/atlas").caps); + } { # The cache's host units create each of its six secrets and read them # back, and replace only the public key: `update` there and nowhere else. diff --git a/nix/module-eval/builder.nix b/nix/module-eval/builder.nix index d6bfcb2c9..0ee138621 100644 --- a/nix/module-eval/builder.nix +++ b/nix/module-eval/builder.nix @@ -561,6 +561,45 @@ let && names atticExternal == names bare && !(lib.elem "cache.example.org" (names atticExternal)); } + { + # The archive on the host that hosts the shared services. Append-only and + # private repos are what make one hive's compromise and one agent's + # login small; the control is the bare host, which runs no archive. + name = "the archive serves append-only with private repositories behind a TLS-only vhost, and a host without it renders none"; + ok = + let + a = atticHere.systemd.services.swarm-archive; + users = atticHere.systemd.services.swarm-archive-users; + prune = atticHere.systemd.services.swarm-archive-prune; + vhost = atticHere.services.hyperhive.gateway.vhosts."archive.t.local"; + cfg = atticHere.services.hyperhive.deploy.archive; + names = m: m.services.hyperhive.swarm.serviceDomains; + in + atticHere.services.hyperhive.deploy.archive.enable + && lib.hasInfix "--append-only" a.serviceConfig.ExecStart + && lib.hasInfix "--private-repos" a.serviceConfig.ExecStart + && lib.hasInfix "--htpasswd /run/swarm-archive-auth/htpasswd" a.serviceConfig.ExecStart + && lib.hasInfix "/var/lib/swarm-archive/repos" a.serviceConfig.ExecStart + && a.serviceConfig.User == "swarm-archive" + && vhost.forceSSL + && !(lib.hasInfix "auth_request" vhost.locations."/".extraConfig) + && vhost.locations."/".proxyPass == "http://127.0.0.1:8911" + && lib.elem "archive.t.local" (names atticHere) + && names atticHere == names bare + && cfg.retention.keepWithin == "3d" + && cfg.retention.keepLast == 3 + && lib.hasInfix "--keep-last 3 --keep-within 3d --prune" prune.script + && prune.serviceConfig.User == "swarm-archive" + && lib.hasInfix "RESTIC_PASSWORD_COMMAND=" prune.script + && lib.hasInfix "secret/swarm/services/archive/users" users.script + && users.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/archive.pem" + && !(lib.hasInfix "password" (lib.toLower (builtins.toJSON a.environment))) + && atticHere.systemd.timers ? swarm-archive-users + && atticHere.systemd.timers.swarm-archive-prune.timerConfig.OnCalendar == "daily" + && lib.elem "swarm-bao-archive-policy.service" users.after + && !(bare.systemd.services ? swarm-archive) + && !(bare.systemd.services ? swarm-archive-users); + } ]; in runGroup "builder" cases diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index 21ba6537c..d965d52b5 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -802,6 +802,31 @@ let noService.systemd.services.hive-priv-cache-facts.environment.BAO_CLIENT_CERT == "/etc/pki/bao-hive.pem"; } + { + # ./hive-archive-backup.nix. The grants are read under this hive's own + # identity and only from its own prefix; the swarm's password lives in no + # file and no unit text. The no-identity host is the control that the + # lookup is not always empty. + name = "the hive backs up its agents to the archive under its own identity, hourly, reading only its own grants"; + ok = + let + u = units.hive-archive-backup; + t = allLocal.systemd.timers.hive-archive-backup; + in + lib.hasInfix "secret/swarm/hives/h1/archive" u.script + && !(lib.hasInfix "swarm/services" u.script) + && !(lib.hasInfix "swarm/hives/h2" u.script) + && lib.hasInfix "rest:https://archive.t.local/" u.script + && lib.hasInfix "RESTIC_PASSWORD_COMMAND=" u.script + && lib.hasInfix "--exclude-caches" u.script + && lib.hasInfix "for target in state claude harness" u.script + && u.environment.BAO_CLIENT_CERT == units.hive-priv-cache-facts.environment.BAO_CLIENT_CERT + && u.serviceConfig.Type == "oneshot" + && t.timerConfig.OnCalendar == "hourly" + && lib.elem "timers.target" t.wantedBy + && !(noIdentity.systemd.services ? hive-archive-backup) + && !(noIdentity.systemd.timers ? hive-archive-backup); + } { # Control: the lookup finds the sibling root unit of the same shape. The # squid self-check's password is read under this hive's own identity and diff --git a/nix/vm-tests.nix b/nix/vm-tests.nix index 96d7bad4a..28870ed92 100644 --- a/nix/vm-tests.nix +++ b/nix/vm-tests.nix @@ -10,6 +10,7 @@ agent-ephemeral-rootfs = import ./vm-tests/agent-ephemeral-rootfs.nix { inherit pkgs hivePriv; }; wg-hive = import ./vm-tests/wg-hive.nix { inherit pkgs self; }; hive-squid = import ./vm-tests/hive-squid.nix { inherit pkgs; }; + swarm-archive = import ./vm-tests/swarm-archive.nix { inherit pkgs; }; swarm-builder-offline = import ./vm-tests/swarm-builder-offline.nix { inherit pkgs self; }; full-setup = import ./vm-tests/full-setup.nix { inherit pkgs self; }; } diff --git a/nix/vm-tests/swarm-archive.nix b/nix/vm-tests/swarm-archive.nix new file mode 100644 index 000000000..c40d4f649 --- /dev/null +++ b/nix/vm-tests/swarm-archive.nix @@ -0,0 +1,273 @@ +# `vmTests..swarm-archive`: runs the archive server's real units +# (../host-modules/swarm-archive.nix) and the hive's real backup script +# (../host-modules/lib/archive-backup.nix) in a VM, with restic and rclone, and +# checks what the module-eval suite can only read: +# +# - a backup of two agents lands in one repository each, and `work/` is left out; +# - one agent's login cannot read another's repository; +# - a client cannot delete or overwrite anything, and the server-side prune +# applies the retention; +# - the users file follows the store: a user removed from it is refused without +# a restart of the server; +# - a hive that has agents and no grant fails its backup. +# +# One node, because the gateway hop is outside what is under test: the +# backup script dials `http://127.0.0.1:` instead of +# `https://archive.`, and the swarm secret store is a stand-in `bao` +# that serves `kv list`, `kv get` and `login` from a directory. The module-eval +# suite covers the gateway vhost and the store grants. +{ pkgs }: +let + port = 8911; + store = "/var/lib/fake-bao"; + hive = "h1"; + agentRoot = "/var/lib/hyperhive/agents"; + + # A directory is a leaf, a file in it a field: `kv get -field= `. + fakeBao = pkgs.writeShellScriptBin "bao" '' + store=${store} + case "$1 $2" in + "login -method=cert") echo faketoken; exit 0 ;; + "kv list") + path="$4" + [ -d "$store/$path" ] || { echo "No value found at $path" >&2; exit 2; } + ls "$store/$path" | ${pkgs.jq}/bin/jq -R . | ${pkgs.jq}/bin/jq -s . + exit 0 ;; + "kv get") + field="''${3#-field=}" + [ -f "$store/$4/$field" ] || { echo "No value found at $4" >&2; exit 2; } + cat "$store/$4/$field" + exit 0 ;; + esac + echo "fake bao: unsupported: $*" >&2 + exit 1 + ''; + + # The stubs for the options the archive module reads from the rest of the + # hive modules. + hiveStubs = + { lib, ... }: + { + options.services.hyperhive = { + swarm.domain = lib.mkOption { + type = lib.types.str; + default = "t.local"; + }; + swarm.bao = { + domain = lib.mkOption { + type = lib.types.str; + default = "bao.t.local"; + }; + port = lib.mkOption { + type = lib.types.port; + default = 8200; + }; + }; + deploy.bao = { + package = lib.mkOption { + type = lib.types.package; + default = fakeBao; + }; + archiveClientCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = "/etc/fake-archive-cert"; + }; + archiveClientKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = "/etc/fake-archive-key"; + }; + serverCaFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + }; + }; + deploy.swarm-otel.enable = lib.mkOption { + type = lib.types.bool; + default = false; + }; + deploy.archive.enable = lib.mkOption { + type = lib.types.bool; + default = true; + }; + otel.journalUnits = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + }; + gateway = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + }; + dns.enable = lib.mkOption { + type = lib.types.bool; + default = false; + }; + localNames = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + }; + vhosts = lib.mkOption { + type = lib.types.attrsOf lib.types.anything; + default = { }; + }; + }; + }; + config.environment.etc."fake-archive-cert".text = ""; + config.environment.etc."fake-archive-key".text = ""; + }; + + backupScript = import ../host-modules/lib/archive-backup.nix { inherit (pkgs) lib; } { + inherit hive agentRoot; + restic = pkgs.restic; + url = "http://127.0.0.1:${toString port}"; + scope = "paths"; + paths = [ + "state" + "claude" + "harness" + ]; + excludes = [ ]; + caBundle = null; + }; +in +pkgs.testers.runNixOSTest { + name = "swarm-archive"; + + nodes.machine = + { pkgs, ... }: + { + imports = [ + hiveStubs + ../host-modules/swarm-archive.nix + ]; + + # Keep only the newest snapshot, so the prune is visible after two backups. + services.hyperhive.deploy.archive.retention = { + keepWithin = null; + keepLast = 1; + }; + + environment.systemPackages = [ + pkgs.curl + pkgs.openssl + pkgs.restic + ]; + + systemd.services.test-archive-backup = { + path = [ + fakeBao + pkgs.btrfs-progs + pkgs.coreutils + pkgs.gnugrep + pkgs.jq + ]; + serviceConfig = { + Type = "oneshot"; + Environment = "RESTIC_CACHE_DIR=/var/cache/test-archive-backup"; + }; + script = backupScript; + }; + }; + + testScript = '' + store = "${store}" + url = "http://127.0.0.1:${toString port}" + agents = ["alpha", "beta"] + + def sha_line(agent, password): + return machine.succeed( + f"printf '%s:{{SHA}}%s' {agent} " + f"\"$(printf %s {password} | openssl dgst -sha1 -binary | base64)\"" + ) + + def put(path, field, value): + machine.succeed(f"mkdir -p {store}/{path} && printf %s '{value}' > {store}/{path}/{field}") + + def status(agent, repo, path="config", method="GET"): + return machine.succeed( + f"curl -s -o /dev/null -w '%{{http_code}}' -X {method} " + f"-u {agent}:restpw-{agent} {url}/{repo}/{path}" + ) + + def restic(agent, repo, args): + return ( + f"RESTIC_PASSWORD=resticpw-{repo} restic " + f"-r rest:http://{agent}:restpw-{agent}@127.0.0.1:${toString port}/{repo}/ {args}" + ) + + machine.wait_for_unit("multi-user.target") + machine.wait_for_open_port(${toString port}) + + with subtest("the server starts with no users and admits nobody"): + machine.succeed("test -e /run/swarm-archive-auth/htpasswd") + assert status("alpha", "alpha") == "401", status("alpha", "alpha") + + # What swarm-controller writes: a user line and a restic password per agent + # for the archive, a REST password and the restic password for the hive. + for agent in agents: + put(f"secret/swarm/services/archive/users/{agent}", "htpasswd", sha_line(agent, f"restpw-{agent}")) + put(f"secret/swarm/services/archive/restic/{agent}", "password", f"resticpw-{agent}") + put(f"secret/swarm/hives/${hive}/archive/{agent}", "rest-password", f"restpw-{agent}") + put(f"secret/swarm/hives/${hive}/archive/{agent}", "restic-password", f"resticpw-{agent}") + machine.succeed( + f"mkdir -p ${agentRoot}/{agent}/state ${agentRoot}/{agent}/claude " + f"${agentRoot}/{agent}/harness ${agentRoot}/{agent}/work" + ) + machine.succeed(f"echo {agent}-state > ${agentRoot}/{agent}/state/data") + machine.succeed(f"echo {agent}-work > ${agentRoot}/{agent}/work/scratch") + + with subtest("the users file follows the store without a restart of the server"): + pid = machine.succeed("systemctl show -p MainPID --value swarm-archive").strip() + machine.succeed("systemctl start swarm-archive-users") + assert machine.succeed("systemctl show -p MainPID --value swarm-archive").strip() == pid + assert status("alpha", "alpha") in ("200", "404"), status("alpha", "alpha") + assert status("beta", "beta") in ("200", "404"), status("beta", "beta") + # Wrong password: the control that the 200 above is a login, not an open door. + machine.succeed(f"test $(curl -s -o /dev/null -w '%{{http_code}}' -u alpha:wrong {url}/alpha/config) = 401") + + with subtest("the users file holds hashes, readable by the server only"): + machine.succeed("grep -q '{SHA}' /run/swarm-archive-auth/htpasswd") + machine.fail("grep -q restpw /run/swarm-archive-auth/htpasswd") + machine.succeed("test \"$(stat -c '%a %U:%G' /run/swarm-archive-auth/htpasswd)\" = '640 root:swarm-archive'") + + with subtest("the hive's backup puts each agent in its own repository, without work/"): + machine.succeed("systemctl start test-archive-backup") + for agent in agents: + snaps = machine.succeed(restic(agent, agent, "snapshots --json")) + assert snaps.count('"id"') == 1, snaps + listing = machine.succeed(restic(agent, agent, "ls latest")) + assert f"/{agent}/state/data" in listing, listing + assert "scratch" not in listing, listing + + with subtest("one agent's login cannot reach another's repository"): + assert status("beta", "alpha") == "403", status("beta", "alpha") + machine.fail(restic("beta", "alpha", "snapshots")) + assert status("alpha", "alpha") == "200", status("alpha", "alpha") + assert status("alpha", "beta") == "403", status("alpha", "beta") + + with subtest("a client cannot delete or overwrite"): + snap = machine.succeed(restic("alpha", "alpha", "snapshots --json")).split('"id":"')[1].split('"')[0] + # The presence control: a lock, which a backup creates and removes, is deletable. + machine.succeed(restic("alpha", "alpha", "backup --host alpha ${agentRoot}/alpha/state")) + assert status("alpha", "alpha", f"snapshots/{snap}", "DELETE") == "403" + assert status("alpha", "alpha", "config", "POST") == "403" + machine.fail(restic("alpha", "alpha", "forget --keep-last 1 --prune")) + assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2 + + with subtest("the server-side prune applies the retention"): + machine.succeed("systemctl start swarm-archive-prune") + for agent in agents: + assert machine.succeed(restic(agent, agent, "snapshots --json")).count('"id"') == 1 + + with subtest("a user removed from the store is refused without a restart"): + machine.succeed(f"rm -r {store}/secret/swarm/services/archive/users/beta") + machine.succeed("systemctl start swarm-archive-users") + assert status("beta", "beta") == "401", status("beta", "beta") + assert status("alpha", "alpha") == "200", status("alpha", "alpha") + + with subtest("a hive with agents and no grant fails its backup"): + machine.succeed(f"rm -r {store}/secret/swarm/hives/${hive}") + machine.fail("systemctl start test-archive-backup") + machine.succeed("journalctl -u test-archive-backup | grep -q 'archive backup failed: swarm-controller has granted this hive no archive credentials'") + ''; +} diff --git a/swarm-controller/Cargo.toml b/swarm-controller/Cargo.toml index 9e17d0a39..90bdbc051 100644 --- a/swarm-controller/Cargo.toml +++ b/swarm-controller/Cargo.toml @@ -68,6 +68,8 @@ opentelemetry-otlp.workspace = true # is departing, not arriving — see that module's docs. hmac.workspace = true sha2.workspace = true +# The `{SHA}` htpasswd line `archive_access.rs` writes for the archive server. +sha1.workspace = true # `forge/pull_diff.rs` diffs a PR's changed files itself, at a context width # the forge's own `.diff` does not offer. similar.workspace = true diff --git a/swarm-controller/src/archive_access.rs b/swarm-controller/src/archive_access.rs new file mode 100644 index 000000000..0df949fb5 --- /dev/null +++ b/swarm-controller/src/archive_access.rs @@ -0,0 +1,572 @@ +//! Each placed agent's archive credentials, and their revocation. +//! +//! The archive (`swarm-archive.nix`) keeps one restic repository per agent and +//! lets a user reach only its own. Every [`RECONCILE_INTERVAL`] this makes the +//! store hold, per `(hive, agent)` that [`crate::jobq::placements`] reports: +//! +//! - the REST password and the restic password at +//! [`archive::grant_path`], which that hive's store identity reads; +//! - the htpasswd line, a hash of the REST password, at [`archive::user_path`], +//! which the archive host renders its users file from; +//! - the restic password at [`archive::restic_path`], created once and never +//! replaced: it opens the agent's repository wherever the agent runs, and the +//! archive host prunes with it. +//! +//! A pass is level-triggered and idempotent. A move to another hive mints a new +//! REST password, which replaces the htpasswd line, so the old hive's password +//! stops working once the archive host re-renders; the old hive's copy is +//! deleted in the same pass. An agent no hive places loses its htpasswd line and +//! every hive's copy. Its restic password stays, so what was backed up can still +//! be read. +//! +//! An agent that two hives place is left as it is: either grant would replace +//! the other's every pass. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use base64::Engine as _; +use sha1::{Digest as _, Sha1}; +use swarm_secret_client::SecretStore; +use swarm_secret_client::archive::{self, HTPASSWD, PASSWORD, REST_PASSWORD, RESTIC_PASSWORD}; + +use crate::wanted::WantedWriter; + +/// How often [`spawn`] reconciles. +const RECONCILE_INTERVAL: Duration = Duration::from_mins(5); + +/// A pass that has not finished by now is given up on; the next tick retries. +const PASS_DEADLINE: Duration = Duration::from_mins(2); + +/// An object in the store: field name to value. +type Fields = BTreeMap; + +/// The store calls a pass makes, so a test can stand in for the store. +pub(crate) trait ArchiveStore { + /// The object at `path`; `None` when nothing is stored there. + async fn read(&self, path: &str) -> Result>; + /// Store `fields` at `path`, replacing what is there. + async fn write(&self, path: &str, fields: &Fields) -> Result<()>; + /// Store `fields` at `path` only while nothing is there. + async fn create(&self, path: &str, fields: &Fields) -> Result<()>; + /// The names stored directly under `path`. + async fn list(&self, path: &str) -> Result>; + /// Remove `path` with every version of it; nothing there is not an error. + async fn delete(&self, path: &str) -> Result<()>; +} + +impl ArchiveStore for SecretStore { + async fn read(&self, path: &str) -> Result> { + Ok(self.read_optional(path).await?) + } + + async fn write(&self, path: &str, fields: &Fields) -> Result<()> { + Ok(SecretStore::write(self, path, fields).await?) + } + + async fn create(&self, path: &str, fields: &Fields) -> Result<()> { + Ok(SecretStore::create(self, path, fields).await?) + } + + async fn list(&self, path: &str) -> Result> { + Ok(SecretStore::list(self, path).await?) + } + + async fn delete(&self, path: &str) -> Result<()> { + Ok(self.delete_all_versions(path).await?) + } +} + +/// What one pass did. +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct Done { + /// Agents whose credentials were written or replaced. + pub granted: usize, + /// Credentials removed: a hive's copy or an htpasswd line. + pub revoked: usize, + /// Agents or hives whose credentials could not all be checked or written. + pub failed: usize, +} + +fn fields(pairs: &[(&str, &str)]) -> Fields { + pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect() +} + +/// The non-empty field `name` of `object`. +fn field<'a>(object: Option<&'a Fields>, name: &str) -> Option<&'a str> { + object + .and_then(|o| o.get(name)) + .map(String::as_str) + .filter(|v| !v.is_empty()) +} + +/// The htpasswd line that accepts `password` for `agent`: apache's `{SHA}` +/// form, which the password's 256 random bits make as strong as a slower hash. +fn htpasswd_line(agent: &str, password: &str) -> String { + let digest = Sha1::digest(password.as_bytes()); + format!( + "{agent}:{{SHA}}{}", + base64::engine::general_purpose::STANDARD.encode(digest) + ) +} + +/// `agent`'s restic password: the stored one, else a new one created at +/// [`archive::restic_path`]. A create that fails is read back, in case another +/// writer's landed first. +async fn ensure_restic(store: &impl ArchiveStore, agent: &str) -> Result { + let path = archive::restic_path(agent)?; + let stored = store + .read(&path) + .await + .with_context(|| format!("reading {path}"))?; + if let Some(password) = field(stored.as_ref(), PASSWORD) { + return Ok(password.to_owned()); + } + let minted = crate::agent_identity::generate_queue_secret()?; + if let Err(e) = store.create(&path, &fields(&[(PASSWORD, &minted)])).await { + let winner = store + .read(&path) + .await + .with_context(|| format!("reading {path} back after a failed create"))?; + return field(winner.as_ref(), PASSWORD) + .map(str::to_owned) + .ok_or_else(|| e.context(format!("creating {path}"))); + } + Ok(minted) +} + +/// Bring `agent`'s credentials on `hive` and at the archive up to date; whether +/// anything was written. +async fn grant(store: &impl ArchiveStore, hive: &str, agent: &str) -> Result { + let restic = ensure_restic(store, agent).await?; + let grant_path = archive::grant_path(hive, agent)?; + let held = store + .read(&grant_path) + .await + .with_context(|| format!("reading {grant_path}"))?; + + let mut written = false; + let rest = match field(held.as_ref(), REST_PASSWORD) { + Some(rest) if field(held.as_ref(), RESTIC_PASSWORD) == Some(restic.as_str()) => { + rest.to_owned() + } + _ => { + let rest = crate::agent_identity::generate_queue_secret()?; + store + .write( + &grant_path, + &fields(&[(REST_PASSWORD, &rest), (RESTIC_PASSWORD, &restic)]), + ) + .await + .with_context(|| format!("writing {grant_path}"))?; + written = true; + rest + } + }; + + let user_path = archive::user_path(agent)?; + let line = htpasswd_line(agent, &rest); + let user = store + .read(&user_path) + .await + .with_context(|| format!("reading {user_path}"))?; + if field(user.as_ref(), HTPASSWD) != Some(line.as_str()) { + store + .write(&user_path, &fields(&[(HTPASSWD, &line)])) + .await + .with_context(|| format!("writing {user_path}"))?; + written = true; + } + Ok(written) +} + +/// The entries under `prefix` that are single leaves. +async fn leaves(store: &impl ArchiveStore, prefix: &str) -> Result> { + Ok(store + .list(prefix) + .await + .with_context(|| format!("listing {prefix}"))? + .into_iter() + .filter(|name| !name.ends_with('/')) + .collect()) +} + +/// One pass: grant every agent in `placed`, then revoke what no placement +/// justifies. `hives` is every hive of the swarm, whose copies are checked. +/// +/// # Errors +/// Never: a failure is counted in [`Done::failed`] and logged, and the rest of +/// the pass still runs. +pub(crate) async fn reconcile( + store: &impl ArchiveStore, + placed: &BTreeSet<(String, String)>, + hives: &[String], +) -> Done { + let mut done = Done::default(); + let mut holders: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for (hive, agent) in placed { + holders.entry(agent).or_default().push(hive); + } + + let mut ambiguous = BTreeSet::new(); + for (agent, on) in &holders { + if let [hive] = on.as_slice() { + match grant(store, hive, agent).await { + Ok(true) => done.granted += 1, + Ok(false) => {} + Err(e) => { + tracing::warn!(hive, agent, error = %format!("{e:#}"), "archive: granting failed"); + done.failed += 1; + } + } + } else { + tracing::warn!(agent, hives = ?on, "archive: placed on more than one hive; credentials left as they are"); + ambiguous.insert(*agent); + } + } + + for hive in hives { + let copies = match archive::grants_prefix(hive) { + Ok(prefix) => leaves(store, &prefix).await, + Err(e) => Err(e.into()), + }; + let copies = match copies { + Ok(copies) => copies, + Err(e) => { + tracing::warn!(hive, error = %format!("{e:#}"), "archive: listing the hive's copies failed"); + done.failed += 1; + continue; + } + }; + for agent in copies { + let keep = ambiguous.contains(agent.as_str()) + || placed.contains(&(hive.clone(), agent.clone())); + if !keep { + revoke(store, archive::grant_path(hive, &agent), &mut done).await; + } + } + } + + match leaves(store, &archive::users_prefix().unwrap_or_default()).await { + Ok(users) => { + for agent in users { + if !holders.contains_key(agent.as_str()) { + revoke(store, archive::user_path(&agent), &mut done).await; + } + } + } + Err(e) => { + tracing::warn!(error = %format!("{e:#}"), "archive: listing the users failed"); + done.failed += 1; + } + } + done +} + +/// Delete the credential at `path`, counting the outcome in `done`. +async fn revoke( + store: &impl ArchiveStore, + path: Result, + done: &mut Done, +) { + let path = match path { + Ok(path) => path, + Err(e) => { + tracing::warn!(error = %e, "archive: a stored name is not a valid path segment"); + done.failed += 1; + return; + } + }; + match store.delete(&path).await { + Ok(()) => { + tracing::info!(%path, "archive: credential revoked"); + done.revoked += 1; + } + Err(e) => { + tracing::warn!(%path, error = %format!("{e:#}"), "archive: revoking failed"); + done.failed += 1; + } + } +} + +/// Reconcile now and every [`RECONCILE_INTERVAL`] after. A pass that fails is +/// logged and retried on the next tick; it never stops the daemon. Nothing is +/// revoked on a pass that could not read where the hives place agents. +pub fn spawn(wanted: Arc, hives: Vec) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); + loop { + ticker.tick().await; + let pass = async { + let placed = crate::jobq::placements(&wanted, &hives).await?; + let store = crate::store::connect() + .await + .context("logging in to the swarm secret store")?; + anyhow::Ok(reconcile(&store, &placed, &hives).await) + }; + match tokio::time::timeout(PASS_DEADLINE, pass).await { + Ok(Ok(done)) if done != Done::default() => tracing::info!( + granted = done.granted, + revoked = done.revoked, + failed = done.failed, + "archive: credentials reconciled" + ), + Ok(Ok(_)) => tracing::debug!("archive: every credential is current"), + Ok(Err(e)) => tracing::warn!( + error = %format!("{e:#}"), + retry_in_s = RECONCILE_INTERVAL.as_secs(), + "archive: pass failed; retrying next tick" + ), + Err(_) => tracing::warn!( + deadline_s = PASS_DEADLINE.as_secs(), + "archive: pass timed out; retrying next tick" + ), + } + } + }); +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::*; + + /// An in-memory store. `create` refuses a held path, `list` answers like + /// the store's: the names directly under a prefix. + #[derive(Default)] + struct Memory { + kv: Mutex>, + /// Paths under this prefix refuse every write and delete. + refuse: Option<&'static str>, + } + + impl Memory { + fn get(&self, path: &str) -> Option { + self.kv.lock().unwrap().get(path).cloned() + } + + fn value(&self, path: &str, name: &str) -> Option { + self.get(path).and_then(|f| f.get(name).cloned()) + } + + fn refused(&self, path: &str) -> bool { + self.refuse.is_some_and(|p| path.starts_with(p)) + } + } + + impl ArchiveStore for Memory { + async fn read(&self, path: &str) -> Result> { + Ok(self.get(path)) + } + + async fn write(&self, path: &str, fields: &Fields) -> Result<()> { + if self.refused(path) { + anyhow::bail!("denied"); + } + self.kv + .lock() + .unwrap() + .insert(path.to_owned(), fields.clone()); + Ok(()) + } + + async fn create(&self, path: &str, fields: &Fields) -> Result<()> { + if self.get(path).is_some() { + anyhow::bail!("exists"); + } + self.write(path, fields).await + } + + async fn list(&self, path: &str) -> Result> { + let prefix = format!("{path}/"); + Ok(self + .kv + .lock() + .unwrap() + .keys() + .filter_map(|k| k.strip_prefix(&prefix)) + .map(|rest| match rest.split_once('/') { + Some((dir, _)) => format!("{dir}/"), + None => rest.to_owned(), + }) + .collect()) + } + + async fn delete(&self, path: &str) -> Result<()> { + if self.refused(path) { + anyhow::bail!("denied"); + } + self.kv.lock().unwrap().remove(path); + Ok(()) + } + } + + fn placed(pairs: &[(&str, &str)]) -> BTreeSet<(String, String)> { + pairs + .iter() + .map(|(h, a)| ((*h).to_owned(), (*a).to_owned())) + .collect() + } + + fn hives() -> Vec { + vec!["alpha".to_owned(), "beta".to_owned()] + } + + const ALPHA_ATLAS: &str = "swarm/hives/alpha/archive/atlas"; + const BETA_ATLAS: &str = "swarm/hives/beta/archive/atlas"; + const USER_ATLAS: &str = "swarm/services/archive/users/atlas"; + const RESTIC_ATLAS: &str = "swarm/services/archive/restic/atlas"; + + #[test] + fn the_htpasswd_line_is_apaches_sha_form() { + // `htpasswd -nbs atlas secret` + assert_eq!( + htpasswd_line("atlas", "secret"), + "atlas:{SHA}5en6G6MezRroT3XKqkdPOmY/BfQ=" + ); + } + + #[tokio::test] + async fn a_placed_agent_gets_its_hive_a_password_and_the_archive_its_hash() { + let store = Memory::default(); + let done = reconcile(&store, &placed(&[("alpha", "atlas")]), &hives()).await; + assert_eq!( + done, + Done { + granted: 1, + revoked: 0, + failed: 0 + } + ); + let rest = store.value(ALPHA_ATLAS, REST_PASSWORD).expect("a password"); + let restic = store.value(ALPHA_ATLAS, RESTIC_PASSWORD).expect("restic"); + assert_eq!( + store.value(USER_ATLAS, HTPASSWD), + Some(htpasswd_line("atlas", &rest)) + ); + assert_eq!(store.value(RESTIC_ATLAS, PASSWORD), Some(restic)); + assert_ne!(Some(rest), store.value(ALPHA_ATLAS, RESTIC_PASSWORD)); + // The hash line is all the archive's user leaf holds. + assert!(!store.get(USER_ATLAS).unwrap().contains_key(REST_PASSWORD)); + // Presence control for the absence checks below: only the placed hive. + assert!(store.get(BETA_ATLAS).is_none()); + } + + #[tokio::test] + async fn a_current_grant_is_not_rewritten() { + let store = Memory::default(); + let one = placed(&[("alpha", "atlas")]); + reconcile(&store, &one, &hives()).await; + let before = store.get(ALPHA_ATLAS); + let done = reconcile(&store, &one, &hives()).await; + assert_eq!(done, Done::default()); + assert_eq!(store.get(ALPHA_ATLAS), before); + } + + #[tokio::test] + async fn a_move_replaces_the_password_and_takes_the_old_hives_copy() { + let store = Memory::default(); + reconcile(&store, &placed(&[("alpha", "atlas")]), &hives()).await; + let old_rest = store.value(ALPHA_ATLAS, REST_PASSWORD).unwrap(); + let restic = store.value(RESTIC_ATLAS, PASSWORD).unwrap(); + + let done = reconcile(&store, &placed(&[("beta", "atlas")]), &hives()).await; + assert_eq!(done.granted, 1); + assert_eq!(done.revoked, 1); + assert!(store.get(ALPHA_ATLAS).is_none(), "the old hive loses read"); + let new_rest = store.value(BETA_ATLAS, REST_PASSWORD).expect("new hive"); + assert_ne!(new_rest, old_rest, "the old password stops working"); + assert_eq!( + store.value(USER_ATLAS, HTPASSWD), + Some(htpasswd_line("atlas", &new_rest)) + ); + assert_ne!( + store.value(USER_ATLAS, HTPASSWD), + Some(htpasswd_line("atlas", &old_rest)) + ); + assert_eq!( + store.value(BETA_ATLAS, RESTIC_PASSWORD).as_deref(), + Some(restic.as_str()), + "the repository stays readable on the new hive" + ); + } + + #[tokio::test] + async fn a_destroyed_agent_loses_every_credential_but_the_restic_password() { + let store = Memory::default(); + reconcile( + &store, + &placed(&[("alpha", "atlas"), ("beta", "argus")]), + &hives(), + ) + .await; + let done = reconcile(&store, &placed(&[("beta", "argus")]), &hives()).await; + assert_eq!(done.revoked, 2); + assert!(store.get(ALPHA_ATLAS).is_none()); + assert!(store.get(USER_ATLAS).is_none()); + assert!( + store.get(RESTIC_ATLAS).is_some(), + "the backups stay readable" + ); + // Control: the agent still placed keeps all of it. + assert!(store.get("swarm/hives/beta/archive/argus").is_some()); + assert!(store.get("swarm/services/archive/users/argus").is_some()); + } + + #[tokio::test] + async fn an_agent_two_hives_place_is_left_as_it_is() { + let store = Memory::default(); + reconcile(&store, &placed(&[("alpha", "atlas")]), &hives()).await; + let before = (store.get(ALPHA_ATLAS), store.get(USER_ATLAS)); + let both = placed(&[("alpha", "atlas"), ("beta", "atlas")]); + let done = reconcile(&store, &both, &hives()).await; + assert_eq!(done, Done::default()); + assert_eq!((store.get(ALPHA_ATLAS), store.get(USER_ATLAS)), before); + assert!(store.get(BETA_ATLAS).is_none()); + + // Neither hive is granted when the agent is new and both place it. + let fresh = Memory::default(); + let done = reconcile(&fresh, &both, &hives()).await; + assert_eq!(done, Done::default()); + assert!(fresh.kv.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_lost_user_line_is_rewritten_from_the_hives_password() { + let store = Memory::default(); + let one = placed(&[("alpha", "atlas")]); + reconcile(&store, &one, &hives()).await; + let line = store.value(USER_ATLAS, HTPASSWD); + store.kv.lock().unwrap().remove(USER_ATLAS); + let done = reconcile(&store, &one, &hives()).await; + assert_eq!(done.granted, 1); + assert_eq!(store.value(USER_ATLAS, HTPASSWD), line); + } + + #[tokio::test] + async fn a_refused_write_counts_as_failed_and_the_other_agents_still_get_theirs() { + let store = Memory { + refuse: Some("swarm/hives/alpha/"), + ..Memory::default() + }; + let done = reconcile( + &store, + &placed(&[("alpha", "atlas"), ("beta", "argus")]), + &hives(), + ) + .await; + assert_eq!(done.failed, 1); + assert_eq!(done.granted, 1); + assert!(store.get("swarm/hives/beta/archive/argus").is_some()); + assert!( + store.get(USER_ATLAS).is_none(), + "no hash without the hive's copy" + ); + } +} diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 3a7388b73..0da2f2bd6 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -39,6 +39,7 @@ mod agents_md; mod agents_md_initial; mod alerts; mod archetypes; +mod archive_access; mod attic_copy; mod auth; mod backend_egress; diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs index 169083aff..39555dff5 100644 --- a/swarm-controller/src/startup.rs +++ b/swarm-controller/src/startup.rs @@ -366,7 +366,8 @@ fn spawn_subagent_streams(status: Option<&Arc>, hives: &[H /// schedule keys, and the permissions bucket and its import, the allowed /// domains derived from backend environments, and the /// archetypes agents are created from. Beside them, the -/// copy of the cache facts to each hive, and the proxy credentials squid checks. +/// copy of the cache facts to each hive, the archive credentials of each placed +/// agent, and the proxy credentials squid checks. /// Lifted out of `main` for `clippy::too_many_lines`. pub(crate) fn spawn_per_agent_upkeep( status: Option<&Arc>, @@ -377,12 +378,22 @@ pub(crate) fn spawn_per_agent_upkeep( spawn_agent_permissions(status, hives); spawn_backend_egress_backfill(status, hives); crate::attic_copy::spawn(hives.iter().map(|h| h.name.clone()).collect()); + spawn_archive_access(status, hives); crate::proxy_credential::spawn(hives.iter().map(|h| h.name.clone()).collect()); if let Some(status) = status { archetypes::spawn(status.queue_client()); } } +/// Grant and revoke each placed agent's archive credentials when a swarm queue +/// is wired up. Same shape as [`spawn_subagent_streams`]. +fn spawn_archive_access(status: Option<&Arc>, hives: &[HiveEntry]) { + let Some(wanted) = wanted_writer(status) else { + return; + }; + crate::archive_access::spawn(wanted, hives.iter().map(|h| h.name.clone()).collect()); +} + /// Create the agent-permissions bucket and answer hives' imports when a swarm /// queue is wired up. Same shape as [`spawn_subagent_streams`]. fn spawn_agent_permissions(status: Option<&Arc>, hives: &[HiveEntry]) { diff --git a/swarm-secret-client/src/archive.rs b/swarm-secret-client/src/archive.rs new file mode 100644 index 000000000..1bf3c18fb --- /dev/null +++ b/swarm-secret-client/src/archive.rs @@ -0,0 +1,124 @@ +//! The archive agreement: where `swarm-controller` keeps each agent's archive +//! credentials, and where its hive and the archive host read them. +//! +//! Three paths per agent, all written by the controller: +//! +//! - [`grant_path`], `swarm/hives//archive/`: the REST password the +//! agent's hive logs in to the archive with, and the restic repository +//! password. The hive's own `swarm/hives//*` stanza reads it, and only +//! the hive that places the agent holds one. +//! - [`user_path`], `swarm/services/archive/users/`: the htpasswd line +//! the archive host renders its users file from. A hash, never the password. +//! - [`restic_path`], `swarm/services/archive/restic/`: the same restic +//! password, kept where it outlives a move. The archive host reads it to +//! prune the repository. +//! +//! The nix half is `swarm-bao.nix`'s archive grants and `swarm-archive.nix`. + +use crate::{ + Error, + path::{Kind, checked_segment, principal_prefix}, +}; + +/// The service segment under `swarm/services/`. +pub const SERVICE: &str = "archive"; + +/// The field of [`grant_path`] holding the REST password. +pub const REST_PASSWORD: &str = "rest-password"; + +/// The field of [`grant_path`] holding the restic repository password. +pub const RESTIC_PASSWORD: &str = "restic-password"; + +/// The field of [`restic_path`] holding the restic repository password. +pub const PASSWORD: &str = "password"; + +/// The field of [`user_path`] holding the htpasswd line. +pub const HTPASSWD: &str = "htpasswd"; + +/// The directory holding every [`user_path`], to list. +/// +/// # Errors +/// Never in practice: the segments are constants. +pub fn users_prefix() -> Result { + Ok(format!( + "{}/users", + principal_prefix(Kind::Service, SERVICE)? + )) +} + +/// Where the archive host reads `agent`'s htpasswd line. +/// +/// # Errors +/// [`Error::PathSegment`] when `agent` contains anything but `[A-Za-z0-9_-]`. +pub fn user_path(agent: &str) -> Result { + checked_segment(Kind::Agent.label(), agent)?; + Ok(format!("{}/{agent}", users_prefix()?)) +} + +/// Where the archive host reads `agent`'s restic password. +/// +/// # Errors +/// [`Error::PathSegment`] when `agent` contains anything but `[A-Za-z0-9_-]`. +pub fn restic_path(agent: &str) -> Result { + checked_segment(Kind::Agent.label(), agent)?; + Ok(format!( + "{}/restic/{agent}", + principal_prefix(Kind::Service, SERVICE)? + )) +} + +/// The directory holding every [`grant_path`] of `hive`, to list. +/// +/// # Errors +/// [`Error::PathSegment`] when `hive` contains anything but `[A-Za-z0-9_-]`. +pub fn grants_prefix(hive: &str) -> Result { + Ok(format!("{}/archive", principal_prefix(Kind::Hive, hive)?)) +} + +/// Where `hive` reads the credentials of `agent`, which it places. +/// +/// # Errors +/// [`Error::PathSegment`] when `hive` or `agent` contains anything but +/// `[A-Za-z0-9_-]`, which keeps one name from addressing another's secret. +pub fn grant_path(hive: &str, agent: &str) -> Result { + checked_segment(Kind::Agent.label(), agent)?; + Ok(format!("{}/{agent}", grants_prefix(hive)?)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_paths_are_the_ones_the_store_grants() { + // Spelled out: `swarm-bao.nix` grants each pattern. + assert_eq!( + user_path("atlas").expect("a plain name"), + "swarm/services/archive/users/atlas" + ); + assert_eq!( + restic_path("atlas").expect("a plain name"), + "swarm/services/archive/restic/atlas" + ); + assert_eq!( + grant_path("alpha", "atlas").expect("plain names"), + "swarm/hives/alpha/archive/atlas" + ); + assert_eq!( + users_prefix().expect("constant"), + "swarm/services/archive/users" + ); + assert_eq!( + grants_prefix("alpha").expect("a plain name"), + "swarm/hives/alpha/archive" + ); + } + + #[test] + fn a_name_cannot_address_another_principals_credentials() { + assert!(grant_path("alpha", "atlas/../argus").is_err()); + assert!(grant_path("alpha/../beta", "atlas").is_err()); + assert!(user_path("../restic/atlas").is_err()); + assert!(restic_path("").is_err()); + } +} diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index 9a53d4404..f499a8ed2 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -30,6 +30,7 @@ //! needs only the provider and the store links nothing else. pub mod acp; +pub mod archive; pub mod attic; pub mod backend_env; pub mod builder;