vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

run attic as a swarm service with deploy.attic.enable

Browse source
This commit is contained in:
atlas 2026-10-27 12:23:28 +01:00
commit 458a046b8d
17 changed files with 1242 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

4
nix/checks.nix
View file

@ -68,6 +68,10 @@ in
# Runs ./host-modules/lib/refresh-consumer.nix's restart decision under a
# comma-decimal locale, against a stub `systemctl`.
script-test-refresh-consumer = import ./script-tests/refresh-consumer.nix { inherit pkgs; };
# Runs ./host-modules/lib/attic-init.nix's two scripts, which create the swarm
# cache's key, cache and tokens, twice over against stub tools: the second
# run must mint nothing.
script-test-attic-init = import ./script-tests/attic-init.nix { inherit pkgs; };
module-eval-nats-tls = import ./module-eval/nats-tls.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;

1
nix/host-modules/default.nix
View file

@ -63,6 +63,7 @@
./glue-victorialogs-otel-oidc-client.nix
./glue-victoriametrics-otel-oidc-client.nix
./swarm-attic-options.nix
./swarm-attic.nix
./swarm-authelia-service.nix
./swarm-authelia.nix
./swarm-bao-service.nix

22
nix/host-modules/deploy.nix
View file

@ -505,6 +505,28 @@ in
'';
};
attic.enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = ''
Run the swarm's binary cache in a `swarm-attic` container on this
host: an attic server behind the gateway, with its signing key, its
cache and its push and pull tokens created in the secret store on first
start. The swarm builder pushes green builds to it. Run it on one host.
On by default where
{option}`services.hyperhive.deploy.allSwarmServices` is set and this
host has the cache's store identity (the pair
{option}`services.hyperhive.deploy.bao.atticClientCertFile` and
`atticClientKeyFile`, minted on the store's host).
With it on, {option}`services.hyperhive.swarm.attic.domain` and `cache`
default to the cache's own name and cache; naming either yourself wins.
With it off, those two options name an external cache or stay unset.
'';
};
hive-controller.enable = lib.mkOption {
type = lib.types.bool;
default = false;

10
nix/host-modules/glue-bao-readers-policy-order.nix
View file

@ -79,6 +79,16 @@ let
here = deployCfg.grafana.enable;
policies = [ "swarm-bao-grafana-oidc-policy" ];
};
# ./swarm-attic.nix, under the cache's leaf, which may create the signing
# key and the two tokens only once that policy is written.
swarm-bao-attic-secret = {
here = deployCfg.attic.enable;
policies = [ "swarm-bao-attic-policy" ];
};
swarm-bao-attic-tokens = {
here = deployCfg.attic.enable;
policies = [ "swarm-bao-attic-policy" ];
};
# ./hive-matrix.nix, under matrix-ctl's leaf.
swarm-bao-matrix-oidc = {
here = deployCfg.matrix.enable;

3
nix/host-modules/glue-bao-readers-store-order.nix
View file

@ -60,6 +60,9 @@ let
swarm-bao-nats-callout-xkey = deployCfg.nats.enable;
# ./swarm-ci-runner.nix
swarm-bao-ci-runner = deployCfg.ci-runner.enable;
# ./swarm-attic.nix
swarm-bao-attic-secret = deployCfg.attic.enable;
swarm-bao-attic-tokens = deployCfg.attic.enable;
# ./swarm-otel.nix
swarm-bao-otel-oidc =
deployCfg.swarm-otel.enable

9
nix/host-modules/glue-bao-tls.nix
View file

@ -124,6 +124,8 @@ in
claudeLoginClientKeyFile = lib.mkDefault "${pkiDir}/claude-login-key.pem";
notifyRouterClientCertFile = lib.mkDefault "${pkiDir}/notify-router.pem";
notifyRouterClientKeyFile = lib.mkDefault "${pkiDir}/notify-router-key.pem";
atticClientCertFile = lib.mkDefault "${pkiDir}/attic.pem";
atticClientKeyFile = lib.mkDefault "${pkiDir}/attic-key.pem";
forgePublisherClientCertFile = lib.mkDefault "${pkiDir}/forge-publisher.pem";
forgePublisherClientKeyFile = lib.mkDefault "${pkiDir}/forge-publisher-key.pem";
servicesIssuerClientCertFile = lib.mkDefault "${pkiDir}/services-issuer.pem";
@ -328,6 +330,13 @@ in
[ -s ${pkiDir}/notify-router.pem ] || ${signLeaf} ${pkiDir} notify-router \
${lib.escapeShellArg deployCfg.bao.notifyRouterCommonName} "" clientAuth
# The in-swarm cache's, which creates and reads its three secrets.
# Minted here for the matrix-ctl leaf's reason: the cache is a swarm
# service, so on a cache host without the store this is the file an
# operator copies.
[ -s ${pkiDir}/attic.pem ] || ${signLeaf} ${pkiDir} attic \
${lib.escapeShellArg deployCfg.bao.atticCommonName} "" clientAuth
# The forge's token publisher. Minted here for the matrix-ctl leaf's
# reason: the forge is a swarm singleton, so on a forge host without the
# store this is the file an operator copies.

165
nix/host-modules/lib/attic-init.nix Normal file
View file

@ -0,0 +1,165 @@
# The shell for the in-swarm cache's first start: what its two host units run
# after they log in to the secret store under the cache's own certificate.
# Each secret is created only while the store holds none, so a second run reads
# what the first one stored and mints nothing.
#
# Pure function — NOT a NixOS module. Call it from a module's `let`:
#
# atticInit = import ./lib/attic-init.nix { inherit lib; } { … };
# script = ''
# ${atticInit.prelude}
# ${atticInit.ensureJwt}
# '';
#
# `prelude` expects `./atomic-write-secret.nix` and `./refresh-consumer.nix`
# above it, and `bao`, `coreutils` and `systemd` on the caller's `path`.
# `ensureJwt` also wants `openssl` there; `provision` wants `atticadm`, `attic`
# and `curl`.
{ lib }:
{
# Store paths in the form `bao kv` takes, mount first.
jwtPath,
pullPath,
pushPath,
# The cache's name on the server.
cache,
# Where the host reaches atticd, scheme included.
endpoint,
# The server's config file, which the host's `atticadm` reads as well.
configFile,
# The file atticd loads its signing key from, and the container that does.
envFile,
machine,
unit,
}:
let
q = lib.escapeShellArg;
in
{
prelude = ''
err="$(mktemp)"
work="$(mktemp -d)"
trap 'rm -rf "$err" "$work"' EXIT
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with the cache's certificate; leaving the cache as it is." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
# <store path> <mint function>: the stored value, created with the
# function's output first while the store holds none. `bao kv get` exits 2
# both for "nothing stored" and for a refusal, so a failed read always
# tries the create, which `-cas=0` refuses when the path exists, and the
# second read decides. The value reaches `bao` on stdin, never in an
# argument.
ensure_secret() {
local path="$1" mint="$2" value minted
if ! value="$(bao kv get -field=value "$path" 2>"$err")"; then
minted="$("$mint")" || return 1
if [ -z "$minted" ]; then
echo "minting $path produced nothing." >&2
return 1
fi
if ! printf '%s' "$minted" | bao kv put -cas=0 "$path" value=- >/dev/null 2>"$err"; then
echo "could not create $path; reading it back in case another run created it." >&2
cat "$err" >&2
fi
if ! value="$(bao kv get -field=value "$path" 2>"$err")"; then
echo "swarm-bao did not return $path." >&2
cat "$err" >&2
return 1
fi
fi
if [ -z "$value" ]; then
echo "swarm-bao returned an empty $path." >&2
return 1
fi
printf '%s' "$value"
}
'';
# The signing key atticd verifies tokens with, and the file it loads it from.
# The key is a base64 RSA PEM in PKCS#1 form, which is what `-traditional`
# asks for.
ensureJwt = ''
mint_jwt() {
openssl genrsa -traditional 4096 2>/dev/null | base64 -w0
}
jwt="$(ensure_secret ${q jwtPath} mint_jwt)" || exit 1
line="ATTIC_SERVER_TOKEN_RS256_SECRET_BASE64=$jwt"
if secret_differs ${q envFile} "$line"; then
atomic_write_secret 0400 "" ${q envFile} "$line"
fi
# atticd reads the key as it starts, and only then.
refresh_consumer ${q machine} ${q unit} ${q envFile}
'';
# The cache and the two tokens, once atticd answers. The tokens are signed
# here with the key the store holds; the cache is created through the API by a
# token that lives for five minutes and is never stored.
provision = ''
up=0
for _ in $(seq 1 30); do
if curl -s -o /dev/null --max-time 2 ${q endpoint}; then
up=1
break
fi
sleep 2
done
if [ "$up" != 1 ]; then
echo "atticd did not answer at ${endpoint}." >&2
exit 1
fi
if ! jwt="$(bao kv get -field=value ${q jwtPath} 2>"$err")" || [ -z "$jwt" ]; then
echo "swarm-bao did not return ${jwtPath}; the cache's tokens are not minted." >&2
cat "$err" >&2
exit 1
fi
export ATTIC_SERVER_TOKEN_RS256_SECRET_BASE64="$jwt"
# <subject> <validity> <grant…>
mint_token() {
local sub="$1" validity="$2"
shift 2
atticadm -f ${q configFile} make-token --sub "$sub" --validity "$validity" "$@"
}
mint_pull() { mint_token swarm-hives-pull 10y --pull ${q cache}; }
mint_push() { mint_token swarm-builder-push 10y --pull ${q cache} --push ${q cache}; }
pull="$(ensure_secret ${q pullPath} mint_pull)" || exit 1
ensure_secret ${q pushPath} mint_push >/dev/null || exit 1
# The client reads its token from a file, so the token is in no argument.
export XDG_CONFIG_HOME="$work"
use_token() {
install -d -m 0700 "$work/attic"
(umask 077 && printf '%s' "$1" >"$work/token")
printf 'default-server = "local"\n\n[servers.local]\nendpoint = "%s"\ntoken-file = "%s"\n' \
${q endpoint} "$work/token" >"$work/attic/config.toml"
}
# The pull token reads the cache's configuration when the cache exists, so
# a cache that is there costs no token. The server's database can be empty
# while the store still holds both tokens.
use_token "$pull"
if ! attic cache info ${q "local:${cache}"} >/dev/null 2>"$err"; then
admin="$(mint_token swarm-attic-init 5m --create-cache ${q cache} --pull ${q cache})" || exit 1
use_token "$admin"
if ! attic cache create ${q "local:${cache}"} >/dev/null 2>"$err"; then
echo "could not create the cache ${cache}; checking whether another run did." >&2
cat "$err" >&2
fi
use_token "$pull"
if ! attic cache info ${q "local:${cache}"} >/dev/null 2>"$err"; then
echo "the cache ${cache} is not there after creating it." >&2
cat "$err" >&2
exit 1
fi
fi
'';
}

43
nix/host-modules/swarm-attic-options.nix
View file

@ -1,9 +1,32 @@
# The binary cache the swarm pushes agent closures to, as every host sees it:
# which attic server, and which cache on it. The swarm runs no attic of its
# own yet; the instance is the operator's, outside the swarm.
{ lib, ... }:
# which attic server, and which cache on it. A host that sets
# `deploy.attic.enable` runs the swarm's own and fills both in (./swarm-attic.nix);
# otherwise the instance is the operator's, outside the swarm.
{ lib, config, ... }:
let
swarmDomain = config.services.hyperhive.swarm.domain;
# Total on a null swarm domain, as every sibling module is: the
# required-domain assertion in hive-network.nix is what an operator should
# see, not a coercion error from here.
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
in
{
options.services.hyperhive.swarm.attic = {
ownDomain = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "attic.${domainBase}";
defaultText = lib.literalExpression ''"attic.''${services.hyperhive.swarm.domain}"'';
description = ''
Read-only: the name the swarm's own cache is served under, which
{option}`services.hyperhive.deploy.attic.enable` defaults
{option}`services.hyperhive.swarm.attic.domain` to. A sibling of the
swarm's other service names, so the swarm-services sub-CA can issue for it,
and in `swarm.serviceDomains` on every host whether or not one runs it.
'';
};
domain = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -18,9 +41,14 @@
Null pushes nothing: the swarm builder does not push, and the closure
record says it is not cached.
The push token is not an option. The operator writes it to
{option}`services.hyperhive.deploy.attic.enable` defaults this to
`attic.<swarm.domain>` on the host that runs the swarm's own cache. Every
other host reads it from the same option, so name it there too.
The push token is not an option. It is at
`secret/swarm/services/attic/push-token` in the swarm secret store
(field `value`), and the builder reads it on every push.
(field `value`), and the builder reads it on every push. The swarm's own
cache creates it; for an external cache the operator writes it.
'';
};
@ -31,8 +59,9 @@
example = "grid";
description = ''
Name of the cache on {option}`services.hyperhive.swarm.attic.domain`
that the swarm builder pushes closures to. The push token must be
allowed to push to it. The name follows attic's rule: 1 to 50 letters,
that the swarm builder pushes closures to, `swarm` by default on the
host that runs the swarm's own cache. The push token must be allowed to
push to it. The name follows attic's rule: 1 to 50 letters,
digits, `-`, `_` or `+`, starting with a letter or digit.
'';
};

332
nix/host-modules/swarm-attic.nix Normal file
View file

@ -0,0 +1,332 @@
# The swarm's binary cache: an attic server in a `swarm-attic` container on the
# host that sets `deploy.attic.enable`, which `deploy.allSwarmServices` defaults
# on where the host has the cache's store identity. The swarm builder pushes
# green builds to it (docs/swarm/builder.md).
#
# Two host units run under the cache's own store leaf, the shape
# ./swarm-grafana.nix gives its admin password. One creates the key atticd signs
# tokens with, if the store holds none, and hands it to the container. The other
# creates the cache and the push and pull tokens, each only while absent
# (./lib/attic-init.nix). Neither needs a manual step.
#
# The container shares the host's network namespace, as ./swarm-victorialogs.nix
# does, and atticd listens on loopback only: the gateway's vhost is the way in,
# and attic's own tokens are the authentication. Storage is local disk under the
# container's state.
#
# The two `swarm.attic` options every host reads (./swarm-attic-options.nix) get
# their values from here, at `mkDefault`: naming them yourself wins, and with
# the switch off nothing is set.
{
pkgs,
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
cfg = hyperhiveCfg.deploy.attic;
atticCfg = hyperhiveCfg.swarm.attic;
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = hyperhiveCfg.deploy.bao;
machine = "swarm-attic";
unit = "swarm-attic";
# Where the container keeps its database and the cache's files.
stateName = "swarm-attic";
stateDir = "/var/lib/${stateName}";
# The host directory that carries the signing key into the container. The
# key's file is the only thing in it.
secretDir = "/run/swarm-attic-secrets";
envFile = "${secretDir}/server.env";
# Where the host units reach atticd: the shared loopback, not the gateway.
endpoint = "http://127.0.0.1:${toString cfg.port}";
configFile = (pkgs.formats.toml { }).generate "swarm-attic.toml" {
listen = "127.0.0.1:${toString cfg.port}";
# What clients are told to use for the cache's API.
api-endpoint = "https://${atticCfg.domain}/";
database.url = "sqlite://${stateDir}/server.db?mode=rwc";
storage = {
type = "local";
path = "${stateDir}/storage";
};
# attic's defaults, which its config requires spelled out.
chunking = {
nar-size-threshold = 65536;
min-size = 16384;
avg-size = 65536;
max-size = 262144;
};
garbage-collection = {
interval = "12 hours";
default-retention-period = cfg.retentionPeriod;
};
};
atticInit = import ./lib/attic-init.nix { inherit lib; } {
jwtPath = "secret/swarm/services/attic/jwt-secret";
pullPath = "secret/swarm/services/attic/pull-token";
pushPath = "secret/swarm/services/attic/push-token";
cache = atticCfg.cache;
inherit
endpoint
configFile
envFile
machine
unit
;
};
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
refreshConsumer = import ./lib/refresh-consumer.nix { };
storeRetry = import ./lib/store-retry.nix { };
haveClientIdentity = baoDeploy.atticClientCertFile != null && baoDeploy.atticClientKeyFile != null;
# Both host units log in as the cache's principal.
storeEnvironment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.atticClientCertFile;
BAO_CLIENT_KEY = baoDeploy.atticClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
in
{
options.services.hyperhive.deploy.attic = {
package = lib.mkOption {
type = lib.types.package;
default = pkgs.attic-server;
defaultText = lib.literalExpression "pkgs.attic-server";
description = "The attic server package: `atticd`, and `atticadm` for the tokens.";
};
clientPackage = lib.mkOption {
type = lib.types.package;
default = pkgs.attic-client;
defaultText = lib.literalExpression "pkgs.attic-client";
description = "The attic client package, which creates the cache.";
};
port = lib.mkOption {
type = lib.types.port;
default = 8910;
description = ''
Loopback port atticd listens on, which the gateway proxies to.
⚠️ Every swarm container shares the host's network namespace, so this
is a swarm-wide claim: two modules picking the same number collide at
runtime with no bind error. No repo-wide check catches it.
'';
};
retentionPeriod = lib.mkOption {
type = lib.types.str;
default = "14 days";
example = "30 days";
description = ''
How long a store path stays in the cache after it was added or last
fetched. A garbage collection every twelve hours removes what is older;
a build the cache no longer holds is rebuilt on the next miss.
'';
};
};
config = lib.mkIf cfg.enable {
services.hyperhive.swarm.attic = {
domain = lib.mkDefault atticCfg.ownDomain;
cache = lib.mkDefault "swarm";
};
assertions = [
{
assertion = haveClientIdentity;
message = ''
services.hyperhive.deploy.attic.enable requires the cache's
swarm-secret-store client identity: set both
services.hyperhive.deploy.bao.atticClientCertFile
services.hyperhive.deploy.bao.atticClientKeyFile
The cache creates its signing key and tokens in the store with it.
Both default to the pair glue-bao-tls.nix mints when
services.hyperhive.deploy.bao.enable is true (the host that runs the
store). Otherwise copy attic.pem and attic-key.pem from the store
host's /var/lib/swarm-bao-pki/ and name them in those two options.
'';
}
{
assertion = atticCfg.domain != null && atticCfg.cache != null;
message = ''
services.hyperhive.deploy.attic.enable needs
services.hyperhive.swarm.attic.domain and
services.hyperhive.swarm.attic.cache: it serves the cache at that name
and creates it by that name. They default when unset, so one of them
is set to null.
'';
}
];
services.hyperhive.gateway.localNames = [ atticCfg.domain ];
# This host serves the vhost, and the host's own resolver finds the name.
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# `forceSSL`: the tokens ride in a header. No `auth_request`: attic checks
# its own tokens, and a nix client cannot follow a login redirect. A store
# path is uploaded as one request that can run for minutes, so the body is
# neither size-limited nor buffered.
services.hyperhive.gateway.vhosts."${atticCfg.domain}" = {
forceSSL = true;
locations."/" = {
proxyPass = "http://127.0.0.1:${toString cfg.port}";
extraConfig = ''
client_max_body_size 0;
proxy_request_buffering off;
proxy_read_timeout 1h;
proxy_send_timeout 1h;
'';
};
};
# The key's directory is the container's read-only bind source, so it has
# to exist before the container starts even when the store does not answer.
# The delivery unit creates it too, which covers a switch.
systemd.tmpfiles.rules = [ "d ${secretDir} 0700 root root -" ];
# The signing key, created in the store if it holds none and written where
# the container reads it. Before the container and wanted by it: a store
# that comes up later fails this run, atticd restarts until the file is
# there, and the retry that lands it restarts atticd.
systemd.services.swarm-bao-attic-secret = {
description = "create the swarm cache's signing key in the secret store if absent and hand it to the container";
# Ordered after the store where it runs here:
# ./glue-bao-readers-store-order.nix and ./glue-bao-readers-policy-order.nix.
before = [ "container@${machine}.service" ];
wantedBy = [
"multi-user.target"
"container@${machine}.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.openssl
pkgs.systemd
];
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-bao-attic-secret";
# Generating a 4096-bit key takes seconds.
TimeoutStartSec = 120;
};
environment = storeEnvironment;
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
install -d -m 0700 ${lib.escapeShellArg secretDir}
${atticInit.prelude}
${atticInit.ensureJwt}
'';
};
# The cache and its two tokens, once atticd answers. The push token is the
# path the swarm builder reads; the pull token is for the hives.
systemd.services.swarm-bao-attic-tokens = {
description = "create the swarm cache, and its push and pull tokens in the secret store, if absent";
after = [
"container@${machine}.service"
"swarm-bao-attic-secret.service"
];
wants = [ "container@${machine}.service" ];
wantedBy = [ "multi-user.target" ];
path = [
baoDeploy.package
cfg.package
cfg.clientPackage
pkgs.coreutils
pkgs.curl
];
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-bao-attic-tokens";
# Up to a minute waiting for atticd.
TimeoutStartSec = 180;
};
environment = storeEnvironment;
script = ''
set -euo pipefail
${atticInit.prelude}
${atticInit.provision}
'';
};
containers.${machine} = {
autoStart = true;
ephemeral = false;
privateNetwork = false;
bindMounts.${secretDir} = {
hostPath = secretDir;
isReadOnly = true;
};
config =
{ ... }:
{
imports = [ ../container-modules/swarm-container.nix ];
services.hyperhive.swarmContainer.privateNetwork = false;
systemd.services.${unit} = {
description = "attic binary cache server";
wantedBy = [ "multi-user.target" ];
serviceConfig = {
ExecStart = "${cfg.package}/bin/atticd -f ${configFile} --mode monolithic";
# The signing key, in an environment file the host unit writes.
EnvironmentFile = envFile;
StateDirectory = stateName;
DynamicUser = true;
Restart = "on-failure";
RestartSec = 10;
SyslogIdentifier = unit;
UMask = "0077";
CapabilityBoundingSet = [ "" ];
NoNewPrivileges = true;
PrivateDevices = true;
PrivateTmp = true;
ProtectSystem = "strict";
ProtectHome = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectControlGroups = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
"AF_UNIX"
];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
LockPersonality = true;
SystemCallArchitectures = "native";
};
};
};
};
};
}

72
nix/host-modules/swarm-bao.nix
View file

@ -693,11 +693,16 @@ let
builderForgeTokenLeaf = "swarm/services/builder/forge/token";
# The token the swarm builder pushes closures to the binary cache with, the nix
# half of `swarm_secret_client::builder::attic_push_token_path`. The operator
# writes it by hand; the builder reads it on every push, and nothing else in
# the swarm reaches it.
# half of `swarm_secret_client::builder::attic_push_token_path`. The builder
# reads it on every push. The operator writes it for an external cache; the
# in-swarm cache's host units create it.
atticPushTokenLeaf = "swarm/services/attic/push-token";
# The in-swarm cache's other two secrets, which its host units create: the
# key atticd signs tokens with, and the read-only token for the hives' pulls.
atticJwtLeaf = "swarm/services/attic/jwt-secret";
atticPullTokenLeaf = "swarm/services/attic/pull-token";
# The CI run poller's forge token, the nix half of
# `swarm_secret_client::ci_metrics::forge_token_path`. The controller mints it
# and reads it back before deciding whether to re-mint; the poller reads it
@ -1166,6 +1171,28 @@ let
}
];
# The in-swarm cache's three secrets, create and read and nothing more: its
# host units create each one only while it is absent, so none needs `update`,
# and a rewrite of a signing key or a token an operator holds is refused.
atticWriters = [
{
name = "swarm-attic";
cn = baoDeploy.atticCommonName;
policyText =
lib.concatMapStrings
(leaf: ''
path "${credentialMountPath}/data/${leaf}" {
capabilities = ["create", "read"]
}
'')
[
atticJwtLeaf
atticPushTokenLeaf
atticPullTokenLeaf
];
}
];
# The CI run poller: its forge token and its own OIDC client secret, both
# read once per pass by the host unit.
ciMetricsReaders = [
@ -2409,6 +2436,43 @@ in
'';
};
atticCommonName = lib.mkOption {
type = lib.types.str;
default = "swarm-attic";
description = ''
Subject the store's `swarm-attic` cert-auth role accepts: the identity
the in-swarm cache's host units present. Its grant is create and read on
three paths, `swarm/services/attic/jwt-secret`, `push-token` and
`pull-token`, and nothing else.
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
'';
};
atticClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/attic.pem";
description = ''
Certificate the in-swarm cache's host units present to the store. Its
subject must be {option}`services.hyperhive.deploy.bao.atticCommonName`.
./glue-bao-tls.nix points it at the leaf it mints on the store's host.
On a cache host without the store, it is a file the operator copies
from the store's host. A path, never a value.
'';
};
atticClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/attic-key.pem";
description = ''
Private key for {option}`services.hyperhive.deploy.bao.atticClientCertFile`.
A path, never a value.
'';
};
claudeLoginClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -3571,6 +3635,8 @@ in
systemd.services.swarm-bao-claude-login-policy = readerPolicyUnit "write the claude login minter's credentials-writer bao policy and cert-auth role" claudeLoginWriters;
systemd.services.swarm-bao-attic-policy = readerPolicyUnit "write the in-swarm cache's secret-creator bao policy and cert-auth role" atticWriters;
systemd.services.swarm-bao-notify-router-policy = readerPolicyUnit "write the notification router's client-secret-reader bao policy and cert-auth role" notifyRouterReaders;
systemd.services.swarm-bao-forge-publisher-policy = readerPolicyUnit "write the forge's controller-token-publisher bao policy and cert-auth role" forgePublisherWriters;

9
nix/host-modules/swarm-required-services.nix
View file

@ -130,6 +130,15 @@ in
&& deployCfg.bao.ciMetricsClientKeyFile != null
);
# The binary cache, on the same terms: its units log in to the store as the
# cache's own principal. An operator running an external cache sets this to
# `false` and names `swarm.attic.{domain,cache}` itself.
config.services.hyperhive.deploy.attic.enable = lib.mkDefault (
deployCfg.allSwarmServices
&& deployCfg.bao.atticClientCertFile != null
&& deployCfg.bao.atticClientKeyFile != null
);
# The claude login minter, defaulted on the same terms as the review
# classifier.
config.services.hyperhive.deploy.claude-login.enable = lib.mkDefault (

5
nix/host-modules/swarm.nix
View file

@ -62,6 +62,7 @@ let
deployCfg.bao.ciMetricsCommonName
deployCfg.bao.claudeLoginCommonName
deployCfg.bao.notifyRouterCommonName
deployCfg.bao.atticCommonName
deployCfg.bao.forgePublisherCommonName
]
# The per-hive subject, spelled out per hive rather than as the prefix.
@ -112,6 +113,10 @@ let
swarmCfg.victoriametrics.domain
swarmCfg.otel.domain
swarmCfg.victorialogs.domain
# The name the swarm's own cache is served under, listed whether or not any
# host runs one, as every name here is. `swarm.attic.domain` is not: it is
# null where there is no cache and may name an external one.
swarmCfg.attic.ownDomain
# The store's browser UI, not the store: `swarm.bao.domain` is mTLS and
# never gets a gateway certificate.
swarmCfg.bao.ui.domain

59
nix/module-eval/bao-grants.nix
View file

@ -163,7 +163,7 @@ let
_: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile
) baoGrantWithConsumers.systemd.services;
# The twenty-one units that write a `swarm-*` grant, by name, for the
# The twenty-two units that write a `swarm-*` grant, by name, for the
# discovery control below.
grantingUnitNames = [
"swarm-bao-controller-policy"
@ -187,6 +187,7 @@ let
"swarm-bao-ci-metrics-policy"
"swarm-bao-claude-login-policy"
"swarm-bao-notify-router-policy"
"swarm-bao-attic-policy"
];
# Comment lines dropped first: both the HCL and the scripts explain
@ -1074,24 +1075,24 @@ let
}
{
# A store host without the granter's pair writes its grants some other
# way, so none of the twenty-one units may exist. Without this arm
# way, so none of the twenty-two units may exist. Without this arm
# `lib.mkIf haveGranter` could be dropped from any of them and every other
# case here would still pass.
name = "without the granter's pair none of the twenty-one granting units render";
name = "without the granter's pair none of the twenty-two granting units render";
ok =
let
s = baoGranterOptOut.systemd.services;
in
lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ])
# The control: the same store with the pair renders all twenty-one.
# The control: the same store with the pair renders all twenty-two.
&& lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames;
}
{
# 🩸 What replaced the silent skip. With no bootstrap token the twenty-one
# 🩸 What replaced the silent skip. With no bootstrap token the twenty-two
# still render, and a refused granter fails them with the step that fixes it.
# A store host that never named a token is told to name one, since the
# unit that sets the granter up renders only where it has.
name = "a store host without a bootstrap token renders the twenty-one, each failing loudly with the one-time step";
name = "a store host without a bootstrap token renders the twenty-two, each failing loudly with the one-time step";
ok =
let
s = baoGranterNoToken.systemd.services;
@ -1937,6 +1938,48 @@ let
in
lib.hasInfix "/notify-router.pem ]" s && lib.hasInfix "swarm-notify-router \"\" clientAuth" s;
}
{
# The cache's host units create each of its three secrets and read them
# back, and may not replace one: no `update`. The notification router's
# grant is the control that `grantsIn` reads a one-path grant too.
name = "the cache's grant is create and read on its signing key and its two tokens, and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-attic-policy.script;
grants = grantsIn s;
caps = [
"create"
"read"
];
in
(grantFor grants "secret/data/swarm/services/attic/jwt-secret").caps == caps
&& (grantFor grants "secret/data/swarm/services/attic/push-token").caps == caps
&& (grantFor grants "secret/data/swarm/services/attic/pull-token").caps == caps
&& lib.length grants == 3
&& lib.length (grantsIn baoGrantHere.systemd.services.swarm-bao-notify-router-policy.script) == 1
&& lib.hasInfix "auth/cert/certs/swarm-attic" s
&& lib.hasInfix "allowed_common_names=swarm-attic" s
&& lib.hasInfix "token_policies=swarm-attic" s;
}
{
# The signing key and the pull token are the cache's alone; the push
# token is also the builder's, and is read there.
name = "no granting unit but the cache's names its signing key or its pull token";
ok =
lib.filter (
u:
lib.hasInfix "services/attic/jwt-secret" baoGrantHere.systemd.services.${u}.script
|| lib.hasInfix "services/attic/pull-token" baoGrantHere.systemd.services.${u}.script
) grantingUnitNames == [ "swarm-bao-attic-policy" ];
}
{
name = "the PKI unit signs the cache's leaf under its own subject";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
in
lib.hasInfix "/attic.pem ]" s && lib.hasInfix "swarm-attic \"\" clientAuth" s;
}
{
name = "the PKI unit signs the CI runner's leaf under its own subject";
ok =
@ -2221,8 +2264,8 @@ let
}
{
# What makes the case above mean something: discovery by the granter's
# certificate reaches all twenty-one units, and each yields calls.
name = "the granter-policy check sees all twenty-one granting units, and parses calls from each";
# certificate reaches all twenty-two units, and each yields calls.
name = "the granter-policy check sees all twenty-two granting units, and parses calls from each";
ok =
lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames
&& lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)

135
nix/module-eval/builder.nix
View file

@ -20,6 +20,7 @@ let
;
})
hive
hiveWithController
runGroup
;
@ -66,7 +67,43 @@ let
svc = inner.systemd.services.swarm-builder;
env = svc.environment;
hasOnPath = name: s: lib.any (p: lib.getName p == name) s.path;
withoutCache = servicesHere.containers.swarm-builder.config.systemd.services.swarm-builder;
# A builder with no cache named anywhere.
withBuilderNoCache = hive {
deploy.bao.enable = true;
deploy.builder.enable = true;
};
withoutCache = withBuilderNoCache.containers.swarm-builder.config.systemd.services.swarm-builder;
# Hosting the shared services runs the swarm's own cache, which names itself.
atticHere = servicesHere;
atticDeploy = atticHere.services.hyperhive.deploy.attic;
atticSwarm = m: m.services.hyperhive.swarm.attic;
atticUnit = atticHere.containers.swarm-attic.config.systemd.services.swarm-attic;
atticSecret = atticHere.systemd.services.swarm-bao-attic-secret;
atticTokens = atticHere.systemd.services.swarm-bao-attic-tokens;
atticVhost = atticHere.services.hyperhive.gateway.vhosts."attic.t.local";
# The cache switched on while the operator names the options by hand.
atticByHand = hive {
deploy.bao.enable = true;
deploy.attic.enable = true;
swarm.attic.domain = "cache.example.org";
swarm.attic.cache = "grid";
};
# An external cache on a host that hosts the shared services: the switch forced
# off, the two options named.
atticExternal = hiveWithController {
deploy.allSwarmServices = true;
deploy.attic.enable = lib.mkForce false;
swarm.attic.domain = "cache.example.org";
swarm.attic.cache = "grid";
};
hasAtticHost =
m:
m.containers ? swarm-attic
|| m.systemd.services ? swarm-bao-attic-secret
|| m.systemd.services ? swarm-bao-attic-tokens;
# The check flake on the inputs the builder's unit names, called as
# `nix build` calls it. `hyperhive` is entered through the filtered source's
@ -271,6 +308,102 @@ let
autheliaHere.containers.swarm-authelia.config.systemd.services.swarm-secret-publish.script
);
}
{
# The switch is on wherever the shared services are and the host holds the
# cache's leaf, and the two options every host reads come from the cache.
# The builder on the same host pushes to it. `bare` is the control that the
# lookup reads null where nothing runs a cache.
name = "hosting the shared services runs the swarm's cache and fills the cache options, which the builder follows";
ok =
atticDeploy.enable
&& atticHere.containers ? swarm-attic
&& atticHere.services.hyperhive.deploy.bao.atticClientCertFile != null
&& atticHere.services.hyperhive.deploy.bao.atticClientKeyFile != null
&& (atticSwarm atticHere).domain == "attic.t.local"
&& (atticSwarm atticHere).cache == "swarm"
&&
atticHere.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_ATTIC_DOMAIN
== "attic.t.local"
&&
atticHere.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_ATTIC_CACHE
== "swarm"
&& (atticSwarm bare).domain == null
&& (atticSwarm bare).cache == null
&& !(hasAtticHost bare);
}
{
# Names the operator gives win over the cache's own, and the gateway then
# serves the cache under the operator's name. `atticHere` is the control
# that the same lookups find the defaults.
name = "options named by hand win over the cache's defaults";
ok =
atticByHand.services.hyperhive.deploy.attic.enable
&& atticByHand.containers ? swarm-attic
&& (atticSwarm atticByHand).domain == "cache.example.org"
&& (atticSwarm atticByHand).cache == "grid"
&& atticByHand.services.hyperhive.gateway.vhosts ? "cache.example.org"
&& !(atticByHand.services.hyperhive.gateway.vhosts ? "attic.t.local")
&& atticHere.services.hyperhive.gateway.vhosts ? "attic.t.local";
}
{
# An operator's external cache on a host that hosts the shared services:
# with the switch forced off nothing of the cache is rendered, and the
# options keep what was named. `atticHere` is the control that the same
# lookups find the cache's pieces where it runs.
name = "with the cache switched off the options are the operator's and no cache runs";
ok =
!atticExternal.services.hyperhive.deploy.attic.enable
&& !(hasAtticHost atticExternal)
&& !(atticExternal.services.hyperhive.gateway.vhosts ? "attic.t.local")
&& (atticSwarm atticExternal).domain == "cache.example.org"
&& (atticSwarm atticExternal).cache == "grid"
&&
atticExternal.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_ATTIC_DOMAIN
== "cache.example.org"
&& hasAtticHost atticHere
&& atticHere.systemd.services ? swarm-bao-attic-secret
&& atticHere.systemd.services ? swarm-bao-attic-tokens;
}
{
# Both host units log in as the cache's principal, not the hive's, and
# the container reaches the key through a read-only directory. The
# unit script holds the create calls only in ./lib/attic-init.nix, which a
# script test runs.
name = "the cache's host units use its own store leaf, and the container gets the key read-only";
ok =
atticSecret.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/attic.pem"
&& atticSecret.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/attic-key.pem"
&& atticTokens.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/attic.pem"
&& atticSecret.serviceConfig.Type == "oneshot"
&& lib.elem "container@swarm-attic.service" atticSecret.before
&& lib.elem "container@swarm-attic.service" atticTokens.after
&& lib.elem "swarm-bao-attic-policy.service" atticSecret.after
&& lib.elem "swarm-bao-attic-policy.service" atticTokens.after
&& atticHere.containers.swarm-attic.bindMounts."/run/swarm-attic-secrets".isReadOnly
&& !atticHere.containers.swarm-attic.privateNetwork
&& atticUnit.serviceConfig.EnvironmentFile == "/run/swarm-attic-secrets/server.env"
&& lib.hasInfix "--mode monolithic" atticUnit.serviceConfig.ExecStart
&& atticDeploy.retentionPeriod == "14 days";
}
{
# attic checks its own tokens and a nix client cannot follow a login
# redirect, so the vhost carries no `auth_request`; the tokens ride in a
# header, so it is TLS only; and a store path is one long upload.
name = "the cache's vhost is TLS only, unauthenticated at the gateway and streams uploads, and its name is a swarm service name on every host";
ok =
let
names = m: m.services.hyperhive.swarm.serviceDomains;
in
atticVhost.forceSSL
&& atticVhost.locations."/".proxyPass == "http://127.0.0.1:8910"
&& !(lib.hasInfix "auth_request" atticVhost.locations."/".extraConfig)
&& lib.hasInfix "client_max_body_size 0" atticVhost.locations."/".extraConfig
&& lib.hasInfix "proxy_request_buffering off" atticVhost.locations."/".extraConfig
&& lib.elem "attic.t.local" (names atticHere)
&& names atticHere == names bare
&& names atticExternal == names bare
&& !(lib.elem "cache.example.org" (names atticExternal));
}
];
in
runGroup "builder" cases

11
nix/module-eval/swarm-services-switch.nix
View file

@ -302,6 +302,17 @@ let
&& !(swarmServicesBaoElsewhere.containers ? swarm-notify-router)
&& swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
}
{
# The cache's units assert the store identity the store's host mints, as
# the router's do; its options stay as unset as they were.
name = "the swarm's cache stays off where the shared services are hosted without the store";
ok =
!swarmServicesBaoElsewhere.services.hyperhive.deploy.attic.enable
&& !(swarmServicesBaoElsewhere.containers ? swarm-attic)
&& swarmServicesBaoElsewhere.services.hyperhive.swarm.attic.domain == null
&& swarmServicesHere.services.hyperhive.deploy.attic.enable
&& swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
}
{
# The CLI is the one the agents run, not the container's own nixpkgs'.
# Unpinned is the control: it falls back to the container's own `claude-code`.

41
nix/script-tests/attic-init.nix Normal file
View file

@ -0,0 +1,41 @@
# `checks.script-test-attic-init` — runs ../host-modules/lib/attic-init.nix's two
# scripts, as the cache's host units run them, against stub `bao`, `openssl`,
# `atticadm`, `attic`, `curl` and `systemctl`. The cases are in ./attic-init.sh.
#
# What runs is the rendered script, behind the same helpers the units put in
# front of it.
{
pkgs,
...
}:
let
init = import ../host-modules/lib/attic-init.nix { inherit (pkgs) lib; } {
jwtPath = "secret/swarm/services/attic/jwt-secret";
pullPath = "secret/swarm/services/attic/pull-token";
pushPath = "secret/swarm/services/attic/push-token";
cache = "grid";
endpoint = "http://127.0.0.1:8910";
configFile = "@DIR@/server.toml";
envFile = "@DIR@/secrets/server.env";
machine = "swarm-attic";
unit = "swarm-attic";
};
render =
name: body:
pkgs.writeText name ''
set -euo pipefail
${import ../host-modules/lib/atomic-write-secret.nix { }}
${import ../host-modules/lib/refresh-consumer.nix { }}
${init.prelude}
${body}
'';
in
pkgs.runCommand "hyperhive-script-test-attic-init"
{
ENSURE_JWT = render "ensure-jwt.sh" init.ensureJwt;
PROVISION = render "provision.sh" init.provision;
}
''
${pkgs.bash}/bin/bash ${./attic-init.sh}
touch "$out"
''

232
nix/script-tests/attic-init.sh Normal file
View file

@ -0,0 +1,232 @@
# Cases for ../host-modules/lib/attic-init.nix. Each case runs the key script or
# the cache script against stubs that record every call, over a store kept in
# files, and compares the record and what the script left behind.
set -uo pipefail
failures=0
count=0
case="setup"
fail() {
echo "FAILED: $case: $*" >&2
failures=$((failures + 1))
}
dir=$(mktemp -d)
mkdir -p "$dir/bin" "$dir/store" "$dir/secrets"
# The scripts name their files under @DIR@, which is this run's directory.
for s in ENSURE_JWT PROVISION; do
sed "s|@DIR@|$dir|g" "${!s}" >"$dir/$s.sh"
export "$s=$dir/$s.sh"
done
calls=$dir/calls
export DIR=$dir
# The store is a directory with one file per path. `get` exits 2 on a miss, as
# `bao kv get` does. `put -cas=0` refuses an existing path; without it a put
# overwrites. $RACE_PATH makes another run create that path just before our put
# lands.
cat >"$dir/bin/bao" <<STUB
#!$BASH
file() { printf '%s/store/%s' "\$DIR" "\$(printf '%s' "\$1" | tr / _)"; }
case "\$1 \$2" in
"login -method=cert")
echo "bao login" >>"$calls"
if [ "\${LOGIN_FAIL:-0}" = 1 ]; then echo denied >&2; exit 2; fi
echo store-token
;;
"kv get")
path=\${4}
f=\$(file "\$path")
echo "bao get \$path" >>"$calls"
[ -f "\$f" ] || exit 2
cat "\$f"
;;
"kv put")
if [ "\$3" = "-cas=0" ]; then cas=1; path=\$4; else cas=0; path=\$3; fi
f=\$(file "\$path")
echo "bao put cas=\$cas \$path" >>"$calls"
value=\$(cat)
if [ "\${RACE_PATH:-}" = "\$path" ] && [ ! -f "\$f" ]; then
printf '%s' raced >"\$f"
fi
if [ "\$cas" = 1 ] && [ -f "\$f" ]; then echo "check-and-set parameter did not match" >&2; exit 1; fi
printf '%s' "\$value" >"\$f"
;;
esac
STUB
# The key is a fixed word, so the file the script writes is predictable.
cat >"$dir/bin/openssl" <<STUB
#!$BASH
echo openssl >>"$calls"
echo FAKEPEM
STUB
# A token carries the subject and a serial, so a case can tell which one a call
# used. The key comes in the environment, as the real tool takes it.
cat >"$dir/bin/atticadm" <<STUB
#!$BASH
[ -n "\${ATTIC_SERVER_TOKEN_RS256_SECRET_BASE64:-}" ] || { echo "no key in the environment" >&2; exit 1; }
echo "atticadm \$*" >>"$calls"
n=\$(grep -c '^atticadm' "$calls")
sub=\$2
[ "\$1" = "-f" ] && sub=\$5
echo "tok-\$sub-\$n"
STUB
# The cache exists once \`cache create\` ran. Each call records the token the
# client config names.
cat >"$dir/bin/attic" <<STUB
#!$BASH
tokenfile=\$(sed -n 's/^token-file = "\(.*\)"$/\1/p' "\$XDG_CONFIG_HOME/attic/config.toml")
token=\$(cat "\$tokenfile")
echo "attic \$* | token=\$token" >>"$calls"
case "\$1 \$2" in
"cache info") [ -f "\$DIR/cache-exists" ] ;;
"cache create") touch "\$DIR/cache-exists" ;;
*) exit 1 ;;
esac
STUB
chmod +x "$dir/bin/"*
cat >"$dir/bin/curl" <<STUB
#!$BASH
echo curl >>"$calls"
[ "\${CURL_FAIL:-0}" != 1 ]
STUB
printf '#!%s\nexit 0\n' "$BASH" >"$dir/bin/sleep"
# A container that is not running has nothing to refresh.
printf '#!%s\nexit 1\n' "$BASH" >"$dir/bin/systemctl"
chmod +x "$dir/bin/"*
export PATH="$dir/bin:$PATH"
# n <pattern>: how many recorded calls match.
n() { grep -c -- "$1" "$calls" || true; }
# run <script> [VAR=value…]: the script's exit status.
run() {
local script=$1
shift
env "$@" bash "$script" >"$dir/out" 2>"$dir/err"
}
reset() {
rm -rf "$dir/store" "$dir/secrets" "$dir/cache-exists" "$calls"
mkdir -p "$dir/store" "$dir/secrets"
: >"$calls"
}
stored() { cat "$dir/store/$1" 2>/dev/null || true; }
JWT=secret_swarm_services_attic_jwt-secret
PULL=secret_swarm_services_attic_pull-token
PUSH=secret_swarm_services_attic_push-token
ENVFILE=$dir/secrets/server.env
expect() { # <description> <got> <want>
count=$((count + 1))
if [ "$2" != "$3" ]; then fail "$1: got '$2', wanted '$3'"; fi
}
# ── the key ──────────────────────────────────────────────────────────────
case="first start creates the key and hands it over"
reset
run "$ENSURE_JWT"
expect "exit status" $? 0
expect "key minted once" "$(n '^openssl')" 1
expect "stored key" "$(stored $JWT)" "$(printf 'FAKEPEM\n' | base64 -w0)"
expect "env file" "$(cat "$ENVFILE")" "ATTIC_SERVER_TOKEN_RS256_SECRET_BASE64=$(stored $JWT)"
expect "env file mode" "$(stat -c %a "$ENVFILE")" 400
case="a second start mints nothing and keeps the key"
first=$(stored $JWT)
run "$ENSURE_JWT"
expect "exit status" $? 0
expect "key minted still once" "$(n '^openssl')" 1
expect "no second create" "$(n '^bao put')" 1
expect "stored key unchanged" "$(stored $JWT)" "$first"
case="a stored key is used, not replaced, when the file is gone"
rm -f "$ENVFILE"
run "$ENSURE_JWT"
expect "exit status" $? 0
expect "key minted still once" "$(n '^openssl')" 1
expect "env file rewritten from the store" "$(cat "$ENVFILE")" "ATTIC_SERVER_TOKEN_RS256_SECRET_BASE64=$first"
case="a create lost to another run uses the winner's key"
reset
run "$ENSURE_JWT" RACE_PATH=secret/swarm/services/attic/jwt-secret
expect "exit status" $? 0
expect "stored key is the winner's" "$(stored $JWT)" raced
expect "env file has the winner's" "$(cat "$ENVFILE")" "ATTIC_SERVER_TOKEN_RS256_SECRET_BASE64=raced"
case="a refused login stops before anything is minted"
reset
run "$ENSURE_JWT" LOGIN_FAIL=1
expect "exit status" $? 1
expect "nothing minted" "$(n '^openssl')" 0
expect "nothing written" "$(ls "$dir/secrets" | wc -l)" 0
# ── the cache and the tokens ─────────────────────────────────────────────
case="first start creates the cache and both tokens"
reset
run "$ENSURE_JWT"
: >"$calls"
run "$PROVISION"
expect "exit status" $? 0
expect "pull token stored" "$(stored $PULL)" "tok-swarm-hives-pull-1"
expect "push token stored" "$(stored $PUSH)" "tok-swarm-builder-push-2"
expect "cache created once" "$(n '^attic cache create')" 1
expect "cache created by the short-lived token" "$(n '^attic cache create local:grid | token=tok-swarm-attic-init-3')" 1
expect "tokens minted: pull, push, the creator" "$(n '^atticadm')" 3
expect "pull token reads only the cache" "$(n 'make-token --sub swarm-hives-pull --validity 10y --pull grid$')" 1
expect "push token pulls and pushes" "$(n 'make-token --sub swarm-builder-push --validity 10y --pull grid --push grid$')" 1
expect "the creator may create and pull, for minutes" "$(n 'make-token --sub swarm-attic-init --validity 5m --create-cache grid --pull grid$')" 1
case="a second start mints nothing and creates nothing"
pull=$(stored $PULL)
push=$(stored $PUSH)
: >"$calls"
run "$PROVISION"
expect "exit status" $? 0
expect "no token minted" "$(n '^atticadm')" 0
expect "no create call" "$(n '^bao put')" 0
expect "no cache create" "$(n '^attic cache create')" 0
expect "the pull token looked the cache up" "$(n "^attic cache info local:grid | token=$pull")" 1
expect "pull token unchanged" "$(stored $PULL)" "$pull"
expect "push token unchanged" "$(stored $PUSH)" "$push"
case="an empty server gets its cache back with the stored tokens"
rm -f "$dir/cache-exists"
: >"$calls"
run "$PROVISION"
expect "exit status" $? 0
expect "only the creator is minted" "$(n '^atticadm')" 1
expect "no secret rewritten" "$(n '^bao put')" 0
expect "cache created" "$(n '^attic cache create')" 1
expect "pull token unchanged" "$(stored $PULL)" "$pull"
case="no tokens are minted while the server is down"
reset
run "$ENSURE_JWT"
: >"$calls"
run "$PROVISION" CURL_FAIL=1
expect "exit status" $? 1
expect "nothing minted" "$(n '^atticadm')" 0
expect "nothing stored" "$(n '^bao put')" 0
case="the client reads its token from a file, not from an argument"
reset
run "$ENSURE_JWT"
run "$PROVISION"
expect "no token among the client's arguments" "$(grep -c -- '^attic [^|]*tok-' "$calls" || true)" 0
# The control: the client did see tokens, through its config file.
expect "tokens seen through the config file" "$(grep -c -- '| token=tok-' "$calls" || true)" 3
if [ "$failures" -ne 0 ]; then
echo "$failures of $count checks failed" >&2
exit 1
fi
echo "$count checks passed"