copy the cache's pull facts to each hive and create them in the in-swarm cache
the in-swarm cache stores its address, name and public key beside its tokens, create-only. swarm-controller copies the four cache facts to swarm/hives/<hive>/attic/* every five minutes, with read on the sources and write on the copies. part 1 of 3: nothing reads the copies until the hive reader and probe land.
This commit is contained in:
atlas
mara
parent
c79c41e004
commit
1e5070796d
8 changed files with 640 additions and 67 deletions
|
|
@ -80,7 +80,8 @@ let
|
|||
policies = [ "swarm-bao-grafana-oidc-policy" ];
|
||||
};
|
||||
# ./swarm-attic.nix, under the cache's leaf, which may create the signing
|
||||
# key and the two tokens only once that policy is written.
|
||||
# key, the two tokens and the cache's public facts only once that policy is
|
||||
# written.
|
||||
swarm-bao-attic-secret = {
|
||||
here = deployCfg.attic.enable;
|
||||
policies = [ "swarm-bao-attic-policy" ];
|
||||
|
|
|
|||
|
|
@ -330,7 +330,7 @@ in
|
|||
[ -s ${pkiDir}/notify-router.pem ] || ${signLeaf} ${pkiDir} notify-router \
|
||||
${lib.escapeShellArg deployCfg.bao.notifyRouterCommonName} "" clientAuth
|
||||
|
||||
# The in-swarm cache's, which creates and reads its three secrets.
|
||||
# The in-swarm cache's, which creates and reads its six secrets.
|
||||
# Minted here for the matrix-ctl leaf's reason: the cache is a swarm
|
||||
# service, so on a cache host without the store this is the file an
|
||||
# operator copies.
|
||||
|
|
|
|||
|
|
@ -13,14 +13,22 @@
|
|||
#
|
||||
# `prelude` expects `./atomic-write-secret.nix` and `./refresh-consumer.nix`
|
||||
# above it, and `bao`, `coreutils` and `systemd` on the caller's `path`.
|
||||
# `ensureJwt` also wants `openssl` there; `provision` wants `atticadm`, `attic`
|
||||
# and `curl`.
|
||||
# `ensureJwt` also wants `openssl` there; `provision` wants `atticadm`, `attic`,
|
||||
# `curl` and `sed`.
|
||||
{ lib }:
|
||||
{
|
||||
# Store paths in the form `bao kv` takes, mount first.
|
||||
jwtPath,
|
||||
pullPath,
|
||||
pushPath,
|
||||
# The cache's public facts, created only while absent: the server's address
|
||||
# (scheme included), the cache's name, and the key its narinfo are signed
|
||||
# with. Hives pull with them.
|
||||
urlPath,
|
||||
cachePath,
|
||||
publicKeyPath,
|
||||
# The address `urlPath` holds, which clients reach the server at.
|
||||
url,
|
||||
# The cache's name on the server.
|
||||
cache,
|
||||
# Where the host reaches atticd, scheme included.
|
||||
|
|
@ -98,9 +106,9 @@ in
|
|||
refresh_consumer ${q machine} ${q unit} ${q envFile}
|
||||
'';
|
||||
|
||||
# The cache and the two tokens, once atticd answers. The tokens are signed
|
||||
# here with the key the store holds; the cache is created through the API by a
|
||||
# token that lives for five minutes and is never stored.
|
||||
# The cache, its two tokens and its public facts, once atticd answers. The
|
||||
# tokens are signed here with the key the store holds; the cache is created
|
||||
# through the API by a token that lives for five minutes and is never stored.
|
||||
provision = ''
|
||||
up=0
|
||||
for _ in $(seq 1 30); do
|
||||
|
|
@ -161,5 +169,16 @@ in
|
|||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# What hives pull with. The key is the one the server reports for the cache;
|
||||
# `attic` prints it on stderr.
|
||||
mint_url() { printf '%s' ${q url}; }
|
||||
mint_cache() { printf '%s' ${q cache}; }
|
||||
mint_public_key() {
|
||||
attic cache info ${q "local:${cache}"} 2>&1 | sed -n 's/^ *Public Key: *//p'
|
||||
}
|
||||
ensure_secret ${q urlPath} mint_url >/dev/null || exit 1
|
||||
ensure_secret ${q cachePath} mint_cache >/dev/null || exit 1
|
||||
ensure_secret ${q publicKeyPath} mint_public_key >/dev/null || exit 1
|
||||
'';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@
|
|||
# Two host units run under the cache's own store leaf, the shape
|
||||
# ./swarm-grafana.nix gives its admin password. One creates the key atticd signs
|
||||
# tokens with, if the store holds none, and hands it to the container. The other
|
||||
# creates the cache and the push and pull tokens, each only while absent
|
||||
# creates the cache, the push and pull tokens, and the facts hives pull with
|
||||
# (its address, name and public key), each only while absent
|
||||
# (./lib/attic-init.nix). Neither needs a manual step.
|
||||
#
|
||||
# The container shares the host's network namespace, as ./swarm-victorialogs.nix
|
||||
|
|
@ -73,6 +74,10 @@ let
|
|||
jwtPath = "secret/swarm/services/attic/jwt-secret";
|
||||
pullPath = "secret/swarm/services/attic/pull-token";
|
||||
pushPath = "secret/swarm/services/attic/push-token";
|
||||
urlPath = "secret/swarm/services/attic/url";
|
||||
cachePath = "secret/swarm/services/attic/cache";
|
||||
publicKeyPath = "secret/swarm/services/attic/public-key";
|
||||
url = "https://${atticCfg.domain}";
|
||||
cache = atticCfg.cache;
|
||||
inherit
|
||||
endpoint
|
||||
|
|
@ -295,10 +300,11 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# The cache and its two tokens, once atticd answers. The push token is the
|
||||
# path the swarm builder reads; the pull token is for the hives.
|
||||
# The cache, its two tokens and its public facts, once atticd answers. The
|
||||
# push token is the path the swarm builder reads; the pull token and the
|
||||
# facts are for the hives, which the controller copies them to.
|
||||
systemd.services.swarm-bao-attic-tokens = {
|
||||
description = "create the swarm cache, and its push and pull tokens in the secret store, if absent";
|
||||
description = "create the swarm cache, and its tokens and public facts in the secret store, if absent";
|
||||
after = [
|
||||
"container@${machine}.service"
|
||||
"swarm-bao-attic-secret.service"
|
||||
|
|
@ -311,6 +317,7 @@ in
|
|||
cfg.clientPackage
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
pkgs.gnused
|
||||
];
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
|
|
|
|||
|
|
@ -482,6 +482,14 @@ let
|
|||
capabilities = ["create", "read", "update"]
|
||||
}
|
||||
|
||||
${lib.concatMapStrings readStanza (
|
||||
map (leaf: "${credentialMountPath}/data/${leaf}") atticFactLeaves
|
||||
)}
|
||||
${lib.concatMapStrings (name: ''
|
||||
path "${credentialMountPath}/data/swarm/hives/+/attic/${name}" {
|
||||
capabilities = ["create", "read", "update"]
|
||||
}
|
||||
'') atticHiveFactNames}
|
||||
path "${credentialMountPath}/data/${ciRunnerTokenLeaf}" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
|
@ -718,6 +726,28 @@ let
|
|||
atticJwtLeaf = "swarm/services/attic/jwt-secret";
|
||||
atticPullTokenLeaf = "swarm/services/attic/pull-token";
|
||||
|
||||
# The cache's three public facts, beside the pull token: its address, its
|
||||
# name, and the key its narinfo are signed with. The in-swarm cache's host
|
||||
# unit creates them; for an external cache the operator writes them.
|
||||
# `swarm-controller` reads all four and copies each to
|
||||
# `swarm/hives/<hive>/attic/<name>`
|
||||
# (`swarm_secret_client::attic::Leaf`).
|
||||
atticUrlLeaf = "swarm/services/attic/url";
|
||||
atticCacheLeaf = "swarm/services/attic/cache";
|
||||
atticPublicKeyLeaf = "swarm/services/attic/public-key";
|
||||
atticFactLeaves = [
|
||||
atticPullTokenLeaf
|
||||
atticUrlLeaf
|
||||
atticCacheLeaf
|
||||
atticPublicKeyLeaf
|
||||
];
|
||||
atticHiveFactNames = [
|
||||
"pull-token"
|
||||
"url"
|
||||
"cache"
|
||||
"public-key"
|
||||
];
|
||||
|
||||
# The CI run poller's forge token, the nix half of
|
||||
# `swarm_secret_client::ci_metrics::forge_token_path`. The controller mints it
|
||||
# and reads it back before deciding whether to re-mint; the poller reads it
|
||||
|
|
@ -1199,7 +1229,7 @@ let
|
|||
}
|
||||
];
|
||||
|
||||
# The in-swarm cache's three secrets, create and read and nothing more: its
|
||||
# The in-swarm cache's six secrets, create and read and nothing more: its
|
||||
# host units create each one only while it is absent, so none needs `update`,
|
||||
# and a rewrite of a signing key or a token an operator holds is refused. The
|
||||
# cache container's journal forwarder reads its client secret under the same
|
||||
|
|
@ -1215,11 +1245,13 @@ let
|
|||
capabilities = ["create", "read"]
|
||||
}
|
||||
'')
|
||||
[
|
||||
atticJwtLeaf
|
||||
atticPushTokenLeaf
|
||||
atticPullTokenLeaf
|
||||
]
|
||||
(
|
||||
[
|
||||
atticJwtLeaf
|
||||
atticPushTokenLeaf
|
||||
]
|
||||
++ atticFactLeaves
|
||||
)
|
||||
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.attic.otel.clientId}/oidc/client";
|
||||
}
|
||||
];
|
||||
|
|
@ -2474,8 +2506,8 @@ in
|
|||
description = ''
|
||||
Subject the store's `swarm-attic` cert-auth role accepts: the identity
|
||||
the in-swarm cache's host units present. Its grant is create and read on
|
||||
three paths, `swarm/services/attic/jwt-secret`, `push-token` and
|
||||
`pull-token`, and nothing else.
|
||||
six paths under `swarm/services/attic/`, `jwt-secret`, `push-token`,
|
||||
`pull-token`, `url`, `cache` and `public-key`, and nothing else.
|
||||
|
||||
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
|
||||
'';
|
||||
|
|
|
|||
|
|
@ -1722,6 +1722,38 @@ let
|
|||
&& !(lib.hasInfix "secret/data/swarm/hives/*" s)
|
||||
&& !(lib.hasInfix "secret/metadata/swarm/hives" s);
|
||||
}
|
||||
{
|
||||
# `attic_copy` reads the cache's four facts and writes each into every
|
||||
# hive's `attic/` prefix. Pinned as whole stanzas, per leaf: a read of
|
||||
# the services prefix would reach every service's secrets, and `*` under
|
||||
# `hives/` every hive's queue credential. The read side has no
|
||||
# `create`/`update`, so the controller cannot replace the cache's token.
|
||||
name = "the controller reads the cache's four facts and writes a copy of each into every hive";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
leaves = [
|
||||
"pull-token"
|
||||
"url"
|
||||
"cache"
|
||||
"public-key"
|
||||
];
|
||||
grants = grantsIn s;
|
||||
in
|
||||
lib.all (
|
||||
l:
|
||||
(grantFor grants "secret/data/swarm/services/attic/${l}").caps == [ "read" ]
|
||||
&&
|
||||
(grantFor grants "secret/data/swarm/hives/+/attic/${l}").caps == [
|
||||
"create"
|
||||
"read"
|
||||
"update"
|
||||
]
|
||||
) leaves
|
||||
&& lib.hasInfix "path \"secret/data/swarm/hives/+/attic/pull-token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
|
||||
&& (grantFor grants "secret/data/swarm/services/attic/jwt-secret") == null
|
||||
&& (grantFor grants "secret/data/swarm/services/attic/push-token") == null;
|
||||
}
|
||||
{
|
||||
# `forge::runner_token` writes the runner's registration token and keeps
|
||||
# the last value in memory, so its stanza has no `read`. Pinned as the
|
||||
|
|
@ -1988,11 +2020,11 @@ let
|
|||
lib.hasInfix "/notify-router.pem ]" s && lib.hasInfix "swarm-notify-router \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
# The cache's host units create each of its three secrets and read them
|
||||
# The cache's host units create each of its six secrets and read them
|
||||
# back, and may not replace one: no `update`. Its container's journal
|
||||
# forwarder reads its own client secret, read only. The notification
|
||||
# router's grant is the control that `grantsIn` reads a one-path grant too.
|
||||
name = "the cache's grant is create and read on its signing key and its two tokens, read on its forwarder client, and nothing else";
|
||||
name = "the cache's grant is create and read on its signing key, its two tokens and its three public facts, read on its forwarder client, and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-attic-policy.script;
|
||||
|
|
@ -2005,24 +2037,33 @@ let
|
|||
(grantFor grants "secret/data/swarm/services/attic/jwt-secret").caps == caps
|
||||
&& (grantFor grants "secret/data/swarm/services/attic/push-token").caps == caps
|
||||
&& (grantFor grants "secret/data/swarm/services/attic/pull-token").caps == caps
|
||||
&& (grantFor grants "secret/data/swarm/services/attic/url").caps == caps
|
||||
&& (grantFor grants "secret/data/swarm/services/attic/cache").caps == caps
|
||||
&& (grantFor grants "secret/data/swarm/services/attic/public-key").caps == caps
|
||||
&&
|
||||
(grantFor grants "secret/data/swarm/services/swarm-attic-collector/oidc/client").caps == [ "read" ]
|
||||
&& lib.length grants == 4
|
||||
&& lib.length grants == 7
|
||||
&& lib.length (grantsIn baoGrantHere.systemd.services.swarm-bao-notify-router-policy.script) == 1
|
||||
&& lib.hasInfix "auth/cert/certs/swarm-attic" s
|
||||
&& lib.hasInfix "allowed_common_names=swarm-attic" s
|
||||
&& lib.hasInfix "token_policies=swarm-attic" s;
|
||||
}
|
||||
{
|
||||
# The signing key and the pull token are the cache's alone; the push
|
||||
# token is also the builder's, and is read there.
|
||||
name = "no granting unit but the cache's names its signing key or its pull token";
|
||||
# The signing key is the cache's alone; the push token is also the
|
||||
# builder's, and is read there. The pull token is also the controller's,
|
||||
# which only reads it (pinned above).
|
||||
name = "no granting unit but the cache's names its signing key, and only it and the controller name its pull token";
|
||||
ok =
|
||||
lib.filter (
|
||||
u:
|
||||
lib.hasInfix "services/attic/jwt-secret" baoGrantHere.systemd.services.${u}.script
|
||||
|| lib.hasInfix "services/attic/pull-token" baoGrantHere.systemd.services.${u}.script
|
||||
) grantingUnitNames == [ "swarm-bao-attic-policy" ];
|
||||
u: lib.hasInfix "services/attic/jwt-secret" baoGrantHere.systemd.services.${u}.script
|
||||
) grantingUnitNames == [ "swarm-bao-attic-policy" ]
|
||||
&&
|
||||
lib.filter (
|
||||
u: lib.hasInfix "services/attic/pull-token" baoGrantHere.systemd.services.${u}.script
|
||||
) grantingUnitNames == [
|
||||
"swarm-bao-controller-policy"
|
||||
"swarm-bao-attic-policy"
|
||||
];
|
||||
}
|
||||
{
|
||||
name = "the PKI unit signs the cache's leaf under its own subject";
|
||||
|
|
|
|||
|
|
@ -13,6 +13,10 @@ let
|
|||
jwtPath = "secret/swarm/services/attic/jwt-secret";
|
||||
pullPath = "secret/swarm/services/attic/pull-token";
|
||||
pushPath = "secret/swarm/services/attic/push-token";
|
||||
urlPath = "secret/swarm/services/attic/url";
|
||||
cachePath = "secret/swarm/services/attic/cache";
|
||||
publicKeyPath = "secret/swarm/services/attic/public-key";
|
||||
url = "http://cache.example";
|
||||
cache = "grid";
|
||||
endpoint = "http://127.0.0.1:8910";
|
||||
configFile = "@DIR@/server.toml";
|
||||
|
|
|
|||
|
|
@ -84,7 +84,9 @@ tokenfile=\$(sed -n 's/^token-file = "\(.*\)"$/\1/p' "\$XDG_CONFIG_HOME/attic/co
|
|||
token=\$(cat "\$tokenfile")
|
||||
echo "attic \$* | token=\$token" >>"$calls"
|
||||
case "\$1 \$2" in
|
||||
"cache info") [ -f "\$DIR/cache-exists" ] ;;
|
||||
"cache info")
|
||||
[ -f "\$DIR/cache-exists" ] && echo " Public Key: grid:FAKEKEY" >&2
|
||||
;;
|
||||
"cache create") touch "\$DIR/cache-exists" ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
|
|
@ -123,6 +125,9 @@ stored() { cat "$dir/store/$1" 2>/dev/null || true; }
|
|||
JWT=secret_swarm_services_attic_jwt-secret
|
||||
PULL=secret_swarm_services_attic_pull-token
|
||||
PUSH=secret_swarm_services_attic_push-token
|
||||
URL=secret_swarm_services_attic_url
|
||||
CACHE=secret_swarm_services_attic_cache
|
||||
KEY=secret_swarm_services_attic_public-key
|
||||
ENVFILE=$dir/secrets/server.env
|
||||
|
||||
expect() { # <description> <got> <want>
|
||||
|
|
@ -179,6 +184,9 @@ expect "exit status" $? 0
|
|||
expect "pull token stored" "$(stored $PULL)" "tok-swarm-hives-pull-1"
|
||||
expect "push token stored" "$(stored $PUSH)" "tok-swarm-builder-push-2"
|
||||
expect "cache created once" "$(n '^attic cache create')" 1
|
||||
expect "the address is stored" "$(stored $URL)" "http://cache.example"
|
||||
expect "the cache name is stored" "$(stored $CACHE)" "grid"
|
||||
expect "the cache's public key is stored" "$(stored $KEY)" "grid:FAKEKEY"
|
||||
expect "cache created by the short-lived token" "$(n '^attic cache create local:grid | token=tok-swarm-attic-init-3')" 1
|
||||
expect "tokens minted: pull, push, the creator" "$(n '^atticadm')" 3
|
||||
expect "pull token reads only the cache" "$(n 'make-token --sub swarm-hives-pull --validity 10y --pull grid$')" 1
|
||||
|
|
@ -197,6 +205,7 @@ expect "no cache create" "$(n '^attic cache create')" 0
|
|||
expect "the pull token looked the cache up" "$(n "^attic cache info local:grid | token=$pull")" 1
|
||||
expect "pull token unchanged" "$(stored $PULL)" "$pull"
|
||||
expect "push token unchanged" "$(stored $PUSH)" "$push"
|
||||
expect "address, name and key unchanged" "$(stored $URL)$(stored $CACHE)$(stored $KEY)" "http://cache.examplegridgrid:FAKEKEY"
|
||||
|
||||
case="an empty server gets its cache back with the stored tokens"
|
||||
rm -f "$dir/cache-exists"
|
||||
|
|
@ -223,7 +232,7 @@ run "$ENSURE_JWT"
|
|||
run "$PROVISION"
|
||||
expect "no token among the client's arguments" "$(grep -c -- '^attic [^|]*tok-' "$calls" || true)" 0
|
||||
# The control: the client did see tokens, through its config file.
|
||||
expect "tokens seen through the config file" "$(grep -c -- '| token=tok-' "$calls" || true)" 3
|
||||
expect "tokens seen through the config file" "$(grep -c -- '| token=tok-' "$calls" || true)" 4
|
||||
|
||||
if [ "$failures" -ne 0 ]; then
|
||||
echo "$failures of $count checks failed" >&2
|
||||
|
|
|
|||
Loading…
Reference in a new issue