vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

keep the cache's key and pull token for hive-priv in a root-only directory

Browse source
This commit is contained in:
atlas 2026-10-28 02:21:01 +01:00
commit b845630023
5 changed files with 519 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

1
nix/host-modules/default.nix
View file

@ -23,6 +23,7 @@
./hive-matrix.nix
./hive-network.nix
./hive-priv.nix
./hive-priv-cache.nix
./hive-squid.nix
./hive-tls.nix
./otel.nix

98
nix/host-modules/hive-priv-cache.nix Normal file
View file

@ -0,0 +1,98 @@
# The binary-cache facts hive-priv (root) trusts, kept where only root can
# change them.
#
# hive-priv downloads an agent's closure from the swarm cache when a rebuild
# asks it to. What decides whether a download is accepted — which cache, and
# which key must have signed it — must not come from a file the unprivileged
# hive-c0re writes, or a compromised hive-c0re could have root accept a
# closure from a cache of its own. So:
#
# - the cache's address is this host's configuration
# ({option}`services.hyperhive.swarm.attic.domain` and `cache`), handed to
# hive-priv in its unit's environment;
# - the cache's public key and the pull token are read from the store by a root
# oneshot, `hive-priv-cache-facts`, under this hive's own store identity, and
# land in `/run/hyperhive-priv/attic/` (root, `0700`; files `0600`).
#
# hive-c0re keeps its own copies under `/run/hyperhive/attic/` for the probe,
# which only reports.
{
pkgs,
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
hiveController = hyperhiveCfg.deploy.hive-controller;
tls = hiveController.tls;
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = hyperhiveCfg.deploy.bao;
atticCfg = hyperhiveCfg.swarm.attic;
# Without a cache to name, a hive name to read under or an identity to log
# in with there is nothing to fetch, and hive-priv has no cache to download
# from.
enabled =
hiveController.enable
&& atticCfg.domain != null
&& atticCfg.cache != null
&& hiveController.hiveName != null
&& tls.baoHiveIdentityCertFile != null
&& tls.baoHiveIdentityKeyFile != null;
dir = "/run/hyperhive-priv/attic";
storeRetry = import ./lib/store-retry.nix { };
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
in
{
config = lib.mkIf enabled {
systemd.tmpfiles.rules = [ "d /run/hyperhive-priv 0700 root root - -" ];
# The URL nix is told is the only substituter for the download; no request
# chooses it.
systemd.services.hive-priv.environment.HIVE_PRIV_CACHE_URL =
"https://${atticCfg.domain}/${atticCfg.cache}";
systemd.services.hive-priv-cache-facts = {
description = "Read the swarm cache's public key and pull token for hive-priv";
wantedBy = [ "multi-user.target" ];
wants = [ "network-online.target" ];
after = [
"network-online.target"
"systemd-tmpfiles-setup.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.gnugrep
pkgs.systemd
];
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = tls.baoHiveIdentityCertFile;
BAO_CLIENT_KEY = tls.baoHiveIdentityKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
# ./lib/store-retry.nix: the login fails for as long as the store is
# sealed or down.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
UMask = "0077";
SyslogIdentifier = "hive-priv-cache-facts";
TimeoutStartSec = "5min";
};
script = import ./lib/attic-root-facts.nix { inherit lib; } {
hive = hiveController.hiveName;
host = atticCfg.domain;
inherit dir;
};
};
systemd.timers.hive-priv-cache-facts = storeRefreshTimer "re-read the swarm cache's facts for hive-priv";
};
}

8
nix/host-modules/hive-tls.nix
View file

@ -127,8 +127,9 @@ let
# The hive's own store identity: its subject is
# `<hiveLeafCommonNamePrefix>-<hiveName>`, and its policy may issue from
# `hiveLeafRole` alone. It also carries the root fetch's handshake. Every
# hive must hold one.
# `hiveLeafRole` and read this hive's own cache public key and pull token
# (`hive-priv-cache-facts`, ./hive-priv-cache.nix), and nothing else. It also
# carries the root fetch's handshake. Every hive must hold one.
haveHiveIdentity = cfg.baoHiveIdentityCertFile != null && cfg.baoHiveIdentityKeyFile != null;
requireHiveIdentity = requireIdentity {
holder = "this hive";
@ -327,7 +328,8 @@ in
Its subject has to be
`<services.hyperhive.deploy.bao.hiveLeafCommonNamePrefix>-<hiveName>`;
that cert-auth role's policy may issue from this hive's gateway-leaf
role and nothing else.
role and read this hive's own cache public key and pull token, and
nothing else. `hive-priv-cache-facts` reads those two under this identity.
Every hive with a self-signed gateway needs it: without it both units
fail, naming this option and

87
nix/host-modules/lib/attic-root-facts.nix Normal file
View file

@ -0,0 +1,87 @@
# The root-side copy of the two cache facts hive-priv trusts: the key a
# downloaded closure must be signed by and the netrc the pull token goes out
# in. The script runs as root on the hive host under the hive's own store
# identity and writes both into `dir`, which only root can read.
#
# Everything hive-priv decides on is here or in the host's configuration, and
# nothing is read from a file hive-c0re (unprivileged) writes: the key and the
# token come from the store, the cache's address from
# `services.hyperhive.swarm.attic.{domain,cache}`.
#
# A store that holds no facts yet, or cannot be reached, leaves what is in `dir`
# as it is. Nothing here removes a file.
#
# Pure function — NOT a NixOS module. Call it from a unit's `script`, with
# `bao`, `coreutils` and `gnugrep` on its `path` and the `BAO_*` client
# environment set:
#
# script = import ./lib/attic-root-facts.nix { inherit lib; } {
# hive = "alpha"; host = "attic.example.org"; dir = "/run/hyperhive-priv/attic";
# };
{ lib }:
{
# The hive's name in the store: facts are read from
# `secret/swarm/hives/<hive>/attic/`.
hive,
# The cache server's host name, which the netrc line names.
host,
# Where the two files go.
dir,
}:
''
set -euo pipefail
${import ./store-seeded-secret.nix { }}
${import ./atomic-write-secret.nix { }}
${import ./refresh-consumer.nix { }}
dir=${lib.escapeShellArg dir}
install -d -m 0700 "$dir"
store_init
store_login
if [ "$STORE_UP" != 1 ]; then
echo "could not read the cache facts; $dir stays as it is." >&2
exit 1
fi
# <leaf> <variable>. Sets the variable to the leaf's value. A leaf the store
# does not hold sets `missing`; any other failure ends the run.
missing=0
fetch() {
local value
if value="$(store_bao kv get -field=value ${lib.escapeShellArg "secret/swarm/hives/${hive}/attic"}/"$1" 2>"$STORE_ERR")"; then
printf -v "$2" '%s' "$value"
elif grep -qi 'no value found' "$STORE_ERR"; then
missing=1
else
cat "$STORE_ERR" >&2
echo "could not read the cache fact $1; $dir stays as it is." >&2
exit 1
fi
}
key=""
token=""
fetch public-key key
fetch pull-token token
if [ "$missing" = 1 ]; then
echo "the swarm has not given this hive its cache facts yet; $dir stays as it is."
exit 0
fi
# One token each: both land in a file nix and hive-priv parse by whitespace.
if [[ ! "$key" =~ ^[^[:space:]:]+:[^[:space:]]+$ ]] || [[ ! "$token" =~ ^[^[:space:]]+$ ]]; then
echo "the stored cache facts are not a <name>:<key> and a single token; $dir stays as it is." >&2
exit 1
fi
# Written only when the value changed, so the files keep their mtime between
# runs. The owner is the running user, root, and the mode is 0600.
if secret_differs "$dir/public-key" "$key"; then
atomic_write_secret 0600 "" "$dir/public-key" "$key"
fi
netrc="machine ${host} password $token"
if secret_differs "$dir/netrc" "$netrc"; then
atomic_write_secret 0600 "" "$dir/netrc" "$netrc"
fi
''

34
nix/host-modules/swarm-bao.nix
View file

@ -935,8 +935,30 @@ let
cn = "${baoDeploy.hiveLeafCommonNamePrefix}-${hiveName}";
inherit (hive) domain;
dashboard = hyperhiveCfg.swarm.dashboardDomains.${hiveName};
inherit hiveName;
}) hyperhiveCfg.swarm.hives;
# What a hive's gateway-leaf identity may do: issue from its own role, and
# read two of its own cache facts. The root unit on the hive host
# (`hive-priv-cache-facts`, ./hive-priv-cache.nix) reads them under this
# identity: the key hive-priv's downloads must be signed by, and the token it
# pulls with. The cache's URL and name come from the host's configuration,
# not from the store.
hiveLeafPolicyText =
obj:
''
path "${servicesPkiMountPath}/issue/${obj.name}" {
capabilities = ["update"]
}
''
+
lib.concatMapStrings
(fact: readStanza "${credentialMountPath}/data/swarm/hives/${obj.hiveName}/attic/${fact}")
[
"public-key"
"pull-token"
];
# The PKI mount agent client certificates are issued from. Its root is
# generated inside the store, so the agent CA's key never exists outside it.
# A mount of its own because the services mount holds exactly one issuer; a
@ -2241,9 +2263,11 @@ in
example = "swarm-bao-hive-leaf.svc";
description = ''
Prefix of the subject the store's per-hive gateway-leaf roles accept.
The role for hive `h` accepts `<prefix>-h` and may only issue from
The role for hive `h` accepts `<prefix>-h`. Its policy may issue from
`<servicesPkiMountPath>/issue/<hiveLeafPkiRolePrefix>-h`, which issues
for `h`'s domain and `*.` that domain and nothing else.
for `h`'s domain and `*.` that domain and nothing else, and read
`secret/swarm/hives/h/attic/public-key` and `…/pull-token`, which the
root unit `hive-priv-cache-facts` fetches for hive-priv.
Per hive because one subject for every hive could ask for any hive's
certificate.
@ -3792,11 +3816,7 @@ in
ttl=${servicesPkiLeafTtl} \
max_ttl=${servicesPkiLeafTtl}
printf '%s' ${lib.escapeShellArg ''
path "${servicesPkiMountPath}/issue/${obj.name}" {
capabilities = ["update"]
}
''} |
printf '%s' ${lib.escapeShellArg (hiveLeafPolicyText obj)} |
bao policy write ${lib.escapeShellArg obj.name} -
bao write auth/cert/certs/${lib.escapeShellArg obj.name} \