hive-c0re pulls closures through the nix daemon; hive-priv verifies the swarm signature before applying
This commit is contained in:
atlas
mara
parent
448b703995
commit
ecd4445680
7 changed files with 1257 additions and 584 deletions
|
|
@ -315,9 +315,11 @@ in
|
|||
# StateDirectory (/var/lib/hyperhive), RuntimeDirectory
|
||||
# (/run/hyperhive), and ReadWritePaths below, which systemd keeps
|
||||
# writable. Beyond the managed directories:
|
||||
# - nix is invoked directly (lifecycle, meta, flake_check), but
|
||||
# NIX_REMOTE=daemon routes all store writes through the host
|
||||
# daemon — hive-c0re never writes to /nix itself.
|
||||
# - the one nix call is the closure pull (src/cache_pull.rs):
|
||||
# `nix-store --realise` with NIX_REMOTE=daemon, so the host daemon
|
||||
# writes the store — hive-c0re never writes to /nix itself. It
|
||||
# connects to the daemon's socket, which RestrictAddressFamilies
|
||||
# (deferred above) must keep allowing.
|
||||
|
||||
# - nix build worktrees live in PrivateTmp, not /tmp.
|
||||
# - /etc writes (bind-mount edits) go through hive-priv via the
|
||||
|
|
|
|||
|
|
@ -1,18 +1,21 @@
|
|||
# The binary-cache facts hive-priv (root) trusts, kept where only root can
|
||||
# change them.
|
||||
# The binary-cache facts the host's nix daemon and hive-priv (root) trust, kept
|
||||
# where only root can change them.
|
||||
#
|
||||
# hive-priv downloads an agent's closure from the swarm cache when a rebuild
|
||||
# asks it to. What decides whether a download is accepted — which cache, and
|
||||
# which key must have signed it — must not come from a file the unprivileged
|
||||
# hive-c0re writes, or a compromised hive-c0re could have root accept a
|
||||
# closure from a cache of its own. So:
|
||||
# hive-c0re downloads an agent's closure from the swarm cache through the
|
||||
# host's nix daemon, and hive-priv applies it only if the swarm's key signed
|
||||
# it. What decides whether a download is accepted — which cache the daemon
|
||||
# lets an unprivileged client name, and which key must have signed it — must
|
||||
# not come from a file the unprivileged hive-c0re writes, or a compromised
|
||||
# hive-c0re could have root accept a closure from a cache of its own. So:
|
||||
#
|
||||
# - the cache's address is this host's configuration
|
||||
# ({option}`services.hyperhive.swarm.attic.domain` and `cache`), handed to
|
||||
# hive-priv in its unit's environment;
|
||||
# ({option}`services.hyperhive.swarm.attic.domain` and `cache`);
|
||||
# - the cache's public key and the pull token are read from the store by a root
|
||||
# oneshot, `hive-priv-cache-facts`, under this hive's own store identity, and
|
||||
# land in `/run/hyperhive-priv/attic/` (root, `0700`; files `0600`).
|
||||
# oneshot, `hive-priv-cache-facts`, under this hive's own store identity. The
|
||||
# key and the token land in `/run/hyperhive-priv/attic/` (root, `0700`; files
|
||||
# `0600`), and the address, the key and the netrc's path in a nix.conf
|
||||
# fragment, `/run/hyperhive-nix/cache.conf`, which the host's nix.conf
|
||||
# includes (`!include`, so a host whose unit has not run yet still starts).
|
||||
#
|
||||
# hive-c0re keeps its own copies under `/run/hyperhive/attic/` for the probe,
|
||||
# which only reports.
|
||||
|
|
@ -42,6 +45,9 @@ let
|
|||
&& tls.baoHiveIdentityKeyFile != null;
|
||||
|
||||
dir = "/run/hyperhive-priv/attic";
|
||||
confDir = "/run/hyperhive-nix";
|
||||
confFile = "${confDir}/cache.conf";
|
||||
url = "https://${atticCfg.domain}/${atticCfg.cache}";
|
||||
|
||||
# Where the store runs on this host, the first login waits for its container.
|
||||
# No `wants`: a pull would start the container mid-switch on the old closure.
|
||||
|
|
@ -52,15 +58,16 @@ let
|
|||
in
|
||||
{
|
||||
config = lib.mkIf enabled {
|
||||
systemd.tmpfiles.rules = [ "d /run/hyperhive-priv 0700 root root - -" ];
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /run/hyperhive-priv 0700 root root - -"
|
||||
"d ${confDir} 0755 root root - -"
|
||||
];
|
||||
|
||||
# The URL nix is told is the only substituter for the download; no request
|
||||
# chooses it.
|
||||
systemd.services.hive-priv.environment.HIVE_PRIV_CACHE_URL =
|
||||
"https://${atticCfg.domain}/${atticCfg.cache}";
|
||||
# `!include` tolerates a fragment that does not exist yet.
|
||||
nix.extraOptions = "!include ${confFile}";
|
||||
|
||||
systemd.services.hive-priv-cache-facts = {
|
||||
description = "Read the swarm cache's public key and pull token for hive-priv";
|
||||
description = "Read the swarm cache's public key and pull token for the host's nix daemon and hive-priv";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
after = [
|
||||
|
|
@ -94,7 +101,7 @@ in
|
|||
script = import ./lib/attic-root-facts.nix { inherit lib; } {
|
||||
hive = hiveController.hiveName;
|
||||
host = atticCfg.domain;
|
||||
inherit dir;
|
||||
inherit dir url confFile;
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -124,8 +124,8 @@ in
|
|||
# hive-priv runs as root. Root nix defaults to store=auto which
|
||||
# resolves to the LOCAL store — bypassing the host daemon, its
|
||||
# remote builders, and prebuilt derivation outputs. Force daemon
|
||||
# routing so the closure `RealiseClosure` downloads is in the same
|
||||
# store the container is then pointed at.
|
||||
# routing so the closure hive-c0re pulls through the daemon is in
|
||||
# the same store the container is then pointed at.
|
||||
NIX_REMOTE = "daemon";
|
||||
# The owner a namespaced agent's data moves to. Which agents are
|
||||
# namespaced is not set here: hive-c0re sends it with each start.
|
||||
|
|
|
|||
|
|
@ -1,22 +1,26 @@
|
|||
# The root-side copy of the two cache facts hive-priv trusts: the key a
|
||||
# downloaded closure must be signed by and the netrc the pull token goes out
|
||||
# in. The script runs as root on the hive host under the hive's own store
|
||||
# identity and writes both into `dir`, which only root can read.
|
||||
# The root-side copy of the cache facts the host's nix daemon and hive-priv
|
||||
# trust: the key a downloaded closure must be signed by and the netrc the pull
|
||||
# token goes out in. The script runs as root on the hive host under the hive's
|
||||
# own store identity and writes both into `dir`, which only root can read, and
|
||||
# a nix.conf fragment at `confFile` that the host's nix.conf includes.
|
||||
#
|
||||
# Everything hive-priv decides on is here or in the host's configuration, and
|
||||
# nothing is read from a file hive-c0re (unprivileged) writes: the key and the
|
||||
# token come from the store, the cache's address from
|
||||
# `services.hyperhive.swarm.attic.{domain,cache}`.
|
||||
# What the daemon and hive-priv decide on is here or in the host's
|
||||
# configuration, and nothing is read from a file hive-c0re (unprivileged)
|
||||
# writes: the key and the token come from the store, the cache's address from
|
||||
# `services.hyperhive.swarm.attic.{domain,cache}`. The fragment holds no secret
|
||||
# (the key is public, the netrc stays in `dir`) and hive-c0re can read it: it
|
||||
# takes the cache's address from there.
|
||||
#
|
||||
# A store that holds no facts yet, or cannot be reached, leaves what is in `dir`
|
||||
# as it is. Nothing here removes a file.
|
||||
# and `confFile` as they are. Nothing here removes a file.
|
||||
#
|
||||
# Pure function — NOT a NixOS module. Call it from a unit's `script`, with
|
||||
# `bao`, `coreutils` and `gnugrep` on its `path` and the `BAO_*` client
|
||||
# environment set:
|
||||
# `bao`, `coreutils`, `gnugrep` and `systemd` on its `path` and the `BAO_*`
|
||||
# client environment set:
|
||||
#
|
||||
# script = import ./lib/attic-root-facts.nix { inherit lib; } {
|
||||
# hive = "alpha"; host = "attic.example.org"; dir = "/run/hyperhive-priv/attic";
|
||||
# url = "https://attic.example.org/swarm"; confFile = "/run/hyperhive-nix/cache.conf";
|
||||
# };
|
||||
{ lib }:
|
||||
{
|
||||
|
|
@ -25,8 +29,12 @@
|
|||
hive,
|
||||
# The cache server's host name, which the netrc line names.
|
||||
host,
|
||||
# Where the two files go.
|
||||
# Where the key and the netrc go.
|
||||
dir,
|
||||
# The cache's address as the daemon lists it in `trusted-substituters`.
|
||||
url,
|
||||
# Where the nix.conf fragment goes; its directory is root's and readable.
|
||||
confFile,
|
||||
}:
|
||||
''
|
||||
set -euo pipefail
|
||||
|
|
@ -35,7 +43,9 @@
|
|||
${import ./refresh-consumer.nix { }}
|
||||
|
||||
dir=${lib.escapeShellArg dir}
|
||||
conf=${lib.escapeShellArg confFile}
|
||||
install -d -m 0700 "$dir"
|
||||
install -d -m 0755 "$(dirname -- "$conf")"
|
||||
|
||||
store_init
|
||||
store_login
|
||||
|
|
@ -84,4 +94,17 @@
|
|||
if secret_differs "$dir/netrc" "$netrc"; then
|
||||
atomic_write_secret 0600 "" "$dir/netrc" "$netrc"
|
||||
fi
|
||||
|
||||
# Last, so a daemon that reads the fragment finds the netrc in place. The
|
||||
# address is the one hive-c0re may name as the substituter. The mtime moves
|
||||
# only with the content. Lix re-reads nix.conf for every connection, CppNix
|
||||
# only at start, so a daemon that started before this write is restarted.
|
||||
printf -v fragment '%s\n%s\n%s' \
|
||||
${lib.escapeShellArg "extra-trusted-substituters = ${url}"} \
|
||||
"extra-trusted-public-keys = $key" \
|
||||
"netrc-file = $dir/netrc"
|
||||
if secret_differs "$conf" "$fragment"; then
|
||||
atomic_write_secret 0644 "" "$conf" "$fragment"
|
||||
fi
|
||||
refresh_unit nix-daemon.service "$conf"
|
||||
''
|
||||
|
|
|
|||
|
|
@ -878,10 +878,23 @@ let
|
|||
inner.ProcSubset == "all" && inner.ProtectProc == "invisible";
|
||||
}
|
||||
{
|
||||
name = "hive-priv is told the cache's address by the host's configuration";
|
||||
# The daemon only learns the cache from the fragment the root unit
|
||||
# writes, and lists as trusted the one address from the host's
|
||||
# configuration. A host with no unit has no include.
|
||||
name = "the host's nix.conf includes the cache fragment the root unit writes, with the cache's address";
|
||||
ok =
|
||||
allLocal.systemd.services.hive-priv.environment.HIVE_PRIV_CACHE_URL == "https://attic.t.local/swarm"
|
||||
&& !(noIdentity.systemd.services.hive-priv.environment ? HIVE_PRIV_CACHE_URL);
|
||||
let
|
||||
u = units.hive-priv-cache-facts;
|
||||
in
|
||||
lib.hasInfix "!include /run/hyperhive-nix/cache.conf" allLocal.nix.extraOptions
|
||||
&& !(lib.hasInfix "hyperhive-nix" noIdentity.nix.extraOptions)
|
||||
&& lib.hasInfix "conf=/run/hyperhive-nix/cache.conf" u.script
|
||||
&& lib.hasInfix "extra-trusted-substituters = https://attic.t.local/swarm" u.script
|
||||
&& lib.hasInfix ''atomic_write_secret 0644 "" "$conf"'' u.script
|
||||
&& lib.elem "d /run/hyperhive-nix 0755 root root - -" allLocal.systemd.tmpfiles.rules
|
||||
# Control for the absence: the lookup finds the unit's other variable.
|
||||
&& allLocal.systemd.services.hive-priv.environment ? NIX_REMOTE
|
||||
&& !(allLocal.systemd.services.hive-priv.environment ? HIVE_PRIV_CACHE_URL);
|
||||
}
|
||||
];
|
||||
in
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ let
|
|||
hive = "alpha";
|
||||
host = "cache.example";
|
||||
dir = "@DIR@/attic";
|
||||
url = "https://cache.example/swarm";
|
||||
confFile = "@DIR@/nix/cache.conf";
|
||||
}
|
||||
);
|
||||
in
|
||||
|
|
|
|||
|
|
@ -45,15 +45,30 @@ case "\$1 \$2" in
|
|||
esac
|
||||
STUB
|
||||
chmod +x "$dir/bin/bao"
|
||||
|
||||
# `systemctl` answers for nix-daemon.service: DAEMON_STARTED is when it last
|
||||
# became active (empty: never) and every restart request is logged.
|
||||
cat >"$dir/bin/systemctl" <<STUB
|
||||
#!$BASH
|
||||
echo "systemctl \$*" >>"$dir/systemctl"
|
||||
case "\$1" in
|
||||
show) printf '%s\n' "\${DAEMON_STARTED-}" ;;
|
||||
is-failed) exit 1 ;;
|
||||
try-restart) ;;
|
||||
*) echo "unexpected call: \$*" >&2; exit 98 ;;
|
||||
esac
|
||||
STUB
|
||||
chmod +x "$dir/bin/systemctl"
|
||||
export PATH="$dir/bin:$PATH"
|
||||
|
||||
KEYLEAF=secret_swarm_hives_alpha_attic_public-key
|
||||
TOKENLEAF=secret_swarm_hives_alpha_attic_pull-token
|
||||
|
||||
reset() {
|
||||
rm -rf "$dir/store" "$dir/attic" "$calls"
|
||||
rm -rf "$dir/store" "$dir/attic" "$dir/nix" "$calls" "$dir/systemctl"
|
||||
mkdir -p "$dir/store"
|
||||
: >"$calls"
|
||||
: >"$dir/systemctl"
|
||||
}
|
||||
|
||||
seed() { # <key> <token>
|
||||
|
|
@ -66,6 +81,7 @@ run() {
|
|||
env "$@" bash "$dir/script.sh" >"$dir/out" 2>"$dir/err"
|
||||
}
|
||||
|
||||
restarts() { grep -c '^systemctl try-restart' "$dir/systemctl"; }
|
||||
mode() { stat -c %a "$1" 2>/dev/null || echo missing; }
|
||||
ino() { stat -c %i "$1" 2>/dev/null || echo missing; }
|
||||
body() { cat "$1" 2>/dev/null || echo missing; }
|
||||
|
|
@ -80,14 +96,22 @@ expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok-
|
|||
expect "key mode" "$(mode "$dir/attic/public-key")" 600
|
||||
expect "netrc mode" "$(mode "$dir/attic/netrc")" 600
|
||||
expect "dir mode" "$(mode "$dir/attic")" 700
|
||||
expect "fragment" "$(body "$dir/nix/cache.conf")" "extra-trusted-substituters = https://cache.example/swarm
|
||||
extra-trusted-public-keys = swarm:S0VZ
|
||||
netrc-file = $dir/attic/netrc"
|
||||
expect "fragment mode" "$(mode "$dir/nix/cache.conf")" 644
|
||||
expect "fragment dir mode" "$(mode "$dir/nix")" 755
|
||||
expect "no secret in the fragment" "$(grep -c tok-1 "$dir/nix/cache.conf")" 0
|
||||
|
||||
case="a second run with the same facts rewrites nothing"
|
||||
k=$(ino "$dir/attic/public-key")
|
||||
n=$(ino "$dir/attic/netrc")
|
||||
c=$(ino "$dir/nix/cache.conf")
|
||||
run
|
||||
expect "exit" $? 0
|
||||
expect "key untouched" "$(ino "$dir/attic/public-key")" "$k"
|
||||
expect "netrc untouched" "$(ino "$dir/attic/netrc")" "$n"
|
||||
expect "fragment untouched" "$(ino "$dir/nix/cache.conf")" "$c"
|
||||
|
||||
case="a rotated token replaces the netrc and leaves the key"
|
||||
seed "swarm:S0VZ" "tok-2"
|
||||
|
|
@ -170,6 +194,54 @@ run
|
|||
expect "exit" $? 1
|
||||
intact
|
||||
|
||||
# A restart of nix-daemon is asked for only when the fragment was written after
|
||||
# the daemon last started; `now` is when the daemon "started", read between runs.
|
||||
now() { date -u '+%a %Y-%m-%d %H:%M:%S.%6N UTC'; }
|
||||
|
||||
case="a new fragment restarts a daemon that started before it"
|
||||
reset
|
||||
seed "swarm:S0VZ" "tok-1"
|
||||
run DAEMON_STARTED="Sat 2000-01-01 00:00:00.000000 UTC"
|
||||
expect "exit" $? 0
|
||||
expect "restart asked" "$(grep -c '^systemctl try-restart --no-block nix-daemon.service' "$dir/systemctl")" 1
|
||||
|
||||
case="the same facts leave the fragment and the daemon alone"
|
||||
started=$(now)
|
||||
: >"$dir/systemctl"
|
||||
run DAEMON_STARTED="$started"
|
||||
expect "exit" $? 0
|
||||
expect "no restart" "$(restarts)" 0
|
||||
|
||||
case="a rotated token changes the netrc but not the fragment, so no restart"
|
||||
started=$(now)
|
||||
seed "swarm:S0VZ" "tok-2"
|
||||
run DAEMON_STARTED="$started"
|
||||
expect "exit" $? 0
|
||||
expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok-2"
|
||||
expect "no restart" "$(restarts)" 0
|
||||
|
||||
case="a rotated key rewrites the fragment and restarts the daemon"
|
||||
started=$(now)
|
||||
seed "swarm:Tk9XCg" "tok-2"
|
||||
run DAEMON_STARTED="$started"
|
||||
expect "exit" $? 0
|
||||
expect "key in the fragment" "$(grep -c 'extra-trusted-public-keys = swarm:Tk9XCg' "$dir/nix/cache.conf")" 1
|
||||
expect "restart asked" "$(restarts)" 1
|
||||
|
||||
case="a store with no facts writes no fragment and restarts nothing"
|
||||
reset
|
||||
run DAEMON_STARTED="Sat 2000-01-01 00:00:00.000000 UTC"
|
||||
expect "exit" $? 0
|
||||
expect "no fragment" "$(body "$dir/nix/cache.conf")" missing
|
||||
expect "no restart" "$(restarts)" 0
|
||||
|
||||
case="a key that is refused writes no fragment"
|
||||
reset
|
||||
seed "swarm:S0 VZ" "tok-1"
|
||||
run
|
||||
expect "exit" $? 1
|
||||
expect "no fragment" "$(body "$dir/nix/cache.conf")" missing
|
||||
|
||||
if [ "$failures" -ne 0 ]; then
|
||||
echo "$failures of $count checks failed" >&2
|
||||
exit 1
|
||||
|
|
|
|||
Loading…
Reference in a new issue