mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

pull swarm closures through the nix daemon: substituters in the fragment, merged root-only netrc

Browse source
This commit is contained in:
atlas 2026-11-08 10:29:46 +01:00 • committed by mara
commit ef68baa00f
4 changed files with 582 additions and 382 deletions
Download patch file Download diff file Expand all files Collapse all files

2
nix/host-modules/hive-c0re/default.nix
View file

@ -316,7 +316,7 @@ in
# (/run/hyperhive), and ReadWritePaths below, which systemd keeps
# writable. Beyond the managed directories:
# - the one nix call is the closure pull (src/cache_pull.rs):
# `nix copy --from` with NIX_REMOTE=daemon, so the host daemon
# `nix-store --realise` with NIX_REMOTE=daemon, so the host daemon
# writes the store — hive-c0re never writes to /nix itself. It
# connects to the daemon's socket, which RestrictAddressFamilies
# (deferred above) must keep allowing.

59
nix/host-modules/hive-priv-cache.nix
View file

@ -1,30 +1,33 @@
# The binary-cache facts the host's nix daemon, hive-priv (root) and hive-c0re
# use, kept where only root can change them.
#
# hive-c0re pulls an agent's closure from the swarm cache with its own
# `nix copy --from`: the client does the HTTP fetch with the pull token, and
# hands the NAR to the host's nix daemon, which stores it only if a key in its
# hive-c0re pulls an agent's closure with `nix-store --realise` through the
# host's nix daemon, which substitutes each path from the swarm cache or from
# its other substituters (cache.nixos.org) and stores it only if a key in its
# `trusted-public-keys` signed it. hive-priv applies a closure only if the
# swarm's key signed it. What decides that must not come from a file the
# unprivileged hive-c0re writes. So:
# swarm's key signed it. Neither decision may come from a file hive-c0re writes:
#
# - the cache's address is this host's configuration
# ({option}`services.hyperhive.swarm.attic.domain` and `cache`), handed to
# hive-c0re in its unit's environment;
# - the cache's address is the host's
# {option}`services.hyperhive.swarm.attic.domain` and `cache`;
# - the cache's public key and the pull token are read from the store by a root
# oneshot, `hive-priv-cache-facts`, under this hive's own store identity. The
# key and the token land in `/run/hyperhive-priv/attic/` (root, `0700`; files
# `0600`). The key also lands in a nix.conf fragment,
# `/run/hyperhive-nix/cache.conf`, as `extra-trusted-public-keys`, which the
# host's nix.conf includes (`!include`, so a host whose unit has not run yet
# still starts). The fragment names no substituter and no netrc, so a client
# of the daemon that is not hive-c0re (an agent container on `nixBuilds =
# host`) cannot fetch from the cache or send the token;
# - the token also lands in `/run/hyperhive-nix/netrc`, `root:hive-core 0640`:
# the one place hive-core, and nothing else unprivileged, can read it.
# key lands in `/run/hyperhive-priv/attic/public-key` (directory `0700`, file
# `0600`).
# The cache's address, the key and a netrc also land in a nix.conf fragment,
# `/run/hyperhive-nix/cache.conf`, as `extra-substituters`,
# `extra-trusted-public-keys` and `netrc-file`, which the host's nix.conf
# includes (`!include`, so a host whose unit has not run yet still starts).
# Every client of the daemon (hive-c0re, agent containers on `nixBuilds =
# host`) can have a swarm closure substituted; closures hold no secret;
# - that netrc, `/run/hyperhive-nix/netrc` (`root:root 0600`), is the pull
# token's entry plus the entries of the host's own netrc, which
# `netrc-file` would otherwise replace. The host's netrc is
# `nix.settings.netrc-file` when the host sets it, else nix's default
# `/etc/nix/netrc`. The unit merges again on every run, so an edit reaches the
# daemon within the timer's ten minutes. A netrc named only in
# `nix.extraOptions` is not seen.
#
# hive-c0re keeps its own copies under `/run/hyperhive/attic/` for the probe,
# which only reports.
# hive-c0re keeps its own copies under `/run/hyperhive/attic/` for the probe.
{
pkgs,
lib,
@ -53,7 +56,10 @@ let
dir = "/run/hyperhive-priv/attic";
confDir = "/run/hyperhive-nix";
confFile = "${confDir}/cache.conf";
clientNetrc = "${confDir}/netrc";
cacheUrl = "https://${atticCfg.domain}/${atticCfg.cache}";
netrcFile = "${confDir}/netrc";
# `netrc-file` is not additive: the fragment's setting replaces the host's.
hostNetrc = toString (config.nix.settings.netrc-file or "/etc/nix/netrc");
# Where the store runs on this host, the first login waits for its container.
# No `wants`: a pull would start the container mid-switch on the old closure.
@ -72,10 +78,6 @@ in
# `!include` tolerates a fragment that does not exist yet.
nix.extraOptions = "!include ${confFile}";
# The address hive-c0re names to `nix copy --from`; no request chooses it.
systemd.services.hive-c0re.environment.HIVE_C0RE_CACHE_URL =
"https://${atticCfg.domain}/${atticCfg.cache}";
systemd.services.hive-priv-cache-facts = {
description = "Read the swarm cache's public key and pull token for the host's nix daemon and hive-priv";
wantedBy = [ "multi-user.target" ];
@ -111,8 +113,13 @@ in
script = import ./lib/attic-root-facts.nix { inherit lib; } {
hive = hiveController.hiveName;
host = atticCfg.domain;
inherit dir confFile clientNetrc;
clientGroup = config.users.users.hive-core.group;
inherit
dir
confFile
netrcFile
hostNetrc
;
url = cacheUrl;
};
};

91
nix/host-modules/lib/attic-root-facts.nix
View file

@ -1,22 +1,23 @@
# The root-side copy of the cache facts the host's nix daemon, hive-priv and
# hive-c0re use: the key a downloaded closure must be signed by and the netrc
# the pull token goes out in. The script runs as root on the hive host under
# the hive's own store identity and writes:
# The root-side copy of the cache facts the host's nix daemon and hive-priv
# use: the key a downloaded closure must be signed by and the netrc the pull
# token goes out in. The script runs as root on the hive host under the hive's
# own store identity and writes:
#
# - the key and the netrc into `dir`, which only root can read (hive-priv's
# check of the key);
# - the key into `dir`, which only root can read;
# - the netrc at `netrcFile`, root 0600: the pull token's entry for `host`
# followed by the entries of the host's own netrc (`hostNetrc`) for other
# machines. `netrc-file` is one path, so a fragment that names this file must
# carry what the host already keeps there;
# - a nix.conf fragment at `confFile` that the host's nix.conf includes, with
# `extra-trusted-public-keys` and nothing else: no trusted substituter and no
# netrc, so no other client of the daemon can fetch from the cache or send
# the token;
# - a second copy of the netrc at `clientNetrc`, `root:<clientGroup> 0640`,
# that hive-c0re passes to its own `nix copy --from` as a client setting.
# `extra-substituters` (the cache's address), `extra-trusted-public-keys` and
# `netrc-file` (that merged netrc), so the daemon
# substitutes from the cache and its other substituters. It holds no secret.
#
# Nothing is read from a file hive-c0re (unprivileged) writes: the key and the
# token come from the store. The fragment holds no secret.
# token come from the store.
#
# A store that holds no facts yet, or cannot be reached, leaves what is in
# `dir`, `confFile` and `clientNetrc` as they are. Nothing here removes a file.
# `dir`, `netrcFile` and `confFile` as they are. Nothing here removes a file.
#
# Pure function — NOT a NixOS module. Call it from a unit's `script`, with
# `bao`, `coreutils`, `gnugrep` and `systemd` on its `path` and the `BAO_*`
@ -24,8 +25,8 @@
#
# script = import ./lib/attic-root-facts.nix { inherit lib; } {
# hive = "alpha"; host = "attic.example.org"; dir = "/run/hyperhive-priv/attic";
# confFile = "/run/hyperhive-nix/cache.conf"; clientNetrc = "/run/hyperhive-nix/netrc";
# clientGroup = "hive-core";
# url = "https://attic.example.org/swarm"; confFile = "/run/hyperhive-nix/cache.conf";
# netrcFile = "/run/hyperhive-nix/netrc"; hostNetrc = "/etc/nix/netrc";
# };
{ lib }:
{
@ -34,13 +35,16 @@
hive,
# The cache server's host name, which the netrc line names.
host,
# Where the key and the netrc go.
# Where the key goes.
dir,
# The cache's address as the daemon lists it in `extra-substituters`.
url,
# Where the nix.conf fragment goes; its directory is root's and readable.
confFile,
# Where the netrc copy for the client goes, and the group that may read it.
clientNetrc,
clientGroup,
# Where the merged netrc goes, and the netrc the host uses without this
# module, which is merged in and never written.
netrcFile,
hostNetrc,
}:
''
set -euo pipefail
@ -50,9 +54,10 @@
dir=${lib.escapeShellArg dir}
conf=${lib.escapeShellArg confFile}
client_netrc=${lib.escapeShellArg clientNetrc}
netrc_file=${lib.escapeShellArg netrcFile}
host_netrc=${lib.escapeShellArg hostNetrc}
install -d -m 0700 "$dir"
install -d -m 0755 "$(dirname -- "$conf")" "$(dirname -- "$client_netrc")"
install -d -m 0755 "$(dirname -- "$conf")" "$(dirname -- "$netrc_file")"
store_init
store_login
@ -97,22 +102,40 @@
if secret_differs "$dir/public-key" "$key"; then
atomic_write_secret 0600 "" "$dir/public-key" "$key"
fi
# The host's netrc without its lines for this cache's host: an entry runs from
# a `machine` or `default` line to the next one. A missing file is no entries.
# Our entry goes first, which is the one curl takes when an entry the filter
# does not recognise names the cache's host too.
other_netrc_entries() {
local line skip=0 tok kept=""
[ -f "$host_netrc" ] || return 0
while IFS= read -r line || [ -n "$line" ]; do
read -ra tok <<<"$line"
case "''${tok[0]-}" in
machine) if [ "''${tok[1]-}" = ${lib.escapeShellArg host} ]; then skip=1; else skip=0; fi ;;
default) skip=0 ;;
esac
[ "$skip" = 1 ] || kept+="$line"$'\n'
done <"$host_netrc"
printf '%s' "$kept"
}
netrc="machine ${host} password $token"
if secret_differs "$dir/netrc" "$netrc"; then
atomic_write_secret 0600 "" "$dir/netrc" "$netrc"
others="$(other_netrc_entries)"
if [ -n "$others" ]; then netrc="$netrc"$'\n'"$others"; fi
# The mode too: an earlier deploy left this path 0640 for a group.
if secret_differs "$netrc_file" "$netrc" || [ "$(stat -c %a "$netrc_file")" != 600 ]; then
atomic_write_secret 0600 "" "$netrc_file" "$netrc"
fi
# The client's copy: readable by the group, which is a deliberate grant of the
# pull token to hive-c0re's user. Written before the fragment, so a host with
# the key has the netrc.
if secret_differs "$client_netrc" "$netrc"; then
atomic_write_secret 0640 ":${clientGroup}" "$client_netrc" "$netrc"
fi
# Last. The mtime moves only with the content. Lix re-reads nix.conf for every
# connection, CppNix only at start, so a daemon that started before this write
# is restarted.
fragment="extra-trusted-public-keys = $key"
# Last, so a daemon that reads the fragment finds the netrc in place. The
# mtime moves only with the content. Lix re-reads nix.conf for every
# connection, CppNix only at start, so a CppNix daemon that started before
# this write is restarted.
printf -v fragment '%s\n%s\n%s' \
${lib.escapeShellArg "extra-substituters = ${url}"} \
"extra-trusted-public-keys = $key" \
"netrc-file = $netrc_file"
if secret_differs "$conf" "$fragment"; then
atomic_write_secret 0644 "" "$conf" "$fragment"
fi

8
nix/host-modules/lib/refresh-consumer.nix
View file

@ -61,7 +61,9 @@
# <unit> <path> [systemctl option…]. Nothing unless `written_after_start`.
# Otherwise a running <unit> is restarted and a failed one — it may have
# hit its start limit without the credential — is reset and started; one
# stopped on purpose stays stopped. `--no-block` because a caller ordered
# that is neither running nor failed (stopped on purpose, never started, or
# a per-connection daemon with no unit of its own) is left alone, and
# nothing is logged for it. `--no-block` because a caller ordered
# `Before=` its consumer must not wait on a job that waits on the caller
# (see `swarm-services-cert`'s propagation in ../hive-tls.nix).
refresh_unit() {
@ -72,8 +74,8 @@
echo "$path changed after $unit last started, and $unit had failed — starting it"
systemctl "$@" reset-failed "$unit"
systemctl "$@" start --no-block "$unit"
else
echo "$path changed after $unit last started — restarting it if it runs"
elif systemctl "$@" is-active --quiet "$unit"; then
echo "$path changed after $unit last started — restarting it"
systemctl "$@" try-restart --no-block "$unit"
fi
}