pull swarm closures through the nix daemon: substituters in the fragment, merged root-only netrc
This commit is contained in:
atlas
mara
parent
63ae5d6cbf
commit
ef68baa00f
11 changed files with 582 additions and 382 deletions
|
|
@ -89,9 +89,16 @@ in
|
|||
# run must mint nothing.
|
||||
script-test-attic-init = import ./script-tests/attic-init.nix { inherit pkgs; };
|
||||
# Runs ./host-modules/lib/attic-root-facts.nix, the root unit that keeps the
|
||||
# cache's key and pull token for hive-priv, against a stub `bao`: file modes,
|
||||
# no rewrite when nothing changed, and no file removed when the store fails.
|
||||
# cache's key and the merged netrc for the host's nix daemon, against a stub
|
||||
# `bao`: the fragment, file modes, the host's own netrc entries kept, no rewrite
|
||||
# when nothing changed, and no file removed when the store fails.
|
||||
script-test-attic-root-facts = import ./script-tests/attic-root-facts.nix { inherit pkgs; };
|
||||
# Runs the closure pull, `nix-store --realise <top> --max-jobs 0`, against a
|
||||
# swarm cache that lacks a dependency and a second cache that has it: the daemon
|
||||
# takes each path from whichever substituter has it.
|
||||
script-test-cache-pull-substitute = import ./script-tests/cache-pull-substitute.nix {
|
||||
inherit pkgs;
|
||||
};
|
||||
# Runs ./host-modules/lib/store-seeded-secret.nix's seed, store-wins and mint
|
||||
# decisions against a stub `bao` that keeps its leaves in files.
|
||||
script-test-store-seeded-secret = import ./script-tests/store-seeded-secret.nix { inherit pkgs; };
|
||||
|
|
|
|||
|
|
@ -316,7 +316,7 @@ in
|
|||
# (/run/hyperhive), and ReadWritePaths below, which systemd keeps
|
||||
# writable. Beyond the managed directories:
|
||||
# - the one nix call is the closure pull (src/cache_pull.rs):
|
||||
# `nix copy --from` with NIX_REMOTE=daemon, so the host daemon
|
||||
# `nix-store --realise` with NIX_REMOTE=daemon, so the host daemon
|
||||
# writes the store — hive-c0re never writes to /nix itself. It
|
||||
# connects to the daemon's socket, which RestrictAddressFamilies
|
||||
# (deferred above) must keep allowing.
|
||||
|
|
|
|||
|
|
@ -1,30 +1,33 @@
|
|||
# The binary-cache facts the host's nix daemon, hive-priv (root) and hive-c0re
|
||||
# use, kept where only root can change them.
|
||||
#
|
||||
# hive-c0re pulls an agent's closure from the swarm cache with its own
|
||||
# `nix copy --from`: the client does the HTTP fetch with the pull token, and
|
||||
# hands the NAR to the host's nix daemon, which stores it only if a key in its
|
||||
# hive-c0re pulls an agent's closure with `nix-store --realise` through the
|
||||
# host's nix daemon, which substitutes each path from the swarm cache or from
|
||||
# its other substituters (cache.nixos.org) and stores it only if a key in its
|
||||
# `trusted-public-keys` signed it. hive-priv applies a closure only if the
|
||||
# swarm's key signed it. What decides that must not come from a file the
|
||||
# unprivileged hive-c0re writes. So:
|
||||
# swarm's key signed it. Neither decision may come from a file hive-c0re writes:
|
||||
#
|
||||
# - the cache's address is this host's configuration
|
||||
# ({option}`services.hyperhive.swarm.attic.domain` and `cache`), handed to
|
||||
# hive-c0re in its unit's environment;
|
||||
# - the cache's address is the host's
|
||||
# {option}`services.hyperhive.swarm.attic.domain` and `cache`;
|
||||
# - the cache's public key and the pull token are read from the store by a root
|
||||
# oneshot, `hive-priv-cache-facts`, under this hive's own store identity. The
|
||||
# key and the token land in `/run/hyperhive-priv/attic/` (root, `0700`; files
|
||||
# `0600`). The key also lands in a nix.conf fragment,
|
||||
# `/run/hyperhive-nix/cache.conf`, as `extra-trusted-public-keys`, which the
|
||||
# host's nix.conf includes (`!include`, so a host whose unit has not run yet
|
||||
# still starts). The fragment names no substituter and no netrc, so a client
|
||||
# of the daemon that is not hive-c0re (an agent container on `nixBuilds =
|
||||
# host`) cannot fetch from the cache or send the token;
|
||||
# - the token also lands in `/run/hyperhive-nix/netrc`, `root:hive-core 0640`:
|
||||
# the one place hive-core, and nothing else unprivileged, can read it.
|
||||
# key lands in `/run/hyperhive-priv/attic/public-key` (directory `0700`, file
|
||||
# `0600`).
|
||||
# The cache's address, the key and a netrc also land in a nix.conf fragment,
|
||||
# `/run/hyperhive-nix/cache.conf`, as `extra-substituters`,
|
||||
# `extra-trusted-public-keys` and `netrc-file`, which the host's nix.conf
|
||||
# includes (`!include`, so a host whose unit has not run yet still starts).
|
||||
# Every client of the daemon (hive-c0re, agent containers on `nixBuilds =
|
||||
# host`) can have a swarm closure substituted; closures hold no secret;
|
||||
# - that netrc, `/run/hyperhive-nix/netrc` (`root:root 0600`), is the pull
|
||||
# token's entry plus the entries of the host's own netrc, which
|
||||
# `netrc-file` would otherwise replace. The host's netrc is
|
||||
# `nix.settings.netrc-file` when the host sets it, else nix's default
|
||||
# `/etc/nix/netrc`. The unit merges again on every run, so an edit reaches the
|
||||
# daemon within the timer's ten minutes. A netrc named only in
|
||||
# `nix.extraOptions` is not seen.
|
||||
#
|
||||
# hive-c0re keeps its own copies under `/run/hyperhive/attic/` for the probe,
|
||||
# which only reports.
|
||||
# hive-c0re keeps its own copies under `/run/hyperhive/attic/` for the probe.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
|
|
@ -53,7 +56,10 @@ let
|
|||
dir = "/run/hyperhive-priv/attic";
|
||||
confDir = "/run/hyperhive-nix";
|
||||
confFile = "${confDir}/cache.conf";
|
||||
clientNetrc = "${confDir}/netrc";
|
||||
cacheUrl = "https://${atticCfg.domain}/${atticCfg.cache}";
|
||||
netrcFile = "${confDir}/netrc";
|
||||
# `netrc-file` is not additive: the fragment's setting replaces the host's.
|
||||
hostNetrc = toString (config.nix.settings.netrc-file or "/etc/nix/netrc");
|
||||
|
||||
# Where the store runs on this host, the first login waits for its container.
|
||||
# No `wants`: a pull would start the container mid-switch on the old closure.
|
||||
|
|
@ -72,10 +78,6 @@ in
|
|||
# `!include` tolerates a fragment that does not exist yet.
|
||||
nix.extraOptions = "!include ${confFile}";
|
||||
|
||||
# The address hive-c0re names to `nix copy --from`; no request chooses it.
|
||||
systemd.services.hive-c0re.environment.HIVE_C0RE_CACHE_URL =
|
||||
"https://${atticCfg.domain}/${atticCfg.cache}";
|
||||
|
||||
systemd.services.hive-priv-cache-facts = {
|
||||
description = "Read the swarm cache's public key and pull token for the host's nix daemon and hive-priv";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
|
@ -111,8 +113,13 @@ in
|
|||
script = import ./lib/attic-root-facts.nix { inherit lib; } {
|
||||
hive = hiveController.hiveName;
|
||||
host = atticCfg.domain;
|
||||
inherit dir confFile clientNetrc;
|
||||
clientGroup = config.users.users.hive-core.group;
|
||||
inherit
|
||||
dir
|
||||
confFile
|
||||
netrcFile
|
||||
hostNetrc
|
||||
;
|
||||
url = cacheUrl;
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,22 +1,23 @@
|
|||
# The root-side copy of the cache facts the host's nix daemon, hive-priv and
|
||||
# hive-c0re use: the key a downloaded closure must be signed by and the netrc
|
||||
# the pull token goes out in. The script runs as root on the hive host under
|
||||
# the hive's own store identity and writes:
|
||||
# The root-side copy of the cache facts the host's nix daemon and hive-priv
|
||||
# use: the key a downloaded closure must be signed by and the netrc the pull
|
||||
# token goes out in. The script runs as root on the hive host under the hive's
|
||||
# own store identity and writes:
|
||||
#
|
||||
# - the key and the netrc into `dir`, which only root can read (hive-priv's
|
||||
# check of the key);
|
||||
# - the key into `dir`, which only root can read;
|
||||
# - the netrc at `netrcFile`, root 0600: the pull token's entry for `host`
|
||||
# followed by the entries of the host's own netrc (`hostNetrc`) for other
|
||||
# machines. `netrc-file` is one path, so a fragment that names this file must
|
||||
# carry what the host already keeps there;
|
||||
# - a nix.conf fragment at `confFile` that the host's nix.conf includes, with
|
||||
# `extra-trusted-public-keys` and nothing else: no trusted substituter and no
|
||||
# netrc, so no other client of the daemon can fetch from the cache or send
|
||||
# the token;
|
||||
# - a second copy of the netrc at `clientNetrc`, `root:<clientGroup> 0640`,
|
||||
# that hive-c0re passes to its own `nix copy --from` as a client setting.
|
||||
# `extra-substituters` (the cache's address), `extra-trusted-public-keys` and
|
||||
# `netrc-file` (that merged netrc), so the daemon
|
||||
# substitutes from the cache and its other substituters. It holds no secret.
|
||||
#
|
||||
# Nothing is read from a file hive-c0re (unprivileged) writes: the key and the
|
||||
# token come from the store. The fragment holds no secret.
|
||||
# token come from the store.
|
||||
#
|
||||
# A store that holds no facts yet, or cannot be reached, leaves what is in
|
||||
# `dir`, `confFile` and `clientNetrc` as they are. Nothing here removes a file.
|
||||
# `dir`, `netrcFile` and `confFile` as they are. Nothing here removes a file.
|
||||
#
|
||||
# Pure function — NOT a NixOS module. Call it from a unit's `script`, with
|
||||
# `bao`, `coreutils`, `gnugrep` and `systemd` on its `path` and the `BAO_*`
|
||||
|
|
@ -24,8 +25,8 @@
|
|||
#
|
||||
# script = import ./lib/attic-root-facts.nix { inherit lib; } {
|
||||
# hive = "alpha"; host = "attic.example.org"; dir = "/run/hyperhive-priv/attic";
|
||||
# confFile = "/run/hyperhive-nix/cache.conf"; clientNetrc = "/run/hyperhive-nix/netrc";
|
||||
# clientGroup = "hive-core";
|
||||
# url = "https://attic.example.org/swarm"; confFile = "/run/hyperhive-nix/cache.conf";
|
||||
# netrcFile = "/run/hyperhive-nix/netrc"; hostNetrc = "/etc/nix/netrc";
|
||||
# };
|
||||
{ lib }:
|
||||
{
|
||||
|
|
@ -34,13 +35,16 @@
|
|||
hive,
|
||||
# The cache server's host name, which the netrc line names.
|
||||
host,
|
||||
# Where the key and the netrc go.
|
||||
# Where the key goes.
|
||||
dir,
|
||||
# The cache's address as the daemon lists it in `extra-substituters`.
|
||||
url,
|
||||
# Where the nix.conf fragment goes; its directory is root's and readable.
|
||||
confFile,
|
||||
# Where the netrc copy for the client goes, and the group that may read it.
|
||||
clientNetrc,
|
||||
clientGroup,
|
||||
# Where the merged netrc goes, and the netrc the host uses without this
|
||||
# module, which is merged in and never written.
|
||||
netrcFile,
|
||||
hostNetrc,
|
||||
}:
|
||||
''
|
||||
set -euo pipefail
|
||||
|
|
@ -50,9 +54,10 @@
|
|||
|
||||
dir=${lib.escapeShellArg dir}
|
||||
conf=${lib.escapeShellArg confFile}
|
||||
client_netrc=${lib.escapeShellArg clientNetrc}
|
||||
netrc_file=${lib.escapeShellArg netrcFile}
|
||||
host_netrc=${lib.escapeShellArg hostNetrc}
|
||||
install -d -m 0700 "$dir"
|
||||
install -d -m 0755 "$(dirname -- "$conf")" "$(dirname -- "$client_netrc")"
|
||||
install -d -m 0755 "$(dirname -- "$conf")" "$(dirname -- "$netrc_file")"
|
||||
|
||||
store_init
|
||||
store_login
|
||||
|
|
@ -97,22 +102,40 @@
|
|||
if secret_differs "$dir/public-key" "$key"; then
|
||||
atomic_write_secret 0600 "" "$dir/public-key" "$key"
|
||||
fi
|
||||
|
||||
# The host's netrc without its lines for this cache's host: an entry runs from
|
||||
# a `machine` or `default` line to the next one. A missing file is no entries.
|
||||
# Our entry goes first, which is the one curl takes when an entry the filter
|
||||
# does not recognise names the cache's host too.
|
||||
other_netrc_entries() {
|
||||
local line skip=0 tok kept=""
|
||||
[ -f "$host_netrc" ] || return 0
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
read -ra tok <<<"$line"
|
||||
case "''${tok[0]-}" in
|
||||
machine) if [ "''${tok[1]-}" = ${lib.escapeShellArg host} ]; then skip=1; else skip=0; fi ;;
|
||||
default) skip=0 ;;
|
||||
esac
|
||||
[ "$skip" = 1 ] || kept+="$line"$'\n'
|
||||
done <"$host_netrc"
|
||||
printf '%s' "$kept"
|
||||
}
|
||||
netrc="machine ${host} password $token"
|
||||
if secret_differs "$dir/netrc" "$netrc"; then
|
||||
atomic_write_secret 0600 "" "$dir/netrc" "$netrc"
|
||||
others="$(other_netrc_entries)"
|
||||
if [ -n "$others" ]; then netrc="$netrc"$'\n'"$others"; fi
|
||||
# The mode too: an earlier deploy left this path 0640 for a group.
|
||||
if secret_differs "$netrc_file" "$netrc" || [ "$(stat -c %a "$netrc_file")" != 600 ]; then
|
||||
atomic_write_secret 0600 "" "$netrc_file" "$netrc"
|
||||
fi
|
||||
|
||||
# The client's copy: readable by the group, which is a deliberate grant of the
|
||||
# pull token to hive-c0re's user. Written before the fragment, so a host with
|
||||
# the key has the netrc.
|
||||
if secret_differs "$client_netrc" "$netrc"; then
|
||||
atomic_write_secret 0640 ":${clientGroup}" "$client_netrc" "$netrc"
|
||||
fi
|
||||
|
||||
# Last. The mtime moves only with the content. Lix re-reads nix.conf for every
|
||||
# connection, CppNix only at start, so a daemon that started before this write
|
||||
# is restarted.
|
||||
fragment="extra-trusted-public-keys = $key"
|
||||
# Last, so a daemon that reads the fragment finds the netrc in place. The
|
||||
# mtime moves only with the content. Lix re-reads nix.conf for every
|
||||
# connection, CppNix only at start, so a CppNix daemon that started before
|
||||
# this write is restarted.
|
||||
printf -v fragment '%s\n%s\n%s' \
|
||||
${lib.escapeShellArg "extra-substituters = ${url}"} \
|
||||
"extra-trusted-public-keys = $key" \
|
||||
"netrc-file = $netrc_file"
|
||||
if secret_differs "$conf" "$fragment"; then
|
||||
atomic_write_secret 0644 "" "$conf" "$fragment"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -61,7 +61,9 @@
|
|||
# <unit> <path> [systemctl option…]. Nothing unless `written_after_start`.
|
||||
# Otherwise a running <unit> is restarted and a failed one — it may have
|
||||
# hit its start limit without the credential — is reset and started; one
|
||||
# stopped on purpose stays stopped. `--no-block` because a caller ordered
|
||||
# that is neither running nor failed (stopped on purpose, never started, or
|
||||
# a per-connection daemon with no unit of its own) is left alone, and
|
||||
# nothing is logged for it. `--no-block` because a caller ordered
|
||||
# `Before=` its consumer must not wait on a job that waits on the caller
|
||||
# (see `swarm-services-cert`'s propagation in ../hive-tls.nix).
|
||||
refresh_unit() {
|
||||
|
|
@ -72,8 +74,8 @@
|
|||
echo "$path changed after $unit last started, and $unit had failed — starting it"
|
||||
systemctl "$@" reset-failed "$unit"
|
||||
systemctl "$@" start --no-block "$unit"
|
||||
else
|
||||
echo "$path changed after $unit last started — restarting it if it runs"
|
||||
elif systemctl "$@" is-active --quiet "$unit"; then
|
||||
echo "$path changed after $unit last started — restarting it"
|
||||
systemctl "$@" try-restart --no-block "$unit"
|
||||
fi
|
||||
}
|
||||
|
|
|
|||
|
|
@ -773,7 +773,6 @@ let
|
|||
&& lib.hasInfix "dir=/run/hyperhive-priv/attic" u.script
|
||||
&& lib.hasInfix "install -d -m 0700 \"$dir\"" u.script
|
||||
&& lib.hasInfix ''atomic_write_secret 0600 "" "$dir/public-key"'' u.script
|
||||
&& lib.hasInfix ''atomic_write_secret 0600 "" "$dir/netrc"'' u.script
|
||||
&& lib.hasInfix "machine attic.t.local password" u.script
|
||||
&& u.environment.BAO_CLIENT_CERT == leafUnit.environment.BAO_CLIENT_CERT
|
||||
&& u.environment.BAO_CLIENT_KEY == leafUnit.environment.BAO_CLIENT_KEY
|
||||
|
|
@ -879,9 +878,9 @@ let
|
|||
}
|
||||
{
|
||||
# The daemon only learns the cache from the fragment the root unit
|
||||
# writes, and lists as trusted the one address from the host's
|
||||
# configuration. A host with no unit has no include.
|
||||
name = "the host's nix.conf includes a fragment that trusts only the swarm key, and only hive-core gets the token";
|
||||
# writes, from the address in the host's configuration. A host with no
|
||||
# unit has no include.
|
||||
name = "the host's nix.conf includes a fragment that gives the daemon the cache, the key and a root-only netrc that keeps the host's own entries";
|
||||
ok =
|
||||
let
|
||||
u = units.hive-priv-cache-facts;
|
||||
|
|
@ -890,23 +889,21 @@ let
|
|||
lib.hasInfix "!include /run/hyperhive-nix/cache.conf" allLocal.nix.extraOptions
|
||||
&& !(lib.hasInfix "hyperhive-nix" noIdentity.nix.extraOptions)
|
||||
&& lib.hasInfix "conf=/run/hyperhive-nix/cache.conf" u.script
|
||||
&& lib.hasInfix ''fragment="extra-trusted-public-keys = $key"'' u.script
|
||||
# The fragment names the cache, the key and the merged netrc.
|
||||
&& lib.hasInfix "extra-substituters = https://attic.t.local/swarm" u.script
|
||||
&& lib.hasInfix "extra-trusted-public-keys = $key" u.script
|
||||
&& lib.hasInfix "netrc-file = $netrc_file" u.script
|
||||
&& lib.hasInfix "netrc_file=/run/hyperhive-nix/netrc" u.script
|
||||
&& lib.hasInfix ''atomic_write_secret 0600 "" "$netrc_file"'' u.script
|
||||
&& lib.hasInfix ''atomic_write_secret 0644 "" "$conf"'' u.script
|
||||
# The daemon is told no substituter and no netrc: no other client of it
|
||||
# can fetch from the cache or send the token.
|
||||
&& !(lib.hasInfix "extra-trusted-substituters" u.script)
|
||||
&& !(lib.hasInfix "trusted-substituters" allLocal.nix.extraOptions)
|
||||
&& !(lib.hasInfix "netrc-file" u.script)
|
||||
# The token's one unprivileged reader is the hive-core group.
|
||||
&& lib.hasInfix "client_netrc=/run/hyperhive-nix/netrc" u.script
|
||||
&& lib.hasInfix ''atomic_write_secret 0640 ":hive-core" "$client_netrc"'' u.script
|
||||
# The host's own netrc is merged in: nix's default path when the host sets none.
|
||||
&& lib.hasInfix "host_netrc=/etc/nix/netrc" u.script
|
||||
# No client copy of the token for hive-core, and hive-c0re is not told the cache.
|
||||
&& !(lib.hasInfix "client_netrc" u.script)
|
||||
&& !(c0re ? HIVE_C0RE_CACHE_URL)
|
||||
&& lib.elem "d /run/hyperhive-nix 0755 root root - -" allLocal.systemd.tmpfiles.rules
|
||||
# hive-c0re is told the address; hive-priv no longer is.
|
||||
&& c0re.HIVE_C0RE_CACHE_URL == "https://attic.t.local/swarm"
|
||||
&& !(noIdentity.systemd.services.hive-c0re.environment ? HIVE_C0RE_CACHE_URL)
|
||||
# Control for the absences: the lookups find the units' other variables.
|
||||
# Control for the absence: the lookup finds the unit's other variables.
|
||||
&& c0re ? HOME
|
||||
&& noIdentity.systemd.services.hive-c0re.environment ? HOME
|
||||
&& allLocal.systemd.services.hive-priv.environment ? NIX_REMOTE
|
||||
&& !(allLocal.systemd.services.hive-priv.environment ? HIVE_PRIV_CACHE_URL);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,9 +15,10 @@ let
|
|||
hive = "alpha";
|
||||
host = "cache.example";
|
||||
dir = "@DIR@/attic";
|
||||
url = "https://cache.example/swarm";
|
||||
confFile = "@DIR@/nix/cache.conf";
|
||||
clientNetrc = "@DIR@/nix/netrc";
|
||||
clientGroup = "@GROUP@";
|
||||
netrcFile = "@DIR@/nix/netrc";
|
||||
hostNetrc = "@DIR@/host-netrc";
|
||||
}
|
||||
);
|
||||
in
|
||||
|
|
|
|||
|
|
@ -19,11 +19,7 @@ expect() { # <description> <got> <want>
|
|||
|
||||
dir=$(mktemp -d)
|
||||
mkdir -p "$dir/bin"
|
||||
# A group other than the primary one when the runner has one, so the group the script sets
|
||||
# differs from the one a new file gets.
|
||||
group=$(id -Gn | tr " " "\n" | grep -vx "$(id -gn)" | head -1 || true)
|
||||
group=${group:-$(id -gn)}
|
||||
sed -e "s|@DIR@|$dir|g" -e "s|@GROUP@|$group|g" "$SCRIPT" >"$dir/script.sh"
|
||||
sed -e "s|@DIR@|$dir|g" "$SCRIPT" >"$dir/script.sh"
|
||||
calls=$dir/calls
|
||||
export DIR=$dir
|
||||
|
||||
|
|
@ -51,13 +47,14 @@ STUB
|
|||
chmod +x "$dir/bin/bao"
|
||||
|
||||
# `systemctl` answers for nix-daemon.service: DAEMON_STARTED is when it last
|
||||
# became active (empty: never) and every restart request is logged.
|
||||
# became active (empty: never, so it is not running) and every call is logged.
|
||||
cat >"$dir/bin/systemctl" <<STUB
|
||||
#!$BASH
|
||||
echo "systemctl \$*" >>"$dir/systemctl"
|
||||
case "\$1" in
|
||||
show) printf '%s\n' "\${DAEMON_STARTED-}" ;;
|
||||
is-failed) exit 1 ;;
|
||||
is-active) [ -n "\${DAEMON_STARTED-}" ] ;;
|
||||
try-restart) ;;
|
||||
*) echo "unexpected call: \$*" >&2; exit 98 ;;
|
||||
esac
|
||||
|
|
@ -69,7 +66,7 @@ KEYLEAF=secret_swarm_hives_alpha_attic_public-key
|
|||
TOKENLEAF=secret_swarm_hives_alpha_attic_pull-token
|
||||
|
||||
reset() {
|
||||
rm -rf "$dir/store" "$dir/attic" "$dir/nix" "$calls" "$dir/systemctl"
|
||||
rm -rf "$dir/store" "$dir/attic" "$dir/nix" "$dir/host-netrc" "$calls" "$dir/systemctl"
|
||||
mkdir -p "$dir/store"
|
||||
: >"$calls"
|
||||
: >"$dir/systemctl"
|
||||
|
|
@ -90,43 +87,41 @@ mode() { stat -c %a "$1" 2>/dev/null || echo missing; }
|
|||
ino() { stat -c %i "$1" 2>/dev/null || echo missing; }
|
||||
body() { cat "$1" 2>/dev/null || echo missing; }
|
||||
|
||||
case="first run writes the key and the netrc, each 0600, in a 0700 directory"
|
||||
case="first run writes the key (0600, in a 0700 directory) and the netrc (0600)"
|
||||
reset
|
||||
seed "swarm:S0VZ" "tok-1"
|
||||
run
|
||||
expect "exit" $? 0
|
||||
expect "key" "$(body "$dir/attic/public-key")" "swarm:S0VZ"
|
||||
expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok-1"
|
||||
expect "netrc" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1"
|
||||
expect "key mode" "$(mode "$dir/attic/public-key")" 600
|
||||
expect "netrc mode" "$(mode "$dir/attic/netrc")" 600
|
||||
expect "netrc mode" "$(mode "$dir/nix/netrc")" 600
|
||||
expect "dir mode" "$(mode "$dir/attic")" 700
|
||||
expect "fragment is the key and nothing else" "$(body "$dir/nix/cache.conf")" "extra-trusted-public-keys = swarm:S0VZ"
|
||||
expect "fragment names the cache, the key and the root-only netrc" "$(body "$dir/nix/cache.conf")" "extra-substituters = https://cache.example/swarm
|
||||
extra-trusted-public-keys = swarm:S0VZ
|
||||
netrc-file = $dir/nix/netrc"
|
||||
expect "fragment mode" "$(mode "$dir/nix/cache.conf")" 644
|
||||
expect "fragment dir mode" "$(mode "$dir/nix")" 755
|
||||
expect "no secret in the fragment" "$(grep -c tok-1 "$dir/nix/cache.conf")" 0
|
||||
expect "no substituter or netrc in the fragment" "$(grep -c -E "substituter|netrc" "$dir/nix/cache.conf")" 0
|
||||
expect "client netrc" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1"
|
||||
expect "client netrc mode" "$(mode "$dir/nix/netrc")" 640
|
||||
expect "client netrc group" "$(stat -c %G "$dir/nix/netrc")" "$group"
|
||||
expect "control: the token is in the netrc" "$(grep -c tok-1 "$dir/nix/netrc")" 1
|
||||
expect "the netrc the fragment names is root-only" "$(mode "$dir/nix/netrc")" 600
|
||||
|
||||
case="a second run with the same facts rewrites nothing"
|
||||
k=$(ino "$dir/attic/public-key")
|
||||
n=$(ino "$dir/attic/netrc")
|
||||
cn=$(ino "$dir/nix/netrc")
|
||||
n=$(ino "$dir/nix/netrc")
|
||||
c=$(ino "$dir/nix/cache.conf")
|
||||
run
|
||||
expect "exit" $? 0
|
||||
expect "key untouched" "$(ino "$dir/attic/public-key")" "$k"
|
||||
expect "netrc untouched" "$(ino "$dir/attic/netrc")" "$n"
|
||||
expect "netrc untouched" "$(ino "$dir/nix/netrc")" "$n"
|
||||
expect "fragment untouched" "$(ino "$dir/nix/cache.conf")" "$c"
|
||||
expect "client netrc untouched" "$(ino "$dir/nix/netrc")" "$cn"
|
||||
|
||||
case="a rotated token replaces the netrc and leaves the key"
|
||||
seed "swarm:S0VZ" "tok-2"
|
||||
run
|
||||
expect "exit" $? 0
|
||||
expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok-2"
|
||||
expect "netrc mode" "$(mode "$dir/attic/netrc")" 600
|
||||
expect "netrc" "$(body "$dir/nix/netrc")" "machine cache.example password tok-2"
|
||||
expect "netrc mode" "$(mode "$dir/nix/netrc")" 600
|
||||
expect "key untouched" "$(ino "$dir/attic/public-key")" "$k"
|
||||
|
||||
case="a rotated key replaces the key file"
|
||||
|
|
@ -140,12 +135,71 @@ expect "reads" "$(grep -c '^bao kv get' "$calls")" "$(grep -c '^bao kv get -fiel
|
|||
expect "login first" "$(head -1 "$calls")" "bao login -method=cert -token-only"
|
||||
expect "something was read" "$([ "$(grep -c '^bao kv get' "$calls")" -ge 2 ] && echo yes || echo no)" yes
|
||||
|
||||
# The merged netrc: the cache's entry first, then the host's other entries as
|
||||
# they are. A missing host file is no entries.
|
||||
HOSTNETRC='machine git.example login bob password hunter2
|
||||
machine other.example
|
||||
login alice
|
||||
password s3cret'
|
||||
|
||||
case="a host netrc with two other machines keeps both, after the cache's entry"
|
||||
reset
|
||||
seed "swarm:S0VZ" "tok-1"
|
||||
printf '%s\n' "$HOSTNETRC" >"$dir/host-netrc"
|
||||
run
|
||||
expect "exit" $? 0
|
||||
expect "merged" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1
|
||||
$HOSTNETRC"
|
||||
expect "mode" "$(mode "$dir/nix/netrc")" 600
|
||||
expect "the host's file is not written" "$(body "$dir/host-netrc")" "$HOSTNETRC"
|
||||
|
||||
case="a host entry for the cache's own machine is replaced and the rest is kept"
|
||||
reset
|
||||
seed "swarm:S0VZ" "tok-1"
|
||||
printf '%s\n%s\n' "$HOSTNETRC" "machine cache.example login old password stale" >"$dir/host-netrc"
|
||||
run
|
||||
expect "exit" $? 0
|
||||
expect "merged" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1
|
||||
$HOSTNETRC"
|
||||
expect "control: the stale entry was in the host file" "$(grep -c stale "$dir/host-netrc")" 1
|
||||
|
||||
case="a multi-line host entry for the cache's machine goes whole"
|
||||
reset
|
||||
seed "swarm:S0VZ" "tok-1"
|
||||
printf '%s\n%s\n' "machine cache.example
|
||||
login old
|
||||
password stale" "$HOSTNETRC" >"$dir/host-netrc"
|
||||
run
|
||||
expect "exit" $? 0
|
||||
expect "merged" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1
|
||||
$HOSTNETRC"
|
||||
|
||||
case="an edit to the host netrc reaches the merged file on the next run"
|
||||
reset
|
||||
seed "swarm:S0VZ" "tok-1"
|
||||
printf '%s\n' "$HOSTNETRC" >"$dir/host-netrc"
|
||||
run
|
||||
printf '%s\n' "machine new.example login n password p" >>"$dir/host-netrc"
|
||||
run
|
||||
expect "exit" $? 0
|
||||
expect "new entry merged" "$(grep -c new.example "$dir/nix/netrc")" 1
|
||||
expect "old entries kept" "$(grep -c git.example "$dir/nix/netrc")" 1
|
||||
|
||||
case="a netrc left 0640 by an earlier deploy is rewritten 0600 though its content is current"
|
||||
reset
|
||||
seed "swarm:S0VZ" "tok-1"
|
||||
run
|
||||
chmod 640 "$dir/nix/netrc"
|
||||
run
|
||||
expect "exit" $? 0
|
||||
expect "mode" "$(mode "$dir/nix/netrc")" 600
|
||||
|
||||
case="a store with no facts exits cleanly and creates nothing"
|
||||
reset
|
||||
run
|
||||
expect "exit" $? 0
|
||||
expect "no key" "$(body "$dir/attic/public-key")" missing
|
||||
expect "no netrc" "$(body "$dir/attic/netrc")" missing
|
||||
expect "no netrc" "$(body "$dir/nix/netrc")" missing
|
||||
|
||||
case="one fact without the other creates nothing"
|
||||
reset
|
||||
|
|
@ -153,7 +207,7 @@ printf '%s' "swarm:S0VZ" >"$dir/store/$KEYLEAF"
|
|||
run
|
||||
expect "exit" $? 0
|
||||
expect "no key" "$(body "$dir/attic/public-key")" missing
|
||||
expect "no netrc" "$(body "$dir/attic/netrc")" missing
|
||||
expect "no netrc" "$(body "$dir/nix/netrc")" missing
|
||||
|
||||
# The failure cases start from a good state and an extra file in the directory.
|
||||
good() {
|
||||
|
|
@ -164,7 +218,7 @@ good() {
|
|||
}
|
||||
intact() {
|
||||
expect "key kept" "$(body "$dir/attic/public-key")" "swarm:S0VZ"
|
||||
expect "netrc kept" "$(body "$dir/attic/netrc")" "machine cache.example password tok-1"
|
||||
expect "netrc kept" "$(body "$dir/nix/netrc")" "machine cache.example password tok-1"
|
||||
expect "other file kept" "$(body "$dir/attic/other")" ""
|
||||
}
|
||||
|
||||
|
|
@ -212,6 +266,16 @@ seed "swarm:S0VZ" "tok-1"
|
|||
run DAEMON_STARTED="Sat 2000-01-01 00:00:00.000000 UTC"
|
||||
expect "exit" $? 0
|
||||
expect "restart asked" "$(grep -c '^systemctl try-restart --no-block nix-daemon.service' "$dir/systemctl")" 1
|
||||
expect "restart claimed" "$(grep -c "cache.conf changed after nix-daemon.service last started — restarting it\$" "$dir/out")" 1
|
||||
|
||||
case="a daemon that never ran is not restarted, and nothing claims it was"
|
||||
reset
|
||||
seed "swarm:S0VZ" "tok-1"
|
||||
run DAEMON_STARTED=
|
||||
expect "exit" $? 0
|
||||
expect "fragment written" "$(grep -c "^extra-substituters" "$dir/nix/cache.conf")" 1
|
||||
expect "no restart" "$(restarts)" 0
|
||||
expect "no change-vs-start claim" "$(grep -c "changed after" "$dir/out")" 0
|
||||
|
||||
case="the same facts leave the fragment and the daemon alone"
|
||||
started=$(now)
|
||||
|
|
@ -225,9 +289,7 @@ started=$(now)
|
|||
seed "swarm:S0VZ" "tok-2"
|
||||
run DAEMON_STARTED="$started"
|
||||
expect "exit" $? 0
|
||||
expect "netrc" "$(body "$dir/attic/netrc")" "machine cache.example password tok-2"
|
||||
expect "client netrc follows" "$(body "$dir/nix/netrc")" "machine cache.example password tok-2"
|
||||
expect "client netrc mode" "$(mode "$dir/nix/netrc")" 640
|
||||
expect "netrc" "$(body "$dir/nix/netrc")" "machine cache.example password tok-2"
|
||||
expect "no restart" "$(restarts)" 0
|
||||
|
||||
case="a rotated key rewrites the fragment and restarts the daemon"
|
||||
|
|
@ -243,7 +305,6 @@ reset
|
|||
run DAEMON_STARTED="Sat 2000-01-01 00:00:00.000000 UTC"
|
||||
expect "exit" $? 0
|
||||
expect "no fragment" "$(body "$dir/nix/cache.conf")" missing
|
||||
expect "no client netrc" "$(body "$dir/nix/netrc")" missing
|
||||
expect "no restart" "$(restarts)" 0
|
||||
|
||||
case="a key that is refused writes no fragment"
|
||||
|
|
@ -252,7 +313,6 @@ seed "swarm:S0 VZ" "tok-1"
|
|||
run
|
||||
expect "exit" $? 1
|
||||
expect "no fragment" "$(body "$dir/nix/cache.conf")" missing
|
||||
expect "no client netrc" "$(body "$dir/nix/netrc")" missing
|
||||
|
||||
if [ "$failures" -ne 0 ]; then
|
||||
echo "$failures of $count checks failed" >&2
|
||||
|
|
|
|||
18
nix/script-tests/cache-pull-substitute.nix
Normal file
18
nix/script-tests/cache-pull-substitute.nix
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
# `checks.script-test-cache-pull-substitute` — runs the closure pull
|
||||
# (`nix-store --realise <top> --max-jobs 0`, hive-c0re/src/cache_pull.rs)
|
||||
# against two file caches: a swarm cache that has the top path and lacks a
|
||||
# dependency, and the host's own substituter that has the dependency. The cases
|
||||
# are in ./cache-pull-substitute.sh.
|
||||
{ pkgs, ... }:
|
||||
pkgs.runCommand "hyperhive-script-test-cache-pull-substitute"
|
||||
{
|
||||
nativeBuildInputs = [
|
||||
pkgs.nix
|
||||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
];
|
||||
}
|
||||
''
|
||||
${pkgs.bash}/bin/bash ${./cache-pull-substitute.sh}
|
||||
touch "$out"
|
||||
''
|
||||
95
nix/script-tests/cache-pull-substitute.sh
Normal file
95
nix/script-tests/cache-pull-substitute.sh
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
# Cases for the closure pull (hive-c0re/src/cache_pull.rs): `nix-store --realise
|
||||
# <top> --max-jobs 0` against a store whose configuration is what the daemon
|
||||
# has on a hive host, the host's own substituter plus the swarm cache from the
|
||||
# nix.conf fragment.
|
||||
#
|
||||
# The swarm cache holds the top path and NOT its dependency, as attic does for
|
||||
# a path cache.nixos.org serves; a second cache, standing for cache.nixos.org,
|
||||
# holds the dependency. The stores are chroot stores, so the realise runs in
|
||||
# this process where a host runs it in the daemon; the substitution logic is
|
||||
# the same.
|
||||
set -uo pipefail
|
||||
|
||||
failures=0
|
||||
count=0
|
||||
case="setup"
|
||||
|
||||
fail() {
|
||||
echo "FAILED: $case: $*" >&2
|
||||
failures=$((failures + 1))
|
||||
}
|
||||
|
||||
expect() { # <description> <got> <want>
|
||||
count=$((count + 1))
|
||||
if [ "$2" != "$3" ]; then fail "$1: got '$2', wanted '$3'"; fi
|
||||
}
|
||||
|
||||
T=$(mktemp -d)
|
||||
export HOME=$T/home XDG_CONFIG_HOME=$T/home NIX_USER_CONF_FILES=
|
||||
mkdir -p "$HOME" "$T/empty"
|
||||
export NIX_CONF_DIR=$T/empty
|
||||
printf 'build-users-group =\nsandbox = false\nexperimental-features = nix-command\n' >"$T/empty/nix.conf"
|
||||
|
||||
src="local?root=$T/src"
|
||||
dep=$(nix eval --raw --store "$src" --expr 'builtins.toFile "dep" "dependency"')
|
||||
top=$(nix eval --raw --store "$src" --expr 'let dep = builtins.toFile "dep" "dependency"; in builtins.toFile "top" "needs ${dep}"')
|
||||
nix-store --generate-binary-cache-key swarm-test-1 "$T/swarm.sec" "$T/swarm.pub" >/dev/null
|
||||
nix-store --generate-binary-cache-key upstream-test-1 "$T/up.sec" "$T/up.pub" >/dev/null
|
||||
attic=file://$T/attic
|
||||
upstream=file://$T/upstream
|
||||
nix copy --from "$src" --to "$attic?secret-key=$T/swarm.sec" "$top"
|
||||
nix copy --from "$src" --to "$upstream?secret-key=$T/up.sec" "$dep"
|
||||
# What attic leaves out because the upstream serves it.
|
||||
rm -f "$T/attic/$(basename "$dep" | cut -c1-32).narinfo"
|
||||
|
||||
# The daemon's nix.conf: its own substituter and key, then the include.
|
||||
# conf <name> <fragment text>: a NIX_CONF_DIR with that fragment.
|
||||
conf() {
|
||||
mkdir -p "$T/$1"
|
||||
printf 'build-users-group =\nsandbox = false\nsubstituters = %s\ntrusted-public-keys = %s\n!include %s/cache.conf\n' \
|
||||
"$upstream" "$(cat "$T/up.pub")" "$T/$1" >"$T/$1/nix.conf"
|
||||
printf '%s\n' "$2" >"$T/$1/cache.conf"
|
||||
}
|
||||
fragment="extra-substituters = $attic
|
||||
extra-trusted-public-keys = $(cat "$T/swarm.pub")"
|
||||
# The paths are content-addressed (`builtins.toFile`), which `require-sigs` does
|
||||
# not apply to, so this test cannot show the signature gate; hive-priv's
|
||||
# `nix store verify` step and the fragment test carry the key.
|
||||
|
||||
# pull <conf dir> <dest name>: the exact pull of cache_pull.rs.
|
||||
pull() {
|
||||
NIX_CONF_DIR=$T/$1 nix-store --store "local?root=$T/$2" --realise "$top" --max-jobs 0 >/dev/null 2>"$T/err"
|
||||
}
|
||||
valid() { # <dest name> <path>
|
||||
nix-store --store "local?root=$T/$1" --check-validity "$2" 2>/dev/null && echo yes || echo no
|
||||
}
|
||||
|
||||
narinfo() { # <cache dir> <store path>: whether the cache has its narinfo
|
||||
if [ -e "$T/$1/$(basename "$2" | cut -c1-32).narinfo" ]; then echo yes; else echo no; fi
|
||||
}
|
||||
|
||||
case="control: the swarm cache has the top path and lacks the dependency"
|
||||
expect "attic has the top path" "$(narinfo attic "$top")" yes
|
||||
expect "attic lacks the dependency" "$(narinfo attic "$dep")" no
|
||||
expect "upstream has the dependency" "$(narinfo upstream "$dep")" yes
|
||||
|
||||
case="the client-side copy that failed on a host fails here too"
|
||||
if nix copy --from "$attic" --to "local?root=$T/d0" "$top" 2>"$T/err"; then
|
||||
fail "nix copy --from the swarm cache alone succeeded"
|
||||
fi
|
||||
expect "no top path" "$(valid d0 "$top")" no
|
||||
|
||||
case="the daemon with both substituters realises the top path and its dependency"
|
||||
conf both "$fragment"
|
||||
pull both d1
|
||||
expect "exit" $? 0
|
||||
expect "top path valid" "$(valid d1 "$top")" yes
|
||||
expect "dependency valid" "$(valid d1 "$dep")" yes
|
||||
|
||||
case="without the fragment the daemon has only the upstream and cannot realise the top path"
|
||||
conf plain ""
|
||||
if pull plain d2; then fail "realised without the swarm cache"; fi
|
||||
expect "no top path" "$(valid d2 "$top")" no
|
||||
|
||||
echo "$count checks, $failures failed"
|
||||
[ "$failures" = 0 ]
|
||||
|
|
@ -26,13 +26,17 @@ cat >"$dir/bin/systemctl" <<STUB
|
|||
args=" \$* "
|
||||
case "\$args" in
|
||||
*" show "*) printf '%s\n' "\$STARTED" ;;
|
||||
*" is-active "*) [ "\$ACTIVE" = 1 ] ;;
|
||||
*" is-active "*)
|
||||
case "\$args" in
|
||||
*" container@"*) [ "\$ACTIVE" = 1 ] ;;
|
||||
*) [ "\$UNIT_ACTIVE" = 1 ] ;;
|
||||
esac ;;
|
||||
*" is-failed "*) [ "\$FAILED" = 1 ] ;;
|
||||
*) printf '%s\n' "\$*" >>"$calls" ;;
|
||||
esac
|
||||
STUB
|
||||
chmod +x "$dir/bin/systemctl"
|
||||
export PATH="$dir/bin:$PATH" ACTIVE=1 FAILED=0 STARTED=
|
||||
export PATH="$dir/bin:$PATH" ACTIVE=1 UNIT_ACTIVE=1 FAILED=0 STARTED=
|
||||
|
||||
export LC_ALL=de_DE.UTF-8
|
||||
|
||||
|
|
@ -106,10 +110,21 @@ check "a failed unit is reset and started" \
|
|||
refresh_unit gitea-runner-swarm.service "$path"
|
||||
FAILED=0
|
||||
|
||||
STARTED=$(systemd_ts 1791232888.193158)
|
||||
got=$(refresh_unit gitea-runner-swarm.service "$path" 2>&1)
|
||||
case="control: a running unit is said to be restarted"
|
||||
count=$((count + 1))
|
||||
if [ "$got" != "$path changed after gitea-runner-swarm.service last started — restarting it" ]; then fail "logged: $got"; fi
|
||||
|
||||
STARTED=
|
||||
check "a unit never active is restarted" \
|
||||
"try-restart --no-block gitea-runner-swarm.service" "" \
|
||||
UNIT_ACTIVE=0
|
||||
check "a unit never active is not asked to restart" "" "" \
|
||||
refresh_unit gitea-runner-swarm.service "$path"
|
||||
got=$(refresh_unit gitea-runner-swarm.service "$path" 2>&1)
|
||||
case="a unit never active is not said to be restarted"
|
||||
count=$((count + 1))
|
||||
if [ -n "$got" ]; then fail "logged: $got"; fi
|
||||
UNIT_ACTIVE=1
|
||||
|
||||
ACTIVE=0
|
||||
check "nothing while the container is down" "" "" \
|
||||
|
|
|
|||
Loading…
Reference in a new issue