hive-priv: create agent socket dirs on start; drop hyperhive-agents.conf
/etc/tmpfiles.d/hyperhive-agents.conf was a boot-time backstop (#2290) that pre-created every agent's bind sources. The start preamble already creates them for every c0re-driven start, and on this host only hive-c0re starts agent containers. The file was also the reason the socket dir's owner had to be declared there, which is how it spent its life at `0777 root root` whenever the uid could not be resolved (#4742). - hive-priv gains `EnsureAgentSocketDir { name }`, called from `set_nspawn_flags` in every start path. It creates `/run/hive-agent/<name>` `0751 root:root` with mkdirat relative to an O_DIRECTORY|O_NOFOLLOW fd for the parent. An existing entry has to be a directory (fstatat AT_SYMLINK_NOFOLLOW); anything else is refused, and a directory is left alone. hive-c0re's own create_dir_all went: its /run is read-only under ProtectSystem=strict. - The container's `hive-agent-user-migrate` activation chowns that dir to the agent user and sets 0751, the same way it already handles state/ and harness/. It refuses a symlink or non-directory there, since `test -d` and chmod follow links. No host-side passwd parse, and no window where the dir is world-writable. - `/run/hyperhive/agents/<name>` stays created by hive-c0re itself (`ensure_agent_runtime_dir`). It holds the `mcp.sock` that hive-c0re binds as hive-core, so it must not become root- or agent-owned. - The `/run/hive-agent` parent is declared in hive-priv.nix, `0755 root:root`, instead of hive-gateway's hive-core rule. hive-priv is its only writer now, and hive-priv's ReadWritePaths needs it to exist. - The manager start in `ensure_root_agent` now goes through `converge_start_preamble` + `start_with_fallback`. It was a bare start, so after a reboot the manager's bind sources existed only because of the tmpfiles file, and its limits drop-in did not exist at all. - Removed: `sync_tmpfiles`, `agent_uid_gid` / `parse_passwd_uid_gid`, `priv_client::sync_agent_tmpfiles`, `AgentTmpfilesEntry`, the tmpfiles body builder and their tests, plus the three call sites. - Legacy: hive-priv unlinks the file at every start, ignoring ENOENT. `SyncAgentTmpfiles` stays one release as a payload-ignoring variant that does the same unlink and returns Ok, for an older hive-c0re. Salvaged from #4752: the boundary.md correction that nginx only dials, because ProtectSystem=strict makes its /run read-only. Behaviour change: a manual `nixos-container start h-<name>` right after a reboot, before hive-c0re has started that agent, now fails on a missing bind source instead of starting. Closes #4742
This commit is contained in:
parent
e7456a49ff
commit
2252c55df8
21 changed files with 369 additions and 421 deletions
|
|
@ -209,6 +209,20 @@ in
|
|||
if [ -d "/agents/$userName/harness" ]; then
|
||||
chown -hR "$userName:$userName" "/agents/$userName/harness" 2>/dev/null || true
|
||||
fi
|
||||
# The socket dir is bind-mounted from the host, where hive-priv creates
|
||||
# it `0751 root`. The harness binds its sockets here as this user; the
|
||||
# 0751 is what lets hive-c0re and nginx dial them without listing
|
||||
# (docs/trust-boundary/boundary.md, "the per-agent socket dir"). Not
|
||||
# recursive: only the harness writes inside it. `test -d` and `chmod`
|
||||
# follow symlinks, so a link here is refused rather than handed over.
|
||||
socketDir="/run/hive-agent/$userName"
|
||||
if [ -L "$socketDir" ] || { [ -e "$socketDir" ] && [ ! -d "$socketDir" ]; }; then
|
||||
echo "hive-agent-user-migrate: $socketDir is a symlink or not a directory; not handing it to $userName" >&2
|
||||
elif [ -d "$socketDir" ]; then
|
||||
chown -h "$userName:$userName" "$socketDir" \
|
||||
&& chmod 0751 "$socketDir" \
|
||||
|| echo "hive-agent-user-migrate: could not hand $socketDir to $userName; the harness cannot bind its sockets" >&2
|
||||
fi
|
||||
# The proposed-config repo is RW-mounted into the editing agent and
|
||||
# owned by it; hive-c0re only pulls from it. Heal
|
||||
# it to this user too — same as state/harness. In an agent's own
|
||||
|
|
|
|||
|
|
@ -132,6 +132,10 @@ in
|
|||
inherit pkgs self nixosSystem;
|
||||
inherit (pkgs) lib;
|
||||
};
|
||||
module-eval-agent-user = import ./module-eval/agent-user.nix {
|
||||
inherit pkgs self nixosSystem;
|
||||
inherit (pkgs) lib;
|
||||
};
|
||||
module-eval-agent-matrix = import ./module-eval/agent-matrix.nix {
|
||||
inherit pkgs self nixosSystem;
|
||||
inherit (pkgs) lib;
|
||||
|
|
|
|||
|
|
@ -19,12 +19,6 @@ let
|
|||
# listing: `hive-admin` members reach `host.sock` without root, while the
|
||||
# socket's own `0660 hive-admin` gates the connection and the per-agent
|
||||
# subdirs keep their own perms.
|
||||
#
|
||||
# hive-priv writes a tmpfiles.d entry for this same path and cannot read
|
||||
# this binding, so the two are kept in step by hand. Divergence is not
|
||||
# cosmetic: tmpfiles then tries to fchmod a directory hive-priv has no
|
||||
# write access to, the whole `--create` run fails, and a single WARN per
|
||||
# sync is the only symptom.
|
||||
runtimeDirMode = "0751";
|
||||
|
||||
baoDeploy = config.services.hyperhive.deploy.bao;
|
||||
|
|
|
|||
|
|
@ -212,13 +212,6 @@ in
|
|||
# before c0re has run, and pin owner + mode rather than leaving it
|
||||
# to whoever creates the path first.
|
||||
#
|
||||
# /run/hive-agent — per-agent UDS socket dir, written by c0re's
|
||||
# set_nspawn_flags when agents start. Owned by `hive-core` (the
|
||||
# unprivileged coordinator user): c0re does the
|
||||
# `create_dir_all(/run/hive-agent/<name>)` itself, so a root-owned
|
||||
# parent would EACCES on the very first agent create on a fresh host
|
||||
# (hive-priv only chowns the subdir afterwards, it doesn't make it).
|
||||
#
|
||||
# ⚠️ There is deliberately NO rule for /var/lib/hyperhive here. One
|
||||
# used to declare it `0755 root root` and could never win:
|
||||
# `hive-c0re.service` sets `StateDirectory = "hyperhive"` with
|
||||
|
|
@ -228,10 +221,6 @@ in
|
|||
# as someone having changed the mode. c0re's own unit owns that dir;
|
||||
# this module no longer has an opinion about it.
|
||||
systemd.tmpfiles.rules = [
|
||||
# Must stay in step with the identical rule hive-priv generates into
|
||||
# /etc/tmpfiles.d/hyperhive-agents.conf — the two used to declare
|
||||
# different owners for this path.
|
||||
"d /run/hive-agent 0755 hive-core hive-core - -"
|
||||
# The gateway's own config dir — NOT under /var/lib/hyperhive. c0re
|
||||
# writes here, nginx reads here, and neither needs any access to the
|
||||
# other's tree: no shared parent to traverse means no group
|
||||
|
|
|
|||
|
|
@ -20,6 +20,11 @@ let
|
|||
in
|
||||
{
|
||||
config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable {
|
||||
# The parent of every agent socket dir. hive-priv is its only writer
|
||||
# (`EnsureAgentSocketDir`), and its unit lists it in `ReadWritePaths`,
|
||||
# which fails the unit when the path is missing.
|
||||
systemd.tmpfiles.rules = [ "d /run/hive-agent 0755 root root - -" ];
|
||||
|
||||
# Socket unit for hive-priv — the narrow root helper that executes
|
||||
# privileged operations on behalf of hive-c0re. Systemd creates and
|
||||
# holds `/run/hive/priv.sock` before the first connection arrives.
|
||||
|
|
@ -108,8 +113,9 @@ in
|
|||
# hive-priv must write to at runtime. Each is a confirmed hive-priv
|
||||
# write that EROFSes (os error 30) without its carve-out:
|
||||
# /etc/nixos-containers — <container>.conf (bind mounts, nspawn flags)
|
||||
# /etc/tmpfiles.d — sync_tmpfiles' hyperhive-agents.conf write
|
||||
# /run/hive-agent — chown/chmod per-agent socket dirs
|
||||
# /etc/tmpfiles.d — unlinks the legacy hyperhive-agents.conf;
|
||||
# drop once every host has run it
|
||||
# /run/hive-agent — creates per-agent socket dirs
|
||||
# /run/systemd — container@ drop-ins + machined state
|
||||
# /run/lock — nixos-container's create/destroy lock file
|
||||
# /run/hive-ci — register_ci_runner's runner-token write
|
||||
|
|
|
|||
49
nix/module-eval/agent-user.nix
Normal file
49
nix/module-eval/agent-user.nix
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
# `checks.module-eval-agent-user` — see ./lib.nix for the shared
|
||||
# rationale (why this suite exists, naming convention, "evaluates
|
||||
# not executes").
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
self,
|
||||
nixosSystem,
|
||||
}:
|
||||
let
|
||||
inherit
|
||||
(import ./lib.nix {
|
||||
inherit
|
||||
pkgs
|
||||
lib
|
||||
self
|
||||
nixosSystem
|
||||
;
|
||||
})
|
||||
agentWith
|
||||
runGroup
|
||||
;
|
||||
|
||||
# A named agent, so the rendered script carries a name no default supplies.
|
||||
migrate =
|
||||
(agentWith { services.hyperhive.agent.user.name = "iris"; })
|
||||
.system.activationScripts.hive-agent-user-migrate.text;
|
||||
cases = [
|
||||
{
|
||||
# hive-priv creates the host socket dir `0751 root` and never chowns it,
|
||||
# so this activation is the only thing that lets the harness bind there.
|
||||
name = "the agent's activation hands its socket dir to the agent user at 0751";
|
||||
ok =
|
||||
lib.hasInfix "userName=${lib.escapeShellArg "iris"}" migrate
|
||||
&& lib.hasInfix ''socketDir="/run/hive-agent/$userName"'' migrate
|
||||
&& lib.hasInfix ''chown -h "$userName:$userName" "$socketDir"'' migrate
|
||||
&& lib.hasInfix ''chmod 0751 "$socketDir"'' migrate;
|
||||
}
|
||||
{
|
||||
# `test -d` and `chmod` follow symlinks, so without this refusal ahead
|
||||
# of the chown branch a link at the socket dir is chmodded at its target.
|
||||
name = "the agent's activation refuses a symlinked or non-directory socket dir";
|
||||
ok =
|
||||
lib.hasInfix ''if [ -L "$socketDir" ] || { [ -e "$socketDir" ] && [ ! -d "$socketDir" ]; }; then'' migrate
|
||||
&& lib.hasInfix ''elif [ -d "$socketDir" ]; then'' migrate;
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "agent-user" cases
|
||||
Loading…
Reference in a new issue