agents: read the hive's name and domain at runtime, not from the build
Part of #5081 (slice S0-1a). An agent's closure no longer depends on the name or domain of the hive that renders it. - hive-priv writes /etc/hyperhive-hive-facts (HYPERHIVE_HIVE_NAME, HYPERHIVE_HIVE_DOMAIN) into the container from the same WriteNspawnFlags call that writes /etc/hyperhive-bridge-dns. hive-c0re passes the values from its own unit env. hive-priv refuses a value outside [A-Za-z0-9._-]. - A systemd environment generator in the agent (hive-facts.nix) loads the file into the container manager's environment, so every unit sees it, as with the systemd.globalEnvironment entries the render used to emit. It also builds CLAUDE_REMOTE_CONTROL_SESSION_NAME_PREFIX (<hive>-<agent>), which left claude's managed settings. - meta.rs stops forwarding HYPERHIVE_HIVE_{NAME,DOMAIN} and stops setting services.hyperhive.agent.hiveName. The option is removed (both spellings now fail with a removal message). - otel.nix drops the agent's hive= resource label. The swarm collector upserts hive per authenticated receiver (swarm-otel.nix resource/<h>, on both the metrics and logs pipelines), so stored series keep it. The otel first hop, otel enable and the dashboard port are still baked; they are S0-1b.
This commit is contained in:
atlas
mara
parent
80e8361da9
commit
c87a4d2c1a
14 changed files with 303 additions and 127 deletions
|
|
@ -22,10 +22,10 @@ use std::path::{Path, PathBuf};
|
|||
|
||||
use anyhow::{Context as _, Result, anyhow, bail};
|
||||
use hive_priv_sock::{
|
||||
AGENT_PREFIX, AGENT_RUNTIME_ROOT, AGENT_STATE_ROOT, BindMount, CredentialMount, InfraAction,
|
||||
InfraContainer, MEASURE_AGENT_DISK_TIMEOUT, META_DIR, MIGRATE_STAGING_ROOT, NetworkIsolation,
|
||||
PAUSED_MARKER_FILE, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream,
|
||||
PrivStreamLine, SIBLING_CONTAINERS,
|
||||
AGENT_PREFIX, AGENT_RUNTIME_ROOT, AGENT_STATE_ROOT, BindMount, CredentialMount, HiveFacts,
|
||||
InfraAction, InfraContainer, MEASURE_AGENT_DISK_TIMEOUT, META_DIR, MIGRATE_STAGING_ROOT,
|
||||
NetworkIsolation, PAUSED_MARKER_FILE, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse,
|
||||
PrivStream, PrivStreamLine, SIBLING_CONTAINERS,
|
||||
};
|
||||
use tokio::io::{AsyncWriteExt, BufReader};
|
||||
use tokio::net::unix::OwnedWriteHalf;
|
||||
|
|
@ -377,7 +377,8 @@ async fn exec(
|
|||
ref binds,
|
||||
ref isolation,
|
||||
ref load_credentials,
|
||||
} => handle_write_nspawn_flags(container, binds, isolation, load_credentials),
|
||||
ref hive_facts,
|
||||
} => handle_write_nspawn_flags(container, binds, isolation, load_credentials, hive_facts),
|
||||
|
||||
PrivRequest::EnsureAgentSocketDir { ref name } => ensure_agent_socket_dir(name),
|
||||
|
||||
|
|
@ -984,6 +985,7 @@ fn handle_write_nspawn_flags(
|
|||
binds: &[BindMount],
|
||||
isolation: &NetworkIsolation,
|
||||
load_credentials: &[CredentialMount],
|
||||
hive_facts: &HiveFacts,
|
||||
) -> Result<(String, String)> {
|
||||
validate_container_system_name(container)?;
|
||||
for bind in binds {
|
||||
|
|
@ -997,7 +999,8 @@ fn handle_write_nspawn_flags(
|
|||
// uses `:` as the name/path separator.
|
||||
validate_bind_path(&cred.host_path)?;
|
||||
}
|
||||
write_nspawn_flags(container, binds, isolation, load_credentials)?;
|
||||
let hive_facts = render_hive_facts(hive_facts)?;
|
||||
write_nspawn_flags(container, binds, isolation, load_credentials, &hive_facts)?;
|
||||
Ok((String::new(), String::new()))
|
||||
}
|
||||
|
||||
|
|
@ -2870,6 +2873,7 @@ fn write_nspawn_flags(
|
|||
binds: &[BindMount],
|
||||
isolation: &NetworkIsolation,
|
||||
load_credentials: &[CredentialMount],
|
||||
hive_facts: &str,
|
||||
) -> Result<()> {
|
||||
use std::fmt::Write as _;
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
|
|
@ -2950,36 +2954,78 @@ fn write_nspawn_flags(
|
|||
// and which unit does it: `docs/networking/network.md` § *How the isolated
|
||||
// container gets its resolver*.
|
||||
write_bridge_dns_marker(container, isolation)?;
|
||||
// Read by the agent's environment generator at boot
|
||||
// (`nix/agent-modules/hive-facts.nix`).
|
||||
write_container_etc_file_in(&container_rootfs(container), HIVE_FACTS_FILE, hive_facts)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Filename of the in-container DNS marker, in the container's own `/etc`.
|
||||
const BRIDGE_DNS_MARKER: &str = "hyperhive-bridge-dns";
|
||||
|
||||
/// Filename of the hive-facts environment file, in the container's own `/etc`.
|
||||
const HIVE_FACTS_FILE: &str = "hyperhive-hive-facts";
|
||||
|
||||
/// The container's `/` on the host.
|
||||
fn container_rootfs(container: &str) -> PathBuf {
|
||||
PathBuf::from(format!("/var/lib/nixos-containers/{container}"))
|
||||
}
|
||||
|
||||
/// Render [`HiveFacts`] as the systemd environment file the agent loads,
|
||||
/// one `KEY=value` line per present fact.
|
||||
///
|
||||
/// A value is refused unless it is DNS-name characters only, so it can carry
|
||||
/// neither a newline (a second assignment) nor anything an environment file
|
||||
/// or a shell would interpret.
|
||||
fn render_hive_facts(facts: &HiveFacts) -> Result<String> {
|
||||
use std::fmt::Write as _;
|
||||
let mut out = String::new();
|
||||
for (key, value) in [
|
||||
("HYPERHIVE_HIVE_NAME", &facts.name),
|
||||
("HYPERHIVE_HIVE_DOMAIN", &facts.domain),
|
||||
] {
|
||||
let Some(value) = value else { continue };
|
||||
if value.is_empty()
|
||||
|| value.len() > 253
|
||||
|| !value
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'_'))
|
||||
{
|
||||
bail!("invalid {key} value {value:?}: must be 1-253 chars of [A-Za-z0-9._-]");
|
||||
}
|
||||
let _ = writeln!(out, "{key}={value}");
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Write the bridge-DNS marker the `hyperhive-isolated-dns` oneshot keys
|
||||
/// off. The marker file contains just the gateway IP. Always written:
|
||||
/// every container is isolated, so there is no host-netns case that
|
||||
/// wants the marker absent.
|
||||
fn write_bridge_dns_marker(container: &str, isolation: &NetworkIsolation) -> Result<()> {
|
||||
let rootfs = PathBuf::from(format!("/var/lib/nixos-containers/{container}"));
|
||||
write_bridge_dns_marker_in(&rootfs, &isolation.gateway_ip)
|
||||
write_bridge_dns_marker_in(&container_rootfs(container), &isolation.gateway_ip)
|
||||
}
|
||||
|
||||
/// Write `<rootfs>/etc/hyperhive-bridge-dns` without following a symlink.
|
||||
/// Write `<rootfs>/etc/hyperhive-bridge-dns`; see [`write_container_etc_file_in`].
|
||||
fn write_bridge_dns_marker_in(rootfs: &Path, gateway_ip: &str) -> Result<()> {
|
||||
write_container_etc_file_in(rootfs, BRIDGE_DNS_MARKER, &format!("{gateway_ip}\n"))
|
||||
}
|
||||
|
||||
/// Write `<rootfs>/etc/<name>` without following a symlink.
|
||||
///
|
||||
/// Root inside the container owns its `/etc`, and this write runs as host
|
||||
/// root, where an absolute symlink planted in the container resolves against
|
||||
/// the host's `/`. So this opens `etc` with `O_DIRECTORY|O_NOFOLLOW` (a
|
||||
/// symlinked `etc` fails with `ELOOP`) and publishes the marker through a
|
||||
/// symlinked `etc` fails with `ELOOP`) and publishes the file through a
|
||||
/// [`StagedFile`] relative to that fd: whatever the container planted at the
|
||||
/// leaf, symlink or FIFO or device node, is replaced by the rename, never
|
||||
/// written through or opened.
|
||||
///
|
||||
/// On a fresh install the container's `/etc` may not exist yet (nixos-container
|
||||
/// materialises the rootfs on the first start), so this creates it first; the
|
||||
/// dir and the marker persist through that start. Mode 0644: the gateway IP
|
||||
/// is not a secret.
|
||||
fn write_bridge_dns_marker_in(rootfs: &Path, gateway_ip: &str) -> Result<()> {
|
||||
/// dir and the file persist through that start. Mode 0644: nothing written
|
||||
/// here is a secret.
|
||||
fn write_container_etc_file_in(rootfs: &Path, name: &str, contents: &str) -> Result<()> {
|
||||
use std::os::unix::fs::OpenOptionsExt as _;
|
||||
|
||||
let etc_path = rootfs.join("etc");
|
||||
|
|
@ -3014,14 +3060,7 @@ fn write_bridge_dns_marker_in(rootfs: &Path, gateway_ip: &str) -> Result<()> {
|
|||
}
|
||||
// SAFETY: `etc_fd` is a fresh, valid fd that nothing else owns.
|
||||
let etc = std::fs::File::from(unsafe { OwnedFd::from_raw_fd(etc_fd) });
|
||||
StagedFile::write(
|
||||
&etc,
|
||||
&etc_path,
|
||||
BRIDGE_DNS_MARKER,
|
||||
format!("{gateway_ip}\n").as_bytes(),
|
||||
0o644,
|
||||
)?
|
||||
.publish()
|
||||
StagedFile::write(&etc, &etc_path, name, contents.as_bytes(), 0o644)?.publish()
|
||||
}
|
||||
|
||||
/// Written by older hive-priv builds. Nothing writes it now, but left on disk
|
||||
|
|
@ -3124,13 +3163,14 @@ fn ensure_socket_dir_in(root: &Path, name: &str) -> Result<()> {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
BindMount, BoundedRun, Command, OwnedFd, PAUSED_MARKER_FILE, PrivRequest, StagedFile,
|
||||
agent_disk_paths, check_fd_agreement, contains_secret_shaped_run, describe_forge_admin,
|
||||
du_command, du_total, ensure_plain_filename, ensure_socket_dir_in, exec_measure_agent_disk,
|
||||
git_overlay_flags, limits_dropin_body, open_dir, open_export_dest, parse_du_total,
|
||||
partial_name, publish_file, redact_secret_line, remove_marker_in, run_bounded,
|
||||
single_output_path, toplevel_attr, validate_credential_name, validate_snapshot_name,
|
||||
write_agent_dir_file, write_bridge_dns_marker_in,
|
||||
BindMount, BoundedRun, Command, HiveFacts, OwnedFd, PAUSED_MARKER_FILE, PrivRequest,
|
||||
StagedFile, agent_disk_paths, check_fd_agreement, contains_secret_shaped_run,
|
||||
describe_forge_admin, du_command, du_total, ensure_plain_filename, ensure_socket_dir_in,
|
||||
exec_measure_agent_disk, git_overlay_flags, limits_dropin_body, open_dir, open_export_dest,
|
||||
parse_du_total, partial_name, publish_file, redact_secret_line, remove_marker_in,
|
||||
render_hive_facts, run_bounded, single_output_path, toplevel_attr,
|
||||
validate_credential_name, validate_snapshot_name, write_agent_dir_file,
|
||||
write_bridge_dns_marker_in,
|
||||
};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
|
|
@ -3782,6 +3822,45 @@ mod tests {
|
|||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
/// Each present fact is one environment-file line; an absent one writes
|
||||
/// nothing, so the agent sees the variable unset rather than empty.
|
||||
#[test]
|
||||
fn hive_facts_render_one_line_per_present_fact() {
|
||||
let both = HiveFacts {
|
||||
name: Some("pr1ma".to_owned()),
|
||||
domain: Some("pr1ma.darkest.space".to_owned()),
|
||||
};
|
||||
assert_eq!(
|
||||
render_hive_facts(&both).unwrap(),
|
||||
"HYPERHIVE_HIVE_NAME=pr1ma\nHYPERHIVE_HIVE_DOMAIN=pr1ma.darkest.space\n"
|
||||
);
|
||||
let domain_only = HiveFacts {
|
||||
name: None,
|
||||
domain: Some("h1.example.test".to_owned()),
|
||||
};
|
||||
assert_eq!(
|
||||
render_hive_facts(&domain_only).unwrap(),
|
||||
"HYPERHIVE_HIVE_DOMAIN=h1.example.test\n"
|
||||
);
|
||||
assert_eq!(render_hive_facts(&HiveFacts::default()).unwrap(), "");
|
||||
}
|
||||
|
||||
/// A value that could end its line or be read as syntax is refused, not
|
||||
/// escaped: a newline would add an assignment of the caller's choosing.
|
||||
#[test]
|
||||
fn hive_facts_refuse_values_outside_dns_characters() {
|
||||
for bad in ["", "pr1ma\nLD_PRELOAD=/x", "a b", "a\"b", "$(x)"] {
|
||||
let facts = HiveFacts {
|
||||
name: Some(bad.to_owned()),
|
||||
domain: None,
|
||||
};
|
||||
assert!(
|
||||
render_hive_facts(&facts).is_err(),
|
||||
"{bad:?} must be refused"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `pid` is gone, or only a zombie awaiting its reaper: either
|
||||
/// way it is no longer running.
|
||||
fn is_dead(pid: &str) -> bool {
|
||||
|
|
|
|||
Loading…
Reference in a new issue