feat(#1930): forward otel headers credential into agent containers via nspawn --load-credential

hive-c0re/src/lifecycle.rs

@@ -3,7 +3,7 @@
 use std::path::Path;
 
 use anyhow::{Context, Result, bail};
-use hive_sh4re::priv_proto::BindMount;
+use hive_sh4re::priv_proto::{BindMount, CredentialMount};
 use tokio::process::Command;
 
 use crate::coordinator::{AgentPaths, HiveEnv};
@@ -1188,6 +1188,42 @@ fn bind_child_agent_dirs(child: &str, binds: &mut Vec<BindMount>) {
     });
 }
 
+/// Hive-wide secrets forwarded into every agent container via nspawn
+/// `--load-credential=<name>:<host_path>`. Currently just the OTEL
+/// auth-header secret, when `services.hyperhive.otel.headersCredential`
+/// is set (surfaced as `HYPERHIVE_OTEL_HEADERS_CREDENTIAL` on hive-c0re's
+/// unit env — the same host option meta.rs reads to inject
+/// `hyperhive.otel.headersCredential`). The inner harness unit reads it
+/// via `LoadCredential=otel-headers` (inherit). The secret never lands in
+/// a bind mount, the nix store, or the generated config.
+///
+/// A configured-but-missing file is skipped with a warning rather than
+/// forwarded (nspawn would refuse to start the container otherwise): a
+/// host-level secret typo shouldn't take down every agent's start; OTEL
+/// just exports without the auth header until the file appears.
+fn hive_load_credentials() -> Vec<CredentialMount> {
+    let mut out = Vec::new();
+    let Ok(path) = std::env::var("HYPERHIVE_OTEL_HEADERS_CREDENTIAL") else {
+        return out;
+    };
+    if path.is_empty() {
+        return out;
+    }
+    if std::path::Path::new(&path).is_file() {
+        out.push(CredentialMount {
+            name: "otel-headers".to_owned(),
+            host_path: path,
+        });
+    } else {
+        tracing::warn!(
+            %path,
+            "HYPERHIVE_OTEL_HEADERS_CREDENTIAL is set but the file is missing; \
+             skipping --load-credential (OTEL will export without the auth header)"
+        );
+    }
+    out
+}
+
 #[allow(
     clippy::too_many_lines,
     reason = "one contiguous nspawn-flag assembly block; the length is the flag \
@@ -1234,6 +1270,10 @@ async fn set_nspawn_flags(
     // is needed here — the bind alone is enough.
     let claude_mount = container_claude_mount(agent_name);
 
+    // Hive-wide secrets forwarded into the container's credential store
+    // (currently just the OTEL auth-header). Same for every agent.
+    let load_creds = hive_load_credentials();
+
     let mut binds: Vec<BindMount> = vec![
         BindMount {
             host_path: runtime_dir.to_string_lossy().into_owned(),
@@ -1367,7 +1407,13 @@ async fn set_nspawn_flags(
                      (bad CIDR? prefix too narrow?); skipping PRIVATE_NETWORK write to \
                      avoid misconfigured isolation"
                 );
-                return crate::priv_client::write_nspawn_flags(container, &binds, None).await;
+                return crate::priv_client::write_nspawn_flags(
+                    container,
+                    &binds,
+                    None,
+                    &load_creds,
+                )
+                .await;
             };
             let Some(gateway_ip) = bridge_gateway_ip(&subnet) else {
                 tracing::warn!(
@@ -1376,7 +1422,13 @@ async fn set_nspawn_flags(
                      skipping PRIVATE_NETWORK write to avoid an isolated container with no \
                      default route or resolver"
                 );
-                return crate::priv_client::write_nspawn_flags(container, &binds, None).await;
+                return crate::priv_client::write_nspawn_flags(
+                    container,
+                    &binds,
+                    None,
+                    &load_creds,
+                )
+                .await;
             };
             tracing::info!(
                 %agent_name, %agent_ip, %gateway_ip, %bridge,
@@ -1393,7 +1445,7 @@ async fn set_nspawn_flags(
     };
 
     // Delegate the actual conf-file rewrite to hive-priv (runs as root).
-    crate::priv_client::write_nspawn_flags(container, &binds, isolation).await
+    crate::priv_client::write_nspawn_flags(container, &binds, isolation, &load_creds).await
 }
 
 /// Build the per-line callback for `create_container_streaming` /

hive-c0re/src/priv_client.rs

@@ -8,8 +8,8 @@
 
 use anyhow::{Context as _, Result, bail};
 use hive_sh4re::priv_proto::{
-    BindMount, InfraAction, InfraContainer, JournalQuery, NetworkIsolation, PRIV_SOCK, PrivEvent,
-    PrivRequest, PrivResponse, PrivStream,
+    BindMount, CredentialMount, InfraAction, InfraContainer, JournalQuery, NetworkIsolation,
+    PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream,
 };
 use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
 use tokio::net::UnixStream;
@@ -179,11 +179,13 @@ pub async fn write_nspawn_flags(
     container: &str,
     binds: &[BindMount],
     isolation: Option<NetworkIsolation>,
+    load_credentials: &[CredentialMount],
 ) -> Result<()> {
     ok(call(&PrivRequest::WriteNspawnFlags {
         container: container.to_owned(),
         binds: binds.to_vec(),
         isolation,
+        load_credentials: load_credentials.to_vec(),
     })
     .await?)
 }

hive-priv/src/main.rs

@@ -21,9 +21,9 @@ use std::path::{Path, PathBuf};
 
 use anyhow::{Context as _, Result, bail};
 use hive_sh4re::priv_proto::{
-    AGENT_PREFIX, AGENT_STATE_ROOT, BindMount, InfraAction, InfraContainer, JournalQuery, META_DIR,
-    NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream, PrivStreamLine,
-    SIBLING_CONTAINERS,
+    AGENT_PREFIX, AGENT_STATE_ROOT, BindMount, CredentialMount, InfraAction, InfraContainer,
+    JournalQuery, META_DIR, NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse,
+    PrivStream, PrivStreamLine, SIBLING_CONTAINERS,
 };
 use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
 use tokio::net::unix::OwnedWriteHalf;
@@ -202,7 +202,8 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String,
             ref container,
             ref binds,
             ref isolation,
-        } => handle_write_nspawn_flags(container, binds, isolation.as_ref()),
+            ref load_credentials,
+        } => handle_write_nspawn_flags(container, binds, isolation.as_ref(), load_credentials),
 
         PrivRequest::WriteResourceLimits {
             ref container,
@@ -323,22 +324,44 @@ async fn container_flake_action(
     }
 }
 
-/// `WriteNspawnFlags` — validate the container + every bind path, then
-/// write the container's nspawn flag overrides.
+/// `WriteNspawnFlags` — validate the container + every bind path + every
+/// credential entry, then write the container's nspawn flag overrides.
 fn handle_write_nspawn_flags(
     container: &str,
     binds: &[BindMount],
     isolation: Option<&NetworkIsolation>,
+    load_credentials: &[CredentialMount],
 ) -> Result<(String, String)> {
     validate_container_system_name(container)?;
     for bind in binds {
         validate_bind_path(&bind.host_path)?;
         validate_bind_path(&bind.container_path)?;
     }
-    write_nspawn_flags(container, binds, isolation)?;
+    for cred in load_credentials {
+        validate_credential_name(&cred.name)?;
+        // Same path rules as binds (absolute, no colon/newline/quote/null):
+        // the colon ban is essential since `--load-credential=name:path`
+        // uses `:` as the name/path separator.
+        validate_bind_path(&cred.host_path)?;
+    }
+    write_nspawn_flags(container, binds, isolation, load_credentials)?;
     Ok((String::new(), String::new()))
 }
 
+/// A systemd credential id must be a short token — restrict to
+/// `[A-Za-z0-9_.-]` so it can't inject extra `--load-credential` argv or
+/// break the `name:path` shape.
+fn validate_credential_name(name: &str) -> Result<()> {
+    if name.is_empty()
+        || !name
+            .bytes()
+            .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'-'))
+    {
+        bail!("invalid credential name {name:?}: must be non-empty [A-Za-z0-9_.-]");
+    }
+    Ok(())
+}
+
 /// `RemoveServiceDropin` — remove the container service's drop-in dir
 /// if present (idempotent).
 fn remove_service_dropin(container: &str) -> Result<(String, String)> {
@@ -1306,6 +1329,7 @@ fn write_nspawn_flags(
     container: &str,
     binds: &[BindMount],
     isolation: Option<&NetworkIsolation>,
+    load_credentials: &[CredentialMount],
 ) -> Result<()> {
     use std::fmt::Write as _;
     let path = format!("/etc/nixos-containers/{container}.conf");
@@ -1350,13 +1374,23 @@ fn write_nspawn_flags(
         out.push_str("LOCAL_ADDRESS6=\n");
         out.push_str("HOST_BRIDGE=\n");
     }
-    let flags: Vec<String> = binds
+    let mut flags: Vec<String> = binds
         .iter()
         .map(|b| {
             let flag = if b.read_only { "--bind-ro" } else { "--bind" };
             format!("{flag}={}:{}", b.host_path, b.container_path)
         })
         .collect();
+    // Credential forwarding: nspawn loads each host secret into the
+    // container's credential store under `<name>`; inner units inherit it
+    // via `LoadCredential=<name>`. Validated (name charset + bind-path
+    // rules) in handle_write_nspawn_flags above.
+    for cred in load_credentials {
+        flags.push(format!(
+            "--load-credential={}:{}",
+            cred.name, cred.host_path
+        ));
+    }
     let flags_joined = flags.join(" ");
     let _ = writeln!(out, "EXTRA_NSPAWN_FLAGS=\"{flags_joined}\"");
     std::fs::write(&path, out).with_context(|| format!("write {path}"))?;

hive-sh4re/src/priv_proto.rs

@@ -177,6 +177,23 @@ pub struct BindMount {
     pub read_only: bool,
 }
 
+/// One credential-forwarding entry for `WriteNspawnFlags`. hive-priv
+/// constructs `--load-credential=<name>:<host_path>` so systemd-nspawn
+/// loads the host secret file into the container's credential store; an
+/// inner unit then reads it via `LoadCredential=<name>` (inherit form).
+/// The secret never lands in a bind mount, the nix store, or the
+/// generated config — only its host path (validated like a bind path)
+/// crosses the wire. Used for the hive-wide OTEL auth-header credential.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CredentialMount {
+    /// systemd credential id (e.g. `otel-headers`); inner units inherit
+    /// it by this name. Restricted to `[A-Za-z0-9_.-]` by hive-priv.
+    pub name: String,
+    /// Host path to the secret file, forwarded via nspawn
+    /// `--load-credential=<name>:<host_path>`.
+    pub host_path: String,
+}
+
 /// Network isolation parameters for `WriteNspawnFlags`. When `Some`,
 /// hive-priv writes `PRIVATE_NETWORK=1` + veth bridge wiring instead
 /// of the default `PRIVATE_NETWORK=0`.
@@ -273,6 +290,12 @@ pub enum PrivRequest {
         /// veth on the specified bridge (`PRIVATE_NETWORK=1`).
         #[serde(default)]
         isolation: Option<NetworkIsolation>,
+        /// Host secrets forwarded into the container's credential store via
+        /// nspawn `--load-credential=<name>:<host_path>`. Empty for agents
+        /// with no credentials configured (the common case). `#[serde(default)]`
+        /// so a hive-priv built before this field deserialises new requests.
+        #[serde(default)]
+        load_credentials: Vec<CredentialMount>,
     },
 
     /// Write `/run/systemd/system/container@<container>.service.d/hyperhive-limits.conf`

nix/modules/hive-c0re.nix

@@ -236,11 +236,14 @@ in
       description = ''
         Absolute path to an operator-provided secret file whose contents
         become `OTEL_EXPORTER_OTLP_HEADERS` (e.g.
-        `Authorization=Bearer <token>`). Loaded via systemd
-        `LoadCredential` into each agent's unit-private credential store
-        at runtime, so the token is never copied into the nix store or
-        exposed in argv. Must be absolute. Leave null if the endpoint
-        needs no auth header.
+        `Authorization=Bearer <token>`). hive-c0re forwards this host
+        file into each agent container's credential store via
+        systemd-nspawn `--load-credential=otel-headers:<path>`; the inner
+        harness unit inherits it by name (`LoadCredential`), so the token
+        is never copied into the nix store, the generated config, a bind
+        mount, or argv. Must be absolute. Leave null if the endpoint
+        needs no auth header. A configured-but-missing file is skipped
+        with a log warning (OTEL still exports, without the auth header).
       '';
     };
 

nix/templates/harness-base.nix

@@ -1786,7 +1786,13 @@ in
           Group = userName;
         }
         // lib.optionalAttrs (otel.enable && otel.headersCredential != null) {
-          LoadCredential = [ "otel-headers:${otel.headersCredential}" ];
+          # Inherit form (no `:path`): hive-c0re forwards the host file at
+          # `headersCredential` into this container's credential store via
+          # nspawn `--load-credential=otel-headers:<host path>` (see
+          # lifecycle.rs::hive_load_credentials). The path isn't reachable
+          # from inside the container, so we inherit the already-loaded
+          # credential by name rather than re-reading the host path here.
+          LoadCredential = [ "otel-headers" ];
         };
       };