mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

remove the old snapshot store; tombstone its options

Browse source
This commit is contained in:
atlas 2026-11-09 23:13:18 +01:00 • committed by mara
commit 20d286f06a
1 changed files with 97 additions and 1308 deletions
Download patch file Download diff file Expand all files Collapse all files

216
hive-c0re/src/priv_client.rs
View file

@ -12,10 +12,9 @@ use hive_priv_sock::{
NetworkIsolation, PRIV_SOCK, PrivEvent, PrivRequest, PrivResponse, PrivStream,
};
use hive_types::NixBuilds;
use std::os::fd::{AsRawFd as _, OwnedFd, RawFd};
use std::path::Path;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Interest};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::net::UnixStream;
/// Send a single request to `hive-priv` and return the response.
@ -48,144 +47,6 @@ async fn call_at(sock: &Path, req: &PrivRequest) -> Result<PrivResponse> {
}
}
/// Ancillary-data buffer sized and aligned for one `SCM_RIGHTS`
/// message. `CMSG_SPACE` is not a `const fn`, so the size is a literal
/// with room to spare; the union member supplies the `cmsghdr`
/// alignment `CMSG_FIRSTHDR` requires.
#[repr(C)]
union CmsgSpace {
_align: libc::cmsghdr,
bytes: [u8; 32],
}
/// `sendmsg` `bytes` with `fd` attached as `SCM_RIGHTS`, returning how
/// many bytes were accepted.
///
/// The descriptor rides on this one call — ancillary data cannot be
/// sent separately from payload — so the caller must not have written
/// any of `bytes` beforehand.
fn send_with_fd(sock: RawFd, bytes: &[u8], fd: RawFd) -> std::io::Result<usize> {
const FD_SIZE: usize = std::mem::size_of::<RawFd>();
let mut iov = libc::iovec {
iov_base: bytes.as_ptr().cast::<libc::c_void>().cast_mut(),
iov_len: bytes.len(),
};
let mut cmsg = CmsgSpace { bytes: [0; 32] };
// SAFETY: msghdr is a plain C struct with no invalid bit patterns;
// every field we rely on is set immediately below.
let mut msg: libc::msghdr = unsafe { std::mem::zeroed() };
msg.msg_iov = &raw mut iov;
msg.msg_iovlen = 1;
msg.msg_control = std::ptr::addr_of_mut!(cmsg.bytes).cast();
// SAFETY: CMSG_SPACE is a pure size computation.
let space = unsafe { libc::CMSG_SPACE(u32::try_from(FD_SIZE).unwrap_or(4)) };
msg.msg_controllen = space as _;
// SAFETY: the control buffer is live, aligned, and long enough for
// the header CMSG_SPACE just sized.
let hdr = unsafe { libc::CMSG_FIRSTHDR(&raw const msg) };
if hdr.is_null() {
return Err(std::io::Error::other(
"control buffer too small for SCM_RIGHTS",
));
}
// SAFETY: CMSG_LEN is a pure size computation; `hdr` points into
// our own buffer, and write_unaligned tolerates its alignment.
unsafe {
let len = libc::CMSG_LEN(u32::try_from(FD_SIZE).unwrap_or(4));
std::ptr::write_unaligned(
hdr,
libc::cmsghdr {
cmsg_len: len as _,
cmsg_level: libc::SOL_SOCKET,
cmsg_type: libc::SCM_RIGHTS,
},
);
// Copied in byte-wise: the control buffer is only cmsghdr-
// aligned, so casting CMSG_DATA to a *mut RawFd would be
// unsound even where it happens to work.
std::ptr::copy_nonoverlapping(
std::ptr::from_ref(&fd).cast::<u8>(),
libc::CMSG_DATA(hdr),
FD_SIZE,
);
}
// SAFETY: `msg` points at a live iovec over `bytes` and the control
// buffer we just filled in.
let n = unsafe { libc::sendmsg(sock, &raw const msg, 0) };
if n < 0 {
return Err(std::io::Error::last_os_error());
}
Ok(usize::try_from(n).unwrap_or_default())
}
/// Send a request to `hive-priv` with an open file descriptor attached,
/// and return the response.
///
/// The helper receives the descriptor itself — not a path or an address
/// — so it can act on something it was handed without being told what
/// that thing is or how to reach it. Used for
/// [`PrivRequest::SendAgentSnapshotToFd`], where the descriptor is a
/// socket already connected to a peer hive's snapshot store.
///
/// ⚠️ Takes the descriptor by value and closes it as soon as the kernel
/// has it, *before* awaiting the response. That is not tidiness: a
/// socket stays open until every copy of it is closed, so a caller
/// holding one back would leave the receiving end waiting for an EOF
/// that never comes — `btrfs receive` blocks, and this side reports
/// success for a transfer the peer has not committed. Passing ownership
/// makes that mistake unrepresentable.
///
/// # Errors
///
/// Fails if the socket is unreachable, the descriptor cannot be
/// attached, or hive-priv answers with something other than a terminal
/// event.
pub async fn call_with_fd(req: &PrivRequest, fd: OwnedFd) -> Result<PrivResponse> {
call_with_fd_at(Path::new(PRIV_SOCK), req, fd).await
}
async fn call_with_fd_at(path: &Path, req: &PrivRequest, fd: OwnedFd) -> Result<PrivResponse> {
let mut stream = UnixStream::connect(path)
.await
.context("connect to hive-priv socket (fd-passing)")?;
let line = serde_json::to_string(req).context("serialise PrivRequest")? + "\n";
let bytes = line.as_bytes();
let sock = stream.as_raw_fd();
let raw_fd = fd.as_raw_fd();
let sent = stream
.async_io(Interest::WRITABLE, || send_with_fd(sock, bytes, raw_fd))
.await
.context("send request + descriptor to hive-priv")?;
// The kernel has duplicated the descriptor into hive-priv's queue,
// so our copy has done its job. Close it now, before waiting on the
// response: see the EOF note on this function.
drop(fd);
// A short sendmsg is legal; the descriptor went with the first
// call, so the tail is an ordinary write.
if sent < bytes.len() {
stream
.write_all(&bytes[sent..])
.await
.context("send remainder of request to hive-priv")?;
}
stream.shutdown().await.context("shutdown write half")?;
let mut resp_line = String::new();
BufReader::new(stream)
.read_line(&mut resp_line)
.await
.context("read response from hive-priv")?;
match serde_json::from_str::<PrivEvent>(&resp_line).context("parse PrivResponse")? {
PrivEvent::Done(resp) => Ok(resp),
PrivEvent::Line(_) => bail!("unexpected stream line from non-streaming priv op"),
}
}
/// Send a streaming request to `hive-priv`, calling `on_line` for each
/// `PrivEvent::Line` as it arrives, then returning the terminal
/// `PrivResponse`. Used for long-running ops (`create` / `update`).
@ -727,16 +588,14 @@ fn ok(resp: PrivResponse) -> Result<()> {
mod tests {
use super::{
BindMount, CredentialMount, HiveFacts, NetworkIsolation, NixBuilds, PrivRequest,
PrivResponse, PrivStream, call_at, call_streaming_at, call_with_fd_at, check,
create_container_request, older_hive_priv_refused, parse_qgroup_usage,
resource_limits_request, send_snapshot_request, update_container_request, within,
write_nspawn_flags_request,
PrivResponse, PrivStream, call_at, call_streaming_at, check, create_container_request,
older_hive_priv_refused, parse_qgroup_usage, resource_limits_request,
send_snapshot_request, update_container_request, within, write_nspawn_flags_request,
};
use anyhow::Context as _;
use serde_json::{Value, json};
use std::io::{Read as _, Write as _};
use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd, RawFd};
use std::os::unix::net::{UnixListener, UnixStream};
use std::os::unix::net::UnixListener;
use std::time::Duration;
fn wire(req: &PrivRequest) -> Value {
@ -1043,69 +902,4 @@ mod tests {
assert!(err.to_string().contains("before sending Done"), "{err}");
server.join().expect("server");
}
/// `recvmsg` the request and the one descriptor that rides with it.
fn recv_with_fd(conn: &UnixStream) -> (Vec<u8>, OwnedFd) {
let mut buf = [0_u8; 256];
let mut iov = libc::iovec {
iov_base: buf.as_mut_ptr().cast(),
iov_len: buf.len(),
};
// u64 elements give the control buffer the 8-byte alignment a
// `cmsghdr` read needs; a byte array would be misaligned.
let mut cmsg = [0_u64; 8];
// SAFETY: an all-zero msghdr is valid; every pointer set below
// refers to a buffer that outlives the recvmsg call.
let (n, fd) = unsafe {
let mut msg: libc::msghdr = std::mem::zeroed();
msg.msg_iov = &raw mut iov;
msg.msg_iovlen = 1;
msg.msg_control = cmsg.as_mut_ptr().cast();
msg.msg_controllen = std::mem::size_of_val(&cmsg);
let n = libc::recvmsg(conn.as_raw_fd(), &raw mut msg, 0);
assert!(n > 0, "recvmsg failed");
let hdr = libc::CMSG_FIRSTHDR(&raw const msg);
assert!(!hdr.is_null(), "no ancillary data arrived");
assert_eq!((*hdr).cmsg_level, libc::SOL_SOCKET);
assert_eq!((*hdr).cmsg_type, libc::SCM_RIGHTS);
let fd = std::ptr::read_unaligned(libc::CMSG_DATA(hdr).cast::<RawFd>());
(n, OwnedFd::from_raw_fd(fd))
};
(buf[..usize::try_from(n).expect("positive")].to_vec(), fd)
}
#[tokio::test]
async fn a_descriptor_arrives_with_the_request_and_is_closed_on_our_side() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("priv.sock");
let listener = UnixListener::bind(&path).expect("bind");
let server = std::thread::spawn(move || {
let (mut conn, _) = listener.accept().expect("accept");
let (request, fd) = recv_with_fd(&conn);
let mut rest = String::new();
conn.read_to_string(&mut rest).expect("read to EOF");
conn.write_all(DONE.as_bytes()).expect("reply");
(request, fd)
});
let (mut read_end, write_end) = std::io::pipe().expect("pipe");
let resp = call_with_fd_at(&path, &PrivRequest::DaemonReload, write_end.into())
.await
.expect("answered");
assert!(resp.ok);
let (request, received) = server.join().expect("server");
assert_eq!(request, b"{\"op\":\"daemon_reload\"}\n");
// The descriptor that arrived is the pipe's write end, and our
// copy is already closed: writing through the received one and
// then dropping it is what ends the read.
let mut write_end = std::fs::File::from(received);
write_end.write_all(b"ping").expect("write via received fd");
drop(write_end);
let mut out = String::new();
read_end
.read_to_string(&mut out)
.expect("EOF after both ends closed");
assert_eq!(out, "ping");
}
}