report per agent whether a hive can pull its latest closure from the cache
for each agent the hive looks up its record in agent-closures, fetches the closure's narinfo from the cache with the pull token and checks that a signature by the cache's public key covers it. the result is a log line and the gauge hyperhive.agent.closure_pullable, with a reason label when it is 0. nothing is downloaded and deploys still build locally. part 3 of 3: the probe needs the cache facts and the closure read grant from parts 1 and 2.
This commit is contained in:
atlas
mara
parent
a55d55e494
commit
5486fc0333
7 changed files with 812 additions and 34 deletions
|
|
@ -48,6 +48,7 @@ listenfd = "1"
|
|||
petgraph.workspace = true
|
||||
hmac.workspace = true
|
||||
sha2.workspace = true
|
||||
ed25519-dalek.workspace = true
|
||||
rusqlite.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
//! The swarm's binary cache, as this hive sees it: the pull credentials it is
|
||||
//! given ([`reader`]).
|
||||
//! given ([`reader`]), the narinfo check ([`narinfo`]), and the probe that
|
||||
//! reports per agent whether the latest closure could be pulled ([`probe`]).
|
||||
|
||||
pub mod narinfo;
|
||||
pub mod probe;
|
||||
pub mod reader;
|
||||
|
|
|
|||
303
hive-c0re/src/attic/narinfo.rs
Normal file
303
hive-c0re/src/attic/narinfo.rs
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
//! A closure's narinfo in the binary cache: fetching it with the pull token,
|
||||
//! and checking its signature against the cache's public key.
|
||||
//!
|
||||
//! The fingerprint is Nix's: `1;<store path>;<nar hash>;<nar size>;<references
|
||||
//! as full paths, comma-separated>`, signed with ed25519. A key is
|
||||
//! `<name>:<base64 of 32 bytes>`, a signature `<name>:<base64 of 64 bytes>`.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context as _, Result, anyhow, bail};
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use ed25519_dalek::{Signature, Verifier as _, VerifyingKey};
|
||||
|
||||
use super::reader::Facts;
|
||||
|
||||
/// How long one narinfo request may take.
|
||||
const FETCH_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
|
||||
/// A client for the cache: no redirects, since the token rides in a header.
|
||||
///
|
||||
/// # Errors
|
||||
/// The TLS roots cannot be loaded.
|
||||
pub fn client() -> Result<reqwest::Client> {
|
||||
Ok(reqwest::Client::builder()
|
||||
.timeout(FETCH_TIMEOUT)
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()?)
|
||||
}
|
||||
|
||||
/// The narinfo of `store_path` from the cache `facts` name.
|
||||
///
|
||||
/// `Ok(None)` is the cache saying it does not hold the path (a 404).
|
||||
///
|
||||
/// # Errors
|
||||
/// A request that failed, or any status but 200 and 404. The message names the
|
||||
/// status, never the token.
|
||||
pub async fn fetch(
|
||||
client: &reqwest::Client,
|
||||
facts: &Facts,
|
||||
store_path: &str,
|
||||
) -> Result<Option<String>> {
|
||||
let hash = path_hash(store_path)?;
|
||||
let url = format!("{}/{hash}.narinfo", facts.substituter());
|
||||
let response = client
|
||||
.get(&url)
|
||||
.basic_auth("hive", Some(facts.token()))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| anyhow!("GET {url}: {}", e.without_url()))?;
|
||||
match response.status() {
|
||||
reqwest::StatusCode::OK => {
|
||||
Ok(Some(response.text().await.map_err(|e| {
|
||||
anyhow!("reading {url}: {}", e.without_url())
|
||||
})?))
|
||||
}
|
||||
reqwest::StatusCode::NOT_FOUND => Ok(None),
|
||||
status => bail!("GET {url}: {status}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The 32-character hash a narinfo is named by: `/nix/store/<hash>-<name>`.
|
||||
fn path_hash(store_path: &str) -> Result<&str> {
|
||||
let base = store_path.rsplit('/').next().unwrap_or_default();
|
||||
match base.split_once('-') {
|
||||
Some((hash, name)) if hash.len() == 32 && !name.is_empty() => Ok(hash),
|
||||
_ => bail!("{store_path:?} is not a store path"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `narinfo` describes `store_path` and carries a signature by
|
||||
/// `public_key` over it.
|
||||
///
|
||||
/// # Errors
|
||||
/// A narinfo that is malformed, describes another path, or has no valid
|
||||
/// signature by the key.
|
||||
pub fn verify(narinfo: &str, store_path: &str, public_key: &str) -> Result<()> {
|
||||
let field = |name: &str| {
|
||||
narinfo
|
||||
.lines()
|
||||
.find_map(|l| l.strip_prefix(name).and_then(|r| r.strip_prefix(": ")))
|
||||
};
|
||||
let path = field("StorePath").context("no StorePath")?;
|
||||
if path != store_path {
|
||||
bail!("the narinfo describes {path}, not {store_path}");
|
||||
}
|
||||
let hash = nar_hash_nix32(field("NarHash").context("no NarHash")?)?;
|
||||
let size: u64 = field("NarSize")
|
||||
.context("no NarSize")?
|
||||
.parse()
|
||||
.context("NarSize is not a number")?;
|
||||
let dir = store_path.rsplit_once('/').map_or("", |(d, _)| d);
|
||||
let mut refs: Vec<&str> = field("References")
|
||||
.unwrap_or_default()
|
||||
.split_whitespace()
|
||||
.collect();
|
||||
refs.sort_unstable();
|
||||
let refs: Vec<String> = refs.iter().map(|r| format!("{dir}/{r}")).collect();
|
||||
let fingerprint = format!("1;{store_path};{hash};{size};{}", refs.join(","));
|
||||
|
||||
let (key_name, key_b64) = public_key
|
||||
.split_once(':')
|
||||
.context("the public key has no name")?;
|
||||
let key = VerifyingKey::from_bytes(
|
||||
&STANDARD
|
||||
.decode(key_b64)?
|
||||
.try_into()
|
||||
.map_err(|_| anyhow!("the public key is not 32 bytes"))?,
|
||||
)?;
|
||||
for sig in narinfo.lines().filter_map(|l| l.strip_prefix("Sig: ")) {
|
||||
let Some((name, b64)) = sig.split_once(':') else {
|
||||
continue;
|
||||
};
|
||||
if name != key_name {
|
||||
continue;
|
||||
}
|
||||
let Ok(bytes) = STANDARD.decode(b64) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(signature) = Signature::from_slice(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
if key.verify(fingerprint.as_bytes(), &signature).is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
bail!("no signature by {key_name} verifies over the fingerprint")
|
||||
}
|
||||
|
||||
/// `sha256:<hash>` as Nix prints it: the hash in its base-32 form. A narinfo
|
||||
/// carries it that way; hex is accepted too.
|
||||
fn nar_hash_nix32(value: &str) -> Result<String> {
|
||||
let digest = value
|
||||
.strip_prefix("sha256:")
|
||||
.context("NarHash is not sha256")?;
|
||||
match digest.len() {
|
||||
52 => Ok(value.to_owned()),
|
||||
64 => {
|
||||
let bytes = (0..32)
|
||||
.map(|i| u8::from_str_radix(&digest[2 * i..2 * i + 2], 16))
|
||||
.collect::<Result<Vec<u8>, _>>()
|
||||
.context("NarHash is not hex")?;
|
||||
Ok(format!("sha256:{}", nix32(&bytes)))
|
||||
}
|
||||
_ => bail!("NarHash is in a form this check does not read"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Nix's base-32: its own alphabet, least significant bits first.
|
||||
fn nix32(bytes: &[u8]) -> String {
|
||||
const ALPHABET: &[u8; 32] = b"0123456789abcdfghijklmnpqrsvwxyz";
|
||||
let len = (bytes.len() * 8 - 1) / 5 + 1;
|
||||
(0..len)
|
||||
.rev()
|
||||
.map(|n| {
|
||||
let (i, j) = (n * 5 / 8, n * 5 % 8);
|
||||
let lo = u16::from(bytes[i]) >> j;
|
||||
let hi = bytes.get(i + 1).map_or(0, |b| u16::from(*b) << (8 - j));
|
||||
char::from(ALPHABET[usize::from((lo | hi) & 0x1f)])
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use ed25519_dalek::{Signer as _, SigningKey};
|
||||
|
||||
use super::*;
|
||||
|
||||
const PATH: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas";
|
||||
const DEP: &str = "ffffffffffffffffffffffffffffffff-glibc";
|
||||
const HASH: &str = "sha256:0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73";
|
||||
|
||||
fn key(seed: u8) -> SigningKey {
|
||||
SigningKey::from_bytes(&[seed; 32])
|
||||
}
|
||||
|
||||
fn public(name: &str, key: &SigningKey) -> String {
|
||||
format!("{name}:{}", STANDARD.encode(key.verifying_key().to_bytes()))
|
||||
}
|
||||
|
||||
/// A narinfo signed by `key` under `name` over the fingerprint of `PATH`.
|
||||
fn signed(name: &str, key: &SigningKey) -> String {
|
||||
let fingerprint = format!("1;{PATH};{HASH};120;/nix/store/{DEP}");
|
||||
let sig = STANDARD.encode(key.sign(fingerprint.as_bytes()).to_bytes());
|
||||
format!(
|
||||
"StorePath: {PATH}\nURL: nar/x.nar\nCompression: none\nNarHash: {HASH}\n\
|
||||
NarSize: 120\nReferences: {DEP}\nSig: {name}:{sig}\n"
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nix32_matches_nixs_own_encoding() {
|
||||
let empty_sha256 = [
|
||||
0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f,
|
||||
0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b,
|
||||
0x78, 0x52, 0xb8, 0x55,
|
||||
];
|
||||
assert_eq!(
|
||||
format!("sha256:{}", nix32(&empty_sha256)),
|
||||
HASH,
|
||||
"from `nix-hash --to-base32`"
|
||||
);
|
||||
let hex = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
||||
assert_eq!(nar_hash_nix32(&format!("sha256:{hex}")).expect("hex"), HASH);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_signature_by_the_key_verifies() {
|
||||
let k = key(7);
|
||||
verify(&signed("swarm", &k), PATH, &public("swarm", &k)).expect("verifies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_signature_by_another_key_or_over_other_content_does_not() {
|
||||
let (ours, theirs) = (key(7), key(9));
|
||||
let err = verify(&signed("swarm", &theirs), PATH, &public("swarm", &ours));
|
||||
assert!(err.is_err(), "a stranger's signature under our key's name");
|
||||
// The key's name must match a Sig line's, whatever the bytes.
|
||||
let err = verify(&signed("other", &ours), PATH, &public("swarm", &ours));
|
||||
assert!(err.is_err(), "a signature under another name");
|
||||
let tampered = signed("swarm", &ours).replace("NarSize: 120", "NarSize: 121");
|
||||
assert!(verify(&tampered, PATH, &public("swarm", &ours)).is_err());
|
||||
let unsigned = signed("swarm", &ours).replace("Sig: ", "Sig-ignored: ");
|
||||
assert!(verify(&unsigned, PATH, &public("swarm", &ours)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_narinfo_for_another_path_is_refused_even_when_signed() {
|
||||
let k = key(7);
|
||||
let other = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-other";
|
||||
assert!(verify(&signed("swarm", &k), other, &public("swarm", &k)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_narinfo_is_named_by_the_paths_hash() {
|
||||
assert_eq!(
|
||||
path_hash(PATH).expect("a store path"),
|
||||
"0123456789abcdfghijklmnpqrsvwxyz"
|
||||
);
|
||||
assert!(path_hash("/nix/store/short-name").is_err());
|
||||
assert!(path_hash("nonsense").is_err());
|
||||
}
|
||||
|
||||
/// A cache on loopback that answers a narinfo only to this basic-auth
|
||||
/// password, 404 for another path, and 500 for a third.
|
||||
async fn serve() -> (String, tokio::task::JoinHandle<()>) {
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
let app = axum::Router::new().route(
|
||||
"/swarm/{file}",
|
||||
axum::routing::get(|axum::extract::Path(file): axum::extract::Path<String>, headers: HeaderMap| async move {
|
||||
let expected = format!("Basic {}", STANDARD.encode("hive:pull-secret"));
|
||||
let ok = headers.get("authorization").is_some_and(|v| *v == *expected);
|
||||
match (ok, file.as_str()) {
|
||||
(false, _) => (StatusCode::UNAUTHORIZED, String::new()),
|
||||
(true, "0123456789abcdfghijklmnpqrsvwxyz.narinfo") => (StatusCode::OK, "narinfo-body".to_owned()),
|
||||
(true, "ffffffffffffffffffffffffffffffff.narinfo") => (StatusCode::INTERNAL_SERVER_ERROR, String::new()),
|
||||
(true, _) => (StatusCode::NOT_FOUND, String::new()),
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind");
|
||||
let base = format!("http://{}", listener.local_addr().expect("addr"));
|
||||
(
|
||||
base,
|
||||
tokio::spawn(async move { axum::serve(listener, app).await.expect("serve") }),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_fetch_sends_the_token_and_tells_missing_from_failed() {
|
||||
let (base, server) = serve().await;
|
||||
// No system roots in the build sandbox; the requests are plain HTTP.
|
||||
let client = reqwest::Client::builder()
|
||||
.tls_certs_only([])
|
||||
.build()
|
||||
.expect("client");
|
||||
let with = |token: &str| Facts::unchecked_for_test(token, &base, "swarm", "swarm:AAAA");
|
||||
|
||||
let found = fetch(&client, &with("pull-secret"), PATH).await;
|
||||
let missing = fetch(
|
||||
&client,
|
||||
&with("pull-secret"),
|
||||
"/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-x",
|
||||
)
|
||||
.await;
|
||||
let broken = fetch(&client, &with("pull-secret"), &format!("/nix/store/{DEP}")).await;
|
||||
let refused = fetch(&client, &with("wrong"), PATH).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(found.expect("200").as_deref(), Some("narinfo-body"));
|
||||
assert!(missing.expect("404 is not an error").is_none());
|
||||
assert!(broken.expect_err("500").to_string().contains("500"));
|
||||
let refused = refused.expect_err("401");
|
||||
assert!(refused.to_string().contains("401"));
|
||||
assert!(
|
||||
!refused.to_string().contains("wrong"),
|
||||
"the token stays out of the message"
|
||||
);
|
||||
}
|
||||
}
|
||||
396
hive-c0re/src/attic/probe.rs
Normal file
396
hive-c0re/src/attic/probe.rs
Normal file
|
|
@ -0,0 +1,396 @@
|
|||
//! Whether this hive could pull the latest closure of each of its agents from
|
||||
//! the swarm's binary cache, and the answer as a log line and a gauge.
|
||||
//!
|
||||
//! Every [`PASS_INTERVAL`] each agent's record in `agent-closures` is looked up
|
||||
//! and its narinfo fetched with the pull token and checked against the cache's
|
||||
//! public key. Nothing is downloaded and nothing is deployed from it.
|
||||
//!
|
||||
//! The verdict is the first of these that fails, in this order:
|
||||
//! no credentials (`no-credentials`), no record (`no-record`), a record built
|
||||
//! from another `internal/meta` commit than the head (`rev-mismatch`), a record
|
||||
//! the builder did not push (`not-cached`), the cache not holding the narinfo
|
||||
//! (`not-cached`), a request that failed or was refused (`fetch-failed`), a
|
||||
//! narinfo no signature of the key covers (`bad-signature`).
|
||||
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use swarm_queue_client::agent_closures::{self, Read};
|
||||
use swarm_wire::agent_closures::AgentClosure;
|
||||
|
||||
use super::narinfo;
|
||||
use super::reader::{self, Facts};
|
||||
use crate::stats::sweep_health::{self, SweepHealth};
|
||||
|
||||
/// How often every agent is probed.
|
||||
pub const PASS_INTERVAL: Duration = Duration::from_mins(5);
|
||||
|
||||
/// At [`PASS_INTERVAL`] this is ~15 minutes of failing passes.
|
||||
const FAILURES_BEFORE_WARNING: u32 = 3;
|
||||
|
||||
/// Why a closure cannot be pulled; the gauge's `reason` label.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Reason {
|
||||
NoRecord,
|
||||
RevMismatch,
|
||||
NotCached,
|
||||
NoCredentials,
|
||||
FetchFailed,
|
||||
BadSignature,
|
||||
}
|
||||
|
||||
impl Reason {
|
||||
/// The label value.
|
||||
pub const fn label(self) -> &'static str {
|
||||
match self {
|
||||
Reason::NoRecord => "no-record",
|
||||
Reason::RevMismatch => "rev-mismatch",
|
||||
Reason::NotCached => "not-cached",
|
||||
Reason::NoCredentials => "no-credentials",
|
||||
Reason::FetchFailed => "fetch-failed",
|
||||
Reason::BadSignature => "bad-signature",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A closure that cannot be pulled, and what the log line says about it.
|
||||
#[derive(Debug)]
|
||||
pub struct Unpullable {
|
||||
pub reason: Reason,
|
||||
pub detail: String,
|
||||
}
|
||||
|
||||
fn no(reason: Reason, detail: impl Into<String>) -> Unpullable {
|
||||
Unpullable {
|
||||
reason,
|
||||
detail: detail.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The cache's narinfo for a store path, so a test can stand in for it.
|
||||
pub(crate) trait Cache {
|
||||
/// `None` when the cache does not hold the path.
|
||||
async fn narinfo(&self, facts: &Facts, store_path: &str) -> Result<Option<String>>;
|
||||
}
|
||||
|
||||
struct Http(reqwest::Client);
|
||||
|
||||
impl Cache for Http {
|
||||
async fn narinfo(&self, facts: &Facts, store_path: &str) -> Result<Option<String>> {
|
||||
narinfo::fetch(&self.0, facts, store_path).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the closure `record` names can be pulled with `facts`. `head` is the
|
||||
/// `internal/meta` commit a deploy would build from.
|
||||
pub(crate) async fn judge(
|
||||
facts: Option<&Facts>,
|
||||
record: Option<&AgentClosure>,
|
||||
head: Option<&str>,
|
||||
cache: &impl Cache,
|
||||
) -> Result<(), Unpullable> {
|
||||
let facts = facts.ok_or_else(|| {
|
||||
no(
|
||||
Reason::NoCredentials,
|
||||
"the swarm has given this hive no cache facts",
|
||||
)
|
||||
})?;
|
||||
let record =
|
||||
record.ok_or_else(|| no(Reason::NoRecord, "no build is recorded for the agent"))?;
|
||||
if head != Some(record.meta_rev.as_str()) {
|
||||
return Err(no(
|
||||
Reason::RevMismatch,
|
||||
format!(
|
||||
"built from {}, internal/meta is at {}",
|
||||
record.meta_rev,
|
||||
head.unwrap_or("nothing")
|
||||
),
|
||||
));
|
||||
}
|
||||
if !record.cached {
|
||||
return Err(no(
|
||||
Reason::NotCached,
|
||||
"the builder did not push the closure",
|
||||
));
|
||||
}
|
||||
let text = match cache.narinfo(facts, &record.store_path).await {
|
||||
Ok(Some(text)) => text,
|
||||
Ok(None) => return Err(no(Reason::NotCached, "the cache does not hold the narinfo")),
|
||||
Err(e) => return Err(no(Reason::FetchFailed, format!("{e:#}"))),
|
||||
};
|
||||
narinfo::verify(&text, &record.store_path, facts.public_key())
|
||||
.map_err(|e| no(Reason::BadSignature, format!("{e:#}")))
|
||||
}
|
||||
|
||||
/// The last pass's verdict per agent: `None` when it can be pulled.
|
||||
static VERDICTS: Mutex<Vec<(String, Option<Reason>)>> = Mutex::new(Vec::new());
|
||||
|
||||
/// The verdicts of the last pass, for the gauge.
|
||||
pub fn verdicts() -> Vec<(String, Option<Reason>)> {
|
||||
VERDICTS.lock().map(|g| g.clone()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Probe every agent now and every [`PASS_INTERVAL`] until shutdown.
|
||||
pub async fn follow(client: async_nats::Client, mut shutdown: tokio::sync::watch::Receiver<bool>) {
|
||||
let mut health = SweepHealth::new("attic_pull_probe", "warn", FAILURES_BEFORE_WARNING);
|
||||
loop {
|
||||
match pass(&client).await {
|
||||
Ok(()) => health.record_ok(),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %format!("{e:#}"), "attic: the pull probe failed; the last verdicts stand");
|
||||
let err = format!("{e:#}");
|
||||
health.record_err(|ctx| {
|
||||
let age = ctx.since_last_ok.map_or_else(
|
||||
|| "no success this session".to_owned(),
|
||||
|d| format!("last ok {} ago", sweep_health::fmt_age(d)),
|
||||
);
|
||||
format!(
|
||||
"checking whether agents' closures can be pulled is failing ({} consecutive, {age}) \
|
||||
— deploys are unaffected: {err}",
|
||||
ctx.consecutive
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(PASS_INTERVAL) => {}
|
||||
_ = shutdown.changed() => {
|
||||
tracing::info!("attic: shutdown signal received");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One pass: publish the facts, then judge every agent.
|
||||
///
|
||||
/// # Errors
|
||||
/// A read that failed (the store, the queue, the forge), which leaves the last
|
||||
/// verdicts as they are: the failure says nothing about any agent.
|
||||
async fn pass(client: &async_nats::Client) -> Result<()> {
|
||||
swarm_queue_client::ensure_connected(client)?;
|
||||
let facts = reader::sync().await?;
|
||||
let agents = crate::lifecycle::agents_for_meta_listing()
|
||||
.await
|
||||
.context("enumerating this hive's agents")?;
|
||||
let head = match &facts {
|
||||
Some(_) => meta_head().await?,
|
||||
None => None,
|
||||
};
|
||||
let http = Http(narinfo::client()?);
|
||||
let mut now = Vec::with_capacity(agents.len());
|
||||
for spec in agents {
|
||||
let record = match &facts {
|
||||
Some(_) => read_record(client, &spec.name).await?,
|
||||
None => None,
|
||||
};
|
||||
let verdict = judge(facts.as_ref(), record.as_ref(), head.as_deref(), &http).await;
|
||||
now.push((spec.name, verdict.as_ref().err().map(|u| u.reason)));
|
||||
report(&now[now.len() - 1], verdict.err().map(|u| u.detail));
|
||||
}
|
||||
if let Ok(mut g) = VERDICTS.lock() {
|
||||
*g = now;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_record(client: &async_nats::Client, agent: &str) -> Result<Option<AgentClosure>> {
|
||||
match agent_closures::read(client, agent)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!(e))
|
||||
.with_context(|| format!("reading the closure record of {agent}"))?
|
||||
{
|
||||
Read::Current(record) => Ok(record),
|
||||
Read::NoBucket => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// The `internal/meta` head, or `None` when the swarm has not published it or
|
||||
/// this hive has no forge token.
|
||||
async fn meta_head() -> Result<Option<String>> {
|
||||
use crate::meta_rev::MetaSource as _;
|
||||
match crate::forge::internal_meta_source().await {
|
||||
Some(source) => source
|
||||
.meta_head()
|
||||
.await
|
||||
.context("reading internal/meta's head"),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// The log line: at info when the verdict differs from the last pass, else
|
||||
/// debug.
|
||||
fn report((agent, verdict): &(String, Option<Reason>), detail: Option<String>) {
|
||||
let changed = verdicts()
|
||||
.iter()
|
||||
.find(|(a, _)| a == agent)
|
||||
.is_none_or(|(_, v)| v != verdict);
|
||||
let reason = verdict.map_or("", Reason::label);
|
||||
let pullable = verdict.is_none();
|
||||
let detail = detail.unwrap_or_default();
|
||||
if changed {
|
||||
tracing::info!(%agent, pullable, reason, %detail, "attic: can this hive pull the agent's latest closure");
|
||||
} else {
|
||||
tracing::debug!(%agent, pullable, reason, %detail, "attic: can this hive pull the agent's latest closure");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use ed25519_dalek::{Signer as _, SigningKey};
|
||||
|
||||
use super::*;
|
||||
|
||||
const PATH: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-atlas";
|
||||
const HASH: &str = "sha256:0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73";
|
||||
const REV: &str = "a1b2c3";
|
||||
|
||||
fn key() -> SigningKey {
|
||||
SigningKey::from_bytes(&[7; 32])
|
||||
}
|
||||
|
||||
fn facts() -> Facts {
|
||||
let public = format!(
|
||||
"swarm:{}",
|
||||
STANDARD.encode(key().verifying_key().to_bytes())
|
||||
);
|
||||
Facts::parse("tok", "https://cache.example", "swarm", &public).expect("facts")
|
||||
}
|
||||
|
||||
fn record() -> AgentClosure {
|
||||
AgentClosure {
|
||||
meta_rev: REV.to_owned(),
|
||||
store_path: PATH.to_owned(),
|
||||
system: "x86_64-linux".to_owned(),
|
||||
built_unix: 1,
|
||||
cached: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn narinfo_signed_by(key: &SigningKey) -> String {
|
||||
let sig = key.sign(format!("1;{PATH};{HASH};120;").as_bytes());
|
||||
format!(
|
||||
"StorePath: {PATH}\nNarHash: {HASH}\nNarSize: 120\nReferences: \nSig: swarm:{}\n",
|
||||
STANDARD.encode(sig.to_bytes())
|
||||
)
|
||||
}
|
||||
|
||||
/// What the cache answers for the narinfo.
|
||||
struct Answers(Result<Option<String>, &'static str>);
|
||||
|
||||
impl Cache for Answers {
|
||||
async fn narinfo(&self, _: &Facts, _: &str) -> Result<Option<String>> {
|
||||
self.0.clone().map_err(|e| anyhow::anyhow!(e))
|
||||
}
|
||||
}
|
||||
|
||||
async fn reason(
|
||||
facts: Option<&Facts>,
|
||||
record: Option<&AgentClosure>,
|
||||
head: Option<&str>,
|
||||
cache: Answers,
|
||||
) -> Option<Reason> {
|
||||
judge(facts, record, head, &cache)
|
||||
.await
|
||||
.err()
|
||||
.map(|u| u.reason)
|
||||
}
|
||||
|
||||
fn good() -> Answers {
|
||||
Answers(Ok(Some(narinfo_signed_by(&key()))))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_signed_narinfo_for_the_head_build_is_pullable() {
|
||||
assert_eq!(
|
||||
reason(Some(&facts()), Some(&record()), Some(REV), good()).await,
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_facts_is_no_credentials_even_with_a_record() {
|
||||
let r = reason(None, Some(&record()), Some(REV), good()).await;
|
||||
assert_eq!(r, Some(Reason::NoCredentials));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_record_is_no_record() {
|
||||
let r = reason(Some(&facts()), None, Some(REV), good()).await;
|
||||
assert_eq!(r, Some(Reason::NoRecord));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_record_from_another_commit_is_a_rev_mismatch() {
|
||||
let r = reason(Some(&facts()), Some(&record()), Some("other"), good()).await;
|
||||
assert_eq!(r, Some(Reason::RevMismatch));
|
||||
let r = reason(Some(&facts()), Some(&record()), None, good()).await;
|
||||
assert_eq!(r, Some(Reason::RevMismatch), "no head to compare with");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unpushed_record_or_a_missing_narinfo_is_not_cached() {
|
||||
let unpushed = AgentClosure {
|
||||
cached: false,
|
||||
..record()
|
||||
};
|
||||
let r = reason(Some(&facts()), Some(&unpushed), Some(REV), good()).await;
|
||||
assert_eq!(r, Some(Reason::NotCached));
|
||||
let r = reason(
|
||||
Some(&facts()),
|
||||
Some(&record()),
|
||||
Some(REV),
|
||||
Answers(Ok(None)),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r, Some(Reason::NotCached));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_request_is_fetch_failed() {
|
||||
let r = reason(
|
||||
Some(&facts()),
|
||||
Some(&record()),
|
||||
Some(REV),
|
||||
Answers(Err("401")),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(r, Some(Reason::FetchFailed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_narinfo_the_key_did_not_sign_is_a_bad_signature() {
|
||||
let forged = Answers(Ok(Some(narinfo_signed_by(&SigningKey::from_bytes(
|
||||
&[9; 32],
|
||||
)))));
|
||||
let r = reason(Some(&facts()), Some(&record()), Some(REV), forged).await;
|
||||
assert_eq!(r, Some(Reason::BadSignature));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_labels_are_the_documented_ones() {
|
||||
let labels = [
|
||||
Reason::NoRecord,
|
||||
Reason::RevMismatch,
|
||||
Reason::NotCached,
|
||||
Reason::NoCredentials,
|
||||
Reason::FetchFailed,
|
||||
Reason::BadSignature,
|
||||
]
|
||||
.map(Reason::label);
|
||||
assert_eq!(
|
||||
labels,
|
||||
[
|
||||
"no-record",
|
||||
"rev-mismatch",
|
||||
"not-cached",
|
||||
"no-credentials",
|
||||
"fetch-failed",
|
||||
"bad-signature"
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -6,7 +6,6 @@
|
|||
//! substituter URL and the public key, where nix and the root helper read them.
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context as _, Result, bail};
|
||||
use swarm_secret_client::attic::{Fact, Leaf, hive_path};
|
||||
|
|
@ -64,6 +63,31 @@ impl Facts {
|
|||
format!("{}/{}", self.url, self.cache)
|
||||
}
|
||||
|
||||
pub fn public_key(&self) -> &str {
|
||||
&self.public_key
|
||||
}
|
||||
|
||||
pub fn token(&self) -> &str {
|
||||
&self.token
|
||||
}
|
||||
|
||||
/// Facts for a test that reaches a server on loopback: no scheme check,
|
||||
/// so the one place an `http` URL gets past [`Facts::parse`] is a test.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn unchecked_for_test(
|
||||
token: &str,
|
||||
url: &str,
|
||||
cache: &str,
|
||||
public_key: &str,
|
||||
) -> Self {
|
||||
Self {
|
||||
token: token.to_owned(),
|
||||
url: url.to_owned(),
|
||||
cache: cache.to_owned(),
|
||||
public_key: public_key.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The netrc line the server's host authenticates with. The token is the
|
||||
/// password; the server ignores the login.
|
||||
fn netrc(&self) -> String {
|
||||
|
|
@ -96,9 +120,6 @@ impl Facts {
|
|||
}
|
||||
}
|
||||
|
||||
/// How often [`follow`] reads the facts.
|
||||
pub const READ_INTERVAL: Duration = Duration::from_mins(5);
|
||||
|
||||
/// Read the facts and put them in [`crate::paths::ATTIC_DIR`]. The facts, or
|
||||
/// `None` when the swarm has none for this hive.
|
||||
///
|
||||
|
|
@ -114,20 +135,6 @@ pub async fn sync() -> Result<Option<Facts>> {
|
|||
Ok(facts)
|
||||
}
|
||||
|
||||
/// [`sync`] now and every [`READ_INTERVAL`] until shutdown. A failed pass is
|
||||
/// logged and leaves the files as they are.
|
||||
pub async fn follow(mut shutdown: tokio::sync::watch::Receiver<bool>) {
|
||||
loop {
|
||||
if let Err(e) = sync().await {
|
||||
tracing::warn!(error = %format!("{e:#}"), "attic: reading the cache facts failed");
|
||||
}
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(READ_INTERVAL) => {}
|
||||
_ = shutdown.changed() => return,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// This hive's facts from the store.
|
||||
///
|
||||
/// `Ok(None)` when the swarm has none for it: no cache configured, the
|
||||
|
|
|
|||
|
|
@ -256,9 +256,29 @@ fn register_instruments(provider: &SdkMeterProvider, snapshot: Arc<Mutex<Vec<Con
|
|||
})
|
||||
.build();
|
||||
|
||||
register_closure_pullable(&meter);
|
||||
register_hive_instruments(&meter);
|
||||
}
|
||||
|
||||
/// 1 when this hive can fetch the agent's latest closure from the swarm's
|
||||
/// cache and verify its signature, else 0 with the `reason` it cannot. Nothing
|
||||
/// is reported until the first probe pass has run.
|
||||
fn register_closure_pullable(meter: &opentelemetry::metrics::Meter) {
|
||||
meter
|
||||
.u64_observable_gauge("hyperhive.agent.closure_pullable")
|
||||
.with_unit("1")
|
||||
.with_callback(|obs| {
|
||||
for (agent, reason) in crate::attic::probe::verdicts() {
|
||||
let mut attrs = vec![KeyValue::new("agent", agent)];
|
||||
if let Some(reason) = reason {
|
||||
attrs.push(KeyValue::new("reason", reason.label()));
|
||||
}
|
||||
obs.observe(u64::from(reason.is_none()), &attrs);
|
||||
}
|
||||
})
|
||||
.build();
|
||||
}
|
||||
|
||||
/// The gauge value for `agent`: 1 when unrestricted, 0 when not. An agent
|
||||
/// missing from `stored` reads 1.
|
||||
fn unrestricted_value(stored: &permissions::Stored, agent: &str) -> u64 {
|
||||
|
|
|
|||
|
|
@ -109,8 +109,12 @@ pub fn spawn(
|
|||
shutdown.clone(),
|
||||
));
|
||||
|
||||
// The cache facts this hive pulls with, read into /run/hyperhive/attic.
|
||||
tokio::spawn(crate::attic::reader::follow(shutdown.clone()));
|
||||
// Whether each agent's latest closure could be pulled from the swarm's
|
||||
// cache, on the same connection.
|
||||
tokio::spawn(crate::attic::probe::follow(
|
||||
client.clone(),
|
||||
shutdown.clone(),
|
||||
));
|
||||
|
||||
// The permissions the swarm grants, and the import of the meta repo's.
|
||||
tokio::spawn(crate::workers::swarm_permissions::follow(
|
||||
|
|
|
|||
Loading…
Reference in a new issue