Watch
0
0
Fork
You've already forked hyperhive
0

Compare commits

...
25 changed files with 1425 additions and 61 deletions

30
Cargo.lock generated
View file

@ -1777,6 +1777,7 @@ dependencies = [
"ed25519-dalek",
"forgejo-api",
"futures-util",
"getrandom 0.3.4",
"hive-core-agent-sock",
"hive-host-sock",
"hive-jobq",
@ -1800,6 +1801,7 @@ dependencies = [
"rusqlite",
"serde",
"serde_json",
"sha-crypt",
"sha2 0.11.0",
"strum",
"swarm-matrix-client",
@ -3082,6 +3084,15 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "mcf"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2044bdbade272ded4a1529324f4f30ca7b0c5b496541f2a9095bbd1c9d03cb06"
dependencies = [
"base64ct",
]
[[package]]
name = "memchr"
version = "2.8.3"
@ -3357,6 +3368,12 @@ dependencies = [
"windows-link",
]
[[package]]
name = "password-hash"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b"
[[package]]
name = "paste"
version = "1.0.15"
@ -4567,6 +4584,19 @@ dependencies = [
"serde",
]
[[package]]
name = "sha-crypt"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32ee31e0190f9dc366d26b032ac36a99328532f40a237a79385a20d2d7f29810"
dependencies = [
"base64ct",
"ctutils",
"mcf",
"password-hash",
"sha2 0.11.0",
]
[[package]]
name = "sha1"
version = "0.10.7"

View file

@ -248,6 +248,9 @@ matrix-sdk = { version = "0.18", default-features = false, features = [
futures-util = "0.3"
hmac = "0.13"
sha2 = "0.11"
# SHA-512 crypt (`$6$`) for the users file hive-c0re renders; squid's `basic_ncsa_auth`
# hands a `$` hash to the system `crypt()`.
sha-crypt = "0.6"
# Verifies the cache signature over a narinfo (hive-c0re). Already in the lock
# through nkeys.
ed25519-dalek = "2"

View file

@ -212,13 +212,27 @@ DNS/gateway), so only expose services safe for any agent to reach.
Every hive host runs squid in the host netns, listening on `<bridgeIp>:3128`
(`nix/host-modules/hive-squid.nix`). The port is open on the bridge interface
only. **No agent uses it yet**: agent traffic still goes straight out through
the NAT. The proxy goes first so you can check its metrics and logs before
anything reroutes.
only. **No agent uses it yet**: none sends a credential, and agent traffic
still goes straight out through the NAT. The proxy goes first so you can check
its metrics and logs before anything reroutes.
Squid filters on the destination hostname: a `CONNECT` target or a plain-HTTP
host. It never decrypts TLS, so it holds no signing key. It caches nothing.
- **Credentials.** A request for a listed name must carry a per-agent proxy
credential (HTTP Basic, the agent's name and its secret). `swarm-controller`
mints one per agent at `swarm/agents/<agent>/proxy-credential`, and never
replaces it. `hive-c0re` hashes the credentials of this hive's agents, and
one more for the self-check, into `services.hyperhive.squid.usersFile`
every 30 seconds. Squid's `basic_ncsa_auth` helper re-reads the file when it
changes. A stopped agent stays in the file. Squid caches a good login for a
minute. Nothing rotates a credential: to replace one, delete its leaf in the
store and the next pass mints a new one. HTTP Basic crosses the bridge in
clear between an agent and the proxy. See
[the credentials reference](../swarm/credentials-reference.md).
A name that isn't on the allow-list gets the 403 whether or not the request
has a credential; a listed name without one gets a 407.
- **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the
swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each
entry matches that exact name. Everything else gets a 403.
@ -237,10 +251,17 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing.
unit and the self-check's to the swarm's log store. A refusal shows as `TCP_DENIED/403` with the
client's source address.
- **Self-check.** `squid-selfcheck.timer` runs `squid-selfcheck.service`
every minute from the bridge IP: `CONNECT` to `services.hyperhive.squid.selfcheck.canary`
(default: the forge's name) must answer 200, and to `deny-canary.invalid`
must answer 403. It writes one `squid-selfcheck ok=… allow_status=…
deny_status=…` line per run and fails the unit when either is wrong. The
every minute from the bridge IP, logging in as `selfcheck.squid`: `CONNECT`
to `services.hyperhive.squid.selfcheck.canary` (default: the forge's name)
must answer 200, to `deny-canary.invalid` 403, and to the canary without a
credential 407. It writes one `squid-selfcheck ok=… allow_status=…
deny_status=… anon_status=… credential=present|absent` line per run and
fails the unit when any is wrong. The password is the hive's
`swarm/hives/<hive>/squid-selfcheck` secret, which
`squid-selfcheck-credential` lands at `/run/credstore/hive-squid.selfcheck`.
Before `swarm-controller` has minted it, or while `hive-c0re` hasn't
written the users file, the line reads `credential=absent` or
`allow_status=407` and `ok=false`, and nothing else breaks. The
host's resolver looks up the canary, so a host whose resolver maps
that name to a loopback or bridge address (a single-host swarm with
`gateway.localHostsEntry`) fails the check.

View file

@ -22,8 +22,10 @@ hive's domain and `*.<hive domain>`, read from
`services.hyperhive.swarm.hives.<name>.domain` on the store's host, with no
IP SANs and no `localhost`. A request for any other name is a refusal from
the store naming that name. The hive's policy grants `update` on that one
issue path and `read` on its own `attic/public-key` and `attic/pull-token`
(see [the cache facts for hive-priv](credentials-reference.md)), and nothing
issue path and `read` on its own `attic/public-key`, `attic/pull-token` and
`squid-selfcheck`
(see [the cache facts for hive-priv](credentials-reference.md) and the
squid self-check password), and nothing
else. `swarm-bao-hive-leaf-policy.service`, on the
store's host, writes the role, the policy and the cert-auth role for every
hive in `swarm.hives`.

View file

@ -36,6 +36,8 @@ the rest of the cell says how.
| `swarm/agents/<agent>/claude-credentials` | `swarm-claude-login`, when you log the agent in from its card; one version per login | the agent, while it waits for a login | ❌ log in again | ✅ every 30 seconds while waiting |
| `swarm/hives/<hive>/matrix/sender-token` | `swarm-controller`, every 5 minutes | `swarm-controller` and the hive | ✅ when missing or rejected by the homeserver | ✅ the hive re-reads on every sweep |
| `swarm/hives/<hive>/forge/token` | `swarm-controller`, every 5 minutes | `swarm-controller` and the hive | ✅ when missing or no longer matching the forge | ✅ read on every forge call |
| `swarm/agents/<agent>/proxy-credential` | `swarm-controller`: 32 random bytes, at start and every 5 minutes for every agent that holds a store identity | the hive (`hive-c0re`); the agent can read it, nothing sends it yet | ❌ never replaced: written only while the path holds nothing | ✅ `hive-c0re` re-reads it every 30 seconds |
| `swarm/hives/<hive>/squid-selfcheck` | `swarm-controller`, in the same pass as the agents' credentials | the hive: `hive-c0re` and `squid-selfcheck-credential` (root) | ❌ never replaced: written only while the path holds nothing | ✅ `hive-c0re` every 30 seconds, the root unit every 10 minutes |
| `swarm/hives/<hive>/oidc/client` | authelia, first boot; `swarm-secret-publish` copies it in | the hive | ❌ minted once; `swarm-secret-publish` republishes a re-mint | ✅ re-read when authelia refuses it |
| `swarm/services/<clientId>/oidc/client` | authelia | that service | must be stated | must be stated |
| `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge, at start and every 5 minutes | `swarm-bao-ci-runner` on the runner's host | ❌ the forge keeps one token until an admin resets it; the controller then writes the new one within 5 minutes | ✅ every 10 minutes; restarts the runner on a change |
@ -154,6 +156,8 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](credentials.
| the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/<id>.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` |
| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/<id>.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` |
| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives/<hive>/forge/token`, for the `hive-<hive>` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity, and copies it (account on the first line, token on the second) to `/run/hyperhive/forge-hive-token` (tmpfs, `0600` `hive-core`) for nix's `git`, which `hive-priv` also runs. Each `internal/meta` read and each boot rewrites the copy when it differs; where they differ after a rotation, a build that starts before the next read fails to authenticate, and a retry reads the new token |
| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `<user>:<hash>` lines, SHA-512 crypt hashes only, renamed into place when it changed. `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; no agent sends a credential, so no traffic changes |
| the squid self-check's password | `swarm-controller` mints `swarm/hives/<hive>/squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks |
### Store identities, one per principal

View file

@ -19,6 +19,8 @@ axum.workspace = true
backon.workspace = true
chrono.workspace = true
base64.workspace = true
getrandom.workspace = true
sha-crypt.workspace = true
reqwest.workspace = true
forgejo-api.workspace = true
url.workspace = true

View file

@ -363,6 +363,8 @@ async fn cmd_serve(
// not a part of the above; see swarm_agent_status's module doc for why
// the two buckets stay separate.
swarm_agent_status::spawn(coord.shutdown_rx());
// Squid's users file: the hashes of this hive's agents' proxy credentials.
workers::squid_users::spawn(coord.shutdown_rx());
// Per-agent events.sqlite + bash-tasks file cleanup now runs
// agent-side in the harness (`hive_agent::vacuum`): the files are
// agent-owned, so host-side deletes hit PermissionDenied / readonly-db

View file

@ -1,7 +1,8 @@
//! [`write_secret_0600`]: the one way hive-c0re puts a credential on disk.
//! [`write_secret_0600`] and [`write_secret_mode`]: the ways hive-c0re puts a
//! credential, or a hash of one, on disk.
use std::io::Write as _;
use std::os::unix::fs::OpenOptionsExt as _;
use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _};
use std::path::Path;
use anyhow::{Context as _, Result};
@ -13,6 +14,12 @@ use anyhow::{Context as _, Result};
/// reader (or a crash) never sees half of it. Errors carry paths, never
/// `contents`.
pub fn write_secret_0600(path: &Path, contents: &[u8]) -> Result<()> {
write_secret_mode(path, contents, 0o600)
}
/// [`write_secret_0600`] with the file's mode given, for a file another user
/// has to read (the group comes from the directory).
pub fn write_secret_mode(path: &Path, contents: &[u8], mode: u32) -> Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
}
@ -27,9 +34,13 @@ pub fn write_secret_0600(path: &Path, contents: &[u8]) -> Result<()> {
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.mode(mode)
.open(&tmp)
.with_context(|| format!("create {}", tmp.display()))?;
// `open` masks the mode with the process umask; the mode is a property of
// the writer, not of the unit's umask.
f.set_permissions(std::fs::Permissions::from_mode(mode))
.with_context(|| format!("chmod {}", tmp.display()))?;
f.write_all(contents)
.and_then(|()| f.sync_all())
.with_context(|| format!("write {}", tmp.display()))?;
@ -38,7 +49,7 @@ pub fn write_secret_0600(path: &Path, contents: &[u8]) -> Result<()> {
#[cfg(test)]
mod tests {
use super::write_secret_0600;
use super::{write_secret_0600, write_secret_mode};
use std::os::unix::fs::PermissionsExt as _;
fn mode(path: &std::path::Path) -> u32 {
@ -76,4 +87,26 @@ mod tests {
write_secret_0600(&path, b"new\n").expect("writes");
assert_eq!(std::fs::read(&path).expect("read"), b"new\n");
}
/// A restrictive umask doesn't change the mode asked for. The umask is
/// process-wide, so the write happens in a child run of this test.
#[test]
fn the_mode_survives_a_restrictive_umask() {
const DIR: &str = "SECRET_FILE_UMASK_TEST_DIR";
if let Some(dir) = std::env::var_os(DIR) {
write_secret_mode(&std::path::Path::new(&dir).join("shared"), b"x\n", 0o640)
.expect("writes");
return;
}
let dir = tempfile::tempdir().expect("tempdir");
let status = std::process::Command::new("sh")
.arg("-c")
.arg(r#"umask 077 && exec "$0" --exact secret_file::tests::the_mode_survives_a_restrictive_umask"#)
.arg(std::env::current_exe().expect("test binary"))
.env(DIR, dir.path())
.status()
.expect("spawns the child");
assert!(status.success());
assert_eq!(mode(&dir.path().join("shared")), 0o640);
}
}

View file

@ -1,7 +1,7 @@
//! Background tasks and periodic sweeps: crash/login watcher, boot-time
//! auto-update reconcile, the agent-sockets.json writer loop, the MCP socket listener
//! reconcile loop, knowledge-repo sync, the swarm wanted-state pull, and the
//! swarm resource-cap and permissions reads.
//! swarm resource-cap and permissions reads, and the squid users file.
//! Each submodule is re-exported
//! at the crate root, so `crate::crash_watch::…` etc. keep working unchanged.
@ -11,6 +11,7 @@ pub mod crash_watch;
pub mod knowledge;
pub mod mcp_sockets;
pub mod permissions_import;
pub mod squid_users;
pub mod swarm_limits;
pub mod swarm_permissions;
pub mod wanted;

View file

@ -0,0 +1,440 @@
//! Squid's users file: the hashes of the proxy credentials this hive's agents
//! authenticate to its egress proxy with.
//!
//! swarm-controller mints one credential per agent and one for this hive's own
//! squid self-check ([`swarm_secret_client::proxy`]). Every [`INTERVAL`] this
//! reads the credentials of the agents this hive has, plus the self-check's,
//! and renders `<user>:<hash>` lines into the file named by
//! [`ENV_USERS_FILE`], which squid's `basic_ncsa_auth` re-reads when its mtime
//! moves. The agents are the container list, not the running containers: a
//! stopped agent stays in the file.
//!
//! Only salted SHA-512 crypt hashes reach the disk. A hash is kept as it is
//! while it still verifies its credential, so the file changes only when a
//! user is added, removed or given a new credential.
//!
//! A failed read leaves the file as it is: dropping a user on a store outage
//! would read exactly like the swarm revoking it.
use std::future::Future;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::{Context, Result, bail};
use sha_crypt::{PasswordHasher as _, PasswordVerifier as _, ShaCrypt};
use swarm_secret_client::{SecretStore, client, proxy};
use crate::stats::sweep_health::{self, SweepHealth};
/// How often the file is re-rendered.
pub const INTERVAL: Duration = Duration::from_secs(30);
/// At [`INTERVAL`] this is ~90 seconds of failing passes.
const FAILURES_BEFORE_WARNING: u32 = 3;
/// The environment variable naming the file. Set by `nix/host-modules/hive-c0re`
/// from `services.hyperhive.squid.usersFile`; unset means this host runs no squid.
pub const ENV_USERS_FILE: &str = "HIVE_SQUID_USERS_FILE";
/// Raw salt bytes: twelve of them encode to the sixteen characters `crypt()`
/// accepts.
const SALT_BYTES: usize = 12;
/// The file's mode. The group is squid's, from the setgid directory
/// `nix/host-modules/hive-squid.nix` creates.
const FILE_MODE: u32 = 0o640;
/// One squid user and the credential it logs in with. No `Debug`: `secret` is
/// live.
struct User {
name: String,
secret: String,
}
/// Start the pass loop, when this host has a squid users file, a store and a
/// hive name.
pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
let Some(path) = std::env::var_os(ENV_USERS_FILE)
.filter(|v| !v.is_empty())
.map(PathBuf::from)
else {
return;
};
let Some(hive) = crate::container_view::hive_swarm_names().0 else {
tracing::info!("squid users: HYPERHIVE_HIVE_NAME is unset; no users file is written");
return;
};
if std::env::var_os(client::ENV_ADDR).is_none_or(|v| v.is_empty()) {
tracing::info!(
"squid users: no swarm secret store is configured; no users file is written"
);
return;
}
tokio::spawn(async move {
let mut health = SweepHealth::new("squid_users_render", "warn", FAILURES_BEFORE_WARNING);
loop {
match pass(&hive, &path).await {
Ok(()) => health.record_ok(),
Err(e) => {
tracing::warn!(error = ?e, "squid users: the pass failed");
let err = format!("{e:#}");
health.record_err(|ctx| {
let age = ctx.since_last_ok.map_or_else(
|| "no success this session".to_owned(),
|d| format!("last ok {} ago", sweep_health::fmt_age(d)),
);
format!(
"rendering squid's users file is failing ({} consecutive, {age}) \
— squid keeps the users last written: {err}",
ctx.consecutive
)
});
}
}
tokio::select! {
() = tokio::time::sleep(INTERVAL) => {}
_ = shutdown.changed() => return,
}
}
});
}
/// Read this hive's credentials from the store and render them into `file`.
async fn pass(hive: &str, file: &Path) -> Result<()> {
let agents: Vec<String> = crate::lifecycle::agents_for_meta_listing()
.await
.context("enumerating this hive's agents")?
.into_iter()
.map(|spec| spec.name)
.collect();
let store = SecretStore::from_env(hive)
.await
.context("connecting to the store")?;
sync_with(hive, &agents, file, |path| {
let store = &store;
async move {
let held: Option<proxy::Credential> = store
.read_optional(&path)
.await
.with_context(|| format!("reading {path}"))?;
Ok(held.map(|c| c.value))
}
})
.await
}
/// [`pass`] with the store read and the file given. Nothing is written unless
/// every read succeeds and the rendered file differs from the current one.
async fn sync_with<F, Fut>(hive: &str, agents: &[String], file: &Path, read: F) -> Result<()>
where
F: Fn(String) -> Fut,
Fut: Future<Output = Result<Option<String>>>,
{
let mut wanted = Vec::new();
for agent in agents {
wanted.push((agent.clone(), proxy::agent_credential_path(agent)?));
}
wanted.push((
proxy::SELFCHECK_USER.to_owned(),
proxy::selfcheck_path(hive)?,
));
let mut users = Vec::new();
for (name, path) in wanted {
if let Some(secret) = read(path).await? {
users.push(User { name, secret });
} else {
tracing::debug!(name, "squid users: no proxy credential minted yet");
}
}
let current = match std::fs::read_to_string(file) {
Ok(text) => text,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(),
Err(e) => return Err(e).with_context(|| format!("reading {}", file.display())),
};
let rendered = render(&current, &users)?;
if rendered == current {
return Ok(());
}
crate::secret_file::write_secret_mode(file, rendered.as_bytes(), FILE_MODE)
.with_context(|| format!("writing {}", file.display()))?;
tracing::info!(
users = users.len(),
agents = users.iter().filter(|u| u.name != proxy::SELFCHECK_USER).count(),
file = %file.display(),
"squid users: wrote the users file"
);
Ok(())
}
/// The users file for `users`, sorted by name, one `<user>:<hash>` line each.
///
/// A user whose line in `current` still verifies against its credential keeps
/// that line. Every other line of `current` is dropped, so a destroyed agent,
/// or one that belongs to another hive, does not outlive the pass.
fn render(current: &str, users: &[User]) -> Result<String> {
let mut lines: Vec<String> = Vec::with_capacity(users.len());
for user in users {
if user.name.contains([':', '\n']) {
bail!("{:?} cannot be a squid user name", user.name);
}
let kept = current
.lines()
.filter_map(|l| l.split_once(':'))
.find(|(name, hash)| *name == user.name && verifies(&user.secret, hash))
.map(|(_, hash)| hash.to_owned());
let hash = match kept {
Some(hash) => hash,
None => hash_secret(&user.secret)?,
};
lines.push(format!("{}:{hash}\n", user.name));
}
lines.sort();
Ok(lines.concat())
}
/// A fresh salted SHA-512 crypt hash of `secret`: `$6$rounds=5000$<salt>$<hash>`.
fn hash_secret(secret: &str) -> Result<String> {
let mut salt = [0u8; SALT_BYTES];
getrandom::fill(&mut salt).context("drawing a salt from the kernel's CSPRNG")?;
let hash = ShaCrypt::SHA512
.hash_password_with_salt(secret.as_bytes(), &salt)
.map_err(|e| anyhow::anyhow!("hashing a proxy credential: {e}"))?;
Ok(hash.as_str().to_owned())
}
/// Whether `hash` is `secret`'s SHA-crypt hash.
fn verifies(secret: &str, hash: &str) -> bool {
ShaCrypt::SHA512
.verify_password(secret.as_bytes(), hash)
.is_ok()
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::os::unix::fs::PermissionsExt as _;
use super::*;
const HIVE: &str = "h1";
/// What the store holds, by path.
fn store(entries: &[(&str, &str)]) -> HashMap<String, String> {
entries
.iter()
.map(|(p, v)| ((*p).to_owned(), (*v).to_owned()))
.collect()
}
fn names(names: &[&str]) -> Vec<String> {
names.iter().map(|n| (*n).to_owned()).collect()
}
async fn sync(held: &HashMap<String, String>, agents: &[&str], file: &Path) -> Result<()> {
sync_with(HIVE, &names(agents), file, |path| {
let found = held.get(&path).cloned();
async move { Ok(found) }
})
.await
}
fn lines(file: &Path) -> Vec<(String, String)> {
std::fs::read_to_string(file)
.expect("the file is there")
.lines()
.map(|l| {
let (name, hash) = l.split_once(':').expect("a name and a hash");
(name.to_owned(), hash.to_owned())
})
.collect()
}
fn two_agents_and_a_stranger() -> HashMap<String, String> {
store(&[
("swarm/agents/atlas/proxy-credential", "atlas-secret"),
("swarm/agents/argus/proxy-credential", "argus-secret"),
("swarm/agents/other-hive-agent/proxy-credential", "stranger"),
("swarm/hives/h1/squid-selfcheck", "check-secret"),
("swarm/hives/h2/squid-selfcheck", "other-check"),
])
}
#[tokio::test]
async fn only_this_hives_agents_and_its_selfcheck_are_rendered() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
.await
.expect("renders");
let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
assert_eq!(users, ["argus", "atlas", "selfcheck.squid"]);
}
#[tokio::test]
async fn each_line_is_a_salted_sha512_crypt_hash_of_its_users_credential() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
.await
.expect("renders");
let by_name: HashMap<_, _> = lines(&file).into_iter().collect();
assert!(by_name["atlas"].starts_with("$6$"), "{}", by_name["atlas"]);
assert!(verifies("atlas-secret", &by_name["atlas"]));
assert!(verifies("argus-secret", &by_name["argus"]));
assert!(verifies("check-secret", &by_name["selfcheck.squid"]));
// Control: a hash is of its own user's credential only.
assert!(!verifies("argus-secret", &by_name["atlas"]));
// Salted: the same credential hashes differently twice.
assert_ne!(
hash_secret("atlas-secret").expect("hashes"),
hash_secret("atlas-secret").expect("hashes")
);
}
#[tokio::test]
async fn no_credential_is_written_to_the_file_or_its_directory() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
.await
.expect("renders");
// Control: the search finds a name that is in the file.
let mut found = 0;
for entry in std::fs::read_dir(dir.path()).expect("lists") {
let text = std::fs::read_to_string(entry.expect("entry").path()).expect("text");
assert!(text.contains("atlas:"), "control: {text}");
for secret in ["atlas-secret", "argus-secret", "check-secret"] {
assert!(!text.contains(secret), "{secret} reached the disk");
}
found += 1;
}
assert_eq!(found, 1, "only the users file is left behind");
}
#[tokio::test]
async fn the_file_is_group_readable_and_replaced_atomically() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas"], &file)
.await
.expect("renders");
let mode = std::fs::metadata(&file).expect("stat").permissions().mode() & 0o777;
assert_eq!(mode, 0o640);
let inode = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
.await
.expect("renders");
let after = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
assert_ne!(
inode, after,
"a rename onto the path, not a rewrite in place"
);
assert!(!dir.path().join("users.tmp").exists());
}
#[tokio::test]
async fn an_unchanged_pass_leaves_the_file_alone() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
let held = two_agents_and_a_stranger();
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
let first = std::fs::read_to_string(&file).expect("read");
let inode = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
assert_eq!(std::fs::read_to_string(&file).expect("read"), first);
let again = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
assert_eq!(inode, again, "nothing was written");
}
#[tokio::test]
async fn a_changed_credential_gets_a_new_hash_and_the_others_keep_theirs() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
let mut held = two_agents_and_a_stranger();
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
let before: HashMap<_, _> = lines(&file).into_iter().collect();
held.insert(
"swarm/agents/atlas/proxy-credential".to_owned(),
"rotated".to_owned(),
);
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
let after: HashMap<_, _> = lines(&file).into_iter().collect();
assert!(verifies("rotated", &after["atlas"]));
assert_ne!(before["atlas"], after["atlas"]);
assert_eq!(before["argus"], after["argus"]);
}
#[tokio::test]
async fn a_stopped_agent_stays_and_a_destroyed_one_goes() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
let held = two_agents_and_a_stranger();
// The agent list is the container list, so an agent whose container
// is down is in it: nothing here looks at whether anything runs.
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
assert!(lines(&file).iter().any(|(n, _)| n == "argus"));
// Control: once the container is gone the user is dropped, and a line
// for an agent this hive never had does not survive a pass either.
let mut text = std::fs::read_to_string(&file).expect("read");
text.push_str("stray:$6$rounds=5000$saltsaltsaltsalt$x\n");
std::fs::write(&file, text).expect("write");
sync(&held, &["atlas"], &file).await.expect("renders");
let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
assert_eq!(users, ["atlas", "selfcheck.squid"]);
}
#[tokio::test]
async fn an_agent_without_a_credential_yet_is_left_out() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
// The empty file squid's unit creates before its helper starts.
std::fs::write(&file, "").expect("write");
sync(&store(&[]), &["atlas"], &file).await.expect("renders");
assert_eq!(std::fs::read_to_string(&file).expect("read"), "");
}
#[tokio::test]
async fn a_failed_read_leaves_the_file_as_it_was() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
.await
.expect("renders");
let before = std::fs::read_to_string(&file).expect("read");
let failed = sync_with(
HIVE,
&names(&["atlas", "argus"]),
&file,
|path| async move {
if path.contains("argus") {
bail!("the store is down")
}
Ok(Some("atlas-secret".to_owned()))
},
)
.await;
assert!(failed.is_err());
assert_eq!(std::fs::read_to_string(&file).expect("read"), before);
}
#[test]
fn a_name_that_would_break_the_line_is_refused() {
for bad in ["a:b", "a\nb"] {
let user = User {
name: bad.to_owned(),
secret: "s".to_owned(),
};
assert!(render("", &[user]).is_err(), "{bad:?}");
}
}
}

View file

@ -25,6 +25,7 @@
./hive-priv.nix
./hive-priv-cache.nix
./hive-squid.nix
./hive-squid-credential.nix
./hive-tls.nix
./otel.nix
./glue-bao-readers-policy-order.nix

View file

@ -347,7 +347,11 @@ in
# needs access to the rest of c0re's state — that separation
# means c0re itself needs an explicit carve-out to write there.
ProtectSystem = "strict";
ReadWritePaths = [ "/var/lib/hive-gateway/conf" ];
# `-` so a host without the squid users directory still starts the daemon.
ReadWritePaths = [
"/var/lib/hive-gateway/conf"
"-${builtins.dirOf config.services.hyperhive.squid.usersFile}"
];
ProtectKernelTunables = true; # no sysctl writes
ProtectKernelLogs = true; # no /dev/kmsg access needed
ProtectControlGroups = true; # cgroup writes go through hive-priv, not c0re directly

View file

@ -271,6 +271,9 @@ in
# See the LoadCredential in ./default.nix.
BAO_CLIENT_CERT = "%d/bao-client.pem";
BAO_CLIENT_KEY = "%d/bao-client-key.pem";
# Where `workers::squid_users` renders the hashes of this hive's agents' proxy
# credentials for squid; it reads them from the store under the identity above.
HIVE_SQUID_USERS_FILE = config.services.hyperhive.squid.usersFile;
}
// lib.optionalAttrs (haveBaoClientIdentity && baoDeploy.serverCaFile != null) {
# Absent means the system trust store — right for a deployment with a real

View file

@ -0,0 +1,103 @@
# The password this hive's squid self-check logs in to squid with, kept where
# only root can read it.
#
# swarm-controller mints it at `swarm/hives/<hive>/squid-selfcheck`
# (`swarm_secret_client::proxy::selfcheck_path`). hive-c0re reads the same path
# to put its hash in squid's users file; this root oneshot reads it under the
# hive's own store identity and lands it as the systemd credential
# `squid-selfcheck` imports (`/run/credstore/<name>`, root, `0400`, tmpfs).
#
# A hive with no minted credential yet leaves the file alone and the self-check
# reports `credential=absent`.
{
pkgs,
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
hiveController = hyperhiveCfg.deploy.hive-controller;
tls = hiveController.tls;
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = hyperhiveCfg.deploy.bao;
enabled =
hiveController.enable
&& hiveController.hiveName != null
&& tls.baoHiveIdentityCertFile != null
&& tls.baoHiveIdentityKeyFile != null;
credentialFile = "/run/credstore/${hyperhiveCfg.squid.selfcheck.credentialName}";
storeRetry = import ./lib/store-retry.nix { };
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
in
{
config = lib.mkIf enabled {
systemd.services.squid-selfcheck-credential = {
description = "Read the squid self-check's password from the swarm secret store";
wantedBy = [ "multi-user.target" ];
before = [ "squid-selfcheck.service" ];
wants = [ "network-online.target" ];
after = [
"network-online.target"
"systemd-tmpfiles-setup.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.gnugrep
];
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = tls.baoHiveIdentityCertFile;
BAO_CLIENT_KEY = tls.baoHiveIdentityKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
# ./lib/store-retry.nix: the login fails for as long as the store is
# sealed or down.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
UMask = "0077";
SyslogIdentifier = "squid-selfcheck-credential";
TimeoutStartSec = "5min";
};
script = ''
set -euo pipefail
${import ./lib/store-seeded-secret.nix { }}
${import ./lib/atomic-write-secret.nix { }}
${import ./lib/refresh-consumer.nix { }}
install -d -m 0700 /run/credstore
store_init
store_login
if [ "$STORE_UP" != 1 ]; then
echo "could not read the squid self-check credential; ${credentialFile} stays as it is." >&2
exit 1
fi
if ! value="$(store_bao kv get -field=value ${lib.escapeShellArg "secret/swarm/hives/${hiveController.hiveName}/squid-selfcheck"} 2>"$STORE_ERR")"; then
if grep -qi 'no value found' "$STORE_ERR"; then
echo "the swarm has not minted this hive's squid self-check credential yet; ${credentialFile} stays as it is."
exit 0
fi
cat "$STORE_ERR" >&2
exit 1
fi
if [[ ! "$value" =~ ^[A-Za-z0-9_-]+$ ]]; then
echo "the stored squid self-check credential is not one base64url word; ${credentialFile} stays as it is." >&2
exit 1
fi
# Written only when the value changed, so the file keeps its mtime.
if secret_differs ${credentialFile} "$value"; then
atomic_write_secret 0400 root:root ${credentialFile} "$value"
fi
'';
};
systemd.timers.squid-selfcheck-credential = storeRefreshTimer "re-read the squid self-check's password";
};
}

View file

@ -1,8 +1,9 @@
# Egress proxy for this hive's agents: squid on the host, bound to the bridge
# IP, default-deny on the destination hostname. It filters CONNECT targets and
# plain-HTTP hosts and never decrypts, so it holds no signing key. No agent is
# pointed at it yet; the exporter, journal forwarding and the self-check below
# are how a deploy shows it works.
# plain-HTTP hosts and never decrypts, so it holds no signing key. A request
# for a listed name must carry a per-agent proxy credential, checked against the
# users file hive-c0re renders. No agent sends one yet; the exporter, journal
# forwarding and the self-check below are how a deploy shows it works.
{
pkgs,
lib,
@ -21,13 +22,20 @@ let
allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);
# The squid user the self-check logs in as: `swarm_secret_client::proxy::SELFCHECK_USER`.
selfcheckUser = "selfcheck.squid";
usersDir = builtins.dirOf cfg.usersFile;
# Evaluated top to bottom, first match wins. The three destination denies
# sit before the only allow: squid resolves names on the host, so a name
# that answers 127.0.0.1, 169.254.169.254 or a bridge address would
# otherwise reach services the bridge firewall keeps agents out of. The
# unlisted-name deny comes first because `dstdomain` needs no lookup while
# the `dst` ACLs do: an unlisted name is refused at once instead of waiting
# on the host resolver, and costs a client no DNS query.
# on the host resolver, and costs a client no DNS query. The credential
# check sits right before the allow, so a refused destination answers 403
# to a client with or without a credential, and only a request that would
# otherwise be let through gets the 407 challenge.
squidConf = ''
http_port ${netCfg.bridgeIp}:${toString port}
pid_filename /run/squid/squid.pid
@ -40,8 +48,17 @@ let
# Cachemgr scrapes are left out.
access_log syslog:daemon.info squid !manager
# The helper reads the users file on every request and reloads it when its
# mtime moves. It exits when the file is missing, which is why the file is
# created before squid starts. A positive answer is cached for a minute, so
# that is also how long a removed user keeps working.
auth_param basic program ${config.services.squid.package}/libexec/basic_ncsa_auth ${cfg.usersFile}
auth_param basic realm hyperhive egress
auth_param basic credentialsttl 1 minute
acl SSL_ports port 443
acl Safe_ports port 80 443
acl authed proxy_auth REQUIRED
# Host-originated requests: the self-check and the exporter. A local
# connect to the bridge IP takes the bridge IP as its source.
@ -69,6 +86,7 @@ let
http_access deny to_localhost
http_access deny to_linklocal
http_access deny to_bridge
http_access deny !authed
${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"}
http_access deny all
'';
@ -101,20 +119,39 @@ let
name = "squid-selfcheck";
runtimeInputs = [ pkgs.curl ];
text = ''
# The status squid answers a CONNECT with; 000 when the proxy is down.
# The credential comes from the unit's imported credential, and goes to
# curl on its stdin so it never appears in a command line. Without one
# the requests are anonymous and the self-check reports it.
secret=""
credential_file="''${CREDENTIALS_DIRECTORY:-/nonexistent}/${cfg.selfcheck.credentialName}"
if [ -s "$credential_file" ]; then
secret=$(< "$credential_file")
fi
# <host> <auth|anon>: the status squid answers a CONNECT with; 000 when
# the proxy is down.
connect_status() {
curl -sS -m 10 --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
{
if [ "$2" = auth ] && [ -n "$secret" ]; then
printf 'proxy-user = "%s:%s"\n' ${selfcheckUser} "$secret"
fi
} | curl -sS -m 10 -K - --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
-o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true
}
allow_status=$(connect_status ${cfg.selfcheck.canary})
deny_status=$(connect_status ${denyCanary})
allow_status=$(connect_status ${cfg.selfcheck.canary} auth)
deny_status=$(connect_status ${denyCanary} auth)
anon_status=$(connect_status ${cfg.selfcheck.canary} anon)
ok=false
if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ]; then
if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ] && [ "$anon_status" = 407 ]; then
ok=true
fi
echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status"
credential=absent
if [ -n "$secret" ]; then
credential=present
fi
echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status anon_status=$anon_status credential=$credential"
[ "$ok" = true ]
'';
};
@ -136,6 +173,31 @@ in
'';
};
usersFile = lib.mkOption {
type = lib.types.path;
default = "/var/lib/hive-squid/users";
description = ''
The `<user>:<hash>` file squid checks proxy credentials against. hive-c0re
renders it from the credentials in the swarm secret store, one line per
agent of this hive and one for the self-check, as SHA-512 crypt hashes.
An empty file, which is what exists until hive-c0re writes one, makes
every login fail, and every request to a listed name answers 407.
Owned by `hive-core` and readable by the `squid` group.
'';
};
selfcheck.credentialName = lib.mkOption {
type = lib.types.str;
default = "hive-squid.selfcheck";
readOnly = true;
description = ''
Name of the systemd credential the self-check imports its squid
password from. `squid-selfcheck-credential` writes it to
`/run/credstore/<name>`. Without one the self-check sends no
credential and reports `ok=false`.
'';
};
selfcheck.canary = lib.mkOption {
type = lib.types.str;
default = hyperhiveCfg.swarm.forge.domain;
@ -167,6 +229,14 @@ in
configText = squidConf;
};
# The directory is setgid so the file hive-c0re renames into it is the
# squid group's. The file exists before squid starts, empty: the helper
# exits on a missing file and squid then rejects every login.
systemd.tmpfiles.rules = [
"d ${usersDir} 2750 hive-core squid - -"
"f ${cfg.usersFile} 0640 hive-core squid - -"
];
systemd.services.squid = {
serviceConfig = sandbox // {
User = "squid";
@ -204,10 +274,13 @@ in
};
systemd.services.squid-selfcheck = {
description = "Squid allow and deny canaries";
description = "Squid allow, deny and credential canaries";
after = [ "squid.service" ];
serviceConfig = sandbox // {
Type = "oneshot";
# A glob that matches nothing is not an error: a host with no
# credential yet still runs the self-check, and it reports that.
ImportCredential = cfg.selfcheck.credentialName;
ExecStart = lib.getExe selfcheck;
DynamicUser = true;
IPAddressAllow = netCfg.bridgeIp;

View file

@ -486,6 +486,10 @@ let
capabilities = ["create", "read", "update"]
}
path "${credentialMountPath}/data/swarm/hives/+/squid-selfcheck" {
capabilities = ["create", "read"]
}
${lib.concatMapStrings readStanza (
map (leaf: "${credentialMountPath}/data/${leaf}") atticFactLeaves
)}
@ -938,12 +942,13 @@ let
inherit hiveName;
}) hyperhiveCfg.swarm.hives;
# What a hive's gateway-leaf identity may do: issue from its own role, and
# read two of its own cache facts. The root unit on the hive host
# (`hive-priv-cache-facts`, ./hive-priv-cache.nix) reads them under this
# identity: the key hive-priv's downloads must be signed by, and the token it
# pulls with. The cache's URL and name come from the host's configuration,
# not from the store.
# What a hive's gateway-leaf identity may do: issue from its own role, read
# two of its own cache facts, and read its own squid self-check credential.
# The root unit on the hive host (`hive-priv-cache-facts`,
# ./hive-priv-cache.nix) reads the facts under this identity: the key
# hive-priv's downloads must be signed by, and the token it pulls with. The
# cache's URL and name come from the host's configuration, not from the
# store. `squid-selfcheck-credential` (./hive-squid.nix) reads the credential.
hiveLeafPolicyText =
obj:
''
@ -957,7 +962,8 @@ let
[
"public-key"
"pull-token"
];
]
+ readStanza "${credentialMountPath}/data/swarm/hives/${obj.hiveName}/squid-selfcheck";
# The PKI mount agent client certificates are issued from. Its root is
# generated inside the store, so the agent CA's key never exists outside it.

View file

@ -1496,9 +1496,10 @@ let
{
# One role, policy and login role per hive in the directory, each
# naming only its own hive: the policy text in the unit is exactly the
# `update` on its own role and `read` on its own two cache facts, and each
# `update` on its own role and `read` on its own two cache facts and its own
# squid self-check credential, and each
# hive's block names neither the other's role nor its subject.
name = "each hive's gateway-leaf identity may issue from its own role and read its own two cache facts, and nothing else";
name = "each hive's gateway-leaf identity may issue from its own role and read its own two cache facts and squid self-check credential, and nothing else";
ok =
let
s = baoGrantTwoHives.systemd.services.swarm-bao-hive-leaf-policy.script;
@ -1536,6 +1537,10 @@ let
path = "secret/data/swarm/hives/h1/attic/pull-token";
caps = [ "read" ];
}
{
path = "secret/data/swarm/hives/h1/squid-selfcheck";
caps = [ "read" ];
}
{
path = "pki/issue/swarm-hive-leaf-h2";
caps = [ "update" ];
@ -1548,6 +1553,10 @@ let
path = "secret/data/swarm/hives/h2/attic/pull-token";
caps = [ "read" ];
}
{
path = "secret/data/swarm/hives/h2/squid-selfcheck";
caps = [ "read" ];
}
]
&& issues "h1" "h1.t.local" h1
&& issues "h2" "h2.example.org" h2
@ -1724,17 +1733,20 @@ let
{
# `matrix_account::hive_sender` writes every hive's sender token and
# `forge::hive_token` every hive's forge token, and nothing else under
# `hives/`, where each hive's queue credential also lives. `+` is one
# `hives/`, where each hive's queue credential also lives. `proxy_credential`
# creates each hive's squid self-check credential and reads it back, and
# has no `update`, so it cannot replace one. `+` is one
# segment, which keeps each stanza to its one leaf; a `*` is a glob only
# at the end of a path. Pinned as whole stanzas, so an added capability
# fails.
name = "the controller writes each hive's sender and forge tokens and nothing else under hives";
name = "the controller writes each hive's sender and forge tokens, creates its squid self-check credential, and nothing else under hives";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/hives/+/matrix/sender-token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/hives/+/forge/token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/hives/+/squid-selfcheck\" {\n capabilities = [\"create\", \"read\"]\n}" s
&& !(lib.hasInfix "secret/data/swarm/hives/*" s)
&& !(lib.hasInfix "secret/metadata/swarm/hives" s);
}

View file

@ -485,26 +485,62 @@ let
&& !(lib.hasInfix "localnet" conf);
}
{
# First match wins, so a destination deny after the allow is dead.
name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, before any allow";
# First match wins, so a destination deny after the allow is dead, and
# a credential check after it never challenges anyone. The two IP-literal
# denies stay ahead of the unlisted-name deny: `dstdomain` on an IP
# literal does a reverse lookup.
name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, then a missing credential, before any allow";
ok =
let
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
at = l: lib.lists.findFirstIndex (x: x == l) null lines;
allow = at "http_access allow allowed_dst";
authed = at "http_access deny !authed";
in
allow != null
&& authed != null
&& at "http_access deny to_localhost" != null
&& at "http_access deny to_localhost" < allow
&& at "http_access deny to_linklocal" != null
&& at "http_access deny to_linklocal" < allow
&& at "http_access deny to_bridge" != null
&& at "http_access deny to_bridge" < allow
&& at "http_access deny to_localhost" < authed
&& at "http_access deny to_linklocal" < authed
&& at "http_access deny to_bridge" < authed
&& authed < allow
&& at "http_access deny ip_v4_host" != null
&& at "http_access deny ip_v4_host" < at "http_access deny !allowed_dst"
&& at "http_access deny ip_v6_host" < at "http_access deny !allowed_dst"
&& at "http_access deny !allowed_dst" != null
&& at "http_access deny !allowed_dst" < at "http_access deny to_localhost"
&& builtins.elem "cache deny all" lines
&& lib.last (lib.filter (l: l != "") lines) == "http_access deny all";
}
{
# The helper is the NCSA one, on the file hive-c0re renders. The file
# exists, empty, before squid starts (the helper exits on a missing one),
# and the self-check imports its credential without failing when there is
# none.
name = "squid authenticates with the NCSA helper on a users file that exists before it starts";
ok =
let
conf = hiveOtel.services.squid.configText;
lines = lib.splitString "\n" conf;
users = hiveOtel.services.hyperhive.squid.usersFile;
rules = hiveOtel.systemd.tmpfiles.rules;
sc = hiveOtel.systemd.services.squid-selfcheck.serviceConfig;
in
lib.any (
l: lib.hasInfix "/libexec/basic_ncsa_auth ${users}" l && lib.hasPrefix "auth_param basic program " l
) lines
&& builtins.elem "acl authed proxy_auth REQUIRED" lines
&& users == "/var/lib/hive-squid/users"
&& builtins.elem "d /var/lib/hive-squid 2750 hive-core squid - -" rules
&& builtins.elem "f ${users} 0640 hive-core squid - -" rules
&& sc.ImportCredential == hiveOtel.services.hyperhive.squid.selfcheck.credentialName
&& hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_USERS_FILE == users;
}
{
name = "squid's port is open on the bridge interface and on no other";
ok =

View file

@ -800,6 +800,28 @@ let
noService.systemd.services.hive-priv-cache-facts.environment.BAO_CLIENT_CERT
== "/etc/pki/bao-hive.pem";
}
{
# Control: the lookup finds the sibling root unit of the same shape. The
# squid self-check's password is read under this hive's own identity and
# lands where only root reads it, as the credential `squid-selfcheck`
# imports.
name = "the squid self-check credential is read by a root oneshot under this hive's own identity into the credstore";
ok =
let
u = units.squid-selfcheck-credential;
in
lib.hasInfix "secret/swarm/hives/h1/squid-selfcheck" u.script
&& !(lib.hasInfix "swarm/hives/h2" u.script)
&& lib.hasInfix "/run/credstore/hive-squid.selfcheck" u.script
&& lib.hasInfix "atomic_write_secret 0400 root:root" u.script
&& u.environment.BAO_CLIENT_CERT == units.hive-priv-cache-facts.environment.BAO_CLIENT_CERT
&& u.environment.BAO_CLIENT_KEY == units.hive-priv-cache-facts.environment.BAO_CLIENT_KEY
&& u.serviceConfig.Type == "oneshot"
&& !(remainsAfterExit u)
&& allLocal.systemd.timers ? squid-selfcheck-credential
&& !(noIdentity.systemd.services ? squid-selfcheck-credential)
&& !(noIdentity.systemd.timers ? squid-selfcheck-credential);
}
{
name = "hive-priv is told the cache's address by the host's configuration";
ok =

View file

@ -55,6 +55,12 @@ let
};
};
};
# The user hive-c0re runs as, which owns the users file squid reads.
config.users.users.hive-core = {
isSystemUser = true;
group = "hive-core";
};
config.users.groups.hive-core = { };
# The firewall hole hive-network.nix opens for `exposeHostPorts`.
config.networking.firewall.interfaces.eth1.allowedTCPPorts =
config.services.hyperhive.network.exposeHostPorts;
@ -77,7 +83,7 @@ pkgs.testers.runNixOSTest {
nodes = {
proxy =
{ lib, ... }:
{ lib, pkgs, ... }:
{
imports = [
hiveStubs
@ -111,6 +117,8 @@ pkgs.testers.runNixOSTest {
${bridgeIp} = [ "bridge.t.local" ];
"169.254.169.254" = [ "meta.t.local" ];
};
# Renders the users file the way hive-c0re does.
environment.systemPackages = [ pkgs.mkpasswd ];
services.hyperhive.squid.allowedDomains = [
"forge.t.local"
"loop.t.local"
@ -148,6 +156,15 @@ pkgs.testers.runNixOSTest {
testScript = ''
import re
import secrets
# Drawn fresh each run. The users file holds only their hashes, as
# hive-c0re renders it.
PW = {u: secrets.token_urlsafe(24) for u in ("atlas", "selfcheck.squid", "ruth")}
ATLAS = "atlas:" + PW["atlas"]
SELFCHECK = "selfcheck.squid:" + PW["selfcheck.squid"]
RUTH = "ruth:" + PW["ruth"]
USERS = "/var/lib/hive-squid/users"
start_all()
proxy.wait_for_unit("squid.service")
@ -156,29 +173,64 @@ pkgs.testers.runNixOSTest {
web.wait_for_unit("nginx.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
def connect(host):
def connect(host, cred=None):
# The status squid answered the CONNECT with; 000 when none came back.
auth = f"-U '{cred}' " if cred else ""
return agent.succeed(
"curl -sS -m 15 -p -x http://${bridgeIp}:3128 -o /dev/null "
"curl -sS -m 15 -p -x http://${bridgeIp}:3128 " + auth + "-o /dev/null "
f"-w '%{{http_connect}}' https://{host}/ 2>/dev/null || true"
).strip()
with subtest("an allowed name is tunnelled and an unlisted one is refused"):
assert connect("forge.t.local") == "200", "allow canary"
assert connect("deny-canary.invalid") == "403", "deny canary"
def write_users(*creds):
# What hive-c0re does: hashes only, written aside and renamed into
# place, in squid's group. The sleep keeps two writes in different
# seconds, which is how the helper tells the file changed.
lines = " ".join(
f"printf '%s:%s\\n' {u} \"$(mkpasswd -m sha-512 -s <<< {p})\";"
for u, p in (c.split(":", 1) for c in creds)
)
proxy.succeed(
f"({lines}) > {USERS}.new && chown hive-core:squid {USERS}.new "
f"&& chmod 640 {USERS}.new && mv {USERS}.new {USERS} && sleep 1"
)
with subtest("squid starts on the empty users file and refuses every login"):
assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640"
assert proxy.succeed("stat -c %a /var/lib/hive-squid").strip() == "2750"
assert proxy.succeed(f"stat -c %s {USERS}").strip() == "0"
assert connect("forge.t.local", ATLAS) == "407", "no such user"
assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name needs no login"
with subtest("the users file holds hashes, never a password"):
write_users(ATLAS, SELFCHECK)
users = proxy.succeed(f"cat {USERS}")
assert re.fullmatch(r"(atlas|selfcheck\.squid):\$6\$\S+\n(atlas|selfcheck\.squid):\$6\$\S+\n", users), users
assert not any(p in users for p in PW.values()), users
with subtest("an allowed name is tunnelled with a credential and challenged without one"):
assert connect("forge.t.local", ATLAS) == "200", "allow canary"
assert connect("forge.t.local") == "407", "no credential"
assert connect("forge.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password"
assert connect("forge.t.local", "nobody:" + PW["atlas"]) == "407", "unknown user"
# DES would accept the first eight characters; the hash must not.
assert connect("forge.t.local", "atlas:" + PW["atlas"][:8]) == "407", "a prefix of the password"
with subtest("an unlisted name is refused with or without a credential"):
assert connect("deny-canary.invalid", ATLAS) == "403", "deny canary"
assert connect("deny-canary.invalid") == "403", "deny canary, anonymous"
with subtest("a name resolving to a host-local destination is refused"):
assert connect("loop.t.local") == "403", "loopback"
assert connect("bridge.t.local") == "403", "bridge address"
assert connect("meta.t.local") == "403", "link-local"
assert connect("loop.t.local", ATLAS) == "403", "loopback"
assert connect("bridge.t.local", ATLAS) == "403", "bridge address"
assert connect("meta.t.local", ATLAS) == "403", "link-local"
with subtest("IP literals are refused even when the address is reachable"):
assert connect("127.0.0.1") == "403"
assert connect("192.168.2.2") == "403"
assert connect("127.0.0.1", ATLAS) == "403"
assert connect("192.168.2.2", ATLAS) == "403"
with subtest("only a port-443 CONNECT is allowed"):
assert connect("forge.t.local:22") == "403"
assert connect("forge.t.local:80") == "403"
assert connect("forge.t.local:22", ATLAS) == "403"
assert connect("forge.t.local:80", ATLAS) == "403"
with subtest("an agent cannot read cachemgr"):
agent.succeed(
@ -186,19 +238,42 @@ pkgs.testers.runNixOSTest {
"http://${bridgeIp}:3128/squid-internal-mgr/counters) = 403"
)
with subtest("the self-check passes against the same squid"):
with subtest("the self-check passes with its credential"):
proxy.succeed("install -d -m 0700 /run/credstore")
proxy.succeed(f"printf '%s\\n' {PW['selfcheck.squid']} > /run/credstore/hive-squid.selfcheck")
proxy.succeed("chmod 400 /run/credstore/hive-squid.selfcheck")
proxy.succeed("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local "
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403'"
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403 "
"anon_status=407 credential=present'"
)
with subtest("squid's access log reaches the journal under squid.service"):
with subtest("the self-check fails, and says why, without a credential or with the wrong one"):
proxy.succeed("mv /run/credstore/hive-squid.selfcheck /run/credstore/kept")
proxy.fail("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local "
"allow_status=407 deny_canary=deny-canary.invalid deny_status=403 "
"anon_status=407 credential=absent'"
)
proxy.succeed(f"printf '%s\\n' {secrets.token_urlsafe(24)} > /run/credstore/hive-squid.selfcheck")
proxy.fail("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'allow_status=407 deny_canary=deny-canary.invalid deny_status=403 "
"anon_status=407 credential=present'"
)
proxy.succeed("mv -f /run/credstore/kept /run/credstore/hive-squid.selfcheck")
with subtest("squid's access log reaches the journal under squid.service, with the user"):
journal = proxy.succeed("journalctl -u squid.service -o cat")
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/407 .* CONNECT forge\.t\.local:443 - ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443 ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443 ", journal), journal
# Exporter scrapes are cachemgr requests and stay out of the log.
assert "squid-internal-mgr" not in journal, journal
@ -215,6 +290,21 @@ pkgs.testers.runNixOSTest {
with subtest("squid survives a reload"):
proxy.succeed("systemctl reload squid.service")
proxy.wait_for_unit("squid.service")
assert connect("forge.t.local") == "200"
assert connect("forge.t.local", ATLAS) == "200"
with subtest("a user added while squid runs can log in, with no reload"):
write_users(ATLAS, SELFCHECK, RUTH)
assert connect("forge.t.local", RUTH) == "200"
assert connect("forge.t.local", ATLAS) == "200"
with subtest("squid starts on a missing users file, refuses logins, and recovers when it appears"):
proxy.succeed(f"rm {USERS}")
proxy.succeed("systemctl restart squid.service")
proxy.wait_for_unit("squid.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
assert connect("forge.t.local", ATLAS) == "407", "no file, no login"
assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name still answers"
write_users(ATLAS, SELFCHECK)
assert connect("forge.t.local", ATLAS) == "200", "the file appeared"
'';
}

Some files were not shown because too many files have changed in this diff Show more