Watch
0
0
Fork
You've already forked hyperhive
0

Compare commits

..
25 changed files with 61 additions and 1425 deletions

30
Cargo.lock generated
View file

@ -1777,7 +1777,6 @@ dependencies = [
"ed25519-dalek",
"forgejo-api",
"futures-util",
"getrandom 0.3.4",
"hive-core-agent-sock",
"hive-host-sock",
"hive-jobq",
@ -1801,7 +1800,6 @@ dependencies = [
"rusqlite",
"serde",
"serde_json",
"sha-crypt",
"sha2 0.11.0",
"strum",
"swarm-matrix-client",
@ -3084,15 +3082,6 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "mcf"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2044bdbade272ded4a1529324f4f30ca7b0c5b496541f2a9095bbd1c9d03cb06"
dependencies = [
"base64ct",
]
[[package]]
name = "memchr"
version = "2.8.3"
@ -3368,12 +3357,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "password-hash"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b"
[[package]]
name = "paste"
version = "1.0.15"
@ -4584,19 +4567,6 @@ dependencies = [
"serde",
]
[[package]]
name = "sha-crypt"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32ee31e0190f9dc366d26b032ac36a99328532f40a237a79385a20d2d7f29810"
dependencies = [
"base64ct",
"ctutils",
"mcf",
"password-hash",
"sha2 0.11.0",
]
[[package]]
name = "sha1"
version = "0.10.7"

View file

@ -248,9 +248,6 @@ matrix-sdk = { version = "0.18", default-features = false, features = [
futures-util = "0.3"
hmac = "0.13"
sha2 = "0.11"
# SHA-512 crypt (`$6$`) for the users file hive-c0re renders; squid's `basic_ncsa_auth`
# hands a `$` hash to the system `crypt()`.
sha-crypt = "0.6"
# Verifies the cache signature over a narinfo (hive-c0re). Already in the lock
# through nkeys.
ed25519-dalek = "2"

View file

@ -212,27 +212,13 @@ DNS/gateway), so only expose services safe for any agent to reach.
Every hive host runs squid in the host netns, listening on `<bridgeIp>:3128`
(`nix/host-modules/hive-squid.nix`). The port is open on the bridge interface
only. **No agent uses it yet**: none sends a credential, and agent traffic
still goes straight out through the NAT. The proxy goes first so you can check
its metrics and logs before anything reroutes.
only. **No agent uses it yet**: agent traffic still goes straight out through
the NAT. The proxy goes first so you can check its metrics and logs before
anything reroutes.
Squid filters on the destination hostname: a `CONNECT` target or a plain-HTTP
host. It never decrypts TLS, so it holds no signing key. It caches nothing.
- **Credentials.** A request for a listed name must carry a per-agent proxy
credential (HTTP Basic, the agent's name and its secret). `swarm-controller`
mints one per agent at `swarm/agents/<agent>/proxy-credential`, and never
replaces it. `hive-c0re` hashes the credentials of this hive's agents, and
one more for the self-check, into `services.hyperhive.squid.usersFile`
every 30 seconds. Squid's `basic_ncsa_auth` helper re-reads the file when it
changes. A stopped agent stays in the file. Squid caches a good login for a
minute. Nothing rotates a credential: to replace one, delete its leaf in the
store and the next pass mints a new one. HTTP Basic crosses the bridge in
clear between an agent and the proxy. See
[the credentials reference](../swarm/credentials-reference.md).
A name that isn't on the allow-list gets the 403 whether or not the request
has a credential; a listed name without one gets a 407.
- **Allow-list.** `services.hyperhive.squid.allowedDomains` defaults to the
swarm's own service names (`services.hyperhive.swarm.serviceDomains`). Each
entry matches that exact name. Everything else gets a 403.
@ -251,17 +237,10 @@ host. It never decrypts TLS, so it holds no signing key. It caches nothing.
unit and the self-check's to the swarm's log store. A refusal shows as `TCP_DENIED/403` with the
client's source address.
- **Self-check.** `squid-selfcheck.timer` runs `squid-selfcheck.service`
every minute from the bridge IP, logging in as `selfcheck.squid`: `CONNECT`
to `services.hyperhive.squid.selfcheck.canary` (default: the forge's name)
must answer 200, to `deny-canary.invalid` 403, and to the canary without a
credential 407. It writes one `squid-selfcheck ok=… allow_status=…
deny_status=… anon_status=… credential=present|absent` line per run and
fails the unit when any is wrong. The password is the hive's
`swarm/hives/<hive>/squid-selfcheck` secret, which
`squid-selfcheck-credential` lands at `/run/credstore/hive-squid.selfcheck`.
Before `swarm-controller` has minted it, or while `hive-c0re` hasn't
written the users file, the line reads `credential=absent` or
`allow_status=407` and `ok=false`, and nothing else breaks. The
every minute from the bridge IP: `CONNECT` to `services.hyperhive.squid.selfcheck.canary`
(default: the forge's name) must answer 200, and to `deny-canary.invalid`
must answer 403. It writes one `squid-selfcheck ok=… allow_status=…
deny_status=…` line per run and fails the unit when either is wrong. The
host's resolver looks up the canary, so a host whose resolver maps
that name to a loopback or bridge address (a single-host swarm with
`gateway.localHostsEntry`) fails the check.

View file

@ -22,10 +22,8 @@ hive's domain and `*.<hive domain>`, read from
`services.hyperhive.swarm.hives.<name>.domain` on the store's host, with no
IP SANs and no `localhost`. A request for any other name is a refusal from
the store naming that name. The hive's policy grants `update` on that one
issue path and `read` on its own `attic/public-key`, `attic/pull-token` and
`squid-selfcheck`
(see [the cache facts for hive-priv](credentials-reference.md) and the
squid self-check password), and nothing
issue path and `read` on its own `attic/public-key` and `attic/pull-token`
(see [the cache facts for hive-priv](credentials-reference.md)), and nothing
else. `swarm-bao-hive-leaf-policy.service`, on the
store's host, writes the role, the policy and the cert-auth role for every
hive in `swarm.hives`.

View file

@ -36,8 +36,6 @@ the rest of the cell says how.
| `swarm/agents/<agent>/claude-credentials` | `swarm-claude-login`, when you log the agent in from its card; one version per login | the agent, while it waits for a login | ❌ log in again | ✅ every 30 seconds while waiting |
| `swarm/hives/<hive>/matrix/sender-token` | `swarm-controller`, every 5 minutes | `swarm-controller` and the hive | ✅ when missing or rejected by the homeserver | ✅ the hive re-reads on every sweep |
| `swarm/hives/<hive>/forge/token` | `swarm-controller`, every 5 minutes | `swarm-controller` and the hive | ✅ when missing or no longer matching the forge | ✅ read on every forge call |
| `swarm/agents/<agent>/proxy-credential` | `swarm-controller`: 32 random bytes, at start and every 5 minutes for every agent that holds a store identity | the hive (`hive-c0re`); the agent can read it, nothing sends it yet | ❌ never replaced: written only while the path holds nothing | ✅ `hive-c0re` re-reads it every 30 seconds |
| `swarm/hives/<hive>/squid-selfcheck` | `swarm-controller`, in the same pass as the agents' credentials | the hive: `hive-c0re` and `squid-selfcheck-credential` (root) | ❌ never replaced: written only while the path holds nothing | ✅ `hive-c0re` every 30 seconds, the root unit every 10 minutes |
| `swarm/hives/<hive>/oidc/client` | authelia, first boot; `swarm-secret-publish` copies it in | the hive | ❌ minted once; `swarm-secret-publish` republishes a re-mint | ✅ re-read when authelia refuses it |
| `swarm/services/<clientId>/oidc/client` | authelia | that service | must be stated | must be stated |
| `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge, at start and every 5 minutes | `swarm-bao-ci-runner` on the runner's host | ❌ the forge keeps one token until an admin resets it; the controller then writes the new one within 5 minutes | ✅ every 10 minutes; restarts the runner on a change |
@ -156,8 +154,6 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](credentials.
| the forge's copy of its OIDC secret | `swarm-bao-forge-oidc.service` reads it out of the swarm secret store under the forge publisher's leaf, wherever authelia runs | `/run/hive-forge-oidc/<id>.secret` on the host (tmpfs), bound into the forge container, handed to `forgejo-sso-source` through `LoadCredential` |
| the homeserver's copy of its OIDC secret | `swarm-bao-matrix-oidc.service` reads it out of the swarm secret store under matrix-ctl's leaf, wherever authelia runs | `/run/hive-matrix-oidc/<id>.secret` on the host (tmpfs), bound into the homeserver container, handed to tuwunel through `LoadCredential` |
| this hive's forge read token | `swarm-controller`, into the secret store at `swarm/hives/<hive>/forge/token`, for the `hive-<hive>` account; a pass at start and every five minutes re-mints a missing or stale token | `hive-c0re` reads it under the hive's own store identity, and copies it (account on the first line, token on the second) to `/run/hyperhive/forge-hive-token` (tmpfs, `0600` `hive-core`) for nix's `git`, which `hive-priv` also runs. Each `internal/meta` read and each boot rewrites the copy when it differs; where they differ after a rotation, a build that starts before the next read fails to authenticate, and a retry reads the new token |
| squid's users file | `hive-c0re` (`workers::squid_users`), every 30 seconds, from the proxy credentials of its agents and the self-check | `services.hyperhive.squid.usersFile` (`/var/lib/hive-squid/users`, `hive-core:squid`, `0640`): `<user>:<hash>` lines, SHA-512 crypt hashes only, renamed into place when it changed. `systemd-tmpfiles` creates it empty so squid's helper never sees a missing file. A hive whose agents have no credential yet (an older controller) has an empty file: every login fails and the self-check reports `ok=false`; no agent sends a credential, so no traffic changes |
| the squid self-check's password | `swarm-controller` mints `swarm/hives/<hive>/squid-selfcheck`; `squid-selfcheck-credential` (root, every 10 minutes) reads it as the hive | `/run/credstore/hive-squid.selfcheck` (tmpfs, root, `0400`), imported by `squid-selfcheck.service` as a systemd credential and handed to `curl` on its stdin. Missing, or different from the users file: the self-check logs `credential=absent` or a 407 and fails, and nothing else breaks |
### Store identities, one per principal

View file

@ -19,8 +19,6 @@ axum.workspace = true
backon.workspace = true
chrono.workspace = true
base64.workspace = true
getrandom.workspace = true
sha-crypt.workspace = true
reqwest.workspace = true
forgejo-api.workspace = true
url.workspace = true

View file

@ -363,8 +363,6 @@ async fn cmd_serve(
// not a part of the above; see swarm_agent_status's module doc for why
// the two buckets stay separate.
swarm_agent_status::spawn(coord.shutdown_rx());
// Squid's users file: the hashes of this hive's agents' proxy credentials.
workers::squid_users::spawn(coord.shutdown_rx());
// Per-agent events.sqlite + bash-tasks file cleanup now runs
// agent-side in the harness (`hive_agent::vacuum`): the files are
// agent-owned, so host-side deletes hit PermissionDenied / readonly-db

View file

@ -1,8 +1,7 @@
//! [`write_secret_0600`] and [`write_secret_mode`]: the ways hive-c0re puts a
//! credential, or a hash of one, on disk.
//! [`write_secret_0600`]: the one way hive-c0re puts a credential on disk.
use std::io::Write as _;
use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _};
use std::os::unix::fs::OpenOptionsExt as _;
use std::path::Path;
use anyhow::{Context as _, Result};
@ -14,12 +13,6 @@ use anyhow::{Context as _, Result};
/// reader (or a crash) never sees half of it. Errors carry paths, never
/// `contents`.
pub fn write_secret_0600(path: &Path, contents: &[u8]) -> Result<()> {
write_secret_mode(path, contents, 0o600)
}
/// [`write_secret_0600`] with the file's mode given, for a file another user
/// has to read (the group comes from the directory).
pub fn write_secret_mode(path: &Path, contents: &[u8], mode: u32) -> Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
}
@ -34,13 +27,9 @@ pub fn write_secret_mode(path: &Path, contents: &[u8], mode: u32) -> Result<()>
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(mode)
.mode(0o600)
.open(&tmp)
.with_context(|| format!("create {}", tmp.display()))?;
// `open` masks the mode with the process umask; the mode is a property of
// the writer, not of the unit's umask.
f.set_permissions(std::fs::Permissions::from_mode(mode))
.with_context(|| format!("chmod {}", tmp.display()))?;
f.write_all(contents)
.and_then(|()| f.sync_all())
.with_context(|| format!("write {}", tmp.display()))?;
@ -49,7 +38,7 @@ pub fn write_secret_mode(path: &Path, contents: &[u8], mode: u32) -> Result<()>
#[cfg(test)]
mod tests {
use super::{write_secret_0600, write_secret_mode};
use super::write_secret_0600;
use std::os::unix::fs::PermissionsExt as _;
fn mode(path: &std::path::Path) -> u32 {
@ -87,26 +76,4 @@ mod tests {
write_secret_0600(&path, b"new\n").expect("writes");
assert_eq!(std::fs::read(&path).expect("read"), b"new\n");
}
/// A restrictive umask doesn't change the mode asked for. The umask is
/// process-wide, so the write happens in a child run of this test.
#[test]
fn the_mode_survives_a_restrictive_umask() {
const DIR: &str = "SECRET_FILE_UMASK_TEST_DIR";
if let Some(dir) = std::env::var_os(DIR) {
write_secret_mode(&std::path::Path::new(&dir).join("shared"), b"x\n", 0o640)
.expect("writes");
return;
}
let dir = tempfile::tempdir().expect("tempdir");
let status = std::process::Command::new("sh")
.arg("-c")
.arg(r#"umask 077 && exec "$0" --exact secret_file::tests::the_mode_survives_a_restrictive_umask"#)
.arg(std::env::current_exe().expect("test binary"))
.env(DIR, dir.path())
.status()
.expect("spawns the child");
assert!(status.success());
assert_eq!(mode(&dir.path().join("shared")), 0o640);
}
}

View file

@ -1,7 +1,7 @@
//! Background tasks and periodic sweeps: crash/login watcher, boot-time
//! auto-update reconcile, the agent-sockets.json writer loop, the MCP socket listener
//! reconcile loop, knowledge-repo sync, the swarm wanted-state pull, and the
//! swarm resource-cap and permissions reads, and the squid users file.
//! swarm resource-cap and permissions reads.
//! Each submodule is re-exported
//! at the crate root, so `crate::crash_watch::…` etc. keep working unchanged.
@ -11,7 +11,6 @@ pub mod crash_watch;
pub mod knowledge;
pub mod mcp_sockets;
pub mod permissions_import;
pub mod squid_users;
pub mod swarm_limits;
pub mod swarm_permissions;
pub mod wanted;

View file

@ -1,440 +0,0 @@
//! Squid's users file: the hashes of the proxy credentials this hive's agents
//! authenticate to its egress proxy with.
//!
//! swarm-controller mints one credential per agent and one for this hive's own
//! squid self-check ([`swarm_secret_client::proxy`]). Every [`INTERVAL`] this
//! reads the credentials of the agents this hive has, plus the self-check's,
//! and renders `<user>:<hash>` lines into the file named by
//! [`ENV_USERS_FILE`], which squid's `basic_ncsa_auth` re-reads when its mtime
//! moves. The agents are the container list, not the running containers: a
//! stopped agent stays in the file.
//!
//! Only salted SHA-512 crypt hashes reach the disk. A hash is kept as it is
//! while it still verifies its credential, so the file changes only when a
//! user is added, removed or given a new credential.
//!
//! A failed read leaves the file as it is: dropping a user on a store outage
//! would read exactly like the swarm revoking it.
use std::future::Future;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::{Context, Result, bail};
use sha_crypt::{PasswordHasher as _, PasswordVerifier as _, ShaCrypt};
use swarm_secret_client::{SecretStore, client, proxy};
use crate::stats::sweep_health::{self, SweepHealth};
/// How often the file is re-rendered.
pub const INTERVAL: Duration = Duration::from_secs(30);
/// At [`INTERVAL`] this is ~90 seconds of failing passes.
const FAILURES_BEFORE_WARNING: u32 = 3;
/// The environment variable naming the file. Set by `nix/host-modules/hive-c0re`
/// from `services.hyperhive.squid.usersFile`; unset means this host runs no squid.
pub const ENV_USERS_FILE: &str = "HIVE_SQUID_USERS_FILE";
/// Raw salt bytes: twelve of them encode to the sixteen characters `crypt()`
/// accepts.
const SALT_BYTES: usize = 12;
/// The file's mode. The group is squid's, from the setgid directory
/// `nix/host-modules/hive-squid.nix` creates.
const FILE_MODE: u32 = 0o640;
/// One squid user and the credential it logs in with. No `Debug`: `secret` is
/// live.
struct User {
name: String,
secret: String,
}
/// Start the pass loop, when this host has a squid users file, a store and a
/// hive name.
pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
let Some(path) = std::env::var_os(ENV_USERS_FILE)
.filter(|v| !v.is_empty())
.map(PathBuf::from)
else {
return;
};
let Some(hive) = crate::container_view::hive_swarm_names().0 else {
tracing::info!("squid users: HYPERHIVE_HIVE_NAME is unset; no users file is written");
return;
};
if std::env::var_os(client::ENV_ADDR).is_none_or(|v| v.is_empty()) {
tracing::info!(
"squid users: no swarm secret store is configured; no users file is written"
);
return;
}
tokio::spawn(async move {
let mut health = SweepHealth::new("squid_users_render", "warn", FAILURES_BEFORE_WARNING);
loop {
match pass(&hive, &path).await {
Ok(()) => health.record_ok(),
Err(e) => {
tracing::warn!(error = ?e, "squid users: the pass failed");
let err = format!("{e:#}");
health.record_err(|ctx| {
let age = ctx.since_last_ok.map_or_else(
|| "no success this session".to_owned(),
|d| format!("last ok {} ago", sweep_health::fmt_age(d)),
);
format!(
"rendering squid's users file is failing ({} consecutive, {age}) \
— squid keeps the users last written: {err}",
ctx.consecutive
)
});
}
}
tokio::select! {
() = tokio::time::sleep(INTERVAL) => {}
_ = shutdown.changed() => return,
}
}
});
}
/// Read this hive's credentials from the store and render them into `file`.
async fn pass(hive: &str, file: &Path) -> Result<()> {
let agents: Vec<String> = crate::lifecycle::agents_for_meta_listing()
.await
.context("enumerating this hive's agents")?
.into_iter()
.map(|spec| spec.name)
.collect();
let store = SecretStore::from_env(hive)
.await
.context("connecting to the store")?;
sync_with(hive, &agents, file, |path| {
let store = &store;
async move {
let held: Option<proxy::Credential> = store
.read_optional(&path)
.await
.with_context(|| format!("reading {path}"))?;
Ok(held.map(|c| c.value))
}
})
.await
}
/// [`pass`] with the store read and the file given. Nothing is written unless
/// every read succeeds and the rendered file differs from the current one.
async fn sync_with<F, Fut>(hive: &str, agents: &[String], file: &Path, read: F) -> Result<()>
where
F: Fn(String) -> Fut,
Fut: Future<Output = Result<Option<String>>>,
{
let mut wanted = Vec::new();
for agent in agents {
wanted.push((agent.clone(), proxy::agent_credential_path(agent)?));
}
wanted.push((
proxy::SELFCHECK_USER.to_owned(),
proxy::selfcheck_path(hive)?,
));
let mut users = Vec::new();
for (name, path) in wanted {
if let Some(secret) = read(path).await? {
users.push(User { name, secret });
} else {
tracing::debug!(name, "squid users: no proxy credential minted yet");
}
}
let current = match std::fs::read_to_string(file) {
Ok(text) => text,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(),
Err(e) => return Err(e).with_context(|| format!("reading {}", file.display())),
};
let rendered = render(&current, &users)?;
if rendered == current {
return Ok(());
}
crate::secret_file::write_secret_mode(file, rendered.as_bytes(), FILE_MODE)
.with_context(|| format!("writing {}", file.display()))?;
tracing::info!(
users = users.len(),
agents = users.iter().filter(|u| u.name != proxy::SELFCHECK_USER).count(),
file = %file.display(),
"squid users: wrote the users file"
);
Ok(())
}
/// The users file for `users`, sorted by name, one `<user>:<hash>` line each.
///
/// A user whose line in `current` still verifies against its credential keeps
/// that line. Every other line of `current` is dropped, so a destroyed agent,
/// or one that belongs to another hive, does not outlive the pass.
fn render(current: &str, users: &[User]) -> Result<String> {
let mut lines: Vec<String> = Vec::with_capacity(users.len());
for user in users {
if user.name.contains([':', '\n']) {
bail!("{:?} cannot be a squid user name", user.name);
}
let kept = current
.lines()
.filter_map(|l| l.split_once(':'))
.find(|(name, hash)| *name == user.name && verifies(&user.secret, hash))
.map(|(_, hash)| hash.to_owned());
let hash = match kept {
Some(hash) => hash,
None => hash_secret(&user.secret)?,
};
lines.push(format!("{}:{hash}\n", user.name));
}
lines.sort();
Ok(lines.concat())
}
/// A fresh salted SHA-512 crypt hash of `secret`: `$6$rounds=5000$<salt>$<hash>`.
fn hash_secret(secret: &str) -> Result<String> {
let mut salt = [0u8; SALT_BYTES];
getrandom::fill(&mut salt).context("drawing a salt from the kernel's CSPRNG")?;
let hash = ShaCrypt::SHA512
.hash_password_with_salt(secret.as_bytes(), &salt)
.map_err(|e| anyhow::anyhow!("hashing a proxy credential: {e}"))?;
Ok(hash.as_str().to_owned())
}
/// Whether `hash` is `secret`'s SHA-crypt hash.
fn verifies(secret: &str, hash: &str) -> bool {
ShaCrypt::SHA512
.verify_password(secret.as_bytes(), hash)
.is_ok()
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::os::unix::fs::PermissionsExt as _;
use super::*;
const HIVE: &str = "h1";
/// What the store holds, by path.
fn store(entries: &[(&str, &str)]) -> HashMap<String, String> {
entries
.iter()
.map(|(p, v)| ((*p).to_owned(), (*v).to_owned()))
.collect()
}
fn names(names: &[&str]) -> Vec<String> {
names.iter().map(|n| (*n).to_owned()).collect()
}
async fn sync(held: &HashMap<String, String>, agents: &[&str], file: &Path) -> Result<()> {
sync_with(HIVE, &names(agents), file, |path| {
let found = held.get(&path).cloned();
async move { Ok(found) }
})
.await
}
fn lines(file: &Path) -> Vec<(String, String)> {
std::fs::read_to_string(file)
.expect("the file is there")
.lines()
.map(|l| {
let (name, hash) = l.split_once(':').expect("a name and a hash");
(name.to_owned(), hash.to_owned())
})
.collect()
}
fn two_agents_and_a_stranger() -> HashMap<String, String> {
store(&[
("swarm/agents/atlas/proxy-credential", "atlas-secret"),
("swarm/agents/argus/proxy-credential", "argus-secret"),
("swarm/agents/other-hive-agent/proxy-credential", "stranger"),
("swarm/hives/h1/squid-selfcheck", "check-secret"),
("swarm/hives/h2/squid-selfcheck", "other-check"),
])
}
#[tokio::test]
async fn only_this_hives_agents_and_its_selfcheck_are_rendered() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
.await
.expect("renders");
let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
assert_eq!(users, ["argus", "atlas", "selfcheck.squid"]);
}
#[tokio::test]
async fn each_line_is_a_salted_sha512_crypt_hash_of_its_users_credential() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
.await
.expect("renders");
let by_name: HashMap<_, _> = lines(&file).into_iter().collect();
assert!(by_name["atlas"].starts_with("$6$"), "{}", by_name["atlas"]);
assert!(verifies("atlas-secret", &by_name["atlas"]));
assert!(verifies("argus-secret", &by_name["argus"]));
assert!(verifies("check-secret", &by_name["selfcheck.squid"]));
// Control: a hash is of its own user's credential only.
assert!(!verifies("argus-secret", &by_name["atlas"]));
// Salted: the same credential hashes differently twice.
assert_ne!(
hash_secret("atlas-secret").expect("hashes"),
hash_secret("atlas-secret").expect("hashes")
);
}
#[tokio::test]
async fn no_credential_is_written_to_the_file_or_its_directory() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
.await
.expect("renders");
// Control: the search finds a name that is in the file.
let mut found = 0;
for entry in std::fs::read_dir(dir.path()).expect("lists") {
let text = std::fs::read_to_string(entry.expect("entry").path()).expect("text");
assert!(text.contains("atlas:"), "control: {text}");
for secret in ["atlas-secret", "argus-secret", "check-secret"] {
assert!(!text.contains(secret), "{secret} reached the disk");
}
found += 1;
}
assert_eq!(found, 1, "only the users file is left behind");
}
#[tokio::test]
async fn the_file_is_group_readable_and_replaced_atomically() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas"], &file)
.await
.expect("renders");
let mode = std::fs::metadata(&file).expect("stat").permissions().mode() & 0o777;
assert_eq!(mode, 0o640);
let inode = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
.await
.expect("renders");
let after = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
assert_ne!(
inode, after,
"a rename onto the path, not a rewrite in place"
);
assert!(!dir.path().join("users.tmp").exists());
}
#[tokio::test]
async fn an_unchanged_pass_leaves_the_file_alone() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
let held = two_agents_and_a_stranger();
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
let first = std::fs::read_to_string(&file).expect("read");
let inode = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
assert_eq!(std::fs::read_to_string(&file).expect("read"), first);
let again = std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&file).expect("stat"));
assert_eq!(inode, again, "nothing was written");
}
#[tokio::test]
async fn a_changed_credential_gets_a_new_hash_and_the_others_keep_theirs() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
let mut held = two_agents_and_a_stranger();
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
let before: HashMap<_, _> = lines(&file).into_iter().collect();
held.insert(
"swarm/agents/atlas/proxy-credential".to_owned(),
"rotated".to_owned(),
);
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
let after: HashMap<_, _> = lines(&file).into_iter().collect();
assert!(verifies("rotated", &after["atlas"]));
assert_ne!(before["atlas"], after["atlas"]);
assert_eq!(before["argus"], after["argus"]);
}
#[tokio::test]
async fn a_stopped_agent_stays_and_a_destroyed_one_goes() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
let held = two_agents_and_a_stranger();
// The agent list is the container list, so an agent whose container
// is down is in it: nothing here looks at whether anything runs.
sync(&held, &["atlas", "argus"], &file)
.await
.expect("renders");
assert!(lines(&file).iter().any(|(n, _)| n == "argus"));
// Control: once the container is gone the user is dropped, and a line
// for an agent this hive never had does not survive a pass either.
let mut text = std::fs::read_to_string(&file).expect("read");
text.push_str("stray:$6$rounds=5000$saltsaltsaltsalt$x\n");
std::fs::write(&file, text).expect("write");
sync(&held, &["atlas"], &file).await.expect("renders");
let users: Vec<String> = lines(&file).into_iter().map(|(n, _)| n).collect();
assert_eq!(users, ["atlas", "selfcheck.squid"]);
}
#[tokio::test]
async fn an_agent_without_a_credential_yet_is_left_out() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
// The empty file squid's unit creates before its helper starts.
std::fs::write(&file, "").expect("write");
sync(&store(&[]), &["atlas"], &file).await.expect("renders");
assert_eq!(std::fs::read_to_string(&file).expect("read"), "");
}
#[tokio::test]
async fn a_failed_read_leaves_the_file_as_it_was() {
let dir = tempfile::tempdir().expect("tempdir");
let file = dir.path().join("users");
sync(&two_agents_and_a_stranger(), &["atlas", "argus"], &file)
.await
.expect("renders");
let before = std::fs::read_to_string(&file).expect("read");
let failed = sync_with(
HIVE,
&names(&["atlas", "argus"]),
&file,
|path| async move {
if path.contains("argus") {
bail!("the store is down")
}
Ok(Some("atlas-secret".to_owned()))
},
)
.await;
assert!(failed.is_err());
assert_eq!(std::fs::read_to_string(&file).expect("read"), before);
}
#[test]
fn a_name_that_would_break_the_line_is_refused() {
for bad in ["a:b", "a\nb"] {
let user = User {
name: bad.to_owned(),
secret: "s".to_owned(),
};
assert!(render("", &[user]).is_err(), "{bad:?}");
}
}
}

View file

@ -25,7 +25,6 @@
./hive-priv.nix
./hive-priv-cache.nix
./hive-squid.nix
./hive-squid-credential.nix
./hive-tls.nix
./otel.nix
./glue-bao-readers-policy-order.nix

View file

@ -347,11 +347,7 @@ in
# needs access to the rest of c0re's state — that separation
# means c0re itself needs an explicit carve-out to write there.
ProtectSystem = "strict";
# `-` so a host without the squid users directory still starts the daemon.
ReadWritePaths = [
"/var/lib/hive-gateway/conf"
"-${builtins.dirOf config.services.hyperhive.squid.usersFile}"
];
ReadWritePaths = [ "/var/lib/hive-gateway/conf" ];
ProtectKernelTunables = true; # no sysctl writes
ProtectKernelLogs = true; # no /dev/kmsg access needed
ProtectControlGroups = true; # cgroup writes go through hive-priv, not c0re directly

View file

@ -271,9 +271,6 @@ in
# See the LoadCredential in ./default.nix.
BAO_CLIENT_CERT = "%d/bao-client.pem";
BAO_CLIENT_KEY = "%d/bao-client-key.pem";
# Where `workers::squid_users` renders the hashes of this hive's agents' proxy
# credentials for squid; it reads them from the store under the identity above.
HIVE_SQUID_USERS_FILE = config.services.hyperhive.squid.usersFile;
}
// lib.optionalAttrs (haveBaoClientIdentity && baoDeploy.serverCaFile != null) {
# Absent means the system trust store — right for a deployment with a real

View file

@ -1,103 +0,0 @@
# The password this hive's squid self-check logs in to squid with, kept where
# only root can read it.
#
# swarm-controller mints it at `swarm/hives/<hive>/squid-selfcheck`
# (`swarm_secret_client::proxy::selfcheck_path`). hive-c0re reads the same path
# to put its hash in squid's users file; this root oneshot reads it under the
# hive's own store identity and lands it as the systemd credential
# `squid-selfcheck` imports (`/run/credstore/<name>`, root, `0400`, tmpfs).
#
# A hive with no minted credential yet leaves the file alone and the self-check
# reports `credential=absent`.
{
pkgs,
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
hiveController = hyperhiveCfg.deploy.hive-controller;
tls = hiveController.tls;
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = hyperhiveCfg.deploy.bao;
enabled =
hiveController.enable
&& hiveController.hiveName != null
&& tls.baoHiveIdentityCertFile != null
&& tls.baoHiveIdentityKeyFile != null;
credentialFile = "/run/credstore/${hyperhiveCfg.squid.selfcheck.credentialName}";
storeRetry = import ./lib/store-retry.nix { };
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
in
{
config = lib.mkIf enabled {
systemd.services.squid-selfcheck-credential = {
description = "Read the squid self-check's password from the swarm secret store";
wantedBy = [ "multi-user.target" ];
before = [ "squid-selfcheck.service" ];
wants = [ "network-online.target" ];
after = [
"network-online.target"
"systemd-tmpfiles-setup.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.gnugrep
];
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = tls.baoHiveIdentityCertFile;
BAO_CLIENT_KEY = tls.baoHiveIdentityKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
# ./lib/store-retry.nix: the login fails for as long as the store is
# sealed or down.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
UMask = "0077";
SyslogIdentifier = "squid-selfcheck-credential";
TimeoutStartSec = "5min";
};
script = ''
set -euo pipefail
${import ./lib/store-seeded-secret.nix { }}
${import ./lib/atomic-write-secret.nix { }}
${import ./lib/refresh-consumer.nix { }}
install -d -m 0700 /run/credstore
store_init
store_login
if [ "$STORE_UP" != 1 ]; then
echo "could not read the squid self-check credential; ${credentialFile} stays as it is." >&2
exit 1
fi
if ! value="$(store_bao kv get -field=value ${lib.escapeShellArg "secret/swarm/hives/${hiveController.hiveName}/squid-selfcheck"} 2>"$STORE_ERR")"; then
if grep -qi 'no value found' "$STORE_ERR"; then
echo "the swarm has not minted this hive's squid self-check credential yet; ${credentialFile} stays as it is."
exit 0
fi
cat "$STORE_ERR" >&2
exit 1
fi
if [[ ! "$value" =~ ^[A-Za-z0-9_-]+$ ]]; then
echo "the stored squid self-check credential is not one base64url word; ${credentialFile} stays as it is." >&2
exit 1
fi
# Written only when the value changed, so the file keeps its mtime.
if secret_differs ${credentialFile} "$value"; then
atomic_write_secret 0400 root:root ${credentialFile} "$value"
fi
'';
};
systemd.timers.squid-selfcheck-credential = storeRefreshTimer "re-read the squid self-check's password";
};
}

View file

@ -1,9 +1,8 @@
# Egress proxy for this hive's agents: squid on the host, bound to the bridge
# IP, default-deny on the destination hostname. It filters CONNECT targets and
# plain-HTTP hosts and never decrypts, so it holds no signing key. A request
# for a listed name must carry a per-agent proxy credential, checked against the
# users file hive-c0re renders. No agent sends one yet; the exporter, journal
# forwarding and the self-check below are how a deploy shows it works.
# plain-HTTP hosts and never decrypts, so it holds no signing key. No agent is
# pointed at it yet; the exporter, journal forwarding and the self-check below
# are how a deploy shows it works.
{
pkgs,
lib,
@ -22,20 +21,13 @@ let
allowed = lib.unique (cfg.allowedDomains ++ [ cfg.selfcheck.canary ]);
# The squid user the self-check logs in as: `swarm_secret_client::proxy::SELFCHECK_USER`.
selfcheckUser = "selfcheck.squid";
usersDir = builtins.dirOf cfg.usersFile;
# Evaluated top to bottom, first match wins. The three destination denies
# sit before the only allow: squid resolves names on the host, so a name
# that answers 127.0.0.1, 169.254.169.254 or a bridge address would
# otherwise reach services the bridge firewall keeps agents out of. The
# unlisted-name deny comes first because `dstdomain` needs no lookup while
# the `dst` ACLs do: an unlisted name is refused at once instead of waiting
# on the host resolver, and costs a client no DNS query. The credential
# check sits right before the allow, so a refused destination answers 403
# to a client with or without a credential, and only a request that would
# otherwise be let through gets the 407 challenge.
# on the host resolver, and costs a client no DNS query.
squidConf = ''
http_port ${netCfg.bridgeIp}:${toString port}
pid_filename /run/squid/squid.pid
@ -48,17 +40,8 @@ let
# Cachemgr scrapes are left out.
access_log syslog:daemon.info squid !manager
# The helper reads the users file on every request and reloads it when its
# mtime moves. It exits when the file is missing, which is why the file is
# created before squid starts. A positive answer is cached for a minute, so
# that is also how long a removed user keeps working.
auth_param basic program ${config.services.squid.package}/libexec/basic_ncsa_auth ${cfg.usersFile}
auth_param basic realm hyperhive egress
auth_param basic credentialsttl 1 minute
acl SSL_ports port 443
acl Safe_ports port 80 443
acl authed proxy_auth REQUIRED
# Host-originated requests: the self-check and the exporter. A local
# connect to the bridge IP takes the bridge IP as its source.
@ -86,7 +69,6 @@ let
http_access deny to_localhost
http_access deny to_linklocal
http_access deny to_bridge
http_access deny !authed
${lib.optionalString (allowed != [ ]) "http_access allow allowed_dst"}
http_access deny all
'';
@ -119,39 +101,20 @@ let
name = "squid-selfcheck";
runtimeInputs = [ pkgs.curl ];
text = ''
# The credential comes from the unit's imported credential, and goes to
# curl on its stdin so it never appears in a command line. Without one
# the requests are anonymous and the self-check reports it.
secret=""
credential_file="''${CREDENTIALS_DIRECTORY:-/nonexistent}/${cfg.selfcheck.credentialName}"
if [ -s "$credential_file" ]; then
secret=$(< "$credential_file")
fi
# <host> <auth|anon>: the status squid answers a CONNECT with; 000 when
# the proxy is down.
# The status squid answers a CONNECT with; 000 when the proxy is down.
connect_status() {
{
if [ "$2" = auth ] && [ -n "$secret" ]; then
printf 'proxy-user = "%s:%s"\n' ${selfcheckUser} "$secret"
fi
} | curl -sS -m 10 -K - --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
curl -sS -m 10 --interface ${netCfg.bridgeIp} -p -x ${proxyUrl} \
-o /dev/null -w '%{http_connect}' "https://$1/" 2>/dev/null || true
}
allow_status=$(connect_status ${cfg.selfcheck.canary} auth)
deny_status=$(connect_status ${denyCanary} auth)
anon_status=$(connect_status ${cfg.selfcheck.canary} anon)
allow_status=$(connect_status ${cfg.selfcheck.canary})
deny_status=$(connect_status ${denyCanary})
ok=false
if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ] && [ "$anon_status" = 407 ]; then
if [ "$allow_status" = 200 ] && [ "$deny_status" = 403 ]; then
ok=true
fi
credential=absent
if [ -n "$secret" ]; then
credential=present
fi
echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status anon_status=$anon_status credential=$credential"
echo "squid-selfcheck ok=$ok allow_canary=${cfg.selfcheck.canary} allow_status=$allow_status deny_canary=${denyCanary} deny_status=$deny_status"
[ "$ok" = true ]
'';
};
@ -173,31 +136,6 @@ in
'';
};
usersFile = lib.mkOption {
type = lib.types.path;
default = "/var/lib/hive-squid/users";
description = ''
The `<user>:<hash>` file squid checks proxy credentials against. hive-c0re
renders it from the credentials in the swarm secret store, one line per
agent of this hive and one for the self-check, as SHA-512 crypt hashes.
An empty file, which is what exists until hive-c0re writes one, makes
every login fail, and every request to a listed name answers 407.
Owned by `hive-core` and readable by the `squid` group.
'';
};
selfcheck.credentialName = lib.mkOption {
type = lib.types.str;
default = "hive-squid.selfcheck";
readOnly = true;
description = ''
Name of the systemd credential the self-check imports its squid
password from. `squid-selfcheck-credential` writes it to
`/run/credstore/<name>`. Without one the self-check sends no
credential and reports `ok=false`.
'';
};
selfcheck.canary = lib.mkOption {
type = lib.types.str;
default = hyperhiveCfg.swarm.forge.domain;
@ -229,14 +167,6 @@ in
configText = squidConf;
};
# The directory is setgid so the file hive-c0re renames into it is the
# squid group's. The file exists before squid starts, empty: the helper
# exits on a missing file and squid then rejects every login.
systemd.tmpfiles.rules = [
"d ${usersDir} 2750 hive-core squid - -"
"f ${cfg.usersFile} 0640 hive-core squid - -"
];
systemd.services.squid = {
serviceConfig = sandbox // {
User = "squid";
@ -274,13 +204,10 @@ in
};
systemd.services.squid-selfcheck = {
description = "Squid allow, deny and credential canaries";
description = "Squid allow and deny canaries";
after = [ "squid.service" ];
serviceConfig = sandbox // {
Type = "oneshot";
# A glob that matches nothing is not an error: a host with no
# credential yet still runs the self-check, and it reports that.
ImportCredential = cfg.selfcheck.credentialName;
ExecStart = lib.getExe selfcheck;
DynamicUser = true;
IPAddressAllow = netCfg.bridgeIp;

View file

@ -486,10 +486,6 @@ let
capabilities = ["create", "read", "update"]
}
path "${credentialMountPath}/data/swarm/hives/+/squid-selfcheck" {
capabilities = ["create", "read"]
}
${lib.concatMapStrings readStanza (
map (leaf: "${credentialMountPath}/data/${leaf}") atticFactLeaves
)}
@ -942,13 +938,12 @@ let
inherit hiveName;
}) hyperhiveCfg.swarm.hives;
# What a hive's gateway-leaf identity may do: issue from its own role, read
# two of its own cache facts, and read its own squid self-check credential.
# The root unit on the hive host (`hive-priv-cache-facts`,
# ./hive-priv-cache.nix) reads the facts under this identity: the key
# hive-priv's downloads must be signed by, and the token it pulls with. The
# cache's URL and name come from the host's configuration, not from the
# store. `squid-selfcheck-credential` (./hive-squid.nix) reads the credential.
# What a hive's gateway-leaf identity may do: issue from its own role, and
# read two of its own cache facts. The root unit on the hive host
# (`hive-priv-cache-facts`, ./hive-priv-cache.nix) reads them under this
# identity: the key hive-priv's downloads must be signed by, and the token it
# pulls with. The cache's URL and name come from the host's configuration,
# not from the store.
hiveLeafPolicyText =
obj:
''
@ -962,8 +957,7 @@ let
[
"public-key"
"pull-token"
]
+ readStanza "${credentialMountPath}/data/swarm/hives/${obj.hiveName}/squid-selfcheck";
];
# The PKI mount agent client certificates are issued from. Its root is
# generated inside the store, so the agent CA's key never exists outside it.

View file

@ -1496,10 +1496,9 @@ let
{
# One role, policy and login role per hive in the directory, each
# naming only its own hive: the policy text in the unit is exactly the
# `update` on its own role and `read` on its own two cache facts and its own
# squid self-check credential, and each
# `update` on its own role and `read` on its own two cache facts, and each
# hive's block names neither the other's role nor its subject.
name = "each hive's gateway-leaf identity may issue from its own role and read its own two cache facts and squid self-check credential, and nothing else";
name = "each hive's gateway-leaf identity may issue from its own role and read its own two cache facts, and nothing else";
ok =
let
s = baoGrantTwoHives.systemd.services.swarm-bao-hive-leaf-policy.script;
@ -1537,10 +1536,6 @@ let
path = "secret/data/swarm/hives/h1/attic/pull-token";
caps = [ "read" ];
}
{
path = "secret/data/swarm/hives/h1/squid-selfcheck";
caps = [ "read" ];
}
{
path = "pki/issue/swarm-hive-leaf-h2";
caps = [ "update" ];
@ -1553,10 +1548,6 @@ let
path = "secret/data/swarm/hives/h2/attic/pull-token";
caps = [ "read" ];
}
{
path = "secret/data/swarm/hives/h2/squid-selfcheck";
caps = [ "read" ];
}
]
&& issues "h1" "h1.t.local" h1
&& issues "h2" "h2.example.org" h2
@ -1733,20 +1724,17 @@ let
{
# `matrix_account::hive_sender` writes every hive's sender token and
# `forge::hive_token` every hive's forge token, and nothing else under
# `hives/`, where each hive's queue credential also lives. `proxy_credential`
# creates each hive's squid self-check credential and reads it back, and
# has no `update`, so it cannot replace one. `+` is one
# `hives/`, where each hive's queue credential also lives. `+` is one
# segment, which keeps each stanza to its one leaf; a `*` is a glob only
# at the end of a path. Pinned as whole stanzas, so an added capability
# fails.
name = "the controller writes each hive's sender and forge tokens, creates its squid self-check credential, and nothing else under hives";
name = "the controller writes each hive's sender and forge tokens and nothing else under hives";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/hives/+/matrix/sender-token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/hives/+/forge/token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/hives/+/squid-selfcheck\" {\n capabilities = [\"create\", \"read\"]\n}" s
&& !(lib.hasInfix "secret/data/swarm/hives/*" s)
&& !(lib.hasInfix "secret/metadata/swarm/hives" s);
}

View file

@ -485,62 +485,26 @@ let
&& !(lib.hasInfix "localnet" conf);
}
{
# First match wins, so a destination deny after the allow is dead, and
# a credential check after it never challenges anyone. The two IP-literal
# denies stay ahead of the unlisted-name deny: `dstdomain` on an IP
# literal does a reverse lookup.
name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, then a missing credential, before any allow";
# First match wins, so a destination deny after the allow is dead.
name = "squid refuses unlisted names, then loopback, link-local and bridge destinations, before any allow";
ok =
let
lines = lib.splitString "\n" hiveOtel.services.squid.configText;
at = l: lib.lists.findFirstIndex (x: x == l) null lines;
allow = at "http_access allow allowed_dst";
authed = at "http_access deny !authed";
in
allow != null
&& authed != null
&& at "http_access deny to_localhost" != null
&& at "http_access deny to_localhost" < allow
&& at "http_access deny to_linklocal" != null
&& at "http_access deny to_linklocal" < allow
&& at "http_access deny to_bridge" != null
&& at "http_access deny to_bridge" < allow
&& at "http_access deny to_localhost" < authed
&& at "http_access deny to_linklocal" < authed
&& at "http_access deny to_bridge" < authed
&& authed < allow
&& at "http_access deny ip_v4_host" != null
&& at "http_access deny ip_v4_host" < at "http_access deny !allowed_dst"
&& at "http_access deny ip_v6_host" < at "http_access deny !allowed_dst"
&& at "http_access deny !allowed_dst" != null
&& at "http_access deny !allowed_dst" < at "http_access deny to_localhost"
&& builtins.elem "cache deny all" lines
&& lib.last (lib.filter (l: l != "") lines) == "http_access deny all";
}
{
# The helper is the NCSA one, on the file hive-c0re renders. The file
# exists, empty, before squid starts (the helper exits on a missing one),
# and the self-check imports its credential without failing when there is
# none.
name = "squid authenticates with the NCSA helper on a users file that exists before it starts";
ok =
let
conf = hiveOtel.services.squid.configText;
lines = lib.splitString "\n" conf;
users = hiveOtel.services.hyperhive.squid.usersFile;
rules = hiveOtel.systemd.tmpfiles.rules;
sc = hiveOtel.systemd.services.squid-selfcheck.serviceConfig;
in
lib.any (
l: lib.hasInfix "/libexec/basic_ncsa_auth ${users}" l && lib.hasPrefix "auth_param basic program " l
) lines
&& builtins.elem "acl authed proxy_auth REQUIRED" lines
&& users == "/var/lib/hive-squid/users"
&& builtins.elem "d /var/lib/hive-squid 2750 hive-core squid - -" rules
&& builtins.elem "f ${users} 0640 hive-core squid - -" rules
&& sc.ImportCredential == hiveOtel.services.hyperhive.squid.selfcheck.credentialName
&& hiveOtelAllLocal.systemd.services.hive-c0re.environment.HIVE_SQUID_USERS_FILE == users;
}
{
name = "squid's port is open on the bridge interface and on no other";
ok =

View file

@ -800,28 +800,6 @@ let
noService.systemd.services.hive-priv-cache-facts.environment.BAO_CLIENT_CERT
== "/etc/pki/bao-hive.pem";
}
{
# Control: the lookup finds the sibling root unit of the same shape. The
# squid self-check's password is read under this hive's own identity and
# lands where only root reads it, as the credential `squid-selfcheck`
# imports.
name = "the squid self-check credential is read by a root oneshot under this hive's own identity into the credstore";
ok =
let
u = units.squid-selfcheck-credential;
in
lib.hasInfix "secret/swarm/hives/h1/squid-selfcheck" u.script
&& !(lib.hasInfix "swarm/hives/h2" u.script)
&& lib.hasInfix "/run/credstore/hive-squid.selfcheck" u.script
&& lib.hasInfix "atomic_write_secret 0400 root:root" u.script
&& u.environment.BAO_CLIENT_CERT == units.hive-priv-cache-facts.environment.BAO_CLIENT_CERT
&& u.environment.BAO_CLIENT_KEY == units.hive-priv-cache-facts.environment.BAO_CLIENT_KEY
&& u.serviceConfig.Type == "oneshot"
&& !(remainsAfterExit u)
&& allLocal.systemd.timers ? squid-selfcheck-credential
&& !(noIdentity.systemd.services ? squid-selfcheck-credential)
&& !(noIdentity.systemd.timers ? squid-selfcheck-credential);
}
{
name = "hive-priv is told the cache's address by the host's configuration";
ok =

View file

@ -55,12 +55,6 @@ let
};
};
};
# The user hive-c0re runs as, which owns the users file squid reads.
config.users.users.hive-core = {
isSystemUser = true;
group = "hive-core";
};
config.users.groups.hive-core = { };
# The firewall hole hive-network.nix opens for `exposeHostPorts`.
config.networking.firewall.interfaces.eth1.allowedTCPPorts =
config.services.hyperhive.network.exposeHostPorts;
@ -83,7 +77,7 @@ pkgs.testers.runNixOSTest {
nodes = {
proxy =
{ lib, pkgs, ... }:
{ lib, ... }:
{
imports = [
hiveStubs
@ -117,8 +111,6 @@ pkgs.testers.runNixOSTest {
${bridgeIp} = [ "bridge.t.local" ];
"169.254.169.254" = [ "meta.t.local" ];
};
# Renders the users file the way hive-c0re does.
environment.systemPackages = [ pkgs.mkpasswd ];
services.hyperhive.squid.allowedDomains = [
"forge.t.local"
"loop.t.local"
@ -156,15 +148,6 @@ pkgs.testers.runNixOSTest {
testScript = ''
import re
import secrets
# Drawn fresh each run. The users file holds only their hashes, as
# hive-c0re renders it.
PW = {u: secrets.token_urlsafe(24) for u in ("atlas", "selfcheck.squid", "ruth")}
ATLAS = "atlas:" + PW["atlas"]
SELFCHECK = "selfcheck.squid:" + PW["selfcheck.squid"]
RUTH = "ruth:" + PW["ruth"]
USERS = "/var/lib/hive-squid/users"
start_all()
proxy.wait_for_unit("squid.service")
@ -173,64 +156,29 @@ pkgs.testers.runNixOSTest {
web.wait_for_unit("nginx.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
def connect(host, cred=None):
def connect(host):
# The status squid answered the CONNECT with; 000 when none came back.
auth = f"-U '{cred}' " if cred else ""
return agent.succeed(
"curl -sS -m 15 -p -x http://${bridgeIp}:3128 " + auth + "-o /dev/null "
"curl -sS -m 15 -p -x http://${bridgeIp}:3128 -o /dev/null "
f"-w '%{{http_connect}}' https://{host}/ 2>/dev/null || true"
).strip()
def write_users(*creds):
# What hive-c0re does: hashes only, written aside and renamed into
# place, in squid's group. The sleep keeps two writes in different
# seconds, which is how the helper tells the file changed.
lines = " ".join(
f"printf '%s:%s\\n' {u} \"$(mkpasswd -m sha-512 -s <<< {p})\";"
for u, p in (c.split(":", 1) for c in creds)
)
proxy.succeed(
f"({lines}) > {USERS}.new && chown hive-core:squid {USERS}.new "
f"&& chmod 640 {USERS}.new && mv {USERS}.new {USERS} && sleep 1"
)
with subtest("squid starts on the empty users file and refuses every login"):
assert proxy.succeed(f"stat -c '%U:%G %a' {USERS}").strip() == "hive-core:squid 640"
assert proxy.succeed("stat -c %a /var/lib/hive-squid").strip() == "2750"
assert proxy.succeed(f"stat -c %s {USERS}").strip() == "0"
assert connect("forge.t.local", ATLAS) == "407", "no such user"
assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name needs no login"
with subtest("the users file holds hashes, never a password"):
write_users(ATLAS, SELFCHECK)
users = proxy.succeed(f"cat {USERS}")
assert re.fullmatch(r"(atlas|selfcheck\.squid):\$6\$\S+\n(atlas|selfcheck\.squid):\$6\$\S+\n", users), users
assert not any(p in users for p in PW.values()), users
with subtest("an allowed name is tunnelled with a credential and challenged without one"):
assert connect("forge.t.local", ATLAS) == "200", "allow canary"
assert connect("forge.t.local") == "407", "no credential"
assert connect("forge.t.local", "atlas:" + secrets.token_urlsafe(24)) == "407", "wrong password"
assert connect("forge.t.local", "nobody:" + PW["atlas"]) == "407", "unknown user"
# DES would accept the first eight characters; the hash must not.
assert connect("forge.t.local", "atlas:" + PW["atlas"][:8]) == "407", "a prefix of the password"
with subtest("an unlisted name is refused with or without a credential"):
assert connect("deny-canary.invalid", ATLAS) == "403", "deny canary"
assert connect("deny-canary.invalid") == "403", "deny canary, anonymous"
with subtest("an allowed name is tunnelled and an unlisted one is refused"):
assert connect("forge.t.local") == "200", "allow canary"
assert connect("deny-canary.invalid") == "403", "deny canary"
with subtest("a name resolving to a host-local destination is refused"):
assert connect("loop.t.local", ATLAS) == "403", "loopback"
assert connect("bridge.t.local", ATLAS) == "403", "bridge address"
assert connect("meta.t.local", ATLAS) == "403", "link-local"
assert connect("loop.t.local") == "403", "loopback"
assert connect("bridge.t.local") == "403", "bridge address"
assert connect("meta.t.local") == "403", "link-local"
with subtest("IP literals are refused even when the address is reachable"):
assert connect("127.0.0.1", ATLAS) == "403"
assert connect("192.168.2.2", ATLAS) == "403"
assert connect("127.0.0.1") == "403"
assert connect("192.168.2.2") == "403"
with subtest("only a port-443 CONNECT is allowed"):
assert connect("forge.t.local:22", ATLAS) == "403"
assert connect("forge.t.local:80", ATLAS) == "403"
assert connect("forge.t.local:22") == "403"
assert connect("forge.t.local:80") == "403"
with subtest("an agent cannot read cachemgr"):
agent.succeed(
@ -238,42 +186,19 @@ pkgs.testers.runNixOSTest {
"http://${bridgeIp}:3128/squid-internal-mgr/counters) = 403"
)
with subtest("the self-check passes with its credential"):
proxy.succeed("install -d -m 0700 /run/credstore")
proxy.succeed(f"printf '%s\\n' {PW['selfcheck.squid']} > /run/credstore/hive-squid.selfcheck")
proxy.succeed("chmod 400 /run/credstore/hive-squid.selfcheck")
with subtest("the self-check passes against the same squid"):
proxy.succeed("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'squid-selfcheck ok=true allow_canary=forge.t.local "
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403 "
"anon_status=407 credential=present'"
"allow_status=200 deny_canary=deny-canary.invalid deny_status=403'"
)
with subtest("the self-check fails, and says why, without a credential or with the wrong one"):
proxy.succeed("mv /run/credstore/hive-squid.selfcheck /run/credstore/kept")
proxy.fail("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'squid-selfcheck ok=false allow_canary=forge.t.local "
"allow_status=407 deny_canary=deny-canary.invalid deny_status=403 "
"anon_status=407 credential=absent'"
)
proxy.succeed(f"printf '%s\\n' {secrets.token_urlsafe(24)} > /run/credstore/hive-squid.selfcheck")
proxy.fail("systemctl start squid-selfcheck.service")
proxy.succeed(
"journalctl -u squid-selfcheck.service -o cat | "
"grep -F 'allow_status=407 deny_canary=deny-canary.invalid deny_status=403 "
"anon_status=407 credential=present'"
)
proxy.succeed("mv -f /run/credstore/kept /run/credstore/hive-squid.selfcheck")
with subtest("squid's access log reaches the journal under squid.service, with the user"):
with subtest("squid's access log reaches the journal under squid.service"):
journal = proxy.succeed("journalctl -u squid.service -o cat")
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443 atlas ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/407 .* CONNECT forge\.t\.local:443 - ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443 ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443 ", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_TUNNEL/200 .* CONNECT forge\.t\.local:443", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT deny-canary\.invalid:443", journal), journal
assert re.search(r"192\.168\.1\.3 TCP_DENIED/403 .* CONNECT loop\.t\.local:443", journal), journal
# Exporter scrapes are cachemgr requests and stay out of the log.
assert "squid-internal-mgr" not in journal, journal
@ -290,21 +215,6 @@ pkgs.testers.runNixOSTest {
with subtest("squid survives a reload"):
proxy.succeed("systemctl reload squid.service")
proxy.wait_for_unit("squid.service")
assert connect("forge.t.local", ATLAS) == "200"
with subtest("a user added while squid runs can log in, with no reload"):
write_users(ATLAS, SELFCHECK, RUTH)
assert connect("forge.t.local", RUTH) == "200"
assert connect("forge.t.local", ATLAS) == "200"
with subtest("squid starts on a missing users file, refuses logins, and recovers when it appears"):
proxy.succeed(f"rm {USERS}")
proxy.succeed("systemctl restart squid.service")
proxy.wait_for_unit("squid.service")
proxy.wait_for_open_port(3128, "${bridgeIp}")
assert connect("forge.t.local", ATLAS) == "407", "no file, no login"
assert connect("deny-canary.invalid", ATLAS) == "403", "an unlisted name still answers"
write_users(ATLAS, SELFCHECK)
assert connect("forge.t.local", ATLAS) == "200", "the file appeared"
assert connect("forge.t.local") == "200"
'';
}

Some files were not shown because too many files have changed in this diff Show more