vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarmctl and the authelia bridge serialise users.yml writes with a shared flock

Browse source
This commit is contained in:
atlas 2026-10-17 14:03:57 +02:00 • committed by mara
commit be744d45b9
7 changed files with 436 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

229
swarmctl/src/main.rs
View file

@ -39,6 +39,7 @@ use std::io::{self, Write as _};
use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _, PermissionsExt as _};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{Duration, Instant};
use anyhow::{Context, Result, bail};
use clap::{Args, Parser, Subcommand};
@ -758,6 +759,7 @@ fn schedule(remote: &RemoteArgs, command: ScheduleVerb) -> Result<()> {
fn user_add(paths: &Paths, args: AddArgs) -> Result<()> {
users::validate_username(&args.username)?;
let _lock = lock_users_file(&paths.users_file)?;
let mut store = load_store(&paths.users_file)?;
if store.users.contains_key(&args.username) {
bail!(
@ -789,6 +791,7 @@ fn user_add(paths: &Paths, args: AddArgs) -> Result<()> {
}
fn user_reset_password(paths: &Paths, args: &ResetPasswordArgs) -> Result<()> {
let _lock = lock_users_file(&paths.users_file)?;
let mut store = load_store(&paths.users_file)?;
let Some(user) = store.users.get_mut(&args.username) else {
bail!(
@ -814,6 +817,7 @@ fn user_reset_password(paths: &Paths, args: &ResetPasswordArgs) -> Result<()> {
}
fn user_update(paths: &Paths, args: UpdateArgs) -> Result<()> {
let _lock = lock_users_file(&paths.users_file)?;
let mut store = load_store(&paths.users_file)?;
let Some(user) = store.users.get_mut(&args.username) else {
bail!(
@ -981,6 +985,81 @@ fn parse_field(stdout: &str, marker: &str) -> Option<String> {
.filter(|value| !value.is_empty())
}
/// How long a writer waits for the other process's `users.yml` update.
const USERS_LOCK_WAIT: Duration = Duration::from_secs(10);
/// Take the exclusive `flock` on `<users_file>.lock`, held until the
/// returned file drops.
///
/// `swarm-authelia-bridge` rewrites the same file from inside the
/// `swarm-authelia` container and takes the same lock on the same inode.
/// Every verb holds it across load → modify → [`publish`]: the rename alone
/// would not stop two writers loading the same snapshot, and the second
/// rename would drop the first one's user.
///
/// The lock file is a sibling of `users_file`, not `users_file` itself,
/// because `write_atomic` replaces that inode on every write. The directory
/// belongs to a less-privileged uid, so the lock file is opened `O_NOFOLLOW`
/// (a planted symlink fails the open) and `O_NONBLOCK` (a planted FIFO cannot
/// hang it), and must be a regular file. A file created here is handed to the
/// directory's owner by descriptor so that uid can open it too.
fn lock_users_file(users_file: &Path) -> Result<File> {
lock_users_file_within(users_file, USERS_LOCK_WAIT)
}
fn lock_users_file_within(users_file: &Path, wait: Duration) -> Result<File> {
let dir = users_file
.parent()
.with_context(|| format!("{} has no parent directory", users_file.display()))?;
fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?;
let name = users_file
.file_name()
.with_context(|| format!("{} has no file name", users_file.display()))?;
let mut lock_name = name.to_os_string();
lock_name.push(".lock");
let lock_path = dir.join(lock_name);
let flags = rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK;
let file = fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(false)
.mode(0o600)
.custom_flags(flags.bits().cast_signed())
.open(&lock_path)
.with_context(|| format!("opening {}", lock_path.display()))?;
let meta = file
.metadata()
.with_context(|| format!("stat {}", lock_path.display()))?;
if !meta.is_file() {
bail!("{} is not a regular file", lock_path.display());
}
let dir_meta = fs::metadata(dir).with_context(|| format!("stat {}", dir.display()))?;
if (meta.uid(), meta.gid()) != (dir_meta.uid(), dir_meta.gid()) {
std::os::unix::fs::fchown(&file, Some(dir_meta.uid()), Some(dir_meta.gid()))
.with_context(|| format!("setting owner on {}", lock_path.display()))?;
}
let deadline = Instant::now() + wait;
loop {
match rustix::fs::flock(&file, rustix::fs::FlockOperation::NonBlockingLockExclusive) {
Ok(()) => return Ok(file),
Err(e) if e == rustix::io::Errno::WOULDBLOCK && Instant::now() < deadline => {
std::thread::sleep(Duration::from_millis(50));
}
Err(e) if e == rustix::io::Errno::WOULDBLOCK => bail!(
"timed out after {wait:?} waiting for {}: another process (the \
authelia bridge or another `swarmctl user` verb) is updating {}",
lock_path.display(),
users_file.display()
),
Err(e) => {
return Err(e).with_context(|| format!("locking {}", lock_path.display()));
}
}
}
}
/// Replace `path`'s contents atomically, preserving the existing owner
/// and mode.
///
@ -1388,6 +1467,156 @@ mod tests {
fs::remove_dir_all(&dir).ok();
}
/// A scratch directory holding a `users.yml` with one user, `carol`, and
/// a stand-in `authelia` that takes `delay` seconds to mint a password.
fn locked_users_fixture(tag: &str, delay: &str) -> Paths {
let dir = std::env::temp_dir().join(format!("swarmctl-lock-{tag}-{}", std::process::id()));
let _ = fs::remove_dir_all(&dir);
fs::create_dir_all(&dir).expect("temp dir");
let authelia_bin = dir.join("authelia");
fs::write(
&authelia_bin,
format!(
"#!/bin/sh\nsleep {delay}\nprintf 'Random Password: pw\\nDigest: $argon2id$x\\n'\n"
),
)
.expect("fake authelia");
fs::set_permissions(&authelia_bin, Permissions::from_mode(0o755)).expect("chmod");
let users_file = dir.join("users.yml");
fs::write(
&users_file,
"users:\n carol:\n displayname: carol\n password: x\n groups: []\n",
)
.expect("seed");
Paths {
authelia_bin,
users_file,
}
}
fn add_args(username: &str) -> AddArgs {
AddArgs {
username: username.to_owned(),
display_name: None,
email: None,
groups: Vec::new(),
}
}
#[test]
fn the_users_lock_excludes_a_second_holder_until_it_is_released() {
let paths = locked_users_fixture("exclusive", "0");
let held = lock_users_file(&paths.users_file).expect("first lock");
let err = lock_users_file_within(&paths.users_file, Duration::from_millis(200))
.expect_err("a held lock must not be granted twice");
assert!(err.to_string().contains("timed out"), "{err}");
drop(held);
lock_users_file_within(&paths.users_file, Duration::from_millis(200))
.expect("a released lock is granted");
fs::remove_dir_all(paths.users_file.parent().unwrap()).ok();
}
#[test]
fn the_users_lock_refuses_a_symlink_planted_at_the_lock_path() {
let paths = locked_users_fixture("symlink", "0");
let dir = paths.users_file.parent().unwrap();
let target = dir.join("victim");
fs::write(&target, "keep").expect("victim");
std::os::unix::fs::symlink(&target, dir.join("users.yml.lock")).expect("plant");
lock_users_file(&paths.users_file).expect_err("a symlinked lock path must be refused");
assert_eq!(fs::read_to_string(&target).expect("read"), "keep");
fs::remove_dir_all(dir).ok();
}
/// Both adds load the file before either publishes (each spends 0.5s in
/// authelia), so without the lock the second rename drops the first user.
#[test]
fn two_concurrent_user_adds_both_land() {
let paths = std::sync::Arc::new(locked_users_fixture("adds", "0.5"));
let handles: Vec<_> = ["alice", "bob"]
.into_iter()
.map(|name| {
let paths = std::sync::Arc::clone(&paths);
std::thread::spawn(move || user_add(&paths, add_args(name)))
})
.collect();
for handle in handles {
handle.join().expect("thread").expect("user add");
}
let store = load_store(&paths.users_file).expect("reload");
let names: Vec<_> = store.users.keys().map(String::as_str).collect();
assert_eq!(names, ["alice", "bob", "carol"]);
fs::remove_dir_all(paths.users_file.parent().unwrap()).ok();
}
#[test]
fn every_user_verb_waits_for_the_users_lock() {
let paths = std::sync::Arc::new(locked_users_fixture("verbs", "0"));
let held = lock_users_file(&paths.users_file).expect("lock");
let (add_tx, add_rx) = std::sync::mpsc::channel();
let (reset_tx, reset_rx) = std::sync::mpsc::channel();
let (update_tx, update_rx) = std::sync::mpsc::channel();
let p = std::sync::Arc::clone(&paths);
std::thread::spawn(move || add_tx.send(user_add(&p, add_args("dave"))));
let p = std::sync::Arc::clone(&paths);
std::thread::spawn(move || {
let args = ResetPasswordArgs {
username: "carol".to_owned(),
};
reset_tx.send(user_reset_password(&p, &args))
});
let p = std::sync::Arc::clone(&paths);
std::thread::spawn(move || {
let args = UpdateArgs {
username: "carol".to_owned(),
display_name: None,
email: None,
add_groups: vec!["ops".to_owned()],
remove_groups: Vec::new(),
};
update_tx.send(user_update(&p, args))
});
let wait = Duration::from_millis(300);
assert!(add_rx.recv_timeout(wait).is_err(), "user add ran unlocked");
assert!(
reset_rx.recv_timeout(wait).is_err(),
"user reset-password ran unlocked"
);
assert!(
update_rx.recv_timeout(wait).is_err(),
"user update ran unlocked"
);
drop(held);
let done = Duration::from_secs(5);
add_rx
.recv_timeout(done)
.expect("add finishes")
.expect("add");
reset_rx
.recv_timeout(done)
.expect("reset finishes")
.expect("reset");
update_rx
.recv_timeout(done)
.expect("update finishes")
.expect("update");
let store = load_store(&paths.users_file).expect("reload");
assert!(store.users.contains_key("dave"));
assert_eq!(store.users["carol"].groups, ["ops"]);
fs::remove_dir_all(paths.users_file.parent().unwrap()).ok();
}
#[test]
fn parses_authelia_hash_output() {
let out = "Random Password: hunter2\nDigest: $argon2id$v=19$m=65536$abc\n";