swarmctl and the authelia bridge serialise users.yml writes with a shared flock

swarm-authelia-bridge/src/store.rs

@@ -19,8 +19,9 @@
 use std::collections::BTreeMap;
 use std::fs::{self, File, Permissions};
 use std::io::Write as _;
-use std::os::unix::fs::PermissionsExt as _;
+use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _};
 use std::path::Path;
+use std::time::Duration;
 
 use anyhow::{Context, Result, bail};
 use serde::{Deserialize, Serialize};
@@ -277,6 +278,72 @@ fn fill_missing_emails(store: &mut UserStore) {
     }
 }
 
+/// How long a writer waits for the other process's `users.yml` update.
+pub const USERS_LOCK_WAIT: Duration = Duration::from_secs(10);
+
+/// Take the exclusive `flock` on `<users_file>.lock`, held until the
+/// returned file drops.
+///
+/// `swarmctl` rewrites the same file from the host and takes the same lock on
+/// the same inode. `handle` holds it across load → modify → [`publish`]:
+/// the in-process `write_lock` does not cover that other process, and the
+/// rename alone would not stop two writers loading the same snapshot.
+///
+/// The lock file is a sibling of `users_file`, not `users_file` itself,
+/// because `write_atomic` replaces that inode on every write. It is opened
+/// `O_NOFOLLOW` and `O_NONBLOCK` (a planted symlink or FIFO cannot redirect or
+/// hang it) and must be a regular file; no ownership is touched by path.
+pub async fn lock_users_file(users_file: &Path, wait: Duration) -> Result<File> {
+    let dir = users_file
+        .parent()
+        .with_context(|| format!("{} has no parent directory", users_file.display()))?;
+    fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?;
+    let name = users_file
+        .file_name()
+        .with_context(|| format!("{} has no file name", users_file.display()))?;
+    let mut lock_name = name.to_os_string();
+    lock_name.push(".lock");
+    let lock_path = dir.join(lock_name);
+
+    let flags = rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK;
+    let file = fs::OpenOptions::new()
+        .write(true)
+        .create(true)
+        .truncate(false)
+        .mode(0o600)
+        .custom_flags(flags.bits().cast_signed())
+        .open(&lock_path)
+        .with_context(|| format!("opening {}", lock_path.display()))?;
+    if !file
+        .metadata()
+        .with_context(|| format!("stat {}", lock_path.display()))?
+        .is_file()
+    {
+        bail!("{} is not a regular file", lock_path.display());
+    }
+
+    let deadline = tokio::time::Instant::now() + wait;
+    loop {
+        match rustix::fs::flock(&file, rustix::fs::FlockOperation::NonBlockingLockExclusive) {
+            Ok(()) => return Ok(file),
+            Err(e)
+                if e == rustix::io::Errno::WOULDBLOCK && tokio::time::Instant::now() < deadline =>
+            {
+                tokio::time::sleep(Duration::from_millis(50)).await;
+            }
+            Err(e) if e == rustix::io::Errno::WOULDBLOCK => bail!(
+                "timed out after {wait:?} waiting for {}: another process (`swarmctl user`) \
+                 is updating {}",
+                lock_path.display(),
+                users_file.display()
+            ),
+            Err(e) => {
+                return Err(e).with_context(|| format!("locking {}", lock_path.display()));
+            }
+        }
+    }
+}
+
 /// Replace `path`'s contents atomically. Unlike `swarmctl::write_atomic`,
 /// this does **not** need to preserve a foreign owner via `chown` — this
 /// process runs as `users_file`'s own owning user (see the module doc:
@@ -637,6 +704,70 @@ some_future_top_level_key: 7
         fs::remove_dir_all(&dir).ok();
     }
 
+    #[tokio::test]
+    async fn the_users_lock_excludes_a_second_holder_until_it_is_released() {
+        let dir = tempdir();
+        let users_file = dir.join("users.yml");
+        let held = lock_users_file(&users_file, Duration::from_secs(1))
+            .await
+            .expect("first lock");
+
+        let err = lock_users_file(&users_file, Duration::from_millis(200))
+            .await
+            .expect_err("a held lock must not be granted twice");
+        assert!(err.to_string().contains("timed out"), "{err}");
+
+        drop(held);
+        lock_users_file(&users_file, Duration::from_millis(200))
+            .await
+            .expect("a released lock is granted");
+
+        fs::remove_dir_all(&dir).ok();
+    }
+
+    #[tokio::test]
+    async fn the_users_lock_refuses_a_symlink_planted_at_the_lock_path() {
+        let dir = tempdir();
+        let users_file = dir.join("users.yml");
+        let target = dir.join("victim");
+        fs::write(&target, "keep").unwrap();
+        std::os::unix::fs::symlink(&target, dir.join("users.yml.lock")).unwrap();
+
+        lock_users_file(&users_file, Duration::from_secs(1))
+            .await
+            .expect_err("a symlinked lock path must be refused");
+        assert_eq!(fs::read_to_string(&target).unwrap(), "keep");
+
+        fs::remove_dir_all(&dir).ok();
+    }
+
+    /// Two read-modify-writes that each sit between load and publish long
+    /// enough for the other to load: without the lock the second `publish`
+    /// drops the first user.
+    #[tokio::test]
+    async fn interleaved_read_modify_writes_both_land() {
+        let dir = tempdir();
+        let users_file = dir.join("users.yml");
+        let add = |name: &'static str| {
+            let users_file = users_file.clone();
+            tokio::spawn(async move {
+                let _lock = lock_users_file(&users_file, Duration::from_secs(5)).await?;
+                let mut store = load_store(&users_file)?;
+                tokio::time::sleep(Duration::from_millis(300)).await;
+                store.users.insert(name.to_owned(), user("$argon2id$x"));
+                publish(&users_file, &mut store)
+            })
+        };
+        let (a, b) = (add("alice"), add("bob"));
+        a.await.unwrap().expect("alice");
+        b.await.unwrap().expect("bob");
+
+        let names: Vec<_> = load_store(&users_file).unwrap().users.into_keys().collect();
+        assert_eq!(names, ["alice", "bob"]);
+
+        fs::remove_dir_all(&dir).ok();
+    }
+
     /// Test-only temp dir under the OS temp root — disposable scratch for
     /// one test's lifetime, not durable state (see `state-not-tmp`: this
     /// is exactly the legitimate use, not a place we persist anything).