vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarmctl: serve login callbacks concurrently, retry a refused token once, test the refresh lock and the socket path

Browse source 
The loopback listener served one connection at a time with no read
timeout, so a connection that never sent a request (a browser preconnect,
any local process) held up the real form_post until the 5-minute wait
ran out. Each connection now runs on its own task and the first final
answer wins. The callback body is capped at 16 KiB; over the cap it reads
as empty and can't complete the login.

A request the gateway answers 401 now triggers one refresh (under the
cache lock, skipped if another process already replaced the refused
token) and one retry; a second 401 says to run `swarmctl login`.

The login cache directory is a `Swarm` field so tests can point it at a
scratch directory. New tests: an idle connection does not block the real
callback; an oversized body is refused without ending the wait; two
concurrent refreshers refresh once and share the token; a refused token
is refreshed once and retried; a second 401 says to log in without a
further retry; with no `--swarm`, the verbs use the controller socket.
This commit is contained in:
atlas 2026-10-12 00:36:48 +02:00 • committed by mara
commit 71e7fd8b20
4 changed files with 473 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

313
swarmctl/src/remote.rs
View file

@ -39,10 +39,12 @@ pub(crate) struct Swarm {
pub origin: url::Url,
/// A PEM CA certificate to trust beside the system roots.
pub ca_file: Option<PathBuf>,
/// Where the login cache lives: `$XDG_STATE_HOME/swarmctl` by default.
pub state_dir: PathBuf,
}
impl Swarm {
pub(crate) fn parse(url: &str, ca_file: Option<PathBuf>) -> Result<Self> {
pub(crate) fn parse(url: &str, ca_file: Option<PathBuf>, state_dir: PathBuf) -> Result<Self> {
let parsed =
url::Url::parse(url).with_context(|| format!("--swarm {url:?} is not a URL"))?;
if parsed.scheme() != "https" || parsed.host_str().is_none() {
@ -50,7 +52,11 @@ impl Swarm {
}
let origin = url::Url::parse(&parsed.origin().ascii_serialization())
.context("re-parsing the swarm's origin")?;
Ok(Self { origin, ca_file })
Ok(Self {
origin,
ca_file,
state_dir,
})
}
/// What the token must name for the gateway to admit it at `/api/`.
@ -73,19 +79,13 @@ impl Swarm {
builder.build().context("building the HTTPS client")
}
pub(crate) fn cache_path(&self) -> Result<PathBuf> {
let state = std::env::var_os("XDG_STATE_HOME")
.map(PathBuf::from)
.or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".local/state")))
.context(
"neither XDG_STATE_HOME nor HOME is set, so there is nowhere to keep the login",
)?;
pub(crate) fn cache_path(&self) -> PathBuf {
let host = self.origin.host_str().unwrap_or_default();
let name = match self.origin.port() {
Some(port) => format!("{host}_{port}.json"),
None => format!("{host}.json"),
};
Ok(state.join("swarmctl").join(name))
self.state_dir.join(name)
}
fn not_logged_in(&self) -> String {
@ -96,6 +96,8 @@ impl Swarm {
}
/// One request to `uri` (a path under the origin) with the cached login.
/// A 401 is answered by one refresh and one retry; a second 401 means the
/// login itself is gone.
pub(crate) async fn request(
&self,
method: hyper::Method,
@ -103,19 +105,35 @@ impl Swarm {
body: Option<Vec<u8>>,
what: &str,
) -> Result<(hyper::StatusCode, bytes::Bytes)> {
let http = self.http()?;
let token = self.access_token(&http).await?;
let url = self.origin.join(uri).context("joining the request path")?;
let mut req = http.request(method, url).bearer_auth(token);
if let Some(body) = body {
req = req
.header(reqwest::header::CONTENT_TYPE, "application/json")
.body(body);
}
let resp = req
.send()
self.request_with(&self.http()?, method, uri, body, what)
.await
.with_context(|| format!("sending the {what} request to {}", self.origin))?;
}
async fn request_with(
&self,
http: &reqwest::Client,
method: hyper::Method,
uri: &str,
body: Option<Vec<u8>>,
what: &str,
) -> Result<(hyper::StatusCode, bytes::Bytes)> {
let url = self.origin.join(uri).context("joining the request path")?;
let send = |token: String| {
let mut req = http.request(method.clone(), url.clone()).bearer_auth(token);
if let Some(body) = &body {
req = req
.header(reqwest::header::CONTENT_TYPE, "application/json")
.body(body.clone());
}
req.send()
};
let context = || format!("sending the {what} request to {}", self.origin);
let token = self.access_token(http, None).await?;
let mut resp = send(token.clone()).await.with_context(context)?;
if resp.status() == reqwest::StatusCode::UNAUTHORIZED {
let fresh = self.access_token(http, Some(&token)).await?;
resp = send(fresh).await.with_context(context)?;
}
let status = resp.status();
if status == reqwest::StatusCode::UNAUTHORIZED {
bail!(
@ -135,17 +153,20 @@ impl Swarm {
Ok((status, body))
}
/// The cached access token, refreshed first if it is about to expire.
async fn access_token(&self, http: &reqwest::Client) -> Result<String> {
let path = self.cache_path()?;
/// The cached access token, refreshed first if it is about to expire or is
/// `rejected`, the one the gateway just refused.
async fn access_token(&self, http: &reqwest::Client, rejected: Option<&str>) -> Result<String> {
let path = self.cache_path();
let now = chrono::Utc::now().timestamp();
let due = |c: &Cached| needs_refresh(c, now) || rejected == Some(c.access_token.as_str());
let cached = read_cache(&path)?.with_context(|| self.not_logged_in())?;
if !needs_refresh(&cached, now) {
if !due(&cached) {
return Ok(cached.access_token);
}
let _lock = lock(&path)?;
// Another process may have refreshed while this one waited for the lock.
let cached = read_cache(&path)?.with_context(|| self.not_logged_in())?;
if !needs_refresh(&cached, now) {
if !due(&cached) {
return Ok(cached.access_token);
}
let token = token_request(
@ -165,6 +186,15 @@ impl Swarm {
}
}
/// `$XDG_STATE_HOME/swarmctl`, else `~/.local/state/swarmctl`.
pub(crate) fn state_dir() -> Result<PathBuf> {
std::env::var_os("XDG_STATE_HOME")
.map(PathBuf::from)
.or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".local/state")))
.map(|state| state.join("swarmctl"))
.context("neither XDG_STATE_HOME nor HOME is set, so there is nowhere to keep the login")
}
/// The token cache: one file per swarm, mode 0600, in a 0700 directory.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct Cached {
@ -288,6 +318,7 @@ pub(crate) fn lock(path: &Path) -> Result<File> {
#[cfg(test)]
mod tests {
use std::os::unix::fs::PermissionsExt as _;
use std::sync::{Arc, Mutex};
use super::*;
@ -345,8 +376,232 @@ mod tests {
#[test]
fn the_audience_is_the_api_prefix_of_the_origin() {
let swarm = Swarm::parse("https://swarm.example.test/some/page", None).expect("parses");
let swarm = Swarm::parse("https://swarm.example.test/some/page", None, "/s".into())
.expect("parses");
assert_eq!(swarm.audience(), "https://swarm.example.test/api/");
assert!(Swarm::parse("http://swarm.example.test", None).is_err());
assert!(Swarm::parse("http://swarm.example.test", None, "/s".into()).is_err());
}
/// authelia's token endpoint and the gateway's `/api/`, as far as these
/// tests need them: a refresh rotates both tokens, a used refresh token
/// revokes the login (as authelia does), and `/api/` admits only the
/// current access token.
struct Mock {
refresh: Option<String>,
access: Option<String>,
refreshes: u32,
api_calls: u32,
api_refuses_all: bool,
}
fn mock() -> Arc<Mutex<Mock>> {
Arc::new(Mutex::new(Mock {
refresh: Some("rt-0".into()),
access: Some("at-0".into()),
refreshes: 0,
api_calls: 0,
api_refuses_all: false,
}))
}
/// Serve `mock` on a loopback port from its own thread, so callers on
/// other threads can block on the cache lock without stalling it.
fn serve(mock: &Arc<Mutex<Mock>>) -> url::Url {
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind");
let addr = listener.local_addr().expect("addr");
listener.set_nonblocking(true).expect("nonblocking");
let mock = Arc::clone(mock);
std::thread::spawn(move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
rt.block_on(async move {
let listener = tokio::net::TcpListener::from_std(listener).expect("listener");
loop {
let (stream, _) = listener.accept().await.expect("accept");
let mock = Arc::clone(&mock);
tokio::spawn(async move {
let service = hyper::service::service_fn(move |req| {
let mock = Arc::clone(&mock);
async move { Ok::<_, std::convert::Infallible>(answer(&mock, req).await) }
});
let _ = hyper::server::conn::http1::Builder::new()
.serve_connection(hyper_util::rt::TokioIo::new(stream), service)
.await;
});
}
});
});
url::Url::parse(&format!("http://{addr}/")).expect("url")
}
async fn answer(
mock: &Mutex<Mock>,
req: hyper::Request<hyper::body::Incoming>,
) -> hyper::Response<http_body_util::Full<bytes::Bytes>> {
use http_body_util::BodyExt as _;
let reply = |status: u16, body: serde_json::Value| {
hyper::Response::builder()
.status(status)
.body(http_body_util::Full::new(body.to_string().into()))
.expect("response")
};
let path = req.uri().path().to_owned();
let bearer = req
.headers()
.get(hyper::header::AUTHORIZATION)
.and_then(|v| v.to_str().ok())
.and_then(|v| v.strip_prefix("Bearer "))
.map(str::to_owned);
let body = req.into_body().collect().await.expect("body").to_bytes();
if path == "/api/oidc/token" {
// Wide enough that two unlocked refreshers are both in flight.
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
let presented = url::form_urlencoded::parse(&body)
.find(|(k, _)| k == "refresh_token")
.map(|(_, v)| v.into_owned());
let mut m = mock.lock().expect("mock");
if presented.is_none() || presented != m.refresh {
m.refresh = None;
m.access = None;
return reply(400, serde_json::json!({ "error": "invalid_grant" }));
}
m.refreshes += 1;
m.access = Some(format!("at-{}", m.refreshes));
m.refresh = Some(format!("rt-{}", m.refreshes));
return reply(
200,
serde_json::json!({
"access_token": m.access, "refresh_token": m.refresh, "expires_in": 3600,
}),
);
}
let mut m = mock.lock().expect("mock");
m.api_calls += 1;
if !m.api_refuses_all && bearer.is_some() && bearer == m.access {
reply(200, serde_json::json!({ "ok": true }))
} else {
reply(401, serde_json::json!({}))
}
}
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("swarmctl-{name}-{}", std::process::id()));
let _ = fs::remove_dir_all(&dir);
dir
}
/// A swarm whose API and authelia are both `origin`, with a cached login.
fn logged_in(origin: &url::Url, dir: &Path, access_token: &str, expires_at: i64) -> Swarm {
let swarm = Swarm {
origin: origin.clone(),
ca_file: None,
state_dir: dir.to_owned(),
};
let cache = Cached {
auth: origin.as_str().trim_end_matches('/').to_owned(),
access_token: access_token.into(),
refresh_token: "rt-0".into(),
expires_at,
};
write_cache(&swarm.cache_path(), &cache).expect("cache");
swarm
}
/// Plain HTTP only: no system roots, which the build sandbox doesn't have.
fn client() -> reqwest::Client {
reqwest::Client::builder()
.tls_certs_only([])
.build()
.expect("client")
}
fn block_on<T>(fut: impl std::future::Future<Output = T>) -> T {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime")
.block_on(fut)
}
/// Two refreshers on one expired cache, as two `swarmctl` processes would
/// be: one refresh, and both get the token it issued. Without the lock
/// and the reread under it, the second presents the used refresh token
/// and authelia revokes the login.
#[test]
fn concurrent_refreshers_refresh_once_and_share_the_token() {
let mock = mock();
let origin = serve(&mock);
let dir = scratch("concurrent");
logged_in(&origin, &dir, "at-0", 0);
let start = Arc::new(std::sync::Barrier::new(2));
let refreshers: Vec<_> = (0..2)
.map(|_| {
let swarm = Swarm {
origin: origin.clone(),
ca_file: None,
state_dir: dir.clone(),
};
let start = Arc::clone(&start);
std::thread::spawn(move || {
let http = client();
start.wait();
block_on(swarm.access_token(&http, None))
})
})
.collect();
let tokens: Vec<String> = refreshers
.into_iter()
.map(|r| r.join().expect("thread").expect("refreshed"))
.collect();
let m = mock.lock().expect("mock");
assert_eq!(m.refreshes, 1);
assert_eq!(tokens, ["at-1", "at-1"]);
assert_eq!(m.access.as_deref(), Some("at-1"), "the login is still live");
fs::remove_dir_all(&dir).expect("cleanup");
}
/// The clock says the token is good and the gateway refuses it: refresh
/// once and retry.
#[test]
fn a_refused_token_is_refreshed_once_and_the_request_retried() {
let mock = mock();
let origin = serve(&mock);
let dir = scratch("retry");
let swarm = logged_in(&origin, &dir, "at-revoked", i64::MAX / 2);
let (status, _) =
block_on(swarm.request_with(&client(), hyper::Method::GET, "/api/x", None, "x"))
.expect("the retry succeeds");
assert_eq!(status, hyper::StatusCode::OK);
let m = mock.lock().expect("mock");
assert_eq!((m.refreshes, m.api_calls), (1, 2));
drop(m);
let cached = read_cache(&swarm.cache_path())
.expect("read")
.expect("cache");
assert_eq!(cached.access_token, "at-1");
fs::remove_dir_all(&dir).expect("cleanup");
}
/// A 401 after the refresh means the login itself is gone: say so, once.
#[test]
fn a_second_401_says_to_log_in_and_does_not_retry_again() {
let mock = mock();
mock.lock().expect("mock").api_refuses_all = true;
let origin = serve(&mock);
let dir = scratch("refused");
let swarm = logged_in(&origin, &dir, "at-0", i64::MAX / 2);
let err = block_on(swarm.request_with(&client(), hyper::Method::GET, "/api/x", None, "x"))
.expect_err("still refused")
.to_string();
assert!(err.contains("(401)") && err.contains("login`"), "{err}");
let m = mock.lock().expect("mock");
assert_eq!((m.refreshes, m.api_calls), (1, 2));
fs::remove_dir_all(&dir).expect("cleanup");
}
}