swarm-controller: end a dead replay, clamp stale resume ids, test on a real server

A replay consumer now has a 5s idle heartbeat, and the row stream ends at
the first consumer error instead of skipping it. When the server drops the
consumer, as on a restart, two missed heartbeats end the SSE response and
the browser reconnects with Last-Event-ID. Before, the response stayed
open and delivered nothing.

A Last-Event-ID above the stream's newest sequence now replays as a fresh
reader. The server accepts a start past the end and delivers nothing
until the stream reaches it, so a stale id stalled the replay.

swarm-controller's stream tests run against a real nats-server on
loopback (`test_nats`, binary from HIVE_NATS_SERVER, exported by
checks.nix and devshell.nix). They cover: ensuring the stream twice, an
existing stream with other limits being opened as it is, the replay
arriving before live rows, a reconnect getting every later row exactly
once, and a dropped consumer ending the row stream.

The limits test now compares against the subagent stream's own config.
Two comments that described the agent's terminal route as live-only are
corrected.

swarm-queue-client/src/agent_term.rs

@@ -37,6 +37,11 @@ const MAX_BYTES: i64 = 64 * 1024 * 1024;
 /// events as its first history page.
 pub const REPLAY_ROWS: u64 = 100;
 
+/// How often the server sends an idle replay consumer a heartbeat. Two missed
+/// in a row end the reader's row stream, so a consumer the server dropped,
+/// as on a restart, ends the SSE response and the browser reconnects.
+pub const IDLE_HEARTBEAT: std::time::Duration = std::time::Duration::from_secs(5);
+
 /// The stream holding `agent`'s terminal rows: `term-main-<agent>`.
 #[must_use]
 pub fn stream_name(agent: &str) -> String {
@@ -75,11 +80,16 @@ pub fn consumer_create_grant() -> String {
 /// it. A fresh reader gets the newest [`REPLAY_ROWS`] below `last_sequence`,
 /// the stream's newest row when it was read. A sequence the stream has since
 /// discarded starts the consumer at the oldest row it still has.
+///
+/// A `resume_after` above `last_sequence` names no row this stream ever held,
+/// as after the stream was created again, and is read as a fresh reader: the
+/// server accepts a start past the end and delivers nothing until the stream
+/// reaches it.
 #[must_use]
 pub fn replay_start(last_sequence: u64, resume_after: Option<u64>) -> u64 {
     match resume_after {
-        Some(seen) => seen.saturating_add(1),
-        None => last_sequence.saturating_sub(REPLAY_ROWS - 1).max(1),
+        Some(seen) if seen <= last_sequence => seen + 1,
+        _ => last_sequence.saturating_sub(REPLAY_ROWS - 1).max(1),
     }
 }
 
@@ -148,6 +158,7 @@ pub fn replay_consumer_config(
             start_sequence,
         },
         ack_policy: async_nats::jetstream::consumer::AckPolicy::None,
+        idle_heartbeat: IDLE_HEARTBEAT,
         ..Default::default()
     }
 }
@@ -243,6 +254,15 @@ mod tests {
         assert_eq!(replay_start(1000, Some(1000)), 1001);
     }
 
+    /// An id past the stream's newest row, as after the stream was created
+    /// again, replays like a fresh reader instead of waiting for that number.
+    #[test]
+    fn an_id_past_the_newest_row_replays_as_a_fresh_reader() {
+        assert_eq!(replay_start(5, Some(99)), replay_start(5, None));
+        assert_eq!(replay_start(1000, Some(1001)), replay_start(1000, None));
+        assert_eq!(replay_start(0, Some(7)), 1);
+    }
+
     /// The consumer starts at a stored sequence, not at the next new row, so
     /// the replay reaches the reader before the live tail, in stream order,
     /// through the one subscription.
@@ -263,19 +283,25 @@ mod tests {
             async_nats::jetstream::consumer::AckPolicy::None
         );
         assert!(cfg.durable_name.is_none());
+        // Without it a dropped consumer is never noticed.
+        assert_eq!(cfg.idle_heartbeat, IDLE_HEARTBEAT);
+        assert!(!cfg.idle_heartbeat.is_zero());
     }
 
-    #[cfg(feature = "agent-term")]
+    /// Every limit is read from the subagent stream's own config, so the two
+    /// fail this test the moment they drift apart.
+    #[cfg(all(feature = "agent-term", feature = "subagent-term"))]
     #[test]
     fn stream_config_matches_the_subagent_streams_limits() {
-        let cfg = stream_config("iris", 8_388_608);
-        assert_eq!(cfg.subjects, ["$SWARM.term.iris"]);
-        assert_eq!(cfg.max_age, crate::subagent_term::MAX_AGE);
-        assert_eq!(cfg.max_bytes, MAX_BYTES);
-        assert_eq!(
-            cfg.discard,
-            async_nats::jetstream::stream::DiscardPolicy::Old
-        );
-        assert_eq!(cfg.max_message_size, 8_388_608);
+        let main = stream_config("iris", 8_388_608);
+        let sub = crate::subagent_term::stream_config("iris", 8_388_608);
+        assert_eq!(main.subjects, ["$SWARM.term.iris"]);
+        assert_eq!(main.max_age, sub.max_age);
+        assert_eq!(main.max_bytes, sub.max_bytes);
+        assert_eq!(main.discard, sub.discard);
+        assert_eq!(main.max_message_size, sub.max_message_size);
+        assert_eq!(main.retention, sub.retention);
+        assert_eq!(main.storage, sub.storage);
+        assert_eq!(main.num_replicas, sub.num_replicas);
     }
 }

swarm-queue-client/src/subagent_term.rs

@@ -99,7 +99,10 @@ pub fn subagent_names<'a>(agent: &str, subjects: impl IntoIterator<Item = &'a st
 /// rows against, so the stream never rejects a row the queue itself
 /// accepted.
 #[cfg(feature = "subagent-term")]
-fn stream_config(agent: &str, max_message_size: i32) -> async_nats::jetstream::stream::Config {
+pub(crate) fn stream_config(
+    agent: &str,
+    max_message_size: i32,
+) -> async_nats::jetstream::stream::Config {
     async_nats::jetstream::stream::Config {
         name: stream_name(agent),
         description: Some(format!("Terminal rows of {agent}'s subagents")),