build a failed meta build once more, and treat a timeout or missing grant as no verdict

swarm-builder/src/nix_build.rs

@@ -80,12 +80,18 @@ impl Builder {
         }
         let mut cmd = self.nix_command(token);
         cmd.args(BUILD_FLAGS).arg(self.meta_attr(job));
-        match run(cmd, self.deadline).await {
-            Reply::Built { out_path } => FromMetaReply::Built {
-                out_path,
-                system: self.system.clone(),
+        // `nix` not finishing (deadline, spawn or wait error) says nothing about
+        // the config, so it is `Unavailable`; only an exit of `nix` itself is a
+        // verdict.
+        match run_raw(cmd, self.deadline).await {
+            Err(reason) => FromMetaReply::Unavailable { reason },
+            Ok((status, out, err)) => match verdict(status, &out, &err) {
+                Reply::Built { out_path } => FromMetaReply::Built {
+                    out_path,
+                    system: self.system.clone(),
+                },
+                Reply::Failed { error_tail } => FromMetaReply::Failed { error_tail },
             },
-            Reply::Failed { error_tail } => FromMetaReply::Failed { error_tail },
         }
     }
 
@@ -274,18 +280,22 @@ fn check_agent_and_rev(agent: &str, rev: &str) -> Result<(), String> {
 async fn run(cmd: tokio::process::Command, deadline: Duration) -> Reply {
     match run_raw(cmd, deadline).await {
         Err(error_tail) => Reply::Failed { error_tail },
-        Ok((status, out, err)) => {
-            let out_path = String::from_utf8_lossy(&out)
-                .lines()
-                .rfind(|l| !l.trim().is_empty())
-                .map(|l| l.trim().to_owned());
-            match out_path {
-                Some(out_path) if status.success() => Reply::Built { out_path },
-                _ => Reply::Failed {
-                    error_tail: tail(&String::from_utf8_lossy(&err), status),
-                },
-            }
-        }
+        Ok((status, out, err)) => verdict(status, &out, &err),
+    }
+}
+
+/// What a `nix build` that exited said: the store path it printed, or the tail
+/// of what it said.
+fn verdict(status: std::process::ExitStatus, out: &[u8], err: &[u8]) -> Reply {
+    let out_path = String::from_utf8_lossy(out)
+        .lines()
+        .rfind(|l| !l.trim().is_empty())
+        .map(|l| l.trim().to_owned());
+    match out_path {
+        Some(out_path) if status.success() => Reply::Built { out_path },
+        _ => Reply::Failed {
+            error_tail: tail(&String::from_utf8_lossy(err), status),
+        },
     }
 }
 
@@ -815,6 +825,40 @@ mod tests {
         (Builder { nix, ..builder() }, dir)
     }
 
+    /// `nix` not finishing says nothing about the config: a build past its
+    /// deadline and a `nix` that cannot start are `Unavailable`. A `nix` that
+    /// exits non-zero is `Failed`, the control for the same builder.
+    #[tokio::test]
+    async fn a_meta_build_that_does_not_finish_is_unavailable_and_one_that_exits_is_failed() {
+        let (slow, _guard) = scripted("sleep 5");
+        let slow = Builder {
+            deadline: Duration::from_millis(200),
+            ..slow
+        };
+        let reply = slow.build_from_meta(&meta_job(), "t").await;
+        assert!(
+            matches!(&reply, FromMetaReply::Unavailable { reason } if reason.contains("did not finish")),
+            "{reply:?}"
+        );
+
+        let missing = Builder {
+            nix: PathBuf::from("/nonexistent/nix"),
+            ..builder()
+        };
+        let reply = missing.build_from_meta(&meta_job(), "t").await;
+        assert!(
+            matches!(&reply, FromMetaReply::Unavailable { reason } if reason.contains("starting nix")),
+            "{reply:?}"
+        );
+
+        let (failing, _guard) = scripted("echo 'error: attribute missing' >&2; exit 1");
+        let reply = failing.build_from_meta(&meta_job(), "t").await;
+        assert!(
+            matches!(&reply, FromMetaReply::Failed { error_tail } if error_tail.ends_with("attribute missing")),
+            "{reply:?}"
+        );
+    }
+
     fn lock_job() -> LockMeta {
         LockMeta {
             flake_nix: "{ outputs = _: { }; }".to_owned(),