Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1961afd9cc | ||
|
|
152d54dcdc |
6 changed files with 166 additions and 41 deletions
|
|
@ -198,6 +198,9 @@ pub async fn run_approval_merge_config_pr(
|
|||
let approval = fetch_approval_for_worker(coord, approval_id, ApprovalKind::MergeConfigPr)?;
|
||||
let agent_dir = crate::paths::agent_runtime_dir(&approval.agent);
|
||||
let applied_dir = crate::paths::applied_dir(&approval.agent);
|
||||
// Captured up front to scope the failure-comment's build-log lookup to
|
||||
// rows this deploy produced (see `post_merge_failure_to_pr`).
|
||||
let since_ts = hive_sh4re::wire_time::now_unix();
|
||||
coord.set_queue_step(queue_entry_id, "merge config pr");
|
||||
let (result, terminal_tag) =
|
||||
run_merge_config_pr(coord, &approval, &agent_dir, &applied_dir, queue_entry_id).await;
|
||||
|
|
@ -210,9 +213,84 @@ pub async fn run_approval_merge_config_pr(
|
|||
if let Err(e) = crate::forge::push_config(&approval.agent).await {
|
||||
tracing::warn!(agent = %approval.agent, error = ?e, "forge: push_config after merge failed");
|
||||
}
|
||||
// On a failed deploy, surface the failing build log back onto the PR so
|
||||
// the manager sees why it was rejected without leaving the forge.
|
||||
if let Err(e) = &result {
|
||||
post_merge_failure_to_pr(coord, &approval, since_ts, e).await;
|
||||
}
|
||||
finish_approval(coord, &approval, result, terminal_tag, false)
|
||||
}
|
||||
|
||||
/// Max stderr bytes to inline in a PR failure comment. Keeps the comment
|
||||
/// readable and under forge's size limits while still carrying the tail
|
||||
/// where the nix/build error actually surfaces.
|
||||
const PR_FAIL_LOG_TAIL_BYTES: usize = 4000;
|
||||
|
||||
/// On a failed `MergeConfigPr` deploy, post the failing build log back to the
|
||||
/// config PR as a comment so the manager sees the rejection reason on the PR
|
||||
/// itself. Best-effort: any error here is logged, never allowed to disturb the
|
||||
/// approval-resolution path.
|
||||
///
|
||||
/// The failing `build_log` row is located heuristically: the most recent `fail`
|
||||
/// row for this agent that started at/after `since_ts` (the caller's function
|
||||
/// entry). Because deploys are serialised per agent through the queue, that is
|
||||
/// the step which just failed — `verify`, `prepare-deploy`, `prebuild`, or the
|
||||
/// container rebuild. Pre-build failures (drift gate, fetch) create no `build_log`
|
||||
/// row, so the comment then carries only the error text.
|
||||
async fn post_merge_failure_to_pr(
|
||||
coord: &Arc<Coordinator>,
|
||||
approval: &hive_sh4re::Approval,
|
||||
since_ts: i64,
|
||||
err: &anyhow::Error,
|
||||
) {
|
||||
let Ok(pr) = approval.commit_ref.parse::<u64>() else {
|
||||
return;
|
||||
};
|
||||
let repo = crate::forge::config_repo(&approval.agent);
|
||||
|
||||
let log_section = coord
|
||||
.build_logs
|
||||
.list_recent_for_agent(&approval.agent, 10)
|
||||
.ok()
|
||||
.and_then(|rows| {
|
||||
rows.into_iter()
|
||||
.find(|r| r.status.as_deref() == Some("fail") && r.started_at >= since_ts)
|
||||
})
|
||||
.and_then(|row| coord.build_logs.get_full(row.id).ok().flatten())
|
||||
.map(|full| {
|
||||
let tail = tail_bytes(full.stderr.trim_end(), PR_FAIL_LOG_TAIL_BYTES);
|
||||
format!(
|
||||
"\n\n**Failing step:** `{}` (build log #{})\n\n```\n{tail}\n```",
|
||||
full.header.kind, full.header.id
|
||||
)
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let body = format!(
|
||||
"## ⚠️ config deploy failed\n\n\
|
||||
Approval #{} to merge this PR could not be deployed:\n\n\
|
||||
```\n{err:#}\n```{log_section}",
|
||||
approval.id
|
||||
);
|
||||
|
||||
if let Err(e) = crate::forge::post_pr_comment(&repo, pr, &body).await {
|
||||
tracing::warn!(agent = %approval.agent, %pr, error = ?e, "post merge-failure comment to PR failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the last `max_bytes` of `s`, snapped to a char boundary, prefixed
|
||||
/// with an elision marker when truncated.
|
||||
fn tail_bytes(s: &str, max_bytes: usize) -> String {
|
||||
if s.len() <= max_bytes {
|
||||
return s.to_owned();
|
||||
}
|
||||
let mut start = s.len() - max_bytes;
|
||||
while start < s.len() && !s.is_char_boundary(start) {
|
||||
start += 1;
|
||||
}
|
||||
format!("[… truncated …]\n{}", &s[start..])
|
||||
}
|
||||
|
||||
/// PR-merge config pipeline. `approval.commit_ref` is the PR number;
|
||||
/// `approval.fetched_sha` is the PR head sha the operator reviewed. Steps:
|
||||
/// 1. drift gate — re-read the live PR head; if it moved since review, abort
|
||||
|
|
|
|||
|
|
@ -93,32 +93,12 @@ pub async fn poll_open_config_prs(core_token: &str, coord: &Arc<Coordinator>) ->
|
|||
};
|
||||
open_prs.insert((agent.to_owned(), pr_number));
|
||||
|
||||
// Skip if a pending approval already exists for this PR.
|
||||
match coord
|
||||
.approvals
|
||||
.has_pending_merge_config_pr(agent, pr_number)
|
||||
{
|
||||
Ok(true) => {
|
||||
tracing::debug!(
|
||||
%agent, %pr_number,
|
||||
"config-pr poll: approval already pending, skipping"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
%agent, %pr_number, error = ?e,
|
||||
"config-pr poll: DB check failed, skipping"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
%agent, %pr_number,
|
||||
"config-pr poll: queuing missed MergeConfigPr approval"
|
||||
);
|
||||
// `submit_merge_config_pr` is idempotent + PR-drift aware: it
|
||||
// no-ops when an approval pinned to this PR's *current* head is
|
||||
// already pending, and cancels+re-queues when the head has drifted.
|
||||
// So the poll can call it unconditionally — it backstops both a
|
||||
// missed `opened` webhook (no approval yet) and a missed
|
||||
// `synchronize` (stale approval whose head moved).
|
||||
let description = format!("PR #{pr_number} on {CONFIG_ORG}/{agent} (poll fallback)");
|
||||
if let Err(e) = crate::socket_server::submit_merge_config_pr(
|
||||
coord,
|
||||
|
|
@ -131,7 +111,7 @@ pub async fn poll_open_config_prs(core_token: &str, coord: &Arc<Coordinator>) ->
|
|||
{
|
||||
tracing::warn!(
|
||||
%agent, %pr_number, error = ?e,
|
||||
"config-pr poll: failed to queue MergeConfigPr approval"
|
||||
"config-pr poll: failed to reconcile MergeConfigPr approval"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ mod repos;
|
|||
mod users;
|
||||
|
||||
pub use pr_merge::{
|
||||
ForgeMergeError, config_repo, fetch_pr_head_into_applied, merge_config_pr_ff, pr_head_sha,
|
||||
pr_is_open,
|
||||
ForgeMergeError, config_repo, fetch_pr_head_into_applied, merge_config_pr_ff, post_pr_comment,
|
||||
pr_head_sha, pr_is_open,
|
||||
};
|
||||
pub use repos::{
|
||||
create_agent_repo, ensure_config_repo, ensure_knowledge_repo, ensure_meta_remote, ensure_repo,
|
||||
|
|
|
|||
|
|
@ -218,6 +218,38 @@ pub async fn merge_config_pr_ff(repo: &str, pr: u64, sha: &str) -> Result<(), Fo
|
|||
}
|
||||
}
|
||||
|
||||
/// Post a comment to PR (= issue) `pr` on `repo` as the core forge user.
|
||||
/// PRs are issues in Forgejo, so the PR number is the issue index. Used to
|
||||
/// surface a failed config-approval deploy's build log back onto the PR so
|
||||
/// the manager sees why it was rejected without leaving the forge. `repo`
|
||||
/// is `owner/name`.
|
||||
///
|
||||
/// # Errors
|
||||
/// `Other` on absent core token, malformed repo, or transport/API failure.
|
||||
pub async fn post_pr_comment(repo: &str, pr: u64, body: &str) -> Result<(), ForgeMergeError> {
|
||||
let token = core_token()
|
||||
.ok_or_else(|| ForgeMergeError::Other(anyhow::anyhow!("forge core token absent")))?;
|
||||
let (owner, name) = repo.split_once('/').ok_or_else(|| {
|
||||
ForgeMergeError::Other(anyhow::anyhow!("forge repo `{repo}` is not owner/name"))
|
||||
})?;
|
||||
let index = i64::try_from(pr)
|
||||
.map_err(|_| ForgeMergeError::Other(anyhow::anyhow!("PR index {pr} overflows i64")))?;
|
||||
let client = api(&token).map_err(ForgeMergeError::Other)?;
|
||||
client
|
||||
.issue_create_comment(
|
||||
owner,
|
||||
name,
|
||||
index,
|
||||
forgejo_api::structs::CreateIssueCommentOption {
|
||||
body: body.to_owned(),
|
||||
updated_at: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| ForgeMergeError::Other(anyhow::Error::from(e).context("post PR comment")))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::repo_agent_name;
|
||||
|
|
|
|||
|
|
@ -157,6 +157,31 @@ pub(crate) async fn submit_merge_config_pr(
|
|||
let sha = crate::forge::pr_head_sha(&repo, pr_number)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("fetch PR head sha for {agent} PR #{pr_number}: {e}"))?;
|
||||
// Both the webhook (`synchronize`) and the poll fallback call this on
|
||||
// every PR update. If an approval for this PR is already pending, reconcile
|
||||
// it against the live head sha rather than blindly queuing another:
|
||||
// - same sha → the PR hasn't moved, so this is a duplicate signal — no-op.
|
||||
// - drifted sha → the reviewed head is stale. Don't mutate the
|
||||
// pending row in place (that races a concurrent approve); cancel it and
|
||||
// fall through to queue a FRESH approval pinned to the new head.
|
||||
if let Some((old_id, old_sha)) = coord.approvals.pending_merge_config_pr(agent, pr_number)? {
|
||||
if old_sha.as_deref() == Some(sha.as_str()) {
|
||||
return Ok(old_id);
|
||||
}
|
||||
let cancelled = coord
|
||||
.approvals
|
||||
.mark_cancelled(old_id, "config PR updated — superseded by a fresh approval")
|
||||
.map_err(|e| anyhow::anyhow!("cancel superseded merge_config_pr approval: {e:#}"))?;
|
||||
coord.emit_approval_resolved(crate::coordinator::ApprovalResolved {
|
||||
id: old_id,
|
||||
agent,
|
||||
approval_kind: "merge_config_pr",
|
||||
sha_short: old_sha.map(|s| s[..s.len().min(12)].to_owned()),
|
||||
status: "cancelled",
|
||||
note: Some("PR head moved; superseded by a fresh approval".to_owned()),
|
||||
description: cancelled.description,
|
||||
});
|
||||
}
|
||||
let id = coord
|
||||
.approvals
|
||||
.submit_kind(
|
||||
|
|
|
|||
|
|
@ -136,19 +136,29 @@ impl Approvals {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Return `true` when there is already a `pending` `merge_config_pr`
|
||||
/// approval for `(agent, pr_number)`. Used by the polling fallback to
|
||||
/// skip re-submitting approvals that were already queued by the webhook.
|
||||
pub fn has_pending_merge_config_pr(&self, agent: &str, pr_number: u64) -> Result<bool> {
|
||||
/// Return the `(id, fetched_sha)` of the pending `merge_config_pr`
|
||||
/// approval for `(agent, pr_number)`, if one exists. Drives
|
||||
/// `submit_merge_config_pr`'s idempotency + PR-drift handling: same
|
||||
/// `fetched_sha` → no new request (the webhook + poll both call submit,
|
||||
/// so re-submits of an unchanged PR must be no-ops); a drifted head →
|
||||
/// cancel this stale row and queue a fresh approval.
|
||||
pub fn pending_merge_config_pr(
|
||||
&self,
|
||||
agent: &str,
|
||||
pr_number: u64,
|
||||
) -> Result<Option<(i64, Option<String>)>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM approvals \
|
||||
WHERE agent = ?1 AND kind = 'merge_config_pr' \
|
||||
AND commit_ref = ?2 AND status = 'pending'",
|
||||
params![agent, pr_number.to_string()],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
Ok(count > 0)
|
||||
let row = conn
|
||||
.query_row(
|
||||
"SELECT id, fetched_sha FROM approvals \
|
||||
WHERE agent = ?1 AND kind = 'merge_config_pr' \
|
||||
AND commit_ref = ?2 AND status = 'pending' \
|
||||
ORDER BY id DESC LIMIT 1",
|
||||
params![agent, pr_number.to_string()],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()?;
|
||||
Ok(row)
|
||||
}
|
||||
|
||||
/// Last `limit` resolved approvals (approved / denied / failed),
|
||||
|
|
|
|||
Loading…
Reference in a new issue