66 lines
1.9 KiB
Shell
66 lines
1.9 KiB
Shell
# Cases for ./retry-in-job.nix. `sleep` is a function that advances `SECONDS`
|
|
# and records the wait, so a case takes no wall time and the budget is
|
|
# measured on the clock the helper reads.
|
|
set -uo pipefail
|
|
|
|
failures=0
|
|
count=0
|
|
|
|
fail() {
|
|
echo "FAILED: $case: $*" >&2
|
|
failures=$((failures + 1))
|
|
}
|
|
|
|
errlog=$(mktemp)
|
|
waits=
|
|
sleep() {
|
|
waits="$waits $1"
|
|
SECONDS=$((SECONDS + $1))
|
|
}
|
|
|
|
# shellcheck source=/dev/null
|
|
. "$HELPER"
|
|
|
|
attempts=0
|
|
# Fails the first $FAIL_FIRST calls, then succeeds.
|
|
flaky() {
|
|
attempts=$((attempts + 1))
|
|
[ "$attempts" -gt "$FAIL_FIRST" ] || return 6
|
|
}
|
|
|
|
# run <description> <fail-first> <budget>
|
|
run() {
|
|
case=$1
|
|
FAIL_FIRST=$2
|
|
attempts=0
|
|
waits=
|
|
SECONDS=0
|
|
count=$((count + 1))
|
|
rc=0
|
|
retry_in_job "$3" flaky 2>"$errlog" || rc=$?
|
|
}
|
|
|
|
run "a command that succeeds at once is not waited for" 0 100
|
|
[ "$rc" = 0 ] || fail "returned $rc"
|
|
[ "$attempts" = 1 ] || fail "ran $attempts times, wanted 1"
|
|
[ -z "$waits" ] || fail "waited:$waits"
|
|
|
|
run "fail, fail, then succeed returns success after waiting 2s and 4s" 2 100
|
|
[ "$rc" = 0 ] || fail "returned $rc"
|
|
[ "$attempts" = 3 ] || fail "ran $attempts times, wanted 3"
|
|
[ "$waits" = " 2 4" ] || fail "waited:$waits, wanted: 2 4"
|
|
grep -q "failed with exit status 6; retrying in 2 seconds" "$errlog" || fail "the cause of a failed attempt is not logged: $(<"$errlog")"
|
|
|
|
run "the wait doubles and stops growing at 60s" 8 1000
|
|
[ "$rc" = 0 ] || fail "returned $rc"
|
|
[ "$waits" = " 2 4 8 16 32 60 60 60" ] || fail "waited:$waits"
|
|
|
|
run "a command that always fails returns 1 once the budget is spent" 1000000 100
|
|
[ "$rc" = 1 ] || fail "returned $rc, wanted 1"
|
|
[ "$waits" = " 2 4 8 16 32" ] || fail "waited:$waits, wanted: 2 4 8 16 32 (the next 60s wait passes the budget)"
|
|
|
|
if [ "$failures" -gt 0 ]; then
|
|
echo "script-test-retry-in-job: $failures failed assertions over $count runs" >&2
|
|
exit 1
|
|
fi
|
|
echo "script-test-retry-in-job: $count runs, all assertions hold"
|