fix(stacks): widen pull retry window and jitter the nightly update timer
Nightly update pulls for coolify/immich kept hitting transient ghcr.io toomanyrequests errors and getting skipped, because the timer fired at exactly 02:00:00 Europe/Amsterdam (the same instant countless other hosts' cron/timer jobs also hit registries) and the retry loop only spanned ~15s (3 attempts, 5/10/15s backoff) — not enough to ride out the burst. - Widen pull retries to 5 attempts with 20/40/60/80s backoff (~3+ min). - Add RandomizedDelaySec=1800 to the update timer (and its install template) so this host's pulls no longer land on the exact top-of-hour thundering herd. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XmtksEEYSBZqUacw3qF1A9
This commit is contained in:
+5
-4
@@ -431,16 +431,16 @@ _update_run() {
|
|||||||
# through to the change check — otherwise a stale image looks
|
# through to the change check — otherwise a stale image looks
|
||||||
# "up to date" forever (this is what stuck coolify). ---
|
# "up to date" forever (this is what stuck coolify). ---
|
||||||
local pull_ok=false attempt
|
local pull_ok=false attempt
|
||||||
for attempt in 1 2 3; do
|
for attempt in 1 2 3 4 5; do
|
||||||
if (cd "$stack_dir" && docker compose "${env_flag[@]}" pull 2>&1); then
|
if (cd "$stack_dir" && docker compose "${env_flag[@]}" pull 2>&1); then
|
||||||
pull_ok=true
|
pull_ok=true
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
warn " Pull failed for $stack (attempt $attempt/3)"
|
warn " Pull failed for $stack (attempt $attempt/5)"
|
||||||
[[ $attempt -lt 3 ]] && sleep $((attempt * 5))
|
[[ $attempt -lt 5 ]] && sleep $((attempt * 20))
|
||||||
done
|
done
|
||||||
if ! $pull_ok; then
|
if ! $pull_ok; then
|
||||||
err " Pull failed for $stack after 3 attempts — skipping (not recreated)"
|
err " Pull failed for $stack after 5 attempts — skipping (not recreated)"
|
||||||
skipped+=("$stack")
|
skipped+=("$stack")
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
@@ -540,6 +540,7 @@ Description=Daily Docker stacks image update at 2:00 AM
|
|||||||
|
|
||||||
[Timer]
|
[Timer]
|
||||||
OnCalendar=*-*-* 02:00:00 Europe/Amsterdam
|
OnCalendar=*-*-* 02:00:00 Europe/Amsterdam
|
||||||
|
RandomizedDelaySec=1800
|
||||||
Persistent=true
|
Persistent=true
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
|
|||||||
Reference in New Issue
Block a user