fix(stacks): widen pull retry window and jitter the nightly update timer
Nightly update pulls for coolify/immich kept hitting transient ghcr.io toomanyrequests errors and getting skipped, because the timer fired at exactly 02:00:00 Europe/Amsterdam (the same instant countless other hosts' cron/timer jobs also hit registries) and the retry loop only spanned ~15s (3 attempts, 5/10/15s backoff) — not enough to ride out the burst. - Widen pull retries to 5 attempts with 20/40/60/80s backoff (~3+ min). - Add RandomizedDelaySec=1800 to the update timer (and its install template) so this host's pulls no longer land on the exact top-of-hour thundering herd. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XmtksEEYSBZqUacw3qF1A9
This commit is contained in:
+5
-4
@@ -431,16 +431,16 @@ _update_run() {
|
||||
# through to the change check — otherwise a stale image looks
|
||||
# "up to date" forever (this is what stuck coolify). ---
|
||||
local pull_ok=false attempt
|
||||
for attempt in 1 2 3; do
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if (cd "$stack_dir" && docker compose "${env_flag[@]}" pull 2>&1); then
|
||||
pull_ok=true
|
||||
break
|
||||
fi
|
||||
warn " Pull failed for $stack (attempt $attempt/3)"
|
||||
[[ $attempt -lt 3 ]] && sleep $((attempt * 5))
|
||||
warn " Pull failed for $stack (attempt $attempt/5)"
|
||||
[[ $attempt -lt 5 ]] && sleep $((attempt * 20))
|
||||
done
|
||||
if ! $pull_ok; then
|
||||
err " Pull failed for $stack after 3 attempts — skipping (not recreated)"
|
||||
err " Pull failed for $stack after 5 attempts — skipping (not recreated)"
|
||||
skipped+=("$stack")
|
||||
continue
|
||||
fi
|
||||
@@ -540,6 +540,7 @@ Description=Daily Docker stacks image update at 2:00 AM
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 02:00:00 Europe/Amsterdam
|
||||
RandomizedDelaySec=1800
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
|
||||
Reference in New Issue
Block a user