fix(stacks): make nightly update check resilient to transient errors
Two opposite failure modes in `update run`: - pulsenode was reported "updated" every night though its image never changed: the pre-pull image-ID snapshot came back empty (the awk fallback used \s, which mawk silently ignores), and empty != populated was read as "changed". - coolify was never recreated despite a new image: its nightly pull kept hitting a transient `toomanyrequests`, and a failed pull only warned before falling through to the comparison, which then saw no change. Changes: - _stack_image_ids: POSIX [[:space:]] instead of \s; return non-zero when images can't be resolved so callers can tell "unknown" from "unchanged". - Retry `docker compose pull` 3x with backoff; if it still fails, add the stack to a new `skipped` list and skip the recreate instead of treating it as up to date. - Detect changes by comparing running containers' image IDs against the post-pull resolved config, so a transient empty read can't be mistaken for "everything changed". - Notify with an amber "finished with warnings" summary listing skipped stacks. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
+75
-21
@@ -379,22 +379,32 @@ cmd_compose() {
|
||||
|
||||
# ─── Update Service ───────────────────────────────────────────────────────────
|
||||
|
||||
# Resolve each service's image tag to its current local SHA256 ID.
|
||||
# Reads from the local image store (updated by 'docker compose pull'),
|
||||
# NOT from running containers — so the comparison reflects the pulled image.
|
||||
# Resolve the compose config's image references to their current local SHA256
|
||||
# IDs, one per line, sorted. Reads from the local image store (updated by
|
||||
# 'docker compose pull'), NOT from running containers.
|
||||
#
|
||||
# Returns non-zero (and prints nothing) if the image list can't be resolved or
|
||||
# none of the images are present locally — callers MUST treat that as "unknown",
|
||||
# never as "everything changed".
|
||||
_stack_image_ids() {
|
||||
local stack_dir="$1"; shift
|
||||
local -a env_flag=("$@")
|
||||
local images
|
||||
local images ids
|
||||
|
||||
# 'config --images' (compose v2.19+) outputs one image name per line;
|
||||
# fall back to parsing the resolved config yaml for older versions.
|
||||
if images=$(cd "$stack_dir" && docker compose "${env_flag[@]}" config --images 2>/dev/null) \
|
||||
&& [[ -n "$images" ]]; then
|
||||
echo "$images"
|
||||
else
|
||||
cd "$stack_dir" && docker compose "${env_flag[@]}" config 2>/dev/null \
|
||||
| awk '/^\s+image:/{print $2}' | sort -u
|
||||
fi | xargs -r docker image inspect --format '{{.Id}}' 2>/dev/null | sort
|
||||
# NOTE: the fallback regex must be POSIX ([[:space:]], not \s) — mawk, the
|
||||
# default awk on Debian, does not understand \s and silently matches nothing.
|
||||
if ! images=$(cd "$stack_dir" && docker compose "${env_flag[@]}" config --images 2>/dev/null) \
|
||||
|| [[ -z "$images" ]]; then
|
||||
images=$(cd "$stack_dir" && docker compose "${env_flag[@]}" config 2>/dev/null \
|
||||
| awk '/^[[:space:]]+image:/{print $2}' | sort -u) || true
|
||||
fi
|
||||
[[ -n "$images" ]] || return 1
|
||||
|
||||
ids=$(echo "$images" | xargs -r docker image inspect --format '{{.Id}}' 2>/dev/null | sort) || true
|
||||
[[ -n "$ids" ]] || return 1
|
||||
echo "$ids"
|
||||
}
|
||||
|
||||
_update_run() {
|
||||
@@ -402,7 +412,7 @@ _update_run() {
|
||||
|
||||
local -a all_stacks=()
|
||||
mapfile -t all_stacks < <(list_all_stacks)
|
||||
local -a updated=() failed=()
|
||||
local -a updated=() failed=() skipped=()
|
||||
local -i checked=0
|
||||
|
||||
log "Starting image update check (${#all_stacks[@]} stacks)"
|
||||
@@ -415,17 +425,56 @@ _update_run() {
|
||||
info "Checking ${BOLD}$stack${RESET}"
|
||||
((checked++)) || true
|
||||
|
||||
local before after
|
||||
before=$(_stack_image_ids "$stack_dir" "${env_flag[@]+"${env_flag[@]}"}") || true
|
||||
if ! (cd "$stack_dir" && docker compose "${env_flag[@]}" pull 2>&1); then
|
||||
warn " Pull reported a non-zero exit for $stack (see output above) — comparison may be unreliable"
|
||||
# --- Pull, with retry. Nightly runs regularly hit transient registry
|
||||
# errors (toomanyrequests, TLS resets, gitea registry restarting) that
|
||||
# a short backoff clears. A pull that never succeeds must NOT fall
|
||||
# through to the change check — otherwise a stale image looks
|
||||
# "up to date" forever (this is what stuck coolify). ---
|
||||
local pull_ok=false attempt
|
||||
for attempt in 1 2 3; do
|
||||
if (cd "$stack_dir" && docker compose "${env_flag[@]}" pull 2>&1); then
|
||||
pull_ok=true
|
||||
break
|
||||
fi
|
||||
warn " Pull failed for $stack (attempt $attempt/3)"
|
||||
[[ $attempt -lt 3 ]] && sleep $((attempt * 5))
|
||||
done
|
||||
if ! $pull_ok; then
|
||||
err " Pull failed for $stack after 3 attempts — skipping (not recreated)"
|
||||
skipped+=("$stack")
|
||||
continue
|
||||
fi
|
||||
after=$(_stack_image_ids "$stack_dir" "${env_flag[@]+"${env_flag[@]}"}") || true
|
||||
|
||||
if [[ "$before" != "$after" ]]; then
|
||||
# --- Decide whether a recreate is needed by comparing the image IDs the
|
||||
# RUNNING containers use against the IDs the (post-pull) config
|
||||
# resolves to. Unlike the old before/after snapshot diff, a transient
|
||||
# empty read here can't be mistaken for "everything changed" — an
|
||||
# unresolvable list is skipped, not acted on. ---
|
||||
local desired current
|
||||
desired=$(_stack_image_ids "$stack_dir" "${env_flag[@]+"${env_flag[@]}"}") || desired=""
|
||||
if [[ -z "$desired" ]]; then
|
||||
warn " Could not resolve images for $stack — skipping (not recreated)"
|
||||
skipped+=("$stack")
|
||||
continue
|
||||
fi
|
||||
current=$(cd "$stack_dir" && docker compose "${env_flag[@]}" ps -q 2>/dev/null \
|
||||
| xargs -r docker inspect --format '{{.Image}}' 2>/dev/null | sort -u)
|
||||
|
||||
if [[ -z "$current" ]]; then
|
||||
info " Up to date (stack not running — left as is)"
|
||||
continue
|
||||
fi
|
||||
|
||||
local stale=false cid
|
||||
while IFS= read -r cid; do
|
||||
[[ -z "$cid" ]] && continue
|
||||
grep -qxF -- "$cid" <<<"$desired" || stale=true
|
||||
done <<<"$current"
|
||||
|
||||
if $stale; then
|
||||
info " Updates found — recreating $stack"
|
||||
info " before: ${before:-<empty>}"
|
||||
info " after: ${after:-<empty>}"
|
||||
info " running: ${current//$'\n'/ }"
|
||||
info " resolved: ${desired//$'\n'/ }"
|
||||
if (cd "$stack_dir" && docker compose "${env_flag[@]}" up -d 2>&1); then
|
||||
updated+=("$stack")
|
||||
ok " $stack updated"
|
||||
@@ -444,9 +493,14 @@ _update_run() {
|
||||
local date_str
|
||||
date_str="$(date '+%Y-%m-%d %H:%M')"
|
||||
|
||||
local skipped_line=""
|
||||
[[ ${#skipped[@]} -gt 0 ]] && skipped_line="\n⚠️ Skipped (transient errors, not recreated): ${skipped[*]}"
|
||||
|
||||
if [[ ${#failed[@]} -gt 0 ]]; then
|
||||
_notify "#cc0000" "❌ **Update failed** — $date_str\nUpdated: ${updated[*]:-none}\nFailed: ${failed[*]}"
|
||||
_notify "#cc0000" "❌ **Update failed** — $date_str\nUpdated: ${updated[*]:-none}\nFailed: ${failed[*]}${skipped_line}"
|
||||
return 1
|
||||
elif [[ ${#skipped[@]} -gt 0 ]]; then
|
||||
_notify "#e8a33d" "⚠️ **Update finished with warnings** — $date_str\nUpdated: ${updated[*]:-none}${skipped_line}"
|
||||
elif [[ ${#updated[@]} -gt 0 ]]; then
|
||||
_notify "#36a64f" "✅ **Updates applied** — $date_str\nStacks updated (${#updated[@]}/$checked): ${updated[*]}"
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user