diff --git a/bin/stacks b/bin/stacks index 0135374..5220d6d 100755 --- a/bin/stacks +++ b/bin/stacks @@ -379,22 +379,32 @@ cmd_compose() { # ─── Update Service ─────────────────────────────────────────────────────────── -# Resolve each service's image tag to its current local SHA256 ID. -# Reads from the local image store (updated by 'docker compose pull'), -# NOT from running containers — so the comparison reflects the pulled image. +# Resolve the compose config's image references to their current local SHA256 +# IDs, one per line, sorted. Reads from the local image store (updated by +# 'docker compose pull'), NOT from running containers. +# +# Returns non-zero (and prints nothing) if the image list can't be resolved or +# none of the images are present locally — callers MUST treat that as "unknown", +# never as "everything changed". _stack_image_ids() { local stack_dir="$1"; shift local -a env_flag=("$@") - local images + local images ids + # 'config --images' (compose v2.19+) outputs one image name per line; # fall back to parsing the resolved config yaml for older versions. - if images=$(cd "$stack_dir" && docker compose "${env_flag[@]}" config --images 2>/dev/null) \ - && [[ -n "$images" ]]; then - echo "$images" - else - cd "$stack_dir" && docker compose "${env_flag[@]}" config 2>/dev/null \ - | awk '/^\s+image:/{print $2}' | sort -u - fi | xargs -r docker image inspect --format '{{.Id}}' 2>/dev/null | sort + # NOTE: the fallback regex must be POSIX ([[:space:]], not \s) — mawk, the + # default awk on Debian, does not understand \s and silently matches nothing. + if ! images=$(cd "$stack_dir" && docker compose "${env_flag[@]}" config --images 2>/dev/null) \ + || [[ -z "$images" ]]; then + images=$(cd "$stack_dir" && docker compose "${env_flag[@]}" config 2>/dev/null \ + | awk '/^[[:space:]]+image:/{print $2}' | sort -u) || true + fi + [[ -n "$images" ]] || return 1 + + ids=$(echo "$images" | xargs -r docker image inspect --format '{{.Id}}' 2>/dev/null | sort) || true + [[ -n "$ids" ]] || return 1 + echo "$ids" } _update_run() { @@ -402,7 +412,7 @@ _update_run() { local -a all_stacks=() mapfile -t all_stacks < <(list_all_stacks) - local -a updated=() failed=() + local -a updated=() failed=() skipped=() local -i checked=0 log "Starting image update check (${#all_stacks[@]} stacks)" @@ -415,17 +425,56 @@ _update_run() { info "Checking ${BOLD}$stack${RESET}" ((checked++)) || true - local before after - before=$(_stack_image_ids "$stack_dir" "${env_flag[@]+"${env_flag[@]}"}") || true - if ! (cd "$stack_dir" && docker compose "${env_flag[@]}" pull 2>&1); then - warn " Pull reported a non-zero exit for $stack (see output above) — comparison may be unreliable" + # --- Pull, with retry. Nightly runs regularly hit transient registry + # errors (toomanyrequests, TLS resets, gitea registry restarting) that + # a short backoff clears. A pull that never succeeds must NOT fall + # through to the change check — otherwise a stale image looks + # "up to date" forever (this is what stuck coolify). --- + local pull_ok=false attempt + for attempt in 1 2 3; do + if (cd "$stack_dir" && docker compose "${env_flag[@]}" pull 2>&1); then + pull_ok=true + break + fi + warn " Pull failed for $stack (attempt $attempt/3)" + [[ $attempt -lt 3 ]] && sleep $((attempt * 5)) + done + if ! $pull_ok; then + err " Pull failed for $stack after 3 attempts — skipping (not recreated)" + skipped+=("$stack") + continue fi - after=$(_stack_image_ids "$stack_dir" "${env_flag[@]+"${env_flag[@]}"}") || true - if [[ "$before" != "$after" ]]; then + # --- Decide whether a recreate is needed by comparing the image IDs the + # RUNNING containers use against the IDs the (post-pull) config + # resolves to. Unlike the old before/after snapshot diff, a transient + # empty read here can't be mistaken for "everything changed" — an + # unresolvable list is skipped, not acted on. --- + local desired current + desired=$(_stack_image_ids "$stack_dir" "${env_flag[@]+"${env_flag[@]}"}") || desired="" + if [[ -z "$desired" ]]; then + warn " Could not resolve images for $stack — skipping (not recreated)" + skipped+=("$stack") + continue + fi + current=$(cd "$stack_dir" && docker compose "${env_flag[@]}" ps -q 2>/dev/null \ + | xargs -r docker inspect --format '{{.Image}}' 2>/dev/null | sort -u) + + if [[ -z "$current" ]]; then + info " Up to date (stack not running — left as is)" + continue + fi + + local stale=false cid + while IFS= read -r cid; do + [[ -z "$cid" ]] && continue + grep -qxF -- "$cid" <<<"$desired" || stale=true + done <<<"$current" + + if $stale; then info " Updates found — recreating $stack" - info " before: ${before:-}" - info " after: ${after:-}" + info " running: ${current//$'\n'/ }" + info " resolved: ${desired//$'\n'/ }" if (cd "$stack_dir" && docker compose "${env_flag[@]}" up -d 2>&1); then updated+=("$stack") ok " $stack updated" @@ -444,9 +493,14 @@ _update_run() { local date_str date_str="$(date '+%Y-%m-%d %H:%M')" + local skipped_line="" + [[ ${#skipped[@]} -gt 0 ]] && skipped_line="\n⚠️ Skipped (transient errors, not recreated): ${skipped[*]}" + if [[ ${#failed[@]} -gt 0 ]]; then - _notify "#cc0000" "❌ **Update failed** — $date_str\nUpdated: ${updated[*]:-none}\nFailed: ${failed[*]}" + _notify "#cc0000" "❌ **Update failed** — $date_str\nUpdated: ${updated[*]:-none}\nFailed: ${failed[*]}${skipped_line}" return 1 + elif [[ ${#skipped[@]} -gt 0 ]]; then + _notify "#e8a33d" "⚠️ **Update finished with warnings** — $date_str\nUpdated: ${updated[*]:-none}${skipped_line}" elif [[ ${#updated[@]} -gt 0 ]]; then _notify "#36a64f" "✅ **Updates applied** — $date_str\nStacks updated (${#updated[@]}/$checked): ${updated[*]}" else