From 4c522961a9af54e9eee083d8c5427b6f16a98ce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Kr=C3=BCger?= Date: Tue, 9 Jun 2026 19:42:25 +0200 Subject: [PATCH] feat(_update): replace watchtower with custom nightly update script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the watchtower container in favour of a host-side script that runs daily at 2:00 AM via systemd timer. Mirrors the _backup pattern: auto-discovers stacks, pulls images, recreates changed containers, prunes dangling images, and notifies via n8n → Telegram. Co-Authored-By: Claude Sonnet 4.6 --- README.md | 31 +++++++++++++++---- _update/.env.example | 1 + _update/stacks-update.service | 10 +++++++ _update/stacks-update.timer | 9 ++++++ _update/update.sh | 56 +++++++++++++++++++++++++++++++++++ affine/compose.yml | 1 - coolify/compose.yml | 2 -- gitea/compose.yml | 2 -- immich/compose.yml | 2 -- mailpit/compose.yml | 1 - n8n/compose.yml | 1 - umami/compose.yml | 1 - vaultwarden/compose.yml | 1 - watchtower/.env.example | 1 - watchtower/compose.yml | 21 ------------- 15 files changed, 102 insertions(+), 38 deletions(-) create mode 100644 _update/.env.example create mode 100644 _update/stacks-update.service create mode 100644 _update/stacks-update.timer create mode 100644 _update/update.sh delete mode 100644 watchtower/.env.example delete mode 100644 watchtower/compose.yml diff --git a/README.md b/README.md index 8717cde..a3aeb8a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ Each stack is independently deployable with its own `compose.yml` and `.env`. Al |---|---|---| | `traefik` | Reverse proxy, TLS termination | traefik | | `mailpit` | SMTP relay (no web UI) | mailpit | -| `watchtower` | Automatic container updates | watchtower | | `umami` | Web analytics | umami, db | | `immich` | Photo & video management | immich, ml, redis, db | | `affine` | Collaborative workspace & notes | affine, redis, db | @@ -24,6 +23,7 @@ Each stack is independently deployable with its own `compose.yml` and `.env`. Al | Directory | Description | |---|---| | `_backup` | Daily restic backups to HiDrive (host script + systemd timer) | +| `_update` | Nightly image update check + prune (host script + systemd timer) | ## Deployment @@ -68,15 +68,36 @@ ssh vps 'systemctl status stacks-backup.timer' ssh vps 'source ~/stacks/_backup/.env && restic -r /mnt/hidrive/users/valknar/Backup/stacks snapshots' ``` +## Updates + +The `_update` script runs nightly at 2:00 AM. It pulls the latest image for every stack, recreates any containers whose image changed, prunes dangling images, and sends a Telegram notification via n8n. + +```bash +# Deploy update stack +rsync -avz _update/ vps:~/stacks/_update/ +ssh vps 'chmod +x ~/stacks/_update/update.sh' + +# Install systemd units +ssh vps 'ln -sf ~/stacks/_update/stacks-update.service /etc/systemd/system/ && \ + ln -sf ~/stacks/_update/stacks-update.timer /etc/systemd/system/ && \ + systemctl daemon-reload && systemctl enable --now stacks-update.timer' + +# Manual test run +ssh vps '~/stacks/_update/update.sh' + +# Check timer status +ssh vps 'systemctl status stacks-update.timer' +``` + ## Notifications -Watchtower and the backup script both POST to an n8n webhook, which forwards messages to Telegram. +The update script and the backup script both POST to an n8n webhook, which forwards messages to Telegram. -The webhook URL is set in two places: +The webhook URL is set in: - `_backup/.env` → `WEBHOOK_URL` -- `watchtower/.env` → `NOTIFICATION_URL` (uses `generic+https://` shoutrrr scheme) +- `_update/.env` → `WEBHOOK_URL` -Both point to the same n8n workflow at `https://n8n.pivoine.art`. The workflow accepts `{ "message": "..." }` from both senders and forwards it to Telegram. +Both point to the same n8n workflow at `https://n8n.pivoine.art`. The workflow accepts `{ "message": "..." }` and forwards it to Telegram. ## Data diff --git a/_update/.env.example b/_update/.env.example new file mode 100644 index 0000000..e2d4ba6 --- /dev/null +++ b/_update/.env.example @@ -0,0 +1 @@ +WEBHOOK_URL=https://n8n.example.com/webhook/change_me diff --git a/_update/stacks-update.service b/_update/stacks-update.service new file mode 100644 index 0000000..a28b39b --- /dev/null +++ b/_update/stacks-update.service @@ -0,0 +1,10 @@ +[Unit] +Description=Docker stacks image update +After=network-online.target docker.service +Wants=network-online.target + +[Service] +Type=oneshot +User=root +ExecStart=/root/stacks/_update/update.sh +Environment=HOME=/root diff --git a/_update/stacks-update.timer b/_update/stacks-update.timer new file mode 100644 index 0000000..91d3bc2 --- /dev/null +++ b/_update/stacks-update.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Daily Docker stacks image update at 2:00 AM + +[Timer] +OnCalendar=*-*-* 02:00:00 Europe/Amsterdam +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/_update/update.sh b/_update/update.sh new file mode 100644 index 0000000..eb30e39 --- /dev/null +++ b/_update/update.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +STACKS_DIR="$(dirname "$SCRIPT_DIR")" +LOG_FILE="$SCRIPT_DIR/update.log" + +set -a; source "$SCRIPT_DIR/.env"; set +a + +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; } +notify() { + local color="$1" text="$2" + curl -sf -o /dev/null -X POST "$WEBHOOK_URL" \ + -H 'Content-Type: application/json' \ + -d "{\"message\":\"$text\",\"color\":\"$color\"}" +} + +: > "$LOG_FILE" +log "Starting image update check" + +updated=(); failed=() + +for stack_dir in "$STACKS_DIR"/*/; do + stack=$(basename "$stack_dir") + [[ "$stack" == _* ]] && continue + [[ ! -f "$stack_dir/compose.yml" ]] && continue + + log "Checking $stack" + cd "$stack_dir" + pull_output=$(docker compose pull 2>&1 | tee -a "$LOG_FILE") + + if echo "$pull_output" | grep -q " Pulled$"; then + log " Updates found — recreating containers" + if docker compose up -d 2>&1 | tee -a "$LOG_FILE"; then + updated+=("$stack") + else + log " FAILED to recreate $stack" + failed+=("$stack") + fi + else + log " Up to date" + fi +done + +log "Pruning dangling images" +docker image prune -f 2>&1 | tee -a "$LOG_FILE" + +if [ ${#updated[@]} -eq 0 ] && [ ${#failed[@]} -eq 0 ]; then + notify "#36a64f" "✅ **Update check complete** — all images up to date" +elif [ ${#failed[@]} -gt 0 ]; then + notify "#cc0000" "❌ **Update failed**\nUpdated: ${updated[*]:-none}\nFailed: ${failed[*]}" +else + notify "#36a64f" "✅ **Updates applied**\nStacks: ${updated[*]}" +fi + +log "Update check complete" diff --git a/affine/compose.yml b/affine/compose.yml index 5359f33..0c50ea3 100644 --- a/affine/compose.yml +++ b/affine/compose.yml @@ -31,7 +31,6 @@ services: - "traefik.http.routers.affine-web-secure.middlewares=security-headers@file,no-index@file" - "traefik.http.services.affine-web-secure.loadbalancer.server.port=3010" - "traefik.docker.network=${NETWORK_NAME}" - - "com.centurylinklabs.watchtower.enable=true" networks: - compose_network affine_migration: diff --git a/coolify/compose.yml b/coolify/compose.yml index 1f46e49..ec79d2e 100644 --- a/coolify/compose.yml +++ b/coolify/compose.yml @@ -54,7 +54,6 @@ services: - "traefik.http.routers.coolify-web-secure.priority=1" - "traefik.http.services.coolify.loadbalancer.server.port=8080" - "traefik.docker.network=${NETWORK_NAME}" - - "com.centurylinklabs.watchtower.enable=true" networks: - compose_network realtime: @@ -91,7 +90,6 @@ services: - "traefik.http.routers.coolify-terminal-ws.priority=100" - "traefik.http.services.coolify-terminal.loadbalancer.server.port=6002" - "traefik.docker.network=${NETWORK_NAME}" - - "com.centurylinklabs.watchtower.enable=true" networks: - compose_network redis: diff --git a/gitea/compose.yml b/gitea/compose.yml index 0dd1677..36e4b48 100644 --- a/gitea/compose.yml +++ b/gitea/compose.yml @@ -58,7 +58,6 @@ services: - "traefik.http.routers.gitea-web-secure.middlewares=security-headers@file" - "traefik.http.services.gitea-web-secure.loadbalancer.server.port=3000" - "traefik.docker.network=${NETWORK_NAME}" - - "com.centurylinklabs.watchtower.enable=true" networks: - compose_network runner: @@ -78,7 +77,6 @@ services: - /var/run/docker.sock:/var/run/docker.sock - ./runner-config.yaml:/data/config.yaml:ro labels: - - "com.centurylinklabs.watchtower.enable=true" restart: always networks: - compose_network diff --git a/immich/compose.yml b/immich/compose.yml index 2ead045..46f3edb 100644 --- a/immich/compose.yml +++ b/immich/compose.yml @@ -33,7 +33,6 @@ services: - "traefik.http.routers.immich-web-secure.middlewares=security-headers@file,no-index@file" - "traefik.http.services.immich-web-secure.loadbalancer.server.port=2283" - "traefik.docker.network=${NETWORK_NAME}" - - "com.centurylinklabs.watchtower.enable=true" networks: - compose_network ml: @@ -45,7 +44,6 @@ services: - ../.data/immich/model-cache:/cache restart: always labels: - - "com.centurylinklabs.watchtower.enable=true" networks: - compose_network redis: diff --git a/mailpit/compose.yml b/mailpit/compose.yml index 806c202..b0ae889 100644 --- a/mailpit/compose.yml +++ b/mailpit/compose.yml @@ -19,7 +19,6 @@ services: - ../.data/mailpit:/data restart: always labels: - - "com.centurylinklabs.watchtower.enable=true" networks: - compose_network networks: diff --git a/n8n/compose.yml b/n8n/compose.yml index 1374a8e..d958f30 100644 --- a/n8n/compose.yml +++ b/n8n/compose.yml @@ -33,7 +33,6 @@ services: - "traefik.http.routers.n8n-web-secure.middlewares=security-headers@file,no-index@file" - "traefik.http.services.n8n-web-secure.loadbalancer.server.port=5678" - "traefik.docker.network=${NETWORK_NAME}" - - "com.centurylinklabs.watchtower.enable=true" networks: - compose_network db: diff --git a/umami/compose.yml b/umami/compose.yml index e4676b6..748fe8a 100644 --- a/umami/compose.yml +++ b/umami/compose.yml @@ -28,7 +28,6 @@ services: - "traefik.http.routers.umami-web-secure.middlewares=security-headers@file,no-index@file" - "traefik.http.services.umami-web-secure.loadbalancer.server.port=3000" - "traefik.docker.network=${NETWORK_NAME}" - - "com.centurylinklabs.watchtower.enable=true" networks: - compose_network db: diff --git a/vaultwarden/compose.yml b/vaultwarden/compose.yml index 1e9e6aa..2db76bc 100644 --- a/vaultwarden/compose.yml +++ b/vaultwarden/compose.yml @@ -29,7 +29,6 @@ services: - "traefik.http.routers.vaultwarden-web-secure.middlewares=security-headers@file,no-index@file" - "traefik.http.services.vaultwarden-web-secure.loadbalancer.server.port=80" - "traefik.docker.network=${NETWORK_NAME}" - - "com.centurylinklabs.watchtower.enable=true" networks: - compose_network networks: diff --git a/watchtower/.env.example b/watchtower/.env.example deleted file mode 100644 index 2eca7de..0000000 --- a/watchtower/.env.example +++ /dev/null @@ -1 +0,0 @@ -NOTIFICATION_URL=generic+https://n8n.example.com/webhook/change_me diff --git a/watchtower/compose.yml b/watchtower/compose.yml deleted file mode 100644 index 4f36e23..0000000 --- a/watchtower/compose.yml +++ /dev/null @@ -1,21 +0,0 @@ -services: - watchtower: - image: containrrr/watchtower:latest - container_name: watchtower - environment: - DOCKER_API_VERSION: "1.44" - WATCHTOWER_POLL_INTERVAL: 300 - WATCHTOWER_LABEL_ENABLE: "true" - WATCHTOWER_CLEANUP: "true" - WATCHTOWER_INCLUDE_STOPPED: "false" - WATCHTOWER_INCLUDE_RESTARTING: "true" - WATCHTOWER_RUN_ONCE: "false" - WATCHTOWER_NOTIFICATIONS: ${NOTIFICATIONS:-} - WATCHTOWER_NOTIFICATION_URL: ${NOTIFICATION_URL:-} - WATCHTOWER_LOG_LEVEL: info - WATCHTOWER_ROLLING_RESTART: "false" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - restart: always - labels: - - "com.centurylinklabs.watchtower.enable=true"