Compare commits

...

2 Commits

Author SHA1 Message Date
valknar 31841d1ac3 fix(compose): remove empty labels keys left after watchtower label removal
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-09 19:44:01 +02:00
valknar 4c522961a9 feat(_update): replace watchtower with custom nightly update script
Removes the watchtower container in favour of a host-side script that
runs daily at 2:00 AM via systemd timer.  Mirrors the _backup pattern:
auto-discovers stacks, pulls images, recreates changed containers,
prunes dangling images, and notifies via n8n → Telegram.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-09 19:42:25 +02:00
15 changed files with 102 additions and 41 deletions
+26 -5
View File
@@ -10,7 +10,6 @@ Each stack is independently deployable with its own `compose.yml` and `.env`. Al
|---|---|---| |---|---|---|
| `traefik` | Reverse proxy, TLS termination | traefik | | `traefik` | Reverse proxy, TLS termination | traefik |
| `mailpit` | SMTP relay (no web UI) | mailpit | | `mailpit` | SMTP relay (no web UI) | mailpit |
| `watchtower` | Automatic container updates | watchtower |
| `umami` | Web analytics | umami, db | | `umami` | Web analytics | umami, db |
| `immich` | Photo & video management | immich, ml, redis, db | | `immich` | Photo & video management | immich, ml, redis, db |
| `affine` | Collaborative workspace & notes | affine, redis, db | | `affine` | Collaborative workspace & notes | affine, redis, db |
@@ -24,6 +23,7 @@ Each stack is independently deployable with its own `compose.yml` and `.env`. Al
| Directory | Description | | Directory | Description |
|---|---| |---|---|
| `_backup` | Daily restic backups to HiDrive (host script + systemd timer) | | `_backup` | Daily restic backups to HiDrive (host script + systemd timer) |
| `_update` | Nightly image update check + prune (host script + systemd timer) |
## Deployment ## Deployment
@@ -68,15 +68,36 @@ ssh vps 'systemctl status stacks-backup.timer'
ssh vps 'source ~/stacks/_backup/.env && restic -r /mnt/hidrive/users/valknar/Backup/stacks snapshots' ssh vps 'source ~/stacks/_backup/.env && restic -r /mnt/hidrive/users/valknar/Backup/stacks snapshots'
``` ```
## Updates
The `_update` script runs nightly at 2:00 AM. It pulls the latest image for every stack, recreates any containers whose image changed, prunes dangling images, and sends a Telegram notification via n8n.
```bash
# Deploy update stack
rsync -avz _update/ vps:~/stacks/_update/
ssh vps 'chmod +x ~/stacks/_update/update.sh'
# Install systemd units
ssh vps 'ln -sf ~/stacks/_update/stacks-update.service /etc/systemd/system/ && \
ln -sf ~/stacks/_update/stacks-update.timer /etc/systemd/system/ && \
systemctl daemon-reload && systemctl enable --now stacks-update.timer'
# Manual test run
ssh vps '~/stacks/_update/update.sh'
# Check timer status
ssh vps 'systemctl status stacks-update.timer'
```
## Notifications ## Notifications
Watchtower and the backup script both POST to an n8n webhook, which forwards messages to Telegram. The update script and the backup script both POST to an n8n webhook, which forwards messages to Telegram.
The webhook URL is set in two places: The webhook URL is set in:
- `_backup/.env``WEBHOOK_URL` - `_backup/.env``WEBHOOK_URL`
- `watchtower/.env``NOTIFICATION_URL` (uses `generic+https://` shoutrrr scheme) - `_update/.env``WEBHOOK_URL`
Both point to the same n8n workflow at `https://n8n.pivoine.art`. The workflow accepts `{ "message": "..." }` from both senders and forwards it to Telegram. Both point to the same n8n workflow at `https://n8n.pivoine.art`. The workflow accepts `{ "message": "..." }` and forwards it to Telegram.
## Data ## Data
+1
View File
@@ -0,0 +1 @@
WEBHOOK_URL=https://n8n.example.com/webhook/change_me
+10
View File
@@ -0,0 +1,10 @@
[Unit]
Description=Docker stacks image update
After=network-online.target docker.service
Wants=network-online.target
[Service]
Type=oneshot
User=root
ExecStart=/root/stacks/_update/update.sh
Environment=HOME=/root
+9
View File
@@ -0,0 +1,9 @@
[Unit]
Description=Daily Docker stacks image update at 2:00 AM
[Timer]
OnCalendar=*-*-* 02:00:00 Europe/Amsterdam
Persistent=true
[Install]
WantedBy=timers.target
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
STACKS_DIR="$(dirname "$SCRIPT_DIR")"
LOG_FILE="$SCRIPT_DIR/update.log"
set -a; source "$SCRIPT_DIR/.env"; set +a
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
notify() {
local color="$1" text="$2"
curl -sf -o /dev/null -X POST "$WEBHOOK_URL" \
-H 'Content-Type: application/json' \
-d "{\"message\":\"$text\",\"color\":\"$color\"}"
}
: > "$LOG_FILE"
log "Starting image update check"
updated=(); failed=()
for stack_dir in "$STACKS_DIR"/*/; do
stack=$(basename "$stack_dir")
[[ "$stack" == _* ]] && continue
[[ ! -f "$stack_dir/compose.yml" ]] && continue
log "Checking $stack"
cd "$stack_dir"
pull_output=$(docker compose pull 2>&1 | tee -a "$LOG_FILE")
if echo "$pull_output" | grep -q " Pulled$"; then
log " Updates found — recreating containers"
if docker compose up -d 2>&1 | tee -a "$LOG_FILE"; then
updated+=("$stack")
else
log " FAILED to recreate $stack"
failed+=("$stack")
fi
else
log " Up to date"
fi
done
log "Pruning dangling images"
docker image prune -f 2>&1 | tee -a "$LOG_FILE"
if [ ${#updated[@]} -eq 0 ] && [ ${#failed[@]} -eq 0 ]; then
notify "#36a64f" "✅ **Update check complete** — all images up to date"
elif [ ${#failed[@]} -gt 0 ]; then
notify "#cc0000" "❌ **Update failed**\nUpdated: ${updated[*]:-none}\nFailed: ${failed[*]}"
else
notify "#36a64f" "✅ **Updates applied**\nStacks: ${updated[*]}"
fi
log "Update check complete"
-1
View File
@@ -31,7 +31,6 @@ services:
- "traefik.http.routers.affine-web-secure.middlewares=security-headers@file,no-index@file" - "traefik.http.routers.affine-web-secure.middlewares=security-headers@file,no-index@file"
- "traefik.http.services.affine-web-secure.loadbalancer.server.port=3010" - "traefik.http.services.affine-web-secure.loadbalancer.server.port=3010"
- "traefik.docker.network=${NETWORK_NAME}" - "traefik.docker.network=${NETWORK_NAME}"
- "com.centurylinklabs.watchtower.enable=true"
networks: networks:
- compose_network - compose_network
affine_migration: affine_migration:
-2
View File
@@ -54,7 +54,6 @@ services:
- "traefik.http.routers.coolify-web-secure.priority=1" - "traefik.http.routers.coolify-web-secure.priority=1"
- "traefik.http.services.coolify.loadbalancer.server.port=8080" - "traefik.http.services.coolify.loadbalancer.server.port=8080"
- "traefik.docker.network=${NETWORK_NAME}" - "traefik.docker.network=${NETWORK_NAME}"
- "com.centurylinklabs.watchtower.enable=true"
networks: networks:
- compose_network - compose_network
realtime: realtime:
@@ -91,7 +90,6 @@ services:
- "traefik.http.routers.coolify-terminal-ws.priority=100" - "traefik.http.routers.coolify-terminal-ws.priority=100"
- "traefik.http.services.coolify-terminal.loadbalancer.server.port=6002" - "traefik.http.services.coolify-terminal.loadbalancer.server.port=6002"
- "traefik.docker.network=${NETWORK_NAME}" - "traefik.docker.network=${NETWORK_NAME}"
- "com.centurylinklabs.watchtower.enable=true"
networks: networks:
- compose_network - compose_network
redis: redis:
-3
View File
@@ -58,7 +58,6 @@ services:
- "traefik.http.routers.gitea-web-secure.middlewares=security-headers@file" - "traefik.http.routers.gitea-web-secure.middlewares=security-headers@file"
- "traefik.http.services.gitea-web-secure.loadbalancer.server.port=3000" - "traefik.http.services.gitea-web-secure.loadbalancer.server.port=3000"
- "traefik.docker.network=${NETWORK_NAME}" - "traefik.docker.network=${NETWORK_NAME}"
- "com.centurylinklabs.watchtower.enable=true"
networks: networks:
- compose_network - compose_network
runner: runner:
@@ -77,8 +76,6 @@ services:
- ../.data/gitea/runner:/data - ../.data/gitea/runner:/data
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock
- ./runner-config.yaml:/data/config.yaml:ro - ./runner-config.yaml:/data/config.yaml:ro
labels:
- "com.centurylinklabs.watchtower.enable=true"
restart: always restart: always
networks: networks:
- compose_network - compose_network
-3
View File
@@ -33,7 +33,6 @@ services:
- "traefik.http.routers.immich-web-secure.middlewares=security-headers@file,no-index@file" - "traefik.http.routers.immich-web-secure.middlewares=security-headers@file,no-index@file"
- "traefik.http.services.immich-web-secure.loadbalancer.server.port=2283" - "traefik.http.services.immich-web-secure.loadbalancer.server.port=2283"
- "traefik.docker.network=${NETWORK_NAME}" - "traefik.docker.network=${NETWORK_NAME}"
- "com.centurylinklabs.watchtower.enable=true"
networks: networks:
- compose_network - compose_network
ml: ml:
@@ -44,8 +43,6 @@ services:
volumes: volumes:
- ../.data/immich/model-cache:/cache - ../.data/immich/model-cache:/cache
restart: always restart: always
labels:
- "com.centurylinklabs.watchtower.enable=true"
networks: networks:
- compose_network - compose_network
redis: redis:
-2
View File
@@ -18,8 +18,6 @@ services:
volumes: volumes:
- ../.data/mailpit:/data - ../.data/mailpit:/data
restart: always restart: always
labels:
- "com.centurylinklabs.watchtower.enable=true"
networks: networks:
- compose_network - compose_network
networks: networks:
-1
View File
@@ -33,7 +33,6 @@ services:
- "traefik.http.routers.n8n-web-secure.middlewares=security-headers@file,no-index@file" - "traefik.http.routers.n8n-web-secure.middlewares=security-headers@file,no-index@file"
- "traefik.http.services.n8n-web-secure.loadbalancer.server.port=5678" - "traefik.http.services.n8n-web-secure.loadbalancer.server.port=5678"
- "traefik.docker.network=${NETWORK_NAME}" - "traefik.docker.network=${NETWORK_NAME}"
- "com.centurylinklabs.watchtower.enable=true"
networks: networks:
- compose_network - compose_network
db: db:
-1
View File
@@ -28,7 +28,6 @@ services:
- "traefik.http.routers.umami-web-secure.middlewares=security-headers@file,no-index@file" - "traefik.http.routers.umami-web-secure.middlewares=security-headers@file,no-index@file"
- "traefik.http.services.umami-web-secure.loadbalancer.server.port=3000" - "traefik.http.services.umami-web-secure.loadbalancer.server.port=3000"
- "traefik.docker.network=${NETWORK_NAME}" - "traefik.docker.network=${NETWORK_NAME}"
- "com.centurylinklabs.watchtower.enable=true"
networks: networks:
- compose_network - compose_network
db: db:
-1
View File
@@ -29,7 +29,6 @@ services:
- "traefik.http.routers.vaultwarden-web-secure.middlewares=security-headers@file,no-index@file" - "traefik.http.routers.vaultwarden-web-secure.middlewares=security-headers@file,no-index@file"
- "traefik.http.services.vaultwarden-web-secure.loadbalancer.server.port=80" - "traefik.http.services.vaultwarden-web-secure.loadbalancer.server.port=80"
- "traefik.docker.network=${NETWORK_NAME}" - "traefik.docker.network=${NETWORK_NAME}"
- "com.centurylinklabs.watchtower.enable=true"
networks: networks:
- compose_network - compose_network
networks: networks:
-1
View File
@@ -1 +0,0 @@
NOTIFICATION_URL=generic+https://n8n.example.com/webhook/change_me
-21
View File
@@ -1,21 +0,0 @@
services:
watchtower:
image: containrrr/watchtower:latest
container_name: watchtower
environment:
DOCKER_API_VERSION: "1.44"
WATCHTOWER_POLL_INTERVAL: 300
WATCHTOWER_LABEL_ENABLE: "true"
WATCHTOWER_CLEANUP: "true"
WATCHTOWER_INCLUDE_STOPPED: "false"
WATCHTOWER_INCLUDE_RESTARTING: "true"
WATCHTOWER_RUN_ONCE: "false"
WATCHTOWER_NOTIFICATIONS: ${NOTIFICATIONS:-}
WATCHTOWER_NOTIFICATION_URL: ${NOTIFICATION_URL:-}
WATCHTOWER_LOG_LEVEL: info
WATCHTOWER_ROLLING_RESTART: "false"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
restart: always
labels:
- "com.centurylinklabs.watchtower.enable=true"