diff --git a/README.md b/README.md index fa4635a..b310860 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,12 @@ Each stack is independently deployable with its own `compose.yml` and `.env`. Al | `sexy` | pivoine.art website | directus, frontend, redis, db | | `vaultwarden` | Password manager | vaultwarden | +## Tools + +| Directory | Description | +|---|---| +| `_backup` | Daily restic backups to HiDrive (host script + systemd timer) | + ## Deployment ```bash @@ -34,6 +40,32 @@ ssh vps 'cd ~/stacks/ && docker compose up -d' All stacks share the external `falcon_network` Docker network for inter-service communication (e.g. traefik routing, mailpit SMTP). +## Backup + +The `_backup` stack runs a daily restic backup at 3:00 AM. It dumps all Postgres databases, then backs up the entire `.data/` directory to HiDrive. Retention: 7 daily, 4 weekly, 6 monthly snapshots. Notifications go to Mattermost. + +```bash +# Deploy backup stack +rsync -avz _backup/ vps:~/stacks/_backup/ + +# Initialize restic repo (first time only) +ssh vps 'source ~/stacks/_backup/.env && restic init -r /mnt/hidrive/users/valknar/Backup/stacks' + +# Install systemd units +ssh vps 'ln -sf ~/stacks/_backup/stacks-backup.service /etc/systemd/system/ && \ + ln -sf ~/stacks/_backup/stacks-backup.timer /etc/systemd/system/ && \ + systemctl daemon-reload && systemctl enable --now stacks-backup.timer' + +# Manual test run +ssh vps '~/stacks/_backup/backup.sh' + +# Check timer status +ssh vps 'systemctl status stacks-backup.timer' + +# View snapshots +ssh vps 'source ~/stacks/_backup/.env && restic -r /mnt/hidrive/users/valknar/Backup/stacks snapshots' +``` + ## Data Persistent data is stored in `~/stacks/.data//` on the VPS using bind mounts. Database stacks use dedicated Postgres instances with simple credentials. diff --git a/_backup/backup.sh b/_backup/backup.sh new file mode 100755 index 0000000..eabd95a --- /dev/null +++ b/_backup/backup.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +set -euo pipefail + +STACKS_DIR="$HOME/stacks" +DATA_DIR="$STACKS_DIR/.data" +DUMP_DIR="$DATA_DIR/backup/dumps" +REPO="/mnt/hidrive/users/valknar/Backup/stacks" +LOG_FILE="$STACKS_DIR/_backup/backup.log" + +# Load environment +set -a +source "$STACKS_DIR/_backup/.env" +set +a + +export RESTIC_REPOSITORY="$REPO" +export RESTIC_PASSWORD + +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; } + +notify() { + local color="$1" text="$2" + curl -sf -o /dev/null -X POST "$MATTERMOST_WEBHOOK" \ + -H 'Content-Type: application/json' \ + -d "{\"attachments\":[{\"color\":\"$color\",\"text\":\"$text\"}]}" +} + +# Truncate log on each run +: > "$LOG_FILE" + +log "Starting backup" + +# --- Postgres dumps --- +mkdir -p "$DUMP_DIR" + +declare -A DATABASES=( + [umami_db]="umami:umami" + [joplin_db]="joplin:joplin" + [gitea_db]="gitea:gitea" + [mattermost_db]="mattermost:mattermost" + [sexy_db]="directus:directus" + [immich_db]="immich:immich" + [coolify_db]="coolify:coolify" +) + +dump_errors=() +for container in "${!DATABASES[@]}"; do + IFS=: read -r user db <<< "${DATABASES[$container]}" + log "Dumping $db from $container" + if docker exec "$container" pg_dump -U "$user" "$db" > "$DUMP_DIR/$db.sql" 2>>"$LOG_FILE"; then + log " OK ($(du -h "$DUMP_DIR/$db.sql" | cut -f1))" + else + log " FAILED: $container" + dump_errors+=("$db") + fi +done + +if [ ${#dump_errors[@]} -gt 0 ]; then + log "WARNING: Failed dumps: ${dump_errors[*]}" +fi + +# --- Restic backup --- +log "Running restic backup" +if ! restic backup "$DATA_DIR" --tag stacks 2>&1 | tee -a "$LOG_FILE"; then + log "FATAL: restic backup failed" + notify "#cc0000" ":x: **Backup failed** — restic backup error. Check \`$LOG_FILE\`." + exit 1 +fi + +# --- Restic prune --- +log "Pruning old snapshots" +if ! restic forget --prune --keep-daily 7 --keep-weekly 4 --keep-monthly 6 2>&1 | tee -a "$LOG_FILE"; then + log "WARNING: restic forget failed" +fi + +# --- Summary --- +snapshot_info=$(restic snapshots --latest 1 --compact 2>/dev/null | tail -3 | head -1) +repo_stats=$(restic stats 2>/dev/null | grep "Total Size" || true) + +summary=":white_check_mark: **Backup complete**" +[ ${#dump_errors[@]} -gt 0 ] && summary+="\n:warning: Failed dumps: ${dump_errors[*]}" +summary+="\nLatest: \`$snapshot_info\`" +[ -n "$repo_stats" ] && summary+="\nRepo: $repo_stats" + +notify "#36a64f" "$summary" +log "Backup complete" diff --git a/_backup/stacks-backup.service b/_backup/stacks-backup.service new file mode 100644 index 0000000..e587538 --- /dev/null +++ b/_backup/stacks-backup.service @@ -0,0 +1,10 @@ +[Unit] +Description=Backup stacks data to HiDrive via restic +After=network-online.target docker.service +Wants=network-online.target + +[Service] +Type=oneshot +User=root +ExecStart=/root/stacks/_backup/backup.sh +Environment=HOME=/root diff --git a/_backup/stacks-backup.timer b/_backup/stacks-backup.timer new file mode 100644 index 0000000..0ecb9b3 --- /dev/null +++ b/_backup/stacks-backup.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Daily stacks backup at 3:00 AM + +[Timer] +OnCalendar=*-*-* 03:00:00 Europe/Amsterdam +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/vaultwarden/compose.yml b/vaultwarden/compose.yml index dd9481e..6a0c621 100644 --- a/vaultwarden/compose.yml +++ b/vaultwarden/compose.yml @@ -11,6 +11,7 @@ services: INVITATIONS_ALLOWED: "true" SHOW_PASSWORD_HINT: "false" SMTP_HOST: mailpit + SMTP_FROM: ${SMTP_FROM} SMTP_FROM_NAME: Vaultwarden SMTP_SECURITY: off SMTP_PORT: 1025