admin/scripts/vds-backup-rsync-kreknin/run.sh

#!/bin/bash
# Daily VDS → kreknin backup. Triggered by /etc/cron.d/vds-backup at 05:00 MSK.
set -euo pipefail

ENV_FILE=/opt/stacks/backup/.env
NTFY_ENV=/opt/stacks/ntfy/.env

# shellcheck disable=SC1090
source "$ENV_FILE"
# shellcheck disable=SC1090
source "$NTFY_ENV"

TODAY=$(date +%Y-%m-%d)
LOG_DIR=/var/log/vds-backup
LOG="$LOG_DIR/$TODAY.log"
LOCK=/tmp/vds-backup.lock
DUMP_DIR=/tmp/vds-backup-dumps/$TODAY
DEST_PATH="$DEST_BASE/$TODAY"
LATEST="$DEST_BASE/latest"

mkdir -p "$LOG_DIR" "$DUMP_DIR"
exec >> "$LOG" 2>&1
echo "===== VDS backup start $TODAY $(date -Is) ====="

# single-instance lock
exec 9>"$LOCK"
if ! flock -n 9; then echo "already running, exiting"; exit 0; fi

START=$(date +%s)
SSH_OPTS="-i $SSH_KEY -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30"

HOST_LABEL=VDS
SOURCE_DISPLAY="vds.kzntsv.site (89.253.255.94)"

notify() {
  local title="$1" msg="$2" tags="${3:-}" prio="${4:-default}"
  curl -sS --max-time 10 -u "$NTFY_USER:$NTFY_PASS" \
    -H "Title: $title" -H "Tags: $tags" -H "Priority: $prio" \
    -d "$msg" "$NTFY_URL/vds-backup" > /dev/null || true
}

email_send() {
  local subj="$1" body="$2"
  printf "From: noreply@snolla.com\nTo: %s\nSubject: %s\n\n%s\n" \
    "$EMAIL" "$subj" "$body" | msmtp "$EMAIL" || true
}

# Duration human-format: Xm YYs (e.g. "21m07s")
fmt_duration() {
  local s=$1
  printf "%dm%02ds" $((s/60)) $((s%60))
}

on_error() {
  local line=$1 rc=$2
  local err="line $line, exit $rc"
  local now=$(date +%s)
  local dur=$((now - ${START:-now}))
  local dur_h=$(fmt_duration "$dur")
  local tail_log
  tail_log=$(tail -40 "$LOG" 2>/dev/null || true)

  echo "FATAL: $HOST_LABEL backup FAILED — $err"

  # ntfy push (single line)
  notify "$HOST_LABEL backup FAILED $TODAY" \
         "After $dur_h: $err. See $LOG" \
         red_circle high

  # email (structured)
  email_send "[$HOST_LABEL] backup FAILED $TODAY" \
"$HOST_LABEL daily backup FAILED.

Date:     $TODAY
Duration: $dur_h
Error:    $err
Source:   $SOURCE_DISPLAY
Log:      $LOG

Tail (last 40 lines):
$tail_log"
  exit "$rc"
}
trap 'on_error $LINENO $?' ERR

# === Step 1: DB dumps ===
echo "--- DB dumps to $DUMP_DIR ---"

docker exec postgres pg_dumpall -U postgres | gzip > "$DUMP_DIR/postgres.sql.gz"
echo "postgres: $(stat -c %s "$DUMP_DIR/postgres.sql.gz") bytes"

docker exec mariadb mariadb-dump -uroot -p"$MARIADB_ROOT_PASSWORD" --all-databases --single-transaction --quick 2>/dev/null | gzip > "$DUMP_DIR/mariadb.sql.gz"
echo "mariadb: $(stat -c %s "$DUMP_DIR/mariadb.sql.gz") bytes"

docker exec mongo mongodump --archive --quiet --ssl --sslAllowInvalidCertificates --uri="mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@localhost:27017/?authSource=admin" 2>/dev/null | gzip > "$DUMP_DIR/mongo.archive.gz"
echo "mongo: $(stat -c %s "$DUMP_DIR/mongo.archive.gz") bytes"

docker exec redis redis-cli --tls --insecure -a "$REDIS_PASSWORD" --no-auth-warning SAVE > /dev/null
docker cp redis:/data/dump.rdb "$DUMP_DIR/redis.rdb"
gzip -f "$DUMP_DIR/redis.rdb"
echo "redis: $(stat -c %s "$DUMP_DIR/redis.rdb.gz") bytes"

# MSSQL: backup 5 prod DBs via sqlcmd into bind-mounted /var/opt/mssql/backups/
# WITH FORMAT (not INIT): Express can't write to a media set previously formatted
# WITH COMPRESSION (e.g. migration-era .bak); FORMAT recreates the media set fresh.
source /opt/stacks/databases/mssql/.env  # injects MSSQL_SA_PASSWORD
MSSQL_BACKUP_DIR=/opt/stacks/databases/mssql/backups
for DB in MoreThenCms StayerCalculator StayerPrice TireService stostayer; do
  docker exec mssql /opt/mssql-tools18/bin/sqlcmd \
    -S localhost -U sa -P "$MSSQL_SA_PASSWORD" -C -b \
    -Q "BACKUP DATABASE [$DB] TO DISK='/var/opt/mssql/backups/${DB}.bak' WITH COPY_ONLY, FORMAT" \
    > /dev/null
  mv "$MSSQL_BACKUP_DIR/${DB}.bak" "$DUMP_DIR/mssql-${DB}.bak"
  echo "mssql $DB: $(stat -c %s "$DUMP_DIR/mssql-${DB}.bak") bytes"
done
unset MSSQL_SA_PASSWORD

# === Step 2: rsync to kreknin ===
echo "--- rsync to $DEST_USER@$DEST_HOST:$DEST_PATH ---"
ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" "mkdir -p '$DEST_PATH'"

LINK_DEST_ARG=""
if ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" "test -L '$LATEST' || test -d '$LATEST'" 2>/dev/null; then
  LINK_DEST_ARG="--link-dest=$LATEST"
fi

rsync -aHh --info=stats2 --delete $LINK_DEST_ARG \
  -e "ssh $SSH_OPTS" \
  /opt/stacks/gitea \
  /opt/stacks/verdaccio/storage \
  /opt/stacks/verdaccio/config \
  /opt/stacks/registry \
  /opt/stacks/traefik \
  /opt/stacks/portainer \
  /opt/stacks/ntfy \
  /opt/stacks/backup \
  /opt/stacks/owncloud \
  /etc/ssh \
  /etc/ufw \
  /etc/hosts \
  /etc/docker \
  "$DUMP_DIR" \
  /home/vitya/.ssh \
  "$DEST_USER@$DEST_HOST:$DEST_PATH/" || true

# Update latest symlink
ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" "rm -f '$LATEST' && ln -s '$DEST_PATH' '$LATEST'"

# === Step 3: retention prune ===
echo "--- prune snapshots older than $RETENTION_DAYS days ---"
ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" "cd '$DEST_BASE' && ls -1d 20*-*-* 2>/dev/null | sort | head -n -$RETENTION_DAYS | xargs -r rm -rf"

# === Step 4: local cleanup ===
rm -rf "$DUMP_DIR"

# === Step 5: report ===
END=$(date +%s)
DURATION=$((END - START))
SIZE=$(ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" "du -sh '$DEST_PATH' 2>/dev/null | cut -f1" || echo "?")
SNAPSHOT_COUNT=$(ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" "ls -1d $DEST_BASE/20*-*-* 2>/dev/null | wc -l" || echo "?")
DURATION_HUMAN=$(fmt_duration "$DURATION")

# ntfy push (single line)
NTFY_BODY="$DURATION_HUMAN, size=$SIZE, snapshots=$SNAPSHOT_COUNT, dest=kreknin:$DEST_PATH"
notify "$HOST_LABEL backup OK $TODAY" "$NTFY_BODY" green_circle

# email (structured)
EMAIL_BODY="$HOST_LABEL daily backup completed successfully.

Date:      $TODAY
Duration:  $DURATION_HUMAN
Size:      $SIZE
Snapshots: $SNAPSHOT_COUNT
Source:    $SOURCE_DISPLAY
Dest:      kreknin:$DEST_PATH

Components:
- /opt/stacks/{gitea,verdaccio,registry,traefik,portainer,ntfy,backup,owncloud}
- /etc/{ssh,ufw,hosts,docker}
- /home/vitya/.ssh
- DB dumps: postgres, mariadb, mongo, redis

Log: $LOG"
email_send "[$HOST_LABEL] backup OK $TODAY" "$EMAIL_BODY"

echo "$HOST_LABEL backup OK: $NTFY_BODY"
echo "===== $HOST_LABEL backup done $(date -Is) ====="