admin/scripts/books-vds-backup-daily-kreknin/run.sh

#!/bin/bash
# Daily books VDS → kreknin backup. Triggered by /etc/cron.d/books-vds-backup at 06:00 MSK.
# Pattern: mirror VDS scripts/vds-backup-rsync-kreknin/run.sh, adapted for books VDS hybrid stack.
set -euo pipefail

ENV_FILE=/opt/stacks/backup/.env

# shellcheck disable=SC1090
source "$ENV_FILE"

TODAY=$(date +%Y-%m-%d)
LOG_DIR=/var/log/books-vds-backup
LOG="$LOG_DIR/$TODAY.log"
LOCK=/tmp/books-vds-backup.lock
DUMP_DIR=/tmp/books-vds-backup-dumps/$TODAY
DEST_PATH="$DEST_BASE/$TODAY"
LATEST="$DEST_BASE/latest"

mkdir -p "$LOG_DIR" "$DUMP_DIR"
exec >> "$LOG" 2>&1
echo "===== BOOKS-VDS backup start $TODAY $(date -Is) ====="

# single-instance lock
exec 9>"$LOCK"
if ! flock -n 9; then echo "already running, exiting"; exit 0; fi

START=$(date +%s)
SSH_OPTS="-i $SSH_KEY -o StrictHostKeyChecking=yes -o ConnectTimeout=30 -o LogLevel=ERROR"
# Note: kreknin host key must be pre-populated in /root/.ssh/known_hosts (CentOS 7 OpenSSH 7.4 has no accept-new option)

HOST_LABEL=BOOKS-VDS
SOURCE_DISPLAY="89.253.255.133 (books VDS)"

notify() {
  local title="$1" msg="$2" tags="${3:-}" prio="${4:-default}"
  curl -sS --max-time 10 -u "$NTFY_USER:$NTFY_PASS" \
    -H "Title: $title" -H "Tags: $tags" -H "Priority: $prio" \
    -d "$msg" "$NTFY_URL/vds-backup" > /dev/null || true
}

email_send() {
  local subj="$1" body="$2"
  local tmpf
  tmpf=$(mktemp)
  # RFC 5322: CRLF headers + Date required by Yandex
  {
    printf 'From: %s\r\n' "$SMTP_FROM"
    printf 'To: %s\r\n' "$OPS_NOTIFY_EMAIL"
    printf 'Subject: %s\r\n' "$subj"
    printf 'Date: %s\r\n' "$(date -R)"
    printf 'MIME-Version: 1.0\r\n'
    printf 'Content-Type: text/plain; charset=UTF-8\r\n'
    printf '\r\n'
    echo "$body" | sed 's/$/\r/'
  } > "$tmpf"
  # smtps:// (implicit TLS) for Yandex port 465; smtp:// (STARTTLS) would be port 587.
  # SMTP_PORT in pass is 465 — use smtps:// scheme.
  curl -sS --max-time 30 \
    --ssl-reqd \
    --url "smtps://$SMTP_HOST:$SMTP_PORT" \
    --user "$SMTP_USER:$SMTP_PASS" \
    --mail-from "$SMTP_FROM" \
    --mail-rcpt "$OPS_NOTIFY_EMAIL" \
    --upload-file "$tmpf" || true
  rm -f "$tmpf"
}

fmt_duration() {
  local s=$1
  printf "%dm%02ds" $((s/60)) $((s%60))
}

on_error() {
  local line=$1 rc=$2
  local err="line $line, exit $rc"
  local now
  now=$(date +%s)
  local dur=$((now - ${START:-now}))
  local dur_h
  dur_h=$(fmt_duration "$dur")
  local tail_log
  tail_log=$(tail -40 "$LOG" 2>/dev/null || true)

  echo "FATAL: $HOST_LABEL backup FAILED — $err"

  notify "$HOST_LABEL backup FAILED $TODAY" \
         "After $dur_h: $err. See $LOG" \
         red_circle high

  email_send "[$HOST_LABEL] backup FAILED $TODAY" \
"$HOST_LABEL daily backup FAILED.

Date:     $TODAY
Duration: $dur_h
Error:    $err
Source:   $SOURCE_DISPLAY
Log:      $LOG

Tail (last 40 lines):
$tail_log"
  exit "$rc"
}
trap 'on_error $LINENO $?' ERR

# === Step 1: DB dumps ===
echo "--- DB dumps to $DUMP_DIR ---"

docker exec books-db mariadb-dump -uroot -p"$BOOKS_DB_ROOT_PASSWORD" \
  --all-databases --single-transaction --quick 2>/dev/null \
  | gzip > "$DUMP_DIR/mariadb.sql.gz"
echo "mariadb: $(stat -c %s "$DUMP_DIR/mariadb.sql.gz") bytes"

docker exec mongo mongodump --archive --quiet \
  --uri="mongodb://$BOOKS_MONGO_USERNAME:$BOOKS_MONGO_PASSWORD@localhost:27017/?authSource=admin" 2>/dev/null \
  | gzip > "$DUMP_DIR/mongo.archive.gz"
echo "mongo (shared): $(stat -c %s "$DUMP_DIR/mongo.archive.gz") bytes"

docker exec books-job-scheduler-mongo mongodump --archive --quiet 2>/dev/null \
  | gzip > "$DUMP_DIR/job-scheduler-mongo.archive.gz"
echo "job-scheduler-mongo: $(stat -c %s "$DUMP_DIR/job-scheduler-mongo.archive.gz") bytes"

# bookva tenant (separate cloned stack): bookva-db shares BOOKS_DB_ROOT_PASSWORD;
# bookva-mongo is no-auth. Containers are named volumes (not bind paths) → logical dumps.
docker exec bookva-db mariadb-dump -uroot -p"$BOOKS_DB_ROOT_PASSWORD" \
  --all-databases --single-transaction --quick 2>/dev/null \
  | gzip > "$DUMP_DIR/bookva-mariadb.sql.gz"
echo "bookva-db (mariadb): $(stat -c %s "$DUMP_DIR/bookva-mariadb.sql.gz") bytes"

docker exec bookva-mongo mongodump --archive --quiet 2>/dev/null \
  | gzip > "$DUMP_DIR/bookva-mongo.archive.gz"
echo "bookva-mongo: $(stat -c %s "$DUMP_DIR/bookva-mongo.archive.gz") bytes"

# === Step 2: ES snapshot via REST ===
echo "--- ES snapshot daily-$TODAY in repo $ES_SNAPSHOT_REPO ---"
ES_SNAP="daily-$TODAY"
SNAP_RESULT=$(docker exec elasticsearch curl -sS -X PUT \
  "http://localhost:9200/_snapshot/$ES_SNAPSHOT_REPO/$ES_SNAP?wait_for_completion=true" \
  -H 'Content-Type: application/json')
echo "ES snapshot result: $SNAP_RESULT"
if ! echo "$SNAP_RESULT" | grep -q '"state":"SUCCESS"'; then
  echo "ES snapshot did NOT report SUCCESS state — failing"
  exit 3
fi

echo "--- prune ES snapshots older than $RETENTION_DAYS ---"
# Pure bash parse — no jq/python dependency
ALL_SNAPS=$(docker exec elasticsearch curl -sS "http://localhost:9200/_snapshot/$ES_SNAPSHOT_REPO/_all" \
  | grep -oE '"snapshot":"daily-[0-9-]+"' \
  | sed -E 's/.*"(daily-[0-9-]+)".*/\1/' \
  | sort)
SNAP_COUNT=$(echo "$ALL_SNAPS" | wc -l)
PRUNE_COUNT=$((SNAP_COUNT - RETENTION_DAYS))
if [ $PRUNE_COUNT -gt 0 ]; then
  echo "$ALL_SNAPS" | head -n "$PRUNE_COUNT" | while read -r snap; do
    [ -z "$snap" ] && continue
    echo "deleting ES snapshot: $snap"
    docker exec elasticsearch curl -sS -X DELETE \
      "http://localhost:9200/_snapshot/$ES_SNAPSHOT_REPO/$snap" || true
    echo
  done
fi

# === Step 3: rsync to kreknin ===
echo "--- rsync to $DEST_USER@$DEST_HOST:$DEST_PATH ---"
ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" "mkdir -p '$DEST_PATH'"

LINK_DEST_ARG=""
if ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" "test -L '$LATEST' || test -d '$LATEST'" 2>/dev/null; then
  LINK_DEST_ARG="--link-dest=$LATEST"
fi

rsync -aHh --info=stats2 --delete $LINK_DEST_ARG \
  -e "ssh $SSH_OPTS" \
  /opt/books \
  /usr/docker/elasticsearch/snapshots \
  /usr/docker/minio/data \
  /var/lib/docker/volumes/bookva-minio-data \
  /usr/docker/traefik/letsencrypt \
  /usr/docker/traefik/data \
  /etc/ssh \
  /etc/hosts \
  /etc/cron.d \
  /root/.ssh \
  "$DUMP_DIR" \
  "$DEST_USER@$DEST_HOST:$DEST_PATH/" || true

ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" "rm -f '$LATEST' && ln -s '$DEST_PATH' '$LATEST'"

# === Step 4: retention prune on kreknin ===
echo "--- prune snapshots older than $RETENTION_DAYS days on kreknin ---"
ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" \
  "cd '$DEST_BASE' && ls -1d 20*-*-* 2>/dev/null | sort | head -n -$RETENTION_DAYS | xargs -r rm -rf"

# === Step 5: local cleanup ===
rm -rf "$DUMP_DIR"

# === Step 6: report ===
END=$(date +%s)
DURATION=$((END - START))
SIZE=$(ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" "du -sh '$DEST_PATH' 2>/dev/null | cut -f1" || echo "?")
SNAPSHOT_COUNT=$(ssh $SSH_OPTS "$DEST_USER@$DEST_HOST" "ls -1d $DEST_BASE/20*-*-* 2>/dev/null | wc -l" || echo "?")
DURATION_HUMAN=$(fmt_duration "$DURATION")

NTFY_BODY="$DURATION_HUMAN, size=$SIZE, snapshots=$SNAPSHOT_COUNT, dest=kreknin:$DEST_PATH"
notify "$HOST_LABEL backup OK $TODAY" "$NTFY_BODY" green_circle

EMAIL_BODY="$HOST_LABEL daily backup completed successfully.

Date:      $TODAY
Duration:  $DURATION_HUMAN
Size:      $SIZE
Snapshots: $SNAPSHOT_COUNT
Source:    $SOURCE_DISPLAY
Dest:      kreknin:$DEST_PATH

Components:
- /opt/books/{api,job-scheduler,task-runner,ntfy,books-ops-mcp}
- /usr/docker/{minio/data, elasticsearch/snapshots, traefik/{letsencrypt,data}}
- /etc/{ssh,hosts,cron.d}
- /root/.ssh
- DB dumps: books-db (mariadb), mongo (shared), books-job-scheduler-mongo, bookva-db (mariadb), bookva-mongo
- bookva-minio: raw volume (bookva-minio-data)
- ES snapshot (slovo): $ES_SNAP  [bookva-es: derivative index == slovo, snapshot pending Portainer path.repo]

Log: $LOG"
email_send "[$HOST_LABEL] backup OK $TODAY" "$EMAIL_BODY"

echo "$HOST_LABEL backup OK: $NTFY_BODY"
echo "===== $HOST_LABEL backup done $(date -Is) ====="