ops: snolla SMTP rollout 2026-08-14 — monitor deployed (VDS cron+ntfy), msmtp/backup-mail on new creds, pilonuxt compose tag 954e29e

scripts/snolla-smtp-monitor/monitor.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+"""snolla-smtp-monitor — health check of snolla sites' SMTP mail sending (lives on vds-kzntsv).
+
+Background: 2026-08-12..14 Yandex disabled SMTP for noreply@snolla.com → site forms kept
+answering "success" while order emails silently died for 2 days (mailer swallows errors).
+This monitor catches that class of failure at the next tick.
+
+Checks (run daily via /etc/cron.d/snolla-smtp-monitor):
+  1. SMTP auth against smtp.yandex.ru:465 with the current snolla mailer creds
+     (catches 525 SMTP disabled / 535 invalid login / connection failure).
+  2. Config parity: SMTP_USER/SMTP_PASSWORD in Portainer stack envs of the env-driven
+     snolla sites must equal the expected creds (catches config drift before it breaks).
+  3. Weekly (Mon): real test send to OPS_NOTIFY_EMAIL (end-to-end proof through Yandex).
+
+On failure: ntfy push (topic vds-ops) + exit 1. On success: log line only (silent).
+
+Config: /root/.snolla-smtp-monitor.env (chmod 600; mirror of `pass snolla-smtp/full-env` +
+ntfy + portainer sections of `pass vds-kzntsv/full-env`). Keys:
+  SMTP_HOST SMTP_PORT SMTP_USER SMTP_PASS OPS_NOTIFY_EMAIL
+  NTFY_URL NTFY_USER NTFY_PASS NTFY_TOPIC
+  PORTAINER_URL PORTAINER_USER PORTAINER_PASS PORTAINER_STACKS(comma list of ids or names)
+"""
+import base64
+import datetime
+import email.message
+import json
+import smtplib
+import ssl
+import sys
+import urllib.request
+
+ENV_PATH = "/root/.snolla-smtp-monitor.env"
+LOG_PATH = "/var/log/snolla-smtp-monitor.log"
+
+# Env-driven snolla stacks (SMTP creds in Portainer stack env). pilonuxt (16) excluded —
+# its config is baked into the image; covered by check 1.
+DEFAULT_STACKS = [17, 18, 19, 20, 21, 22, 23]
+
+
+def load_env(path):
+    cfg = {}
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            k, _, v = line.partition("=")
+            cfg[k.strip()] = v.strip()
+    return cfg
+
+
+def log(cfg, msg):
+    ts = datetime.datetime.now().astimezone().strftime("%Y-%m-%dT%H:%M:%S%z")
+    line = f"{ts} {msg}"
+    print(line, flush=True)
+    try:
+        with open(LOG_PATH, "a", encoding="utf-8") as f:
+            f.write(line + "\n")
+    except Exception as e:
+        print(f"log write failed: {e}", flush=True)
+
+
+def ntfy_alert(cfg, title, body):
+    try:
+        url = f"{cfg['NTFY_URL']}/{cfg.get('NTFY_TOPIC', 'vds-ops')}"
+        req = urllib.request.Request(url, data=body.encode(), method="POST")
+        token = base64.b64encode(f"{cfg['NTFY_USER']}:{cfg['NTFY_PASS']}".encode()).decode()
+        req.add_header("Authorization", "Basic " + token)
+        # urllib encodes header VALUES as latin-1 → keep title ASCII (emoji goes in body)
+        req.add_header("Title", title.encode("ascii", "replace").decode("ascii"))
+        req.add_header("Priority", "high")
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            log(cfg, f"ntfy alert sent: HTTP {resp.status}")
+    except Exception as e:
+        log(cfg, f"ntfy push FAILED: {e}")
+
+
+def smtp_login(cfg):
+    ctx = ssl.create_default_context()
+    s = smtplib.SMTP_SSL(cfg["SMTP_HOST"], int(cfg.get("SMTP_PORT", 465)),
+                         context=ctx, timeout=25)
+    try:
+        s.login(cfg["SMTP_USER"], cfg["SMTP_PASS"])
+    finally:
+        s.quit()
+    return True
+
+
+def smtp_send_test(cfg):
+    msg = email.message.EmailMessage()
+    msg["From"] = cfg["SMTP_USER"]
+    msg["To"] = cfg["OPS_NOTIFY_EMAIL"]
+    msg["Subject"] = "[snolla-monitor] weekly SMTP send test OK"
+    msg.set_content(
+        "Automated weekly test send from snolla-smtp-monitor (vds-kzntsv). "
+        "If you see this, SMTP sending for snolla sites works.\n"
+        f"ts={datetime.datetime.now().astimezone().isoformat()}"
+    )
+    ctx = ssl.create_default_context()
+    s = smtplib.SMTP_SSL(cfg["SMTP_HOST"], int(cfg.get("SMTP_PORT", 465)),
+                         context=ctx, timeout=25)
+    try:
+        s.login(cfg["SMTP_USER"], cfg["SMTP_PASS"])
+        s.send_message(msg)
+    finally:
+        s.quit()
+    return True
+
+
+def portainer_parity(cfg, expected_user, expected_pass):
+    """Every env-driven snolla stack must carry the expected SMTP creds in its env."""
+    bad = []
+    # JWT auth (API key gave 401 historically — use admin password JWT)
+    auth_req = urllib.request.Request(
+        f"{cfg['PORTAINER_URL']}/api/auth",
+        data=json.dumps({"username": cfg["PORTAINER_USER"],
+                         "password": cfg["PORTAINER_PASS"]}).encode(),
+        headers={"Content-Type": "application/json"}, method="POST")
+    with urllib.request.urlopen(auth_req, timeout=15) as r:
+        jwt = json.loads(r.read())["jwt"]
+    stacks_req = urllib.request.Request(f"{cfg['PORTAINER_URL']}/api/stacks",
+                                        headers={"Authorization": "Bearer " + jwt})
+    with urllib.request.urlopen(stacks_req, timeout=15) as r:
+        stacks = json.loads(r.read())
+    by_id = {s["Id"]: s for s in stacks}
+    for sid in cfg.get("PORTAINER_STACKS", DEFAULT_STACKS):
+        s = by_id.get(int(sid))
+        if s is None:
+            bad.append(f"stack {sid}: NOT FOUND")
+            continue
+        env = {e.get("name"): e.get("value") for e in s.get("Env", [])}
+        u, p = env.get("SMTP_USER"), env.get("SMTP_PASSWORD")
+        if u != expected_user or p != expected_pass:
+            bad.append(f"stack {sid} ({s.get('Name')}): SMTP env MISMATCH "
+                       f"(user={u!r} pass_len={len(p) if p else 0})")
+    return bad
+
+
+def main():
+    cfg = load_env(ENV_PATH)
+    problems = []
+
+    # 1. SMTP auth
+    try:
+        smtp_login(cfg)
+        log(cfg, "OK smtp-auth")
+    except Exception as e:
+        problems.append(f"SMTP auth FAILED: {e}")
+
+    # 3. weekly real send (Monday)
+    if datetime.date.today().weekday() == 0:
+        try:
+            smtp_send_test(cfg)
+            log(cfg, "OK weekly-send")
+        except Exception as e:
+            problems.append(f"weekly test send FAILED: {e}")
+
+    # 2. Portainer env parity
+    try:
+        bad = portainer_parity(cfg, cfg["SMTP_USER"], cfg["SMTP_PASS"])
+        if bad:
+            problems.append("parity: " + "; ".join(bad))
+        else:
+            log(cfg, "OK parity")
+    except Exception as e:
+        problems.append(f"parity check FAILED: {e}")
+
+    if problems:
+        body = "\n".join(problems)
+        log(cfg, "ALERT: " + body)
+        ntfy_alert(cfg, "snolla SMTP monitor: FAIL", body)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())