diff --git a/.gitignore b/.gitignore index 55d2f7c..d8ce631 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,8 @@ pilonuxt-home-smoke.jpeg # task-runner runtime lock (not project content) .tasks/.lock + +# sched-pipelines local stack — runtime artifacts & rendered secrets +host-stacks/local/sched-pipelines/tasks.generated.json +host-stacks/local/sched-pipelines/data-ym/ +host-stacks/local/sched-pipelines/data-ntfy/ diff --git a/host-stacks/local/sched-pipelines/README.md b/host-stacks/local/sched-pipelines/README.md new file mode 100644 index 0000000..99e759e --- /dev/null +++ b/host-stacks/local/sched-pipelines/README.md @@ -0,0 +1,64 @@ +# sched-pipelines — локальный стенд + +sched daemon + 2 HTTP-воркера (Yandex Market, Ozon Seller) + CDP-browser + локальный ntfy + alert-bridge. + +## Расписание (sched daemon) + +| Задача | Cron | URL | +|---|---|---| +| `ozon-seller-pipeline` | `0 2 * * *` (02:00) | http://ozon-seller-builder:8080/run | +| `yandex-market-pipeline` | `30 2 * * *` (02:30) | http://ym-client-builder:8080/run | + +Воркеры в **DRY-RUN**: реальные npm publish / GitHub push не выполняются (озон — `DRY_RUN=1` + publish-стадии нет на sched-пути; яндекс — stages без publish/githubDistro). После отмашки оператора — снять DRY_RUN и добавить стадии. + +## Состав + +| Сервис | Порт на хосте | Заметки | +|---|---|---| +| `schedd` | 127.0.0.1:18080 | admin API, ключ `SCHED_ADMIN_KEY` | +| `ym-client-builder` | 127.0.0.1:18090 | воркер Yandex (envelope-контракт ✅) | +| `ozon-seller-builder` | 127.0.0.1:18091 | воркер Ozon (sched-путь без publish/notify — ТЗ отправлено) | +| `browser` | — | Chrome headless + CDP :9222 (антибот-стадии ozon) | +| `ntfy` | 127.0.0.1:8096 | локальный ntfy на период локального теста | +| `alert-bridge` | 127.0.0.1:19090 | sched webhook → ntfy + email (unisender go2) | + +## Секреты + +- `.env` — gitignored; источник правды — `pass sched-pipelines/local/*`: + `sched-api-key`, `admin-key`, `webhook-secret`, `unisender-go-api-key`. +- `SCHED_API_KEY` — общий токен sched→воркеры (x-sched-api-key). Сгенерирован, в pass. +- `WEBHOOK_SECRET` — HMAC-подпись sched-алертов (X-Sched-Signature-256). + +## Команды + +```bash +# рендер tasks.generated.json из tasks.json + .env (после смены секретов/расписания) +node render-tasks.cjs + +# сборка + запуск +docker compose up -d --build + +# логи +docker compose logs -f schedd + +# health sched +curl -H "Authorization: Bearer $(pass show sched-pipelines/local/admin-key)" http://127.0.0.1:18080/api/health + +# ручной триггер (dry-run безопасен: воркеры в DRY-RUN) +# через sched admin API: POST /api/tasks/:name/run — см. sched docs 09.admin-api.md +``` + +## Миграция на VDS + +1. Запушить образы в registry.kzntsv.site, compose → Portainer-стек. +2. Сеть: `proxy` (traefik), ntfy → `https://ntfy.vds.kzntsv.site` (+ auth-топик), порты → traefik-правила. +3. Секреты → stack.env на Portainer (или pass, как принято). +4. `CRON_SCHEDULE` ozon остаётся выключенным (рулит sched). +5. Проверить IP-доступность: VDS IP для docs.ozon.ru / антибот — не заблокировать. + +## Известные дыры (ТЗ отправлены командам apilki) + +- **ozon**: sched-путь не отдаёт sched-envelope (всегда 200 + `{runId, verdict}`) → sched видит «succeeded» на фейле; нет notify на sched-пути; нет publish-стадии на sched-пути (publish только в внутреннем кроне, который тут выключен). +- **yandex**: email-эндпоинт unisender `go2.unisender.ru/api/v1/sendEmail` → 404; правильный — `go2.unisender.ru/ru/transactional/api/v1/email/send.json` (JSON body). +- **ozon**: email-эндпоинт `go.unisender.ru/...` (старый домен) + массив `to` → go2 + одиночный получатель. +- **unisender go2**: аккаунт требует настроенный backend/tracking domain («Custom backend domain or tracking domain required») — конфигурация в аккаунте go2.unisender.ru (руками оператора). diff --git a/host-stacks/local/sched-pipelines/bridge/Dockerfile b/host-stacks/local/sched-pipelines/bridge/Dockerfile new file mode 100644 index 0000000..0550f63 --- /dev/null +++ b/host-stacks/local/sched-pipelines/bridge/Dockerfile @@ -0,0 +1,5 @@ +FROM node:22-alpine +WORKDIR /app +COPY index.cjs ./ +EXPOSE 9090 +CMD ["node", "index.cjs"] diff --git a/host-stacks/local/sched-pipelines/bridge/index.cjs b/host-stacks/local/sched-pipelines/bridge/index.cjs new file mode 100644 index 0000000..18601f3 --- /dev/null +++ b/host-stacks/local/sched-pipelines/bridge/index.cjs @@ -0,0 +1,126 @@ +'use strict'; +/** + * alert-bridge — приёмник sched webhook-алертов (универсальный outbound webhook). + * Форматирует пейлоад sched → ntfy + email (Unisender Go, go2). + * + * sched шлёт: POST {version, event, task, run|schedule}, подпись + * X-Sched-Signature-256: sha256= (HMAC-SHA256, GitHub-модель) если WEBHOOK_SECRET задан. + * + * Env: NTFY_URL (http://ntfy:8096), NTFY_TOPIC, WEBHOOK_SECRET, + * UNISENDER_API_KEY, UNISENDER_FROM, EMAIL_TO, PORT (9090) + */ +const http = require('node:http'); +const crypto = require('node:crypto'); + +const { + NTFY_URL = 'http://ntfy', + NTFY_TOPIC = 'sched-alerts', + WEBHOOK_SECRET = '', + UNISENDER_API_KEY = '', + UNISENDER_FROM = '', + EMAIL_TO = '', + PORT = '9090', +} = process.env; + +const UNISENDER_ENDPOINT = 'https://go2.unisender.ru/ru/transactional/api/v1/email/send.json'; + +function verify(req, raw) { + if (!WEBHOOK_SECRET) return true; + const sig = req.headers['x-sched-signature-256'] || ''; + const expected = 'sha256=' + crypto.createHmac('sha256', WEBHOOK_SECRET).update(raw).digest('hex'); + if (sig.length !== expected.length) return false; + return crypto.timingSafeEqual(Buffer.from(sig), Buffer.from(expected)); +} + +function fmt(payload) { + const task = payload.task ?? {}; + const name = task.name ?? '?'; + if (payload.event === 'missed-slot') { + const s = payload.schedule ?? {}; + return { + title: `⏰ missed-slot: ${name}`, + body: `Задача ${name} не успела на слот ${s.scheduledAt ?? '?'} (задержка ${s.delayMs ?? '?'} ms).`, + prio: 4, + alert: true, + }; + } + const status = (payload.event ?? '').replace(/^run\./, '') || '?'; + const run = payload.run ?? {}; + const err = run.error ? `\nОшибка: ${run.error}` : ''; + const failed = status === 'failed'; + return { + title: `${failed ? '🚨' : 'ℹ️'} run.${status}: ${name}`, + body: `Рана ${(run.id ?? '?').slice(0, 12)} · статус ${status} · attempt ${run.attempt ?? 1}${err}`, + prio: failed ? 5 : 3, + alert: failed, + }; +} + +async function sendNtfy(m) { + const base = NTFY_URL.replace(/\/+$/, ''); + const res = await fetch(`${base}/${encodeURIComponent(NTFY_TOPIC)}`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + topic: NTFY_TOPIC, + title: m.title, + message: m.body, + priority: m.prio, + tags: [m.prio >= 4 ? 'rotating_light' : 'info'], + }), + }); + if (!res.ok) throw new Error(`ntfy ${res.status}`); +} + +async function sendEmail(m) { + if (!UNISENDER_API_KEY || !EMAIL_TO) return { skipped: 'no UNISENDER_API_KEY/EMAIL_TO' }; + const res = await fetch(UNISENDER_ENDPOINT, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + api_key: UNISENDER_API_KEY, + email: EMAIL_TO, + sender_name: 'sched-pipeline', + sender_email: UNISENDER_FROM, + subject: m.title, + body: m.body, + }), + }); + const text = await res.text(); + if (!res.ok) throw new Error(`unisender ${res.status}: ${text.slice(0, 200)}`); + return text; +} + +const server = http.createServer(async (req, res) => { + const json = (code, body) => { + const payload = JSON.stringify(body); + res.writeHead(code, { 'content-type': 'application/json' }); + res.end(payload); + }; + if (req.method !== 'POST' || req.url !== '/webhook') return json(404, { ok: false, error: 'not found' }); + + const chunks = []; + for await (const c of req) chunks.push(c); + const raw = Buffer.concat(chunks); + + if (!verify(req, raw)) return json(401, { ok: false, error: 'bad signature' }); + + let payload; + try { + payload = JSON.parse(raw.toString('utf8')); + } catch { + return json(400, { ok: false, error: 'bad json' }); + } + + const m = fmt(payload); + const errs = []; + try { await sendNtfy(m); } catch (e) { errs.push(`ntfy: ${e.message}`); } + if (m.alert) { + try { await sendEmail(m); } catch (e) { errs.push(`email: ${e.message}`); } + } + json(200, { ok: true, errors: errs }); +}); + +server.listen(Number(PORT), '0.0.0.0', () => { + console.log(`alert-bridge on :${PORT} (ntfy=${NTFY_URL}/${NTFY_TOPIC}, email=${EMAIL_TO ? 'on' : 'off'})`); +}); diff --git a/host-stacks/local/sched-pipelines/docker-compose.yml b/host-stacks/local/sched-pipelines/docker-compose.yml new file mode 100644 index 0000000..928f8b2 --- /dev/null +++ b/host-stacks/local/sched-pipelines/docker-compose.yml @@ -0,0 +1,150 @@ +# sched-pipelines — локальный стенд: sched daemon + 2 HTTP-воркера (ym/ozon) +# + CDP-browser (ozon антибот) + локальный ntfy + alert-bridge (sched webhook → ntfy/email). +# +# Расписание (sched daemon, tasks.json): ozon 2:00, yandex 2:30 — ежедневно. +# Воркеры в DRY-RUN: реальные publish/push НЕ выполняются до отмашки оператора. +# +# Запуск: docker compose up -d --build +# Секреты: .env (gitignored; значения в pass: sched-pipelines/local/*) +# Миграция на VDS: образы в registry, сеть proxy, ntfy → ntfy.vds.kzntsv.site. +name: sched-pipelines + +services: + # --- sched daemon (schedd) --- + schedd: + build: + context: . + dockerfile: schedd.Dockerfile + args: + VERDACCIO_TOKEN: ${VERDACCIO_TOKEN:-} + image: sched-pipelines/schedd:local + command: + - --tasks + - /app/tasks.json + - --db + - /data/sched.db + - --admin-port + - "8080" + - --admin-host + - 0.0.0.0 + - --tick-interval + - "1000" + environment: + SCHED_ADMIN_KEY: ${SCHED_ADMIN_KEY:?задайте в .env} + volumes: + - ./tasks.generated.json:/app/tasks.json:ro + - sched-data:/data + ports: + - "127.0.0.1:18080:8080" + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8080/api/health',{headers:{authorization:'Bearer '+process.env.SCHED_ADMIN_KEY}}).then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + interval: 5s + timeout: 3s + retries: 10 + restart: unless-stopped + networks: [sched-local] + + # --- воркер Yandex Market (HTTP-воркер sched, envelope) --- + ym-client-builder: + build: + context: ../../../../yandex-market-partner-api-client + dockerfile: Dockerfile + image: sched-pipelines/ym-client-builder:local + environment: + PORT: "8080" + SCHED_API_KEY: ${SCHED_API_KEY:?задайте в .env} + UNISENDER_API_KEY: ${UNISENDER_API_KEY:-} + UNISENDER_FROM: ${UNISENDER_FROM:-} + NPM_TOKEN: ${NPM_TOKEN:-} + GITHUB_TOKEN: ${GITHUB_TOKEN:-} + volumes: + - ./data-ym:/app/data + ports: + - "127.0.0.1:18090:8080" + restart: unless-stopped + networks: [sched-local] + + # --- воркер Ozon Seller (HTTP-воркер sched + внутренний крон ОТКЛЮЧЁН) --- + ozon-seller-builder: + build: + context: ../../../../ozon-seller-api-client + dockerfile: C:/Users/vitya/projects/.admin/host-stacks/local/sched-pipelines/ozon.Dockerfile + args: + VERDACCIO_TOKEN: ${VERDACCIO_TOKEN:-} + image: sched-pipelines/ozon-seller-builder:local + environment: + WORKER_PORT: "8080" + SCHED_API_KEY: ${SCHED_API_KEY:?задайте в .env} + # внутренний крон не нужен — расписанием рулит sched (ozon 2:00). + # 31 февраля не наступает никогда: страховая от двойного запуска. + CRON_SCHEDULE: "0 0 29 2 *" + STATE_FILE: /app/data/state.json + LOCK_FILE: /app/data/pipeline.lock + DRY_RUN: "1" + BROWSER_CDP_URL: http://browser:9222 + NTFY_URL: http://ntfy + NTFY_TOPIC: ozon-seller-pipeline + UNISENDER_API_KEY: ${UNISENDER_API_KEY:-} + NOTIFY_EMAIL_TO: ${NOTIFY_EMAIL_TO:-} + GH_TOKEN: ${GITHUB_TOKEN:-} + NPM_TOKEN: ${NPM_TOKEN:-} + volumes: + - ozon-data:/app/data + ports: + - "127.0.0.1:18091:8080" + restart: unless-stopped + depends_on: + - browser + networks: [sched-local] + + # --- headless Chrome + CDP (для антибот-fetch стадий ozon) --- + browser: + build: + context: ../../../../ozon-seller-api-client/service/browser + dockerfile: Dockerfile + image: sched-pipelines/browser-cdp:local + container_name: browser-cdp + shm_size: "1gb" + restart: unless-stopped + networks: [sched-local] + + # --- локальный ntfy (на время локального периода; VDS-ntfy требует auth-топика) --- + ntfy: + image: binwiederhier/ntfy:v2.11.0 + command: serve + environment: + NTFY_LISTEN_HTTP: ":80" + NTFY_CACHE_FILE: /var/cache/ntfy/cache.db + volumes: + - ./data-ntfy:/var/cache/ntfy + ports: + - "127.0.0.1:8096:80" + restart: unless-stopped + networks: [sched-local] + + # --- alert-bridge: sched webhook-алерты → ntfy + email (unisender go2) --- + alert-bridge: + build: ./bridge + image: sched-pipelines/alert-bridge:local + environment: + PORT: "9090" + NTFY_URL: http://ntfy + NTFY_TOPIC: sched-alerts + WEBHOOK_SECRET: ${WEBHOOK_SECRET:?задайте в .env} + UNISENDER_API_KEY: ${UNISENDER_API_KEY:-} + UNISENDER_FROM: ${UNISENDER_FROM:-} + EMAIL_TO: ${NOTIFY_EMAIL_TO:-} + ports: + - "127.0.0.1:19090:9090" + restart: unless-stopped + depends_on: + - ntfy + networks: [sched-local] + +volumes: + sched-data: + ozon-data: + +networks: + sched-local: + driver: bridge diff --git a/host-stacks/local/sched-pipelines/ozon.Dockerfile b/host-stacks/local/sched-pipelines/ozon.Dockerfile new file mode 100644 index 0000000..ae475a7 --- /dev/null +++ b/host-stacks/local/sched-pipelines/ozon.Dockerfile @@ -0,0 +1,32 @@ +# Локальная копия service/Dockerfile из ozon-seller-api-client с фиксом: +# `COPY state.json ./state.json 2>/dev/null || true` — шелл-синтаксис в COPY не валиден, +# docker падает «"/2>/dev/null": not found». state.json в контексте есть, копируем напрямую. +FROM node:22-alpine + +RUN apk add --no-cache openjdk21-jre-headless git + +WORKDIR /app + +COPY package.json package-lock.json ./ +# lock пинит yaml на verdaccio (локальный .npmrc хоста имеет токен, контейнер — нет) +ARG VERDACCIO_TOKEN +RUN npm config set //verdaccio.kzntsv.site/:_authToken=${VERDACCIO_TOKEN} \ + && npm ci --no-audit --no-fund || npm install --no-audit --no-fund + +COPY service/ ./service/ +COPY generator/ ./generator/ +COPY test/ ./test/ +COPY LICENSE ./ +COPY state.json ./state.json + +ENV NODE_ENV=production +ENV WORKER_PORT=8080 +ENV WORKER_TOKEN= + +# крон-расписание (формат node-cron): в стеке переопределяется на «никогда» — +# расписанием рулит sched daemon +ENV CRON_SCHEDULE="0 0 31 2 *" + +EXPOSE 8080 + +CMD ["node", "service/entrypoint.js"] diff --git a/host-stacks/local/sched-pipelines/render-tasks.cjs b/host-stacks/local/sched-pipelines/render-tasks.cjs new file mode 100644 index 0000000..816c7bd --- /dev/null +++ b/host-stacks/local/sched-pipelines/render-tasks.cjs @@ -0,0 +1,43 @@ +'use strict'; +/** + * Рендер tasks.generated.json из tasks.json (шаблон) + .env. + * Подставляет __SCHED_API_KEY__ / __WEBHOOK_SECRET__; после рендера валидирует + * «ни одного плейсхолдера не осталось». + * + * Запуск: node render-tasks.mjs (в этой папке; .env рядом) + */ +const fs = require('node:fs'); +const path = require('node:path'); + +const envPath = path.join(__dirname, '.env'); +const tplPath = path.join(__dirname, 'tasks.json'); +const outPath = path.join(__dirname, 'tasks.generated.json'); + +function loadEnv(file) { + const out = {}; + for (const line of fs.readFileSync(file, 'utf8').split(/\r?\n/)) { + const m = /^\s*([A-Z0-9_]+)\s*=\s*(.*)\s*$/.exec(line); + if (m && !line.trim().startsWith('#')) out[m[1]] = m[2].replace(/^["']|["']$/g, ''); + } + return out; +} + +const env = loadEnv(envPath); +const tpl = fs.readFileSync(tplPath, 'utf8'); + +const subs = { + __SCHED_API_KEY__: env.SCHED_API_KEY, + __WEBHOOK_SECRET__: env.WEBHOOK_SECRET, +}; + +let out = tpl; +for (const [k, v] of Object.entries(subs)) { + if (!v) throw new Error(`.env не содержит ${k.slice(2, -2)} — рендер невозможен`); + out = out.split(k).join(v); +} + +const leftovers = [...out.matchAll(/__[A-Z_]+__/g)].map((m) => m[0]); +if (leftovers.length) throw new Error(`незаменённые плейсхолдеры: ${[...new Set(leftovers)].join(', ')}`); + +fs.writeFileSync(outPath, out); +console.log(`tasks.generated.json записан (${outPath})`); diff --git a/host-stacks/local/sched-pipelines/schedd.Dockerfile b/host-stacks/local/sched-pipelines/schedd.Dockerfile new file mode 100644 index 0000000..bcae1aa --- /dev/null +++ b/host-stacks/local/sched-pipelines/schedd.Dockerfile @@ -0,0 +1,13 @@ +# schedd — опубликованный daemon из приватного реестра (verdaccio), как на проде. +# @sched/daemon@0.4.2 (latest) + deps (core 0.44.0, admin-api 0.2.0, mcp 0.4.1). +# node >= 24 (engines у daemon; node:sqlite). +FROM node:24-alpine +RUN apk add --no-cache git +WORKDIR /app +ENV NODE_ENV=production +ARG VERDACCIO_TOKEN +RUN npm config set //verdaccio.kzntsv.site/:_authToken=${VERDACCIO_TOKEN} \ + && npm install -g @sched/daemon@0.4.2 --registry=https://verdaccio.kzntsv.site --no-audit --no-fund +EXPOSE 8080 +VOLUME ["/data"] +ENTRYPOINT ["schedd"] diff --git a/host-stacks/local/sched-pipelines/tasks.json b/host-stacks/local/sched-pipelines/tasks.json new file mode 100644 index 0000000..d1b9690 --- /dev/null +++ b/host-stacks/local/sched-pipelines/tasks.json @@ -0,0 +1,53 @@ +{ + "tasks": [ + { + "name": "ozon-seller-pipeline", + "runner": "http", + "schedules": [{ "cron": "0 2 * * *" }], + "config": { + "url": "http://ozon-seller-builder:8080/run", + "method": "POST", + "envelope": true, + "auth": { "apiKey": "__SCHED_API_KEY__" }, + "data": { + "stages": ["fetch", "hash", "patch", "diff", "classify", "generate", "fix", "build", "postman", "state"] + } + } + }, + { + "name": "yandex-market-pipeline", + "runner": "http", + "schedules": [{ "cron": "30 2 * * *" }], + "config": { + "url": "http://ym-client-builder:8080/run", + "method": "POST", + "envelope": true, + "auth": { "apiKey": "__SCHED_API_KEY__" }, + "data": { + "stages": ["fetch", "detect", "diff", "classify", "patch", "generate", "fix", "build", "test", "postmanGenerate", "postman"], + "repo": "https://github.com/yandex-market/yandex-market-partner-api", + "branch": "main", + "workDir": "data/clone", + "specDir": "data/openapi/snapshots", + "snapshotsDir": "data/openapi/snapshots", + "statePath": "data/state.json", + "outputDir": "data/output/typescript", + "testsDir": "tests/client", + "notify": { + "ntfy": { "url": "http://ntfy", "topic": "ym-client-builder" }, + "email": { "to": "vitya.kuznetsov@gmail.com", "on": ["published", "report", "failed"] } + } + } + } + } + ], + "alerts": { + "on": ["failed"], + "onMissed": true, + "webhook": { + "url": "http://alert-bridge:9090/webhook", + "secret": "__WEBHOOK_SECRET__", + "timeoutMs": 10000 + } + } +}