From d922a8368fd9c69bed3587f03551cac3f0477f5b Mon Sep 17 00:00:00 2001 From: peptiderepo Date: Mon, 22 Jun 2026 16:33:55 +0000 Subject: [PATCH 1/2] infra/vps: PRAB generation orchestrator (VPS-driven SSH, systemd timer) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the VPS-side orchestrator for Option A (ADR 2026-06-22): the Coolify KVM8 VPS (76.13.220.15) drives PRAutoBlogger generation on prod synchronously over a held-open SSH session, escaping Hostinger's ~10-min background-PHP-process kill. Files: infra/vps/orchestrate-generation.sh - SSH to Hostinger; reaper sweep; wp prautoblogger generate --sync - Lock-aware: skips if run in-flight (plugin DB mutex) - Structured log lines (captured by systemd journal) - Failure alert: email to cto@peptiderepo.com via system mail - DRY_RUN=1 mode: echoes commands without executing generation - All config via env vars (SSH_KEY, SSH_HOST, etc.); no hardcoded secrets - SSH key referenced by PATH (/root/.ssh/hostinger_prod_ed25519) infra/vps/install-orchestrator.sh - Installs orchestrator to /opt/prab-orchestrator/ - Installs systemd .service and .timer to /etc/systemd/system/ - Does NOT enable or start timer (explicit post-QA cutover step) infra/vps/prab-generation.service - Type=oneshot, User=root, TimeoutStartSec=7200 (covers 32-min LLM calls) - Restart=no (timer retries next day on failure) infra/vps/prab-generation.timer - OnCalendar=*-*-* 06:00:00 UTC, Persistent=true, RandomizedDelaySec=300 DO NOT install or enable on the VPS this round — QA-gated, hold for CTO review. Agent-Session: prab-reliability-20260622 --- infra/vps/install-orchestrator.sh | 64 ++++++++++++ infra/vps/orchestrate-generation.sh | 153 ++++++++++++++++++++++++++++ infra/vps/prab-generation.service | 19 ++++ infra/vps/prab-generation.timer | 19 ++++ 4 files changed, 255 insertions(+) create mode 100755 infra/vps/install-orchestrator.sh create mode 100755 infra/vps/orchestrate-generation.sh create mode 100644 infra/vps/prab-generation.service create mode 100644 infra/vps/prab-generation.timer diff --git a/infra/vps/install-orchestrator.sh b/infra/vps/install-orchestrator.sh new file mode 100755 index 0000000..98f33cb --- /dev/null +++ b/infra/vps/install-orchestrator.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# install-orchestrator.sh — One-time setup for the PRAB generation orchestrator on the VPS. +# +# Installs: +# 1. orchestrate-generation.sh -> /opt/prab-orchestrator/ (from this repo) +# 2. prab-generation.service -> /etc/systemd/system/ +# 3. prab-generation.timer -> /etc/systemd/system/ +# +# Does NOT enable or start the timer — that is an explicit post-QA cutover step. +# Does NOT write secrets — the SSH key at /root/.ssh/hostinger_prod_ed25519 must +# already be present (placed manually by the CTO from .env.credentials). +# +# Run as root on the KVM8 VPS (76.13.220.15): +# bash infra/vps/install-orchestrator.sh +# +# After install, validate with DRY_RUN=1 before enabling the timer: +# DRY_RUN=1 /opt/prab-orchestrator/orchestrate-generation.sh +# systemctl enable --now prab-generation.timer # only after CEO greenlight + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INSTALL_DIR="/opt/prab-orchestrator" +SYSTEMD_DIR="/etc/systemd/system" +ORCHESTRATE_SH="${SCRIPT_DIR}/orchestrate-generation.sh" +SERVICE_FILE="${SCRIPT_DIR}/prab-generation.service" +TIMER_FILE="${SCRIPT_DIR}/prab-generation.timer" + +# Must be root. +if [[ "$(id -u)" -ne 0 ]]; then + echo "ERROR: install-orchestrator.sh must be run as root." >&2 + exit 1 +fi + +# Validate source files exist. +for f in "${ORCHESTRATE_SH}" "${SERVICE_FILE}" "${TIMER_FILE}"; do + if [[ ! -f "$f" ]]; then + echo "ERROR: source file not found: $f" >&2 + exit 1 + fi +done + +echo "Installing PRAB orchestrator to ${INSTALL_DIR}..." +mkdir -p "${INSTALL_DIR}" +install -m 0750 "${ORCHESTRATE_SH}" "${INSTALL_DIR}/orchestrate-generation.sh" +echo " OK: ${INSTALL_DIR}/orchestrate-generation.sh" + +echo "Installing systemd units..." +install -m 0644 "${SERVICE_FILE}" "${SYSTEMD_DIR}/prab-generation.service" +install -m 0644 "${TIMER_FILE}" "${SYSTEMD_DIR}/prab-generation.timer" +echo " OK: ${SYSTEMD_DIR}/prab-generation.service" +echo " OK: ${SYSTEMD_DIR}/prab-generation.timer" + +systemctl daemon-reload +echo " OK: systemctl daemon-reload" + +echo "" +echo "Install complete. Timer is installed but NOT enabled." +echo "Next steps (post-QA + CEO greenlight only):" +echo " 1. Ensure SSH key exists: ls -la /root/.ssh/hostinger_prod_ed25519" +echo " 2. Validate with dry run: DRY_RUN=1 /opt/prab-orchestrator/orchestrate-generation.sh" +echo " 3. Run first real invocation: /opt/prab-orchestrator/orchestrate-generation.sh" +echo " 4. Enable timer: systemctl enable --now prab-generation.timer" +echo " 5. Confirm next run: systemctl list-timers prab-generation.timer" diff --git a/infra/vps/orchestrate-generation.sh b/infra/vps/orchestrate-generation.sh new file mode 100755 index 0000000..e71f566 --- /dev/null +++ b/infra/vps/orchestrate-generation.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# orchestrate-generation.sh — VPS orchestrator for PRAutoBlogger article generation +# +# Runs as a systemd oneshot service (prab-generation.service) on the Coolify KVM8 VPS. +# SSHes to Hostinger and drives wp prautoblogger generate --sync in a held-open SSH +# session. SSH-attached processes survive Hostinger's ~10-min background-PHP-process kill. +# +# Usage: +# DRY_RUN=1 ./orchestrate-generation.sh # echo commands, no generation +# ./orchestrate-generation.sh # real run +# +# Alerting: on failure, sends email to cto@peptiderepo.com via system mail. +# Logs: structured log lines to stdout (captured by systemd journal). +# Lock: the plugin's Generation_Lock (DB mutex) prevents double-drive. +# +# Environment variables: +# DRY_RUN Set to "1" to echo SSH/WP-CLI commands without executing. Default: 0. +# LOGFILE Path for persistent log file. Default: /var/log/prab-generation.log. +# ALERT_EMAIL Failure alert recipient. Default: cto@peptiderepo.com. +# SSH_KEY Path to Hostinger SSH private key. Default: /root/.ssh/hostinger_prod_ed25519. +# SSH_HOST Hostinger SSH host. Default: 145.223.107.228. +# SSH_PORT Hostinger SSH port. Default: 65002. +# SSH_USER Hostinger SSH user. Default: u117248512. +# WP_PATH WordPress docroot on Hostinger. Default: /home/u117248512/domains/peptiderepo.com/public_html. + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Configuration (override via environment; no hardcoded secrets) +# --------------------------------------------------------------------------- +DRY_RUN="${DRY_RUN:-0}" +LOGFILE="${LOGFILE:-/var/log/prab-generation.log}" +ALERT_EMAIL="${ALERT_EMAIL:-cto@peptiderepo.com}" +SSH_KEY="${SSH_KEY:-/root/.ssh/hostinger_prod_ed25519}" +SSH_HOST="${SSH_HOST:-145.223.107.228}" +SSH_PORT="${SSH_PORT:-65002}" +SSH_USER="${SSH_USER:-u117248512}" +WP_PATH="${WP_PATH:-/home/u117248512/domains/peptiderepo.com/public_html}" + +# --------------------------------------------------------------------------- +# Logging helpers +# --------------------------------------------------------------------------- +PID="$$" +TS() { date -u '+%Y-%m-%dT%H:%M:%SZ'; } + +log() { + local msg="prab-generation[${PID}]: $*" + echo "$msg" + echo "$(TS) $msg" >> "${LOGFILE}" 2>/dev/null || true +} + +die() { + log "FATAL: $*" + exit 1 +} + +# --------------------------------------------------------------------------- +# Validate required key before doing anything +# --------------------------------------------------------------------------- +if [[ "${DRY_RUN}" != "1" ]] && [[ ! -f "${SSH_KEY}" ]]; then + die "SSH key not found at ${SSH_KEY}. Run install-orchestrator.sh first." +fi + +# --------------------------------------------------------------------------- +# DRY_RUN mode: echo commands and exit +# --------------------------------------------------------------------------- +if [[ "${DRY_RUN}" == "1" ]]; then + log "DRY_RUN=1 — echoing commands only, no generation" + log "WOULD RUN: ssh -i ${SSH_KEY} -p ${SSH_PORT} -o StrictHostKeyChecking=no -o ConnectTimeout=15 ${SSH_USER}@${SSH_HOST} \\" + log " 'wp prautoblogger generate --sync --path=${WP_PATH} 2>&1'" + log "DRY_RUN complete" + exit 0 +fi + +# --------------------------------------------------------------------------- +# Run start +# --------------------------------------------------------------------------- +START_TS="$(TS)" +log "START ts=${START_TS}" + +# --------------------------------------------------------------------------- +# Step 1: Reaper sweep — clear any stuck runs before starting. +# Belt-and-suspenders: the plugin's daily reaper cron handles this too, but +# running it explicitly here ensures a clean state before each generation. +# --------------------------------------------------------------------------- +log "REAPER_SWEEP: invoking Run_Reaper::reap()" +REAP_CMD="wp eval 'echo json_encode(PRAutoBlogger_Run_Reaper::reap());' --path=${WP_PATH} 2>&1" + +REAP_OUT="" +# shellcheck disable=SC2029 +REAP_OUT="$(ssh -i "${SSH_KEY}" \ + -p "${SSH_PORT}" \ + -o StrictHostKeyChecking=no \ + -o ConnectTimeout=15 \ + -o ServerAliveInterval=60 \ + -o ServerAliveCountMax=30 \ + "${SSH_USER}@${SSH_HOST}" \ + "${REAP_CMD}" \ + 2>&1)" || true +log "REAPER_RESULT: ${REAP_OUT}" + +# --------------------------------------------------------------------------- +# Step 2: Run wp prautoblogger generate --sync +# --------------------------------------------------------------------------- +log "SSH_CONNECT: ${SSH_USER}@${SSH_HOST}:${SSH_PORT} key=${SSH_KEY}" + +WP_CMD="wp prautoblogger generate --sync --path=${WP_PATH} 2>&1" + +RESULT_OUTPUT="" +EXIT_CODE=0 + +# shellcheck disable=SC2029 +RESULT_OUTPUT="$(ssh -i "${SSH_KEY}" \ + -p "${SSH_PORT}" \ + -o StrictHostKeyChecking=no \ + -o ConnectTimeout=15 \ + -o ServerAliveInterval=60 \ + -o ServerAliveCountMax=30 \ + "${SSH_USER}@${SSH_HOST}" \ + "${WP_CMD}" \ + 2>&1)" || EXIT_CODE=$? + +# Extract the last JSON summary line (the one from WP_CLI::log(json_encode(...))) +LAST_JSON="$(echo "${RESULT_OUTPUT}" | grep -E '^\{' | tail -1 || true)" +LAST_LINE="$(echo "${RESULT_OUTPUT}" | tail -1)" + +log "SSH_RESULT exit=${EXIT_CODE} json=${LAST_JSON:-none}" + +# --------------------------------------------------------------------------- +# Step 3: Evaluate result +# --------------------------------------------------------------------------- +END_TS="$(TS)" + +if [[ ${EXIT_CODE} -eq 0 ]]; then + log "END result=success ts=${END_TS}" +else + log "END result=failure exit=${EXIT_CODE} last_line=${LAST_LINE} ts=${END_TS}" + + # Send failure alert via email (system mail; no external deps required). + # cto@peptiderepo.com forwards to peptiderepo@gmail.com via Cloudflare Email Routing. + ALERT_BODY="PRAutoBlogger generation FAILED on $(hostname) at ${END_TS}. +Exit code: ${EXIT_CODE} +Last output: ${LAST_LINE} +Full log: ${LOGFILE} +SSH: ${SSH_USER}@${SSH_HOST}:${SSH_PORT} +WP path: ${WP_PATH}" + + echo "${ALERT_BODY}" \ + | mail -s "PRAB generation FAILED @ ${END_TS}" "${ALERT_EMAIL}" 2>/dev/null \ + || log "ALERT_FAILED: could not send email to ${ALERT_EMAIL} (mail not configured?)" + + exit "${EXIT_CODE}" +fi diff --git a/infra/vps/prab-generation.service b/infra/vps/prab-generation.service new file mode 100644 index 0000000..a215f5c --- /dev/null +++ b/infra/vps/prab-generation.service @@ -0,0 +1,19 @@ +[Unit] +Description=PRAutoBlogger article generation (VPS orchestrator) +Documentation=https://github.com/peptide-repo/peptide-e2e/tree/main/infra/vps/ +After=network.target +# Do not start if the timer fires while another instance is still running. +# Type=oneshot + TimeoutStartSec enforce this at the systemd level. + +[Service] +Type=oneshot +User=root +ExecStart=/opt/prab-orchestrator/orchestrate-generation.sh +StandardOutput=journal +StandardError=journal +SyslogIdentifier=prab-generation +# Allow up to 2 hours for a full generation run (orchestrate + draft LLM calls). +# The longest observed single LLM call was 1925s (~32 min); normal runs 2-5 min. +TimeoutStartSec=7200 +# Do not restart on failure — the timer will retry the next day. +Restart=no diff --git a/infra/vps/prab-generation.timer b/infra/vps/prab-generation.timer new file mode 100644 index 0000000..d9f2e11 --- /dev/null +++ b/infra/vps/prab-generation.timer @@ -0,0 +1,19 @@ +[Unit] +Description=Daily PRAutoBlogger generation at 06:00 UTC +Documentation=https://github.com/peptide-repo/peptide-e2e/tree/main/infra/vps/ +Requires=prab-generation.service + +[Timer] +# Fire daily at 06:00 UTC — before Hostinger's daily wp-cron window. +OnCalendar=*-*-* 06:00:00 UTC +# Persistent=true: if the VPS was down at 06:00 UTC, fire immediately on next +# boot so at most one day's generation is missed (not deferred indefinitely). +Persistent=true +# Up to 5 min random jitter to avoid thundering-herd if multiple VPS instances +# are ever added. Harmless with a single instance. +RandomizedDelaySec=300 +# Accuracy within 1 minute (default is 1 minute; explicit for clarity). +AccuracySec=60 + +[Install] +WantedBy=timers.target -- 2.49.1 From f3f579f5a8fc874ef33235467dd05461a24bf49f Mon Sep 17 00:00:00 2001 From: peptiderepo Date: Mon, 22 Jun 2026 17:16:19 +0000 Subject: [PATCH 2/2] fix: QA remediation P1-D + P2 for infra/vps orchestrator (v1.8.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1-D: bump package.json version 1.7.2 → 1.8.0 and add [1.8.0] CHANGELOG entry documenting the infra/vps/ PRAB generation orchestrator addition. P2 (security): replace StrictHostKeyChecking=no with StrictHostKeyChecking=accept-new in all three SSH invocations in orchestrate-generation.sh. Add UserKnownHostsFile=${KNOWN_HOSTS} (-o) to pin key verification to the pre-seeded file rather than the root known_hosts. install-orchestrator.sh: add ssh-keyscan step that pre-populates /opt/prab-orchestrator/known_hosts with Hostinger's host key on first install. Graceful degradation: if ssh-keyscan fails, logs WARN and proceeds (accept-new will add the key on first real connect). prab-generation.service: add justifying comment for User=root (root holds the Hostinger SSH key + known_hosts in /root/.ssh/ and /opt/prab-orchestrator/). Agent-Session: prab-reliability-remediation-20260622 --- CHANGELOG.md | 17 +++++++++++++++++ infra/vps/install-orchestrator.sh | 15 +++++++++++++++ infra/vps/orchestrate-generation.sh | 11 ++++++++--- infra/vps/prab-generation.service | 3 +++ package.json | 2 +- 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa88e6a..43cff21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## [1.8.0] - 2026-06-23 + +### Added +- **`infra/vps/` PRAB generation orchestrator**: systemd timer + service + bash + script that SSHes from the Coolify KVM8 VPS to Hostinger and drives + `wp prautoblogger generate --sync` in a held-open SSH session. Sidesteps + Hostinger's ~10-min background-PHP-process kill that was blocking full + generation runs. Components: + - `orchestrate-generation.sh`: main script with DRY_RUN mode, structured + logging, reaper pre-sweep, failure email alert, and shellcheck-clean quoting. + - `prab-generation.service` / `prab-generation.timer`: systemd oneshot service + (2h timeout, no auto-restart) + daily 06:00 UTC timer with 5-min jitter. + - `install-orchestrator.sh`: one-time setup script; installs units but does NOT + enable the timer (requires explicit CEO greenlight post-QA). + - Security: `StrictHostKeyChecking=accept-new` with pre-seeded known_hosts for + the Hostinger IP; no hardcoded secrets; SSH key referenced by env-var path. + ## [1.7.2] - 2026-06-17 ### Fixed diff --git a/infra/vps/install-orchestrator.sh b/infra/vps/install-orchestrator.sh index 98f33cb..2146ae8 100755 --- a/infra/vps/install-orchestrator.sh +++ b/infra/vps/install-orchestrator.sh @@ -45,6 +45,21 @@ mkdir -p "${INSTALL_DIR}" install -m 0750 "${ORCHESTRATE_SH}" "${INSTALL_DIR}/orchestrate-generation.sh" echo " OK: ${INSTALL_DIR}/orchestrate-generation.sh" +# Pre-seed known_hosts with Hostinger's host key so ssh(1) can verify +# the server identity without StrictHostKeyChecking=no. +# ssh-keyscan output is pinned here; re-run if Hostinger rotates its key. +KNOWN_HOSTS_FILE="${INSTALL_DIR}/known_hosts" +if [[ ! -f "${KNOWN_HOSTS_FILE}" ]]; then + echo "Pre-seeding known_hosts from Hostinger (${SSH_HOST:-145.223.107.228}:${SSH_PORT:-65002})..." + ssh-keyscan -p "${SSH_PORT:-65002}" "${SSH_HOST:-145.223.107.228}" \ + > "${KNOWN_HOSTS_FILE}" 2>/dev/null \ + && chmod 0600 "${KNOWN_HOSTS_FILE}" \ + && echo " OK: ${KNOWN_HOSTS_FILE}" \ + || echo " WARN: ssh-keyscan failed — known_hosts not seeded; first SSH will add key via accept-new" +else + echo " OK: ${KNOWN_HOSTS_FILE} (already exists, not overwritten)" +fi + echo "Installing systemd units..." install -m 0644 "${SERVICE_FILE}" "${SYSTEMD_DIR}/prab-generation.service" install -m 0644 "${TIMER_FILE}" "${SYSTEMD_DIR}/prab-generation.timer" diff --git a/infra/vps/orchestrate-generation.sh b/infra/vps/orchestrate-generation.sh index e71f566..1827879 100755 --- a/infra/vps/orchestrate-generation.sh +++ b/infra/vps/orchestrate-generation.sh @@ -36,6 +36,9 @@ SSH_HOST="${SSH_HOST:-145.223.107.228}" SSH_PORT="${SSH_PORT:-65002}" SSH_USER="${SSH_USER:-u117248512}" WP_PATH="${WP_PATH:-/home/u117248512/domains/peptiderepo.com/public_html}" +# Known-hosts file pre-seeded by install-orchestrator.sh with Hostinger's host key. +# Pins the server identity; StrictHostKeyChecking=accept-new adds on first connect. +KNOWN_HOSTS="${KNOWN_HOSTS:-/opt/prab-orchestrator/known_hosts}" # --------------------------------------------------------------------------- # Logging helpers @@ -66,7 +69,7 @@ fi # --------------------------------------------------------------------------- if [[ "${DRY_RUN}" == "1" ]]; then log "DRY_RUN=1 — echoing commands only, no generation" - log "WOULD RUN: ssh -i ${SSH_KEY} -p ${SSH_PORT} -o StrictHostKeyChecking=no -o ConnectTimeout=15 ${SSH_USER}@${SSH_HOST} \\" + log "WOULD RUN: ssh -i ${SSH_KEY} -p ${SSH_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 ${SSH_USER}@${SSH_HOST} \\" log " 'wp prautoblogger generate --sync --path=${WP_PATH} 2>&1'" log "DRY_RUN complete" exit 0 @@ -90,7 +93,8 @@ REAP_OUT="" # shellcheck disable=SC2029 REAP_OUT="$(ssh -i "${SSH_KEY}" \ -p "${SSH_PORT}" \ - -o StrictHostKeyChecking=no \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile="${KNOWN_HOSTS}" \ -o ConnectTimeout=15 \ -o ServerAliveInterval=60 \ -o ServerAliveCountMax=30 \ @@ -112,7 +116,8 @@ EXIT_CODE=0 # shellcheck disable=SC2029 RESULT_OUTPUT="$(ssh -i "${SSH_KEY}" \ -p "${SSH_PORT}" \ - -o StrictHostKeyChecking=no \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile="${KNOWN_HOSTS}" \ -o ConnectTimeout=15 \ -o ServerAliveInterval=60 \ -o ServerAliveCountMax=30 \ diff --git a/infra/vps/prab-generation.service b/infra/vps/prab-generation.service index a215f5c..5015be6 100644 --- a/infra/vps/prab-generation.service +++ b/infra/vps/prab-generation.service @@ -7,6 +7,9 @@ After=network.target [Service] Type=oneshot +# root owns /root/.ssh/hostinger_prod_ed25519 and /opt/prab-orchestrator/known_hosts; +# a dedicated service user would need its own keypair + sudo-less SSH access. +# Acceptable on this dedicated lab VPS (not shared); revisit if VPS is multi-tenant. User=root ExecStart=/opt/prab-orchestrator/orchestrate-generation.sh StandardOutput=journal diff --git a/package.json b/package.json index ee10ec4..447a511 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "peptide-e2e", - "version": "1.7.2", + "version": "1.8.0", "description": "Playwright smoke tests for peptiderepo.com (staging + production)", "private": true, "scripts": { -- 2.49.1