2026-06-22 17:54:36 +00:00
6 changed files with 296 additions and 1 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,22 @@
 # Changelog

+## [1.8.0] - 2026-06-23
+
+### Added
+- **`infra/vps/` PRAB generation orchestrator**: systemd timer + service + bash
+  script that SSHes from the Coolify KVM8 VPS to Hostinger and drives
+  `wp prautoblogger generate --sync` in a held-open SSH session. Sidesteps
+  Hostinger's ~10-min background-PHP-process kill that was blocking full
+  generation runs. Components:
+  - `orchestrate-generation.sh`: main script with DRY_RUN mode, structured
+    logging, reaper pre-sweep, failure email alert, and shellcheck-clean quoting.
+  - `prab-generation.service` / `prab-generation.timer`: systemd oneshot service
+    (2h timeout, no auto-restart) + daily 06:00 UTC timer with 5-min jitter.
+  - `install-orchestrator.sh`: one-time setup script; installs units but does NOT
+    enable the timer (requires explicit CEO greenlight post-QA).
+  - Security: `StrictHostKeyChecking=accept-new` with pre-seeded known_hosts for
+    the Hostinger IP; no hardcoded secrets; SSH key referenced by env-var path.
+
 ## [1.7.2] - 2026-06-17

 ### Fixed
--- a/infra/vps/install-orchestrator.sh
+++ b/infra/vps/install-orchestrator.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# install-orchestrator.sh — One-time setup for the PRAB generation orchestrator on the VPS.
+#
+# Installs:
+#   1. orchestrate-generation.sh  -> /opt/prab-orchestrator/ (from this repo)
+#   2. prab-generation.service    -> /etc/systemd/system/
+#   3. prab-generation.timer      -> /etc/systemd/system/
+#
+# Does NOT enable or start the timer — that is an explicit post-QA cutover step.
+# Does NOT write secrets — the SSH key at /root/.ssh/hostinger_prod_ed25519 must
+# already be present (placed manually by the CTO from .env.credentials).
+#
+# Run as root on the KVM8 VPS (76.13.220.15):
+#   bash infra/vps/install-orchestrator.sh
+#
+# After install, validate with DRY_RUN=1 before enabling the timer:
+#   DRY_RUN=1 /opt/prab-orchestrator/orchestrate-generation.sh
+#   systemctl enable --now prab-generation.timer   # only after CEO greenlight
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INSTALL_DIR="/opt/prab-orchestrator"
+SYSTEMD_DIR="/etc/systemd/system"
+ORCHESTRATE_SH="${SCRIPT_DIR}/orchestrate-generation.sh"
+SERVICE_FILE="${SCRIPT_DIR}/prab-generation.service"
+TIMER_FILE="${SCRIPT_DIR}/prab-generation.timer"
+
+# Must be root.
+if [[ "$(id -u)" -ne 0 ]]; then
+  echo "ERROR: install-orchestrator.sh must be run as root." >&2
+  exit 1
+fi
+
+# Validate source files exist.
+for f in "${ORCHESTRATE_SH}" "${SERVICE_FILE}" "${TIMER_FILE}"; do
+  if [[ ! -f "$f" ]]; then
+    echo "ERROR: source file not found: $f" >&2
+    exit 1
+  fi
+done
+
+echo "Installing PRAB orchestrator to ${INSTALL_DIR}..."
+mkdir -p "${INSTALL_DIR}"
+install -m 0750 "${ORCHESTRATE_SH}" "${INSTALL_DIR}/orchestrate-generation.sh"
+echo "  OK: ${INSTALL_DIR}/orchestrate-generation.sh"
+
+# Pre-seed known_hosts with Hostinger's host key so ssh(1) can verify
+# the server identity without StrictHostKeyChecking=no.
+# ssh-keyscan output is pinned here; re-run if Hostinger rotates its key.
+KNOWN_HOSTS_FILE="${INSTALL_DIR}/known_hosts"
+if [[ ! -f "${KNOWN_HOSTS_FILE}" ]]; then
+  echo "Pre-seeding known_hosts from Hostinger (${SSH_HOST:-145.223.107.228}:${SSH_PORT:-65002})..."
+  ssh-keyscan -p "${SSH_PORT:-65002}" "${SSH_HOST:-145.223.107.228}" \
+    > "${KNOWN_HOSTS_FILE}" 2>/dev/null \
+    && chmod 0600 "${KNOWN_HOSTS_FILE}" \
+    && echo "  OK: ${KNOWN_HOSTS_FILE}" \
+    || echo "  WARN: ssh-keyscan failed — known_hosts not seeded; first SSH will add key via accept-new"
+else
+  echo "  OK: ${KNOWN_HOSTS_FILE} (already exists, not overwritten)"
+fi
+
+echo "Installing systemd units..."
+install -m 0644 "${SERVICE_FILE}" "${SYSTEMD_DIR}/prab-generation.service"
+install -m 0644 "${TIMER_FILE}" "${SYSTEMD_DIR}/prab-generation.timer"
+echo "  OK: ${SYSTEMD_DIR}/prab-generation.service"
+echo "  OK: ${SYSTEMD_DIR}/prab-generation.timer"
+
+systemctl daemon-reload
+echo "  OK: systemctl daemon-reload"
+
+echo ""
+echo "Install complete. Timer is installed but NOT enabled."
+echo "Next steps (post-QA + CEO greenlight only):"
+echo "  1. Ensure SSH key exists: ls -la /root/.ssh/hostinger_prod_ed25519"
+echo "  2. Validate with dry run: DRY_RUN=1 /opt/prab-orchestrator/orchestrate-generation.sh"
+echo "  3. Run first real invocation: /opt/prab-orchestrator/orchestrate-generation.sh"
+echo "  4. Enable timer: systemctl enable --now prab-generation.timer"
+echo "  5. Confirm next run: systemctl list-timers prab-generation.timer"
--- a/infra/vps/orchestrate-generation.sh
+++ b/infra/vps/orchestrate-generation.sh
@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+# orchestrate-generation.sh — VPS orchestrator for PRAutoBlogger article generation
+#
+# Runs as a systemd oneshot service (prab-generation.service) on the Coolify KVM8 VPS.
+# SSHes to Hostinger and drives wp prautoblogger generate --sync in a held-open SSH
+# session. SSH-attached processes survive Hostinger's ~10-min background-PHP-process kill.
+#
+# Usage:
+#   DRY_RUN=1 ./orchestrate-generation.sh   # echo commands, no generation
+#   ./orchestrate-generation.sh             # real run
+#
+# Alerting: on failure, sends email to cto@peptiderepo.com via system mail.
+# Logs:     structured log lines to stdout (captured by systemd journal).
+# Lock:     the plugin's Generation_Lock (DB mutex) prevents double-drive.
+#
+# Environment variables:
+#   DRY_RUN       Set to "1" to echo SSH/WP-CLI commands without executing. Default: 0.
+#   LOGFILE       Path for persistent log file. Default: /var/log/prab-generation.log.
+#   ALERT_EMAIL   Failure alert recipient. Default: cto@peptiderepo.com.
+#   SSH_KEY       Path to Hostinger SSH private key. Default: /root/.ssh/hostinger_prod_ed25519.
+#   SSH_HOST      Hostinger SSH host. Default: 145.223.107.228.
+#   SSH_PORT      Hostinger SSH port. Default: 65002.
+#   SSH_USER      Hostinger SSH user. Default: u117248512.
+#   WP_PATH       WordPress docroot on Hostinger. Default: /home/u117248512/domains/peptiderepo.com/public_html.
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# Configuration (override via environment; no hardcoded secrets)
+# ---------------------------------------------------------------------------
+DRY_RUN="${DRY_RUN:-0}"
+LOGFILE="${LOGFILE:-/var/log/prab-generation.log}"
+ALERT_EMAIL="${ALERT_EMAIL:-cto@peptiderepo.com}"
+SSH_KEY="${SSH_KEY:-/root/.ssh/hostinger_prod_ed25519}"
+SSH_HOST="${SSH_HOST:-145.223.107.228}"
+SSH_PORT="${SSH_PORT:-65002}"
+SSH_USER="${SSH_USER:-u117248512}"
+WP_PATH="${WP_PATH:-/home/u117248512/domains/peptiderepo.com/public_html}"
+# Known-hosts file pre-seeded by install-orchestrator.sh with Hostinger's host key.
+# Pins the server identity; StrictHostKeyChecking=accept-new adds on first connect.
+KNOWN_HOSTS="${KNOWN_HOSTS:-/opt/prab-orchestrator/known_hosts}"
+
+# ---------------------------------------------------------------------------
+# Logging helpers
+# ---------------------------------------------------------------------------
+PID="$$"
+TS() { date -u '+%Y-%m-%dT%H:%M:%SZ'; }
+
+log() {
+  local msg="prab-generation[${PID}]: $*"
+  echo "$msg"
+  echo "$(TS) $msg" >> "${LOGFILE}" 2>/dev/null || true
+}
+
+die() {
+  log "FATAL: $*"
+  exit 1
+}
+
+# ---------------------------------------------------------------------------
+# Validate required key before doing anything
+# ---------------------------------------------------------------------------
+if [[ "${DRY_RUN}" != "1" ]] && [[ ! -f "${SSH_KEY}" ]]; then
+  die "SSH key not found at ${SSH_KEY}. Run install-orchestrator.sh first."
+fi
+
+# ---------------------------------------------------------------------------
+# DRY_RUN mode: echo commands and exit
+# ---------------------------------------------------------------------------
+if [[ "${DRY_RUN}" == "1" ]]; then
+  log "DRY_RUN=1 — echoing commands only, no generation"
+  log "WOULD RUN: ssh -i ${SSH_KEY} -p ${SSH_PORT} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 ${SSH_USER}@${SSH_HOST} \\"
+  log "  'wp prautoblogger generate --sync --path=${WP_PATH} 2>&1'"
+  log "DRY_RUN complete"
+  exit 0
+fi
+
+# ---------------------------------------------------------------------------
+# Run start
+# ---------------------------------------------------------------------------
+START_TS="$(TS)"
+log "START ts=${START_TS}"
+
+# ---------------------------------------------------------------------------
+# Step 1: Reaper sweep — clear any stuck runs before starting.
+# Belt-and-suspenders: the plugin's daily reaper cron handles this too, but
+# running it explicitly here ensures a clean state before each generation.
+# ---------------------------------------------------------------------------
+log "REAPER_SWEEP: invoking Run_Reaper::reap()"
+REAP_CMD="wp eval 'echo json_encode(PRAutoBlogger_Run_Reaper::reap());' --path=${WP_PATH} 2>&1"
+
+REAP_OUT=""
+# shellcheck disable=SC2029
+REAP_OUT="$(ssh -i "${SSH_KEY}" \
+    -p "${SSH_PORT}" \
+    -o StrictHostKeyChecking=accept-new \
+    -o UserKnownHostsFile="${KNOWN_HOSTS}" \
+    -o ConnectTimeout=15 \
+    -o ServerAliveInterval=60 \
+    -o ServerAliveCountMax=30 \
+    "${SSH_USER}@${SSH_HOST}" \
+    "${REAP_CMD}" \
+    2>&1)" || true
+log "REAPER_RESULT: ${REAP_OUT}"
+
+# ---------------------------------------------------------------------------
+# Step 2: Run wp prautoblogger generate --sync
+# ---------------------------------------------------------------------------
+log "SSH_CONNECT: ${SSH_USER}@${SSH_HOST}:${SSH_PORT} key=${SSH_KEY}"
+
+WP_CMD="wp prautoblogger generate --sync --path=${WP_PATH} 2>&1"
+
+RESULT_OUTPUT=""
+EXIT_CODE=0
+
+# shellcheck disable=SC2029
+RESULT_OUTPUT="$(ssh -i "${SSH_KEY}" \
+    -p "${SSH_PORT}" \
+    -o StrictHostKeyChecking=accept-new \
+    -o UserKnownHostsFile="${KNOWN_HOSTS}" \
+    -o ConnectTimeout=15 \
+    -o ServerAliveInterval=60 \
+    -o ServerAliveCountMax=30 \
+    "${SSH_USER}@${SSH_HOST}" \
+    "${WP_CMD}" \
+    2>&1)" || EXIT_CODE=$?
+
+# Extract the last JSON summary line (the one from WP_CLI::log(json_encode(...)))
+LAST_JSON="$(echo "${RESULT_OUTPUT}" | grep -E '^\{' | tail -1 || true)"
+LAST_LINE="$(echo "${RESULT_OUTPUT}" | tail -1)"
+
+log "SSH_RESULT exit=${EXIT_CODE} json=${LAST_JSON:-none}"
+
+# ---------------------------------------------------------------------------
+# Step 3: Evaluate result
+# ---------------------------------------------------------------------------
+END_TS="$(TS)"
+
+if [[ ${EXIT_CODE} -eq 0 ]]; then
+  log "END result=success ts=${END_TS}"
+else
+  log "END result=failure exit=${EXIT_CODE} last_line=${LAST_LINE} ts=${END_TS}"
+
+  # Send failure alert via email (system mail; no external deps required).
+  # cto@peptiderepo.com forwards to peptiderepo@gmail.com via Cloudflare Email Routing.
+  ALERT_BODY="PRAutoBlogger generation FAILED on $(hostname) at ${END_TS}.
+Exit code: ${EXIT_CODE}
+Last output: ${LAST_LINE}
+Full log: ${LOGFILE}
+SSH: ${SSH_USER}@${SSH_HOST}:${SSH_PORT}
+WP path: ${WP_PATH}"
+
+  echo "${ALERT_BODY}" \
+    | mail -s "PRAB generation FAILED @ ${END_TS}" "${ALERT_EMAIL}" 2>/dev/null \
+    || log "ALERT_FAILED: could not send email to ${ALERT_EMAIL} (mail not configured?)"
+
+  exit "${EXIT_CODE}"
+fi
--- a/infra/vps/prab-generation.service
+++ b/infra/vps/prab-generation.service
@@ -0,0 +1,22 @@
+[Unit]
+Description=PRAutoBlogger article generation (VPS orchestrator)
+Documentation=https://github.com/peptide-repo/peptide-e2e/tree/main/infra/vps/
+After=network.target
+# Do not start if the timer fires while another instance is still running.
+# Type=oneshot + TimeoutStartSec enforce this at the systemd level.
+
+[Service]
+Type=oneshot
+# root owns /root/.ssh/hostinger_prod_ed25519 and /opt/prab-orchestrator/known_hosts;
+# a dedicated service user would need its own keypair + sudo-less SSH access.
+# Acceptable on this dedicated lab VPS (not shared); revisit if VPS is multi-tenant.
+User=root
+ExecStart=/opt/prab-orchestrator/orchestrate-generation.sh
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=prab-generation
+# Allow up to 2 hours for a full generation run (orchestrate + draft LLM calls).
+# The longest observed single LLM call was 1925s (~32 min); normal runs 2-5 min.
+TimeoutStartSec=7200
+# Do not restart on failure — the timer will retry the next day.
+Restart=no
--- a/infra/vps/prab-generation.timer
+++ b/infra/vps/prab-generation.timer
@@ -0,0 +1,19 @@
+[Unit]
+Description=Daily PRAutoBlogger generation at 06:00 UTC
+Documentation=https://github.com/peptide-repo/peptide-e2e/tree/main/infra/vps/
+Requires=prab-generation.service
+
+[Timer]
+# Fire daily at 06:00 UTC — before Hostinger's daily wp-cron window.
+OnCalendar=*-*-* 06:00:00 UTC
+# Persistent=true: if the VPS was down at 06:00 UTC, fire immediately on next
+# boot so at most one day's generation is missed (not deferred indefinitely).
+Persistent=true
+# Up to 5 min random jitter to avoid thundering-herd if multiple VPS instances
+# are ever added. Harmless with a single instance.
+RandomizedDelaySec=300
+# Accuracy within 1 minute (default is 1 minute; explicit for clarity).
+AccuracySec=60
+
+[Install]
+WantedBy=timers.target
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
  "name": "peptide-e2e",
-  "version": "1.7.2",
+  "version": "1.8.0",
  "description": "Playwright smoke tests for peptiderepo.com (staging + production)",
  "private": true,
  "scripts": {