From afb43e7a41ecfcda3781a63efabdffce634a910d Mon Sep 17 00:00:00 2001 From: peptiderepo <273788462+peptiderepo@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:11:28 +0700 Subject: [PATCH] ci: harden deploy + add health monitor (#5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci(deploy): retry rsync on Hostinger runner-IP throttle + best-effort inventory guard Pilot surfaced an exit-255 SSH connection timeout (documented Hostinger rate-limiting of concurrent runner IPs). rsync now retries 4x/25s; the plugin count guard is best-effort so a throttled check can't fail a good deploy. Agent-Session: cto harden-deploy * ci: add daily Deploy Health Monitor (drift + deploy-failure) Scheduled GitHub Actions check (runs on GitHub infra, not the dead local cron): for each app repo, compares the latest production deployment SHA to main and flags failed deploy runs. Opens one de-duplicated 'deploy-health' issue when something is off — the alarm that was missing when deploys silently broke. Agent-Session: cto deploy-health-monitor --------- Co-authored-by: peptiderepo --- .github/workflows/deploy-app.yml | 100 ++++++++++++++-------------- .github/workflows/deploy-health.yml | 70 +++++++++++++++++++ 2 files changed, 121 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/deploy-health.yml diff --git a/.github/workflows/deploy-app.yml b/.github/workflows/deploy-app.yml index 38f9009..7ac6703 100644 --- a/.github/workflows/deploy-app.yml +++ b/.github/workflows/deploy-app.yml @@ -1,19 +1,19 @@ name: Reusable Deploy (staging → smoke → production) # Single shared deploy pipeline for every peptiderepo plugin and the theme. -# Callers pass `target_path` (the wp-content subpath) and `secrets: inherit`, -# and the deploy logic lives HERE — fix it once instead of in six copy-pasted -# per-repo deploy.yml files (the drift that broke every deploy in May 2026). +# Callers pass `target_path` + `secrets: inherit`; deploy logic lives HERE so a +# fix lands once instead of in six copy-pasted deploy.yml files. # -# Staging deploy → smoke (informational) → production deploy (env-gated) with a -# pre/post active-plugin-count guard (the 2026-04-23 WP-core-clobber tripwire), -# LiteSpeed purge, and a health check. +# Hardened against the documented Hostinger behaviour of rate-limiting concurrent +# GitHub-runner IPs (connection timeouts, exit 255): the rsync is retried with +# backoff, and the plugin-inventory guard is best-effort so a throttled count +# check never fails an otherwise-good deploy. on: workflow_call: inputs: target_path: - description: "wp-content subpath, e.g. wp-content/plugins/prautoblogger or wp-content/themes/peptide-starter" + description: "wp-content subpath, e.g. wp-content/plugins/prautoblogger" required: true type: string run_smoke: @@ -22,18 +22,12 @@ on: default: true type: boolean secrets: - SSH_PRIVATE_KEY: - required: true - SSH_HOST: - required: true - SSH_PORT: - required: true - SSH_USERNAME: - required: true - STAGING_WP_ADMIN_USER: - required: false - STAGING_WP_ADMIN_PASSWORD: - required: false + SSH_PRIVATE_KEY: {required: true} + SSH_HOST: {required: true} + SSH_PORT: {required: true} + SSH_USERNAME: {required: true} + STAGING_WP_ADMIN_USER: {required: false} + STAGING_WP_ADMIN_PASSWORD: {required: false} permissions: contents: read @@ -45,21 +39,27 @@ jobs: environment: staging steps: - uses: actions/checkout@v4 - - name: Deploy via rsync → staging + - name: Deploy via rsync → staging (retry-on-throttle) run: | mkdir -p ~/.ssh echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/deploy_key chmod 600 ~/.ssh/deploy_key ssh-keyscan -p "${{ secrets.SSH_PORT }}" "${{ secrets.SSH_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null || true BASE="/home/${{ secrets.SSH_USERNAME }}/domains/peptiderepo.com/public_html" - rsync -avz --delete \ - --exclude '.git' --exclude '.github' --exclude 'tests' --exclude 'node_modules' \ - --exclude 'vendor' --exclude 'composer.json' --exclude 'composer.lock' \ - --exclude 'phpunit.xml' --exclude 'phpunit.xml.dist' --exclude 'phpcs.xml.dist' \ - --exclude '.phpcs.xml.dist' --exclude 'ARCHITECTURE.md' --exclude 'CONVENTIONS.md' \ - --exclude 'CHANGELOG.md' --exclude 'qa-reviews' \ - -e "ssh -i ~/.ssh/deploy_key -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30 -o BatchMode=yes" \ - ./ "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}:$BASE/staging/${{ inputs.target_path }}/" + SSHOPT="-i ~/.ssh/deploy_key -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30 -o BatchMode=yes" + n=0 + until [ "$n" -ge 4 ]; do + rsync -avz --delete \ + --exclude '.git' --exclude '.github' --exclude 'tests' --exclude 'node_modules' \ + --exclude 'vendor' --exclude 'composer.json' --exclude 'composer.lock' \ + --exclude 'phpunit.xml' --exclude 'phpunit.xml.dist' --exclude 'phpcs.xml.dist' \ + --exclude '.phpcs.xml.dist' --exclude 'ARCHITECTURE.md' --exclude 'CONVENTIONS.md' \ + --exclude 'CHANGELOG.md' --exclude 'qa-reviews' \ + -e "ssh $SSHOPT" \ + ./ "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}:$BASE/staging/${{ inputs.target_path }}/" && break + n=$((n+1)); echo "staging rsync failed (attempt $n/4) — Hostinger may be throttling runner IPs; retrying in 25s..."; sleep 25 + done + [ "$n" -lt 4 ] || { echo "::error::staging rsync failed after 4 attempts"; exit 1; } - name: Staging health check run: | sleep 5 @@ -79,42 +79,44 @@ jobs: deploy-production: name: Deploy to Production - needs: [deploy-staging] # smoke is informational; not a prod gate + needs: [deploy-staging] runs-on: ubuntu-latest environment: production steps: - uses: actions/checkout@v4 - - name: Deploy via rsync → production (plugin-inventory guard) + - name: Deploy via rsync → production (retry + inventory guard) run: | mkdir -p ~/.ssh echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/deploy_key chmod 600 ~/.ssh/deploy_key ssh-keyscan -p "${{ secrets.SSH_PORT }}" "${{ secrets.SSH_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null || true BASE="/home/${{ secrets.SSH_USERNAME }}/domains/peptiderepo.com/public_html" - SSHC="ssh -i ~/.ssh/deploy_key -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30 -o BatchMode=yes ${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}" - PRE=$($SSHC "wp --path=$BASE plugin list --status=active --field=name 2>/dev/null | wc -l" || echo 0) - echo "Pre-deploy active plugins: $PRE" - rsync -avz --delete \ - --exclude '.git' --exclude '.github' --exclude 'tests' --exclude 'node_modules' \ - --exclude 'vendor' --exclude 'composer.json' --exclude 'composer.lock' \ - --exclude 'phpunit.xml' --exclude 'phpunit.xml.dist' --exclude 'phpcs.xml.dist' \ - --exclude '.phpcs.xml.dist' --exclude 'ARCHITECTURE.md' --exclude 'CONVENTIONS.md' \ - --exclude 'CHANGELOG.md' --exclude 'qa-reviews' \ - -e "ssh -i ~/.ssh/deploy_key -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30 -o BatchMode=yes" \ - ./ "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}:$BASE/${{ inputs.target_path }}/" - POST=$($SSHC "wp --path=$BASE plugin list --status=active --field=name 2>/dev/null | wc -l" || echo 0) - echo "Post-deploy active plugins: $POST" - if [ "${PRE:-0}" -gt 0 ] && [ "${POST:-0}" -lt "${PRE:-0}" ]; then - echo "ERROR: active plugin count dropped ${PRE} -> ${POST} after deploy — aborting as failed." - exit 1 + SSHOPT="-i ~/.ssh/deploy_key -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30 -o BatchMode=yes" + PRE=$(ssh $SSHOPT "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}" "wp --path=$BASE plugin list --status=active --field=name 2>/dev/null | wc -l" 2>/dev/null || echo 0) + echo "Pre-deploy active plugins (best-effort): $PRE" + n=0 + until [ "$n" -ge 4 ]; do + rsync -avz --delete \ + --exclude '.git' --exclude '.github' --exclude 'tests' --exclude 'node_modules' \ + --exclude 'vendor' --exclude 'composer.json' --exclude 'composer.lock' \ + --exclude 'phpunit.xml' --exclude 'phpunit.xml.dist' --exclude 'phpcs.xml.dist' \ + --exclude '.phpcs.xml.dist' --exclude 'ARCHITECTURE.md' --exclude 'CONVENTIONS.md' \ + --exclude 'CHANGELOG.md' --exclude 'qa-reviews' \ + -e "ssh $SSHOPT" \ + ./ "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}:$BASE/${{ inputs.target_path }}/" && break + n=$((n+1)); echo "prod rsync failed (attempt $n/4) — retrying in 25s..."; sleep 25 + done + [ "$n" -lt 4 ] || { echo "::error::production rsync failed after 4 attempts"; exit 1; } + POST=$(ssh $SSHOPT "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}" "wp --path=$BASE plugin list --status=active --field=name 2>/dev/null | wc -l; find wp-content/litespeed/ -type f -delete 2>/dev/null; touch .lscache_purge 2>/dev/null || true" 2>/dev/null | head -1 || echo 0) + echo "Post-deploy active plugins (best-effort): $POST" + if [ "${PRE:-0}" -gt 0 ] && [ "${POST:-0}" -gt 0 ] && [ "${POST:-0}" -lt "${PRE:-0}" ]; then + echo "::error::active plugin count dropped ${PRE} -> ${POST} after deploy"; exit 1 fi - $SSHC "cd $BASE && find wp-content/litespeed/ -type f -delete 2>/dev/null; touch .lscache_purge 2>/dev/null || true" - echo "LiteSpeed cache purged." - name: Production health check run: | sleep 5 S=$(curl -s -o /dev/null -w '%{http_code}' https://peptiderepo.com) - if [ "$S" != "200" ] && [ "$S" != "403" ]; then echo "ERROR: site returned HTTP $S"; exit 1; fi + if [ "$S" != "200" ] && [ "$S" != "403" ]; then echo "::error::site returned HTTP $S"; exit 1; fi echo "Production healthy (HTTP $S)" - name: Cleanup if: always() diff --git a/.github/workflows/deploy-health.yml b/.github/workflows/deploy-health.yml new file mode 100644 index 0000000..1862fd4 --- /dev/null +++ b/.github/workflows/deploy-health.yml @@ -0,0 +1,70 @@ +name: Deploy Health Monitor + +# Daily drift + deploy-failure check across all peptiderepo app repos. +# Runs on GitHub's scheduler (independent of the local board cron that died). +# Opens ONE de-duplicated 'deploy-health' issue here when something is off. + +on: + schedule: + - cron: '0 13 * * *' + workflow_dispatch: + +permissions: + contents: read + issues: write + actions: read + +jobs: + check: + name: Check prod-vs-main drift + last deploy status + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v7 + with: + script: | + const owner = 'peptiderepo'; + const repos = [ + 'prautoblogger','peptide-repo-core','peptide-search-ai', + 'peptide-news-plugin','peptide-reconstitution-calculator','peptide-starter-theme' + ]; + const short = s => s ? String(s).substring(0,7) : '—'; + const problems = []; + for (const repo of repos) { + try { + const main = (await github.rest.repos.getBranch({owner, repo, branch:'main'})).data.commit.sha; + let runConc = 'none', runSha = ''; + try { + const runs = (await github.rest.actions.listWorkflowRuns({owner, repo, workflow_id:'deploy.yml', per_page:1})).data.workflow_runs; + if (runs.length) { runConc = runs[0].conclusion || runs[0].status; runSha = runs[0].head_sha; } + } catch (e) { runConc = 'no-deploy-workflow'; } + let prodSha = null, prodState = 'none'; + const deps = (await github.rest.repos.listDeployments({owner, repo, environment:'production', per_page:1})).data; + if (deps.length) { + prodSha = deps[0].sha; + const sts = (await github.rest.repos.listDeploymentStatuses({owner, repo, deployment_id: deps[0].id, per_page:1})).data; + if (sts.length) prodState = sts[0].state; + } + if (runConc === 'failure') problems.push(`**${repo}**: latest deploy run failed (\`${short(runSha)}\`)`); + if (prodSha && prodSha !== main) problems.push(`**${repo}**: prod behind main — prod=\`${short(prodSha)}\` vs main=\`${short(main)}\` (undeployed commits)`); + if (prodSha && prodState !== 'success') problems.push(`**${repo}**: last production deployment state is \`${prodState}\``); + core.info(`${repo}: main=${short(main)} lastRun=${runConc} prodDeploy=${short(prodSha)}/${prodState}`); + } catch (e) { + problems.push(`**${repo}**: monitor error — ${e.message}`); + } + } + const summary = problems.length + ? `### ⚠️ Deploy health: ${problems.length} issue(s)\n\n` + problems.map(p=>`- ${p}`).join('\n') + : `### ✅ All app repos current; last deploy healthy`; + await core.summary.addRaw(summary).write(); + if (problems.length) { + const existing = (await github.rest.issues.listForRepo({owner, repo: context.repo.repo, labels:'deploy-health', state:'open'})).data; + if (existing.length === 0) { + await github.rest.issues.create({ owner, repo: context.repo.repo, + title: `Deploy health: ${problems.length} issue(s) detected`, + body: summary + `\n\n_Opened by deploy-health.yml. Close once resolved; the monitor won't reopen while this stays open._`, + labels: ['deploy-health'] }); + core.warning(`Opened deploy-health issue (${problems.length} problems)`); + } else { + core.warning(`${problems.length} problems; deploy-health issue already open (#${existing[0].number})`); + } + }