ci: harden deploy + add health monitor (#5)
* ci(deploy): retry rsync on Hostinger runner-IP throttle + best-effort inventory guard Pilot surfaced an exit-255 SSH connection timeout (documented Hostinger rate-limiting of concurrent runner IPs). rsync now retries 4x/25s; the plugin count guard is best-effort so a throttled check can't fail a good deploy. Agent-Session: cto harden-deploy * ci: add daily Deploy Health Monitor (drift + deploy-failure) Scheduled GitHub Actions check (runs on GitHub infra, not the dead local cron): for each app repo, compares the latest production deployment SHA to main and flags failed deploy runs. Opens one de-duplicated 'deploy-health' issue when something is off — the alarm that was missing when deploys silently broke. Agent-Session: cto deploy-health-monitor --------- Co-authored-by: peptiderepo <peptiderepo@users.noreply.github.com>
This commit was merged in pull request #5.
This commit is contained in:
peptiderepo
100
.github/workflows/deploy-app.yml
vendored
100
.github/workflows/deploy-app.yml
vendored
@@ -1,19 +1,19 @@
|
||||
name: Reusable Deploy (staging → smoke → production)
|
||||
|
||||
# Single shared deploy pipeline for every peptiderepo plugin and the theme.
|
||||
# Callers pass `target_path` (the wp-content subpath) and `secrets: inherit`,
|
||||
# and the deploy logic lives HERE — fix it once instead of in six copy-pasted
|
||||
# per-repo deploy.yml files (the drift that broke every deploy in May 2026).
|
||||
# Callers pass `target_path` + `secrets: inherit`; deploy logic lives HERE so a
|
||||
# fix lands once instead of in six copy-pasted deploy.yml files.
|
||||
#
|
||||
# Staging deploy → smoke (informational) → production deploy (env-gated) with a
|
||||
# pre/post active-plugin-count guard (the 2026-04-23 WP-core-clobber tripwire),
|
||||
# LiteSpeed purge, and a health check.
|
||||
# Hardened against the documented Hostinger behaviour of rate-limiting concurrent
|
||||
# GitHub-runner IPs (connection timeouts, exit 255): the rsync is retried with
|
||||
# backoff, and the plugin-inventory guard is best-effort so a throttled count
|
||||
# check never fails an otherwise-good deploy.
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
target_path:
|
||||
description: "wp-content subpath, e.g. wp-content/plugins/prautoblogger or wp-content/themes/peptide-starter"
|
||||
description: "wp-content subpath, e.g. wp-content/plugins/prautoblogger"
|
||||
required: true
|
||||
type: string
|
||||
run_smoke:
|
||||
@@ -22,18 +22,12 @@ on:
|
||||
default: true
|
||||
type: boolean
|
||||
secrets:
|
||||
SSH_PRIVATE_KEY:
|
||||
required: true
|
||||
SSH_HOST:
|
||||
required: true
|
||||
SSH_PORT:
|
||||
required: true
|
||||
SSH_USERNAME:
|
||||
required: true
|
||||
STAGING_WP_ADMIN_USER:
|
||||
required: false
|
||||
STAGING_WP_ADMIN_PASSWORD:
|
||||
required: false
|
||||
SSH_PRIVATE_KEY: {required: true}
|
||||
SSH_HOST: {required: true}
|
||||
SSH_PORT: {required: true}
|
||||
SSH_USERNAME: {required: true}
|
||||
STAGING_WP_ADMIN_USER: {required: false}
|
||||
STAGING_WP_ADMIN_PASSWORD: {required: false}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -45,21 +39,27 @@ jobs:
|
||||
environment: staging
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Deploy via rsync → staging
|
||||
- name: Deploy via rsync → staging (retry-on-throttle)
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/deploy_key
|
||||
chmod 600 ~/.ssh/deploy_key
|
||||
ssh-keyscan -p "${{ secrets.SSH_PORT }}" "${{ secrets.SSH_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null || true
|
||||
BASE="/home/${{ secrets.SSH_USERNAME }}/domains/peptiderepo.com/public_html"
|
||||
rsync -avz --delete \
|
||||
--exclude '.git' --exclude '.github' --exclude 'tests' --exclude 'node_modules' \
|
||||
--exclude 'vendor' --exclude 'composer.json' --exclude 'composer.lock' \
|
||||
--exclude 'phpunit.xml' --exclude 'phpunit.xml.dist' --exclude 'phpcs.xml.dist' \
|
||||
--exclude '.phpcs.xml.dist' --exclude 'ARCHITECTURE.md' --exclude 'CONVENTIONS.md' \
|
||||
--exclude 'CHANGELOG.md' --exclude 'qa-reviews' \
|
||||
-e "ssh -i ~/.ssh/deploy_key -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30 -o BatchMode=yes" \
|
||||
./ "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}:$BASE/staging/${{ inputs.target_path }}/"
|
||||
SSHOPT="-i ~/.ssh/deploy_key -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30 -o BatchMode=yes"
|
||||
n=0
|
||||
until [ "$n" -ge 4 ]; do
|
||||
rsync -avz --delete \
|
||||
--exclude '.git' --exclude '.github' --exclude 'tests' --exclude 'node_modules' \
|
||||
--exclude 'vendor' --exclude 'composer.json' --exclude 'composer.lock' \
|
||||
--exclude 'phpunit.xml' --exclude 'phpunit.xml.dist' --exclude 'phpcs.xml.dist' \
|
||||
--exclude '.phpcs.xml.dist' --exclude 'ARCHITECTURE.md' --exclude 'CONVENTIONS.md' \
|
||||
--exclude 'CHANGELOG.md' --exclude 'qa-reviews' \
|
||||
-e "ssh $SSHOPT" \
|
||||
./ "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}:$BASE/staging/${{ inputs.target_path }}/" && break
|
||||
n=$((n+1)); echo "staging rsync failed (attempt $n/4) — Hostinger may be throttling runner IPs; retrying in 25s..."; sleep 25
|
||||
done
|
||||
[ "$n" -lt 4 ] || { echo "::error::staging rsync failed after 4 attempts"; exit 1; }
|
||||
- name: Staging health check
|
||||
run: |
|
||||
sleep 5
|
||||
@@ -79,42 +79,44 @@ jobs:
|
||||
|
||||
deploy-production:
|
||||
name: Deploy to Production
|
||||
needs: [deploy-staging] # smoke is informational; not a prod gate
|
||||
needs: [deploy-staging]
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Deploy via rsync → production (plugin-inventory guard)
|
||||
- name: Deploy via rsync → production (retry + inventory guard)
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/deploy_key
|
||||
chmod 600 ~/.ssh/deploy_key
|
||||
ssh-keyscan -p "${{ secrets.SSH_PORT }}" "${{ secrets.SSH_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null || true
|
||||
BASE="/home/${{ secrets.SSH_USERNAME }}/domains/peptiderepo.com/public_html"
|
||||
SSHC="ssh -i ~/.ssh/deploy_key -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30 -o BatchMode=yes ${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}"
|
||||
PRE=$($SSHC "wp --path=$BASE plugin list --status=active --field=name 2>/dev/null | wc -l" || echo 0)
|
||||
echo "Pre-deploy active plugins: $PRE"
|
||||
rsync -avz --delete \
|
||||
--exclude '.git' --exclude '.github' --exclude 'tests' --exclude 'node_modules' \
|
||||
--exclude 'vendor' --exclude 'composer.json' --exclude 'composer.lock' \
|
||||
--exclude 'phpunit.xml' --exclude 'phpunit.xml.dist' --exclude 'phpcs.xml.dist' \
|
||||
--exclude '.phpcs.xml.dist' --exclude 'ARCHITECTURE.md' --exclude 'CONVENTIONS.md' \
|
||||
--exclude 'CHANGELOG.md' --exclude 'qa-reviews' \
|
||||
-e "ssh -i ~/.ssh/deploy_key -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30 -o BatchMode=yes" \
|
||||
./ "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}:$BASE/${{ inputs.target_path }}/"
|
||||
POST=$($SSHC "wp --path=$BASE plugin list --status=active --field=name 2>/dev/null | wc -l" || echo 0)
|
||||
echo "Post-deploy active plugins: $POST"
|
||||
if [ "${PRE:-0}" -gt 0 ] && [ "${POST:-0}" -lt "${PRE:-0}" ]; then
|
||||
echo "ERROR: active plugin count dropped ${PRE} -> ${POST} after deploy — aborting as failed."
|
||||
exit 1
|
||||
SSHOPT="-i ~/.ssh/deploy_key -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=30 -o BatchMode=yes"
|
||||
PRE=$(ssh $SSHOPT "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}" "wp --path=$BASE plugin list --status=active --field=name 2>/dev/null | wc -l" 2>/dev/null || echo 0)
|
||||
echo "Pre-deploy active plugins (best-effort): $PRE"
|
||||
n=0
|
||||
until [ "$n" -ge 4 ]; do
|
||||
rsync -avz --delete \
|
||||
--exclude '.git' --exclude '.github' --exclude 'tests' --exclude 'node_modules' \
|
||||
--exclude 'vendor' --exclude 'composer.json' --exclude 'composer.lock' \
|
||||
--exclude 'phpunit.xml' --exclude 'phpunit.xml.dist' --exclude 'phpcs.xml.dist' \
|
||||
--exclude '.phpcs.xml.dist' --exclude 'ARCHITECTURE.md' --exclude 'CONVENTIONS.md' \
|
||||
--exclude 'CHANGELOG.md' --exclude 'qa-reviews' \
|
||||
-e "ssh $SSHOPT" \
|
||||
./ "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}:$BASE/${{ inputs.target_path }}/" && break
|
||||
n=$((n+1)); echo "prod rsync failed (attempt $n/4) — retrying in 25s..."; sleep 25
|
||||
done
|
||||
[ "$n" -lt 4 ] || { echo "::error::production rsync failed after 4 attempts"; exit 1; }
|
||||
POST=$(ssh $SSHOPT "${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }}" "wp --path=$BASE plugin list --status=active --field=name 2>/dev/null | wc -l; find wp-content/litespeed/ -type f -delete 2>/dev/null; touch .lscache_purge 2>/dev/null || true" 2>/dev/null | head -1 || echo 0)
|
||||
echo "Post-deploy active plugins (best-effort): $POST"
|
||||
if [ "${PRE:-0}" -gt 0 ] && [ "${POST:-0}" -gt 0 ] && [ "${POST:-0}" -lt "${PRE:-0}" ]; then
|
||||
echo "::error::active plugin count dropped ${PRE} -> ${POST} after deploy"; exit 1
|
||||
fi
|
||||
$SSHC "cd $BASE && find wp-content/litespeed/ -type f -delete 2>/dev/null; touch .lscache_purge 2>/dev/null || true"
|
||||
echo "LiteSpeed cache purged."
|
||||
- name: Production health check
|
||||
run: |
|
||||
sleep 5
|
||||
S=$(curl -s -o /dev/null -w '%{http_code}' https://peptiderepo.com)
|
||||
if [ "$S" != "200" ] && [ "$S" != "403" ]; then echo "ERROR: site returned HTTP $S"; exit 1; fi
|
||||
if [ "$S" != "200" ] && [ "$S" != "403" ]; then echo "::error::site returned HTTP $S"; exit 1; fi
|
||||
echo "Production healthy (HTTP $S)"
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
|
||||
70
.github/workflows/deploy-health.yml
vendored
Normal file
70
.github/workflows/deploy-health.yml
vendored
Normal file
@@ -0,0 +1,70 @@
|
||||
name: Deploy Health Monitor
|
||||
|
||||
# Daily drift + deploy-failure check across all peptiderepo app repos.
|
||||
# Runs on GitHub's scheduler (independent of the local board cron that died).
|
||||
# Opens ONE de-duplicated 'deploy-health' issue here when something is off.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 13 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Check prod-vs-main drift + last deploy status
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const owner = 'peptiderepo';
|
||||
const repos = [
|
||||
'prautoblogger','peptide-repo-core','peptide-search-ai',
|
||||
'peptide-news-plugin','peptide-reconstitution-calculator','peptide-starter-theme'
|
||||
];
|
||||
const short = s => s ? String(s).substring(0,7) : '—';
|
||||
const problems = [];
|
||||
for (const repo of repos) {
|
||||
try {
|
||||
const main = (await github.rest.repos.getBranch({owner, repo, branch:'main'})).data.commit.sha;
|
||||
let runConc = 'none', runSha = '';
|
||||
try {
|
||||
const runs = (await github.rest.actions.listWorkflowRuns({owner, repo, workflow_id:'deploy.yml', per_page:1})).data.workflow_runs;
|
||||
if (runs.length) { runConc = runs[0].conclusion || runs[0].status; runSha = runs[0].head_sha; }
|
||||
} catch (e) { runConc = 'no-deploy-workflow'; }
|
||||
let prodSha = null, prodState = 'none';
|
||||
const deps = (await github.rest.repos.listDeployments({owner, repo, environment:'production', per_page:1})).data;
|
||||
if (deps.length) {
|
||||
prodSha = deps[0].sha;
|
||||
const sts = (await github.rest.repos.listDeploymentStatuses({owner, repo, deployment_id: deps[0].id, per_page:1})).data;
|
||||
if (sts.length) prodState = sts[0].state;
|
||||
}
|
||||
if (runConc === 'failure') problems.push(`**${repo}**: latest deploy run failed (\`${short(runSha)}\`)`);
|
||||
if (prodSha && prodSha !== main) problems.push(`**${repo}**: prod behind main — prod=\`${short(prodSha)}\` vs main=\`${short(main)}\` (undeployed commits)`);
|
||||
if (prodSha && prodState !== 'success') problems.push(`**${repo}**: last production deployment state is \`${prodState}\``);
|
||||
core.info(`${repo}: main=${short(main)} lastRun=${runConc} prodDeploy=${short(prodSha)}/${prodState}`);
|
||||
} catch (e) {
|
||||
problems.push(`**${repo}**: monitor error — ${e.message}`);
|
||||
}
|
||||
}
|
||||
const summary = problems.length
|
||||
? `### ⚠️ Deploy health: ${problems.length} issue(s)\n\n` + problems.map(p=>`- ${p}`).join('\n')
|
||||
: `### ✅ All app repos current; last deploy healthy`;
|
||||
await core.summary.addRaw(summary).write();
|
||||
if (problems.length) {
|
||||
const existing = (await github.rest.issues.listForRepo({owner, repo: context.repo.repo, labels:'deploy-health', state:'open'})).data;
|
||||
if (existing.length === 0) {
|
||||
await github.rest.issues.create({ owner, repo: context.repo.repo,
|
||||
title: `Deploy health: ${problems.length} issue(s) detected`,
|
||||
body: summary + `\n\n_Opened by deploy-health.yml. Close once resolved; the monitor won't reopen while this stays open._`,
|
||||
labels: ['deploy-health'] });
|
||||
core.warning(`Opened deploy-health issue (${problems.length} problems)`);
|
||||
} else {
|
||||
core.warning(`${problems.length} problems; deploy-health issue already open (#${existing[0].number})`);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user