fix: VPS-orchestrated generation reliability — generate --sync, reaper TZ fix, maintenance-cron self-heal (v0.22.1) (#171)

Synchronous --sync CLI tick loop for VPS-driven generation over a held-open SSH session (survives Hostinger's ~10-min background-kill); $sync_mode guard on both tick handlers prevents double-drive; Run_Reaper UTC-vs-site-tz fix; one-shot self-heal migration restoring the maintenance crons that vanished after the 06-15/16 deploy (daily-generation excluded — VPS owns it).

QA: APPROVE @ 0689989 (qa-reviews/prautoblogger/2026-06-23-0689989.md). CEO greenlight 2026-06-22.

Agent-Session: prab-reliability-merge-20260622

includes/admin/class-wp-cli-commands.php

@@ -8,6 +8,14 @@ declare(strict_types=1);
  */
 class PRAutoBlogger_WP_CLI_Commands {
 
+	/**
+	 * Max generate-tick iterations for the --sync loop (covers daily_article_target = 1
+	 * with ample headroom; prevents infinite loops if the queue never drains).
+	 *
+	 * @var int
+	 */
+	private const SYNC_MAX_TICKS = 20;
+
 	/**
 	 * Register WP-CLI commands.
 	 *
@@ -18,12 +26,19 @@ class PRAutoBlogger_WP_CLI_Commands {
 			return;
 		}
 
-		// Command: wp prautoblogger opik:eval
 		\WP_CLI::add_command(
 			'prautoblogger generate',
 			array( self::class, 'generate_command' ),
 			array(
-				'shortdesc' => 'Queue a new article generation run on chained-cron checkpoints',
+				'shortdesc' => 'Queue (or synchronously run) a new article generation run',
+				'synopsis'  => array(
+					array(
+						'type'        => 'flag',
+						'name'        => 'sync',
+						'description' => 'Run the full pipeline synchronously in this process (VPS orchestrator mode). Blocks until complete.',
+						'optional'    => true,
+					),
+				),
 			)
 		);
 
@@ -51,6 +66,148 @@ class PRAutoBlogger_WP_CLI_Commands {
 		);
 	}
 
+	/**
+	 * Queue or synchronously run a new article generation run.
+	 *
+	 * Without --sync: routes through kick_off() (same as the board "New Article"
+	 * button). Returns immediately; all pipeline work happens in cron.
+	 *
+	 * With --sync: runs the pipeline synchronously in this SSH/CLI process.
+	 *   1. Acquires the generation lock (skips if a live run is in progress).
+	 *   2. Calls on_orchestrate_tick() to collect/score ideas in-process.
+	 *   3. Loops calling on_generate_tick() until the queue is empty or the run
+	 *      reaches a terminal state, up to SYNC_MAX_TICKS iterations.
+	 *   4. Emits a JSON summary line on stdout and exits 0 (success) or 1 (error).
+	 *
+	 * The --sync path suppresses wp-cron reschedule inside on_generate_tick() so
+	 * the external loop cannot race with a background cron tick.
+	 *
+	 * ## OPTIONS
+	 *
+	 * [--sync]
+	 * : Run synchronously (VPS orchestrator mode).
+	 *
+	 * ## EXAMPLES
+	 *
+	 *     wp prautoblogger generate
+	 *     wp prautoblogger generate --sync
+	 *
+	 * @param array $args       Positional args (unused).
+	 * @param array $assoc_args Associative args: sync.
+	 * @return void
+	 */
+	public static function generate_command( array $args, array $assoc_args ): void {
+		if ( isset( $assoc_args['sync'] ) ) {
+			self::run_sync();
+			return;
+		}
+
+		PRAutoBlogger_Generation_Checkpoint_Runner::kick_off();
+		\WP_CLI::success( 'Generation queued on chained-cron checkpoints. Check the board or Activity Log for progress.' );
+	}
+
+	/**
+	 * Run the full pipeline synchronously in this process (--sync mode).
+	 *
+	 * Called only from generate_command(); never from wp-cron or AJAX paths.
+	 *
+	 * @return void
+	 */
+	private static function run_sync(): void {
+		// Honour ignore_user_abort so a dropped SSH session does not kill us.
+		// phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged
+		@ignore_user_abort( true );
+
+		// Check for a live (non-expired) lock. Generation_Lock::acquire() already
+		// cleans expired locks on its DELETE pass, so if acquire() returns false
+		// there is a genuinely live concurrent run — skip cleanly.
+		if ( ! PRAutoBlogger_Generation_Lock::is_locked() ) {
+			// Lock is not held; proceed. acquire() is called inside on_orchestrate_tick().
+		} else {
+			$acquired_at = PRAutoBlogger_Generation_Lock::get_acquired_at();
+			$age         = null !== $acquired_at ? ( time() - $acquired_at ) : HOUR_IN_SECONDS;
+			if ( $age < HOUR_IN_SECONDS ) {
+				\WP_CLI::log(
+					json_encode(
+						array(
+							'status' => 'skipped',
+							'reason' => 'run_in_progress',
+							'lock_age_s' => $age,
+						)
+					)
+				);
+				exit( 0 );
+			}
+			// Expired lock — on_orchestrate_tick()'s acquire() will clean it.
+		}
+
+		// Enable sync-mode: on_generate_tick() will skip the wp-cron reschedule.
+		PRAutoBlogger_Generation_Checkpoint_Runner::set_sync_mode( true );
+
+		\WP_CLI::log( 'PRAB-SYNC: starting orchestrate tick' );
+		PRAutoBlogger_Generation_Checkpoint_Runner::on_orchestrate_tick();
+
+		// If the orchestrate tick failed to acquire the lock or found no ideas,
+		// the RUN_ID option will be absent. Detect and exit cleanly.
+		$run_id = (string) get_option( 'prautoblogger_checkpoint_run_id', '' );
+		if ( '' === $run_id ) {
+			$queue = get_option( 'prautoblogger_article_queue' );
+			if ( ! is_array( $queue ) || empty( $queue['ideas'] ) ) {
+				\WP_CLI::log(
+					json_encode(
+						array(
+							'status' => 'done',
+							'generated' => 0,
+							'reason' => 'no_ideas_or_lock_failed',
+						)
+					)
+				);
+				exit( 0 );
+			}
+		}
+
+		// Tick loop: drive generate ticks synchronously.
+		$ticks = 0;
+		while ( $ticks < self::SYNC_MAX_TICKS ) {
+			$queue = get_option( 'prautoblogger_article_queue' );
+			if ( ! is_array( $queue ) || empty( $queue['ideas'] ) ) {
+				break; // Queue drained or finalized.
+			}
+			++$ticks;
+			\WP_CLI::log( sprintf( 'PRAB-SYNC: generate tick %d (ideas remaining: %d)', $ticks, count( $queue['ideas'] ) ) );
+			PRAutoBlogger_Generation_Checkpoint_Runner::on_generate_tick();
+		}
+
+		PRAutoBlogger_Generation_Checkpoint_Runner::set_sync_mode( false );
+
+		if ( $ticks >= self::SYNC_MAX_TICKS ) {
+			\WP_CLI::warning( 'PRAB-SYNC: hit max-tick guard (' . self::SYNC_MAX_TICKS . '). Queue may not be fully drained.' );
+		}
+
+		// Read final status transient for summary.
+		$status = get_transient( 'prautoblogger_generation_status' );
+		$result = is_array( $status ) ? $status : array();
+
+		$summary = array(
+			'status'     => 'done',
+			'generated'  => (int) ( $result['generated'] ?? 0 ),
+			'published'  => (int) ( $result['published'] ?? 0 ),
+			'rejected'   => (int) ( $result['rejected'] ?? 0 ),
+			'cost_usd'   => round( (float) ( $result['cost'] ?? 0.0 ), 4 ),
+			'ticks'      => $ticks,
+		);
+
+		\WP_CLI::log( json_encode( $summary ) );
+
+		if ( 'error' === ( $result['status'] ?? '' ) ) {
+			\WP_CLI::error( 'PRAB-SYNC: generation completed with error: ' . ( $result['message'] ?? 'unknown' ), false );
+			exit( 1 );
+		}
+
+		\WP_CLI::success( 'PRAB-SYNC: generation complete.' );
+		exit( 0 );
+	}
+
 	/**
 	 * Run eval on the frozen dataset.
 	 *
@@ -77,27 +234,4 @@ class PRAutoBlogger_WP_CLI_Commands {
 
 		exit( $result['items_run'] > 0 ? 0 : 1 );
 	}
-
-	/**
-	 * Queue a new article generation run on chained-cron checkpoints.
-	 *
-	 * Routes through PRAutoBlogger_Generation_Checkpoint_Runner::kick_off()
-	 * (same path as the board "New Article" button). Does NOT execute
-	 * synchronously -- check the board or Activity Log for progress.
-	 * This retires the `do_action("prautoblogger_manual_generation")` workaround.
-	 *
-	 * ## OPTIONS
-	 *
-	 * ## EXAMPLES
-	 *
-	 *     wp prautoblogger generate
-	 *
-	 * @param array $args       Positional args (unused).
-	 * @param array $assoc_args Associative args (unused).
-	 * @return void
-	 */
-	public static function generate_command( array $args, array $assoc_args ): void {
-		PRAutoBlogger_Generation_Checkpoint_Runner::kick_off();
-		\WP_CLI::success( 'Generation queued on chained-cron checkpoints. Check the board or Activity Log for progress.' );
-	}
 }

includes/class-db-migrations.php

@@ -125,5 +125,32 @@ class PRAutoBlogger_DB_Migrations {
 			}
 			update_option( 'prautoblogger_migrated_enc_prefix', '1' );
 		}
+
+		// v0.22.1 self-heal: restore low-stakes maintenance cron events that were
+		// lost when the WP-Cron schedule vanished after the 06-15 deploy cycle.
+		// NOTE: wp-cron reliability on this Hostinger host is suspect (background
+		// PHP processes are killed at ~10 min). These low-stakes events (metrics,
+		// reaper, Runware sync) are acceptable on wp-cron; daily GENERATION is NOT
+		// re-registered here — the VPS systemd timer owns generation scheduling.
+		// The prautoblogger_migrated_schedule_tz_v082 guard is intentionally NOT
+		// re-used here; this migration targets the missing events themselves.
+		if ( ! get_option( 'prautoblogger_migrated_maintenance_crons_v0221' ) ) {
+			// Re-register maintenance crons only — daily_generation deliberately excluded.
+			// Activator::schedule_cron() uses wp_next_scheduled() guards so it is
+			// idempotent and will not overwrite events that are already present.
+			if ( ! wp_next_scheduled( 'prautoblogger_reap_orphan_research_rows' ) ) {
+				$tomorrow_reap = strtotime( 'tomorrow 03:15' );
+				if ( false !== $tomorrow_reap ) {
+					wp_schedule_event( (int) $tomorrow_reap, 'daily', 'prautoblogger_reap_orphan_research_rows' );
+				}
+			}
+			if ( ! wp_next_scheduled( 'prautoblogger_collect_metrics' ) ) {
+				wp_schedule_event( time() + HOUR_IN_SECONDS, 'prautoblogger_six_hours', 'prautoblogger_collect_metrics' );
+			}
+			if ( ! wp_next_scheduled( 'prautoblogger_sync_runware_models' ) ) {
+				wp_schedule_event( time() + HOUR_IN_SECONDS, 'daily', 'prautoblogger_sync_runware_models' );
+			}
+			update_option( 'prautoblogger_migrated_maintenance_crons_v0221', '1' );
+		}
 	}
 }

includes/core/class-generation-checkpoint-runner.php

@@ -52,6 +52,26 @@ class PRAutoBlogger_Generation_Checkpoint_Runner {
 	/** WP-Cron action for each article-generation tick. */
 	public const GENERATE_ACTION = 'prautoblogger_gen_tick';
 
+	/**
+	 * When true, on_generate_tick() skips wp-cron rescheduling so the VPS
+	 * orchestrator's external loop is the sole driver. Set via set_sync_mode().
+	 * Never true on the wp-cron path; reset to false after the sync run.
+	 *
+	 * @var bool
+	 */
+	private static bool $sync_mode = false;
+
+	/**
+	 * Enable or disable synchronous-driver mode.
+	 * Called by WP_CLI_Commands::run_sync() before and after the tick loop.
+	 *
+	 * @param bool $enabled True to suppress internal cron reschedule.
+	 * @return void
+	 */
+	public static function set_sync_mode( bool $enabled ): void {
+		self::$sync_mode = $enabled;
+	}
+
 	/**
 	 * Schedule the orchestration tick and write the initial status transient.
 	 * Called by on_ajax_generate_now() (via Executor) and by the WP-CLI command.
@@ -155,11 +175,15 @@ class PRAutoBlogger_Generation_Checkpoint_Runner {
 				)
 			);
 
-			if ( ! wp_next_scheduled( self::GENERATE_ACTION ) ) {
-				wp_schedule_single_event( time(), self::GENERATE_ACTION );
+			// In sync mode the VPS tick loop is the external driver; skip
+			// wp-cron reschedule to prevent a competing background tick.
+			// on_generate_tick() has the same guard for subsequent reschedules.
+			if ( ! self::$sync_mode ) {
+				if ( ! wp_next_scheduled( self::GENERATE_ACTION ) ) {
+					wp_schedule_single_event( time(), self::GENERATE_ACTION );
+				}
+				PRAutoBlogger_Generation_Checkpoint_Helpers::fire_cron_now();
 			}
-			PRAutoBlogger_Generation_Checkpoint_Helpers::fire_cron_now();
-
 		} catch ( \Throwable $e ) {
 			PRAutoBlogger_Logger::instance()->error(
 				sprintf( 'Orchestrate tick %s: %s', get_class( $e ), $e->getMessage() ),
@@ -245,10 +269,14 @@ class PRAutoBlogger_Generation_Checkpoint_Runner {
 			if ( ! empty( $queue['ideas'] ) ) {
 				update_option( self::QUEUE_KEY, $queue, false );
 				PRAutoBlogger_Pipeline_Status::update_queue_progress( $queue );
-				if ( ! wp_next_scheduled( self::GENERATE_ACTION ) ) {
-					wp_schedule_single_event( time(), self::GENERATE_ACTION );
+				// In sync mode the VPS tick loop is the external driver; skip
+				// wp-cron reschedule to prevent a competing background tick.
+				if ( ! self::$sync_mode ) {
+					if ( ! wp_next_scheduled( self::GENERATE_ACTION ) ) {
+						wp_schedule_single_event( time(), self::GENERATE_ACTION );
+					}
+					PRAutoBlogger_Generation_Checkpoint_Helpers::fire_cron_now();
 				}
-				PRAutoBlogger_Generation_Checkpoint_Helpers::fire_cron_now();
 			} else {
 				PRAutoBlogger_Post_Assembler::amortize_research_costs( $run_id );
 				PRAutoBlogger_Generation_Checkpoint_Helpers::finalize( $queue['results'], $run_id );

includes/core/class-run-reaper.php

@@ -133,7 +133,9 @@ class PRAutoBlogger_Run_Reaper {
 		 * @param int $seconds Expected seconds (default 300).
 		 */
 		$expected = (int) apply_filters( 'prautoblogger_filter_stage_expected_seconds', self::EXPECTED_STAGE_SECONDS );
-		$cutoff   = gmdate( 'Y-m-d H:i:s', time() - ( 2 * $expected ) );
+		// Use wp_date() so the cutoff is expressed in the site's local timezone,
+		// matching the updated_at values written by current_time('mysql') (v0.22.1 TZ fix).
+		$cutoff   = wp_date( 'Y-m-d H:i:s', time() - ( 2 * $expected ) );
 		$now      = current_time( 'mysql' );
 
 		// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery
@@ -169,7 +171,8 @@ class PRAutoBlogger_Run_Reaper {
 		 * @param int $seconds Expected seconds (default 1800).
 		 */
 		$expected = (int) apply_filters( 'prautoblogger_filter_expected_run_seconds', self::EXPECTED_RUN_SECONDS );
-		$cutoff   = gmdate( 'Y-m-d H:i:s', time() - ( 2 * $expected ) );
+		// Use wp_date() so the cutoff matches updated_at's local-timezone storage (v0.22.1 TZ fix).
+		$cutoff   = wp_date( 'Y-m-d H:i:s', time() - ( 2 * $expected ) );
 
 		// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching
 		$stuck = $wpdb->get_col(