feat(runs): state machine, idempotent resume, reaper extension, retention
Pipeline v2 Phase 1, commit 4/4. Run_Stage_State persists per-run per-stage state keyed run_id + stage + agent_role + item_key (item_key scopes articles because one run_id spans a whole batch; agent_role is the Phase-2 quorum dimension, schema-only). Done stages snapshot output and are reused on re-entry — writer stages via Content_Generator::set_run_item, the review via its stored verdict, a published item skipped outright — never re-run, never re-charged. Post creation is keyed by _prautoblogger_run_id + new _prautoblogger_idea_hash (check-before-insert in Publisher) so retries cannot duplicate posts. Run-level research/analysis checkpoints + done/failed lifecycle marks ride Pipeline_Runner/Executor (incl. orphan-abort + budget-abort paths). Run_Reaper rides the EXISTING #19 daily cron: stages stuck running >2x expected stage wall-clock and runs silent >2x expected run wall-clock -> failed (filterable), open stages reaped, queryable as 'incomplete'. Retention: request_json + stage output payloads NULLed after R days, R from the new Audit Payload Retention setting (default constant 14, 0 = keep forever, no literal at point of use). Audit_Writer records editorial verdicts (run_decisions) and per-article kept sources (run_sources) for new runs. Under-300: status-transient/summary helpers extracted verbatim to Pipeline_Status (Pipeline_Runner 271; removed the worker's duplicate). Agent-Session: pipeline-v2-phase1-engineer-20260612
This commit is contained in:
@@ -113,6 +113,16 @@ class PRAutoBlogger_Settings_Fields_Extended {
|
||||
'step' => '0.01',
|
||||
'description' => __( 'Hard stop when reached. 0 = unlimited.', 'prautoblogger' ),
|
||||
),
|
||||
array(
|
||||
'id' => 'prautoblogger_request_json_retention_days',
|
||||
'label' => __( 'Audit Payload Retention (days)', 'prautoblogger' ),
|
||||
'type' => 'number',
|
||||
'section' => 'prautoblogger_schedule',
|
||||
'default' => PRAUTOBLOGGER_DEFAULT_REQUEST_JSON_RETENTION_DAYS,
|
||||
'min' => 0,
|
||||
'step' => '1',
|
||||
'description' => __( 'Heavy audit payloads (request_json on the generation log, stage output snapshots) are emptied after this many days by the daily cleanup cron. Cost/token rows are kept forever. 0 = keep payloads forever.', 'prautoblogger' ),
|
||||
),
|
||||
array(
|
||||
'id' => 'prautoblogger_per_run_cost_ceiling_usd',
|
||||
'label' => __( 'Per-Run Cost Ceiling (USD)', 'prautoblogger' ),
|
||||
|
||||
@@ -69,6 +69,7 @@ class PRAutoBlogger_Activator {
|
||||
'prautoblogger_target_subreddits' => '[]',
|
||||
'prautoblogger_monthly_budget_usd' => 50.00,
|
||||
'prautoblogger_per_run_cost_ceiling_usd' => PRAUTOBLOGGER_DEFAULT_RUN_CEILING_USD,
|
||||
'prautoblogger_request_json_retention_days' => PRAUTOBLOGGER_DEFAULT_REQUEST_JSON_RETENTION_DAYS,
|
||||
'prautoblogger_tone' => 'informational',
|
||||
'prautoblogger_min_word_count' => 800,
|
||||
'prautoblogger_max_word_count' => 2000,
|
||||
|
||||
@@ -53,10 +53,25 @@ class PRAutoBlogger_Executor {
|
||||
sprintf( 'Daily generation %s: %s', get_class( $e ), $e->getMessage() ),
|
||||
'scheduler'
|
||||
);
|
||||
$this->mark_current_run_failed();
|
||||
PRAutoBlogger_Generation_Lock::release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark this process's run failed after a fatal pipeline error
|
||||
* (no-op outside a run; a governor-halted run stays halted — final
|
||||
* states are sticky).
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function mark_current_run_failed(): void {
|
||||
$run_id = PRAutoBlogger_Run_Context::current_run_id();
|
||||
if ( null !== $run_id ) {
|
||||
PRAutoBlogger_Run_State::mark_status( $run_id, 'failed' );
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle the metrics collection cron event.
|
||||
* Side effects: API calls to GA4, database writes to ab_content_scores.
|
||||
@@ -209,6 +224,7 @@ class PRAutoBlogger_Executor {
|
||||
),
|
||||
self::STATUS_TTL
|
||||
);
|
||||
$this->mark_current_run_failed();
|
||||
PRAutoBlogger_Generation_Lock::release();
|
||||
}
|
||||
}
|
||||
@@ -286,6 +302,12 @@ class PRAutoBlogger_Executor {
|
||||
|
||||
/** Clean up an orphaned generation run and report error. */
|
||||
private function abort_orphaned_run( string $message ): void {
|
||||
// v0.18.0: the dead run renders as failed/"incomplete" in audit
|
||||
// queries instead of silently lingering as 'running'.
|
||||
$queue = get_option( 'prautoblogger_article_queue' );
|
||||
if ( is_array( $queue ) && ! empty( $queue['run_id'] ) ) {
|
||||
PRAutoBlogger_Run_State::mark_status( (string) $queue['run_id'], 'failed' );
|
||||
}
|
||||
delete_transient( self::STATUS_TRANSIENT );
|
||||
delete_option( 'prautoblogger_article_queue' );
|
||||
PRAutoBlogger_Generation_Lock::release();
|
||||
|
||||
@@ -150,6 +150,10 @@ class PRAutoBlogger {
|
||||
// pipeline dies before amortize_research_costs runs).
|
||||
add_action( 'prautoblogger_reap_orphan_research_rows', array( 'PRAutoBlogger_Research_Reaper', 'on_cron' ) );
|
||||
|
||||
// v0.18.0: stuck-run sweep + audit-payload retention rides the SAME
|
||||
// existing daily cron (no new schedule). See class-run-reaper.php.
|
||||
add_action( 'prautoblogger_reap_orphan_research_rows', array( 'PRAutoBlogger_Run_Reaper', 'on_cron' ) );
|
||||
|
||||
// v0.8.1: WP-CLI manual trigger for the reaper. Only registers when
|
||||
// WP-CLI is present; no-op in normal HTTP requests.
|
||||
if ( defined( 'WP_CLI' ) && WP_CLI ) {
|
||||
|
||||
@@ -9,8 +9,15 @@ declare(strict_types=1);
|
||||
* Extracted from Pipeline_Runner so each chained cron job has a focused,
|
||||
* single-responsibility worker. One worker = one article = one PHP process.
|
||||
*
|
||||
* v0.18.0: per-article stage state (run_stages, item-scoped) makes resume
|
||||
* idempotent — an already-published item is skipped outright, a done
|
||||
* review is reused from its snapshot, and the writer reuses done LLM
|
||||
* stages via Content_Generator::set_run_item(). Editorial verdicts and
|
||||
* the sources that fed the article are persisted to the audit tables.
|
||||
*
|
||||
* Triggered by: Pipeline_Runner (directly for article 1, via cron for 2..N).
|
||||
* Dependencies: Content_Generator, Chief_Editor, Publisher, Cost_Tracker.
|
||||
* Dependencies: Content_Generator, Chief_Editor, Publisher, Cost_Tracker,
|
||||
* Run_Stage_State, Audit_Writer.
|
||||
*
|
||||
* Opik instrumentation: Initializes trace context at start, creates spans
|
||||
* for content generation and editorial review. Feature-flag gated (default off).
|
||||
@@ -48,6 +55,27 @@ class PRAutoBlogger_Article_Worker {
|
||||
* @return array{generated: int, published: int, rejected: int, cost: float}
|
||||
*/
|
||||
public function generate( PRAutoBlogger_Article_Idea $idea ): array {
|
||||
$run_id = (string) ( $this->cost_tracker->get_run_id() ?? '' );
|
||||
$item = PRAutoBlogger_Run_Stage_State::item_key_for_idea( $idea );
|
||||
|
||||
$result = array(
|
||||
'generated' => 0,
|
||||
'published' => 0,
|
||||
'rejected' => 0,
|
||||
'cost' => 0.0,
|
||||
);
|
||||
|
||||
// Idempotent resume: this item already completed its publish stage
|
||||
// in this run — re-dispatching it must not create a second post or
|
||||
// charge a second cent.
|
||||
if ( '' !== $run_id && PRAutoBlogger_Run_Stage_State::is_done( $run_id, 'publish', '', $item ) ) {
|
||||
PRAutoBlogger_Logger::instance()->info(
|
||||
sprintf( 'Skipping "%s" — already completed in run %s (idempotent resume).', $idea->get_topic(), $run_id ),
|
||||
'pipeline'
|
||||
);
|
||||
return $result;
|
||||
}
|
||||
|
||||
// Initialize Opik trace context for this article generation.
|
||||
$opik = $this->should_trace_with_opik();
|
||||
if ( $opik ) {
|
||||
@@ -56,6 +84,7 @@ class PRAutoBlogger_Article_Worker {
|
||||
|
||||
$llm = new PRAutoBlogger_OpenRouter_Provider();
|
||||
$generator = new PRAutoBlogger_Content_Generator( $llm, $this->cost_tracker );
|
||||
$generator->set_run_item( '' !== $run_id ? $run_id : null, $item );
|
||||
$editor = new PRAutoBlogger_Chief_Editor( $llm, $this->cost_tracker );
|
||||
$publisher = new PRAutoBlogger_Publisher();
|
||||
|
||||
@@ -65,22 +94,16 @@ class PRAutoBlogger_Article_Worker {
|
||||
true
|
||||
);
|
||||
|
||||
$result = array(
|
||||
'generated' => 0,
|
||||
'published' => 0,
|
||||
'rejected' => 0,
|
||||
'cost' => 0.0,
|
||||
);
|
||||
|
||||
try {
|
||||
$this->broadcast_stage( __( 'Generating article draft via AI…', 'prautoblogger' ) );
|
||||
PRAutoBlogger_Pipeline_Status::broadcast( __( 'Generating article draft via AI…', 'prautoblogger' ) );
|
||||
$content = $generator->generate( $idea );
|
||||
$result['generated'] = 1;
|
||||
|
||||
$this->broadcast_stage( __( 'Running editorial pass…', 'prautoblogger' ) );
|
||||
$review = $editor->review( $content, $idea );
|
||||
PRAutoBlogger_Pipeline_Status::broadcast( __( 'Running editorial pass…', 'prautoblogger' ) );
|
||||
$review = $this->review_with_state( $editor, $content, $idea, $run_id, $item );
|
||||
|
||||
$this->broadcast_stage( __( 'Saving and publishing…', 'prautoblogger' ) );
|
||||
PRAutoBlogger_Pipeline_Status::broadcast( __( 'Saving and publishing…', 'prautoblogger' ) );
|
||||
PRAutoBlogger_Run_Stage_State::start( $run_id, 'publish', '', $item );
|
||||
$this->publish_or_draft(
|
||||
$content,
|
||||
$idea,
|
||||
@@ -89,7 +112,13 @@ class PRAutoBlogger_Article_Worker {
|
||||
$auto_publish,
|
||||
$result
|
||||
);
|
||||
PRAutoBlogger_Run_Stage_State::done( $run_id, 'publish', '', $item );
|
||||
|
||||
PRAutoBlogger_Audit_Writer::record_idea_sources( $run_id, $idea );
|
||||
} catch ( \Throwable $e ) {
|
||||
if ( '' !== $run_id ) {
|
||||
PRAutoBlogger_Run_Stage_State::fail_open_for_item( $run_id, $item );
|
||||
}
|
||||
PRAutoBlogger_Logger::instance()->error(
|
||||
sprintf(
|
||||
'Article generation %s for "%s": %s',
|
||||
@@ -111,6 +140,63 @@ class PRAutoBlogger_Article_Worker {
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Editorial review with stage state: a done review is reused from its
|
||||
* snapshot (never re-charged); a fresh review is checkpointed and its
|
||||
* verdict recorded to the run_decisions audit table.
|
||||
*
|
||||
* Side effects: run_stages upserts, one run_decisions insert, and —
|
||||
* when not resuming — one LLM call via Chief_Editor.
|
||||
*
|
||||
* @param PRAutoBlogger_Chief_Editor $editor Editor agent.
|
||||
* @param string $content Generated HTML content.
|
||||
* @param PRAutoBlogger_Article_Idea $idea The idea under review.
|
||||
* @param string $run_id Run UUID ('' outside a run).
|
||||
* @param string $item Stage item key for this idea.
|
||||
* @return PRAutoBlogger_Editorial_Review
|
||||
*/
|
||||
private function review_with_state(
|
||||
PRAutoBlogger_Chief_Editor $editor,
|
||||
string $content,
|
||||
PRAutoBlogger_Article_Idea $idea,
|
||||
string $run_id,
|
||||
string $item
|
||||
): PRAutoBlogger_Editorial_Review {
|
||||
if ( '' !== $run_id ) {
|
||||
$snapshot = PRAutoBlogger_Run_Stage_State::get_output( $run_id, 'review', '', $item );
|
||||
if ( null !== $snapshot ) {
|
||||
$data = json_decode( $snapshot, true );
|
||||
if ( is_array( $data ) && isset( $data['verdict'] ) ) {
|
||||
PRAutoBlogger_Logger::instance()->info(
|
||||
sprintf( 'Reusing completed review for "%s" (idempotent resume).', $idea->get_topic() ),
|
||||
'pipeline'
|
||||
);
|
||||
return new PRAutoBlogger_Editorial_Review( $data );
|
||||
}
|
||||
}
|
||||
PRAutoBlogger_Run_Stage_State::start( $run_id, 'review', '', $item );
|
||||
}
|
||||
|
||||
$review = $editor->review( $content, $idea );
|
||||
|
||||
if ( '' !== $run_id ) {
|
||||
$snapshot = (string) wp_json_encode(
|
||||
array(
|
||||
'verdict' => $review->get_verdict(),
|
||||
'notes' => $review->get_notes(),
|
||||
'revised_content' => $review->get_revised_content(),
|
||||
'quality_score' => $review->get_quality_score(),
|
||||
'seo_score' => $review->get_seo_score(),
|
||||
'issues' => $review->get_issues(),
|
||||
)
|
||||
);
|
||||
PRAutoBlogger_Run_Stage_State::done( $run_id, 'review', '', $item, $snapshot );
|
||||
PRAutoBlogger_Audit_Writer::record_decision( $run_id, 'review', $review->get_verdict(), $review->get_notes() );
|
||||
}
|
||||
|
||||
return $review;
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish approved content or save as draft.
|
||||
*
|
||||
@@ -153,21 +239,6 @@ class PRAutoBlogger_Article_Worker {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the generation status transient with the current stage.
|
||||
*
|
||||
* @param string $stage Human-readable stage description.
|
||||
*/
|
||||
private function broadcast_stage( string $stage ): void {
|
||||
$transient_key = 'prautoblogger_generation_status';
|
||||
$current = get_transient( $transient_key );
|
||||
if ( is_array( $current ) && 'running' === ( $current['status'] ?? '' ) ) {
|
||||
$current['stage'] = $stage;
|
||||
$current['last_updated'] = time();
|
||||
set_transient( $transient_key, $current, 600 );
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if Opik tracing is enabled and has credentials.
|
||||
*
|
||||
|
||||
132
includes/core/class-audit-writer.php
Normal file
132
includes/core/class-audit-writer.php
Normal file
@@ -0,0 +1,132 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* phpcs:ignore WordPress.Files.FileName.InvalidClassFileName -- class naming convention differs from WordPress standard
|
||||
*
|
||||
* Writes run-scoped audit rows: sources considered + stage decisions.
|
||||
*
|
||||
* What: Thin, self-healing insert layer over the v0.18.0 audit child
|
||||
* tables. `record_decision()` persists a stage verdict (Phase 1:
|
||||
* the chief editor's review verdict; Phase 2 adds curate/editorial/
|
||||
* seo decisions and citation_score). `record_idea_sources()`
|
||||
* consolidates which collected sources fed a generated article —
|
||||
* the run_sources replacement for the historical source_ids_json /
|
||||
* `_prautoblogger_research_sources` scatter — for NEW runs only (no
|
||||
* backfill). Every method no-ops when the tables are missing.
|
||||
* Who triggers it: Article_Worker (decision after review, sources after
|
||||
* publish), Phase-2 agents.
|
||||
* Dependencies: WordPress $wpdb, Run_State (availability probe pattern).
|
||||
*
|
||||
* @see core/class-article-worker.php — Phase-1 call sites.
|
||||
* @see ARCHITECTURE.md #21 — Audit substrate design.
|
||||
*/
|
||||
class PRAutoBlogger_Audit_Writer {
|
||||
|
||||
/** @var bool|null Per-request "audit tables exist" probe result. */
|
||||
private static ?bool $tables_ok = null;
|
||||
|
||||
/**
|
||||
* Whether the audit child tables exist. Cached per request; never throws.
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public static function is_available(): bool {
|
||||
if ( null !== self::$tables_ok ) {
|
||||
return self::$tables_ok;
|
||||
}
|
||||
global $wpdb;
|
||||
if ( null === $wpdb ) {
|
||||
return false; // Not cached: $wpdb may appear later in boot.
|
||||
}
|
||||
$table = $wpdb->prefix . 'prautoblogger_run_decisions';
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching
|
||||
$found = $wpdb->get_var( $wpdb->prepare( 'SHOW TABLES LIKE %s', $table ) );
|
||||
self::$tables_ok = ( $found === $table );
|
||||
return self::$tables_ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a stage decision for a run.
|
||||
*
|
||||
* @param string $run_id Run UUID.
|
||||
* @param string $stage Deciding stage (see Stage_Display_Map).
|
||||
* @param string $verdict e.g. 'approved', 'revised', 'rejected'.
|
||||
* @param string $rationale Why (editor notes, governor message, …).
|
||||
* @param float|null $citation_score Phase-2 citation score (null in Phase 1).
|
||||
* @return void
|
||||
*/
|
||||
public static function record_decision( string $run_id, string $stage, string $verdict, string $rationale = '', ?float $citation_score = null ): void {
|
||||
if ( '' === $run_id || ! self::is_available() ) {
|
||||
return;
|
||||
}
|
||||
global $wpdb;
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery
|
||||
$wpdb->insert(
|
||||
$wpdb->prefix . 'prautoblogger_run_decisions',
|
||||
array(
|
||||
'run_id' => $run_id,
|
||||
'stage' => $stage,
|
||||
'verdict' => $verdict,
|
||||
'rationale' => $rationale,
|
||||
'citation_score' => $citation_score,
|
||||
'created_at' => current_time( 'mysql' ),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record the collected sources that fed a generated article (kept = 1;
|
||||
* Phase-2 curation adds discard rows with reasons + quality scores).
|
||||
*
|
||||
* Resolves the idea's source_data ids to permalinks in one query.
|
||||
*
|
||||
* @param string $run_id Run UUID.
|
||||
* @param PRAutoBlogger_Article_Idea $idea The generated idea.
|
||||
* @return void
|
||||
*/
|
||||
public static function record_idea_sources( string $run_id, PRAutoBlogger_Article_Idea $idea ): void {
|
||||
if ( '' === $run_id || ! self::is_available() ) {
|
||||
return;
|
||||
}
|
||||
$source_ids = array_filter( array_map( 'intval', $idea->get_source_ids() ) );
|
||||
if ( empty( $source_ids ) ) {
|
||||
return;
|
||||
}
|
||||
|
||||
global $wpdb;
|
||||
$source_table = $wpdb->prefix . 'prautoblogger_source_data';
|
||||
$placeholders = implode( ',', array_fill( 0, count( $source_ids ), '%d' ) );
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching,WordPress.DB.PreparedSQL.NotPrepared
|
||||
$rows = $wpdb->get_results(
|
||||
// phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared -- $placeholders is a fixed %d list.
|
||||
$wpdb->prepare( "SELECT id, source_type, permalink FROM {$source_table} WHERE id IN ({$placeholders})", $source_ids ),
|
||||
ARRAY_A
|
||||
);
|
||||
if ( ! is_array( $rows ) ) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ( $rows as $row ) {
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery
|
||||
$wpdb->insert(
|
||||
$wpdb->prefix . 'prautoblogger_run_sources',
|
||||
array(
|
||||
'run_id' => $run_id,
|
||||
'agent_role' => 'analyst',
|
||||
'source_url' => (string) ( $row['permalink'] ?? '' ),
|
||||
'doi' => null,
|
||||
'kept' => 1,
|
||||
'reason' => sprintf( 'Cited by analyzer for "%s" (%s #%d)', $idea->get_topic(), (string) $row['source_type'], (int) $row['id'] ),
|
||||
'quality_score' => null,
|
||||
'created_at' => current_time( 'mysql' ),
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reset per-request caches (tests). */
|
||||
public static function flush_cache(): void {
|
||||
self::$tables_ok = null;
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,12 @@ class PRAutoBlogger_Content_Generator {
|
||||
private PRAutoBlogger_LLM_Provider_Interface $llm;
|
||||
private PRAutoBlogger_Cost_Tracker $cost_tracker;
|
||||
|
||||
/** @var string|null Run UUID for stage-state checkpointing (null = off). */
|
||||
private ?string $run_id = null;
|
||||
|
||||
/** @var string|null Stage item key scoping this article within the run. */
|
||||
private ?string $item_key = null;
|
||||
|
||||
public function __construct(
|
||||
PRAutoBlogger_LLM_Provider_Interface $llm,
|
||||
PRAutoBlogger_Cost_Tracker $cost_tracker
|
||||
@@ -27,6 +33,25 @@ class PRAutoBlogger_Content_Generator {
|
||||
$this->cost_tracker = $cost_tracker;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable per-stage state checkpointing for one article item (v0.18.0).
|
||||
*
|
||||
* When set, every LLM stage (outline/draft/polish or the single-pass
|
||||
* draft) records pending->running->done state and snapshots its output;
|
||||
* a re-entered done stage returns the snapshot WITHOUT a new LLM call
|
||||
* (never re-charged). When unset (default), behavior is exactly
|
||||
* pre-v0.18.0 — used by callers outside a run (e.g. eval mode).
|
||||
*
|
||||
* @param string|null $run_id Run UUID, or null to disable.
|
||||
* @param string|null $item_key Item key from Run_Stage_State::item_key_for_idea().
|
||||
* @return $this
|
||||
*/
|
||||
public function set_run_item( ?string $run_id, ?string $item_key ): self {
|
||||
$this->run_id = $run_id;
|
||||
$this->item_key = $item_key;
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a blog post from an article idea.
|
||||
*
|
||||
@@ -159,6 +184,19 @@ class PRAutoBlogger_Content_Generator {
|
||||
array $options,
|
||||
?string $prompt_key = null
|
||||
): string {
|
||||
// Idempotent resume: a done stage returns its snapshot, no LLM call.
|
||||
if ( null !== $this->run_id && null !== $this->item_key ) {
|
||||
$cached = PRAutoBlogger_Run_Stage_State::get_output( $this->run_id, $stage, '', $this->item_key );
|
||||
if ( null !== $cached ) {
|
||||
PRAutoBlogger_Logger::instance()->info(
|
||||
sprintf( 'Reusing completed %s stage output (idempotent resume).', $stage ),
|
||||
'pipeline'
|
||||
);
|
||||
return $cached;
|
||||
}
|
||||
PRAutoBlogger_Run_Stage_State::start( $this->run_id, $stage, '', $this->item_key );
|
||||
}
|
||||
|
||||
$ctx = PRAutoBlogger_Opik_Trace_Context::current();
|
||||
|
||||
$span_id = $ctx->start_span(
|
||||
@@ -209,6 +247,17 @@ class PRAutoBlogger_Content_Generator {
|
||||
$prompt_key
|
||||
);
|
||||
|
||||
if ( null !== $this->run_id && null !== $this->item_key ) {
|
||||
PRAutoBlogger_Run_Stage_State::done(
|
||||
$this->run_id,
|
||||
$stage,
|
||||
'',
|
||||
$this->item_key,
|
||||
(string) $response['content'],
|
||||
$this->llm->estimate_cost( $response['model'], (int) $response['prompt_tokens'], (int) $response['completion_tokens'] )
|
||||
);
|
||||
}
|
||||
|
||||
return $response['content'];
|
||||
}
|
||||
|
||||
|
||||
@@ -65,6 +65,7 @@ class PRAutoBlogger_Pipeline_Runner {
|
||||
|
||||
$ideas = $this->orchestrate( $cost_tracker );
|
||||
if ( empty( $ideas ) ) {
|
||||
PRAutoBlogger_Run_State::mark_status( (string) $cost_tracker->get_run_id(), 'done' );
|
||||
return array(
|
||||
'generated' => 0,
|
||||
'published' => 0,
|
||||
@@ -84,7 +85,8 @@ class PRAutoBlogger_Pipeline_Runner {
|
||||
} else {
|
||||
// Single-article run — amortize research costs now (1 article = full cost).
|
||||
PRAutoBlogger_Post_Assembler::amortize_research_costs( $cost_tracker->get_run_id() );
|
||||
$this->log_summary( $result );
|
||||
PRAutoBlogger_Pipeline_Status::log_summary( $result );
|
||||
PRAutoBlogger_Run_State::mark_status( (string) $cost_tracker->get_run_id(), 'done' );
|
||||
}
|
||||
|
||||
return $result;
|
||||
@@ -115,6 +117,7 @@ class PRAutoBlogger_Pipeline_Runner {
|
||||
'Budget limit reached. Aborting remaining queued articles.',
|
||||
'pipeline'
|
||||
);
|
||||
PRAutoBlogger_Run_State::mark_status( (string) $queue['run_id'], 'failed' );
|
||||
$this->finish_queue();
|
||||
return;
|
||||
}
|
||||
@@ -152,15 +155,16 @@ class PRAutoBlogger_Pipeline_Runner {
|
||||
|
||||
if ( ! empty( $queue['ideas'] ) ) {
|
||||
update_option( self::QUEUE_KEY, $queue, false );
|
||||
$this->update_queue_status( $queue );
|
||||
PRAutoBlogger_Pipeline_Status::update_queue_progress( $queue );
|
||||
$this->schedule_next();
|
||||
} else {
|
||||
// All articles done — amortize shared research costs across them.
|
||||
PRAutoBlogger_Post_Assembler::amortize_research_costs( $queue['run_id'] );
|
||||
$this->write_final_status( $queue['results'] );
|
||||
PRAutoBlogger_Pipeline_Status::write_final( $queue['results'] );
|
||||
delete_option( self::QUEUE_KEY );
|
||||
PRAutoBlogger_Generation_Lock::release();
|
||||
$this->log_summary( $queue['results'] );
|
||||
PRAutoBlogger_Pipeline_Status::log_summary( $queue['results'] );
|
||||
PRAutoBlogger_Run_State::mark_status( (string) $queue['run_id'], 'done' );
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,20 +179,26 @@ class PRAutoBlogger_Pipeline_Runner {
|
||||
private function orchestrate( PRAutoBlogger_Cost_Tracker $cost_tracker ): array {
|
||||
$target = absint( get_option( 'prautoblogger_daily_article_target', 1 ) );
|
||||
|
||||
$run_id = (string) $cost_tracker->get_run_id();
|
||||
|
||||
$enabled = json_decode( get_option( 'prautoblogger_enabled_sources', '["reddit"]' ), true );
|
||||
$source_labels = is_array( $enabled ) ? implode( ', ', $enabled ) : 'reddit';
|
||||
/* translators: %s is a comma-separated list of enabled source names, e.g. "reddit, llm_research". */
|
||||
$this->broadcast_stage( sprintf( __( 'Collecting sources from %s…', 'prautoblogger' ), $source_labels ) );
|
||||
PRAutoBlogger_Pipeline_Status::broadcast( sprintf( __( 'Collecting sources from %s…', 'prautoblogger' ), $source_labels ) );
|
||||
PRAutoBlogger_Run_Stage_State::start( $run_id, 'research' );
|
||||
( new PRAutoBlogger_Source_Collector() )
|
||||
->set_cost_tracker( $cost_tracker )
|
||||
->collect_from_all_sources();
|
||||
PRAutoBlogger_Run_Stage_State::done( $run_id, 'research' );
|
||||
|
||||
$this->broadcast_stage( __( 'Analyzing topics and scoring…', 'prautoblogger' ) );
|
||||
PRAutoBlogger_Pipeline_Status::broadcast( __( 'Analyzing topics and scoring…', 'prautoblogger' ) );
|
||||
PRAutoBlogger_Run_Stage_State::start( $run_id, 'analysis' );
|
||||
$llm = new PRAutoBlogger_OpenRouter_Provider();
|
||||
$analyzer = new PRAutoBlogger_Content_Analyzer( $llm, $cost_tracker );
|
||||
$analysis = $analyzer->analyze_recent_data( max( $target * 2, 6 ) );
|
||||
PRAutoBlogger_Run_Stage_State::done( $run_id, 'analysis' );
|
||||
|
||||
$this->broadcast_stage( __( 'Selecting best topic…', 'prautoblogger' ) );
|
||||
PRAutoBlogger_Pipeline_Status::broadcast( __( 'Selecting best topic…', 'prautoblogger' ) );
|
||||
$scorer = new PRAutoBlogger_Idea_Scorer();
|
||||
$scorer->set_skip_dedup( $this->skip_dedup );
|
||||
$ideas = $scorer->score_and_rank( $analysis, $target );
|
||||
@@ -223,7 +233,7 @@ class PRAutoBlogger_Pipeline_Runner {
|
||||
);
|
||||
|
||||
update_option( self::QUEUE_KEY, $queue, false );
|
||||
$this->update_queue_status( $queue );
|
||||
PRAutoBlogger_Pipeline_Status::update_queue_progress( $queue );
|
||||
$this->schedule_next();
|
||||
|
||||
PRAutoBlogger_Logger::instance()->info(
|
||||
@@ -248,71 +258,13 @@ class PRAutoBlogger_Pipeline_Runner {
|
||||
);
|
||||
}
|
||||
|
||||
/** Update the frontend status transient with queue progress. */
|
||||
private function update_queue_status( array $queue ): void {
|
||||
$done = $queue['results']['generated'] + $queue['results']['rejected'];
|
||||
$total = $done + count( $queue['ideas'] );
|
||||
|
||||
$current = get_transient( 'prautoblogger_generation_status' );
|
||||
set_transient(
|
||||
'prautoblogger_generation_status',
|
||||
array(
|
||||
'status' => 'running',
|
||||
'stage' => sprintf( __( 'Generating article %1$d of %2$d…', 'prautoblogger' ), $done + 1, $total ),
|
||||
'started' => is_array( $current ) ? ( $current['started'] ?? time() ) : time(),
|
||||
'last_updated' => time(),
|
||||
),
|
||||
600
|
||||
);
|
||||
}
|
||||
|
||||
/** Write the final "complete" status transient. */
|
||||
private function write_final_status( array $r ): void {
|
||||
set_transient(
|
||||
'prautoblogger_generation_status',
|
||||
array(
|
||||
'status' => 'complete',
|
||||
'generated' => $r['generated'],
|
||||
'published' => $r['published'],
|
||||
'rejected' => $r['rejected'],
|
||||
'cost' => $r['cost'],
|
||||
),
|
||||
600
|
||||
);
|
||||
}
|
||||
|
||||
/** Clean up a stale or empty queue. */
|
||||
private function finish_queue(): void {
|
||||
$queue = get_option( self::QUEUE_KEY );
|
||||
if ( is_array( $queue ) && isset( $queue['results'] ) ) {
|
||||
$this->write_final_status( $queue['results'] );
|
||||
PRAutoBlogger_Pipeline_Status::write_final( $queue['results'] );
|
||||
}
|
||||
delete_option( self::QUEUE_KEY );
|
||||
PRAutoBlogger_Generation_Lock::release();
|
||||
}
|
||||
|
||||
/** Log the pipeline summary. */
|
||||
private function log_summary( array $r ): void {
|
||||
PRAutoBlogger_Logger::instance()->info(
|
||||
sprintf(
|
||||
'Pipeline complete: %d generated, %d published, %d rejected. Cost: $%.4f',
|
||||
$r['generated'],
|
||||
$r['published'],
|
||||
$r['rejected'],
|
||||
$r['cost']
|
||||
),
|
||||
'pipeline'
|
||||
);
|
||||
}
|
||||
|
||||
/** Update the generation status transient with the current stage. */
|
||||
private function broadcast_stage( string $stage ): void {
|
||||
$key = 'prautoblogger_generation_status';
|
||||
$current = get_transient( $key );
|
||||
if ( is_array( $current ) && 'running' === ( $current['status'] ?? '' ) ) {
|
||||
$current['stage'] = $stage;
|
||||
$current['last_updated'] = time();
|
||||
set_transient( $key, $current, 600 );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
106
includes/core/class-pipeline-status.php
Normal file
106
includes/core/class-pipeline-status.php
Normal file
@@ -0,0 +1,106 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* phpcs:ignore WordPress.Files.FileName.InvalidClassFileName -- class naming convention differs from WordPress standard
|
||||
*
|
||||
* Frontend status-transient + summary-log helpers for pipeline runs.
|
||||
*
|
||||
* What: The generation-status transient plumbing (stage broadcast, queue
|
||||
* progress, final result) and the pipeline summary log line,
|
||||
* extracted verbatim from Pipeline_Runner / Article_Worker in
|
||||
* v0.18.0 (300-line cap; the two classes carried duplicate
|
||||
* broadcast helpers). Pure presentation/state — no business logic.
|
||||
* Who triggers it: Pipeline_Runner, Article_Worker.
|
||||
* Dependencies: WordPress transients, PRAutoBlogger_Logger.
|
||||
*
|
||||
* @see core/class-pipeline-runner.php — Queue progress + final status writes.
|
||||
* @see core/class-article-worker.php — Per-stage broadcasts.
|
||||
* @see class-executor.php — Poller that reads this transient.
|
||||
*/
|
||||
class PRAutoBlogger_Pipeline_Status {
|
||||
|
||||
/** Transient key polled by the admin frontend. */
|
||||
private const KEY = 'prautoblogger_generation_status';
|
||||
|
||||
/** Seconds the status transient stays readable. */
|
||||
private const TTL = 600;
|
||||
|
||||
/**
|
||||
* Update the live stage text (only while a run is broadcasting).
|
||||
*
|
||||
* @param string $stage Human-readable stage description.
|
||||
* @return void
|
||||
*/
|
||||
public static function broadcast( string $stage ): void {
|
||||
$current = get_transient( self::KEY );
|
||||
if ( is_array( $current ) && 'running' === ( $current['status'] ?? '' ) ) {
|
||||
$current['stage'] = $stage;
|
||||
$current['last_updated'] = time();
|
||||
set_transient( self::KEY, $current, self::TTL );
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update queue progress ("Generating article X of Y…").
|
||||
*
|
||||
* @param array $queue Queue array with 'results' + 'ideas'.
|
||||
* @return void
|
||||
*/
|
||||
public static function update_queue_progress( array $queue ): void {
|
||||
$done = $queue['results']['generated'] + $queue['results']['rejected'];
|
||||
$total = $done + count( $queue['ideas'] );
|
||||
|
||||
$current = get_transient( self::KEY );
|
||||
set_transient(
|
||||
self::KEY,
|
||||
array(
|
||||
'status' => 'running',
|
||||
/* translators: 1: current article number, 2: total articles. */
|
||||
'stage' => sprintf( __( 'Generating article %1$d of %2$d…', 'prautoblogger' ), $done + 1, $total ),
|
||||
'started' => is_array( $current ) ? ( $current['started'] ?? time() ) : time(),
|
||||
'last_updated' => time(),
|
||||
),
|
||||
self::TTL
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the final "complete" status.
|
||||
*
|
||||
* @param array $r Result counters (generated/published/rejected/cost).
|
||||
* @return void
|
||||
*/
|
||||
public static function write_final( array $r ): void {
|
||||
set_transient(
|
||||
self::KEY,
|
||||
array(
|
||||
'status' => 'complete',
|
||||
'generated' => $r['generated'],
|
||||
'published' => $r['published'],
|
||||
'rejected' => $r['rejected'],
|
||||
'cost' => $r['cost'],
|
||||
),
|
||||
self::TTL
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log the pipeline summary line.
|
||||
*
|
||||
* @param array $r Result counters (generated/published/rejected/cost).
|
||||
* @return void
|
||||
*/
|
||||
public static function log_summary( array $r ): void {
|
||||
PRAutoBlogger_Logger::instance()->info(
|
||||
sprintf(
|
||||
'Pipeline complete: %d generated, %d published, %d rejected. Cost: $%.4f',
|
||||
$r['generated'],
|
||||
$r['published'],
|
||||
$r['rejected'],
|
||||
$r['cost']
|
||||
),
|
||||
'pipeline'
|
||||
);
|
||||
}
|
||||
}
|
||||
57
includes/core/class-publisher.php
Executable file → Normal file
57
includes/core/class-publisher.php
Executable file → Normal file
@@ -82,6 +82,22 @@ class PRAutoBlogger_Publisher {
|
||||
?string $run_id = null,
|
||||
?PRAutoBlogger_Cost_Tracker $cost_tracker = null
|
||||
): int {
|
||||
// v0.18.0 idempotency: post creation is keyed by run + idea
|
||||
// (check-before-insert) so a retried/resumed run cannot create a
|
||||
// duplicate post. One run_id spans all N articles of a batch, so
|
||||
// the idea hash disambiguates within the run.
|
||||
$idea_hash = PRAutoBlogger_Run_Stage_State::idea_hash( $idea );
|
||||
if ( null !== $run_id && '' !== $run_id ) {
|
||||
$existing_id = $this->find_existing_post( $run_id, $idea_hash );
|
||||
if ( $existing_id > 0 ) {
|
||||
PRAutoBlogger_Logger::instance()->info(
|
||||
sprintf( 'Post for run %s / idea %s already exists (ID=%d) — skipping duplicate insert.', $run_id, $idea_hash, $existing_id ),
|
||||
'publisher'
|
||||
);
|
||||
return $existing_id;
|
||||
}
|
||||
}
|
||||
|
||||
// Clean LLM artifacts, then inject peptide hyperlinks deterministically
|
||||
// before the content enters WordPress. Peptide linker no-ops gracefully
|
||||
// if PR Core is not active.
|
||||
@@ -94,7 +110,7 @@ class PRAutoBlogger_Publisher {
|
||||
'post_status' => $post_status,
|
||||
'post_type' => 'post',
|
||||
'post_author' => PRAutoBlogger_Post_Assembler::get_default_author_id(),
|
||||
'meta_input' => $this->build_meta( $idea, $review, $run_id ),
|
||||
'meta_input' => $this->build_meta( $idea, $review, $run_id, $idea_hash ),
|
||||
);
|
||||
|
||||
/** @see class-prautoblogger.php — listeners registered in main loader. */
|
||||
@@ -140,7 +156,8 @@ class PRAutoBlogger_Publisher {
|
||||
private function build_meta(
|
||||
PRAutoBlogger_Article_Idea $idea,
|
||||
PRAutoBlogger_Editorial_Review $review,
|
||||
?string $run_id = null
|
||||
?string $run_id = null,
|
||||
string $idea_hash = ''
|
||||
): array {
|
||||
$meta = array(
|
||||
'_prautoblogger_generated' => '1',
|
||||
@@ -160,6 +177,42 @@ class PRAutoBlogger_Publisher {
|
||||
if ( null !== $run_id && '' !== $run_id ) {
|
||||
$meta['_prautoblogger_run_id'] = $run_id;
|
||||
}
|
||||
if ( '' !== $idea_hash ) {
|
||||
$meta['_prautoblogger_idea_hash'] = $idea_hash;
|
||||
}
|
||||
return $meta;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a post already created for this (run, idea) pair.
|
||||
*
|
||||
* Direct postmeta self-join (any post_status, including trash) — the
|
||||
* idempotency check must see drafts and pending posts too.
|
||||
*
|
||||
* @param string $run_id Pipeline run UUID.
|
||||
* @param string $idea_hash Idea hash from Run_Stage_State::idea_hash().
|
||||
* @return int Existing post ID, or 0 when none.
|
||||
*/
|
||||
private function find_existing_post( string $run_id, string $idea_hash ): int {
|
||||
global $wpdb;
|
||||
if ( null === $wpdb ) {
|
||||
return 0;
|
||||
}
|
||||
$postmeta = $wpdb->prefix . 'postmeta';
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching
|
||||
$post_id = $wpdb->get_var(
|
||||
$wpdb->prepare(
|
||||
"SELECT pm_run.post_id FROM {$postmeta} pm_run
|
||||
INNER JOIN {$postmeta} pm_hash
|
||||
ON pm_hash.post_id = pm_run.post_id
|
||||
AND pm_hash.meta_key = '_prautoblogger_idea_hash'
|
||||
AND pm_hash.meta_value = %s
|
||||
WHERE pm_run.meta_key = '_prautoblogger_run_id' AND pm_run.meta_value = %s
|
||||
LIMIT 1",
|
||||
$idea_hash,
|
||||
$run_id
|
||||
)
|
||||
);
|
||||
return null !== $post_id ? (int) $post_id : 0;
|
||||
}
|
||||
}
|
||||
|
||||
270
includes/core/class-run-reaper.php
Normal file
270
includes/core/class-run-reaper.php
Normal file
@@ -0,0 +1,270 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* phpcs:ignore WordPress.Files.FileName.InvalidClassFileName -- class naming convention differs from WordPress standard
|
||||
*
|
||||
* Daily sweep for stuck runs + audit-payload retention (extends #19).
|
||||
*
|
||||
* What: Rides the EXISTING `prautoblogger_reap_orphan_research_rows` cron
|
||||
* (no new schedule) alongside the v0.8.1 Research_Reaper. Three jobs:
|
||||
* 1. Stage sweep — run_stages rows stuck `running` longer than
|
||||
* 2× the stage's expected wall-clock are marked `failed`
|
||||
* (Hostinger kills PHP at ~120s; a stage "running" for 10+
|
||||
* minutes is dead, its process gone).
|
||||
* 2. Run sweep — runs rows still open (pending/running) with no
|
||||
* ledger/state activity for 2× the expected run wall-clock are
|
||||
* marked `failed` and their non-done stages reaped to `failed`;
|
||||
* such runs render as "incomplete" via incomplete_runs().
|
||||
* 3. Retention — `request_json` on generation_log and stage output
|
||||
* payloads (`meta_json` on run_stages) are NULLed after R days;
|
||||
* R comes from the `prautoblogger_request_json_retention_days`
|
||||
* SETTING (default constant 14; 0 = keep forever) — never a
|
||||
* literal at the point of use.
|
||||
* Who triggers it: WP-Cron `prautoblogger_reap_orphan_research_rows`
|
||||
* (daily 03:15, registered in class-prautoblogger.php).
|
||||
* Dependencies: Run_State, Run_Stage_State, WordPress $wpdb, Logger.
|
||||
*
|
||||
* @see core/class-research-reaper.php — The #19 orphan-cost reaper (same cron).
|
||||
* @see core/class-run-state.php — Run rows swept here.
|
||||
* @see ARCHITECTURE.md #21 — Design rationale.
|
||||
*/
|
||||
class PRAutoBlogger_Run_Reaper {
|
||||
|
||||
/**
|
||||
* Expected per-stage wall-clock seconds (sweep threshold = 2×). An
|
||||
* engineering constant (like Research_Reaper::GRACE_WINDOW_SECONDS),
|
||||
* overridable per stage via the
|
||||
* `prautoblogger_filter_stage_expected_seconds` filter.
|
||||
*/
|
||||
private const EXPECTED_STAGE_SECONDS = 300;
|
||||
|
||||
/**
|
||||
* Expected whole-run wall-clock seconds (collect + analyze + up to 10
|
||||
* chained articles ≈ 30 min worst case). Sweep threshold = 2×.
|
||||
* Overridable via `prautoblogger_filter_expected_run_seconds`.
|
||||
*/
|
||||
private const EXPECTED_RUN_SECONDS = 1800;
|
||||
|
||||
/**
|
||||
* Cron action handler — entrypoint. Pure delegate, never throws.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public static function on_cron(): void {
|
||||
self::reap();
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all three sweeps. Returns a summary (cron path ignores it).
|
||||
*
|
||||
* Side effects: UPDATEs on run_stages / runs / generation_log; INFO
|
||||
* log lines for swept runs.
|
||||
*
|
||||
* @return array{stages_failed: int, runs_failed: int, payloads_pruned: int}
|
||||
*/
|
||||
public static function reap(): array {
|
||||
$stats = array(
|
||||
'stages_failed' => 0,
|
||||
'runs_failed' => 0,
|
||||
'payloads_pruned' => 0,
|
||||
);
|
||||
|
||||
try {
|
||||
$stats['stages_failed'] = self::sweep_stuck_stages();
|
||||
$stats['runs_failed'] = self::sweep_stuck_runs();
|
||||
$stats['payloads_pruned'] = self::prune_payloads();
|
||||
} catch ( \Throwable $e ) {
|
||||
PRAutoBlogger_Logger::instance()->warning(
|
||||
'Run reaper encountered an error: ' . $e->getMessage(),
|
||||
'run-reaper'
|
||||
);
|
||||
}
|
||||
|
||||
return $stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs that died mid-pipeline, for audit surfaces: failed/halted runs
|
||||
* that still have non-done stage rows ("incomplete").
|
||||
*
|
||||
* @param int $limit Max rows (default 20).
|
||||
* @return array<int, array<string, mixed>>
|
||||
*/
|
||||
public static function incomplete_runs( int $limit = 20 ): array {
|
||||
if ( ! PRAutoBlogger_Run_State::is_available() || ! PRAutoBlogger_Run_Stage_State::is_available() ) {
|
||||
return array();
|
||||
}
|
||||
global $wpdb;
|
||||
$runs = PRAutoBlogger_Run_State::table_name();
|
||||
$stages = PRAutoBlogger_Run_Stage_State::table_name();
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching
|
||||
$rows = $wpdb->get_results(
|
||||
$wpdb->prepare(
|
||||
"SELECT r.* FROM {$runs} r
|
||||
WHERE r.status IN ('failed','halted')
|
||||
AND EXISTS (SELECT 1 FROM {$stages} s WHERE s.run_id = r.run_id AND s.status != 'done')
|
||||
ORDER BY r.updated_at DESC LIMIT %d",
|
||||
$limit
|
||||
),
|
||||
ARRAY_A
|
||||
);
|
||||
return is_array( $rows ) ? $rows : array();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage sweep: `running` rows older than 2× expected stage wall-clock
|
||||
* become `failed`.
|
||||
*
|
||||
* @return int Rows swept.
|
||||
*/
|
||||
private static function sweep_stuck_stages(): int {
|
||||
if ( ! PRAutoBlogger_Run_Stage_State::is_available() ) {
|
||||
return 0;
|
||||
}
|
||||
global $wpdb;
|
||||
$table = PRAutoBlogger_Run_Stage_State::table_name();
|
||||
|
||||
/**
|
||||
* Filter the expected per-stage wall-clock seconds used by the
|
||||
* stuck-stage sweep (threshold is 2× this value).
|
||||
*
|
||||
* @param int $seconds Expected seconds (default 300).
|
||||
*/
|
||||
$expected = (int) apply_filters( 'prautoblogger_filter_stage_expected_seconds', self::EXPECTED_STAGE_SECONDS );
|
||||
$cutoff = gmdate( 'Y-m-d H:i:s', time() - ( 2 * $expected ) );
|
||||
$now = current_time( 'mysql' );
|
||||
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery
|
||||
$swept = $wpdb->query(
|
||||
$wpdb->prepare(
|
||||
"UPDATE {$table} SET status = 'failed', updated_at = %s, finished_at = %s
|
||||
WHERE status = 'running' AND updated_at < %s",
|
||||
$now,
|
||||
$now,
|
||||
$cutoff
|
||||
)
|
||||
);
|
||||
return is_numeric( $swept ) ? (int) $swept : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run sweep: open runs with no activity for 2× expected run wall-clock
|
||||
* are marked `failed`; their pending/running stages are reaped.
|
||||
*
|
||||
* @return int Runs swept.
|
||||
*/
|
||||
private static function sweep_stuck_runs(): int {
|
||||
if ( ! PRAutoBlogger_Run_State::is_available() ) {
|
||||
return 0;
|
||||
}
|
||||
global $wpdb;
|
||||
$runs = PRAutoBlogger_Run_State::table_name();
|
||||
|
||||
/**
|
||||
* Filter the expected whole-run wall-clock seconds used by the
|
||||
* stuck-run sweep (threshold is 2× this value).
|
||||
*
|
||||
* @param int $seconds Expected seconds (default 1800).
|
||||
*/
|
||||
$expected = (int) apply_filters( 'prautoblogger_filter_expected_run_seconds', self::EXPECTED_RUN_SECONDS );
|
||||
$cutoff = gmdate( 'Y-m-d H:i:s', time() - ( 2 * $expected ) );
|
||||
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching
|
||||
$stuck = $wpdb->get_col(
|
||||
$wpdb->prepare(
|
||||
"SELECT run_id FROM {$runs} WHERE status IN ('pending','running') AND updated_at < %s",
|
||||
$cutoff
|
||||
)
|
||||
);
|
||||
if ( ! is_array( $stuck ) || empty( $stuck ) ) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$swept = 0;
|
||||
foreach ( $stuck as $run_id ) {
|
||||
$run_id = (string) $run_id;
|
||||
if ( ! PRAutoBlogger_Run_State::mark_status( $run_id, 'failed' ) ) {
|
||||
continue;
|
||||
}
|
||||
self::reap_open_stages( $run_id );
|
||||
PRAutoBlogger_Logger::instance()->info(
|
||||
sprintf( 'Run %s stuck past 2x expected wall-clock — marked failed (renders as incomplete).', $run_id ),
|
||||
'run-reaper'
|
||||
);
|
||||
++$swept;
|
||||
}
|
||||
return $swept;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a failed run's pending/running stage rows as failed.
|
||||
*
|
||||
* @param string $run_id Run UUID.
|
||||
* @return void
|
||||
*/
|
||||
private static function reap_open_stages( string $run_id ): void {
|
||||
if ( ! PRAutoBlogger_Run_Stage_State::is_available() ) {
|
||||
return;
|
||||
}
|
||||
global $wpdb;
|
||||
$table = PRAutoBlogger_Run_Stage_State::table_name();
|
||||
$now = current_time( 'mysql' );
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery
|
||||
$wpdb->query(
|
||||
$wpdb->prepare(
|
||||
"UPDATE {$table} SET status = 'failed', updated_at = %s, finished_at = %s
|
||||
WHERE run_id = %s AND status IN ('pending','running')",
|
||||
$now,
|
||||
$now,
|
||||
$run_id
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retention: NULL heavy payloads older than R days. R is read from the
|
||||
* `prautoblogger_request_json_retention_days` setting at every call
|
||||
* (default = PRAUTOBLOGGER_DEFAULT_REQUEST_JSON_RETENTION_DAYS;
|
||||
* 0 or negative = keep forever).
|
||||
*
|
||||
* @return int Rows pruned (both tables combined).
|
||||
*/
|
||||
private static function prune_payloads(): int {
|
||||
$days = (int) get_option(
|
||||
'prautoblogger_request_json_retention_days',
|
||||
PRAUTOBLOGGER_DEFAULT_REQUEST_JSON_RETENTION_DAYS
|
||||
);
|
||||
if ( $days <= 0 ) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
global $wpdb;
|
||||
$cutoff = gmdate( 'Y-m-d H:i:s', time() - ( $days * DAY_IN_SECONDS ) );
|
||||
$pruned = 0;
|
||||
|
||||
$gen_log = $wpdb->prefix . 'prautoblogger_generation_log';
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery
|
||||
$result = $wpdb->query(
|
||||
$wpdb->prepare(
|
||||
"UPDATE {$gen_log} SET request_json = NULL WHERE request_json IS NOT NULL AND created_at < %s",
|
||||
$cutoff
|
||||
)
|
||||
);
|
||||
$pruned += is_numeric( $result ) ? (int) $result : 0;
|
||||
|
||||
if ( PRAutoBlogger_Run_Stage_State::is_available() ) {
|
||||
$stages = PRAutoBlogger_Run_Stage_State::table_name();
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery
|
||||
$result = $wpdb->query(
|
||||
$wpdb->prepare(
|
||||
"UPDATE {$stages} SET meta_json = NULL WHERE meta_json IS NOT NULL AND updated_at < %s",
|
||||
$cutoff
|
||||
)
|
||||
);
|
||||
$pruned += is_numeric( $result ) ? (int) $result : 0;
|
||||
}
|
||||
|
||||
return $pruned;
|
||||
}
|
||||
}
|
||||
293
includes/core/class-run-stage-state.php
Normal file
293
includes/core/class-run-stage-state.php
Normal file
@@ -0,0 +1,293 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* phpcs:ignore WordPress.Files.FileName.InvalidClassFileName -- class naming convention differs from WordPress standard
|
||||
*
|
||||
* Per-run per-stage state machine over `wp_prautoblogger_run_stages`.
|
||||
*
|
||||
* What: One row per (run_id, stage, agent_role, item_key) holding
|
||||
* pending|running|done|failed|halted plus the stage's output
|
||||
* snapshot (meta_json). This is what makes resume idempotent on the
|
||||
* chained-cron architecture: a `done` stage is NEVER re-run or
|
||||
* re-charged — re-entry reuses its stored output; `done` is sticky
|
||||
* (an upsert cannot demote it). agent_role is the Phase-2 fan-out
|
||||
* dimension (quorum members; the quorum logic itself is Phase 2);
|
||||
* item_key scopes article-level stages because one run_id spans all
|
||||
* N articles of a batch run ('' for run-level stages). Self-healing:
|
||||
* a missing table degrades every method to a no-op/false.
|
||||
* Who triggers it: Pipeline_Runner (run-level research/analysis),
|
||||
* Article_Worker (per-article generate/review/publish),
|
||||
* Content_Generator (per-LLM-stage outline/draft/polish),
|
||||
* Run_Reaper (stuck-stage sweep + payload pruning).
|
||||
* Dependencies: WordPress $wpdb, wp_json_encode.
|
||||
*
|
||||
* @see core/class-run-state.php — Run-level lifecycle + ledger row.
|
||||
* @see core/class-run-reaper.php — Sweeps stages stuck in 'running'.
|
||||
* @see ARCHITECTURE.md #21 — Idempotency-key design.
|
||||
*/
|
||||
class PRAutoBlogger_Run_Stage_State {
|
||||
|
||||
/** @var bool|null Per-request "run_stages table exists" probe result. */
|
||||
private static ?bool $table_ok = null;
|
||||
|
||||
/** Fully-qualified run_stages table name. */
|
||||
public static function table_name(): string {
|
||||
global $wpdb;
|
||||
return $wpdb->prefix . 'prautoblogger_run_stages';
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the run_stages table exists and is queryable. Cached per
|
||||
* request; never throws.
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public static function is_available(): bool {
|
||||
if ( null !== self::$table_ok ) {
|
||||
return self::$table_ok;
|
||||
}
|
||||
global $wpdb;
|
||||
if ( null === $wpdb ) {
|
||||
return false; // Not cached: $wpdb may appear later in boot.
|
||||
}
|
||||
$table = self::table_name();
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching
|
||||
$found = $wpdb->get_var( $wpdb->prepare( 'SHOW TABLES LIKE %s', $table ) );
|
||||
self::$table_ok = ( $found === $table );
|
||||
return self::$table_ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable hash identifying one article idea within a run (post-creation
|
||||
* idempotency + stage item scoping both key off this).
|
||||
*
|
||||
* @param PRAutoBlogger_Article_Idea $idea The idea.
|
||||
* @return string 16-hex-char hash.
|
||||
*/
|
||||
public static function idea_hash( PRAutoBlogger_Article_Idea $idea ): string {
|
||||
return substr( md5( $idea->get_suggested_title() . '|' . $idea->get_topic() ), 0, 16 );
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage item_key for an article idea ('' is reserved for run-level stages).
|
||||
*
|
||||
* @param PRAutoBlogger_Article_Idea $idea The idea.
|
||||
* @return string e.g. 'idea:1a2b3c4d5e6f7a8b'.
|
||||
*/
|
||||
public static function item_key_for_idea( PRAutoBlogger_Article_Idea $idea ): string {
|
||||
return 'idea:' . self::idea_hash( $idea );
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a stage as entered (running). Upsert: first entry inserts the
|
||||
* row; re-entry bumps `attempt` — but a `done` stage is sticky and is
|
||||
* neither demoted nor re-counted.
|
||||
*
|
||||
* Side effects: one INSERT … ON DUPLICATE KEY UPDATE.
|
||||
*
|
||||
* @param string $run_id Run UUID.
|
||||
* @param string $stage Stage name (see Stage_Display_Map).
|
||||
* @param string $agent_role Fan-out dimension ('' default in Phase 1).
|
||||
* @param string $item_key Article scope ('' for run-level stages).
|
||||
* @return void
|
||||
*/
|
||||
public static function start( string $run_id, string $stage, string $agent_role = '', string $item_key = '' ): void {
|
||||
if ( '' === $run_id || ! self::is_available() ) {
|
||||
return;
|
||||
}
|
||||
global $wpdb;
|
||||
$table = self::table_name();
|
||||
$now = current_time( 'mysql' );
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery
|
||||
$wpdb->query(
|
||||
$wpdb->prepare(
|
||||
"INSERT INTO {$table}
|
||||
(run_id, stage, agent_role, item_key, status, attempt, started_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, 'running', 1, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
attempt = IF(status = 'done', attempt, attempt + 1),
|
||||
status = IF(status = 'done', 'done', 'running'),
|
||||
updated_at = VALUES(updated_at)",
|
||||
$run_id,
|
||||
$stage,
|
||||
$agent_role,
|
||||
$item_key,
|
||||
$now,
|
||||
$now
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a stage done, persisting its output for resume-without-recharge.
|
||||
*
|
||||
* Side effects: one INSERT … ON DUPLICATE KEY UPDATE.
|
||||
*
|
||||
* @param string $run_id Run UUID.
|
||||
* @param string $stage Stage name.
|
||||
* @param string $agent_role Fan-out dimension.
|
||||
* @param string $item_key Article scope.
|
||||
* @param string|null $output Stage output to snapshot (pruned after R days).
|
||||
* @param float $cost_usd Cost attributed to the stage.
|
||||
* @return void
|
||||
*/
|
||||
public static function done( string $run_id, string $stage, string $agent_role = '', string $item_key = '', ?string $output = null, float $cost_usd = 0.0 ): void {
|
||||
if ( '' === $run_id || ! self::is_available() ) {
|
||||
return;
|
||||
}
|
||||
global $wpdb;
|
||||
$table = self::table_name();
|
||||
$now = current_time( 'mysql' );
|
||||
$meta = null !== $output ? wp_json_encode( array( 'output' => $output ) ) : null;
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery
|
||||
$wpdb->query(
|
||||
$wpdb->prepare(
|
||||
"INSERT INTO {$table}
|
||||
(run_id, stage, agent_role, item_key, status, attempt, cost_usd, meta_json, started_at, updated_at, finished_at)
|
||||
VALUES (%s, %s, %s, %s, 'done', 1, %f, %s, %s, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
status = 'done',
|
||||
cost_usd = VALUES(cost_usd),
|
||||
meta_json = VALUES(meta_json),
|
||||
updated_at = VALUES(updated_at),
|
||||
finished_at = VALUES(finished_at)",
|
||||
$run_id,
|
||||
$stage,
|
||||
$agent_role,
|
||||
$item_key,
|
||||
$cost_usd,
|
||||
$meta,
|
||||
$now,
|
||||
$now,
|
||||
$now
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a stage failed (does not demote a `done` stage).
|
||||
*
|
||||
* @param string $run_id Run UUID.
|
||||
* @param string $stage Stage name.
|
||||
* @param string $agent_role Fan-out dimension.
|
||||
* @param string $item_key Article scope.
|
||||
* @return void
|
||||
*/
|
||||
public static function fail( string $run_id, string $stage, string $agent_role = '', string $item_key = '' ): void {
|
||||
if ( '' === $run_id || ! self::is_available() ) {
|
||||
return;
|
||||
}
|
||||
global $wpdb;
|
||||
$table = self::table_name();
|
||||
$now = current_time( 'mysql' );
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery
|
||||
$wpdb->query(
|
||||
$wpdb->prepare(
|
||||
"UPDATE {$table} SET status = 'failed', updated_at = %s, finished_at = %s
|
||||
WHERE run_id = %s AND stage = %s AND agent_role = %s AND item_key = %s AND status != 'done'",
|
||||
$now,
|
||||
$now,
|
||||
$run_id,
|
||||
$stage,
|
||||
$agent_role,
|
||||
$item_key
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch one stage row.
|
||||
*
|
||||
* @param string $run_id Run UUID.
|
||||
* @param string $stage Stage name.
|
||||
* @param string $agent_role Fan-out dimension.
|
||||
* @param string $item_key Article scope.
|
||||
* @return array<string, mixed>|null Row or null.
|
||||
*/
|
||||
public static function get( string $run_id, string $stage, string $agent_role = '', string $item_key = '' ): ?array {
|
||||
if ( '' === $run_id || ! self::is_available() ) {
|
||||
return null;
|
||||
}
|
||||
global $wpdb;
|
||||
$table = self::table_name();
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.DirectDatabaseQuery.NoCaching
|
||||
$row = $wpdb->get_row(
|
||||
$wpdb->prepare(
|
||||
"SELECT * FROM {$table} WHERE run_id = %s AND stage = %s AND agent_role = %s AND item_key = %s",
|
||||
$run_id,
|
||||
$stage,
|
||||
$agent_role,
|
||||
$item_key
|
||||
),
|
||||
ARRAY_A
|
||||
);
|
||||
return is_array( $row ) ? $row : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a stage already completed (resume entry-point check).
|
||||
*
|
||||
* @param string $run_id Run UUID.
|
||||
* @param string $stage Stage name.
|
||||
* @param string $agent_role Fan-out dimension.
|
||||
* @param string $item_key Article scope.
|
||||
* @return bool
|
||||
*/
|
||||
public static function is_done( string $run_id, string $stage, string $agent_role = '', string $item_key = '' ): bool {
|
||||
$row = self::get( $run_id, $stage, $agent_role, $item_key );
|
||||
return null !== $row && 'done' === $row['status'];
|
||||
}
|
||||
|
||||
/**
|
||||
* A done stage's stored output (null when absent, not done, or pruned).
|
||||
*
|
||||
* @param string $run_id Run UUID.
|
||||
* @param string $stage Stage name.
|
||||
* @param string $agent_role Fan-out dimension.
|
||||
* @param string $item_key Article scope.
|
||||
* @return string|null
|
||||
*/
|
||||
public static function get_output( string $run_id, string $stage, string $agent_role = '', string $item_key = '' ): ?string {
|
||||
$row = self::get( $run_id, $stage, $agent_role, $item_key );
|
||||
if ( null === $row || 'done' !== $row['status'] || empty( $row['meta_json'] ) ) {
|
||||
return null;
|
||||
}
|
||||
$meta = json_decode( (string) $row['meta_json'], true );
|
||||
return is_array( $meta ) && isset( $meta['output'] ) ? (string) $meta['output'] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fail every still-open (pending/running) stage of one item — used by
|
||||
* the worker's catch-all so a crashed article leaves no stage stuck
|
||||
* in 'running' until the reaper. Done stages are untouched.
|
||||
*
|
||||
* @param string $run_id Run UUID.
|
||||
* @param string $item_key Stage item key ('' = run-level stages).
|
||||
* @return void
|
||||
*/
|
||||
public static function fail_open_for_item( string $run_id, string $item_key ): void {
|
||||
if ( '' === $run_id || ! self::is_available() ) {
|
||||
return;
|
||||
}
|
||||
global $wpdb;
|
||||
$table = self::table_name();
|
||||
$now = current_time( 'mysql' );
|
||||
// phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery
|
||||
$wpdb->query(
|
||||
$wpdb->prepare(
|
||||
"UPDATE {$table} SET status = 'failed', updated_at = %s, finished_at = %s
|
||||
WHERE run_id = %s AND item_key = %s AND status IN ('pending','running')",
|
||||
$now,
|
||||
$now,
|
||||
$run_id,
|
||||
$item_key
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/** Reset per-request caches (tests). */
|
||||
public static function flush_cache(): void {
|
||||
self::$table_ok = null;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user