peptide-repo/prautoblogger
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: empty-content guard + finish_reason=length handling + reasoning budget (v0.18.1)

Browse Source 
Reasoning models at high effort could spend the entire max_tokens budget
on thinking and return zero visible content with finish_reason=length;
the empty article flowed writer -> editor (reject) -> empty draft post,
with the cost booked as success (2026-06-11 incident: prod run acf24029,
draft post 921; diagnosis: convo pipeline-v2-phase1 seq 12).

- Empty-completion guard at the provider seam (new
  OpenRouter_Completion_Guard): empty/whitespace content is a failure;
  one retry with reasoning disabled (per-call override) when
  finish_reason=length or reasoning was active, then the stage fails.
- finish_reason=length always warning-logged (model, stage, tokens).
- Failed/retried attempts booked to generation_log with
  response_status=error + real token burn; run_decisions verdict rows
  (retry_reasoning_disabled / failed_empty_completion) for v0.18.0 runs.
- Writer-path guard in Content_Generator::execute_stage and
  belt-and-braces empty check in Publisher::create_post.
- Reasoning budget sanity: reasoning.max_tokens cap + completion
  headroom via new prautoblogger_reasoning_max_tokens setting (default
  constant 2048, 0 = legacy uncapped effort mode); cap replaces effort
  when active; governor reservations use the effective ceiling.
- Body assembly moved to Request_Builder::build_body (300-line cap).
- Tests: guard verdict matrix, builder budget shaping, writer/publisher
  guards, full provider retry path.

Agent-Session: prab-empty-draft-guard-20260611
This commit is contained in:
peptiderepo
2026-06-11 13:15:58 +00:00
parent f5658f1c97
commit 2bdc763dc0
19 changed files with 1196 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
includes/admin/class-settings-fields-extended.php
View File

@@ -78,7 +78,17 @@ class PRAutoBlogger_Settings_Fields_Extended {
'low' => __( 'Low — light reasoning, lower cost', 'prautoblogger' ),
'minimal' => __( 'Minimal — barely any reasoning', 'prautoblogger' ),
),
'description' => __( 'How much effort the model spends reasoning before answering. Higher effort = more reasoning tokens = higher cost. Only applies when reasoning is enabled above.', 'prautoblogger' ),
'description' => __( 'How much effort the model spends reasoning before answering. Higher effort = more reasoning tokens = higher cost. Only applies when reasoning is enabled above. When the Reasoning Token Cap below is set (default), the cap is sent instead of the effort level — set the cap to 0 to use effort only.', 'prautoblogger' ),
),
array(
'id' => 'prautoblogger_reasoning_max_tokens',
'label' => __( 'Reasoning Token Cap', 'prautoblogger' ),
'type' => 'number',
'section' => 'prautoblogger_models',
'default' => PRAUTOBLOGGER_DEFAULT_REASONING_MAX_TOKENS,
'min' => 0,
'max' => 32000,
'description' => __( 'Hard ceiling on thinking tokens per call when reasoning is enabled. The completion budget is raised by this amount so reasoning can never crowd out the article text — an uncapped high effort can spend the entire budget thinking and return an empty draft. 0 = no cap (uncapped effort mode, not recommended).', 'prautoblogger' ),
),
// ── Schedule & Budget ───────────────────────────────────────

1
includes/core/class-chief-editor.php
View File

@@ -64,6 +64,7 @@ class PRAutoBlogger_Chief_Editor {
'temperature' => 0.3,
'max_tokens' => 5000,
'response_format' => array( 'type' => 'json_object' ),
'stage' => 'review',
)
);

1
includes/core/class-content-analyzer.php
View File

@@ -78,6 +78,7 @@ class PRAutoBlogger_Content_Analyzer {
'temperature' => 0.5,
'max_tokens' => 8000,
'response_format' => array( 'type' => 'json_object' ),
'stage' => 'analysis',
)
);

21
includes/core/class-content-generator.php
View File

@@ -197,6 +197,12 @@ class PRAutoBlogger_Content_Generator {
PRAutoBlogger_Run_Stage_State::start( $this->run_id, $stage, '', $this->item_key );
}
// v0.18.1: stage metadata for the provider's empty-completion
// guard — warning logs and failed-attempt audit rows carry the
// real stage + registry key (never sent upstream).
$options['stage'] = $stage;
$options['prompt_key'] = $prompt_key;
$ctx = PRAutoBlogger_Opik_Trace_Context::current();
$span_id = $ctx->start_span(
@@ -223,6 +229,21 @@ class PRAutoBlogger_Content_Generator {
$options
);
// v0.18.1 writer-path guard (belt over the provider seam, which
// already throws for OpenRouter): no empty article may ever leave
// a writing stage — it would flow to the editor, be rejected as
// "draft text is missing", and land as an empty draft post.
if ( '' === trim( (string) $response['content'] ) ) {
throw new \RuntimeException(
sprintf(
/* translators: 1: writing stage name, 2: model identifier. */
__( 'Writing stage "%1$s" produced empty content (model %2$s) — failing the stage instead of passing an empty article downstream.', 'prautoblogger' ),
$stage,
(string) $response['model']
)
);
}
$ctx->end_span(
$span_id,
array(

1
includes/core/class-image-prompt-builder.php
View File

@@ -200,6 +200,7 @@ PROMPT;
array(
'temperature' => 0.7,
'max_tokens' => self::REWRITER_MAX_TOKENS,
'stage' => 'image_prompt_rewrite',
)
);

17
includes/core/class-publisher.php
View File

@@ -82,6 +82,23 @@ class PRAutoBlogger_Publisher {
?string $run_id = null,
?PRAutoBlogger_Cost_Tracker $cost_tracker = null
): int {
// v0.18.1 belt-and-braces: never create a post whose content is
// empty once tags and whitespace are stripped — an empty "draft
// for human review" (2026-06-11 incident, post 921) reviews
// nothing and hides the underlying generation failure. The
// provider/writer guards make this unreachable on OpenRouter
// paths; this protects every other path into post creation.
if ( '' === trim( wp_strip_all_tags( $content ) ) ) {
throw new \RuntimeException(
sprintf(
/* translators: 1: intended post status, 2: article title. */
__( 'Refusing to create %1$s post "%2$s": generated content is empty.', 'prautoblogger' ),
$post_status,
$idea->get_suggested_title()
)
);
}
// v0.18.0 idempotency: post creation is keyed by run + idea
// (check-before-insert) so a retried/resumed run cannot create a
// duplicate post. One run_id spans all N articles of a batch, so

1
includes/providers/class-llm-research-provider.php
View File

@@ -70,6 +70,7 @@ class PRAutoBlogger_LLM_Research_Provider implements PRAutoBlogger_Source_Provid
'temperature' => 0.7,
'max_tokens' => 8000,
'response_format' => array( 'type' => 'json_object' ),
'stage' => 'llm_research',
)
);

168
includes/providers/class-open-router-completion-guard.php Normal file
View File

@@ -0,0 +1,168 @@
<?php
declare(strict_types=1);
/**
* phpcs:ignore WordPress.Files.FileName.InvalidClassFileName -- class naming convention differs from WordPress standard
*
* Empty-completion guard for OpenRouter chat responses (v0.18.1).
*
* What: Treats a chat completion whose visible content is empty/whitespace
* as a FAILURE, never a success — per the LLM-call discipline:
* schema-validate every output, retry/fall back, never silently
* pass. The incident this prevents (2026-06-11, prod run acf24029,
* draft post 921): a true reasoning model at a high reasoning effort
* spent the entire max_tokens completion budget on thinking,
* finished with finish_reason='length', and emitted zero visible
* content — which flowed writer → editor (reject: "draft text is
* missing") → an empty draft post in the Review Queue, with the
* cost booked as 'success'.
*
* Behavior on every parsed response:
* - finish_reason='length' is warning-logged (model, stage, prompt/
* completion/reasoning tokens) whether or not content survived.
* - Empty content + (finish_reason='length' OR reasoning active on
* the request) → ONE retry with reasoning disabled for that call
* (per-call `reasoning` override), then fail the stage.
* - Every failed/retried attempt lands in generation_log with its
* REAL response_status ('error') and token burn, and — inside a
* v0.18.0 run — a run_decisions row records the verdict.
* Who triggers it: PRAutoBlogger_OpenRouter_Provider::send_chat_completion()
* on every successfully parsed response.
* Dependencies: Logger, Cost_Tracker (failure rows), Run_Context (run
* attribution), Audit_Writer (run_decisions verdicts).
*
* @see class-open-router-provider.php — Integration point; supplies the retry callable.
* @see class-open-router-response-parser.php — Produces the parsed array assessed here.
* @see class-open-router-request-builder.php — Reasoning token cap (first line of defense).
* @see core/class-audit-writer.php — run_decisions verdict rows.
*/
class PRAutoBlogger_OpenRouter_Completion_Guard {
/** Verdict recorded when an empty attempt is retried with reasoning disabled. */
public const VERDICT_RETRY = 'retry_reasoning_disabled';
/** Verdict recorded when a call finally fails for empty content. */
public const VERDICT_FAIL = 'failed_empty_completion';
/**
* Assess a parsed completion: pass it through, retry once, or fail loudly.
*
* @param array<string, mixed> $parsed Parsed response from Response_Parser::parse_success().
* @param array<string, mixed> $options Original call options. Reads the caller-metadata keys
* 'stage' / 'prompt_key' (audit attribution) and
* 'empty_retry' (marks the retry attempt — never retried twice).
* @param bool $reasoning_active Whether the dispatched request had reasoning enabled.
* @param callable $retry Re-dispatches the call with the options it is given.
* Invoked at most once, with reasoning disabled.
*
* @return array<string, mixed> A completion whose visible content is non-empty.
*
* @throws \RuntimeException When content is empty and the retry budget is spent or a retry
* cannot plausibly help.
*/
public function finalize( array $parsed, array $options, bool $reasoning_active, callable $retry ): array {
$stage = (string) ( $options['stage'] ?? 'unknown' );
$finish_reason = (string) ( $parsed['finish_reason'] ?? 'unknown' );
if ( 'length' === $finish_reason ) {
PRAutoBlogger_Logger::instance()->warning(
sprintf(
'Completion hit its max_tokens ceiling (finish_reason=length): model=%s, stage=%s, prompt_tokens=%d, completion_tokens=%d, reasoning_tokens=%d.',
(string) $parsed['model'],
$stage,
(int) $parsed['prompt_tokens'],
(int) $parsed['completion_tokens'],
(int) $parsed['reasoning_tokens']
),
'openrouter'
);
}
if ( '' !== trim( (string) ( $parsed['content'] ?? '' ) ) ) {
return $parsed; // Healthy completion — visible content present.
}
$already_retried = ! empty( $options['empty_retry'] );
// A reasoning-disabled retry can plausibly help when the budget was
// exhausted (length) or when reasoning was eating the budget.
$retry_can_help = 'length' === $finish_reason || $reasoning_active;
if ( $retry_can_help && ! $already_retried ) {
$note = sprintf(
'Empty completion (finish_reason=%s, completion_tokens=%d, reasoning_tokens=%d) — retrying once with reasoning disabled.',
$finish_reason,
(int) $parsed['completion_tokens'],
(int) $parsed['reasoning_tokens']
);
$this->record_attempt( $parsed, $options, $stage, self::VERDICT_RETRY, $note );
PRAutoBlogger_Logger::instance()->warning(
sprintf( 'Stage %s: %s (model %s)', $stage, $note, (string) $parsed['model'] ),
'openrouter'
);
$retry_options = $options;
$retry_options['reasoning'] = array( 'enabled' => false );
$retry_options['empty_retry'] = true;
return $retry( $retry_options );
}
$note = sprintf(
'Empty completion (finish_reason=%s, completion_tokens=%d, reasoning_tokens=%d)%s — failing the call.',
$finish_reason,
(int) $parsed['completion_tokens'],
(int) $parsed['reasoning_tokens'],
$already_retried ? ' after a reasoning-disabled retry' : ''
);
$this->record_attempt( $parsed, $options, $stage, self::VERDICT_FAIL, $note );
throw new \RuntimeException(
sprintf(
/* translators: 1: pipeline stage, 2: model identifier, 3: finish_reason value, 4: retry note. */
__( 'LLM call for stage "%1$s" returned empty content (model %2$s, finish_reason=%3$s)%4$s.', 'prautoblogger' ),
$stage,
(string) $parsed['model'],
$finish_reason,
$already_retried ? __( ' after a reasoning-disabled retry', 'prautoblogger' ) : ''
)
);
}
/**
* Book a failed/retried attempt: a generation_log row with the REAL
* response_status and token burn (the empty call still cost money),
* plus a run_decisions verdict for new-style runs. Both writes are
* self-healing no-ops on a half-migrated schema; neither can throw.
*
* @param array<string, mixed> $parsed Parsed response of the failed attempt.
* @param array<string, mixed> $options Call options ('prompt_key' attribution).
* @param string $stage Stage label for the log row.
* @param string $verdict VERDICT_RETRY or VERDICT_FAIL.
* @param string $note Human-readable failure note.
* @return void
*/
private function record_attempt( array $parsed, array $options, string $stage, string $verdict, string $note ): void {
$run_id = PRAutoBlogger_Run_Context::current_run_id();
$tracker = new PRAutoBlogger_Cost_Tracker();
if ( null !== $run_id && '' !== $run_id ) {
// Attribute the row to the surrounding run. Idempotent: the
// run context, ledger row, and prompt pins already exist.
$tracker->set_run_id( $run_id );
}
$tracker->log_api_call(
null,
$stage,
'OpenRouter',
(string) $parsed['model'],
(int) $parsed['prompt_tokens'],
(int) $parsed['completion_tokens'],
'error',
$note,
null,
isset( $options['prompt_key'] ) ? (string) $options['prompt_key'] : null
);
// run_decisions verdict — no-op outside a run or pre-1.2.0 schema.
PRAutoBlogger_Audit_Writer::record_decision( (string) ( $run_id ?? '' ), $stage, $verdict, $note );
}
}

76
includes/providers/class-open-router-provider.php
View File

@@ -19,8 +19,9 @@ declare(strict_types=1);
* @see class-open-router-config.php — Config helpers (base URL, cache TTL).
* @see class-open-router-pricing.php — Pricing and model list lookups.
* @see class-open-router-validator.php — Credential validation (delegated).
* @see class-open-router-request-builder.php — Request header building + cURL filter.
* @see class-open-router-request-builder.php — Request body/header building + cURL filter.
* @see class-open-router-response-parser.php — Response parsing + error classification.
* @see class-open-router-completion-guard.php — Empty-completion guard + reasoning-off retry (v0.18.1).
* @see class-cost-tracker.php — Called after every request to log token usage.
* @see ARCHITECTURE.md — Data flow diagram showing where this fits.
*/
@@ -40,8 +41,15 @@ class PRAutoBlogger_OpenRouter_Provider implements PRAutoBlogger_LLM_Provider_In
* temperature?: float,
* max_tokens?: int,
* response_format?: array{type: string},
* reasoning?: array{enabled: bool, effort?: string},
* } $options Optional parameters. Pass 'reasoning' to override the global setting.
* reasoning?: array{enabled: bool, effort?: string, max_tokens?: int},
* stage?: string,
* prompt_key?: string,
* empty_retry?: bool,
* } $options Optional parameters. Pass 'reasoning' to override the global
* setting. 'stage'/'prompt_key' are caller metadata for the
* empty-completion guard's logging + audit rows (never sent
* upstream); 'empty_retry' is set internally by the guard's
* single reasoning-disabled retry.
*
* @return array{
* content: string,
@@ -52,46 +60,39 @@ class PRAutoBlogger_OpenRouter_Provider implements PRAutoBlogger_LLM_Provider_In
* finish_reason: string,
* }
*
* @throws \RuntimeException On API error after retries exhausted.
* @throws \RuntimeException On API error after retries exhausted, or when the
* completion's visible content is empty/whitespace
* (v0.18.1 guard — after at most one reasoning-disabled retry).
* @throws PRAutoBlogger_Cost_Ceiling_Exception When the per-run cost governor blocks
* the call (run already halted).
*/
public function send_chat_completion( array $messages, string $model, array $options = array() ): array {
$builder = new PRAutoBlogger_OpenRouter_Request_Builder();
// Fetch + format-validate the key (moved to Request_Builder in
// v0.18.0 for the 300-line cap; behavior unchanged — throws the
// same messages on missing/corrupted keys).
$api_key = ( new PRAutoBlogger_OpenRouter_Request_Builder() )->resolve_api_key();
$api_key = $builder->resolve_api_key();
$body = array(
'model' => $model,
'messages' => $messages,
);
// Body assembly — incl. global/per-call reasoning and the v0.18.1
// reasoning token cap + completion headroom — lives in the builder.
$body = $builder->build_body( $messages, $model, $options );
if ( isset( $options['temperature'] ) ) {
$body['temperature'] = $options['temperature'];
}
if ( isset( $options['max_tokens'] ) ) {
$body['max_tokens'] = $options['max_tokens'];
}
if ( isset( $options['response_format'] ) ) {
$body['response_format'] = $options['response_format'];
}
// Reasoning mode: explicit caller override takes priority, then global setting.
if ( isset( $options['reasoning'] ) ) {
$body['reasoning'] = $options['reasoning'];
} elseif ( '1' === get_option( 'prautoblogger_reasoning_enabled', '0' ) ) {
$body['reasoning'] = array(
'enabled' => true,
'effort' => get_option( 'prautoblogger_reasoning_effort', 'medium' ),
);
}
// Whether the outgoing request has reasoning enabled — drives the
// empty-completion retry decision after parse.
$reasoning_active = isset( $body['reasoning'] ) && false !== ( $body['reasoning']['enabled'] ?? true );
// v0.18.0 — per-run cost governor: reserve the worst-case estimate
// BEFORE any dispatch. Throws (and the run is already halted) when
// the reservation would breach the run ceiling; returns null when
// this process is not inside a governed run (historical behavior).
$reservation = PRAutoBlogger_Cost_Governor::open_chat_reservation( $model, $messages, $options );
// v0.18.1: the estimate uses the EFFECTIVE completion ceiling from
// the built body (incl. reasoning headroom), not the raw option.
$govern_options = $options;
if ( isset( $body['max_tokens'] ) ) {
$govern_options['max_tokens'] = $body['max_tokens'];
}
$reservation = PRAutoBlogger_Cost_Governor::open_chat_reservation( $model, $messages, $govern_options );
$last_error = '';
@@ -101,7 +102,6 @@ class PRAutoBlogger_OpenRouter_Provider implements PRAutoBlogger_LLM_Provider_In
$cache_ttl = $config->get_cache_ttl_seconds();
$via_gateway = $config->is_via_gateway();
$builder = new PRAutoBlogger_OpenRouter_Request_Builder();
$request_headers = $builder->build_headers( $api_key, $via_gateway, $cache_ttl );
$curl_auth_filter = $builder->register_curl_auth_filter( $request_headers, $base_host );
@@ -190,7 +190,21 @@ class PRAutoBlogger_OpenRouter_Provider implements PRAutoBlogger_LLM_Provider_In
$this->estimate_cost( $model, (int) $parsed['prompt_tokens'], (int) $parsed['completion_tokens'] )
);
$reservation = null;
return $parsed;
// v0.18.1 — empty-completion guard: empty visible content
// is a FAILURE, never a success. Warns on finish_reason=
// length, retries ONCE with reasoning disabled when that
// can plausibly help, books failed attempts with their
// real status, and otherwise throws. The recursive retry
// manages its own reservation and cURL filter.
return ( new PRAutoBlogger_OpenRouter_Completion_Guard() )->finalize(
$parsed,
$options,
$reasoning_active,
function ( array $retry_options ) use ( $messages, $model ): array {
return $this->send_chat_completion( $messages, $model, $retry_options );
}
);
}
// All retries exhausted.

94
includes/providers/class-open-router-request-builder.php
View File

@@ -18,6 +18,100 @@ declare(strict_types=1);
*/
class PRAutoBlogger_OpenRouter_Request_Builder {
/**
* Build the JSON request body for a chat-completion call.
*
* Centralizes option handling (v0.18.1, moved from the provider for
* the 300-line cap): temperature, max_tokens, response_format, and
* the reasoning block — an explicit caller override takes priority
* over the global setting (that per-call override is how the
* empty-completion retry disables reasoning for its second attempt).
*
* Caller-metadata keys ('stage', 'prompt_key', 'empty_retry') are
* deliberately NOT copied into the HTTP body.
*
* @param array<int, array{role: string, content: string}> $messages Chat messages.
* @param string $model Model identifier.
* @param array<string, mixed> $options Call options.
*
* @return array<string, mixed> Request body ready for wp_json_encode().
*/
public function build_body( array $messages, string $model, array $options ): array {
$body = array(
'model' => $model,
'messages' => $messages,
);
if ( isset( $options['temperature'] ) ) {
$body['temperature'] = $options['temperature'];
}
if ( isset( $options['max_tokens'] ) ) {
$body['max_tokens'] = $options['max_tokens'];
}
if ( isset( $options['response_format'] ) ) {
$body['response_format'] = $options['response_format'];
}
// Reasoning mode: explicit caller override takes priority, then global setting.
if ( isset( $options['reasoning'] ) ) {
$body['reasoning'] = $options['reasoning'];
} elseif ( '1' === get_option( 'prautoblogger_reasoning_enabled', '0' ) ) {
$body['reasoning'] = array(
'enabled' => true,
'effort' => get_option( 'prautoblogger_reasoning_effort', 'medium' ),
);
}
return $this->apply_reasoning_budget( $body );
}
/**
* Reasoning budget sanity (v0.18.1): cap thinking, protect content.
*
* When the outgoing request enables reasoning and carries a completion
* ceiling, the thinking budget is capped via OpenRouter's
* `reasoning.max_tokens` and the completion ceiling is raised by the
* same amount — so no reasoning effort (incl. xhigh) can consume the
* entire completion budget and emit an empty message (the 2026-06-11
* empty-draft incident, prod run acf24029). The cap is SETTINGS-backed
* (`prautoblogger_reasoning_max_tokens`, default
* PRAUTOBLOGGER_DEFAULT_REASONING_MAX_TOKENS, 0 = pure effort mode:
* no cap, no headroom — the historical behavior). When the cap is
* active it replaces `effort` — OpenRouter treats the two as
* alternative budget controls and normalizes max_tokens per model.
*
* No-op when reasoning is absent/disabled on the request, when the
* request has no completion ceiling, or when the cap setting is 0.
*
* @param array<string, mixed> $body Assembled request body.
*
* @return array<string, mixed> Body with the reasoning budget applied.
*/
private function apply_reasoning_budget( array $body ): array {
if ( ! isset( $body['reasoning'] ) || false === ( $body['reasoning']['enabled'] ?? true ) ) {
return $body;
}
if ( ! isset( $body['max_tokens'] ) ) {
return $body; // No completion ceiling to protect.
}
if ( ! isset( $body['reasoning']['max_tokens'] ) ) {
$cap = absint( get_option( 'prautoblogger_reasoning_max_tokens', PRAUTOBLOGGER_DEFAULT_REASONING_MAX_TOKENS ) );
if ( $cap < 1 ) {
return $body; // Cap disabled — pure effort mode.
}
$body['reasoning']['max_tokens'] = $cap;
unset( $body['reasoning']['effort'] );
}
// Headroom: models that spend reasoning from the completion budget
// (e.g. deepseek/deepseek-v4-flash) must still have the caller's
// full max_tokens available for visible content.
$body['max_tokens'] = (int) $body['max_tokens'] + (int) $body['reasoning']['max_tokens'];
return $body;
}
/**
* Build request headers for OpenRouter API call.
*

1
includes/services/opik/class-opik-eval-scorer.php
View File

@@ -82,6 +82,7 @@ class PRAutoBlogger_Opik_Eval_Scorer {
array(
'temperature' => 0.1,
'max_tokens' => 200,
'stage' => 'opik_eval_judge',
)
);