fix: empty-content guard + finish_reason=length handling + reasoning budget (v0.18.1)
Reasoning models at high effort could spend the entire max_tokens budget on thinking and return zero visible content with finish_reason=length; the empty article flowed writer -> editor (reject) -> empty draft post, with the cost booked as success (2026-06-11 incident: prod run acf24029, draft post 921; diagnosis: convo pipeline-v2-phase1 seq 12). - Empty-completion guard at the provider seam (new OpenRouter_Completion_Guard): empty/whitespace content is a failure; one retry with reasoning disabled (per-call override) when finish_reason=length or reasoning was active, then the stage fails. - finish_reason=length always warning-logged (model, stage, tokens). - Failed/retried attempts booked to generation_log with response_status=error + real token burn; run_decisions verdict rows (retry_reasoning_disabled / failed_empty_completion) for v0.18.0 runs. - Writer-path guard in Content_Generator::execute_stage and belt-and-braces empty check in Publisher::create_post. - Reasoning budget sanity: reasoning.max_tokens cap + completion headroom via new prautoblogger_reasoning_max_tokens setting (default constant 2048, 0 = legacy uncapped effort mode); cap replaces effort when active; governor reservations use the effective ceiling. - Body assembly moved to Request_Builder::build_body (300-line cap). - Tests: guard verdict matrix, builder budget shaping, writer/publisher guards, full provider retry path. Agent-Session: prab-empty-draft-guard-20260611
This commit is contained in:
peptiderepo
@@ -166,4 +166,153 @@ class OpenRouterProviderTest extends BaseTestCase {
|
||||
$this->expectException( \RuntimeException::class );
|
||||
$provider->send_chat_completion( $messages, 'model/test', $options );
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire the provider for a full send_chat_completion run: decryptable
|
||||
* API key, reasoning settings, HTTP stubs that replay scripted
|
||||
* responses, and a wpdb mock for the guard's failure bookkeeping.
|
||||
*
|
||||
* @param array $option_overrides Extra get_option values.
|
||||
* @param array $responses Scripted JSON-decodable response bodies.
|
||||
* @param array $requests (by ref) captures decoded request bodies.
|
||||
*/
|
||||
private function wire_full_provider( array $option_overrides, array $responses, array &$requests ): void {
|
||||
\PRAutoBlogger_Run_Context::clear();
|
||||
|
||||
$encrypted_key = \PRAutoBlogger_Encryption::encrypt( 'sk-or-v1-unit-test-key-0000' );
|
||||
|
||||
$this->stub_get_option( array_merge( [
|
||||
'prautoblogger_openrouter_api_key' => $encrypted_key,
|
||||
'prautoblogger_log_level' => 'error',
|
||||
], $option_overrides ) );
|
||||
|
||||
Functions\when( 'absint' )->alias( function ( $val ) {
|
||||
return abs( (int) $val );
|
||||
} );
|
||||
|
||||
Functions\when( 'wp_remote_post' )->alias(
|
||||
function ( $url, $args ) use ( &$requests, $responses ) {
|
||||
$requests[] = json_decode( $args['body'], true );
|
||||
$body = $responses[ count( $requests ) - 1 ];
|
||||
return [
|
||||
'body' => $body,
|
||||
'response' => [ 'code' => 200 ],
|
||||
];
|
||||
}
|
||||
);
|
||||
Functions\when( 'wp_remote_retrieve_response_code' )->alias(
|
||||
function ( $response ) {
|
||||
return $response['response']['code'];
|
||||
}
|
||||
);
|
||||
Functions\when( 'wp_remote_retrieve_body' )->alias(
|
||||
function ( $response ) {
|
||||
return $response['body'];
|
||||
}
|
||||
);
|
||||
|
||||
// The guard books failed attempts through Cost_Tracker + Logger.
|
||||
$wpdb = $this->create_mock_wpdb();
|
||||
$wpdb->method( 'insert' )->willReturn( 1 );
|
||||
$wpdb->insert_id = 11;
|
||||
$GLOBALS['wpdb'] = $wpdb;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.18.1 incident shape, end to end: reasoning enabled (uncapped
|
||||
* effort mode), the first response burns the whole completion budget
|
||||
* on thinking and returns empty content with finish_reason=length —
|
||||
* the provider retries exactly once with reasoning disabled and
|
||||
* returns the recovered completion.
|
||||
*/
|
||||
public function test_empty_length_completion_retries_once_with_reasoning_disabled(): void {
|
||||
$requests = [];
|
||||
|
||||
$empty_response = json_encode( [
|
||||
'model' => 'deepseek/deepseek-v4-flash',
|
||||
'choices' => [ [ 'message' => [ 'content' => '', 'reasoning' => 'endless thinking…' ], 'finish_reason' => 'length' ] ],
|
||||
'usage' => [ 'prompt_tokens' => 1200, 'completion_tokens' => 4000, 'reasoning_tokens' => 4000, 'total_tokens' => 5200 ],
|
||||
] );
|
||||
$good_response = json_encode( [
|
||||
'model' => 'deepseek/deepseek-v4-flash',
|
||||
'choices' => [ [ 'message' => [ 'content' => 'Recovered article.' ], 'finish_reason' => 'stop' ] ],
|
||||
'usage' => [ 'prompt_tokens' => 1200, 'completion_tokens' => 900, 'total_tokens' => 2100 ],
|
||||
] );
|
||||
|
||||
$this->wire_full_provider(
|
||||
[
|
||||
'prautoblogger_reasoning_enabled' => '1',
|
||||
'prautoblogger_reasoning_effort' => 'xhigh',
|
||||
'prautoblogger_reasoning_max_tokens' => 0,
|
||||
],
|
||||
[ $empty_response, $good_response ],
|
||||
$requests
|
||||
);
|
||||
|
||||
try {
|
||||
$provider = new \PRAutoBlogger_OpenRouter_Provider();
|
||||
$result = $provider->send_chat_completion(
|
||||
[ [ 'role' => 'user', 'content' => 'Write the article.' ] ],
|
||||
'deepseek/deepseek-v4-flash',
|
||||
[ 'temperature' => 0.7, 'max_tokens' => 4000, 'stage' => 'draft' ]
|
||||
);
|
||||
|
||||
$this->assertSame( 'Recovered article.', $result['content'] );
|
||||
$this->assertCount( 2, $requests );
|
||||
// First attempt: global reasoning, uncapped effort mode.
|
||||
$this->assertSame( [ 'enabled' => true, 'effort' => 'xhigh' ], $requests[0]['reasoning'] );
|
||||
$this->assertSame( 4000, $requests[0]['max_tokens'] );
|
||||
// Retry: reasoning disabled per call.
|
||||
$this->assertSame( [ 'enabled' => false ], $requests[1]['reasoning'] );
|
||||
// Caller metadata never reaches the HTTP body.
|
||||
$this->assertArrayNotHasKey( 'stage', $requests[0] );
|
||||
$this->assertArrayNotHasKey( 'empty_retry', $requests[1] );
|
||||
} finally {
|
||||
unset( $GLOBALS['wpdb'] );
|
||||
\PRAutoBlogger_Run_Context::clear();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.18.1 reasoning budget: with the token cap active (default), the
|
||||
* outgoing request caps thinking via reasoning.max_tokens and raises
|
||||
* max_tokens by the cap — and a healthy completion needs no retry.
|
||||
*/
|
||||
public function test_reasoning_cap_shapes_the_outgoing_request(): void {
|
||||
$requests = [];
|
||||
|
||||
$good_response = json_encode( [
|
||||
'model' => 'deepseek/deepseek-v4-flash',
|
||||
'choices' => [ [ 'message' => [ 'content' => 'Article text.' ], 'finish_reason' => 'stop' ] ],
|
||||
'usage' => [ 'prompt_tokens' => 1200, 'completion_tokens' => 900, 'total_tokens' => 2100 ],
|
||||
] );
|
||||
|
||||
$this->wire_full_provider(
|
||||
[
|
||||
'prautoblogger_reasoning_enabled' => '1',
|
||||
'prautoblogger_reasoning_effort' => 'xhigh',
|
||||
'prautoblogger_reasoning_max_tokens' => 2048,
|
||||
],
|
||||
[ $good_response ],
|
||||
$requests
|
||||
);
|
||||
|
||||
try {
|
||||
$provider = new \PRAutoBlogger_OpenRouter_Provider();
|
||||
$result = $provider->send_chat_completion(
|
||||
[ [ 'role' => 'user', 'content' => 'Write the article.' ] ],
|
||||
'deepseek/deepseek-v4-flash',
|
||||
[ 'temperature' => 0.7, 'max_tokens' => 4000, 'stage' => 'draft' ]
|
||||
);
|
||||
|
||||
$this->assertSame( 'Article text.', $result['content'] );
|
||||
$this->assertCount( 1, $requests );
|
||||
$this->assertSame( 2048, $requests[0]['reasoning']['max_tokens'] );
|
||||
$this->assertArrayNotHasKey( 'effort', $requests[0]['reasoning'] );
|
||||
$this->assertSame( 6048, $requests[0]['max_tokens'] );
|
||||
} finally {
|
||||
unset( $GLOBALS['wpdb'] );
|
||||
\PRAutoBlogger_Run_Context::clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user