fix: empty-content guard + finish_reason=length handling + reasoning budget (v0.18.1)
Reasoning models at high effort could spend the entire max_tokens budget on thinking and return zero visible content with finish_reason=length; the empty article flowed writer -> editor (reject) -> empty draft post, with the cost booked as success (2026-06-11 incident: prod run acf24029, draft post 921; diagnosis: convo pipeline-v2-phase1 seq 12). - Empty-completion guard at the provider seam (new OpenRouter_Completion_Guard): empty/whitespace content is a failure; one retry with reasoning disabled (per-call override) when finish_reason=length or reasoning was active, then the stage fails. - finish_reason=length always warning-logged (model, stage, tokens). - Failed/retried attempts booked to generation_log with response_status=error + real token burn; run_decisions verdict rows (retry_reasoning_disabled / failed_empty_completion) for v0.18.0 runs. - Writer-path guard in Content_Generator::execute_stage and belt-and-braces empty check in Publisher::create_post. - Reasoning budget sanity: reasoning.max_tokens cap + completion headroom via new prautoblogger_reasoning_max_tokens setting (default constant 2048, 0 = legacy uncapped effort mode); cap replaces effort when active; governor reservations use the effective ceiling. - Body assembly moved to Request_Builder::build_body (300-line cap). - Tests: guard verdict matrix, builder budget shaping, writer/publisher guards, full provider retry path. Agent-Session: prab-empty-draft-guard-20260611
This commit is contained in:
peptiderepo
@@ -28,6 +28,9 @@ if ( ! defined( 'PRAUTOBLOGGER_DEFAULT_RUN_CEILING_USD' ) ) {
|
||||
if ( ! defined( 'PRAUTOBLOGGER_DEFAULT_REQUEST_JSON_RETENTION_DAYS' ) ) {
|
||||
define( 'PRAUTOBLOGGER_DEFAULT_REQUEST_JSON_RETENTION_DAYS', 14 );
|
||||
}
|
||||
if ( ! defined( 'PRAUTOBLOGGER_DEFAULT_REASONING_MAX_TOKENS' ) ) {
|
||||
define( 'PRAUTOBLOGGER_DEFAULT_REASONING_MAX_TOKENS', 2048 );
|
||||
}
|
||||
|
||||
// WordPress time constants used by CostTracker and ContentAnalyzer.
|
||||
if ( ! defined( 'DAY_IN_SECONDS' ) ) {
|
||||
|
||||
164
tests/unit/Core/ContentGeneratorTest.php
Normal file
164
tests/unit/Core/ContentGeneratorTest.php
Normal file
@@ -0,0 +1,164 @@
|
||||
<?php
|
||||
/**
|
||||
* Tests for PRAutoBlogger_Content_Generator (v0.18.1 writer-path guard).
|
||||
*
|
||||
* Validates that a writing stage whose LLM output is empty/whitespace
|
||||
* throws instead of passing an empty article downstream, that healthy
|
||||
* output passes through, and that the stage metadata for the provider's
|
||||
* empty-completion guard rides the options array.
|
||||
*
|
||||
* execute_stage() is exercised via reflection: generate() defines the
|
||||
* PRAUTOBLOGGER_EVAL_MODE constant, which cannot be redefined across
|
||||
* tests in one PHP process.
|
||||
*
|
||||
* @package PRAutoBlogger\Tests\Core
|
||||
*/
|
||||
|
||||
namespace PRAutoBlogger\Tests\Core;
|
||||
|
||||
use PRAutoBlogger\Tests\BaseTestCase;
|
||||
use Brain\Monkey\Functions;
|
||||
|
||||
class ContentGeneratorTest extends BaseTestCase {
|
||||
|
||||
private \PRAutoBlogger_Content_Request $request;
|
||||
|
||||
protected function setUp(): void {
|
||||
parent::setUp();
|
||||
|
||||
\PRAutoBlogger_Prompt_Registry::flush_cache();
|
||||
|
||||
$this->stub_get_option( [
|
||||
'prautoblogger_log_level' => 'error',
|
||||
] );
|
||||
|
||||
// Content_Prompts::build_system() lists recent posts for the
|
||||
// linking rules — none exist in the unit environment.
|
||||
Functions\when( 'get_posts' )->justReturn( [] );
|
||||
|
||||
$idea = new \PRAutoBlogger_Article_Idea( $this->get_article_idea_fixture() );
|
||||
|
||||
$this->request = new \PRAutoBlogger_Content_Request(
|
||||
$idea,
|
||||
'single_pass',
|
||||
'informational',
|
||||
800,
|
||||
2000,
|
||||
'',
|
||||
[],
|
||||
''
|
||||
);
|
||||
}
|
||||
|
||||
protected function tearDown(): void {
|
||||
\PRAutoBlogger_Prompt_Registry::flush_cache();
|
||||
parent::tearDown();
|
||||
}
|
||||
|
||||
/**
|
||||
* Invoke the private execute_stage() on a generator wired to the
|
||||
* given LLM response.
|
||||
*
|
||||
* @param array $llm_response Response the mock provider returns.
|
||||
* @param array|null $captured Receives the options passed to the LLM.
|
||||
* @return string Stage output.
|
||||
*/
|
||||
private function run_stage( array $llm_response, ?array &$captured = null ): string {
|
||||
$llm = $this->createMock( \PRAutoBlogger_LLM_Provider_Interface::class );
|
||||
$llm->method( 'send_chat_completion' )->willReturnCallback(
|
||||
function ( array $messages, string $model, array $options ) use ( $llm_response, &$captured ) {
|
||||
$captured = $options;
|
||||
return $llm_response;
|
||||
}
|
||||
);
|
||||
$llm->method( 'get_provider_name' )->willReturn( 'OpenRouter' );
|
||||
$llm->method( 'estimate_cost' )->willReturn( 0.001 );
|
||||
|
||||
$cost_tracker = $this->createMock( \PRAutoBlogger_Cost_Tracker::class );
|
||||
|
||||
$generator = new \PRAutoBlogger_Content_Generator( $llm, $cost_tracker );
|
||||
|
||||
$method = new \ReflectionMethod( \PRAutoBlogger_Content_Generator::class, 'execute_stage' );
|
||||
$method->setAccessible( true );
|
||||
|
||||
return $method->invoke(
|
||||
$generator,
|
||||
'draft',
|
||||
'draft_generation',
|
||||
'Write the article.',
|
||||
$this->request,
|
||||
'model/test',
|
||||
[
|
||||
'temperature' => 0.7,
|
||||
'max_tokens' => 4000,
|
||||
],
|
||||
'content.single_pass'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Healthy stage output passes through unchanged.
|
||||
*/
|
||||
public function test_non_empty_stage_output_passes_through(): void {
|
||||
$output = $this->run_stage( [
|
||||
'content' => '<p>A real article.</p>',
|
||||
'model' => 'model/test',
|
||||
'prompt_tokens' => 100,
|
||||
'completion_tokens' => 200,
|
||||
] );
|
||||
|
||||
$this->assertSame( '<p>A real article.</p>', $output );
|
||||
}
|
||||
|
||||
/**
|
||||
* Empty stage output throws — no empty article may leave the writer.
|
||||
*/
|
||||
public function test_empty_stage_output_throws(): void {
|
||||
$this->expectException( \RuntimeException::class );
|
||||
$this->expectExceptionMessage( 'produced empty content' );
|
||||
|
||||
$this->run_stage( [
|
||||
'content' => '',
|
||||
'model' => 'model/test',
|
||||
'prompt_tokens' => 100,
|
||||
'completion_tokens' => 4000,
|
||||
] );
|
||||
}
|
||||
|
||||
/**
|
||||
* Whitespace-only output counts as empty (the guard trims).
|
||||
*/
|
||||
public function test_whitespace_stage_output_throws(): void {
|
||||
$this->expectException( \RuntimeException::class );
|
||||
$this->expectExceptionMessage( 'draft' );
|
||||
|
||||
$this->run_stage( [
|
||||
'content' => " \n\t ",
|
||||
'model' => 'model/test',
|
||||
'prompt_tokens' => 100,
|
||||
'completion_tokens' => 4000,
|
||||
] );
|
||||
}
|
||||
|
||||
/**
|
||||
* The stage + prompt_key metadata for the provider's empty-completion
|
||||
* guard rides the LLM options (v0.18.1).
|
||||
*/
|
||||
public function test_stage_metadata_rides_the_llm_options(): void {
|
||||
$captured = null;
|
||||
|
||||
$this->run_stage(
|
||||
[
|
||||
'content' => 'ok',
|
||||
'model' => 'model/test',
|
||||
'prompt_tokens' => 1,
|
||||
'completion_tokens' => 1,
|
||||
],
|
||||
$captured
|
||||
);
|
||||
|
||||
$this->assertSame( 'draft', $captured['stage'] );
|
||||
$this->assertSame( 'content.single_pass', $captured['prompt_key'] );
|
||||
$this->assertSame( 4000, $captured['max_tokens'] );
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,10 @@ class PublisherTest extends BaseTestCase {
|
||||
|
||||
Functions\when( 'sanitize_text_field' )->returnArg();
|
||||
Functions\when( 'wp_kses_post' )->returnArg();
|
||||
// v0.18.1 empty-content guard strips tags before the emptiness check.
|
||||
Functions\when( 'wp_strip_all_tags' )->alias( function ( $text ) {
|
||||
return trim( strip_tags( (string) $text ) );
|
||||
} );
|
||||
Functions\when( 'is_wp_error' )->justReturn( false );
|
||||
Functions\when( 'apply_filters' )->returnArg( 2 );
|
||||
Functions\when( 'absint' )->alias( function ( $val ) { return abs( (int) $val ); } );
|
||||
@@ -295,4 +299,35 @@ class PublisherTest extends BaseTestCase {
|
||||
$publisher = new \PRAutoBlogger_Publisher();
|
||||
$publisher->publish( '<p>Content</p>', $this->idea, $this->review );
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.18.1 belt-and-braces: save_as_draft() with empty content throws
|
||||
* and never reaches wp_insert_post — no empty draft post can be
|
||||
* created (the 2026-06-11 post-921 incident shape).
|
||||
*/
|
||||
public function test_save_as_draft_refuses_empty_content(): void {
|
||||
Functions\expect( 'wp_insert_post' )->never();
|
||||
|
||||
$publisher = new \PRAutoBlogger_Publisher();
|
||||
|
||||
$this->expectException( \RuntimeException::class );
|
||||
$this->expectExceptionMessage( 'generated content is empty' );
|
||||
|
||||
$publisher->save_as_draft( '', $this->idea, $this->review, 'run_empty_draft' );
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.18.1 belt-and-braces: publish() refuses content that is empty
|
||||
* once tags and whitespace are stripped (an empty HTML shell).
|
||||
*/
|
||||
public function test_publish_refuses_whitespace_only_html_content(): void {
|
||||
Functions\expect( 'wp_insert_post' )->never();
|
||||
|
||||
$publisher = new \PRAutoBlogger_Publisher();
|
||||
|
||||
$this->expectException( \RuntimeException::class );
|
||||
$this->expectExceptionMessage( 'generated content is empty' );
|
||||
|
||||
$publisher->publish( "<p> \n\t </p>", $this->idea, $this->review );
|
||||
}
|
||||
}
|
||||
|
||||
240
tests/unit/Providers/OpenRouterCompletionGuardTest.php
Normal file
240
tests/unit/Providers/OpenRouterCompletionGuardTest.php
Normal file
@@ -0,0 +1,240 @@
|
||||
<?php
|
||||
/**
|
||||
* Tests for PRAutoBlogger_OpenRouter_Completion_Guard (v0.18.1).
|
||||
*
|
||||
* Validates the empty-completion verdict matrix: healthy pass-through,
|
||||
* the single reasoning-disabled retry on length/reasoning empties, the
|
||||
* never-retry-twice rule, and the hard failure for unexplained empties.
|
||||
*
|
||||
* @package PRAutoBlogger\Tests\Providers
|
||||
*/
|
||||
|
||||
namespace PRAutoBlogger\Tests\Providers;
|
||||
|
||||
use PRAutoBlogger\Tests\BaseTestCase;
|
||||
use Brain\Monkey\Functions;
|
||||
|
||||
class OpenRouterCompletionGuardTest extends BaseTestCase {
|
||||
|
||||
protected function setUp(): void {
|
||||
parent::setUp();
|
||||
|
||||
\PRAutoBlogger_Run_Context::clear();
|
||||
\PRAutoBlogger_Audit_Writer::flush_cache();
|
||||
|
||||
$this->stub_get_option( [
|
||||
'prautoblogger_log_level' => 'error',
|
||||
] );
|
||||
|
||||
// Cost_Tracker::log_api_call() (failed-attempt booking) inserts a
|
||||
// generation_log row and prices the burn via the pricing chain.
|
||||
$wpdb = $this->create_mock_wpdb();
|
||||
$wpdb->method( 'insert' )->willReturn( 1 );
|
||||
$wpdb->insert_id = 7;
|
||||
$GLOBALS['wpdb'] = $wpdb;
|
||||
}
|
||||
|
||||
protected function tearDown(): void {
|
||||
unset( $GLOBALS['wpdb'] );
|
||||
\PRAutoBlogger_Run_Context::clear();
|
||||
\PRAutoBlogger_Audit_Writer::flush_cache();
|
||||
parent::tearDown();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a parsed-response fixture in the Response_Parser shape.
|
||||
*
|
||||
* @param string $content Visible content.
|
||||
* @param string $finish finish_reason value.
|
||||
* @return array
|
||||
*/
|
||||
private function parsed( string $content, string $finish ): array {
|
||||
return [
|
||||
'content' => $content,
|
||||
'model' => 'deepseek/deepseek-v4-flash',
|
||||
'prompt_tokens' => 1200,
|
||||
'completion_tokens' => 4000,
|
||||
'reasoning_tokens' => 4000,
|
||||
'total_tokens' => 5200,
|
||||
'finish_reason' => $finish,
|
||||
'reasoning_content' => 'thinking…',
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Healthy completion passes through untouched; the retry callable is
|
||||
* never invoked.
|
||||
*/
|
||||
public function test_healthy_completion_passes_through(): void {
|
||||
$parsed = $this->parsed( 'A full article.', 'stop' );
|
||||
$invoked = 0;
|
||||
|
||||
$result = ( new \PRAutoBlogger_OpenRouter_Completion_Guard() )->finalize(
|
||||
$parsed,
|
||||
[ 'stage' => 'draft' ],
|
||||
false,
|
||||
function () use ( &$invoked ) {
|
||||
++$invoked;
|
||||
return [];
|
||||
}
|
||||
);
|
||||
|
||||
$this->assertSame( $parsed, $result );
|
||||
$this->assertSame( 0, $invoked );
|
||||
}
|
||||
|
||||
/**
|
||||
* The incident shape — empty content + finish_reason=length — retries
|
||||
* exactly once with reasoning disabled and the empty_retry marker set,
|
||||
* and returns the retry's result.
|
||||
*/
|
||||
public function test_empty_plus_length_retries_once_with_reasoning_disabled(): void {
|
||||
$recovered = $this->parsed( 'Recovered article.', 'stop' );
|
||||
$captured_opts = null;
|
||||
|
||||
$result = ( new \PRAutoBlogger_OpenRouter_Completion_Guard() )->finalize(
|
||||
$this->parsed( '', 'length' ),
|
||||
[ 'stage' => 'draft', 'prompt_key' => 'content.single_pass' ],
|
||||
true,
|
||||
function ( array $retry_options ) use ( &$captured_opts, $recovered ) {
|
||||
$captured_opts = $retry_options;
|
||||
return $recovered;
|
||||
}
|
||||
);
|
||||
|
||||
$this->assertSame( $recovered, $result );
|
||||
$this->assertSame( [ 'enabled' => false ], $captured_opts['reasoning'] );
|
||||
$this->assertTrue( $captured_opts['empty_retry'] );
|
||||
$this->assertSame( 'draft', $captured_opts['stage'] );
|
||||
}
|
||||
|
||||
/**
|
||||
* Whitespace-only content counts as empty (the guard trims).
|
||||
*/
|
||||
public function test_whitespace_only_content_counts_as_empty(): void {
|
||||
$invoked = 0;
|
||||
|
||||
( new \PRAutoBlogger_OpenRouter_Completion_Guard() )->finalize(
|
||||
$this->parsed( " \n\t ", 'length' ),
|
||||
[ 'stage' => 'polish' ],
|
||||
true,
|
||||
function ( array $retry_options ) use ( &$invoked ) {
|
||||
++$invoked;
|
||||
return $this->parsed( 'ok', 'stop' );
|
||||
}
|
||||
);
|
||||
|
||||
$this->assertSame( 1, $invoked );
|
||||
}
|
||||
|
||||
/**
|
||||
* An attempt already marked empty_retry is never retried again — the
|
||||
* guard fails the call instead (no infinite retry loops).
|
||||
*/
|
||||
public function test_already_retried_empty_fails_without_second_retry(): void {
|
||||
$invoked = 0;
|
||||
|
||||
try {
|
||||
( new \PRAutoBlogger_OpenRouter_Completion_Guard() )->finalize(
|
||||
$this->parsed( '', 'length' ),
|
||||
[ 'stage' => 'draft', 'empty_retry' => true ],
|
||||
false,
|
||||
function () use ( &$invoked ) {
|
||||
++$invoked;
|
||||
return [];
|
||||
}
|
||||
);
|
||||
$this->fail( 'Expected RuntimeException for retried-and-still-empty completion.' );
|
||||
} catch ( \RuntimeException $e ) {
|
||||
$this->assertStringContainsString( 'empty content', $e->getMessage() );
|
||||
$this->assertStringContainsString( 'after a reasoning-disabled retry', $e->getMessage() );
|
||||
}
|
||||
|
||||
$this->assertSame( 0, $invoked );
|
||||
}
|
||||
|
||||
/**
|
||||
* Empty content with finish_reason=stop and no reasoning active: a
|
||||
* reasoning-disabled retry cannot plausibly help — fail immediately.
|
||||
*/
|
||||
public function test_unexplained_empty_fails_without_retry(): void {
|
||||
$invoked = 0;
|
||||
|
||||
try {
|
||||
( new \PRAutoBlogger_OpenRouter_Completion_Guard() )->finalize(
|
||||
$this->parsed( '', 'stop' ),
|
||||
[ 'stage' => 'review' ],
|
||||
false,
|
||||
function () use ( &$invoked ) {
|
||||
++$invoked;
|
||||
return [];
|
||||
}
|
||||
);
|
||||
$this->fail( 'Expected RuntimeException for unexplained empty completion.' );
|
||||
} catch ( \RuntimeException $e ) {
|
||||
$this->assertStringContainsString( 'stage "review"', $e->getMessage() );
|
||||
$this->assertStringContainsString( 'finish_reason=stop', $e->getMessage() );
|
||||
}
|
||||
|
||||
$this->assertSame( 0, $invoked );
|
||||
}
|
||||
|
||||
/**
|
||||
* Empty content with reasoning active retries even when finish_reason
|
||||
* is not 'length' (reasoning can eat the budget without a length flag).
|
||||
*/
|
||||
public function test_empty_with_reasoning_active_retries_on_any_finish_reason(): void {
|
||||
$invoked = 0;
|
||||
|
||||
( new \PRAutoBlogger_OpenRouter_Completion_Guard() )->finalize(
|
||||
$this->parsed( '', 'stop' ),
|
||||
[ 'stage' => 'draft' ],
|
||||
true,
|
||||
function ( array $retry_options ) use ( &$invoked ) {
|
||||
++$invoked;
|
||||
return $this->parsed( 'ok', 'stop' );
|
||||
}
|
||||
);
|
||||
|
||||
$this->assertSame( 1, $invoked );
|
||||
}
|
||||
|
||||
/**
|
||||
* The failed attempt is booked to generation_log with its REAL status:
|
||||
* response_status='error', the stage, the model, and the token burn.
|
||||
*/
|
||||
public function test_failed_attempt_books_generation_log_row_with_error_status(): void {
|
||||
$captured_row = null;
|
||||
|
||||
$wpdb = $this->create_mock_wpdb();
|
||||
$wpdb->method( 'insert' )->willReturnCallback(
|
||||
function ( $table, $row ) use ( &$captured_row ) {
|
||||
if ( false !== strpos( (string) $table, 'generation_log' ) ) {
|
||||
$captured_row = $row;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
);
|
||||
$wpdb->insert_id = 9;
|
||||
$GLOBALS['wpdb'] = $wpdb;
|
||||
|
||||
try {
|
||||
( new \PRAutoBlogger_OpenRouter_Completion_Guard() )->finalize(
|
||||
$this->parsed( '', 'stop' ),
|
||||
[ 'stage' => 'draft' ],
|
||||
false,
|
||||
function () {
|
||||
return [];
|
||||
}
|
||||
);
|
||||
$this->fail( 'Expected RuntimeException.' );
|
||||
} catch ( \RuntimeException $e ) {
|
||||
$this->assertIsArray( $captured_row );
|
||||
$this->assertSame( 'error', $captured_row['response_status'] );
|
||||
$this->assertSame( 'draft', $captured_row['stage'] );
|
||||
$this->assertSame( 'deepseek/deepseek-v4-flash', $captured_row['model'] );
|
||||
$this->assertSame( 4000, $captured_row['completion_tokens'] );
|
||||
$this->assertStringContainsString( 'Empty completion', $captured_row['error_message'] );
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -166,4 +166,153 @@ class OpenRouterProviderTest extends BaseTestCase {
|
||||
$this->expectException( \RuntimeException::class );
|
||||
$provider->send_chat_completion( $messages, 'model/test', $options );
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire the provider for a full send_chat_completion run: decryptable
|
||||
* API key, reasoning settings, HTTP stubs that replay scripted
|
||||
* responses, and a wpdb mock for the guard's failure bookkeeping.
|
||||
*
|
||||
* @param array $option_overrides Extra get_option values.
|
||||
* @param array $responses Scripted JSON-decodable response bodies.
|
||||
* @param array $requests (by ref) captures decoded request bodies.
|
||||
*/
|
||||
private function wire_full_provider( array $option_overrides, array $responses, array &$requests ): void {
|
||||
\PRAutoBlogger_Run_Context::clear();
|
||||
|
||||
$encrypted_key = \PRAutoBlogger_Encryption::encrypt( 'sk-or-v1-unit-test-key-0000' );
|
||||
|
||||
$this->stub_get_option( array_merge( [
|
||||
'prautoblogger_openrouter_api_key' => $encrypted_key,
|
||||
'prautoblogger_log_level' => 'error',
|
||||
], $option_overrides ) );
|
||||
|
||||
Functions\when( 'absint' )->alias( function ( $val ) {
|
||||
return abs( (int) $val );
|
||||
} );
|
||||
|
||||
Functions\when( 'wp_remote_post' )->alias(
|
||||
function ( $url, $args ) use ( &$requests, $responses ) {
|
||||
$requests[] = json_decode( $args['body'], true );
|
||||
$body = $responses[ count( $requests ) - 1 ];
|
||||
return [
|
||||
'body' => $body,
|
||||
'response' => [ 'code' => 200 ],
|
||||
];
|
||||
}
|
||||
);
|
||||
Functions\when( 'wp_remote_retrieve_response_code' )->alias(
|
||||
function ( $response ) {
|
||||
return $response['response']['code'];
|
||||
}
|
||||
);
|
||||
Functions\when( 'wp_remote_retrieve_body' )->alias(
|
||||
function ( $response ) {
|
||||
return $response['body'];
|
||||
}
|
||||
);
|
||||
|
||||
// The guard books failed attempts through Cost_Tracker + Logger.
|
||||
$wpdb = $this->create_mock_wpdb();
|
||||
$wpdb->method( 'insert' )->willReturn( 1 );
|
||||
$wpdb->insert_id = 11;
|
||||
$GLOBALS['wpdb'] = $wpdb;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.18.1 incident shape, end to end: reasoning enabled (uncapped
|
||||
* effort mode), the first response burns the whole completion budget
|
||||
* on thinking and returns empty content with finish_reason=length —
|
||||
* the provider retries exactly once with reasoning disabled and
|
||||
* returns the recovered completion.
|
||||
*/
|
||||
public function test_empty_length_completion_retries_once_with_reasoning_disabled(): void {
|
||||
$requests = [];
|
||||
|
||||
$empty_response = json_encode( [
|
||||
'model' => 'deepseek/deepseek-v4-flash',
|
||||
'choices' => [ [ 'message' => [ 'content' => '', 'reasoning' => 'endless thinking…' ], 'finish_reason' => 'length' ] ],
|
||||
'usage' => [ 'prompt_tokens' => 1200, 'completion_tokens' => 4000, 'reasoning_tokens' => 4000, 'total_tokens' => 5200 ],
|
||||
] );
|
||||
$good_response = json_encode( [
|
||||
'model' => 'deepseek/deepseek-v4-flash',
|
||||
'choices' => [ [ 'message' => [ 'content' => 'Recovered article.' ], 'finish_reason' => 'stop' ] ],
|
||||
'usage' => [ 'prompt_tokens' => 1200, 'completion_tokens' => 900, 'total_tokens' => 2100 ],
|
||||
] );
|
||||
|
||||
$this->wire_full_provider(
|
||||
[
|
||||
'prautoblogger_reasoning_enabled' => '1',
|
||||
'prautoblogger_reasoning_effort' => 'xhigh',
|
||||
'prautoblogger_reasoning_max_tokens' => 0,
|
||||
],
|
||||
[ $empty_response, $good_response ],
|
||||
$requests
|
||||
);
|
||||
|
||||
try {
|
||||
$provider = new \PRAutoBlogger_OpenRouter_Provider();
|
||||
$result = $provider->send_chat_completion(
|
||||
[ [ 'role' => 'user', 'content' => 'Write the article.' ] ],
|
||||
'deepseek/deepseek-v4-flash',
|
||||
[ 'temperature' => 0.7, 'max_tokens' => 4000, 'stage' => 'draft' ]
|
||||
);
|
||||
|
||||
$this->assertSame( 'Recovered article.', $result['content'] );
|
||||
$this->assertCount( 2, $requests );
|
||||
// First attempt: global reasoning, uncapped effort mode.
|
||||
$this->assertSame( [ 'enabled' => true, 'effort' => 'xhigh' ], $requests[0]['reasoning'] );
|
||||
$this->assertSame( 4000, $requests[0]['max_tokens'] );
|
||||
// Retry: reasoning disabled per call.
|
||||
$this->assertSame( [ 'enabled' => false ], $requests[1]['reasoning'] );
|
||||
// Caller metadata never reaches the HTTP body.
|
||||
$this->assertArrayNotHasKey( 'stage', $requests[0] );
|
||||
$this->assertArrayNotHasKey( 'empty_retry', $requests[1] );
|
||||
} finally {
|
||||
unset( $GLOBALS['wpdb'] );
|
||||
\PRAutoBlogger_Run_Context::clear();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.18.1 reasoning budget: with the token cap active (default), the
|
||||
* outgoing request caps thinking via reasoning.max_tokens and raises
|
||||
* max_tokens by the cap — and a healthy completion needs no retry.
|
||||
*/
|
||||
public function test_reasoning_cap_shapes_the_outgoing_request(): void {
|
||||
$requests = [];
|
||||
|
||||
$good_response = json_encode( [
|
||||
'model' => 'deepseek/deepseek-v4-flash',
|
||||
'choices' => [ [ 'message' => [ 'content' => 'Article text.' ], 'finish_reason' => 'stop' ] ],
|
||||
'usage' => [ 'prompt_tokens' => 1200, 'completion_tokens' => 900, 'total_tokens' => 2100 ],
|
||||
] );
|
||||
|
||||
$this->wire_full_provider(
|
||||
[
|
||||
'prautoblogger_reasoning_enabled' => '1',
|
||||
'prautoblogger_reasoning_effort' => 'xhigh',
|
||||
'prautoblogger_reasoning_max_tokens' => 2048,
|
||||
],
|
||||
[ $good_response ],
|
||||
$requests
|
||||
);
|
||||
|
||||
try {
|
||||
$provider = new \PRAutoBlogger_OpenRouter_Provider();
|
||||
$result = $provider->send_chat_completion(
|
||||
[ [ 'role' => 'user', 'content' => 'Write the article.' ] ],
|
||||
'deepseek/deepseek-v4-flash',
|
||||
[ 'temperature' => 0.7, 'max_tokens' => 4000, 'stage' => 'draft' ]
|
||||
);
|
||||
|
||||
$this->assertSame( 'Article text.', $result['content'] );
|
||||
$this->assertCount( 1, $requests );
|
||||
$this->assertSame( 2048, $requests[0]['reasoning']['max_tokens'] );
|
||||
$this->assertArrayNotHasKey( 'effort', $requests[0]['reasoning'] );
|
||||
$this->assertSame( 6048, $requests[0]['max_tokens'] );
|
||||
} finally {
|
||||
unset( $GLOBALS['wpdb'] );
|
||||
\PRAutoBlogger_Run_Context::clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
189
tests/unit/Providers/OpenRouterRequestBuilderTest.php
Normal file
189
tests/unit/Providers/OpenRouterRequestBuilderTest.php
Normal file
@@ -0,0 +1,189 @@
|
||||
<?php
|
||||
/**
|
||||
* Tests for PRAutoBlogger_OpenRouter_Request_Builder::build_body() (v0.18.1).
|
||||
*
|
||||
* Validates body assembly, the global-vs-per-call reasoning precedence, the
|
||||
* reasoning token cap + completion headroom, and that caller-metadata keys
|
||||
* never leak into the HTTP body.
|
||||
*
|
||||
* @package PRAutoBlogger\Tests\Providers
|
||||
*/
|
||||
|
||||
namespace PRAutoBlogger\Tests\Providers;
|
||||
|
||||
use PRAutoBlogger\Tests\BaseTestCase;
|
||||
use Brain\Monkey\Functions;
|
||||
|
||||
class OpenRouterRequestBuilderTest extends BaseTestCase {
|
||||
|
||||
private const MESSAGES = [
|
||||
[ 'role' => 'system', 'content' => 'sys' ],
|
||||
[ 'role' => 'user', 'content' => 'usr' ],
|
||||
];
|
||||
|
||||
protected function setUp(): void {
|
||||
parent::setUp();
|
||||
Functions\when( 'absint' )->alias( function ( $val ) {
|
||||
return abs( (int) $val );
|
||||
} );
|
||||
}
|
||||
|
||||
/**
|
||||
* Reasoning enabled globally + cap set: thinking is capped via
|
||||
* reasoning.max_tokens, effort is dropped, and the completion budget
|
||||
* is raised by the cap so content keeps its full allowance.
|
||||
*/
|
||||
public function test_reasoning_cap_replaces_effort_and_raises_headroom(): void {
|
||||
$this->stub_get_option( [
|
||||
'prautoblogger_reasoning_enabled' => '1',
|
||||
'prautoblogger_reasoning_effort' => 'xhigh',
|
||||
'prautoblogger_reasoning_max_tokens' => 2048,
|
||||
] );
|
||||
|
||||
$body = ( new \PRAutoBlogger_OpenRouter_Request_Builder() )
|
||||
->build_body( self::MESSAGES, 'deepseek/deepseek-v4-flash', [ 'max_tokens' => 4000 ] );
|
||||
|
||||
$this->assertSame( 2048, $body['reasoning']['max_tokens'] );
|
||||
$this->assertArrayNotHasKey( 'effort', $body['reasoning'] );
|
||||
$this->assertSame( 6048, $body['max_tokens'] );
|
||||
$this->assertTrue( $body['reasoning']['enabled'] );
|
||||
}
|
||||
|
||||
/**
|
||||
* Cap setting 0 = pure effort mode: the historical body shape is
|
||||
* preserved exactly (effort kept, no reasoning.max_tokens, no headroom).
|
||||
*/
|
||||
public function test_cap_zero_keeps_pure_effort_mode(): void {
|
||||
$this->stub_get_option( [
|
||||
'prautoblogger_reasoning_enabled' => '1',
|
||||
'prautoblogger_reasoning_effort' => 'xhigh',
|
||||
'prautoblogger_reasoning_max_tokens' => 0,
|
||||
] );
|
||||
|
||||
$body = ( new \PRAutoBlogger_OpenRouter_Request_Builder() )
|
||||
->build_body( self::MESSAGES, 'deepseek/deepseek-v4-flash', [ 'max_tokens' => 4000 ] );
|
||||
|
||||
$this->assertSame( [ 'enabled' => true, 'effort' => 'xhigh' ], $body['reasoning'] );
|
||||
$this->assertSame( 4000, $body['max_tokens'] );
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-call reasoning override (the guard's retry) beats the global
|
||||
* setting; a disabled override gets no cap and no headroom.
|
||||
*/
|
||||
public function test_per_call_disabled_override_beats_global_setting(): void {
|
||||
$this->stub_get_option( [
|
||||
'prautoblogger_reasoning_enabled' => '1',
|
||||
'prautoblogger_reasoning_effort' => 'xhigh',
|
||||
'prautoblogger_reasoning_max_tokens' => 2048,
|
||||
] );
|
||||
|
||||
$body = ( new \PRAutoBlogger_OpenRouter_Request_Builder() )
|
||||
->build_body(
|
||||
self::MESSAGES,
|
||||
'deepseek/deepseek-v4-flash',
|
||||
[
|
||||
'max_tokens' => 4000,
|
||||
'reasoning' => [ 'enabled' => false ],
|
||||
]
|
||||
);
|
||||
|
||||
$this->assertSame( [ 'enabled' => false ], $body['reasoning'] );
|
||||
$this->assertSame( 4000, $body['max_tokens'] );
|
||||
}
|
||||
|
||||
/**
|
||||
* Global reasoning off + no override: no reasoning key at all and no
|
||||
* budget math (historical default path).
|
||||
*/
|
||||
public function test_reasoning_off_produces_no_reasoning_key(): void {
|
||||
$this->stub_get_option( [
|
||||
'prautoblogger_reasoning_enabled' => '0',
|
||||
] );
|
||||
|
||||
$body = ( new \PRAutoBlogger_OpenRouter_Request_Builder() )
|
||||
->build_body(
|
||||
self::MESSAGES,
|
||||
'google/gemini-2.5-flash-lite',
|
||||
[
|
||||
'temperature' => 0.7,
|
||||
'max_tokens' => 4000,
|
||||
'response_format' => [ 'type' => 'json_object' ],
|
||||
]
|
||||
);
|
||||
|
||||
$this->assertArrayNotHasKey( 'reasoning', $body );
|
||||
$this->assertSame( 0.7, $body['temperature'] );
|
||||
$this->assertSame( 4000, $body['max_tokens'] );
|
||||
$this->assertSame( [ 'type' => 'json_object' ], $body['response_format'] );
|
||||
}
|
||||
|
||||
/**
|
||||
* An explicit caller reasoning config with its own max_tokens is
|
||||
* respected (no cap overwrite) but still gets completion headroom.
|
||||
*/
|
||||
public function test_caller_reasoning_max_tokens_respected_with_headroom(): void {
|
||||
$this->stub_get_option( [
|
||||
'prautoblogger_reasoning_enabled' => '0',
|
||||
'prautoblogger_reasoning_max_tokens' => 2048,
|
||||
] );
|
||||
|
||||
$body = ( new \PRAutoBlogger_OpenRouter_Request_Builder() )
|
||||
->build_body(
|
||||
self::MESSAGES,
|
||||
'deepseek/deepseek-v4-flash',
|
||||
[
|
||||
'max_tokens' => 1000,
|
||||
'reasoning' => [ 'enabled' => true, 'max_tokens' => 512 ],
|
||||
]
|
||||
);
|
||||
|
||||
$this->assertSame( 512, $body['reasoning']['max_tokens'] );
|
||||
$this->assertSame( 1512, $body['max_tokens'] );
|
||||
}
|
||||
|
||||
/**
|
||||
* Caller-metadata keys (stage, prompt_key, empty_retry) must never
|
||||
* appear in the HTTP body.
|
||||
*/
|
||||
public function test_caller_metadata_keys_never_leak_into_body(): void {
|
||||
$this->stub_get_option( [
|
||||
'prautoblogger_reasoning_enabled' => '0',
|
||||
] );
|
||||
|
||||
$body = ( new \PRAutoBlogger_OpenRouter_Request_Builder() )
|
||||
->build_body(
|
||||
self::MESSAGES,
|
||||
'google/gemini-2.5-flash-lite',
|
||||
[
|
||||
'max_tokens' => 4000,
|
||||
'stage' => 'draft',
|
||||
'prompt_key' => 'content.single_pass',
|
||||
'empty_retry' => true,
|
||||
]
|
||||
);
|
||||
|
||||
$this->assertArrayNotHasKey( 'stage', $body );
|
||||
$this->assertArrayNotHasKey( 'prompt_key', $body );
|
||||
$this->assertArrayNotHasKey( 'empty_retry', $body );
|
||||
$this->assertSame( [ 'model', 'messages', 'max_tokens' ], array_keys( $body ) );
|
||||
}
|
||||
|
||||
/**
|
||||
* Reasoning enabled but the call has no completion ceiling: nothing to
|
||||
* protect, body passes through with effort untouched.
|
||||
*/
|
||||
public function test_no_max_tokens_means_no_budget_math(): void {
|
||||
$this->stub_get_option( [
|
||||
'prautoblogger_reasoning_enabled' => '1',
|
||||
'prautoblogger_reasoning_effort' => 'high',
|
||||
'prautoblogger_reasoning_max_tokens' => 2048,
|
||||
] );
|
||||
|
||||
$body = ( new \PRAutoBlogger_OpenRouter_Request_Builder() )
|
||||
->build_body( self::MESSAGES, 'deepseek/deepseek-v4-flash', [] );
|
||||
|
||||
$this->assertSame( [ 'enabled' => true, 'effort' => 'high' ], $body['reasoning'] );
|
||||
$this->assertArrayNotHasKey( 'max_tokens', $body );
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user