peptide-repo/prautoblogger
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Wave 2 eval harness + image pipeline instrumentation (v0.13.0)

Browse Source 
Closes #119

- Extract PRAutoBlogger_Image_Scene_Parser from image prompt builder (331→300 lines)
- Add frozen eval dataset (20 peptide topics at eval/dataset.json)
- Add LLM-as-judge scorer (gemini-flash-1.5 via OpenRouter, 3 axes)
- Add eval runner orchestrator with dry-run and Opik push support
- Add WP-CLI command: wp prautoblogger opik:eval [--limit=<n>] [--dry-run]
- Add image pipeline Opik instrumentation (image_prompt_rewrite span)
- Add content generator eval mode (suppresses publish + image gen)
- Version: v0.13.0
This commit is contained in:
peptiderepo
2026-04-24 12:03:33 +07:00
committed by GitHub
parent c1d22fd275
commit a8ba7a8ea9
11 changed files with 876 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
includes/core/class-content-generator.php
View File

@@ -29,7 +29,10 @@ class PRAutoBlogger_Content_Generator {
* @param PRAutoBlogger_Article_Idea $idea The scored idea to generate content for.
* @return string Generated HTML content.
*/
public function generate( PRAutoBlogger_Article_Idea $idea ): string {
public function generate( PRAutoBlogger_Article_Idea $idea, bool $eval_mode = false ): string {
// In eval mode, suppress publishing and image generation side effects.
define( 'PRAUTOBLOGGER_EVAL_MODE', $eval_mode );
$mode = get_option( 'prautoblogger_writing_pipeline', 'multi_step' );
$request = new PRAutoBlogger_Content_Request(
@@ -285,4 +288,14 @@ class PRAutoBlogger_Content_Generator {
return $response['content'];
}
/**
* Generate content in eval mode (no publishing, no image generation).
*
* @param PRAutoBlogger_Article_Idea $idea The scored idea to generate content for.
* @return string Generated HTML content.
*/
public function generate_eval( PRAutoBlogger_Article_Idea $idea ): string {
return $this->generate( $idea, true );
}
}

6
includes/core/class-image-pipeline.php
View File

@@ -47,12 +47,14 @@ class PRAutoBlogger_Image_Pipeline {
*/
public function __construct(
?PRAutoBlogger_Image_Provider_Interface $provider = null,
?PRAutoBlogger_Cost_Tracker $cost_tracker = null
?PRAutoBlogger_Cost_Tracker $cost_tracker = null,
?PRAutoBlogger_Opik_Trace_Context $trace_context = null
) {
$this->provider = $provider ?? self::create_default_provider();
$this->prompt_builder = new PRAutoBlogger_Image_Prompt_Builder();
$this->prompt_builder = new PRAutoBlogger_Image_Prompt_Builder( $trace_context );
$this->sideloader = new PRAutoBlogger_Image_Media_Sideloader();
$this->cost_tracker = $cost_tracker ?? new PRAutoBlogger_Cost_Tracker();
$this->trace_context = $trace_context;
}
/**

136
includes/core/class-image-prompt-builder.php
View File

@@ -65,6 +65,22 @@ PROMPT;
*/
private ?PRAutoBlogger_OpenRouter_Provider $llm = null;
/**
* Opik trace context for instrumentation.
*
* @var PRAutoBlogger_Opik_Trace_Context|null
*/
private ?PRAutoBlogger_Opik_Trace_Context $trace_context = null;
/**
* Constructor.
*
* @param PRAutoBlogger_Opik_Trace_Context|null $trace_context Optional trace context for instrumentation.
*/
public function __construct( ?PRAutoBlogger_Opik_Trace_Context $trace_context = null ) {
$this->trace_context = $trace_context;
}
/**
* Build a visual prompt from finished article content.
*
@@ -78,7 +94,7 @@ PROMPT;
public function build_article_prompt( array $article_data ): array {
$title = $article_data['post_title'] ?? $article_data['suggested_title'] ?? 'Product';
$content = $article_data['post_content'] ?? '';
$first_para = $this->extract_first_paragraph( $content );
$first_para = PRAutoBlogger_Image_Scene_Parser::extract_first_paragraph( $content );
$parsed = $this->rewrite_via_llm( $title, $first_para );
@@ -142,7 +158,24 @@ PROMPT;
$user_message .= "\n\nSummary: {$context}";
}
$span_id = null;
try {
if ( null !== $this->trace_context ) {
$span_id = $this->trace_context->start_span(
array(
'name' => 'image_prompt_rewrite',
'type' => 'llm',
'model' => get_option( 'prautoblogger_analysis_model', PRAUTOBLOGGER_DEFAULT_ANALYSIS_MODEL ),
'provider' => 'openrouter',
'input' => array(
'title' => $title,
'context' => substr( $context, 0, 200 ),
),
)
);
}
$llm = $this->get_llm_provider();
$model = get_option( 'prautoblogger_analysis_model', PRAUTOBLOGGER_DEFAULT_ANALYSIS_MODEL );
$system = $this->resolve_system_prompt();
@@ -188,8 +221,24 @@ PROMPT;
'image_prompt_builder'
);
if ( null !== $span_id && null !== $this->trace_context ) {
$this->trace_context->end_span(
$span_id,
array(
'output' => array(
'raw_response' => substr( $raw, 0, 200 ),
),
'usage' => array(
'prompt_tokens' => $result['prompt_tokens'] ?? 0,
'completion_tokens' => $result['completion_tokens'] ?? 0,
'total_tokens' => ( $result['prompt_tokens'] ?? 0 ) + ( $result['completion_tokens'] ?? 0 ),
),
)
);
}
if ( '' !== $raw ) {
return $this->parse_scene_and_caption( $raw );
return PRAutoBlogger_Image_Scene_Parser::parse_scene_and_caption( $raw );
}
} catch ( \Throwable $e ) {
// LLM failure is not fatal — fall back to rule-based synthesis.
@@ -199,64 +248,7 @@ PROMPT;
);
}
return $this->synthesize_visual_concepts_fallback( $title, $context );
}
/**
* Parse the LLM response into separate scene and caption parts.
*
* Expected format from the LLM:
* Scene description text here.
*
* "Caption punchline here."
*
* Falls back gracefully: if no blank-line separator is found, the entire
* response becomes the scene with an empty caption.
*
* @param string $raw Raw LLM response text.
* @return array{scene: string, caption: string}
*/
private function parse_scene_and_caption( string $raw ): array {
// Split on blank line (the format the system prompt requests).
$parts = preg_split( '/\n\s*\n/', $raw, 2 );
if ( count( $parts ) >= 2 ) {
$scene = trim( $parts[0] );
$caption = trim( $parts[1] );
// Strip surrounding quotes from caption — the LLM wraps it in quotes.
$caption = trim( $caption, '"\'' );
return array(
'scene' => $scene,
'caption' => $caption,
);
}
// No blank-line separator found — treat entire response as scene.
return array(
'scene' => trim( $raw ),
'caption' => '',
);
}
/**
* Extract the first paragraph from HTML content.
*
* Removes all tags and captures the text up to the first double-newline
* or 200 characters, whichever comes first.
*
* @param string $html HTML content.
* @return string Plain text of the first paragraph.
*/
private function extract_first_paragraph( string $html ): string {
$text = wp_strip_all_tags( $html );
$paras = preg_split( '/\n\n+/', $text );
$para = isset( $paras[0] ) ? trim( $paras[0] ) : '';
if ( strlen( $para ) > 200 ) {
$para = substr( $para, 0, 200 ) . '...';
}
return $para;
return PRAutoBlogger_Image_Scene_Parser::synthesize_visual_concepts_fallback( $title, $context );
}
/**
@@ -268,7 +260,7 @@ PROMPT;
* @return array{prompt: string, caption: string}
*/
public function build_fallback_prompt( string $title ): array {
$parsed = $this->synthesize_visual_concepts_fallback( $title, '' );
$parsed = PRAutoBlogger_Image_Scene_Parser::synthesize_visual_concepts_fallback( $title, '' );
$style_suffix = $this->get_style_suffix();
return array(
'prompt' => trim( $parsed['scene'] . ' ' . $style_suffix ),
@@ -276,28 +268,6 @@ PROMPT;
);
}
/**
* Rule-based fallback when LLM rewriting is unavailable.
*
* Kept deliberately simple — this only runs if OpenRouter is down.
*
* @param string $title Main heading / topic.
* @param string $supporting_text Additional context.
* @return array{scene: string, caption: string} Scene for image gen, caption for HTML.
*/
private function synthesize_visual_concepts_fallback( string $title, string $supporting_text ): array {
// Fallback produces a simple comic concept when the LLM is unavailable.
$scene = sprintf(
'A cartoon scientist in a lab coat looking bewildered while examining something related to: %s.',
$title
);
return array(
'scene' => trim( $scene ),
'caption' => 'Science is full of surprises.',
);
}
/** Lazy-load the OpenRouter provider. */
private function get_llm_provider(): PRAutoBlogger_OpenRouter_Provider {
if ( null === $this->llm ) {

91
includes/core/class-image-scene-parser.php Normal file
View File

@@ -0,0 +1,91 @@
<?php
declare(strict_types=1);
/**
* Parses LLM output into scene/caption pairs and synthesises fallback prompts
* from article titles when the LLM call fails.
*
* Who calls it: PRAutoBlogger_Image_Prompt_Builder (static calls).
* Dependencies: None.
*/
class PRAutoBlogger_Image_Scene_Parser {
/**
* Parse the LLM response into separate scene and caption parts.
*
* Expected format from the LLM:
* Scene description text here.
*
* "Caption punchline here."
*
* Falls back gracefully: if no blank-line separator is found, the entire
* response becomes the scene with an empty caption.
*
* @param string $raw Raw LLM response text.
* @return array{scene: string, caption: string}
*/
public static function parse_scene_and_caption( string $raw ): array {
// Split on blank line (the format the system prompt requests).
$parts = preg_split( '/\n\s*\n/', $raw, 2 );
if ( count( $parts ) >= 2 ) {
$scene = trim( $parts[0] );
$caption = trim( $parts[1] );
// Strip surrounding quotes from caption — the LLM wraps it in quotes.
$caption = trim( $caption, '"\'' );
return array(
'scene' => $scene,
'caption' => $caption,
);
}
// No blank-line separator found — treat entire response as scene.
return array(
'scene' => trim( $raw ),
'caption' => '',
);
}
/**
* Extract the first paragraph from HTML content.
*
* Removes all tags and captures the text up to the first double-newline
* or 200 characters, whichever comes first.
*
* @param string $html HTML content.
* @return string Plain text of the first paragraph.
*/
public static function extract_first_paragraph( string $html ): string {
$text = wp_strip_all_tags( $html );
$paras = preg_split( '/\n\n+/', $text );
$para = isset( $paras[0] ) ? trim( $paras[0] ) : '';
if ( strlen( $para ) > 200 ) {
$para = substr( $para, 0, 200 ) . '...';
}
return $para;
}
/**
* Rule-based fallback when LLM rewriting is unavailable.
*
* Kept deliberately simple — this only runs if OpenRouter is down.
*
* @param string $title Main heading / topic.
* @param string $supporting_text Additional context.
* @return array{scene: string, caption: string} Scene for image gen, caption for HTML.
*/
public static function synthesize_visual_concepts_fallback( string $title, string $supporting_text ): array {
// Fallback produces a simple comic concept when the LLM is unavailable.
$scene = sprintf(
'A cartoon scientist in a lab coat looking bewildered while examining something related to: %s.',
$title
);
return array(
'scene' => trim( $scene ),
'caption' => 'Science is full of surprises.',
);
}
}