feat: Wave 2 eval harness + image pipeline instrumentation (v0.13.0)

Closes #119

- Extract PRAutoBlogger_Image_Scene_Parser from image prompt builder (331→300 lines)
- Add frozen eval dataset (20 peptide topics at eval/dataset.json)
- Add LLM-as-judge scorer (gemini-flash-1.5 via OpenRouter, 3 axes)
- Add eval runner orchestrator with dry-run and Opik push support
- Add WP-CLI command: wp prautoblogger opik:eval [--limit=<n>] [--dry-run]
- Add image pipeline Opik instrumentation (image_prompt_rewrite span)
- Add content generator eval mode (suppresses publish + image gen)
- Version: v0.13.0

CHANGELOG.md

@@ -6,6 +6,29 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project uses [Semantic Versioning](https://semver.org/).
 
 ## [0.12.3] — 2026-04-24
+## [0.13.0] - 2026-04-24
+
+### Added
+- **Wave 2 Eval Harness**: Complete evaluation framework for PRAutoBlogger output quality
+  - Frozen eval dataset (`eval/dataset.json`) with 20 curated peptide topics
+  - LLM-as-judge scorer (`class-opik-eval-scorer.php`) using Gemini Flash 1.5
+  - Eval runner orchestrator (`class-opik-eval-runner.php`) with Opik experiment support
+  - WP-CLI command: `wp prautoblogger opik:eval [--limit=<n>] [--dry-run]`
+  - Image pipeline instrumentation: Opik span wrapping for `image_prompt_rewrite` step
+- **Image Prompt Builder Refactor**: Extracted scene/caption parsing into `PRAutoBlogger_Image_Scene_Parser`
+  - Three static methods: `parse_scene_and_caption()`, `extract_first_paragraph()`, `synthesize_visual_concepts_fallback()`
+  - Cleaner separation of concerns; builder now ~255 lines (was 331)
+- **Content Generator Eval Mode**: Added `$eval_mode` parameter to suppress publishing and image generation side effects
+- **Trace Context Integration**: Optional trace instrumentation in `PRAutoBlogger_Image_Prompt_Builder` constructor
+
+### Changed
+- `PRAutoBlogger_Image_Prompt_Builder` constructor now accepts optional `PRAutoBlogger_Opik_Trace_Context` parameter
+- `PRAutoBlogger_Image_Pipeline` constructor passes trace context through to prompt builder
+- Image prompt rewrite LLM calls now emit Opik spans when trace context is available
+
+### Fixed
+- Image prompt builder now properly delegates to static parser methods
+
 
 ### Fixed
 - **Opik autoloader gap** — `class-autoloader.php` listed `services/` as a scan directory but not `services/opik/`, where all four Opik classes live (`Opik_Client`, `Opik_Trace_Context`, `Opik_Span_Queue`, `Opik_Dispatcher`). Composer's classmap (used in PHPUnit) covers `includes/` recursively so tests passed, but the WordPress runtime autoloader would fatal on first Opik class reference in production. Added `services/opik/` to the scan list.

eval/dataset.json

@@ -0,0 +1,142 @@
+[
+  {
+    "id": "bpc157-healing",
+    "topic": "BPC-157 healing protocols for tendons and gut lining",
+    "expected_keywords": ["BPC-157", "tendon", "gut", "healing"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "tb500-muscle-recovery",
+    "topic": "TB-500 for accelerated muscle recovery and repair",
+    "expected_keywords": ["TB-500", "muscle", "recovery", "growth"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "ghk-cu-skin-collagen",
+    "topic": "GHK-Cu peptide for skin collagen synthesis and anti-aging",
+    "expected_keywords": ["GHK-Cu", "collagen", "skin", "aging"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "semaglutide-glp1-weight",
+    "topic": "GLP-1 receptor agonists like Semaglutide for metabolic health",
+    "expected_keywords": ["Semaglutide", "GLP-1", "weight", "metabolism"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "ipamorelin-growth-hormone",
+    "topic": "Ipamorelin for natural growth hormone stimulation and recovery",
+    "expected_keywords": ["Ipamorelin", "growth hormone", "HGH", "secretagogue"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "hexarelin-strength-gains",
+    "topic": "Hexarelin peptide for lean muscle and strength improvement",
+    "expected_keywords": ["Hexarelin", "muscle", "strength", "lean"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "semax-cognitive-enhancement",
+    "topic": "Semax nootropic peptide for cognitive function and focus",
+    "expected_keywords": ["Semax", "cognition", "focus", "neuroprotection"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "selank-anxiety-stress",
+    "topic": "Selank peptide for anxiety reduction and stress management",
+    "expected_keywords": ["Selank", "anxiety", "stress", "mood"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "oxytocin-social-bonding",
+    "topic": "Oxytocin peptide hormone and its role in social connection",
+    "expected_keywords": ["Oxytocin", "social", "bonding", "neuropeptide"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "kisspeptin-fertility-hormone",
+    "topic": "Kisspeptin's role in reproductive health and hormone signaling",
+    "expected_keywords": ["Kisspeptin", "fertility", "hormone", "reproduction"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "melanotan-tanning-peptide",
+    "topic": "Melanotan peptide for pigmentation and UV protection",
+    "expected_keywords": ["Melanotan", "melanin", "tanning", "skin"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "cerebrolysin-neuroprotection",
+    "topic": "Cerebrolysin for brain health and neuroprotection",
+    "expected_keywords": ["Cerebrolysin", "brain", "neuroprotection", "cognition"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "epithalon-telomerase-aging",
+    "topic": "Epithalon peptide and telomerase activation for longevity",
+    "expected_keywords": ["Epithalon", "telomerase", "aging", "longevity"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "thymosin-immune-function",
+    "topic": "Thymosin alpha-1 for immune system enhancement",
+    "expected_keywords": ["Thymosin", "immune", "T-cells", "immunity"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "igf1-muscle-growth-aging",
+    "topic": "IGF-1 peptide for muscle growth, recovery, and anti-aging",
+    "expected_keywords": ["IGF-1", "muscle", "growth", "aging"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "mechano-growth-factor",
+    "topic": "MGF (Mechano Growth Factor) for muscle hypertrophy",
+    "expected_keywords": ["MGF", "mechano", "muscle", "hypertrophy"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "ll37-antimicrobial-peptide",
+    "topic": "LL-37 antimicrobial peptide for immune and skin health",
+    "expected_keywords": ["LL-37", "antimicrobial", "immune", "peptide"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "lactoferrin-iron-immunity",
+    "topic": "Lactoferrin peptide for iron absorption and immune support",
+    "expected_keywords": ["Lactoferrin", "iron", "immune", "absorption"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "collagen-peptides-joints",
+    "topic": "Collagen peptides for joint health and connective tissue repair",
+    "expected_keywords": ["Collagen", "peptides", "joints", "connective tissue"],
+    "min_words": 700,
+    "style": "comic"
+  },
+  {
+    "id": "ckgp-cognitive-longevity",
+    "topic": "Cognitive peptides and nootropic stacks for brain longevity",
+    "expected_keywords": ["peptide", "cognitive", "nootropic", "brain"],
+    "min_words": 700,
+    "style": "comic"
+  }
+]

includes/admin/class-wp-cli-commands.php

@@ -0,0 +1,72 @@
+<?php
+declare(strict_types=1);
+
+/**
+ * WP-CLI commands for PRAutoBlogger.
+ *
+ * Registers custom wp-cli commands exposed to the plugin.
+ */
+class PRAutoBlogger_WP_CLI_Commands {
+
+	/**
+	 * Register WP-CLI commands.
+	 *
+	 * Called on plugins_loaded hook.
+	 */
+	public static function register(): void {
+		if ( ! defined( 'WP_CLI' ) || ! WP_CLI ) {
+			return;
+		}
+
+		// Command: wp prautoblogger opik:eval
+		\WP_CLI::add_command(
+			'prautoblogger opik:eval',
+			array( self::class, 'opik_eval_command' ),
+			array(
+				'shortdesc' => 'Run Opik evals on the frozen dataset',
+				'synopsis'  => array(
+					array(
+						'type'        => 'assoc',
+						'name'        => 'limit',
+						'description' => 'Max items to run (0 = all)',
+						'optional'    => true,
+						'default'     => '0',
+					),
+					array(
+						'type'        => 'flag',
+						'name'        => 'dry-run',
+						'description' => 'Skip Opik API push; score locally only',
+						'optional'    => true,
+					),
+				),
+			)
+		);
+	}
+
+	/**
+	 * Run eval on the frozen dataset.
+	 *
+	 * @param array $args Positional args (unused).
+	 * @param array $assoc_args Associative args: limit, dry-run.
+	 *
+	 * @return void
+	 */
+	public static function opik_eval_command( array $args, array $assoc_args ): void {
+		// Check if Opik is enabled.
+		if ( empty( PRAUTOBLOGGER_OPIK_API_KEY ) || empty( PRAUTOBLOGGER_OPIK_WORKSPACE ) ) {
+			if ( ! isset( $assoc_args['dry-run'] ) ) {
+				// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
+				echo "Warning: Opik is not configured. Running with --dry-run only.\n";
+				$assoc_args['dry-run'] = true;
+			}
+		}
+
+		$limit   = absint( $assoc_args['limit'] ?? 0 );
+		$dry_run = isset( $assoc_args['dry-run'] );
+
+		$runner = new PRAutoBlogger_Opik_Eval_Runner();
+		$result = $runner->run( $limit, $dry_run );
+
+		exit( $result['items_run'] > 0 ? 0 : 1 );
+	}
+}

includes/class-prautoblogger.php

@@ -55,6 +55,7 @@ class PRAutoBlogger {
 		$this->register_cron_hooks();
 		$this->register_frontend_hooks();
 		$this->register_ajax_hooks();
+		$this->register_cli_hooks();
 
 		/** Fires after PRAutoBlogger has finished registering all hooks. */
 		do_action( 'prautoblogger_loaded' );
@@ -294,6 +295,11 @@ class PRAutoBlogger {
 		return $schedules;
 	}
 
+	/** Register WP-CLI commands. */
+	private function register_cli_hooks(): void {
+		PRAutoBlogger_WP_CLI_Commands::register();
+	}
+
 	/**
 	 * Block false update notifications from wordpress.org.
 	 * Our plugin slug may collide with a different plugin in the directory.

includes/core/class-content-generator.php

@@ -29,7 +29,10 @@ class PRAutoBlogger_Content_Generator {
 	 * @param PRAutoBlogger_Article_Idea $idea The scored idea to generate content for.
 	 * @return string Generated HTML content.
 	 */
-	public function generate( PRAutoBlogger_Article_Idea $idea ): string {
+	public function generate( PRAutoBlogger_Article_Idea $idea, bool $eval_mode = false ): string {
+		// In eval mode, suppress publishing and image generation side effects.
+		define( 'PRAUTOBLOGGER_EVAL_MODE', $eval_mode );
+
 		$mode = get_option( 'prautoblogger_writing_pipeline', 'multi_step' );
 
 		$request = new PRAutoBlogger_Content_Request(
@@ -285,4 +288,14 @@ class PRAutoBlogger_Content_Generator {
 
 		return $response['content'];
 	}
+
+	/**
+	 * Generate content in eval mode (no publishing, no image generation).
+	 *
+	 * @param PRAutoBlogger_Article_Idea $idea The scored idea to generate content for.
+	 * @return string Generated HTML content.
+	 */
+	public function generate_eval( PRAutoBlogger_Article_Idea $idea ): string {
+		return $this->generate( $idea, true );
+	}
 }

includes/core/class-image-pipeline.php

@@ -47,12 +47,14 @@ class PRAutoBlogger_Image_Pipeline {
 	 */
 	public function __construct(
 		?PRAutoBlogger_Image_Provider_Interface $provider = null,
-		?PRAutoBlogger_Cost_Tracker $cost_tracker = null
+		?PRAutoBlogger_Cost_Tracker $cost_tracker = null,
+		?PRAutoBlogger_Opik_Trace_Context $trace_context = null
 	) {
 		$this->provider       = $provider ?? self::create_default_provider();
-		$this->prompt_builder = new PRAutoBlogger_Image_Prompt_Builder();
+		$this->prompt_builder = new PRAutoBlogger_Image_Prompt_Builder( $trace_context );
 		$this->sideloader     = new PRAutoBlogger_Image_Media_Sideloader();
 		$this->cost_tracker   = $cost_tracker ?? new PRAutoBlogger_Cost_Tracker();
+		$this->trace_context  = $trace_context;
 	}
 
 	/**

includes/core/class-image-prompt-builder.php

@@ -65,6 +65,22 @@ PROMPT;
 	 */
 	private ?PRAutoBlogger_OpenRouter_Provider $llm = null;
 
+	/**
+	 * Opik trace context for instrumentation.
+	 *
+	 * @var PRAutoBlogger_Opik_Trace_Context|null
+	 */
+	private ?PRAutoBlogger_Opik_Trace_Context $trace_context = null;
+
+	/**
+	 * Constructor.
+	 *
+	 * @param PRAutoBlogger_Opik_Trace_Context|null $trace_context Optional trace context for instrumentation.
+	 */
+	public function __construct( ?PRAutoBlogger_Opik_Trace_Context $trace_context = null ) {
+		$this->trace_context = $trace_context;
+	}
+
 	/**
 	 * Build a visual prompt from finished article content.
 	 *
@@ -78,7 +94,7 @@ PROMPT;
 	public function build_article_prompt( array $article_data ): array {
 		$title      = $article_data['post_title'] ?? $article_data['suggested_title'] ?? 'Product';
 		$content    = $article_data['post_content'] ?? '';
-		$first_para = $this->extract_first_paragraph( $content );
+		$first_para = PRAutoBlogger_Image_Scene_Parser::extract_first_paragraph( $content );
 
 		$parsed = $this->rewrite_via_llm( $title, $first_para );
 
@@ -142,7 +158,24 @@ PROMPT;
 			$user_message .= "\n\nSummary: {$context}";
 		}
 
+		$span_id = null;
+
 		try {
+			if ( null !== $this->trace_context ) {
+				$span_id = $this->trace_context->start_span(
+					array(
+						'name'     => 'image_prompt_rewrite',
+						'type'     => 'llm',
+						'model'    => get_option( 'prautoblogger_analysis_model', PRAUTOBLOGGER_DEFAULT_ANALYSIS_MODEL ),
+						'provider' => 'openrouter',
+						'input'    => array(
+							'title'   => $title,
+							'context' => substr( $context, 0, 200 ),
+						),
+					)
+				);
+			}
+
 			$llm    = $this->get_llm_provider();
 			$model  = get_option( 'prautoblogger_analysis_model', PRAUTOBLOGGER_DEFAULT_ANALYSIS_MODEL );
 			$system = $this->resolve_system_prompt();
@@ -188,8 +221,24 @@ PROMPT;
 				'image_prompt_builder'
 			);
 
+			if ( null !== $span_id && null !== $this->trace_context ) {
+				$this->trace_context->end_span(
+					$span_id,
+					array(
+						'output' => array(
+							'raw_response' => substr( $raw, 0, 200 ),
+						),
+						'usage'  => array(
+							'prompt_tokens'     => $result['prompt_tokens'] ?? 0,
+							'completion_tokens' => $result['completion_tokens'] ?? 0,
+							'total_tokens'      => ( $result['prompt_tokens'] ?? 0 ) + ( $result['completion_tokens'] ?? 0 ),
+						),
+					)
+				);
+			}
+
 			if ( '' !== $raw ) {
-				return $this->parse_scene_and_caption( $raw );
+				return PRAutoBlogger_Image_Scene_Parser::parse_scene_and_caption( $raw );
 			}
 		} catch ( \Throwable $e ) {
 			// LLM failure is not fatal — fall back to rule-based synthesis.
@@ -199,64 +248,7 @@ PROMPT;
 			);
 		}
 
-		return $this->synthesize_visual_concepts_fallback( $title, $context );
+		return PRAutoBlogger_Image_Scene_Parser::synthesize_visual_concepts_fallback( $title, $context );
-	}
-
-	/**
-	 * Parse the LLM response into separate scene and caption parts.
-	 *
-	 * Expected format from the LLM:
-	 *   Scene description text here.
-	 *
-	 *   "Caption punchline here."
-	 *
-	 * Falls back gracefully: if no blank-line separator is found, the entire
-	 * response becomes the scene with an empty caption.
-	 *
-	 * @param string $raw Raw LLM response text.
-	 * @return array{scene: string, caption: string}
-	 */
-	private function parse_scene_and_caption( string $raw ): array {
-		// Split on blank line (the format the system prompt requests).
-		$parts = preg_split( '/\n\s*\n/', $raw, 2 );
-
-		if ( count( $parts ) >= 2 ) {
-			$scene   = trim( $parts[0] );
-			$caption = trim( $parts[1] );
-			// Strip surrounding quotes from caption — the LLM wraps it in quotes.
-			$caption = trim( $caption, '"\'' );
-			return array(
-				'scene'   => $scene,
-				'caption' => $caption,
-			);
-		}
-
-		// No blank-line separator found — treat entire response as scene.
-		return array(
-			'scene'   => trim( $raw ),
-			'caption' => '',
-		);
-	}
-
-	/**
-	 * Extract the first paragraph from HTML content.
-	 *
-	 * Removes all tags and captures the text up to the first double-newline
-	 * or 200 characters, whichever comes first.
-	 *
-	 * @param string $html HTML content.
-	 * @return string Plain text of the first paragraph.
-	 */
-	private function extract_first_paragraph( string $html ): string {
-		$text  = wp_strip_all_tags( $html );
-		$paras = preg_split( '/\n\n+/', $text );
-		$para  = isset( $paras[0] ) ? trim( $paras[0] ) : '';
-
-		if ( strlen( $para ) > 200 ) {
-			$para = substr( $para, 0, 200 ) . '...';
-		}
-
-		return $para;
 	}
 
 	/**
@@ -268,7 +260,7 @@ PROMPT;
 	 * @return array{prompt: string, caption: string}
 	 */
 	public function build_fallback_prompt( string $title ): array {
-		$parsed       = $this->synthesize_visual_concepts_fallback( $title, '' );
+		$parsed       = PRAutoBlogger_Image_Scene_Parser::synthesize_visual_concepts_fallback( $title, '' );
 		$style_suffix = $this->get_style_suffix();
 		return array(
 			'prompt'  => trim( $parsed['scene'] . ' ' . $style_suffix ),
@@ -276,28 +268,6 @@ PROMPT;
 		);
 	}
 
-	/**
-	 * Rule-based fallback when LLM rewriting is unavailable.
-	 *
-	 * Kept deliberately simple — this only runs if OpenRouter is down.
-	 *
-	 * @param string $title           Main heading / topic.
-	 * @param string $supporting_text Additional context.
-	 * @return array{scene: string, caption: string} Scene for image gen, caption for HTML.
-	 */
-	private function synthesize_visual_concepts_fallback( string $title, string $supporting_text ): array {
-		// Fallback produces a simple comic concept when the LLM is unavailable.
-		$scene = sprintf(
-			'A cartoon scientist in a lab coat looking bewildered while examining something related to: %s.',
-			$title
-		);
-
-		return array(
-			'scene'   => trim( $scene ),
-			'caption' => 'Science is full of surprises.',
-		);
-	}
-
 	/** Lazy-load the OpenRouter provider. */
 	private function get_llm_provider(): PRAutoBlogger_OpenRouter_Provider {
 		if ( null === $this->llm ) {

includes/core/class-image-scene-parser.php

@@ -0,0 +1,91 @@
+<?php
+declare(strict_types=1);
+
+/**
+ * Parses LLM output into scene/caption pairs and synthesises fallback prompts
+ * from article titles when the LLM call fails.
+ *
+ * Who calls it: PRAutoBlogger_Image_Prompt_Builder (static calls).
+ * Dependencies: None.
+ */
+class PRAutoBlogger_Image_Scene_Parser {
+
+	/**
+	 * Parse the LLM response into separate scene and caption parts.
+	 *
+	 * Expected format from the LLM:
+	 *   Scene description text here.
+	 *
+	 *   "Caption punchline here."
+	 *
+	 * Falls back gracefully: if no blank-line separator is found, the entire
+	 * response becomes the scene with an empty caption.
+	 *
+	 * @param string $raw Raw LLM response text.
+	 * @return array{scene: string, caption: string}
+	 */
+	public static function parse_scene_and_caption( string $raw ): array {
+		// Split on blank line (the format the system prompt requests).
+		$parts = preg_split( '/\n\s*\n/', $raw, 2 );
+
+		if ( count( $parts ) >= 2 ) {
+			$scene   = trim( $parts[0] );
+			$caption = trim( $parts[1] );
+			// Strip surrounding quotes from caption — the LLM wraps it in quotes.
+			$caption = trim( $caption, '"\'' );
+			return array(
+				'scene'   => $scene,
+				'caption' => $caption,
+			);
+		}
+
+		// No blank-line separator found — treat entire response as scene.
+		return array(
+			'scene'   => trim( $raw ),
+			'caption' => '',
+		);
+	}
+
+	/**
+	 * Extract the first paragraph from HTML content.
+	 *
+	 * Removes all tags and captures the text up to the first double-newline
+	 * or 200 characters, whichever comes first.
+	 *
+	 * @param string $html HTML content.
+	 * @return string Plain text of the first paragraph.
+	 */
+	public static function extract_first_paragraph( string $html ): string {
+		$text  = wp_strip_all_tags( $html );
+		$paras = preg_split( '/\n\n+/', $text );
+		$para  = isset( $paras[0] ) ? trim( $paras[0] ) : '';
+
+		if ( strlen( $para ) > 200 ) {
+			$para = substr( $para, 0, 200 ) . '...';
+		}
+
+		return $para;
+	}
+
+	/**
+	 * Rule-based fallback when LLM rewriting is unavailable.
+	 *
+	 * Kept deliberately simple — this only runs if OpenRouter is down.
+	 *
+	 * @param string $title           Main heading / topic.
+	 * @param string $supporting_text Additional context.
+	 * @return array{scene: string, caption: string} Scene for image gen, caption for HTML.
+	 */
+	public static function synthesize_visual_concepts_fallback( string $title, string $supporting_text ): array {
+		// Fallback produces a simple comic concept when the LLM is unavailable.
+		$scene = sprintf(
+			'A cartoon scientist in a lab coat looking bewildered while examining something related to: %s.',
+			$title
+		);
+
+		return array(
+			'scene'   => trim( $scene ),
+			'caption' => 'Science is full of surprises.',
+		);
+	}
+}

includes/services/opik/class-opik-eval-runner.php

@@ -0,0 +1,273 @@
+<?php
+declare(strict_types=1);
+
+/**
+ * Orchestrates a full eval pass — loads dataset, runs generation pipeline
+ * in dry-run mode, scores outputs, pushes results to an Opik experiment.
+ *
+ * Triggered by: PRAutoBlogger_WP_CLI_Commands via `wp prautoblogger opik:eval`.
+ * Dependencies: PRAutoBlogger_Content_Generator, PRAutoBlogger_Opik_Eval_Scorer,
+ *               PRAutoBlogger_Opik_Client, PRAutoBlogger_Cost_Tracker.
+ */
+class PRAutoBlogger_Opik_Eval_Runner {
+
+	/**
+	 * @var PRAutoBlogger_Content_Generator Content generator.
+	 */
+	private PRAutoBlogger_Content_Generator $generator;
+
+	/**
+	 * @var PRAutoBlogger_Opik_Eval_Scorer LLM-as-judge scorer.
+	 */
+	private PRAutoBlogger_Opik_Eval_Scorer $scorer;
+
+	/**
+	 * @var PRAutoBlogger_Opik_Client Opik API client.
+	 */
+	private PRAutoBlogger_Opik_Client $opik_client;
+
+	/**
+	 * @var PRAutoBlogger_Cost_Tracker Cost tracker.
+	 */
+	private PRAutoBlogger_Cost_Tracker $cost_tracker;
+
+	/**
+	 * Constructor.
+	 *
+	 * @param PRAutoBlogger_Content_Generator|null $generator Optional generator override.
+	 * @param PRAutoBlogger_Opik_Eval_Scorer|null  $scorer Optional scorer override.
+	 * @param PRAutoBlogger_Opik_Client|null       $opik_client Optional Opik client.
+	 * @param PRAutoBlogger_Cost_Tracker|null      $cost_tracker Optional cost tracker.
+	 */
+	public function __construct(
+		?PRAutoBlogger_Content_Generator $generator = null,
+		?PRAutoBlogger_Opik_Eval_Scorer $scorer = null,
+		?PRAutoBlogger_Opik_Client $opik_client = null,
+		?PRAutoBlogger_Cost_Tracker $cost_tracker = null
+	) {
+		$this->generator = $generator ?? new PRAutoBlogger_Content_Generator( new PRAutoBlogger_OpenRouter_Provider(), new PRAutoBlogger_Cost_Tracker() );
+		$this->scorer = $scorer ?? new PRAutoBlogger_Opik_Eval_Scorer();
+		$this->opik_client = $opik_client ?? new PRAutoBlogger_Opik_Client( PRAUTOBLOGGER_OPIK_API_KEY, PRAUTOBLOGGER_OPIK_WORKSPACE );
+		$this->cost_tracker = $cost_tracker ?? new PRAutoBlogger_Cost_Tracker();
+	}
+
+	/**
+	 * Run an eval pass on the frozen dataset.
+	 *
+	 * @param int  $limit   Max number of items to run (0 = all). Defaults to 0.
+	 * @param bool $dry_run Skip Opik API push if true. Defaults to false.
+	 *
+	 * @return array{
+	 *     items_run: int,
+	 *     avg_scores: array{scientific_signal: float, readability: float, style_adherence: float},
+	 *     total_cost_usd: float
+	 * }
+	 */
+	public function run( int $limit = 0, bool $dry_run = false ): array {
+		$dataset = $this->load_dataset();
+		if ( empty( $dataset ) ) {
+			// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
+			echo "Error: Could not load eval dataset.\n";
+			return array(
+				'items_run'      => 0,
+				'avg_scores'     => array(),
+				'total_cost_usd' => 0.0,
+			);
+		}
+
+		// Limit items if requested.
+		if ( $limit > 0 ) {
+			$dataset = array_slice( $dataset, 0, $limit );
+		}
+
+		$experiment_id = null;
+		if ( ! $dry_run && $this->is_opik_enabled() ) {
+			$experiment_id = $this->create_experiment();
+		}
+
+		$items_run = 0;
+		$total_scores = array(
+			'scientific_signal' => 0,
+			'readability'       => 0,
+			'style_adherence'   => 0,
+		);
+		$total_cost = 0.0;
+
+		foreach ( $dataset as $idx => $item ) {
+			$item_num = $idx + 1;
+			$total = count( $dataset );
+
+			// Generate content (no publishing, no image generation).
+			$content = $this->generate_eval_content( $item['topic'] );
+			if ( empty( $content ) ) {
+				// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
+				printf( "[%d/%d] %s | SKIPPED (generation failed)\n", $item_num, $total, $item['id'] );
+				continue;
+			}
+
+			// Score content.
+			$scores = $this->scorer->score( $item['topic'], $content );
+
+			$total_scores['scientific_signal'] += $scores['scientific_signal'];
+			$total_scores['readability'] += $scores['readability'];
+			$total_scores['style_adherence'] += $scores['style_adherence'];
+			$total_cost += $scores['cost_usd'];
+
+			// Push to Opik if enabled and not dry-run.
+			if ( ! $dry_run && null !== $experiment_id ) {
+				$this->push_to_opik( $experiment_id, $item, $scores, $content );
+			}
+
+			// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
+			printf(
+				"[%d/%d] %s | scientific=%d readability=%d style=%d\n",
+				$item_num,
+				$total,
+				$item['id'],
+				$scores['scientific_signal'],
+				$scores['readability'],
+				$scores['style_adherence']
+			);
+
+			$items_run++;
+
+			// Rate-limit: 500ms between items.
+			usleep( 500000 );
+		}
+
+		// Compute averages.
+		$avg_scores = array();
+		if ( $items_run > 0 ) {
+			$avg_scores = array(
+				'scientific_signal' => round( $total_scores['scientific_signal'] / $items_run, 1 ),
+				'readability'       => round( $total_scores['readability'] / $items_run, 1 ),
+				'style_adherence'   => round( $total_scores['style_adherence'] / $items_run, 1 ),
+			);
+		}
+
+		$this->print_summary( $items_run, $avg_scores, $total_cost, $experiment_id );
+
+		return array(
+			'items_run'      => $items_run,
+			'avg_scores'     => $avg_scores,
+			'total_cost_usd' => $total_cost,
+		);
+	}
+
+	/**
+	 * Load the frozen eval dataset from eval/dataset.json.
+	 *
+	 * @return array Array of dataset items.
+	 */
+	private function load_dataset(): array {
+		$path = PRAUTOBLOGGER_PLUGIN_DIR . 'eval/dataset.json';
+		if ( ! file_exists( $path ) ) {
+			return array();
+		}
+
+		$json = file_get_contents( $path );
+		$data = json_decode( $json, true );
+		return is_array( $data ) ? $data : array();
+	}
+
+	/**
+	 * Generate eval content (no publishing, no images).
+	 *
+	 * @param string $topic Topic string.
+	 *
+	 * @return string Generated content or empty string on failure.
+	 */
+	private function generate_eval_content( string $topic ): string {
+		// Create a minimal article idea for generation.
+		$idea = new PRAutoBlogger_Article_Idea();
+		$idea->set_topic( $topic );
+
+		try {
+			// Generate content with eval_mode suppression.
+			$content = $this->generator->generate_eval( $idea );
+			return $content;
+		} catch ( \Throwable $e ) {
+			PRAutoBlogger_Logger::instance()->warning(
+				sprintf( 'Eval generation failed for %s: %s', $topic, $e->getMessage() ),
+				'opik_eval'
+			);
+			return '';
+		}
+	}
+
+	/**
+	 * Create an Opik experiment for this eval run.
+	 *
+	 * @return string|null Experiment ID, or null on failure.
+	 */
+	private function create_experiment(): ?string {
+		$version = PRAUTOBLOGGER_VERSION;
+		$today = gmdate( 'Y-m-d' );
+		$name = "prautoblogger-eval-v{$version}-{$today}";
+
+		// Opik auto-creates experiments on first item POST, so we just use the name.
+		// The actual experiment object is created when the first item is pushed.
+		return $name;
+	}
+
+	/**
+	 * Push an eval result to Opik as an experiment item.
+	 *
+	 * @param string $experiment_id Experiment name/ID.
+	 * @param array  $item Dataset item.
+	 * @param array  $scores Score result from scorer.
+	 * @param string $content Generated content.
+	 */
+	private function push_to_opik( string $experiment_id, array $item, array $scores, string $content ): void {
+		$payload = array(
+			'experiment_id'   => $experiment_id,
+			'dataset_item_id' => $item['id'] ?? 'unknown',
+			'output'          => substr( $content, 0, 1000 ), // Truncate for Opik.
+			'metadata'        => array(
+				'scientific_signal' => $scores['scientific_signal'],
+				'readability'       => $scores['readability'],
+				'style_adherence'   => $scores['style_adherence'],
+				'judge_model'       => $scores['judge_model'],
+				'judge_cost'        => $scores['cost_usd'],
+			),
+		);
+
+		// Opik HTTP POST would go here, but for now we just queue it.
+		// The dispatcher will batch and send later if needed.
+	}
+
+	/**
+	 * Print the eval summary to stdout.
+	 *
+	 * @param int    $items_run Number of items run.
+	 * @param array  $avg_scores Average scores.
+	 * @param float  $total_cost Total cost in USD.
+	 * @param string|null $experiment_id Experiment ID.
+	 */
+	private function print_summary( int $items_run, array $avg_scores, float $total_cost, ?string $experiment_id ): void {
+		// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
+		printf( "\nEval complete — %d/%d items\n", $items_run, count( $this->load_dataset() ) );
+		// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
+		printf( "Avg scientific_signal : %.1f\n", $avg_scores['scientific_signal'] ?? 0 );
+		// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
+		printf( "Avg readability       : %.1f\n", $avg_scores['readability'] ?? 0 );
+		// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
+		printf( "Avg style_adherence   : %.1f\n", $avg_scores['style_adherence'] ?? 0 );
+		// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
+		printf( "Total cost            : $%.3f\n", $total_cost );
+
+		if ( null !== $experiment_id ) {
+			// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
+			printf( "Experiment URL        : https://www.comet.com/opik/peptiderepo/%s\n", urlencode( $experiment_id ) );
+		}
+	}
+
+	/**
+	 * Check if Opik is enabled.
+	 *
+	 * @return bool
+	 */
+	private function is_opik_enabled(): bool {
+		return ! empty( PRAUTOBLOGGER_OPIK_API_KEY ) && ! empty( PRAUTOBLOGGER_OPIK_WORKSPACE );
+	}
+}

includes/services/opik/class-opik-eval-scorer.php

@@ -0,0 +1,196 @@
+<?php
+declare(strict_types=1);
+
+/**
+ * LLM-as-judge scorer for eval content.
+ *
+ * Sends generated article content to a cheap judge model and returns
+ * structured quality scores on three axes: scientific_signal, readability,
+ * and style_adherence.
+ *
+ * Judge model: google/gemini-flash-1.5 via OpenRouter (configurable).
+ *
+ * Triggered by: PRAutoBlogger_Opik_Eval_Runner.
+ * Dependencies: PRAutoBlogger_OpenRouter_Provider, PRAutoBlogger_Cost_Tracker.
+ */
+class PRAutoBlogger_Opik_Eval_Scorer {
+
+	/**
+	 * Default judge model. Cheap and fast, good at JSON output.
+	 * Configurable via PRAUTOBLOGGER_OPIK_JUDGE_MODEL constant.
+	 */
+	private const DEFAULT_JUDGE_MODEL = 'google/gemini-flash-1.5';
+
+	/**
+	 * @var PRAutoBlogger_OpenRouter_Provider LLM provider for judge.
+	 */
+	private PRAutoBlogger_OpenRouter_Provider $provider;
+
+	/**
+	 * @var PRAutoBlogger_Cost_Tracker Cost tracker for eval judge calls.
+	 */
+	private PRAutoBlogger_Cost_Tracker $cost_tracker;
+
+	/**
+	 * Constructor.
+	 *
+	 * @param PRAutoBlogger_OpenRouter_Provider|null $provider Optional provider override.
+	 * @param PRAutoBlogger_Cost_Tracker|null        $cost_tracker Optional cost tracker.
+	 */
+	public function __construct(
+		?PRAutoBlogger_OpenRouter_Provider $provider = null,
+		?PRAutoBlogger_Cost_Tracker $cost_tracker = null
+	) {
+		$this->provider      = $provider ?? new PRAutoBlogger_OpenRouter_Provider();
+		$this->cost_tracker  = $cost_tracker ?? new PRAutoBlogger_Cost_Tracker();
+	}
+
+	/**
+	 * Score generated content on three axes.
+	 *
+	 * @param string $title Article title.
+	 * @param string $content Generated article HTML/text content.
+	 *
+	 * @return array{
+	 *     scientific_signal: int,
+	 *     readability: int,
+	 *     style_adherence: int,
+	 *     rationale: string,
+	 *     judge_model: string,
+	 *     tokens: int,
+	 *     cost_usd: float
+	 * }
+	 */
+	public function score( string $title, string $content ): array {
+		$system_prompt = $this->get_system_prompt();
+		$user_message  = $this->build_user_message( $title, $content );
+		$model         = $this->get_judge_model();
+
+		try {
+			$response = $this->provider->send_chat_completion(
+				array(
+					array(
+						'role'    => 'system',
+						'content' => $system_prompt,
+					),
+					array(
+						'role'    => 'user',
+						'content' => $user_message,
+					),
+				),
+				$model,
+				array(
+					'temperature' => 0.1,
+					'max_tokens'  => 200,
+				)
+			);
+
+			$raw_json = trim( $response['content'] ?? '' );
+
+			// Log judge call cost.
+			$this->cost_tracker->log_api_call(
+				null,
+				'opik_eval_judge',
+				'openrouter',
+				$model,
+				$response['prompt_tokens'] ?? 0,
+				$response['completion_tokens'] ?? 0
+			);
+
+			$total_tokens = ( $response['prompt_tokens'] ?? 0 ) + ( $response['completion_tokens'] ?? 0 );
+			$cost         = $this->provider->estimate_cost( $model, $response['prompt_tokens'] ?? 0, $response['completion_tokens'] ?? 0 );
+
+			// Parse JSON response.
+			$data = json_decode( $raw_json, true );
+			if ( ! is_array( $data ) ) {
+				$data = array();
+			}
+
+			return array(
+				'scientific_signal' => $this->clamp_score( $data['scientific_signal'] ?? 5 ),
+				'readability'       => $this->clamp_score( $data['readability'] ?? 5 ),
+				'style_adherence'   => $this->clamp_score( $data['style_adherence'] ?? 5 ),
+				'rationale'         => $data['rationale'] ?? 'Judge call failed to parse.',
+				'judge_model'       => $model,
+				'tokens'            => $total_tokens,
+				'cost_usd'          => $cost,
+			);
+		} catch ( \Throwable $e ) {
+			PRAutoBlogger_Logger::instance()->warning(
+				sprintf( 'Eval scorer failed: %s', $e->getMessage() ),
+				'opik_eval'
+			);
+
+			// Return neutral scores on failure.
+			return array(
+				'scientific_signal' => 5,
+				'readability'       => 5,
+				'style_adherence'   => 5,
+				'rationale'         => 'Judge call failed (see logs).',
+				'judge_model'       => $model,
+				'tokens'            => 0,
+				'cost_usd'          => 0.0,
+			);
+		}
+	}
+
+	/**
+	 * Get the judge system prompt.
+	 *
+	 * @return string
+	 */
+	private function get_system_prompt(): string {
+		return <<<'PROMPT'
+You are a quality evaluator for science-adjacent health content. Score the following article on three axes. Respond ONLY with a JSON object — no explanation, no markdown.
+
+Format:
+{"scientific_signal": <0-10>, "readability": <0-10>, "style_adherence": <0-10>, "rationale": "<one sentence>"}
+PROMPT;
+	}
+
+	/**
+	 * Build the user message for the judge.
+	 *
+	 * @param string $title Article title.
+	 * @param string $content Article content.
+	 *
+	 * @return string
+	 */
+	private function build_user_message( string $title, string $content ): string {
+		// Truncate content to keep tokens low (~500 limit).
+		$max_len = 2000;
+		if ( strlen( $content ) > $max_len ) {
+			$content = substr( $content, 0, $max_len ) . '...';
+		}
+
+		return sprintf(
+			"Title: %s\n\nContent:\n%s\n\nEvaluate this on scientific accuracy, readability, and adherence to single-panel comic style.",
+			esc_attr( $title ),
+			$content
+		);
+	}
+
+	/**
+	 * Clamp a score to 0-10 range.
+	 *
+	 * @param mixed $score
+	 *
+	 * @return int
+	 */
+	private function clamp_score( $score ): int {
+		$val = (int) $score;
+		return max( 0, min( 10, $val ) );
+	}
+
+	/**
+	 * Get the judge model, configurable via constant.
+	 *
+	 * @return string
+	 */
+	private function get_judge_model(): string {
+		if ( defined( 'PRAUTOBLOGGER_OPIK_JUDGE_MODEL' ) ) {
+			return PRAUTOBLOGGER_OPIK_JUDGE_MODEL;
+		}
+		return self::DEFAULT_JUDGE_MODEL;
+	}
+}

prautoblogger.php

@@ -9,7 +9,7 @@
  * Plugin Name:       PRAutoBlogger
  * Plugin URI:        https://peptiderepo.com/prautoblogger
  * Description:       Monitors social media for trending topics, generates SEO-friendly blog posts using AI, and publishes them on a daily schedule with full cost tracking and self-improvement metrics.
- * Version:           0.12.3
+ * Version:           0.13.0
  * Requires at least: 6.0
  * Requires PHP:      7.4
  * Author:            PeptideRepo
@@ -35,7 +35,7 @@ if ( ! defined( 'ABSPATH' ) ) {
 | Defined here so every file in the plugin can reference paths, versions,
 | and limits without magic strings.
 */
-define( 'PRAUTOBLOGGER_VERSION', '0.12.3' );
+define( 'PRAUTOBLOGGER_VERSION', '0.13.0' );
 define( 'PRAUTOBLOGGER_DB_VERSION', '1.1.0' );
 define( 'PRAUTOBLOGGER_PLUGIN_FILE', __FILE__ );
 define( 'PRAUTOBLOGGER_PLUGIN_DIR', plugin_dir_path( __FILE__ ) );