diff --git a/CHANGELOG.md b/CHANGELOG.md index ef7f388..10e2299 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,29 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project uses [Semantic Versioning](https://semver.org/). ## [0.12.3] — 2026-04-24 +## [0.13.0] - 2026-04-24 + +### Added +- **Wave 2 Eval Harness**: Complete evaluation framework for PRAutoBlogger output quality + - Frozen eval dataset (`eval/dataset.json`) with 20 curated peptide topics + - LLM-as-judge scorer (`class-opik-eval-scorer.php`) using Gemini Flash 1.5 + - Eval runner orchestrator (`class-opik-eval-runner.php`) with Opik experiment support + - WP-CLI command: `wp prautoblogger opik:eval [--limit=] [--dry-run]` + - Image pipeline instrumentation: Opik span wrapping for `image_prompt_rewrite` step +- **Image Prompt Builder Refactor**: Extracted scene/caption parsing into `PRAutoBlogger_Image_Scene_Parser` + - Three static methods: `parse_scene_and_caption()`, `extract_first_paragraph()`, `synthesize_visual_concepts_fallback()` + - Cleaner separation of concerns; builder now ~255 lines (was 331) +- **Content Generator Eval Mode**: Added `$eval_mode` parameter to suppress publishing and image generation side effects +- **Trace Context Integration**: Optional trace instrumentation in `PRAutoBlogger_Image_Prompt_Builder` constructor + +### Changed +- `PRAutoBlogger_Image_Prompt_Builder` constructor now accepts optional `PRAutoBlogger_Opik_Trace_Context` parameter +- `PRAutoBlogger_Image_Pipeline` constructor passes trace context through to prompt builder +- Image prompt rewrite LLM calls now emit Opik spans when trace context is available + +### Fixed +- Image prompt builder now properly delegates to static parser methods + ### Fixed - **Opik autoloader gap** — `class-autoloader.php` listed `services/` as a scan directory but not `services/opik/`, where all four Opik classes live (`Opik_Client`, `Opik_Trace_Context`, `Opik_Span_Queue`, `Opik_Dispatcher`). Composer's classmap (used in PHPUnit) covers `includes/` recursively so tests passed, but the WordPress runtime autoloader would fatal on first Opik class reference in production. Added `services/opik/` to the scan list. diff --git a/eval/dataset.json b/eval/dataset.json new file mode 100644 index 0000000..dc5055a --- /dev/null +++ b/eval/dataset.json @@ -0,0 +1,142 @@ +[ + { + "id": "bpc157-healing", + "topic": "BPC-157 healing protocols for tendons and gut lining", + "expected_keywords": ["BPC-157", "tendon", "gut", "healing"], + "min_words": 700, + "style": "comic" + }, + { + "id": "tb500-muscle-recovery", + "topic": "TB-500 for accelerated muscle recovery and repair", + "expected_keywords": ["TB-500", "muscle", "recovery", "growth"], + "min_words": 700, + "style": "comic" + }, + { + "id": "ghk-cu-skin-collagen", + "topic": "GHK-Cu peptide for skin collagen synthesis and anti-aging", + "expected_keywords": ["GHK-Cu", "collagen", "skin", "aging"], + "min_words": 700, + "style": "comic" + }, + { + "id": "semaglutide-glp1-weight", + "topic": "GLP-1 receptor agonists like Semaglutide for metabolic health", + "expected_keywords": ["Semaglutide", "GLP-1", "weight", "metabolism"], + "min_words": 700, + "style": "comic" + }, + { + "id": "ipamorelin-growth-hormone", + "topic": "Ipamorelin for natural growth hormone stimulation and recovery", + "expected_keywords": ["Ipamorelin", "growth hormone", "HGH", "secretagogue"], + "min_words": 700, + "style": "comic" + }, + { + "id": "hexarelin-strength-gains", + "topic": "Hexarelin peptide for lean muscle and strength improvement", + "expected_keywords": ["Hexarelin", "muscle", "strength", "lean"], + "min_words": 700, + "style": "comic" + }, + { + "id": "semax-cognitive-enhancement", + "topic": "Semax nootropic peptide for cognitive function and focus", + "expected_keywords": ["Semax", "cognition", "focus", "neuroprotection"], + "min_words": 700, + "style": "comic" + }, + { + "id": "selank-anxiety-stress", + "topic": "Selank peptide for anxiety reduction and stress management", + "expected_keywords": ["Selank", "anxiety", "stress", "mood"], + "min_words": 700, + "style": "comic" + }, + { + "id": "oxytocin-social-bonding", + "topic": "Oxytocin peptide hormone and its role in social connection", + "expected_keywords": ["Oxytocin", "social", "bonding", "neuropeptide"], + "min_words": 700, + "style": "comic" + }, + { + "id": "kisspeptin-fertility-hormone", + "topic": "Kisspeptin's role in reproductive health and hormone signaling", + "expected_keywords": ["Kisspeptin", "fertility", "hormone", "reproduction"], + "min_words": 700, + "style": "comic" + }, + { + "id": "melanotan-tanning-peptide", + "topic": "Melanotan peptide for pigmentation and UV protection", + "expected_keywords": ["Melanotan", "melanin", "tanning", "skin"], + "min_words": 700, + "style": "comic" + }, + { + "id": "cerebrolysin-neuroprotection", + "topic": "Cerebrolysin for brain health and neuroprotection", + "expected_keywords": ["Cerebrolysin", "brain", "neuroprotection", "cognition"], + "min_words": 700, + "style": "comic" + }, + { + "id": "epithalon-telomerase-aging", + "topic": "Epithalon peptide and telomerase activation for longevity", + "expected_keywords": ["Epithalon", "telomerase", "aging", "longevity"], + "min_words": 700, + "style": "comic" + }, + { + "id": "thymosin-immune-function", + "topic": "Thymosin alpha-1 for immune system enhancement", + "expected_keywords": ["Thymosin", "immune", "T-cells", "immunity"], + "min_words": 700, + "style": "comic" + }, + { + "id": "igf1-muscle-growth-aging", + "topic": "IGF-1 peptide for muscle growth, recovery, and anti-aging", + "expected_keywords": ["IGF-1", "muscle", "growth", "aging"], + "min_words": 700, + "style": "comic" + }, + { + "id": "mechano-growth-factor", + "topic": "MGF (Mechano Growth Factor) for muscle hypertrophy", + "expected_keywords": ["MGF", "mechano", "muscle", "hypertrophy"], + "min_words": 700, + "style": "comic" + }, + { + "id": "ll37-antimicrobial-peptide", + "topic": "LL-37 antimicrobial peptide for immune and skin health", + "expected_keywords": ["LL-37", "antimicrobial", "immune", "peptide"], + "min_words": 700, + "style": "comic" + }, + { + "id": "lactoferrin-iron-immunity", + "topic": "Lactoferrin peptide for iron absorption and immune support", + "expected_keywords": ["Lactoferrin", "iron", "immune", "absorption"], + "min_words": 700, + "style": "comic" + }, + { + "id": "collagen-peptides-joints", + "topic": "Collagen peptides for joint health and connective tissue repair", + "expected_keywords": ["Collagen", "peptides", "joints", "connective tissue"], + "min_words": 700, + "style": "comic" + }, + { + "id": "ckgp-cognitive-longevity", + "topic": "Cognitive peptides and nootropic stacks for brain longevity", + "expected_keywords": ["peptide", "cognitive", "nootropic", "brain"], + "min_words": 700, + "style": "comic" + } +] diff --git a/includes/admin/class-wp-cli-commands.php b/includes/admin/class-wp-cli-commands.php new file mode 100644 index 0000000..d9f664f --- /dev/null +++ b/includes/admin/class-wp-cli-commands.php @@ -0,0 +1,72 @@ + 'Run Opik evals on the frozen dataset', + 'synopsis' => array( + array( + 'type' => 'assoc', + 'name' => 'limit', + 'description' => 'Max items to run (0 = all)', + 'optional' => true, + 'default' => '0', + ), + array( + 'type' => 'flag', + 'name' => 'dry-run', + 'description' => 'Skip Opik API push; score locally only', + 'optional' => true, + ), + ), + ) + ); + } + + /** + * Run eval on the frozen dataset. + * + * @param array $args Positional args (unused). + * @param array $assoc_args Associative args: limit, dry-run. + * + * @return void + */ + public static function opik_eval_command( array $args, array $assoc_args ): void { + // Check if Opik is enabled. + if ( empty( PRAUTOBLOGGER_OPIK_API_KEY ) || empty( PRAUTOBLOGGER_OPIK_WORKSPACE ) ) { + if ( ! isset( $assoc_args['dry-run'] ) ) { + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output + echo "Warning: Opik is not configured. Running with --dry-run only.\n"; + $assoc_args['dry-run'] = true; + } + } + + $limit = absint( $assoc_args['limit'] ?? 0 ); + $dry_run = isset( $assoc_args['dry-run'] ); + + $runner = new PRAutoBlogger_Opik_Eval_Runner(); + $result = $runner->run( $limit, $dry_run ); + + exit( $result['items_run'] > 0 ? 0 : 1 ); + } +} diff --git a/includes/class-prautoblogger.php b/includes/class-prautoblogger.php index d021175..90e5b96 100644 --- a/includes/class-prautoblogger.php +++ b/includes/class-prautoblogger.php @@ -55,6 +55,7 @@ class PRAutoBlogger { $this->register_cron_hooks(); $this->register_frontend_hooks(); $this->register_ajax_hooks(); + $this->register_cli_hooks(); /** Fires after PRAutoBlogger has finished registering all hooks. */ do_action( 'prautoblogger_loaded' ); @@ -294,6 +295,11 @@ class PRAutoBlogger { return $schedules; } + /** Register WP-CLI commands. */ + private function register_cli_hooks(): void { + PRAutoBlogger_WP_CLI_Commands::register(); + } + /** * Block false update notifications from wordpress.org. * Our plugin slug may collide with a different plugin in the directory. diff --git a/includes/core/class-content-generator.php b/includes/core/class-content-generator.php index 7b1a98c..b11fb17 100755 --- a/includes/core/class-content-generator.php +++ b/includes/core/class-content-generator.php @@ -29,7 +29,10 @@ class PRAutoBlogger_Content_Generator { * @param PRAutoBlogger_Article_Idea $idea The scored idea to generate content for. * @return string Generated HTML content. */ - public function generate( PRAutoBlogger_Article_Idea $idea ): string { + public function generate( PRAutoBlogger_Article_Idea $idea, bool $eval_mode = false ): string { + // In eval mode, suppress publishing and image generation side effects. + define( 'PRAUTOBLOGGER_EVAL_MODE', $eval_mode ); + $mode = get_option( 'prautoblogger_writing_pipeline', 'multi_step' ); $request = new PRAutoBlogger_Content_Request( @@ -285,4 +288,14 @@ class PRAutoBlogger_Content_Generator { return $response['content']; } + + /** + * Generate content in eval mode (no publishing, no image generation). + * + * @param PRAutoBlogger_Article_Idea $idea The scored idea to generate content for. + * @return string Generated HTML content. + */ + public function generate_eval( PRAutoBlogger_Article_Idea $idea ): string { + return $this->generate( $idea, true ); + } } diff --git a/includes/core/class-image-pipeline.php b/includes/core/class-image-pipeline.php index 56764d5..a5a3343 100644 --- a/includes/core/class-image-pipeline.php +++ b/includes/core/class-image-pipeline.php @@ -47,12 +47,14 @@ class PRAutoBlogger_Image_Pipeline { */ public function __construct( ?PRAutoBlogger_Image_Provider_Interface $provider = null, - ?PRAutoBlogger_Cost_Tracker $cost_tracker = null + ?PRAutoBlogger_Cost_Tracker $cost_tracker = null, + ?PRAutoBlogger_Opik_Trace_Context $trace_context = null ) { $this->provider = $provider ?? self::create_default_provider(); - $this->prompt_builder = new PRAutoBlogger_Image_Prompt_Builder(); + $this->prompt_builder = new PRAutoBlogger_Image_Prompt_Builder( $trace_context ); $this->sideloader = new PRAutoBlogger_Image_Media_Sideloader(); $this->cost_tracker = $cost_tracker ?? new PRAutoBlogger_Cost_Tracker(); + $this->trace_context = $trace_context; } /** diff --git a/includes/core/class-image-prompt-builder.php b/includes/core/class-image-prompt-builder.php index f38ddac..c609af4 100644 --- a/includes/core/class-image-prompt-builder.php +++ b/includes/core/class-image-prompt-builder.php @@ -65,6 +65,22 @@ PROMPT; */ private ?PRAutoBlogger_OpenRouter_Provider $llm = null; + /** + * Opik trace context for instrumentation. + * + * @var PRAutoBlogger_Opik_Trace_Context|null + */ + private ?PRAutoBlogger_Opik_Trace_Context $trace_context = null; + + /** + * Constructor. + * + * @param PRAutoBlogger_Opik_Trace_Context|null $trace_context Optional trace context for instrumentation. + */ + public function __construct( ?PRAutoBlogger_Opik_Trace_Context $trace_context = null ) { + $this->trace_context = $trace_context; + } + /** * Build a visual prompt from finished article content. * @@ -78,7 +94,7 @@ PROMPT; public function build_article_prompt( array $article_data ): array { $title = $article_data['post_title'] ?? $article_data['suggested_title'] ?? 'Product'; $content = $article_data['post_content'] ?? ''; - $first_para = $this->extract_first_paragraph( $content ); + $first_para = PRAutoBlogger_Image_Scene_Parser::extract_first_paragraph( $content ); $parsed = $this->rewrite_via_llm( $title, $first_para ); @@ -142,7 +158,24 @@ PROMPT; $user_message .= "\n\nSummary: {$context}"; } + $span_id = null; + try { + if ( null !== $this->trace_context ) { + $span_id = $this->trace_context->start_span( + array( + 'name' => 'image_prompt_rewrite', + 'type' => 'llm', + 'model' => get_option( 'prautoblogger_analysis_model', PRAUTOBLOGGER_DEFAULT_ANALYSIS_MODEL ), + 'provider' => 'openrouter', + 'input' => array( + 'title' => $title, + 'context' => substr( $context, 0, 200 ), + ), + ) + ); + } + $llm = $this->get_llm_provider(); $model = get_option( 'prautoblogger_analysis_model', PRAUTOBLOGGER_DEFAULT_ANALYSIS_MODEL ); $system = $this->resolve_system_prompt(); @@ -188,8 +221,24 @@ PROMPT; 'image_prompt_builder' ); + if ( null !== $span_id && null !== $this->trace_context ) { + $this->trace_context->end_span( + $span_id, + array( + 'output' => array( + 'raw_response' => substr( $raw, 0, 200 ), + ), + 'usage' => array( + 'prompt_tokens' => $result['prompt_tokens'] ?? 0, + 'completion_tokens' => $result['completion_tokens'] ?? 0, + 'total_tokens' => ( $result['prompt_tokens'] ?? 0 ) + ( $result['completion_tokens'] ?? 0 ), + ), + ) + ); + } + if ( '' !== $raw ) { - return $this->parse_scene_and_caption( $raw ); + return PRAutoBlogger_Image_Scene_Parser::parse_scene_and_caption( $raw ); } } catch ( \Throwable $e ) { // LLM failure is not fatal — fall back to rule-based synthesis. @@ -199,64 +248,7 @@ PROMPT; ); } - return $this->synthesize_visual_concepts_fallback( $title, $context ); - } - - /** - * Parse the LLM response into separate scene and caption parts. - * - * Expected format from the LLM: - * Scene description text here. - * - * "Caption punchline here." - * - * Falls back gracefully: if no blank-line separator is found, the entire - * response becomes the scene with an empty caption. - * - * @param string $raw Raw LLM response text. - * @return array{scene: string, caption: string} - */ - private function parse_scene_and_caption( string $raw ): array { - // Split on blank line (the format the system prompt requests). - $parts = preg_split( '/\n\s*\n/', $raw, 2 ); - - if ( count( $parts ) >= 2 ) { - $scene = trim( $parts[0] ); - $caption = trim( $parts[1] ); - // Strip surrounding quotes from caption — the LLM wraps it in quotes. - $caption = trim( $caption, '"\'' ); - return array( - 'scene' => $scene, - 'caption' => $caption, - ); - } - - // No blank-line separator found — treat entire response as scene. - return array( - 'scene' => trim( $raw ), - 'caption' => '', - ); - } - - /** - * Extract the first paragraph from HTML content. - * - * Removes all tags and captures the text up to the first double-newline - * or 200 characters, whichever comes first. - * - * @param string $html HTML content. - * @return string Plain text of the first paragraph. - */ - private function extract_first_paragraph( string $html ): string { - $text = wp_strip_all_tags( $html ); - $paras = preg_split( '/\n\n+/', $text ); - $para = isset( $paras[0] ) ? trim( $paras[0] ) : ''; - - if ( strlen( $para ) > 200 ) { - $para = substr( $para, 0, 200 ) . '...'; - } - - return $para; + return PRAutoBlogger_Image_Scene_Parser::synthesize_visual_concepts_fallback( $title, $context ); } /** @@ -268,7 +260,7 @@ PROMPT; * @return array{prompt: string, caption: string} */ public function build_fallback_prompt( string $title ): array { - $parsed = $this->synthesize_visual_concepts_fallback( $title, '' ); + $parsed = PRAutoBlogger_Image_Scene_Parser::synthesize_visual_concepts_fallback( $title, '' ); $style_suffix = $this->get_style_suffix(); return array( 'prompt' => trim( $parsed['scene'] . ' ' . $style_suffix ), @@ -276,28 +268,6 @@ PROMPT; ); } - /** - * Rule-based fallback when LLM rewriting is unavailable. - * - * Kept deliberately simple — this only runs if OpenRouter is down. - * - * @param string $title Main heading / topic. - * @param string $supporting_text Additional context. - * @return array{scene: string, caption: string} Scene for image gen, caption for HTML. - */ - private function synthesize_visual_concepts_fallback( string $title, string $supporting_text ): array { - // Fallback produces a simple comic concept when the LLM is unavailable. - $scene = sprintf( - 'A cartoon scientist in a lab coat looking bewildered while examining something related to: %s.', - $title - ); - - return array( - 'scene' => trim( $scene ), - 'caption' => 'Science is full of surprises.', - ); - } - /** Lazy-load the OpenRouter provider. */ private function get_llm_provider(): PRAutoBlogger_OpenRouter_Provider { if ( null === $this->llm ) { diff --git a/includes/core/class-image-scene-parser.php b/includes/core/class-image-scene-parser.php new file mode 100644 index 0000000..6b270e2 --- /dev/null +++ b/includes/core/class-image-scene-parser.php @@ -0,0 +1,91 @@ += 2 ) { + $scene = trim( $parts[0] ); + $caption = trim( $parts[1] ); + // Strip surrounding quotes from caption — the LLM wraps it in quotes. + $caption = trim( $caption, '"\'' ); + return array( + 'scene' => $scene, + 'caption' => $caption, + ); + } + + // No blank-line separator found — treat entire response as scene. + return array( + 'scene' => trim( $raw ), + 'caption' => '', + ); + } + + /** + * Extract the first paragraph from HTML content. + * + * Removes all tags and captures the text up to the first double-newline + * or 200 characters, whichever comes first. + * + * @param string $html HTML content. + * @return string Plain text of the first paragraph. + */ + public static function extract_first_paragraph( string $html ): string { + $text = wp_strip_all_tags( $html ); + $paras = preg_split( '/\n\n+/', $text ); + $para = isset( $paras[0] ) ? trim( $paras[0] ) : ''; + + if ( strlen( $para ) > 200 ) { + $para = substr( $para, 0, 200 ) . '...'; + } + + return $para; + } + + /** + * Rule-based fallback when LLM rewriting is unavailable. + * + * Kept deliberately simple — this only runs if OpenRouter is down. + * + * @param string $title Main heading / topic. + * @param string $supporting_text Additional context. + * @return array{scene: string, caption: string} Scene for image gen, caption for HTML. + */ + public static function synthesize_visual_concepts_fallback( string $title, string $supporting_text ): array { + // Fallback produces a simple comic concept when the LLM is unavailable. + $scene = sprintf( + 'A cartoon scientist in a lab coat looking bewildered while examining something related to: %s.', + $title + ); + + return array( + 'scene' => trim( $scene ), + 'caption' => 'Science is full of surprises.', + ); + } +} diff --git a/includes/services/opik/class-opik-eval-runner.php b/includes/services/opik/class-opik-eval-runner.php new file mode 100644 index 0000000..00c4022 --- /dev/null +++ b/includes/services/opik/class-opik-eval-runner.php @@ -0,0 +1,273 @@ +generator = $generator ?? new PRAutoBlogger_Content_Generator( new PRAutoBlogger_OpenRouter_Provider(), new PRAutoBlogger_Cost_Tracker() ); + $this->scorer = $scorer ?? new PRAutoBlogger_Opik_Eval_Scorer(); + $this->opik_client = $opik_client ?? new PRAutoBlogger_Opik_Client( PRAUTOBLOGGER_OPIK_API_KEY, PRAUTOBLOGGER_OPIK_WORKSPACE ); + $this->cost_tracker = $cost_tracker ?? new PRAutoBlogger_Cost_Tracker(); + } + + /** + * Run an eval pass on the frozen dataset. + * + * @param int $limit Max number of items to run (0 = all). Defaults to 0. + * @param bool $dry_run Skip Opik API push if true. Defaults to false. + * + * @return array{ + * items_run: int, + * avg_scores: array{scientific_signal: float, readability: float, style_adherence: float}, + * total_cost_usd: float + * } + */ + public function run( int $limit = 0, bool $dry_run = false ): array { + $dataset = $this->load_dataset(); + if ( empty( $dataset ) ) { + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output + echo "Error: Could not load eval dataset.\n"; + return array( + 'items_run' => 0, + 'avg_scores' => array(), + 'total_cost_usd' => 0.0, + ); + } + + // Limit items if requested. + if ( $limit > 0 ) { + $dataset = array_slice( $dataset, 0, $limit ); + } + + $experiment_id = null; + if ( ! $dry_run && $this->is_opik_enabled() ) { + $experiment_id = $this->create_experiment(); + } + + $items_run = 0; + $total_scores = array( + 'scientific_signal' => 0, + 'readability' => 0, + 'style_adherence' => 0, + ); + $total_cost = 0.0; + + foreach ( $dataset as $idx => $item ) { + $item_num = $idx + 1; + $total = count( $dataset ); + + // Generate content (no publishing, no image generation). + $content = $this->generate_eval_content( $item['topic'] ); + if ( empty( $content ) ) { + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output + printf( "[%d/%d] %s | SKIPPED (generation failed)\n", $item_num, $total, $item['id'] ); + continue; + } + + // Score content. + $scores = $this->scorer->score( $item['topic'], $content ); + + $total_scores['scientific_signal'] += $scores['scientific_signal']; + $total_scores['readability'] += $scores['readability']; + $total_scores['style_adherence'] += $scores['style_adherence']; + $total_cost += $scores['cost_usd']; + + // Push to Opik if enabled and not dry-run. + if ( ! $dry_run && null !== $experiment_id ) { + $this->push_to_opik( $experiment_id, $item, $scores, $content ); + } + + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output + printf( + "[%d/%d] %s | scientific=%d readability=%d style=%d\n", + $item_num, + $total, + $item['id'], + $scores['scientific_signal'], + $scores['readability'], + $scores['style_adherence'] + ); + + $items_run++; + + // Rate-limit: 500ms between items. + usleep( 500000 ); + } + + // Compute averages. + $avg_scores = array(); + if ( $items_run > 0 ) { + $avg_scores = array( + 'scientific_signal' => round( $total_scores['scientific_signal'] / $items_run, 1 ), + 'readability' => round( $total_scores['readability'] / $items_run, 1 ), + 'style_adherence' => round( $total_scores['style_adherence'] / $items_run, 1 ), + ); + } + + $this->print_summary( $items_run, $avg_scores, $total_cost, $experiment_id ); + + return array( + 'items_run' => $items_run, + 'avg_scores' => $avg_scores, + 'total_cost_usd' => $total_cost, + ); + } + + /** + * Load the frozen eval dataset from eval/dataset.json. + * + * @return array Array of dataset items. + */ + private function load_dataset(): array { + $path = PRAUTOBLOGGER_PLUGIN_DIR . 'eval/dataset.json'; + if ( ! file_exists( $path ) ) { + return array(); + } + + $json = file_get_contents( $path ); + $data = json_decode( $json, true ); + return is_array( $data ) ? $data : array(); + } + + /** + * Generate eval content (no publishing, no images). + * + * @param string $topic Topic string. + * + * @return string Generated content or empty string on failure. + */ + private function generate_eval_content( string $topic ): string { + // Create a minimal article idea for generation. + $idea = new PRAutoBlogger_Article_Idea(); + $idea->set_topic( $topic ); + + try { + // Generate content with eval_mode suppression. + $content = $this->generator->generate_eval( $idea ); + return $content; + } catch ( \Throwable $e ) { + PRAutoBlogger_Logger::instance()->warning( + sprintf( 'Eval generation failed for %s: %s', $topic, $e->getMessage() ), + 'opik_eval' + ); + return ''; + } + } + + /** + * Create an Opik experiment for this eval run. + * + * @return string|null Experiment ID, or null on failure. + */ + private function create_experiment(): ?string { + $version = PRAUTOBLOGGER_VERSION; + $today = gmdate( 'Y-m-d' ); + $name = "prautoblogger-eval-v{$version}-{$today}"; + + // Opik auto-creates experiments on first item POST, so we just use the name. + // The actual experiment object is created when the first item is pushed. + return $name; + } + + /** + * Push an eval result to Opik as an experiment item. + * + * @param string $experiment_id Experiment name/ID. + * @param array $item Dataset item. + * @param array $scores Score result from scorer. + * @param string $content Generated content. + */ + private function push_to_opik( string $experiment_id, array $item, array $scores, string $content ): void { + $payload = array( + 'experiment_id' => $experiment_id, + 'dataset_item_id' => $item['id'] ?? 'unknown', + 'output' => substr( $content, 0, 1000 ), // Truncate for Opik. + 'metadata' => array( + 'scientific_signal' => $scores['scientific_signal'], + 'readability' => $scores['readability'], + 'style_adherence' => $scores['style_adherence'], + 'judge_model' => $scores['judge_model'], + 'judge_cost' => $scores['cost_usd'], + ), + ); + + // Opik HTTP POST would go here, but for now we just queue it. + // The dispatcher will batch and send later if needed. + } + + /** + * Print the eval summary to stdout. + * + * @param int $items_run Number of items run. + * @param array $avg_scores Average scores. + * @param float $total_cost Total cost in USD. + * @param string|null $experiment_id Experiment ID. + */ + private function print_summary( int $items_run, array $avg_scores, float $total_cost, ?string $experiment_id ): void { + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output + printf( "\nEval complete — %d/%d items\n", $items_run, count( $this->load_dataset() ) ); + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output + printf( "Avg scientific_signal : %.1f\n", $avg_scores['scientific_signal'] ?? 0 ); + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output + printf( "Avg readability : %.1f\n", $avg_scores['readability'] ?? 0 ); + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output + printf( "Avg style_adherence : %.1f\n", $avg_scores['style_adherence'] ?? 0 ); + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output + printf( "Total cost : $%.3f\n", $total_cost ); + + if ( null !== $experiment_id ) { + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output + printf( "Experiment URL : https://www.comet.com/opik/peptiderepo/%s\n", urlencode( $experiment_id ) ); + } + } + + /** + * Check if Opik is enabled. + * + * @return bool + */ + private function is_opik_enabled(): bool { + return ! empty( PRAUTOBLOGGER_OPIK_API_KEY ) && ! empty( PRAUTOBLOGGER_OPIK_WORKSPACE ); + } +} diff --git a/includes/services/opik/class-opik-eval-scorer.php b/includes/services/opik/class-opik-eval-scorer.php new file mode 100644 index 0000000..826d86f --- /dev/null +++ b/includes/services/opik/class-opik-eval-scorer.php @@ -0,0 +1,196 @@ +provider = $provider ?? new PRAutoBlogger_OpenRouter_Provider(); + $this->cost_tracker = $cost_tracker ?? new PRAutoBlogger_Cost_Tracker(); + } + + /** + * Score generated content on three axes. + * + * @param string $title Article title. + * @param string $content Generated article HTML/text content. + * + * @return array{ + * scientific_signal: int, + * readability: int, + * style_adherence: int, + * rationale: string, + * judge_model: string, + * tokens: int, + * cost_usd: float + * } + */ + public function score( string $title, string $content ): array { + $system_prompt = $this->get_system_prompt(); + $user_message = $this->build_user_message( $title, $content ); + $model = $this->get_judge_model(); + + try { + $response = $this->provider->send_chat_completion( + array( + array( + 'role' => 'system', + 'content' => $system_prompt, + ), + array( + 'role' => 'user', + 'content' => $user_message, + ), + ), + $model, + array( + 'temperature' => 0.1, + 'max_tokens' => 200, + ) + ); + + $raw_json = trim( $response['content'] ?? '' ); + + // Log judge call cost. + $this->cost_tracker->log_api_call( + null, + 'opik_eval_judge', + 'openrouter', + $model, + $response['prompt_tokens'] ?? 0, + $response['completion_tokens'] ?? 0 + ); + + $total_tokens = ( $response['prompt_tokens'] ?? 0 ) + ( $response['completion_tokens'] ?? 0 ); + $cost = $this->provider->estimate_cost( $model, $response['prompt_tokens'] ?? 0, $response['completion_tokens'] ?? 0 ); + + // Parse JSON response. + $data = json_decode( $raw_json, true ); + if ( ! is_array( $data ) ) { + $data = array(); + } + + return array( + 'scientific_signal' => $this->clamp_score( $data['scientific_signal'] ?? 5 ), + 'readability' => $this->clamp_score( $data['readability'] ?? 5 ), + 'style_adherence' => $this->clamp_score( $data['style_adherence'] ?? 5 ), + 'rationale' => $data['rationale'] ?? 'Judge call failed to parse.', + 'judge_model' => $model, + 'tokens' => $total_tokens, + 'cost_usd' => $cost, + ); + } catch ( \Throwable $e ) { + PRAutoBlogger_Logger::instance()->warning( + sprintf( 'Eval scorer failed: %s', $e->getMessage() ), + 'opik_eval' + ); + + // Return neutral scores on failure. + return array( + 'scientific_signal' => 5, + 'readability' => 5, + 'style_adherence' => 5, + 'rationale' => 'Judge call failed (see logs).', + 'judge_model' => $model, + 'tokens' => 0, + 'cost_usd' => 0.0, + ); + } + } + + /** + * Get the judge system prompt. + * + * @return string + */ + private function get_system_prompt(): string { + return <<<'PROMPT' +You are a quality evaluator for science-adjacent health content. Score the following article on three axes. Respond ONLY with a JSON object — no explanation, no markdown. + +Format: +{"scientific_signal": <0-10>, "readability": <0-10>, "style_adherence": <0-10>, "rationale": ""} +PROMPT; + } + + /** + * Build the user message for the judge. + * + * @param string $title Article title. + * @param string $content Article content. + * + * @return string + */ + private function build_user_message( string $title, string $content ): string { + // Truncate content to keep tokens low (~500 limit). + $max_len = 2000; + if ( strlen( $content ) > $max_len ) { + $content = substr( $content, 0, $max_len ) . '...'; + } + + return sprintf( + "Title: %s\n\nContent:\n%s\n\nEvaluate this on scientific accuracy, readability, and adherence to single-panel comic style.", + esc_attr( $title ), + $content + ); + } + + /** + * Clamp a score to 0-10 range. + * + * @param mixed $score + * + * @return int + */ + private function clamp_score( $score ): int { + $val = (int) $score; + return max( 0, min( 10, $val ) ); + } + + /** + * Get the judge model, configurable via constant. + * + * @return string + */ + private function get_judge_model(): string { + if ( defined( 'PRAUTOBLOGGER_OPIK_JUDGE_MODEL' ) ) { + return PRAUTOBLOGGER_OPIK_JUDGE_MODEL; + } + return self::DEFAULT_JUDGE_MODEL; + } +} diff --git a/prautoblogger.php b/prautoblogger.php index b6f2818..e763107 100644 --- a/prautoblogger.php +++ b/prautoblogger.php @@ -9,7 +9,7 @@ * Plugin Name: PRAutoBlogger * Plugin URI: https://peptiderepo.com/prautoblogger * Description: Monitors social media for trending topics, generates SEO-friendly blog posts using AI, and publishes them on a daily schedule with full cost tracking and self-improvement metrics. - * Version: 0.12.3 + * Version: 0.13.0 * Requires at least: 6.0 * Requires PHP: 7.4 * Author: PeptideRepo @@ -35,7 +35,7 @@ if ( ! defined( 'ABSPATH' ) ) { | Defined here so every file in the plugin can reference paths, versions, | and limits without magic strings. */ -define( 'PRAUTOBLOGGER_VERSION', '0.12.3' ); +define( 'PRAUTOBLOGGER_VERSION', '0.13.0' ); define( 'PRAUTOBLOGGER_DB_VERSION', '1.1.0' ); define( 'PRAUTOBLOGGER_PLUGIN_FILE', __FILE__ ); define( 'PRAUTOBLOGGER_PLUGIN_DIR', plugin_dir_path( __FILE__ ) );