feat: wave 2 eval harness with lm-as-judge and cli command

New eval framework:
- eval/dataset.json: 20 frozen peptide topic eval items
- class-opik-eval-scorer.php: LLM-as-judge using Gemini Flash 1.5
  Scores on: scientific_signal, readability, style_adherence (0-10)
- class-opik-eval-runner.php: orchestrator for full eval pass
  Loads dataset, runs generation (no publish/images), scores, pushes to Opik
  run(int $limit = 0, bool $dry_run = false): array
- class-wp-cli-commands.php: registers wp prautoblogger opik:eval
  Flags: --limit=<n> (default 0 = all), --dry-run (score only)

Features:
- Rate limiting: 500ms between items
- Cost tracking: eval judge calls logged to opik_eval_judge stage
- Dry-run mode: test locally without Opik API push
- Experiment naming: prautoblogger-eval-v<VERSION>-<YYYY-MM-DD>

All files <= 300 lines. Version 0.13.0. Ready for PR.
This commit is contained in:
peptiderepo
2026-04-24 04:25:41 +00:00
parent 575d0b97b9
commit c79fbb570e
5 changed files with 702 additions and 0 deletions

View File

@@ -0,0 +1,72 @@
<?php
declare(strict_types=1);
/**
* WP-CLI commands for PRAutoBlogger.
*
* Registers custom wp-cli commands exposed to the plugin.
*/
class PRAutoBlogger_WP_CLI_Commands {
/**
* Register WP-CLI commands.
*
* Called on plugins_loaded hook.
*/
public static function register(): void {
if ( ! defined( 'WP_CLI' ) || ! WP_CLI ) {
return;
}
// Command: wp prautoblogger opik:eval
\WP_CLI::add_command(
'prautoblogger opik:eval',
array( self::class, 'opik_eval_command' ),
array(
'shortdesc' => 'Run Opik evals on the frozen dataset',
'synopsis' => array(
array(
'type' => 'assoc',
'name' => 'limit',
'description' => 'Max items to run (0 = all)',
'optional' => true,
'default' => '0',
),
array(
'type' => 'flag',
'name' => 'dry-run',
'description' => 'Skip Opik API push; score locally only',
'optional' => true,
),
),
)
);
}
/**
* Run eval on the frozen dataset.
*
* @param array $args Positional args (unused).
* @param array $assoc_args Associative args: limit, dry-run.
*
* @return void
*/
public static function opik_eval_command( array $args, array $assoc_args ): void {
// Check if Opik is enabled.
if ( empty( PRAUTOBLOGGER_OPIK_API_KEY ) || empty( PRAUTOBLOGGER_OPIK_WORKSPACE ) ) {
if ( ! isset( $assoc_args['dry-run'] ) ) {
// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
echo "Warning: Opik is not configured. Running with --dry-run only.\n";
$assoc_args['dry-run'] = true;
}
}
$limit = absint( $assoc_args['limit'] ?? 0 );
$dry_run = isset( $assoc_args['dry-run'] );
$runner = new PRAutoBlogger_Opik_Eval_Runner();
$result = $runner->run( $limit, $dry_run );
exit( $result['items_run'] > 0 ? 0 : 1 );
}
}