feat: wave 2 eval harness with lm-as-judge and cli command
New eval framework: - eval/dataset.json: 20 frozen peptide topic eval items - class-opik-eval-scorer.php: LLM-as-judge using Gemini Flash 1.5 Scores on: scientific_signal, readability, style_adherence (0-10) - class-opik-eval-runner.php: orchestrator for full eval pass Loads dataset, runs generation (no publish/images), scores, pushes to Opik run(int $limit = 0, bool $dry_run = false): array - class-wp-cli-commands.php: registers wp prautoblogger opik:eval Flags: --limit=<n> (default 0 = all), --dry-run (score only) Features: - Rate limiting: 500ms between items - Cost tracking: eval judge calls logged to opik_eval_judge stage - Dry-run mode: test locally without Opik API push - Experiment naming: prautoblogger-eval-v<VERSION>-<YYYY-MM-DD> All files <= 300 lines. Version 0.13.0. Ready for PR.
This commit is contained in:
72
includes/admin/class-wp-cli-commands.php
Normal file
72
includes/admin/class-wp-cli-commands.php
Normal file
@@ -0,0 +1,72 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* WP-CLI commands for PRAutoBlogger.
|
||||
*
|
||||
* Registers custom wp-cli commands exposed to the plugin.
|
||||
*/
|
||||
class PRAutoBlogger_WP_CLI_Commands {
|
||||
|
||||
/**
|
||||
* Register WP-CLI commands.
|
||||
*
|
||||
* Called on plugins_loaded hook.
|
||||
*/
|
||||
public static function register(): void {
|
||||
if ( ! defined( 'WP_CLI' ) || ! WP_CLI ) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Command: wp prautoblogger opik:eval
|
||||
\WP_CLI::add_command(
|
||||
'prautoblogger opik:eval',
|
||||
array( self::class, 'opik_eval_command' ),
|
||||
array(
|
||||
'shortdesc' => 'Run Opik evals on the frozen dataset',
|
||||
'synopsis' => array(
|
||||
array(
|
||||
'type' => 'assoc',
|
||||
'name' => 'limit',
|
||||
'description' => 'Max items to run (0 = all)',
|
||||
'optional' => true,
|
||||
'default' => '0',
|
||||
),
|
||||
array(
|
||||
'type' => 'flag',
|
||||
'name' => 'dry-run',
|
||||
'description' => 'Skip Opik API push; score locally only',
|
||||
'optional' => true,
|
||||
),
|
||||
),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run eval on the frozen dataset.
|
||||
*
|
||||
* @param array $args Positional args (unused).
|
||||
* @param array $assoc_args Associative args: limit, dry-run.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public static function opik_eval_command( array $args, array $assoc_args ): void {
|
||||
// Check if Opik is enabled.
|
||||
if ( empty( PRAUTOBLOGGER_OPIK_API_KEY ) || empty( PRAUTOBLOGGER_OPIK_WORKSPACE ) ) {
|
||||
if ( ! isset( $assoc_args['dry-run'] ) ) {
|
||||
// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
|
||||
echo "Warning: Opik is not configured. Running with --dry-run only.\n";
|
||||
$assoc_args['dry-run'] = true;
|
||||
}
|
||||
}
|
||||
|
||||
$limit = absint( $assoc_args['limit'] ?? 0 );
|
||||
$dry_run = isset( $assoc_args['dry-run'] );
|
||||
|
||||
$runner = new PRAutoBlogger_Opik_Eval_Runner();
|
||||
$result = $runner->run( $limit, $dry_run );
|
||||
|
||||
exit( $result['items_run'] > 0 ? 0 : 1 );
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user