c79fbb570e
New eval framework: - eval/dataset.json: 20 frozen peptide topic eval items - class-opik-eval-scorer.php: LLM-as-judge using Gemini Flash 1.5 Scores on: scientific_signal, readability, style_adherence (0-10) - class-opik-eval-runner.php: orchestrator for full eval pass Loads dataset, runs generation (no publish/images), scores, pushes to Opik run(int $limit = 0, bool $dry_run = false): array - class-wp-cli-commands.php: registers wp prautoblogger opik:eval Flags: --limit=<n> (default 0 = all), --dry-run (score only) Features: - Rate limiting: 500ms between items - Cost tracking: eval judge calls logged to opik_eval_judge stage - Dry-run mode: test locally without Opik API push - Experiment naming: prautoblogger-eval-v<VERSION>-<YYYY-MM-DD> All files <= 300 lines. Version 0.13.0. Ready for PR.
73 lines
1.8 KiB
PHP
73 lines
1.8 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
/**
|
|
* WP-CLI commands for PRAutoBlogger.
|
|
*
|
|
* Registers custom wp-cli commands exposed to the plugin.
|
|
*/
|
|
class PRAutoBlogger_WP_CLI_Commands {
|
|
|
|
/**
|
|
* Register WP-CLI commands.
|
|
*
|
|
* Called on plugins_loaded hook.
|
|
*/
|
|
public static function register(): void {
|
|
if ( ! defined( 'WP_CLI' ) || ! WP_CLI ) {
|
|
return;
|
|
}
|
|
|
|
// Command: wp prautoblogger opik:eval
|
|
\WP_CLI::add_command(
|
|
'prautoblogger opik:eval',
|
|
array( self::class, 'opik_eval_command' ),
|
|
array(
|
|
'shortdesc' => 'Run Opik evals on the frozen dataset',
|
|
'synopsis' => array(
|
|
array(
|
|
'type' => 'assoc',
|
|
'name' => 'limit',
|
|
'description' => 'Max items to run (0 = all)',
|
|
'optional' => true,
|
|
'default' => '0',
|
|
),
|
|
array(
|
|
'type' => 'flag',
|
|
'name' => 'dry-run',
|
|
'description' => 'Skip Opik API push; score locally only',
|
|
'optional' => true,
|
|
),
|
|
),
|
|
)
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Run eval on the frozen dataset.
|
|
*
|
|
* @param array $args Positional args (unused).
|
|
* @param array $assoc_args Associative args: limit, dry-run.
|
|
*
|
|
* @return void
|
|
*/
|
|
public static function opik_eval_command( array $args, array $assoc_args ): void {
|
|
// Check if Opik is enabled.
|
|
if ( empty( PRAUTOBLOGGER_OPIK_API_KEY ) || empty( PRAUTOBLOGGER_OPIK_WORKSPACE ) ) {
|
|
if ( ! isset( $assoc_args['dry-run'] ) ) {
|
|
// phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_echo -- CLI output
|
|
echo "Warning: Opik is not configured. Running with --dry-run only.\n";
|
|
$assoc_args['dry-run'] = true;
|
|
}
|
|
}
|
|
|
|
$limit = absint( $assoc_args['limit'] ?? 0 );
|
|
$dry_run = isset( $assoc_args['dry-run'] );
|
|
|
|
$runner = new PRAutoBlogger_Opik_Eval_Runner();
|
|
$result = $runner->run( $limit, $dry_run );
|
|
|
|
exit( $result['items_run'] > 0 ? 0 : 1 );
|
|
}
|
|
}
|