peptide-repo/pr-vision
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.3.0 — cost accountability + LLM call audit trail (#7)

Browse Source 
Per-call recording of cost and I/O for every LLM probe. New tables prv_call_meta (schema v3) and prv_call_io. New admin pages: Costs (spend drill-down vs $5 cap) and Call Log (paginated + detail drawer). 90-day I/O retention prune via prv_daily_prune cron. P0 security: key never stored; gateway client exposes HTTP status only.

QA APPROVED 2026-06-15 SHA 09dc6db9ce
This commit is contained in:
peptiderepo
2026-06-15 17:07:08 +07:00
committed by GitHub
parent 2f4c1b2c72
commit 2e2c72cac6
37 changed files with 3246 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
tests/bootstrap.php
View File

@@ -27,41 +27,51 @@ if ( ! defined( 'ARRAY_N' ) ) {
define( 'ARRAY_N', 'ARRAY_N' );
}
define( 'PRV_VERSION', '0.2.3' );
define( 'PRV_VERSION', '0.3.0' );
define( 'PRV_PLUGIN_FILE', __DIR__ . '/../pr-vision.php' );
define( 'PRV_PLUGIN_DIR', realpath( __DIR__ . '/..' ) . '/' );
define( 'PRV_PLUGIN_URL', 'http://example.test/wp-content/plugins/pr-vision/' );
define( 'PRV_SCHEMA_VERSION', 2 );
define( 'PRV_SCHEMA_VERSION', 3 );
define( 'PRV_MAX_RETRIES', 3 );
define( 'PRV_API_TIMEOUT_SECONDS', 60 );
define( 'PRV_RETRY_BASE_DELAY_SECONDS', 2 );
define( 'PRV_DEFAULT_MONTHLY_BUDGET_USD', 5.0 );
define( 'PRV_CRON_HOOK', 'prv_weekly_probe' );
define( 'PRV_TARGET_DOMAIN', 'peptiderepo.com' );
define( 'PRV_IO_RETENTION_DEFAULT_DAYS', 90 );
define( 'PRV_DAILY_PRUNE_HOOK', 'prv_daily_prune' );
/* ── Global test state ─────────────────────────────────────────────────── */
$GLOBALS['prv_test_state'] = [
'options' => [],
'transients' => [],
'actions' => [],
'wpdb_insert_id' => 1,
'wpdb_results' => [],
'wpdb_var' => null,
'cron_events' => [],
'remote_posts' => [],
'options' => [],
'transients' => [],
'actions' => [],
'wpdb_insert_id' => 1,
'wpdb_results' => [],
'wpdb_var' => null,
'wpdb_row' => null,
'cron_events' => [],
'remote_posts' => [],
'wpdb_call_meta_rows' => [],
'wpdb_call_io_rows' => [],
'wpdb_dropped_tables' => [],
];
function prv_test_reset(): void {
$GLOBALS['prv_test_state'] = [
'options' => [],
'transients' => [],
'actions' => [],
'wpdb_insert_id' => 1,
'wpdb_results' => [],
'wpdb_var' => null,
'cron_events' => [],
'remote_posts' => [],
'options' => [],
'transients' => [],
'actions' => [],
'wpdb_insert_id' => 1,
'wpdb_results' => [],
'wpdb_var' => null,
'wpdb_row' => null,
'cron_events' => [],
'remote_posts' => [],
'wpdb_call_meta_rows' => [],
'wpdb_call_io_rows' => [],
'wpdb_dropped_tables' => [],
];
PRV_Collector_Registry::reset_for_testing();
}
@@ -182,6 +192,7 @@ function wp_create_nonce( string $action ): string { return 'test_nonce'; }
function submit_button( string $text, string $type = 'primary', string $name = 'submit', bool $wrap = true ): void {}
function add_query_arg( array $args, string $url ): string { return $url . '?' . http_build_query( $args ); }
function admin_url( string $path ): string { return 'http://example.test/wp-admin/' . ltrim( $path, '/' ); }
function plugin_dir_path( string $file ): string { return dirname( $file ) . '/'; }
function wp_safe_redirect( string $url ): void {}
function wp_send_json_success( $data = null ): void { throw new \RuntimeException( 'json_success:' . json_encode( $data ) ); }
function wp_send_json_error( $data = null ): void { throw new \RuntimeException( 'json_error:' . json_encode( $data ) ); }
@@ -230,15 +241,28 @@ class stdClass_wpdb {
public function prepare( string $query, ...$args ): string {
return vsprintf( str_replace( '%s', "'%s'", $query ), array_map( 'addslashes', $args ) );
}
public function query( string $sql ): bool { return true; }
public function query( string $sql ): bool {
if ( preg_match( '/DROP TABLE.*?(prv_\w+)/i', $sql, $m ) ) {
$GLOBALS['prv_test_state']['wpdb_dropped_tables'][] = $m[1];
}
return true;
}
public function insert( string $table, array $data, array $format = [] ): int {
$this->insert_id = $GLOBALS['prv_test_state']['wpdb_insert_id']++;
// Track call_meta and call_io rows for test assertions.
if ( str_contains( $table, 'prv_call_meta' ) ) {
$GLOBALS['prv_test_state']['wpdb_call_meta_rows'][] = $data;
}
if ( str_contains( $table, 'prv_call_io' ) ) {
$GLOBALS['prv_test_state']['wpdb_call_io_rows'][] = $data;
}
return 1;
}
public function update( string $table, array $data, array $where, array $df = [], array $wf = [] ): int { return 1; }
public function get_results( string $query, string $output = 'OBJECT' ): array {
return is_array( $GLOBALS['prv_test_state']['wpdb_results'] ) ? $GLOBALS['prv_test_state']['wpdb_results'] : [];
}
public function get_row( string $query, string $output = 'OBJECT' ): mixed { return $GLOBALS['prv_test_state']['wpdb_row'] ?? null; }
public function get_var( string $query ): mixed { return $GLOBALS['prv_test_state']['wpdb_var']; }
}

142
tests/unit/test-capture-writer.php Normal file
View File

@@ -0,0 +1,142 @@
<?php
/**
* Tests: PRV_Capture_Writer — metadata + I/O writes; security test.
*
* @package PrVision
*/
declare(strict_types=1);
require_once __DIR__ . '/../bootstrap.php';
echo "=== test-capture-writer ===\n";
prv_test_reset();
// ── Test 1: write_meta writes a row to prv_call_meta ─────────────────────
$writer = new PRV_Capture_Writer();
$call_id = $writer->write_meta(
array(
'visibility_row' => 10,
'run_id' => 'abc-123',
'peptide_slug' => 'bpc-157',
'model' => 'perplexity/sonar',
'intent_label' => 'what is {peptide}',
'tokens_in' => 100,
'tokens_out' => 250,
'cost_usd' => 0.00123,
'latency_ms' => 800,
'cited' => 1,
'http_status' => 200,
'config_version' => 3,
)
);
prv_assert( $call_id > 0, 'write_meta returns a positive row ID' );
$meta_rows = $GLOBALS['prv_test_state']['wpdb_call_meta_rows'];
prv_assert( count( $meta_rows ) === 1, 'write_meta inserts exactly one call_meta row' );
prv_assert_equals( 'bpc-157', $meta_rows[0]['peptide_slug'], 'write_meta stores peptide_slug' );
prv_assert_equals( 100, $meta_rows[0]['tokens_in'], 'write_meta stores tokens_in' );
prv_assert_equals( 250, $meta_rows[0]['tokens_out'], 'write_meta stores tokens_out' );
prv_assert_equals( 0, $meta_rows[0]['io_captured'], 'write_meta sets io_captured=0 initially' );
// ── Test 2: write_io writes a row to prv_call_io ─────────────────────────
$ok = $writer->write_io( $call_id, 'what is BPC-157?', 'BPC-157 is a peptide...' );
prv_assert( $ok, 'write_io returns true on success' );
$io_rows = $GLOBALS['prv_test_state']['wpdb_call_io_rows'];
prv_assert( count( $io_rows ) === 1, 'write_io inserts exactly one call_io row' );
prv_assert_equals( 'what is BPC-157?', $io_rows[0]['prompt_text'], 'write_io stores prompt_text' );
prv_assert_equals( 'BPC-157 is a peptide...', $io_rows[0]['response_text'], 'write_io stores response_text' );
// ── Test 3 (P0 SECURITY — direct writer): 401 error record does NOT store key/Bearer ──
// Direct-writer path: simulate the probe-runner error path parameters.
prv_test_reset();
$writer2 = new PRV_Capture_Writer();
$writer2->write_meta(
array(
'visibility_row' => null,
'run_id' => 'error-run-1',
'peptide_slug' => 'tb-500',
'model' => 'openai/gpt-4o',
'intent_label' => 'what is {peptide}',
'tokens_in' => null,
'tokens_out' => null,
'cost_usd' => 0.0,
'latency_ms' => 150,
'cited' => null,
'http_status' => 401,
'config_version' => 3,
)
);
$writer2->write_io( 1, 'what is TB-500?', '' );
$err_meta = $GLOBALS['prv_test_state']['wpdb_call_meta_rows'][0];
$err_io = $GLOBALS['prv_test_state']['wpdb_call_io_rows'][0];
prv_assert_equals( 401, $err_meta['http_status'], 'Error record stores http_status 401' );
$all_stored = wp_json_encode( $err_meta ) . wp_json_encode( $err_io );
prv_assert( false === strpos( $all_stored, 'Bearer' ), 'Stored data does not contain "Bearer"' );
prv_assert( false === strpos( $all_stored, 'sk-or-' ), 'Stored data does not contain "sk-or-"' );
prv_assert( false === strpos( $all_stored, 'Authorization' ), 'Stored data does not contain "Authorization"' );
prv_assert( null === $err_meta['cited'], 'Error record has cited=null' );
prv_assert_equals( 'what is TB-500?', $err_io['prompt_text'], 'Error record stores rendered query as prompt_text' );
// ── Test 4 (P0 SECURITY — full executor path): sentinel key must not leak ─
// Inject a known sentinel as the resolved API key; drive a mocked 401 error
// path end-to-end through PRV_Probe_Run_Executor (via PRV_Probe_Runner).
// Assert the sentinel, "Bearer", and "Authorization" appear in NO stored row.
prv_test_reset();
if ( ! defined( 'PRV_OPENROUTER_API_KEY' ) ) {
define( 'PRV_OPENROUTER_API_KEY', 'sk-or-TEST-LEAK-CANARY' );
}
$GLOBALS['prv_test_state']['options']['prv_monthly_budget_usd'] = 5.0;
$GLOBALS['prv_test_state']['options']['prv_peptides'] = array( array( 'slug' => 'bpc-157', 'label' => 'BPC-157' ) );
$GLOBALS['prv_test_state']['options']['prv_prompt_intents'] = array( 'what is {peptide}' );
$GLOBALS['prv_test_state']['options']['prv_models_v2'] = array(
array( 'slug' => 'openai/gpt-4o', 'enabled' => true ),
);
$GLOBALS['prv_test_state']['options']['prv_active_config_version'] = 3;
$GLOBALS['prv_test_state']['wpdb_var'] = '0.0'; // No prior spend.
// Queue a 401 response for wp_remote_post.
$GLOBALS['prv_test_state']['remote_posts'][] = array(
'response' => array( 'code' => 401 ),
'body' => '{"error":{"message":"Invalid API key","code":401}}',
);
$sentinel_capture = new PRV_Capture_Writer();
$sentinel_ledger = new PRV_Cost_Ledger();
$sentinel_runner = new PRV_Probe_Runner( $sentinel_ledger, $sentinel_capture );
try {
$sentinel_counts = $sentinel_runner->run();
} catch ( \Throwable $ex ) {
$sentinel_counts = array( 'skipped_error' => -99 );
}
// The executor must have recorded the 401 error call (skipped_error >= 1).
prv_assert( isset( $sentinel_counts['skipped_error'] ) && $sentinel_counts['skipped_error'] >= 1, 'Executor recorded a skipped_error for the 401 path' );
// Now verify NO stored row (meta or io) contains the sentinel key or auth material.
$all_meta = $GLOBALS['prv_test_state']['wpdb_call_meta_rows'];
$all_io = $GLOBALS['prv_test_state']['wpdb_call_io_rows'];
$all_json = wp_json_encode( $all_meta ) . wp_json_encode( $all_io );
prv_assert( false === strpos( $all_json, 'sk-or-TEST-LEAK-CANARY' ), 'Sentinel key does not appear in any stored row (executor path)' );
prv_assert( false === strpos( $all_json, 'Bearer' ), 'Bearer does not appear in any stored row (executor path)' );
prv_assert( false === strpos( $all_json, 'Authorization' ), 'Authorization does not appear in any stored row (executor path)' );
prv_assert( count( $all_meta ) >= 1, 'Executor path wrote at least one call_meta row on 401' );
if ( ! empty( $all_meta ) ) {
prv_assert_equals( 401, $all_meta[0]['http_status'], 'Executor 401 path stores http_status=401 in call_meta' );
}
// ── Test 5: update_meta_cost writes back the cost ────────────────────────
prv_test_reset();
$writer3 = new PRV_Capture_Writer();
$result = $writer3->update_meta_cost( 5, 0.004567 );
prv_assert( $result, 'update_meta_cost returns true' );
exit( prv_test_summary() );

91
tests/unit/test-cost-audit.php Normal file
View File

@@ -0,0 +1,91 @@
<?php
/**
* Tests: cost audit trail — rollup reconciliation, prune, and probe-runner
* best-effort capture.
*
* @package PrVision
*/
declare(strict_types=1);
require_once __DIR__ . '/../bootstrap.php';
echo "=== test-cost-audit ===\n";
prv_test_reset();
// ── Test 1: per-call cost sum reconciles to MTD via PRV_Cost_Rollup_Query ─
// Seed get_row mock (used by get_mtd_summary) and get_var (used by PRV_Cost_Ledger).
$GLOBALS['prv_test_state']['wpdb_row'] = array( 'total_cost' => '1.23456789', 'total_calls' => '5' );
$GLOBALS['prv_test_state']['wpdb_var'] = '1.23456789';
$rollup = new PRV_Cost_Rollup_Query();
$summary = $rollup->get_mtd_summary();
prv_assert( $summary['total_cost'] > 0, 'get_mtd_summary returns positive cost when rows present' );
// Reconcile: PRV_Cost_Ledger reads get_var; confirm it agrees with the seeded mock.
$ledger = new PRV_Cost_Ledger();
$mtd_via_ledger = $ledger->get_month_to_date_usd();
prv_assert_equals( (float) '1.23456789', $mtd_via_ledger, 'PRV_Cost_Ledger MTD matches mock value' );
// ── Test 2: project_month_end returns a numeric value ────────────────────
$projected = $rollup->project_month_end( 1.23 );
prv_assert( $projected >= 1.23, 'project_month_end returns at least MTD value' );
prv_assert( is_float( $projected ), 'project_month_end returns a float' );
// ── Test 3: prune_now deletes only prv_call_io (not prv_call_meta) ───────
// PRV_Prune_Cron::prune_now() runs a DELETE on prv_call_io. The wpdb stub
// returns true for all queries; confirm it does NOT drop prv_call_meta.
prv_test_reset();
$deleted = PRV_Prune_Cron::prune_now();
// The stub returns (int)true = 1; just confirm no exception and correct return type.
prv_assert( is_int( $deleted ), 'prune_now returns an integer' );
// Confirm prv_call_meta was not among dropped tables.
prv_assert(
! in_array( 'prv_call_meta', $GLOBALS['prv_test_state']['wpdb_dropped_tables'], true ),
'prune_now does not drop prv_call_meta'
);
// ── Test 4: forced capture_io exception does not break probe ──────────────
// Create a mock capture writer that throws in write_io.
class PRV_Capture_Writer_Throwing extends PRV_Capture_Writer {
public function write_io( int $call_id, string $prompt_text, string $response_text ): bool {
throw new \RuntimeException( 'Simulated capture_io failure' );
}
}
prv_test_reset();
$GLOBALS['prv_test_state']['options']['prv_monthly_budget_usd'] = 5.0;
$GLOBALS['prv_test_state']['options']['prv_peptides'] = array( array( 'slug' => 'bpc-157', 'label' => 'BPC-157' ) );
$GLOBALS['prv_test_state']['options']['prv_prompt_intents'] = array( 'what is {peptide}' );
$GLOBALS['prv_test_state']['options']['prv_models_v2'] = array(
array( 'slug' => 'perplexity/sonar', 'enabled' => true ),
);
$GLOBALS['prv_test_state']['options']['prv_active_config_version'] = 3;
$GLOBALS['prv_test_state']['wpdb_var'] = '0.0'; // No prior spend.
$throwing_capture = new PRV_Capture_Writer_Throwing();
$ledger2 = new PRV_Cost_Ledger();
$runner = new PRV_Probe_Runner( $ledger2, $throwing_capture );
$exception_escaped = false;
try {
$counts = $runner->run();
// If we got here, the exception was swallowed.
} catch ( \Throwable $e ) {
$exception_escaped = true;
}
prv_assert( ! $exception_escaped, 'forced capture_io exception does not propagate to probe runner caller' );
prv_assert( is_array( $counts ?? null ), 'runner returns counts array even when capture_io throws' );
// ── Test 5: table drop methods work correctly ─────────────────────────────
prv_test_reset();
// Simulate what uninstall.php does: call the static drop methods directly.
PRV_Table_Manager::drop_table();
PRV_Call_Io_Table::drop_table();
PRV_Call_Meta_Table::drop_table();
$dropped = $GLOBALS['prv_test_state']['wpdb_dropped_tables'];
prv_assert( in_array( 'prv_call_io', $dropped, true ), 'drop_table() drops prv_call_io' );
prv_assert( in_array( 'prv_call_meta', $dropped, true ), 'drop_table() drops prv_call_meta' );
prv_assert( in_array( 'prv_ai_visibility', $dropped, true ), 'drop_table() drops prv_ai_visibility (existing table)' );
exit( prv_test_summary() );