Compare commits
1 Commits
fix/admin-
...
claude/wor
| Author | SHA1 | Date | |
|---|---|---|---|
|
5ece95f236 |
124
.github/workflows/cto-review.yml
vendored
124
.github/workflows/cto-review.yml
vendored
@@ -1,20 +1,15 @@
|
||||
name: CTO Review
|
||||
|
||||
# Automated PR review powered by Claude + DeepSeek + Kimi (via OpenRouter)
|
||||
# plus Neurometric PR-Summarizer (a Qwen3-4B task-specialist SLM).
|
||||
# Automated PR review powered by Claude + DeepSeek + Kimi via OpenRouter.
|
||||
#
|
||||
# During the A/B window (until further notice), every PR gets reviewed by FOUR models
|
||||
# During the A/B window (until further notice), every PR gets reviewed by THREE models
|
||||
# in parallel. The PR comment shows each model's verdict side-by-side so we can compare
|
||||
# quality/cost on real PRs before picking a permanent default.
|
||||
#
|
||||
# Neurometric is a "prose voice": it does not support OpenAI tool-calling, so its
|
||||
# contribution is raw markdown + a heuristic verdict. It is never the primary voice.
|
||||
#
|
||||
# Does NOT merge. Does NOT post formal APPROVE/REQUEST_CHANGES reviews — always COMMENT
|
||||
# (GITHUB_TOKEN can't formally approve PRs without repo-setting toggle).
|
||||
#
|
||||
# Requires repo secrets: OPENROUTER_API_KEY, NEUROMETRIC_API_KEY (optional; the
|
||||
# Neurometric voice self-skips with an inline error note if the key is missing).
|
||||
# Requires repo secret: OPENROUTER_API_KEY
|
||||
#
|
||||
# Cost cap: tracked in Repository Variables (CTO_REVIEW_SPEND_CENTS, CTO_REVIEW_CAP_CENTS).
|
||||
# Default cap is 1000 cents ($10/month). Auto-resets on the 1st of each month.
|
||||
@@ -150,12 +145,11 @@ jobs:
|
||||
echo "EOF_CONV"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Call 4 models in parallel (OpenRouter + Neurometric)
|
||||
- name: Call 3 models in parallel via OpenRouter
|
||||
if: steps.budget.outputs.skip != 'true'
|
||||
id: review
|
||||
env:
|
||||
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||
NEUROMETRIC_API_KEY: ${{ secrets.NEUROMETRIC_API_KEY }}
|
||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||
PR_BODY: ${{ github.event.pull_request.body }}
|
||||
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
|
||||
@@ -187,19 +181,14 @@ jobs:
|
||||
|
||||
api_key = os.environ["OPENROUTER_API_KEY"]
|
||||
|
||||
# A/B harness: four models. Each gets the same prompt.
|
||||
# A/B harness: three models. Each gets the same prompt.
|
||||
# Primary model's verdict drives the heading. Others are labeled.
|
||||
# Edit this list to change the roster (or drop to 1 after A/B window closes).
|
||||
#
|
||||
# provider_type semantics:
|
||||
# - "openrouter_tools": OpenAI tool-calling via OpenRouter; structured verdict.
|
||||
# - "neurometric_prose": Neurometric marketplace SLM; prose output + heuristic verdict.
|
||||
MODELS = [
|
||||
# (slug, display name, input_per_M_USD, output_per_M_USD, is_primary, provider_type)
|
||||
("anthropic/claude-haiku-4.5", "Claude Haiku 4.5", 1.00, 5.00, True, "openrouter_tools"),
|
||||
("deepseek/deepseek-chat-v3.1", "DeepSeek V3.1", 0.15, 0.75, False, "openrouter_tools"),
|
||||
("moonshotai/kimi-k2-thinking", "Kimi K2 Thinking", 0.60, 2.50, False, "openrouter_tools"),
|
||||
("neurometric/pr-summarizer", "Neurometric PR-Summarizer (Qwen3-4B)", 0.00, 0.00, False, "neurometric_prose"),
|
||||
# (openrouter slug, display name, input_per_M_USD, output_per_M_USD, is_primary)
|
||||
("anthropic/claude-haiku-4.5", "Claude Haiku 4.5", 1.00, 5.00, True),
|
||||
("deepseek/deepseek-chat-v3.1", "DeepSeek V3.1", 0.15, 0.75, False),
|
||||
("moonshotai/kimi-k2-thinking", "Kimi K2 Thinking", 0.60, 2.50, False),
|
||||
]
|
||||
|
||||
system = """You are the CTO and lead reviewer for the peptiderepo project. You are reviewing a pull request opened by an autonomous agent or human contributor.
|
||||
@@ -262,28 +251,7 @@ jobs:
|
||||
|
||||
PER_MODEL_TIMEOUT = 60 # seconds; one hung model should not block siblings
|
||||
|
||||
# Heuristic verdict inference for prose-voice models that can't do tool-calling.
|
||||
# Keyword order matters: block markers beat approve markers.
|
||||
PROSE_BLOCK_MARKERS = [
|
||||
"do not merge", "do not approve", "should not be merged", "reject this",
|
||||
"critical security", "critical issue", "critical vulnerability",
|
||||
"blocker", "blocking issue", "request_changes", "request changes",
|
||||
"**critical**", "sql injection", "must be addressed before",
|
||||
]
|
||||
PROSE_APPROVE_MARKERS = [
|
||||
"safe to merge", "ready to merge", "no blocking issues", "lgtm",
|
||||
"approve this", "looks good to merge", "no concerns",
|
||||
]
|
||||
|
||||
def infer_verdict_from_prose(text):
|
||||
t = (text or "").lower()
|
||||
if any(m in t for m in PROSE_BLOCK_MARKERS):
|
||||
return "REQUEST_CHANGES"
|
||||
if any(m in t for m in PROSE_APPROVE_MARKERS):
|
||||
return "APPROVE"
|
||||
return "COMMENT"
|
||||
|
||||
def call_openrouter_tools(slug, display, in_rate, out_rate, is_primary):
|
||||
def call_model(slug, display, in_rate, out_rate, is_primary):
|
||||
body = {
|
||||
"model": slug,
|
||||
"max_tokens": 2048,
|
||||
@@ -311,7 +279,6 @@ jobs:
|
||||
except Exception as e:
|
||||
return {
|
||||
"slug": slug, "display": display, "is_primary": is_primary,
|
||||
"provider_type": "openrouter_tools",
|
||||
"error": str(e)[:300],
|
||||
"cost_cents": 0,
|
||||
}
|
||||
@@ -319,9 +286,9 @@ jobs:
|
||||
choice = data.get("choices", [{}])[0].get("message", {})
|
||||
tool_calls = choice.get("tool_calls", [])
|
||||
if not tool_calls:
|
||||
# Some non-Anthropic providers occasionally put the tool args in content.
|
||||
return {
|
||||
"slug": slug, "display": display, "is_primary": is_primary,
|
||||
"provider_type": "openrouter_tools",
|
||||
"error": f"No tool_call in response. Raw content: {str(choice)[:300]}",
|
||||
"cost_cents": 0,
|
||||
}
|
||||
@@ -331,7 +298,6 @@ jobs:
|
||||
except Exception as e:
|
||||
return {
|
||||
"slug": slug, "display": display, "is_primary": is_primary,
|
||||
"provider_type": "openrouter_tools",
|
||||
"error": f"Malformed tool args: {e}",
|
||||
"cost_cents": 0,
|
||||
}
|
||||
@@ -343,62 +309,10 @@ jobs:
|
||||
cost_cents = cost_usd * 100
|
||||
return {
|
||||
"slug": slug, "display": display, "is_primary": is_primary,
|
||||
"provider_type": "openrouter_tools",
|
||||
"review": review, "prompt_tokens": ptok, "completion_tokens": ctok,
|
||||
"cost_cents": cost_cents,
|
||||
}
|
||||
|
||||
def call_neurometric_prose(slug, display, is_primary):
|
||||
nm_key = os.environ.get("NEUROMETRIC_API_KEY", "")
|
||||
if not nm_key:
|
||||
return {
|
||||
"slug": slug, "display": display, "is_primary": is_primary,
|
||||
"provider_type": "neurometric_prose",
|
||||
"error": "NEUROMETRIC_API_KEY secret not set on this repo",
|
||||
"cost_cents": 0,
|
||||
}
|
||||
body = {
|
||||
"model": slug,
|
||||
"max_tokens": 2048,
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user_message},
|
||||
],
|
||||
}
|
||||
req = urllib.request.Request(
|
||||
"https://api.neurometric.ai/v1/chat/completions",
|
||||
data=json.dumps(body).encode(),
|
||||
headers={
|
||||
"Authorization": f"Bearer {nm_key}",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=PER_MODEL_TIMEOUT) as resp:
|
||||
data = json.loads(resp.read())
|
||||
except Exception as e:
|
||||
return {
|
||||
"slug": slug, "display": display, "is_primary": is_primary,
|
||||
"provider_type": "neurometric_prose",
|
||||
"error": str(e)[:300],
|
||||
"cost_cents": 0,
|
||||
}
|
||||
prose = data.get("choices", [{}])[0].get("message", {}).get("content") or ""
|
||||
verdict = infer_verdict_from_prose(prose)
|
||||
return {
|
||||
"slug": slug, "display": display, "is_primary": is_primary,
|
||||
"provider_type": "neurometric_prose",
|
||||
"review": {"verdict": verdict, "prose": prose},
|
||||
"prompt_tokens": 0, "completion_tokens": 0,
|
||||
"cost_cents": 0,
|
||||
}
|
||||
|
||||
def call_model(slug, display, in_rate, out_rate, is_primary, provider_type="openrouter_tools"):
|
||||
if provider_type == "neurometric_prose":
|
||||
return call_neurometric_prose(slug, display, is_primary)
|
||||
return call_openrouter_tools(slug, display, in_rate, out_rate, is_primary)
|
||||
|
||||
results = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(MODELS)) as pool:
|
||||
futures = [pool.submit(call_model, *m) for m in MODELS]
|
||||
@@ -428,21 +342,9 @@ jobs:
|
||||
continue
|
||||
rev = r["review"]
|
||||
lines.append("")
|
||||
if r.get("provider_type") == "neurometric_prose":
|
||||
lines.append(f"**Verdict:** {rev['verdict']} _(heuristic, inferred from prose)_")
|
||||
lines.append("")
|
||||
# Cap at 6KB to keep PR comments readable; 4B model output is rarely over this
|
||||
prose = rev.get("prose", "")
|
||||
if len(prose) > 6000:
|
||||
prose = prose[:6000] + "\n\n_…output truncated for comment length…_"
|
||||
lines.append(prose)
|
||||
lines.append("")
|
||||
lines.append("_free tier; per-call token usage not tracked_")
|
||||
lines.append("")
|
||||
continue
|
||||
lines.append(f"**Verdict:** {rev.get('verdict', 'COMMENT')}")
|
||||
lines.append(f"**Verdict:** {rev['verdict']}")
|
||||
lines.append("")
|
||||
lines.append(rev.get("summary") or "_(model returned no summary)_")
|
||||
lines.append(rev["summary"])
|
||||
lines.append("")
|
||||
if rev.get("concerns"):
|
||||
lines.append("**Concerns:**")
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<?php
|
||||
declare( strict_types=1 );
|
||||
/**
|
||||
* Admin-specific functionality.
|
||||
*
|
||||
@@ -16,16 +17,6 @@
|
||||
* @see Peptide_News_Admin_Settings_Page — Settings page + log viewer
|
||||
* @see Peptide_News_Admin_Dashboard_Pages — Dashboard/articles/costs pages
|
||||
*/
|
||||
|
||||
// Load delegate classes that this orchestrator depends on.
|
||||
require_once __DIR__ . '/class-pn-admin-assets.php';
|
||||
require_once __DIR__ . '/class-pn-admin-menu.php';
|
||||
require_once __DIR__ . '/class-pn-admin-field-renderers.php';
|
||||
require_once __DIR__ . '/class-pn-admin-settings.php';
|
||||
require_once __DIR__ . '/class-pn-admin-log-viewer.php';
|
||||
require_once __DIR__ . '/class-pn-admin-settings-page.php';
|
||||
require_once __DIR__ . '/class-pn-admin-dashboard-pages.php';
|
||||
|
||||
class Peptide_News_Admin {
|
||||
|
||||
/** @var string Plugin name/slug */
|
||||
@@ -149,31 +140,4 @@ class Peptide_News_Admin {
|
||||
public function render_cost_dashboard_page(): void {
|
||||
$this->dashboard_pages->render_cost_dashboard_page();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize and validate an OpenRouter model ID.
|
||||
*
|
||||
* Delegates to the settings class. Kept on the orchestrator because
|
||||
* WordPress register_setting() callbacks may reference $this.
|
||||
*
|
||||
* @param string $value Raw model ID from form input.
|
||||
* @return string Sanitized model ID or empty string on failure.
|
||||
*/
|
||||
public function sanitize_model_id( string $value ): string {
|
||||
return $this->settings->sanitize_model_id( $value );
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize an API key value — encrypt if new, preserve if masked.
|
||||
*
|
||||
* Delegates to the settings class. Kept on the orchestrator because
|
||||
* WordPress register_setting() callbacks may reference $this.
|
||||
*
|
||||
* @param string $value Raw API key from form input.
|
||||
* @param string $option_name The WP option being saved.
|
||||
* @return string Encrypted key or previous value if input was masked.
|
||||
*/
|
||||
public function sanitize_api_key( string $value, string $option_name = '' ): string {
|
||||
return $this->settings->sanitize_api_key( $value, $option_name );
|
||||
}
|
||||
}
|
||||
|
||||
@@ -300,55 +300,6 @@ if ( ! function_exists( 'selected' ) ) {
|
||||
return $result;
|
||||
}
|
||||
}
|
||||
if ( ! function_exists( 'checked' ) ) {
|
||||
function checked( $checked, $current = true, $echo = true ) {
|
||||
$result = (string) $checked === (string) $current ? ' checked="checked"' : '';
|
||||
if ( $echo ) {
|
||||
echo $result;
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
}
|
||||
if ( ! function_exists( 'wp_enqueue_style' ) ) {
|
||||
function wp_enqueue_style( $handle, $src = '', $deps = array(), $ver = false, $media = 'all' ) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if ( ! function_exists( 'wp_enqueue_script' ) ) {
|
||||
function wp_enqueue_script( $handle, $src = '', $deps = array(), $ver = false, $args = array() ) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if ( ! function_exists( 'wp_localize_script' ) ) {
|
||||
function wp_localize_script( $handle, $object_name, $l10n ) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if ( ! function_exists( 'add_menu_page' ) ) {
|
||||
function add_menu_page( $page_title, $menu_title, $capability, $menu_slug, $callback = '', $icon_url = '', $position = null ) {
|
||||
return $menu_slug;
|
||||
}
|
||||
}
|
||||
if ( ! function_exists( 'add_submenu_page' ) ) {
|
||||
function add_submenu_page( $parent_slug, $page_title, $menu_title, $capability, $menu_slug, $callback = '', $position = null ) {
|
||||
return $menu_slug;
|
||||
}
|
||||
}
|
||||
if ( ! function_exists( 'register_setting' ) ) {
|
||||
function register_setting( $option_group, $option_name, $args = array() ) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if ( ! function_exists( 'add_settings_section' ) ) {
|
||||
function add_settings_section( $id, $title, $callback, $page, $args = array() ) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if ( ! function_exists( 'add_settings_field' ) ) {
|
||||
function add_settings_field( $id, $title, $callback, $page, $section = 'default', $args = array() ) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Cost tracker provides calculate_cost(), budget checks used by tests.
|
||||
require_once PEPTIDE_NEWS_PLUGIN_DIR . 'includes/class-peptide-news-cost-tracker.php';
|
||||
|
||||
Reference in New Issue
Block a user