Compare commits
1 Commits
fix/admin-
...
claude/wor
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5ece95f236 |
124
.github/workflows/cto-review.yml
vendored
124
.github/workflows/cto-review.yml
vendored
@@ -1,20 +1,15 @@
|
|||||||
name: CTO Review
|
name: CTO Review
|
||||||
|
|
||||||
# Automated PR review powered by Claude + DeepSeek + Kimi (via OpenRouter)
|
# Automated PR review powered by Claude + DeepSeek + Kimi via OpenRouter.
|
||||||
# plus Neurometric PR-Summarizer (a Qwen3-4B task-specialist SLM).
|
|
||||||
#
|
#
|
||||||
# During the A/B window (until further notice), every PR gets reviewed by FOUR models
|
# During the A/B window (until further notice), every PR gets reviewed by THREE models
|
||||||
# in parallel. The PR comment shows each model's verdict side-by-side so we can compare
|
# in parallel. The PR comment shows each model's verdict side-by-side so we can compare
|
||||||
# quality/cost on real PRs before picking a permanent default.
|
# quality/cost on real PRs before picking a permanent default.
|
||||||
#
|
#
|
||||||
# Neurometric is a "prose voice": it does not support OpenAI tool-calling, so its
|
|
||||||
# contribution is raw markdown + a heuristic verdict. It is never the primary voice.
|
|
||||||
#
|
|
||||||
# Does NOT merge. Does NOT post formal APPROVE/REQUEST_CHANGES reviews — always COMMENT
|
# Does NOT merge. Does NOT post formal APPROVE/REQUEST_CHANGES reviews — always COMMENT
|
||||||
# (GITHUB_TOKEN can't formally approve PRs without repo-setting toggle).
|
# (GITHUB_TOKEN can't formally approve PRs without repo-setting toggle).
|
||||||
#
|
#
|
||||||
# Requires repo secrets: OPENROUTER_API_KEY, NEUROMETRIC_API_KEY (optional; the
|
# Requires repo secret: OPENROUTER_API_KEY
|
||||||
# Neurometric voice self-skips with an inline error note if the key is missing).
|
|
||||||
#
|
#
|
||||||
# Cost cap: tracked in Repository Variables (CTO_REVIEW_SPEND_CENTS, CTO_REVIEW_CAP_CENTS).
|
# Cost cap: tracked in Repository Variables (CTO_REVIEW_SPEND_CENTS, CTO_REVIEW_CAP_CENTS).
|
||||||
# Default cap is 1000 cents ($10/month). Auto-resets on the 1st of each month.
|
# Default cap is 1000 cents ($10/month). Auto-resets on the 1st of each month.
|
||||||
@@ -150,12 +145,11 @@ jobs:
|
|||||||
echo "EOF_CONV"
|
echo "EOF_CONV"
|
||||||
} >> "$GITHUB_OUTPUT"
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
- name: Call 4 models in parallel (OpenRouter + Neurometric)
|
- name: Call 3 models in parallel via OpenRouter
|
||||||
if: steps.budget.outputs.skip != 'true'
|
if: steps.budget.outputs.skip != 'true'
|
||||||
id: review
|
id: review
|
||||||
env:
|
env:
|
||||||
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||||
NEUROMETRIC_API_KEY: ${{ secrets.NEUROMETRIC_API_KEY }}
|
|
||||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||||
PR_BODY: ${{ github.event.pull_request.body }}
|
PR_BODY: ${{ github.event.pull_request.body }}
|
||||||
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
|
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
|
||||||
@@ -187,19 +181,14 @@ jobs:
|
|||||||
|
|
||||||
api_key = os.environ["OPENROUTER_API_KEY"]
|
api_key = os.environ["OPENROUTER_API_KEY"]
|
||||||
|
|
||||||
# A/B harness: four models. Each gets the same prompt.
|
# A/B harness: three models. Each gets the same prompt.
|
||||||
# Primary model's verdict drives the heading. Others are labeled.
|
# Primary model's verdict drives the heading. Others are labeled.
|
||||||
# Edit this list to change the roster (or drop to 1 after A/B window closes).
|
# Edit this list to change the roster (or drop to 1 after A/B window closes).
|
||||||
#
|
|
||||||
# provider_type semantics:
|
|
||||||
# - "openrouter_tools": OpenAI tool-calling via OpenRouter; structured verdict.
|
|
||||||
# - "neurometric_prose": Neurometric marketplace SLM; prose output + heuristic verdict.
|
|
||||||
MODELS = [
|
MODELS = [
|
||||||
# (slug, display name, input_per_M_USD, output_per_M_USD, is_primary, provider_type)
|
# (openrouter slug, display name, input_per_M_USD, output_per_M_USD, is_primary)
|
||||||
("anthropic/claude-haiku-4.5", "Claude Haiku 4.5", 1.00, 5.00, True, "openrouter_tools"),
|
("anthropic/claude-haiku-4.5", "Claude Haiku 4.5", 1.00, 5.00, True),
|
||||||
("deepseek/deepseek-chat-v3.1", "DeepSeek V3.1", 0.15, 0.75, False, "openrouter_tools"),
|
("deepseek/deepseek-chat-v3.1", "DeepSeek V3.1", 0.15, 0.75, False),
|
||||||
("moonshotai/kimi-k2-thinking", "Kimi K2 Thinking", 0.60, 2.50, False, "openrouter_tools"),
|
("moonshotai/kimi-k2-thinking", "Kimi K2 Thinking", 0.60, 2.50, False),
|
||||||
("neurometric/pr-summarizer", "Neurometric PR-Summarizer (Qwen3-4B)", 0.00, 0.00, False, "neurometric_prose"),
|
|
||||||
]
|
]
|
||||||
|
|
||||||
system = """You are the CTO and lead reviewer for the peptiderepo project. You are reviewing a pull request opened by an autonomous agent or human contributor.
|
system = """You are the CTO and lead reviewer for the peptiderepo project. You are reviewing a pull request opened by an autonomous agent or human contributor.
|
||||||
@@ -262,28 +251,7 @@ jobs:
|
|||||||
|
|
||||||
PER_MODEL_TIMEOUT = 60 # seconds; one hung model should not block siblings
|
PER_MODEL_TIMEOUT = 60 # seconds; one hung model should not block siblings
|
||||||
|
|
||||||
# Heuristic verdict inference for prose-voice models that can't do tool-calling.
|
def call_model(slug, display, in_rate, out_rate, is_primary):
|
||||||
# Keyword order matters: block markers beat approve markers.
|
|
||||||
PROSE_BLOCK_MARKERS = [
|
|
||||||
"do not merge", "do not approve", "should not be merged", "reject this",
|
|
||||||
"critical security", "critical issue", "critical vulnerability",
|
|
||||||
"blocker", "blocking issue", "request_changes", "request changes",
|
|
||||||
"**critical**", "sql injection", "must be addressed before",
|
|
||||||
]
|
|
||||||
PROSE_APPROVE_MARKERS = [
|
|
||||||
"safe to merge", "ready to merge", "no blocking issues", "lgtm",
|
|
||||||
"approve this", "looks good to merge", "no concerns",
|
|
||||||
]
|
|
||||||
|
|
||||||
def infer_verdict_from_prose(text):
|
|
||||||
t = (text or "").lower()
|
|
||||||
if any(m in t for m in PROSE_BLOCK_MARKERS):
|
|
||||||
return "REQUEST_CHANGES"
|
|
||||||
if any(m in t for m in PROSE_APPROVE_MARKERS):
|
|
||||||
return "APPROVE"
|
|
||||||
return "COMMENT"
|
|
||||||
|
|
||||||
def call_openrouter_tools(slug, display, in_rate, out_rate, is_primary):
|
|
||||||
body = {
|
body = {
|
||||||
"model": slug,
|
"model": slug,
|
||||||
"max_tokens": 2048,
|
"max_tokens": 2048,
|
||||||
@@ -311,7 +279,6 @@ jobs:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {
|
return {
|
||||||
"slug": slug, "display": display, "is_primary": is_primary,
|
"slug": slug, "display": display, "is_primary": is_primary,
|
||||||
"provider_type": "openrouter_tools",
|
|
||||||
"error": str(e)[:300],
|
"error": str(e)[:300],
|
||||||
"cost_cents": 0,
|
"cost_cents": 0,
|
||||||
}
|
}
|
||||||
@@ -319,9 +286,9 @@ jobs:
|
|||||||
choice = data.get("choices", [{}])[0].get("message", {})
|
choice = data.get("choices", [{}])[0].get("message", {})
|
||||||
tool_calls = choice.get("tool_calls", [])
|
tool_calls = choice.get("tool_calls", [])
|
||||||
if not tool_calls:
|
if not tool_calls:
|
||||||
|
# Some non-Anthropic providers occasionally put the tool args in content.
|
||||||
return {
|
return {
|
||||||
"slug": slug, "display": display, "is_primary": is_primary,
|
"slug": slug, "display": display, "is_primary": is_primary,
|
||||||
"provider_type": "openrouter_tools",
|
|
||||||
"error": f"No tool_call in response. Raw content: {str(choice)[:300]}",
|
"error": f"No tool_call in response. Raw content: {str(choice)[:300]}",
|
||||||
"cost_cents": 0,
|
"cost_cents": 0,
|
||||||
}
|
}
|
||||||
@@ -331,7 +298,6 @@ jobs:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {
|
return {
|
||||||
"slug": slug, "display": display, "is_primary": is_primary,
|
"slug": slug, "display": display, "is_primary": is_primary,
|
||||||
"provider_type": "openrouter_tools",
|
|
||||||
"error": f"Malformed tool args: {e}",
|
"error": f"Malformed tool args: {e}",
|
||||||
"cost_cents": 0,
|
"cost_cents": 0,
|
||||||
}
|
}
|
||||||
@@ -343,62 +309,10 @@ jobs:
|
|||||||
cost_cents = cost_usd * 100
|
cost_cents = cost_usd * 100
|
||||||
return {
|
return {
|
||||||
"slug": slug, "display": display, "is_primary": is_primary,
|
"slug": slug, "display": display, "is_primary": is_primary,
|
||||||
"provider_type": "openrouter_tools",
|
|
||||||
"review": review, "prompt_tokens": ptok, "completion_tokens": ctok,
|
"review": review, "prompt_tokens": ptok, "completion_tokens": ctok,
|
||||||
"cost_cents": cost_cents,
|
"cost_cents": cost_cents,
|
||||||
}
|
}
|
||||||
|
|
||||||
def call_neurometric_prose(slug, display, is_primary):
|
|
||||||
nm_key = os.environ.get("NEUROMETRIC_API_KEY", "")
|
|
||||||
if not nm_key:
|
|
||||||
return {
|
|
||||||
"slug": slug, "display": display, "is_primary": is_primary,
|
|
||||||
"provider_type": "neurometric_prose",
|
|
||||||
"error": "NEUROMETRIC_API_KEY secret not set on this repo",
|
|
||||||
"cost_cents": 0,
|
|
||||||
}
|
|
||||||
body = {
|
|
||||||
"model": slug,
|
|
||||||
"max_tokens": 2048,
|
|
||||||
"messages": [
|
|
||||||
{"role": "system", "content": system},
|
|
||||||
{"role": "user", "content": user_message},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
req = urllib.request.Request(
|
|
||||||
"https://api.neurometric.ai/v1/chat/completions",
|
|
||||||
data=json.dumps(body).encode(),
|
|
||||||
headers={
|
|
||||||
"Authorization": f"Bearer {nm_key}",
|
|
||||||
"content-type": "application/json",
|
|
||||||
},
|
|
||||||
method="POST",
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(req, timeout=PER_MODEL_TIMEOUT) as resp:
|
|
||||||
data = json.loads(resp.read())
|
|
||||||
except Exception as e:
|
|
||||||
return {
|
|
||||||
"slug": slug, "display": display, "is_primary": is_primary,
|
|
||||||
"provider_type": "neurometric_prose",
|
|
||||||
"error": str(e)[:300],
|
|
||||||
"cost_cents": 0,
|
|
||||||
}
|
|
||||||
prose = data.get("choices", [{}])[0].get("message", {}).get("content") or ""
|
|
||||||
verdict = infer_verdict_from_prose(prose)
|
|
||||||
return {
|
|
||||||
"slug": slug, "display": display, "is_primary": is_primary,
|
|
||||||
"provider_type": "neurometric_prose",
|
|
||||||
"review": {"verdict": verdict, "prose": prose},
|
|
||||||
"prompt_tokens": 0, "completion_tokens": 0,
|
|
||||||
"cost_cents": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
def call_model(slug, display, in_rate, out_rate, is_primary, provider_type="openrouter_tools"):
|
|
||||||
if provider_type == "neurometric_prose":
|
|
||||||
return call_neurometric_prose(slug, display, is_primary)
|
|
||||||
return call_openrouter_tools(slug, display, in_rate, out_rate, is_primary)
|
|
||||||
|
|
||||||
results = []
|
results = []
|
||||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(MODELS)) as pool:
|
with concurrent.futures.ThreadPoolExecutor(max_workers=len(MODELS)) as pool:
|
||||||
futures = [pool.submit(call_model, *m) for m in MODELS]
|
futures = [pool.submit(call_model, *m) for m in MODELS]
|
||||||
@@ -428,21 +342,9 @@ jobs:
|
|||||||
continue
|
continue
|
||||||
rev = r["review"]
|
rev = r["review"]
|
||||||
lines.append("")
|
lines.append("")
|
||||||
if r.get("provider_type") == "neurometric_prose":
|
lines.append(f"**Verdict:** {rev['verdict']}")
|
||||||
lines.append(f"**Verdict:** {rev['verdict']} _(heuristic, inferred from prose)_")
|
|
||||||
lines.append("")
|
|
||||||
# Cap at 6KB to keep PR comments readable; 4B model output is rarely over this
|
|
||||||
prose = rev.get("prose", "")
|
|
||||||
if len(prose) > 6000:
|
|
||||||
prose = prose[:6000] + "\n\n_…output truncated for comment length…_"
|
|
||||||
lines.append(prose)
|
|
||||||
lines.append("")
|
|
||||||
lines.append("_free tier; per-call token usage not tracked_")
|
|
||||||
lines.append("")
|
|
||||||
continue
|
|
||||||
lines.append(f"**Verdict:** {rev.get('verdict', 'COMMENT')}")
|
|
||||||
lines.append("")
|
lines.append("")
|
||||||
lines.append(rev.get("summary") or "_(model returned no summary)_")
|
lines.append(rev["summary"])
|
||||||
lines.append("")
|
lines.append("")
|
||||||
if rev.get("concerns"):
|
if rev.get("concerns"):
|
||||||
lines.append("**Concerns:**")
|
lines.append("**Concerns:**")
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
<?php
|
<?php
|
||||||
|
declare( strict_types=1 );
|
||||||
/**
|
/**
|
||||||
* Admin-specific functionality.
|
* Admin-specific functionality.
|
||||||
*
|
*
|
||||||
@@ -16,16 +17,6 @@
|
|||||||
* @see Peptide_News_Admin_Settings_Page — Settings page + log viewer
|
* @see Peptide_News_Admin_Settings_Page — Settings page + log viewer
|
||||||
* @see Peptide_News_Admin_Dashboard_Pages — Dashboard/articles/costs pages
|
* @see Peptide_News_Admin_Dashboard_Pages — Dashboard/articles/costs pages
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// Load delegate classes that this orchestrator depends on.
|
|
||||||
require_once __DIR__ . '/class-pn-admin-assets.php';
|
|
||||||
require_once __DIR__ . '/class-pn-admin-menu.php';
|
|
||||||
require_once __DIR__ . '/class-pn-admin-field-renderers.php';
|
|
||||||
require_once __DIR__ . '/class-pn-admin-settings.php';
|
|
||||||
require_once __DIR__ . '/class-pn-admin-log-viewer.php';
|
|
||||||
require_once __DIR__ . '/class-pn-admin-settings-page.php';
|
|
||||||
require_once __DIR__ . '/class-pn-admin-dashboard-pages.php';
|
|
||||||
|
|
||||||
class Peptide_News_Admin {
|
class Peptide_News_Admin {
|
||||||
|
|
||||||
/** @var string Plugin name/slug */
|
/** @var string Plugin name/slug */
|
||||||
@@ -149,31 +140,4 @@ class Peptide_News_Admin {
|
|||||||
public function render_cost_dashboard_page(): void {
|
public function render_cost_dashboard_page(): void {
|
||||||
$this->dashboard_pages->render_cost_dashboard_page();
|
$this->dashboard_pages->render_cost_dashboard_page();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Sanitize and validate an OpenRouter model ID.
|
|
||||||
*
|
|
||||||
* Delegates to the settings class. Kept on the orchestrator because
|
|
||||||
* WordPress register_setting() callbacks may reference $this.
|
|
||||||
*
|
|
||||||
* @param string $value Raw model ID from form input.
|
|
||||||
* @return string Sanitized model ID or empty string on failure.
|
|
||||||
*/
|
|
||||||
public function sanitize_model_id( string $value ): string {
|
|
||||||
return $this->settings->sanitize_model_id( $value );
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sanitize an API key value — encrypt if new, preserve if masked.
|
|
||||||
*
|
|
||||||
* Delegates to the settings class. Kept on the orchestrator because
|
|
||||||
* WordPress register_setting() callbacks may reference $this.
|
|
||||||
*
|
|
||||||
* @param string $value Raw API key from form input.
|
|
||||||
* @param string $option_name The WP option being saved.
|
|
||||||
* @return string Encrypted key or previous value if input was masked.
|
|
||||||
*/
|
|
||||||
public function sanitize_api_key( string $value, string $option_name = '' ): string {
|
|
||||||
return $this->settings->sanitize_api_key( $value, $option_name );
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -300,55 +300,6 @@ if ( ! function_exists( 'selected' ) ) {
|
|||||||
return $result;
|
return $result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ( ! function_exists( 'checked' ) ) {
|
|
||||||
function checked( $checked, $current = true, $echo = true ) {
|
|
||||||
$result = (string) $checked === (string) $current ? ' checked="checked"' : '';
|
|
||||||
if ( $echo ) {
|
|
||||||
echo $result;
|
|
||||||
}
|
|
||||||
return $result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ( ! function_exists( 'wp_enqueue_style' ) ) {
|
|
||||||
function wp_enqueue_style( $handle, $src = '', $deps = array(), $ver = false, $media = 'all' ) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ( ! function_exists( 'wp_enqueue_script' ) ) {
|
|
||||||
function wp_enqueue_script( $handle, $src = '', $deps = array(), $ver = false, $args = array() ) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ( ! function_exists( 'wp_localize_script' ) ) {
|
|
||||||
function wp_localize_script( $handle, $object_name, $l10n ) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ( ! function_exists( 'add_menu_page' ) ) {
|
|
||||||
function add_menu_page( $page_title, $menu_title, $capability, $menu_slug, $callback = '', $icon_url = '', $position = null ) {
|
|
||||||
return $menu_slug;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ( ! function_exists( 'add_submenu_page' ) ) {
|
|
||||||
function add_submenu_page( $parent_slug, $page_title, $menu_title, $capability, $menu_slug, $callback = '', $position = null ) {
|
|
||||||
return $menu_slug;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ( ! function_exists( 'register_setting' ) ) {
|
|
||||||
function register_setting( $option_group, $option_name, $args = array() ) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ( ! function_exists( 'add_settings_section' ) ) {
|
|
||||||
function add_settings_section( $id, $title, $callback, $page, $args = array() ) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ( ! function_exists( 'add_settings_field' ) ) {
|
|
||||||
function add_settings_field( $id, $title, $callback, $page, $section = 'default', $args = array() ) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cost tracker provides calculate_cost(), budget checks used by tests.
|
// Cost tracker provides calculate_cost(), budget checks used by tests.
|
||||||
require_once PEPTIDE_NEWS_PLUGIN_DIR . 'includes/class-peptide-news-cost-tracker.php';
|
require_once PEPTIDE_NEWS_PLUGIN_DIR . 'includes/class-peptide-news-cost-tracker.php';
|
||||||
|
|||||||
Reference in New Issue
Block a user