From 000c21a9e82891598d9ae8b305047fc32c5e6e1f Mon Sep 17 00:00:00 2001 From: peptiderepo <273788462+peptiderepo@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:12:20 +0700 Subject: [PATCH] feat(ci): add Neurometric PR-Summarizer as 4th QA voice (#9) Neurometric markets a task-specialized PR-Summarizer SLM (Qwen3-4B + specialist system prompt) on a free tier. Adding it as a 4th voice in the existing A/B reviewer panel to see whether a specialized small model catches things the 3 general-purpose voices miss. The SLM does not honor OpenAI tool-calling, so it cannot return our structured submit_review schema. Wired as a "prose voice": - MODELS tuple gains a 6th provider_type field ("openrouter_tools" | "neurometric_prose") - call_model() dispatches; new call_neurometric_prose() path hits api.neurometric.ai/v1 without tools, embeds raw markdown output, and infers verdict from keyword heuristic - Render branch in PR comment marks the verdict as heuristic and notes tokens/cost are not tracked (free tier) - Never the primary voice; heading still driven by Haiku Also hardens the shared render path against models that omit the optional 'summary' field (rev.get('summary') fallback). Requires new repo secret NEUROMETRIC_API_KEY (already set on all four peptiderepo repos). Voice self-skips with inline error if missing. Agent-Session: cowork-neurometric-qa-voice-20260415 Co-authored-by: peptiderepo --- .github/workflows/cto-review.yml | 124 +++++++++++++++++++++++++++---- 1 file changed, 111 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cto-review.yml b/.github/workflows/cto-review.yml index 0c501f3..b43838e 100644 --- a/.github/workflows/cto-review.yml +++ b/.github/workflows/cto-review.yml @@ -1,15 +1,20 @@ name: CTO Review -# Automated PR review powered by Claude + DeepSeek + Kimi via OpenRouter. +# Automated PR review powered by Claude + DeepSeek + Kimi (via OpenRouter) +# plus Neurometric PR-Summarizer (a Qwen3-4B task-specialist SLM). # -# During the A/B window (until further notice), every PR gets reviewed by THREE models +# During the A/B window (until further notice), every PR gets reviewed by FOUR models # in parallel. The PR comment shows each model's verdict side-by-side so we can compare # quality/cost on real PRs before picking a permanent default. # +# Neurometric is a "prose voice": it does not support OpenAI tool-calling, so its +# contribution is raw markdown + a heuristic verdict. It is never the primary voice. +# # Does NOT merge. Does NOT post formal APPROVE/REQUEST_CHANGES reviews — always COMMENT # (GITHUB_TOKEN can't formally approve PRs without repo-setting toggle). # -# Requires repo secret: OPENROUTER_API_KEY +# Requires repo secrets: OPENROUTER_API_KEY, NEUROMETRIC_API_KEY (optional; the +# Neurometric voice self-skips with an inline error note if the key is missing). # # Cost cap: tracked in Repository Variables (CTO_REVIEW_SPEND_CENTS, CTO_REVIEW_CAP_CENTS). # Default cap is 1000 cents ($10/month). Auto-resets on the 1st of each month. @@ -145,11 +150,12 @@ jobs: echo "EOF_CONV" } >> "$GITHUB_OUTPUT" - - name: Call 3 models in parallel via OpenRouter + - name: Call 4 models in parallel (OpenRouter + Neurometric) if: steps.budget.outputs.skip != 'true' id: review env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + NEUROMETRIC_API_KEY: ${{ secrets.NEUROMETRIC_API_KEY }} PR_TITLE: ${{ github.event.pull_request.title }} PR_BODY: ${{ github.event.pull_request.body }} PR_AUTHOR: ${{ github.event.pull_request.user.login }} @@ -181,14 +187,19 @@ jobs: api_key = os.environ["OPENROUTER_API_KEY"] - # A/B harness: three models. Each gets the same prompt. + # A/B harness: four models. Each gets the same prompt. # Primary model's verdict drives the heading. Others are labeled. # Edit this list to change the roster (or drop to 1 after A/B window closes). + # + # provider_type semantics: + # - "openrouter_tools": OpenAI tool-calling via OpenRouter; structured verdict. + # - "neurometric_prose": Neurometric marketplace SLM; prose output + heuristic verdict. MODELS = [ - # (openrouter slug, display name, input_per_M_USD, output_per_M_USD, is_primary) - ("anthropic/claude-haiku-4.5", "Claude Haiku 4.5", 1.00, 5.00, True), - ("deepseek/deepseek-chat-v3.1", "DeepSeek V3.1", 0.15, 0.75, False), - ("moonshotai/kimi-k2-thinking", "Kimi K2 Thinking", 0.60, 2.50, False), + # (slug, display name, input_per_M_USD, output_per_M_USD, is_primary, provider_type) + ("anthropic/claude-haiku-4.5", "Claude Haiku 4.5", 1.00, 5.00, True, "openrouter_tools"), + ("deepseek/deepseek-chat-v3.1", "DeepSeek V3.1", 0.15, 0.75, False, "openrouter_tools"), + ("moonshotai/kimi-k2-thinking", "Kimi K2 Thinking", 0.60, 2.50, False, "openrouter_tools"), + ("neurometric/pr-summarizer", "Neurometric PR-Summarizer (Qwen3-4B)", 0.00, 0.00, False, "neurometric_prose"), ] system = """You are the CTO and lead reviewer for the peptiderepo project. You are reviewing a pull request opened by an autonomous agent or human contributor. @@ -251,7 +262,28 @@ jobs: PER_MODEL_TIMEOUT = 60 # seconds; one hung model should not block siblings - def call_model(slug, display, in_rate, out_rate, is_primary): + # Heuristic verdict inference for prose-voice models that can't do tool-calling. + # Keyword order matters: block markers beat approve markers. + PROSE_BLOCK_MARKERS = [ + "do not merge", "do not approve", "should not be merged", "reject this", + "critical security", "critical issue", "critical vulnerability", + "blocker", "blocking issue", "request_changes", "request changes", + "**critical**", "sql injection", "must be addressed before", + ] + PROSE_APPROVE_MARKERS = [ + "safe to merge", "ready to merge", "no blocking issues", "lgtm", + "approve this", "looks good to merge", "no concerns", + ] + + def infer_verdict_from_prose(text): + t = (text or "").lower() + if any(m in t for m in PROSE_BLOCK_MARKERS): + return "REQUEST_CHANGES" + if any(m in t for m in PROSE_APPROVE_MARKERS): + return "APPROVE" + return "COMMENT" + + def call_openrouter_tools(slug, display, in_rate, out_rate, is_primary): body = { "model": slug, "max_tokens": 2048, @@ -279,6 +311,7 @@ jobs: except Exception as e: return { "slug": slug, "display": display, "is_primary": is_primary, + "provider_type": "openrouter_tools", "error": str(e)[:300], "cost_cents": 0, } @@ -286,9 +319,9 @@ jobs: choice = data.get("choices", [{}])[0].get("message", {}) tool_calls = choice.get("tool_calls", []) if not tool_calls: - # Some non-Anthropic providers occasionally put the tool args in content. return { "slug": slug, "display": display, "is_primary": is_primary, + "provider_type": "openrouter_tools", "error": f"No tool_call in response. Raw content: {str(choice)[:300]}", "cost_cents": 0, } @@ -298,6 +331,7 @@ jobs: except Exception as e: return { "slug": slug, "display": display, "is_primary": is_primary, + "provider_type": "openrouter_tools", "error": f"Malformed tool args: {e}", "cost_cents": 0, } @@ -309,10 +343,62 @@ jobs: cost_cents = cost_usd * 100 return { "slug": slug, "display": display, "is_primary": is_primary, + "provider_type": "openrouter_tools", "review": review, "prompt_tokens": ptok, "completion_tokens": ctok, "cost_cents": cost_cents, } + def call_neurometric_prose(slug, display, is_primary): + nm_key = os.environ.get("NEUROMETRIC_API_KEY", "") + if not nm_key: + return { + "slug": slug, "display": display, "is_primary": is_primary, + "provider_type": "neurometric_prose", + "error": "NEUROMETRIC_API_KEY secret not set on this repo", + "cost_cents": 0, + } + body = { + "model": slug, + "max_tokens": 2048, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user_message}, + ], + } + req = urllib.request.Request( + "https://api.neurometric.ai/v1/chat/completions", + data=json.dumps(body).encode(), + headers={ + "Authorization": f"Bearer {nm_key}", + "content-type": "application/json", + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=PER_MODEL_TIMEOUT) as resp: + data = json.loads(resp.read()) + except Exception as e: + return { + "slug": slug, "display": display, "is_primary": is_primary, + "provider_type": "neurometric_prose", + "error": str(e)[:300], + "cost_cents": 0, + } + prose = data.get("choices", [{}])[0].get("message", {}).get("content") or "" + verdict = infer_verdict_from_prose(prose) + return { + "slug": slug, "display": display, "is_primary": is_primary, + "provider_type": "neurometric_prose", + "review": {"verdict": verdict, "prose": prose}, + "prompt_tokens": 0, "completion_tokens": 0, + "cost_cents": 0, + } + + def call_model(slug, display, in_rate, out_rate, is_primary, provider_type="openrouter_tools"): + if provider_type == "neurometric_prose": + return call_neurometric_prose(slug, display, is_primary) + return call_openrouter_tools(slug, display, in_rate, out_rate, is_primary) + results = [] with concurrent.futures.ThreadPoolExecutor(max_workers=len(MODELS)) as pool: futures = [pool.submit(call_model, *m) for m in MODELS] @@ -342,9 +428,21 @@ jobs: continue rev = r["review"] lines.append("") - lines.append(f"**Verdict:** {rev['verdict']}") + if r.get("provider_type") == "neurometric_prose": + lines.append(f"**Verdict:** {rev['verdict']} _(heuristic, inferred from prose)_") + lines.append("") + # Cap at 6KB to keep PR comments readable; 4B model output is rarely over this + prose = rev.get("prose", "") + if len(prose) > 6000: + prose = prose[:6000] + "\n\n_…output truncated for comment length…_" + lines.append(prose) + lines.append("") + lines.append("_free tier; per-call token usage not tracked_") + lines.append("") + continue + lines.append(f"**Verdict:** {rev.get('verdict', 'COMMENT')}") lines.append("") - lines.append(rev["summary"]) + lines.append(rev.get("summary") or "_(model returned no summary)_") lines.append("") if rev.get("concerns"): lines.append("**Concerns:**")