feat: semantic dedup + LLM-aware topic avoidance
Two-pronged fix for repeated article topics: 1. Feed last 30 days of published article titles into the LLM analysis prompt with explicit "avoid these" instruction. Zero cost — just extra context in an existing API call. 2. Replace keyword-overlap dedup (60% word match) with embedding-based cosine similarity via sentence-transformers/all-MiniLM-L12-v2 on OpenRouter. Catches rephrasings that keyword matching misses (e.g. "BPC-157 dosing" vs "how much BPC-157 to take"). Falls back to keyword matching if the embedding API is unavailable. Cost: ~$0.00001 per generation run ($0.02/M tokens, ~500 tokens/run). New files: - class-semantic-dedup.php — Embedding cosine similarity + keyword fallback - class-open-router-embedding-provider.php — OpenRouter /embeddings client Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
peptiderepo
272
includes/core/class-semantic-dedup.php
Normal file
272
includes/core/class-semantic-dedup.php
Normal file
@@ -0,0 +1,272 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* Semantic deduplication using embedding cosine similarity.
|
||||
*
|
||||
* Embeds candidate topics and recent post titles in a single batch API call,
|
||||
* then compares via cosine similarity. Falls back to keyword-overlap matching
|
||||
* if the embedding API is unavailable or fails.
|
||||
*
|
||||
* Triggered by: PRAutoBlogger_Idea_Scorer::score_and_rank().
|
||||
* Dependencies: OpenRouter_Embedding_Provider, WP_Query.
|
||||
*
|
||||
* @see core/class-idea-scorer.php — Consumer.
|
||||
* @see providers/class-open-router-embedding-provider.php — Embedding API.
|
||||
*/
|
||||
class PRAutoBlogger_Semantic_Dedup {
|
||||
|
||||
/**
|
||||
* Cosine similarity threshold for "too similar". MiniLM embeddings treat
|
||||
* ~0.7+ as semantically equivalent for short texts. We use 0.72 to avoid
|
||||
* over-aggressive blocking while still catching rephrasings.
|
||||
*/
|
||||
private const SIMILARITY_THRESHOLD = 0.72;
|
||||
|
||||
/** @var float[][] Cached embeddings for recent post titles, keyed by index. */
|
||||
private array $title_embeddings = [];
|
||||
|
||||
/** @var string[] Recent post titles. */
|
||||
private array $recent_titles = [];
|
||||
|
||||
/** @var float[][] Cached embeddings for accepted ideas, accumulated per-batch. */
|
||||
private array $accepted_embeddings = [];
|
||||
|
||||
/** @var bool Whether embeddings are available (false after API failure). */
|
||||
private bool $embeddings_available = true;
|
||||
|
||||
/** @var PRAutoBlogger_OpenRouter_Embedding_Provider|null Reused across calls. */
|
||||
private ?PRAutoBlogger_OpenRouter_Embedding_Provider $provider = null;
|
||||
|
||||
/**
|
||||
* Initialize by fetching recent titles and pre-computing their embeddings.
|
||||
*
|
||||
* Called once at the start of scoring. If the embedding API fails,
|
||||
* sets a flag so all subsequent checks fall back to keyword matching.
|
||||
*
|
||||
* Side effects: HTTP request to OpenRouter embedding API, WP_Query.
|
||||
*/
|
||||
public function initialize(): void {
|
||||
$this->recent_titles = $this->fetch_recent_post_titles();
|
||||
if ( empty( $this->recent_titles ) ) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
$this->provider = new PRAutoBlogger_OpenRouter_Embedding_Provider();
|
||||
$this->title_embeddings = $this->provider->get_embeddings( $this->recent_titles );
|
||||
} catch ( \Throwable $e ) {
|
||||
PRAutoBlogger_Logger::instance()->warning(
|
||||
'Semantic dedup unavailable, falling back to keyword matching: ' . $e->getMessage(),
|
||||
'scorer'
|
||||
);
|
||||
$this->embeddings_available = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a topic is too similar to any already-accepted idea in this batch.
|
||||
*
|
||||
* @param string $topic Candidate topic text.
|
||||
* @param string[] $topic_keywords Pre-extracted keywords (for fallback).
|
||||
* @param array $keyword_sets Accepted keyword sets (for fallback).
|
||||
*
|
||||
* @return bool True if the topic should be skipped.
|
||||
*/
|
||||
public function is_batch_duplicate( string $topic, array $topic_keywords, array $keyword_sets ): bool {
|
||||
if ( ! $this->embeddings_available || empty( $this->accepted_embeddings ) ) {
|
||||
return $this->keyword_overlaps_any( $topic_keywords, $keyword_sets );
|
||||
}
|
||||
|
||||
$embedding = $this->embed_single( $topic );
|
||||
if ( null === $embedding ) {
|
||||
return $this->keyword_overlaps_any( $topic_keywords, $keyword_sets );
|
||||
}
|
||||
|
||||
foreach ( $this->accepted_embeddings as $accepted ) {
|
||||
$sim = PRAutoBlogger_OpenRouter_Embedding_Provider::cosine_similarity( $embedding, $accepted );
|
||||
if ( $sim >= self::SIMILARITY_THRESHOLD ) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a topic is too similar to any recently published post.
|
||||
*
|
||||
* @param string $topic Candidate topic text.
|
||||
* @param string[] $topic_keywords Pre-extracted keywords (for fallback).
|
||||
*
|
||||
* @return bool True if a recent post is semantically too similar.
|
||||
*/
|
||||
public function is_recent_duplicate( string $topic, array $topic_keywords ): bool {
|
||||
if ( ! $this->embeddings_available || empty( $this->title_embeddings ) ) {
|
||||
return $this->keyword_matches_recent( $topic_keywords );
|
||||
}
|
||||
|
||||
$embedding = $this->embed_single( $topic );
|
||||
if ( null === $embedding ) {
|
||||
return $this->keyword_matches_recent( $topic_keywords );
|
||||
}
|
||||
|
||||
foreach ( $this->title_embeddings as $i => $title_emb ) {
|
||||
$sim = PRAutoBlogger_OpenRouter_Embedding_Provider::cosine_similarity( $embedding, $title_emb );
|
||||
if ( $sim >= self::SIMILARITY_THRESHOLD ) {
|
||||
PRAutoBlogger_Logger::instance()->info(
|
||||
sprintf(
|
||||
'Semantic match (%.2f): "%s" ≈ "%s"',
|
||||
$sim,
|
||||
substr( $topic, 0, 60 ),
|
||||
substr( $this->recent_titles[ $i ] ?? '', 0, 60 )
|
||||
),
|
||||
'scorer'
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record an accepted idea's embedding for intra-batch dedup.
|
||||
*
|
||||
* @param string $topic The accepted topic text.
|
||||
*/
|
||||
public function record_accepted( string $topic ): void {
|
||||
if ( ! $this->embeddings_available ) {
|
||||
return;
|
||||
}
|
||||
$embedding = $this->embed_single( $topic );
|
||||
if ( null !== $embedding ) {
|
||||
$this->accepted_embeddings[] = $embedding;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Private helpers ─────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Embed a single text, with error handling.
|
||||
*
|
||||
* @param string $text Text to embed.
|
||||
* @return float[]|null Embedding vector, or null on failure.
|
||||
*/
|
||||
private function embed_single( string $text ): ?array {
|
||||
try {
|
||||
if ( null === $this->provider ) {
|
||||
$this->provider = new PRAutoBlogger_OpenRouter_Embedding_Provider();
|
||||
}
|
||||
$result = $this->provider->get_embeddings( [ $text ] );
|
||||
return $result[0] ?? null;
|
||||
} catch ( \Throwable $e ) {
|
||||
// Degrade gracefully — one failed embed shouldn't kill the pipeline.
|
||||
$this->embeddings_available = false;
|
||||
PRAutoBlogger_Logger::instance()->warning(
|
||||
'Embedding failed, disabling semantic dedup for this run: ' . $e->getMessage(),
|
||||
'scorer'
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Keyword-overlap fallback for intra-batch dedup.
|
||||
*
|
||||
* @param string[] $keywords Keywords to check.
|
||||
* @param array $keyword_sets List of previously accepted keyword arrays.
|
||||
* @return bool
|
||||
*/
|
||||
private function keyword_overlaps_any( array $keywords, array $keyword_sets ): bool {
|
||||
if ( count( $keywords ) < 2 ) {
|
||||
return false;
|
||||
}
|
||||
foreach ( $keyword_sets as $existing ) {
|
||||
$overlap = count( array_intersect( $keywords, $existing ) );
|
||||
if ( $overlap / count( $keywords ) >= 0.6 ) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keyword-overlap fallback for recent-post dedup.
|
||||
*
|
||||
* @param string[] $topic_keywords Pre-extracted keywords.
|
||||
* @return bool
|
||||
*/
|
||||
private function keyword_matches_recent( array $topic_keywords ): bool {
|
||||
if ( count( $topic_keywords ) < 2 ) {
|
||||
return false;
|
||||
}
|
||||
foreach ( $this->recent_titles as $title ) {
|
||||
$title_keywords = $this->extract_meaningful_keywords( $title );
|
||||
$overlap = count( array_intersect( $topic_keywords, $title_keywords ) );
|
||||
if ( $overlap / count( $topic_keywords ) >= 0.6 ) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch titles of generated posts from the last 30 days.
|
||||
*
|
||||
* Widened from 7 days (the old keyword window) because semantic similarity
|
||||
* catches rephrasings that keywords miss — a wider net is safe here.
|
||||
*
|
||||
* @return string[]
|
||||
*/
|
||||
private function fetch_recent_post_titles(): array {
|
||||
$query = new \WP_Query( [
|
||||
'post_type' => 'post',
|
||||
'post_status' => [ 'publish', 'draft', 'pending' ],
|
||||
'meta_key' => '_prautoblogger_generated',
|
||||
'meta_value' => '1',
|
||||
'date_query' => [ [ 'after' => '30 days ago' ] ],
|
||||
'posts_per_page' => 100,
|
||||
'fields' => 'ids',
|
||||
'no_found_rows' => true,
|
||||
] );
|
||||
|
||||
$titles = [];
|
||||
foreach ( $query->posts as $post_id ) {
|
||||
$titles[] = get_the_title( $post_id );
|
||||
}
|
||||
return $titles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract meaningful keywords from text (fallback dedup).
|
||||
*
|
||||
* @param string $text Input text.
|
||||
* @return string[] Unique lowercase keywords.
|
||||
*/
|
||||
private function extract_meaningful_keywords( string $text ): array {
|
||||
static $stopwords = [
|
||||
'a', 'an', 'the', 'and', 'or', 'but', 'is', 'are', 'was', 'were',
|
||||
'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'from', 'as',
|
||||
'it', 'its', 'this', 'that', 'your', 'you', 'how', 'what', 'when',
|
||||
'why', 'where', 'which', 'who', 'do', 'does', 'did', 'not', 'no',
|
||||
'can', 'will', 'should', 'could', 'would', 'may', 'might', 'have',
|
||||
'has', 'had', 'be', 'been', 'being', 'about', 'into', 'through',
|
||||
'during', 'before', 'after', 'above', 'below', 'between', 'same',
|
||||
'up', 'down', 'out', 'off', 'over', 'under', 'again', 'then',
|
||||
'here', 'there', 'all', 'each', 'every', 'both', 'few', 'more',
|
||||
'most', 'other', 'some', 'such', 'than', 'too', 'very', 'just',
|
||||
'also', 'only', 'own', 'so', 'if', 'while', 'because', 'until',
|
||||
'vs', 'versus', 'guide', 'complete', 'ultimate', 'best', 'top',
|
||||
'new', 'first', 'need', 'know', 'everything',
|
||||
];
|
||||
|
||||
$text = strtolower( $text );
|
||||
$words = preg_split( '/[^a-z0-9-]+/', $text, -1, PREG_SPLIT_NO_EMPTY );
|
||||
$words = array_filter( $words, static function ( string $w ) use ( $stopwords ): bool {
|
||||
return strlen( $w ) >= 2 && ! in_array( $w, $stopwords, true );
|
||||
} );
|
||||
|
||||
return array_values( array_unique( $words ) );
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user