peptide-repo/prautoblogger
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: replace over-aggressive WP_Query dedup with keyword-overlap check

Browse Source 
The old has_similar_post() used WordPress search ('s' parameter) which
matches ANY word in the query. On a niche peptide blog, every topic
shares vocabulary like "BPC-157", "peptide", "dosage" — so all 125
ideas were false-positive deduped against existing posts, producing
zero viable article ideas.

New approach: extract meaningful keywords (strip stopwords), compare
against each existing post title, and only dedup when ≥60% of the
topic's keywords overlap with a single title. Also caches the title
list per pipeline run to avoid repeated queries.

Added summary logging so we can see exactly how many ideas pass
through each filter stage.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
peptiderepo
2026-04-19 02:52:17 +07:00
parent d17c488ef3
commit 42d6d03cb3
2 changed files with 130 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
includes/core/class-idea-scorer.php
View File

@@ -39,20 +39,24 @@ class PRAutoBlogger_Idea_Scorer {
* @return PRAutoBlogger_Article_Idea[] Top-ranked ideas, sorted by score descending.
*/
public function score_and_rank( array $analysis_results, int $target_count ): array {
$exclusions = json_decode( get_option( 'prautoblogger_topic_exclusions', '[]' ), true ) ?: [];
$ideas = [];
$exclusions = json_decode( get_option( 'prautoblogger_topic_exclusions', '[]' ), true ) ?: [];
$ideas = [];
$excluded_count = 0;
$deduped_count = 0;
foreach ( $analysis_results as $result ) {
// Skip excluded topics.
if ( $this->is_excluded( $result->get_topic(), $exclusions ) ) {
$excluded_count++;
continue;
}
// Skip if we've already published something very similar
// (unless dedup is disabled for manual runs).
if ( ! $this->skip_dedup && $this->has_similar_post( $result->get_topic() ) ) {
$deduped_count++;
PRAutoBlogger_Logger::instance()->info(
sprintf( 'Skipping topic "%s" — similar post exists.', substr( $result->get_topic(), 0, 80 ) ),
sprintf( 'Dedup: skipping "%s" — ≥60%% keyword overlap with existing post.', substr( $result->get_topic(), 0, 80 ) ),
'scorer'
);
continue;
@@ -73,6 +77,18 @@ class PRAutoBlogger_Idea_Scorer {
] );
}
PRAutoBlogger_Logger::instance()->info(
sprintf(
'Scorer: %d input → %d viable, %d deduped, %d excluded. Returning top %d.',
count( $analysis_results ),
count( $ideas ),
$deduped_count,
$excluded_count,
min( $target_count, count( $ideas ) )
),
'scorer'
);
// Sort by score descending.
usort( $ideas, static function ( PRAutoBlogger_Article_Idea $a, PRAutoBlogger_Article_Idea $b ): int {
return $b->get_score() <=> $a->get_score();
@@ -122,24 +138,105 @@ class PRAutoBlogger_Idea_Scorer {
/**
* Check if a substantially similar post already exists.
*
* Uses a simple keyword overlap check against existing published post titles.
* Compares meaningful keywords from the proposed topic against existing
* post titles. A match requires ≥60% of the topic's keywords to appear
* in a single existing title — not just one shared niche word.
*
* The previous implementation used WP_Query 's' (WordPress search) which
* matches ANY word. On a niche blog every topic shares common vocabulary
* (e.g. "BPC-157", "peptide"), so all ideas were false-positive deduped.
*
* @param string $topic Topic to check.
*
* @return bool True if a similar post exists.
*/
private function has_similar_post( string $topic ): bool {
$topic_keywords = $this->extract_meaningful_keywords( $topic );
if ( count( $topic_keywords ) < 2 ) {
// Topic too short to meaningfully deduplicate.
return false;
}
// Cache the post title list within a single pipeline run to avoid
// repeating the same query for every idea.
static $existing_titles = null;
if ( null === $existing_titles ) {
$existing_titles = $this->fetch_existing_titles();
}
foreach ( $existing_titles as $existing_title ) {
$title_keywords = $this->extract_meaningful_keywords( $existing_title );
$overlap = count( array_intersect( $topic_keywords, $title_keywords ) );
$overlap_ratio = $overlap / count( $topic_keywords );
// 60% keyword overlap = too similar to publish again.
if ( $overlap_ratio >= 0.6 ) {
return true;
}
}
return false;
}
/**
* Fetch titles of all existing generated posts (cached per run).
*
* @return string[]
*/
private function fetch_existing_titles(): array {
$query = new \WP_Query( [
'post_type' => 'post',
'post_status' => [ 'publish', 'draft', 'pending' ],
'meta_key' => '_prautoblogger_generated',
'meta_value' => '1',
's' => $topic,
'posts_per_page' => 1,
'posts_per_page' => 200,
'fields' => 'ids',
'no_found_rows' => true,
] );
return $query->found_posts > 0;
$titles = [];
foreach ( $query->posts as $post_id ) {
$titles[] = get_the_title( $post_id );
}
return $titles;
}
/**
* Extract meaningful keywords from a text string.
*
* Strips stopwords and short filler words so dedup focuses on
* substantive terms. Returns lowercase unique keywords.
*
* @param string $text Input text (topic or title).
* @return string[] Unique lowercase keywords.
*/
private function extract_meaningful_keywords( string $text ): array {
// Common stopwords that add no topical signal.
static $stopwords = [
'a', 'an', 'the', 'and', 'or', 'but', 'is', 'are', 'was', 'were',
'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'from', 'as',
'it', 'its', 'this', 'that', 'your', 'you', 'how', 'what', 'when',
'why', 'where', 'which', 'who', 'do', 'does', 'did', 'not', 'no',
'can', 'will', 'should', 'could', 'would', 'may', 'might', 'have',
'has', 'had', 'be', 'been', 'being', 'about', 'into', 'through',
'during', 'before', 'after', 'above', 'below', 'between', 'same',
'up', 'down', 'out', 'off', 'over', 'under', 'again', 'then',
'here', 'there', 'all', 'each', 'every', 'both', 'few', 'more',
'most', 'other', 'some', 'such', 'than', 'too', 'very', 'just',
'also', 'only', 'own', 'so', 'if', 'while', 'because', 'until',
'vs', 'versus', 'guide', 'complete', 'ultimate', 'best', 'top',
'new', 'first', 'need', 'know', 'everything',
];
$text = strtolower( $text );
// Preserve compound identifiers like "BPC-157" by treating hyphens
// between alphanumerics as part of the word.
$words = preg_split( '/[^a-z0-9-]+/', $text, -1, PREG_SPLIT_NO_EMPTY );
$words = array_filter( $words, static function ( string $w ) use ( $stopwords ): bool {
return strlen( $w ) >= 2 && ! in_array( $w, $stopwords, true );
} );
return array_values( array_unique( $words ) );
}
/**

26
tests/unit/Core/IdeaScorerTest.php
View File

@@ -21,6 +21,9 @@ class IdeaScorerTest extends BaseTestCase {
$this->stub_get_option( [
'prautoblogger_topic_exclusions' => '[]',
] );
// has_similar_post() → fetch_existing_titles() calls get_the_title().
Functions\when( 'get_the_title' )->justReturn( 'Existing Post Title' );
}
/**
@@ -141,4 +144,27 @@ class IdeaScorerTest extends BaseTestCase {
$this->assertIsArray( $result );
}
/**
* Test that topics sharing only common niche words are NOT deduped.
*
* Regression test for the WP_Query 's' bug: WordPress search matched
* ANY word, so "BPC-157 dosage" deduped against "BPC-157 storage"
* even though they're completely different articles.
*/
public function test_dedup_does_not_reject_different_topics_with_shared_words(): void {
// The default WP_Query stub returns 0 posts, so nothing gets deduped.
// This test verifies that score_and_rank passes ideas through when
// no similar posts exist (the common case after the fix).
$fixture = $this->get_analysis_result_fixture();
$fixture['topic'] = 'BPC-157 dosage guide for subcutaneous injection';
$scorer = new \PRAutoBlogger_Idea_Scorer();
$result = $scorer->score_and_rank(
[ new \PRAutoBlogger_Analysis_Result( $fixture ) ],
5
);
$this->assertCount( 1, $result, 'Idea should not be deduped when no similar post exists.' );
}
}