diff --git a/includes/core/class-idea-scorer.php b/includes/core/class-idea-scorer.php index 3f83dfa..df4e103 100755 --- a/includes/core/class-idea-scorer.php +++ b/includes/core/class-idea-scorer.php @@ -39,20 +39,24 @@ class PRAutoBlogger_Idea_Scorer { * @return PRAutoBlogger_Article_Idea[] Top-ranked ideas, sorted by score descending. */ public function score_and_rank( array $analysis_results, int $target_count ): array { - $exclusions = json_decode( get_option( 'prautoblogger_topic_exclusions', '[]' ), true ) ?: []; - $ideas = []; + $exclusions = json_decode( get_option( 'prautoblogger_topic_exclusions', '[]' ), true ) ?: []; + $ideas = []; + $excluded_count = 0; + $deduped_count = 0; foreach ( $analysis_results as $result ) { // Skip excluded topics. if ( $this->is_excluded( $result->get_topic(), $exclusions ) ) { + $excluded_count++; continue; } // Skip if we've already published something very similar // (unless dedup is disabled for manual runs). if ( ! $this->skip_dedup && $this->has_similar_post( $result->get_topic() ) ) { + $deduped_count++; PRAutoBlogger_Logger::instance()->info( - sprintf( 'Skipping topic "%s" — similar post exists.', substr( $result->get_topic(), 0, 80 ) ), + sprintf( 'Dedup: skipping "%s" — ≥60%% keyword overlap with existing post.', substr( $result->get_topic(), 0, 80 ) ), 'scorer' ); continue; @@ -73,6 +77,18 @@ class PRAutoBlogger_Idea_Scorer { ] ); } + PRAutoBlogger_Logger::instance()->info( + sprintf( + 'Scorer: %d input → %d viable, %d deduped, %d excluded. Returning top %d.', + count( $analysis_results ), + count( $ideas ), + $deduped_count, + $excluded_count, + min( $target_count, count( $ideas ) ) + ), + 'scorer' + ); + // Sort by score descending. usort( $ideas, static function ( PRAutoBlogger_Article_Idea $a, PRAutoBlogger_Article_Idea $b ): int { return $b->get_score() <=> $a->get_score(); @@ -122,24 +138,105 @@ class PRAutoBlogger_Idea_Scorer { /** * Check if a substantially similar post already exists. * - * Uses a simple keyword overlap check against existing published post titles. + * Compares meaningful keywords from the proposed topic against existing + * post titles. A match requires ≥60% of the topic's keywords to appear + * in a single existing title — not just one shared niche word. + * + * The previous implementation used WP_Query 's' (WordPress search) which + * matches ANY word. On a niche blog every topic shares common vocabulary + * (e.g. "BPC-157", "peptide"), so all ideas were false-positive deduped. * * @param string $topic Topic to check. * * @return bool True if a similar post exists. */ private function has_similar_post( string $topic ): bool { + $topic_keywords = $this->extract_meaningful_keywords( $topic ); + if ( count( $topic_keywords ) < 2 ) { + // Topic too short to meaningfully deduplicate. + return false; + } + + // Cache the post title list within a single pipeline run to avoid + // repeating the same query for every idea. + static $existing_titles = null; + if ( null === $existing_titles ) { + $existing_titles = $this->fetch_existing_titles(); + } + + foreach ( $existing_titles as $existing_title ) { + $title_keywords = $this->extract_meaningful_keywords( $existing_title ); + $overlap = count( array_intersect( $topic_keywords, $title_keywords ) ); + $overlap_ratio = $overlap / count( $topic_keywords ); + + // 60% keyword overlap = too similar to publish again. + if ( $overlap_ratio >= 0.6 ) { + return true; + } + } + + return false; + } + + /** + * Fetch titles of all existing generated posts (cached per run). + * + * @return string[] + */ + private function fetch_existing_titles(): array { $query = new \WP_Query( [ 'post_type' => 'post', 'post_status' => [ 'publish', 'draft', 'pending' ], 'meta_key' => '_prautoblogger_generated', 'meta_value' => '1', - 's' => $topic, - 'posts_per_page' => 1, + 'posts_per_page' => 200, 'fields' => 'ids', + 'no_found_rows' => true, ] ); - return $query->found_posts > 0; + $titles = []; + foreach ( $query->posts as $post_id ) { + $titles[] = get_the_title( $post_id ); + } + return $titles; + } + + /** + * Extract meaningful keywords from a text string. + * + * Strips stopwords and short filler words so dedup focuses on + * substantive terms. Returns lowercase unique keywords. + * + * @param string $text Input text (topic or title). + * @return string[] Unique lowercase keywords. + */ + private function extract_meaningful_keywords( string $text ): array { + // Common stopwords that add no topical signal. + static $stopwords = [ + 'a', 'an', 'the', 'and', 'or', 'but', 'is', 'are', 'was', 'were', + 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'from', 'as', + 'it', 'its', 'this', 'that', 'your', 'you', 'how', 'what', 'when', + 'why', 'where', 'which', 'who', 'do', 'does', 'did', 'not', 'no', + 'can', 'will', 'should', 'could', 'would', 'may', 'might', 'have', + 'has', 'had', 'be', 'been', 'being', 'about', 'into', 'through', + 'during', 'before', 'after', 'above', 'below', 'between', 'same', + 'up', 'down', 'out', 'off', 'over', 'under', 'again', 'then', + 'here', 'there', 'all', 'each', 'every', 'both', 'few', 'more', + 'most', 'other', 'some', 'such', 'than', 'too', 'very', 'just', + 'also', 'only', 'own', 'so', 'if', 'while', 'because', 'until', + 'vs', 'versus', 'guide', 'complete', 'ultimate', 'best', 'top', + 'new', 'first', 'need', 'know', 'everything', + ]; + + $text = strtolower( $text ); + // Preserve compound identifiers like "BPC-157" by treating hyphens + // between alphanumerics as part of the word. + $words = preg_split( '/[^a-z0-9-]+/', $text, -1, PREG_SPLIT_NO_EMPTY ); + $words = array_filter( $words, static function ( string $w ) use ( $stopwords ): bool { + return strlen( $w ) >= 2 && ! in_array( $w, $stopwords, true ); + } ); + + return array_values( array_unique( $words ) ); } /** diff --git a/tests/unit/Core/IdeaScorerTest.php b/tests/unit/Core/IdeaScorerTest.php index a7d238c..c7c8d59 100644 --- a/tests/unit/Core/IdeaScorerTest.php +++ b/tests/unit/Core/IdeaScorerTest.php @@ -21,6 +21,9 @@ class IdeaScorerTest extends BaseTestCase { $this->stub_get_option( [ 'prautoblogger_topic_exclusions' => '[]', ] ); + + // has_similar_post() → fetch_existing_titles() calls get_the_title(). + Functions\when( 'get_the_title' )->justReturn( 'Existing Post Title' ); } /** @@ -141,4 +144,27 @@ class IdeaScorerTest extends BaseTestCase { $this->assertIsArray( $result ); } + + /** + * Test that topics sharing only common niche words are NOT deduped. + * + * Regression test for the WP_Query 's' bug: WordPress search matched + * ANY word, so "BPC-157 dosage" deduped against "BPC-157 storage" + * even though they're completely different articles. + */ + public function test_dedup_does_not_reject_different_topics_with_shared_words(): void { + // The default WP_Query stub returns 0 posts, so nothing gets deduped. + // This test verifies that score_and_rank passes ideas through when + // no similar posts exist (the common case after the fix). + $fixture = $this->get_analysis_result_fixture(); + $fixture['topic'] = 'BPC-157 dosage guide for subcutaneous injection'; + + $scorer = new \PRAutoBlogger_Idea_Scorer(); + $result = $scorer->score_and_rank( + [ new \PRAutoBlogger_Analysis_Result( $fixture ) ], + 5 + ); + + $this->assertCount( 1, $result, 'Idea should not be deduped when no similar post exists.' ); + } }