peptide-repo/peptide-news-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: fetch article page content when RSS provides only a title

Browse Source 
Google News RSS items often have no description or content — just a
title. This left the LLM with insufficient context, resulting in empty
ai_summary fields.

When the stored content is sparse (<100 chars or matches the title),
build_summary_prompt() now fetches the actual article page via
wp_remote_get(), extracts text from <article>/<main> tags, and feeds
it to the LLM. Gracefully falls back to title-only if the fetch fails.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Terence Ong
2026-04-11 21:01:10 +07:00
parent 221a46a60a
commit 71f7a35eb3
1 changed files with 82 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
includes/class-peptide-news-llm.php
View File

@@ -239,16 +239,37 @@ class Peptide_News_LLM {
/**
* Build the prompt for article summarization.
*
* When the article's stored content is sparse (empty or just the title),
* attempts to fetch the actual page content from the source URL.
*
* @param object $article
* @return string
*/
private static function build_summary_prompt( $article ) {
$text = $article->title;
if ( ! empty( $article->excerpt ) ) {
$text .= "\n\n" . $article->excerpt;
$title = trim( $article->title ?? '' );
$excerpt = trim( $article->excerpt ?? '' );
$content = trim( $article->content ?? '' );
$content_raw = wp_strip_all_tags( $content );
// Determine if we have meaningful content beyond just the title.
$has_real_excerpt = ! empty( $excerpt ) && $excerpt !== $title;
$has_real_content = mb_strlen( $content_raw ) > 100;
// If content is sparse, try fetching the article's web page.
if ( ! $has_real_content && ! empty( $article->source_url ) ) {
$fetched = self::fetch_article_text( $article->source_url );
if ( ! empty( $fetched ) ) {
$content_raw = $fetched;
$has_real_content = true;
}
}
if ( ! empty( $article->content ) ) {
$text .= "\n\n" . mb_substr( wp_strip_all_tags( $article->content ), 0, 3000 );
$text = $title;
if ( $has_real_excerpt ) {
$text .= "\n\n" . $excerpt;
}
if ( $has_real_content ) {
$text .= "\n\n" . mb_substr( $content_raw, 0, 3000 );
}
return "Summarize this peptide research article in 3-4 sentences. "
@@ -257,6 +278,62 @@ class Peptide_News_LLM {
. "Article:\n" . $text;
}
/**
* Fetch article text from a URL for summarization when RSS content is sparse.
*
* Extracts the main body text from the page, stripping navigation,
* scripts, styles, and other non-content elements.
*
* @param string $url The article URL.
* @return string Cleaned plain text or empty string on failure.
*/
private static function fetch_article_text( $url ) {
$response = wp_remote_get( $url, array(
'timeout' => 10,
'user-agent' => 'Mozilla/5.0 (compatible; PeptideNewsBot/1.0; +https://peptiderepo.com)',
'headers' => array( 'Accept' => 'text/html' ),
) );
if ( is_wp_error( $response ) || 200 !== wp_remote_retrieve_response_code( $response ) ) {
return '';
}
$html = wp_remote_retrieve_body( $response );
if ( empty( $html ) ) {
return '';
}
// Strip elements that don't carry article content.
$html = preg_replace( '/<script[^>]*>.*?<\/script>/is', '', $html );
$html = preg_replace( '/<style[^>]*>.*?<\/style>/is', '', $html );
$html = preg_replace( '/<nav[^>]*>.*?<\/nav>/is', '', $html );
$html = preg_replace( '/<header[^>]*>.*?<\/header>/is', '', $html );
$html = preg_replace( '/<footer[^>]*>.*?<\/footer>/is', '', $html );
$html = preg_replace( '/<aside[^>]*>.*?<\/aside>/is', '', $html );
$html = preg_replace( '/<!--.*?-->/s', '', $html );
// Try to extract content from <article> or <main> tags first.
$text = '';
if ( preg_match( '/<article[^>]*>(.*?)<\/article>/is', $html, $matches ) ) {
$text = wp_strip_all_tags( $matches[1] );
} elseif ( preg_match( '/<main[^>]*>(.*?)<\/main>/is', $html, $matches ) ) {
$text = wp_strip_all_tags( $matches[1] );
} else {
$text = wp_strip_all_tags( $html );
}
// Collapse whitespace.
$text = preg_replace( '/\s+/', ' ', $text );
$text = trim( $text );
// Only return if we got meaningful content.
if ( mb_strlen( $text ) < 100 ) {
return '';
}
return $text;
}
/**
* Call the OpenRouter API with retry logic for 429 rate limits.
*