' . esc_html__( 'OpenRouter model ID for AI thumbnail generation (e.g., openai/dall-e-3, stabilityai/stable-diffusion-xl). Leave blank to disable AI thumbnails.', 'peptide-news' ) . '
';
+ }
+
public function render_llm_max_articles_field() {
$value = get_option( 'peptide_news_llm_max_articles', 10 );
printf(
@@ -609,8 +620,12 @@ class Peptide_News_Admin {
$btn.prop('disabled', false).text('');
if (response.success) {
var d = response.data;
- var msg = d.updated + ' of ' + d.total + ' thumbnails found.';
- if (d.failed > 0) { msg += ' ' + d.failed + ' could not be resolved.'; }
+ var parts = [];
+ if (d.scraped > 0) { parts.push(d.scraped + ' scraped'); }
+ if (d.generated > 0) { parts.push(d.generated + ' AI-generated'); }
+ if (d.failed > 0) { parts.push(d.failed + ' remaining'); }
+ var msg = (d.scraped + d.generated) + ' of ' + d.total + ' thumbnails resolved';
+ if (parts.length) { msg += ' (' + parts.join(', ') + ')'; }
$result.text(msg);
if (d.samples && d.samples.length) { console.log('Thumbnail backfill:', JSON.stringify(d.samples, null, 2)); }
} else {
diff --git a/admin/partials/articles-list.php b/admin/partials/articles-list.php
index 200dd47..f770cb1 100644
--- a/admin/partials/articles-list.php
+++ b/admin/partials/articles-list.php
@@ -73,8 +73,16 @@ $total_pages = ceil( $total / $per_page );
published_at ) ) ); ?>
fetched_at ) ) ); ?>
- thumbnail_url ) ) : ?>
-
+ thumbnail_url ) && '_no_image' !== $article->thumbnail_url ) {
+ $thumb_src = $article->thumbnail_url;
+ } elseif ( ! empty( $article->thumbnail_local ) ) {
+ $upload_dir = wp_upload_dir();
+ $thumb_src = $upload_dir['baseurl'] . '/' . $article->thumbnail_local;
+ }
+ if ( $thumb_src ) : ?>
+
diff --git a/includes/class-peptide-news-activator.php b/includes/class-peptide-news-activator.php
index 3a82ec9..67da210 100644
--- a/includes/class-peptide-news-activator.php
+++ b/includes/class-peptide-news-activator.php
@@ -36,6 +36,7 @@ class Peptide_News_Activator {
content LONGTEXT NOT NULL,
author VARCHAR(255) NOT NULL DEFAULT '',
thumbnail_url VARCHAR(2048) NOT NULL DEFAULT '',
+ thumbnail_local VARCHAR(512) NOT NULL DEFAULT '',
published_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
categories VARCHAR(500) NOT NULL DEFAULT '',
diff --git a/includes/class-peptide-news-fetcher.php b/includes/class-peptide-news-fetcher.php
index 5bd26d1..c8d6a7a 100644
--- a/includes/class-peptide-news-fetcher.php
+++ b/includes/class-peptide-news-fetcher.php
@@ -328,11 +328,27 @@ class Peptide_News_Fetcher {
return 0;
}
- $updated = 0;
+ $aggregators = array( 'news.google.com', 'news.yahoo.com', 'msn.com' );
+ $updated = 0;
foreach ( $articles as $article ) {
- $scrape_url = $this->resolve_article_for_scraping( $article->source_url, $article->title, $article->source );
- $image_url = $this->scrape_og_image( $scrape_url );
+ $domain = $this->extract_domain( $article->source_url );
+
+ // Skip aggregator URLs — they can't be scraped for OG images.
+ // These will be handled by AI image generation instead.
+ if ( in_array( $domain, $aggregators, true ) ) {
+ $wpdb->update(
+ $table,
+ array( 'thumbnail_url' => '_no_image' ),
+ array( 'id' => $article->id ),
+ array( '%s' ),
+ array( '%d' )
+ );
+ continue;
+ }
+
+ // Direct OG scrape for non-aggregator URLs.
+ $image_url = $this->scrape_og_image( $article->source_url );
if ( ! empty( $image_url ) ) {
$local_path = $this->download_thumbnail( $image_url, $article->id, $article->title );
@@ -345,7 +361,6 @@ class Peptide_News_Fetcher {
$wpdb->update( $table, $update_data, array( 'id' => $article->id ) );
$updated++;
} else {
- // Mark as attempted so we don't retry on every cron cycle.
$wpdb->update(
$table,
array( 'thumbnail_url' => '_no_image' ),
@@ -355,7 +370,6 @@ class Peptide_News_Fetcher {
);
}
- // Throttle: 0.3s for direct scrapes, search adds its own delay.
usleep( 300000 );
}
@@ -366,52 +380,6 @@ class Peptide_News_Fetcher {
return $updated;
}
- /**
- * Resolve an article URL to a scrapeable page.
- *
- * For Google News and other aggregators, tries redirect resolution first,
- * then falls back to search-based lookup using article title and publisher.
- *
- * @param string $source_url The stored article URL.
- * @param string $title The article title.
- * @param string $source_name The publisher name.
- * @return string URL suitable for OG image scraping.
- */
- private function resolve_article_for_scraping( $source_url, $title, $source_name ) {
- if ( empty( $source_url ) ) {
- return $source_url;
- }
-
- $domain = $this->extract_domain( $source_url );
-
- // Non-aggregator URLs can be scraped directly.
- $aggregators = array( 'news.google.com', 'news.yahoo.com', 'msn.com' );
- if ( ! in_array( $domain, $aggregators, true ) ) {
- return $source_url;
- }
-
- // Try HTTP redirect resolution first (works for Yahoo, MSN).
- $resolved = $this->resolve_redirect_url( $source_url );
- if ( $resolved && $resolved !== $source_url ) {
- $resolved_domain = $this->extract_domain( $resolved );
- if ( ! in_array( $resolved_domain, $aggregators, true ) ) {
- return $resolved;
- }
- }
-
- // For Google News: search for the actual article URL.
- if ( 'news.google.com' === $domain ) {
- $search_result = $this->find_article_url_via_search( $title, $source_name );
- if ( $search_result ) {
- // Throttle between search requests.
- usleep( 1500000 ); // 1.5s
- return $search_result;
- }
- }
-
- return $source_url;
- }
-
/**
* Scrape Open Graph / meta image tags from a URL.
*
@@ -769,143 +737,21 @@ class Peptide_News_Fetcher {
}
- /**
- * Find the actual article URL via DuckDuckGo search.
- *
- * Google News RSS stores encrypted article URLs that cannot be resolved
- * server-side. This method uses the article title and publisher name
- * to find the real article page via search engine lookup.
- *
- * @param string $title The article title (may include " - Source" suffix).
- * @param string $source_name The publisher name (e.g., "STAT News").
- * @return string|false The actual article URL, or false on failure.
- */
- private function find_article_url_via_search( $title, $source_name = '' ) {
- // Strip " - Source" suffix from title if present.
- $clean_title = preg_replace( '/\s[-\x{2013}\x{2014}]\s[^-\x{2013}\x{2014}]+$/u', '', $title );
- $clean_title = trim( $clean_title );
-
- if ( empty( $clean_title ) || mb_strlen( $clean_title ) < 10 ) {
- return false;
- }
-
- // Build search query with quoted title for exact match + source name.
- $query = '"' . $clean_title . '"';
- if ( ! empty( $source_name ) ) {
- $query .= ' ' . $source_name;
- }
-
- $search_url = 'https://html.duckduckgo.com/html/?q=' . urlencode( $query );
-
- $response = wp_remote_get( $search_url, array(
- 'timeout' => 12,
- 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
- 'headers' => array(
- 'Accept' => 'text/html,application/xhtml+xml',
- 'Accept-Language' => 'en-US,en;q=0.9',
- ),
- ) );
-
- if ( is_wp_error( $response ) ) {
- $this->log_error( 'Search lookup failed: ' . $response->get_error_message() );
- return false;
- }
-
- $status = wp_remote_retrieve_response_code( $response );
- if ( $status < 200 || $status >= 400 ) {
- return false;
- }
-
- $body = wp_remote_retrieve_body( $response );
- if ( empty( $body ) ) {
- return false;
- }
-
- // Domains to skip in search results.
- $skip_domains = array(
- 'news.google.com', 'google.com', 'duckduckgo.com', 'bing.com',
- 'yahoo.com', 'msn.com', 'youtube.com', 'facebook.com', 'twitter.com',
- );
-
- // DuckDuckGo HTML results: links with class "result__a" or redirect URLs with uddg param.
- if ( preg_match_all( '/class=["\']result__a["\'][^>]*href=["\']([^"\']+)["\']/i', $body, $matches ) ) {
- foreach ( $matches[1] as $href ) {
- $actual_url = $this->extract_ddg_url( $href );
- if ( ! $actual_url ) {
- continue;
- }
-
- $domain = $this->extract_domain( $actual_url );
- if ( in_array( $domain, $skip_domains, true ) ) {
- continue;
- }
-
- return esc_url_raw( $actual_url );
- }
- }
-
- // Fallback: look for uddg-encoded redirect links anywhere.
- if ( preg_match_all( '/uddg=([^&"\'>\s]+)/i', $body, $matches ) ) {
- foreach ( $matches[1] as $encoded_url ) {
- $actual_url = urldecode( $encoded_url );
- if ( ! filter_var( $actual_url, FILTER_VALIDATE_URL ) ) {
- continue;
- }
- $domain = $this->extract_domain( $actual_url );
- if ( ! in_array( $domain, $skip_domains, true ) ) {
- return esc_url_raw( $actual_url );
- }
- }
- }
-
- return false;
- }
-
- /**
- * Extract the actual URL from a DuckDuckGo redirect link.
- *
- * DuckDuckGo HTML results use redirect URLs with the actual URL
- * encoded in the 'uddg' query parameter.
- *
- * @param string $href The DuckDuckGo result href.
- * @return string|false The actual URL or false.
- */
- private function extract_ddg_url( $href ) {
- $href = html_entity_decode( $href );
-
- // Direct URL (no redirect wrapper).
- if ( strpos( $href, 'duckduckgo.com' ) === false && filter_var( $href, FILTER_VALIDATE_URL ) ) {
- return $href;
- }
-
- // Extract from redirect: //duckduckgo.com/l/?uddg=&...
- if ( strpos( $href, 'uddg=' ) !== false ) {
- $query_string = wp_parse_url( $href, PHP_URL_QUERY );
- if ( $query_string ) {
- parse_str( $query_string, $params );
- if ( ! empty( $params['uddg'] ) && filter_var( $params['uddg'], FILTER_VALIDATE_URL ) ) {
- return $params['uddg'];
- }
- }
- }
-
- return false;
- }
-
/**
* Backfill thumbnails for existing articles that have none.
*
- * Re-runs OG scraping with search-based URL resolution for articles
- * missing thumbnails. Also retries articles previously marked '_no_image'.
+ * Attempts OG scraping for non-aggregator URLs, then uses AI image
+ * generation for any remaining articles without thumbnails.
*
- * @return array Summary with 'updated' count and 'samples' for diagnostics.
+ * @return array Summary with counts and diagnostic samples.
*/
public function backfill_article_thumbnails() {
global $wpdb;
- $table = $wpdb->prefix . 'peptide_news_articles';
+ $table = $wpdb->prefix . 'peptide_news_articles';
+ $aggregators = array( 'news.google.com', 'news.yahoo.com', 'msn.com' );
- // Reset '_no_image' sentinel on previously failed articles so we retry them.
+ // Reset sentinels so we retry everything.
$wpdb->query(
"UPDATE {$table}
SET thumbnail_url = ''
@@ -914,70 +760,83 @@ class Peptide_News_Fetcher {
);
$articles = $wpdb->get_results(
- "SELECT id, source, source_url, title
+ "SELECT id, source, source_url, title, excerpt, ai_summary, tags, categories
FROM {$table}
WHERE is_active = 1
AND ( thumbnail_url = '' OR thumbnail_url IS NULL )
+ AND ( thumbnail_local = '' OR thumbnail_local IS NULL )
ORDER BY fetched_at DESC
LIMIT 20"
);
$result = array(
- 'total' => count( $articles ),
- 'updated' => 0,
- 'failed' => 0,
- 'samples' => array(),
+ 'total' => count( $articles ),
+ 'scraped' => 0,
+ 'generated' => 0,
+ 'failed' => 0,
+ 'samples' => array(),
);
if ( empty( $articles ) ) {
return $result;
}
+ // Phase 1: OG scrape non-aggregator URLs.
foreach ( $articles as $article ) {
- $sample = array(
- 'id' => $article->id,
- 'title' => mb_substr( $article->title, 0, 60 ),
- );
-
- $scrape_url = $this->resolve_article_for_scraping( $article->source_url, $article->title, $article->source );
- $sample['resolved'] = ( $scrape_url !== $article->source_url );
- $sample['scrape_url'] = mb_substr( $scrape_url, 0, 100 );
-
- $image_url = $this->scrape_og_image( $scrape_url );
+ $domain = $this->extract_domain( $article->source_url );
+ if ( in_array( $domain, $aggregators, true ) ) {
+ continue; // Skip — will be handled by AI generation.
+ }
+ $image_url = $this->scrape_og_image( $article->source_url );
if ( ! empty( $image_url ) ) {
- $local_path = $this->download_thumbnail( $image_url, $article->id, $article->title );
-
+ $local_path = $this->download_thumbnail( $image_url, $article->id, $article->title );
$update_data = array( 'thumbnail_url' => $image_url );
if ( $local_path ) {
$update_data['thumbnail_local'] = $local_path;
}
-
$wpdb->update( $table, $update_data, array( 'id' => $article->id ) );
- $result['updated']++;
- $sample['status'] = 'ok';
- $sample['image'] = mb_substr( $image_url, 0, 80 );
- } else {
- $wpdb->update(
- $table,
- array( 'thumbnail_url' => '_no_image' ),
- array( 'id' => $article->id ),
- array( '%s' ),
- array( '%d' )
- );
- $result['failed']++;
- $sample['status'] = 'no_image';
+ $result['scraped']++;
+
+ if ( count( $result['samples'] ) < 5 ) {
+ $result['samples'][] = array(
+ 'id' => $article->id,
+ 'title' => mb_substr( $article->title, 0, 60 ),
+ 'method' => 'og_scrape',
+ 'status' => 'ok',
+ );
+ }
}
- // Only include first 5 samples in diagnostics.
- if ( count( $result['samples'] ) < 5 ) {
- $result['samples'][] = $sample;
- }
-
- usleep( 500000 );
+ usleep( 300000 );
}
- if ( $result['updated'] > 0 ) {
+ // Phase 2: AI generation for articles still missing thumbnails.
+ if ( class_exists( 'Peptide_News_LLM' ) && Peptide_News_LLM::is_enabled() ) {
+ $ai_generated = Peptide_News_LLM::generate_missing_thumbnails( 20 );
+ $result['generated'] = $ai_generated;
+ }
+
+ // Count remaining failures.
+ $still_missing = $wpdb->get_var(
+ "SELECT COUNT(*)
+ FROM {$table}
+ WHERE is_active = 1
+ AND ( thumbnail_url = '' OR thumbnail_url IS NULL )
+ AND ( thumbnail_local = '' OR thumbnail_local IS NULL )"
+ );
+ $result['failed'] = (int) $still_missing;
+
+ // Mark remaining as '_no_image' so cron doesn't retry every cycle.
+ $wpdb->query(
+ "UPDATE {$table}
+ SET thumbnail_url = '_no_image'
+ WHERE is_active = 1
+ AND ( thumbnail_url = '' OR thumbnail_url IS NULL )
+ AND ( thumbnail_local = '' OR thumbnail_local IS NULL )"
+ );
+
+ if ( $result['scraped'] > 0 || $result['generated'] > 0 ) {
$this->clear_article_cache();
}
diff --git a/includes/class-peptide-news-llm.php b/includes/class-peptide-news-llm.php
index 89bfba7..3d1e4ff 100644
--- a/includes/class-peptide-news-llm.php
+++ b/includes/class-peptide-news-llm.php
@@ -390,12 +390,12 @@ class Peptide_News_LLM {
global $wpdb;
$table = $wpdb->prefix . 'peptide_news_articles';
- // Find active articles with no thumbnail (neither URL nor local).
+ // Find active articles with no usable thumbnail (including _no_image sentinel).
$articles = $wpdb->get_results( $wpdb->prepare(
"SELECT id, title, excerpt, ai_summary, tags, categories
FROM {$table}
WHERE is_active = 1
- AND ( thumbnail_url = '' OR thumbnail_url IS NULL )
+ AND ( thumbnail_url = '' OR thumbnail_url IS NULL OR thumbnail_url = '_no_image' )
AND ( thumbnail_local = '' OR thumbnail_local IS NULL )
ORDER BY fetched_at DESC
LIMIT %d",
diff --git a/includes/class-peptide-news-rest-api.php b/includes/class-peptide-news-rest-api.php
index e0c468e..cc895f9 100644
--- a/includes/class-peptide-news-rest-api.php
+++ b/includes/class-peptide-news-rest-api.php
@@ -134,7 +134,7 @@ class Peptide_News_Rest_API {
$articles = $wpdb->get_results( $wpdb->prepare(
"SELECT id, source, source_url, title, excerpt, author, thumbnail_url,
- published_at, categories, tags
+ thumbnail_local, published_at, categories, tags
FROM {$table}
WHERE is_active = 1
ORDER BY published_at DESC
diff --git a/public/class-peptide-news-public.php b/public/class-peptide-news-public.php
index 1783fef..caec701 100644
--- a/public/class-peptide-news-public.php
+++ b/public/class-peptide-news-public.php
@@ -95,9 +95,16 @@ class Peptide_News_Public {
?>