From 5918a7d4467b40f0b9cbe610656fdb5d38840de1 Mon Sep 17 00:00:00 2001 From: Terence Ong Date: Mon, 6 Apr 2026 19:32:09 +0700 Subject: [PATCH] Fix missing thumbnails by resolving Google News redirect URLs before OG scraping The OG image scraper was using Google News redirect URLs (source_url) directly, which don't serve og:image tags. Added resolve_scrape_url() to follow redirects (including meta-refresh and JS redirects) to reach the actual publisher page before scraping. Also added a backfill AJAX endpoint and admin button to re-scrape thumbnails for existing articles. Co-Authored-By: Claude Opus 4.6 --- admin/class-peptide-news-admin.php | 28 +++++ includes/class-peptide-news-fetcher.php | 155 +++++++++++++++++++++++- includes/class-peptide-news.php | 1 + 3 files changed, 183 insertions(+), 1 deletion(-) diff --git a/admin/class-peptide-news-admin.php b/admin/class-peptide-news-admin.php index 90b4b65..0a40681 100644 --- a/admin/class-peptide-news-admin.php +++ b/admin/class-peptide-news-admin.php @@ -567,6 +567,14 @@ class Peptide_News_Admin { echo esc_html__( 'Fix Article Sources', 'peptide-news' ); echo ''; echo ''; + + // Backfill Thumbnails button. + echo '
'; + echo ''; + echo ''; + echo '
'; ?> scrape_og_image( $article->source_url ); + // Resolve redirect URLs (e.g. Google News) to the actual article page. + $scrape_url = $this->resolve_scrape_url( $article->source_url ); + $image_url = $this->scrape_og_image( $scrape_url ); if ( ! empty( $image_url ) ) { // Download and store the image locally for reliability. @@ -730,6 +732,157 @@ class Peptide_News_Fetcher { return ! empty( $final_url ) ? $final_url : false; } + /** + * Resolve a URL to a scrapeable destination. + * + * For aggregator redirect URLs (Google News, Yahoo News, etc.), + * follows redirects to reach the actual publisher page where + * og:image tags can be found. + * + * @param string $url The stored article URL. + * @return string The resolved URL suitable for OG scraping. + */ + private function resolve_scrape_url( $url ) { + if ( empty( $url ) ) { + return $url; + } + + $domain = $this->extract_domain( $url ); + $aggregators = array( 'news.google.com', 'news.yahoo.com', 'msn.com' ); + + if ( ! in_array( $domain, $aggregators, true ) ) { + return $url; + } + + // Google News URLs need a full GET request — HEAD often doesn't resolve. + $response = wp_remote_get( $url, array( + 'timeout' => 15, + 'redirection' => 10, + 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'headers' => array( 'Accept' => 'text/html,application/xhtml+xml' ), + 'limit_response_size' => 200000, + ) ); + + if ( is_wp_error( $response ) ) { + return $url; + } + + // Check if WordPress followed redirects and landed on a different URL. + if ( isset( $response['http_response'] ) ) { + $http_response = $response['http_response']; + if ( method_exists( $http_response, 'get_response_object' ) ) { + $raw = $http_response->get_response_object(); + if ( isset( $raw->url ) && $raw->url !== $url ) { + return $raw->url; + } + } + } + + // Check for meta refresh or JavaScript redirects in the HTML body. + $body = wp_remote_retrieve_body( $response ); + if ( ! empty( $body ) ) { + // Meta refresh: + if ( preg_match( '/]+http-equiv=["\']refresh["\'][^>]+content=["\'][^"\']*url=([^"\'>\s]+)/is', $body, $matches ) ) { + $redirect_url = trim( $matches[1] ); + if ( filter_var( $redirect_url, FILTER_VALIDATE_URL ) ) { + return $redirect_url; + } + } + + // JavaScript redirect: window.location = "..." or location.href = "..." + if ( preg_match( '/(?:window\.)?location(?:\.href)?\s*=\s*["\']([^"\']+)["\']/i', $body, $matches ) ) { + $redirect_url = trim( $matches[1] ); + if ( filter_var( $redirect_url, FILTER_VALIDATE_URL ) ) { + return $redirect_url; + } + } + } + + return $url; + } + + /** + * Backfill thumbnails for existing articles that have none. + * + * Re-runs OG scraping with redirect resolution for articles + * missing both thumbnail_url and thumbnail_local. + * + * @return int Number of articles updated. + */ + public function backfill_article_thumbnails() { + global $wpdb; + + $table = $wpdb->prefix . 'peptide_news_articles'; + $articles = $wpdb->get_results( + "SELECT id, source_url, title + FROM {$table} + WHERE is_active = 1 + AND ( thumbnail_url = '' OR thumbnail_url IS NULL ) + AND ( thumbnail_local = '' OR thumbnail_local IS NULL ) + ORDER BY fetched_at DESC + LIMIT 50" + ); + + if ( empty( $articles ) ) { + return 0; + } + + $updated = 0; + + foreach ( $articles as $article ) { + $scrape_url = $this->resolve_scrape_url( $article->source_url ); + $image_url = $this->scrape_og_image( $scrape_url ); + + if ( ! empty( $image_url ) ) { + $local_path = $this->download_thumbnail( $image_url, $article->id, $article->title ); + + if ( $local_path ) { + $wpdb->update( + $table, + array( + 'thumbnail_url' => $image_url, + 'thumbnail_local' => $local_path, + ), + array( 'id' => $article->id ), + array( '%s', '%s' ), + array( '%d' ) + ); + } else { + $wpdb->update( + $table, + array( 'thumbnail_url' => $image_url ), + array( 'id' => $article->id ), + array( '%s' ), + array( '%d' ) + ); + } + $updated++; + } + + usleep( 500000 ); // 0.5s throttle — full GET requests are heavier. + } + + if ( $updated > 0 ) { + $this->clear_article_cache(); + } + + return $updated; + } + + /** + * AJAX handler for the thumbnail backfill action. + */ + public function ajax_backfill_thumbnails() { + check_ajax_referer( 'peptide_news_admin', 'nonce' ); + + if ( ! current_user_can( 'manage_options' ) ) { + wp_send_json_error( 'Unauthorized', 403 ); + } + + $updated = $this->backfill_article_thumbnails(); + wp_send_json_success( array( 'updated' => $updated ) ); + } + /** * Backfill source names for existing articles that show an aggregator domain. * diff --git a/includes/class-peptide-news.php b/includes/class-peptide-news.php index 7dee51b..7b92530 100644 --- a/includes/class-peptide-news.php +++ b/includes/class-peptide-news.php @@ -81,6 +81,7 @@ class Peptide_News { $this->loader->add_action( 'peptide_news_cron_fetch', $fetcher, 'fetch_all_sources' ); $this->loader->add_filter( 'cron_schedules', $fetcher, 'add_custom_cron_schedules' ); $this->loader->add_action( 'wp_ajax_peptide_news_backfill_sources', $fetcher, 'ajax_backfill_sources' ); + $this->loader->add_action( 'wp_ajax_peptide_news_backfill_thumbnails', $fetcher, 'ajax_backfill_thumbnails' ); } /**