peptide-repo/prautoblogger
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: remove PullPush.io, use Reddit RSS as primary data source

Browse Source 
PullPush.io index was frequently stale/unavailable. Reddit RSS/Atom feeds
work reliably from datacenter IPs (Hostinger) where .json gets 403.

- RSS is now the primary source for post fetching
- .json endpoints kept as fallback for posts and sole source for comments
- Deleted class-pull-push-client.php
- Updated admin UI, test connection handler, docs, and ARCHITECTURE.md

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
peptiderepo
2026-04-12 14:00:55 +07:00
parent b5cbbef804
commit d9985f6dc2
10 changed files with 263 additions and 800 deletions
Download Patch File Download Diff File Expand all files Collapse all files

382
includes/providers/class-pull-push-client.php
View File

@@ -1,382 +0,0 @@
<?php
declare(strict_types=1);
/**
* PullPush.io API HTTP client for fetching Reddit data without OAuth.
*
* PullPush.io is a free, no-auth-required API that mirrors Reddit's submission
* and comment data. We use it as the PRIMARY data source because Reddit rejected
* our OAuth application. Rate limit: 30 requests/minute, 1000/hour.
*
* Triggered by: PRAutoBlogger_Reddit_Provider calls fetch methods here.
* Dependencies: wp_remote_get(), PRAutoBlogger_Logger.
*
* @see providers/class-reddit-provider.php — Instantiates this class.
* @see providers/class-reddit-json-client.php — Fallback client if PullPush is down.
* @see ARCHITECTURE.md — External API integrations table.
*/
class PRAutoBlogger_PullPush_Client {
/**
* PullPush API base URL.
*
* @var string
*/
private const API_BASE = 'https://api.pullpush.io/reddit';
/**
* User agent sent with every request so PullPush can identify us.
*
* @var string
*/
private const USER_AGENT = 'PRAutoBlogger/0.1.0 (WordPress Plugin; +https://peptiderepo.com)';
/**
* HTTP timeout in seconds. PullPush can be slow under load.
*
* @var int
*/
private const TIMEOUT_SECONDS = 30;
/**
* Maximum results PullPush returns per request.
*
* @var int
*/
private const MAX_SIZE = 100;
/**
* Transient key for tracking rate limit state.
*
* @var string
*/
private const RATE_LIMIT_TRANSIENT = 'prautoblogger_pullpush_rate_limit';
/**
* Minimum seconds between requests to stay under 30 req/min.
*
* @var float
*/
private const MIN_REQUEST_INTERVAL = 2.0;
/**
* Fetch recent submissions (posts) from a subreddit, sorted by score.
*
* PullPush returns submissions as a flat array (not nested like Reddit's
* data.children[].data structure). Each item has Reddit-standard field names:
* id, title, selftext, author, score, num_comments, permalink, created_utc, etc.
*
* Side effects: HTTP request to PullPush API, updates rate limit transient.
*
* @param string $subreddit Subreddit name (without r/ prefix).
* @param int $limit Max posts to fetch (capped at 100 by API).
* @param string $time_filter Time window: 'hour', 'day', 'week', 'month', 'year', 'all'.
*
* @return array<int, array<string, mixed>> Array of submission data arrays.
*/
public function fetch_posts( string $subreddit, int $limit, string $time_filter = 'day' ): array {
$size = min( $limit, self::MAX_SIZE );
$after = $this->time_filter_to_epoch( $time_filter );
$url = sprintf(
'%s/search/submission?subreddit=%s&sort=score&sort_type=desc&size=%d&after=%d',
self::API_BASE,
rawurlencode( $subreddit ),
$size,
$after
);
$response = $this->api_get( $url );
if ( null === $response ) {
return [];
}
// PullPush returns { "data": [...] } or just [...] depending on endpoint version.
$items = $response['data'] ?? $response;
if ( ! is_array( $items ) ) {
return [];
}
return $items;
}
/**
* Fetch recent comments from a subreddit, sorted by score.
*
* Unlike the Reddit API which fetches comments per-post, PullPush lets us
* search comments across an entire subreddit. This is more efficient for
* finding high-quality discussion content.
*
* Side effects: HTTP request to PullPush API, updates rate limit transient.
*
* @param string $subreddit Subreddit name (without r/ prefix).
* @param int $limit Max comments to fetch (capped at 100 by API).
* @param string $time_filter Time window for filtering.
*
* @return array<int, array<string, mixed>> Array of comment data arrays.
*/
public function fetch_comments( string $subreddit, int $limit, string $time_filter = 'day' ): array {
$size = min( $limit, self::MAX_SIZE );
$after = $this->time_filter_to_epoch( $time_filter );
$url = sprintf(
'%s/search/comment?subreddit=%s&sort=score&sort_type=desc&size=%d&after=%d',
self::API_BASE,
rawurlencode( $subreddit ),
$size,
$after
);
$response = $this->api_get( $url );
if ( null === $response ) {
return [];
}
$items = $response['data'] ?? $response;
if ( ! is_array( $items ) ) {
return [];
}
return $items;
}
/**
* Fetch comments for a specific post by its Reddit ID.
*
* Uses the link_id filter (t3_ prefixed) to get comments for one post.
*
* Side effects: HTTP request to PullPush API.
*
* @param string $post_id Reddit post ID (without t3_ prefix).
* @param int $limit Max comments to fetch.
*
* @return array<int, array<string, mixed>> Array of comment data arrays.
*/
public function fetch_comments_for_post( string $post_id, int $limit = 5 ): array {
$size = min( $limit, self::MAX_SIZE );
$url = sprintf(
'%s/search/comment?link_id=t3_%s&sort=score&sort_type=desc&size=%d',
self::API_BASE,
rawurlencode( $post_id ),
$size
);
$response = $this->api_get( $url );
if ( null === $response ) {
return [];
}
$items = $response['data'] ?? $response;
return is_array( $items ) ? $items : [];
}
/**
* Check if PullPush API is reachable AND has reasonably fresh data.
*
* PullPush can return HTTP 200 but have a stale index (months behind Reddit).
* We check that the most recent post is no older than 7 days. If the index
* is stale, we return false so the caller falls back to Reddit .json.
*
* Side effects: HTTP request.
*
* @return bool True if API responded with HTTP 200 and has data < 7 days old.
*/
public function is_available(): bool {
// Fetch 1 recent submission sorted by date to check freshness.
$url = self::API_BASE . '/search/submission?subreddit=all&size=1&sort_type=created_utc&sort=desc';
$response = wp_remote_get(
$url,
[
'timeout' => 10,
'headers' => [
'User-Agent' => self::USER_AGENT,
],
]
);
if ( is_wp_error( $response ) ) {
return false;
}
if ( 200 !== wp_remote_retrieve_response_code( $response ) ) {
return false;
}
$body = json_decode( wp_remote_retrieve_body( $response ), true );
$items = $body['data'] ?? [];
if ( empty( $items ) ) {
return false;
}
// Check that the most recent post is within the last 7 days.
// If PullPush's index is stale (e.g. months behind), we should
// fall back to Reddit .json which always has live data.
$latest_utc = (int) ( $items[0]['created_utc'] ?? 0 );
$seven_days = 7 * DAY_IN_SECONDS;
$is_fresh = ( time() - $latest_utc ) < $seven_days;
if ( ! $is_fresh ) {
PRAutoBlogger_Logger::instance()->warning(
sprintf(
'PullPush index is stale — latest post is from %s (%d days old). Falling back to Reddit .json.',
gmdate( 'Y-m-d', $latest_utc ),
(int) ( ( time() - $latest_utc ) / DAY_IN_SECONDS )
),
'pullpush'
);
}
return $is_fresh;
}
/**
* Get current rate limit status.
*
* PullPush doesn't return rate limit headers, so we track our own
* request count using a transient with a 60-second TTL.
*
* @return array{remaining: int, limit: int, resets_at: string}
*/
public function get_rate_limit_status(): array {
$status = get_transient( self::RATE_LIMIT_TRANSIENT );
if ( is_array( $status ) ) {
return $status;
}
// No tracking data yet — assume fresh window.
return [
'remaining' => 30,
'limit' => 30,
'resets_at' => '',
];
}
/**
* Make an HTTP GET request to the PullPush API.
*
* Handles error logging, response parsing, and self-imposed rate limit tracking.
* No authentication required — PullPush is a free public API.
*
* Side effects: HTTP request, sets rate limit transient, logs errors.
*
* @param string $url Full API URL with query parameters.
*
* @return array<string, mixed>|null Decoded JSON response, or null on failure.
*/
private function api_get( string $url ): ?array {
// Self-imposed rate limiting: track requests per minute.
$this->track_rate_limit();
$response = wp_remote_get(
$url,
[
'timeout' => self::TIMEOUT_SECONDS,
'headers' => [
'User-Agent' => self::USER_AGENT,
'Accept' => 'application/json',
],
]
);
if ( is_wp_error( $response ) ) {
PRAutoBlogger_Logger::instance()->error(
'PullPush API GET failed: ' . $response->get_error_message(),
'pullpush'
);
return null;
}
$status_code = wp_remote_retrieve_response_code( $response );
if ( 200 !== $status_code ) {
PRAutoBlogger_Logger::instance()->error(
sprintf(
'PullPush API HTTP %d: %s',
$status_code,
substr( wp_remote_retrieve_body( $response ), 0, 300 )
),
'pullpush'
);
return null;
}
$body = wp_remote_retrieve_body( $response );
$data = json_decode( $body, true );
if ( ! is_array( $data ) ) {
PRAutoBlogger_Logger::instance()->error(
'PullPush API returned non-JSON response.',
'pullpush'
);
return null;
}
return $data;
}
/**
* Track our own request count to avoid exceeding PullPush's 30 req/min limit.
*
* Uses a transient that stores the request count and window start time.
* If we're close to the limit, we don't block — the caller should check
* get_rate_limit_status() before making bulk requests.
*
* Side effects: Sets transient.
*/
private function track_rate_limit(): void {
$status = get_transient( self::RATE_LIMIT_TRANSIENT );
if ( ! is_array( $status ) ) {
$status = [
'remaining' => 29, // This request uses one.
'limit' => 30,
'resets_at' => gmdate( 'Y-m-d H:i:s', time() + 60 ),
];
} else {
$status['remaining'] = max( 0, (int) $status['remaining'] - 1 );
}
// TTL of 60 seconds — the window auto-resets.
set_transient( self::RATE_LIMIT_TRANSIENT, $status, 60 );
}
/**
* Convert a human-readable time filter to a Unix epoch timestamp.
*
* PullPush uses `after` parameter as a Unix timestamp, not Reddit's
* `t=day` style. This maps between the two conventions.
*
* @param string $time_filter One of: 'hour', 'day', 'week', 'month', 'year', 'all'.
*
* @return int Unix timestamp to use as the `after` parameter.
*/
private function time_filter_to_epoch( string $time_filter ): int {
$now = time();
switch ( $time_filter ) {
case 'hour':
return $now - 3600;
case 'day':
return $now - 86400;
case 'week':
return $now - 604800;
case 'month':
return $now - 2592000;
case 'year':
return $now - 31536000;
case 'all':
return 0;
default:
return $now - 86400; // Default to 1 day.
}
}
}

233
includes/providers/class-reddit-json-client.php
View File

@@ -2,24 +2,23 @@
declare(strict_types=1);
/**
* Reddit .json endpoint client — fallback when PullPush.io is unavailable.
* Reddit HTTP client — fetches posts via RSS (primary) and .json (fallback).
*
* Appends `.json` to standard Reddit URLs to get JSON responses without OAuth.
* No authentication required, but heavily rate-limited (~10 req/min for
* unauthenticated requests) and datacenter IPs may be blocked aggressively.
* This is a FALLBACK only — PullPush is the primary data source.
* RSS feeds are the primary data source because they work reliably from datacenter
* IPs (Hostinger), while .json endpoints return 403 for server IPs. The .json
* endpoints are kept as a fallback in case RSS ever breaks, and are always used
* for comment fetching (comments are not available in RSS feeds).
*
* Triggered by: PRAutoBlogger_Reddit_Provider when PullPush is_available() returns false.
* Triggered by: PRAutoBlogger_Reddit_Provider::collect_data().
* Dependencies: wp_remote_get(), PRAutoBlogger_Logger.
*
* @see providers/class-pullpush-client.php — Primary client (preferred).
* @see providers/class-reddit-provider.php — Instantiates this class as fallback.
* @see ARCHITECTURE.md — External API integrations table.
* @see providers/class-reddit-provider.php — Instantiates and calls this class.
* @see ARCHITECTURE.md — External API integrations table.
*/
class PRAutoBlogger_Reddit_JSON_Client {
/**
* Reddit base URL for .json endpoints.
* Reddit base URL for all endpoints.
*
* @var string
*/
@@ -31,7 +30,7 @@ class PRAutoBlogger_Reddit_JSON_Client {
*
* @var string
*/
private const USER_AGENT = 'Mozilla/5.0 (compatible; PRAutoBlogger/0.1.0; +https://peptiderepo.com)';
private const USER_AGENT = 'Mozilla/5.0 (compatible; PRAutoBlogger/0.2.1; +https://peptiderepo.com)';
/**
* HTTP timeout in seconds.
@@ -41,63 +40,50 @@ class PRAutoBlogger_Reddit_JSON_Client {
private const TIMEOUT_SECONDS = 30;
/**
* Transient key for rate limit tracking.
* Transient key for rate limit tracking (.json endpoints only).
*
* @var string
*/
private const RATE_LIMIT_TRANSIENT = 'prautoblogger_reddit_json_rate_limit';
/**
* Fetch hot posts from a subreddit via Reddit's .json endpoint.
* Fetch hot posts from a subreddit. Tries RSS first, .json fallback.
*
* Returns data in the same format as the old OAuth client's fetch_posts():
* an array of post data arrays with standard Reddit field names.
* RSS is the primary source because it works from datacenter IPs where
* .json returns 403. If RSS fails, we fall back to .json. Both return
* data in the same normalized format.
*
* Side effects: HTTP request, updates rate limit transient.
* Side effects: HTTP request to Reddit.
*
* @param string $subreddit Subreddit name (without r/ prefix).
* @param int $limit Max posts to fetch (Reddit caps at 100).
* @param string $time_filter Time filter for /top (hour, day, week, month, year, all).
* @param string $time_filter Time filter (unused by RSS, used by .json).
*
* @return array<int, array<string, mixed>> Array of post data.
*/
public function fetch_posts( string $subreddit, int $limit, string $time_filter = 'day' ): array {
$url = sprintf(
'%s/r/%s/hot.json?limit=%d&t=%s&raw_json=1',
self::BASE_URL,
rawurlencode( $subreddit ),
min( $limit, 100 ),
rawurlencode( $time_filter )
// Try RSS first — most reliable from datacenter IPs.
$posts = $this->fetch_posts_via_rss( $subreddit, $limit );
if ( ! empty( $posts ) ) {
return $posts;
}
// RSS failed — fall back to .json endpoints.
PRAutoBlogger_Logger::instance()->info(
sprintf( 'Reddit RSS returned 0 posts for r/%s — trying .json fallback.', $subreddit ),
'reddit'
);
$response = $this->api_get( $url );
if ( null === $response || ! isset( $response['data']['children'] ) ) {
// .json endpoint failed (likely 403 IP block) — try RSS fallback.
PRAutoBlogger_Logger::instance()->info(
sprintf( 'Reddit .json failed for r/%s — trying RSS fallback.', $subreddit ),
'reddit_json'
);
return $this->fetch_posts_via_rss( $subreddit, $limit );
}
$posts = [];
foreach ( $response['data']['children'] as $child ) {
if ( isset( $child['data'] ) ) {
$posts[] = $child['data'];
}
}
return $posts;
return $this->fetch_posts_via_json( $subreddit, $limit, $time_filter );
}
/**
* Fetch posts from a subreddit via Reddit's RSS/Atom feed.
*
* RSS endpoints are less commonly blocked by Reddit than .json.
* Returns data in the same format as fetch_posts() for compatibility,
* but some fields (score, num_comments) are unavailable in RSS and
* set to defaults.
* RSS endpoints work reliably from datacenter IPs where .json gets blocked.
* Returns data normalized to the same format as .json for pipeline compatibility.
* Some fields (score, num_comments) are unavailable in RSS and set to defaults.
*
* Side effects: HTTP request to Reddit.
*
@@ -106,7 +92,7 @@ class PRAutoBlogger_Reddit_JSON_Client {
*
* @return array<int, array<string, mixed>> Array of post data.
*/
private function fetch_posts_via_rss( string $subreddit, int $limit ): array {
public function fetch_posts_via_rss( string $subreddit, int $limit ): array {
$url = sprintf(
'%s/r/%s/hot.rss?limit=%d',
self::BASE_URL,
@@ -128,7 +114,7 @@ class PRAutoBlogger_Reddit_JSON_Client {
if ( is_wp_error( $response ) ) {
PRAutoBlogger_Logger::instance()->error(
'Reddit RSS GET failed: ' . $response->get_error_message(),
'reddit_json'
'reddit'
);
return [];
}
@@ -137,7 +123,7 @@ class PRAutoBlogger_Reddit_JSON_Client {
if ( 200 !== $status_code ) {
PRAutoBlogger_Logger::instance()->error(
sprintf( 'Reddit RSS HTTP %d for r/%s', $status_code, $subreddit ),
'reddit_json'
'reddit'
);
return [];
}
@@ -146,6 +132,47 @@ class PRAutoBlogger_Reddit_JSON_Client {
return $this->parse_atom_feed( $body, $subreddit );
}
/**
* Fetch hot posts from a subreddit via Reddit's .json endpoint (fallback).
*
* Datacenter IPs are often blocked by Reddit for .json endpoints (HTTP 403).
* This method is only used as a fallback when RSS fails.
*
* Side effects: HTTP request, updates rate limit transient.
*
* @param string $subreddit Subreddit name (without r/ prefix).
* @param int $limit Max posts to fetch (Reddit caps at 100).
* @param string $time_filter Time filter for /hot (hour, day, week, month, year, all).
*
* @return array<int, array<string, mixed>> Array of post data.
*/
public function fetch_posts_via_json( string $subreddit, int $limit, string $time_filter = 'day' ): array {
$url = sprintf(
'%s/r/%s/hot.json?limit=%d&t=%s&raw_json=1',
self::BASE_URL,
rawurlencode( $subreddit ),
min( $limit, 100 ),
rawurlencode( $time_filter )
);
$response = $this->api_get( $url );
if ( null === $response || ! isset( $response['data']['children'] ) ) {
return [];
}
$posts = [];
foreach ( $response['data']['children'] as $child ) {
if ( isset( $child['data'] ) ) {
$post = $child['data'];
$post['data_source'] = 'reddit_json';
$posts[] = $post;
}
}
return $posts;
}
/**
* Parse a Reddit Atom feed into post-data arrays matching .json format.
*
@@ -165,7 +192,7 @@ class PRAutoBlogger_Reddit_JSON_Client {
libxml_use_internal_errors( $prev_errors );
if ( false === $feed ) {
PRAutoBlogger_Logger::instance()->error( 'Failed to parse Reddit RSS XML.', 'reddit_json' );
PRAutoBlogger_Logger::instance()->error( 'Failed to parse Reddit RSS XML.', 'reddit' );
return [];
}
@@ -193,11 +220,10 @@ class PRAutoBlogger_Reddit_JSON_Client {
// Content is in <content> tag as HTML.
if ( isset( $entry->content ) ) {
// Strip HTML tags to get plain text for content field.
$content = wp_strip_all_tags( (string) $entry->content );
}
// Extract post ID from the entry id (format: /r/subreddit/comments/ID/...)
// Extract post ID from the entry id (format: /r/subreddit/comments/ID/...).
$entry_id = (string) $entry->id;
$post_id = '';
if ( preg_match( '#/comments/([a-z0-9]+)#', $entry_id, $m ) ) {
@@ -217,30 +243,30 @@ class PRAutoBlogger_Reddit_JSON_Client {
// Extract permalink (relative path) from full URL.
$permalink = '';
if ( '' !== $link ) {
$parsed = wp_parse_url( $link );
$parsed = wp_parse_url( $link );
$permalink = $parsed['path'] ?? '';
}
$posts[] = [
'id' => $post_id,
'title' => $title,
'selftext' => $content,
'author' => $author,
'score' => 1, // Not available in RSS.
'num_comments' => 0, // Not available in RSS.
'permalink' => $permalink,
'created_utc' => $created_utc,
'is_self' => true,
'link_flair_text' => null,
'upvote_ratio' => null,
'id' => $post_id,
'title' => $title,
'selftext' => $content,
'author' => $author,
'score' => 1, // Not available in RSS.
'num_comments' => 0, // Not available in RSS.
'permalink' => $permalink,
'created_utc' => $created_utc,
'is_self' => true,
'link_flair_text' => null,
'upvote_ratio' => null,
'is_original_content' => false,
'data_source' => 'reddit_rss', // Tag for debugging.
'data_source' => 'reddit_rss',
];
}
PRAutoBlogger_Logger::instance()->info(
sprintf( 'Parsed %d posts from Reddit RSS for r/%s.', count( $posts ), $subreddit ),
'reddit_json'
'reddit'
);
return $posts;
@@ -249,8 +275,9 @@ class PRAutoBlogger_Reddit_JSON_Client {
/**
* Fetch top comments for a specific post via Reddit's .json endpoint.
*
* Returns data in the same format as the old OAuth client's fetch_comments():
* an array of comment data arrays with standard Reddit field names.
* Comments are always fetched via .json because RSS feeds don't include them.
* Datacenter IP blocks may prevent this from working, in which case we
* return an empty array (articles can still be generated from post titles alone).
*
* Side effects: HTTP request, updates rate limit transient.
*
@@ -288,22 +315,49 @@ class PRAutoBlogger_Reddit_JSON_Client {
}
/**
* Check if Reddit .json or RSS endpoints are reachable from this server.
* Check if at least one Reddit endpoint is reachable from this server.
*
* Datacenter IPs are often blocked by Reddit for .json endpoints.
* We check .json first, then fall back to RSS availability.
* Tries RSS first (most reliable), then .json as fallback.
*
* Side effects: HTTP request.
*
* @return bool True if we got a valid response from either endpoint.
*/
public function is_available(): bool {
return $this->is_json_available() || $this->is_rss_available();
return $this->is_rss_available() || $this->is_json_available();
}
/**
* Check if Reddit RSS endpoints are reachable.
*
* RSS endpoints are rarely blocked by Reddit, even for datacenter IPs.
*
* @return bool True if RSS endpoint returns HTTP 200.
*/
public function is_rss_available(): bool {
$url = self::BASE_URL . '/r/all/hot.rss?limit=1';
$response = wp_remote_get(
$url,
[
'timeout' => 10,
'headers' => [
'User-Agent' => self::USER_AGENT,
],
]
);
if ( is_wp_error( $response ) ) {
return false;
}
return 200 === wp_remote_retrieve_response_code( $response );
}
/**
* Check if Reddit .json endpoints are reachable.
*
* Datacenter IPs are often blocked by Reddit for .json endpoints.
*
* @return bool True if .json endpoint returns HTTP 200.
*/
public function is_json_available(): bool {
@@ -325,37 +379,10 @@ class PRAutoBlogger_Reddit_JSON_Client {
return 200 === wp_remote_retrieve_response_code( $response );
}
/**
* Check if Reddit RSS endpoints are reachable.
*
* RSS endpoints are less commonly blocked by Reddit than .json.
*
* @return bool True if RSS endpoint returns HTTP 200 with XML content.
*/
private function is_rss_available(): bool {
$url = self::BASE_URL . '/r/all/hot.rss?limit=1';
$response = wp_remote_get(
$url,
[
'timeout' => 10,
'headers' => [
'User-Agent' => self::USER_AGENT,
],
]
);
if ( is_wp_error( $response ) ) {
return false;
}
return 200 === wp_remote_retrieve_response_code( $response );
}
/**
* Get current rate limit status.
*
* Reddit .json endpoints return rate limit headers similar to the OAuth API.
* We track them in a transient.
* Reddit .json endpoints return rate limit headers. RSS has no rate limits.
*
* @return array{remaining: int, limit: int, resets_at: string}
*/
@@ -400,12 +427,12 @@ class PRAutoBlogger_Reddit_JSON_Client {
if ( is_wp_error( $response ) ) {
PRAutoBlogger_Logger::instance()->error(
'Reddit .json GET failed: ' . $response->get_error_message(),
'reddit_json'
'reddit'
);
return null;
}
// Track rate limits from response headers (same as OAuth API).
// Track rate limits from response headers.
$rate_remaining = wp_remote_retrieve_header( $response, 'x-ratelimit-remaining' );
$rate_used = wp_remote_retrieve_header( $response, 'x-ratelimit-used' );
$rate_reset = wp_remote_retrieve_header( $response, 'x-ratelimit-reset' );
@@ -431,7 +458,7 @@ class PRAutoBlogger_Reddit_JSON_Client {
$status_code,
substr( wp_remote_retrieve_body( $response ), 0, 300 )
),
'reddit_json'
'reddit'
);
return null;
}

337
includes/providers/class-reddit-provider.php
View File

@@ -4,47 +4,30 @@ declare(strict_types=1);
/**
* Reddit data provider — collects posts and comments from target subreddits.
*
* Uses PullPush.io as the PRIMARY data source (free, no auth required, 30 req/min).
* Falls back to Reddit .json endpoints if PullPush is unavailable.
* Reddit OAuth was removed because Reddit rejected our API application.
* Uses Reddit RSS/Atom feeds as the PRIMARY data source (reliable from datacenter IPs,
* no auth required, no rate-limit headers). Falls back to Reddit .json endpoints if
* RSS fails. PullPush.io was removed because its index was frequently stale/unavailable.
*
* Triggered by: Source_Collector calls collect_data() on each enabled provider.
* Dependencies: PRAutoBlogger_PullPush_Client (primary), PRAutoBlogger_Reddit_JSON_Client (fallback).
* Dependencies: PRAutoBlogger_Reddit_JSON_Client (handles both RSS and .json).
*
* @see interface-source-provider.php — Interface this class implements.
* @see providers/class-pullpush-client.php — Primary HTTP client (PullPush.io).
* @see providers/class-reddit-json-client.php — Fallback HTTP client (Reddit .json).
* @see providers/class-reddit-json-client.php — HTTP client for RSS and .json endpoints.
* @see core/class-source-collector.php — Orchestrates collection from all providers.
* @see ARCHITECTURE.md — External API integrations table.
*/
class PRAutoBlogger_Reddit_Provider implements PRAutoBlogger_Source_Provider_Interface {
/**
* Transient key for caching PullPush availability check.
* Avoids re-checking on every request within a collection run.
*
* @var string
*/
private const PULLPUSH_AVAILABLE_TRANSIENT = 'prautoblogger_pullpush_available';
/**
* How long to cache PullPush availability status (5 minutes).
*
* @var int
*/
private const AVAILABILITY_CACHE_TTL = 300;
/**
* Collect posts and comments from configured subreddits.
*
* Strategy: Try PullPush first (better API, subreddit-wide comment search).
* If PullPush is down, fall back to Reddit .json endpoints (same response
* format as the old OAuth API, but unauthenticated and rate-limited).
* Strategy: RSS first (works reliably from Hostinger datacenter IPs).
* If RSS fails, falls back to .json endpoints. Comment fetching always
* uses .json since comments are not available in RSS feeds.
*
* For each subreddit: fetches top posts, then fetches top comments for
* the most-engaged posts. Returns an array of Source_Data objects.
* For each subreddit: fetches posts via RSS, then fetches top comments
* for high-engagement posts via .json. Returns Source_Data objects.
*
* Side effects: HTTP requests to PullPush or Reddit, uses/sets transients.
* Side effects: HTTP requests to Reddit, uses/sets transients.
*
* @param array{
* subreddits?: string[],
@@ -63,16 +46,14 @@ class PRAutoBlogger_Reddit_Provider implements PRAutoBlogger_Source_Provider_Int
$include_comments = $config['include_comments'] ?? true;
$comments_limit = $config['comments_per_post'] ?? 5;
// Determine which client to use.
$use_pullpush = $this->is_pullpush_available();
$collected = [];
$client = new PRAutoBlogger_Reddit_JSON_Client();
$collected = [];
PRAutoBlogger_Logger::instance()->info(
sprintf(
'Reddit collect_data: %d subreddits=%s, use_pullpush=%s, limit=%d, time=%s',
'Reddit collect_data: %d subreddits=%s, limit=%d, time=%s',
count( $subreddits ),
wp_json_encode( $subreddits ),
$use_pullpush ? 'yes' : 'no',
$limit,
$time_filter
),
@@ -84,24 +65,70 @@ class PRAutoBlogger_Reddit_Provider implements PRAutoBlogger_Source_Provider_Int
try {
$before_count = count( $collected );
if ( $use_pullpush ) {
$collected = array_merge(
$collected,
$this->collect_from_pullpush( $subreddit, $limit, $time_filter, $include_comments, $comments_limit )
);
} else {
$collected = array_merge(
$collected,
$this->collect_from_reddit_json( $subreddit, $limit, $time_filter, $include_comments, $comments_limit )
);
// Fetch posts — client tries RSS first, .json fallback internally.
$posts = $client->fetch_posts( $subreddit, $limit, $time_filter );
foreach ( $posts as $post ) {
$data_source = $post['data_source'] ?? 'reddit';
$collected[] = new PRAutoBlogger_Source_Data( [
'source_type' => 'reddit',
'source_id' => 't3_' . ( $post['id'] ?? '' ),
'subreddit' => $subreddit,
'title' => $post['title'] ?? '',
'content' => $post['selftext'] ?? '',
'author' => $post['author'] ?? '[deleted]',
'score' => (int) ( $post['score'] ?? 0 ),
'comment_count' => (int) ( $post['num_comments'] ?? 0 ),
'permalink' => 'https://reddit.com' . ( $post['permalink'] ?? '' ),
'collected_at' => current_time( 'mysql' ),
'metadata' => [
'post_type' => ( $post['is_self'] ?? false ) ? 'self' : 'link',
'flair' => $post['link_flair_text'] ?? null,
'created_utc' => $post['created_utc'] ?? null,
'upvote_ratio' => $post['upvote_ratio'] ?? null,
'is_original' => $post['is_original_content'] ?? false,
'data_source' => $data_source,
],
] );
// Fetch comments for high-engagement posts via .json
// (comments are not available in RSS feeds).
if ( $include_comments && ( $post['num_comments'] ?? 0 ) > 0 ) {
$post_id = $post['id'] ?? '';
$comments = $client->fetch_comments( $subreddit, $post_id, $comments_limit );
foreach ( $comments as $comment ) {
$collected[] = new PRAutoBlogger_Source_Data( [
'source_type' => 'reddit',
'source_id' => 't1_' . ( $comment['id'] ?? '' ),
'subreddit' => $subreddit,
'title' => null,
'content' => $comment['body'] ?? '',
'author' => $comment['author'] ?? '[deleted]',
'score' => (int) ( $comment['score'] ?? 0 ),
'comment_count' => 0,
'permalink' => 'https://reddit.com' . ( $comment['permalink'] ?? '' ),
'collected_at' => current_time( 'mysql' ),
'metadata' => [
'parent_post_id' => 't3_' . $post_id,
'parent_title' => $post['title'] ?? '',
'is_comment' => true,
'created_utc' => $comment['created_utc'] ?? null,
'data_source' => 'reddit_json',
],
] );
}
}
}
$added = count( $collected ) - $before_count;
PRAutoBlogger_Logger::instance()->info(
sprintf( 'Reddit r/%s: fetched %d items via %s', $subreddit, $added, $use_pullpush ? 'pullpush' : 'reddit_json' ),
sprintf( 'Reddit r/%s: fetched %d items', $subreddit, $added ),
'reddit'
);
} catch ( \Exception $e ) {
// Log but continue with other subreddits.
PRAutoBlogger_Logger::instance()->error(
sprintf(
'Reddit: Failed to collect from r/%s: %s',
@@ -126,236 +153,28 @@ class PRAutoBlogger_Reddit_Provider implements PRAutoBlogger_Source_Provider_Int
/**
* Validate that at least one data source is reachable.
*
* No credentials needed anymore — PullPush and Reddit .json are both
* unauthenticated. We just check if either endpoint is responding.
* No credentials needed — RSS and .json are both unauthenticated.
* We just check if either endpoint is responding.
*
* Side effects: HTTP requests to PullPush and/or Reddit.
* Side effects: HTTP requests to Reddit.
*
* @return bool True if at least one data source is available.
*/
public function validate_credentials(): bool {
$pullpush = new PRAutoBlogger_PullPush_Client();
if ( $pullpush->is_available() ) {
return true;
}
$reddit_json = new PRAutoBlogger_Reddit_JSON_Client();
return $reddit_json->is_available();
$client = new PRAutoBlogger_Reddit_JSON_Client();
return $client->is_available();
}
/**
* Get current rate limit status from the active data source.
* Get current rate limit status from the Reddit client.
*
* @return array{remaining: int, limit: int, resets_at: string}
*/
public function get_rate_limit_status(): array {
if ( $this->is_pullpush_available() ) {
$client = new PRAutoBlogger_PullPush_Client();
return $client->get_rate_limit_status();
}
$client = new PRAutoBlogger_Reddit_JSON_Client();
return $client->get_rate_limit_status();
}
/**
* Collect data from PullPush.io API.
*
* PullPush returns flat arrays of Reddit objects with standard field names.
* We can search comments subreddit-wide (more efficient than per-post).
*
* @param string $subreddit Subreddit name.
* @param int $limit Max posts.
* @param string $time_filter Time window.
* @param bool $include_comments Whether to fetch comments.
* @param int $comments_limit Max comments per post.
*
* @return PRAutoBlogger_Source_Data[]
*/
private function collect_from_pullpush(
string $subreddit,
int $limit,
string $time_filter,
bool $include_comments,
int $comments_limit
): array {
$client = new PRAutoBlogger_PullPush_Client();
$collected = [];
// Fetch posts.
$posts = $client->fetch_posts( $subreddit, $limit, $time_filter );
foreach ( $posts as $post ) {
$collected[] = new PRAutoBlogger_Source_Data( [
'source_type' => 'reddit',
'source_id' => 't3_' . ( $post['id'] ?? '' ),
'subreddit' => $subreddit,
'title' => $post['title'] ?? '',
'content' => $post['selftext'] ?? '',
'author' => $post['author'] ?? '[deleted]',
'score' => (int) ( $post['score'] ?? 0 ),
'comment_count' => (int) ( $post['num_comments'] ?? 0 ),
'permalink' => 'https://reddit.com' . ( $post['permalink'] ?? '' ),
'collected_at' => current_time( 'mysql' ),
'metadata' => [
'post_type' => ( $post['is_self'] ?? false ) ? 'self' : 'link',
'flair' => $post['link_flair_text'] ?? null,
'created_utc' => $post['created_utc'] ?? null,
'upvote_ratio' => $post['upvote_ratio'] ?? null,
'is_original' => $post['is_original_content'] ?? false,
'data_source' => 'pullpush',
],
] );
// Fetch comments for high-engagement posts.
if ( $include_comments && ( $post['num_comments'] ?? 0 ) > 0 ) {
$post_id = $post['id'] ?? '';
$comments = $client->fetch_comments_for_post( $post_id, $comments_limit );
foreach ( $comments as $comment ) {
$collected[] = new PRAutoBlogger_Source_Data( [
'source_type' => 'reddit',
'source_id' => 't1_' . ( $comment['id'] ?? '' ),
'subreddit' => $subreddit,
'title' => null,
'content' => $comment['body'] ?? '',
'author' => $comment['author'] ?? '[deleted]',
'score' => (int) ( $comment['score'] ?? 0 ),
'comment_count' => 0,
'permalink' => 'https://reddit.com' . ( $comment['permalink'] ?? '' ),
'collected_at' => current_time( 'mysql' ),
'metadata' => [
'parent_post_id' => 't3_' . $post_id,
'parent_title' => $post['title'] ?? '',
'is_comment' => true,
'created_utc' => $comment['created_utc'] ?? null,
'data_source' => 'pullpush',
],
] );
}
}
}
return $collected;
}
/**
* Collect data from Reddit .json endpoints (fallback).
*
* Response format matches the old OAuth API: data.children[].data.
* Per-post comment fetching (not subreddit-wide like PullPush).
*
* @param string $subreddit Subreddit name.
* @param int $limit Max posts.
* @param string $time_filter Time window.
* @param bool $include_comments Whether to fetch comments.
* @param int $comments_limit Max comments per post.
*
* @return PRAutoBlogger_Source_Data[]
*/
private function collect_from_reddit_json(
string $subreddit,
int $limit,
string $time_filter,
bool $include_comments,
int $comments_limit
): array {
$client = new PRAutoBlogger_Reddit_JSON_Client();
$collected = [];
// Fetch posts — same response format as old OAuth API.
$posts = $client->fetch_posts( $subreddit, $limit, $time_filter );
foreach ( $posts as $post ) {
$collected[] = new PRAutoBlogger_Source_Data( [
'source_type' => 'reddit',
'source_id' => 't3_' . ( $post['id'] ?? '' ),
'subreddit' => $subreddit,
'title' => $post['title'] ?? '',
'content' => $post['selftext'] ?? '',
'author' => $post['author'] ?? '[deleted]',
'score' => (int) ( $post['score'] ?? 0 ),
'comment_count' => (int) ( $post['num_comments'] ?? 0 ),
'permalink' => 'https://reddit.com' . ( $post['permalink'] ?? '' ),
'collected_at' => current_time( 'mysql' ),
'metadata' => [
'post_type' => ( $post['is_self'] ?? false ) ? 'self' : 'link',
'flair' => $post['link_flair_text'] ?? null,
'created_utc' => $post['created_utc'] ?? null,
'upvote_ratio' => $post['upvote_ratio'] ?? null,
'is_original' => $post['is_original_content'] ?? false,
'data_source' => 'reddit_json',
],
] );
// Collect top comments for high-engagement posts.
if ( $include_comments && ( $post['num_comments'] ?? 0 ) > 0 ) {
$post_id = $post['id'] ?? '';
$comments = $client->fetch_comments( $subreddit, $post_id, $comments_limit );
foreach ( $comments as $comment ) {
$collected[] = new PRAutoBlogger_Source_Data( [
'source_type' => 'reddit',
'source_id' => 't1_' . ( $comment['id'] ?? '' ),
'subreddit' => $subreddit,
'title' => null,
'content' => $comment['body'] ?? '',
'author' => $comment['author'] ?? '[deleted]',
'score' => (int) ( $comment['score'] ?? 0 ),
'comment_count' => 0,
'permalink' => 'https://reddit.com' . ( $comment['permalink'] ?? '' ),
'collected_at' => current_time( 'mysql' ),
'metadata' => [
'parent_post_id' => 't3_' . $post_id,
'parent_title' => $post['title'] ?? '',
'is_comment' => true,
'created_utc' => $comment['created_utc'] ?? null,
'data_source' => 'reddit_json',
],
] );
}
}
}
return $collected;
}
/**
* Check if PullPush.io is currently available, with caching.
*
* Caches the result for 5 minutes to avoid re-checking availability
* on every subreddit within a single collection run.
*
* Side effects: May make HTTP request, sets transient.
*
* @return bool True if PullPush is reachable.
*/
private function is_pullpush_available(): bool {
$cached = get_transient( self::PULLPUSH_AVAILABLE_TRANSIENT );
if ( false !== $cached ) {
return '1' === $cached;
}
$client = new PRAutoBlogger_PullPush_Client();
$available = $client->is_available();
set_transient(
self::PULLPUSH_AVAILABLE_TRANSIENT,
$available ? '1' : '0',
self::AVAILABILITY_CACHE_TTL
);
if ( ! $available ) {
PRAutoBlogger_Logger::instance()->warning(
'PullPush.io unavailable — falling back to Reddit .json endpoints.',
'reddit'
);
}
return $available;
}
/**
* Get the list of subreddits configured in plugin settings.
*