mirror of
https://github.com/RSS-Bridge/rss-bridge.git
synced 2025-08-30 11:30:11 +02:00
fix: rewrite and improve caching (#3594)
This commit is contained in:
Dag
GitHub
@@ -34,7 +34,7 @@ class ConnectivityAction implements ActionInterface
|
||||
public function execute(array $request)
|
||||
{
|
||||
if (!Debug::isEnabled()) {
|
||||
throw new \Exception('This action is only available in debug mode!');
|
||||
return new Response('This action is only available in debug mode!');
|
||||
}
|
||||
|
||||
$bridgeName = $request['bridge'] ?? null;
|
||||
@@ -43,7 +43,7 @@ class ConnectivityAction implements ActionInterface
|
||||
}
|
||||
$bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName);
|
||||
if (!$bridgeClassName) {
|
||||
throw new \Exception(sprintf('Bridge not found: %s', $bridgeName));
|
||||
return new Response('Bridge not found', 404);
|
||||
}
|
||||
return $this->reportBridgeConnectivity($bridgeClassName);
|
||||
}
|
||||
@@ -54,29 +54,25 @@ class ConnectivityAction implements ActionInterface
|
||||
throw new \Exception('Bridge is not whitelisted!');
|
||||
}
|
||||
|
||||
$retVal = [
|
||||
'bridge' => $bridgeClassName,
|
||||
'successful' => false,
|
||||
'http_code' => 200,
|
||||
];
|
||||
|
||||
$bridge = $this->bridgeFactory->create($bridgeClassName);
|
||||
$curl_opts = [
|
||||
CURLOPT_CONNECTTIMEOUT => 5
|
||||
CURLOPT_CONNECTTIMEOUT => 5,
|
||||
CURLOPT_FOLLOWLOCATION => true,
|
||||
];
|
||||
$result = [
|
||||
'bridge' => $bridgeClassName,
|
||||
'successful' => false,
|
||||
'http_code' => null,
|
||||
];
|
||||
try {
|
||||
$reply = getContents($bridge::URI, [], $curl_opts, true);
|
||||
|
||||
if ($reply['code'] === 200) {
|
||||
$retVal['successful'] = true;
|
||||
if (strpos(implode('', $reply['status_lines']), '301 Moved Permanently')) {
|
||||
$retVal['http_code'] = 301;
|
||||
}
|
||||
$response = getContents($bridge::URI, [], $curl_opts, true);
|
||||
$result['http_code'] = $response['code'];
|
||||
if (in_array($response['code'], [200])) {
|
||||
$result['successful'] = true;
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
$retVal['successful'] = false;
|
||||
}
|
||||
|
||||
return new Response(Json::encode($retVal), 200, ['Content-Type' => 'text/json']);
|
||||
return new Response(Json::encode($result), 200, ['content-type' => 'text/json']);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ class DetectAction implements ActionInterface
|
||||
$bridgeParams['format'] = $format;
|
||||
|
||||
$url = '?action=display&' . http_build_query($bridgeParams);
|
||||
return new Response('', 301, ['Location' => $url]);
|
||||
return new Response('', 301, ['location' => $url]);
|
||||
}
|
||||
|
||||
throw new \Exception('No bridge found for given URL: ' . $targetURL);
|
||||
|
||||
@@ -10,50 +10,41 @@ class DisplayAction implements ActionInterface
|
||||
return new Response('503 Service Unavailable', 503);
|
||||
}
|
||||
$this->cache = RssBridge::getCache();
|
||||
$this->cache->setScope('http');
|
||||
$this->cache->setKey($request);
|
||||
// avg timeout of 20m
|
||||
$timeout = 60 * 15 + rand(1, 60 * 10);
|
||||
$cacheKey = 'http_' . json_encode($request);
|
||||
/** @var Response $cachedResponse */
|
||||
$cachedResponse = $this->cache->loadData($timeout);
|
||||
if ($cachedResponse && !Debug::isEnabled()) {
|
||||
//Logger::info(sprintf('Returning cached (http) response: %s', $cachedResponse->getBody()));
|
||||
$cachedResponse = $this->cache->get($cacheKey);
|
||||
if ($cachedResponse) {
|
||||
$ifModifiedSince = $_SERVER['HTTP_IF_MODIFIED_SINCE'] ?? null;
|
||||
$lastModified = $cachedResponse->getHeader('last-modified');
|
||||
if ($ifModifiedSince && $lastModified) {
|
||||
$lastModified = new \DateTimeImmutable($lastModified);
|
||||
$lastModifiedTimestamp = $lastModified->getTimestamp();
|
||||
$modifiedSince = strtotime($ifModifiedSince);
|
||||
if ($lastModifiedTimestamp <= $modifiedSince) {
|
||||
$modificationTimeGMT = gmdate('D, d M Y H:i:s ', $lastModifiedTimestamp);
|
||||
return new Response('', 304, ['last-modified' => $modificationTimeGMT . 'GMT']);
|
||||
}
|
||||
}
|
||||
return $cachedResponse;
|
||||
}
|
||||
$response = $this->createResponse($request);
|
||||
if (in_array($response->getCode(), [429, 503])) {
|
||||
//Logger::info(sprintf('Storing cached (http) response: %s', $response->getBody()));
|
||||
$this->cache->setScope('http');
|
||||
$this->cache->setKey($request);
|
||||
$this->cache->saveData($response);
|
||||
}
|
||||
return $response;
|
||||
}
|
||||
|
||||
private function createResponse(array $request)
|
||||
{
|
||||
$bridgeFactory = new BridgeFactory();
|
||||
$formatFactory = new FormatFactory();
|
||||
|
||||
$bridgeName = $request['bridge'] ?? null;
|
||||
$format = $request['format'] ?? null;
|
||||
|
||||
if (!$bridgeName) {
|
||||
return new Response('Missing bridge param', 400);
|
||||
}
|
||||
$bridgeFactory = new BridgeFactory();
|
||||
$bridgeClassName = $bridgeFactory->createBridgeClassName($bridgeName);
|
||||
if (!$bridgeClassName) {
|
||||
throw new \Exception(sprintf('Bridge not found: %s', $bridgeName));
|
||||
return new Response('Bridge not found', 404);
|
||||
}
|
||||
$format = $request['format'] ?? null;
|
||||
if (!$format) {
|
||||
throw new \Exception('You must specify a format!');
|
||||
return new Response('You must specify a format!', 400);
|
||||
}
|
||||
if (!$bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
throw new \Exception('This bridge is not whitelisted');
|
||||
return new Response('This bridge is not whitelisted', 400);
|
||||
}
|
||||
|
||||
$format = $formatFactory->create($format);
|
||||
|
||||
$bridge = $bridgeFactory->create($bridgeClassName);
|
||||
$bridge->loadConfiguration();
|
||||
|
||||
$noproxy = $request['_noproxy'] ?? null;
|
||||
if (
|
||||
Configuration::getConfig('proxy', 'url')
|
||||
@@ -64,147 +55,100 @@ class DisplayAction implements ActionInterface
|
||||
define('NOPROXY', true);
|
||||
}
|
||||
|
||||
$cacheTimeout = $request['_cache_timeout'] ?? null;
|
||||
if (Configuration::getConfig('cache', 'custom_timeout') && $cacheTimeout) {
|
||||
$cacheTimeout = (int) $cacheTimeout;
|
||||
} else {
|
||||
// At this point the query argument might still be in the url but it won't be used
|
||||
$cacheTimeout = $bridge->getCacheTimeout();
|
||||
$bridge = $bridgeFactory->create($bridgeClassName);
|
||||
$formatFactory = new FormatFactory();
|
||||
$format = $formatFactory->create($format);
|
||||
|
||||
$response = $this->createResponse($request, $bridge, $format);
|
||||
|
||||
if ($response->getCode() === 200) {
|
||||
$ttl = $request['_cache_timeout'] ?? null;
|
||||
if (Configuration::getConfig('cache', 'custom_timeout') && $ttl) {
|
||||
$ttl = (int) $ttl;
|
||||
} else {
|
||||
$ttl = $bridge->getCacheTimeout();
|
||||
}
|
||||
$this->cache->set($cacheKey, $response, $ttl);
|
||||
}
|
||||
|
||||
// Remove parameters that don't concern bridges
|
||||
$bridge_params = array_diff_key(
|
||||
$request,
|
||||
array_fill_keys(
|
||||
[
|
||||
'action',
|
||||
'bridge',
|
||||
'format',
|
||||
'_noproxy',
|
||||
'_cache_timeout',
|
||||
'_error_time'
|
||||
],
|
||||
''
|
||||
)
|
||||
);
|
||||
if (in_array($response->getCode(), [429, 503])) {
|
||||
$this->cache->set($cacheKey, $response, 60 * 15 + rand(1, 60 * 10)); // average 20m
|
||||
}
|
||||
|
||||
// Remove parameters that don't concern caches
|
||||
$cache_params = array_diff_key(
|
||||
$request,
|
||||
array_fill_keys(
|
||||
[
|
||||
'action',
|
||||
'format',
|
||||
'_noproxy',
|
||||
'_cache_timeout',
|
||||
'_error_time'
|
||||
],
|
||||
''
|
||||
)
|
||||
);
|
||||
|
||||
$this->cache->setScope('');
|
||||
$this->cache->setKey($cache_params);
|
||||
if ($response->getCode() === 500) {
|
||||
$this->cache->set($cacheKey, $response, 60 * 15);
|
||||
}
|
||||
if (rand(1, 100) === 2) {
|
||||
$this->cache->prune();
|
||||
}
|
||||
return $response;
|
||||
}
|
||||
|
||||
private function createResponse(array $request, BridgeInterface $bridge, FormatInterface $format)
|
||||
{
|
||||
$items = [];
|
||||
$infos = [];
|
||||
|
||||
$feed = $this->cache->loadData($cacheTimeout);
|
||||
|
||||
if ($feed && !Debug::isEnabled()) {
|
||||
if (isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) {
|
||||
$modificationTime = $this->cache->getTime();
|
||||
// The client wants to know if the feed has changed since its last check
|
||||
$modifiedSince = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']);
|
||||
if ($modificationTime <= $modifiedSince) {
|
||||
$modificationTimeGMT = gmdate('D, d M Y H:i:s ', $modificationTime);
|
||||
return new Response('', 304, ['Last-Modified' => $modificationTimeGMT . 'GMT']);
|
||||
try {
|
||||
$bridge->loadConfiguration();
|
||||
// Remove parameters that don't concern bridges
|
||||
$bridgeData = array_diff_key($request, array_fill_keys(['action', 'bridge', 'format', '_noproxy', '_cache_timeout', '_error_time'], ''));
|
||||
$bridge->setDatas($bridgeData);
|
||||
$bridge->collectData();
|
||||
$items = $bridge->getItems();
|
||||
if (isset($items[0]) && is_array($items[0])) {
|
||||
$feedItems = [];
|
||||
foreach ($items as $item) {
|
||||
$feedItems[] = new FeedItem($item);
|
||||
}
|
||||
$items = $feedItems;
|
||||
}
|
||||
$infos = [
|
||||
'name' => $bridge->getName(),
|
||||
'uri' => $bridge->getURI(),
|
||||
'donationUri' => $bridge->getDonationURI(),
|
||||
'icon' => $bridge->getIcon()
|
||||
];
|
||||
} catch (\Exception $e) {
|
||||
$errorOutput = Configuration::getConfig('error', 'output');
|
||||
$reportLimit = Configuration::getConfig('error', 'report_limit');
|
||||
if ($e instanceof HttpException) {
|
||||
// Reproduce (and log) these responses regardless of error output and report limit
|
||||
if ($e->getCode() === 429) {
|
||||
Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
|
||||
return new Response('429 Too Many Requests', 429);
|
||||
}
|
||||
if ($e->getCode() === 503) {
|
||||
Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
|
||||
return new Response('503 Service Unavailable', 503);
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($feed['items']) && isset($feed['extraInfos'])) {
|
||||
foreach ($feed['items'] as $item) {
|
||||
$items[] = new FeedItem($item);
|
||||
}
|
||||
$infos = $feed['extraInfos'];
|
||||
Logger::error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]);
|
||||
$errorCount = 1;
|
||||
if ($reportLimit > 1) {
|
||||
$errorCount = $this->logBridgeError($bridge->getName(), $e->getCode());
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
$bridge->setDatas($bridge_params);
|
||||
$bridge->collectData();
|
||||
$items = $bridge->getItems();
|
||||
if (isset($items[0]) && is_array($items[0])) {
|
||||
$feedItems = [];
|
||||
foreach ($items as $item) {
|
||||
$feedItems[] = new FeedItem($item);
|
||||
}
|
||||
$items = $feedItems;
|
||||
}
|
||||
$infos = [
|
||||
'name' => $bridge->getName(),
|
||||
'uri' => $bridge->getURI(),
|
||||
'donationUri' => $bridge->getDonationURI(),
|
||||
'icon' => $bridge->getIcon()
|
||||
];
|
||||
} catch (\Exception $e) {
|
||||
$errorOutput = Configuration::getConfig('error', 'output');
|
||||
$reportLimit = Configuration::getConfig('error', 'report_limit');
|
||||
if ($e instanceof HttpException) {
|
||||
// Reproduce (and log) these responses regardless of error output and report limit
|
||||
if ($e->getCode() === 429) {
|
||||
Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e)));
|
||||
return new Response('429 Too Many Requests', 429);
|
||||
}
|
||||
if ($e->getCode() === 503) {
|
||||
Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e)));
|
||||
return new Response('503 Service Unavailable', 503);
|
||||
}
|
||||
// Might want to cache other codes such as 504 Gateway Timeout
|
||||
}
|
||||
if (in_array($errorOutput, ['feed', 'none'])) {
|
||||
Logger::error(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e)), ['e' => $e]);
|
||||
}
|
||||
$errorCount = 1;
|
||||
if ($reportLimit > 1) {
|
||||
$errorCount = $this->logBridgeError($bridge->getName(), $e->getCode());
|
||||
}
|
||||
// Let clients know about the error if we are passed the report limit
|
||||
if ($errorCount >= $reportLimit) {
|
||||
if ($errorOutput === 'feed') {
|
||||
// Render the exception as a feed item
|
||||
$items[] = $this->createFeedItemFromException($e, $bridge);
|
||||
} elseif ($errorOutput === 'http') {
|
||||
// Rethrow so that the main exception handler in RssBridge.php produces an HTTP 500
|
||||
throw $e;
|
||||
} elseif ($errorOutput === 'none') {
|
||||
// Do nothing (produces an empty feed)
|
||||
} else {
|
||||
// Do nothing, unknown error output? Maybe throw exception or validate in Configuration.php
|
||||
}
|
||||
// Let clients know about the error if we are passed the report limit
|
||||
if ($errorCount >= $reportLimit) {
|
||||
if ($errorOutput === 'feed') {
|
||||
// Render the exception as a feed item
|
||||
$items[] = $this->createFeedItemFromException($e, $bridge);
|
||||
} elseif ($errorOutput === 'http') {
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['e' => $e]), 500);
|
||||
} elseif ($errorOutput === 'none') {
|
||||
// Do nothing (produces an empty feed)
|
||||
}
|
||||
}
|
||||
|
||||
// Unfortunately need to set scope and key again because they might be modified
|
||||
$this->cache->setScope('');
|
||||
$this->cache->setKey($cache_params);
|
||||
$this->cache->saveData([
|
||||
'items' => array_map(function (FeedItem $item) {
|
||||
return $item->toArray();
|
||||
}, $items),
|
||||
'extraInfos' => $infos
|
||||
]);
|
||||
$this->cache->purgeCache();
|
||||
}
|
||||
|
||||
$format->setItems($items);
|
||||
$format->setExtraInfos($infos);
|
||||
$newModificationTime = $this->cache->getTime();
|
||||
$format->setLastModified($newModificationTime);
|
||||
$headers = [];
|
||||
if ($newModificationTime) {
|
||||
$headers['Last-Modified'] = gmdate('D, d M Y H:i:s ', $newModificationTime) . 'GMT';
|
||||
}
|
||||
$headers['Content-Type'] = $format->getMimeType() . '; charset=' . $format->getCharset();
|
||||
$now = time();
|
||||
$format->setLastModified($now);
|
||||
$headers = [
|
||||
'last-modified' => gmdate('D, d M Y H:i:s ', $now) . 'GMT',
|
||||
'content-type' => $format->getMimeType() . '; charset=' . $format->getCharset(),
|
||||
];
|
||||
return new Response($format->stringify(), 200, $headers);
|
||||
}
|
||||
|
||||
@@ -234,9 +178,8 @@ class DisplayAction implements ActionInterface
|
||||
|
||||
private function logBridgeError($bridgeName, $code)
|
||||
{
|
||||
$this->cache->setScope('error_reporting');
|
||||
$this->cache->setkey([$bridgeName . '_' . $code]);
|
||||
$report = $this->cache->loadData();
|
||||
$cacheKey = 'error_reporting_' . $bridgeName . '_' . $code;
|
||||
$report = $this->cache->get($cacheKey);
|
||||
if ($report) {
|
||||
$report = Json::decode($report);
|
||||
$report['time'] = time();
|
||||
@@ -248,7 +191,8 @@ class DisplayAction implements ActionInterface
|
||||
'count' => 1,
|
||||
];
|
||||
}
|
||||
$this->cache->saveData(Json::encode($report));
|
||||
$ttl = 86400 * 5;
|
||||
$this->cache->set($cacheKey, Json::encode($report), $ttl);
|
||||
return $report['count'];
|
||||
}
|
||||
|
||||
|
||||
@@ -37,6 +37,6 @@ class ListAction implements ActionInterface
|
||||
];
|
||||
}
|
||||
$list->total = count($list->bridges);
|
||||
return new Response(Json::encode($list), 200, ['Content-Type' => 'application/json']);
|
||||
return new Response(Json::encode($list), 200, ['content-type' => 'application/json']);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,10 @@ class SetBridgeCacheAction implements ActionInterface
|
||||
$authenticationMiddleware = new ApiAuthenticationMiddleware();
|
||||
$authenticationMiddleware($request);
|
||||
|
||||
$key = $request['key'] or returnClientError('You must specify key!');
|
||||
$key = $request['key'] ?? null;
|
||||
if (!$key) {
|
||||
returnClientError('You must specify key!');
|
||||
}
|
||||
|
||||
$bridgeFactory = new BridgeFactory();
|
||||
|
||||
@@ -40,13 +43,10 @@ class SetBridgeCacheAction implements ActionInterface
|
||||
$value = $request['value'];
|
||||
|
||||
$cache = RssBridge::getCache();
|
||||
$cache->setScope(get_class($bridge));
|
||||
if (!is_array($key)) {
|
||||
// not sure if $key is an array when it comes in from request
|
||||
$key = [$key];
|
||||
}
|
||||
$cache->setKey($key);
|
||||
$cache->saveData($value);
|
||||
|
||||
$cacheKey = get_class($bridge) . '_' . $key;
|
||||
$ttl = 86400 * 3;
|
||||
$cache->set($cacheKey, $value, $ttl);
|
||||
|
||||
header('Content-Type: text/plain');
|
||||
echo 'done';
|
||||
|
||||
Reference in New Issue
Block a user