fix: give better error message when feed can't be parsed

* New Bridge for Twitter using v2 API

* Top comment block, tweaks to match contributing guide

* [TwitterV2Bridge] new Bridge (sort of)

* Discovered the point of, and re-added, no image scaling option

* Fix the phpcs sniff violations (I hope)

* More linter fixes, I figured out how to use phpcs locally

* Removed unnecessary custom version of getContents function

* Limit query to 100 tweets, valid example query, improved error handling

* Added config doc (correctly, I hope) with link from DESCRIPTION

* little tweak to doc

.github/prtester-requirements.txt

@@ -0,0 +1,2 @@
+beautifulsoup4>=4.10.0
+requests>=2.26.0

.github/prtester.py

@@ -0,0 +1,101 @@
+import requests
+from bs4 import BeautifulSoup
+from datetime import datetime
+import os.path
+
+# This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge
+#
+# This will scrape the whitelisted bridges in the current state (port 3000) and the PR state (port 3001) of
+# RSS-Bridge, generate a feed for each of the bridges and save the output as html files.
+# It also replaces the default static CSS link with a hardcoded link to @em92's public instance, so viewing
+# the HTML file locally will actually work as designed.
+
+def testBridges(bridges,status):
+    for bridge in bridges:
+        if bridge.get('data-ref'): # Some div entries are empty, this ignores those
+            bridgeid = bridge.get('id')
+            bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata
+            bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html'
+            forms = bridge.find_all("form")
+            formid = 1
+            for form in forms:
+                # a bridge can have multiple contexts, named 'forms' in html
+                # this code will produce a fully working formstring that should create a working feed when called
+                # this will create an example feed for every single context, to test them all
+                formstring = ''
+                errormessages = []
+                parameters = form.find_all("input")
+                lists = form.find_all("select")
+                # this for/if mess cycles through all available input parameters, checks if it required, then pulls
+                # the default or examplevalue and then combines it all together into the formstring
+                # if an example or default value is missing for a required attribute, it will throw an error
+                # any non-required fields are not tested!!!
+                for parameter in parameters:
+                    if parameter.get('type') == 'hidden' and parameter.get('name') == 'context':
+                        cleanvalue = parameter.get('value').replace(" ","+")
+                        formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue
+                    if parameter.get('type') == 'number' or parameter.get('type') == 'text':
+                        if parameter.has_attr('required'):
+                            if parameter.get('placeholder') == '':
+                                if parameter.get('value') == '':
+                                    errormessages.append(parameter.get('name'))
+                                else:
+                                    formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value')
+                            else:
+                                formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder')
+                    # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring
+                    if parameter.get('type') == 'checkbox':
+                        if parameter.has_attr('checked'):
+                            formstring = formstring + '&' + parameter.get('name') + '=on'
+                for list in lists:
+                    selectionvalue = ''
+                    for selectionentry in list.contents:
+                        if 'selected' in selectionentry.attrs:
+                            selectionvalue = selectionentry.get('value')
+                            break
+                    if selectionvalue == '':
+                        selectionvalue = list.contents[0].get('value')
+                    formstring = formstring + '&' + list.get('name') + '=' + selectionvalue
+                if not errormessages:
+                    # if all example/default values are present, form the full request string, run the request, replace the static css
+                    # file with the url of em's public instance and then upload it to termpad.com, a pastebin-like-site.
+                    r = requests.get(URL + bridgestring + formstring)
+                    pagetext = r.text.replace('static/HtmlFormat.css','https://feed.eugenemolotov.ru/static/HtmlFormat.css')
+                    pagetext = pagetext.encode("utf_8")
+                    termpad = requests.post(url="https://termpad.com/", data=pagetext)
+                    termpadurl = termpad.text
+                    termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/')
+                    termpadurl = termpadurl.replace('\n','')
+                    with open(os.getcwd() + '/comment.txt', 'a+') as file:
+                        file.write("\n")
+                        file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |")
+                else:
+                    # if there are errors (which means that a required value has no example or default value), log out which error appeared
+                    termpad = requests.post(url="https://termpad.com/", data=str(errormessages))
+                    termpadurl = termpad.text
+                    termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/')
+                    termpadurl = termpadurl.replace('\n','')
+                    with open(os.getcwd() + '/comment.txt', 'a+') as file:
+                        file.write("\n")
+                        file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |")
+                formid += 1
+
+gitstatus = ["current", "pr"]
+now = datetime.now()
+date_time = now.strftime("%Y-%m-%d, %H:%M:%S")
+
+with open(os.getcwd() + '/comment.txt', 'w+') as file:
+    file.write(''' ## Pull request artifacts
+| file | last change |
+| ---- | ------ |''')
+
+for status in gitstatus: # run this twice, once for the current version, once for the PR version
+    if status == "current":
+        port = "3000" # both ports are defined in the corresponding workflow .yml file
+    elif status == "pr":
+        port = "3001"
+    URL = "http://localhost:" + port
+    page = requests.get(URL) # Use python requests to grab the rss-bridge main page
+    soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup
+    bridges = soup.find_all("section") # get a soup-formatted list of all bridges on the rss-bridge page
+    testBridges(bridges,status) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version

.github/workflows/prhtmlgenerator.yml

@@ -0,0 +1,69 @@
+name: 'PR Testing'
+
+on:
+  pull_request_target:
+    branches: [ master ]
+
+jobs:
+  test-pr:
+    name: Generate HTML
+    runs-on: ubuntu-latest
+    # Needs additional permissions https://github.com/actions/first-interaction/issues/10#issuecomment-1041402989
+    steps:
+      - name: Check out self
+        uses: actions/checkout@v2.3.2
+        with:
+          ref: ${{github.event.pull_request.head.ref}}
+          repository: ${{github.event.pull_request.head.repo.full_name}}
+      - name: Check out rss-bridge
+        run: |
+          PR=${{github.event.number}};
+          wget -O requirements.txt https://raw.githubusercontent.com/RSS-Bridge/rss-bridge/master/.github/prtester-requirements.txt;
+          wget https://raw.githubusercontent.com/RSS-Bridge/rss-bridge/master/.github/prtester.py;
+          wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch;
+          touch DEBUG;
+          cat $PR.patch | grep " bridges/.*\.php" | sed "s= bridges/\(.*\)Bridge.php.*=\1=g" | sort | uniq > whitelist.txt
+      - name: Start Docker - Current
+        run: |
+          docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3000:80 ghcr.io/rss-bridge/rss-bridge:latest
+      - name: Start Docker - PR
+        run: |
+          docker build -t prbuild .;
+          docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3001:80 prbuild
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+          cache: 'pip'
+      - name: Install requirements
+        run: |
+          cd $GITHUB_WORKSPACE
+          pip install -r requirements.txt
+      - name: Run bridge tests
+        id: testrun
+        run: |
+          mkdir results;
+          python prtester.py;
+          body="$(cat comment.txt)";
+          body="${body//'%'/'%25'}";
+          body="${body//$'\n'/'%0A'}";
+          body="${body//$'\r'/'%0D'}";
+          echo "::set-output name=bodylength::${#body}"
+          echo "::set-output name=body::$body"
+      - name: Find Comment
+        if: ${{ steps.testrun.outputs.bodylength > 130 }}
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Pull request artifacts
+      - name: Create or update comment
+        if: ${{ steps.testrun.outputs.bodylength > 130 }}
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ${{ steps.testrun.outputs.body }}
+          edit-mode: replace

README.md

@@ -22,7 +22,7 @@ Supported sites/pages (examples)
 * `FlickrExplore` : [Latest interesting images](http://www.flickr.com/explore) from Flickr
 * `GoogleSearch` : Most recent results from Google Search
 * `Identi.ca` : Identica user timeline (Should be compatible with other Pump.io instances)
-* `Instagram`: Most recent photos from an Instagram user (There is an [issue](https://github.com/RSS-Bridge/rss-bridge/issues/1891) for public instances)
+* `Instagram`: Most recent photos from an Instagram user (It is recommended to [configure](https://rss-bridge.github.io/rss-bridge/Bridge_Specific/Instagram.html) this bridge to work)
 * `OpenClassrooms`: Lastest tutorials from [fr.openclassrooms.com](http://fr.openclassrooms.com/)
 * `Pinterest`: Most recent photos from user or search
 * `ScmbBridge`: Newest stories from [secouchermoinsbete.fr](http://secouchermoinsbete.fr/)
@@ -88,9 +88,11 @@ Deploy
 ===
 
 Thanks to the community, hosting your own instance of RSS-Bridge is as easy as clicking a button!
+*Note: External providers' applications are packaged by 3rd parties. Use at your own discretion.*
 
 [![Deploy on Scalingo](https://cdn.scalingo.com/deploy/button.svg)](https://my.scalingo.com/deploy?source=https://github.com/sebsauvage/rss-bridge)
 [![Deploy to Heroku](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy)
+[![Deploy to Cloudron](https://cloudron.io/img/button.svg)](https://www.cloudron.io/store/com.rssbridgeapp.cloudronapp.html)
 
 Getting involved
 ===

bridges/ABCTabsBridge.php

@@ -1,42 +0,0 @@
-<?php
-class ABCTabsBridge extends BridgeAbstract {
-
-	const MAINTAINER = 'kranack';
-	const NAME = 'ABC Tabs Bridge';
-	const URI = 'https://www.abc-tabs.com/';
-	const DESCRIPTION = 'Returns 22 newest tabs';
-
-	public function collectData(){
-		$html = '';
-		$html = getSimpleHTMLDOM(static::URI . 'tablatures/nouveautes.html')
-			or returnClientError('No results for this query.');
-
-		$table = $html->find('table#myTable', 0)->children(1);
-
-		foreach ($table->find('tr') as $tab) {
-			$item = array();
-			$item['author'] = $tab->find('td', 1)->plaintext
-			. ' - '
-			. $tab->find('td', 2)->plaintext;
-
-			$item['title'] = $tab->find('td', 1)->plaintext
-			. ' - '
-			. $tab->find('td', 2)->plaintext;
-
-			$item['content'] = 'Le '
-			. $tab->find('td', 0)->plaintext
-			. '<br> Par: '
-			. $tab->find('td', 5)->plaintext
-			. '<br> Type: '
-			. $tab->find('td', 3)->plaintext;
-
-			$item['id'] = static::URI
-			. $tab->find('td', 2)->find('a', 0)->getAttribute('href');
-
-			$item['uri'] = static::URI
-			. $tab->find('td', 2)->find('a', 0)->getAttribute('href');
-
-			$this->items[] = $item;
-		}
-	}
-}

bridges/AcrimedBridge.php

@@ -7,8 +7,21 @@ class AcrimedBridge extends FeedExpander {
 	const CACHE_TIMEOUT = 4800; //2hours
 	const DESCRIPTION = 'Returns the newest articles';
 
+	const PARAMETERS = [
+		[
+			'limit' => [
+				'name' => 'limit',
+				'type' => 'number',
+				'defaultValue' => -1,
+			]
+		]
+	];
+
 	public function collectData(){
-		$this->collectExpandableDatas(static::URI . 'spip.php?page=backend');
+		$this->collectExpandableDatas(
+			static::URI . 'spip.php?page=backend',
+			$this->getInput('limit')
+		);
 	}
 
 	protected function parseItem($newsItem){

bridges/AlfaBankByBridge.php

@@ -0,0 +1,83 @@
+<?php
+class AlfaBankByBridge extends BridgeAbstract {
+
+	const MAINTAINER = 'lassana';
+	const NAME = 'AlfaBank.by Новости';
+	const URI = 'https://www.alfabank.by';
+	const DESCRIPTION = 'Уведомления Alfa-Now — новости от Альфа-Банка';
+	const CACHE_TIMEOUT = 3600; // 1 hour
+	const PARAMETERS = array(
+		'News' => array(
+			'business' => array(
+				'name' => 'Альфа Бизнес',
+				'type' => 'list',
+				'title' => 'В зависимости от выбора, возращает уведомления для" .
+					" клиентов физ. лиц либо для клиентов-юридических лиц и ИП',
+				'values' => array(
+					'Новости' => 'news',
+					'Новости бизнеса' => 'newsBusiness'
+				),
+				'defaultValue' => 'news'
+			),
+			'fullContent' => array(
+				'name' => 'Включать содержимое',
+				'type' => 'checkbox',
+				'title' => 'Если выбрано, содержимое уведомлений вставляется в поток (работает медленно)'
+			)
+		)
+	);
+
+	public function collectData() {
+		$business = $this->getInput('business') == 'newsBusiness';
+		$fullContent = $this->getInput('fullContent') == 'on';
+
+		$mainPageUrl = self::URI . '/about/articles/uvedomleniya/';
+		if($business) {
+			$mainPageUrl .= '?business=true';
+		}
+		$html = getSimpleHTMLDOM($mainPageUrl);
+		$limit = 0;
+
+		foreach($html->find('a.notifications__item') as $element) {
+			if($limit < 10) {
+				$item = array();
+				$item['uid'] = 'urn:sha1:' . hash('sha1', $element->getAttribute('data-notification-id'));
+				$item['title'] = $element->find('div.item-title', 0)->innertext;
+				$item['timestamp'] = DateTime::createFromFormat(
+					'd M Y',
+					$this->ruMonthsToEn($element->find('div.item-date', 0)->innertext)
+					)->getTimestamp();
+
+				$itemUrl = self::URI . $element->href;
+				if($business) {
+					$itemUrl = str_replace('?business=true', '', $itemUrl);
+				}
+				$item['uri'] = $itemUrl;
+
+				if($fullContent) {
+					$itemHtml = getSimpleHTMLDOM($itemUrl);
+					if($itemHtml) {
+						$item['content'] = $itemHtml->find('div.now-p__content-text', 0)->innertext;
+					}
+				}
+
+				$this->items[] = $item;
+				$limit++;
+			}
+		}
+	}
+
+	public function getIcon() {
+		return static::URI . '/local/images/favicon.ico';
+	}
+
+	private function ruMonthsToEn($date) {
+		$ruMonths = array(
+			'Января', 'Февраля', 'Марта', 'Апреля', 'Мая', 'Июня',
+			'Июля', 'Августа', 'Сентября', 'Октября', 'Ноября', 'Декабря' );
+		$enMonths = array(
+			'January', 'February', 'March', 'April', 'May', 'June',
+			'July', 'August', 'September', 'October', 'November', 'December' );
+		return str_replace($ruMonths, $enMonths, $date);
+	}
+}

bridges/AmazonPriceTrackerBridge.php

@@ -49,6 +49,8 @@ class AmazonPriceTrackerBridge extends BridgeAbstract {
 		'.a-color-price',
 	);
 
+	const WHITESPACE = " \t\n\r\0\x0B\xC2\xA0";
+
 	protected $title;
 
 	/**
@@ -154,6 +156,22 @@ EOT;
 		return false;
 	}
 
+	private function scrapePriceTwister($html) {
+		$str = $html->find('.twister-plus-buying-options-price-data', 0);
+
+		$data = json_decode($str->innertext, true);
+		if(count($data) === 1) {
+			$data = $data[0];
+			return array(
+				'displayPrice' => $data['displayPrice'],
+				'currency' => $data['currency'],
+				'shipping' => '0',
+			);
+		}
+
+		return false;
+	}
+
 	private function scrapePriceGeneric($html) {
 		$priceDiv = null;
 
@@ -168,12 +186,11 @@ EOT;
 			return false;
 		}
 
-		$priceString = $priceDiv->plaintext;
-
-		preg_match('/[\d.,]+/', $priceString, $matches);
+		$priceString = str_replace(str_split(self::WHITESPACE), '', $priceDiv->plaintext);
+		preg_match('/(\d+\.\d{0,2})/', $priceString, $matches);
 
 		$price = $matches[0];
-		$currency = trim(str_replace($price, '', $priceString), " \t\n\r\0\x0B\xC2\xA0");
+		$currency = str_replace($price, '', $priceString);
 
 		if ($price != null && $currency != null) {
 			return array(
@@ -186,6 +203,21 @@ EOT;
 		return false;
 	}
 
+	private function renderContent($image, $data) {
+		$price = $data['displayPrice'];
+		if (!$price) {
+			$price = "{$data['price']} {$data['currency']}";
+		}
+
+		$html = "$image<br>Price: $price";
+
+		if ($data['shipping'] !== '0') {
+			$html .= "<br>Shipping: {$data['shipping']} {$data['currency']}</br>";
+		}
+
+		return $html;
+	}
+
 	/**
 	 * Scrape method for Amazon product page
 	 * @return [type] [description]
@@ -195,20 +227,16 @@ EOT;
 		$this->title = $this->getTitle($html);
 		$imageTag = $this->getImage($html);
 
-		$data = $this->scrapePriceFromMetrics($html) ?: $this->scrapePriceGeneric($html);
+		$data = $this->scrapePriceGeneric($html);
 
 		$item = array(
 			'title' 	=> $this->title,
 			'uri' 		=> $this->getURI(),
-			'content' 	=> "$imageTag<br/>Price: {$data['price']} {$data['currency']}",
+			'content' 	=> $this->renderContent($imageTag, $data),
 			// This is to ensure that feed readers notice the price change
 			'uid'		=> md5($data['price'])
 		);
 
-		if ($data['shipping'] !== '0') {
-			$item['content'] .= "<br>Shipping: {$data['shipping']} {$data['currency']}</br>";
-		}
-
 		$this->items[] = $item;
 	}
 }

bridges/AnimeUltimeBridge.php

@@ -37,9 +37,11 @@ class AnimeUltimeBridge extends BridgeAbstract {
 		$processedOK = 0;
 		foreach (array($thismonth, $lastmonth) as $requestFilter) {
 
-			//Retrive page contents
 			$url = self::URI . 'history-0-1/' . $requestFilter;
-			$html = getSimpleHTMLDOM($url);
+			$html = getContents($url);
+			// Convert html from iso-8859-1 => utf8
+			$html = utf8_encode($html);
+			$html = str_get_html($html);
 
 			//Relases are sorted by day : process each day individually
 			foreach($html->find('div.history', 0)->find('h3') as $daySection) {
@@ -87,6 +89,8 @@ class AnimeUltimeBridge extends BridgeAbstract {
 
 							// Retrieve description from description page
 							$html_item = getContents($item_uri);
+							// Convert html from iso-8859-1 => utf8
+							$html_item = utf8_encode($html_item);
 							$item_description = substr(
 								$html_item,
 								strpos($html_item, 'class="principal_contain" align="center">') + 41

bridges/AppleAppStoreBridge.php

@@ -94,6 +94,7 @@ class AppleAppStoreBridge extends BridgeAbstract {
 
 		$headers = array(
 			"Authorization: Bearer $token",
+			'Origin: https://apps.apple.com',
 		);
 
 		$json = json_decode(getContents($uri, $headers), true);

bridges/Arte7Bridge.php

@@ -10,52 +10,60 @@ class Arte7Bridge extends BridgeAbstract {
 	const API_TOKEN = 'Nzc1Yjc1ZjJkYjk1NWFhN2I2MWEwMmRlMzAzNjI5NmU3NWU3ODg4ODJjOWMxNTMxYzEzZGRjYjg2ZGE4MmIwOA';
 
 	const PARAMETERS = array(
-		'Catégorie (Français)' => array(
-			'catfr' => array(
+		'global' => [
+			'video_duration_filter' => [
+				'name' => 'Exclude short videos',
+				'type' => 'checkbox',
+				'title' => 'Exclude videos that are shorter than 3 minutes',
+				'defaultValue'	=> false,
+			],
+		],
+		'Category' => array(
+			'lang' => array(
 				'type' => 'list',
-				'name' => 'Catégorie',
+				'name' => 'Language',
 				'values' => array(
-					'Toutes les vidéos (français)' => null,
-					'Actu & société' => 'ACT',
-					'Séries & fiction' => 'SER',
-					'Cinéma' => 'CIN',
-					'Arts & spectacles classiques' => 'ARS',
-					'Culture pop' => 'CPO',
-					'Découverte' => 'DEC',
-					'Histoire' => 'HIST',
-					'Science' => 'SCI',
-					'Autre' => 'AUT'
-				)
-			)
-		),
-		'Collection (Français)' => array(
-			'colfr' => array(
-				'name' => 'Collection id',
-				'required' => true,
+					'Français' => 'fr',
+					'Deutsch' => 'de',
+					'English' => 'en',
+					'Español' => 'es',
+					'Polski' => 'pl',
+					'Italiano' => 'it'
+				),
 				'title' => 'ex. RC-014095 pour https://www.arte.tv/fr/videos/RC-014095/blow-up/',
 				'exampleValue'	=> 'RC-014095'
-			)
-		),
-		'Catégorie (Allemand)' => array(
-			'catde' => array(
+			),
+			'cat' => array(
 				'type' => 'list',
-				'name' => 'Catégorie',
+				'name' => 'Category',
 				'values' => array(
-					'Alle Videos (deutsch)' => null,
-					'Aktuelles & Gesellschaft' => 'ACT',
-					'Fernsehfilme & Serien' => 'SER',
-					'Kino' => 'CIN',
-					'Kunst & Kultur' => 'ARS',
-					'Popkultur & Alternativ' => 'CPO',
-					'Entdeckung' => 'DEC',
-					'Geschichte' => 'HIST',
-					'Wissenschaft' => 'SCI',
-					'Sonstiges' => 'AUT'
+					'All videos' => null,
+					'News & society' => 'ACT',
+					'Series & fiction' => 'SER',
+					'Cinema' => 'CIN',
+					'Culture' => 'ARS',
+					'Culture pop' => 'CPO',
+					'Discovery' => 'DEC',
+					'History' => 'HIST',
+					'Science' => 'SCI',
+					'Other' => 'AUT'
 				)
-			)
+			),
 		),
-		'Collection (Allemand)' => array(
-			'colde' => array(
+		'Collection' => array(
+			'lang' => array(
+				'type' => 'list',
+				'name' => 'Language',
+				'values' => array(
+					'Français' => 'fr',
+					'Deutsch' => 'de',
+					'English' => 'en',
+					'Español' => 'es',
+					'Polski' => 'pl',
+					'Italiano' => 'it'
+				)
+			),
+			'col' => array(
 				'name' => 'Collection id',
 				'required' => true,
 				'title' => 'ex. RC-014095 pour https://www.arte.tv/de/videos/RC-014095/blow-up/',
@@ -65,26 +73,19 @@ class Arte7Bridge extends BridgeAbstract {
 	);
 
 	public function collectData(){
+		$lang = $this->getInput('lang');
 		switch($this->queriedContext) {
-		case 'Catégorie (Français)':
-			$category = $this->getInput('catfr');
-			$lang = 'fr';
+		case 'Category':
+			$category = $this->getInput('cat');
+			$collectionId = null;
 			break;
-		case 'Collection (Français)':
-			$lang = 'fr';
-			$collectionId = $this->getInput('colfr');
-			break;
-		case 'Catégorie (Allemand)':
-			$category = $this->getInput('catde');
-			$lang = 'de';
-			break;
-		case 'Collection (Allemand)':
-			$lang = 'de';
-			$collectionId = $this->getInput('colde');
+		case 'Collection':
+			$collectionId = $this->getInput('col');
+			$category = null;
 			break;
 		}
 
-		$url = 'https://api.arte.tv/api/opa/v3/videos?sort=-lastModified&limit=10&language='
+		$url = 'https://api.arte.tv/api/opa/v3/videos?sort=-lastModified&limit=15&language='
 			. $lang
 			. ($category != null ? '&category.code=' . $category : '')
 			. ($collectionId != null ? '&collections.collectionId=' . $collectionId : '');
@@ -97,6 +98,11 @@ class Arte7Bridge extends BridgeAbstract {
 		$input_json = json_decode($input, true);
 
 		foreach($input_json['videos'] as $element) {
+			$durationSeconds = $element['durationSeconds'];
+
+			if ($this->getInput('video_duration_filter') && $durationSeconds < 60 * 3) {
+				continue;
+			}
 
 			$item = array();
 			$item['uri'] = $element['url'];
@@ -108,10 +114,10 @@ class Arte7Bridge extends BridgeAbstract {
 			if(!empty($element['subtitle']))
 				$item['title'] = $element['title'] . ' | ' . $element['subtitle'];
 
-			$item['duration'] = round((int)$element['durationSeconds'] / 60);
+			$durationMinutes = round((int)$durationSeconds / 60);
 			$item['content'] = $element['teaserText']
 			. '<br><br>'
-			. $item['duration']
+			. $durationMinutes
 			. 'min<br><a href="'
 			. $item['uri']
 			. '"><img src="'

bridges/AssociatedPressNewsBridge.php

@@ -0,0 +1,270 @@
+<?php
+class AssociatedPressNewsBridge extends BridgeAbstract {
+	const NAME = 'Associated Press News Bridge';
+	const URI = 'https://apnews.com/';
+	const DESCRIPTION = 'Returns newest articles by topic';
+	const MAINTAINER = 'VerifiedJoseph';
+	const PARAMETERS = array(
+		'Standard Topics' => array(
+			'topic' => array(
+				'name' => 'Topic',
+				'type' => 'list',
+				'values' => array(
+					'AP Top News' => 'apf-topnews',
+					'Sports' => 'apf-sports',
+					'Entertainment' => 'apf-entertainment',
+					'Oddities' => 'apf-oddities',
+					'Travel' => 'apf-Travel',
+					'Technology' => 'apf-technology',
+					'Lifestyle' => 'apf-lifestyle',
+					'Business' => 'apf-business',
+					'U.S. News' => 'apf-usnews',
+					'Health' => 'apf-Health',
+					'Science' => 'apf-science',
+					'World News' => 'apf-WorldNews',
+					'Politics' => 'apf-politics',
+					'Religion' => 'apf-religion',
+					'Photo Galleries' => 'PhotoGalleries',
+					'Fact Checks' => 'APFactCheck',
+					'Videos' => 'apf-videos',
+				),
+				'defaultValue' => 'apf-topnews',
+			),
+		),
+		'Custom Topic' => array(
+			'topic' => array(
+				'name' => 'Topic',
+				'type' => 'text',
+				'required' => true,
+				'exampleValue' => 'europe'
+			),
+		)
+	);
+
+	const CACHE_TIMEOUT = 900; // 15 mins
+
+	private $detectParamRegex = '/^https?:\/\/(?:www\.)?apnews\.com\/(?:[tag|hub]+\/)?([\w-]+)$/';
+	private $tagEndpoint = 'https://afs-prod.appspot.com/api/v2/feed/tag?tags=';
+	private $feedName = '';
+
+	public function detectParameters($url) {
+		$params = array();
+
+		if(preg_match($this->detectParamRegex, $url, $matches) > 0) {
+			$params['topic'] = $matches[1];
+			$params['context'] = 'Custom Topic';
+			return $params;
+		}
+
+		return null;
+	}
+
+	public function collectData() {
+		switch($this->getInput('topic')) {
+			case 'Podcasts':
+				returnClientError('Podcasts topic feed is not supported');
+				break;
+			case 'PressReleases':
+				returnClientError('PressReleases topic feed is not supported');
+				break;
+			default:
+				$this->collectCardData();
+		}
+	}
+
+	public function getURI() {
+		if (!is_null($this->getInput('topic'))) {
+			return self::URI . $this->getInput('topic');
+		}
+
+		return parent::getURI();
+	}
+
+	public function getName() {
+		if (!empty($this->feedName)) {
+			return $this->feedName . ' - Associated Press';
+		}
+
+		return parent::getName();
+	}
+
+	private function getTagURI() {
+		if (!is_null($this->getInput('topic'))) {
+			return $this->tagEndpoint . $this->getInput('topic');
+		}
+
+		return parent::getURI();
+	}
+
+	private function collectCardData() {
+		$json = getContents($this->getTagURI())
+			or returnServerError('Could not request: ' . $this->getTagURI());
+
+		$tagContents = json_decode($json, true);
+
+		if (empty($tagContents['tagObjs'])) {
+			returnClientError('Topic not found: ' . $this->getInput('topic'));
+		}
+
+		$this->feedName = $tagContents['tagObjs'][0]['name'];
+
+		foreach ($tagContents['cards'] as $card) {
+			$item = array();
+
+			 // skip hub peeks & Notifications
+			if ($card['cardType'] == 'Hub Peek' || $card['cardType'] == 'Notification') {
+				continue;
+			}
+
+			$storyContent = $card['contents'][0];
+
+			switch($storyContent['contentType']) {
+				case 'web': // Skip link only content
+					continue 2;
+
+				case 'video':
+					$html = $this->processVideo($storyContent);
+
+					$item['enclosures'][] = 'https://storage.googleapis.com/afs-prod/media/'
+						. $storyContent['media'][0]['id'] . '/800.jpeg';
+					break;
+				default:
+					if (empty($storyContent['storyHTML'])) { // Skip if no storyHTML
+						continue 2;
+					}
+
+					$html = defaultLinkTo($storyContent['storyHTML'], self::URI);
+					$html = str_get_html($html);
+
+					$this->processMediaPlaceholders($html, $storyContent['id']);
+					$this->processHubLinks($html, $storyContent);
+					$this->processIframes($html);
+
+					if (!is_null($storyContent['leadPhotoId'])) {
+						$item['enclosures'][] = 'https://storage.googleapis.com/afs-prod/media/'
+							. $storyContent['leadPhotoId'] . '/800.jpeg';
+					}
+			}
+
+			$item['title'] = $card['contents'][0]['headline'];
+			$item['uri'] = self::URI . $card['shortId'];
+
+			if ($card['contents'][0]['localLinkUrl']) {
+				$item['uri'] = $card['contents'][0]['localLinkUrl'];
+			}
+
+			$item['timestamp'] = $storyContent['published'];
+
+			if (is_null($storyContent['bylines']) === false) {
+				// Remove 'By' from the bylines
+				if (substr($storyContent['bylines'], 0, 2) == 'By') {
+					$item['author'] = ltrim($storyContent['bylines'], 'By ');
+				} else {
+					$item['author'] = $storyContent['bylines'];
+				}
+			}
+
+			$item['content'] = $html;
+
+			foreach ($storyContent['tagObjs'] as $tag) {
+				$item['categories'][] = $tag['name'];
+			}
+
+			$this->items[] = $item;
+
+			if (count($this->items) >= 15) {
+				break;
+			}
+		}
+	}
+
+	private function processMediaPlaceholders($html, $id) {
+
+		if ($html->find('div.media-placeholder', 0)) {
+			// Fetch page content
+			$json = getContents('https://afs-prod.appspot.com/api/v2/content/' . $id);
+			$storyContent = json_decode($json, true);
+
+			foreach ($html->find('div.media-placeholder') as $div) {
+				$key = array_search($div->id, $storyContent['mediumIds']);
+
+				if (!isset($storyContent['media'][$key])) {
+					continue;
+				}
+
+				$media = $storyContent['media'][$key];
+
+				if ($media['type'] === 'Photo') {
+					$mediaUrl = $media['gcsBaseUrl'] . $media['imageRenderedSizes'][0] . $media['imageFileExtension'];
+					$mediaCaption = $media['caption'];
+
+					$div->outertext = <<<EOD
+	<figure><img loading="lazy" src="{$mediaUrl}"/><figcaption>{$mediaCaption}</figcaption></figure>
+EOD;
+				}
+
+				if ($media['type'] === 'YouTube') {
+					$div->outertext = <<<EOD
+	<iframe src="https://www.youtube.com/embed/{$media['externalId']}" width="560" height="315">
+	</iframe>
+EOD;
+				}
+			}
+		}
+	}
+
+	/*
+		Create full coverage links (HubLinks)
+	*/
+	private function processHubLinks($html, $storyContent) {
+
+		if (!empty($storyContent['richEmbeds'])) {
+			foreach ($storyContent['richEmbeds'] as $embed) {
+
+				if ($embed['type'] === 'Hub Link') {
+					$url = self::URI . $embed['tag']['id'];
+					$div = $html->find('div[id=' . $embed['id'] . ']', 0);
+
+					if ($div) {
+						$div->outertext = <<<EOD
+<p><a href="{$url}">{$embed['calloutText']} {$embed['displayName']}</a></p>
+EOD;
+					}
+				}
+			}
+		}
+	}
+
+	private function processVideo($storyContent) {
+		$video = $storyContent['media'][0];
+
+		if ($video['type'] === 'YouTube') {
+			$url = 'https://www.youtube.com/embed/' . $video['externalId'];
+			$html = <<<EOD
+<iframe width="560" height="315" src="{$url}" frameborder="0" allowfullscreen></iframe>
+EOD;
+		} else {
+			$html = <<<EOD
+<video controls poster="https://storage.googleapis.com/afs-prod/media/{$video['id']}/800.jpeg" preload="none">
+	<source src="{$video['gcsBaseUrl']} {$video['videoRenderedSizes'][0]} {$video['videoFileExtension']}" type="video/mp4">
+</video>
+EOD;
+		}
+
+		return $html;
+	}
+
+	// Remove datawrapper.dwcdn.net iframes and related javaScript
+	private function processIframes($html) {
+
+		foreach ($html->find('iframe') as $index => $iframe) {
+			if (preg_match('/datawrapper\.dwcdn\.net/', $iframe->src)) {
+				$iframe->outertext = '';
+
+				if ($html->find('script', $index)) {
+					$html->find('script', $index)->outertext = '';
+				}
+			}
+		}
+	}
+}

bridges/BinanceBridge.php

@@ -1,41 +1,18 @@
 <?php
 class BinanceBridge extends BridgeAbstract {
-	const NAME = 'Binance';
-	const URI = 'https://www.binance.com';
-	const DESCRIPTION = 'Subscribe to the Binance blog or the Binance Zendesk announcements.';
+	const NAME = 'Binance Blog';
+	const URI = 'https://www.binance.com/en/blog';
+	const DESCRIPTION = 'Subscribe to the Binance blog.';
 	const MAINTAINER = 'thefranke';
 	const CACHE_TIMEOUT = 3600; // 1h
 
-	const PARAMETERS = array( array(
-		'category' => array(
-			'name' => 'category',
-			'type' => 'list',
-			'exampleValue' => 'Blog',
-			'title' => 'Select a category',
-			'values' => array(
-				'Blog' => 'Blog',
-				'Announcements' => 'Announcements'
-			)
-		)
-	));
-
 	public function getIcon() {
 		return 'https://bin.bnbstatic.com/static/images/common/favicon.ico';
 	}
 
-	public function getName() {
-		return self::NAME . ' ' . $this->getInput('category');
-	}
-
-	public function getURI() {
-		if ($this->getInput('category') == 'Blog')
-			return self::URI . '/en/blog';
-		else
-			return 'https://binance.zendesk.com/hc/en-us/categories/115000056351-Announcements';
-	}
-
-	protected function collectBlogData() {
-		$html = getSimpleHTMLDOM($this->getURI());
+	public function collectData() {
+		$html = getSimpleHTMLDOM(self::URI)
+			or returnServerError('Could not fetch Binance blog data.');
 
 		$appData = $html->find('script[id="__APP_DATA"]');
 		$appDataJson = json_decode($appData[0]->innertext);
@@ -61,37 +38,4 @@ class BinanceBridge extends BridgeAbstract {
 				break;
 		}
 	}
-
-	protected function collectAnnouncementData() {
-		$html = getSimpleHTMLDOM($this->getURI());
-
-		foreach($html->find('a.article-list-link') as $a) {
-			$title = $a->innertext;
-			$uri = 'https://binance.zendesk.com' . $a->href;
-
-			$full = getSimpleHTMLDOMCached($uri);
-			$content = $full->find('div.article-body', 0);
-			$date = $full->find('time', 0)->getAttribute('datetime');
-
-			$item = array();
-
-			$item['title'] = $title;
-			$item['uri'] = $uri;
-			$item['timestamp'] = strtotime($date);
-			$item['author'] = 'Binance';
-			$item['content'] = $content;
-
-			$this->items[] = $item;
-
-			if (count($this->items) >= 10)
-				break;
-		}
-	}
-
-	public function collectData() {
-		if ($this->getInput('category') == 'Blog')
-			$this->collectBlogData();
-		else
-			$this->collectAnnouncementData();
-	}
 }

bridges/BooruprojectBridge.php

@@ -1,7 +1,7 @@
 <?php
-require_once('GelbooruBridge.php');
+require_once('DanbooruBridge.php');
 
-class BooruprojectBridge extends GelbooruBridge {
+class BooruprojectBridge extends DanbooruBridge {
 
 	const MAINTAINER = 'mitsukarenai';
 	const NAME = 'Booruproject';
@@ -11,6 +11,7 @@ class BooruprojectBridge extends GelbooruBridge {
 		'global' => array(
 			'p' => array(
 				'name' => 'page',
+				'defaultValue' => 0,
 				'type' => 'number'
 			),
 			't' => array(
@@ -29,8 +30,30 @@ class BooruprojectBridge extends GelbooruBridge {
 		)
 	);
 
+	const PATHTODATA = '.thumb';
+	const IDATTRIBUTE = 'id';
+	const TAGATTRIBUTE = 'title';
 	const PIDBYPAGE = 20;
 
+	protected function getFullURI(){
+		return $this->getURI()
+		. 'index.php?page=post&s=list&pid='
+		. ($this->getInput('p') ? ($this->getInput('p') - 1) * static::PIDBYPAGE : '')
+		. '&tags=' . urlencode($this->getInput('t'));
+	}
+
+	protected function getTags($element){
+		$tags = parent::getTags($element);
+		$tags = explode(' ', $tags);
+
+		// Remove statistics from the tags list (identified by colon)
+		foreach($tags as $key => $tag) {
+			if(strpos($tag, ':') !== false) unset($tags[$key]);
+		}
+
+		return implode(' ', $tags);
+	}
+
 	public function getURI(){
 		if(!is_null($this->getInput('i'))) {
 			return 'https://' . $this->getInput('i') . '.booru.org/';

bridges/CBCEditorsBlogBridge.php

@@ -0,0 +1,36 @@
+<?php
+class CBCEditorsBlogBridge extends BridgeAbstract {
+
+	const MAINTAINER = 'quickwick';
+	const NAME = 'CBC Editors Blog';
+	const URI = 'https://www.cbc.ca/news/editorsblog';
+	const DESCRIPTION = 'Recent CBC Editor\'s Blog posts';
+
+	public function collectData(){
+		$html = getSimpleHTMLDOM(self::URI);
+
+		// Loop on each blog post entry
+		foreach($html->find('div.contentListCards', 0)->find('a[data-test=type-story]') as $element) {
+			$headline = ($element->find('.headline', 0))->innertext;
+			$timestamp = ($element->find('time', 0))->datetime;
+			$articleUri = 'https://www.cbc.ca' . $element->href;
+			$summary = ($element->find('div.description', 0))->innertext;
+			$thumbnailUris = ($element->find('img[loading=lazy]', 0))->srcset;
+			$thumbnailUri = rtrim(explode(',', $thumbnailUris)[0], ' 300w');
+
+			// Fill item
+			$item = array();
+			$item['uri'] = $articleUri;
+			$item['id'] = $item['uri'];
+			$item['timestamp'] = $timestamp;
+			$item['title'] = $headline;
+			$item['content'] = '<img src="'
+			. $thumbnailUri . '" /><br>' . $summary;
+			$item['author'] = 'Editor\'s Blog';
+
+			if(isset($item['title'])) {
+				$this->items[] = $item;
+			}
+		}
+	}
+}

bridges/CourrierInternationalBridge.php

@@ -15,11 +15,10 @@ class CourrierInternationalBridge extends FeedExpander {
 		$item = parent::parseItem($feedItem);
 
 		$articlePage = getSimpleHTMLDOMCached($feedItem->link);
-		$content = $articlePage->find('.article-text', 0);
-		if(!$content) {
-			$content = $articlePage->find('.depeche-text', 0);
+		$content = $articlePage->find('.article-text, depeche-text', 0);
+		if (!$content) {
+			return $item;
 		}
-
 		$item['content'] = sanitize($content);
 
 		return $item;

bridges/CraigslistBridge.php

@@ -74,6 +74,7 @@ class CraigslistBridge extends BridgeAbstract {
 		foreach($results as $post) {
 
 			// Skip "nearby results" banner and results
+			// This only appears when searchNearby is not specified
 			if ($post->tag == 'h4') {
 				break;
 			}
@@ -86,7 +87,8 @@ class CraigslistBridge extends BridgeAbstract {
 			$item['timestamp'] = $post->find('.result-date', 0)->datetime;
 			$item['uid'] = $heading->id;
 			$item['content'] = $post->find('.result-price', 0)->plaintext . ' '
-							 . $post->find('.result-hood', 0)->plaintext;
+							 // Find the location (local and nearby results if searchNearby=1)
+							 . $post->find('.result-hood, span.nearby', 0)->plaintext;
 
 			$images = $post->find('.result-image[data-ids]', 0);
 			if (!is_null($images)) {

bridges/CryptomeBridge.php

@@ -4,7 +4,7 @@ class CryptomeBridge extends BridgeAbstract {
 	const MAINTAINER = 'BoboTiG';
 	const NAME = 'Cryptome';
 	const URI = 'https://cryptome.org/';
-	const CACHE_TIMEOUT = 21600; //6h
+	const CACHE_TIMEOUT = 21600; // 6h
 	const DESCRIPTION = 'Returns the N most recent documents.';
 	const PARAMETERS = array( array(
 		'n' => array(

bridges/DanbooruBridge.php

@@ -57,79 +57,10 @@ class DanbooruBridge extends BridgeAbstract {
 	}
 
 	public function collectData(){
-		$content = getContents($this->getFullURI());
-
-		$html = Fix_Simple_Html_Dom::str_get_html($content);
+		$html = getSimpleHTMLDOMCached($this->getFullURI());
 
 		foreach($html->find(static::PATHTODATA) as $element) {
 			$this->items[] = $this->getItemFromElement($element);
 		}
 	}
 }
-
-/**
- * This class is a monkey patch to 'extend' simplehtmldom to recognize <source>
- * tags (HTML5) as self closing tag. This patch should be removed once
- * simplehtmldom was fixed. This seems to be a issue with more tags:
- * https://sourceforge.net/p/simplehtmldom/bugs/83/
- *
- * The tag itself is valid according to Mozilla:
- *
- * The HTML <picture> element serves as a container for zero or more <source>
- * elements and one <img> element to provide versions of an image for different
- * display device scenarios. The browser will consider each of the child <source>
- * elements and select one corresponding to the best match found; if no matches
- * are found among the <source> elements, the file specified by the <img>
- * element's src attribute is selected. The selected image is then presented in
- * the space occupied by the <img> element.
- *
- * -- https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture
- *
- * Notice: This class uses parts of the original simplehtmldom, adjusted to pass
- * the guidelines of RSS-Bridge (formatting)
- */
-final class Fix_Simple_Html_Dom extends simple_html_dom {
-
-	/* copy from simple_html_dom, added 'source' at the end */
-	protected $self_closing_tags = array(
-		'img' => 1,
-		'br' => 1,
-		'input' => 1,
-		'meta' => 1,
-		'link' => 1,
-		'hr' => 1,
-		'base' => 1,
-		'embed' => 1,
-		'spacer' => 1,
-		'source' => 1
-	);
-
-	/* copy from simplehtmldom, changed 'simple_html_dom' to 'Fix_Simple_Html_Dom' */
-	public static function str_get_html($str,
-	$lowercase = true,
-	$forceTagsClosed = true,
-	$target_charset = DEFAULT_TARGET_CHARSET,
-	$stripRN = true,
-	$defaultBRText = DEFAULT_BR_TEXT,
-	$defaultSpanText = DEFAULT_SPAN_TEXT)
-	{
-		$dom = new Fix_Simple_Html_Dom(null,
-		$lowercase,
-		$forceTagsClosed,
-		$target_charset,
-		$stripRN,
-		$defaultBRText,
-		$defaultSpanText);
-
-		if (empty($str) || strlen($str) > MAX_FILE_SIZE) {
-
-			$dom->clear();
-			return false;
-
-		}
-
-		$dom->load($str, $lowercase, $stripRN);
-
-		return $dom;
-	}
-}

bridges/DaveRamseyBlogBridge.php

@@ -1,23 +0,0 @@
-<?php
-
-class DaveRamseyBlogBridge extends BridgeAbstract {
-	const MAINTAINER = 'johnpc';
-	const NAME = 'Dave Ramsey Blog';
-	const URI = 'https://www.daveramsey.com/blog';
-	const CACHE_TIMEOUT = 7200; // 2h
-	const DESCRIPTION = 'Returns blog posts from daveramsey.com';
-
-	public function collectData()
-	{
-		$html = getSimpleHTMLDOM(self::URI);
-
-		foreach ($html->find('.Post') as $element) {
-			$this->items[] = array(
-				'uri' => 'https://www.daveramsey.com' . $element->find('header > a', 0)->href,
-				'title' => $element->find('header > h2 > a', 0)->plaintext,
-				'tags' => $element->find('.Post-topic', 0)->plaintext,
-				'content' => $element->find('.Post-body', 0)->plaintext,
-			);
-		}
-	}
-}

bridges/DownDetectorBridge.php

@@ -1,194 +0,0 @@
-<?php
-class DownDetectorBridge extends BridgeAbstract {
-	const MAINTAINER = 'teromene';
-	const NAME = 'DownDetector Bridge';
-	const URI = 'https://downdetector.com/';
-	const DESCRIPTION = 'Returns most recent downtimes from DownDetector';
-	const CACHE_TIMEOUT = 300; // 5 min
-
-	const PARAMETERS = array(
-		'All Websites' => array(
-			'country' => array(
-				'type' => 'list',
-				'name' => 'Country',
-				'values' => array(
-					'Argentina' => 'https://downdetector.com.ar',
-					'Australia' => 'https://downdetector.com.au',
-					'België' => 'https://allestoringen.be',
-					'Brasil' => 'https://downdetector.com.br',
-					'Canada' => 'https://downdetector.ca',
-					'Chile' => 'https://downdetector.cl',
-					'Colombia' => 'https://downdetector.com.co',
-					'Danmark' => 'https://downdetector.dk',
-					'Deutschland' => 'https://allestörungen.de',
-					'Ecuador' => 'https://downdetector.ec',
-					'España' => 'https://downdetector.es',
-					'France' => 'https://downdetector.fr',
-					'Hong Kong' => 'https://downdetector.hk',
-					'Hrvatska' => 'https://downdetector.hr',
-					'India' => 'https://downdetector.in',
-					'Indonesia' => 'https://downdetector.id',
-					'Ireland' => 'https://downdetector.ie',
-					'Italia' => 'https://downdetector.it',
-					'Magyarország' => 'https://downdetector.hu',
-					'Malaysia' => 'https://downdetector.my',
-					'México' => 'https://downdetector.mx',
-					'Nederland' => 'https://allestoringen.nl',
-					'New Zealand' => 'https://downdetector.co.nz',
-					'Norge' => 'https://downdetector.no',
-					'Pakistan' => 'https://downdetector.pk',
-					'Perú' => 'https://downdetector.pe',
-					'Pilipinas' => 'https://downdetector.ph',
-					'Polska' => 'https://downdetector.pl',
-					'Portugal' => 'https://downdetector.pt',
-					'România' => 'https://downdetector.ro',
-					'Schweiz' => 'https://allestörungen.ch',
-					'Singapore' => 'https://downdetector.sg',
-					'Slovensko' => 'https://downdetector.sk',
-					'South Africa' => 'https://downdetector.co.za',
-					'Suomi' => 'https://downdetector.fi',
-					'Sverige' => 'https://downdetector.se',
-					'Türkiye' => 'https://downdetector.web.tr',
-					'UAE' => 'https://downdetector.ae',
-					'UK' => 'https://downdetector.co.uk',
-					'United States' => 'https://downdetector.com',
-					'Österreich' => 'https://allestörungen.at',
-					'Česko' => 'https://downdetector.cz',
-					'Ελλάς' => 'https://downdetector.gr',
-					'Россия' => 'https://downdetector.ru',
-					'日本' => 'https://downdetector.jp'
-				)
-			)
-		),
-		'Specific Website' => array(
-			'page' => array(
-				'type' => 'text',
-				'name' => 'Status page',
-				'required' => true,
-				'exampleValue' => 'https://downdetector.com/status/rainbow-six',
-				'title' => 'URL of a DownDetector status page e.g: https://downdetector.com/status/rainbow-six/',
-			)
-		),
-	);
-
-	private $hostname = '';
-	private $statusPageId = '';
-	private $feedname = '';
-
-	private	$statusUrlRegex = '/\/([a-zA-z0-9ö.]+)\/(?:statu(?:s|t)|problemas?|nu-merge
-|(?:feil-)?problem(y|i)?(?:-storningar)?(?:-fejl)?|stoerung|durum|storing|fora-do-ar|ne-rabotaet
-|masalah|shougai|ei-toimi)\/([a-zA-Z0-9-]+)/';
-
-	public function collectData(){
-		if ($this->queriedContext == 'Specific Website') {
-			preg_match($this->statusUrlRegex, $this->getInput('page'), $match)
-				or returnClientError('Given URL does not seem to at a DownDetector status page!');
-
-			$this->hostname = $match[1];
-			$this->statusPageId = $match[3];
-		}
-
-		$html = getSimpleHTMLDOM($this->getURI() . '/archive/')
-			or returnClientError('Could not request website!.');
-
-		$html = defaultLinkTo($html, $this->getURI());
-
-		if ($this->getInput('page')) {
-			$this->feedname = $html->find('li.breadcrumb-item.active', 0)->plaintext;
-		}
-
-		$table = $html->find('table.table-striped', 0);
-
-		if ($table) {
-			foreach ($table->find('tr') as $event) {
-				$td = $event->find('td', 0);
-
-				if (is_null($td)) {
-					continue;
-				}
-
-				$item['uri'] = $event->find('td', 0)->find('a', 0)->href;
-				$item['title'] = $event->find('td', 0)->find('a', 0)->plaintext .
-					'(' . trim($event->find('td', 1)->plaintext) . ' ' . trim($event->find('td', 2)->plaintext) . ')';
-				$item['content'] = 'User reports indicate problems at' . $event->find('td', 0)->find('a', 0)->plaintext .
-					' since ' . $event->find('td', 2)->plaintext;
-				$item['timestamp'] = $this->formatDate(
-					trim($event->find('td', 1)->plaintext),
-					trim($event->find('td', 2)->plaintext)
-				);
-
-				$this->items[] = $item;
-			}
-		}
-	}
-
-	public function getURI() {
-		if($this->getInput('country')) {
-			return $this->getInput('country');
-		}
-
-		if ($this->getInput('page')) {
-			return 'https://' . $this->hostname . '/status/' . $this->statusPageId;
-		}
-
-		return self::URI;
-	}
-
-	public function getName() {
-		if($this->getInput('country')) {
-			$country = $this->getCountry($this->getInput('country'));
-			return $country . ' - DownDetector';
-		}
-
-		if ($this->getInput('page')) {
-			$country = $this->getCountry($this->hostname);
-			return $this->feedname . ' - ' . $country . '  - DownDetector';
-		}
-
-		return self::NAME;
-	}
-
-	private function formatDate($date, $time) {
-		switch($this->getCountry()) {
-			case 'Australia':
-			case 'UK':
-				$date = DateTime::createFromFormat('d/m/Y', $date);
-				return $date->format('Y-m-d') . $time;
-			case 'Brasil':
-			case 'Chile':
-			case 'Colombia':
-			case 'Ecuador':
-			case 'España':
-			case 'Italia':
-			case 'Perú':
-			case 'Portugal':
-				$date = DateTime::createFromFormat('d/m/Y', $date);
-				return $date->format('Y-m-d') . $time;
-			case 'Magyarország':
-				$date = DateTime::createFromFormat('Y.m.d.', $date);
-				return $date->format('Y-m-d') . $time;
-			default:
-				return $date . $time;
-		}
-	}
-
-	private function getCountry() {
-		if($this->getInput('country')) {
-			$input = $this->getInput('country');
-		}
-
-		if ($this->getInput('page')) {
-			if (empty($this->hostname)) {
-				return 'N/A';
-			}
-
-			$input = 'https://' . $this->hostname;
-		}
-
-		$parameters = $this->getParameters();
-		$countryValues = array_flip($parameters['All Websites']['country']['values']);
-		$country = $countryValues[$input];
-
-		return $country;
-	}
-}

bridges/EZTVBridge.php

@@ -3,65 +3,109 @@ class EZTVBridge extends BridgeAbstract {
 
 	const MAINTAINER = 'alexAubin';
 	const NAME = 'EZTV';
-	const URI = 'https://eztv.ch/';
-	const DESCRIPTION = 'Returns list of *recent* torrents for a specific show
-on EZTV. Get showID from URLs in https://eztv.ch/shows/showID/show-full-name.';
+	const URI = 'https://eztv.re/';
+	const DESCRIPTION = 'Returns list of torrents for specific show(s)
+on EZTV. Get IMDB IDs from IMDB.';
 
-	const PARAMETERS = array( array(
-		'i' => array(
-			'name' => 'Show ids',
-			'exampleValue' => '1017,249',
-			'title' => 'One of more showids as a comma separated list',
-			'required' => true
+	const PARAMETERS = array(
+		array(
+			'ids' => array(
+				'name' => 'Show IMDB IDs',
+				'exampleValue' => '8740790,1733785',
+				'required' => true,
+				'title' => 'One or more IMDB show IDs (can be found in the IMDB show URL)'
+			),
+			'no480' => array(
+				'name' => 'No 480p',
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude 480p torrents'
+			),
+			'no720' => array(
+				'name' => 'No 720p',
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude 720p torrents'
+			),
+			'no1080' => array(
+				'name' => 'No 1080p',
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude 1080p torrents'
+			),
+			'no2160' => array(
+				'name' => 'No 2160p',
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude 2160p torrents'
+			),
+			'noUnknownRes' => array(
+				'name' => 'No Unknown resolution',
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude unknown resolution torrents'
+			),
 		)
-	));
+	);
+
+	// Shamelessly lifted from https://stackoverflow.com/a/2510459
+	protected function formatBytes($bytes, $precision = 2) {
+		$units = array('B', 'KB', 'MB', 'GB', 'TB');
+
+		$bytes = max($bytes, 0);
+		$pow = floor(($bytes ? log($bytes) : 0) / log(1024));
+		$pow = min($pow, count($units) - 1);
+		$bytes /= pow(1024, $pow);
+
+		return round($bytes, $precision) . ' ' . $units[$pow];
+	}
+
+	protected function getItemFromTorrent($torrent){
+		$item = array();
+		$item['uri'] = $torrent->episode_url;
+		$item['author'] = $torrent->imdb_id;
+		$item['timestamp'] = date('d F Y H:i:s', $torrent->date_released_unix);
+		$item['title'] = $torrent->title;
+		$item['enclosures'][] = $torrent->torrent_url;
+
+		$thumbnailUri = 'https:' . $torrent->small_screenshot;
+		$torrentSize = $this->formatBytes($torrent->size_bytes);
+
+		$item['content'] = $torrent->filename . '<br>File size: '
+		. $torrentSize . '<br><a href="' . $torrent->magnet_url
+		. '">magnet link</a><br><a href="' . $torrent->torrent_url
+		. '">torrent link</a><br><img src="' . $thumbnailUri . '" />';
+
+		return $item;
+	}
+
+	private static function compareDate($torrent1, $torrent2) {
+		return (strtotime($torrent1['timestamp']) < strtotime($torrent2['timestamp']) ? 1 : -1);
+	}
 
 	public function collectData(){
+		$showIds = explode(',', $this->getInput('ids'));
 
-		// Make timestamp from relative released time in table
-		function makeTimestamp($relativeReleaseTime){
+		foreach($showIds as $showId) {
+			$eztvUri = $this->getURI() . 'api/get-torrents?imdb_id=' . $showId;
+			$content = getContents($eztvUri);
+			$torrents = json_decode($content)->torrents;
+			foreach($torrents as $torrent) {
+				$title = $torrent->title;
+				$regex480 = '/480p/';
+				$regex720 = '/720p/';
+				$regex1080 = '/1080p/';
+				$regex2160 = '/2160p/';
+				$regexUnknown = '/(480p|720p|1080p|2160p)/';
+				// Skip unwanted resolution torrents
+				if ((preg_match($regex480, $title) === 1 && $this->getInput('no480'))
+				|| (preg_match($regex720, $title) === 1 && $this->getInput('no720'))
+				|| (preg_match($regex1080, $title) === 1 && $this->getInput('no1080'))
+				|| (preg_match($regex2160, $title) === 1 && $this->getInput('no2160'))
+				|| (preg_match($regexUnknown, $title) !== 1 && $this->getInput('noUnknownRes'))) {
+					continue;
+				}
 
-			$relativeDays = 0;
-			$relativeHours = 0;
-
-			foreach(explode(' ', $relativeReleaseTime) as $relativeTimeElement) {
-				if(substr($relativeTimeElement, -1) == 'd') $relativeDays = substr($relativeTimeElement, 0, -1);
-				if(substr($relativeTimeElement, -1) == 'h') $relativeHours = substr($relativeTimeElement, 0, -1);
-			}
-			return mktime(date('h') - $relativeHours, 0, 0, date('m'), date('d') - $relativeDays, date('Y'));
-		}
-
-		// Loop on show ids
-		$showList = explode(',', $this->getInput('i'));
-		foreach($showList as $showID) {
-
-			// Get show page
-			$html = getSimpleHTMLDOM(self::URI . 'shows/' . rawurlencode($showID) . '/');
-
-			// Loop on each element that look like an episode entry...
-			foreach($html->find('.forum_header_border') as $element) {
-
-				// Filter entries that are not episode entries
-				$ep = $element->find('td', 1);
-				if(empty($ep)) continue;
-				$epinfo = $ep->find('.epinfo', 0);
-				$released = $element->find('td', 3);
-				if(empty($epinfo)) continue;
-				if(empty($released->plaintext)) continue;
-
-				// Filter entries that are older than 1 week
-				if($released->plaintext == '&gt;1 week') continue;
-
-				// Fill item
-				$item = array();
-				$item['uri'] = self::URI . $epinfo->href;
-				$item['id'] = $item['uri'];
-				$item['timestamp'] = makeTimestamp($released->plaintext);
-				$item['title'] = $epinfo->plaintext;
-				$item['content'] = $epinfo->alt;
-				if(isset($item['title']))
-					$this->items[] = $item;
+				$this->items[] = $this->getItemFromTorrent($torrent);
 			}
 		}
+
+		// Sort all torrents in array by date
+		usort($this->items, array('EZTVBridge', 'compareDate'));
 	}
 }

bridges/ElsevierBridge.php

@@ -1,7 +1,7 @@
 <?php
 class ElsevierBridge extends BridgeAbstract {
 
-	const MAINTAINER = 'Pierre Mazière';
+	const MAINTAINER = 'dvikan';
 	const NAME = 'Elsevier journals recent articles';
 	const URI = 'https://www.journals.elsevier.com/';
 	const CACHE_TIMEOUT = 43200; //12h
@@ -16,63 +16,26 @@ class ElsevierBridge extends BridgeAbstract {
 		)
 	));
 
-	// Extracts the list of names from an article as string
-	private function extractArticleName($article){
-		$names = $article->find('small', 0);
-		if($names)
-			return trim($names->plaintext);
-		return '';
-	}
-
-	// Extracts the timestamp from an article
-	private function extractArticleTimestamp($article){
-		$time = $article->find('.article-info', 0);
-		if($time) {
-			$timestring = trim($time->plaintext);
-			/*
-				The format depends on the age of an article:
-				- Available online 29 July 2016
-				- July 2016
-				- May–June 2016
-			*/
-			if(preg_match('/\S*(\d+\s\S+\s\d{4})/ims', $timestring, $matches)) {
-				return strtotime($matches[0]);
-			} elseif (preg_match('/[A-Za-z]+\-([A-Za-z]+\s\d{4})/ims', $timestring, $matches)) {
-				return strtotime($matches[0]);
-			} elseif (preg_match('/([A-Za-z]+\s\d{4})/ims', $timestring, $matches)) {
-				return strtotime($matches[0]);
-			} else {
-				return 0;
-			}
-		}
-		return 0;
-	}
-
-	// Extracts the content from an article
-	private function extractArticleContent($article){
-		$content = $article->find('.article-content', 0);
-		if($content) {
-			return trim($content->plaintext);
-		}
-		return '';
-	}
-
-	public function getIcon() {
-		return 'https://cdn.elsevier.io/verona/includes/favicons/favicon-32x32.png';
-	}
-
 	public function collectData(){
-		$uri = self::URI . $this->getInput('j') . '/recent-articles/';
-		$html = getSimpleHTMLDOM($uri);
+		// Not all journals have the /recent-articles page
+		$url = sprintf('https://www.journals.elsevier.com/%s/recent-articles/', $this->getInput('j'));
+		$html = getSimpleHTMLDOM($url);
 
-		foreach($html->find('.pod-listing') as $article) {
-			$item = array();
-			$item['uri'] = $article->find('.pod-listing-header>a', 0)->getAttribute('href') . '?np=y';
-			$item['title'] = $article->find('.pod-listing-header>a', 0)->plaintext;
-			$item['author'] = $this->extractArticleName($article);
-			$item['timestamp'] = $this->extractArticleTimestamp($article);
-			$item['content'] = $this->extractArticleContent($article);
+		foreach($html->find('article') as $recentArticle) {
+			$item = [];
+			$item['uri'] = $recentArticle->find('a', 0)->getAttribute('href');
+			$item['title'] = $recentArticle->find('h2', 0)->plaintext;
+			$item['author'] = $recentArticle->find('p > span', 0)->plaintext;
+			$publicationDateString = trim($recentArticle->find('p > span', 1)->plaintext);
+			$publicationDate = DateTimeImmutable::createFromFormat('F d, Y', $publicationDateString);
+			if ($publicationDate) {
+				$item['timestamp'] = $publicationDate->getTimestamp();
+			}
 			$this->items[] = $item;
 		}
 	}
+
+	public function getIcon(): string {
+		return 'https://cdn.elsevier.io/verona/includes/favicons/favicon-32x32.png';
+	}
 }

bridges/EtsyBridge.php

@@ -22,12 +22,10 @@ class EtsyBridge extends BridgeAbstract {
 (anything after ?search=<your search query>)',
 				'exampleValue' => '&explicit=1&locationQuery=2921044'
 			),
-			'showimage' => array(
-				'name' => 'Show image in content',
+			'hideimage' => array(
+				'name' => 'Hide image in content',
 				'type' => 'checkbox',
-				'required' => false,
-				'title' => 'Activate to show the image in the content',
-				'defaultValue' => 'checked'
+				'title' => 'Activate to hide the image in the content',
 			)
 		)
 	);
@@ -35,29 +33,29 @@ class EtsyBridge extends BridgeAbstract {
 	public function collectData(){
 		$html = getSimpleHTMLDOM($this->getURI());
 
-		$results = $html->find('li.block-grid-item');
+		$results = $html->find('li.wt-list-unstyled');
 
 		foreach($results as $result) {
-			// Skip banner cards (ads for categories)
-			if($result->find('span.ad-indicator'))
+			// Remove Lazy loading
+			if($result->find('.wt-skeleton-ui', 0))
 				continue;
 
 			$item = array();
 
 			$item['title'] = $result->find('a', 0)->title;
 			$item['uri'] = $result->find('a', 0)->href;
-			$item['author'] = $result->find('p.text-gray-lighter', 0)->plaintext;
+			$item['author'] = $result->find('p.wt-text-gray > span', 2)->plaintext;
 
 			$item['content'] = '<p>'
-			. $result->find('span.currency-value', 0)->plaintext . ' '
 			. $result->find('span.currency-symbol', 0)->plaintext
+			. $result->find('span.currency-value', 0)->plaintext
 			. '</p><p>'
 			. $result->find('a', 0)->title
 			. '</p>';
 
-			$image = $result->find('img.display-block', 0)->src;
+			$image = $result->find('img.wt-display-block', 0)->src;
 
-			if($this->getInput('showimage')) {
+			if(!$this->getInput('hideimage')) {
 				$item['content'] .= '<img src="' . $image . '">';
 			}
 

bridges/FB2Bridge.php

@@ -229,7 +229,7 @@ EOD
 
 		$ctx = stream_context_create(array(
 			'http' => array(
-				'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0',
+				'user_agent' => Configuration::getConfig('http', 'useragent'),
 				'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
 				)
 			)
@@ -254,7 +254,7 @@ EOD
 
 		$context = stream_context_create(array(
 			'http' => array(
-				'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0',
+				'user_agent' => Configuration::getConfig('http', 'useragent'),
 				'header' => 'Cookie: ' . $cookies
 				)
 			)

bridges/FeedReducerBridge.php

@@ -0,0 +1,60 @@
+<?php
+class FeedReducerBridge extends FeedExpander {
+
+	const MAINTAINER = 'mdemoss';
+	const NAME = 'Feed Reducer';
+	const URI = 'http://github.com/RSS-Bridge/rss-bridge/';
+	const DESCRIPTION = 'Choose a percentage of a feed you want to see.';
+	const PARAMETERS = array( array(
+		'url' => array(
+			'name' => 'Feed URI',
+			'exampleValue' => 'https://lorem-rss.herokuapp.com/feed?length=42',
+			'required' => true
+		),
+		'percentage' => array(
+			'name' => 'percentage',
+			'type' => 'number',
+			'exampleValue' => 50,
+			'required' => true
+		)
+	));
+	const CACHE_TIMEOUT = 3600;
+
+	public function collectData(){
+		if(preg_match('#^http(s?)://#i', $this->getInput('url'))) {
+			$this->collectExpandableDatas($this->getInput('url'));
+		} else {
+			throw new Exception('URI must begin with http(s)://');
+		}
+	}
+
+	public function getItems(){
+		$filteredItems = array();
+		$intPercentage = (int)preg_replace('/[^0-9]/', '', $this->getInput('percentage'));
+
+		foreach ($this->items as $thisItem) {
+			// The URL is included in the hash:
+			//  - so you can change the output by adding a local-part to the URL
+			//  - so items with the same URI in different feeds won't be correlated
+
+			// $pseudoRandomInteger will be a 16 bit unsigned int mod 100.
+			// This won't be uniformly distributed 1-100, but should be close enough.
+
+			$pseudoRandomInteger = unpack(
+				'S', // unsigned 16-bit int
+				hash( 'sha256', $thisItem['uri'] . '::' . $this->getInput('url'), true )
+			)[1] % 100;
+
+			if ($pseudoRandomInteger < $intPercentage) {
+				$filteredItems[] = $thisItem;
+			}
+		}
+
+		return $filteredItems;
+	}
+
+	public function getName(){
+		$trimmedPercentage = preg_replace('/[^0-9]/', '', $this->getInput('percentage') ?? '');
+		return parent::getName() . ' [' . $trimmedPercentage . '%]';
+	}
+}

bridges/FilterBridge.php

@@ -11,6 +11,8 @@ class FilterBridge extends FeedExpander {
 	const PARAMETERS = array(array(
 		'url' => array(
 			'name' => 'Feed URL',
+			'type'	=> 'text',
+			'defaultValue' => 'https://lorem-rss.herokuapp.com/feed?unit=day',
 			'required' => true,
 		),
 		'filter' => array(

bridges/FindACrewBridge.php

@@ -62,11 +62,11 @@ class FindACrewBridge extends BridgeAbstract {
 		foreach ($annonces as $annonce) {
 			$item = array();
 
-			$link = parent::getURI() . $annonce->find('.lst-ctrls a', 0)->href;
+			$link = parent::getURI() . $annonce->find('.lstsum-btn-con a', 0)->href;
 			$htmlDetail = getSimpleHTMLDOMCached($link . '?mdl=2'); // add ?mdl=2 for xhr content not full html page
 
 			$img = parent::getURI() . $htmlDetail->find('img.img-responsive', 0)->getAttribute('src');
-			$item['title'] = $annonce->find('.lst-tags span', 0)->plaintext;
+			$item['title'] = $htmlDetail->find('div.label-account', 0)->plaintext;
 			$item['uri'] = $link;
 			$content = $htmlDetail->find('.panel-body div.clearfix.row > div', 1)->innertext;
 			$content .= $htmlDetail->find('.panel-body > div', 1)->innertext;

bridges/FlickrBridge.php

@@ -51,6 +51,15 @@ class FlickrBridge extends BridgeAbstract {
 				'title' => 'Insert username (as shown in the address bar)',
 				'exampleValue' => 'flickr'
 			),
+			'content' => array(
+				'name' => 'Content',
+				'type' => 'list',
+				'values' => array(
+					'Uploads' => 'uploads',
+					'Favorites' => 'faves',
+				),
+				'defaultValue' => 'uploads',
+			),
 			'media' => array(
 				'name' => 'Media',
 				'type' => 'list',
@@ -156,8 +165,14 @@ class FlickrBridge extends BridgeAbstract {
 					. '&sort=' . $this->getInput('sort') . '&media=' . $this->getInput('media');
 				break;
 			case 'By username':
-				return self::URI . 'search/?user_id=' . urlencode($this->getInput('u'))
-					. '&sort=' . $this->getInput('sort') . '&media=' . $this->getInput('media');
+				$uri = self::URI . 'search/?user_id=' . urlencode($this->getInput('u'))
+					. '&sort=date-posted-desc&media=' . $this->getInput('media');
+
+				if ($this->getInput('content') === 'faves') {
+					return $uri . '&faves=1';
+				}
+
+				return $uri;
 				break;
 
 			default:
@@ -175,6 +190,11 @@ class FlickrBridge extends BridgeAbstract {
 				return $this->getInput('q') . ' - keyword - ' . self::NAME;
 				break;
 			case 'By username':
+
+				if ($this->getInput('content') === 'faves') {
+					return $this->username . ' - favorites - ' . self::NAME;
+				}
+
 				return $this->username . ' - ' . self::NAME;
 				break;
 

bridges/FootitoBridge.php

@@ -1,74 +0,0 @@
-<?php
-class FootitoBridge extends BridgeAbstract {
-
-	const MAINTAINER = 'superbaillot.net';
-	const NAME = 'Footito';
-	const URI = 'http://www.footito.fr/';
-	const DESCRIPTION = 'Footito';
-
-	public function collectData(){
-		$html = getSimpleHTMLDOM(self::URI);
-
-		foreach($html->find('div.post') as $element) {
-			$item = array();
-
-			$content = trim($element->innertext);
-			$content = str_replace(
-				'<img',
-				"<img style='float : left;'",
-				$content );
-
-			$content = str_replace(
-				'class="logo"',
-				"style='float : left;'",
-				$content );
-
-			$content = str_replace(
-				'class="contenu"',
-				"style='margin-left : 60px;'",
-				$content );
-
-			$content = str_replace(
-				'class="responsive-comment"',
-				"style='border-top : 1px #DDD solid; background-color : white; padding : 10px;'",
-				$content );
-
-			$content = str_replace(
-				'class="jaime"',
-				"style='display : none;'",
-				$content );
-
-			$content = str_replace(
-				'class="auteur-event responsive"',
-				"style='display : none;'",
-				$content );
-
-			$content = str_replace(
-				'class="report-abuse-button"',
-				"style='display : none;'",
-				$content );
-
-			$content = str_replace(
-				'class="reaction clearfix"',
-				"style='margin : 10px 0px; padding : 5px; border-bottom : 1px #DDD solid;'",
-				$content );
-
-			$content = str_replace(
-				'class="infos"',
-				"style='font-size : 0.7em;'",
-				$content );
-
-			$item['content'] = $content;
-
-			$title = $element->find('.contenu .texte ', 0)->plaintext;
-			$item['title'] = $title;
-
-			$info = $element->find('div.infos', 0);
-
-			$item['timestamp'] = strtotime($info->find('time', 0)->datetime);
-			$item['author'] = $info->find('a.auteur', 0)->plaintext;
-
-			$this->items[] = $item;
-		}
-	}
-}

bridges/FurAffinityUserBridge.php

@@ -3,21 +3,23 @@ class FurAffinityUserBridge extends BridgeAbstract {
 	const NAME = 'FurAffinity User Gallery';
 	const URI = 'https://www.furaffinity.net';
 	const MAINTAINER = 'CyberJacob';
+	const DESCRIPTION = 'See https://rss-bridge.github.io/rss-bridge/Bridge_Specific/Furaffinityuser.html for explanation';
 	const PARAMETERS = array(
 		array(
 			'searchUsername' => array(
 				'name' => 'Search Username',
 				'type' => 'text',
 				'required' => true,
-				'title' => 'Username to fetch the gallery for'
+				'title' => 'Username to fetch the gallery for',
+				'exampleValue' => 'armundy',
 			),
-			'loginUsername' => array(
-				'name' => 'Login Username',
+			'aCookie' => array(
+				'name' => 'Login cookie \'a\'',
 				'type' => 'text',
 				'required' => true
 			),
-			'loginPassword' => array(
-				'name' => 'Login Password',
+			'bCookie' => array(
+				'name' => 'Login cookie \'b\'',
 				'type' => 'text',
 				'required' => true
 			)
@@ -25,10 +27,12 @@ class FurAffinityUserBridge extends BridgeAbstract {
 	);
 
 	public function collectData() {
-		$cookies = self::login();
+		$opt = array(CURLOPT_COOKIE => 'b=' . $this->getInput('bCookie') . '; a=' . $this->getInput('aCookie'));
+
 		$url = self::URI . '/gallery/' . $this->getInput('searchUsername');
 
-		$html = getSimpleHTMLDOM($url, $cookies);
+		$html = getSimpleHTMLDOM($url, array(), $opt)
+			or returnServerError('Could not load the user\'s gallery page.');
 
 		$submissions = $html->find('section[id=gallery-gallery]', 0)->find('figure');
 		foreach($submissions as $submission) {
@@ -51,59 +55,4 @@ class FurAffinityUserBridge extends BridgeAbstract {
 	public function getURI() {
 		return self::URI . '/user/' . $this->getInput('searchUsername');
 	}
-
-	private function login() {
-		$ch = curl_init(self::URI . '/login/');
-		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
-
-		curl_setopt($ch, CURLOPT_USERAGENT, ini_get('user_agent'));
-		curl_setopt($ch, CURLOPT_ENCODING, '');
-		curl_setopt($ch, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
-
-		$fields = implode('&', array(
-			'action=login',
-			'retard_protection=1',
-			'name=' . urlencode($this->getInput('loginUsername')),
-			'pass=' . urlencode($this->getInput('loginPassword')),
-			'login=Login to Faraffinity'
-		));
-
-		curl_setopt($ch, CURLOPT_POST, 5);
-		curl_setopt($ch, CURLOPT_POSTFIELDS, $fields);
-
-		if(defined('PROXY_URL') && !defined('NOPROXY')) {
-			curl_setopt($ch, CURLOPT_PROXY, PROXY_URL);
-		}
-
-		curl_setopt($ch, CURLOPT_HEADER, true);
-		curl_setopt($ch, CURLINFO_HEADER_OUT, true);
-
-		$data = curl_exec($ch);
-
-		$errorCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
-
-		$curlError = curl_error($ch);
-		$curlErrno = curl_errno($ch);
-		$curlInfo = curl_getinfo($ch);
-
-		if($data === false)
-			fDebug::log("Cant't download {$url} cUrl error: {$curlError} ({$curlErrno})");
-
-		curl_close($ch);
-
-		if($errorCode != 200) {
-			returnServerError(error_get_last());
-		} else {
-			preg_match_all('/^Set-Cookie:\s*([^;]*)/mi', $data, $matches);
-			$cookies = array();
-
-			foreach($matches[1] as $item) {
-				parse_str($item, $cookie);
-				$cookies = array_merge($cookies, $cookie);
-			}
-
-			return $cookies;
-		}
-	}
 }

bridges/GelbooruBridge.php

@@ -1,35 +1,87 @@
 <?php
-require_once('DanbooruBridge.php');
-
-class GelbooruBridge extends DanbooruBridge {
+class GelbooruBridge extends BridgeAbstract {
 
 	const MAINTAINER = 'mitsukarenai';
 	const NAME = 'Gelbooru';
-	const URI = 'http://gelbooru.com/';
+	const URI = 'https://gelbooru.com/';
 	const DESCRIPTION = 'Returns images from given page';
 
-	const PATHTODATA = '.thumb';
-	const IDATTRIBUTE = 'id';
-	const TAGATTRIBUTE = 'title';
-
-	const PIDBYPAGE = 63;
+	const PARAMETERS = array(
+		'global' => array(
+			'p' => array(
+				'name' => 'page',
+				'defaultValue' => 0,
+				'type' => 'number'
+			),
+			't' => array(
+				'name' => 'tags',
+				'exampleValue' => 'pinup',
+				'title' => 'Tags to search for'
+			),
+			'l' => array(
+				'name' => 'limit',
+				'exampleValue' => 100,
+				'title' => 'How many posts to retrieve (hard limit of 1000)'
+			)
+		),
+		0 => array()
+	);
 
 	protected function getFullURI(){
 		return $this->getURI()
-		. 'index.php?page=post&s=list&pid='
-		. ($this->getInput('p') ? ($this->getInput('p') - 1) * static::PIDBYPAGE : '')
+		. 'index.php?&page=dapi&s=post&q=index&json=1&pid=' . $this->getInput('p')
+		. '&limit=' . $this->getInput('l')
 		. '&tags=' . urlencode($this->getInput('t'));
 	}
 
-	protected function getTags($element){
-		$tags = parent::getTags($element);
-		$tags = explode(' ', $tags);
+	/*
+	This function is superfluous for GelbooruBridge, but useful
+	for Bridges that inherit from it
+	*/
+	protected function buildThumbnailURI($element){
+		return $this->getURI() . 'thumbnails/' . $element->directory
+		. '/thumbnail_' . $element->md5 . '.jpg';
+	}
 
-		// Remove statistics from the tags list (identified by colon)
-		foreach($tags as $key => $tag) {
-			if(strpos($tag, ':') !== false) unset($tags[$key]);
+	protected function getItemFromElement($element){
+		$item = array();
+		$item['uri'] = $this->getURI() . 'index.php?page=post&s=view&id='
+		. $element->id;
+		$item['postid'] = $element->id;
+		$item['author'] = $element->owner;
+		$item['timestamp'] = date('d F Y H:i:s', $element->change);
+		$item['tags'] = $element->tags;
+		$item['title'] = $this->getName() . ' | ' . $item['postid'];
+
+		if (isset($element->preview_url)) {
+			$thumbnailUri = $element->preview_url;
+		} else{
+			$thumbnailUri = $this->buildThumbnailURI($element);
 		}
 
-		return implode(' ', $tags);
+		$item['content'] = '<a href="' . $item['uri'] . '"><img src="'
+		. $thumbnailUri	. '" /></a><br><br><b>Tags:</b> '
+		. $item['tags'] . '<br><br>' . $item['timestamp'];
+
+		return $item;
+	}
+
+	public function collectData(){
+		$content = getContents($this->getFullURI());
+
+		// Most other Gelbooru-based boorus put their content in the root of
+		// the JSON. This check is here for Bridges that inherit from this one
+		$posts = json_decode($content);
+		if (isset($posts->post)) {
+			$posts = $posts->post;
+		}
+
+		if (is_null($posts)) {
+			returnServerError('No posts found.');
+		}
+
+		foreach($posts as $post) {
+			$this->items[] = $this->getItemFromElement($post);
+		}
 	}
 }

bridges/GithubIssueBridge.php

@@ -4,7 +4,7 @@ class GithubIssueBridge extends BridgeAbstract {
 	const MAINTAINER = 'Pierre Mazière';
 	const NAME = 'Github Issue';
 	const URI = 'https://github.com/';
-	const CACHE_TIMEOUT = 600; // 10min
+	const CACHE_TIMEOUT = 0; // 10min
 	const DESCRIPTION = 'Returns the issues or comments of an issue of a github project';
 
 	const PARAMETERS = array(
@@ -24,6 +24,11 @@ class GithubIssueBridge extends BridgeAbstract {
 			'c' => array(
 				'name' => 'Show Issues Comments',
 				'type' => 'checkbox'
+			),
+			'q' => array(
+				'name' => 'Search Query',
+				'defaultValue' => 'is:issue is:open sort:updated-desc',
+				'required' => true
 			)
 		),
 		'Issue comments' => array(
@@ -40,7 +45,6 @@ class GithubIssueBridge extends BridgeAbstract {
 	const BRIDGE_OPTIONS = array(0 => 'Project Issues', 1 => 'Issue comments');
 	const URL_PATH = 'issues';
 	const SEARCH_QUERY_PATH = 'issues';
-	const SEARCH_QUERY = '?q=is%3Aissue+sort%3Aupdated-desc';
 
 	public function getName(){
 		$name = $this->getInput('u') . '/' . $this->getInput('p');
@@ -67,7 +71,7 @@ class GithubIssueBridge extends BridgeAbstract {
 			if($this->queriedContext === static::BRIDGE_OPTIONS[1]) {
 				$uri .= static::URL_PATH . '/' . $this->getInput('i');
 			} else {
-				$uri .= static::SEARCH_QUERY_PATH . static::SEARCH_QUERY;
+				$uri .= static::SEARCH_QUERY_PATH . '?q=' . urlencode($this->getInput('q'));
 			}
 			return $uri;
 		}
@@ -128,9 +132,8 @@ class GithubIssueBridge extends BridgeAbstract {
 
 		$author = $comment->find('.author', 0)->plaintext;
 
-		$title .= ' / ' . trim(
-			$comment->find('.timeline-comment-header-text', 0)->plaintext
-		);
+		$header = $comment->find('.timeline-comment-header > h3', 0);
+		$title .= ' / ' . ($header ? $header->plaintext : 'Activity');
 
 		$time = $comment->find('relative-time', 0);
 		if ($time === null) {

bridges/GithubPullRequestBridge.php

@@ -22,6 +22,11 @@ class GitHubPullRequestBridge extends GithubIssueBridge {
 			'c' => array(
 				'name' => 'Show Pull Request Comments',
 				'type' => 'checkbox'
+			),
+			'q' => array(
+				'name' => 'Search Query',
+				'defaultValue' => 'is:pr is:open sort:created-desc',
+				'required' => true
 			)
 		),
 		'Pull Request comments' => array(
@@ -37,5 +42,4 @@ class GitHubPullRequestBridge extends GithubIssueBridge {
 	const BRIDGE_OPTIONS = array(0 => 'Project Pull Requests', 1 => 'Pull Request comments');
 	const URL_PATH = 'pull';
 	const SEARCH_QUERY_PATH = 'pulls';
-	const SEARCH_QUERY = '?q=is%3Apr+sort%3Acreated-desc';
 }

bridges/GithubTrendingBridge.php

@@ -8,41 +8,39 @@ class GithubTrendingBridge extends BridgeAbstract {
 	const CACHE_TIMEOUT = 43200; // 12hr
 	const DESCRIPTION = 'See what the GitHub community is most excited repos.';
 	const PARAMETERS = array(
+	// If you are changing context and/or parameter names, change them also in getName().
 		'By language' => array(
 			'language' => array(
 				'name' => 'Select language',
 				'type' => 'list',
 				'values' => array(
 					'All languages' => '',
-					'C++' => 'c++',
 					'HTML' => 'html',
-					'Java' => 'java',
-					'JavaScript' => 'javascript',
 					'PHP' => 'php',
 					'Python' => 'python',
-					'Ruby' => 'ruby',
-					'Unknown languages' => 'unknown languages',
-					'1C Enterprise' => '1c enterprise',
+					'Shell' => 'shell',
+					'Unknown languages' => 'unknown',
+					'1C Enterprise' => '1c-enterprise',
 					'4D' => '4d',
 					'ABAP' => 'abap',
 					'ABNF' => 'abnf',
 					'ActionScript' => 'actionscript',
 					'Ada' => 'ada',
-					'Adobe Font Metrics' => 'adobe font metrics',
+					'Adobe Font Metrics' => 'adobe-font-metrics',
 					'Agda' => 'agda',
-					'AGS Script' => 'ags script',
+					'AGS Script' => 'ags-script',
 					'Alloy' => 'alloy',
-					'Alpine Abuild' => 'alpine abuild',
-					'Altium Designer' => 'altium designer',
+					'Alpine Abuild' => 'alpine-abuild',
+					'Altium Designer' => 'altium-designer',
 					'AMPL' => 'ampl',
 					'AngelScript' => 'angelscript',
-					'Ant Build System' => 'ant build system',
+					'Ant Build System' => 'ant-build-system',
 					'ANTLR' => 'antlr',
 					'ApacheConf' => 'apacheconf',
 					'Apex' => 'apex',
-					'API Blueprint' => 'api blueprint',
+					'API Blueprint' => 'api-blueprint',
 					'APL' => 'apl',
-					'Apollo Guidance Computer' => 'apollo guidance computer',
+					'Apollo Guidance Computer' => 'apollo-guidance-computer',
 					'AppleScript' => 'applescript',
 					'Arc' => 'arc',
 					'AsciiDoc' => 'asciidoc',
@@ -71,11 +69,12 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Brightscript' => 'brightscript',
 					'Zeek' => 'zeek',
 					'C' => 'c',
-					'C#' => 'c#',
+					'C#' => 'c%23',		// already URL encoded
 					'C++' => 'c++',
 					'C-ObjDump' => 'c-objdump',
-					'C2hs Haskell' => 'c2hs haskell',
-					'Cabal Config' => 'cabal config',
+					'C2hs Haskell' => 'c2hs-haskell',
+					'Cabal Config' => 'cabal-config',
+					'Cap\'n Proto' => 'cap\'n-proto',
 					'CartoCSS' => 'cartocss',
 					'Ceylon' => 'ceylon',
 					'Chapel' => 'chapel',
@@ -87,18 +86,18 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Click' => 'click',
 					'CLIPS' => 'clips',
 					'Clojure' => 'clojure',
-					'Closure Templates' => 'closure templates',
-					'Cloud Firestore Security Rules' => 'cloud firestore security rules',
+					'Closure Templates' => 'closure-templates',
+					'Cloud Firestore Security Rules' => 'cloud-firestore-security-rules',
 					'CMake' => 'cmake',
 					'COBOL' => 'cobol',
 					'CodeQL' => 'codeql',
 					'CoffeeScript' => 'coffeescript',
 					'ColdFusion' => 'coldfusion',
-					'ColdFusion CFC' => 'coldfusion cfc',
+					'ColdFusion CFC' => 'coldfusion-cfc',
 					'COLLADA' => 'collada',
-					'Common Lisp' => 'common lisp',
-					'Common Workflow Language' => 'common workflow language',
-					'Component Pascal' => 'component pascal',
+					'Common Lisp' => 'common-lisp',
+					'Common Workflow Language' => 'common-workflow-language',
+					'Component Pascal' => 'component-pascal',
 					'CoNLL-U' => 'conll-u',
 					'Cool' => 'cool',
 					'Coq' => 'coq',
@@ -107,28 +106,28 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Crystal' => 'crystal',
 					'CSON' => 'cson',
 					'Csound' => 'csound',
-					'Csound Document' => 'csound document',
-					'Csound Score' => 'csound score',
+					'Csound Document' => 'csound-document',
+					'Csound Score' => 'csound-score',
 					'CSS' => 'css',
 					'CSV' => 'csv',
 					'Cuda' => 'cuda',
-					'cURL Config' => 'curl config',
+					'cURL Config' => 'curl-config',
 					'CWeb' => 'cweb',
 					'Cycript' => 'cycript',
 					'Cython' => 'cython',
 					'D' => 'd',
 					'D-ObjDump' => 'd-objdump',
-					'Darcs Patch' => 'darcs patch',
+					'Darcs Patch' => 'darcs-patch',
 					'Dart' => 'dart',
 					'DataWeave' => 'dataweave',
 					'desktop' => 'desktop',
 					'Dhall' => 'dhall',
 					'Diff' => 'diff',
-					'DIGITAL Command Language' => 'digital command language',
+					'DIGITAL Command Language' => 'digital-command-language',
 					'dircolors' => 'dircolors',
-					'DirectX 3D File' => 'directx 3d file',
+					'DirectX 3D File' => 'directx-3d-file',
 					'DM' => 'dm',
-					'DNS Zone' => 'dns zone',
+					'DNS Zone' => 'dns-zone',
 					'Dockerfile' => 'dockerfile',
 					'Dogescript' => 'dogescript',
 					'DTrace' => 'dtrace',
@@ -138,29 +137,29 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Easybuild' => 'easybuild',
 					'EBNF' => 'ebnf',
 					'eC' => 'ec',
-					'Ecere Projects' => 'ecere projects',
+					'Ecere Projects' => 'ecere-projects',
 					'ECL' => 'ecl',
 					'ECLiPSe' => 'eclipse',
 					'EditorConfig' => 'editorconfig',
-					'Edje Data Collection' => 'edje data collection',
+					'Edje Data Collection' => 'edje-data-collection',
 					'edn' => 'edn',
 					'Eiffel' => 'eiffel',
 					'EJS' => 'ejs',
 					'Elixir' => 'elixir',
 					'Elm' => 'elm',
-					'Emacs Lisp' => 'emacs lisp',
+					'Emacs Lisp' => 'emacs-lisp',
 					'EmberScript' => 'emberscript',
 					'EML' => 'eml',
 					'EQ' => 'eq',
 					'Erlang' => 'erlang',
-					'F#' => 'f#',
+					'F#' => 'f%23',		// already URL encoded
 					'F*' => 'f*',
 					'Factor' => 'factor',
 					'Fancy' => 'fancy',
 					'Fantom' => 'fantom',
 					'Faust' => 'faust',
-					'FIGlet Font' => 'figlet font',
-					'Filebench WML' => 'filebench wml',
+					'FIGlet Font' => 'figlet-font',
+					'Filebench WML' => 'filebench-wml',
 					'Filterscript' => 'filterscript',
 					'fish' => 'fish',
 					'FLUX' => 'flux',
@@ -170,25 +169,25 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'FreeMarker' => 'freemarker',
 					'Frege' => 'frege',
 					'G-code' => 'g-code',
-					'Game Maker Language' => 'game maker language',
+					'Game Maker Language' => 'game-maker-language',
 					'GAML' => 'gaml',
 					'GAMS' => 'gams',
 					'GAP' => 'gap',
-					'GCC Machine Description' => 'gcc machine description',
+					'GCC Machine Description' => 'gcc-machine-description',
 					'GDB' => 'gdb',
 					'GDScript' => 'gdscript',
 					'Genie' => 'genie',
 					'Genshi' => 'genshi',
-					'Gentoo Ebuild' => 'gentoo ebuild',
-					'Gentoo Eclass' => 'gentoo eclass',
-					'Gerber Image' => 'gerber image',
-					'Gettext Catalog' => 'gettext catalog',
+					'Gentoo Ebuild' => 'gentoo-ebuild',
+					'Gentoo Eclass' => 'gentoo-eclass',
+					'Gerber Image' => 'gerber-image',
+					'Gettext Catalog' => 'gettext-catalog',
 					'Gherkin' => 'gherkin',
-					'Git Attributes' => 'git attributes',
-					'Git Config' => 'git config',
+					'Git Attributes' => 'git-attributes',
+					'Git Config' => 'git-config',
 					'GLSL' => 'glsl',
 					'Glyph' => 'glyph',
-					'Glyph Bitmap Distribution Format' => 'glyph bitmap distribution format',
+					'Glyph Bitmap Distribution Format' => 'glyph-bitmap-distribution-format',
 					'GN' => 'gn',
 					'Gnuplot' => 'gnuplot',
 					'Go' => 'go',
@@ -196,12 +195,12 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Gosu' => 'gosu',
 					'Grace' => 'grace',
 					'Gradle' => 'gradle',
-					'Grammatical Framework' => 'grammatical framework',
-					'Graph Modeling Language' => 'graph modeling language',
+					'Grammatical Framework' => 'grammatical-framework',
+					'Graph Modeling Language' => 'graph-modeling-language',
 					'GraphQL' => 'graphql',
-					'Graphviz (DOT)' => 'graphviz (dot)',
+					'Graphviz (DOT)' => 'graphviz-(dot)',
 					'Groovy' => 'groovy',
-					'Groovy Server Pages' => 'groovy server pages',
+					'Groovy Server Pages' => 'groovy-server-pages',
 					'Hack' => 'hack',
 					'Haml' => 'haml',
 					'Handlebars' => 'handlebars',
@@ -213,7 +212,6 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'HiveQL' => 'hiveql',
 					'HLSL' => 'hlsl',
 					'HolyC' => 'holyc',
-					'HTML' => 'html',
 					'HTML+Django' => 'html+django',
 					'HTML+ECR' => 'html+ecr',
 					'HTML+EEX' => 'html+eex',
@@ -226,39 +224,39 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'HyPhy' => 'hyphy',
 					'IDL' => 'idl',
 					'Idris' => 'idris',
-					'Ignore List' => 'ignore list',
-					'IGOR Pro' => 'igor pro',
-					'Inform 7' => 'inform 7',
+					'Ignore List' => 'ignore-list',
+					'IGOR Pro' => 'igor-pro',
+					'Inform 7' => 'inform-7',
 					'INI' => 'ini',
-					'Inno Setup' => 'inno setup',
+					'Inno Setup' => 'inno-setup',
 					'Io' => 'io',
 					'Ioke' => 'ioke',
-					'IRC log' => 'irc log',
+					'IRC log' => 'irc-log',
 					'Isabelle' => 'isabelle',
-					'Isabelle ROOT' => 'isabelle root',
+					'Isabelle ROOT' => 'isabelle-root',
 					'J' => 'j',
 					'Jasmin' => 'jasmin',
 					'Java' => 'java',
-					'Java Properties' => 'java properties',
-					'Java Server Pages' => 'java server pages',
+					'Java Properties' => 'java-properties',
+					'Java Server Pages' => 'java-server-pages',
 					'JavaScript' => 'javascript',
 					'JavaScript+ERB' => 'javascript+erb',
 					'JFlex' => 'jflex',
 					'Jison' => 'jison',
-					'Jison Lex' => 'jison lex',
+					'Jison Lex' => 'jison-lex',
 					'Jolie' => 'jolie',
 					'JSON' => 'json',
-					'JSON with Comments' => 'json with comments',
+					'JSON with Comments' => 'json-with-comments',
 					'JSON5' => 'json5',
 					'JSONiq' => 'jsoniq',
 					'JSONLD' => 'jsonld',
 					'Jsonnet' => 'jsonnet',
 					'JSX' => 'jsx',
 					'Julia' => 'julia',
-					'Jupyter Notebook' => 'jupyter notebook',
-					'KiCad Layout' => 'kicad layout',
-					'KiCad Legacy Layout' => 'kicad legacy layout',
-					'KiCad Schematic' => 'kicad schematic',
+					'Jupyter Notebook' => 'jupyter-notebook',
+					'KiCad Layout' => 'kicad-layout',
+					'KiCad Legacy Layout' => 'kicad-legacy-layout',
+					'KiCad Schematic' => 'kicad-schematic',
 					'Kit' => 'kit',
 					'Kotlin' => 'kotlin',
 					'KRL' => 'krl',
@@ -271,12 +269,12 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'LFE' => 'lfe',
 					'LilyPond' => 'lilypond',
 					'Limbo' => 'limbo',
-					'Linker Script' => 'linker script',
-					'Linux Kernel Module' => 'linux kernel module',
+					'Linker Script' => 'linker-script',
+					'Linux Kernel Module' => 'linux-kernel-module',
 					'Liquid' => 'liquid',
-					'Literate Agda' => 'literate agda',
-					'Literate CoffeeScript' => 'literate coffeescript',
-					'Literate Haskell' => 'literate haskell',
+					'Literate Agda' => 'literate-agda',
+					'Literate CoffeeScript' => 'literate-coffeescript',
+					'Literate Haskell' => 'literate-haskell',
 					'LiveScript' => 'livescript',
 					'LLVM' => 'llvm',
 					'Logos' => 'logos',
@@ -285,7 +283,7 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'LookML' => 'lookml',
 					'LoomScript' => 'loomscript',
 					'LSL' => 'lsl',
-					'LTspice Symbol' => 'ltspice symbol',
+					'LTspice Symbol' => 'ltspice-symbol',
 					'Lua' => 'lua',
 					'M' => 'm',
 					'M4' => 'm4',
@@ -297,7 +295,7 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Mask' => 'mask',
 					'Mathematica' => 'mathematica',
 					'MATLAB' => 'matlab',
-					'Maven POM' => 'maven pom',
+					'Maven POM' => 'maven-pom',
 					'Max' => 'max',
 					'MAXScript' => 'maxscript',
 					'mcfunction' => 'mcfunction',
@@ -305,19 +303,19 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Mercury' => 'mercury',
 					'Meson' => 'meson',
 					'Metal' => 'metal',
-					'Microsoft Developer Studio Project' => 'microsoft developer studio project',
+					'Microsoft Developer Studio Project' => 'microsoft-developer-studio-project',
 					'MiniD' => 'minid',
 					'Mirah' => 'mirah',
-					'mIRC Script' => 'mirc script',
+					'mIRC Script' => 'mirc-script',
 					'MLIR' => 'mlir',
 					'Modelica' => 'modelica',
 					'Modula-2' => 'modula-2',
 					'Modula-3' => 'modula-3',
-					'Module Management System' => 'module management system',
+					'Module Management System' => 'module-management-system',
 					'Monkey' => 'monkey',
 					'Moocode' => 'moocode',
 					'MoonScript' => 'moonscript',
-					'Motorola 68K Assembly' => 'motorola 68k assembly',
+					'Motorola 68K Assembly' => 'motorola-68k-assembly',
 					'MQL4' => 'mql4',
 					'MQL5' => 'mql5',
 					'MTML' => 'mtml',
@@ -342,12 +340,12 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Nit' => 'nit',
 					'Nix' => 'nix',
 					'NL' => 'nl',
-					'NPM Config' => 'npm config',
+					'NPM Config' => 'npm-config',
 					'NSIS' => 'nsis',
 					'Nu' => 'nu',
 					'NumPy' => 'numpy',
 					'ObjDump' => 'objdump',
-					'Object Data Instance Notation' => 'object data instance notation',
+					'Object Data Instance Notation' => 'object-data-instance-notation',
 					'Objective-C' => 'objective-c',
 					'Objective-C++' => 'objective-c++',
 					'Objective-J' => 'objective-j',
@@ -358,14 +356,14 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'ooc' => 'ooc',
 					'Opa' => 'opa',
 					'Opal' => 'opal',
-					'Open Policy Agent' => 'open policy agent',
+					'Open Policy Agent' => 'open-policy-agent',
 					'OpenCL' => 'opencl',
-					'OpenEdge ABL' => 'openedge abl',
+					'OpenEdge ABL' => 'openedge-abl',
 					'OpenQASM' => 'openqasm',
-					'OpenRC runscript' => 'openrc runscript',
+					'OpenRC runscript' => 'openrc-runscript',
 					'OpenSCAD' => 'openscad',
-					'OpenStep Property List' => 'openstep property list',
-					'OpenType Feature File' => 'opentype feature file',
+					'OpenStep Property List' => 'openstep-property-list',
+					'OpenType Feature File' => 'opentype-feature-file',
 					'Org' => 'org',
 					'Ox' => 'ox',
 					'Oxygene' => 'oxygene',
@@ -374,13 +372,12 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Pan' => 'pan',
 					'Papyrus' => 'papyrus',
 					'Parrot' => 'parrot',
-					'Parrot Assembly' => 'parrot assembly',
-					'Parrot Internal Representation' => 'parrot internal representation',
+					'Parrot Assembly' => 'parrot-assembly',
+					'Parrot Internal Representation' => 'parrot-internal-representation',
 					'Pascal' => 'pascal',
 					'Pawn' => 'pawn',
 					'Pep8' => 'pep8',
 					'Perl' => 'perl',
-					'PHP' => 'php',
 					'Pic' => 'pic',
 					'Pickle' => 'pickle',
 					'PicoLisp' => 'picolisp',
@@ -389,29 +386,28 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'PLpgSQL' => 'plpgsql',
 					'PLSQL' => 'plsql',
 					'Pod' => 'pod',
-					'Pod 6' => 'pod 6',
+					'Pod 6' => 'pod-6',
 					'PogoScript' => 'pogoscript',
 					'Pony' => 'pony',
 					'PostCSS' => 'postcss',
 					'PostScript' => 'postscript',
-					'POV-Ray SDL' => 'pov-ray sdl',
+					'POV-Ray SDL' => 'pov-ray-sdl',
 					'PowerBuilder' => 'powerbuilder',
 					'PowerShell' => 'powershell',
 					'Prisma' => 'prisma',
 					'Processing' => 'processing',
 					'Proguard' => 'proguard',
 					'Prolog' => 'prolog',
-					'Propeller Spin' => 'propeller spin',
-					'Protocol Buffer' => 'protocol buffer',
-					'Public Key' => 'public key',
+					'Propeller Spin' => 'propeller-spin',
+					'Protocol Buffer' => 'protocol-buffer',
+					'Public Key' => 'public-key',
 					'Pug' => 'pug',
 					'Puppet' => 'puppet',
-					'Pure Data' => 'pure data',
+					'Pure Data' => 'pure-data',
 					'PureBasic' => 'purebasic',
 					'PureScript' => 'purescript',
-					'Python' => 'python',
-					'Python console' => 'python console',
-					'Python traceback' => 'python traceback',
+					'Python console' => 'python-console',
+					'Python traceback' => 'python-traceback',
 					'q' => 'q',
 					'QMake' => 'qmake',
 					'QML' => 'qml',
@@ -422,30 +418,30 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Raku' => 'raku',
 					'RAML' => 'raml',
 					'Rascal' => 'rascal',
-					'Raw token data' => 'raw token data',
+					'Raw token data' => 'raw-token-data',
 					'RDoc' => 'rdoc',
-					'Readline Config' => 'readline config',
+					'Readline Config' => 'readline-config',
 					'REALbasic' => 'realbasic',
 					'Reason' => 'reason',
 					'Rebol' => 'rebol',
 					'Red' => 'red',
 					'Redcode' => 'redcode',
-					'Regular Expression' => 'regular expression',
-					// 'Ren'Py' => 'ren'py',
+					'Regular Expression' => 'regular-expression',
+					'Ren\'Py' => 'ren\'py',
 					'RenderScript' => 'renderscript',
 					'reStructuredText' => 'restructuredtext',
 					'REXX' => 'rexx',
 					'RHTML' => 'rhtml',
-					'Rich Text Format' => 'rich text format',
+					'Rich Text Format' => 'rich-text-format',
 					'Ring' => 'ring',
 					'Riot' => 'riot',
 					'RMarkdown' => 'rmarkdown',
 					'RobotFramework' => 'robotframework',
 					'Roff' => 'roff',
-					'Roff Manpage' => 'roff manpage',
+					'Roff Manpage' => 'roff-manpage',
 					'Rouge' => 'rouge',
 					'RPC' => 'rpc',
-					'RPM Spec' => 'rpm spec',
+					'RPM Spec' => 'rpm-spec',
 					'Ruby' => 'ruby',
 					'RUNOFF' => 'runoff',
 					'Rust' => 'rust',
@@ -461,7 +457,6 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'sed' => 'sed',
 					'Self' => 'self',
 					'ShaderLab' => 'shaderlab',
-					'Shell' => 'shell',
 					'ShellSession' => 'shellsession',
 					'Shen' => 'shen',
 					'Slash' => 'slash',
@@ -475,20 +470,20 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Solidity' => 'solidity',
 					'SourcePawn' => 'sourcepawn',
 					'SPARQL' => 'sparql',
-					'Spline Font Database' => 'spline font database',
+					'Spline Font Database' => 'spline-font-database',
 					'SQF' => 'sqf',
 					'SQL' => 'sql',
 					'SQLPL' => 'sqlpl',
 					'Squirrel' => 'squirrel',
-					'SRecode Template' => 'srecode template',
-					'SSH Config' => 'ssh config',
+					'SRecode Template' => 'srecode-template',
+					'SSH Config' => 'ssh-config',
 					'Stan' => 'stan',
-					'Standard ML' => 'standard ml',
+					'Standard ML' => 'standard-ml',
 					'Starlark' => 'starlark',
 					'Stata' => 'stata',
 					'STON' => 'ston',
 					'Stylus' => 'stylus',
-					'SubRip Text' => 'subrip text',
+					'SubRip Text' => 'subrip-text',
 					'SugarSS' => 'sugarss',
 					'SuperCollider' => 'supercollider',
 					'Svelte' => 'svelte',
@@ -505,7 +500,7 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Text' => 'text',
 					'Textile' => 'textile',
 					'Thrift' => 'thrift',
-					'TI Program' => 'ti program',
+					'TI Program' => 'ti-program',
 					'TLA' => 'tla',
 					'TOML' => 'toml',
 					'TSQL' => 'tsql',
@@ -514,11 +509,11 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Turtle' => 'turtle',
 					'Twig' => 'twig',
 					'TXL' => 'txl',
-					'Type Language' => 'type language',
+					'Type Language' => 'type-language',
 					'TypeScript' => 'typescript',
-					'Unified Parallel C' => 'unified parallel c',
-					'Unity3D Asset' => 'unity3d asset',
-					'Unix Assembly' => 'unix assembly',
+					'Unified Parallel C' => 'unified-parallel-c',
+					'Unity3D Asset' => 'unity3d-asset',
+					'Unix Assembly' => 'unix-assembly',
 					'Uno' => 'uno',
 					'UnrealScript' => 'unrealscript',
 					'UrWeb' => 'urweb',
@@ -529,33 +524,33 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'VCL' => 'vcl',
 					'Verilog' => 'verilog',
 					'VHDL' => 'vhdl',
-					'Vim script' => 'vim script',
-					'Vim Snippet' => 'vim snippet',
-					'Visual Basic .NET' => 'visual basic .net',
-					'Visual Basic .NET' => 'visual basic .net',
+					'Vim script' => 'vim-script',
+					'Vim Snippet' => 'vim-snippet',
+					'Visual Basic .NET' => 'visual-basic-.net',
+					'Visual Basic .NET' => 'visual-basic-.net',
 					'Volt' => 'volt',
 					'Vue' => 'vue',
-					'Wavefront Material' => 'wavefront material',
-					'Wavefront Object' => 'wavefront object',
+					'Wavefront Material' => 'wavefront-material',
+					'Wavefront Object' => 'wavefront-object',
 					'wdl' => 'wdl',
-					'Web Ontology Language' => 'web ontology language',
+					'Web Ontology Language' => 'web-ontology-language',
 					'WebAssembly' => 'webassembly',
 					'WebIDL' => 'webidl',
 					'WebVTT' => 'webvtt',
-					'Wget Config' => 'wget config',
-					'Windows Registry Entries' => 'windows registry entries',
+					'Wget Config' => 'wget-config',
+					'Windows Registry Entries' => 'windows-registry-entries',
 					'wisp' => 'wisp',
 					'Wollok' => 'wollok',
-					'World of Warcraft Addon Data' => 'world of warcraft addon data',
-					'X BitMap' => 'x bitmap',
-					'X Font Directory Index' => 'x font directory index',
-					'X PixMap' => 'x pixmap',
+					'World of Warcraft Addon Data' => 'world-of-warcraft-addon-data',
+					'X BitMap' => 'x-bitmap',
+					'X Font Directory Index' => 'x-font-directory-index',
+					'X PixMap' => 'x-pixmap',
 					'X10' => 'x10',
 					'xBase' => 'xbase',
 					'XC' => 'xc',
 					'XCompose' => 'xcompose',
 					'XML' => 'xml',
-					'XML Property List' => 'xml property list',
+					'XML Property List' => 'xml-property-list',
 					'Xojo' => 'xojo',
 					'XPages' => 'xpages',
 					'XProc' => 'xproc',
@@ -612,7 +607,9 @@ class GithubTrendingBridge extends BridgeAbstract {
 			$item['title'] = str_replace('  ', '', trim(strip_tags($element->find('h1 a', 0)->plaintext)));
 
 			// Description
-			$item['content'] = trim(strip_tags($element->find('p', 0)->innertext));
+			$description = $element->find('p', 0);
+			if ($description != null)
+				$item['content'] = trim(strip_tags($description->innertext));
 
 			// Time
 			$item['timestamp'] = time();
@@ -623,10 +620,9 @@ class GithubTrendingBridge extends BridgeAbstract {
 	}
 
 	public function getName(){
-		if($this->getInput('language') == '') {
-			return self::NAME . ': all';
-		} elseif (!is_null($this->getInput('language'))) {
-			return self::NAME . ': ' . $this->getInput('language');
+		if (!is_null($this->getInput('language'))) {
+			$language = array_search($this->getInput('language'), self::PARAMETERS['By language']['language']['values']);
+			return self::NAME . ': ' . $language;
 		}
 
 		return parent::getName();

bridges/GlassdoorBridge.php

@@ -1,4 +1,5 @@
 <?php
+
 class GlassdoorBridge extends BridgeAbstract {
 
 	// Contexts
@@ -17,7 +18,6 @@ class GlassdoorBridge extends BridgeAbstract {
 	const BLOG_TYPE_COMPANIES_HIRING = 'Companies Hiring';
 	const BLOG_TYPE_CAREER_ADVICE	 = 'Career Advice';
 	const BLOG_TYPE_INTERVIEWS		 = 'Interviews';
-	const BLOG_TYPE_GUIDE			 = 'Guides';
 
 	// Review context parameters
 	const PARAM_REVIEW_COMPANY = 'company';
@@ -39,7 +39,6 @@ class GlassdoorBridge extends BridgeAbstract {
 					self::BLOG_TYPE_COMPANIES_HIRING	=> 'blog/companies-hiring/',
 					self::BLOG_TYPE_CAREER_ADVICE		=> 'blog/career-advice/',
 					self::BLOG_TYPE_INTERVIEWS			=> 'blog/interviews/',
-					self::BLOG_TYPE_GUIDE				=> 'blog/guide/'
 				)
 			),
 			self::PARAM_BLOG_FULL => array(
@@ -67,9 +66,6 @@ class GlassdoorBridge extends BridgeAbstract {
 		)
 	);
 
-	private $host = self::URI; // They redirect without notice :/
-	private $title = '';
-
 	public function getURI() {
 		switch($this->queriedContext) {
 			case self::CONTEXT_BLOG:
@@ -81,18 +77,10 @@ class GlassdoorBridge extends BridgeAbstract {
 		return parent::getURI();
 	}
 
-	public function getName() {
-		return $this->title ? $this->title . ' - ' . self::NAME : parent::getName();
-	}
-
 	public function collectData() {
-		$html = getSimpleHTMLDOM($this->getURI());
-
-		$this->host = $html->find('link[rel="canonical"]', 0)->href;
-
-		$html = defaultLinkTo($html, $this->host);
-
-		$this->title = $html->find('meta[property="og:title"]', 0)->content;
+		$url = $this->getURI();
+		$html = getSimpleHTMLDOM($url);
+		$html = defaultLinkTo($html, $url);
 		$limit = $this->getInput(self::PARAM_LIMIT);
 
 		switch($this->queriedContext) {
@@ -106,35 +94,24 @@ class GlassdoorBridge extends BridgeAbstract {
 	}
 
 	private function collectBlogData($html, $limit) {
-		$posts = $html->find('section')
+		$posts = $html->find('div.post')
 			or returnServerError('Unable to find blog posts!');
 
 		foreach($posts as $post) {
-			$item = array();
+			$item = [];
 
-			$item['uri'] = $post->find('header a', 0)->href;
-			$item['title'] = $post->find('header', 0)->plaintext;
-			$item['content'] = $post->find('div[class="excerpt-content"]', 0)->plaintext;
-			$item['enclosures'] = array(
-				$this->getFullSizeImageURI($post->find('div[class*="post-thumb"]', 0)->{'data-original'})
-			);
-
-			// optionally load full articles
-			if($this->getInput(self::PARAM_BLOG_FULL)) {
-				$full_html = getSimpleHTMLDOMCached($item['uri']);
-
-				$full_html = defaultLinkTo($full_html, $this->host);
-
-				$item['author'] = $full_html->find('a[rel="author"]', 0);
-				$item['content'] = $full_html->find('article', 0);
-				$item['timestamp'] = strtotime($full_html->find('time.updated', 0)->datetime);
-				$item['categories'] = $full_html->find('span[class="post_tag"]');
-			}
+			$item['uri'] = $post->find('a', 0)->href;
+			$item['title'] = $post->find('h3', 0)->plaintext;
+			$item['content'] = $post->find('p', 0)->plaintext;
+			$item['author'] = $post->find('p', -2)->plaintext;
+			$item['timestamp'] = strtotime($post->find('p', -1)->plaintext);
 
+			// TODO: fetch entire blog post content
 			$this->items[] = $item;
 
-			if($limit > 0 && count($this->items) >= $limit)
+			if ($limit > 0 && count($this->items) >= $limit) {
 				return;
+			}
 		}
 	}
 
@@ -143,53 +120,32 @@ class GlassdoorBridge extends BridgeAbstract {
 			or returnServerError('Unable to find reviews!');
 
 		foreach($reviews as $review) {
-			$item = array();
+			$item = [];
 
 			$item['uri'] = $review->find('a.reviewLink', 0)->href;
-			$item['title'] = $review->find('[class="summary"]', 0)->plaintext;
-			$item['author'] = $review->find('div.author span', 0)->plaintext;
-			$item['timestamp'] = strtotime($review->find('time', 0)->datetime);
 
-			$mainText = $review->find('p.mainText', 0)->plaintext;
+			// Not all reviews have a title
+			$item['title'] = $review->find('h2', 0)->plaintext ?? 'Glassdoor review';
 
-			$description = '';
-			foreach($review->find('div.description p') as $p) {
+			[$date, $author] = explode('-', $review->find('span.authorInfo', 0)->plaintext);
 
-				if ($p->hasClass('strong')) {
-					$p->tag = 'strong';
-					$p->removeClass('strong');
-				}
-
-				$description .= $p;
+			$item['author'] = trim($author);
 
+			$createdAt = DateTimeImmutable::createFromFormat('F m, Y', trim($date));
+			if ($createdAt) {
+				$item['timestamp'] = $createdAt->getTimestamp();
 			}
 
-			$item['content'] = "<p>{$mainText}</p><p>{$description}</p>";
+			$item['content'] = $review->find('.px-std', 2)->text();
 
 			$this->items[] = $item;
 
-			if($limit > 0 && count($this->items) >= $limit)
+			if($limit > 0 && count($this->items) >= $limit) {
 				return;
+			}
 		}
 	}
 
-	private function getFullSizeImageURI($uri) {
-		/* Images are scaled for display on the website. The scaling takes place
-		 * on the host, who provides images in different sizes.
-		 *
-		 * For example:
-		 * https://www.glassdoor.com/blog/app/uploads/sites/2/GettyImages-982402074-e1538092065712-390x193.jpg
-		 *
-		 * By removing the size information we receive the full sized image.
-		 *
-		 * For example:
-		 * https://www.glassdoor.com/blog/app/uploads/sites/2/GettyImages-982402074-e1538092065712.jpg
-		 */
-
-		$uri = filter_var($uri, FILTER_SANITIZE_URL);
-		return preg_replace('/(.*)(\-\d+x\d+)(\.jpg)/', '$1$3', $uri);
-	}
-
 	private function filterCompanyURI($uri) {
 		/* Make sure the URI is a valid review page. Unfortunately there is no
 		 * simple way to determine if the URI is valid, because of automagic

bridges/GoogleGroupsBridge.php

@@ -0,0 +1,68 @@
+<?php
+
+class GoogleGroupsBridge extends XPathAbstract {
+	const NAME = 'Google Groups Bridge';
+	const DESCRIPTION = 'Returns the latest posts on a Google Group';
+	const URI = 'https://groups.google.com';
+	const MAINTAINER = 'Yaman Qalieh';
+	const PARAMETERS = array( array(
+		'group' => array(
+			'name' => 'Group id',
+			'title' => 'The string that follows /g/ in the URL',
+			'exampleValue' => 'governance',
+			'required' => true
+		),
+		'account' => array(
+			'name' => 'Account id',
+			'title' => 'Some Google groups have an additional id following /a/ in the URL',
+			'exampleValue' => 'mozilla.org',
+			'required' => false
+		)
+	));
+	const CACHE_TIMEOUT = 3600;
+
+	const TEST_DETECT_PARAMETERS = array(
+		'https://groups.google.com/a/mozilla.org/g/announce' => array(
+			'account' => 'mozilla.org', 'group' => 'announce'
+		),
+		'https://groups.google.com/g/ansible-project' => array(
+			'account' => null, 'group' => 'ansible-project'
+		),
+	);
+
+	const XPATH_EXPRESSION_ITEM = '//div[@class="yhgbKd"]';
+	const XPATH_EXPRESSION_ITEM_TITLE = './/span[@class="o1DPKc"]';
+	const XPATH_EXPRESSION_ITEM_CONTENT = './/span[@class="WzoK"]';
+	const XPATH_EXPRESSION_ITEM_URI = './/a[@class="ZLl54"]/@href';
+	const XPATH_EXPRESSION_ITEM_AUTHOR = './/span[@class="z0zUgf"][last()]';
+	const XPATH_EXPRESSION_ITEM_TIMESTAMP = './/div[@class="tRlaM"]';
+	const XPATH_EXPRESSION_ITEM_ENCLOSURES = '';
+	const XPATH_EXPRESSION_ITEM_CATEGORIES = '';
+	const SETTING_FIX_ENCODING = true;
+
+	protected function getSourceUrl() {
+		$source = self::URI;
+
+		$account = $this->getInput('account');
+		if($account) {
+			$source = $source . '/a/' . $account;
+		}
+		return $source . '/g/' . $this->getInput('group');
+	}
+
+	protected function provideWebsiteContent() {
+		return defaultLinkTo(getContents($this->getSourceUrl()), self::URI);
+	}
+
+	const URL_REGEX = '#^https://groups.google.com(?:/a/(?<account>\S+))?(?:/g/(?<group>\S+))#';
+
+	public function detectParameters($url) {
+		$params = array();
+		if(preg_match(self::URL_REGEX, $url, $matches)) {
+			$params['group'] = $matches['group'];
+			$params['account'] = $matches['account'];
+			return $params;
+		}
+		return null;
+	}
+}

bridges/GroupBundNaturschutzBridge.php

@@ -0,0 +1,107 @@
+<?php
+
+class GroupBundNaturschutzBridge extends XPathAbstract
+{
+	const NAME = 'BUND Naturschutz in Bayern e.V. - Kreisgruppen';
+	const URI = 'https://www.bund-naturschutz.de/ueber-uns/organisation/kreisgruppen-ortsgruppen';
+	const DESCRIPTION = 'Returns the latest news from specified BUND Naturschutz in Bayern e.V. local group (Germany)';
+	const MAINTAINER = 'dweipert';
+
+	const PARAMETERS = array(
+		array(
+			'group' => array(
+				'name' => 'Group',
+				'type' => 'list',
+				'values' => array(
+					// 'Aichach-Friedberg' => 'bn-aic.de', # non-uniform page
+					'Altötting' => 'altoetting',
+					'Amberg-Sulzbach' => 'amberg-sulzbach',
+					'Ansbach' => 'ansbach',
+					'Aschaffenburg' => 'aschaffenburg',
+					'Augsburg' => 'augsburg',
+					'Bad Kissingen' => 'bad-kissingen',
+					'Bad Tölz' => 'bad-toelz',
+					'Bamberg' => 'bamberg',
+					'Bayreuth' => 'bayreuth', # single entry # different layout
+					'Berchtesgadener Land' => 'berchtesgadener-land',
+					'Cham' => 'cham',
+					// 'Coburg' => 'coburg', # no real entries # different layout
+					'Dachau' => 'dachau',
+					'Deggendorf' => 'Deggendorf',
+					'Dillingen' => 'dillingen',
+					'Dingolfing-Landau' => 'dingolfing-landau',
+					'Donau-Ries' => 'donauries',
+					'Ebersberg' => 'ebersberg',
+					'Eichstätt' => 'eichstaett', # single entry since 2020
+					'Erding' => 'erding',
+					'Erlangen' => 'erlangen',
+					'Forchheim' => 'forchheim',
+					'Freising' => 'freising',
+					'Freyung-Grafenau' => 'freyung-grafenau',
+					'Fürstenfeldbruck' => 'fuerstenfeldbruck',
+					'Fürth-Land' => 'fuerth-land',
+					'Fürth-Stadt' => 'fuerth',
+					'Garmisch-Partenkirchen' => 'garmisch-partenkirchen',
+					'Günzburg' => 'guenzburg',
+					'Hassberge' => 'hassberge',
+					'Höchstadt-Herzogenaurach' => 'hoechstadt-herzogenaurach',
+					// 'Hof' => 'kreisgruppehof.bund-naturschutz.com', # non-uniform page
+					'Ingolstadt' => 'ingolstadt',
+					'Kelheim' => 'kelheim',
+					'Kempten' => 'kempten',
+					'Kitzingen' => 'kitzingen',
+					'Kronach' => 'kronach',
+					'Kulmbach' => 'kulmbach',
+					'Landsberg' => 'landsberg',
+					'Landshut' => 'landshut',
+					'Lichtenfeld' => 'lichtenfels',
+					'Lindau' => 'lindau',
+					'Main-Spessart' => 'main-spessart',
+					'Memmingen-Unterallgäu' => 'memmingen-unterallgaeu',
+					'Miesbach' => 'miesbach',
+					'Miltenberg' => 'miltenberg',
+					'Mühldorf am Inn' => 'muehldorf',
+					// 'München' => 'bn-muenchen.de', # non-uniform page
+					'Neu-Ulm' => 'neu-ulm',
+					'Neuburg-Schrobenhausen' => 'neuburg-schrobenhausen',
+					'Neumarkt' => 'neumarkt',
+					'Neustadt/Aisch-Bad Windsheim' => 'neustadt-aisch',
+					'Neustadt/Waldnaab-Weiden' => 'neustadt-weiden',
+					'Nürnberg Stadt' => 'nuernberg-stadt',
+					'Nürnberger Land' => 'nuernberger-land',
+					'Ostallgäu-Kaufbeuren' => 'Ostallgäu-Kaufbeuren',
+					'Passau' => 'passau',
+					'Pfaffenhofen/Ilm' => 'pfaffenhofen',
+					'Regen' => 'regen',
+					'Regensburg' => 'regensburg',
+					'Rhön-Grabfeld' => 'rhoen-grabfeld',
+					'Rosenheim' => 'rosenheim',
+					'Roth' => 'roth',
+					'Rottal-Inn' => 'rottal-inn',
+					'Schwabach' => 'schwabach',
+					'Schwandorf' => 'schwandorf',
+					'Schweinfurt' => 'schweinfurt',
+					'Starnberg' => 'starnberg',
+					'Straubing-Bogen' => 'straubing',
+					'Tirschenreuth' => 'tirschenreuth',
+					'Traunstein' => 'traunstein',
+					'Weilheim-Schongau' => 'weilheim-schongau',
+					'Weißenburg-Gunzenhausen' => 'weissenburg-gunzenhausen',
+					'Wunsiedel' => 'wunsiedel',
+					'Würzburg' => 'wuerzburg',
+				),
+			),
+		),
+	);
+
+	const XPATH_EXPRESSION_ITEM = '//div[@itemtype="http://schema.org/Article"]';
+	const XPATH_EXPRESSION_ITEM_TITLE = './/*[@itemprop="headline"]';
+	const XPATH_EXPRESSION_ITEM_CONTENT = './/*[@itemprop="description"]/text()';
+	const XPATH_EXPRESSION_ITEM_URI = './/a/@href';
+	const XPATH_EXPRESSION_ITEM_TIMESTAMP = './/*[@itemprop="datePublished"]/@datetime';
+	const XPATH_EXPRESSION_ITEM_ENCLOSURES = './/img/@src';
+
+	protected function getSourceUrl() {
+		return 'https://' . $this->getInput('group') . '.bund-naturschutz.de/aktuelles';
+	}
+}

bridges/HeiseBridge.php

@@ -42,6 +42,10 @@ class HeiseBridge extends FeedExpander {
 		$item = parent::parseItem($feedItem);
 		$item['uri'] = explode('?', $item['uri'])[0] . '?seite=all';
 
+		if (strpos($item['uri'], 'https://www.heise.de') !== 0) {
+			return $item;
+		}
+
 		$article = getSimpleHTMLDOMCached($item['uri']);
 
 		if ($article) {

bridges/HentaiHavenBridge.php

@@ -1,36 +0,0 @@
-<?php
-class HentaiHavenBridge extends BridgeAbstract {
-
-	const MAINTAINER = 'albirew';
-	const NAME = 'Hentai Haven';
-	const URI = 'https://hentaihaven.com/';
-	const CACHE_TIMEOUT = 21600; // 6h
-	const DESCRIPTION = 'Returns releases from Hentai Haven';
-
-	public function collectData(){
-		$html = getSimpleHTMLDOM(self::URI);
-
-		foreach($html->find('div.zoe-grid') as $element) {
-			$item = array();
-			$item['uri'] = $element->find('div.brick-content h3 a', 0)->href;
-			$thumbnailUri = $element->find('a.thumbnail-image img', 0)->getAttribute('src');
-			$item['title'] = mb_convert_encoding(
-				trim($element->find('div.brick-content h3 a', 0)->innertext),
-				'UTF-8',
-				'HTML-ENTITIES'
-			);
-
-			$item['tags'] = $element->find('div.oFlyout_bg div.oFlyout div.flyoutContent span.tags', 0)->plaintext;
-			$item['content'] = 'Tags: '
-			. $item['tags']
-			. '<br><br><a href="'
-			. $item['uri']
-			. '"><img width="300" height="169" src="'
-			. $thumbnailUri
-			. '" /></a><br>'
-			. $element->find('div.oFlyout_bg div.oFlyout div.flyoutContent p.description', 0)->innertext;
-
-			$this->items[] = $item;
-		}
-	}
-}

bridges/IPBBridge.php

@@ -55,7 +55,7 @@ class IPBBridge extends FeedExpander {
 		$headers = get_headers($uri . '.xml');
 
 		if($headers[0] === 'HTTP/1.1 200 OK') { // Heureka! It's a valid feed!
-			return $this->collectExpandableDatas($uri);
+			return $this->collectExpandableDatas($uri . '.xml');
 		}
 
 		// No valid feed, so do it the hard way

bridges/LWNprevBridge.php

@@ -109,7 +109,7 @@ EOD;
 
 	private function getFeatureContents(&$html){
 		$items = array();
-		foreach($html->getElementsByTagName('h2') as $title) {
+		foreach($html->getElementsByTagName('h3') as $title) {
 			if($title->getAttribute('class') !== 'SummaryHL') {
 				continue;
 			}

bridges/LichessBridge.php

@@ -1,31 +0,0 @@
-<?php
-class LichessBridge  extends FeedExpander {
-
-	const MAINTAINER = 'AmauryCarrade';
-	const NAME = 'Lichess Blog';
-	const URI = 'http://fr.lichess.org/blog';
-	const DESCRIPTION = 'Returns the 5 newest posts from the Lichess blog (full text)';
-
-	public function collectData(){
-		$this->collectExpandableDatas(self::URI . '.atom', 5);
-	}
-
-	protected function parseItem($newsItem){
-		$item = parent::parseItem($newsItem);
-		$item['content'] = $this->retrieveLichessPost($item['uri']);
-		return $item;
-	}
-
-	private function retrieveLichessPost($blog_post_uri){
-		$blog_post_html = getSimpleHTMLDOMCached($blog_post_uri);
-		$blog_post_div  = $blog_post_html->find('#lichess_blog', 0);
-
-		$post_chapo = $blog_post_div->find('.shortlede', 0)->innertext;
-		$post_content = $blog_post_div->find('.body', 0)->innertext;
-
-		$content = '<p><em>' . $post_chapo . '</em></p>';
-		$content .= '<div>' . $post_content . '</div>';
-
-		return $content;
-	}
-}

bridges/MangaDexBridge.php

@@ -0,0 +1,174 @@
+<?php
+class MangaDexBridge extends BridgeAbstract {
+	const MAINTAINER = 'Yaman Qalieh';
+	const NAME = 'MangaDex Bridge';
+	const URI = 'https://mangadex.org/';
+	const API_ROOT = 'https://api.mangadex.org/';
+	const DESCRIPTION = 'Returns MangaDex items using the API';
+
+	const PARAMETERS = array(
+		'global' => array(
+			'limit' => array(
+				'name' => 'Item Limit',
+				'type' => 'number',
+				'defaultValue' => 10,
+				'required' => true
+			),
+			'lang' => array(
+				'name' => 'Chapter Languages',
+				'title' => 'comma-separated, two-letter language codes (example "en,jp")',
+				'exampleValue' => 'en,jp',
+				'required' => false
+			),
+		),
+		'Title Chapters' => array(
+			'url' => array(
+				'name' => 'URL to title page',
+				'exampleValue' => 'https://mangadex.org/title/f9c33607-9180-4ba6-b85c-e4b5faee7192/official-test-manga',
+				'required' => true
+			),
+			'external' => array(
+				'name' => 'Allow external feed items',
+				'type' => 'checkbox',
+				'title' => 'Some chapters are inaccessible or only available on an external site. Include these?'
+			)
+		)
+	);
+
+	const TITLE_REGEX = '#title/(?<uuid>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})#';
+
+	protected $feedName = '';
+	protected $feedURI = '';
+
+	protected function buildArrayQuery($name, $array) {
+		$query = '';
+		foreach($array as $item) {
+			$query .= '&' . $name . '=' . $item;
+		}
+		return $query;
+	}
+
+	protected function getAPI() {
+		$params = array(
+			'limit' => $this->getInput('limit')
+		);
+
+		$array_params = array();
+		if (!empty($this->getInput('lang'))) {
+			$array_params['translatedLanguage[]'] = explode(',', $this->getInput('lang'));
+		}
+
+		switch($this->queriedContext) {
+		case 'Title Chapters':
+			preg_match(self::TITLE_REGEX, $this->getInput('url'), $matches)
+				or returnClientError('Invalid URL Parameter');
+			$this->feedURI = self::URI . 'title/' . $matches['uuid'];
+			$params['order[updatedAt]'] = 'desc';
+			if (!$this->getInput('external')) {
+				$params['includeFutureUpdates'] = '0';
+			}
+			$array_params['includes[]'] = array('manga', 'scanlation_group', 'user');
+			$uri = self::API_ROOT . 'manga/' . $matches['uuid'] . '/feed';
+			break;
+		default:
+			returnServerError('Unimplemented Context (getAPI)');
+		}
+
+		$uri .= '?' . http_build_query($params);
+
+		// Arrays are passed as repeated keys to MangaDex
+		// This cannot be handled by http_build_query
+		foreach($array_params as $name => $array_param) {
+			$uri .= $this->buildArrayQuery($name, $array_param);
+		}
+
+		return $uri;
+
+	}
+
+	public function getName() {
+		switch($this->queriedContext) {
+		case 'Title Chapters':
+			return $this->feedName . ' Chapters';
+		default:
+			return parent::getName();
+		}
+	}
+
+	public function getURI() {
+		switch($this->queriedContext) {
+		case 'Title Chapters':
+			return $this->feedURI;
+		default:
+			return parent::getURI();
+		}
+	}
+
+	public function collectData() {
+		$api_uri = $this->getApi();
+		$header = array(
+			'Content-Type: application/json'
+		);
+		$content = json_decode(getContents($api_uri, $header), true);
+		if ($content['result'] == 'ok') {
+			$content = $content['data'];
+		} else {
+			returnServerError('Could not retrieve API results');
+		}
+
+		switch($this->queriedContext) {
+		case 'Title Chapters':
+			$this->getChapters($content);
+			break;
+		default:
+			returnServerError('Unimplemented Context (collectData)');
+		}
+	}
+
+	protected function getChapters($content) {
+		foreach($content as $chapter) {
+			$item = array();
+			$item['uid'] = $chapter['id'];
+			$item['uri'] = self::URI . 'chapter/' . $chapter['id'];
+
+			// Preceding space accounts for Manga title added later
+			$item['title'] = ' Chapter ' . $chapter['attributes']['chapter'];
+			if (!empty($chapter['attributes']['title'])) {
+				$item['title'] .= ' - ' . $chapter['attributes']['title'];
+			}
+			$item['title'] .= ' [' . $chapter['attributes']['translatedLanguage'] . ']';
+
+			$item['timestamp'] = $chapter['attributes']['updatedAt'];
+
+			$groups = array();
+			$users = array();
+			foreach($chapter['relationships'] as $rel) {
+				switch($rel['type']) {
+				case 'scanlation_group':
+					$groups[] = $rel['attributes']['name'];
+					break;
+				case 'manga':
+					if (empty($this->feedName)) {
+						$this->feedName = reset($rel['attributes']['title']);
+					}
+					$item['title'] = reset($rel['attributes']['title']) . $item['title'];
+					break;
+				case 'user':
+					if (isset($item['author'])) {
+						$users[] = $rel['attributes']['username'];
+					} else {
+						$item['author'] = $rel['attributes']['username'];
+					}
+					break;
+				}
+			}
+			$item['content'] = 'Groups: ' .
+							 (empty($groups) ? 'No Group' : implode(', ', $groups));
+			if (!empty($users)) {
+				$item['content'] .= '<br>Other Users: ' . implode(', ', $users);
+			}
+
+			$this->items[] = $item;
+		}
+	}
+}

bridges/MangareaderBridge.php

@@ -1,248 +0,0 @@
-<?php
-class MangareaderBridge extends BridgeAbstract {
-
-	const MAINTAINER = 'logmanoriginal';
-	const NAME = 'Mangareader Bridge';
-	const URI = 'https://www.mangareader.net';
-	const CACHE_TIMEOUT = 10800; // 3h
-	const DESCRIPTION = 'Returns the latest updates, popular mangas or manga updates (new chapters)';
-
-	const PARAMETERS = array(
-		'Get latest updates' => array(),
-		'Get popular mangas' => array(
-			'category' => array(
-				'name' => 'Category',
-				'type' => 'list',
-				'values' => array(
-					'All' => 'all',
-					'Action' => 'action',
-					'Adventure' => 'adventure',
-					'Comedy' => 'comedy',
-					'Demons' => 'demons',
-					'Drama' => 'drama',
-					'Ecchi' => 'ecchi',
-					'Fantasy' => 'fantasy',
-					'Gender Bender' => 'gender-bender',
-					'Harem' => 'harem',
-					'Historical' => 'historical',
-					'Horror' => 'horror',
-					'Josei' => 'josei',
-					'Magic' => 'magic',
-					'Martial Arts' => 'martial-arts',
-					'Mature' => 'mature',
-					'Mecha' => 'mecha',
-					'Military' => 'military',
-					'Mystery' => 'mystery',
-					'One Shot' => 'one-shot',
-					'Psychological' => 'psychological',
-					'Romance' => 'romance',
-					'School Life' => 'school-life',
-					'Sci-Fi' => 'sci-fi',
-					'Seinen' => 'seinen',
-					'Shoujo' => 'shoujo',
-					'Shoujoai' => 'shoujoai',
-					'Shounen' => 'shounen',
-					'Shounenai' => 'shounenai',
-					'Slice of Life' => 'slice-of-life',
-					'Smut' => 'smut',
-					'Sports' => 'sports',
-					'Super Power' => 'super-power',
-					'Supernatural' => 'supernatural',
-					'Tragedy' => 'tragedy',
-					'Vampire' => 'vampire',
-					'Yaoi' => 'yaoi',
-					'Yuri' => 'yuri'
-				),
-			'exampleValue' => 'All',
-			'title' => 'Select your category'
-			)
-		),
-		'Get manga updates' => array(
-			'path' => array(
-				'name' => 'Path',
-				'required' => true,
-				'pattern' => '[a-zA-Z0-9-_]*',
-				'exampleValue' => 'bleach',
-				'title' => 'URL part of desired manga. e.g= bleach OR umi-no-kishidan'
-			),
-			'limit' => array(
-				'name' => 'Limit',
-				'type' => 'number',
-				'defaultValue' => 10,
-				'title' => 'Number of items to return [-1 returns all]'
-			)
-		)
-	);
-
-	private $request = '';
-
-	public function collectData(){
-		// We'll use the DOM parser for this as it makes navigation easier
-		$html = getContents($this->getURI());
-		if(!$html) {
-			returnClientError('Could not receive data for ' . $path . '!');
-		}
-		libxml_use_internal_errors(true);
-		$doc = new DomDocument;
-		@$doc->loadHTML($html);
-		libxml_clear_errors();
-
-		// Navigate via XPath
-		$xpath = new DomXPath($doc);
-
-		$this->request = '';
-		switch($this->queriedContext) {
-		case 'Get latest updates':
-			$this->request = 'Latest updates';
-			$this->getLatestUpdates($xpath);
-			break;
-		case 'Get popular mangas':
-			// Find manga name within "Popular mangas for ..."
-			$pagetitle = $xpath->query(".//*[@id='bodyalt']/h1")->item(0)->nodeValue;
-			$this->request = substr($pagetitle, 0, strrpos($pagetitle, ' -'));
-			$this->getPopularMangas($xpath);
-			break;
-		case 'Get manga updates':
-			$limit = $this->getInput('limit');
-			if(empty($limit)) {
-				$limit = self::PARAMETERS[$this->queriedContext]['limit']['defaultValue'];
-			}
-
-			$this->request = $xpath->query(".//*[@id='mangaproperties']//*[@class='aname']")
-				->item(0)
-				->nodeValue;
-
-			$this->getMangaUpdates($xpath, $limit);
-			break;
-		}
-
-		// Return some dummy-data if no content available
-		if(empty($this->items)) {
-			$item = array();
-			$item['content'] = '<p>No updates available</p>';
-
-			$this->items[] = $item;
-		}
-	}
-
-	private function getLatestUpdates($xpath){
-		// Query each item (consists of Manga + chapters)
-		$nodes = $xpath->query("//*[@id='latestchapters']/table//td");
-
-		foreach ($nodes as $node) {
-			// Query the manga
-			$manga = $xpath->query("a[@class='chapter']", $node)->item(0);
-
-			// Collect the chapters for each Manga
-			$chapters = $xpath->query("a[@class='chaptersrec']", $node);
-
-			if (isset($manga) && $chapters->length >= 1) {
-				$item = array();
-				$item['uri'] = self::URI . htmlspecialchars($manga->getAttribute('href'));
-				$item['title'] = htmlspecialchars($manga->nodeValue);
-
-				// Add each chapter to the feed
-				$item['content'] = '';
-
-				foreach ($chapters as $chapter) {
-					if($item['content'] <> '') {
-						$item['content'] .= '<br>';
-					}
-					$item['content'] .= "<a href='"
-						. self::URI
-						. htmlspecialchars($chapter->getAttribute('href'))
-						. "'>"
-						. htmlspecialchars($chapter->nodeValue)
-						. '</a>';
-				}
-
-				$this->items[] = $item;
-			}
-		}
-	}
-
-	private function getPopularMangas($xpath){
-		// Query all mangas
-		$mangas = $xpath->query("//*[@id='mangaresults']/*[@class='mangaresultitem']");
-
-		foreach ($mangas as $manga) {
-
-			// The thumbnail is encrypted in a css-style...
-			// format: "background-image:url('<the part which is actually interesting>')"
-			$mangaimgelement = $xpath->query(".//*[@class='imgsearchresults']", $manga)
-				->item(0)
-				->getAttribute('style');
-			$thumbnail = substr($mangaimgelement, 22, strlen($mangaimgelement) - 24);
-
-			$item = array();
-			$item['title'] = htmlspecialchars($xpath->query(".//*[@class='manga_name']//a", $manga)
-				->item(0)
-				->nodeValue);
-			$item['uri'] = self::URI . $xpath->query(".//*[@class='manga_name']//a", $manga)
-				->item(0)
-				->getAttribute('href');
-			$item['author'] = htmlspecialchars($xpath->query("//*[@class='author_name']", $manga)
-				->item(0)
-				->nodeValue);
-			$item['chaptercount'] = $xpath->query(".//*[@class='chapter_count']", $manga)
-				->item(0)
-				->nodeValue;
-			$item['genre'] = htmlspecialchars($xpath->query(".//*[@class='manga_genre']", $manga)
-				->item(0)
-				->nodeValue);
-			$item['content'] = <<<EOD
-<a href="{$item['uri']}"><img src="{$thumbnail}" alt="{$item['title']}" /></a>
-<p>{$item['genre']}</p>
-<p>{$item['chaptercount']}</p>
-EOD;
-			$this->items[] = $item;
-		}
-	}
-
-	private function getMangaUpdates($xpath, $limit){
-		$query = "(.//*[@id='listing']//tr)[position() > 1]";
-
-		if($limit !== -1) {
-			$query = "(.//*[@id='listing']//tr)[position() > 1][position() > last() - {$limit}]";
-		}
-
-		$chapters = $xpath->query($query);
-
-		foreach ($chapters as $chapter) {
-			$item = array();
-			$item['title'] = htmlspecialchars($xpath->query('td[1]', $chapter)
-				->item(0)
-				->nodeValue);
-			$item['uri'] = self::URI . $xpath->query('td[1]/a', $chapter)
-				->item(0)
-				->getAttribute('href');
-			$item['timestamp'] = strtotime($xpath->query('td[2]', $chapter)
-				->item(0)
-				->nodeValue);
-			array_unshift($this->items, $item);
-		}
-	}
-
-	public function getURI(){
-		switch($this->queriedContext) {
-		case 'Get latest updates':
-			$path = 'latest';
-			break;
-		case 'Get popular mangas':
-			$path = 'popular';
-			if($this->getInput('category') !== 'all') {
-				$path .= '/' . $this->getInput('category');
-			}
-			break;
-		case 'Get manga updates':
-			$path = $this->getInput('path');
-			break;
-		default: return parent::getURI();
-		}
-		return self::URI . '/' . $path;
-	}
-
-	public function getName(){
-		return (!empty($this->request) ? $this->request . ' - ' : '') . 'Mangareader Bridge';
-	}
-}

bridges/MixCloudBridge.php

@@ -5,6 +5,7 @@ class MixCloudBridge extends BridgeAbstract {
 	const MAINTAINER = 'Alexis CHEMEL';
 	const NAME = 'MixCloud';
 	const URI = 'https://www.mixcloud.com';
+	const API_URI = 'https://api.mixcloud.com/';
 	const CACHE_TIMEOUT = 3600; // 1h
 	const DESCRIPTION = 'Returns latest musics on user stream';
 
@@ -24,30 +25,37 @@ class MixCloudBridge extends BridgeAbstract {
 		return parent::getName();
 	}
 
+	private static function compareDate($stream1, $stream2) {
+		return (strtotime($stream1['timestamp']) < strtotime($stream2['timestamp']) ? 1 : -1);
+	}
+
 	public function collectData(){
-		ini_set('user_agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0');
+		$user = urlencode($this->getInput('u'));
+		// Get Cloudcasts
+		$mixcloudUri = self::API_URI . $user . '/cloudcasts/';
+		$content = getContents($mixcloudUri);
+		$casts = json_decode($content)->data;
 
-		$html = getSimpleHTMLDOM(self::URI . '/' . $this->getInput('u'));
+		// Get Listens
+		$mixcloudUri = self::API_URI . $user . '/listens/';
+		$content = getContents($mixcloudUri);
+		$listens = json_decode($content)->data;
 
-		foreach($html->find('section.card') as $element) {
+		$streams = array_merge($casts, $listens);
 
+		foreach($streams as $stream) {
 			$item = array();
 
-			$item['uri'] = self::URI . $element->find('hgroup.card-title h1 a', 0)->getAttribute('href');
-			$item['title'] = html_entity_decode(
-				$element->find('hgroup.card-title h1 a span', 0)->getAttribute('title'),
-				ENT_QUOTES
-			);
-
-			$image = $element->find('a.album-art img', 0);
-
-			if($image) {
-				$item['content'] = '<img src="' . $image->getAttribute('src') . '" />';
-			}
-
-			$item['author'] = trim($element->find('hgroup.card-title h2 a', 0)->innertext);
+			$item['uri'] = $stream->url;
+			$item['title'] = $stream->name;
+			$item['content'] = '<img src="' . $stream->pictures->thumbnail . '" />';
+			$item['author'] = $stream->user->name;
+			$item['timestamp'] = $stream->created_time;
 
 			$this->items[] = $item;
 		}
+
+		// Sort items by date
+		usort($this->items, array('MixCloudBridge', 'compareDate'));
 	}
 }

bridges/MspabooruBridge.php

@@ -5,8 +5,11 @@ class MspabooruBridge extends GelbooruBridge {
 
 	const MAINTAINER = 'mitsukarenai';
 	const NAME = 'Mspabooru';
-	const URI = 'http://mspabooru.com/';
+	const URI = 'https://mspabooru.com/';
 	const DESCRIPTION = 'Returns images from given page';
-	const PIDBYPAGE = 50;
 
+	protected function buildThumbnailURI($element){
+		return $this->getURI() . 'thumbnails/' . $element->directory
+		. '/thumbnail_' . $element->image;
+	}
 }

bridges/OpenClassroomsBridge.php

@@ -1,47 +0,0 @@
-<?php
-class OpenClassroomsBridge extends BridgeAbstract {
-
-	//const MAINTAINER = 'sebsauvage';
-	const NAME = 'OpenClassrooms Bridge';
-	const URI = 'https://openclassrooms.com/';
-	const CACHE_TIMEOUT = 21600; // 6h
-	const DESCRIPTION = 'Returns latest tutorials from OpenClassrooms.';
-
-	const PARAMETERS = array( array(
-		'u' => array(
-			'name' => 'Catégorie',
-			'type' => 'list',
-			'values' => array(
-				'Arts & Culture' => 'arts',
-				'Code' => 'code',
-				'Design' => 'design',
-				'Entreprise' => 'business',
-				'Numérique' => 'digital',
-				'Sciences' => 'sciences',
-				'Sciences Humaines' => 'humainities',
-				'Systèmes d\'information' => 'it',
-				'Autres' => 'others'
-			)
-		)
-	));
-
-	public function getURI(){
-		if(!is_null($this->getInput('u'))) {
-			return self::URI . '/courses?categories=' . $this->getInput('u') . '&title=&sort=updatedAt+desc';
-		}
-
-		return parent::getURI();
-	}
-
-	public function collectData(){
-		$html = getSimpleHTMLDOM($this->getURI());
-
-		foreach($html->find('.courseListItem') as $element) {
-				$item = array();
-				$item['uri'] = self::URI . $element->find('a', 0)->href;
-				$item['title'] = $element->find('h3', 0)->plaintext;
-				$item['content'] = $element->find('slidingItem__descriptionContent', 0)->plaintext;
-				$this->items[] = $item;
-		}
-	}
-}

bridges/OtrkeyFinderBridge.php

@@ -10,7 +10,7 @@ class OtrkeyFinderBridge extends BridgeAbstract {
 		array(
 			'searchterm' => array(
 				'name' => 'Search term',
-				'exampleValue' => 'Terminator',
+				'exampleValue' => 'Tatort',
 				'title' => 'The search term is case-insensitive',
 			),
 			'station' => array(
@@ -155,8 +155,14 @@ class OtrkeyFinderBridge extends BridgeAbstract {
 
 		if ($file == null)
 			return null;
-		else
-			return trim($file->innertext);
+
+		// Sometimes there is HTML in the filename - we don't want that.
+		// To filter that out, enumerate to the node which contains the text only.
+		foreach($file->nodes as $node)
+			if ($node->nodetype == HDOM_TYPE_TEXT)
+				return trim($node->innertext);
+
+		return null;
 	}
 
 	private function buildContent(simple_html_dom_node $node) {

bridges/ParksOnTheAirBridge.php

@@ -0,0 +1,41 @@
+<?php
+
+class ParksOnTheAirBridge extends BridgeAbstract {
+	const MAINTAINER = 's0lesurviv0r';
+	const NAME = 'Parks On The Air Spots';
+	const URI = 'https://pota.app/#';
+	const API_URI = 'https://api.pota.app/spot/activator';
+	const CACHE_TIMEOUT = 60; // 1m
+	const DESCRIPTION = 'Parks On The Air Activator Spots';
+
+	public function collectData() {
+
+		$header = array('Content-type:application/json');
+		$opts = array(CURLOPT_HTTPGET => 1);
+		$json = getContents(self::API_URI, $header, $opts);
+
+		$spots = json_decode($json, true);
+
+		foreach ($spots as $spot) {
+			$title = $spot['activator'] . ' @ ' . $spot['reference'] . ' ' .
+				$spot['frequency'] . ' kHz';
+			$park_link = self::URI . '/park/' . $spot['reference'];
+
+			$content = <<<EOL
+<a href="{$park_link}">
+{$spot['reference']}, {$spot['name']}</a><br />
+Location: {$spot['locationDesc']}<br />
+Frequency: {$spot['frequency']} kHz<br />
+Spotter: {$spot['spotter']}<br />
+Comments: {$spot['comments']}
+EOL;
+
+			$this->items[] = array(
+				'uri' => $park_link,
+				'title' => $title,
+				'content' => $content,
+				'timestamp' => $spot['spotTime']
+			);
+		}
+	}
+}

bridges/PhoronixBridge.php

@@ -6,17 +6,60 @@ class PhoronixBridge extends FeedExpander {
 	const URI = 'https://www.phoronix.com';
 	const CACHE_TIMEOUT = 3600;
 	const DESCRIPTION = 'RSS feed for Linux news website Phoronix';
+	const PARAMETERS = array(array(
+		'n' => array(
+			'name' => 'Limit',
+			'type' => 'number',
+			'required' => false,
+			'title' => 'Maximum number of items to return',
+			'defaultValue' => 10
+		),
+		'svgAsImg' => array(
+			'name' => 'SVG in "image" tag',
+			'type' => 'checkbox',
+			'title' => 'Some benchmarks are exported as SVG with "object" tag,
+but some RSS readers don\'t support this. "img" tag are supported by most browsers',
+			'defaultValue' => false
+		),
+	));
 
 	public function collectData(){
-		$this->collectExpandableDatas('https://www.phoronix.com/rss.php', 15);
+		$this->collectExpandableDatas('https://www.phoronix.com/rss.php', $this->getInput('n'));
 	}
 
 	protected function parseItem($newsItem){
 		$item = parent::parseItem($newsItem);
 		// $articlePage gets the entire page's contents
 		$articlePage = getSimpleHTMLDOM($newsItem->link);
-		$article = $articlePage->find('.content', 0);
-		$item['content'] = $article;
+		$articlePage = defaultLinkTo($articlePage, $this->getURI());
+		// Extract final link. From Facebook's like plugin.
+		parse_str(parse_url($articlePage->find('iframe[src^=//www.facebook.com/plugins]', 0), PHP_URL_QUERY), $facebookQuery);
+		if (array_key_exists('href', $facebookQuery)) {
+			$newsItem->link = $facebookQuery['href'];
+		}
+		$item['content'] = $this->extractContent($articlePage);
+
+		$pages = $articlePage->find('.pagination a[!title]');
+		foreach ($pages as $page) {
+			$pageURI = urljoin($newsItem->link, html_entity_decode($page->href));
+			$page = getSimpleHTMLDOM($pageURI);
+			$item['content'] .= $this->extractContent($page);
+		}
 		return $item;
 	}
+
+	private function extractContent($page){
+		$content = $page->find('.content', 0);
+		$objects = $content->find('script[src^=//openbenchmarking.org]');
+		foreach ($objects as $object) {
+			$objectSrc = preg_replace('/p=0/', 'p=2', $object->src);
+			if ($this->getInput('svgAsImg')) {
+				$object->outertext = '<a href="' . $objectSrc . '"><img src="' . $objectSrc . '"/></a>';
+			} else {
+				$object->outertext = '<object data="' . $objectSrc . '" type="image/svg+xml"></object>';
+			}
+		}
+		$content = stripWithDelimiters($content, '<script', '</script>');
+		return $content;
+	}
 }

bridges/PillowfortBridge.php

@@ -8,7 +8,8 @@ class PillowfortBridge extends BridgeAbstract {
 		'username' => array(
 			'name' => 'Username',
 			'type' => 'text',
-			'required' => true
+			'required' => true,
+			'exampleValue' => 'vaxis2',
 		),
 		'noava' => array(
 			'name' => 'Hide avatar',

bridges/PlantUMLReleasesBridge.php

@@ -5,62 +5,41 @@
  * @author nicolas-delsaux
  *
  */
-class PlantUMLReleasesBridge extends BridgeAbstract
-{
+class PlantUMLReleasesBridge extends BridgeAbstract {
 	const MAINTAINER = 'Riduidel';
-
 	const NAME = 'PlantUML Releases';
-
 	const AUTHOR = 'PlantUML team';
-
-	// URI is no more valid, since we can address the whole gq galaxy
-	const URI = 'http://plantuml.com/fr/changes';
+	const URI = 'https://plantuml.com/changes';
 
 	const CACHE_TIMEOUT = 7200; // 2h
 	const DESCRIPTION = 'PlantUML releases bridge, showing for each release the changelog';
+	const ITEM_LIMIT = 10;
 
-	const DEFAULT_DOMAIN = 'plantuml.com';
-
-	const PARAMETERS = array( array(
-	));
-
-	const REPLACED_ATTRIBUTES = array(
-		'href' => 'href',
-		'src' => 'src',
-		'data-original' => 'src'
-	);
-
-	private function getDomain() {
-		$domain = $this->getInput('domain');
-		if (empty($domain))
-			$domain = self::DEFAULT_DOMAIN;
-		if (strpos($domain, '://') === false)
-			$domain = 'https://' . $domain;
-		return $domain;
-	}
-
-	public function getURI()
-	{
+	public function getURI() {
 		return self::URI;
 	}
 
-	public function collectData()
-	{
-		$html = getSimpleHTMLDOM($this->getURI());
+	public function collectData() {
+		$html = defaultLinkTo(getSimpleHTMLDOM($this->getURI()), self::URI);
 
-		// Since GQ don't want simple class scrapping, let's do it the hard way and ... discover content !
+		$num_items = 0;
 		$main = $html->find('div[id=root]', 0);
 		foreach ($main->find('h2') as $release) {
+			// Limit to $ITEM_LIMIT number of results
+			if ($num_items++ >= self::ITEM_LIMIT) {
+				break;
+			}
 			$item = array();
 			$item['author'] = self::AUTHOR;
 			$release_text = $release->innertext;
 			if (preg_match('/(.+) \((.*)\)/', $release_text, $matches)) {
 				$item['title'] = $matches[1];
-				// And now, build the date from the date text
-				$item['timestamp'] = strtotime($matches[2]);
+				$item['timestamp'] = preg_replace('/(\d+) (\w{3})\w*, (\d+)/', '${1} ${2} ${3}', $matches[2]);
+			} else {
+				$item['title'] = $release_text;
 			}
 			$item['uri'] = $this->getURI();
-			$item['content'] = $release->next_sibling ();
+			$item['content'] = $release->next_sibling();
 			$this->items[] = $item;
 		}
 	}

bridges/PokemonTVBridge.php

@@ -36,7 +36,7 @@ class PokemonTVBridge extends BridgeAbstract {
 		),
 		'filterseason' => array(
 			'name' => 'Series Season Filter',
-			'exampleValue' => '5',
+			'exampleValue' => '22',
 			'required' => false
 		)
 	));

bridges/QPlayBridge.php

@@ -1,128 +0,0 @@
-<?php
-class QPlayBridge extends BridgeAbstract {
-	const NAME = 'Q Play';
-	const URI = 'https://www.qplay.pt';
-	const DESCRIPTION = 'Entretenimento e humor em Português';
-	const MAINTAINER = 'somini';
-	const PARAMETERS = array(
-		'Program' => array(
-			'program' => array(
-				'name' => 'Program Name',
-				'type' => 'text',
-				'exampleValue' => 'bridgebroken',
-				'required' => true,
-			),
-		),
-		'Catalog' => array(
-			'all_pages' => array(
-				'name' => 'All Pages',
-				'type' => 'checkbox',
-				'defaultValue' => false,
-			),
-		),
-	);
-
-	public function getIcon() {
-		# This should be the favicon served on `self::URI`
-		return 'https://s3.amazonaws.com/unode1/assets/4957/r3T9Lm9LTLmpAEX6FlSA_apple-touch-icon.png';
-	}
-
-	public function getURI() {
-		switch ($this->queriedContext) {
-			case 'Program':
-				return self::URI . '/programs/' . $this->getInput('program');
-			case 'Catalog':
-				return self::URI . '/catalog';
-		}
-		return parent::getURI();
-	}
-
-	public function getName() {
-		switch ($this->queriedContext) {
-			case 'Program':
-				$html = getSimpleHTMLDOMCached($this->getURI());
-
-				return $html->find('h1.program--title', 0)->innertext;
-			case 'Catalog':
-				return self::NAME . ' | Programas';
-		}
-
-		return parent::getName();
-	}
-
-	/* This uses the uscreen platform, other sites can adapt this. https://www.uscreen.tv/ */
-	public function collectData() {
-		switch ($this->queriedContext) {
-		case 'Program':
-			$program = $this->getInput('program');
-			$html = getSimpleHTMLDOMCached($this->getURI());
-
-			foreach($html->find('.cce--thumbnails-video-chapter') as $element) {
-				$cid = $element->getAttribute('data-id');
-				$item['title'] = $element->find('.cce--chapter-title', 0)->innertext;
-				$item['content'] = $element->find('.cce--thumbnails-image-block', 0)
-					. $element->find('.cce--chapter-body', 0)->innertext;
-				$item['uri'] = $this->getURI() . '?cid=' . $cid;
-
-				/* TODO: Suport login credentials? */
-				/* # Get direct video URL */
-				/* $json_source = getContents(self::URI . '/chapters/' . $cid, array('Cookie: _uscreen2_session=???;')); */
-				/* $json = json_decode($json_source); */
-
-				/* $item['enclosures'] = [$json->fallback]; */
-
-				$this->items[] = $item;
-			}
-
-			break;
-		case 'Catalog':
-			$json_raw = getContents($this->getCatalogURI(1));
-
-			$json = json_decode($json_raw);
-			$total_pages = $json->total_pages;
-
-			foreach($this->parseCatalogPage($json) as $item) {
-				$this->items[] = $item;
-			}
-
-			if ($this->getInput('all_pages') === true) {
-				foreach(range(2, $total_pages) as $page) {
-					$json_raw = getContents($this->getCatalogURI($page));
-
-					$json = json_decode($json_raw);
-
-					foreach($this->parseCatalogPage($json) as $item) {
-						$this->items[] = $item;
-					}
-				}
-			}
-
-			break;
-		}
-	}
-
-	private function getCatalogURI($page) {
-		return self::URI . '/catalog.json?page=' . $page;
-	}
-
-	private function parseCatalogPage($json) {
-		$items = array();
-
-		foreach($json->records as $record) {
-			$item = array();
-
-			$item['title'] = $record->title;
-			$item['content'] = $record->description
-				. '<div>Duration: ' . $record->duration . '</div>';
-			$item['timestamp'] = strtotime($record->release_date);
-			$item['uri'] = self::URI . $record->url;
-			$item['enclosures'] = array(
-				$record->main_poster,
-			);
-
-			$items[] = $item;
-		}
-
-		return $items;
-	}
-}

bridges/RTBFBridge.php

@@ -1,65 +0,0 @@
-<?php
-class RTBFBridge extends BridgeAbstract {
-	const NAME = 'RTBF Bridge';
-	const URI = 'http://www.rtbf.be/auvio/';
-	const CACHE_TIMEOUT = 21600; //6h
-	const DESCRIPTION = 'Returns the newest RTBF videos by series ID';
-	const MAINTAINER = 'Frenzie';
-
-	const PARAMETERS = array( array(
-		'c' => array(
-			'name' => 'series id',
-			'exampleValue' => 9500,
-			'required' => true
-		)
-	));
-
-	public function collectData(){
-		$html = '';
-		$limit = 10;
-		$count = 0;
-
-		$html = getSimpleHTMLDOM($this->getURI());
-
-		foreach($html->find('section[id!=widget-ml-avoiraussi-] .rtbf-media-grid article') as $element) {
-			if($count >= $limit) {
-				break;
-			}
-
-			$item = array();
-			$item['id'] = $element->getAttribute('data-id');
-			$item['uri'] = self::URI . 'detail?id=' . $item['id'];
-			$thumbnailUriSrcSet = explode(
-				',',
-				$element->find('figure .www-img-16by9 img', 0)->getAttribute('data-srcset')
-			);
-
-			$thumbnailUriLastSrc = end($thumbnailUriSrcSet);
-			$thumbnailUri = explode(' ', $thumbnailUriLastSrc)[0];
-			$item['title'] = trim($element->find('h3', 0)->plaintext)
-			. ' - '
-			. trim($element->find('h4', 0)->plaintext);
-
-			$item['timestamp'] = strtotime($element->find('time', 0)->getAttribute('datetime'));
-			$item['content'] = '<a href="' . $item['uri'] . '"><img src="' . $thumbnailUri . '" /></a>';
-			$this->items[] = $item;
-			$count++;
-		}
-	}
-
-	public function getURI(){
-		if(!is_null($this->getInput('c'))) {
-			return self::URI . 'emissions/detail?id=' . $this->getInput('c');
-		}
-
-		return parent::getURI() . 'emissions/';
-	}
-
-	public function getName(){
-		if(!is_null($this->getInput('c'))) {
-			return $this->getInput('c') . ' - RTBF Bridge';
-		}
-
-		return parent::getName();
-	}
-}

bridges/ReutersBridge.php

@@ -131,6 +131,7 @@ class ReutersBridge extends BridgeAbstract
 		'sports' => '/lifestyle/sports',
 		'life' => '/lifestyle',
 		'science' => '/lifestyle/science',
+		'home/topnews' => '/home',
 	);
 
 	const OLD_WIRE_SECTION = array(
@@ -211,11 +212,12 @@ class ReutersBridge extends BridgeAbstract
 	}
 
 	/**
-	* @param string $endpoint - Provide section's endpoint to Reuters API.
+	* @param string $endpoint - A endpoint is provided could be article URI or ID.
 	* @param string $fetch_type - Provide what kind of fetch do you want? Article or Section.
+	* @param boolean $is_article_uid {true|false} - A boolean flag to determined if using UID instead of url to fetch.
 	* @return string A completed API URL to fetch data
 	*/
-	private function getAPIURL($endpoint, $fetch_type) {
+	private function getAPIURL($endpoint, $fetch_type, $is_article_uid = false) {
 		$base_url = self::URI . '/pf/api/v3/content/fetch/';
 		$wire_url = 'https://wireapi.reuters.com/v8';
 		switch($fetch_type) {
@@ -223,10 +225,23 @@ class ReutersBridge extends BridgeAbstract
 				if($this->useWireAPI) {
 					return $wire_url . $endpoint;
 				}
-				$query = array(
-					'website_url' => $endpoint,
-					'website' => 'reuters'
+
+				$base_query = array(
+					'website' => 'reuters',
 				);
+				$query = array();
+
+				if ($is_article_uid) {
+					$query = array(
+						'id' => $endpoint
+					);
+				} else {
+					$query = array(
+						'website_url' => $endpoint,
+					);
+				}
+
+				$query = array_merge($base_query, $query);
 				$json_query = json_encode($query);
 				return $base_url . 'article-by-id-or-url-v1?query=' . $json_query;
 				break;
@@ -241,11 +256,17 @@ class ReutersBridge extends BridgeAbstract
 					return $wire_url . $feed_uri;
 				}
 				$query = array(
-					'fetch_type' => 'section',
 					'section_id' => $endpoint,
 					'size' => 30,
 					'website' => 'reuters'
 				);
+
+				if ($endpoint != '/home') {
+					$query = array_merge($query, array(
+						'fetch_type' => 'section',
+					));
+				}
+
 				$json_query = json_encode($query);
 				return $base_url . 'articles-by-section-alias-or-id-v1?query=' . $json_query;
 				break;
@@ -253,11 +274,27 @@ class ReutersBridge extends BridgeAbstract
 		returnServerError('unsupported endpoint');
 	}
 
-	private function getArticle($feed_uri)
+	private function addStories($title, $content, $timestamp, $author, $url, $category) {
+		$item = array();
+		$item['categories'] = $category;
+		$item['author'] = $author;
+		$item['content'] = $content;
+		$item['title'] = $title;
+		$item['timestamp'] = $timestamp;
+		$item['uri'] = $url;
+		$this->items[] = $item;
+	}
+
+	private function getArticle($feed_uri, $is_article_uid = false)
 	{
 		// This will make another request to API to get full detail of article and author's name.
-		$url = $this->getAPIURL($feed_uri, 'article');
+		$url = $this->getAPIURL($feed_uri, 'article', $is_article_uid);
 		$rawData = $this->getJson($url);
+
+		if(json_last_error() != JSON_ERROR_NONE) { // Checking whether a valid JSON or not
+			return $this->handleRedirectedArticle($url);
+		}
+
 		$article_content = '';
 		$authorlist = '';
 		$category = array();
@@ -299,6 +336,40 @@ class ReutersBridge extends BridgeAbstract
 		return $content_detail;
 	}
 
+	private function handleRedirectedArticle($url) {
+		$html = getSimpleHTMLDOMCached($url, 86400); // Duration 24h
+
+		$description = '';
+		$author = '';
+		$images = '';
+		$meta_items = $html->find('meta');
+		foreach($meta_items as $meta) {
+			switch ($meta->name) {
+				case 'description':
+					$description = $meta->content;
+					break;
+				case 'author':
+				case 'twitter:creator':
+					$author = $meta->content;
+					break;
+				case 'twitter:image:src':
+				case 'twitter:image':
+					$url = $meta->content;
+					$images = "<img src=$url" . '>';
+					break;
+			}
+		}
+
+		return array(
+			'content' => $description,
+			'author' => $author,
+			'category' => '',
+			'images' => $images,
+			'published_at' => '',
+			'status' => 'redirected'
+		);
+	}
+
 	private function handleImage($images) {
 		$img_placeholder = '';
 
@@ -444,6 +515,27 @@ EOD;
 		return $description;
 	}
 
+	/**
+	 * @param array $stories
+	 */
+	private function addRelatedStories($stories) {
+		foreach($stories as $story) {
+			$story_data = $this->getArticle($story['url']);
+			$title = $story['caption'];
+			$url = self::URI . $story['url'];
+			if(isset($story_data['status']) && $story_data['status'] != 'redirected') {
+				$article_body = defaultLinkTo($story_data['content'], $this->getURI());
+			} else {
+				$article_body = $story_data['content'];
+			}
+			$content = $article_body . $story_data['images'];
+			$timestamp = $story_data['published_at'];
+			$category = $story_data['category'];
+			$author = $story_data['author'];
+			$this->addStories($title, $content, $timestamp, $author, $url, $category);
+		}
+	}
+
 	public function getName() {
 		return $this->feedName;
 	}
@@ -500,11 +592,14 @@ EOD;
 				$title = $story['title'];
 				$article_uri = $story['canonical_url'];
 				$source_type = $story['source']['name'];
+				if (isset($story['related_stories'])) {
+					$this->addRelatedStories($story['related_stories']);
+				}
 			}
 
 			// Some article cause unexpected behaviour like redirect to another site not API.
 			// Attempt to check article source type to avoid this.
-			if($source_type == 'composer') { // Only Reuters PF api have this, Wire don't.
+			if(!$this->useWireAPI && $source_type != 'Package') { // Only Reuters PF api have this, Wire don't.
 				$author = $this->handleAuthorName($story['authors']);
 				$timestamp = $story['published_time'];
 				$image_placeholder = '';
@@ -512,6 +607,7 @@ EOD;
 					$image_placeholder = $this->handleImage(array($story['thumbnail']));
 				}
 				$content = $story['description'] . $image_placeholder;
+				$category = array($story['primary_section']['name']);
 			} else {
 				$content_detail = $this->getArticle($article_uri);
 				$description = $content_detail['content'];
@@ -524,13 +620,8 @@ EOD;
 				$timestamp = $content_detail['published_at'];
 			}
 
-			$item['categories'] = $category;
-			$item['author'] = $author;
-			$item['content'] = $content;
-			$item['title'] = $title;
-			$item['timestamp'] = $timestamp;
-			$item['uri'] = $url;
-			$this->items[] = $item;
+			$this->addStories($title, $content, $timestamp, $author, $url, $category);
+
 		}
 	}
 }

bridges/Rule34Bridge.php

@@ -8,5 +8,4 @@ class Rule34Bridge extends GelbooruBridge {
 	const URI = 'https://rule34.xxx/';
 	const DESCRIPTION = 'Returns images from given page';
 
-	const PIDBYPAGE = 50;
 }

bridges/SafebooruBridge.php

@@ -8,5 +8,9 @@ class SafebooruBridge extends GelbooruBridge {
 	const URI = 'https://safebooru.org/';
 	const DESCRIPTION = 'Returns images from given page';
 
-	const PIDBYPAGE = 40;
+	protected function buildThumbnailURI($element){
+		$regex = '/\.\w+$/';
+		return $this->getURI() . 'thumbnails/' . $element->directory
+		. '/thumbnail_' . preg_replace($regex, '.jpg', $element->image);
+	}
 }

bridges/ScribdBridge.php

@@ -10,7 +10,7 @@ class ScribdBridge extends BridgeAbstract {
 			'type' => 'text',
 			'required' => true,
 			'title' => 'Profile URL. Example: https://www.scribd.com/user/164147088/Ars-Technica',
-			'exampleValue' => 'https://www.scribd.com/user/'
+			'exampleValue' => 'https://www.scribd.com/user/164147088/Ars-Technica'
 		),
 	));
 

bridges/SupInfoBridge.php

@@ -1,57 +0,0 @@
-<?php
-class SupInfoBridge extends BridgeAbstract {
-
-	const MAINTAINER = 'teromene';
-	const NAME = 'SupInfoBridge';
-	const URI = 'https://www.supinfo.com';
-	const DESCRIPTION = 'Returns the newest articles.';
-
-	const PARAMETERS = array(array(
-		'tag' => array(
-			'name' => 'Category (not mandatory)',
-			'type' => 'text',
-		)
-	));
-
-	public function getIcon() {
-		return self::URI . '/favicon.png';
-	}
-
-	public function collectData() {
-
-		if(empty($this->getInput('tag'))) {
-			$html = getSimpleHTMLDOM(self::URI . '/articles/');
-		} else {
-			$html = getSimpleHTMLDOM(self::URI . '/articles/tag/' . $this->getInput('tag'));
-		}
-		$content = $html->find('#latest', 0)->find('ul[class=courseContent]', 0);
-
-		for($i = 0; $i < 5; $i++) {
-
-			$this->items[] = $this->fetchArticle($content->find('h4', $i)->find('a', 0)->href);
-
-		}
-	}
-
-	private function fetchArticle($link) {
-
-		$articleHTML = getSimpleHTMLDOM(self::URI . $link);
-
-		$article = $articleHTML->find('div[id=courseDocZero]', 0);
-		$item = array();
-		$item['author'] = $article->find('#courseMetas', 0)->find('a', 0)->plaintext;
-		$item['id'] = $link;
-		$item['uri'] = self::URI . $link;
-		$item['title'] = $article->find('h1', 0)->plaintext;
-		$date = explode(' ', $article->find('#courseMetas', 0)->find('span', 1)->plaintext);
-		$item['timestamp'] = DateTime::createFromFormat('d/m/Y H:i:s', $date[2] . ' ' . $date[4])->getTimestamp();
-
-		$article->find('div[id=courseHeader]', 0)->innertext = '';
-		$article->find('div[id=author-infos]', 0)->innertext = '';
-		$article->find('div[id=cartouche-tete]', 0)->innertext = '';
-		$item['content'] = $article;
-
-		return $item;
-
-	}
-}

bridges/TbibBridge.php

@@ -8,5 +8,9 @@ class TbibBridge extends GelbooruBridge {
 	const URI = 'https://tbib.org/';
 	const DESCRIPTION = 'Returns images from given page';
 
-	const PIDBYPAGE = 50;
+	protected function buildThumbnailURI($element){
+		$regex = '/\.\w+$/';
+		return $this->getURI() . 'thumbnails/' . $element->directory
+		. '/thumbnail_' . preg_replace($regex, '.jpg', $element->image);
+	}
 }

bridges/TelegramBridge.php

@@ -9,7 +9,7 @@ class TelegramBridge extends BridgeAbstract {
 				'name' => 'Username',
 				'type' => 'text',
 				'required' => true,
-				'exampleValue' => '@telegram',
+				'exampleValue' => '@rssbridge',
 			)
 		)
 	);
@@ -47,6 +47,7 @@ class TelegramBridge extends BridgeAbstract {
 			$html->find('div.tgme_channel_info_header_title span', 0)->plaintext,
 			ENT_QUOTES
 		);
+
 		$this->feedName = $channelTitle . ' (@' . $this->processUsername() . ')';
 
 		foreach($html->find('div.tgme_widget_message_wrap.js-widget_message_wrap') as $index => $messageDiv) {
@@ -68,7 +69,6 @@ class TelegramBridge extends BridgeAbstract {
 	}
 
 	public function getURI() {
-
 		if (!is_null($this->getInput('username'))) {
 			return self::URI . '/s/' . $this->processUsername();
 		}
@@ -77,7 +77,6 @@ class TelegramBridge extends BridgeAbstract {
 	}
 
 	public function getName() {
-
 		if (!empty($this->feedName)) {
 			return $this->feedName . ' - Telegram';
 		}
@@ -86,7 +85,6 @@ class TelegramBridge extends BridgeAbstract {
 	}
 
 	private function processUsername() {
-
 		if (substr($this->getInput('username'), 0, 1) === '@') {
 			return substr($this->getInput('username'), 1);
 		}
@@ -98,6 +96,11 @@ class TelegramBridge extends BridgeAbstract {
 		return $messageDiv->find('a.tgme_widget_message_date', 0)->href;
 	}
 
+	private function processDate($messageDiv) {
+		$messageMeta = $messageDiv->find('span.tgme_widget_message_meta', 0);
+		return $messageMeta->find('time', 0)->datetime;
+	}
+
 	private function processContent($messageDiv) {
 		$message = '';
 
@@ -106,7 +109,7 @@ class TelegramBridge extends BridgeAbstract {
 		}
 
 		if ($messageDiv->find('a.tgme_widget_message_reply', 0)) {
-			$message = $this->processReply($messageDiv);
+			$message .= $this->processReply($messageDiv);
 		}
 
 		if ($messageDiv->find('div.tgme_widget_message_sticker_wrap', 0)) {
@@ -136,46 +139,62 @@ class TelegramBridge extends BridgeAbstract {
 				$messageDiv->find('div.tgme_widget_message_text.js-message_text', 0)->plaintext
 			);
 		}
+
 		if ($messageDiv->find('div.tgme_widget_message_document', 0)) {
-			$message .= 'Attachments:';
-			foreach ($messageDiv->find('div.tgme_widget_message_document') as $attachments) {
-				$message .= $attachments->find('div.tgme_widget_message_document_title.accent_color', 0);
-			}
+			$message .= $this->processAttachment($messageDiv);
 		}
 
 		if ($messageDiv->find('a.tgme_widget_message_link_preview', 0)) {
 			$message .= $this->processLinkPreview($messageDiv);
 		}
 
+		if ($messageDiv->find('a.tgme_widget_message_location_wrap', 0)) {
+			$message .= $this->processLocation($messageDiv);
+		}
+
 		return $message;
 	}
 
 	private function processReply($messageDiv) {
-
 		$reply = $messageDiv->find('a.tgme_widget_message_reply', 0);
+		$author = $reply->find('span.tgme_widget_message_author_name', 0)->plaintext;
+		$text = '';
+
+		if ($reply->find('div.tgme_widget_message_metatext', 0)) {
+			$text = $reply->find('div.tgme_widget_message_metatext', 0)->innertext;
+		}
+
+		if ($reply->find('div.tgme_widget_message_text', 0)) {
+			$text = $reply->find('div.tgme_widget_message_text', 0)->innertext;
+		}
 
 		return <<<EOD
-<blockquote>{$reply->find('span.tgme_widget_message_author_name', 0)->plaintext}<br>
-{$reply->find('div.tgme_widget_message_text', 0)->innertext} 
+<blockquote>{$author}<br>
+{$text}
 <a href="{$reply->href}">{$reply->href}</a></blockquote><hr>
 EOD;
 	}
 
 	private function processSticker($messageDiv) {
-
 		if (empty($this->itemTitle)) {
 			$this->itemTitle = '@' . $this->processUsername() . ' posted a sticker';
 		}
 
 		$stickerDiv = $messageDiv->find('div.tgme_widget_message_sticker_wrap', 0);
 
-		preg_match($this->backgroundImageRegex, $stickerDiv->find('i', 0)->style, $sticker);
+		if ($stickerDiv->find('picture', 0)) {
+			$stickerDiv->find('picture', 0)->find('div', 0)->style = '';
+			$stickerDiv->find('picture', 0)->style = '';
 
-		$this->enclosures[] = $sticker[1];
+			return $stickerDiv;
 
-		return <<<EOD
-<a href="{$stickerDiv->children(0)->herf}"><img src="{$sticker[1]}"></a>
+		} elseif (preg_match($this->backgroundImageRegex, $stickerDiv->find('i', 0)->style, $sticker)) {
+			$this->enclosures[] = $sticker[1];
+
+			return <<<EOD
+				<a href="{$stickerDiv->children(0)->herf}"><img src="{$sticker[1]}"></a>
 EOD;
+		}
 	}
 
 	private function processPoll($messageDiv) {
@@ -203,7 +222,6 @@ EOD;
 	}
 
 	private function processLinkPreview($messageDiv) {
-
 		$image = '';
 		$title = '';
 		$site = '';
@@ -235,13 +253,12 @@ EOD;
 		}
 
 		return <<<EOD
-<blockquote><a href="{$preview->href}">$image</a><br><a href="{$preview->href}">
+<blockquote><a href="{$preview->href}">{$image}</a><br><a href="{$preview->href}">
 {$title} - {$site}</a><br>{$description}</blockquote>
 EOD;
 	}
 
 	private function processVideo($messageDiv) {
-
 		if (empty($this->itemTitle)) {
 			$this->itemTitle = '@' . $this->processUsername() . ' posted a video';
 		}
@@ -255,14 +272,13 @@ EOD;
 		$this->enclosures[] = $photo[1];
 
 		return <<<EOD
-<video controls="" poster="{$photo[1]}" preload="none">
+<video controls="" poster="{$photo[1]}" style="max-width:100%;" preload="none">
 	<source src="{$messageDiv->find('video', 0)->src}" type="video/mp4">
 </video>
 EOD;
 	}
 
 	private function processPhoto($messageDiv) {
-
 		if (empty($this->itemTitle)) {
 			$this->itemTitle = '@' . $this->processUsername() . ' posted a photo';
 		}
@@ -282,7 +298,6 @@ EOD;
 	}
 
 	private function processNotSupported($messageDiv) {
-
 		if (empty($this->itemTitle)) {
 			$this->itemTitle = '@' . $this->processUsername() . ' posted a video';
 		}
@@ -303,15 +318,40 @@ EOD;
 EOD;
 	}
 
-	private function processDate($messageDiv) {
+	private function processAttachment($messageDiv) {
+		$attachments = 'File attachments:<br>';
 
-		$messageMeta = $messageDiv->find('span.tgme_widget_message_meta', 0);
-		return $messageMeta->find('time', 0)->datetime;
+		if (empty($this->itemTitle)) {
+			$this->itemTitle = '@' . $this->processUsername() . ' posted an attachment';
+		}
 
+		foreach ($messageDiv->find('div.tgme_widget_message_document') as $document) {
+			$attachments .= <<<EOD
+{$document->find('div.tgme_widget_message_document_title', 0)->plaintext} -
+{$document->find('div.tgme_widget_message_document_extra', 0)->plaintext}<br>
+EOD;
+		}
+
+		return $attachments;
+	}
+
+	private function processLocation($messageDiv) {
+		if (empty($this->itemTitle)) {
+			$this->itemTitle = '@' . $this->processUsername() . ' posted a location';
+		}
+
+		preg_match($this->backgroundImageRegex, $messageDiv->find('div.tgme_widget_message_location', 0)->style, $image);
+
+		$link = $messageDiv->find('a.tgme_widget_message_location_wrap', 0)->href;
+
+		$this->enclosures[] = $image[1];
+
+		return <<<EOD
+			<a href="{$link}"><img src="{$image[1]}"></a>
+EOD;
 	}
 
 	private function ellipsisTitle($text) {
-
 		$length = 100;
 
 		if (strlen($text) > $length) {

bridges/ThingiverseBridge.php

@@ -1,164 +0,0 @@
-<?php
-class ThingiverseBridge extends BridgeAbstract {
-
-	const NAME = 'Thingiverse Search';
-	const URI = 'https://thingiverse.com';
-	const DESCRIPTION = 'Returns feeds for search results';
-	const MAINTAINER = 'AntoineTurmel';
-	const PARAMETERS = array(
-		array(
-			'query' => array(
-				'name' => 'Search query',
-				'type' => 'text',
-				'required' => true,
-				'title' => 'Insert your search term here',
-				'exampleValue' => 'vase'
-			),
-			'sortby' => array(
-				'name' => 'Sort by',
-				'type' => 'list',
-				'required' => false,
-				'values' => array(
-					'Relevant' => 'relevant',
-					'Text' => 'text',
-					'Popular' => 'popular',
-					'# of Makes' => 'makes',
-					'Newest' => 'newest',
-				),
-				'defaultValue' => 'newest'
-			),
-			'category' => array(
-				'name' => 'Category',
-				'type' => 'list',
-				'required' => false,
-				'values' => array(
-					'Any' => '',
-					'3D Printing' => '73',
-					'Art' => '63',
-					'Fashion' => '64',
-					'Gadgets' => '65',
-					'Hobby' => '66',
-					'Household' => '67',
-					'Learning' => '69',
-					'Models' => '70',
-					'Tools' => '71',
-					'Toys &amp; Games' => '72',
-					'2D Art' => '144',
-					'Art Tools' => '75',
-					'Coins &amp; Badges' => '143',
-					'Interactive Art' => '78',
-					'Math Art' => '79',
-					'Scans &amp; Replicas' => '145',
-					'Sculptures' => '80',
-					'Signs &amp; Logos' => '76',
-					'Accessories' => '81',
-					'Bracelets' => '82',
-					'Costume' => '142',
-					'Earrings' => '139',
-					'Glasses' => '83',
-					'Jewelry' => '84',
-					'Keychains' => '130',
-					'Rings' => '85',
-					'Audio' => '141',
-					'Camera' => '86',
-					'Computer' => '87',
-					'Mobile Phone' => '88',
-					'Tablet' => '90',
-					'Video Games' => '91',
-					'Automotive' => '155',
-					'DIY' => '93',
-					'Electronics' => '92',
-					'Music' => '94',
-					'R/C Vehicles' => '95',
-					'Robotics' => '96',
-					'Sport &amp; Outdoors' => '140',
-					'Bathroom' => '147',
-					'Containers' => '146',
-					'Decor' => '97',
-					'Household Supplies' => '99',
-					'Kitchen &amp; Dining' => '100',
-					'Office' => '101',
-					'Organization' => '102',
-					'Outdoor &amp; Garden' => '98',
-					'Pets' => '103',
-					'Replacement Parts' => '153',
-					'Biology' => '106',
-					'Engineering' => '104',
-					'Math' => '105',
-					'Physics &amp; Astronomy' => '148',
-					'Animals' => '107',
-					'Buildings &amp; Structures' => '108',
-					'Creatures' => '109',
-					'Food &amp; Drink' => '110',
-					'Model Furniture' => '111',
-					'Model Robots' => '115',
-					'People' => '112',
-					'Props' => '114',
-					'Vehicles' => '116',
-					'Hand Tools' => '118',
-					'Machine Tools' => '117',
-					'Parts' => '119',
-					'Tool Holders &amp; Boxes' => '120',
-					'Chess' => '151',
-					'Construction Toys' => '121',
-					'Dice' => '122',
-					'Games' => '123',
-					'Mechanical Toys' => '124',
-					'Playsets' => '113',
-					'Puzzles' => '125',
-					'Toy &amp; Game Accessories' => '149',
-					'3D Printer Accessories' => '127',
-					'3D Printer Extruders' => '152',
-					'3D Printer Parts' => '128',
-					'3D Printers' => '126',
-					'3D Printing Tests' => '129',
-				)
-			),
-			'showimage' => array(
-				'name' => 'Show image in content',
-				'type' => 'checkbox',
-				'required' => false,
-				'title' => 'Activate to show the image in the content',
-				'defaultValue' => 'checked'
-			)
-		)
-	);
-
-	public function collectData(){
-		$html = getSimpleHTMLDOM($this->getURI());
-
-		$results = $html->find('div.thing-card');
-
-		foreach($results as $result) {
-
-			$item = array();
-
-			$item['title'] = $result->find('span.ellipsis', 0);
-			$item['uri'] = self::URI . $result->find('a', 1)->href;
-			$item['author'] = $result->find('span.item-creator', 0);
-			$item['content'] = '';
-
-			$image = $result->find('img.card-img', 0)->src;
-
-			if($this->getInput('showimage')) {
-				$item['content'] .= '<img src="' . $image . '">';
-			}
-
-			$item['enclosures'] = array($image);
-
-			$this->items[] = $item;
-		}
-	}
-
-	public function getURI(){
-		if(!is_null($this->getInput('query'))) {
-			$uri = self::URI . '/search?q=' . urlencode($this->getInput('query'));
-			$uri .= '&sort=' . $this->getInput('sortby');
-			$uri .= '&category_id=' . $this->getInput('category');
-
-			return $uri;
-		}
-
-		return parent::getURI();
-	}
-}

bridges/TwitterBridge.php

@@ -30,7 +30,7 @@ class TwitterBridge extends BridgeAbstract {
 			'q' => array(
 				'name' => 'Keyword or #hashtag',
 				'required' => true,
-				'exampleValue' => 'rss-bridge',
+				'exampleValue' => 'rss-bridge OR rssbridge',
 				'title' => <<<EOD
 * To search for multiple words (must contain all of these words), put a space between them.
 
@@ -119,6 +119,10 @@ EOD
 		)
 	);
 
+	private $apiKey     = null;
+	private $guestToken = null;
+	private $authHeader = array();
+
 	public function detectParameters($url){
 		$params = array();
 
@@ -198,48 +202,65 @@ EOD
 		}
 	}
 
-	private function getApiURI() {
-		switch($this->queriedContext) {
-		case 'By keyword or hashtag':
-			return self::API_URI
-			. '/2/search/adaptive.json?q='
-			. urlencode($this->getInput('q'))
-			. '&tweet_mode=extended&tweet_search_mode=live';
-		case 'By username':
-			// use search endpoint if without replies or without retweets enabled
-			if ($this->getInput('noretweet') || $this->getInput('norep')) {
-				$query = 'from:' . $this->getInput('u');
-				// Twitter's from: search excludes retweets by default
-				if (!$this->getInput('noretweet')) $query .= ' include:nativeretweets';
-				if ($this->getInput('norep')) $query .= ' exclude:replies';
-				return self::API_URI
-				. '/2/search/adaptive.json?q='
-				. urlencode($query)
-				. '&tweet_mode=extended&tweet_search_mode=live';
-			} else {
-				return self::API_URI
-				. '/2/timeline/profile/'
-				. $this->getRestId($this->getInput('u'))
-				. '.json?tweet_mode=extended';
-			}
-		case 'By list':
-			return self::API_URI
-			. '/2/timeline/list.json?list_id='
-			. $this->getListId($this->getInput('user'), $this->getInput('list'))
-			. '&tweet_mode=extended';
-		case 'By list ID':
-			return self::API_URI
-			. '/2/timeline/list.json?list_id='
-			. $this->getInput('listid')
-			. '&tweet_mode=extended';
-		default: returnServerError('Invalid query context !');
-		}
-	}
-
 	public function collectData(){
-		$html = '';
-		$page = $this->getURI();
-		$data = json_decode($this->getApiContents($this->getApiURI()));
+		// $data will contain an array of all found tweets (unfiltered)
+		$data = null;
+		// Contains user data (when in by username context)
+		$user = null;
+		// Array of all found tweets
+		$tweets = array();
+
+		// Get authentication information
+		$this->getApiKey();
+
+		// Try to get all tweets
+		switch($this->queriedContext) {
+		case 'By username':
+			$user = $this->makeApiCall('/1.1/users/show.json', array('screen_name' => $this->getInput('u')));
+			if (!$user) {
+				returnServerError('Requested username can\'t be found.');
+			}
+
+			$params = array(
+				'user_id'       => $user->id_str,
+				'tweet_mode'    => 'extended'
+			);
+
+			$data = $this->makeApiCall('/1.1/statuses/user_timeline.json', $params);
+			break;
+
+		case 'By keyword or hashtag':
+			$params = array(
+				'q'                 => urlencode($this->getInput('q')),
+				'tweet_mode'        => 'extended',
+				'tweet_search_mode' => 'live',
+			);
+
+			$data = $this->makeApiCall('/1.1/search/tweets.json', $params)->statuses;
+			break;
+
+		case 'By list':
+			$params = array(
+				'slug'              => strtolower($this->getInput('list')),
+				'owner_screen_name' => strtolower($this->getInput('user')),
+				'tweet_mode'        => 'extended',
+			);
+
+			$data = $this->makeApiCall('/1.1/lists/statuses.json', $params);
+			break;
+
+		case 'By list ID':
+			$params = array(
+				'list_id'           => $this->getInput('listid'),
+				'tweet_mode'        => 'extended',
+			);
+
+			$data = $this->makeApiCall('/1.1/lists/statuses.json', $params);
+			break;
+
+		default:
+			returnServerError('Invalid query context !');
+		}
 
 		if(!$data) {
 			switch($this->queriedContext) {
@@ -252,65 +273,33 @@ EOD
 			}
 		}
 
-		$hidePictures = $this->getInput('nopic');
+		// Filter out unwanted tweets
+		foreach ($data as $tweet) {
+			// Filter out retweets to remove possible duplicates of original tweet
+			switch($this->queriedContext) {
+			case 'By keyword or hashtag':
+				if (isset($tweet->retweeted_status) && substr($tweet->full_text, 0, 4) === 'RT @') {
+					continue 2;
+				}
+				break;
+			}
+			$tweets[] = $tweet;
+		}
 
-		$promotedTweetIds = array_reduce($data->timeline->instructions[0]->addEntries->entries, function($carry, $entry) {
-			if (!isset($entry->content->item)) {
-				return $carry;
-			}
-			$tweet = $entry->content->item->content->tweet;
-			if (isset($tweet->promotedMetadata)) {
-				$carry[] = $tweet->id;
-			}
-			return $carry;
-		}, array());
+		$hidePictures = $this->getInput('nopic');
 
 		$hidePinned = $this->getInput('nopinned');
 		if ($hidePinned) {
 			$pinnedTweetId = null;
-			if (isset($data->timeline->instructions[1]) && isset($data->timeline->instructions[1]->pinEntry)) {
-				$pinnedTweetId = $data->timeline->instructions[1]->pinEntry->entry->content->item->content->tweet->id;
-			}
-		}
-
-		$tweets = array();
-
-		// Extract tweets from timeline property when in username mode
-		// This fixes number of issues:
-		// * If there's a retweet of a quote tweet, the quoted tweet will not appear in results (since it wasn't retweeted directly)
-		// * Pinned tweets do not get stuck at the bottom
-		if ($this->queriedContext === 'By username') {
-			foreach($data->timeline->instructions[0]->addEntries->entries as $tweet) {
-				if (!isset($tweet->content->item)) continue;
-				$tweetId = $tweet->content->item->content->tweet->id;
-				$selectedTweet = $this->getTweet($tweetId, $data->globalObjects);
-				if (!$selectedTweet) continue;
-				// If this is a retweet, it will contain shorter text and will point to the original full tweet (retweeted_status_id_str).
-				// Let's use the original tweet text.
-				if (isset($selectedTweet->retweeted_status_id_str)) {
-					$tweetId = $selectedTweet->retweeted_status_id_str;
-					$selectedTweet = $this->getTweet($tweetId, $data->globalObjects);
-					if (!$selectedTweet) continue;
-				}
-				// use $tweetId as key to avoid duplicates (e.g. user retweeting their own tweet)
-				$tweets[$tweetId] = $selectedTweet;
-			}
-		} else {
-			foreach($data->globalObjects->tweets as $tweet) {
-				$tweets[] = $tweet;
+			if ($user && $user->pinned_tweet_ids_str) {
+				$pinnedTweetId = $user->pinned_tweet_ids_str;
 			}
 		}
 
 		foreach($tweets as $tweet) {
 
-			/* Debug::log('>>> ' . json_encode($tweet)); */
-			// Skip spurious retweets
-			if (isset($tweet->retweeted_status_id_str) && substr($tweet->full_text, 0, 4) === 'RT @') {
-				continue;
-			}
-
-			// Skip promoted tweets
-			if (in_array($tweet->id_str, $promotedTweetIds)) {
+			// Skip own Retweets...
+			if (isset($tweet->retweeted_status) && $tweet->retweeted_status->user->id_str === $tweet->user->id_str) {
 				continue;
 			}
 
@@ -319,37 +308,52 @@ EOD
 				continue;
 			}
 
-			$item = array();
-			// extract username and sanitize
-			$user_info = $this->getUserInformation($tweet->user_id_str, $data->globalObjects);
-
-			$item['username'] = $user_info->screen_name;
-			$item['fullname'] = $user_info->name;
-			$item['author'] = $item['fullname'] . ' (@' . $item['username'] . ')';
-			if (null !== $this->getInput('u') && strtolower($item['username']) != strtolower($this->getInput('u'))) {
-				$item['author'] .= ' RT: @' . $this->getInput('u');
+			switch($this->queriedContext) {
+				case 'By username':
+					if ($this->getInput('norep') && isset($tweet->in_reply_to_status_id))
+						continue 2;
+					break;
 			}
-			$item['avatar'] = $user_info->profile_image_url_https;
 
-			$item['id'] = $tweet->id_str;
-			$item['uri'] = self::URI . $item['username'] . '/status/' . $item['id'];
-			// extract tweet timestamp
-			$item['timestamp'] = $tweet->created_at;
+			$item = array();
+
+			$realtweet = $tweet;
+			if (isset($tweet->retweeted_status)) {
+				// Tweet is a Retweet, so set author based on original tweet and set realtweet for reference to the right content
+				$realtweet = $tweet->retweeted_status;
+			}
+
+			$item['username']  = $realtweet->user->screen_name;
+			$item['fullname']  = $realtweet->user->name;
+			$item['avatar']    = $realtweet->user->profile_image_url_https;
+			$item['timestamp'] = $realtweet->created_at;
+			$item['id']        = $realtweet->id_str;
+			$item['uri']       = self::URI . $item['username'] . '/status/' . $item['id'];
+			$item['author']    = (isset($tweet->retweeted_status) ? 'RT: ' : '' )
+						 . $item['fullname']
+						 . ' (@'
+						 . $item['username'] . ')';
 
 			// Convert plain text URLs into HTML hyperlinks
-			$cleanedTweet = $tweet->full_text;
+			$fulltext = $realtweet->full_text;
+			$cleanedTweet = $fulltext;
+
 			$foundUrls = false;
 
-			if (isset($tweet->entities->media)) {
-				foreach($tweet->entities->media as $media) {
+			if (substr($cleanedTweet, 0, 4) === 'RT @') {
+				$cleanedTweet = substr($cleanedTweet, 3);
+			}
+
+			if (isset($realtweet->entities->media)) {
+				foreach($realtweet->entities->media as $media) {
 					$cleanedTweet = str_replace($media->url,
 						'<a href="' . $media->expanded_url . '">' . $media->display_url . '</a>',
 						$cleanedTweet);
 					$foundUrls = true;
 				}
 			}
-			if (isset($tweet->entities->urls)) {
-				foreach($tweet->entities->urls as $url) {
+			if (isset($realtweet->entities->urls)) {
+				foreach($realtweet->entities->urls as $url) {
 					$cleanedTweet = str_replace($url->url,
 						'<a href="' . $url->expanded_url . '">' . $url->display_url . '</a>',
 						$cleanedTweet);
@@ -359,7 +363,7 @@ EOD
 			if ($foundUrls === false) {
 				// fallback to regex'es
 				$reg_ex = '/(http|https|ftp|ftps)\:\/\/[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(\/\S*)?/';
-				if(preg_match($reg_ex, $tweet->full_text, $url)) {
+				if(preg_match($reg_ex, $realtweet->full_text, $url)) {
 					$cleanedTweet = preg_replace($reg_ex,
 						"<a href='{$url[0]}' target='_blank'>{$url[0]}</a> ",
 						$cleanedTweet);
@@ -384,8 +388,8 @@ EOD;
 
 			// Get images
 			$media_html = '';
-			if(isset($tweet->extended_entities->media) && !$this->getInput('noimg')) {
-				foreach($tweet->extended_entities->media as $media) {
+			if(isset($realtweet->extended_entities->media) && !$this->getInput('noimg')) {
+				foreach($realtweet->extended_entities->media as $media) {
 					switch($media->type) {
 					case 'photo':
 						$image = $media->media_url_https . '?name=orig';
@@ -480,7 +484,7 @@ EOD;
 
 	//The aim of this function is to get an API key and a guest token
 	//This function takes 2 requests, and therefore is cached
-	private function getApiKey() {
+	private function getApiKey($forceNew = 0) {
 
 		$cacheFac = new CacheFactory();
 		$cacheFac->setWorkingDir(PATH_LIB_CACHES);
@@ -505,7 +509,7 @@ EOD;
 		$data = $cache->loadData();
 
 		$apiKey = null;
-		if($data === null || (time() - $refresh) > self::GUEST_TOKEN_EXPIRY) {
+		if($forceNew || $data === null || (time() - $refresh) > self::GUEST_TOKEN_EXPIRY) {
 			$twitterPage = getContents('https://twitter.com');
 
 			$jsLink = false;
@@ -542,7 +546,7 @@ EOD;
 		$guestTokenUses = $gt_cache->loadData();
 
 		$guestToken = null;
-		if($guestTokenUses === null || !is_array($guestTokenUses) || count($guestTokenUses) != 2
+		if($forceNew || $guestTokenUses === null || !is_array($guestTokenUses) || count($guestTokenUses) != 2
 		|| $guestTokenUses[0] <= 0 || (time() - $refresh) > self::GUEST_TOKEN_EXPIRY) {
 			$guestToken = $this->getGuestToken($apiKey);
 			if ($guestToken === null) {
@@ -561,8 +565,14 @@ EOD;
 			$guestToken = $guestTokenUses[1];
 		}
 
-		return array($apiKey, $guestToken);
+		$this->apiKey	   = $apiKey;
+		$this->guestToken  = $guestToken;
+		$this->authHeaders = array(
+			'authorization: Bearer ' . $apiKey,
+			'x-guest-token: ' . $guestToken,
+		);
 
+		return array($apiKey, $guestToken);
 	}
 
 	// Get a guest token. This is different to an API key,
@@ -584,47 +594,48 @@ EOD;
 		return $guestToken;
 	}
 
-	private function getApiContents($uri) {
-		$apiKeys = $this->getApiKey();
-		$headers = array('authorization: Bearer ' . $apiKeys[0],
-				 'x-guest-token: ' . $apiKeys[1],
-			   );
-		return getContents($uri, $headers);
-	}
+	/**
+	 * Tries to make an API call to twitter.
+	 * @param $api string API entry point
+	 * @param $params array additional URI parmaeters
+	 * @return object json data
+	 */
+	private function makeApiCall($api, $params) {
+		$uri = self::API_URI . $api . '?' . http_build_query($params);
 
-	private function getRestId($username) {
-		$searchparams = urlencode('{"screen_name":"' . strtolower($username) . '", "withHighlightedLabel":true}');
-		$searchURL = self::API_URI . '/graphql/-xfUfZsnR_zqjFd-IfrN5A/UserByScreenName?variables=' . $searchparams;
-		$searchResult = $this->getApiContents($searchURL);
-		$searchResult = json_decode($searchResult);
-		return $searchResult->data->user->rest_id;
-	}
+		$retries = 1;
+		$retry = 0;
+		do {
+			$retry = 0;
 
-	private function getListId($username, $listName) {
-		$searchparams = urlencode('{"screenName":"'
-				. strtolower($username)
-				. '", "listSlug": "'
-				. $listName
-				. '", "withHighlightedLabel":false}');
-		$searchURL = self::API_URI . '/graphql/ErWsz9cObLel1BF-HjuBlA/ListBySlug?variables=' . $searchparams;
-		$searchResult = $this->getApiContents($searchURL);
-		$searchResult = json_decode($searchResult);
-		return $searchResult->data->user_by_screen_name->list->id_str;
-	}
-
-	private function getUserInformation($userId, $apiData) {
-		foreach($apiData->users as $user) {
-			if($user->id_str == $userId) {
-				return $user;
+			try {
+				$result = getContents($uri, $this->authHeaders, array(), true);
+			} catch (UnexpectedResponseException $e) {
+				switch ($e->getResponseCode()) {
+				case 401:
+				case 403:
+					if ($retries) {
+						$retries--;
+						$retry = 1;
+						$this->getApiKey(1);
+						continue 2;
+					}
+				default:
+					$code = $e->getResponseCode();
+					$data = $e->getResponseBody();
+					returnServerError(<<<EOD
+Failed to make api call: $api
+HTTP Status: $code
+Errormessage: $data
+EOD
+					);
+					break;
+				}
 			}
-		}
-	}
+		} while ($retry);
 
-	private function getTweet($tweetId, $apiData) {
-		if (property_exists($apiData->tweets, $tweetId)) {
-			return $apiData->tweets->$tweetId;
-		} else {
-			return null;
-		}
+		$data = json_decode($result['content']);
+
+		return $data;
 	}
 }

bridges/TwitterV2Bridge.php

@@ -0,0 +1,592 @@
+<?php
+/**
+ * TwitterV2Bridge leverages Twitter API v2, and requires
+ * a unique API Bearer Token, which requires creation of
+ * a Twitter Dev account. Link to instructions in DESCRIPTION.
+ */
+class TwitterV2Bridge extends BridgeAbstract {
+	const NAME = 'Twitter V2 Bridge';
+	const URI = 'https://twitter.com/';
+	const API_URI = 'https://api.twitter.com/2';
+	const DESCRIPTION = 'Returns tweets (using Twitter API v2). See the 
+	<a href="https://rss-bridge.github.io/rss-bridge/Bridge_Specific/TwitterV2.html">
+	Configuration Instructions</a>.';
+	const MAINTAINER = 'quickwick';
+	const CONFIGURATION = array(
+		'twitterv2apitoken' => array(
+			'required' => true,
+		)
+	);
+	const PARAMETERS = array(
+		'global' => array(
+			'filter' => array(
+				'name' => 'Filter',
+				'exampleValue' => 'rss-bridge',
+				'required' => false,
+				'title' => 'Specify a single term to search for'
+			),
+			'norep' => array(
+				'name' => 'Without replies',
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude reply tweets'
+			),
+			'noretweet' => array(
+				'name' => 'Without retweets',
+				'required' => false,
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude retweets'
+			),
+			'nopinned' => array(
+				'name' => 'Without pinned tweet',
+				'required' => false,
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude pinned tweets'
+			),
+			'maxresults' => array(
+				'name' => 'Maximum results',
+				'required' => false,
+				'exampleValue' => '20',
+				'title' => 'Maximum number of tweets to retrieve (limit is 100)'
+			),
+			'imgonly' => array(
+				'name' => 'Only media tweets',
+				'type' => 'checkbox',
+				'title' => 'Activate to show only tweets with media (photo/video)'
+			),
+			'nopic' => array(
+				'name' => 'Hide profile pictures',
+				'type' => 'checkbox',
+				'title' => 'Activate to hide profile pictures in content'
+			),
+			'noimg' => array(
+				'name' => 'Hide images in tweets',
+				'type' => 'checkbox',
+				'title' => 'Activate to hide images in tweets'
+			),
+			'noimgscaling' => array(
+				'name' => 'Disable image scaling',
+				'type' => 'checkbox',
+				'title' => 'Activate to display original sized images (no thumbnails)'
+			),
+			'idastitle' => array(
+				'name' => 'Use tweet id as title',
+				'type' => 'checkbox',
+				'title' => 'Activate to use tweet id as title (instead of tweet text)'
+			)
+		),
+		'By username' => array(
+			'u' => array(
+				'name' => 'username',
+				'required' => true,
+				'exampleValue' => 'sebsauvage',
+				'title' => 'Insert a user name'
+			)
+		),
+		'By keyword or hashtag' => array(
+			'query' => array(
+				'name' => 'Keyword or #hashtag',
+				'required' => true,
+				'exampleValue' => 'rss-bridge OR #rss-bridge',
+				'title' => <<<EOD
+* To search for multiple words (must contain all of these words), put a space between them.
+
+Example: `rss-bridge release`.
+
+* To search for multiple words (contains any of these words), put "OR" between them.
+
+Example: `rss-bridge OR rssbridge`.
+
+* To search for an exact phrase (including whitespace), put double-quotes around them.
+
+Example: `"rss-bridge release"`
+
+* If you want to search for anything **but** a specific word, put a hyphen before it.
+
+Example: `rss-bridge -release` (ignores "release")
+
+* Of course, this also works for hashtags.
+
+Example: `#rss-bridge OR #rssbridge`
+
+* And you can combine them in any shape or form you like.
+
+Example: `#rss-bridge OR #rssbridge -release`
+EOD
+			)
+		),
+		'By list ID' => array(
+			'listid' => array(
+				'name' => 'List ID',
+				'exampleValue' => '31748',
+				'required' => true,
+				'title' => 'Enter a list id'
+			)
+		)
+	);
+
+	private $apiToken     = null;
+	private $authHeaders = array();
+
+	public function getName() {
+		switch($this->queriedContext) {
+			case 'By keyword or hashtag':
+				$specific = 'search ';
+				$param = 'query';
+				break;
+			case 'By username':
+				$specific = '@';
+				$param = 'u';
+				break;
+			case 'By list ID':
+				return 'Twitter List #' . $this->getInput('listid');
+			default:
+				return parent::getName();
+		}
+		return 'Twitter ' . $specific . $this->getInput($param);
+	}
+
+	public function collectData() {
+		// $data will contain an array of all found tweets
+		$data = null;
+		// Contains user data (when in by username context)
+		$user = null;
+		// Array of all found tweets
+		$tweets = array();
+
+		$hideProfilePic = $this->getInput('nopic');
+		$hideImages = $this->getInput('noimg');
+		$hideReplies = $this->getInput('norep');
+		$hideRetweets = $this->getInput('noretweet');
+		$hidePinned = $this->getInput('nopinned');
+		$tweetFilter = $this->getInput('filter');
+		$maxResults = $this->getInput('maxresults');
+		if ($maxResults > 100) {
+			$maxResults = 100;
+		}
+		$idAsTitle = $this->getInput('idastitle');
+		$onlyMediaTweets = $this->getInput('imgonly');
+
+		// Read API token from config.ini.php, put into Header
+		$this->apiToken     = $this->getOption('twitterv2apitoken');
+		$this->authHeaders = array(
+			'authorization: Bearer ' . $this->apiToken,
+		);
+
+		// Try to get all tweets
+		switch($this->queriedContext) {
+		case 'By username':
+			//Get id from username
+			$params = array(
+				'user.fields'	=> 'pinned_tweet_id,profile_image_url'
+			);
+			$user = $this->makeApiCall('/users/by/username/'
+			. $this->getInput('u'), $params);
+
+			if(isset($user->errors)) {
+				Debug::log('User JSON: ' . json_encode($user));
+				returnServerError('Requested username can\'t be found.');
+			}
+
+			// Set default params
+			$params = array(
+				'max_results'	=> (empty($maxResults) ? '10' : $maxResults ),
+				'tweet.fields'
+				=> 'created_at,referenced_tweets,entities,attachments',
+				'user.fields'	=> 'pinned_tweet_id',
+				'expansions'
+				=> 'referenced_tweets.id.author_id,entities.mentions.username,attachments.media_keys',
+				'media.fields'	=> 'type,url,preview_image_url'
+			);
+
+			// Set params to filter out replies and/or retweets
+			if($hideReplies && $hideRetweets) {
+				$params['exclude'] = 'replies,retweets';
+			} elseif($hideReplies) {
+				$params['exclude'] = 'replies';
+			} elseif($hideRetweets) {
+				$params['exclude'] = 'retweets';
+			}
+
+			// Get the tweets
+			$data = $this->makeApiCall('/users/' . $user->data->id
+			. '/tweets', $params);
+			break;
+
+		case 'By keyword or hashtag':
+			$params = array(
+				'query'			=> $this->getInput('query'),
+				'max_results'	=> (empty($maxResults) ? '10' : $maxResults ),
+				'tweet.fields'
+				=> 'created_at,referenced_tweets,entities,attachments',
+				'expansions'
+				=> 'referenced_tweets.id.author_id,entities.mentions.username,attachments.media_keys',
+				'media.fields'	=> 'type,url,preview_image_url'
+			);
+
+			// Set params to filter out replies and/or retweets
+			if($hideReplies) {
+				$params['query'] = $params['query'] . ' -is:reply';
+			}
+			if($hideRetweets) {
+				$params['query'] = $params['query'] . ' -is:retweet';
+			}
+
+			$data = $this->makeApiCall('/tweets/search/recent', $params);
+			break;
+
+		case 'By list ID':
+			// Set default params
+			$params = array(
+				'max_results' => (empty($maxResults) ? '10' : $maxResults ),
+				'tweet.fields'
+				=> 'created_at,referenced_tweets,entities,attachments',
+				'expansions'
+				=> 'referenced_tweets.id.author_id,entities.mentions.username,attachments.media_keys',
+				'media.fields'	=> 'type,url,preview_image_url'
+			);
+
+			$data = $this->makeApiCall('/lists/' . $this->getInput('listid') .
+			'/tweets', $params);
+			break;
+
+		default:
+			returnServerError('Invalid query context !');
+		}
+
+		if((isset($data->errors) && !isset($data->data)) ||
+		(isset($data->meta) && $data->meta->result_count === 0)) {
+			Debug::log('Data JSON: ' . json_encode($data));
+			switch($this->queriedContext) {
+			case 'By keyword or hashtag':
+				returnServerError('No results for this query.');
+			case 'By username':
+				returnServerError('Requested username cannnot be found.');
+			case 'By list ID':
+				returnServerError('Requested list cannnot be found');
+			}
+		}
+
+		// figure out the Pinned Tweet Id
+		if($hidePinned) {
+			$pinnedTweetId = null;
+			if(isset($user) && isset($user->data->pinned_tweet_id)) {
+				$pinnedTweetId = $user->data->pinned_tweet_id;
+			}
+		}
+
+		// Extract Media data into array
+		isset($data->includes->media) ? $includesMedia = $data->includes->media : $includesMedia = null;
+
+		// Extract additional Users data into array
+		isset($data->includes->users) ? $includesUsers = $data->includes->users : $includesUsers = null;
+
+		// Extract additional Tweets data into array
+		isset($data->includes->tweets) ? $includesTweets = $data->includes->tweets : $includesTweets = null;
+
+		// Extract main Tweets data into array
+		$tweets = $data->data;
+
+		// Make another API call to get user and media info for retweets
+		// Is there some way to get this info included in original API call?
+		$retweetedData = null;
+		$retweetedMedia = null;
+		$retweetedUsers = null;
+		if(!$hideImages && !$hideRetweets && isset($includesTweets)) {
+			// There has to be a better PHP way to extract the tweet Ids?
+			$includesTweetsIds = array();
+			foreach($includesTweets as $includesTweet) {
+				$includesTweetsIds[] = $includesTweet->id;
+			}
+			//Debug::log('includesTweetsIds: ' . join(',', $includesTweetsIds));
+
+			// Set default params for API query
+			$params = array(
+				'ids'			=> join(',', $includesTweetsIds),
+				'tweet.fields'  => 'entities,attachments',
+				'expansions'	=> 'author_id,attachments.media_keys',
+				'media.fields'	=> 'type,url,preview_image_url',
+				'user.fields'	=> 'id,profile_image_url'
+			);
+
+			// Get the retweeted tweets
+			$retweetedData = $this->makeApiCall('/tweets', $params);
+
+			// Extract retweets Media data into array
+			isset($retweetedData->includes->media) ? $retweetedMedia
+			= $retweetedData->includes->media : $retweetedMedia = null;
+
+			// Extract retweets additional Users data into array
+			isset($retweetedData->includes->users) ? $retweetedUsers
+			= $retweetedData->includes->users : $retweetedUsers = null;
+		}
+
+		// Create output array with all required elements for each tweet
+		foreach($tweets as $tweet) {
+			//Debug::log('Tweet JSON: ' . json_encode($tweet));
+
+			// Skip pinned tweet (if selected)
+			if($hidePinned && $tweet->id === $pinnedTweetId) {
+				continue;
+			}
+
+			// Check if Retweet or Reply
+			$retweetTypes = array('retweeted', 'quoted');
+			$isRetweet = false;
+			$isReply = false;
+			if(isset($tweet->referenced_tweets)) {
+				if(in_array($tweet->referenced_tweets[0]->type, $retweetTypes)) {
+					$isRetweet = true;
+				} elseif ($tweet->referenced_tweets[0]->type === 'replied_to') {
+					$isReply = true;
+				}
+			}
+
+			// Skip replies and/or retweets (if selected). This check is primarily for lists
+			// These should already be pre-filtered for username and keyword queries
+			if (($hideRetweets && $isRetweet) || ($hideReplies && $isReply)) {
+				continue;
+			}
+
+			$cleanedTweet = $tweet->text;
+
+			// Perform filtering (skip tweets that don't contain desired word, if provided)
+			if (! empty($tweetFilter)) {
+				if(stripos($cleanedTweet, $this->getInput('filter')) === false) {
+					continue;
+				}
+			}
+
+			// Initialize empty array to hold eventual HTML output
+			$item = array();
+
+			// Start setting values needed for HTML output
+			if($isRetweet || is_null($user)) {
+				// Replace tweet object with original retweeted object
+				if($isRetweet) {
+					foreach($includesTweets as $includesTweet) {
+						if($includesTweet->id === $tweet->referenced_tweets[0]->id) {
+							$tweet = $includesTweet;
+							break;
+						}
+					}
+				}
+
+				// Skip self-Retweets (can cause duplicate entries in output)
+				if(isset($user) && $tweet->author_id === $user->data->id) {
+					continue;
+				}
+
+				// Get user object for retweeted tweet
+				$originalUser = new stdClass(); // make the linters stop complaining
+				if(isset($retweetedUsers)) {
+					foreach($retweetedUsers as $retweetedUser) {
+						if($retweetedUser->id === $tweet->author_id) {
+							$originalUser = $retweetedUser;
+							break;
+						}
+					}
+				}
+				if(isset($includesUsers)) {
+					foreach($includesUsers as $includesUser) {
+						if($includesUser->id === $tweet->author_id) {
+							$originalUser = $includesUser;
+							break;
+						}
+					}
+				}
+
+				$item['username']  = $originalUser->username;
+				$item['fullname']  = $originalUser->name;
+				if(isset($originalUser->profile_image_url)) {
+					$item['avatar']    = $originalUser->profile_image_url;
+				} else{
+					$item['avatar'] = null;
+				}
+			} else{
+				$item['username']  = $user->data->username;
+				$item['fullname']  = $user->data->name;
+				$item['avatar']    = $user->data->profile_image_url;
+			}
+			$item['id']        = $tweet->id;
+			$item['timestamp'] = $tweet->created_at;
+			$item['uri']
+			= self::URI . $item['username'] . '/status/' . $item['id'];
+			$item['author']    = ($isRetweet ? 'RT: ' : '' )
+						 . $item['fullname']
+						 . ' (@'
+						 . $item['username'] . ')';
+
+			// Skip non-media tweet (if selected)
+			// This check must wait until after retweets are identified
+			if ($onlyMediaTweets && !isset($tweet->attachments->media_keys)) {
+				continue;
+			}
+
+			// Search for and replace URLs in Tweet text
+			$foundUrls = false;
+			if(isset($tweet->entities->urls)) {
+				foreach($tweet->entities->urls as $url) {
+					$cleanedTweet = str_replace($url->url,
+						'<a href="' . $url->expanded_url
+						. '">' . $url->display_url . '</a>',
+						$cleanedTweet);
+					$foundUrls = true;
+				}
+			}
+			if($foundUrls === false) {
+				// fallback to regex'es
+				$reg_ex = '/(http|https|ftp|ftps)\:\/\/[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(\/\S*)?/';
+				if(preg_match($reg_ex, $cleanedTweet, $url)) {
+					$cleanedTweet = preg_replace($reg_ex,
+						"<a href='{$url[0]}' target='_blank'>{$url[0]}</a> ",
+						$cleanedTweet);
+				}
+			}
+
+			// generate the title
+			if ($idAsTitle) {
+				$titleText = $tweet->id;
+			} else{
+				$titleText = strip_tags($cleanedTweet);
+			}
+
+			if($isRetweet && substr($titleText, 0, 4) === 'RT @') {
+				$titleText = substr_replace($titleText, ':', 2, 0 );
+			} elseif ($isReply  && !$idAsTitle) {
+				$titleText = 'R: ' . $titleText;
+			}
+
+			$item['title'] = $titleText;
+
+			// Add avatar
+			$picture_html = '';
+			if(!$hideProfilePic && isset($item['avatar'])) {
+				$picture_html = <<<EOD
+<a href="https://twitter.com/{$item['username']}">
+<img
+	style="align:top; width:75px; border:1px solid black;"
+	alt="{$item['username']}"
+	src="{$item['avatar']}"
+	title="{$item['fullname']}" />
+</a>
+EOD;
+			}
+
+			// Get images
+			$media_html = '';
+			if(!$hideImages && isset($tweet->attachments->media_keys)) {
+
+				// Match media_keys in tweet to media list from, put matches
+				// into new array
+				$tweetMedia = array();
+				// Start by checking the original list of tweet Media includes
+				if(isset($includesMedia)) {
+					foreach($includesMedia as $includesMedium) {
+						if(in_array ($includesMedium->media_key,
+						$tweet->attachments->media_keys)) {
+							$tweetMedia[] = $includesMedium;
+						}
+					}
+				}
+				// If no matches found, check the retweet Media includes
+				if(empty($tweetMedia) && isset($retweetedMedia)) {
+					foreach($retweetedMedia as $retweetedMedium) {
+						if(in_array ($retweetedMedium->media_key,
+						$tweet->attachments->media_keys)) {
+							$tweetMedia[] = $retweetedMedium;
+						}
+					}
+				}
+
+				foreach($tweetMedia as $media) {
+					switch($media->type) {
+					case 'photo':
+						if ($this->getInput('noimgscaling')) {
+							$image = $media->url;
+							$display_image = $media->url;
+						} else{
+							$image = $media->url . '?name=orig';
+							$display_image = $media->url . '?name=thumb';
+						}
+						// add enclosures
+						$item['enclosures'][] = $image;
+
+						$media_html .= <<<EOD
+<a href="{$image}">
+<img
+	style="align:top; max-width:558px; border:1px solid black;"
+	referrerpolicy="no-referrer"
+	src="{$display_image}" />
+</a>
+EOD;
+						break;
+					case 'video':
+						// To Do: Is there a way to easily match this
+						// to a URL for a link?
+						$display_image = $media->preview_image_url;
+
+						$media_html .= <<<EOD
+<img
+	style="align:top; max-width:558px; border:1px solid black;"
+	referrerpolicy="no-referrer"
+	src="{$display_image}" />
+EOD;
+						break;
+					case 'animated_gif':
+						// To Do: Is there a way to easily match this to a
+						// URL for a link?
+						$display_image = $media->preview_image_url;
+
+						$media_html .= <<<EOD
+<img
+	style="align:top; max-width:558px; border:1px solid black;"
+	referrerpolicy="no-referrer"
+	src="{$display_image}" />
+EOD;
+						break;
+					default:
+						Debug::log('Missing support for media type: '
+						. $media->type);
+					}
+				}
+			}
+
+			$item['content'] = <<<EOD
+<div style="display: inline-block; vertical-align: top;">
+	{$picture_html}
+</div>
+<div style="display: inline-block; vertical-align: top;">
+	<blockquote>{$cleanedTweet}</blockquote>
+</div>
+<div style="display: block; vertical-align: top;">
+	<blockquote>{$media_html}</blockquote>
+</div>
+EOD;
+
+			$item['content'] = htmlspecialchars_decode($item['content'], ENT_QUOTES);
+
+			// put out item
+			$this->items[] = $item;
+		}
+
+		// Sort all tweets in array by date
+		usort($this->items, array('TwitterV2Bridge', 'compareTweetDate'));
+	}
+
+	private static function compareTweetDate($tweet1, $tweet2) {
+		return (strtotime($tweet1['timestamp']) < strtotime($tweet2['timestamp']) ? 1 : -1);
+	}
+
+	/**
+	 * Tries to make an API call to Twitter.
+	 * @param $api string API entry point
+	 * @param $params array additional URI parmaeters
+	 * @return object json data
+	 */
+	private function makeApiCall($api, $params) {
+		$uri = self::API_URI . $api . '?' . http_build_query($params);
+		$result = getContents($uri, $this->authHeaders, array(), false);
+		$data = json_decode($result);
+		return $data;
+	}
+}

bridges/UnsplashBridge.php

@@ -1,67 +1,113 @@
 <?php
-class UnsplashBridge extends BridgeAbstract {
 
-	const MAINTAINER = 'nel50n';
+class UnsplashBridge extends BridgeAbstract
+{
+	const MAINTAINER = 'nel50n, langfingaz';
 	const NAME = 'Unsplash Bridge';
 	const URI = 'https://unsplash.com/';
 	const CACHE_TIMEOUT = 43200; // 12h
-	const DESCRIPTION = 'Returns the latests photos from Unsplash';
+	const DESCRIPTION = 'Returns the latest photos from Unsplash';
 
-	const PARAMETERS = array( array(
+	const PARAMETERS = array(array(
+		'u' => array(
+			'name' => 'Filter by username (optional)',
+			'type' => 'text',
+			'defaultValue' => 'unsplash'
+		),
 		'm' => array(
 			'name' => 'Max number of photos',
 			'type' => 'number',
-			'defaultValue' => 20
+			'defaultValue' => 20,
+			'required' => true
+		),
+		'prev_q' => array(
+			'name' => 'Preview quality',
+			'type' => 'list',
+			'values' => array(
+				'full' => 'full',
+				'regular' => 'regular',
+				'small' => 'small',
+				'thumb' => 'thumb',
+			),
+			'defaultValue' => 'regular'
 		),
 		'w' => array(
-			'name' => 'Width',
-			'exampleValue' => '1920, 1680, …',
-			'defaultValue' => '1920'
-		),
-		'q' => array(
-			'name' => 'JPEG quality',
+			'name' => 'Max download width (optional)',
+			'exampleValue' => 1920,
 			'type' => 'number',
-			'defaultValue' => 75
+			'defaultValue' => 1920,
+		),
+		'jpg_q' => array(
+			'name' => 'Max JPEG quality (optional)',
+			'exampleValue' => 75,
+			'type' => 'number',
+			'defaultValue' => 75,
 		)
 	));
 
-	public function collectData(){
+	public function collectData()
+	{
+		$filteredUser = $this->getInput('u');
 		$width = $this->getInput('w');
 		$max = $this->getInput('m');
-		$quality = $this->getInput('q');
+		$previewQuality = $this->getInput('prev_q');
+		$jpgQuality = $this->getInput('jpg_q');
+
+		$url = 'https://unsplash.com/napi';
+		if (strlen($filteredUser) > 0) $url .= '/users/' . $filteredUser;
+		$url .= '/photos?page=1&per_page=' . $max;
+		$api_response = getContents($url);
 
-		$api_response = getContents('https://unsplash.com/napi/photos?page=1&per_page=' . $max);
 		$json = json_decode($api_response, true);
 
 		foreach ($json as $json_item) {
 			$item = array();
 
 			// Get image URI
-			$uri = $json_item['urls']['regular'] . '.jpg'; // '.jpg' only for format hint
-			$uri = str_replace('q=80', 'q=' . $quality, $uri);
-			$uri = str_replace('w=1080', 'w=' . $width, $uri);
+			$uri = $json_item['urls']['raw'] . '&fm=jpg';
+			if ($jpgQuality > 0) $uri .= '&q=' . $jpgQuality;
+			if ($width > 0) $uri .= '&w=' . $width . '&fit=max';
+			$uri .= '.jpg'; // only for format hint
 			$item['uri'] = $uri;
 
 			// Get title from description
-			if (is_null($json_item['alt_description'])) {
-				if (is_null($json_item['description'])) {
-					$item['title'] = 'Unsplash picture from ' . $json_item['user']['name'];
-				} else {
-					$item['title'] = $json_item['description'];
-				}
+			if (is_null($json_item['description'])) {
+				$item['title'] = 'Unsplash picture from ' . $json_item['user']['name'];
 			} else {
-				$item['title'] = $json_item['alt_description'];
+				$item['title'] = $json_item['description'];
 			}
 
-			$item['timestamp'] = time();
-			$item['content'] = $item['title']
-				. '<br><a href="'
-				. $item['uri']
+			$item['timestamp'] = $json_item['created_at'];
+			$content = 'User: <a href="'
+				. $json_item['user']['links']['html']
+				. '">@'
+				. $json_item['user']['username']
+				. '</a>';
+			if (isset($json_item['location']['name'])) {
+				$content .= ' | Location: ' . $json_item['location']['name'];
+			}
+			$content .= ' | Image on <a href="'
+				. $json_item['links']['html']
+				. '">Unsplash</a><br><a href="'
+				. $uri
 				. '"><img src="'
-				. $json_item['urls']['thumb']
+				. $json_item['urls'][$previewQuality]
+				. '" alt="Image from '
+				. $filteredUser
 				. '" /></a>';
+			$item['content'] = $content;
 
 			$this->items[] = $item;
 		}
 	}
+
+	public function getName()
+	{
+		$filteredUser = $this->getInput('u');
+		if (strlen($filteredUser) > 0) {
+			return $filteredUser . ' - ' . self::NAME;
+		} else {
+			return self::NAME;
+		}
+	}
 }

bridges/VieDeMerdeBridge.php

@@ -3,7 +3,7 @@ class VieDeMerdeBridge extends BridgeAbstract {
 
 	const MAINTAINER = 'floviolleau';
 	const NAME = 'VieDeMerde Bridge';
-	const URI = 'https://viedemerde.fr';
+	const URI = 'https://www.viedemerde.fr';
 	const DESCRIPTION = 'Returns latest quotes from VieDeMerde.';
 	const CACHE_TIMEOUT = 7200;
 
@@ -23,16 +23,15 @@ class VieDeMerdeBridge extends BridgeAbstract {
 		}
 
 		$html = getSimpleHTMLDOM(self::URI, array());
-
-		$quotes = $html->find('article.article-panel');
+		$quotes = $html->find('article.bg-white');
 		if(sizeof($quotes) === 0) {
 			return;
 		}
 
 		foreach($quotes as $quote) {
 			$item = array();
-			$item['uri'] = self::URI . $quote->find('.article-contents a', 0)->href;
-			$titleContent = $quote->find('.article-contents a h2.classic-title', 0);
+			$item['uri'] = self::URI . $quote->find('a', 0)->href;
+			$titleContent = $quote->find('h2', 0);
 
 			if($titleContent) {
 				$item['title'] = html_entity_decode($titleContent->plaintext, ENT_QUOTES);
@@ -40,9 +39,11 @@ class VieDeMerdeBridge extends BridgeAbstract {
 				continue;
 			}
 
-			$quote->find('.article-contents a h2.classic-title', 0)->outertext = '';
-			$item['content'] = $quote->find('.article-contents a', 0)->innertext;
-			$item['author'] = $quote->find('.article-topbar', 0)->innertext;
+			$quoteText = $quote->find('a', 1)->plaintext;
+			$isAVDM = $quote->find('.vote-btn', 0)->plaintext;
+			$isNotAVDM = $quote->find('.vote-btn', 1)->plaintext;
+			$item['content'] = $quoteText . '<br>' . $isAVDM . '<br>' . $isNotAVDM;
+			$item['author'] = $quote->find('p', 0)->plaintext;
 			$item['uid'] = hash('sha256', $item['title']);
 
 			$this->items[] = $item;

bridges/VkBridge.php

@@ -409,8 +409,6 @@ class VkBridge extends BridgeAbstract
 
 	private function getContents()
 	{
-		ini_set('user-agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0');
-
 		$header = array('Accept-language: en', 'Cookie: remixlang=3');
 
 		return getContents($this->getURI(), $header);

bridges/WebfailBridge.php

@@ -41,9 +41,6 @@ class WebfailBridge extends BridgeAbstract {
 	}
 
 	public function collectData(){
-
-		ini_set('user_agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0');
-
 		$html = getSimpleHTMLDOM($this->getURI() . $this->getInput('type'));
 
 		$type = array_search($this->getInput('type'),
@@ -84,6 +81,16 @@ class WebfailBridge extends BridgeAbstract {
 				$description = $element->find('div.wf-news-description', 0)->innertext;
 			}
 
+			$infoElement = $element->find('div.wf-small', 0);
+			if (!is_null($infoElement)) {
+				if (preg_match('/(\d{2}\.\d{2}\.\d{4})/m', $infoElement->innertext, $matches) === 1 && count($matches) == 2) {
+					$dt = DateTime::createFromFormat('!d.m.Y', $matches[1]);
+					if ($dt !== false) {
+						$item['timestamp'] = $dt->getTimestamp();
+					}
+				}
+			}
+
 			$item['content'] = '<p>'
 			. $description
 			. '</p><br><a href="'

bridges/WordPressBridge.php

@@ -26,6 +26,11 @@ class WordPressBridge extends FeedExpander {
 
 		$article = null;
 		switch(true) {
+
+		// Custom fix for theme in https://jungefreiheit.de/politik/deutschland/2022/wahl-im-saarland/
+		case !is_null($article_html->find('div[data-widget_type="theme-post-content.default"]', 0)):
+			$article = $article_html->find('div[data-widget_type="theme-post-content.default"]', 0);
+			break;
 		case !is_null($article_html->find('[itemprop=articleBody]', 0)):
 			// highest priority content div
 			$article = $article_html->find('[itemprop=articleBody]', 0);
@@ -73,6 +78,7 @@ class WordPressBridge extends FeedExpander {
 
 		if(!is_null($article)) {
 			$item['content'] = $this->cleanContent($article->innertext);
+			$item['content'] = defaultLinkTo($item['content'], $item['uri']);
 		}
 
 		return $item;

bridges/WordPressPluginUpdateBridge.php

@@ -1,74 +1,62 @@
 <?php
-class WordPressPluginUpdateBridge extends BridgeAbstract {
 
-	const MAINTAINER = 'teromene';
+final class WordPressPluginUpdateBridge extends BridgeAbstract {
+	const MAINTAINER = 'dvikan';
 	const NAME = 'WordPress Plugins Update Bridge';
 	const URI = 'https://wordpress.org/plugins/';
-	const CACHE_TIMEOUT = 86400; // 24h = 86400s
-	const DESCRIPTION = 'Returns latest updates of WordPress.com plugins.';
+	const DESCRIPTION = 'Returns latest updates of wordpress.org plugins.';
 
-	const PARAMETERS = array(
-		array(
-			'pluginUrl' => array(
-				'name' => 'URL to the plugin',
-				'exampleValue' => 'https://wordpress.org/plugins/wp-rss-aggregator/',
-				'required' => true
-			)
-		)
-	);
-
-	public function collectData(){
-
-		$request = str_replace('/', '', $this->getInput('pluginUrl'));
-		$page = self::URI . $request . '/changelog/';
-
-		$html = getSimpleHTMLDOM($page);
-
-		$content = $html->find('.block-content', 0);
-
-		$item = array();
-		$item['content'] = '';
-		$version = null;
-
-		foreach($content->children() as $element) {
-
-			if($element->tag != 'h4') {
-
-				$item['content'] .= $element;
-
-			} else {
-
-				if($version == null) {
-
-					$version = $element;
-
-				} else {
-
-					$item['title'] = $version;
-					$item['uri'] = 'https://downloads.wordpress.org/plugin/' . $request . '.' . strip_tags($version) . '.zip';
-					$this->items[] = $item;
-
-					$version = $element;
-					$item = array();
-					$item['content'] = '';
-
-				}
-
-			}
+	const PARAMETERS = [
+		[
+			// The incorrectly named pluginUrl is kept for BC
+			'pluginUrl' => [
+				'name' => 'Plugin slug',
+				'exampleValue' => 'akismet',
+				'required' => true,
+				'title' => 'Slug or url',
+			]
+		]
+	];
 
+	public function collectData() {
+		$input = trim($this->getInput('pluginUrl'));
+		if (preg_match('#https://wordpress\.org/plugins/([\w-]+)#', $input, $m)) {
+			$slug = $m[1];
+		} else {
+			$slug = str_replace(['/'], '', $input);
 		}
 
-		$item['uri'] = 'https://downloads.wordpress.org/plugin/' . $request . '.' . strip_tags($version) . '.zip';
-		$item['title'] = $version;
-		$this->items[] = $item;
+		$pluginData = self::fetchPluginData($slug);
 
+		if ($pluginData->versions === []) {
+			throw new \Exception('This plugin does not have versioning data');
+		}
+
+		// We don't need trunk. I think it's the latest commit.
+		unset($pluginData->versions->trunk);
+
+		foreach ($pluginData->versions as $version => $downloadUrl) {
+			$this->items[] = [
+				'title'     => $version,
+				'uri'       => sprintf('https://wordpress.org/plugins/%s/#developers', $slug),
+				'uid'       => $downloadUrl,
+			];
+		}
+
+		usort($this->items, function($a, $b) {
+			return version_compare($b['title'], $a['title']);
+		});
 	}
 
-	public function getName(){
-		if(!is_null($this->getInput('q'))) {
-			return $this->getInput('q') . ' : ' . self::NAME;
-		}
-
-		return parent::getName();
+	/**
+	 * Fetch plugin data from wordpress.org json api
+	 *
+	 * https://codex.wordpress.org/WordPress.org_API#Plugins
+	 * https://wordpress.org/support/topic/using-the-wordpress-org-api/
+	 */
+	private static function fetchPluginData(string $slug): \stdClass
+	{
+		$api = 'https://api.wordpress.org/plugins/info/1.2/?action=plugin_information&request[slug]=%s';
+		return json_decode(getContents(sprintf($api, $slug)));
 	}
 }

bridges/WosckerBridge.php

@@ -1,50 +0,0 @@
-<?php
-class WosckerBridge extends BridgeAbstract {
-	const NAME = 'Woscker Bridge';
-	const URI = 'https://woscker.com/';
-	const DESCRIPTION = 'Returns news of the day';
-	const MAINTAINER = 'VerifiedJoseph';
-	const PARAMETERS = array();
-
-	const CACHE_TIMEOUT = 1800; // 30 mins
-
-	public function collectData() {
-		$html = getSimpleHTMLDOM($this->getURI());
-
-		$date = $html->find('h1', 0)->plaintext;
-		$timestamp = $html->find('span.dateFont', 0)->plaintext . ' ' . $html->find('span.dateFont', 1)->plaintext;
-
-		$item = array();
-		$item['title'] = $date;
-		$item['content'] = $this->formatContent($html);
-		$item['timestamp'] = $timestamp;
-
-		$this->items[] = $item;
-	}
-
-	private function formatContent($html) {
-		$html->find('h1', 0)->outertext = '';
-
-		foreach ($html->find('hr') as $hr) {
-			$hr->outertext = '';
-		}
-
-		foreach ($html->find('div.betweenHeadline') as $div) {
-			$div->outertext = '';
-		}
-
-		foreach ($html->find('div.dividingBarrier') as $div) {
-			$div->outertext = '';
-		}
-
-		foreach ($html->find('h2') as $h2) {
-			$h2->outertext = '<br><strong>' . $h2->innertext . '</strong><br>';
-		}
-
-		foreach ($html->find('h3') as $h3) {
-			$h3->outertext = $h3->innertext . '<br>';
-		}
-
-		return $html->find('div.fullContentPiece', 0)->innertext;
-	}
-}

bridges/XbooruBridge.php

@@ -8,5 +8,8 @@ class XbooruBridge extends GelbooruBridge {
 	const URI = 'https://xbooru.com/';
 	const DESCRIPTION = 'Returns images from given page';
 
-	const PIDBYPAGE = 50;
+	protected function buildThumbnailURI($element){
+		return $this->getURI() . 'thumbnails/' . $element->directory
+		. '/thumbnail_' . $element->hash . '.jpg';
+	}
 }

bridges/YahtzeeDevDiaryBridge.php

@@ -1,20 +0,0 @@
-<?php
-class YahtzeeDevDiaryBridge extends BridgeAbstract {
-	const MAINTAINER = 'somini';
-	const NAME = "Yahtzee's Dev Diary";
-	const URI = 'https://www.escapistmagazine.com/v2/yahtzees-dev-diary-completed-games-list/';
-	const DESCRIPTION = 'Yahtzee’s Dev Diary Series';
-
-	public function collectData(){
-		$html = getSimpleHTMLDOM($this->getURI());
-
-		foreach($html->find('blockquote.wp-embedded-content a') as $element) {
-			$item = array();
-
-			$item['title'] = $element->innertext;
-			$item['uri'] = $element->href;
-
-			$this->items[] = $item;
-		}
-	}
-}

bridges/YeggiBridge.php

@@ -3,7 +3,7 @@ class YeggiBridge extends BridgeAbstract {
 
 	const NAME = 'Yeggi Search';
 	const URI = 'https://www.yeggi.com';
-	const DESCRIPTION = 'Returns feeds for search results';
+	const DESCRIPTION = 'Returns 3D Models from Thingiverse, MyMiniFactory, Cults3D, and more';
 	const MAINTAINER = 'AntoineTurmel';
 	const PARAMETERS = array(
 		array(
@@ -63,13 +63,19 @@ class YeggiBridge extends BridgeAbstract {
 			}
 			$item['uri'] = self::URI . $result->find('a', 0)->href;
 			$item['author'] = 'Yeggi';
-			$item['content'] = '';
+
+			$text = $result->find('i');
+			$item['content'] = $text[0]->plaintext . ' on ' . $text[1]->plaintext;
 			$item['uid'] = hash('md5', $item['title']);
 
+			foreach($result->find('.item_3_B_2 > a[href^=/q/]') as $tag) {
+				$item['tags'][] = $tag->plaintext;
+			}
+
 			$image = $result->find('img', 0)->src;
 
 			if($this->getInput('showimage')) {
-				$item['content'] .= '<img src="' . $image . '">';
+				$item['content'] .= '<br><img src="' . $image . '">';
 			}
 
 			$item['enclosures'] = array($image);

bridges/YouTubeCommunityTabBridge.php

@@ -0,0 +1,267 @@
+<?php
+class YouTubeCommunityTabBridge extends BridgeAbstract {
+	const NAME = 'YouTube Community Tab Bridge';
+	const URI = 'https://www.youtube.com';
+	const DESCRIPTION = 'Returns posts from a channel\'s community tab';
+	const MAINTAINER = 'VerifiedJoseph';
+	const PARAMETERS = array(
+		'By channel ID' => array(
+			'channel' => array(
+				'name' => 'Channel ID',
+				'type' => 'text',
+				'required' => true,
+				'exampleValue' => 'UCULkRHBdLC5ZcEQBaL0oYHQ'
+			)
+		),
+		'By username' => array(
+			'username' => array(
+				'name' => 'Username',
+				'type' => 'text',
+				'required' => true,
+				'exampleValue' => 'YouTubeUK'
+			),
+		)
+	);
+
+	const CACHE_TIMEOUT = 3600; // 1 hour
+
+	private $feedUrl = '';
+	private $feedName = '';
+	private $itemTitle = '';
+
+	private $urlRegex = '/youtube\.com\/(channel|user|c)\/([\w]+)\/community/';
+	private $jsonRegex = '/var ytInitialData = (.*);<\/script><link rel="canonical"/';
+
+	public function detectParameters($url) {
+		$params = array();
+
+		if(preg_match($this->urlRegex, $url, $matches)) {
+			if ($matches[1] === 'channel') {
+				$params['context'] = 'By channel ID';
+				$params['channel'] = $matches[2];
+			}
+
+			if ($matches[1] === 'user') {
+				$params['context'] = 'By username';
+				$params['username'] = $matches[2];
+			}
+
+			return $params;
+		}
+
+		return null;
+	}
+
+	public function collectData() {
+
+		if (is_null($this->getInput('username')) === false) {
+			try {
+				$this->feedUrl = $this->buildCommunityUri($this->getInput('username'), 'c');
+				$html = getSimpleHTMLDOM($this->feedUrl);
+
+			} catch (Exception $e) {
+				$this->feedUrl = $this->buildCommunityUri($this->getInput('username'), 'user');
+				$html = getSimpleHTMLDOM($this->feedUrl);
+			}
+		} else {
+			$this->feedUrl = $this->buildCommunityUri($this->getInput('channel'), 'channel');
+			$html = getSimpleHTMLDOM($this->feedUrl);
+		}
+
+		$json = $this->extractJson($html->find('body', 0)->innertext);
+
+		$this->feedName = $json->header->c4TabbedHeaderRenderer->title;
+
+		if ($this->hasCommunityTab($json) === false) {
+			returnServerError('Channel does not have a community tab');
+		}
+
+		foreach ($this->getCommunityPosts($json) as $post) {
+			$this->itemTitle = '';
+
+			if (!isset($post->backstagePostThreadRenderer)) {
+				continue;
+			}
+
+			$details = $post->backstagePostThreadRenderer->post->backstagePostRenderer;
+
+			$item = array();
+			$item['uri'] = self::URI . '/post/' . $details->postId;
+			$item['author'] = $details->authorText->runs[0]->text;
+			$item['content'] = '';
+
+			if (isset($details->contentText)) {
+				$text = $this->getText($details->contentText->runs);
+
+				$this->itemTitle = $this->ellipsisTitle($text);
+				$item['content'] = $text;
+			}
+
+			$item['content'] .= $this->getAttachments($details);
+			$item['title'] = $this->itemTitle;
+
+			$this->items[] = $item;
+		}
+	}
+
+	public function getURI() {
+
+		if (!empty($this->feedUri)) {
+			return $this->feedUri;
+		}
+
+		return parent::getURI();
+	}
+
+	public function getName() {
+
+		if (!empty($this->feedName)) {
+			return $this->feedName . ' - YouTube Community Tab';
+		}
+
+		return parent::getName();
+	}
+
+	/**
+	 * Build Community URI
+	 */
+	private function buildCommunityUri($value, $type) {
+		return self::URI . '/' . $type . '/' . $value . '/community';
+	}
+
+	/**
+	 * Extract JSON from page
+	 */
+	private function extractJson($html) {
+
+		if (!preg_match($this->jsonRegex, $html, $parts)) {
+			returnServerError('Failed to extract data from page');
+		}
+
+		$data = json_decode($parts[1]);
+
+		if ($data === false) {
+			returnServerError('Failed to decode extracted data');
+		}
+
+		return $data;
+	}
+
+	/**
+	 * Check if channel has a community tab
+	 */
+	private function hasCommunityTab($json) {
+
+		foreach ($json->contents->twoColumnBrowseResultsRenderer->tabs as $tab) {
+			if (isset($tab->tabRenderer) && $tab->tabRenderer->title === 'Community') {
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	/**
+	 * Get community tab posts
+	 */
+	private function getCommunityPosts($json) {
+
+		foreach ($json->contents->twoColumnBrowseResultsRenderer->tabs as $tab) {
+			if ($tab->tabRenderer->title === 'Community') {
+				return $tab->tabRenderer->content->sectionListRenderer->contents[0]->itemSectionRenderer->contents;
+			}
+		}
+	}
+
+	/**
+	 * Get text content for a post
+	 */
+	private function getText($runs) {
+		$text = '';
+
+		foreach ($runs as $part) {
+			$text .= $this->formatUrls($part->text);
+		}
+
+		return nl2br($text);
+	}
+
+	/**
+	 * Get attachments for posts
+	 */
+	private function getAttachments($details) {
+		$content = '';
+
+		if (isset($details->backstageAttachment)) {
+			$attachments = $details->backstageAttachment;
+
+			// Video
+			if (isset($attachments->videoRenderer) && isset($attachments->videoRenderer->videoId)) {
+				if (empty($this->itemTitle)) {
+					$this->itemTitle = $this->feedName . ' posted a video';
+				}
+
+				$content = <<<EOD
+<iframe width="100%" height="410" src="https://www.youtube.com/embed/{$attachments->videoRenderer->videoId}" 
+frameborder="0" allow="encrypted-media;" allowfullscreen></iframe>
+EOD;
+			}
+
+			// Image
+			if (isset($attachments->backstageImageRenderer)) {
+				if (empty($this->itemTitle)) {
+					$this->itemTitle = $this->feedName . ' posted an image';
+				}
+
+				$lastThumb = end($attachments->backstageImageRenderer->image->thumbnails);
+
+				$content = <<<EOD
+<p><img src="{$lastThumb->url}"></p>
+EOD;
+			}
+
+			// Poll
+			if (isset($attachments->pollRenderer)) {
+				if (empty($this->itemTitle)) {
+					$this->itemTitle = $this->feedName . ' posted a poll';
+				}
+
+				$pollChoices = '';
+
+				foreach ($attachments->pollRenderer->choices as $choice) {
+					$pollChoices .= <<<EOD
+<li>{$choice->text->runs[0]->text}</li>
+EOD;
+				}
+
+				$content = <<<EOD
+<hr><p>Poll ({$attachments->pollRenderer->totalVotes->simpleText})<br><ul>{$pollChoices}</ul><p>
+EOD;
+			}
+		}
+
+		return $content;
+	}
+
+	/*
+		Ellipsis text for title
+	*/
+	private function ellipsisTitle($text) {
+		$length = 100;
+
+		if (strlen($text) > $length) {
+			$text = explode('<br>', wordwrap($text, $length, '<br>'));
+			return $text[0] . '...';
+		}
+
+		return $text;
+	}
+
+	private function formatUrls($content) {
+		return preg_replace(
+			'/(http[s]{0,1}\:\/\/[a-zA-Z0-9.\/\?\&=\-_]{4,})/ims',
+			'<a target="_blank" href="$1" target="_blank">$1</a> ',
+			$content
+		);
+	}
+}

bridges/ZoneTelechargementBridge.php

@@ -47,7 +47,8 @@ class ZoneTelechargementBridge extends BridgeAbstract {
 		$header = array();
 		// Parse the URL to extract the hostname
 		$parse = parse_url($url);
-		$opts = array(CURLOPT_USERAGENT => 'curl/7.64.0',
+		$opts = array(
+			CURLOPT_USERAGENT => Configuration::getConfig('http', 'useragent'),
 			CURLOPT_RESOLVE => $this->getResolve($parse['host'])
 		);
 

config.default.ini.php

@@ -12,6 +12,9 @@
 ; timezone = "UTC" (default)
 timezone = "UTC"
 
+[http]
+useragent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
+
 [cache]
 
 ; Defines the cache type used by RSS-Bridge

docker-entrypoint.sh

@@ -5,7 +5,7 @@
 # This will overwrite previous configs and bridges of same name
 # If there are no matching files, rss-bridge works like default.
 
-find /config/ -type f -name '*.*' -print0 | 
+find /config/ -type f -name '*' -print0 | 
 while IFS= read -r -d '' file; do
     file_name="$(basename "$file")" # Strip leading path
     if [[ $file_name = *" "* ]]; then

docs/05_Bridge_API/02_BridgeAbstract.md

@@ -453,4 +453,62 @@ public function detectParameters($url){
 		return null;
 	}
 }
-```
+```
+
+***
+
+# Helper Methods
+`BridgeAbstract` implements helper methods to make it easier for bridge maintainers to create bridges. Use these methods whenever possible instead of writing your own.
+
+- [saveCacheValue](#savecachevalue)
+- [loadCacheValue](#loadcachevalue)
+
+## saveCacheValue
+Within the context of the current bridge, stores a value by key in the cache. The value can later be retrieved with [loadCacheValue](#loadcachevalue).
+
+```php
+protected function saveCacheValue($key, $value)
+```
+
+- `$key` - the name under which the value is stored in the cache.
+- `$value` - the value to store in the cache.
+
+Usage example:
+
+```php
+const MY_KEY = 'MyKey';
+
+public function collectData()
+{
+    $value = 'my value';
+    $this->saveCacheValue(MY_KEY, $value);
+}
+```
+
+## loadCacheValue
+Within the context of the current bridge, loads a value by key from cache. Optionally specifies the cache duration for the key. Returns `null` if the key doesn't exist or the value is expired.
+
+```php
+protected function loadCacheValue($key, $duration = 86400)
+```
+
+- `$key` - the name under which the value is stored in the cache.
+- `$duration` - the maximum time in seconds after which the value expires. The default duration is 86400 (24 hours).
+
+Usage example:
+
+```php
+const MY_KEY = 'MyKey';
+
+public function collectData()
+{
+  $value = $this->loadCacheValue(MY_KEY, 1800 /* 30 minutes */);
+
+  if (!isset($value)){
+      // load value
+      $this->saveCacheValue(MY_KEY, $value);
+  }
+
+  // ...
+}
+```

docs/10_Bridge_Specific/Furaffinityuser.md

@@ -0,0 +1,21 @@
+# FuraffinityuserBridge
+
+## How to retrieve and use cookie values
+---
+> The following steps describe how to get the session cookies using a Chromium-based browser. Other browser may require slightly different steps. Keyword search "how to find cookie values" in your favorite search engine.
+
+### Retreiving session cookies.
+
+- Login to Furaffinity
+
+- Open DevTools by pressing F12
+
+- Open "Application"
+
+- On the left side, select "Cookies" -> "Furaffinity.net"
+
+- There will be (at least) two cookie informations in the main window. You need the values of the "a" key and "b" key
+
+### Configuring RSS-Bridge
+
+- Copy/Paste the values from cookies "a" and "b" into their respective fields in the bridge config and generate the feed

docs/10_Bridge_Specific/Instagram.md

@@ -0,0 +1,45 @@
+InstagramBridge
+===============
+
+To somehow bypass the [rate limiting issue](https://github.com/RSS-Bridge/rss-bridge/issues/1891)
+it is suggested to deploy a private RSS-Bridge instance that uses a working Instagram account.
+
+Configuration
+-------------
+
+1. Retreiving session id.
+The following steps describe how to get the session id using a Chromium-based browser.
+
+- Create an Instagram account, that you will use for your RSS-Bridge instance.
+It is NOT recommended to use your existing account that is used for common interaction with Instagram services.
+
+- Login to Instagram
+
+- Open DevTools by pressing F12
+
+- Open "Networks tab"
+
+- In the "Filter" field input "i.instagram.com"
+
+- Click on "Fetch/XHR"
+
+- Refresh web page
+
+- Click on any item from the table of http requests
+
+- In the new frame open the "Headers" tab and scroll to "Request Headers"
+
+- There will be a cookie param will lots of `<key>=<value>;` text. You need the value of the "sessionid" key. Copy it.
+
+2. Configuring RSS-Bridge
+
+- In config.ini.php add following configuration:
+
+```
+[InstagramBridge]
+session_id = %sessionid from step 1%
+cache_timeout = %cache timeout in seconds%
+```
+
+The bigger the cache_timeout value, the smaller the chance for RSS-Bridge to throw 429 errors.
+Default cache_timeout is 3600 seconds (1 hour).

docs/10_Bridge_Specific/TwitterV2.md

@@ -0,0 +1,37 @@
+TwitterV2Bridge
+===============
+
+To automatically retrieve Tweets containing potentially sensitive/age-restricted content, you'll need to acquire your own unique API Bearer token, which will be used by this Bridge to query Twitter's API v2.
+
+Configuration
+-------------
+
+1. Make a Twitter Developer account
+
+	- Developer Portal: https://dev.twitter.com
+
+	- I will not detail exactly how to do this, as the specific process will likely change over time. You should easily be able to find guides using your search engine of choice.
+
+	- A basic free developer account grants Essential access to the Twitter API v2, which should be sufficient for this bridge.
+
+2. Create a Twitter Project and App, get Bearer Token
+
+	- Once you have an active Twitter Developer account, sign in to the dev portal
+
+	- Create a new Project (name doesn't matter)
+
+	- Create an App within the Project (again, name doesn't matter)
+
+	- Go to the **Keys and tokens** tab
+
+	- Generate a **Bearer Token** (you don't want the API Key and Secret, or the Access Token and Secret)
+
+3. Configure RSS-Bridge
+
+	- In **config.ini.php** (in rss-bridge root directory) add following lines at the end:
+
+	```
+	[TwitterV2Bridge]
+	twitterv2apitoken = %Bearer Token from step 2%
+	```
+	- If you don't have a **config.ini.php**, create one by making a copy of **config.default.ini.php**

formats/JsonFormat.php

@@ -111,12 +111,15 @@ class JsonFormat extends FormatAbstract {
 		}
 		$data['items'] = $items;
 
-		$toReturn = json_encode($data, JSON_PRETTY_PRINT);
+		/**
+		 * The intention here is to discard non-utf8 byte sequences.
+		 * But the JSON_PARTIAL_OUTPUT_ON_ERROR also discards lots of other errors.
+		 * So consider this a hack.
+		 * Switch to JSON_INVALID_UTF8_IGNORE when PHP 7.2 is the latest platform requirement.
+		 */
+		$json = json_encode($data, JSON_PRETTY_PRINT | JSON_PARTIAL_OUTPUT_ON_ERROR);
 
-		// Remove invalid non-UTF8 characters
-		ini_set('mbstring.substitute_character', 'none');
-		$toReturn = mb_convert_encoding($toReturn, $this->getCharset(), 'UTF-8');
-		return $toReturn;
+		return $json;
 	}
 
 	public function display(){

index.php

@@ -17,16 +17,6 @@ if (isset($argv)) {
 	$params = $_GET;
 }
 
-define('USER_AGENT',
-	'Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0(rss-bridge/'
-	. Configuration::$VERSION
-	. ';+'
-	. REPOSITORY
-	. ')'
-);
-
-ini_set('user_agent', USER_AGENT);
-
 try {
 
 	$actionFac = new \ActionFactory();

lib/BridgeAbstract.php

@@ -370,4 +370,37 @@ abstract class BridgeAbstract implements BridgeInterface {
 			return null;
 		}
 	}
+
+	/**
+	 * Loads a cached value for the specified key
+	 *
+	 * @param string $key Key name
+	 * @param int $duration Cache duration (optional, default: 24 hours)
+	 * @return mixed Cached value or null if the key doesn't exist or has expired
+	 */
+	protected function loadCacheValue($key, $duration = 86400){
+		$cacheFac = new CacheFactory();
+		$cacheFac->setWorkingDir(PATH_LIB_CACHES);
+		$cache = $cacheFac->create(Configuration::getConfig('cache', 'type'));
+		$cache->setScope(get_called_class());
+		$cache->setKey($key);
+		if($cache->getTime() < time() - $duration)
+			return null;
+		return $cache->loadData();
+	}
+
+	/**
+	 * Stores a value to cache with the specified key
+	 *
+	 * @param string $key Key name
+	 * @param mixed $value Value to cache
+	 */
+	protected function saveCacheValue($key, $value){
+		$cacheFac = new CacheFactory();
+		$cacheFac->setWorkingDir(PATH_LIB_CACHES);
+		$cache = $cacheFac->create(Configuration::getConfig('cache', 'type'));
+		$cache->setScope(get_called_class());
+		$cache->setKey($key);
+		$cache->saveData($value);
+	}
 }

lib/BridgeCard.php

@@ -88,7 +88,7 @@ This bridge is not fetching its content through a secure connection</div>';
 				$form .= '<label for="'
 					. $idArg
 					. '">'
-					. filter_var($inputEntry['name'], FILTER_SANITIZE_STRING)
+					. filter_var($inputEntry['name'], FILTER_SANITIZE_FULL_SPECIAL_CHARS)
 					. '</label>'
 					. PHP_EOL;
 
@@ -102,10 +102,12 @@ This bridge is not fetching its content through a secure connection</div>';
 					$form .= self::getCheckboxInput($inputEntry, $idArg, $id);
 				}
 
-				if(isset($inputEntry['title']))
-					$form .= '<i class="info" title="' . filter_var($inputEntry['title'], FILTER_SANITIZE_STRING) . '">i</i>';
-				else
+				if(isset($inputEntry['title'])) {
+					$title_filtered = filter_var($inputEntry['title'], FILTER_SANITIZE_FULL_SPECIAL_CHARS);
+					$form .= '<i class="info" title="' . $title_filtered . '">i</i>';
+				} else {
 					$form .= '<i class="no-info"></i>';
+				}
 			}
 
 			$form .= '</div>';
@@ -153,9 +155,9 @@ This bridge is not fetching its content through a secure connection</div>';
 		. ' id="'
 		. $id
 		. '" type="text" value="'
-		. filter_var($entry['defaultValue'], FILTER_SANITIZE_STRING)
+		. filter_var($entry['defaultValue'], FILTER_SANITIZE_FULL_SPECIAL_CHARS)
 		. '" placeholder="'
-		. filter_var($entry['exampleValue'], FILTER_SANITIZE_STRING)
+		. filter_var($entry['exampleValue'], FILTER_SANITIZE_FULL_SPECIAL_CHARS)
 		. '" name="'
 		. $name
 		. '" />'

lib/FeedExpander.php

@@ -57,6 +57,12 @@ abstract class FeedExpander extends BridgeAbstract {
 	 */
 	private $uri;
 
+	/**
+	 * Holds the icon of the feed
+	 *
+	 */
+	private $icon;
+
 	/**
 	 * Holds the feed type during internal operations.
 	 *
@@ -89,6 +95,10 @@ abstract class FeedExpander extends BridgeAbstract {
 			or returnServerError('Could not request ' . $url);
 		$rssContent = simplexml_load_string(trim($content));
 
+		if ($rssContent === false) {
+			throw new \Exception('Unable to parse string as xml');
+		}
+
 		Debug::log('Detecting feed format/version');
 		switch(true) {
 		case isset($rssContent->item[0]):
@@ -216,6 +226,10 @@ abstract class FeedExpander extends BridgeAbstract {
 	protected function load_RSS_2_0_feed_data($rssContent){
 		$this->title = trim((string)$rssContent->title);
 		$this->uri = trim((string)$rssContent->link);
+
+		if (!empty($rssContent->image)) {
+			$this->icon = trim((string)$rssContent->image->url);
+		}
 	}
 
 	/**
@@ -241,6 +255,12 @@ abstract class FeedExpander extends BridgeAbstract {
 				}
 			}
 		}
+
+		if(!empty($content->icon)) {
+			$this->icon = (string)$content->icon;
+		} elseif(!empty($content->logo)) {
+			$this->icon = (string)$content->logo;
+		}
 	}
 
 	/**

lib/contents.php

@@ -168,7 +168,7 @@ function getContents($url, $header = array(), $opts = array(), $returnHeader = f
 
 		}
 
-		curl_setopt($ch, CURLOPT_USERAGENT, ini_get('user_agent'));
+		curl_setopt($ch, CURLOPT_USERAGENT, Configuration::getConfig('http', 'useragent'));
 		curl_setopt($ch, CURLOPT_ENCODING, '');
 		curl_setopt($ch, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
 

phpcs.xml

@@ -3,6 +3,7 @@
   <description>Created with the PHP Coding Standard Generator. http://edorian.github.com/php-coding-standard-generator/</description>
   <exclude-pattern>./static</exclude-pattern>
   <exclude-pattern>./vendor</exclude-pattern>
+  <exclude-pattern>./config.default.ini.php</exclude-pattern>
   <!-- Duplicate class names are not allowed -->
   <rule ref="Generic.Classes.DuplicateClassName"/>
   <!-- Statements must not be empty -->

vendor/simplehtmldom/simple_html_dom.php

@@ -18,7 +18,7 @@
  *   Vadim Voituk
  *   Antcs
  *
- * Version Rev. 1.9 (290)
+ * Version Rev. 1.9.1 (291)
  */
 
 define('HDOM_TYPE_ELEMENT', 1);
@@ -609,6 +609,13 @@ class simple_html_dom_node
 				$pass = false;
 			}
 
+			// Handle 'text' selector
+			if($pass && $tag === 'text' && $node->tag === 'text') {
+				$ret[array_search($node, $this->dom->nodes, true)] = 1;
+				unset($node);
+				continue;
+			}
+
 			// Skip if node isn't a child node (i.e. text nodes)
 			if($pass && !in_array($node, $node->parent->children, true)) {
 				$pass = false;