[CyanideAndHappiness] Add bridge (#2807)

Without this, format tests fail on systems without `/etc/mime.types`.

.gitattributes

@@ -1,5 +1,6 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+*.sh text eol=lf
 
 # Custom for Visual Studio
 *.cs     diff=csharp

.github/CONTRIBUTING.md

@@ -1,49 +1,7 @@
 ### Pull request policy
 
-* [Fix one issue per pull request](https://github.com/RSS-Bridge/rss-bridge/wiki/Pull-request-policy#fix-one-issue-per-pull-request)
-* [Respect the coding style policy](https://github.com/RSS-Bridge/rss-bridge/wiki/Pull-request-policy#respect-the-coding-style-policy)
-* [Properly name your commits](https://github.com/RSS-Bridge/rss-bridge/wiki/Pull-request-policy#properly-name-your-commits)
-  * When fixing a bridge (located in the `bridges` directory), write `[BridgeName] Feature` <br>(i.e. `[YoutubeBridge] Fix typo in video titles`).
-  * When fixing other files, use `[FileName] Feature` <br>(i.e. `[index.php] Add multilingual support`).
-  * When fixing a general problem that applies to multiple files, write `category: feature` <br>(i.e. `bridges: Fix various typos`).
-
-Note that all pull-requests must pass all tests before they can be merged.
+See the [Pull request policy page on the documentation](https://rss-bridge.github.io/rss-bridge/For_Developers/Pull_Request_policy.html) for more information on the pull request policy.
 
 ### Coding style
 
-* [Whitespace](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitespace)
-  * [Add a new line at the end of a file](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitespace#add-a-new-line-at-the-end-of-a-file)
-  * [Do not add a whitespace before a semicolon](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitespace#add-a-new-line-at-the-end-of-a-file)
-  * [Do not add whitespace at start or end of a file or end of a line](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitespace#do-not-add-whitespace-at-start-or-end-of-a-file-or-end-of-a-line)
-* [Indentation](https://github.com/RSS-Bridge/rss-bridge/wiki/Indentation)
-  * [Use tabs for indentation](https://github.com/RSS-Bridge/rss-bridge/wiki/Indentation#use-tabs-for-indentation)
-* [Maximum line length](https://github.com/RSS-Bridge/rss-bridge/wiki/Maximum-line-length)
-  * [The maximum line length should not exceed 80 characters](https://github.com/RSS-Bridge/rss-bridge/wiki/Maximum-line-length#the-maximum-line-length-should-not-exceed-80-characters)
-* [Strings](https://github.com/RSS-Bridge/rss-bridge/wiki/Strings)
-  * [Whenever possible use single quoted strings](https://github.com/RSS-Bridge/rss-bridge/wiki/Strings#whenever-possible-use-single-quote-strings)
-  * [Add spaces around the concatenation operator](https://github.com/RSS-Bridge/rss-bridge/wiki/Strings#add-spaces-around-the-concatenation-operator)
-  * [Use a single string instead of concatenating](https://github.com/RSS-Bridge/rss-bridge/wiki/Strings#use-a-single-string-instead-of-concatenating)
-* [Constants](https://github.com/RSS-Bridge/rss-bridge/wiki/Constants)
-  * [Use UPPERCASE for constants](https://github.com/RSS-Bridge/rss-bridge/wiki/Constants#use-uppercase-for-constants)
-* [Keywords](https://github.com/RSS-Bridge/rss-bridge/wiki/Keywords)
-  * [Use lowercase for `true`, `false` and `null`](https://github.com/RSS-Bridge/rss-bridge/wiki/Keywords#use-lowercase-for-true-false-and-null)
-* [Operators](https://github.com/RSS-Bridge/rss-bridge/wiki/Operators)
-  * [Operators must have a space around them](https://github.com/RSS-Bridge/rss-bridge/wiki/Operators#operators-must-have-a-space-around-them)
-* [Functions](https://github.com/RSS-Bridge/rss-bridge/wiki/Functions)
-  * [Parameters with default values must appear last in functions](https://github.com/RSS-Bridge/rss-bridge/wiki/Functions#parameters-with-default-values-must-appear-last-in-functions)
-  * [Calling functions](https://github.com/RSS-Bridge/rss-bridge/wiki/Functions#calling-functions)
-  * [Do not add spaces after opening or before closing bracket](https://github.com/RSS-Bridge/rss-bridge/wiki/Functions#do-not-add-spaces-after-opening-or-before-closing-bracket)
-* [Structures](https://github.com/RSS-Bridge/rss-bridge/wiki/Structures)
-  * [Structures must always be formatted as multi-line blocks](https://github.com/RSS-Bridge/rss-bridge/wiki/Structures#structures-must-always-be-formatted-as-multi-line-blocks)
-* [If-Statement](https://github.com/RSS-Bridge/rss-bridge/wiki/if-Statement)
-  * [Use `elseif` instead of `else if`](https://github.com/RSS-Bridge/rss-bridge/wiki/if-Statement#use-elseif-instead-of-else-if)
-  * [Do not write empty statements](https://github.com/RSS-Bridge/rss-bridge/wiki/if-Statement#do-not-write-empty-statements)
-  * [Do not write unconditional if-statements](https://github.com/RSS-Bridge/rss-bridge/wiki/if-Statement#do-not-write-unconditional-if-statements)
-* [Classes](https://github.com/RSS-Bridge/rss-bridge/wiki/Classes)
-  * [Use PascalCase for class names](https://github.com/RSS-Bridge/rss-bridge/wiki/Classes#use-pascalcase-for-class-names)
-  * [Do not use final statements inside final classes](https://github.com/RSS-Bridge/rss-bridge/wiki/Classes#do-not-use-final-statements-inside-final-classes)
-  * [Do not override methods to call their parent](https://github.com/RSS-Bridge/rss-bridge/wiki/Classes#do-not-override-methods-to-call-their-parent)
-  * [abstract and final declarations MUST precede the visibility declaration](https://github.com/RSS-Bridge/rss-bridge/wiki/Classes#abstract-and-final-declarations-must-precede-the-visibility-declaration)
-  * [static declaration MUST come after the visibility declaration](https://github.com/RSS-Bridge/rss-bridge/wiki/Classes#static-declaration-must-come-after-the-visibility-declaration)
-* [Casting](https://github.com/RSS-Bridge/rss-bridge/wiki/Casting)
-  * [Do not add spaces when casting](https://github.com/RSS-Bridge/rss-bridge/wiki/Casting#do-not-add-spaces-when-casting)
+See the [Coding style policy page on the documentation](https://rss-bridge.github.io/rss-bridge/For_Developers/Coding_style_policy.html) for more information on the coding style of the project.

.github/ISSUE_TEMPLATE/bridge-request.md

@@ -60,5 +60,5 @@ Please describe what you expect from the bridge. Whenever possible provide sampl
 
 Keep in mind that opening a request does not guarantee the bridge being implemented! That depends entirely on the interest and time of others to make the bridge for you.
 
-You can also implement your own bridge (with support of the community if needed). Find more information in the [RSS-Bridge Wiki](https://github.com/RSS-Bridge/rss-bridge/wiki/For-developers) developer section.
+You can also implement your own bridge (with support of the community if needed). Find more information in the [RSS-Bridge Documentation](https://rss-bridge.github.io/rss-bridge/For_Developers/index.html) developer section.
 -->

.github/prtester-requirements.txt

@@ -0,0 +1,2 @@
+beautifulsoup4>=4.10.0
+requests>=2.26.0

.github/prtester.py

@@ -0,0 +1,105 @@
+import requests
+import itertools
+from bs4 import BeautifulSoup
+from datetime import datetime
+import os.path
+
+# This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge
+#
+# This will scrape the whitelisted bridges in the current state (port 3000) and the PR state (port 3001) of
+# RSS-Bridge, generate a feed for each of the bridges and save the output as html files.
+# It also replaces the default static CSS link with a hardcoded link to @em92's public instance, so viewing
+# the HTML file locally will actually work as designed.
+
+def testBridges(bridges,status):
+    for bridge in bridges:
+        if bridge.get('data-ref'): # Some div entries are empty, this ignores those
+            bridgeid = bridge.get('id')
+            bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata
+            bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html'
+            forms = bridge.find_all("form")
+            formid = 1
+            for form in forms:
+                # a bridge can have multiple contexts, named 'forms' in html
+                # this code will produce a fully working formstring that should create a working feed when called
+                # this will create an example feed for every single context, to test them all
+                formstring = ''
+                errormessages = []
+                parameters = form.find_all("input")
+                lists = form.find_all("select")
+                # this for/if mess cycles through all available input parameters, checks if it required, then pulls
+                # the default or examplevalue and then combines it all together into the formstring
+                # if an example or default value is missing for a required attribute, it will throw an error
+                # any non-required fields are not tested!!!
+                for parameter in parameters:
+                    if parameter.get('type') == 'hidden' and parameter.get('name') == 'context':
+                        cleanvalue = parameter.get('value').replace(" ","+")
+                        formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue
+                    if parameter.get('type') == 'number' or parameter.get('type') == 'text':
+                        if parameter.has_attr('required'):
+                            if parameter.get('placeholder') == '':
+                                if parameter.get('value') == '':
+                                    errormessages.append(parameter.get('name'))
+                                else:
+                                    formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value')
+                            else:
+                                formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder')
+                    # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring
+                    if parameter.get('type') == 'checkbox':
+                        if parameter.has_attr('checked'):
+                            formstring = formstring + '&' + parameter.get('name') + '=on'
+                for listing in lists:
+                    selectionvalue = ''
+                    listname = listing.get('name')
+                    if 'optgroup' in listing.contents[0].name:
+                        listing = list(itertools.chain.from_iterable(listing))
+                    for selectionentry in listing:
+                        if 'selected' in selectionentry.attrs:
+                            selectionvalue = selectionentry.get('value')
+                            break
+                    if selectionvalue == '':
+                        selectionvalue = listing.contents[0].get('value')
+                    formstring = formstring + '&' + listname + '=' + selectionvalue
+                if not errormessages:
+                    # if all example/default values are present, form the full request string, run the request, replace the static css
+                    # file with the url of em's public instance and then upload it to termpad.com, a pastebin-like-site.
+                    r = requests.get(URL + bridgestring + formstring)
+                    pagetext = r.text.replace('static/HtmlFormat.css','https://feed.eugenemolotov.ru/static/HtmlFormat.css')
+                    pagetext = pagetext.encode("utf_8")
+                    termpad = requests.post(url="https://termpad.com/", data=pagetext)
+                    termpadurl = termpad.text
+                    termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/')
+                    termpadurl = termpadurl.replace('\n','')
+                    with open(os.getcwd() + '/comment.txt', 'a+') as file:
+                        file.write("\n")
+                        file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |")
+                else:
+                    # if there are errors (which means that a required value has no example or default value), log out which error appeared
+                    termpad = requests.post(url="https://termpad.com/", data=str(errormessages))
+                    termpadurl = termpad.text
+                    termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/')
+                    termpadurl = termpadurl.replace('\n','')
+                    with open(os.getcwd() + '/comment.txt', 'a+') as file:
+                        file.write("\n")
+                        file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |")
+                formid += 1
+
+gitstatus = ["current", "pr"]
+now = datetime.now()
+date_time = now.strftime("%Y-%m-%d, %H:%M:%S")
+
+with open(os.getcwd() + '/comment.txt', 'w+') as file:
+    file.write(''' ## Pull request artifacts
+| file | last change |
+| ---- | ------ |''')
+
+for status in gitstatus: # run this twice, once for the current version, once for the PR version
+    if status == "current":
+        port = "3000" # both ports are defined in the corresponding workflow .yml file
+    elif status == "pr":
+        port = "3001"
+    URL = "http://localhost:" + port
+    page = requests.get(URL) # Use python requests to grab the rss-bridge main page
+    soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup
+    bridges = soup.find_all("section") # get a soup-formatted list of all bridges on the rss-bridge page
+    testBridges(bridges,status) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version

.github/workflows/prhtmlgenerator.yml

@@ -0,0 +1,69 @@
+name: 'PR Testing'
+
+on:
+  pull_request_target:
+    branches: [ master ]
+
+jobs:
+  test-pr:
+    name: Generate HTML
+    runs-on: ubuntu-latest
+    # Needs additional permissions https://github.com/actions/first-interaction/issues/10#issuecomment-1041402989
+    steps:
+      - name: Check out self
+        uses: actions/checkout@v2.3.2
+        with:
+          ref: ${{github.event.pull_request.head.ref}}
+          repository: ${{github.event.pull_request.head.repo.full_name}}
+      - name: Check out rss-bridge
+        run: |
+          PR=${{github.event.number}};
+          wget -O requirements.txt https://raw.githubusercontent.com/RSS-Bridge/rss-bridge/master/.github/prtester-requirements.txt;
+          wget https://raw.githubusercontent.com/RSS-Bridge/rss-bridge/master/.github/prtester.py;
+          wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch;
+          touch DEBUG;
+          cat $PR.patch | grep " bridges/.*\.php" | sed "s= bridges/\(.*\)Bridge.php.*=\1=g" | sort | uniq > whitelist.txt
+      - name: Start Docker - Current
+        run: |
+          docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3000:80 ghcr.io/rss-bridge/rss-bridge:latest
+      - name: Start Docker - PR
+        run: |
+          docker build -t prbuild .;
+          docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3001:80 prbuild
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+          cache: 'pip'
+      - name: Install requirements
+        run: |
+          cd $GITHUB_WORKSPACE
+          pip install -r requirements.txt
+      - name: Run bridge tests
+        id: testrun
+        run: |
+          mkdir results;
+          python prtester.py;
+          body="$(cat comment.txt)";
+          body="${body//'%'/'%25'}";
+          body="${body//$'\n'/'%0A'}";
+          body="${body//$'\r'/'%0D'}";
+          echo "::set-output name=bodylength::${#body}"
+          echo "::set-output name=body::$body"
+      - name: Find Comment
+        if: ${{ steps.testrun.outputs.bodylength > 130 }}
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Pull request artifacts
+      - name: Create or update comment
+        if: ${{ steps.testrun.outputs.bodylength > 130 }}
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ${{ steps.testrun.outputs.body }}
+          edit-mode: replace

.gitignore

@@ -229,6 +229,7 @@ pip-log.txt
 /whitelist.txt
 DEBUG
 config.ini.php
+config/*
 
 ######################
 ## VisualStudioCode ##

Dockerfile

@@ -1,24 +1,24 @@
-FROM php:7-apache-buster
+FROM php:7.4.29-fpm
 
 LABEL description="RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites that don't have one."
 LABEL repository="https://github.com/RSS-Bridge/rss-bridge"
 LABEL website="https://github.com/RSS-Bridge/rss-bridge"
 
-ENV APACHE_DOCUMENT_ROOT=/app
+RUN apt-get update && \
+    apt-get install --yes --no-install-recommends \
+      nginx \
+      zlib1g-dev \
+      libzip-dev \
+      libmemcached-dev && \
+    docker-php-ext-install zip && \
+    pecl install memcached && \
+    docker-php-ext-enable memcached && \
+    mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini"
 
-RUN mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" \
-	&& apt-get --yes update \
-	&& apt-get --yes --no-install-recommends install \
-		zlib1g-dev \
-		libmemcached-dev \
-	&& rm -rf /var/lib/apt/lists/* \
-	&& pecl install memcached \
-	&& docker-php-ext-enable memcached \
-	&& sed -ri -e 's!/var/www/html!${APACHE_DOCUMENT_ROOT}!g' /etc/apache2/sites-available/*.conf \
-	&& sed -ri -e 's!/var/www/!${APACHE_DOCUMENT_ROOT}!g' /etc/apache2/apache2.conf /etc/apache2/conf-available/*.conf \
-	&& sed -ri -e 's/(MinProtocol\s*=\s*)TLSv1\.2/\1None/' /etc/ssl/openssl.cnf \
-	&& sed -ri -e 's/(CipherString\s*=\s*DEFAULT)@SECLEVEL=2/\1/' /etc/ssl/openssl.cnf
+COPY ./config/nginx.conf /etc/nginx/sites-enabled/default
 
 COPY --chown=www-data:www-data ./ /app/
 
-CMD ["/app/docker-entrypoint.sh"]
+EXPOSE 80
+
+ENTRYPOINT ["/app/docker-entrypoint.sh"]

README.md

@@ -15,17 +15,17 @@ Supported sites/pages (examples)
 ===
 
 * `Bandcamp` : Returns last release from [bandcamp](https://bandcamp.com/) for a tag
-* `Cryptome` : Returns the most recent documents from [Cryptome.org](http://cryptome.org/)
-* `DansTonChat`: Most recent quotes from [danstonchat.com](http://danstonchat.com/)
+* `Cryptome` : Returns the most recent documents from [Cryptome.org](https://cryptome.org/)
+* `DansTonChat`: Most recent quotes from [danstonchat.com](https://danstonchat.com/)
 * `DuckDuckGo`: Most recent results from [DuckDuckGo.com](https://duckduckgo.com/)
 * `Facebook` : Returns the latest posts on a page or profile on [Facebook](https://facebook.com/) (There is an [issue](https://github.com/RSS-Bridge/rss-bridge/issues/2047) for public instances)
-* `FlickrExplore` : [Latest interesting images](http://www.flickr.com/explore) from Flickr
+* `FlickrExplore` : [Latest interesting images](https://www.flickr.com/explore) from Flickr
 * `GoogleSearch` : Most recent results from Google Search
 * `Identi.ca` : Identica user timeline (Should be compatible with other Pump.io instances)
-* `Instagram`: Most recent photos from an Instagram user (There is an [issue](https://github.com/RSS-Bridge/rss-bridge/issues/1891) for public instances)
-* `OpenClassrooms`: Lastest tutorials from [fr.openclassrooms.com](http://fr.openclassrooms.com/)
+* `Instagram`: Most recent photos from an Instagram user (It is recommended to [configure](https://rss-bridge.github.io/rss-bridge/Bridge_Specific/Instagram.html) this bridge to work)
+* `OpenClassrooms`: Lastest tutorials from [openclassrooms.com](https://openclassrooms.com/)
 * `Pinterest`: Most recent photos from user or search
-* `ScmbBridge`: Newest stories from [secouchermoinsbete.fr](http://secouchermoinsbete.fr/)
+* `ScmbBridge`: Newest stories from [secouchermoinsbete.fr](https://secouchermoinsbete.fr/)
 * `ThePirateBay` : Returns the newest indexed torrents from [The Pirate Bay](https://thepiratebay.se/) with keywords
 * `Twitter` : Return keyword/hashtag search or user timeline
 * `Wikipedia`: highlighted articles from [Wikipedia](https://wikipedia.org/) in English, German, French or Esperanto
@@ -44,20 +44,18 @@ RSS-Bridge is capable of producing several output formats:
 * `Mrss` : MRSS feed, for use in feed readers
 * `Plaintext` : Raw text, for consumption by other applications
 
-You can extend RSS-Bridge with your own format, using the [Format API](https://github.com/RSS-Bridge/rss-bridge/wiki/Format-API)!
+You can extend RSS-Bridge with your own format, using the [Format API](https://rss-bridge.github.io/rss-bridge/Format_API/index.html)!
 
 Screenshot
 ===
 
 Welcome screen:
 
-![Screenshot](https://github.com/RSS-Bridge/rss-bridge/wiki/images/screenshot_rss-bridge_welcome.png)
-
-***
+![Screenshot](/static/screenshot_rss-bridge_welcome.png?raw=true)
 
 RSS-Bridge hashtag (#rss-bridge) search on Twitter, in Atom format (as displayed by Firefox):
 
-![Screenshot](https://github.com/RSS-Bridge/rss-bridge/wiki/images/screenshot_twitterbridge_atom.png)
+![Screenshot](/static/screenshot_twitterbridge_atom.png?raw=true)
 
 Requirements
 ===
@@ -71,26 +69,29 @@ RSS-Bridge requires PHP 7.1 or higher with following extensions enabled:
   - [`curl`](https://secure.php.net/manual/en/book.curl.php)
   - [`json`](https://secure.php.net/manual/en/book.json.php)
   - [`filter`](https://secure.php.net/manual/en/book.filter.php)
-  - [`sqlite3`](http://php.net/manual/en/book.sqlite3.php) (only when using SQLiteCache)
+  - [`zip`](https://secure.php.net/manual/en/book.zip.php) (for some bridges)
+  - [`sqlite3`](https://www.php.net/manual/en/book.sqlite3.php) (only when using SQLiteCache)
 
-Find more information on our [Wiki](https://github.com/rss-bridge/rss-bridge/wiki)
+Find more information on our [Documentation](https://rss-bridge.github.io/rss-bridge/index.html)
 
 Enable / Disable bridges
 ===
 
 RSS-Bridge allows you to take full control over which bridges are displayed to the user. That way you can host your own RSS-Bridge service with your favorite collection of bridges!
 
-Find more information on the [Wiki](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitelisting)
+Find more information on the [Documentation](https://rss-bridge.github.io/rss-bridge/For_Hosts/Whitelisting.html)
 
-**Notice**: By default, RSS-Bridge will only show a small subset of bridges. Make sure to read up on [whitelisting](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitelisting) to unlock the full potential of RSS-Bridge!
+**Notice**: By default, RSS-Bridge will only show a small subset of bridges. Make sure to read up on [whitelisting](https://rss-bridge.github.io/rss-bridge/For_Hosts/Whitelisting.html) to unlock the full potential of RSS-Bridge!
 
 Deploy
 ===
 
 Thanks to the community, hosting your own instance of RSS-Bridge is as easy as clicking a button!
+*Note: External providers' applications are packaged by 3rd parties. Use at your own discretion.*
 
 [![Deploy on Scalingo](https://cdn.scalingo.com/deploy/button.svg)](https://my.scalingo.com/deploy?source=https://github.com/sebsauvage/rss-bridge)
 [![Deploy to Heroku](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy)
+[![Deploy to Cloudron](https://cloudron.io/img/button.svg)](https://www.cloudron.io/store/com.rssbridgeapp.cloudronapp.html)
 
 Getting involved
 ===
@@ -102,13 +103,13 @@ There are many ways for you to getting involved with RSS-Bridge. Here are a few
 - Request new features or suggest ideas (via [Issues](https://github.com/RSS-Bridge/rss-bridge/issues))
 - Discuss bugs, features, ideas or [issues](https://github.com/RSS-Bridge/rss-bridge/issues)
 - Add new bridges or improve the API
-- Improve the [Wiki](https://github.com/RSS-Bridge/rss-bridge/wiki)
+- Improve the [Documentation](https://rss-bridge.github.io/rss-bridge/)
 - Host an instance of RSS-Bridge for your personal use or make it available to the community :sparkling_heart:
 
 Authors
 ===
 
-We are RSS-Bridge community, a group of developers continuing the project initiated by sebsauvage, webmaster of [sebsauvage.net](http://sebsauvage.net), author of [Shaarli](http://sebsauvage.net/wiki/doku.php?id=php:shaarli) and [ZeroBin](http://sebsauvage.net/wiki/doku.php?id=php:zerobin).
+We are RSS-Bridge community, a group of developers continuing the project initiated by sebsauvage, webmaster of [sebsauvage.net](https://sebsauvage.net), author of [Shaarli](https://sebsauvage.net/wiki/doku.php?id=php:shaarli) and [ZeroBin](https://sebsauvage.net/wiki/doku.php?id=php:zerobin).
 
 **Contributors** (sorted alphabetically):
 <!--
@@ -316,16 +317,18 @@ The source code for RSS-Bridge is [Public Domain](UNLICENSE).
 
 RSS-Bridge uses third party libraries with their own license:
 
-  * [`Parsedown`](https://github.com/erusev/parsedown) licensed under the [MIT License](http://opensource.org/licenses/MIT)
-  * [`PHP Simple HTML DOM Parser`](http://simplehtmldom.sourceforge.net/) licensed under the [MIT License](http://opensource.org/licenses/MIT)
-  * [`php-urljoin`](https://github.com/fluffy-critter/php-urljoin) licensed under the [MIT License](http://opensource.org/licenses/MIT)
+  * [`Parsedown`](https://github.com/erusev/parsedown) licensed under the [MIT License](https://opensource.org/licenses/MIT)
+  * [`PHP Simple HTML DOM Parser`](https://simplehtmldom.sourceforge.io/docs/1.9/index.html) licensed under the [MIT License](https://opensource.org/licenses/MIT)
+  * [`php-urljoin`](https://github.com/fluffy-critter/php-urljoin) licensed under the [MIT License](https://opensource.org/licenses/MIT)
+  * [php polyfills](https://github.com/symfony/polyfill) licensed under the [MIT License](https://opensource.org/licenses/MIT)
+  * [`Laravel framework`](https://github.com/laravel/framework/) licensed under the [MIT License](https://opensource.org/licenses/MIT)
 
 Technical notes
 ===
 
   * RSS-Bridge uses caching to prevent services from banning your server for repeatedly updating feeds. The specific cache duration can be different between bridges. Cached files are deleted automatically after 24 hours.
-  * You can implement your own bridge, [following these instructions](https://github.com/RSS-Bridge/rss-bridge/wiki/Bridge-API).
-  * You can enable debug mode to disable caching. Find more information on the [Wiki](https://github.com/RSS-Bridge/rss-bridge/wiki/Debug-mode)
+  * You can implement your own bridge, [following these instructions](https://rss-bridge.github.io/rss-bridge/Bridge_API/index.html).
+  * You can enable debug mode to disable caching. Find more information on the [Wiki](https://rss-bridge.github.io/rss-bridge/For_Developers/Debug_mode.html)
 
 Rant
 ===

actions/ConnectivityAction.php

@@ -25,7 +25,7 @@ class ConnectivityAction extends ActionAbstract {
 	public function execute() {
 
 		if(!Debug::isEnabled()) {
-			returnError('This action is only available in debug mode!');
+			returnError('This action is only available in debug mode!', 400);
 		}
 
 		if(!isset($this->userData['bridge'])) {
@@ -55,7 +55,6 @@ class ConnectivityAction extends ActionAbstract {
 	private function reportBridgeConnectivity($bridgeName) {
 
 		$bridgeFac = new \BridgeFactory();
-		$bridgeFac->setWorkingDir(PATH_LIB_BRIDGES);
 
 		if(!$bridgeFac->isWhitelisted($bridgeName)) {
 			header('Content-Type: text/html');
@@ -84,12 +83,10 @@ class ConnectivityAction extends ActionAbstract {
 		try {
 			$reply = getContents($bridge::URI, array(), $curl_opts, true);
 
-			if($reply) {
+			if($reply['code'] === 200) {
 				$retVal['successful'] = true;
-				if (isset($reply['header'])) {
-					if (strpos($reply['header'], 'HTTP/1.1 301 Moved Permanently') !== false) {
-						$retVal['http_code'] = 301;
-					}
+				if (strpos(implode('', $reply['status_lines']), '301 Moved Permanently')) {
+					$retVal['http_code'] = 301;
 				}
 			}
 		} catch(Exception $e) {

actions/DetectAction.php

@@ -20,7 +20,6 @@ class DetectAction extends ActionAbstract {
 			or returnClientError('You must specify a format!');
 
 		$bridgeFac = new \BridgeFactory();
-		$bridgeFac->setWorkingDir(PATH_LIB_BRIDGES);
 
 		foreach($bridgeFac->getBridgeNames() as $bridgeName) {
 

actions/DisplayAction.php

@@ -28,7 +28,6 @@ class DisplayAction extends ActionAbstract {
 			or returnClientError('You must specify a format!');
 
 		$bridgeFac = new \BridgeFactory();
-		$bridgeFac->setWorkingDir(PATH_LIB_BRIDGES);
 
 		// whitelist control
 		if(!$bridgeFac->isWhitelisted($bridge)) {
@@ -245,8 +244,14 @@ class DisplayAction extends ActionAbstract {
 			$format = $formatFac->create($format);
 			$format->setItems($items);
 			$format->setExtraInfos($infos);
-			$format->setLastModified($cache->getTime());
-			$format->display();
+			$lastModified = $cache->getTime();
+			$format->setLastModified($lastModified);
+			if ($lastModified) {
+				header('Last-Modified: ' . gmdate('D, d M Y H:i:s ', $lastModified) . 'GMT');
+			}
+			header('Content-Type: ' . $format->getMimeType() . '; charset=' . $format->getCharset());
+
+			echo $format->stringify();
 		} catch(Error $e) {
 			error_log($e);
 			header('Content-Type: text/html', true, $e->getCode());

actions/ListAction.php

@@ -18,7 +18,6 @@ class ListAction extends ActionAbstract {
 		$list->total = 0;
 
 		$bridgeFac = new \BridgeFactory();
-		$bridgeFac->setWorkingDir(PATH_LIB_BRIDGES);
 
 		foreach($bridgeFac->getBridgeNames() as $bridgeName) {
 

app.json

@@ -1,8 +1,8 @@
 {
   "service": "Heroku",
-  "name": "RSS-Bridge",
+  "name": "rss-bridge-heroku",
   "description": "RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites which don't have one.",
-  "repository": "https://github.com/RSS-Bridge/rss-bridge",
+  "repository": "https://github.com/RSS-Bridge/rss-bridge?1651005770",
   "keywords": ["php", "rss-bridge", "rss"]
 }
 

bridges/ABCTabsBridge.php

@@ -1,42 +0,0 @@
-<?php
-class ABCTabsBridge extends BridgeAbstract {
-
-	const MAINTAINER = 'kranack';
-	const NAME = 'ABC Tabs Bridge';
-	const URI = 'https://www.abc-tabs.com/';
-	const DESCRIPTION = 'Returns 22 newest tabs';
-
-	public function collectData(){
-		$html = '';
-		$html = getSimpleHTMLDOM(static::URI . 'tablatures/nouveautes.html')
-			or returnClientError('No results for this query.');
-
-		$table = $html->find('table#myTable', 0)->children(1);
-
-		foreach ($table->find('tr') as $tab) {
-			$item = array();
-			$item['author'] = $tab->find('td', 1)->plaintext
-			. ' - '
-			. $tab->find('td', 2)->plaintext;
-
-			$item['title'] = $tab->find('td', 1)->plaintext
-			. ' - '
-			. $tab->find('td', 2)->plaintext;
-
-			$item['content'] = 'Le '
-			. $tab->find('td', 0)->plaintext
-			. '<br> Par: '
-			. $tab->find('td', 5)->plaintext
-			. '<br> Type: '
-			. $tab->find('td', 3)->plaintext;
-
-			$item['id'] = static::URI
-			. $tab->find('td', 2)->find('a', 0)->getAttribute('href');
-
-			$item['uri'] = static::URI
-			. $tab->find('td', 2)->find('a', 0)->getAttribute('href');
-
-			$this->items[] = $item;
-		}
-	}
-}

bridges/AcrimedBridge.php

@@ -7,8 +7,21 @@ class AcrimedBridge extends FeedExpander {
 	const CACHE_TIMEOUT = 4800; //2hours
 	const DESCRIPTION = 'Returns the newest articles';
 
+	const PARAMETERS = [
+		[
+			'limit' => [
+				'name' => 'limit',
+				'type' => 'number',
+				'defaultValue' => -1,
+			]
+		]
+	];
+
 	public function collectData(){
-		$this->collectExpandableDatas(static::URI . 'spip.php?page=backend');
+		$this->collectExpandableDatas(
+			static::URI . 'spip.php?page=backend',
+			$this->getInput('limit')
+		);
 	}
 
 	protected function parseItem($newsItem){

bridges/AlfaBankByBridge.php

@@ -0,0 +1,83 @@
+<?php
+class AlfaBankByBridge extends BridgeAbstract {
+
+	const MAINTAINER = 'lassana';
+	const NAME = 'AlfaBank.by Новости';
+	const URI = 'https://www.alfabank.by';
+	const DESCRIPTION = 'Уведомления Alfa-Now — новости от Альфа-Банка';
+	const CACHE_TIMEOUT = 3600; // 1 hour
+	const PARAMETERS = array(
+		'News' => array(
+			'business' => array(
+				'name' => 'Альфа Бизнес',
+				'type' => 'list',
+				'title' => 'В зависимости от выбора, возращает уведомления для" .
+					" клиентов физ. лиц либо для клиентов-юридических лиц и ИП',
+				'values' => array(
+					'Новости' => 'news',
+					'Новости бизнеса' => 'newsBusiness'
+				),
+				'defaultValue' => 'news'
+			),
+			'fullContent' => array(
+				'name' => 'Включать содержимое',
+				'type' => 'checkbox',
+				'title' => 'Если выбрано, содержимое уведомлений вставляется в поток (работает медленно)'
+			)
+		)
+	);
+
+	public function collectData() {
+		$business = $this->getInput('business') == 'newsBusiness';
+		$fullContent = $this->getInput('fullContent') == 'on';
+
+		$mainPageUrl = self::URI . '/about/articles/uvedomleniya/';
+		if($business) {
+			$mainPageUrl .= '?business=true';
+		}
+		$html = getSimpleHTMLDOM($mainPageUrl);
+		$limit = 0;
+
+		foreach($html->find('a.notifications__item') as $element) {
+			if($limit < 10) {
+				$item = array();
+				$item['uid'] = 'urn:sha1:' . hash('sha1', $element->getAttribute('data-notification-id'));
+				$item['title'] = $element->find('div.item-title', 0)->innertext;
+				$item['timestamp'] = DateTime::createFromFormat(
+					'd M Y',
+					$this->ruMonthsToEn($element->find('div.item-date', 0)->innertext)
+					)->getTimestamp();
+
+				$itemUrl = self::URI . $element->href;
+				if($business) {
+					$itemUrl = str_replace('?business=true', '', $itemUrl);
+				}
+				$item['uri'] = $itemUrl;
+
+				if($fullContent) {
+					$itemHtml = getSimpleHTMLDOM($itemUrl);
+					if($itemHtml) {
+						$item['content'] = $itemHtml->find('div.now-p__content-text', 0)->innertext;
+					}
+				}
+
+				$this->items[] = $item;
+				$limit++;
+			}
+		}
+	}
+
+	public function getIcon() {
+		return static::URI . '/local/images/favicon.ico';
+	}
+
+	private function ruMonthsToEn($date) {
+		$ruMonths = array(
+			'Января', 'Февраля', 'Марта', 'Апреля', 'Мая', 'Июня',
+			'Июля', 'Августа', 'Сентября', 'Октября', 'Ноября', 'Декабря' );
+		$enMonths = array(
+			'January', 'February', 'March', 'April', 'May', 'June',
+			'July', 'August', 'September', 'October', 'November', 'December' );
+		return str_replace($ruMonths, $enMonths, $date);
+	}
+}

bridges/AllocineFRBridge.php

@@ -4,7 +4,7 @@ class AllocineFRBridge extends BridgeAbstract {
 	const MAINTAINER = 'superbaillot.net';
 	const NAME = 'Allo Cine Bridge';
 	const CACHE_TIMEOUT = 25200; // 7h
-	const URI = 'https://www.allocine.fr/';
+	const URI = 'https://www.allocine.fr';
 	const DESCRIPTION = 'Bridge for allocine.fr';
 	const PARAMETERS = array( array(
 		'category' => array(
@@ -35,26 +35,26 @@ class AllocineFRBridge extends BridgeAbstract {
 		if(!is_null($this->getInput('category'))) {
 
 			$categories = array(
-				'faux-raccord' => 'video/programme-12284/saison-37054/',
-				'fanzone' => 'video/programme-12298/saison-37059/',
-				'game-in-cine' => 'video/programme-12288/saison-22971/',
-				'pour-la-faire-courte' => 'video/programme-20960/saison-29678/',
-				'home-cinema' => 'video/programme-12287/saison-34703/',
-				'pils-par-ici-les-sorties' => 'video/programme-25789/saison-37253/',
-				'allocine-lemission-sur-lestream' => 'video/programme-25123/saison-36067/',
-				'give-me-five' => 'video/programme-21919/saison-34518/',
-				'aviez-vous-remarque' => 'video/programme-19518/saison-37084/',
-				'et-paf-il-est-mort' => 'video/programme-25113/saison-36657/',
-				'the-big-fan-theory' => 'video/programme-20403/saison-37419/',
-				'cliches' => 'video/programme-24834/saison-35591/',
-				'completement' => 'video/programme-23859/saison-34102/',
-				'fun-facts' => 'video/programme-23040/saison-32686/',
-				'origin-story' => 'video/programme-25667/saison-37041/'
+				'faux-raccord' => '/video/programme-12284/',
+				'fanzone' => '/video/programme-12298/',
+				'game-in-cine' => '/video/programme-12288/',
+				'pour-la-faire-courte' => '/video/programme-20960/',
+				'home-cinema' => '/video/programme-12287/',
+				'pils-par-ici-les-sorties' => '/video/programme-25789/',
+				'allocine-lemission-sur-lestream' => '/video/programme-25123/',
+				'give-me-five' => '/video/programme-21919/saison-34518/',
+				'aviez-vous-remarque' => '/video/programme-19518/',
+				'et-paf-il-est-mort' => '/video/programme-25113/',
+				'the-big-fan-theory' => '/video/programme-20403/',
+				'cliches' => '/video/programme-24834/',
+				'completement' => '/video/programme-23859/',
+				'fun-facts' => '/video/programme-23040/',
+				'origin-story' => '/video/programme-25667/'
 			);
 
 			$category = $this->getInput('category');
 			if(array_key_exists($category, $categories)) {
-				return static::URI . $categories[$category];
+				return static::URI . $this->getLastSeasonURI($categories[$category]);
 			} else {
 				returnClientError('Emission inconnue');
 			}
@@ -63,6 +63,14 @@ class AllocineFRBridge extends BridgeAbstract {
 		return parent::getURI();
 	}
 
+	private function getLastSeasonURI($category)
+	{
+		$html = getSimpleHTMLDOMCached(static::URI . $category, 86400);
+		$seasonLink = $html->find('section[class=section-wrap section]', 0)->find('div[class=cf]', 0)->find('a', 0);
+		$URI = $seasonLink->href;
+		return $URI;
+	}
+
 	public function getName(){
 		if(!is_null($this->getInput('category'))) {
 			return self::NAME . ' : '
@@ -83,12 +91,11 @@ class AllocineFRBridge extends BridgeAbstract {
 				$this->getInput('category'),
 				self::PARAMETERS[$this->queriedContext]['category']['values']
 			);
-
 		foreach($html->find('div[class=gd-col-left]', 0)->find('div[class*=video-card]') as $element) {
 			$item = array();
 
 			$title = $element->find('a[class*=meta-title-link]', 0);
-			$content = trim($element->outertext);
+			$content = trim(defaultLinkTo($element->outertext, static::URI));
 
 			// Replace image 'src' with the one in 'data-src'
 			$content = preg_replace('@src="data:image/gif;base64,[A-Za-z0-9+\/]*"@', '', $content);
@@ -99,7 +106,7 @@ class AllocineFRBridge extends BridgeAbstract {
 
 			$item['content'] = $content;
 			$item['title'] = trim($title->innertext);
-			$item['uri'] = static::URI . substr($title->href, 1);
+			$item['uri'] = static::URI . '/' . substr($title->href, 1);
 			$this->items[] = $item;
 		}
 	}

bridges/AmazonBridge.php

@@ -42,6 +42,8 @@ class AmazonBridge extends BridgeAbstract {
 				'Mexico' => 'com.mx',
 				'Netherlands' => 'nl',
 				'Spain' => 'es',
+				'Sweden' => 'se',
+				'Turkey' => 'com.tr',
 				'United Kingdom' => 'co.uk',
 				'United States' => 'com',
 			),
@@ -49,6 +51,48 @@ class AmazonBridge extends BridgeAbstract {
 		),
 	));
 
+	public function collectData() {
+
+		$baseUrl = sprintf('https://www.amazon.%s', $this->getInput('tld'));
+
+		$url = sprintf(
+			'%s/s/?field-keywords=%s&sort=%s',
+			$baseUrl,
+			urlencode($this->getInput('q')),
+			$this->getInput('sort')
+		);
+
+		$dom = getSimpleHTMLDOM($url);
+
+		$elements = $dom->find('div.s-result-item');
+
+		foreach($elements as $element) {
+			$item = [];
+
+			$title = $element->find('h2', 0);
+			if (!$title) {
+				continue;
+			}
+
+			$item['title'] = $title->innertext;
+
+			$itemUrl = $element->find('a', 0)->href;
+			$item['uri'] = urljoin($baseUrl, $itemUrl);
+
+			$image = $element->find('img', 0);
+			if ($image) {
+				$item['content'] = '<img src="' . $image->getAttribute('src') . '" /><br />';
+			}
+
+			$price = $element->find('span.a-price > .a-offscreen', 0);
+			if ($price) {
+				$item['content'] .= $price->innertext;
+			}
+
+			$this->items[] = $item;
+		}
+	}
+
 	public function getName(){
 		if(!is_null($this->getInput('tld')) && !is_null($this->getInput('q'))) {
 			return 'Amazon.' . $this->getInput('tld') . ': ' . $this->getInput('q');
@@ -56,40 +100,4 @@ class AmazonBridge extends BridgeAbstract {
 
 		return parent::getName();
 	}
-
-	public function collectData() {
-
-		$uri = 'https://www.amazon.' . $this->getInput('tld') . '/';
-		$uri .= 's/?field-keywords=' . urlencode($this->getInput('q')) . '&sort=' . $this->getInput('sort');
-
-		$html = getSimpleHTMLDOM($uri);
-
-		foreach($html->find('li.s-result-item') as $element) {
-
-			$item = array();
-
-			// Title
-			$title = $element->find('h2', 0);
-			if (is_null($title)) {
-				continue;
-			}
-
-			$item['title'] = html_entity_decode($title->innertext, ENT_QUOTES);
-
-			// Url
-			$uri = $title->parent()->getAttribute('href');
-			$uri = substr($uri, 0, strrpos($uri, '/'));
-
-			$item['uri'] = substr($uri, 0, strrpos($uri, '/'));
-
-			// Content
-			$image = $element->find('img', 0);
-			$price = $element->find('span.s-price', 0);
-			$price = ($price) ? $price->innertext : '';
-
-			$item['content'] = '<img src="' . $image->getAttribute('src') . '" /><br />' . $price;
-
-			$this->items[] = $item;
-		}
-	}
 }

bridges/AmazonPriceTrackerBridge.php

@@ -20,7 +20,7 @@ class AmazonPriceTrackerBridge extends BridgeAbstract {
 			'name' => 'Country',
 			'type' => 'list',
 			'values' => array(
-				'Australia' 	=> 'com.au',
+				'Australia' 		=> 'com.au',
 				'Brazil' 		=> 'com.br',
 				'Canada' 		=> 'ca',
 				'China' 		=> 'cn',
@@ -30,9 +30,10 @@ class AmazonPriceTrackerBridge extends BridgeAbstract {
 				'Italy' 		=> 'it',
 				'Japan' 		=> 'co.jp',
 				'Mexico' 		=> 'com.mx',
-				'Netherlands'	=> 'nl',
+				'Netherlands'		=> 'nl',
 				'Spain' 		=> 'es',
 				'Sweden' 		=> 'se',
+				'Turkey'		=> 'com.tr',
 				'United Kingdom'	=> 'co.uk',
 				'United States'		=> 'com',
 			),
@@ -49,6 +50,8 @@ class AmazonPriceTrackerBridge extends BridgeAbstract {
 		'.a-color-price',
 	);
 
+	const WHITESPACE = " \t\n\r\0\x0B\xC2\xA0";
+
 	protected $title;
 
 	/**
@@ -154,6 +157,22 @@ EOT;
 		return false;
 	}
 
+	private function scrapePriceTwister($html) {
+		$str = $html->find('.twister-plus-buying-options-price-data', 0);
+
+		$data = json_decode($str->innertext, true);
+		if(count($data) === 1) {
+			$data = $data[0];
+			return array(
+				'displayPrice' => $data['displayPrice'],
+				'currency' => $data['currency'],
+				'shipping' => '0',
+			);
+		}
+
+		return false;
+	}
+
 	private function scrapePriceGeneric($html) {
 		$priceDiv = null;
 
@@ -168,12 +187,11 @@ EOT;
 			return false;
 		}
 
-		$priceString = $priceDiv->plaintext;
-
-		preg_match('/[\d.,]+/', $priceString, $matches);
+		$priceString = str_replace(str_split(self::WHITESPACE), '', $priceDiv->plaintext);
+		preg_match('/(\d+\.\d{0,2})/', $priceString, $matches);
 
 		$price = $matches[0];
-		$currency = trim(str_replace($price, '', $priceString), " \t\n\r\0\x0B\xC2\xA0");
+		$currency = str_replace($price, '', $priceString);
 
 		if ($price != null && $currency != null) {
 			return array(
@@ -186,6 +204,21 @@ EOT;
 		return false;
 	}
 
+	private function renderContent($image, $data) {
+		$price = $data['displayPrice'];
+		if (!$price) {
+			$price = "{$data['price']} {$data['currency']}";
+		}
+
+		$html = "$image<br>Price: $price";
+
+		if ($data['shipping'] !== '0') {
+			$html .= "<br>Shipping: {$data['shipping']} {$data['currency']}</br>";
+		}
+
+		return $html;
+	}
+
 	/**
 	 * Scrape method for Amazon product page
 	 * @return [type] [description]
@@ -195,20 +228,16 @@ EOT;
 		$this->title = $this->getTitle($html);
 		$imageTag = $this->getImage($html);
 
-		$data = $this->scrapePriceFromMetrics($html) ?: $this->scrapePriceGeneric($html);
+		$data = $this->scrapePriceGeneric($html);
 
 		$item = array(
 			'title' 	=> $this->title,
 			'uri' 		=> $this->getURI(),
-			'content' 	=> "$imageTag<br/>Price: {$data['price']} {$data['currency']}",
+			'content' 	=> $this->renderContent($imageTag, $data),
 			// This is to ensure that feed readers notice the price change
 			'uid'		=> md5($data['price'])
 		);
 
-		if ($data['shipping'] !== '0') {
-			$item['content'] .= "<br>Shipping: {$data['shipping']} {$data['currency']}</br>";
-		}
-
 		$this->items[] = $item;
 	}
 }

bridges/AnimeUltimeBridge.php

@@ -37,9 +37,11 @@ class AnimeUltimeBridge extends BridgeAbstract {
 		$processedOK = 0;
 		foreach (array($thismonth, $lastmonth) as $requestFilter) {
 
-			//Retrive page contents
 			$url = self::URI . 'history-0-1/' . $requestFilter;
-			$html = getSimpleHTMLDOM($url);
+			$html = getContents($url);
+			// Convert html from iso-8859-1 => utf8
+			$html = utf8_encode($html);
+			$html = str_get_html($html);
 
 			//Relases are sorted by day : process each day individually
 			foreach($html->find('div.history', 0)->find('h3') as $daySection) {
@@ -87,6 +89,8 @@ class AnimeUltimeBridge extends BridgeAbstract {
 
 							// Retrieve description from description page
 							$html_item = getContents($item_uri);
+							// Convert html from iso-8859-1 => utf8
+							$html_item = utf8_encode($html_item);
 							$item_description = substr(
 								$html_item,
 								strpos($html_item, 'class="principal_contain" align="center">') + 41

bridges/AppleAppStoreBridge.php

@@ -94,6 +94,7 @@ class AppleAppStoreBridge extends BridgeAbstract {
 
 		$headers = array(
 			"Authorization: Bearer $token",
+			'Origin: https://apps.apple.com',
 		);
 
 		$json = json_decode(getContents($uri, $headers), true);

bridges/Arte7Bridge.php

@@ -1,61 +1,95 @@
 <?php
 class Arte7Bridge extends BridgeAbstract {
 
-	// const MAINTAINER = 'mitsukarenai';
 	const NAME = 'Arte +7';
 	const URI = 'https://www.arte.tv/';
+	const MAINTAINER = 'imagoiq';
 	const CACHE_TIMEOUT = 1800; // 30min
 	const DESCRIPTION = 'Returns newest videos from ARTE +7';
 
 	const API_TOKEN = 'Nzc1Yjc1ZjJkYjk1NWFhN2I2MWEwMmRlMzAzNjI5NmU3NWU3ODg4ODJjOWMxNTMxYzEzZGRjYjg2ZGE4MmIwOA';
 
 	const PARAMETERS = array(
-		'Catégorie (Français)' => array(
-			'catfr' => array(
+		'global' => [
+			'sort_by' => array(
 				'type' => 'list',
-				'name' => 'Catégorie',
+				'name' => 'Sort by',
+				'required' => false,
+				'defaultValue' => null,
 				'values' => array(
-					'Toutes les vidéos (français)' => null,
-					'Actu & société' => 'ACT',
-					'Séries & fiction' => 'SER',
-					'Cinéma' => 'CIN',
-					'Arts & spectacles classiques' => 'ARS',
+					'Default' => null,
+					'Video rights start date' => 'videoRightsBegin',
+					'Video rights end date' => 'videoRightsEnd',
+					'Brodcast date' => 'broadcastBegin',
+					'Creation date' => 'creationDate',
+					'Last modified' => 'lastModified',
+					'Number of views' => 'views',
+					'Number of views per period' => 'viewsPeriod',
+					'Available screens' => 'availableScreens',
+					'Episode' => 'episode'
+				),
+			),
+			'sort_direction' => array(
+				'type' => 'list',
+				'name' => 'Sort direction',
+				'required' => false,
+				'defaultValue' => 'DESC',
+				'values' => array(
+					'Ascending' => 'ASC',
+					'Descending' => 'DESC'
+				),
+			),
+			'exclude_trailers' => [
+				'name' => 'Exclude trailers',
+				'type' => 'checkbox',
+				'required' => false,
+				'defaultValue' => false
+			],
+		],
+		'Category' => array(
+			'lang' => array(
+				'type' => 'list',
+				'name' => 'Language',
+				'values' => array(
+					'Français' => 'fr',
+					'Deutsch' => 'de',
+					'English' => 'en',
+					'Español' => 'es',
+					'Polski' => 'pl',
+					'Italiano' => 'it'
+				),
+			),
+			'cat' => array(
+				'type' => 'list',
+				'name' => 'Category',
+				'values' => array(
+					'All videos' => null,
+					'News & society' => 'ACT',
+					'Series & fiction' => 'SER',
+					'Cinema' => 'CIN',
+					'Culture' => 'ARS',
 					'Culture pop' => 'CPO',
-					'Découverte' => 'DEC',
-					'Histoire' => 'HIST',
+					'Discovery' => 'DEC',
+					'History' => 'HIST',
 					'Science' => 'SCI',
-					'Autre' => 'AUT'
+					'Other' => 'AUT'
 				)
-			)
+			),
 		),
-		'Collection (Français)' => array(
-			'colfr' => array(
-				'name' => 'Collection id',
-				'required' => true,
-				'title' => 'ex. RC-014095 pour https://www.arte.tv/fr/videos/RC-014095/blow-up/',
-				'exampleValue'	=> 'RC-014095'
-			)
-		),
-		'Catégorie (Allemand)' => array(
-			'catde' => array(
+		'Collection' => array(
+			'lang' => array(
 				'type' => 'list',
-				'name' => 'Catégorie',
+				'name' => 'Language',
 				'values' => array(
-					'Alle Videos (deutsch)' => null,
-					'Aktuelles & Gesellschaft' => 'ACT',
-					'Fernsehfilme & Serien' => 'SER',
-					'Kino' => 'CIN',
-					'Kunst & Kultur' => 'ARS',
-					'Popkultur & Alternativ' => 'CPO',
-					'Entdeckung' => 'DEC',
-					'Geschichte' => 'HIST',
-					'Wissenschaft' => 'SCI',
-					'Sonstiges' => 'AUT'
+					'Français' => 'fr',
+					'Deutsch' => 'de',
+					'English' => 'en',
+					'Español' => 'es',
+					'Polski' => 'pl',
+					'Italiano' => 'it'
 				)
-			)
-		),
-		'Collection (Allemand)' => array(
-			'colde' => array(
+			),
+			'col' => array(
 				'name' => 'Collection id',
 				'required' => true,
 				'title' => 'ex. RC-014095 pour https://www.arte.tv/de/videos/RC-014095/blow-up/',
@@ -66,26 +100,23 @@ class Arte7Bridge extends BridgeAbstract {
 
 	public function collectData(){
 		switch($this->queriedContext) {
-		case 'Catégorie (Français)':
-			$category = $this->getInput('catfr');
-			$lang = 'fr';
+		case 'Category':
+			$category = $this->getInput('cat');
+			$collectionId = null;
 			break;
-		case 'Collection (Français)':
-			$lang = 'fr';
-			$collectionId = $this->getInput('colfr');
-			break;
-		case 'Catégorie (Allemand)':
-			$category = $this->getInput('catde');
-			$lang = 'de';
-			break;
-		case 'Collection (Allemand)':
-			$lang = 'de';
-			$collectionId = $this->getInput('colde');
+		case 'Collection':
+			$collectionId = $this->getInput('col');
+			$category = null;
 			break;
 		}
 
-		$url = 'https://api.arte.tv/api/opa/v3/videos?sort=-lastModified&limit=10&language='
+		$lang = $this->getInput('lang');
+		$sort_by = $this->getInput('sort_by');
+		$sort_direction = $this->getInput('sort_direction') == 'ASC' ? '' : '-';
+
+		$url = 'https://api.arte.tv/api/opa/v3/videos?limit=15&language='
 			. $lang
+			. ($sort_by != null ? '&sort=' . $sort_direction . $sort_by : '')
 			. ($category != null ? '&category.code=' . $category : '')
 			. ($collectionId != null ? '&collections.collectionId=' . $collectionId : '');
 
@@ -97,6 +128,11 @@ class Arte7Bridge extends BridgeAbstract {
 		$input_json = json_decode($input, true);
 
 		foreach($input_json['videos'] as $element) {
+			if($this->getInput('exclude_trailers') && $element['platform'] == 'EXTRAIT') {
+				continue;
+			}
+
+			$durationSeconds = $element['durationSeconds'];
 
 			$item = array();
 			$item['uri'] = $element['url'];
@@ -108,10 +144,10 @@ class Arte7Bridge extends BridgeAbstract {
 			if(!empty($element['subtitle']))
 				$item['title'] = $element['title'] . ' | ' . $element['subtitle'];
 
-			$item['duration'] = round((int)$element['durationSeconds'] / 60);
+			$durationMinutes = round((int)$durationSeconds / 60);
 			$item['content'] = $element['teaserText']
 			. '<br><br>'
-			. $item['duration']
+			. $durationMinutes
 			. 'min<br><a href="'
 			. $item['uri']
 			. '"><img src="'

bridges/AsahiShimbunAJWBridge.php

@@ -20,9 +20,10 @@ class AsahiShimbunAJWBridge extends BridgeAbstract {
 					'Culture » Style' => 'culture/style',
 					'Culture » Movies' => 'culture/movies',
 					'Culture » Manga & Anime' => 'culture/manga_anime',
-					'Asia » China' => 'asia/china',
-					'Asia » Korean Peninsula' => 'asia/korean_peninsula',
-					'Asia » Around Asia' => 'asia/around_asia',
+					'Asia » China' => 'asia_world/china',
+					'Asia » Korean Peninsula' => 'asia_world/korean_peninsula',
+					'Asia » Around Asia' => 'asia_world/around_asia',
+					'Asia » World' => 'asia_world/world',
 					'Opinion » Editorial' => 'opinion/editorial',
 					'Opinion » Vox Populi' => 'opinion/vox',
 				),

bridges/AssociatedPressNewsBridge.php

@@ -0,0 +1,270 @@
+<?php
+class AssociatedPressNewsBridge extends BridgeAbstract {
+	const NAME = 'Associated Press News Bridge';
+	const URI = 'https://apnews.com/';
+	const DESCRIPTION = 'Returns newest articles by topic';
+	const MAINTAINER = 'VerifiedJoseph';
+	const PARAMETERS = array(
+		'Standard Topics' => array(
+			'topic' => array(
+				'name' => 'Topic',
+				'type' => 'list',
+				'values' => array(
+					'AP Top News' => 'apf-topnews',
+					'Sports' => 'apf-sports',
+					'Entertainment' => 'apf-entertainment',
+					'Oddities' => 'apf-oddities',
+					'Travel' => 'apf-Travel',
+					'Technology' => 'apf-technology',
+					'Lifestyle' => 'apf-lifestyle',
+					'Business' => 'apf-business',
+					'U.S. News' => 'apf-usnews',
+					'Health' => 'apf-Health',
+					'Science' => 'apf-science',
+					'World News' => 'apf-WorldNews',
+					'Politics' => 'apf-politics',
+					'Religion' => 'apf-religion',
+					'Photo Galleries' => 'PhotoGalleries',
+					'Fact Checks' => 'APFactCheck',
+					'Videos' => 'apf-videos',
+				),
+				'defaultValue' => 'apf-topnews',
+			),
+		),
+		'Custom Topic' => array(
+			'topic' => array(
+				'name' => 'Topic',
+				'type' => 'text',
+				'required' => true,
+				'exampleValue' => 'europe'
+			),
+		)
+	);
+
+	const CACHE_TIMEOUT = 900; // 15 mins
+
+	private $detectParamRegex = '/^https?:\/\/(?:www\.)?apnews\.com\/(?:[tag|hub]+\/)?([\w-]+)$/';
+	private $tagEndpoint = 'https://afs-prod.appspot.com/api/v2/feed/tag?tags=';
+	private $feedName = '';
+
+	public function detectParameters($url) {
+		$params = array();
+
+		if(preg_match($this->detectParamRegex, $url, $matches) > 0) {
+			$params['topic'] = $matches[1];
+			$params['context'] = 'Custom Topic';
+			return $params;
+		}
+
+		return null;
+	}
+
+	public function collectData() {
+		switch($this->getInput('topic')) {
+			case 'Podcasts':
+				returnClientError('Podcasts topic feed is not supported');
+				break;
+			case 'PressReleases':
+				returnClientError('PressReleases topic feed is not supported');
+				break;
+			default:
+				$this->collectCardData();
+		}
+	}
+
+	public function getURI() {
+		if (!is_null($this->getInput('topic'))) {
+			return self::URI . $this->getInput('topic');
+		}
+
+		return parent::getURI();
+	}
+
+	public function getName() {
+		if (!empty($this->feedName)) {
+			return $this->feedName . ' - Associated Press';
+		}
+
+		return parent::getName();
+	}
+
+	private function getTagURI() {
+		if (!is_null($this->getInput('topic'))) {
+			return $this->tagEndpoint . $this->getInput('topic');
+		}
+
+		return parent::getURI();
+	}
+
+	private function collectCardData() {
+		$json = getContents($this->getTagURI())
+			or returnServerError('Could not request: ' . $this->getTagURI());
+
+		$tagContents = json_decode($json, true);
+
+		if (empty($tagContents['tagObjs'])) {
+			returnClientError('Topic not found: ' . $this->getInput('topic'));
+		}
+
+		$this->feedName = $tagContents['tagObjs'][0]['name'];
+
+		foreach ($tagContents['cards'] as $card) {
+			$item = array();
+
+			 // skip hub peeks & Notifications
+			if ($card['cardType'] == 'Hub Peek' || $card['cardType'] == 'Notification') {
+				continue;
+			}
+
+			$storyContent = $card['contents'][0];
+
+			switch($storyContent['contentType']) {
+				case 'web': // Skip link only content
+					continue 2;
+
+				case 'video':
+					$html = $this->processVideo($storyContent);
+
+					$item['enclosures'][] = 'https://storage.googleapis.com/afs-prod/media/'
+						. $storyContent['media'][0]['id'] . '/800.jpeg';
+					break;
+				default:
+					if (empty($storyContent['storyHTML'])) { // Skip if no storyHTML
+						continue 2;
+					}
+
+					$html = defaultLinkTo($storyContent['storyHTML'], self::URI);
+					$html = str_get_html($html);
+
+					$this->processMediaPlaceholders($html, $storyContent['id']);
+					$this->processHubLinks($html, $storyContent);
+					$this->processIframes($html);
+
+					if (!is_null($storyContent['leadPhotoId'])) {
+						$item['enclosures'][] = 'https://storage.googleapis.com/afs-prod/media/'
+							. $storyContent['leadPhotoId'] . '/800.jpeg';
+					}
+			}
+
+			$item['title'] = $card['contents'][0]['headline'];
+			$item['uri'] = self::URI . $card['shortId'];
+
+			if ($card['contents'][0]['localLinkUrl']) {
+				$item['uri'] = $card['contents'][0]['localLinkUrl'];
+			}
+
+			$item['timestamp'] = $storyContent['published'];
+
+			if (is_null($storyContent['bylines']) === false) {
+				// Remove 'By' from the bylines
+				if (substr($storyContent['bylines'], 0, 2) == 'By') {
+					$item['author'] = ltrim($storyContent['bylines'], 'By ');
+				} else {
+					$item['author'] = $storyContent['bylines'];
+				}
+			}
+
+			$item['content'] = $html;
+
+			foreach ($storyContent['tagObjs'] as $tag) {
+				$item['categories'][] = $tag['name'];
+			}
+
+			$this->items[] = $item;
+
+			if (count($this->items) >= 15) {
+				break;
+			}
+		}
+	}
+
+	private function processMediaPlaceholders($html, $id) {
+
+		if ($html->find('div.media-placeholder', 0)) {
+			// Fetch page content
+			$json = getContents('https://afs-prod.appspot.com/api/v2/content/' . $id);
+			$storyContent = json_decode($json, true);
+
+			foreach ($html->find('div.media-placeholder') as $div) {
+				$key = array_search($div->id, $storyContent['mediumIds']);
+
+				if (!isset($storyContent['media'][$key])) {
+					continue;
+				}
+
+				$media = $storyContent['media'][$key];
+
+				if ($media['type'] === 'Photo') {
+					$mediaUrl = $media['gcsBaseUrl'] . $media['imageRenderedSizes'][0] . $media['imageFileExtension'];
+					$mediaCaption = $media['caption'];
+
+					$div->outertext = <<<EOD
+	<figure><img loading="lazy" src="{$mediaUrl}"/><figcaption>{$mediaCaption}</figcaption></figure>
+EOD;
+				}
+
+				if ($media['type'] === 'YouTube') {
+					$div->outertext = <<<EOD
+	<iframe src="https://www.youtube.com/embed/{$media['externalId']}" width="560" height="315">
+	</iframe>
+EOD;
+				}
+			}
+		}
+	}
+
+	/*
+		Create full coverage links (HubLinks)
+	*/
+	private function processHubLinks($html, $storyContent) {
+
+		if (!empty($storyContent['richEmbeds'])) {
+			foreach ($storyContent['richEmbeds'] as $embed) {
+
+				if ($embed['type'] === 'Hub Link') {
+					$url = self::URI . $embed['tag']['id'];
+					$div = $html->find('div[id=' . $embed['id'] . ']', 0);
+
+					if ($div) {
+						$div->outertext = <<<EOD
+<p><a href="{$url}">{$embed['calloutText']} {$embed['displayName']}</a></p>
+EOD;
+					}
+				}
+			}
+		}
+	}
+
+	private function processVideo($storyContent) {
+		$video = $storyContent['media'][0];
+
+		if ($video['type'] === 'YouTube') {
+			$url = 'https://www.youtube.com/embed/' . $video['externalId'];
+			$html = <<<EOD
+<iframe width="560" height="315" src="{$url}" frameborder="0" allowfullscreen></iframe>
+EOD;
+		} else {
+			$html = <<<EOD
+<video controls poster="https://storage.googleapis.com/afs-prod/media/{$video['id']}/800.jpeg" preload="none">
+	<source src="{$video['gcsBaseUrl']} {$video['videoRenderedSizes'][0]} {$video['videoFileExtension']}" type="video/mp4">
+</video>
+EOD;
+		}
+
+		return $html;
+	}
+
+	// Remove datawrapper.dwcdn.net iframes and related javaScript
+	private function processIframes($html) {
+
+		foreach ($html->find('iframe') as $index => $iframe) {
+			if (preg_match('/datawrapper\.dwcdn\.net/', $iframe->src)) {
+				$iframe->outertext = '';
+
+				if ($html->find('script', $index)) {
+					$html->find('script', $index)->outertext = '';
+				}
+			}
+		}
+	}
+}

bridges/BandcampDailyBridge.php

@@ -7,7 +7,7 @@ class BandcampDailyBridge extends BridgeAbstract {
 	const PARAMETERS = array(
 		'Latest articles' => array(),
 		'Best of' => array(
-			'content' => array(
+			'best-content' => array(
 				'name' => 'content',
 				'type' => 'list',
 				'values' => array(
@@ -28,7 +28,7 @@ class BandcampDailyBridge extends BridgeAbstract {
 			),
 		),
 		'Genres' => array(
-			'content' => array(
+			'genres-content' => array(
 				'name' => 'content',
 				'type' => 'list',
 				'values' => array(
@@ -62,7 +62,7 @@ class BandcampDailyBridge extends BridgeAbstract {
 			),
 		),
 		'Franchises' => array(
-			'content' => array(
+			'franchises-content' => array(
 				'name' => 'content',
 				'type' => 'list',
 				'values' => array(
@@ -133,23 +133,28 @@ class BandcampDailyBridge extends BridgeAbstract {
 			case 'Best of':
 			case 'Genres':
 			case 'Franchises':
-				return self::URI . '/' . $this->getInput('content');
+				// TODO Switch to array_key_first once php >= 7.3
+				$contentKey = key(self::PARAMETERS[$this->queriedContext]);
+				return self::URI . '/' . $this->getInput($contentKey);
 			default:
 				return parent::getURI();
 		}
 	}
 
 	public function getName() {
-		if ($this->queriedContext === 'Latest articles') {
-			return $this->queriedContext . ' - Bandcamp Daily';
+		switch($this->queriedContext) {
+			case 'Latest articles':
+				return $this->queriedContext . ' - Bandcamp Daily';
+			case 'Best of':
+			case 'Genres':
+			case 'Franchises':
+				// TODO Switch to array_key_first once php >= 7.3
+				$contentKey = key(self::PARAMETERS[$this->queriedContext]);
+				$contentValues = array_flip(self::PARAMETERS[$this->queriedContext][$contentKey]['values']);
+
+				return $contentValues[$this->getInput($contentKey)] . ' - Bandcamp Daily';
+			default:
+				return parent::getName();
 		}
-
-		if (!is_null($this->getInput('content'))) {
-			$contentValues = array_flip(self::PARAMETERS[$this->queriedContext]['content']['values']);
-
-			return $contentValues[$this->getInput('content')] . ' - Bandcamp Daily';
-		}
-
-		return parent::getName();
 	}
 }

bridges/BinanceBridge.php

@@ -1,41 +1,18 @@
 <?php
 class BinanceBridge extends BridgeAbstract {
-	const NAME = 'Binance';
-	const URI = 'https://www.binance.com';
-	const DESCRIPTION = 'Subscribe to the Binance blog or the Binance Zendesk announcements.';
+	const NAME = 'Binance Blog';
+	const URI = 'https://www.binance.com/en/blog';
+	const DESCRIPTION = 'Subscribe to the Binance blog.';
 	const MAINTAINER = 'thefranke';
 	const CACHE_TIMEOUT = 3600; // 1h
 
-	const PARAMETERS = array( array(
-		'category' => array(
-			'name' => 'category',
-			'type' => 'list',
-			'exampleValue' => 'Blog',
-			'title' => 'Select a category',
-			'values' => array(
-				'Blog' => 'Blog',
-				'Announcements' => 'Announcements'
-			)
-		)
-	));
-
 	public function getIcon() {
 		return 'https://bin.bnbstatic.com/static/images/common/favicon.ico';
 	}
 
-	public function getName() {
-		return self::NAME . ' ' . $this->getInput('category');
-	}
-
-	public function getURI() {
-		if ($this->getInput('category') == 'Blog')
-			return self::URI . '/en/blog';
-		else
-			return 'https://binance.zendesk.com/hc/en-us/categories/115000056351-Announcements';
-	}
-
-	protected function collectBlogData() {
-		$html = getSimpleHTMLDOM($this->getURI());
+	public function collectData() {
+		$html = getSimpleHTMLDOM(self::URI)
+			or returnServerError('Could not fetch Binance blog data.');
 
 		$appData = $html->find('script[id="__APP_DATA"]');
 		$appDataJson = json_decode($appData[0]->innertext);
@@ -61,37 +38,4 @@ class BinanceBridge extends BridgeAbstract {
 				break;
 		}
 	}
-
-	protected function collectAnnouncementData() {
-		$html = getSimpleHTMLDOM($this->getURI());
-
-		foreach($html->find('a.article-list-link') as $a) {
-			$title = $a->innertext;
-			$uri = 'https://binance.zendesk.com' . $a->href;
-
-			$full = getSimpleHTMLDOMCached($uri);
-			$content = $full->find('div.article-body', 0);
-			$date = $full->find('time', 0)->getAttribute('datetime');
-
-			$item = array();
-
-			$item['title'] = $title;
-			$item['uri'] = $uri;
-			$item['timestamp'] = strtotime($date);
-			$item['author'] = 'Binance';
-			$item['content'] = $content;
-
-			$this->items[] = $item;
-
-			if (count($this->items) >= 10)
-				break;
-		}
-	}
-
-	public function collectData() {
-		if ($this->getInput('category') == 'Blog')
-			$this->collectBlogData();
-		else
-			$this->collectAnnouncementData();
-	}
 }

bridges/BooruprojectBridge.php

@@ -1,7 +1,6 @@
 <?php
-require_once('GelbooruBridge.php');
 
-class BooruprojectBridge extends GelbooruBridge {
+class BooruprojectBridge extends DanbooruBridge {
 
 	const MAINTAINER = 'mitsukarenai';
 	const NAME = 'Booruproject';
@@ -11,6 +10,7 @@ class BooruprojectBridge extends GelbooruBridge {
 		'global' => array(
 			'p' => array(
 				'name' => 'page',
+				'defaultValue' => 0,
 				'type' => 'number'
 			),
 			't' => array(
@@ -29,8 +29,30 @@ class BooruprojectBridge extends GelbooruBridge {
 		)
 	);
 
+	const PATHTODATA = '.thumb';
+	const IDATTRIBUTE = 'id';
+	const TAGATTRIBUTE = 'title';
 	const PIDBYPAGE = 20;
 
+	protected function getFullURI(){
+		return $this->getURI()
+		. 'index.php?page=post&s=list&pid='
+		. ($this->getInput('p') ? ($this->getInput('p') - 1) * static::PIDBYPAGE : '')
+		. '&tags=' . urlencode($this->getInput('t'));
+	}
+
+	protected function getTags($element){
+		$tags = parent::getTags($element);
+		$tags = explode(' ', $tags);
+
+		// Remove statistics from the tags list (identified by colon)
+		foreach($tags as $key => $tag) {
+			if(strpos($tag, ':') !== false) unset($tags[$key]);
+		}
+
+		return implode(' ', $tags);
+	}
+
 	public function getURI(){
 		if(!is_null($this->getInput('i'))) {
 			return 'https://' . $this->getInput('i') . '.booru.org/';

bridges/BrutBridge.php

@@ -2,7 +2,7 @@
 class BrutBridge extends BridgeAbstract {
 	const NAME = 'Brut Bridge';
 	const URI = 'https://www.brut.media';
-	const DESCRIPTION = 'Returns 5 newest videos by category and edition';
+	const DESCRIPTION = 'Returns 10 newest videos by category and edition';
 	const MAINTAINER = 'VerifiedJoseph';
 	const PARAMETERS = array(array(
 			'category' => array(
@@ -38,9 +38,7 @@ class BrutBridge extends BridgeAbstract {
 
 	const CACHE_TIMEOUT = 1800; // 30 mins
 
-	private $videoId = '';
-	private $videoType = '';
-	private $videoImage = '';
+	private $jsonRegex = '/window\.__PRELOADED_STATE__ = ((?:.*)});/';
 
 	public function collectData() {
 
@@ -48,36 +46,38 @@ class BrutBridge extends BridgeAbstract {
 
 		$results = $html->find('div.results', 0);
 
-		foreach($results->find('li.col-6.col-sm-4.col-md-3.col-lg-2.px-2.pb-4') as $index => $li) {
+		foreach($results->find('li.col-6.col-sm-4.col-md-3.col-lg-2.px-2.pb-4') as $li) {
 			$item = array();
 
 			$videoPath = self::URI . $li->children(0)->href;
-
 			$videoPageHtml = getSimpleHTMLDOMCached($videoPath, 3600);
 
-			$this->videoImage = $videoPageHtml->find('meta[name="twitter:image"]', 0)->content;
-
-			$this->processTwitterImage();
-
-			$description = $videoPageHtml->find('div.description', 0);
+			$json = $this->extractJson($videoPageHtml);
+			$id = array_keys((array) $json->media->index)[0];
 
 			$item['uri'] = $videoPath;
-			$item['title'] = $description->find('h1', 0)->plaintext;
+			$item['title'] = $json->media->index->$id->title;
+			$item['timestamp'] = $json->media->index->$id->published_at;
+			$item['enclosures'][] = $json->media->index->$id->media->thumbnail;
 
-			if ($description->find('div.date', 0)->children(0)) {
-				$description->find('div.date', 0)->children(0)->outertext = '';
+			$description = $json->media->index->$id->description;
+			$article = '';
+
+			if (is_null($json->media->index->$id->media->seo_article) === false) {
+				$article = markdownToHtml($json->media->index->$id->media->seo_article);
 			}
 
-			$item['content'] = $this->processContent(
-				$description
-			);
-
-			$item['timestamp'] = $this->processDate($description);
-			$item['enclosures'][] = $this->videoImage;
+			$item['content'] = <<<EOD
+			<video controls poster="{$json->media->index->$id->media->thumbnail}" preload="none">
+				<source src="{$json->media->index->$id->media->mp4_url}" type="video/mp4">
+			</video>
+			<p>{$description}</p>
+			{$article}
+EOD;
 
 			$this->items[] = $item;
 
-			if (count($this->items) >= 5) {
+			if (count($this->items) >= 10) {
 				break;
 			}
 		}
@@ -107,51 +107,21 @@ class BrutBridge extends BridgeAbstract {
 		return parent::getName();
 	}
 
-	private function processDate($description) {
+	/**
+	 * Extract JSON from page
+	 */
+   private function extractJson($html) {
 
-		if ($this->getInput('edition') === 'uk') {
-			$date = DateTime::createFromFormat('d/m/Y H:i', $description->find('div.date', 0)->innertext);
-			return strtotime($date->format('Y-m-d H:i:s'));
+		if (!preg_match($this->jsonRegex, $html, $parts)) {
+			returnServerError('Failed to extract data from page');
 		}
 
-		return strtotime($description->find('div.date', 0)->innertext);
-	}
+		$data = json_decode($parts[1]);
 
-	private function processContent($description) {
-
-		$content = '<video controls poster="' . $this->videoImage . '" preload="none">
-			<source src="https://content.brut.media/video/' . $this->videoId . '-' . $this->videoType . '-web.mp4"
-            type="video/mp4">
-			</video>';
-		$content .= '<p>' . $description->find('h2.mb-1', 0)->innertext . '</p>';
-
-		if ($description->find('div.text.pb-3', 0)->children(1)->class != 'date') {
-			$content .= '<p>' . $description->find('div.text.pb-3', 0)->children(1)->innertext . '</p>';
+		if ($data === false) {
+			returnServerError('Failed to decode extracted data');
 		}
 
-		return $content;
-	}
-
-	private function processTwitterImage() {
-		/**
-		 * Extract video ID + type from twitter image
-		 *
-		 * Example (wrapped):
-		 *  https://img.brut.media/thumbnail/
-		 *  the-life-of-rita-moreno-2cce75b5-d448-44d2-a97c-ca50d6470dd4-square.jpg
-		 *  ?ts=1559337892
-		 */
-		$fpath = parse_url($this->videoImage, PHP_URL_PATH);
-		$fname = basename($fpath);
-		$fname = substr($fname, 0, strrpos($fname, '.'));
-		$parts = explode('-', $fname);
-
-		if (end($parts) === 'auto') {
-			$key = array_search('auto', $parts);
-			unset($parts[$key]);
-		}
-
-		$this->videoId = implode('-', array_splice($parts, -6, 5));
-		$this->videoType = end($parts);
+		return $data;
 	}
 }

bridges/CBCEditorsBlogBridge.php

@@ -0,0 +1,36 @@
+<?php
+class CBCEditorsBlogBridge extends BridgeAbstract {
+
+	const MAINTAINER = 'quickwick';
+	const NAME = 'CBC Editors Blog';
+	const URI = 'https://www.cbc.ca/news/editorsblog';
+	const DESCRIPTION = 'Recent CBC Editor\'s Blog posts';
+
+	public function collectData(){
+		$html = getSimpleHTMLDOM(self::URI);
+
+		// Loop on each blog post entry
+		foreach($html->find('div.contentListCards', 0)->find('a[data-test=type-story]') as $element) {
+			$headline = ($element->find('.headline', 0))->innertext;
+			$timestamp = ($element->find('time', 0))->datetime;
+			$articleUri = 'https://www.cbc.ca' . $element->href;
+			$summary = ($element->find('div.description', 0))->innertext;
+			$thumbnailUris = ($element->find('img[loading=lazy]', 0))->srcset;
+			$thumbnailUri = rtrim(explode(',', $thumbnailUris)[0], ' 300w');
+
+			// Fill item
+			$item = array();
+			$item['uri'] = $articleUri;
+			$item['id'] = $item['uri'];
+			$item['timestamp'] = $timestamp;
+			$item['title'] = $headline;
+			$item['content'] = '<img src="'
+			. $thumbnailUri . '" /><br>' . $summary;
+			$item['author'] = 'Editor\'s Blog';
+
+			if(isset($item['title'])) {
+				$this->items[] = $item;
+			}
+		}
+	}
+}

bridges/CastorusBridge.php

@@ -35,7 +35,7 @@ class CastorusBridge extends BridgeAbstract {
 		if(!$title)
 			returnServerError('Cannot find title!');
 
-		return htmlspecialchars(trim($title->plaintext));
+		return trim($title->plaintext);
 	}
 
 	// Extracts the url from an actitiy

bridges/CourrierInternationalBridge.php

@@ -15,11 +15,10 @@ class CourrierInternationalBridge extends FeedExpander {
 		$item = parent::parseItem($feedItem);
 
 		$articlePage = getSimpleHTMLDOMCached($feedItem->link);
-		$content = $articlePage->find('.article-text', 0);
-		if(!$content) {
-			$content = $articlePage->find('.depeche-text', 0);
+		$content = $articlePage->find('.article-text, depeche-text', 0);
+		if (!$content) {
+			return $item;
 		}
-
 		$item['content'] = sanitize($content);
 
 		return $item;

bridges/CraigslistBridge.php

@@ -74,6 +74,7 @@ class CraigslistBridge extends BridgeAbstract {
 		foreach($results as $post) {
 
 			// Skip "nearby results" banner and results
+			// This only appears when searchNearby is not specified
 			if ($post->tag == 'h4') {
 				break;
 			}
@@ -86,7 +87,8 @@ class CraigslistBridge extends BridgeAbstract {
 			$item['timestamp'] = $post->find('.result-date', 0)->datetime;
 			$item['uid'] = $heading->id;
 			$item['content'] = $post->find('.result-price', 0)->plaintext . ' '
-							 . $post->find('.result-hood', 0)->plaintext;
+							 // Find the location (local and nearby results if searchNearby=1)
+							 . $post->find('.result-hood, span.nearby', 0)->plaintext;
 
 			$images = $post->find('.result-image[data-ids]', 0);
 			if (!is_null($images)) {

bridges/CryptomeBridge.php

@@ -4,7 +4,7 @@ class CryptomeBridge extends BridgeAbstract {
 	const MAINTAINER = 'BoboTiG';
 	const NAME = 'Cryptome';
 	const URI = 'https://cryptome.org/';
-	const CACHE_TIMEOUT = 21600; //6h
+	const CACHE_TIMEOUT = 21600; // 6h
 	const DESCRIPTION = 'Returns the N most recent documents.';
 	const PARAMETERS = array( array(
 		'n' => array(

bridges/CubariBridge.php

@@ -0,0 +1,98 @@
+<?php
+class CubariBridge extends BridgeAbstract
+{
+	const NAME = 'Cubari';
+	const URI = 'https://cubari.moe';
+	const DESCRIPTION = 'Parses given cubari-formatted JSON file for updates.';
+	const MAINTAINER = 'KamaleiZestri';
+	const PARAMETERS = array(array(
+		'gist' => array(
+			'name' => 'Gist/Raw Url',
+			'type' => 'text',
+			'required' => true,
+			'exampleValue' => 'https://raw.githubusercontent.com/kurisumx/baka/main/ikedan'
+		)
+	));
+
+	private $mangaTitle = '';
+
+	public function getName()
+	{
+		if (!empty($this->mangaTitle))
+			return $this->mangaTitle . ' - ' . self::NAME;
+		else
+			return self::NAME;
+	}
+
+	public function getURI()
+	{
+		if ($this->getInput('gist') != '')
+			return self::URI . '/read/gist/' . $this->getEncodedGist();
+		else
+			return self::URI;
+	}
+
+	/**
+	 * The Cubari bridge.
+	 *
+	 * Cubari urls are base64 encodes of a given github raw or gist link described as below:
+	 * https://cubari.moe/read/gist/${bаse64.url_encode(raw/<rest of the url...>)}/
+	 * https://cubari.moe/read/gist/${bаse64.url_encode(gist/<rest of the url...>)}/
+	 * https://cubari.moe/read/gist/${gitio shortcode}
+	 *
+	 * This bridge uses just the raw/gist and generates matching cubari urls.
+	 */
+	public function collectData()
+	{
+		$jsonSite = getContents($this->getInput('gist'));
+		$jsonFile = json_decode($jsonSite, true);
+
+		$this->mangaTitle = $jsonFile['title'];
+
+		$chapters = $jsonFile['chapters'];
+
+		foreach ($chapters as $chapnum => $chapter) {
+			$item = $this->getItemFromChapter($chapnum, $chapter);
+			$this->items[] = $item;
+		}
+
+		array_multisort(array_column($this->items, 'timestamp'), SORT_DESC, $this->items);
+	}
+
+	protected function getEncodedGist()
+	{
+		$url = $this->getInput('gist');
+
+		preg_match('/\/([a-z]*)\.githubusercontent.com(.*)/', $url, $matches);
+
+		// raw or gist is first match.
+		$unencoded = $matches[1] . $matches[2];
+
+		return base64_encode($unencoded);
+	}
+
+	private function getSanitizedHash($string)
+	{
+		return hash('sha1', preg_replace('/[^a-zA-Z0-9\-\.]/', '', ucwords(strtolower($string))));
+	}
+
+	protected function getItemFromChapter($chapnum, $chapter)
+	{
+		$item = array();
+
+		$item['uri'] = $this->getURI() . '/' . $chapnum;
+		$item['title'] = 'Chapter ' . $chapnum . ' - ' . $chapter['title'] . ' - ' . $this->mangaTitle;
+		foreach ($chapter['groups'] as $key => $value)
+			$item['author'] = $key;
+		$item['timestamp'] = $chapter['last_updated'];
+
+		$item['content'] = '<p>Manga: <a href=' . $this->getURI() . '>' . $this->mangaTitle . '</a> </p>
+			<p>Chapter Number: ' . $chapnum . '</p>
+			<p>Chapter Title: <a href=' . $item['uri'] . '>' . $chapter['title'] . '</a></p>
+			<p>Group: ' . $item['author'] . '</p>';
+
+		$item['uid'] = $this->getSanitizedHash($item['title'] . $item['author']);
+
+		return $item;
+	}
+}

bridges/CyanideAndHappinessBridge.php

@@ -0,0 +1,37 @@
+<?php
+class CyanideAndHappinessBridge extends BridgeAbstract {
+	const NAME = 'Cyanide & Happiness';
+	const URI = 'https://explosm.net/';
+	const DESCRIPTION = 'The Webcomic from Explosm.';
+	const MAINTAINER = 'sal0max';
+	const CACHE_TIMEOUT = 60 * 60 * 2; // 2 hours
+
+	public function getIcon() {
+		return self::URI . 'favicon-32x32.png';
+	}
+
+	public function getURI(){
+		return self::URI . 'comics/latest#comic';
+	}
+
+	public function collectData() {
+		$html = getSimpleHTMLDOM($this->getUri());
+
+		foreach ($html->find('[class*=ComicImage]') as $element) {
+			$date        = $element->find('[class^=Author__Right] p', 0)->plaintext;
+			$author      = str_replace('by ', '', $element->find('[class^=Author__Right] p', 1)->plaintext);
+			$image       = $element->find('img', 0)->src;
+			$link        = $html->find('[rel=canonical]', 0)->href;
+
+			$item = array(
+				'uid'       => $link,
+				'author'    => $author,
+				'title'     => $date,
+				'uri'       => $link . '#comic',
+				'timestamp' => str_replace('.', '-', $date) . 'T00:00:00Z',
+				'content'   => "<img src=\"$image\" />"
+			);
+			$this->items[] = $item;
+		}
+	}
+}

bridges/DanbooruBridge.php

@@ -15,7 +15,9 @@ class DanbooruBridge extends BridgeAbstract {
 				'type' => 'number'
 			),
 			't' => array(
-				'name' => 'tags'
+				'type' => 'text',
+				'name' => 'tags',
+				'exampleValue' => 'cosplay',
 			)
 		),
 		0 => array()
@@ -57,79 +59,10 @@ class DanbooruBridge extends BridgeAbstract {
 	}
 
 	public function collectData(){
-		$content = getContents($this->getFullURI());
-
-		$html = Fix_Simple_Html_Dom::str_get_html($content);
+		$html = getSimpleHTMLDOMCached($this->getFullURI());
 
 		foreach($html->find(static::PATHTODATA) as $element) {
 			$this->items[] = $this->getItemFromElement($element);
 		}
 	}
 }
-
-/**
- * This class is a monkey patch to 'extend' simplehtmldom to recognize <source>
- * tags (HTML5) as self closing tag. This patch should be removed once
- * simplehtmldom was fixed. This seems to be a issue with more tags:
- * https://sourceforge.net/p/simplehtmldom/bugs/83/
- *
- * The tag itself is valid according to Mozilla:
- *
- * The HTML <picture> element serves as a container for zero or more <source>
- * elements and one <img> element to provide versions of an image for different
- * display device scenarios. The browser will consider each of the child <source>
- * elements and select one corresponding to the best match found; if no matches
- * are found among the <source> elements, the file specified by the <img>
- * element's src attribute is selected. The selected image is then presented in
- * the space occupied by the <img> element.
- *
- * -- https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture
- *
- * Notice: This class uses parts of the original simplehtmldom, adjusted to pass
- * the guidelines of RSS-Bridge (formatting)
- */
-final class Fix_Simple_Html_Dom extends simple_html_dom {
-
-	/* copy from simple_html_dom, added 'source' at the end */
-	protected $self_closing_tags = array(
-		'img' => 1,
-		'br' => 1,
-		'input' => 1,
-		'meta' => 1,
-		'link' => 1,
-		'hr' => 1,
-		'base' => 1,
-		'embed' => 1,
-		'spacer' => 1,
-		'source' => 1
-	);
-
-	/* copy from simplehtmldom, changed 'simple_html_dom' to 'Fix_Simple_Html_Dom' */
-	public static function str_get_html($str,
-	$lowercase = true,
-	$forceTagsClosed = true,
-	$target_charset = DEFAULT_TARGET_CHARSET,
-	$stripRN = true,
-	$defaultBRText = DEFAULT_BR_TEXT,
-	$defaultSpanText = DEFAULT_SPAN_TEXT)
-	{
-		$dom = new Fix_Simple_Html_Dom(null,
-		$lowercase,
-		$forceTagsClosed,
-		$target_charset,
-		$stripRN,
-		$defaultBRText,
-		$defaultSpanText);
-
-		if (empty($str) || strlen($str) > MAX_FILE_SIZE) {
-
-			$dom->clear();
-			return false;
-
-		}
-
-		$dom->load($str, $lowercase, $stripRN);
-
-		return $dom;
-	}
-}

bridges/DarkReadingBridge.php

@@ -33,7 +33,8 @@ class DarkReadingBridge extends FeedExpander {
 				'Insider Threats' => '663_Insider%20Threats',
 				'Vulnerability Management' => '664_Vulnerability%20Management',
 			)
-		)
+		),
+		'limit' => self::LIMIT,
 	));
 
 	public function collectData(){
@@ -48,7 +49,8 @@ class DarkReadingBridge extends FeedExpander {
 		if ($feed_id != '000') {
 			$feed_url .= '?f_n=' . $feed_id . '&f_ln=' . $feed_name;
 		}
-		$this->collectExpandableDatas($feed_url, 20);
+		$limit = $this->getInput('limit') ?? 10;
+		$this->collectExpandableDatas($feed_url, $limit);
 	}
 
 	protected function parseItem($newsItem){

bridges/DaveRamseyBlogBridge.php

@@ -1,23 +0,0 @@
-<?php
-
-class DaveRamseyBlogBridge extends BridgeAbstract {
-	const MAINTAINER = 'johnpc';
-	const NAME = 'Dave Ramsey Blog';
-	const URI = 'https://www.daveramsey.com/blog';
-	const CACHE_TIMEOUT = 7200; // 2h
-	const DESCRIPTION = 'Returns blog posts from daveramsey.com';
-
-	public function collectData()
-	{
-		$html = getSimpleHTMLDOM(self::URI);
-
-		foreach ($html->find('.Post') as $element) {
-			$this->items[] = array(
-				'uri' => 'https://www.daveramsey.com' . $element->find('header > a', 0)->href,
-				'title' => $element->find('header > h2 > a', 0)->plaintext,
-				'tags' => $element->find('.Post-topic', 0)->plaintext,
-				'content' => $element->find('.Post-body', 0)->plaintext,
-			);
-		}
-	}
-}

bridges/DealabsBridge.php

@@ -10,7 +10,7 @@ class DealabsBridge extends PepperBridgeAbstract {
 			'q' => array(
 				'name' => 'Mot(s) clé(s)',
 				'type' => 'text',
-				'exampleValue' => 'lamp',
+				'exampleValue' => 'lampe',
 				'required' => true
 			),
 			'hide_expired' => array(
@@ -1886,7 +1886,7 @@ class DealabsBridge extends PepperBridgeAbstract {
 				'type' => 'text',
 				'required' => true,
 				'title' => 'URL discussion à surveiller: https://www.dealabs.com/discussions/titre-1234',
-				'exampleValue' => 'https://www.dealabs.com/discussions/titre-1234',
+				'exampleValue' => 'https://www.dealabs.com/discussions/jeux-steam-gratuits-gleam-woobox-etc-1071415',
 				),
 
 			'only_with_url' => array(
@@ -1963,682 +1963,3 @@ class DealabsBridge extends PepperBridgeAbstract {
 
 
 }
-
-class PepperBridgeAbstract extends BridgeAbstract {
-
-	const CACHE_TIMEOUT = 3600;
-
-	public function collectData(){
-		switch($this->queriedContext) {
-		case $this->i8n('context-keyword'):
-			return $this->collectDataKeywords();
-			break;
-		case $this->i8n('context-group'):
-			return $this->collectDataGroup();
-			break;
-		case $this->i8n('context-talk'):
-			return $this->collectDataTalk();
-			break;
-		}
-	}
-
-	/**
-	 * Get the Deal data from the choosen group in the choosed order
-	 */
-	protected function collectDataGroup()
-	{
-		$url = $this->getGroupURI();
-		$this->collectDeals($url);
-	}
-
-	/**
-	 * Get the Deal data from the choosen keywords and parameters
-	 */
-	protected function collectDataKeywords()
-	{
-		/* Even if the original website uses POST with the search page, GET works too */
-		$url = $this->getSearchURI();
-		$this->collectDeals($url);
-	}
-
-	/**
-	 * Get the Deal data using the given URL
-	 */
-	protected function collectDeals($url){
-		$html = getSimpleHTMLDOM($url);
-		$list = $html->find('article[id]');
-
-		// Deal Image Link CSS Selector
-		$selectorImageLink = implode(
-			' ', /* Notice this is a space! */
-			array(
-				'cept-thread-image-link',
-				'imgFrame',
-				'imgFrame--noBorder',
-				'thread-listImgCell',
-			)
-		);
-
-		// Deal Link CSS Selector
-		$selectorLink = implode(
-			' ', /* Notice this is a space! */
-			array(
-				'cept-tt',
-				'thread-link',
-				'linkPlain',
-			)
-		);
-
-		// Deal Hotness CSS Selector
-		$selectorHot = implode(
-			' ', /* Notice this is a space! */
-			array(
-				'cept-vote-box',
-				'vote-box'
-			)
-		);
-
-		// Deal Description CSS Selector
-		$selectorDescription = implode(
-			' ', /* Notice this is a space! */
-			array(
-				'cept-description-container',
-				'overflow--wrap-break'
-			)
-		);
-
-		// Deal Date CSS Selector
-		$selectorDate = implode(
-			' ', /* Notice this is a space! */
-			array(
-				'size--all-s',
-				'flex',
-				'boxAlign-jc--all-fe'
-			)
-		);
-
-		// If there is no results, we don't parse the content because it display some random deals
-		$noresult = $html->find('h3[class=size--all-l size--fromW2-xl size--fromW3-xxl]', 0);
-		if ($noresult != null && strpos($noresult->plaintext, $this->i8n('no-results')) !== false) {
-			$this->items = array();
-		} else {
-			foreach ($list as $deal) {
-				$item = array();
-				$item['uri'] = $this->getDealURI($deal);
-				$item['title'] = $this->GetTitle($deal);
-				$item['author'] = $deal->find('span.thread-username', 0)->plaintext;
-
-				$item['content'] = '<table><tr><td><a href="'
-					. $item['uri']
-						. '"><img src="'
-						. $this->getImage($deal)
-						. '"/></td><td>'
-						. $this->getHTMLTitle($item)
-						. $this->getPrice($deal)
-						. $this->getDiscount($deal)
-						. $this->getShipsFrom($deal)
-						. $this->getShippingCost($deal)
-						. $this->GetSource($deal)
-						. $deal->find('div[class*=' . $selectorDescription . ']', 0)->innertext
-						. '</td><td>'
-						. $deal->find('div[class*=' . $selectorHot . ']', 0)
-						->find('span', 1)->outertext
-						. '</td></table>';
-				$dealDateDiv = $deal->find('div[class*=' . $selectorDate . ']', 0)
-					->find('span[class=hide--toW3]');
-				$itemDate = end($dealDateDiv)->plaintext;
-				// In case of a Local deal, there is no date, but we can use
-				// this case for other reason (like date not in the last field)
-				if ($this->contains($itemDate, $this->i8n('localdeal'))) {
-					$item['timestamp'] = time();
-				} else if ($this->contains($itemDate, $this->i8n('relative-date-indicator'))) {
-					$item['timestamp'] = $this->relativeDateToTimestamp($itemDate);
-				} else {
-					$item['timestamp'] = $this->parseDate($itemDate);
-				}
-				$this->items[] = $item;
-			}
-		}
-	}
-
-	/**
-	 * Get the Talk lastest comments
-	 */
-	protected function collectDataTalk(){
-		$threadURL = $this->getInput('url');
-		$onlyWithUrl = $this->getInput('only_with_url');
-
-		// Get Thread ID from url passed in parameter
-		$threadSearch = preg_match('/-([0-9]{1,20})$/', $threadURL, $matches);
-
-		// Show an error message if we can't find the thread ID in the URL sent by the user
-		if($threadSearch !== 1) {
-			returnClientError($this->i8n('thread-error'));
-		}
-		$threadID = $matches[1];
-
-		$url = $this->i8n('bridge-uri') . 'graphql';
-
-		// Get Cookies header to do the query
-		$cookies = $this->getCookies($url);
-
-		// GraphQL String
-		// This was extracted from https://www.dealabs.com/assets/js/modern/common_211b99.js
-		// This string was extracted during a Website visit, and minified using this neat tool :
-		// https://codepen.io/dangodev/pen/Baoqmoy
-		$graphqlString = <<<'HEREDOC'
-query comments($filter:CommentFilter!,$limit:Int,$page:Int){comments(filter:$filter,limit:$limit,page:$page){
-items{...commentFields}pagination{...paginationFields}}}fragment commentFields on Comment{commentId threadId url 
-preparedHtmlContent user{...userMediumAvatarFields...userNameFields...userPersonaFields bestBadge{...badgeFields}}
-reactionCounts{type count}deletable currentUserReaction{type}reported reportable source status createdAt updatedAt 
-ignored popular deletedBy{username}notes{content createdAt user{username}}lastEdit{reason timeAgo userId}}fragment 
-userMediumAvatarFields on User{userId isDeletedOrPendingDeletion imageUrls(slot:"default",variations:
-["user_small_avatar"])}fragment userNameFields on User{userId username isUserProfileHidden isDeletedOrPendingDeletion}
-fragment userPersonaFields on User{persona{type text}}fragment badgeFields on Badge{badgeId level{...badgeLevelFields}}
-fragment badgeLevelFields on BadgeLevel{key name description}fragment paginationFields on Pagination{count current last
- next previous size order}
-HEREDOC;
-
-		// Construct the JSON object to send to the Website
-		$queryArray = array (
-			'query' => $graphqlString,
-			'variables' => array (
-				'filter' => array (
-					'threadId' => array (
-						'eq' => $threadID,
-						),
-					'order' => array (
-						'direction' => 'Descending',
-						),
-
-					),
-				'page' => 1,
-				),
-			);
-		$queryJSON = json_encode($queryArray);
-
-		// HTTP headers
-		$header = array(
-				'Content-Type: application/json',
-				'Accept: application/json, text/plain, */*',
-				'X-Pepper-Txn: threads.show',
-				'X-Request-Type: application/vnd.pepper.v1+json',
-				'X-Requested-With: XMLHttpRequest',
-				$cookies,
-			);
-		// CURL Options
-		$opts = array(
-				CURLOPT_POST => 1,
-				CURLOPT_POSTFIELDS => $queryJSON
-				);
-		$json = getContents($url, $header, $opts);
-		$objects = json_decode($json);
-		foreach($objects->data->comments->items as $comment) {
-			$item = array();
-			$item['uri'] = $comment->url;
-			$item['title'] = $comment->user->username . ' - ' . $comment->createdAt;
-			$item['author'] = $comment->user->username;
-			$item['content'] = $comment->preparedHtmlContent;
-			$item['uid'] = $comment->commentId;
-			// Timestamp handling needs a new parsing function
-			if($onlyWithUrl == true) {
-				// Count Links and Quote Links
-				$content = str_get_html($item['content']);
-				$countLinks = count($content->find('a[href]'));
-				$countQuoteLinks = count($content->find('a[href][class=userHtml-quote-source]'));
-				// Only add element if there are Links ans more links tant Quote links
-				if($countLinks > 0 && $countLinks > $countQuoteLinks) {
-					$this->items[] = $item;
-				}
-			} else {
-				$this->items[] = $item;
-			}
-		}
-	}
-
-	/**
-	 * Extract the cookies obtained from the URL
-	 * @return array the array containing the cookies set by the URL
-	 */
-	private function getCookies($url)
-	{
-		$ch = curl_init($url);
-		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
-		// get headers too with this line
-		curl_setopt($ch, CURLOPT_HEADER, 1);
-		$result = curl_exec($ch);
-		// get cookie
-		// multi-cookie variant contributed by @Combuster in comments
-		preg_match_all('/^Set-Cookie:\s*([^;]*)/mi', $result, $matches);
-		$cookies = array();
-		foreach($matches[1] as $item) {
-			parse_str($item, $cookie);
-			$cookies = array_merge($cookies, $cookie);
-		}
-		$header = 'Cookie: ';
-		foreach($cookies as $name => $content) {
-			$header .= $name . '=' . $content . '; ';
-		}
-		return $header;
-	}
-
-	/**
-	 * Check if the string $str contains any of the string of the array $arr
-	 * @return boolean true if the string matched anything otherwise false
-	 */
-	private function contains($str, array $arr)
-	{
-		foreach ($arr as $a) {
-			if (stripos($str, $a) !== false) {
-				return true;
-			}
-		}
-		return false;
-	}
-
-	/**
-	 * Get the Price from a Deal if it exists
-	 * @return string String of the deal price
-	 */
-	private function getPrice($deal)
-	{
-		if ($deal->find(
-			'span[class*=thread-price]', 0) != null) {
-			return '<div>' . $this->i8n('price') . ' : '
-				. $deal->find(
-					'span[class*=thread-price]', 0
-				)->plaintext
-				. '</div>';
-		} else {
-			return '';
-		}
-	}
-
-	/**
-	 * Get the Title from a Deal if it exists
-	 * @return string String of the deal title
-	 */
-	private function getTitle($deal)
-	{
-
-		$titleRoot = $deal->find('div[class*=threadGrid-title]', 0);
-		$titleA = $titleRoot->find('a[class*=thread-link]', 0);
-		$titleFirstChild = $titleRoot->first_child();
-		if($titleA !== null) {
-			$title = $titleA->plaintext;
-		} else {
-		// Inb ssome case, expired deals have a different format
-			$title = $titleRoot->find('span', 0)->plaintext;
-		}
-
-		return $title;
-
-	}
-
-	/**
-	 * Get the Title from a Talk if it exists
-	 * @return string String of the Talk title
-	 */
-	private function getTalkTitle()
-	{
-		$html = getSimpleHTMLDOMCached($this->getInput('url'));
-		$title = $html->find('h1[class=thread-title]', 0)->plaintext;
-		return $title;
-
-	}
-
-	/**
-	 * Get the HTML Title code from an item
-	 * @return string String of the deal title
-	 */
-	private function getHTMLTitle($item)
-	{
-		if($item['uri'] == '') {
-			$html = '<h2>' . $item['title'] . '</h2>';
-		} else {
-			$html = '<h2><a href="' . $item['uri'] . '">'
-				. $item['title'] . '</a></h2>';
-		}
-
-		return $html;
-
-	}
-
-	/**
-	 * Get the URI from a Deal if it exists
-	 * @return string String of the deal URI
-	 */
-	private function getDealURI($deal)
-	{
-
-		$uriA = $deal->find('div[class*=threadGrid-title]', 0)->find('a[class*=thread-link]', 0);
-		if($uriA === null) {
-			$uri = '';
-		} else 	{
-			$uri = $uriA->href;
-		}
-
-		return $uri;
-
-	}
-
-	/**
-	 * Get the Shipping costs from a Deal if it exists
-	 * @return string String of the deal shipping Cost
-	 */
-	private function getShippingCost($deal)
-	{
-		if ($deal->find('span[class*=cept-shipping-price]', 0) != null) {
-			if ($deal->find('span[class*=cept-shipping-price]', 0)->children(0) != null) {
-				return '<div>' . $this->i8n('shipping') . ' : '
-					. $deal->find('span[class*=cept-shipping-price]', 0)->children(0)->innertext
-					. '</div>';
-			} else {
-				return '<div>' . $this->i8n('shipping') . ' : '
-					. $deal->find('span[class*=cept-shipping-price]', 0)->innertext
-					. '</div>';
-			}
-		} else {
-			return '';
-		}
-	}
-
-	/**
-	 * Get the source of a Deal if it exists
-	 * @return string String of the deal source
-	 */
-	private function GetSource($deal)
-	{
-		if ($deal->find('a[class=text--color-greyShade]', 0) != null) {
-			return '<div>' . $this->i8n('origin') . ' : '
-				. $deal->find('a[class=text--color-greyShade]', 0)->outertext
-				. '</div>';
-		} else {
-			return '';
-		}
-	}
-
-	/**
-	 * Get the original Price and discout from a Deal if it exists
-	 * @return string String of the deal original price and discount
-	 */
-	private function getDiscount($deal)
-	{
-		if ($deal->find('span[class*=mute--text text--lineThrough]', 0) != null) {
-			$discountHtml = $deal->find('span[class=space--ml-1 size--all-l size--fromW3-xl]', 0);
-			if ($discountHtml != null) {
-				$discount = $discountHtml->plaintext;
-			} else {
-				$discount = '';
-			}
-			return '<div>' . $this->i8n('discount') . ' : <span style="text-decoration: line-through;">'
-				. $deal->find(
-					'span[class*=mute--text text--lineThrough]', 0
-				)->plaintext
-				. '</span>&nbsp;'
-				. $discount
-				. '</div>';
-		} else {
-			return '';
-		}
-	}
-
-	/**
-	 * Get the Picture URL from a Deal if it exists
-	 * @return string String of the deal Picture URL
-	 */
-	private function getImage($deal)
-	{
-		$selectorLazy = implode(
-			' ', /* Notice this is a space! */
-			array(
-				'thread-image',
-				'width--all-auto',
-				'height--all-auto',
-				'imgFrame-img',
-				'cept-thread-img',
-				'img--dummy',
-				'js-lazy-img'
-			)
-		);
-
-		$selectorPlain = implode(
-			' ', /* Notice this is a space! */
-			array(
-				'thread-image',
-				'width--all-auto',
-				'height--all-auto',
-				'imgFrame-img',
-				'cept-thread-img'
-			)
-		);
-		if ($deal->find('img[class=' . $selectorLazy . ']', 0) != null) {
-			return json_decode(
-				html_entity_decode(
-					$deal->find('img[class=' . $selectorLazy . ']', 0)
-					->getAttribute('data-lazy-img')))->{'src'};
-		} else {
-			return $deal->find('img[class*=' . $selectorPlain . ']', 0	)->src;
-		}
-	}
-
-	/**
-	 * Get the originating country from a Deal if it exists
-	 * @return string String of the deal originating country
-	 */
-	private function getShipsFrom($deal)
-	{
-		$selector = implode(
-			' ', /* Notice this is a space! */
-			array(
-				'meta-ribbon',
-				'overflow--wrap-off',
-				'space--l-3',
-				'text--color-greyShade'
-			)
-		);
-		if ($deal->find('span[class=' . $selector . ']', 0) != null) {
-			return '<div>'
-				. $deal->find('span[class=' . $selector . ']', 0)->children(2)->plaintext
-				. '</div>';
-		} else {
-			return '';
-		}
-	}
-
-	/**
-	 * Transforms a local date into a timestamp
-	 * @return int timestamp of the input date
-	 */
-	private function parseDate($string)
-	{
-		$month_local = $this->i8n('local-months');
-		$month_en = array(
-			'January',
-			'February',
-			'March',
-			'April',
-			'May',
-			'June',
-			'July',
-			'August',
-			'September',
-			'October',
-			'November',
-			'December'
-		);
-
-		// A date can be prfixed with some words, we remove theme
-		$string = $this->removeDatePrefixes($string);
-		// We translate the local months name in the english one
-		$date_str = trim(str_replace($month_local, $month_en, $string));
-
-		// If the date does not contain any year, we add the current year
-		if (!preg_match('/[0-9]{4}/', $string)) {
-			$date_str .= ' ' . date('Y');
-		}
-
-		// Add the Hour and minutes
-		$date_str .= ' 00:00';
-		$date = DateTime::createFromFormat('j F Y H:i', $date_str);
-		// In some case, the date is not recognized : as a workaround the actual date is taken
-		if($date === false) {
-			$date = new DateTime();
-		}
-		return $date->getTimestamp();
-	}
-
-	/**
-	 * Remove the prefix of a date if it has one
-	 * @return the date without prefiux
-	 */
-	private function removeDatePrefixes($string)
-	{
-		$string = str_replace($this->i8n('date-prefixes'), array(), $string);
-		return $string;
-	}
-
-	/**
-	 * Remove the suffix of a relative date if it has one
-	 * @return the relative date without suffixes
-	 */
-	private function removeRelativeDateSuffixes($string)
-	{
-		if (count($this->i8n('relative-date-ignore-suffix')) > 0) {
-			$string = preg_replace($this->i8n('relative-date-ignore-suffix'), '', $string);
-		}
-		return $string;
-	}
-
-	/**
-	 * Transforms a relative local date into a timestamp
-	 * @return int timestamp of the input date
-	 */
-	private function relativeDateToTimestamp($str) {
-		$date = new DateTime();
-
-		// In case of update date, replace it by the regular relative date first word
-		$str = str_replace($this->i8n('relative-date-alt-prefixes'), $this->i8n('local-time-relative')[0], $str);
-
-		$str = $this->removeRelativeDateSuffixes($str);
-
-		$search = $this->i8n('local-time-relative');
-
-		$replace = array(
-			'-',
-			'minute',
-			'hour',
-			'day',
-			'month',
-			'year',
-			''
-		);
-
-		$date->modify(str_replace($search, $replace, $str));
-		return $date->getTimestamp();
-	}
-
-	/**
-	 * Returns the RSS Feed title according to the parameters
-	 * @return string the RSS feed Tiyle
-	 */
-	public function getName(){
-		switch($this->queriedContext) {
-			case $this->i8n('context-keyword'):
-				return $this->i8n('bridge-name') . ' - ' . $this->i8n('title-keyword') . ' : ' . $this->getInput('q');
-				break;
-			case $this->i8n('context-group'):
-				$values = $this->getParameters()[$this->i8n('context-group')]['group']['values'];
-				$group = array_search($this->getInput('group'), $values);
-				return $this->i8n('bridge-name') . ' - ' . $this->i8n('title-group') . ' : ' . $group;
-				break;
-			case $this->i8n('context-talk'):
-				return $this->i8n('bridge-name') . ' - ' . $this->i8n('title-talk') . ' : ' . $this->getTalkTitle();
-				break;
-			default: // Return default value
-				return static::NAME;
-		}
-	}
-
-	/**
-	 * Returns the RSS Feed URI according to the parameters
-	 * @return string the RSS feed Title
-	 */
-	public function getURI(){
-		switch($this->queriedContext) {
-			case $this->i8n('context-keyword'):
-				return $this->getSearchURI();
-				break;
-			case $this->i8n('context-group'):
-				return $this->getGroupURI();
-				break;
-			case $this->i8n('context-talk'):
-				return $this->getTalkURI();
-				break;
-			default: // Return default value
-				return static::URI;
-		}
-	}
-
-	/**
-	 * Returns the RSS Feed URI for a keyword Feed
-	 * @return string the RSS feed URI
-	 */
-	private function getSearchURI(){
-		$q = $this->getInput('q');
-		$hide_expired = $this->getInput('hide_expired');
-		$hide_local = $this->getInput('hide_local');
-		$priceFrom = $this->getInput('priceFrom');
-		$priceTo = $this->getInput('priceTo');
-		$url = $this->i8n('bridge-uri')
-			. 'search/advanced?q='
-			. urlencode($q)
-			. '&hide_expired=' . $hide_expired
-			. '&hide_local=' . $hide_local
-			. '&priceFrom=' . $priceFrom
-			. '&priceTo=' . $priceTo
-			/* Some default parameters
-			 * search_fields : Search in Titres & Descriptions & Codes
-			 * sort_by : Sort the search by new deals
-			 * time_frame : Search will not be on a limited timeframe
-			 */
-			. '&search_fields[]=1&search_fields[]=2&search_fields[]=3&sort_by=new&time_frame=0';
-		return $url;
-	}
-
-	/**
-	 * Returns the RSS Feed URI for a group Feed
-	 * @return string the RSS feed URI
-	 */
-	private function getGroupURI(){
-		$group = $this->getInput('group');
-		$order = $this->getInput('order');
-
-		$url = $this->i8n('bridge-uri')
-			. $this->i8n('uri-group') . $group . $order;
-		return $url;
-	}
-
-	/**
-	 * Returns the RSS Feed URI for a Talk Feed
-	 * @return string the RSS feed URI
-	 */
-	private function getTalkURI(){
-		$url = $this->getInput('url');
-		return $url;
-	}
-
-	/**
-	 * This is some "localisation" function that returns the needed content using
-	 * the "$lang" class variable in the local class
-	 * @return various the local content needed
-	 */
-	protected function i8n($key)
-	{
-		if (array_key_exists($key, $this->lang)) {
-			return $this->lang[$key];
-		} else {
-			return null;
-		}
-	}
-}

bridges/DeveloppezDotComBridge.php

@@ -1,47 +1,407 @@
 <?php
-class DeveloppezDotComBridge extends FeedExpander {
 
-	const MAINTAINER = 'polopollo';
+class DeveloppezDotComBridge extends FeedExpander
+{
+
+	const MAINTAINER = 'Binnette';
 	const NAME = 'Developpez.com Actus (FR)';
 	const URI = 'https://www.developpez.com/';
+	const DOMAIN = '.developpez.com/';
+	const RSS_URL = 'index/rss';
 	const CACHE_TIMEOUT = 1800; // 30min
-	const DESCRIPTION = 'Returns the 15 newest posts from DeveloppezDotCom (full text).';
+	const DESCRIPTION = 'Returns complete posts from developpez.com';
+	// Encodings used by Developpez.com in their articles body
+	const ENCONDINGS = array('Windows-1252', 'UTF-8');
+	const PARAMETERS = array(
+		array(
+			'limit' => array(
+				'name' => 'Max items',
+				'type' => 'number',
+				'defaultValue' => 5,
+			),
+			// list of the differents RSS availables
+			'domain' => array(
+				'type' => 'list',
+				'name' => 'Domaine',
+				'title' => 'Chosissez un sous-domaine',
+				'values' => array(
+					'= Domaine principal =' => 'www',
+					'4d' => '4d',
+					'abbyy' => 'abbyy',
+					'access' => 'access',
+					'agile' => 'agile',
+					'ajax' => 'ajax',
+					'algo' => 'algo',
+					'alm' => 'alm',
+					'android' => 'android',
+					'apache' => 'apache',
+					'applications' => 'applications',
+					'arduino' => 'arduino',
+					'asm' => 'asm',
+					'asp' => 'asp',
+					'aspose' => 'aspose',
+					'bacasable' => 'bacasable',
+					'big-data' => 'big-data',
+					'bpm' => 'bpm',
+					'bsd' => 'bsd',
+					'business-intelligence' => 'business-intelligence',
+					'c' => 'c',
+					'cloud-computing' => 'cloud-computing',
+					'club' => 'club',
+					'cms' => 'cms',
+					'cpp' => 'cpp',
+					'crm' => 'crm',
+					'css' => 'css',
+					'd' => 'd',
+					'dart' => 'dart',
+					'data-science' => 'data-science',
+					'db2' => 'db2',
+					'delphi' => 'delphi',
+					'dotnet' => 'dotnet',
+					'droit' => 'droit',
+					'eclipse' => 'eclipse',
+					'edi' => 'edi',
+					'embarque' => 'embarque',
+					'emploi' => 'emploi',
+					'etudes' => 'etudes',
+					'excel' => 'excel',
+					'firebird' => 'firebird',
+					'flash' => 'flash',
+					'go' => 'go',
+					'green-it' => 'green-it',
+					'gtk' => 'gtk',
+					'hardware' => 'hardware',
+					'hpc' => 'hpc',
+					'humour' => 'humour',
+					'ibmcloud' => 'ibmcloud',
+					'intelligence-artificielle' => 'intelligence-artificielle',
+					'interbase' => 'interbase',
+					'ios' => 'ios',
+					'java' => 'java',
+					'javascript' => 'javascript',
+					'javaweb' => 'javaweb',
+					'jetbrains' => 'jetbrains',
+					'jeux' => 'jeux',
+					'kotlin' => 'kotlin',
+					'labview' => 'labview',
+					'laravel' => 'laravel',
+					'latex' => 'latex',
+					'lazarus' => 'lazarus',
+					'linux' => 'linux',
+					'mac' => 'mac',
+					'matlab' => 'matlab',
+					'megaoffice' => 'megaoffice',
+					'merise' => 'merise',
+					'microsoft' => 'microsoft',
+					'mobiles' => 'mobiles',
+					'mongodb' => 'mongodb',
+					'mysql' => 'mysql',
+					'netbeans' => 'netbeans',
+					'nodejs' => 'nodejs',
+					'nosql' => 'nosql',
+					'objective-c' => 'objective-c',
+					'office' => 'office',
+					'open-source' => 'open-source',
+					'openoffice-libreoffice' => 'openoffice-libreoffice',
+					'oracle' => 'oracle',
+					'outlook' => 'outlook',
+					'pascal' => 'pascal',
+					'perl' => 'perl',
+					'php' => 'php',
+					'portail-emploi' => 'portail-emploi',
+					'portail-projets' => 'portail-projets',
+					'postgresql' => 'postgresql',
+					'powerpoint' => 'powerpoint',
+					'preprod-emploi' => 'preprod-emploi',
+					'programmation' => 'programmation',
+					'project' => 'project',
+					'purebasic' => 'purebasic',
+					'pyqt' => 'pyqt',
+					'python' => 'python',
+					'qt-creator' => 'qt-creator',
+					'qt' => 'qt',
+					'r' => 'r',
+					'raspberry-pi' => 'raspberry-pi',
+					'reseau' => 'reseau',
+					'ruby' => 'ruby',
+					'rust' => 'rust',
+					'sap' => 'sap',
+					'sas' => 'sas',
+					'scilab' => 'scilab',
+					'securite' => 'securite',
+					'sgbd' => 'sgbd',
+					'sharepoint' => 'sharepoint',
+					'solutions-entreprise' => 'solutions-entreprise',
+					'spring' => 'spring',
+					'sqlserver' => 'sqlserver',
+					'stages' => 'stages',
+					'supervision' => 'supervision',
+					'swift' => 'swift',
+					'sybase' => 'sybase',
+					'symfony' => 'symfony',
+					'systeme' => 'systeme',
+					'talend' => 'talend',
+					'typescript' => 'typescript',
+					'uml' => 'uml',
+					'unix' => 'unix',
+					'vb' => 'vb',
+					'vba' => 'vba',
+					'virtualisation' => 'virtualisation',
+					'visualstudio' => 'visualstudio',
+					'web-semantique' => 'web-semantique',
+					'web' => 'web',
+					'webmarketing' => 'webmarketing',
+					'wind' => 'wind',
+					'windows-azure' => 'windows-azure',
+					'windows' => 'windows',
+					'windowsphone' => 'windowsphone',
+					'word' => 'word',
+					'xhtml' => 'xhtml',
+					'xml' => 'xml',
+					'zend-framework' => 'zend-framework'
+				),
+			)
+		)
+	);
 
-	public function collectData(){
-		$this->collectExpandableDatas(self::URI . 'index/rss', 15);
+	/**
+	 * Return the RSS url for selected domain
+	 */
+	private function getRssUrl()
+	{
+		$domain = $this->getInput('domain');
+		if (!empty($domain)) {
+			return 'https://' . $domain . self::DOMAIN . self::RSS_URL;
+		}
+
+		return self::URI . self::RSS_URL;
 	}
 
-	protected function parseItem($newsItem){
+	/**
+	 * Grabs the RSS item from Developpez.com
+	 */
+	public function collectData()
+	{
+		$url = $this->getRssUrl();
+		$this->collectExpandableDatas($url, 20);
+	}
+
+	/**
+	 * Parse the content of every RSS item. And will try to get the full article
+	 * pointed by the item URL intead of the default abstract.
+	 */
+	protected function parseItem($newsItem)
+	{
+		if (count($this->items) >= $this->getInput('limit')) {
+			return null;
+		}
+
+		// This function parse each entry in the RSS with the default parse
 		$item = parent::parseItem($newsItem);
-		$item['content'] = $this->extractContent($item['uri']);
+
+		// There is a bug in Developpez RSS, coma are writtent as '~?' in the
+		// title, so I have to fix it manually
+		$item['title'] = $this->fixComaInTitle($item['title']);
+
+		// We get the content of the full article behind the RSS item URL
+		$articleHTMLContent = getSimpleHTMLDOMCached($item['uri']);
+
+		// Here we call our custom parser
+		$fullText = $this->extractFullText($articleHTMLContent);
+		if (!is_null($fullText)) {
+			// if we manage to parse the page behind the url of the RSS item
+			// then we set it as the new content. Otherwise we keep the default
+			// content to avoid RSS Bridge to return an empty item
+			$item['content'] = $fullText;
+		}
+
+		// Now we will attach video url in item
+		$videosUrl = $this->getAllVideoUrl($articleHTMLContent);
+		if (!empty($videosUrl)) {
+			$item['enclosures'] = array_merge($item['enclosures'], $videosUrl);
+		}
+
+		// Now we can look for the blog writer/creator
+		$author = $articleHTMLContent->find('[itemprop="creator"]', 0);
+		if (!empty($author)) {
+			$item['author'] = $author->outertext;
+		}
+
 		return $item;
 	}
 
-	// F***ing quotes from Microsoft Word badly encoded, here was the trick:
-	// http://stackoverflow.com/questions/1262038/how-to-replace-microsoft-encoded-quotes-in-php
-	private function convertSmartQuotes($string)
+	/**
+	 * Replace '~?' by a proper coma ','
+	 */
+	private function fixComaInTitle($txt)
 	{
-		$search = array(chr(145),
-						chr(146),
-						chr(147),
-						chr(148),
-						chr(151));
-
-		$replace = array(
-			"'",
-			"'",
-			'"',
-			'"',
-			'-'
-		);
-
-		return str_replace($search, $replace, $string);
+		return str_replace('~?', ',', $txt);
 	}
 
-	private function extractContent($url){
-		$articleHTMLContent = getSimpleHTMLDOMCached($url);
-		$text = $this->convertSmartQuotes($articleHTMLContent->find('div.content', 0)->innertext);
-		$text = utf8_encode($text);
-		return trim($text);
+	/**
+	 * Return the full article pointed by the url in the RSS item
+	 * Since Developpez.com only provides a short abstract of the article, we
+	 * use the url to retrieve the complete article and return it as the content
+	 */
+	private function extractFullText($articleHTMLContent)
+	{
+		// All blog entry contains a div with the class 'content'. This div
+		// contains the complete blog article. But the RSS can also return
+		// announcement and not a blog article. So the next if, should take
+		// care of the "non blog" entry
+		$divArticleEntry = $articleHTMLContent->find('div.content', 0);
+		if (is_null($divArticleEntry)) {
+			// Didn't find the div with class content. It is probably not a blog
+			// entry. It is probably just an announcement for an ebook, a PDF,
+			// etc. So we can use the default RSS item content.
+			return null;
+		}
+
+		// The following code is a bit hacky, but I really manage to get the
+		// full content of articles without any encoding issues. What is very
+		// weird and ugly in Developpez.com is the fact the some paragraphs of
+		// the article will be encoded as UTF-8 and some other paragraphs will
+		// be encoded as Windows-1252. So we can NOT decode the full article
+		// with only one encoding. We have to check every paragraph and
+		// determine its encoding
+
+		// This contains all the 'paragraphs' of the article. It includes the
+		// pictures, the text and the links at the bottom of the article
+		$paragraphs = $divArticleEntry->nodes;
+		// This will store the complete decoded content
+		$fullText = '';
+
+		// For each paragraph, we will identify the encoding, then decode it
+		// and finally store the decoded content in $text
+		foreach ($paragraphs as $paragraph) {
+			// We have to recreate a new DOM document from the current node
+			// otherwise the find function will look in the complet article and
+			// not only in the current paragraph. This is an ugly behavior of
+			// the library Simple HTML DOM Parser...
+			$html = str_get_html($paragraph->outertext);
+			$fullText .= $this->decodeParagraph($html);
+		}
+
+		// Finally we return the full 'well' enconded content of the article
+		return $fullText;
+	}
+
+	/**
+	 *
+	 */
+	private function decodeParagraph($p)
+	{
+		// First we check if this paragraph is a video
+		$videoUrl = $this->getVideoUrl($p);
+		if (!empty($videoUrl)) {
+			// If this is a video, we just return a link to the video
+			// &#128250; => 🎞️
+			return	'<p>
+						<b>&#128250; <a href="' . $videoUrl . '">Voir la vidéo</a></b>
+					</p>';
+		}
+
+		// We take outertext to get the complete paragraph not only the text
+		// inside it. That way we still graph block <img> and so on.
+		$pTxt = $p->outertext;
+		// This will store the decoded text if we manage to decode it
+		$decodedTxt = '';
+
+		// This is the only way to properly decode each paragraph. I tried
+		// many stuffs but this is the only working way I found.
+		foreach (self::ENCONDINGS as $enc) {
+			// We check the encoding of the current paragraph
+			if (mb_check_encoding($pTxt, $enc)) {
+				// If the encoding is well recognized, we can convert from
+				// this encoding to UTF-8
+				$decodedTxt = iconv($enc, 'UTF-8', $pTxt);
+			}
+		}
+
+		// We should not trim the strings to avoid the <a> to be glued to the
+		// text like: the software<a href="...">started</a>to...
+		if (!empty($decodedTxt)) {
+			// We manage to decode the text, so we take the decoded version
+			return $this->formatParagraph($decodedTxt);
+		} else {
+			// Otherwise we take the non decoded version and hope it will
+			// be displayed not too ugly in the fulltext content
+			return $this->formatParagraph($pTxt);
+		}
+	}
+
+	/**
+	 * Return true in $txt is a HTML tag and not plain text
+	 */
+	private function isHtmlTagNotTxt($txt)
+	{
+		$html = str_get_html($txt);
+		return $html && $html->root && count($html->root->children) > 0;
+	}
+
+	/**
+	 * Will add a space before paragraph when needed
+	 */
+	private function formatParagraph($txt)
+	{
+		// If the paragraph is an html tag, we add a space before
+		if ($this->isHtmlTagNotTxt($txt)) {
+			// the first element is an html tag and not a text, so we can add a
+			// space before it
+			return ' ' . $txt;
+		}
+		// If the text start with word (not punctation), we had a space
+		$pattern = '/^\w/';
+		if (preg_match($pattern, $txt)) {
+			return ' ' . $txt;
+		}
+		return $txt;
+	}
+
+	/**
+	 * Retrieve all video url in the article
+	 */
+	private function getAllVideoUrl($item)
+	{
+		// Array of video url
+		$url = array();
+
+		// Developpez use a div with the class video-container
+		$divsVideo = $item->find('div.video-container');
+		if (empty($divsVideo)) {
+			return $url;
+		}
+
+		// get the url of the video
+		foreach ($divsVideo as $div) {
+			$html = str_get_html($div->outertext);
+			$url[] = $this->getVideoUrl($html);
+		}
+
+		return $url;
+	}
+
+	/**
+	 * Retrieve URL video. We have to check for the src of an iframe
+	 * Work for Youtube. Will have to test for other video platform
+	 */
+	private function getVideoUrl($p)
+	{
+		$divVideo = $p->find('div.video-container', 0);
+		if (empty($divVideo)) {
+			return null;
+		}
+		$iframe = $divVideo->find('iframe', 0);
+		if (empty($iframe)) {
+			return null;
+		}
+		$src = trim($iframe->getAttribute('src'));
+		if (empty($src)) {
+			return null;
+		}
+		if (str_starts_with($src, '//')) {
+			$src = 'https:' . $src;
+		}
+		return $src;
 	}
 }

bridges/DownDetectorBridge.php

@@ -1,194 +0,0 @@
-<?php
-class DownDetectorBridge extends BridgeAbstract {
-	const MAINTAINER = 'teromene';
-	const NAME = 'DownDetector Bridge';
-	const URI = 'https://downdetector.com/';
-	const DESCRIPTION = 'Returns most recent downtimes from DownDetector';
-	const CACHE_TIMEOUT = 300; // 5 min
-
-	const PARAMETERS = array(
-		'All Websites' => array(
-			'country' => array(
-				'type' => 'list',
-				'name' => 'Country',
-				'values' => array(
-					'Argentina' => 'https://downdetector.com.ar',
-					'Australia' => 'https://downdetector.com.au',
-					'België' => 'https://allestoringen.be',
-					'Brasil' => 'https://downdetector.com.br',
-					'Canada' => 'https://downdetector.ca',
-					'Chile' => 'https://downdetector.cl',
-					'Colombia' => 'https://downdetector.com.co',
-					'Danmark' => 'https://downdetector.dk',
-					'Deutschland' => 'https://allestörungen.de',
-					'Ecuador' => 'https://downdetector.ec',
-					'España' => 'https://downdetector.es',
-					'France' => 'https://downdetector.fr',
-					'Hong Kong' => 'https://downdetector.hk',
-					'Hrvatska' => 'https://downdetector.hr',
-					'India' => 'https://downdetector.in',
-					'Indonesia' => 'https://downdetector.id',
-					'Ireland' => 'https://downdetector.ie',
-					'Italia' => 'https://downdetector.it',
-					'Magyarország' => 'https://downdetector.hu',
-					'Malaysia' => 'https://downdetector.my',
-					'México' => 'https://downdetector.mx',
-					'Nederland' => 'https://allestoringen.nl',
-					'New Zealand' => 'https://downdetector.co.nz',
-					'Norge' => 'https://downdetector.no',
-					'Pakistan' => 'https://downdetector.pk',
-					'Perú' => 'https://downdetector.pe',
-					'Pilipinas' => 'https://downdetector.ph',
-					'Polska' => 'https://downdetector.pl',
-					'Portugal' => 'https://downdetector.pt',
-					'România' => 'https://downdetector.ro',
-					'Schweiz' => 'https://allestörungen.ch',
-					'Singapore' => 'https://downdetector.sg',
-					'Slovensko' => 'https://downdetector.sk',
-					'South Africa' => 'https://downdetector.co.za',
-					'Suomi' => 'https://downdetector.fi',
-					'Sverige' => 'https://downdetector.se',
-					'Türkiye' => 'https://downdetector.web.tr',
-					'UAE' => 'https://downdetector.ae',
-					'UK' => 'https://downdetector.co.uk',
-					'United States' => 'https://downdetector.com',
-					'Österreich' => 'https://allestörungen.at',
-					'Česko' => 'https://downdetector.cz',
-					'Ελλάς' => 'https://downdetector.gr',
-					'Россия' => 'https://downdetector.ru',
-					'日本' => 'https://downdetector.jp'
-				)
-			)
-		),
-		'Specific Website' => array(
-			'page' => array(
-				'type' => 'text',
-				'name' => 'Status page',
-				'required' => true,
-				'exampleValue' => 'https://downdetector.com/status/rainbow-six',
-				'title' => 'URL of a DownDetector status page e.g: https://downdetector.com/status/rainbow-six/',
-			)
-		),
-	);
-
-	private $hostname = '';
-	private $statusPageId = '';
-	private $feedname = '';
-
-	private	$statusUrlRegex = '/\/([a-zA-z0-9ö.]+)\/(?:statu(?:s|t)|problemas?|nu-merge
-|(?:feil-)?problem(y|i)?(?:-storningar)?(?:-fejl)?|stoerung|durum|storing|fora-do-ar|ne-rabotaet
-|masalah|shougai|ei-toimi)\/([a-zA-Z0-9-]+)/';
-
-	public function collectData(){
-		if ($this->queriedContext == 'Specific Website') {
-			preg_match($this->statusUrlRegex, $this->getInput('page'), $match)
-				or returnClientError('Given URL does not seem to at a DownDetector status page!');
-
-			$this->hostname = $match[1];
-			$this->statusPageId = $match[3];
-		}
-
-		$html = getSimpleHTMLDOM($this->getURI() . '/archive/')
-			or returnClientError('Could not request website!.');
-
-		$html = defaultLinkTo($html, $this->getURI());
-
-		if ($this->getInput('page')) {
-			$this->feedname = $html->find('li.breadcrumb-item.active', 0)->plaintext;
-		}
-
-		$table = $html->find('table.table-striped', 0);
-
-		if ($table) {
-			foreach ($table->find('tr') as $event) {
-				$td = $event->find('td', 0);
-
-				if (is_null($td)) {
-					continue;
-				}
-
-				$item['uri'] = $event->find('td', 0)->find('a', 0)->href;
-				$item['title'] = $event->find('td', 0)->find('a', 0)->plaintext .
-					'(' . trim($event->find('td', 1)->plaintext) . ' ' . trim($event->find('td', 2)->plaintext) . ')';
-				$item['content'] = 'User reports indicate problems at' . $event->find('td', 0)->find('a', 0)->plaintext .
-					' since ' . $event->find('td', 2)->plaintext;
-				$item['timestamp'] = $this->formatDate(
-					trim($event->find('td', 1)->plaintext),
-					trim($event->find('td', 2)->plaintext)
-				);
-
-				$this->items[] = $item;
-			}
-		}
-	}
-
-	public function getURI() {
-		if($this->getInput('country')) {
-			return $this->getInput('country');
-		}
-
-		if ($this->getInput('page')) {
-			return 'https://' . $this->hostname . '/status/' . $this->statusPageId;
-		}
-
-		return self::URI;
-	}
-
-	public function getName() {
-		if($this->getInput('country')) {
-			$country = $this->getCountry($this->getInput('country'));
-			return $country . ' - DownDetector';
-		}
-
-		if ($this->getInput('page')) {
-			$country = $this->getCountry($this->hostname);
-			return $this->feedname . ' - ' . $country . '  - DownDetector';
-		}
-
-		return self::NAME;
-	}
-
-	private function formatDate($date, $time) {
-		switch($this->getCountry()) {
-			case 'Australia':
-			case 'UK':
-				$date = DateTime::createFromFormat('d/m/Y', $date);
-				return $date->format('Y-m-d') . $time;
-			case 'Brasil':
-			case 'Chile':
-			case 'Colombia':
-			case 'Ecuador':
-			case 'España':
-			case 'Italia':
-			case 'Perú':
-			case 'Portugal':
-				$date = DateTime::createFromFormat('d/m/Y', $date);
-				return $date->format('Y-m-d') . $time;
-			case 'Magyarország':
-				$date = DateTime::createFromFormat('Y.m.d.', $date);
-				return $date->format('Y-m-d') . $time;
-			default:
-				return $date . $time;
-		}
-	}
-
-	private function getCountry() {
-		if($this->getInput('country')) {
-			$input = $this->getInput('country');
-		}
-
-		if ($this->getInput('page')) {
-			if (empty($this->hostname)) {
-				return 'N/A';
-			}
-
-			$input = 'https://' . $this->hostname;
-		}
-
-		$parameters = $this->getParameters();
-		$countryValues = array_flip($parameters['All Websites']['country']['values']);
-		$country = $countryValues[$input];
-
-		return $country;
-	}
-}

bridges/EZTVBridge.php

@@ -3,65 +3,109 @@ class EZTVBridge extends BridgeAbstract {
 
 	const MAINTAINER = 'alexAubin';
 	const NAME = 'EZTV';
-	const URI = 'https://eztv.ch/';
-	const DESCRIPTION = 'Returns list of *recent* torrents for a specific show
-on EZTV. Get showID from URLs in https://eztv.ch/shows/showID/show-full-name.';
+	const URI = 'https://eztv.re/';
+	const DESCRIPTION = 'Returns list of torrents for specific show(s)
+on EZTV. Get IMDB IDs from IMDB.';
 
-	const PARAMETERS = array( array(
-		'i' => array(
-			'name' => 'Show ids',
-			'exampleValue' => '1017,249',
-			'title' => 'One of more showids as a comma separated list',
-			'required' => true
+	const PARAMETERS = array(
+		array(
+			'ids' => array(
+				'name' => 'Show IMDB IDs',
+				'exampleValue' => '8740790,1733785',
+				'required' => true,
+				'title' => 'One or more IMDB show IDs (can be found in the IMDB show URL)'
+			),
+			'no480' => array(
+				'name' => 'No 480p',
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude 480p torrents'
+			),
+			'no720' => array(
+				'name' => 'No 720p',
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude 720p torrents'
+			),
+			'no1080' => array(
+				'name' => 'No 1080p',
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude 1080p torrents'
+			),
+			'no2160' => array(
+				'name' => 'No 2160p',
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude 2160p torrents'
+			),
+			'noUnknownRes' => array(
+				'name' => 'No Unknown resolution',
+				'type' => 'checkbox',
+				'title' => 'Activate to exclude unknown resolution torrents'
+			),
 		)
-	));
+	);
+
+	// Shamelessly lifted from https://stackoverflow.com/a/2510459
+	protected function formatBytes($bytes, $precision = 2) {
+		$units = array('B', 'KB', 'MB', 'GB', 'TB');
+
+		$bytes = max($bytes, 0);
+		$pow = floor(($bytes ? log($bytes) : 0) / log(1024));
+		$pow = min($pow, count($units) - 1);
+		$bytes /= pow(1024, $pow);
+
+		return round($bytes, $precision) . ' ' . $units[$pow];
+	}
+
+	protected function getItemFromTorrent($torrent){
+		$item = array();
+		$item['uri'] = $torrent->episode_url;
+		$item['author'] = $torrent->imdb_id;
+		$item['timestamp'] = date('d F Y H:i:s', $torrent->date_released_unix);
+		$item['title'] = $torrent->title;
+		$item['enclosures'][] = $torrent->torrent_url;
+
+		$thumbnailUri = 'https:' . $torrent->small_screenshot;
+		$torrentSize = $this->formatBytes($torrent->size_bytes);
+
+		$item['content'] = $torrent->filename . '<br>File size: '
+		. $torrentSize . '<br><a href="' . $torrent->magnet_url
+		. '">magnet link</a><br><a href="' . $torrent->torrent_url
+		. '">torrent link</a><br><img src="' . $thumbnailUri . '" />';
+
+		return $item;
+	}
+
+	private static function compareDate($torrent1, $torrent2) {
+		return (strtotime($torrent1['timestamp']) < strtotime($torrent2['timestamp']) ? 1 : -1);
+	}
 
 	public function collectData(){
+		$showIds = explode(',', $this->getInput('ids'));
 
-		// Make timestamp from relative released time in table
-		function makeTimestamp($relativeReleaseTime){
+		foreach($showIds as $showId) {
+			$eztvUri = $this->getURI() . 'api/get-torrents?imdb_id=' . $showId;
+			$content = getContents($eztvUri);
+			$torrents = json_decode($content)->torrents;
+			foreach($torrents as $torrent) {
+				$title = $torrent->title;
+				$regex480 = '/480p/';
+				$regex720 = '/720p/';
+				$regex1080 = '/1080p/';
+				$regex2160 = '/2160p/';
+				$regexUnknown = '/(480p|720p|1080p|2160p)/';
+				// Skip unwanted resolution torrents
+				if ((preg_match($regex480, $title) === 1 && $this->getInput('no480'))
+				|| (preg_match($regex720, $title) === 1 && $this->getInput('no720'))
+				|| (preg_match($regex1080, $title) === 1 && $this->getInput('no1080'))
+				|| (preg_match($regex2160, $title) === 1 && $this->getInput('no2160'))
+				|| (preg_match($regexUnknown, $title) !== 1 && $this->getInput('noUnknownRes'))) {
+					continue;
+				}
 
-			$relativeDays = 0;
-			$relativeHours = 0;
-
-			foreach(explode(' ', $relativeReleaseTime) as $relativeTimeElement) {
-				if(substr($relativeTimeElement, -1) == 'd') $relativeDays = substr($relativeTimeElement, 0, -1);
-				if(substr($relativeTimeElement, -1) == 'h') $relativeHours = substr($relativeTimeElement, 0, -1);
-			}
-			return mktime(date('h') - $relativeHours, 0, 0, date('m'), date('d') - $relativeDays, date('Y'));
-		}
-
-		// Loop on show ids
-		$showList = explode(',', $this->getInput('i'));
-		foreach($showList as $showID) {
-
-			// Get show page
-			$html = getSimpleHTMLDOM(self::URI . 'shows/' . rawurlencode($showID) . '/');
-
-			// Loop on each element that look like an episode entry...
-			foreach($html->find('.forum_header_border') as $element) {
-
-				// Filter entries that are not episode entries
-				$ep = $element->find('td', 1);
-				if(empty($ep)) continue;
-				$epinfo = $ep->find('.epinfo', 0);
-				$released = $element->find('td', 3);
-				if(empty($epinfo)) continue;
-				if(empty($released->plaintext)) continue;
-
-				// Filter entries that are older than 1 week
-				if($released->plaintext == '&gt;1 week') continue;
-
-				// Fill item
-				$item = array();
-				$item['uri'] = self::URI . $epinfo->href;
-				$item['id'] = $item['uri'];
-				$item['timestamp'] = makeTimestamp($released->plaintext);
-				$item['title'] = $epinfo->plaintext;
-				$item['content'] = $epinfo->alt;
-				if(isset($item['title']))
-					$this->items[] = $item;
+				$this->items[] = $this->getItemFromTorrent($torrent);
 			}
 		}
+
+		// Sort all torrents in array by date
+		usort($this->items, array('EZTVBridge', 'compareDate'));
 	}
 }

bridges/EconomistWorldInBriefBridge.php

@@ -0,0 +1,141 @@
+<?php
+class EconomistWorldInBriefBridge extends BridgeAbstract
+{
+	const MAINTAINER = 'sqrtminusone';
+	const NAME = 'Economist the World in Brief Bridge';
+	const URI = 'https://www.economist.com/the-world-in-brief';
+
+	const CACHE_TIMEOUT = 3600; // 1 hour
+	const DESCRIPTION = 'Returns stories from the World in Brief section';
+
+	const PARAMETERS = array(
+		'' => array(
+			'splitGobbets' => array(
+				'name' => 'Split the short stories',
+				'type' => 'checkbox',
+				'defaultValue' => false,
+				'title' => 'Whether to split the short stories into separate entries'
+			),
+			'limit' => array(
+				'name' => 'Truncate headers for the short stories',
+				'type' => 'number',
+				'defaultValue' => 100
+			),
+			'agenda' => array(
+				'name' => 'Add agenda for the day',
+				'type' => 'checkbox',
+				'defaultValue' => 'checked'
+			),
+			'agendaPictures' => array(
+				'name' => 'Include pictures to the agenda',
+				'type' => 'checkbox',
+				'defaultValue' => 'checked'
+			),
+			'quote' => array(
+				'name' => 'Include the quote of the day',
+				'type' => 'checkbox'
+			)
+		)
+	);
+
+	public function collectData()
+	{
+		$html = getSimpleHTMLDOM(self::URI);
+		$gobbets = $html->find('._gobbets', 0);
+		if ($this->getInput('splitGobbets') == 1) {
+			$this->splitGobbets($gobbets);
+		} else {
+			$this->mergeGobbets($gobbets);
+		};
+		if ($this->getInput('agenda') == 1) {
+			$articles = $html->find('._articles', 0);
+			$this->collectArticles($articles);
+		}
+		if ($this->getInput('quote') == 1) {
+			$quote = $html->find('._quote-container', 0);
+			$this->addQuote($quote);
+		}
+	}
+
+	private function splitGobbets($gobbets)
+	{
+		$today = new Datetime();
+		$today->setTime(0, 0, 0, 0);
+		$limit = $this->getInput('limit');
+		foreach ($gobbets->find('._gobbet') as $gobbet) {
+			$title = $gobbet->plaintext;
+			$match = preg_match('/[\.,]/', $title, $matches, PREG_OFFSET_CAPTURE);
+			if ($match > 0) {
+				$point = $matches[0][1];
+				$title = mb_substr($title, 0, $point);
+			}
+			if ($limit && mb_strlen($title) > $limit) {
+				$title = mb_substr($title, 0, $limit) . '...';
+			}
+			$item = array(
+				'uri' => self::URI,
+				'title' => $title,
+				'content' => $gobbet->innertext,
+				'timestamp' => $today->format('U'),
+				'uid' => md5($gobbet->plaintext)
+			);
+			$this->items[] = $item;
+		}
+	}
+
+	private function mergeGobbets($gobbets)
+	{
+		$today = new Datetime();
+		$today->setTime(0, 0, 0, 0);
+		$contents = '';
+		foreach ($gobbets->find('._gobbet') as $gobbet) {
+			$contents .= "<p>{$gobbet->innertext}";
+		}
+		$this->items[] = array(
+			'uri' => self::URI,
+			'title' => 'World in brief at ' . $today->format('Y.m.d'),
+			'content' => $contents,
+			'timestamp' => $today->format('U'),
+			'uid' => 'world-in-brief-' . $today->format('U')
+		);
+	}
+
+	private function collectArticles($articles)
+	{
+		$i = 0;
+		$today = new Datetime();
+		$today->setTime(0, 0, 0, 0);
+		foreach ($articles->find('._article') as $article) {
+			$title = $article->find('._headline', 0)->plaintext;
+			$image = $article->find('._main-image', 0);
+			$content = $article->find('._content', 0);
+
+			$res_content = '';
+			if ($image != null && $this->getInput('agendaPictures') == 1) {
+				$img = $image->find('img', 0);
+				$res_content .= '<img src="' . $img->src . '" />';
+			}
+			$res_content .= $content->innertext;
+			$this->items[] = array(
+				'uri' => self::URI,
+				'title' => $title,
+				'content' => $res_content,
+				'timestamp' => $today->format('U'),
+				'uid' => 'story-' . $today->format('U') . "{$i}",
+			);
+			$i++;
+		}
+	}
+
+	private function addQuote($quote) {
+		$today = new Datetime();
+		$today->setTime(0, 0, 0, 0);
+		$this->items[] = array(
+			'uri' => self::URI,
+			'title' => 'Quote of the day ' . $today->format('Y.m.d'),
+			'content' => $quote->innertext,
+			'timestamp' => $today->format('U'),
+			'uid' => 'quote-' . $today->format('U')
+		);
+	}
+}

bridges/ElsevierBridge.php

@@ -1,7 +1,7 @@
 <?php
 class ElsevierBridge extends BridgeAbstract {
 
-	const MAINTAINER = 'Pierre Mazière';
+	const MAINTAINER = 'dvikan';
 	const NAME = 'Elsevier journals recent articles';
 	const URI = 'https://www.journals.elsevier.com/';
 	const CACHE_TIMEOUT = 43200; //12h
@@ -16,63 +16,26 @@ class ElsevierBridge extends BridgeAbstract {
 		)
 	));
 
-	// Extracts the list of names from an article as string
-	private function extractArticleName($article){
-		$names = $article->find('small', 0);
-		if($names)
-			return trim($names->plaintext);
-		return '';
-	}
-
-	// Extracts the timestamp from an article
-	private function extractArticleTimestamp($article){
-		$time = $article->find('.article-info', 0);
-		if($time) {
-			$timestring = trim($time->plaintext);
-			/*
-				The format depends on the age of an article:
-				- Available online 29 July 2016
-				- July 2016
-				- May–June 2016
-			*/
-			if(preg_match('/\S*(\d+\s\S+\s\d{4})/ims', $timestring, $matches)) {
-				return strtotime($matches[0]);
-			} elseif (preg_match('/[A-Za-z]+\-([A-Za-z]+\s\d{4})/ims', $timestring, $matches)) {
-				return strtotime($matches[0]);
-			} elseif (preg_match('/([A-Za-z]+\s\d{4})/ims', $timestring, $matches)) {
-				return strtotime($matches[0]);
-			} else {
-				return 0;
-			}
-		}
-		return 0;
-	}
-
-	// Extracts the content from an article
-	private function extractArticleContent($article){
-		$content = $article->find('.article-content', 0);
-		if($content) {
-			return trim($content->plaintext);
-		}
-		return '';
-	}
-
-	public function getIcon() {
-		return 'https://cdn.elsevier.io/verona/includes/favicons/favicon-32x32.png';
-	}
-
 	public function collectData(){
-		$uri = self::URI . $this->getInput('j') . '/recent-articles/';
-		$html = getSimpleHTMLDOM($uri);
+		// Not all journals have the /recent-articles page
+		$url = sprintf('https://www.journals.elsevier.com/%s/recent-articles/', $this->getInput('j'));
+		$html = getSimpleHTMLDOM($url);
 
-		foreach($html->find('.pod-listing') as $article) {
-			$item = array();
-			$item['uri'] = $article->find('.pod-listing-header>a', 0)->getAttribute('href') . '?np=y';
-			$item['title'] = $article->find('.pod-listing-header>a', 0)->plaintext;
-			$item['author'] = $this->extractArticleName($article);
-			$item['timestamp'] = $this->extractArticleTimestamp($article);
-			$item['content'] = $this->extractArticleContent($article);
+		foreach($html->find('article') as $recentArticle) {
+			$item = [];
+			$item['uri'] = $recentArticle->find('a', 0)->getAttribute('href');
+			$item['title'] = $recentArticle->find('h2', 0)->plaintext;
+			$item['author'] = $recentArticle->find('p > span', 0)->plaintext;
+			$publicationDateString = trim($recentArticle->find('p > span', 1)->plaintext);
+			$publicationDate = DateTimeImmutable::createFromFormat('F d, Y', $publicationDateString);
+			if ($publicationDate) {
+				$item['timestamp'] = $publicationDate->getTimestamp();
+			}
 			$this->items[] = $item;
 		}
 	}
+
+	public function getIcon(): string {
+		return 'https://cdn.elsevier.io/verona/includes/favicons/favicon-32x32.png';
+	}
 }

bridges/EtsyBridge.php

@@ -22,12 +22,10 @@ class EtsyBridge extends BridgeAbstract {
 (anything after ?search=<your search query>)',
 				'exampleValue' => '&explicit=1&locationQuery=2921044'
 			),
-			'showimage' => array(
-				'name' => 'Show image in content',
+			'hideimage' => array(
+				'name' => 'Hide image in content',
 				'type' => 'checkbox',
-				'required' => false,
-				'title' => 'Activate to show the image in the content',
-				'defaultValue' => 'checked'
+				'title' => 'Activate to hide the image in the content',
 			)
 		)
 	);
@@ -35,29 +33,29 @@ class EtsyBridge extends BridgeAbstract {
 	public function collectData(){
 		$html = getSimpleHTMLDOM($this->getURI());
 
-		$results = $html->find('li.block-grid-item');
+		$results = $html->find('li.wt-list-unstyled');
 
 		foreach($results as $result) {
-			// Skip banner cards (ads for categories)
-			if($result->find('span.ad-indicator'))
+			// Remove Lazy loading
+			if($result->find('.wt-skeleton-ui', 0))
 				continue;
 
 			$item = array();
 
 			$item['title'] = $result->find('a', 0)->title;
 			$item['uri'] = $result->find('a', 0)->href;
-			$item['author'] = $result->find('p.text-gray-lighter', 0)->plaintext;
+			$item['author'] = $result->find('p.wt-text-gray > span', 2)->plaintext;
 
 			$item['content'] = '<p>'
-			. $result->find('span.currency-value', 0)->plaintext . ' '
 			. $result->find('span.currency-symbol', 0)->plaintext
+			. $result->find('span.currency-value', 0)->plaintext
 			. '</p><p>'
 			. $result->find('a', 0)->title
 			. '</p>';
 
-			$image = $result->find('img.display-block', 0)->src;
+			$image = $result->find('img.wt-display-block', 0)->src;
 
-			if($this->getInput('showimage')) {
+			if(!$this->getInput('hideimage')) {
 				$item['content'] .= '<img src="' . $image . '">';
 			}
 

bridges/EuronewsBridge.php

@@ -0,0 +1,209 @@
+<?php
+class EuronewsBridge extends BridgeAbstract
+{
+	const MAINTAINER = 'sqrtminusone';
+	const NAME = 'Euronews Bridge';
+	const URI = 'https://www.euronews.com/';
+	const CACHE_TIMEOUT = 600; // 10 minutes
+	const DESCRIPTION = 'Return articles from the "Just In" feed of Euronews.';
+
+	const PARAMETERS = array(
+		'' => array(
+			'lang' => array(
+				'name' => 'Language',
+				'type' => 'list',
+				'defaultValue' => 'euronews.com',
+				'values' => array(
+					'English' => 'euronews.com',
+					'French' => 'fr.euronews.com',
+					'German' => 'de.euronews.com',
+					'Italian' => 'it.euronews.com',
+					'Spanish' => 'es.euronews.com',
+					'Portuguese' => 'pt.euronews.com',
+					'Russian' => 'ru.euronews.com',
+					'Turkish' => 'tr.euronews.com',
+					'Greek' => 'gr.euronews.com',
+					'Hungarian' => 'hu.euronews.com',
+					'Persian' => 'per.euronews.com',
+					'Arabic' => 'arabic.euronews.com',
+					/* These versions don't have timeline.json */
+					// 'Albanian' => 'euronews.al',
+					// 'Romanian' => 'euronews.ro',
+					// 'Georigian' => 'euronewsgeorgia.com',
+					// 'Bulgarian' => 'euronewsbulgaria.com'
+					// 'Serbian' => 'euronews.rs'
+				)
+			),
+			'limit' => array(
+				'name' => 'Limit of items per feed',
+				'required' => true,
+				'type' => 'number',
+				'defaultValue' => 10,
+				'title' => 'Maximum number of returned feed items. Maximum 50, default 10'
+			),
+		)
+	);
+
+	public function collectData()
+	{
+		$limit = $this->getInput('limit');
+		$root_url = 'https://' . $this->getInput('lang');
+		$url = $root_url . '/api/timeline.json?limit=' . $limit;
+		$json = getContents($url);
+		$data = json_decode($json, true);
+
+		foreach ($data as $datum) {
+			$datum_uri = $root_url . $datum['fullUrl'];
+			$url_datum = $this->getItemContent($datum_uri);
+			$categories = array();
+			if (array_key_exists('program', $datum)) {
+				if (array_key_exists('title', $datum['program'])) {
+					$categories[] = $datum['program']['title'];
+				}
+			}
+			if (array_key_exists('themes', $datum)) {
+				foreach ($datum['themes'] as $theme) {
+					$categories[] = $theme['title'];
+				}
+			}
+			$item = array(
+				'uri' => $datum_uri,
+				'title' => $datum['title'],
+				'uid' => strval($datum['id']),
+				'timestamp' => $datum['publishedAt'],
+				'content' => $url_datum['content'],
+				'author' => $url_datum['author'],
+				'enclosures' => $url_datum['enclosures'],
+				'categories' => array_unique($categories)
+			);
+			$this->items[] = $item;
+		}
+	}
+
+	private function getItemContent($url)
+	{
+		try {
+			$html = getSimpleHTMLDOMCached($url);
+		} catch (Exception $e) {
+			// Every once in a while it fails with too many redirects
+			return array('author' => null, 'content' => null, 'enclosures' => null);
+		}
+		$data = $html->find('script[type="application/ld+json"]', 0)->innertext;
+		$json = json_decode($data, true);
+		$author = 'Euronews';
+		$content = '';
+		$enclosures = array();
+		if (array_key_exists('@graph', $json)) {
+			foreach ($json['@graph'] as $item) {
+				if ($item['@type'] == 'NewsArticle') {
+					if (array_key_exists('author', $item)) {
+						$author = $item['author']['name'];
+					}
+					if (array_key_exists('image', $item)) {
+						$content .= '<figure>';
+						$content .= '<img src="' . $item['image']['url'] . '">';
+						$content .= '<figcaption>' . $item['image']['caption'] . '</figcaption>';
+						$content .= '</figure><br>';
+					}
+					if (array_key_exists('video', $item)) {
+						$enclosures[] = $item['video']['contentUrl'];
+					}
+				}
+			}
+		}
+
+		// Normal article
+		$article_content = $html->find('.c-article-content', 0);
+		if ($article_content) {
+			// Usually the .c-article-content is the root of the
+			// content, but once in a blue moon the root is the second
+			// div
+			if ((count($article_content->children()) == 2)
+				&& ($article_content->children(1)->tag == 'div')
+			) {
+				$article_content = $article_content->children(1);
+			}
+			// The content is interspersed with links and stuff, so we
+			// iterate over the children
+			foreach ($article_content->children() as $element) {
+				if ($element->tag == 'p') {
+					$scribble_live = $element->find('#scribblelive-items', 0);
+					if (is_null($scribble_live)) {
+						// A normal paragraph
+						$content .= '<p>' . $element->innertext . '</p>';
+					} else {
+						// LIVE mode
+						foreach ($scribble_live->children() as $child) {
+							if ($child->tag == 'div') {
+								$content .= '<div>' . $child->innertext . '</div>';
+							}
+						}
+					}
+				} elseif (preg_match('/h[1-6]/', $element->tag)) {
+					// Header
+					$content .= '<h' . $element->tag[1] . '>' . $element->innertext . '</h' . $element->tag[1] . '>';
+				} elseif ($element->tag == 'div') {
+					if (preg_match('/.*widget--type-image.*/', $element->class)) {
+						// Image
+						$content .= '<figure>';
+						$content .= '<img src="' . $element->find('img', 0)->src . '">';
+						$caption = $element->find('figcaption', 0);
+						if ($caption) {
+							$content .= '<figcaption>' . $element->plaintext . '</figcaption>';
+						}
+						$content .= '</figure><br>';
+					} elseif (preg_match('/.*widget--type-quotation.*/', $element->class)) {
+						// Quotation
+						$quote = $element->find('.widget__quoteText', 0);
+						$author = $element->find('.widget__author', 0);
+						$content .= '<figure>';
+						$content .= '<blockquote>' . $quote->plaintext . '</blockquote>';
+						if ($author) {
+							$content .= '<figcaption>' . $author->plaintext . '</figcaption>';
+						}
+						$content .= '</figure><br>';
+					}
+				}
+			}
+		}
+
+		// Video article
+		if (is_null($article_content)) {
+			$image = $html->find('.c-article-media__img', 0);
+			if ($image) {
+				$content .= '<figure>';
+				$content .= '<img src="' . $image->src . '">';
+				$content .= '</figure><br>';
+			}
+
+			$description = $html->find('.m-object__description', 0);
+			if ($description) {
+				// In some editions the description is a link to the
+				// current page
+				$content .= '<div>' . $description->plaintext . '</div>';
+			}
+
+			// Euronews usually hosts videos on dailymotion...
+			$player_div = $html->find('.dmPlayer', 0);
+			if ($player_div) {
+				$video_id = $player_div->getAttribute('data-video-id');
+				$video_url = 'https://www.dailymotion.com/video/' . $video_id;
+				$content .= '<a href="' . $video_url . '">' . $video_url . '</a>';
+			}
+
+			// ...or on YouTube
+			$player_div = $html->find('.js-player-pfp', 0);
+			if ($player_div) {
+				$video_id = $player_div->getAttribute('data-video-id');
+				$video_url = 'https://www.youtube.com/watch?v=' . $video_id;
+				$content .= '<a href="' . $video_url . '">' . $video_url . '</a>';
+			}
+		}
+
+		return array(
+			'author' => $author,
+			'content' => $content,
+			'enclosures' => $enclosures
+		);
+	}
+}

bridges/FB2Bridge.php

@@ -229,7 +229,7 @@ EOD
 
 		$ctx = stream_context_create(array(
 			'http' => array(
-				'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0',
+				'user_agent' => Configuration::getConfig('http', 'useragent'),
 				'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
 				)
 			)
@@ -254,7 +254,7 @@ EOD
 
 		$context = stream_context_create(array(
 			'http' => array(
-				'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0',
+				'user_agent' => Configuration::getConfig('http', 'useragent'),
 				'header' => 'Cookie: ' . $cookies
 				)
 			)

bridges/FDroidRepoBridge.php

@@ -0,0 +1,197 @@
+<?php
+class FDroidRepoBridge extends BridgeAbstract {
+	const NAME = 'F-Droid Repository Bridge';
+	const URI = 'https://f-droid.org/';
+	const DESCRIPTION = 'Query any F-Droid Repository for its latest updates.';
+	const MAINTAINER = 'Yaman Qalieh';
+
+	const ITEM_LIMIT = 50;
+
+	const PARAMETERS = array(
+		'global' => array(
+			'url' => array(
+				'name' => 'Repository URL',
+				'title' => 'Usually ends with /repo/',
+				'required' => true,
+				'exampleValue' => 'https://srv.tt-rss.org/fdroid/repo'
+			)
+		),
+		'Latest Updates' => array(
+			'sorting' => array(
+				'name' => 'Sort By',
+				'type' => 'list',
+				'values' => array(
+					'Latest added apps' => 'added',
+					'Latest updated apps' => 'lastUpdated'
+				)
+			),
+			'locale' => array(
+				'name' => 'Locale',
+				'defaultValue' => 'en-US'
+			)
+		),
+		'Follow Package' => array(
+			'package' => array(
+				'name' => 'Package Identifier',
+				'required' => true,
+				'exampleValue' => 'org.fox.ttrss'
+			)
+		)
+	);
+
+	// Stores repo information
+	private $repo;
+
+	public function getURI() {
+		if (empty($this->queriedContext))
+			return parent::getURI();
+
+		$url = rtrim($this->GetInput('url'), '/');
+		return strstr($url, '?', true) ?: $url;
+	}
+
+	public function getName() {
+		if (empty($this->queriedContext))
+			return parent::getName();
+
+		$name = $this->repo['repo']['name'];
+		switch($this->queriedContext) {
+			case 'Latest Updates':
+				return $name;
+			case 'Follow Package':
+				return $this->getInput('package') . ' - ' . $name;
+			default:
+				returnServerError('Unimplemented Context (getName)');
+		}
+	}
+
+	public function collectData() {
+		$this->repo = $this->getRepo();
+		switch($this->queriedContext) {
+			case 'Latest Updates':
+				$this->getAllUpdates();
+				break;
+			case 'Follow Package':
+				$this->getPackage($this->getInput('package'));
+				break;
+			default:
+				returnServerError('Unimplemented Context (collectData)');
+		}
+	}
+
+	private function getRepo() {
+		$url = $this->getURI();
+
+		// Get repo information (only available as JAR)
+		$jar = getContents($url . '/index-v1.jar');
+		$jar_loc = tempnam(sys_get_temp_dir(), '');
+		file_put_contents($jar_loc, $jar);
+
+		// JAR files are specially formatted ZIP files
+		$jar = new ZipArchive;
+		if ($jar->open($jar_loc) !== true) {
+			returnServerError('Failed to extract archive');
+		}
+
+		// Get file pointer to the relevant JSON inside
+		$fp = $jar->getStream('index-v1.json');
+		if (!$fp) {
+			returnServerError('Failed to get file pointer');
+		}
+
+		$data = json_decode(stream_get_contents($fp), true);
+		fclose($fp);
+		$jar->close();
+		return $data;
+	}
+
+	private function getAllUpdates() {
+		$apps = $this->repo['apps'];
+		usort($apps, function($a, $b) {
+			return $b[$this->getInput('sorting')] <=> $a[$this->getInput('sorting')];
+		});
+		$apps = array_slice($apps, 0, self::ITEM_LIMIT);
+		foreach($apps as $app) {
+			$latest = reset($this->repo['packages'][$app['packageName']]);
+
+			if (isset($app['localized'])) {
+				// Try provided locale, then en-US, then any
+				$lang = $app['localized'];
+				$lang = $lang[$this->getInput('locale')] ?? $lang['en-US'] ?? reset($lang);
+			} else
+				$lang = array();
+
+			$item = array();
+			$item['uri'] = $this->getURI() . '/' . $latest['apkName'];
+			$item['title'] = $lang['name'] ?? $app['packageName'];
+			$item['title'] .= ' ' . $latest['versionName'];
+			$item['timestamp'] = date(DateTime::ISO8601, (int) ($app['lastUpdated'] / 1000));
+			if (isset($app['authorName']))
+				$item['author'] = $app['authorName'];
+			if (isset($app['categories']))
+				$item['categories'] = $app['categories'];
+
+			// Adding Content
+			$icon = $app['icon'] ?? '';
+			if (!empty($icon)) {
+				$icon = $this->getURI() . '/icons-320/' . $icon;
+				$item['enclosures'] = array($icon);
+				$icon = '<img src="' . $icon . '">';
+			}
+			$summary = $lang['summary'] ?? $app['summary'] ?? '';
+			$description = markdownToHtml(trim($lang['description'] ?? $app['description'] ?? 'None'));
+			$whatsNew = markdownToHtml(trim($lang['whatsNew'] ?? 'None'));
+			$website = $this->link($lang['webSite'] ?? $app['webSite'] ?? $app['authorWebSite'] ?? null);
+			$source = $this->link($app['sourceCode'] ?? null);
+			$issueTracker = $this->link($app['issueTracker'] ?? null);
+			$license = $app['license'] ?? 'None';
+			$item['content'] = <<<EOD
+{$icon}
+<p>{$summary}</p>
+<h1>Description</h1>
+{$description}
+<h1>What's New</h1>
+{$whatsNew}
+<h1>Information</h1>
+<p>Website: {$website}</p>
+<p>Source Code: {$source}</p>
+<p>Issue Tracker: {$issueTracker}</p>
+<p>license: {$app['license']}</p>
+EOD;
+			$this->items[] = $item;
+		}
+	}
+
+	private function getPackage($package) {
+		if (!isset($this->repo['packages'][$package])) {
+			returnClientError('Invalid Package Name');
+		}
+		$package = $this->repo['packages'][$package];
+
+		$count = self::ITEM_LIMIT;
+		foreach($package as $version) {
+			$item = array();
+			$item['uri'] = $this->getURI() . '/' . $version['apkName'];
+			$item['title'] = $version['versionName'];
+			$item['timestamp'] = date(DateTime::ISO8601, (int) ($version['added'] / 1000));
+			$item['uid'] = $version['versionCode'];
+			$size = round($version['size'] / 1048576, 1); // Bytes -> MB
+			$sdk_link = 'https://developer.android.com/studio/releases/platforms';
+			$item['content'] = <<<EOD
+<p>size: {$size}MB</p>
+<p>Minimum SDK: {$version['minSdkVersion']}
+(<a href="{$sdk_link}">SDK to Android Version List</a>)</p>
+<p>hash ({$version['hashType']}): {$version['hash']}</p>
+EOD;
+			$this->items[] = $item;
+			if (--$count <= 0)
+				break;
+		}
+	}
+
+	private function link($url) {
+		if (empty($url))
+			return null;
+		return '<a href="' . $url . '">' . $url . '</a>';
+	}
+}

bridges/FSecureBlogBridge.php

@@ -18,7 +18,7 @@ class FSecureBlogBridge extends BridgeAbstract {
 			),
 			'oldest_date' => array(
 				'name' => 'Oldest article date',
-				'exampleValue' => '-2 months',
+				'exampleValue' => '-6 months',
 			),
 		)
 	);

bridges/FeedMergeBridge.php

@@ -0,0 +1,54 @@
+<?php
+
+class FeedMergeBridge extends FeedExpander {
+	const MAINTAINER = 'dvikan';
+	const NAME = 'FeedMerge';
+	const URI = 'https://github.com/RSS-Bridge/rss-bridge';
+	const DESCRIPTION = <<<'TEXT'
+This bridge merges two or more feeds into a single feed. Max 10 items are fetched from each feed.
+TEXT;
+
+	const PARAMETERS = [
+		[
+			'feed_name' => [
+				'name' => 'Feed name',
+				'type' => 'text',
+				'exampleValue' => 'rss-bridge/FeedMerger',
+			],
+			'feed_1' => [
+				'name' => 'Feed url',
+				'type' => 'text',
+				'required' => true,
+				'exampleValue' => 'https://lorem-rss.herokuapp.com/feed?unit=day'
+			],
+			'feed_2' => ['name' => 'Feed url', 'type' => 'text'],
+			'feed_3' => ['name' => 'Feed url', 'type' => 'text'],
+			'feed_4' => ['name' => 'Feed url', 'type' => 'text'],
+			'feed_5' => ['name' => 'Feed url', 'type' => 'text'],
+		]
+	];
+
+	public function collectData() {
+		$limit = 10;
+		$feeds = [
+			$this->getInput('feed_1'),
+			$this->getInput('feed_2'),
+			$this->getInput('feed_3'),
+			$this->getInput('feed_4'),
+			$this->getInput('feed_5'),
+		];
+		// Remove empty values
+		$feeds = array_filter($feeds);
+		foreach ($feeds as $feed) {
+			$this->collectExpandableDatas($feed, $limit);
+		}
+	}
+
+	public function getIcon() {
+		return 'https://cdn.jsdelivr.net/npm/famfamfam-silk@1.0.0/dist/png/folder_feed.png';
+	}
+
+	public function getName() {
+		return $this->getInput('feed_name') ?: 'rss-bridge/FeedMerger';
+	}
+}

bridges/FeedReducerBridge.php

@@ -0,0 +1,60 @@
+<?php
+class FeedReducerBridge extends FeedExpander {
+
+	const MAINTAINER = 'mdemoss';
+	const NAME = 'Feed Reducer';
+	const URI = 'http://github.com/RSS-Bridge/rss-bridge/';
+	const DESCRIPTION = 'Choose a percentage of a feed you want to see.';
+	const PARAMETERS = array( array(
+		'url' => array(
+			'name' => 'Feed URI',
+			'exampleValue' => 'https://lorem-rss.herokuapp.com/feed?length=42',
+			'required' => true
+		),
+		'percentage' => array(
+			'name' => 'percentage',
+			'type' => 'number',
+			'exampleValue' => 50,
+			'required' => true
+		)
+	));
+	const CACHE_TIMEOUT = 3600;
+
+	public function collectData(){
+		if(preg_match('#^http(s?)://#i', $this->getInput('url'))) {
+			$this->collectExpandableDatas($this->getInput('url'));
+		} else {
+			throw new Exception('URI must begin with http(s)://');
+		}
+	}
+
+	public function getItems(){
+		$filteredItems = array();
+		$intPercentage = (int)preg_replace('/[^0-9]/', '', $this->getInput('percentage'));
+
+		foreach ($this->items as $thisItem) {
+			// The URL is included in the hash:
+			//  - so you can change the output by adding a local-part to the URL
+			//  - so items with the same URI in different feeds won't be correlated
+
+			// $pseudoRandomInteger will be a 16 bit unsigned int mod 100.
+			// This won't be uniformly distributed 1-100, but should be close enough.
+
+			$pseudoRandomInteger = unpack(
+				'S', // unsigned 16-bit int
+				hash( 'sha256', $thisItem['uri'] . '::' . $this->getInput('url'), true )
+			)[1] % 100;
+
+			if ($pseudoRandomInteger < $intPercentage) {
+				$filteredItems[] = $thisItem;
+			}
+		}
+
+		return $filteredItems;
+	}
+
+	public function getName(){
+		$trimmedPercentage = preg_replace('/[^0-9]/', '', $this->getInput('percentage') ?? '');
+		return parent::getName() . ' [' . $trimmedPercentage . '%]';
+	}
+}

bridges/FilterBridge.php

@@ -11,6 +11,8 @@ class FilterBridge extends FeedExpander {
 	const PARAMETERS = array(array(
 		'url' => array(
 			'name' => 'Feed URL',
+			'type'	=> 'text',
+			'defaultValue' => 'https://lorem-rss.herokuapp.com/feed?unit=day',
 			'required' => true,
 		),
 		'filter' => array(

bridges/FindACrewBridge.php

@@ -26,7 +26,8 @@ class FindACrewBridge extends BridgeAbstract {
 			'distance' => array(
 				'name' => 'Limit boundary of search in KM',
 				'title' => 'Boundary of the search in kilometers when using longitude and latitude'
-			)
+			),
+			'limit' => self::LIMIT,
 		)
 	);
 
@@ -59,14 +60,15 @@ class FindACrewBridge extends BridgeAbstract {
 		$html = getSimpleHTMLDOM($url, $header, $opts) or returnClientError('No results for this query.');
 
 		$annonces = $html->find('.css_SrhRst');
-		foreach ($annonces as $annonce) {
+		$limit = $this->getInput('limit') ?? 10;
+		foreach (array_slice($annonces, 0, $limit) as $annonce) {
 			$item = array();
 
-			$link = parent::getURI() . $annonce->find('.lst-ctrls a', 0)->href;
+			$link = parent::getURI() . $annonce->find('.lstsum-btn-con a', 0)->href;
 			$htmlDetail = getSimpleHTMLDOMCached($link . '?mdl=2'); // add ?mdl=2 for xhr content not full html page
 
 			$img = parent::getURI() . $htmlDetail->find('img.img-responsive', 0)->getAttribute('src');
-			$item['title'] = $annonce->find('.lst-tags span', 0)->plaintext;
+			$item['title'] = $htmlDetail->find('div.label-account', 0)->plaintext;
 			$item['uri'] = $link;
 			$content = $htmlDetail->find('.panel-body div.clearfix.row > div', 1)->innertext;
 			$content .= $htmlDetail->find('.panel-body > div', 1)->innertext;

bridges/FlickrBridge.php

@@ -51,6 +51,15 @@ class FlickrBridge extends BridgeAbstract {
 				'title' => 'Insert username (as shown in the address bar)',
 				'exampleValue' => 'flickr'
 			),
+			'content' => array(
+				'name' => 'Content',
+				'type' => 'list',
+				'values' => array(
+					'Uploads' => 'uploads',
+					'Favorites' => 'faves',
+				),
+				'defaultValue' => 'uploads',
+			),
 			'media' => array(
 				'name' => 'Media',
 				'type' => 'list',
@@ -156,8 +165,14 @@ class FlickrBridge extends BridgeAbstract {
 					. '&sort=' . $this->getInput('sort') . '&media=' . $this->getInput('media');
 				break;
 			case 'By username':
-				return self::URI . 'search/?user_id=' . urlencode($this->getInput('u'))
-					. '&sort=' . $this->getInput('sort') . '&media=' . $this->getInput('media');
+				$uri = self::URI . 'search/?user_id=' . urlencode($this->getInput('u'))
+					. '&sort=date-posted-desc&media=' . $this->getInput('media');
+
+				if ($this->getInput('content') === 'faves') {
+					return $uri . '&faves=1';
+				}
+
+				return $uri;
 				break;
 
 			default:
@@ -175,6 +190,11 @@ class FlickrBridge extends BridgeAbstract {
 				return $this->getInput('q') . ' - keyword - ' . self::NAME;
 				break;
 			case 'By username':
+
+				if ($this->getInput('content') === 'faves') {
+					return $this->username . ' - favorites - ' . self::NAME;
+				}
+
 				return $this->username . ' - ' . self::NAME;
 				break;
 

bridges/FolhaDeSaoPauloBridge.php

@@ -63,6 +63,7 @@ class FolhaDeSaoPauloBridge extends FeedExpander {
 			$feed_url = self::URI . '/' . $this->getInput('feed');
 		}
 		Debug::log('URL: ' . $feed_url);
-		$this->collectExpandableDatas($feed_url, $this->getInput('amount'));
+		$limit = $this->getInput('amount');
+		$this->collectExpandableDatas($feed_url, $limit);
 	}
 }

bridges/FootitoBridge.php

@@ -1,74 +0,0 @@
-<?php
-class FootitoBridge extends BridgeAbstract {
-
-	const MAINTAINER = 'superbaillot.net';
-	const NAME = 'Footito';
-	const URI = 'http://www.footito.fr/';
-	const DESCRIPTION = 'Footito';
-
-	public function collectData(){
-		$html = getSimpleHTMLDOM(self::URI);
-
-		foreach($html->find('div.post') as $element) {
-			$item = array();
-
-			$content = trim($element->innertext);
-			$content = str_replace(
-				'<img',
-				"<img style='float : left;'",
-				$content );
-
-			$content = str_replace(
-				'class="logo"',
-				"style='float : left;'",
-				$content );
-
-			$content = str_replace(
-				'class="contenu"',
-				"style='margin-left : 60px;'",
-				$content );
-
-			$content = str_replace(
-				'class="responsive-comment"',
-				"style='border-top : 1px #DDD solid; background-color : white; padding : 10px;'",
-				$content );
-
-			$content = str_replace(
-				'class="jaime"',
-				"style='display : none;'",
-				$content );
-
-			$content = str_replace(
-				'class="auteur-event responsive"',
-				"style='display : none;'",
-				$content );
-
-			$content = str_replace(
-				'class="report-abuse-button"',
-				"style='display : none;'",
-				$content );
-
-			$content = str_replace(
-				'class="reaction clearfix"',
-				"style='margin : 10px 0px; padding : 5px; border-bottom : 1px #DDD solid;'",
-				$content );
-
-			$content = str_replace(
-				'class="infos"',
-				"style='font-size : 0.7em;'",
-				$content );
-
-			$item['content'] = $content;
-
-			$title = $element->find('.contenu .texte ', 0)->plaintext;
-			$item['title'] = $title;
-
-			$info = $element->find('div.infos', 0);
-
-			$item['timestamp'] = strtotime($info->find('time', 0)->datetime);
-			$item['author'] = $info->find('a.auteur', 0)->plaintext;
-
-			$this->items[] = $item;
-		}
-	}
-}

bridges/FurAffinityUserBridge.php

@@ -3,21 +3,23 @@ class FurAffinityUserBridge extends BridgeAbstract {
 	const NAME = 'FurAffinity User Gallery';
 	const URI = 'https://www.furaffinity.net';
 	const MAINTAINER = 'CyberJacob';
+	const DESCRIPTION = 'See https://rss-bridge.github.io/rss-bridge/Bridge_Specific/Furaffinityuser.html for explanation';
 	const PARAMETERS = array(
 		array(
 			'searchUsername' => array(
 				'name' => 'Search Username',
 				'type' => 'text',
 				'required' => true,
-				'title' => 'Username to fetch the gallery for'
+				'title' => 'Username to fetch the gallery for',
+				'exampleValue' => 'armundy',
 			),
-			'loginUsername' => array(
-				'name' => 'Login Username',
+			'aCookie' => array(
+				'name' => 'Login cookie \'a\'',
 				'type' => 'text',
 				'required' => true
 			),
-			'loginPassword' => array(
-				'name' => 'Login Password',
+			'bCookie' => array(
+				'name' => 'Login cookie \'b\'',
 				'type' => 'text',
 				'required' => true
 			)
@@ -25,10 +27,12 @@ class FurAffinityUserBridge extends BridgeAbstract {
 	);
 
 	public function collectData() {
-		$cookies = self::login();
+		$opt = array(CURLOPT_COOKIE => 'b=' . $this->getInput('bCookie') . '; a=' . $this->getInput('aCookie'));
+
 		$url = self::URI . '/gallery/' . $this->getInput('searchUsername');
 
-		$html = getSimpleHTMLDOM($url, $cookies);
+		$html = getSimpleHTMLDOM($url, array(), $opt)
+			or returnServerError('Could not load the user\'s gallery page.');
 
 		$submissions = $html->find('section[id=gallery-gallery]', 0)->find('figure');
 		foreach($submissions as $submission) {
@@ -51,59 +55,4 @@ class FurAffinityUserBridge extends BridgeAbstract {
 	public function getURI() {
 		return self::URI . '/user/' . $this->getInput('searchUsername');
 	}
-
-	private function login() {
-		$ch = curl_init(self::URI . '/login/');
-		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
-
-		curl_setopt($ch, CURLOPT_USERAGENT, ini_get('user_agent'));
-		curl_setopt($ch, CURLOPT_ENCODING, '');
-		curl_setopt($ch, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
-
-		$fields = implode('&', array(
-			'action=login',
-			'retard_protection=1',
-			'name=' . urlencode($this->getInput('loginUsername')),
-			'pass=' . urlencode($this->getInput('loginPassword')),
-			'login=Login to Faraffinity'
-		));
-
-		curl_setopt($ch, CURLOPT_POST, 5);
-		curl_setopt($ch, CURLOPT_POSTFIELDS, $fields);
-
-		if(defined('PROXY_URL') && !defined('NOPROXY')) {
-			curl_setopt($ch, CURLOPT_PROXY, PROXY_URL);
-		}
-
-		curl_setopt($ch, CURLOPT_HEADER, true);
-		curl_setopt($ch, CURLINFO_HEADER_OUT, true);
-
-		$data = curl_exec($ch);
-
-		$errorCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
-
-		$curlError = curl_error($ch);
-		$curlErrno = curl_errno($ch);
-		$curlInfo = curl_getinfo($ch);
-
-		if($data === false)
-			fDebug::log("Cant't download {$url} cUrl error: {$curlError} ({$curlErrno})");
-
-		curl_close($ch);
-
-		if($errorCode != 200) {
-			returnServerError(error_get_last());
-		} else {
-			preg_match_all('/^Set-Cookie:\s*([^;]*)/mi', $data, $matches);
-			$cookies = array();
-
-			foreach($matches[1] as $item) {
-				parse_str($item, $cookie);
-				$cookies = array_merge($cookies, $cookie);
-			}
-
-			return $cookies;
-		}
-	}
 }

bridges/GQMagazineBridge.php

@@ -32,6 +32,7 @@ class GQMagazineBridge extends BridgeAbstract
 			'required' => true,
 			'exampleValue' => 'sexe/news'
 		),
+		'limit' => self::LIMIT,
 	));
 
 	const REPLACED_ATTRIBUTES = array(
@@ -76,7 +77,12 @@ class GQMagazineBridge extends BridgeAbstract
 
 		// Since GQ don't want simple class scrapping, let's do it the hard way and ... discover content !
 		$main = $html->find('main', 0);
+		$limit = $this->getInput('limit') ?? 10;
 		foreach ($main->find('a') as $link) {
+			if (count($this->items) >= $limit) {
+				break;
+			}
+
 			$uri = $link->href;
 			$date = $link->parent()->find('time', 0);
 

bridges/GatesNotesBridge.php

@@ -0,0 +1,54 @@
+<?php
+class GatesNotesBridge extends FeedExpander {
+
+	const MAINTAINER = 'corenting';
+	const NAME = 'Gates Notes';
+	const URI = 'https://www.gatesnotes.com';
+	const DESCRIPTION = 'Returns the newest articles.';
+	const CACHE_TIMEOUT = 21600; // 6h
+
+	protected function parseItem($item){
+		$item = parent::parseItem($item);
+
+		$article_html = getSimpleHTMLDOMCached($item['uri']);
+		if(!$article_html) {
+			$item['content'] .= '<p><em>Could not request ' . $this->getName() . ': ' . $item['uri'] . '</em></p>';
+			return $item;
+		}
+		$article_html = defaultLinkTo($article_html, $this->getURI());
+
+		$top_description = '<p>' . $article_html->find('div.article_top_description', 0)->innertext . '</p>';
+		$hero_image = '<img src=' . $article_html->find('img.article_top_DMT_Image', 0)->getAttribute('data-src') . '>';
+
+		$article_body = $article_html->find('div.TGN_Article_ReadTimeSection', 0);
+		// Convert iframe of Youtube videos to link
+		foreach($article_body->find('iframe') as $found) {
+
+			$iframeUrl = $found->getAttribute('src');
+
+			if ($iframeUrl) {
+				$text = 'Embedded Youtube video, click here to watch on Youtube.com';
+				$found->outertext = '<p><a href="' . $iframeUrl . '">' . $text . '</a></p>';
+			}
+		}
+		// Remove <link> CSS ressources
+		foreach($article_body->find('link') as $found) {
+
+			$linkedRessourceUrl = $found->getAttribute('href');
+
+			if (str_ends_with($linkedRessourceUrl, '.css')) {
+				$found->outertext = '';
+			}
+		}
+		$article_body = sanitize($article_body->innertext);
+
+		$item['content'] = $top_description . $hero_image . $article_body;
+
+		return $item;
+	}
+
+	public function collectData(){
+		$feed = static::URI . '/rss';
+		$this->collectExpandableDatas($feed);
+	}
+}

bridges/GelbooruBridge.php

@@ -1,35 +1,88 @@
 <?php
-require_once('DanbooruBridge.php');
-
-class GelbooruBridge extends DanbooruBridge {
+class GelbooruBridge extends BridgeAbstract {
 
 	const MAINTAINER = 'mitsukarenai';
 	const NAME = 'Gelbooru';
-	const URI = 'http://gelbooru.com/';
+	const URI = 'https://gelbooru.com/';
 	const DESCRIPTION = 'Returns images from given page';
 
-	const PATHTODATA = '.thumb';
-	const IDATTRIBUTE = 'id';
-	const TAGATTRIBUTE = 'title';
-
-	const PIDBYPAGE = 63;
+	const PARAMETERS = array(
+		'global' => array(
+			'p' => array(
+				'name' => 'page',
+				'defaultValue' => 0,
+				'type' => 'number'
+			),
+			't' => array(
+				'name' => 'tags',
+				'exampleValue' => 'solo',
+				'title' => 'Tags to search for'
+			),
+			'l' => array(
+				'name' => 'limit',
+				'exampleValue' => 100,
+				'title' => 'How many posts to retrieve (hard limit of 1000)'
+			)
+		),
+		0 => array()
+	);
 
 	protected function getFullURI(){
 		return $this->getURI()
-		. 'index.php?page=post&s=list&pid='
-		. ($this->getInput('p') ? ($this->getInput('p') - 1) * static::PIDBYPAGE : '')
+		. 'index.php?&page=dapi&s=post&q=index&json=1&pid=' . $this->getInput('p')
+		. '&limit=' . $this->getInput('l')
 		. '&tags=' . urlencode($this->getInput('t'));
 	}
 
-	protected function getTags($element){
-		$tags = parent::getTags($element);
-		$tags = explode(' ', $tags);
+	/*
+	This function is superfluous for GelbooruBridge, but useful
+	for Bridges that inherit from it
+	*/
+	protected function buildThumbnailURI($element){
+		return $this->getURI() . 'thumbnails/' . $element->directory
+		. '/thumbnail_' . $element->md5 . '.jpg';
+	}
 
-		// Remove statistics from the tags list (identified by colon)
-		foreach($tags as $key => $tag) {
-			if(strpos($tag, ':') !== false) unset($tags[$key]);
+	protected function getItemFromElement($element){
+		$item = array();
+		$item['uri'] = $this->getURI() . 'index.php?page=post&s=view&id='
+		. $element->id;
+		$item['postid'] = $element->id;
+		$item['author'] = $element->owner;
+		$item['timestamp'] = date('d F Y H:i:s', $element->change);
+		$item['tags'] = $element->tags;
+		$item['title'] = $this->getName() . ' | ' . $item['postid'];
+
+		if (isset($element->preview_url)) {
+			$thumbnailUri = $element->preview_url;
+		} else{
+			$thumbnailUri = $this->buildThumbnailURI($element);
 		}
 
-		return implode(' ', $tags);
+		$item['content'] = '<a href="' . $item['uri'] . '"><img src="'
+		. $thumbnailUri	. '" /></a><br><br><b>Tags:</b> '
+		. $item['tags'] . '<br><br>' . $item['timestamp'];
+
+		return $item;
+	}
+
+	public function collectData(){
+		$content = getContents($this->getFullURI());
+		// $content is empty string
+
+		// Most other Gelbooru-based boorus put their content in the root of
+		// the JSON. This check is here for Bridges that inherit from this one
+		$posts = json_decode($content);
+		if (isset($posts->post)) {
+			$posts = $posts->post;
+		}
+
+		if (is_null($posts)) {
+			returnServerError('No posts found.');
+		}
+
+		foreach($posts as $post) {
+			$this->items[] = $this->getItemFromElement($post);
+		}
 	}
 }

bridges/GettrBridge.php

@@ -30,7 +30,7 @@ class GettrBridge extends BridgeAbstract
 		$api = sprintf(
 			'https://api.gettr.com/u/user/%s/posts?offset=0&max=%s&dir=fwd&incl=posts&fp=f_uo',
 			$this->getInput('user'),
-			max($this->getInput('limit'), 20)
+			min($this->getInput('limit'), 20)
 		);
 		$data = json_decode(getContents($api), false);
 

bridges/GiphyBridge.php

@@ -31,6 +31,15 @@ class GiphyBridge extends BridgeAbstract {
 		)
 	));
 
+	public function getName()
+	{
+		if (!is_null($this->getInput('s'))) {
+			return $this->getInput('s') . ' - ' . parent::getName();
+		}
+
+		return parent::getName();
+	}
+
 	protected function getGiphyItems($entries){
 		foreach($entries as $entry) {
 			$createdAt = new \DateTime($entry->import_datetime);
@@ -56,12 +65,16 @@ HTML
 
 	public function collectData() {
 		/**
-		 * This uses a public beta key which has severe rate limiting.
+		 * This uses Giphy's own undocumented public prod api key,
+		 * which should not have any rate limiting.
+		 * There is a documented public beta api key (dc6zaTOxFJmzC),
+		 * but it has severe rate limiting.
 		 *
 		 * https://giphy.api-docs.io/1.0/welcome/access-and-api-keys
-		 * https://giphy.api-docs.io/1.0/gifs/search-1
+		 * https://developers.giphy.com/branch/master/docs/api/endpoint/#search
 		 */
-		$apiKey = 'dc6zaTOxFJmzC';
+		$apiKey = 'Gc7131jiJuvI7IdN0HZ1D7nh0ow5BU6g';
+		$bundle = 'low_bandwidth';
 		$limit = min($this->getInput('n') ?: 10, 50);
 		$endpoints = array();
 		if (empty($this->getInput('noGif'))) {
@@ -73,10 +86,11 @@ HTML
 
 		foreach ($endpoints as $endpoint) {
 			$uri = sprintf(
-				'https://api.giphy.com/v1/%s/search?q=%s&limit=%s&api_key=%s',
+				'https://api.giphy.com/v1/%s/search?q=%s&limit=%s&bundle=%s&api_key=%s',
 				$endpoint,
 				rawurlencode($this->getInput('s')),
 				$limit,
+				$bundle,
 				$apiKey
 			);
 

bridges/GiteaBridge.php

@@ -1,27 +1,305 @@
 <?php
 /**
- * Gitea is a fork of Gogs which may diverge in the future.
+ * Gitea is a community managed lightweight code hosting solution.
  * https://docs.gitea.io/en-us/
  */
-require_once 'GogsBridge.php';
 
-class GiteaBridge extends GogsBridge {
+class GiteaBridge extends BridgeAbstract {
 
 	const NAME = 'Gitea';
 	const URI = 'https://gitea.io';
 	const DESCRIPTION = 'Returns the latest issues, commits or releases';
-	const MAINTAINER = 'logmanoriginal';
+	const MAINTAINER = 'gileri';
 	const CACHE_TIMEOUT = 300; // 5 minutes
 
+	const PARAMETERS = array(
+		'global' => array(
+			'host' => array(
+				'name' => 'Host',
+				'exampleValue' => 'https://gitea.com',
+				'required' => true,
+				'title' => 'Host name with its protocol, without trailing slash',
+			),
+			'user' => array(
+				'name' => 'Username',
+				'exampleValue' => 'gitea',
+				'required' => true,
+				'title' => 'User name as it appears in the URL',
+			),
+			'project' => array(
+				'name' => 'Project name',
+				'exampleValue' => 'helm-chart',
+				'required' => true,
+				'title' => 'Project name as it appears in the URL',
+			),
+		),
+		'Commits' => array(
+			'branch' => array(
+				'name' => 'Branch name',
+				'defaultValue' => 'master',
+				'required' => true,
+				'title' => 'Branch name as it appears in the URL',
+			),
+		),
+		'Issues' => array(
+			'include_description' => array(
+				'name' => 'Include issue description',
+				'type' => 'checkbox',
+				'title' => 'Activate to include the issue description',
+			),
+		),
+		'Single issue' => array(
+			'issue' => array(
+				'name' => 'Issue number',
+				'type' => 'number',
+				'exampleValue' => 100,
+				'required' => true,
+				'title' => 'Issue number from the issues list',
+			),
+		),
+		'Single pull request' => array(
+			'pull_request' => array(
+				'name' => 'Pull request number',
+				'type' => 'number',
+				'exampleValue' => 100,
+				'required' => true,
+				'title' => 'Pull request number from the issues list',
+			),
+		),
+		'Pull requests' => array(
+			'include_description' => array(
+				'name' => 'Include pull request description',
+				'type' => 'checkbox',
+				'title' => 'Activate to include the pull request description',
+			),
+		),
+		'Releases' => array(),
+		'Tags' => array(),
+	);
+
+	private $title = '';
+
+	public function getIcon() {
+		return 'https://gitea.io/images/gitea.png';
+	}
+
+	public function getName() {
+		switch($this->queriedContext) {
+			case 'Commits':
+			case 'Issues':
+			case 'Pull requests':
+			case 'Releases':
+			case 'Tags': return $this->title . ' ' . $this->queriedContext;
+			case 'Single issue': return 'Issue ' . $this->getInput('issue') . ': ' . $this->title;
+			case 'Single pull request': return 'Pull request ' . $this->getInput('pull_request') . ': ' . $this->title;
+			default: return parent::getName();
+		}
+	}
+
+	public function getURI() {
+		switch($this->queriedContext) {
+			case 'Commits': {
+				return $this->getInput('host')
+				. '/' . $this->getInput('user')
+				. '/' . $this->getInput('project')
+				. '/commits/' . $this->getInput('branch');
+			} break;
+			case 'Issues': {
+				return $this->getInput('host')
+				. '/' . $this->getInput('user')
+				. '/' . $this->getInput('project')
+				. '/issues/';
+			} break;
+			case 'Single issue': {
+				return $this->getInput('host')
+				. '/' . $this->getInput('user')
+				. '/' . $this->getInput('project')
+				. '/issues/' . $this->getInput('issue');
+			} break;
+			case 'Releases': {
+				return $this->getInput('host')
+				. '/' . $this->getInput('user')
+				. '/' . $this->getInput('project')
+				. '/releases/';
+			} break;
+			case 'Tags': {
+				return $this->getInput('host')
+				. '/' . $this->getInput('user')
+				. '/' . $this->getInput('project')
+				. '/tags/';
+			} break;
+			case 'Pull requests': {
+				return $this->getInput('host')
+				. '/' . $this->getInput('user')
+				. '/' . $this->getInput('project')
+				. '/pulls/';
+			} break;
+			case 'Single pull request': {
+				return $this->getInput('host')
+				. '/' . $this->getInput('user')
+				. '/' . $this->getInput('project')
+				. '/pulls/' . $this->getInput('pull_request');
+			} break;
+			default: return parent::getURI();
+		}
+	}
+
+	public function collectData() {
+		$html = getSimpleHTMLDOM($this->getURI())
+			or returnServerError('Could not request ' . $this->getURI());
+		$html = defaultLinkTo($html, $this->getURI());
+
+		$this->title = $html->find('[property="og:title"]', 0)->content;
+
+		switch($this->queriedContext) {
+			case 'Commits': {
+				$this->collectCommitsData($html);
+			} break;
+			case 'Issues': {
+				$this->collectIssuesData($html);
+			} break;
+			case 'Pull requests': {
+				$this->collectPullRequestsData($html);
+			} break;
+			case 'Single issue': {
+				$this->collectSingleIssueOrPrData($html);
+			} break;
+			case 'Single pull request': {
+				$this->collectSingleIssueOrPrData($html);
+			} break;
+			case 'Releases': {
+				$this->collectReleasesData($html);
+			} break;
+			case 'Tags': {
+				$this->collectTagsData($html);
+			} break;
+		}
+	}
+
 	protected function collectReleasesData($html) {
 		$releases = $html->find('#release-list > li')
 			or returnServerError('Unable to find releases');
 
 		foreach($releases as $release) {
 			$this->items[] = array(
+				'author' => $release->find('.author', 0)->plaintext,
 				'uri' => $release->find('a', 0)->href,
-				'title' => 'Release ' . $release->find('h3', 0)->plaintext,
+				'title' => 'Release ' . $release->find('h4', 0)->plaintext,
+				'timestamp' => $release->find('.time-since', 0)->title,
 			);
 		}
 	}
+
+	protected function collectTagsData($html) {
+		$tags = $html->find('table#tags-table > tbody > tr')
+			or returnServerError('Unable to find tags');
+
+		foreach($tags as $tag) {
+			$this->items[] = array(
+				'uri' => $tag->find('a', 0)->href,
+				'title' => 'Tag ' . $tag->find('.release-tag-name > a', 0)->plaintext,
+			);
+		}
+	}
+
+	protected function collectCommitsData($html) {
+		$commits = $html->find('#commits-table tbody tr')
+			or returnServerError('Unable to find commits');
+
+		foreach($commits as $commit) {
+			$this->items[] = array(
+				'uri' => $commit->find('a.sha', 0)->href,
+				'title' => $commit->find('.message span', 0)->plaintext,
+				'author' => $commit->find('.author', 0)->plaintext,
+				'timestamp' => $commit->find('.time-since', 0)->title,
+				'uid' => $commit->find('.sha', 0)->plaintext,
+			);
+		}
+	}
+
+	protected function collectIssuesData($html) {
+		$issues = $html->find('.issue.list li')
+			or returnServerError('Unable to find issues');
+
+		foreach($issues as $issue) {
+			$uri = $issue->find('a', 0)->href;
+
+			$item = array(
+				'uri' => $uri,
+				'title' => trim($issue->find('a.index', 0)->plaintext) . ' | ' . $issue->find('a.title', 0)->plaintext,
+				'author' => $issue->find('.desc a', 1)->plaintext,
+				'timestamp' => $issue->find('.time-since', 0)->title,
+			);
+
+			if($this->getInput('include_description')) {
+				$issue_html = getSimpleHTMLDOMCached($uri, 3600)
+					or returnServerError('Unable to load issue description');
+
+				$issue_html = defaultLinkTo($issue_html, $uri);
+
+				$item['content'] = $issue_html->find('.comment .markup', 0);
+			}
+
+			$this->items[] = $item;
+		}
+	}
+
+	protected function collectSingleIssueOrPrData($html) {
+		$comments = $html->find('.comment')
+			or returnServerError('Unable to find comments');
+
+		foreach($comments as $comment) {
+			if (strpos($comment->getAttribute('class'), 'form') !== false
+				|| strpos($comment->getAttribute('class'), 'merge') !== false
+			) {
+				// Ignore comment form and merge information
+				continue;
+			}
+			$commentLink = $comment->find('a[href*="#issue"]', 0);
+			$item = array(
+				'author' => $comment->find('a.author', 0)->plaintext,
+				'content' => $comment->find('.render-content', 0),
+			);
+			if ($commentLink !== null) {
+				// Regular comment
+				$item['uri'] = $commentLink->href;
+				$item['title'] = str_replace($commentLink->plaintext, '', $comment->find('span', 0)->plaintext);
+				$item['timestamp'] = $comment->find('.time-since', 0)->title;
+			} else {
+				// Change request comment
+				$item['uri'] = $this->getURI() . '#' . $comment->getAttribute('id');
+				$item['title'] = $comment->find('.comment-header .text', 0)->plaintext;
+			}
+			$this->items[] = $item;
+		}
+
+		$this->items = array_reverse($this->items);
+	}
+
+	protected function collectPullRequestsData($html) {
+		$issues = $html->find('.issue.list li')
+			or returnServerError('Unable to find pull requests');
+
+		foreach($issues as $issue) {
+			$uri = $issue->find('a', 0)->href;
+
+			$item = array(
+				'uri' => $uri,
+				'title' => trim($issue->find('a.index', 0)->plaintext) . ' | ' . $issue->find('a.title', 0)->plaintext,
+				'author' => $issue->find('.desc a', 1)->plaintext,
+				'timestamp' => $issue->find('.time-since', 0)->title,
+			);
+
+			if($this->getInput('include_description')) {
+				$issue_html = getSimpleHTMLDOMCached($uri, 3600)
+					or returnServerError('Unable to load issue description');
+
+				$issue_html = defaultLinkTo($issue_html, $uri);
+
+				$item['content'] = $issue_html->find('.comment .markup', 0);
+			}
+
+			$this->items[] = $item;
+		}
+	}
 }

bridges/GithubIssueBridge.php

@@ -4,7 +4,7 @@ class GithubIssueBridge extends BridgeAbstract {
 	const MAINTAINER = 'Pierre Mazière';
 	const NAME = 'Github Issue';
 	const URI = 'https://github.com/';
-	const CACHE_TIMEOUT = 600; // 10min
+	const CACHE_TIMEOUT = 0; // 10min
 	const DESCRIPTION = 'Returns the issues or comments of an issue of a github project';
 
 	const PARAMETERS = array(
@@ -24,6 +24,11 @@ class GithubIssueBridge extends BridgeAbstract {
 			'c' => array(
 				'name' => 'Show Issues Comments',
 				'type' => 'checkbox'
+			),
+			'q' => array(
+				'name' => 'Search Query',
+				'defaultValue' => 'is:issue is:open sort:updated-desc',
+				'required' => true
 			)
 		),
 		'Issue comments' => array(
@@ -40,7 +45,6 @@ class GithubIssueBridge extends BridgeAbstract {
 	const BRIDGE_OPTIONS = array(0 => 'Project Issues', 1 => 'Issue comments');
 	const URL_PATH = 'issues';
 	const SEARCH_QUERY_PATH = 'issues';
-	const SEARCH_QUERY = '?q=is%3Aissue+sort%3Aupdated-desc';
 
 	public function getName(){
 		$name = $this->getInput('u') . '/' . $this->getInput('p');
@@ -67,7 +71,7 @@ class GithubIssueBridge extends BridgeAbstract {
 			if($this->queriedContext === static::BRIDGE_OPTIONS[1]) {
 				$uri .= static::URL_PATH . '/' . $this->getInput('i');
 			} else {
-				$uri .= static::SEARCH_QUERY_PATH . static::SEARCH_QUERY;
+				$uri .= static::SEARCH_QUERY_PATH . '?q=' . urlencode($this->getInput('q'));
 			}
 			return $uri;
 		}
@@ -128,9 +132,8 @@ class GithubIssueBridge extends BridgeAbstract {
 
 		$author = $comment->find('.author', 0)->plaintext;
 
-		$title .= ' / ' . trim(
-			$comment->find('.timeline-comment-header-text', 0)->plaintext
-		);
+		$header = $comment->find('.timeline-comment-header > h3', 0);
+		$title .= ' / ' . ($header ? $header->plaintext : 'Activity');
 
 		$time = $comment->find('relative-time', 0);
 		if ($time === null) {

bridges/GithubPullRequestBridge.php

@@ -1,5 +1,5 @@
 <?php
-require_once('GithubIssueBridge.php');
+
 class GitHubPullRequestBridge extends GithubIssueBridge {
 	const MAINTAINER = 'Yaman Qalieh';
 	const NAME = 'GitHub Pull Request';
@@ -22,6 +22,11 @@ class GitHubPullRequestBridge extends GithubIssueBridge {
 			'c' => array(
 				'name' => 'Show Pull Request Comments',
 				'type' => 'checkbox'
+			),
+			'q' => array(
+				'name' => 'Search Query',
+				'defaultValue' => 'is:pr is:open sort:created-desc',
+				'required' => true
 			)
 		),
 		'Pull Request comments' => array(
@@ -37,5 +42,4 @@ class GitHubPullRequestBridge extends GithubIssueBridge {
 	const BRIDGE_OPTIONS = array(0 => 'Project Pull Requests', 1 => 'Pull Request comments');
 	const URL_PATH = 'pull';
 	const SEARCH_QUERY_PATH = 'pulls';
-	const SEARCH_QUERY = '?q=is%3Apr+sort%3Acreated-desc';
 }

bridges/GithubTrendingBridge.php

@@ -8,41 +8,39 @@ class GithubTrendingBridge extends BridgeAbstract {
 	const CACHE_TIMEOUT = 43200; // 12hr
 	const DESCRIPTION = 'See what the GitHub community is most excited repos.';
 	const PARAMETERS = array(
+	// If you are changing context and/or parameter names, change them also in getName().
 		'By language' => array(
 			'language' => array(
 				'name' => 'Select language',
 				'type' => 'list',
 				'values' => array(
 					'All languages' => '',
-					'C++' => 'c++',
 					'HTML' => 'html',
-					'Java' => 'java',
-					'JavaScript' => 'javascript',
 					'PHP' => 'php',
 					'Python' => 'python',
-					'Ruby' => 'ruby',
-					'Unknown languages' => 'unknown languages',
-					'1C Enterprise' => '1c enterprise',
+					'Shell' => 'shell',
+					'Unknown languages' => 'unknown',
+					'1C Enterprise' => '1c-enterprise',
 					'4D' => '4d',
 					'ABAP' => 'abap',
 					'ABNF' => 'abnf',
 					'ActionScript' => 'actionscript',
 					'Ada' => 'ada',
-					'Adobe Font Metrics' => 'adobe font metrics',
+					'Adobe Font Metrics' => 'adobe-font-metrics',
 					'Agda' => 'agda',
-					'AGS Script' => 'ags script',
+					'AGS Script' => 'ags-script',
 					'Alloy' => 'alloy',
-					'Alpine Abuild' => 'alpine abuild',
-					'Altium Designer' => 'altium designer',
+					'Alpine Abuild' => 'alpine-abuild',
+					'Altium Designer' => 'altium-designer',
 					'AMPL' => 'ampl',
 					'AngelScript' => 'angelscript',
-					'Ant Build System' => 'ant build system',
+					'Ant Build System' => 'ant-build-system',
 					'ANTLR' => 'antlr',
 					'ApacheConf' => 'apacheconf',
 					'Apex' => 'apex',
-					'API Blueprint' => 'api blueprint',
+					'API Blueprint' => 'api-blueprint',
 					'APL' => 'apl',
-					'Apollo Guidance Computer' => 'apollo guidance computer',
+					'Apollo Guidance Computer' => 'apollo-guidance-computer',
 					'AppleScript' => 'applescript',
 					'Arc' => 'arc',
 					'AsciiDoc' => 'asciidoc',
@@ -71,11 +69,12 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Brightscript' => 'brightscript',
 					'Zeek' => 'zeek',
 					'C' => 'c',
-					'C#' => 'c#',
+					'C#' => 'c%23',		// already URL encoded
 					'C++' => 'c++',
 					'C-ObjDump' => 'c-objdump',
-					'C2hs Haskell' => 'c2hs haskell',
-					'Cabal Config' => 'cabal config',
+					'C2hs Haskell' => 'c2hs-haskell',
+					'Cabal Config' => 'cabal-config',
+					'Cap\'n Proto' => 'cap\'n-proto',
 					'CartoCSS' => 'cartocss',
 					'Ceylon' => 'ceylon',
 					'Chapel' => 'chapel',
@@ -87,18 +86,18 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Click' => 'click',
 					'CLIPS' => 'clips',
 					'Clojure' => 'clojure',
-					'Closure Templates' => 'closure templates',
-					'Cloud Firestore Security Rules' => 'cloud firestore security rules',
+					'Closure Templates' => 'closure-templates',
+					'Cloud Firestore Security Rules' => 'cloud-firestore-security-rules',
 					'CMake' => 'cmake',
 					'COBOL' => 'cobol',
 					'CodeQL' => 'codeql',
 					'CoffeeScript' => 'coffeescript',
 					'ColdFusion' => 'coldfusion',
-					'ColdFusion CFC' => 'coldfusion cfc',
+					'ColdFusion CFC' => 'coldfusion-cfc',
 					'COLLADA' => 'collada',
-					'Common Lisp' => 'common lisp',
-					'Common Workflow Language' => 'common workflow language',
-					'Component Pascal' => 'component pascal',
+					'Common Lisp' => 'common-lisp',
+					'Common Workflow Language' => 'common-workflow-language',
+					'Component Pascal' => 'component-pascal',
 					'CoNLL-U' => 'conll-u',
 					'Cool' => 'cool',
 					'Coq' => 'coq',
@@ -107,28 +106,28 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Crystal' => 'crystal',
 					'CSON' => 'cson',
 					'Csound' => 'csound',
-					'Csound Document' => 'csound document',
-					'Csound Score' => 'csound score',
+					'Csound Document' => 'csound-document',
+					'Csound Score' => 'csound-score',
 					'CSS' => 'css',
 					'CSV' => 'csv',
 					'Cuda' => 'cuda',
-					'cURL Config' => 'curl config',
+					'cURL Config' => 'curl-config',
 					'CWeb' => 'cweb',
 					'Cycript' => 'cycript',
 					'Cython' => 'cython',
 					'D' => 'd',
 					'D-ObjDump' => 'd-objdump',
-					'Darcs Patch' => 'darcs patch',
+					'Darcs Patch' => 'darcs-patch',
 					'Dart' => 'dart',
 					'DataWeave' => 'dataweave',
 					'desktop' => 'desktop',
 					'Dhall' => 'dhall',
 					'Diff' => 'diff',
-					'DIGITAL Command Language' => 'digital command language',
+					'DIGITAL Command Language' => 'digital-command-language',
 					'dircolors' => 'dircolors',
-					'DirectX 3D File' => 'directx 3d file',
+					'DirectX 3D File' => 'directx-3d-file',
 					'DM' => 'dm',
-					'DNS Zone' => 'dns zone',
+					'DNS Zone' => 'dns-zone',
 					'Dockerfile' => 'dockerfile',
 					'Dogescript' => 'dogescript',
 					'DTrace' => 'dtrace',
@@ -138,29 +137,29 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Easybuild' => 'easybuild',
 					'EBNF' => 'ebnf',
 					'eC' => 'ec',
-					'Ecere Projects' => 'ecere projects',
+					'Ecere Projects' => 'ecere-projects',
 					'ECL' => 'ecl',
 					'ECLiPSe' => 'eclipse',
 					'EditorConfig' => 'editorconfig',
-					'Edje Data Collection' => 'edje data collection',
+					'Edje Data Collection' => 'edje-data-collection',
 					'edn' => 'edn',
 					'Eiffel' => 'eiffel',
 					'EJS' => 'ejs',
 					'Elixir' => 'elixir',
 					'Elm' => 'elm',
-					'Emacs Lisp' => 'emacs lisp',
+					'Emacs Lisp' => 'emacs-lisp',
 					'EmberScript' => 'emberscript',
 					'EML' => 'eml',
 					'EQ' => 'eq',
 					'Erlang' => 'erlang',
-					'F#' => 'f#',
+					'F#' => 'f%23',		// already URL encoded
 					'F*' => 'f*',
 					'Factor' => 'factor',
 					'Fancy' => 'fancy',
 					'Fantom' => 'fantom',
 					'Faust' => 'faust',
-					'FIGlet Font' => 'figlet font',
-					'Filebench WML' => 'filebench wml',
+					'FIGlet Font' => 'figlet-font',
+					'Filebench WML' => 'filebench-wml',
 					'Filterscript' => 'filterscript',
 					'fish' => 'fish',
 					'FLUX' => 'flux',
@@ -170,25 +169,25 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'FreeMarker' => 'freemarker',
 					'Frege' => 'frege',
 					'G-code' => 'g-code',
-					'Game Maker Language' => 'game maker language',
+					'Game Maker Language' => 'game-maker-language',
 					'GAML' => 'gaml',
 					'GAMS' => 'gams',
 					'GAP' => 'gap',
-					'GCC Machine Description' => 'gcc machine description',
+					'GCC Machine Description' => 'gcc-machine-description',
 					'GDB' => 'gdb',
 					'GDScript' => 'gdscript',
 					'Genie' => 'genie',
 					'Genshi' => 'genshi',
-					'Gentoo Ebuild' => 'gentoo ebuild',
-					'Gentoo Eclass' => 'gentoo eclass',
-					'Gerber Image' => 'gerber image',
-					'Gettext Catalog' => 'gettext catalog',
+					'Gentoo Ebuild' => 'gentoo-ebuild',
+					'Gentoo Eclass' => 'gentoo-eclass',
+					'Gerber Image' => 'gerber-image',
+					'Gettext Catalog' => 'gettext-catalog',
 					'Gherkin' => 'gherkin',
-					'Git Attributes' => 'git attributes',
-					'Git Config' => 'git config',
+					'Git Attributes' => 'git-attributes',
+					'Git Config' => 'git-config',
 					'GLSL' => 'glsl',
 					'Glyph' => 'glyph',
-					'Glyph Bitmap Distribution Format' => 'glyph bitmap distribution format',
+					'Glyph Bitmap Distribution Format' => 'glyph-bitmap-distribution-format',
 					'GN' => 'gn',
 					'Gnuplot' => 'gnuplot',
 					'Go' => 'go',
@@ -196,12 +195,12 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Gosu' => 'gosu',
 					'Grace' => 'grace',
 					'Gradle' => 'gradle',
-					'Grammatical Framework' => 'grammatical framework',
-					'Graph Modeling Language' => 'graph modeling language',
+					'Grammatical Framework' => 'grammatical-framework',
+					'Graph Modeling Language' => 'graph-modeling-language',
 					'GraphQL' => 'graphql',
-					'Graphviz (DOT)' => 'graphviz (dot)',
+					'Graphviz (DOT)' => 'graphviz-(dot)',
 					'Groovy' => 'groovy',
-					'Groovy Server Pages' => 'groovy server pages',
+					'Groovy Server Pages' => 'groovy-server-pages',
 					'Hack' => 'hack',
 					'Haml' => 'haml',
 					'Handlebars' => 'handlebars',
@@ -213,7 +212,6 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'HiveQL' => 'hiveql',
 					'HLSL' => 'hlsl',
 					'HolyC' => 'holyc',
-					'HTML' => 'html',
 					'HTML+Django' => 'html+django',
 					'HTML+ECR' => 'html+ecr',
 					'HTML+EEX' => 'html+eex',
@@ -226,39 +224,39 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'HyPhy' => 'hyphy',
 					'IDL' => 'idl',
 					'Idris' => 'idris',
-					'Ignore List' => 'ignore list',
-					'IGOR Pro' => 'igor pro',
-					'Inform 7' => 'inform 7',
+					'Ignore List' => 'ignore-list',
+					'IGOR Pro' => 'igor-pro',
+					'Inform 7' => 'inform-7',
 					'INI' => 'ini',
-					'Inno Setup' => 'inno setup',
+					'Inno Setup' => 'inno-setup',
 					'Io' => 'io',
 					'Ioke' => 'ioke',
-					'IRC log' => 'irc log',
+					'IRC log' => 'irc-log',
 					'Isabelle' => 'isabelle',
-					'Isabelle ROOT' => 'isabelle root',
+					'Isabelle ROOT' => 'isabelle-root',
 					'J' => 'j',
 					'Jasmin' => 'jasmin',
 					'Java' => 'java',
-					'Java Properties' => 'java properties',
-					'Java Server Pages' => 'java server pages',
+					'Java Properties' => 'java-properties',
+					'Java Server Pages' => 'java-server-pages',
 					'JavaScript' => 'javascript',
 					'JavaScript+ERB' => 'javascript+erb',
 					'JFlex' => 'jflex',
 					'Jison' => 'jison',
-					'Jison Lex' => 'jison lex',
+					'Jison Lex' => 'jison-lex',
 					'Jolie' => 'jolie',
 					'JSON' => 'json',
-					'JSON with Comments' => 'json with comments',
+					'JSON with Comments' => 'json-with-comments',
 					'JSON5' => 'json5',
 					'JSONiq' => 'jsoniq',
 					'JSONLD' => 'jsonld',
 					'Jsonnet' => 'jsonnet',
 					'JSX' => 'jsx',
 					'Julia' => 'julia',
-					'Jupyter Notebook' => 'jupyter notebook',
-					'KiCad Layout' => 'kicad layout',
-					'KiCad Legacy Layout' => 'kicad legacy layout',
-					'KiCad Schematic' => 'kicad schematic',
+					'Jupyter Notebook' => 'jupyter-notebook',
+					'KiCad Layout' => 'kicad-layout',
+					'KiCad Legacy Layout' => 'kicad-legacy-layout',
+					'KiCad Schematic' => 'kicad-schematic',
 					'Kit' => 'kit',
 					'Kotlin' => 'kotlin',
 					'KRL' => 'krl',
@@ -271,12 +269,12 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'LFE' => 'lfe',
 					'LilyPond' => 'lilypond',
 					'Limbo' => 'limbo',
-					'Linker Script' => 'linker script',
-					'Linux Kernel Module' => 'linux kernel module',
+					'Linker Script' => 'linker-script',
+					'Linux Kernel Module' => 'linux-kernel-module',
 					'Liquid' => 'liquid',
-					'Literate Agda' => 'literate agda',
-					'Literate CoffeeScript' => 'literate coffeescript',
-					'Literate Haskell' => 'literate haskell',
+					'Literate Agda' => 'literate-agda',
+					'Literate CoffeeScript' => 'literate-coffeescript',
+					'Literate Haskell' => 'literate-haskell',
 					'LiveScript' => 'livescript',
 					'LLVM' => 'llvm',
 					'Logos' => 'logos',
@@ -285,7 +283,7 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'LookML' => 'lookml',
 					'LoomScript' => 'loomscript',
 					'LSL' => 'lsl',
-					'LTspice Symbol' => 'ltspice symbol',
+					'LTspice Symbol' => 'ltspice-symbol',
 					'Lua' => 'lua',
 					'M' => 'm',
 					'M4' => 'm4',
@@ -297,7 +295,7 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Mask' => 'mask',
 					'Mathematica' => 'mathematica',
 					'MATLAB' => 'matlab',
-					'Maven POM' => 'maven pom',
+					'Maven POM' => 'maven-pom',
 					'Max' => 'max',
 					'MAXScript' => 'maxscript',
 					'mcfunction' => 'mcfunction',
@@ -305,19 +303,19 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Mercury' => 'mercury',
 					'Meson' => 'meson',
 					'Metal' => 'metal',
-					'Microsoft Developer Studio Project' => 'microsoft developer studio project',
+					'Microsoft Developer Studio Project' => 'microsoft-developer-studio-project',
 					'MiniD' => 'minid',
 					'Mirah' => 'mirah',
-					'mIRC Script' => 'mirc script',
+					'mIRC Script' => 'mirc-script',
 					'MLIR' => 'mlir',
 					'Modelica' => 'modelica',
 					'Modula-2' => 'modula-2',
 					'Modula-3' => 'modula-3',
-					'Module Management System' => 'module management system',
+					'Module Management System' => 'module-management-system',
 					'Monkey' => 'monkey',
 					'Moocode' => 'moocode',
 					'MoonScript' => 'moonscript',
-					'Motorola 68K Assembly' => 'motorola 68k assembly',
+					'Motorola 68K Assembly' => 'motorola-68k-assembly',
 					'MQL4' => 'mql4',
 					'MQL5' => 'mql5',
 					'MTML' => 'mtml',
@@ -342,12 +340,12 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Nit' => 'nit',
 					'Nix' => 'nix',
 					'NL' => 'nl',
-					'NPM Config' => 'npm config',
+					'NPM Config' => 'npm-config',
 					'NSIS' => 'nsis',
 					'Nu' => 'nu',
 					'NumPy' => 'numpy',
 					'ObjDump' => 'objdump',
-					'Object Data Instance Notation' => 'object data instance notation',
+					'Object Data Instance Notation' => 'object-data-instance-notation',
 					'Objective-C' => 'objective-c',
 					'Objective-C++' => 'objective-c++',
 					'Objective-J' => 'objective-j',
@@ -358,14 +356,14 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'ooc' => 'ooc',
 					'Opa' => 'opa',
 					'Opal' => 'opal',
-					'Open Policy Agent' => 'open policy agent',
+					'Open Policy Agent' => 'open-policy-agent',
 					'OpenCL' => 'opencl',
-					'OpenEdge ABL' => 'openedge abl',
+					'OpenEdge ABL' => 'openedge-abl',
 					'OpenQASM' => 'openqasm',
-					'OpenRC runscript' => 'openrc runscript',
+					'OpenRC runscript' => 'openrc-runscript',
 					'OpenSCAD' => 'openscad',
-					'OpenStep Property List' => 'openstep property list',
-					'OpenType Feature File' => 'opentype feature file',
+					'OpenStep Property List' => 'openstep-property-list',
+					'OpenType Feature File' => 'opentype-feature-file',
 					'Org' => 'org',
 					'Ox' => 'ox',
 					'Oxygene' => 'oxygene',
@@ -374,13 +372,12 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Pan' => 'pan',
 					'Papyrus' => 'papyrus',
 					'Parrot' => 'parrot',
-					'Parrot Assembly' => 'parrot assembly',
-					'Parrot Internal Representation' => 'parrot internal representation',
+					'Parrot Assembly' => 'parrot-assembly',
+					'Parrot Internal Representation' => 'parrot-internal-representation',
 					'Pascal' => 'pascal',
 					'Pawn' => 'pawn',
 					'Pep8' => 'pep8',
 					'Perl' => 'perl',
-					'PHP' => 'php',
 					'Pic' => 'pic',
 					'Pickle' => 'pickle',
 					'PicoLisp' => 'picolisp',
@@ -389,29 +386,28 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'PLpgSQL' => 'plpgsql',
 					'PLSQL' => 'plsql',
 					'Pod' => 'pod',
-					'Pod 6' => 'pod 6',
+					'Pod 6' => 'pod-6',
 					'PogoScript' => 'pogoscript',
 					'Pony' => 'pony',
 					'PostCSS' => 'postcss',
 					'PostScript' => 'postscript',
-					'POV-Ray SDL' => 'pov-ray sdl',
+					'POV-Ray SDL' => 'pov-ray-sdl',
 					'PowerBuilder' => 'powerbuilder',
 					'PowerShell' => 'powershell',
 					'Prisma' => 'prisma',
 					'Processing' => 'processing',
 					'Proguard' => 'proguard',
 					'Prolog' => 'prolog',
-					'Propeller Spin' => 'propeller spin',
-					'Protocol Buffer' => 'protocol buffer',
-					'Public Key' => 'public key',
+					'Propeller Spin' => 'propeller-spin',
+					'Protocol Buffer' => 'protocol-buffer',
+					'Public Key' => 'public-key',
 					'Pug' => 'pug',
 					'Puppet' => 'puppet',
-					'Pure Data' => 'pure data',
+					'Pure Data' => 'pure-data',
 					'PureBasic' => 'purebasic',
 					'PureScript' => 'purescript',
-					'Python' => 'python',
-					'Python console' => 'python console',
-					'Python traceback' => 'python traceback',
+					'Python console' => 'python-console',
+					'Python traceback' => 'python-traceback',
 					'q' => 'q',
 					'QMake' => 'qmake',
 					'QML' => 'qml',
@@ -422,30 +418,30 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Raku' => 'raku',
 					'RAML' => 'raml',
 					'Rascal' => 'rascal',
-					'Raw token data' => 'raw token data',
+					'Raw token data' => 'raw-token-data',
 					'RDoc' => 'rdoc',
-					'Readline Config' => 'readline config',
+					'Readline Config' => 'readline-config',
 					'REALbasic' => 'realbasic',
 					'Reason' => 'reason',
 					'Rebol' => 'rebol',
 					'Red' => 'red',
 					'Redcode' => 'redcode',
-					'Regular Expression' => 'regular expression',
-					// 'Ren'Py' => 'ren'py',
+					'Regular Expression' => 'regular-expression',
+					'Ren\'Py' => 'ren\'py',
 					'RenderScript' => 'renderscript',
 					'reStructuredText' => 'restructuredtext',
 					'REXX' => 'rexx',
 					'RHTML' => 'rhtml',
-					'Rich Text Format' => 'rich text format',
+					'Rich Text Format' => 'rich-text-format',
 					'Ring' => 'ring',
 					'Riot' => 'riot',
 					'RMarkdown' => 'rmarkdown',
 					'RobotFramework' => 'robotframework',
 					'Roff' => 'roff',
-					'Roff Manpage' => 'roff manpage',
+					'Roff Manpage' => 'roff-manpage',
 					'Rouge' => 'rouge',
 					'RPC' => 'rpc',
-					'RPM Spec' => 'rpm spec',
+					'RPM Spec' => 'rpm-spec',
 					'Ruby' => 'ruby',
 					'RUNOFF' => 'runoff',
 					'Rust' => 'rust',
@@ -461,7 +457,6 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'sed' => 'sed',
 					'Self' => 'self',
 					'ShaderLab' => 'shaderlab',
-					'Shell' => 'shell',
 					'ShellSession' => 'shellsession',
 					'Shen' => 'shen',
 					'Slash' => 'slash',
@@ -475,20 +470,20 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Solidity' => 'solidity',
 					'SourcePawn' => 'sourcepawn',
 					'SPARQL' => 'sparql',
-					'Spline Font Database' => 'spline font database',
+					'Spline Font Database' => 'spline-font-database',
 					'SQF' => 'sqf',
 					'SQL' => 'sql',
 					'SQLPL' => 'sqlpl',
 					'Squirrel' => 'squirrel',
-					'SRecode Template' => 'srecode template',
-					'SSH Config' => 'ssh config',
+					'SRecode Template' => 'srecode-template',
+					'SSH Config' => 'ssh-config',
 					'Stan' => 'stan',
-					'Standard ML' => 'standard ml',
+					'Standard ML' => 'standard-ml',
 					'Starlark' => 'starlark',
 					'Stata' => 'stata',
 					'STON' => 'ston',
 					'Stylus' => 'stylus',
-					'SubRip Text' => 'subrip text',
+					'SubRip Text' => 'subrip-text',
 					'SugarSS' => 'sugarss',
 					'SuperCollider' => 'supercollider',
 					'Svelte' => 'svelte',
@@ -505,7 +500,7 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Text' => 'text',
 					'Textile' => 'textile',
 					'Thrift' => 'thrift',
-					'TI Program' => 'ti program',
+					'TI Program' => 'ti-program',
 					'TLA' => 'tla',
 					'TOML' => 'toml',
 					'TSQL' => 'tsql',
@@ -514,11 +509,11 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'Turtle' => 'turtle',
 					'Twig' => 'twig',
 					'TXL' => 'txl',
-					'Type Language' => 'type language',
+					'Type Language' => 'type-language',
 					'TypeScript' => 'typescript',
-					'Unified Parallel C' => 'unified parallel c',
-					'Unity3D Asset' => 'unity3d asset',
-					'Unix Assembly' => 'unix assembly',
+					'Unified Parallel C' => 'unified-parallel-c',
+					'Unity3D Asset' => 'unity3d-asset',
+					'Unix Assembly' => 'unix-assembly',
 					'Uno' => 'uno',
 					'UnrealScript' => 'unrealscript',
 					'UrWeb' => 'urweb',
@@ -529,33 +524,33 @@ class GithubTrendingBridge extends BridgeAbstract {
 					'VCL' => 'vcl',
 					'Verilog' => 'verilog',
 					'VHDL' => 'vhdl',
-					'Vim script' => 'vim script',
-					'Vim Snippet' => 'vim snippet',
-					'Visual Basic .NET' => 'visual basic .net',
-					'Visual Basic .NET' => 'visual basic .net',
+					'Vim script' => 'vim-script',
+					'Vim Snippet' => 'vim-snippet',
+					'Visual Basic .NET' => 'visual-basic-.net',
+					'Visual Basic .NET' => 'visual-basic-.net',
 					'Volt' => 'volt',
 					'Vue' => 'vue',
-					'Wavefront Material' => 'wavefront material',
-					'Wavefront Object' => 'wavefront object',
+					'Wavefront Material' => 'wavefront-material',
+					'Wavefront Object' => 'wavefront-object',
 					'wdl' => 'wdl',
-					'Web Ontology Language' => 'web ontology language',
+					'Web Ontology Language' => 'web-ontology-language',
 					'WebAssembly' => 'webassembly',
 					'WebIDL' => 'webidl',
 					'WebVTT' => 'webvtt',
-					'Wget Config' => 'wget config',
-					'Windows Registry Entries' => 'windows registry entries',
+					'Wget Config' => 'wget-config',
+					'Windows Registry Entries' => 'windows-registry-entries',
 					'wisp' => 'wisp',
 					'Wollok' => 'wollok',
-					'World of Warcraft Addon Data' => 'world of warcraft addon data',
-					'X BitMap' => 'x bitmap',
-					'X Font Directory Index' => 'x font directory index',
-					'X PixMap' => 'x pixmap',
+					'World of Warcraft Addon Data' => 'world-of-warcraft-addon-data',
+					'X BitMap' => 'x-bitmap',
+					'X Font Directory Index' => 'x-font-directory-index',
+					'X PixMap' => 'x-pixmap',
 					'X10' => 'x10',
 					'xBase' => 'xbase',
 					'XC' => 'xc',
 					'XCompose' => 'xcompose',
 					'XML' => 'xml',
-					'XML Property List' => 'xml property list',
+					'XML Property List' => 'xml-property-list',
 					'Xojo' => 'xojo',
 					'XPages' => 'xpages',
 					'XProc' => 'xproc',
@@ -612,7 +607,9 @@ class GithubTrendingBridge extends BridgeAbstract {
 			$item['title'] = str_replace('  ', '', trim(strip_tags($element->find('h1 a', 0)->plaintext)));
 
 			// Description
-			$item['content'] = trim(strip_tags($element->find('p', 0)->innertext));
+			$description = $element->find('p', 0);
+			if ($description != null)
+				$item['content'] = trim(strip_tags($description->innertext));
 
 			// Time
 			$item['timestamp'] = time();
@@ -623,10 +620,9 @@ class GithubTrendingBridge extends BridgeAbstract {
 	}
 
 	public function getName(){
-		if($this->getInput('language') == '') {
-			return self::NAME . ': all';
-		} elseif (!is_null($this->getInput('language'))) {
-			return self::NAME . ': ' . $this->getInput('language');
+		if (!is_null($this->getInput('language'))) {
+			$language = array_search($this->getInput('language'), self::PARAMETERS['By language']['language']['values']);
+			return self::NAME . ': ' . $language;
 		}
 
 		return parent::getName();

bridges/GitlabIssueBridge.php

@@ -0,0 +1,205 @@
+<?php
+class GitlabIssueBridge extends BridgeAbstract {
+
+	const MAINTAINER = 'Mynacol';
+	const NAME = 'Gitlab Issue/Merge Request';
+	const URI = 'https://gitlab.com/';
+	const CACHE_TIMEOUT = 1800; // 30min
+	const DESCRIPTION = 'Returns  comments of an issue/MR of a gitlab project';
+
+	const PARAMETERS = array(
+		'global' => array(
+			'h' => array(
+				'name' => 'Gitlab instance host name',
+				'exampleValue' => 'gitlab.com',
+				'defaultValue' => 'gitlab.com',
+				'required' => true
+			),
+			'u' => array(
+				'name' => 'User/Organization name',
+				'exampleValue' => 'fdroid',
+				'required' => true
+			),
+			'p' => array(
+				'name' => 'Project name',
+				'exampleValue' => 'fdroidclient',
+				'required' => true
+			)
+
+		),
+		'Issue comments' => array(
+			'i' => array(
+				'name' => 'Issue number',
+				'type' => 'number',
+				'exampleValue' => '2099',
+				'required' => true
+			)
+		),
+		'Merge Request comments' => array(
+			'i' => array(
+				'name' => 'Merge Request number',
+				'type' => 'number',
+				'exampleValue' => '2099',
+				'required' => true
+			)
+		)
+	);
+
+	public function getName(){
+		$name = $this->getInput('h') . '/' . $this->getInput('u') . '/' . $this->getInput('p');
+		switch ($this->queriedContext) {
+			case 'Issue comments':
+				$name .= ' Issue #' . $this->getInput('i');
+				break;
+			case 'Merge Request comments':
+				$name .= ' MR !' . $this->getInput('i');
+				break;
+			default:
+				return parent::getName();
+		}
+		return $name;
+	}
+
+	public function getURI() {
+		$host = $this->getInput('h') ?? 'gitlab.com';
+		$uri = 'https://' . $host . '/' . $this->getInput('u') . '/'
+			 . $this->getInput('p') . '/';
+		switch ($this->queriedContext) {
+			case 'Issue comments':
+				$uri .= '-/issues';
+				break;
+			case 'Merge Request comments':
+				$uri .= '-/merge_requests';
+				break;
+			default:
+				return $uri;
+		}
+		$uri .= '/' . $this->getInput('i');
+		return $uri;
+	}
+
+	public function getIcon() {
+		return 'https://' . $this->getInput('h') . '/favicon.ico';
+	}
+
+	public function collectData() {
+		switch ($this->queriedContext) {
+			case 'Issue comments':
+				$this->items[] = $this->parseIssueDescription();
+				break;
+			case 'Merge Request comments':
+				$this->items[] = $this->parseMergeRequestDescription();
+				break;
+			default:
+				break;
+		}
+
+		/* parse issue/MR comments */
+		$comments_uri = $this->getURI() . '/discussions.json';
+		$comments = getContents($comments_uri);
+		$comments = json_decode($comments, false);
+
+		foreach ($comments as $value) {
+			foreach ($value->notes as $comment) {
+				$item = array();
+				$item['uri'] = $comment->noteable_note_url;
+				$item['uid'] = $item['uri'];
+
+				// TODO fix invalid timestamps (fdroid bot)
+				$item['timestamp'] = $comment->created_at ?? $comment->updated_at ?? $comment->last_edited_at;
+				$author = $comment->author ?? $comment->last_edited_by;
+				$item['author'] = '<img src="' . $author->avatar_url . '" width=24></img> <a href="https://' .
+					$this->getInput('h') . $author->path . '">' . $author->name . ' @' . $author->username . '</a>';
+
+				$content = '';
+				if ($comment->system) {
+					$content = $comment->note_html;
+					if ($comment->type === 'StateNote') {
+						$content .= ' the issue';
+					} elseif ($comment->type === null) {
+						// e.g. "added 900 commits\n800 from master\n175h4d - commit message\n..."
+						$content = str_get_html($comment->note_html)->find('p', 0);
+					}
+				} else {
+					// no switch-case to do strict comparison
+					if ($comment->type === null || $comment->type === 'DiscussionNote') {
+						$content = 'commented';
+					} elseif ($comment->type === 'DiffNote') {
+						$content = 'commented on a thread';
+					} else {
+						$content = $comment->note_html;
+					}
+				}
+				$item['title'] = $author->name . " $content";
+
+				$content = $this->fixImgSrc($comment->note_html);
+				$item['content'] = defaultLinkTo($content, 'https://' . $this->getInput('h') . '/');
+
+				$this->items[] = $item;
+			}
+		}
+	}
+
+	private function parseIssueDescription() {
+		$description_uri = $this->getURI() . '.json';
+		$description = getContents($description_uri);
+		$description = json_decode($description, false);
+		$description_html = getSimpleHtmlDomCached($this->getURI());
+
+		$item = array();
+		$item['uri'] = $this->getURI();
+		$item['uid'] = $item['uri'];
+
+		$item['timestamp'] = $description->created_at ?? $description->updated_at;
+
+		$item['author'] = $this->parseAuthor($description_html);
+
+		$item['title'] = $description->title;
+		$item['content'] = markdownToHtml($description->description);
+
+		return $item;
+	}
+
+	private function parseMergeRequestDescription() {
+		$description_uri = $this->getURI() . '/cached_widget.json';
+		$description = getContents($description_uri);
+		$description = json_decode($description, false);
+		$description_html = getSimpleHtmlDomCached($this->getURI());
+
+		$item = array();
+		$item['uri'] = $this->getURI();
+		$item['uid'] = $item['uri'];
+
+		$item['timestamp'] = $description_html->find('.merge-request-details time', 0)->datetime;
+
+		$item['author'] = $this->parseAuthor($description_html);
+
+		$item['title'] = 'Merge Request ' . $description->title;
+		$item['content'] = markdownToHtml($description->description);
+
+		return $item;
+	}
+
+	private function fixImgSrc($html) {
+		if (is_string($html)) {
+			$html = str_get_html($html);
+		}
+
+		foreach ($html->find('img') as $img) {
+			$img->src = $img->getAttribute('data-src');
+		}
+		return $html;
+	}
+
+	private function parseAuthor($description_html) {
+		$description_html = $this->fixImgSrc($description_html);
+
+		$authors = $description_html->find('.issuable-meta a.author-link, .merge-request a.author-link');
+		$editors = $description_html->find('.edited-text a.author-link');
+		$author_str = implode(' ', $authors);
+		if ($editors) {
+			$author_str .= ', ' . implode(' ', $editors);
+		}
+		return defaultLinkTo($author_str, 'https://' . $this->getInput('h') . '/');
+	}
+}

bridges/GlassdoorBridge.php

@@ -1,4 +1,5 @@
 <?php
+
 class GlassdoorBridge extends BridgeAbstract {
 
 	// Contexts
@@ -17,7 +18,6 @@ class GlassdoorBridge extends BridgeAbstract {
 	const BLOG_TYPE_COMPANIES_HIRING = 'Companies Hiring';
 	const BLOG_TYPE_CAREER_ADVICE	 = 'Career Advice';
 	const BLOG_TYPE_INTERVIEWS		 = 'Interviews';
-	const BLOG_TYPE_GUIDE			 = 'Guides';
 
 	// Review context parameters
 	const PARAM_REVIEW_COMPANY = 'company';
@@ -39,7 +39,6 @@ class GlassdoorBridge extends BridgeAbstract {
 					self::BLOG_TYPE_COMPANIES_HIRING	=> 'blog/companies-hiring/',
 					self::BLOG_TYPE_CAREER_ADVICE		=> 'blog/career-advice/',
 					self::BLOG_TYPE_INTERVIEWS			=> 'blog/interviews/',
-					self::BLOG_TYPE_GUIDE				=> 'blog/guide/'
 				)
 			),
 			self::PARAM_BLOG_FULL => array(
@@ -67,9 +66,6 @@ class GlassdoorBridge extends BridgeAbstract {
 		)
 	);
 
-	private $host = self::URI; // They redirect without notice :/
-	private $title = '';
-
 	public function getURI() {
 		switch($this->queriedContext) {
 			case self::CONTEXT_BLOG:
@@ -81,18 +77,10 @@ class GlassdoorBridge extends BridgeAbstract {
 		return parent::getURI();
 	}
 
-	public function getName() {
-		return $this->title ? $this->title . ' - ' . self::NAME : parent::getName();
-	}
-
 	public function collectData() {
-		$html = getSimpleHTMLDOM($this->getURI());
-
-		$this->host = $html->find('link[rel="canonical"]', 0)->href;
-
-		$html = defaultLinkTo($html, $this->host);
-
-		$this->title = $html->find('meta[property="og:title"]', 0)->content;
+		$url = $this->getURI();
+		$html = getSimpleHTMLDOM($url);
+		$html = defaultLinkTo($html, $url);
 		$limit = $this->getInput(self::PARAM_LIMIT);
 
 		switch($this->queriedContext) {
@@ -106,35 +94,24 @@ class GlassdoorBridge extends BridgeAbstract {
 	}
 
 	private function collectBlogData($html, $limit) {
-		$posts = $html->find('section')
+		$posts = $html->find('div.post')
 			or returnServerError('Unable to find blog posts!');
 
 		foreach($posts as $post) {
-			$item = array();
+			$item = [];
 
-			$item['uri'] = $post->find('header a', 0)->href;
-			$item['title'] = $post->find('header', 0)->plaintext;
-			$item['content'] = $post->find('div[class="excerpt-content"]', 0)->plaintext;
-			$item['enclosures'] = array(
-				$this->getFullSizeImageURI($post->find('div[class*="post-thumb"]', 0)->{'data-original'})
-			);
-
-			// optionally load full articles
-			if($this->getInput(self::PARAM_BLOG_FULL)) {
-				$full_html = getSimpleHTMLDOMCached($item['uri']);
-
-				$full_html = defaultLinkTo($full_html, $this->host);
-
-				$item['author'] = $full_html->find('a[rel="author"]', 0);
-				$item['content'] = $full_html->find('article', 0);
-				$item['timestamp'] = strtotime($full_html->find('time.updated', 0)->datetime);
-				$item['categories'] = $full_html->find('span[class="post_tag"]');
-			}
+			$item['uri'] = $post->find('a', 0)->href;
+			$item['title'] = $post->find('h3', 0)->plaintext;
+			$item['content'] = $post->find('p', 0)->plaintext;
+			$item['author'] = $post->find('p', -2)->plaintext;
+			$item['timestamp'] = strtotime($post->find('p', -1)->plaintext);
 
+			// TODO: fetch entire blog post content
 			$this->items[] = $item;
 
-			if($limit > 0 && count($this->items) >= $limit)
+			if ($limit > 0 && count($this->items) >= $limit) {
 				return;
+			}
 		}
 	}
 
@@ -143,53 +120,32 @@ class GlassdoorBridge extends BridgeAbstract {
 			or returnServerError('Unable to find reviews!');
 
 		foreach($reviews as $review) {
-			$item = array();
+			$item = [];
 
 			$item['uri'] = $review->find('a.reviewLink', 0)->href;
-			$item['title'] = $review->find('[class="summary"]', 0)->plaintext;
-			$item['author'] = $review->find('div.author span', 0)->plaintext;
-			$item['timestamp'] = strtotime($review->find('time', 0)->datetime);
 
-			$mainText = $review->find('p.mainText', 0)->plaintext;
+			// Not all reviews have a title
+			$item['title'] = $review->find('h2', 0)->plaintext ?? 'Glassdoor review';
 
-			$description = '';
-			foreach($review->find('div.description p') as $p) {
+			[$date, $author] = explode('-', $review->find('span.authorInfo', 0)->plaintext);
 
-				if ($p->hasClass('strong')) {
-					$p->tag = 'strong';
-					$p->removeClass('strong');
-				}
-
-				$description .= $p;
+			$item['author'] = trim($author);
 
+			$createdAt = DateTimeImmutable::createFromFormat('F m, Y', trim($date));
+			if ($createdAt) {
+				$item['timestamp'] = $createdAt->getTimestamp();
 			}
 
-			$item['content'] = "<p>{$mainText}</p><p>{$description}</p>";
+			$item['content'] = $review->find('.px-std', 2)->text();
 
 			$this->items[] = $item;
 
-			if($limit > 0 && count($this->items) >= $limit)
+			if($limit > 0 && count($this->items) >= $limit) {
 				return;
+			}
 		}
 	}
 
-	private function getFullSizeImageURI($uri) {
-		/* Images are scaled for display on the website. The scaling takes place
-		 * on the host, who provides images in different sizes.
-		 *
-		 * For example:
-		 * https://www.glassdoor.com/blog/app/uploads/sites/2/GettyImages-982402074-e1538092065712-390x193.jpg
-		 *
-		 * By removing the size information we receive the full sized image.
-		 *
-		 * For example:
-		 * https://www.glassdoor.com/blog/app/uploads/sites/2/GettyImages-982402074-e1538092065712.jpg
-		 */
-
-		$uri = filter_var($uri, FILTER_SANITIZE_URL);
-		return preg_replace('/(.*)(\-\d+x\d+)(\.jpg)/', '$1$3', $uri);
-	}
-
 	private function filterCompanyURI($uri) {
 		/* Make sure the URI is a valid review page. Unfortunately there is no
 		 * simple way to determine if the URI is valid, because of automagic

bridges/GogsBridge.php

@@ -11,9 +11,9 @@ class GogsBridge extends BridgeAbstract {
 		'global' => array(
 			'host' => array(
 				'name' => 'Host',
-				'exampleValue' => 'notabug.org',
+				'exampleValue' => 'https://notabug.org',
 				'required' => true,
-				'title' => 'Host name without trailing slash',
+				'title' => 'Host name with its protocol, without trailing slash',
 			),
 			'user' => array(
 				'name' => 'Username',

bridges/GolemBridge.php

@@ -0,0 +1,125 @@
+<?php
+
+class GolemBridge extends FeedExpander {
+	const MAINTAINER = 'Mynacol';
+	const NAME = 'Golem Bridge';
+	const URI = 'https://www.golem.de/';
+	const CACHE_TIMEOUT = 1800; // 30min
+	const DESCRIPTION = 'Returns the full articles instead of only the intro';
+	const PARAMETERS = array(array(
+		'category' => array(
+			'name' => 'Category',
+			'type' => 'list',
+			'values' => array(
+				'Alle News'
+				=> 'https://rss.golem.de/rss.php?feed=ATOM1.0',
+				'Audio/Video'
+				=> 'https://rss.golem.de/rss.php?ms=audio-video&feed=ATOM1.0',
+				'Auto'
+				=> 'https://rss.golem.de/rss.php?ms=auto&feed=ATOM1.0',
+				'Foto'
+				=> 'https://rss.golem.de/rss.php?ms=foto&feed=ATOM1.0',
+				'Games'
+				=> 'https://rss.golem.de/rss.php?ms=games&feed=ATOM1.0',
+				'Handy'
+				=> 'https://rss.golem.de/rss.php?ms=handy&feed=ATOM1.0',
+				'Internet'
+				=> 'https://rss.golem.de/rss.php?ms=internet&feed=ATOM1.0',
+				'Mobil'
+				=> 'https://rss.golem.de/rss.php?ms=mobil&feed=ATOM1.0',
+				'Open Source'
+				=> 'https://rss.golem.de/rss.php?ms=open-source&feed=ATOM1.0',
+				'Politik/Recht'
+				=> 'https://rss.golem.de/rss.php?ms=politik-recht&feed=ATOM1.0',
+				'Security'
+				=> 'https://rss.golem.de/rss.php?ms=security&feed=ATOM1.0',
+				'Desktop-Applikationen'
+				=> 'https://rss.golem.de/rss.php?ms=desktop-applikationen&feed=ATOM1.0',
+				'Software-Entwicklung'
+				=> 'https://rss.golem.de/rss.php?ms=softwareentwicklung&feed=ATOM1.0',
+				'Wirtschaft'
+				=> 'https://rss.golem.de/rss.php?ms=wirtschaft&feed=ATOM1.0',
+				'Wissenschaft'
+				=> 'https://rss.golem.de/rss.php?ms=wissenschaft&feed=ATOM1.0'
+			)
+		),
+		'limit' => array(
+			'name' => 'Limit',
+			'type' => 'number',
+			'required' => false,
+			'title' => 'Specify number of full articles to return',
+			'defaultValue' => 5
+		)
+	));
+	const LIMIT = 5;
+	const HEADERS = array('Cookie: golem_consent20=simple|220101;');
+
+	public function collectData() {
+		$this->collectExpandableDatas(
+			$this->getInput('category'),
+			$this->getInput('limit') ?: static::LIMIT
+		);
+	}
+
+	protected function parseItem($item) {
+		$item = parent::parseItem($item);
+		$item['content'] = $item['content'] ?? '';
+		$uri = $item['uri'];
+
+		while ($uri) {
+			$articlePage = getSimpleHTMLDOMCached($uri, static::CACHE_TIMEOUT, static::HEADERS);
+
+			// URI without RSS feed reference
+			$item['uri'] = $articlePage->find('head meta[name="twitter:url"]', 0)->content;
+
+			$author = $articlePage->find('article header .authors .authors__name', 0);
+			if ($author) {
+				$item['author'] = $author->innertext;
+			}
+
+			$item['content'] .= $this->extractContent($articlePage);
+
+			// next page
+			$nextUri = $articlePage->find('link[rel="next"]', 0);
+			$uri = $nextUri ? static::URI . $nextUri->href : null;
+		}
+
+		return $item;
+	}
+
+	private function extractContent($page) {
+		$item = '';
+
+		$article = $page->find('article', 0);
+
+		// delete known bad elements
+		foreach($article->find('div[id*="adtile"], #job-market, #seminars,
+			div.gbox_affiliate, div.toc, .embedcontent') as $bad) {
+			$bad->remove();
+		}
+		// reload html, as remove() is buggy
+		$article = str_get_html($article->outertext);
+
+		if ($pageHeader = $article->find('header.paged-cluster-header h1', 0)) {
+			$item .= $pageHeader;
+		}
+
+		$header = $article->find('header', 0);
+		foreach($header->find('p, figure') as $element) {
+			$item .= $element;
+		}
+
+		$content = $article->find('div.formatted', 0);
+
+		// full image quality
+		foreach($content->find('img[data-src-full][src*="."]') as $img) {
+			$img->src = $img->getAttribute('data-src-full');
+		}
+
+		foreach($content->find('p, h1, h3, img[src*="."]') as $element) {
+			$item .= $element;
+		}
+
+		return $item;
+	}
+}

bridges/GoodreadsBridge.php

@@ -0,0 +1,95 @@
+<?php
+
+class GoodreadsBridge extends BridgeAbstract {
+
+	const MAINTAINER = 'captn3m0';
+	const NAME = 'Goodreads Bridge';
+	const URI = 'https://www.goodreads.com/';
+	const CACHE_TIMEOUT = 0; // 30min
+	const DESCRIPTION = 'Various RSS feeds from Goodreads';
+
+	const CONTEXT_AUTHOR_BOOKS = 'Books by Author';
+
+	// Using a specific context because I plan to expand this soon
+	const PARAMETERS = array(
+		'Books by Author' => array(
+			'author_url' => array(
+				'name' => 'Link to author\'s page on Goodreads',
+				'type' => 'text',
+				'required' => true,
+				'title' => 'Should look somewhat like goodreads.com/author/show/',
+				'pattern' => '^(https:\/\/)?(www.)?goodreads\.com\/author\/show\/\d+\..*$',
+				'exampleValue' => 'https://www.goodreads.com/author/show/38550.Brandon_Sanderson'
+			),
+			'published_only' => array(
+				'name' => 'Show published books only',
+				'type' => 'checkbox',
+				'required' => false,
+				'title' => 'If left unchecked, this will return unpublished books as well',
+				'defaultValue' => 'checked',
+			),
+		),
+	);
+
+	private function collectAuthorBooks($url) {
+
+		$regex = '/goodreads\.com\/author\/show\/(\d+)/';
+
+		preg_match($regex, $url, $matches);
+
+		$authorId = $matches[1];
+
+		$authorListUrl = "https://www.goodreads.com/author/list/$authorId?sort=original_publication_year";
+
+		$html = getSimpleHTMLDOMCached($authorListUrl, self::CACHE_TIMEOUT);
+
+		foreach($html->find('tr[itemtype="http://schema.org/Book"]') as $row) {
+		$dateSpan = $row->find('.uitext', 0)->plaintext;
+		$date = null;
+
+		// If book is not yet published, ignore for now
+		if(preg_match('/published\s+(\d{4})/', $dateSpan, $matches) === 1) {
+			// Goodreads doesn't give us exact publication date here, only a year
+			// We are skipping future dates anyway, so this is def published
+			// but we can't pick a dynamic date either to keep clients from getting
+			// confused. So we pick a guaranteed date of 1st-Jan instead.
+			$date = $matches[1] . '-01-01';
+		} else if ($this->getInput('published_only') !== 'checked') {
+			// We can return unpublished books as well
+			$date = date('Y-01-01');
+		} else {
+			continue;
+		}
+
+		$row = defaultLinkTo($row, $this->getURI());
+
+		$item['title'] = $row->find('.bookTitle', 0)->plaintext;
+		$item['uri'] = $row->find('.bookTitle', 0)->getAttribute('href');
+		$item['author'] = $row->find('.authorName', 0)->plaintext;
+		$item['content'] = '<a href="'
+		. $row->find('.bookTitle', 0)->getAttribute('href')
+		. '"><img src="'
+		. $row->find('.bookCover', 0)->getAttribute('src')
+		. '"></a>';
+		$item['timestamp'] = $date;
+		$item['enclosures'] = array(
+			$row->find('.bookCover', 0)->getAttribute('src')
+		);
+
+		$this->items[] = $item; // Add item to the list
+		}
+	}
+
+	public function collectData() {
+
+		switch ($this->queriedContext) {
+		case self::CONTEXT_AUTHOR_BOOKS:
+			$this->collectAuthorBooks($this->getInput('author_url'));
+			break;
+
+		default:
+			throw new Exception('Invalid context', 1);
+			break;
+		}
+	}
+}

bridges/GoogleGroupsBridge.php

@@ -0,0 +1,68 @@
+<?php
+
+class GoogleGroupsBridge extends XPathAbstract {
+	const NAME = 'Google Groups Bridge';
+	const DESCRIPTION = 'Returns the latest posts on a Google Group';
+	const URI = 'https://groups.google.com';
+	const MAINTAINER = 'Yaman Qalieh';
+	const PARAMETERS = array( array(
+		'group' => array(
+			'name' => 'Group id',
+			'title' => 'The string that follows /g/ in the URL',
+			'exampleValue' => 'governance',
+			'required' => true
+		),
+		'account' => array(
+			'name' => 'Account id',
+			'title' => 'Some Google groups have an additional id following /a/ in the URL',
+			'exampleValue' => 'mozilla.org',
+			'required' => false
+		)
+	));
+	const CACHE_TIMEOUT = 3600;
+
+	const TEST_DETECT_PARAMETERS = array(
+		'https://groups.google.com/a/mozilla.org/g/announce' => array(
+			'account' => 'mozilla.org', 'group' => 'announce'
+		),
+		'https://groups.google.com/g/ansible-project' => array(
+			'account' => null, 'group' => 'ansible-project'
+		),
+	);
+
+	const XPATH_EXPRESSION_ITEM = '//div[@class="yhgbKd"]';
+	const XPATH_EXPRESSION_ITEM_TITLE = './/span[@class="o1DPKc"]';
+	const XPATH_EXPRESSION_ITEM_CONTENT = './/span[@class="WzoK"]';
+	const XPATH_EXPRESSION_ITEM_URI = './/a[@class="ZLl54"]/@href';
+	const XPATH_EXPRESSION_ITEM_AUTHOR = './/span[@class="z0zUgf"][last()]';
+	const XPATH_EXPRESSION_ITEM_TIMESTAMP = './/div[@class="tRlaM"]';
+	const XPATH_EXPRESSION_ITEM_ENCLOSURES = '';
+	const XPATH_EXPRESSION_ITEM_CATEGORIES = '';
+	const SETTING_FIX_ENCODING = true;
+
+	protected function getSourceUrl() {
+		$source = self::URI;
+
+		$account = $this->getInput('account');
+		if($account) {
+			$source = $source . '/a/' . $account;
+		}
+		return $source . '/g/' . $this->getInput('group');
+	}
+
+	protected function provideWebsiteContent() {
+		return defaultLinkTo(getContents($this->getSourceUrl()), self::URI);
+	}
+
+	const URL_REGEX = '#^https://groups.google.com(?:/a/(?<account>\S+))?(?:/g/(?<group>\S+))#';
+
+	public function detectParameters($url) {
+		$params = array();
+		if(preg_match(self::URL_REGEX, $url, $matches)) {
+			$params['group'] = $matches['group'];
+			$params['account'] = $matches['account'];
+			return $params;
+		}
+		return null;
+	}
+}

bridges/GoogleSearchBridge.php

@@ -24,25 +24,35 @@ class GoogleSearchBridge extends BridgeAbstract {
 	));
 
 	public function collectData(){
-		$html = '';
-
-		$html = getSimpleHTMLDOM($this->getURI());
+		$header = array('Accept-language: en-US');
+		$html = getSimpleHTMLDOM($this->getURI(), $header)
+			or returnServerError('No results for this query.');
 
 		$emIsRes = $html->find('div[id=res]', 0);
 
 		if(!is_null($emIsRes)) {
-			foreach($emIsRes->find('div[class=g]') as $element) {
-
+			foreach($emIsRes->find('div[class~=g]') as $element) {
 				$item = array();
 
 				$t = $element->find('a[href]', 0)->href;
 				$item['uri'] = htmlspecialchars_decode($t);
 				$item['title'] = $element->find('h3', 0)->plaintext;
-				$item['content'] = $element->find('span[class=aCOpRe]', 0)->plaintext;
+				$resultComponents = explode(' — ', $element->find('div[data-content-feature=1]', 0)->plaintext);
+				$item['content'] = $resultComponents[1];
+
+				if(strpos($resultComponents[0], 'day') === true) {
+					$daysago = explode(' ', $resultComponents[0])[0];
+					$item['timestamp'] = date('d M Y', strtotime('-' . $daysago . ' days'));
+				} else {
+					$item['timestamp'] = $resultComponents[0];
+				}
 
 				$this->items[] = $item;
 			}
 		}
+		usort($this->items, function($a, $b) {
+			return $a['timestamp'] < $b['timestamp'];
+		});
 	}
 
 	public function getURI() {
@@ -50,7 +60,7 @@ class GoogleSearchBridge extends BridgeAbstract {
 			return self::URI
 				. 'search?q='
 				. urlencode($this->getInput('q'))
-				. '&num=100&complete=0&tbs=qdr:y,sbd:1';
+				. '&hl=en&num=100&complete=0&tbs=qdr:y,sbd:1';
 		}
 
 		return parent::getURI();

bridges/GroupBundNaturschutzBridge.php

@@ -0,0 +1,107 @@
+<?php
+
+class GroupBundNaturschutzBridge extends XPathAbstract
+{
+	const NAME = 'BUND Naturschutz in Bayern e.V. - Kreisgruppen';
+	const URI = 'https://www.bund-naturschutz.de/ueber-uns/organisation/kreisgruppen-ortsgruppen';
+	const DESCRIPTION = 'Returns the latest news from specified BUND Naturschutz in Bayern e.V. local group (Germany)';
+	const MAINTAINER = 'dweipert';
+
+	const PARAMETERS = array(
+		array(
+			'group' => array(
+				'name' => 'Group',
+				'type' => 'list',
+				'values' => array(
+					// 'Aichach-Friedberg' => 'bn-aic.de', # non-uniform page
+					'Altötting' => 'altoetting',
+					'Amberg-Sulzbach' => 'amberg-sulzbach',
+					'Ansbach' => 'ansbach',
+					'Aschaffenburg' => 'aschaffenburg',
+					'Augsburg' => 'augsburg',
+					'Bad Kissingen' => 'bad-kissingen',
+					'Bad Tölz' => 'bad-toelz',
+					'Bamberg' => 'bamberg',
+					'Bayreuth' => 'bayreuth', # single entry # different layout
+					'Berchtesgadener Land' => 'berchtesgadener-land',
+					'Cham' => 'cham',
+					// 'Coburg' => 'coburg', # no real entries # different layout
+					'Dachau' => 'dachau',
+					'Deggendorf' => 'Deggendorf',
+					'Dillingen' => 'dillingen',
+					'Dingolfing-Landau' => 'dingolfing-landau',
+					'Donau-Ries' => 'donauries',
+					'Ebersberg' => 'ebersberg',
+					'Eichstätt' => 'eichstaett', # single entry since 2020
+					'Erding' => 'erding',
+					'Erlangen' => 'erlangen',
+					'Forchheim' => 'forchheim',
+					'Freising' => 'freising',
+					'Freyung-Grafenau' => 'freyung-grafenau',
+					'Fürstenfeldbruck' => 'fuerstenfeldbruck',
+					'Fürth-Land' => 'fuerth-land',
+					'Fürth-Stadt' => 'fuerth',
+					'Garmisch-Partenkirchen' => 'garmisch-partenkirchen',
+					'Günzburg' => 'guenzburg',
+					'Hassberge' => 'hassberge',
+					'Höchstadt-Herzogenaurach' => 'hoechstadt-herzogenaurach',
+					// 'Hof' => 'kreisgruppehof.bund-naturschutz.com', # non-uniform page
+					'Ingolstadt' => 'ingolstadt',
+					'Kelheim' => 'kelheim',
+					'Kempten' => 'kempten',
+					'Kitzingen' => 'kitzingen',
+					'Kronach' => 'kronach',
+					'Kulmbach' => 'kulmbach',
+					'Landsberg' => 'landsberg',
+					'Landshut' => 'landshut',
+					'Lichtenfeld' => 'lichtenfels',
+					'Lindau' => 'lindau',
+					'Main-Spessart' => 'main-spessart',
+					'Memmingen-Unterallgäu' => 'memmingen-unterallgaeu',
+					'Miesbach' => 'miesbach',
+					'Miltenberg' => 'miltenberg',
+					'Mühldorf am Inn' => 'muehldorf',
+					// 'München' => 'bn-muenchen.de', # non-uniform page
+					'Neu-Ulm' => 'neu-ulm',
+					'Neuburg-Schrobenhausen' => 'neuburg-schrobenhausen',
+					'Neumarkt' => 'neumarkt',
+					'Neustadt/Aisch-Bad Windsheim' => 'neustadt-aisch',
+					'Neustadt/Waldnaab-Weiden' => 'neustadt-weiden',
+					'Nürnberg Stadt' => 'nuernberg-stadt',
+					'Nürnberger Land' => 'nuernberger-land',
+					'Ostallgäu-Kaufbeuren' => 'Ostallgäu-Kaufbeuren',
+					'Passau' => 'passau',
+					'Pfaffenhofen/Ilm' => 'pfaffenhofen',
+					'Regen' => 'regen',
+					'Regensburg' => 'regensburg',
+					'Rhön-Grabfeld' => 'rhoen-grabfeld',
+					'Rosenheim' => 'rosenheim',
+					'Roth' => 'roth',
+					'Rottal-Inn' => 'rottal-inn',
+					'Schwabach' => 'schwabach',
+					'Schwandorf' => 'schwandorf',
+					'Schweinfurt' => 'schweinfurt',
+					'Starnberg' => 'starnberg',
+					'Straubing-Bogen' => 'straubing',
+					'Tirschenreuth' => 'tirschenreuth',
+					'Traunstein' => 'traunstein',
+					'Weilheim-Schongau' => 'weilheim-schongau',
+					'Weißenburg-Gunzenhausen' => 'weissenburg-gunzenhausen',
+					'Wunsiedel' => 'wunsiedel',
+					'Würzburg' => 'wuerzburg',
+				),
+			),
+		),
+	);
+
+	const XPATH_EXPRESSION_ITEM = '//div[@itemtype="http://schema.org/Article"]';
+	const XPATH_EXPRESSION_ITEM_TITLE = './/*[@itemprop="headline"]';
+	const XPATH_EXPRESSION_ITEM_CONTENT = './/*[@itemprop="description"]/text()';
+	const XPATH_EXPRESSION_ITEM_URI = './/a/@href';
+	const XPATH_EXPRESSION_ITEM_TIMESTAMP = './/*[@itemprop="datePublished"]/@datetime';
+	const XPATH_EXPRESSION_ITEM_ENCLOSURES = './/img/@src';
+
+	protected function getSourceUrl() {
+		return 'https://' . $this->getInput('group') . '.bund-naturschutz.de/aktuelles';
+	}
+}

bridges/HaveIBeenPwnedBridge.php

@@ -1,4 +1,11 @@
 <?php
+/**
+ * Uses the API as documented here:
+ * https://haveibeenpwned.com/API/v3#AllBreaches
+ *
+ * Gets the latest breaches by the date of the breach or when it was added to
+ * HIBP.
+ * */
 class HaveIBeenPwnedBridge extends BridgeAbstract {
 	const NAME = 'Have I Been Pwned (HIBP) Bridge';
 	const URI = 'https://haveibeenpwned.com';
@@ -21,52 +28,41 @@ class HaveIBeenPwnedBridge extends BridgeAbstract {
 			'defaultValue' => 20,
 		)
 	));
+	const API_URI = 'https://haveibeenpwned.com/api/v3';
 
 	const CACHE_TIMEOUT = 3600;
 
-	private $breachDateRegex = '/Breach date: ([0-9]{1,2} [A-Z-a-z]+ [0-9]{4})/';
-	private $dateAddedRegex = '/Date added to HIBP: ([0-9]{1,2} [A-Z-a-z]+ [0-9]{4})/';
-	private $accountsRegex = '/Compromised accounts: ([0-9,]+)/';
-
 	private $breaches = array();
 
 	public function collectData() {
 
-		$html = getSimpleHTMLDOM(self::URI . '/PwnedWebsites');
+		$data = json_decode(getContents(self::API_URI . '/breaches'), true);
 
-		$breaches = array();
-
-		foreach($html->find('div.row') as $breach) {
+		foreach($data as $breach) {
 			$item = array();
 
-			if ($breach->class != 'row') {
-				continue;
-			}
+			$pwnCount = number_format($breach['PwnCount']);
+			$item['title'] = $breach['Title'] . ' - '
+						   . $pwnCount . ' breached accounts';
+			$item['dateAdded'] = $breach['AddedDate'];
+			$item['breachDate'] = $breach['BreachDate'];
+			$item['uri'] = self::URI . '/PwnedWebsites#' . $breach['Name'];
 
-			preg_match($this->breachDateRegex, $breach->find('p', 1)->plaintext, $breachDate)
-				or returnServerError('Could not extract details');
-
-			preg_match($this->dateAddedRegex, $breach->find('p', 1)->plaintext, $dateAdded)
-				or returnServerError('Could not extract details');
-
-			preg_match($this->accountsRegex, $breach->find('p', 1)->plaintext, $accounts)
-				or returnServerError('Could not extract details');
-
-			$permalink = $breach->find('p', 1)->find('a', 0)->href;
-
-			// Remove permalink
-			$breach->find('p', 1)->find('a', 0)->outertext = '';
-
-			$item['title'] = html_entity_decode($breach->find('h3', 0)->plaintext, ENT_QUOTES)
-				. ' - ' . $accounts[1] . ' breached accounts';
-			$item['dateAdded'] = strtotime($dateAdded[1]);
-			$item['breachDate'] = strtotime($breachDate[1]);
-			$item['uri'] = self::URI . '/PwnedWebsites' . $permalink;
-
-			$item['content'] = '<p>' . $breach->find('p', 0)->innertext . '</p>';
+			$item['content'] = '<p>' . $breach['Description'] . '</p>';
 			$item['content'] .= '<p>' . $this->breachType($breach) . '</p>';
-			$item['content'] .= '<p>' . $breach->find('p', 1)->innertext . '</p>';
 
+			$breachDate = date('j F Y', strtotime($breach['BreachDate']));
+			$addedDate = date('j F Y', strtotime($breach['AddedDate']));
+			$compData = implode(', ', $breach['DataClasses']);
+
+			$item['content'] .= <<<EOD
+<p>
+<strong>Breach date:</strong> {$breachDate}<br>
+<strong>Date added to HIBP:</strong> {$addedDate}<br>
+<strong>Compromised accounts:</strong> {$pwnCount}<br>
+<strong>Compromised data:</strong> {$compData}<br>
+EOD;
+			$item['uid'] = $breach['Name'];
 			$this->breaches[] = $item;
 		}
 
@@ -74,6 +70,27 @@ class HaveIBeenPwnedBridge extends BridgeAbstract {
 		$this->createItems();
 	}
 
+	private const BREACH_TYPES = array(
+		'IsVerified' => array(
+			false => 'Unverified breach, may be sourced from elsewhere'
+		),
+		'IsFabricated' => array(
+			true => 'Fabricated breach, likely not legitimate'
+		),
+		'IsSensitive' => array(
+			true => 'Sensitive breach, not publicly searchable'
+		),
+		'IsRetired' => array(
+			true => 'Retired breach, removed from system'
+		),
+		'IsSpamList' => array(
+			true => 'Spam list, used for spam marketing'
+		),
+		'IsMalware' => array(
+			true => 'Malware breach'
+		),
+	);
+
 	/**
 	 * Extract data breach type(s)
 	 */
@@ -81,12 +98,10 @@ class HaveIBeenPwnedBridge extends BridgeAbstract {
 
 		$content = '';
 
-		if ($breach->find('h3 > i', 0)) {
-
-			foreach ($breach->find('h3 > i') as $i) {
-				$content .= $i->title . '.<br>';
+		foreach (self::BREACH_TYPES as $type => $message) {
+			if (isset($message[$breach[$type]])) {
+				$content .= $message[$breach[$type]] . '.<br>';
 			}
-
 		}
 
 		return $content;
@@ -127,6 +142,7 @@ class HaveIBeenPwnedBridge extends BridgeAbstract {
 			$item['timestamp'] = $breach[$this->getInput('order')];
 			$item['uri'] = $breach['uri'];
 			$item['content'] = $breach['content'];
+			$item['uid'] = $breach['uid'];
 
 			$this->items[] = $item;
 

bridges/HeiseBridge.php

@@ -42,6 +42,10 @@ class HeiseBridge extends FeedExpander {
 		$item = parent::parseItem($feedItem);
 		$item['uri'] = explode('?', $item['uri'])[0] . '?seite=all';
 
+		if (strpos($item['uri'], 'https://www.heise.de') !== 0) {
+			return $item;
+		}
+
 		$article = getSimpleHTMLDOMCached($item['uri']);
 
 		if ($article) {

bridges/HentaiHavenBridge.php

@@ -1,36 +0,0 @@
-<?php
-class HentaiHavenBridge extends BridgeAbstract {
-
-	const MAINTAINER = 'albirew';
-	const NAME = 'Hentai Haven';
-	const URI = 'https://hentaihaven.com/';
-	const CACHE_TIMEOUT = 21600; // 6h
-	const DESCRIPTION = 'Returns releases from Hentai Haven';
-
-	public function collectData(){
-		$html = getSimpleHTMLDOM(self::URI);
-
-		foreach($html->find('div.zoe-grid') as $element) {
-			$item = array();
-			$item['uri'] = $element->find('div.brick-content h3 a', 0)->href;
-			$thumbnailUri = $element->find('a.thumbnail-image img', 0)->getAttribute('src');
-			$item['title'] = mb_convert_encoding(
-				trim($element->find('div.brick-content h3 a', 0)->innertext),
-				'UTF-8',
-				'HTML-ENTITIES'
-			);
-
-			$item['tags'] = $element->find('div.oFlyout_bg div.oFlyout div.flyoutContent span.tags', 0)->plaintext;
-			$item['content'] = 'Tags: '
-			. $item['tags']
-			. '<br><br><a href="'
-			. $item['uri']
-			. '"><img width="300" height="169" src="'
-			. $thumbnailUri
-			. '" /></a><br>'
-			. $element->find('div.oFlyout_bg div.oFlyout div.flyoutContent p.description', 0)->innertext;
-
-			$this->items[] = $item;
-		}
-	}
-}

bridges/HotUKDealsBridge.php

@@ -1,6 +1,5 @@
 <?php
 
-require_once(__DIR__ . '/DealabsBridge.php');
 class HotUKDealsBridge extends PepperBridgeAbstract {
 
 	const NAME = 'HotUKDeals bridge';
@@ -3253,7 +3252,7 @@ class HotUKDealsBridge extends PepperBridgeAbstract {
 				'name' => 'Discussion URL',
 				'type' => 'text',
 				'required' => true,
-				'title' => 'Discussion URL to monitor. Ex: https://www.hotukdeals.com/discussions/the-hukd-lego-thread-3599357',
+				'title' => 'Discussion URL to monitor. Ex: https://www.hotukdeals.com/discussions/title-123',
 				'exampleValue' => 'https://www.hotukdeals.com/discussions/the-hukd-lego-thread-3599357',
 				),
 			'only_with_url' => array(
@@ -3306,7 +3305,7 @@ class HotUKDealsBridge extends PepperBridgeAbstract {
 			'th'
 		),
 		'local-time-relative' => array(
-			'Found ',
+			'Posted ',
 			'm',
 			'h,',
 			'day',

bridges/IPBBridge.php

@@ -55,7 +55,7 @@ class IPBBridge extends FeedExpander {
 		$headers = get_headers($uri . '.xml');
 
 		if($headers[0] === 'HTTP/1.1 200 OK') { // Heureka! It's a valid feed!
-			return $this->collectExpandableDatas($uri);
+			return $this->collectExpandableDatas($uri . '.xml');
 		}
 
 		// No valid feed, so do it the hard way

bridges/InstagramBridge.php

@@ -160,6 +160,11 @@ class InstagramBridge extends BridgeAbstract {
 				$mediaURI = self::URI . 'p/' . $media->shortcode . '/media?size=l';
 			}
 
+			$pattern = array('/\@([\w\.]+)/', '/#([\w\.]+)/');
+			$replace = array(
+				'<a href="https://www.instagram.com/$1">@$1</a>',
+				'<a href="https://www.instagram.com/explore/tags/$1">#$1</a>');
+
 			switch($media->__typename) {
 				case 'GraphSidecar':
 					$data = $this->getInstagramSidecarData($item['uri'], $item['title'], $media, $textContent);
@@ -169,7 +174,7 @@ class InstagramBridge extends BridgeAbstract {
 				case 'GraphImage':
 					$item['content'] = '<a href="' . htmlentities($item['uri']) . '" target="_blank">';
 					$item['content'] .= '<img src="' . htmlentities($mediaURI) . '" alt="' . $item['title'] . '" />';
-					$item['content'] .= '</a><br><br>' . nl2br(htmlentities($textContent));
+					$item['content'] .= '</a><br><br>' . nl2br(preg_replace($pattern, $replace, htmlentities($textContent)));
 					$item['enclosures'] = array($mediaURI);
 					break;
 				case 'GraphVideo':

bridges/InternetArchiveBridge.php

@@ -23,7 +23,8 @@ class InternetArchiveBridge extends BridgeAbstract {
 					'Web Archives' => 'web-archive',
 				),
 				'defaultValue' => 'uploads',
-			)
+			),
+			'limit' => self::LIMIT,
 		)
 	);
 
@@ -72,7 +73,8 @@ class InternetArchiveBridge extends BridgeAbstract {
 		if ($this->getInput('content') !== 'posts') {
 			$detailsDivNumber = 0;
 
-			foreach ($html->find('div.results > div[data-id]') as $index => $result) {
+			$results = $html->find('div.results > div[data-id]');
+			foreach ($results as $index => $result) {
 				$item = array();
 
 				if (in_array($result->class, $this->skipClasses)) {
@@ -110,6 +112,11 @@ class InternetArchiveBridge extends BridgeAbstract {
 				}
 
 				$detailsDivNumber++;
+
+				$limit = $this->getInput('limit') ?? 10;
+				if (count($this->items) >= $limit) {
+					break;
+				}
 			}
 		}
 
@@ -302,7 +309,7 @@ EOD;
 
 			$items[] = $item;
 
-			if (count($items) >= 10) {
+			if (count($items) >= $this->getInput('limit') ?? 10) {
 				break;
 			}
 		}

bridges/KhinsiderBridge.php

@@ -12,17 +12,23 @@ class KhinsiderBridge extends BridgeAbstract
 	{
 		$html = getSimpleHTMLDOM(self::URI);
 
-		$dates = $html->find('#EchoTopic h3');
-		foreach ($dates as $date) {
+		$dates = $html->find('.latestSoundtrackHeading');
+		$tables = $html->find('.albumList');
+		// $dates is empty
+		foreach ($dates as $i => $date) {
 			$item = array();
 			$item['uri'] = self::URI;
-			$item['timestamp'] = DateTime::createFromFormat('F jS, Y', $date->plaintext)->format('U');
+			$item['timestamp'] = DateTime::createFromFormat('F jS, Y', $date->plaintext)->setTime(1, 1)->format('U');
 			$item['title'] = sprintf('OST for %s', $date->plaintext);
 			$item['author'] = 'Khinsider';
-			$links = $date->next_sibling()->find('a');
+			$trs = $tables[$i]->find('tr');
 			$content = '<ul>';
-			foreach ($links as $link) {
-				$content .= sprintf('<li><a href="%s">%s</a></li>', $link->href, $link->plaintext);
+			foreach ($trs as $tr) {
+				$td = $tr->find('td', 1);
+				if (null !== $td) {
+					$link = $td->find('a', 0);
+					$content .= sprintf('<li><a href="%s">%s</a></li>', $link->href, $link->plaintext);
+				}
 			}
 			$content .= '</ul>';
 			$item['content'] = $content;

bridges/KonachanBridge.php

@@ -1,5 +1,4 @@
 <?php
-require_once('MoebooruBridge.php');
 
 class KonachanBridge extends MoebooruBridge {
 

bridges/KununuBridge.php

@@ -17,7 +17,8 @@ class KununuBridge extends BridgeAbstract {
 					'Germany' => 'de',
 					'Switzerland' => 'ch',
 					'United States' => 'us'
-				)
+				),
+				'exampleValue' => 'de',
 			),
 			'full' => array(
 				'name' => 'Load full article',
@@ -46,7 +47,7 @@ class KununuBridge extends BridgeAbstract {
 			'company' => array(
 				'name' => 'Company',
 				'required' => true,
-				'exampleValue' => 'kununu-us',
+				'exampleValue' => 'adesso',
 				'title' => 'Insert company name (i.e. Kununu US) or URI path (i.e. kununu-us)'
 			)
 		)
@@ -72,7 +73,8 @@ class KununuBridge extends BridgeAbstract {
 				break;
 			}
 
-			return self::URI . $site . '/' . $company . '/' . $section . '?sort=update_time_desc';
+			$url = sprintf('%s%s/%s/%s?sort=update_time_desc', self::URI, $site, $company, $section);
+			return $url;
 		}
 
 		return parent::getURI();
@@ -91,6 +93,9 @@ class KununuBridge extends BridgeAbstract {
 		return 'https://www.kununu.com/favicon-196x196.png';
 	}
 
+	/**
+	 * All css selectors need rework
+	 */
 	public function collectData(){
 		$full = $this->getInput('full');
 

bridges/LWNprevBridge.php

@@ -109,7 +109,7 @@ EOD;
 
 	private function getFeatureContents(&$html){
 		$items = array();
-		foreach($html->getElementsByTagName('h2') as $title) {
+		foreach($html->getElementsByTagName('h3') as $title) {
 			if($title->getAttribute('class') !== 'SummaryHL') {
 				continue;
 			}

bridges/LaCentraleBridge.php

@@ -1,4 +1,5 @@
 <?php
+
 class LaCentraleBridge extends BridgeAbstract {
 
 	const MAINTAINER = 'jacknumber';
@@ -414,9 +415,9 @@ class LaCentraleBridge extends BridgeAbstract {
 	));
 
 	public function collectData(){
-		// check data
 		if(!empty($this->getInput('distance'))
-		&& is_null($this->getInput('location'))) {
+			&& is_null($this->getInput('location'))
+		) {
 			returnClientError('You need a place ("CP ou département") to search arround.');
 		}
 
@@ -442,35 +443,31 @@ class LaCentraleBridge extends BridgeAbstract {
 			'doors' => $this->getInput('doors'),
 			'sortBy' => $this->getInput('sort')
 		);
-		$url = self::URI . 'listing?' . http_build_query($params);
+		$url = sprintf('%slisting?%s', self::URI, http_build_query($params));
 		$html = getSimpleHTMLDOM($url);
 
-		foreach($html->find('.linkAd') as $element) {
+		$elements = $html->find('.adLineContainer');
+		foreach($elements as $element) {
 
 			$item = array();
-			$item['uri'] = trim(self::URI, '/') . $element->href;
-			$item['title'] = $element->find('.brandModel', 0)->plaintext;
-			$item['sellerType'] = $element->find('.typeSeller', 0)->plaintext;
+			$item['uri'] = trim(self::URI, '/') . $element->find('div > a', 0)->href;
+			$item['title'] = $element->find('.searchCard__makeModel', 0)->plaintext;
+			$item['sellerType'] = $element->find('.searchCard__customer', 0)->plaintext;
 			$item['author'] = $item['sellerType'];
-			$item['version'] = $element->find('.version', 0)->plaintext;
-			$item['price'] = $element->find('.fieldPrice', 0)->plaintext;
-			$item['year'] = $element->find('.fieldYear', 0)->plaintext;
-			$item['mileage'] = $element->find('.fieldMileage', 0)->plaintext;
-			$item['departement'] = str_replace(',', '', $element->find('.dptCont', 0)->plaintext);
-			$item['thumbnail'] = $element->find('.imgContent img', 0)->src;
-			$item['enclosures'] = array($item['thumbnail']);
+			$item['version'] = $element->find('.searchCard__version', 0)->plaintext;
+			$item['price'] = $element->find('.searchCard__fieldPrice', 0)->plaintext;
+			$item['year'] = $element->find('.searchCard__year', 0)->plaintext;
+			$item['mileage'] = $element->find('.searchCard__mileage', 0)->plaintext;
+			// The image is lazyloaded with ajax
 
 			$item['content'] = '
-			<img src="' . $item['thumbnail'] . '">
 			<br>Variation : ' . $item['version']
 			. '<br>Prix : ' . $item['price']
 			. '<br>Année : ' . $item['year']
 			. '<br>Kilométrage : ' . $item['mileage']
-			. '<br>Département : ' . $item['departement']
 			. '<br>Type de vendeur : ' . $item['sellerType'];
 
 			$this->items[] = $item;
-
 		}
 	}
 }

bridges/LegifranceJOBridge.php

@@ -3,6 +3,7 @@ class LegifranceJOBridge extends BridgeAbstract {
 
 	const MAINTAINER = 'Pierre Mazière';
 	const NAME = 'Journal Officiel de la République Française';
+	// This uri returns a snippet of js. Should probably be https://www.legifrance.gouv.fr/jorf/jo/
 	const URI = 'https://www.legifrance.gouv.fr/affichJO.do';
 	const DESCRIPTION = 'Returns the laws and decrees officially registered daily in France';
 

bridges/LichessBridge.php

@@ -1,31 +0,0 @@
-<?php
-class LichessBridge  extends FeedExpander {
-
-	const MAINTAINER = 'AmauryCarrade';
-	const NAME = 'Lichess Blog';
-	const URI = 'http://fr.lichess.org/blog';
-	const DESCRIPTION = 'Returns the 5 newest posts from the Lichess blog (full text)';
-
-	public function collectData(){
-		$this->collectExpandableDatas(self::URI . '.atom', 5);
-	}
-
-	protected function parseItem($newsItem){
-		$item = parent::parseItem($newsItem);
-		$item['content'] = $this->retrieveLichessPost($item['uri']);
-		return $item;
-	}
-
-	private function retrieveLichessPost($blog_post_uri){
-		$blog_post_html = getSimpleHTMLDOMCached($blog_post_uri);
-		$blog_post_div  = $blog_post_html->find('#lichess_blog', 0);
-
-		$post_chapo = $blog_post_div->find('.shortlede', 0)->innertext;
-		$post_content = $blog_post_div->find('.body', 0)->innertext;
-
-		$content = '<p><em>' . $post_chapo . '</em></p>';
-		$content .= '<div>' . $post_content . '</div>';
-
-		return $content;
-	}
-}

bridges/LolibooruBridge.php

@@ -1,5 +1,4 @@
 <?php
-require_once('MoebooruBridge.php');
 
 class LolibooruBridge extends MoebooruBridge {
 

bridges/MangaDexBridge.php

@@ -0,0 +1,233 @@
+<?php
+class MangaDexBridge extends BridgeAbstract {
+	const MAINTAINER = 'Yaman Qalieh';
+	const NAME = 'MangaDex Bridge';
+	const URI = 'https://mangadex.org/';
+	const API_ROOT = 'https://api.mangadex.org/';
+	const DESCRIPTION = 'Returns MangaDex items using the API';
+
+	const PARAMETERS = array(
+		'global' => array(
+			'limit' => array(
+				'name' => 'Item Limit',
+				'type' => 'number',
+				'defaultValue' => 10,
+				'required' => true
+			),
+			'lang' => array(
+				'name' => 'Chapter Languages (default=all)',
+				'title' => 'comma-separated, two-letter language codes (example "en,jp")',
+				'exampleValue' => 'en,jp',
+				'required' => false
+			),
+		),
+		'Title Chapters' => array(
+			'url' => array(
+				'name' => 'URL to title page',
+				'exampleValue' => 'https://mangadex.org/title/f9c33607-9180-4ba6-b85c-e4b5faee7192/official-test-manga',
+				'required' => true
+			),
+			'external' => array(
+				'name' => 'Allow external feed items',
+				'type' => 'checkbox',
+				'title' => 'Some chapters are inaccessible or only available on an external site. Include these?'
+			)
+		),
+		'Search Chapters' => array(
+			'chapter' => array(
+				'name' => 'Chapter Number (default=all)',
+				'title' => 'The example value finds the newest first chapters',
+				'exampleValue' => 1,
+				'required' => false
+			),
+			'groups' => array(
+				'name' => 'Group UUID (default=all)',
+				'title' => 'This can be found in the MangaDex Group Page URL',
+				'exampleValue' => '00e03853-1b96-4f41-9542-c71b8692033b',
+				'required' => false,
+			),
+			'uploader' => array(
+				'name' => 'User UUID (default=all)',
+				'title' => 'This can be found in the MangaDex User Page URL',
+				'exampleValue' => 'd2ae45e0-b5e2-4e7f-a688-17925c2d7d6b',
+				'required' => false,
+			),
+			'external' => array(
+				'name' => 'Allow external feed items',
+				'type' => 'checkbox',
+				'title' => 'Some chapters are inaccessible or only available on an external site. Include these?'
+			)
+		)
+		// Future Manga Contexts:
+		// Manga List (by author or tags): https://api.mangadex.org/swagger.html#/Manga/get-search-manga
+		// Random Manga: https://api.mangadex.org/swagger.html#/Manga/get-manga-random
+		// Future Chapter Contexts:
+		// User Lists https://api.mangadex.org/swagger.html#/Feed/get-list-id-feed
+		//
+		// https://api.mangadex.org/docs/get-covers/
+	);
+
+	const TITLE_REGEX = '#title/(?<uuid>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})#';
+
+	protected $feedName = '';
+	protected $feedURI = '';
+
+	protected function buildArrayQuery($name, $array) {
+		$query = '';
+		foreach($array as $item) {
+			$query .= '&' . $name . '=' . $item;
+		}
+		return $query;
+	}
+
+	protected function getAPI() {
+		$params = array(
+			'limit' => $this->getInput('limit')
+		);
+
+		$array_params = array();
+		if (!empty($this->getInput('lang'))) {
+			$array_params['translatedLanguage[]'] = explode(',', $this->getInput('lang'));
+		}
+
+		switch($this->queriedContext) {
+		case 'Title Chapters':
+			preg_match(self::TITLE_REGEX, $this->getInput('url'), $matches)
+				or returnClientError('Invalid URL Parameter');
+			$this->feedURI = self::URI . 'title/' . $matches['uuid'];
+			$params['order[updatedAt]'] = 'desc';
+			if (!$this->getInput('external')) {
+				$params['includeFutureUpdates'] = '0';
+			}
+			$array_params['includes[]'] = array('manga', 'scanlation_group', 'user');
+			$uri = self::API_ROOT . 'manga/' . $matches['uuid'] . '/feed';
+			break;
+		case 'Search Chapters':
+			$params['chapter'] = $this->getInput('chapter');
+			$params['groups[]'] = $this->getInput('groups');
+			$params['uploader'] = $this->getInput('uploader');
+			$params['order[updatedAt]'] = 'desc';
+			if (!$this->getInput('external')) {
+				$params['includeFutureUpdates'] = '0';
+			}
+			$array_params['includes[]'] = array('manga', 'scanlation_group', 'user');
+			$uri = self::API_ROOT . 'chapter';
+			break;
+		default:
+			returnServerError('Unimplemented Context (getAPI)');
+		}
+
+		// Remove null keys
+		$params = array_filter($params, function($v) {
+			return !empty($v);
+		});
+
+		$uri .= '?' . http_build_query($params);
+
+		// Arrays are passed as repeated keys to MangaDex
+		// This cannot be handled by http_build_query
+		foreach($array_params as $name => $array_param) {
+			$uri .= $this->buildArrayQuery($name, $array_param);
+		}
+
+		return $uri;
+	}
+
+	public function getName() {
+		switch($this->queriedContext) {
+		case 'Title Chapters':
+			return $this->feedName . ' Chapters';
+		case 'Search Chapters':
+			return 'MangaDex Chapter Search';
+		default:
+			return parent::getName();
+		}
+	}
+
+	public function getURI() {
+		switch($this->queriedContext) {
+		case 'Title Chapters':
+			return $this->feedURI;
+		default:
+			return parent::getURI();
+		}
+	}
+
+	public function collectData() {
+		$api_uri = $this->getAPI();
+		$header = array(
+			'Content-Type: application/json'
+		);
+		$content = json_decode(getContents($api_uri, $header), true);
+		if ($content['result'] == 'ok') {
+			$content = $content['data'];
+		} else {
+			returnServerError('Could not retrieve API results');
+		}
+
+		switch($this->queriedContext) {
+		case 'Title Chapters':
+			$this->getChapters($content);
+			break;
+		case 'Search Chapters':
+			$this->getChapters($content);
+			break;
+		default:
+			returnServerError('Unimplemented Context (collectData)');
+		}
+	}
+
+	protected function getChapters($content) {
+		foreach($content as $chapter) {
+			$item = array();
+			$item['uid'] = $chapter['id'];
+			$item['uri'] = self::URI . 'chapter/' . $chapter['id'];
+
+			// External chapter
+			if (!$this->getInput('external') && $chapter['attributes']['pages'] == 0)
+				continue;
+
+			$item['title'] = '';
+			if (isset($chapter['attributes']['volume']))
+				$item['title'] .= 'Volume ' . $chapter['attributes']['volume'] . ' ';
+			if (isset($chapter['attributes']['chapter']))
+				$item['title'] .= 'Chapter ' . $chapter['attributes']['chapter'];
+			if (!empty($chapter['attributes']['title'])) {
+				$item['title'] .= ' - ' . $chapter['attributes']['title'];
+			}
+			$item['title'] .= ' [' . $chapter['attributes']['translatedLanguage'] . ']';
+
+			$item['timestamp'] = $chapter['attributes']['updatedAt'];
+
+			$groups = array();
+			$users = array();
+			foreach($chapter['relationships'] as $rel) {
+				switch($rel['type']) {
+				case 'scanlation_group':
+					$groups[] = $rel['attributes']['name'];
+					break;
+				case 'manga':
+					if (empty($this->feedName))
+						$this->feedName = reset($rel['attributes']['title']);
+					if ($this->queriedContext !== 'Title Chapters')
+						$item['title'] = reset($rel['attributes']['title']) . ' ' . $item['title'];
+					break;
+				case 'user':
+					if (isset($item['author'])) {
+						$users[] = $rel['attributes']['username'];
+					} else {
+						$item['author'] = $rel['attributes']['username'];
+					}
+					break;
+				}
+			}
+			$item['content'] = 'Groups: ' .
+							 (empty($groups) ? 'No Group' : implode(', ', $groups));
+			if (!empty($users)) {
+				$item['content'] .= '<br>Other Users: ' . implode(', ', $users);
+			}
+
+			$this->items[] = $item;
+		}
+	}
+}

bridges/MangareaderBridge.php

@@ -1,248 +0,0 @@
-<?php
-class MangareaderBridge extends BridgeAbstract {
-
-	const MAINTAINER = 'logmanoriginal';
-	const NAME = 'Mangareader Bridge';
-	const URI = 'https://www.mangareader.net';
-	const CACHE_TIMEOUT = 10800; // 3h
-	const DESCRIPTION = 'Returns the latest updates, popular mangas or manga updates (new chapters)';
-
-	const PARAMETERS = array(
-		'Get latest updates' => array(),
-		'Get popular mangas' => array(
-			'category' => array(
-				'name' => 'Category',
-				'type' => 'list',
-				'values' => array(
-					'All' => 'all',
-					'Action' => 'action',
-					'Adventure' => 'adventure',
-					'Comedy' => 'comedy',
-					'Demons' => 'demons',
-					'Drama' => 'drama',
-					'Ecchi' => 'ecchi',
-					'Fantasy' => 'fantasy',
-					'Gender Bender' => 'gender-bender',
-					'Harem' => 'harem',
-					'Historical' => 'historical',
-					'Horror' => 'horror',
-					'Josei' => 'josei',
-					'Magic' => 'magic',
-					'Martial Arts' => 'martial-arts',
-					'Mature' => 'mature',
-					'Mecha' => 'mecha',
-					'Military' => 'military',
-					'Mystery' => 'mystery',
-					'One Shot' => 'one-shot',
-					'Psychological' => 'psychological',
-					'Romance' => 'romance',
-					'School Life' => 'school-life',
-					'Sci-Fi' => 'sci-fi',
-					'Seinen' => 'seinen',
-					'Shoujo' => 'shoujo',
-					'Shoujoai' => 'shoujoai',
-					'Shounen' => 'shounen',
-					'Shounenai' => 'shounenai',
-					'Slice of Life' => 'slice-of-life',
-					'Smut' => 'smut',
-					'Sports' => 'sports',
-					'Super Power' => 'super-power',
-					'Supernatural' => 'supernatural',
-					'Tragedy' => 'tragedy',
-					'Vampire' => 'vampire',
-					'Yaoi' => 'yaoi',
-					'Yuri' => 'yuri'
-				),
-			'exampleValue' => 'All',
-			'title' => 'Select your category'
-			)
-		),
-		'Get manga updates' => array(
-			'path' => array(
-				'name' => 'Path',
-				'required' => true,
-				'pattern' => '[a-zA-Z0-9-_]*',
-				'exampleValue' => 'bleach',
-				'title' => 'URL part of desired manga. e.g= bleach OR umi-no-kishidan'
-			),
-			'limit' => array(
-				'name' => 'Limit',
-				'type' => 'number',
-				'defaultValue' => 10,
-				'title' => 'Number of items to return [-1 returns all]'
-			)
-		)
-	);
-
-	private $request = '';
-
-	public function collectData(){
-		// We'll use the DOM parser for this as it makes navigation easier
-		$html = getContents($this->getURI());
-		if(!$html) {
-			returnClientError('Could not receive data for ' . $path . '!');
-		}
-		libxml_use_internal_errors(true);
-		$doc = new DomDocument;
-		@$doc->loadHTML($html);
-		libxml_clear_errors();
-
-		// Navigate via XPath
-		$xpath = new DomXPath($doc);
-
-		$this->request = '';
-		switch($this->queriedContext) {
-		case 'Get latest updates':
-			$this->request = 'Latest updates';
-			$this->getLatestUpdates($xpath);
-			break;
-		case 'Get popular mangas':
-			// Find manga name within "Popular mangas for ..."
-			$pagetitle = $xpath->query(".//*[@id='bodyalt']/h1")->item(0)->nodeValue;
-			$this->request = substr($pagetitle, 0, strrpos($pagetitle, ' -'));
-			$this->getPopularMangas($xpath);
-			break;
-		case 'Get manga updates':
-			$limit = $this->getInput('limit');
-			if(empty($limit)) {
-				$limit = self::PARAMETERS[$this->queriedContext]['limit']['defaultValue'];
-			}
-
-			$this->request = $xpath->query(".//*[@id='mangaproperties']//*[@class='aname']")
-				->item(0)
-				->nodeValue;
-
-			$this->getMangaUpdates($xpath, $limit);
-			break;
-		}
-
-		// Return some dummy-data if no content available
-		if(empty($this->items)) {
-			$item = array();
-			$item['content'] = '<p>No updates available</p>';
-
-			$this->items[] = $item;
-		}
-	}
-
-	private function getLatestUpdates($xpath){
-		// Query each item (consists of Manga + chapters)
-		$nodes = $xpath->query("//*[@id='latestchapters']/table//td");
-
-		foreach ($nodes as $node) {
-			// Query the manga
-			$manga = $xpath->query("a[@class='chapter']", $node)->item(0);
-
-			// Collect the chapters for each Manga
-			$chapters = $xpath->query("a[@class='chaptersrec']", $node);
-
-			if (isset($manga) && $chapters->length >= 1) {
-				$item = array();
-				$item['uri'] = self::URI . htmlspecialchars($manga->getAttribute('href'));
-				$item['title'] = htmlspecialchars($manga->nodeValue);
-
-				// Add each chapter to the feed
-				$item['content'] = '';
-
-				foreach ($chapters as $chapter) {
-					if($item['content'] <> '') {
-						$item['content'] .= '<br>';
-					}
-					$item['content'] .= "<a href='"
-						. self::URI
-						. htmlspecialchars($chapter->getAttribute('href'))
-						. "'>"
-						. htmlspecialchars($chapter->nodeValue)
-						. '</a>';
-				}
-
-				$this->items[] = $item;
-			}
-		}
-	}
-
-	private function getPopularMangas($xpath){
-		// Query all mangas
-		$mangas = $xpath->query("//*[@id='mangaresults']/*[@class='mangaresultitem']");
-
-		foreach ($mangas as $manga) {
-
-			// The thumbnail is encrypted in a css-style...
-			// format: "background-image:url('<the part which is actually interesting>')"
-			$mangaimgelement = $xpath->query(".//*[@class='imgsearchresults']", $manga)
-				->item(0)
-				->getAttribute('style');
-			$thumbnail = substr($mangaimgelement, 22, strlen($mangaimgelement) - 24);
-
-			$item = array();
-			$item['title'] = htmlspecialchars($xpath->query(".//*[@class='manga_name']//a", $manga)
-				->item(0)
-				->nodeValue);
-			$item['uri'] = self::URI . $xpath->query(".//*[@class='manga_name']//a", $manga)
-				->item(0)
-				->getAttribute('href');
-			$item['author'] = htmlspecialchars($xpath->query("//*[@class='author_name']", $manga)
-				->item(0)
-				->nodeValue);
-			$item['chaptercount'] = $xpath->query(".//*[@class='chapter_count']", $manga)
-				->item(0)
-				->nodeValue;
-			$item['genre'] = htmlspecialchars($xpath->query(".//*[@class='manga_genre']", $manga)
-				->item(0)
-				->nodeValue);
-			$item['content'] = <<<EOD
-<a href="{$item['uri']}"><img src="{$thumbnail}" alt="{$item['title']}" /></a>
-<p>{$item['genre']}</p>
-<p>{$item['chaptercount']}</p>
-EOD;
-			$this->items[] = $item;
-		}
-	}
-
-	private function getMangaUpdates($xpath, $limit){
-		$query = "(.//*[@id='listing']//tr)[position() > 1]";
-
-		if($limit !== -1) {
-			$query = "(.//*[@id='listing']//tr)[position() > 1][position() > last() - {$limit}]";
-		}
-
-		$chapters = $xpath->query($query);
-
-		foreach ($chapters as $chapter) {
-			$item = array();
-			$item['title'] = htmlspecialchars($xpath->query('td[1]', $chapter)
-				->item(0)
-				->nodeValue);
-			$item['uri'] = self::URI . $xpath->query('td[1]/a', $chapter)
-				->item(0)
-				->getAttribute('href');
-			$item['timestamp'] = strtotime($xpath->query('td[2]', $chapter)
-				->item(0)
-				->nodeValue);
-			array_unshift($this->items, $item);
-		}
-	}
-
-	public function getURI(){
-		switch($this->queriedContext) {
-		case 'Get latest updates':
-			$path = 'latest';
-			break;
-		case 'Get popular mangas':
-			$path = 'popular';
-			if($this->getInput('category') !== 'all') {
-				$path .= '/' . $this->getInput('category');
-			}
-			break;
-		case 'Get manga updates':
-			$path = $this->getInput('path');
-			break;
-		default: return parent::getURI();
-		}
-		return self::URI . '/' . $path;
-	}
-
-	public function getName(){
-		return (!empty($this->request) ? $this->request . ' - ' : '') . 'Mangareader Bridge';
-	}
-}

bridges/MarktplaatsBridge.php

@@ -17,12 +17,14 @@ class MarktplaatsBridge extends BridgeAbstract {
 				'name' => 'zipcode',
 				'type' => 'text',
 				'required' => false,
+				'exampleValue' => '1013AA',
 				'title' => 'Zip code for location limited searches',
 			),
 			'd' => array(
 				'name' => 'distance',
 				'type' => 'number',
 				'required' => false,
+				'exampleValue' => '100000',
 				'title' => 'The distance in meters from the zipcode',
 			),
 			'f' => array(
@@ -77,7 +79,7 @@ class MarktplaatsBridge extends BridgeAbstract {
 			}
 		}
 		$url = 'https://www.marktplaats.nl/lrp/api/search?query=' . urlencode($this->getInput('q')) . $query;
-		$jsonString = getSimpleHTMLDOM($url, 900);
+		$jsonString = getSimpleHTMLDOM($url);
 		$jsonObj = json_decode($jsonString);
 		foreach($jsonObj->listings as $listing) {
 			if(!$excludeGlobal || $listing->location->distanceMeters >= 0) {

bridges/MilbooruBridge.php

@@ -1,5 +1,4 @@
 <?php
-require_once('Shimmie2Bridge.php');
 
 class MilbooruBridge extends Shimmie2Bridge {
 

bridges/MixCloudBridge.php

@@ -5,6 +5,7 @@ class MixCloudBridge extends BridgeAbstract {
 	const MAINTAINER = 'Alexis CHEMEL';
 	const NAME = 'MixCloud';
 	const URI = 'https://www.mixcloud.com';
+	const API_URI = 'https://api.mixcloud.com/';
 	const CACHE_TIMEOUT = 3600; // 1h
 	const DESCRIPTION = 'Returns latest musics on user stream';
 
@@ -24,30 +25,37 @@ class MixCloudBridge extends BridgeAbstract {
 		return parent::getName();
 	}
 
+	private static function compareDate($stream1, $stream2) {
+		return (strtotime($stream1['timestamp']) < strtotime($stream2['timestamp']) ? 1 : -1);
+	}
+
 	public function collectData(){
-		ini_set('user_agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0');
+		$user = urlencode($this->getInput('u'));
+		// Get Cloudcasts
+		$mixcloudUri = self::API_URI . $user . '/cloudcasts/';
+		$content = getContents($mixcloudUri);
+		$casts = json_decode($content)->data;
 
-		$html = getSimpleHTMLDOM(self::URI . '/' . $this->getInput('u'));
+		// Get Listens
+		$mixcloudUri = self::API_URI . $user . '/listens/';
+		$content = getContents($mixcloudUri);
+		$listens = json_decode($content)->data;
 
-		foreach($html->find('section.card') as $element) {
+		$streams = array_merge($casts, $listens);
 
+		foreach($streams as $stream) {
 			$item = array();
 
-			$item['uri'] = self::URI . $element->find('hgroup.card-title h1 a', 0)->getAttribute('href');
-			$item['title'] = html_entity_decode(
-				$element->find('hgroup.card-title h1 a span', 0)->getAttribute('title'),
-				ENT_QUOTES
-			);
-
-			$image = $element->find('a.album-art img', 0);
-
-			if($image) {
-				$item['content'] = '<img src="' . $image->getAttribute('src') . '" />';
-			}
-
-			$item['author'] = trim($element->find('hgroup.card-title h2 a', 0)->innertext);
+			$item['uri'] = $stream->url;
+			$item['title'] = $stream->name;
+			$item['content'] = '<img src="' . $stream->pictures->thumbnail . '" />';
+			$item['author'] = $stream->user->name;
+			$item['timestamp'] = $stream->created_time;
 
 			$this->items[] = $item;
 		}
+
+		// Sort items by date
+		usort($this->items, array('MixCloudBridge', 'compareDate'));
 	}
 }

bridges/MsnMondeBridge.php

@@ -20,6 +20,8 @@ class MsnMondeBridge extends BridgeAbstract {
 		$html = getSimpleHTMLDOM($this->getURI());
 
 		$limit = 0;
+
+		// TODO: fix why articles is empty
 		foreach($html->find('.smalla') as $article) {
 			if($limit < 10) {
 				$item = array();

bridges/MspabooruBridge.php

@@ -1,12 +1,14 @@
 <?php
-require_once('GelbooruBridge.php');
 
 class MspabooruBridge extends GelbooruBridge {
 
 	const MAINTAINER = 'mitsukarenai';
 	const NAME = 'Mspabooru';
-	const URI = 'http://mspabooru.com/';
+	const URI = 'https://mspabooru.com/';
 	const DESCRIPTION = 'Returns images from given page';
-	const PIDBYPAGE = 50;
 
+	protected function buildThumbnailURI($element){
+		return $this->getURI() . 'thumbnails/' . $element->directory
+		. '/thumbnail_' . $element->image;
+	}
 }

bridges/MydealsBridge.php

@@ -1,6 +1,5 @@
 <?php
 
-require_once(__DIR__ . '/DealabsBridge.php');
 class MydealsBridge extends PepperBridgeAbstract {
 
 	const NAME = 'Mydeals bridge';
@@ -12,7 +11,7 @@ class MydealsBridge extends PepperBridgeAbstract {
 			'q' => array(
 				'name' => 'Stichworten',
 				'type' => 'text',
-				'exampleValue' => 'watch',
+				'exampleValue' => 'lamp',
 				'required' => true
 			),
 			'hide_expired' => array(
@@ -2002,8 +2001,8 @@ class MydealsBridge extends PepperBridgeAbstract {
 				'name' => 'URL der Diskussion',
 				'type' => 'text',
 				'required' => true,
-				'title' => 'URL-Diskussion zu überwachen:  https://www.mydealz.de/diskussion/title-123',
-				'exampleValue' => '://www.mydealz.de/diskussion/title-123',
+				'title' => 'URL-Diskussion zu überwachen: https://www.mydealz.de/diskussion/title-123',
+				'exampleValue' => 'https://www.mydealz.de/diskussion/anleitung-wie-schreibe-ich-einen-deal-1658317',
 				),
 			'only_with_url' => array(
 				'name' => 'Kommentare ohne URL ausschließen',

bridges/N26Bridge.php

@@ -8,28 +8,36 @@ class N26Bridge extends BridgeAbstract
 	const CACHE_TIMEOUT = 1800;
 	const DESCRIPTION = 'Returns recent blog posts from N26.';
 
+	public function collectData()
+	{
+		$limit = 5;
+		$url = 'https://n26.com/en-eu/blog/all';
+		$html = getSimpleHTMLDOM($url);
+
+		$articles = $html->find('div[class="bl bm"]');
+
+		foreach($articles as $article) {
+			$item = array();
+
+			$itemUrl = self::URI . $article->find('a', 1)->href;
+			$item['uri'] = $itemUrl;
+
+			$item['title'] = $article->find('a', 1)->plaintext;
+
+			$fullArticle = getSimpleHTMLDOM($item['uri']);
+
+			$createdAt = $fullArticle->find('time', 0);
+			$item['timestamp'] = strtotime($createdAt->plaintext);
+
+			$this->items[] = $item;
+			if (count($this->items) >= $limit) {
+				break;
+			}
+		}
+	}
+
 	public function getIcon()
 	{
 		return 'https://n26.com/favicon.ico';
 	}
-
-	public function collectData()
-	{
-		$html = getSimpleHTMLDOM(self::URI . '/en-eu/blog-archive');
-
-		foreach($html->find('div[class="ag ah ai aj bs bt dx ea fo gx ie if ih ii ij ik s"]') as $article) {
-			$item = array();
-
-			$item['uri'] = self::URI . $article->find('h2 a', 0)->href;
-			$item['title'] = $article->find('h2 a', 0)->plaintext;
-
-			$fullArticle = getSimpleHTMLDOM($item['uri']);
-
-			$dateElement = $fullArticle->find('time', 0);
-			$item['timestamp'] = strtotime($dateElement->plaintext);
-			$item['content'] = $fullArticle->find('div[class="af ag ah ai an"]', 1)->innertext;
-
-			$this->items[] = $item;
-		}
-	}
 }