mirror/archivebox
1
0
Fork 0
mirror of https://github.com/pirate/ArchiveBox.git synced 2025-08-27 00:05:27 +02:00
Code Issues Releases Wiki Activity

exclude links that are in blacklist

Browse Source
This commit is contained in:
mlazana
2019-03-24 14:40:26 +02:00
parent 417ee9e302
commit 4d10568477
2 changed files with 18 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
archivebox/config.py
View File

@@ -47,7 +47,7 @@ WGET_BINARY = os.getenv('WGET_BINARY', 'wget')
YOUTUBEDL_BINARY = os.getenv('YOUTUBEDL_BINARY', 'youtube-dl') YOUTUBEDL_BINARY = os.getenv('YOUTUBEDL_BINARY', 'youtube-dl')
CHROME_BINARY = os.getenv('CHROME_BINARY', None) CHROME_BINARY = os.getenv('CHROME_BINARY', None)
URL_BLACKLIST = os.getenv('URL_BLACKLIST', '.*youtube.com.*,.*facebook.com/.*,.*.exe') URL_BLACKLIST = os.getenv('URL_BLACKLIST', '.*youtube.com.*,.*facebook.com/.*,.*.exe')
try: try:
OUTPUT_DIR = os.path.abspath(os.getenv('OUTPUT_DIR')) OUTPUT_DIR = os.path.abspath(os.getenv('OUTPUT_DIR'))
@@ -270,6 +270,7 @@ except:
URL_BLACKLIST = re.compile( URL_BLACKLIST = re.compile(
r'(.*\.youtube\.com)|' r'(.*\.youtube\.com)|'
r'(.*\.facebook\.com)|'
r'(.*\.amazon\.com)|' r'(.*\.amazon\.com)|'
r'(.*\.reddit\.com)', r'(.*\.reddit\.com)',
re.IGNORECASE, re.IGNORECASE,

18
archivebox/links.py
View File

@@ -28,13 +28,19 @@ from util import (
check_links_structure, check_links_structure,
) )
from config import (
URL_BLACKLIST,
)
def validate_links(links): def validate_links(links):
check_links_structure(links) check_links_structure(links)
links = archivable_links(links) # remove chrome://, about:, mailto: etc. links = archivable_links(links) # remove chrome://, about:, mailto: etc.
links = uniquefied_links(links) # merge/dedupe duplicate timestamps & urls links = uniquefied_links(links) # merge/dedupe duplicate timestamps & urls
links = sorted_links(links) # deterministically sort the links based on timstamp, url links = sorted_links(links) # deterministically sort the links based on timstamp, url
links = exclude_links(links) # exclude links that are in blacklist
print(links)
if not links: if not links:
print('[X] No links found :(') print('[X] No links found :(')
raise SystemExit(1) raise SystemExit(1)
@@ -42,7 +48,8 @@ def validate_links(links):
for link in links: for link in links:
link['title'] = unescape(link['title'].strip()) if link['title'] else None link['title'] = unescape(link['title'].strip()) if link['title'] else None
check_link_structure(link) check_link_structure(link)
print("FINAL LIST", list(links))
return list(links) return list(links)
@@ -115,3 +122,10 @@ def lowest_uniq_timestamp(used_timestamps, timestamp):
new_timestamp = '{}.{}'.format(timestamp, nonce) new_timestamp = '{}.{}'.format(timestamp, nonce)
return new_timestamp return new_timestamp
def exclude_links(links):
""" exclude links that are in blacklist"""
links = [link for link in links if not URL_BLACKLIST.match(link['url'])]
return links