mirror/archivebox
1
0
Fork 0
mirror of https://github.com/pirate/ArchiveBox.git synced 2025-08-21 05:41:54 +02:00
Code Issues Releases Wiki Activity

ignore robots.txt when using wget

Browse Source
This commit is contained in:
Nick Sweeting
2019-02-06 22:06:28 -08:00
parent 56d382235f
commit e6d5cd4432
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
archivebox/archive_methods.py
View File

@@ -217,6 +217,7 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, warc=FETCH_WARC
'--backup-converted', '--backup-converted',
'--span-hosts', '--span-hosts',
'--no-parent', '--no-parent',
'-e', 'robots=off',
'--restrict-file-names=unix', '--restrict-file-names=unix',
'--timeout={}'.format(timeout), '--timeout={}'.format(timeout),
*(() if warc else ('--timestamping',)), *(() if warc else ('--timestamping',)),