ignore robots.txt when using wget

2025-08-18 20:31:38 +02:00 · 2019-02-06 22:06:28 -08:00
parent 56d382235f
commit e6d5cd4432
1 changed files with 1 additions and 0 deletions
--- a/archivebox/archive_methods.py
+++ b/archivebox/archive_methods.py
@@ -217,6 +217,7 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, warc=FETCH_WARC
        '--backup-converted',
        '--span-hosts',
        '--no-parent',
+        '-e', 'robots=off',
        '--restrict-file-names=unix',
        '--timeout={}'.format(timeout),
        *(() if warc else ('--timestamping',)),