From 050cd9c8616cae31e388ecb4a312e107decc1f57 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 May 2019 15:28:55 -0400 Subject: [PATCH 01/12] add license to manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index ddb780e6..9100b772 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ +include LICENSE include archivebox/VERSION graft archivebox/themes graft archivebox/themes/static From 2440c1c1bf5901dc058baee0a9aeac78f2babcc8 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 May 2019 15:30:42 -0400 Subject: [PATCH 02/12] just use simple version instead of git hash --- setup.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/setup.py b/setup.py index adca4887..310c9691 100644 --- a/setup.py +++ b/setup.py @@ -8,19 +8,10 @@ with open("README.md", "r") as fh: script_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) VERSION = open(os.path.join(script_dir, 'archivebox', 'VERSION'), 'r').read().strip() -try: - GIT_HEAD = open(os.path.join(script_dir, '.git', 'HEAD'), 'r').read().strip().split(': ')[1] - GIT_SHA = open(os.path.join(script_dir, '.git', GIT_HEAD), 'r').read().strip()[:9] - PYPI_VERSION = "{}+{}".format(VERSION, GIT_SHA) -except: - PYPI_VERSION = VERSION - -with open(os.path.join(script_dir, 'archivebox', 'VERSION'), 'w+') as f: - f.write(PYPI_VERSION) setuptools.setup( name="archivebox", - version=PYPI_VERSION, + version=VERSION, author="Nick Sweeting", author_email="git@nicksweeting.com", description="The self-hosted internet archive.", From 7ee837c12784e196865a06ce09b84a91f1c1274d Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 May 2019 15:32:18 -0400 Subject: [PATCH 03/12] add twine to packages --- Pipfile | 1 + Pipfile.lock | 117 ++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 84 insertions(+), 34 deletions(-) diff --git a/Pipfile b/Pipfile index 194f81db..7c7e05ce 100644 --- a/Pipfile +++ b/Pipfile @@ -12,6 +12,7 @@ setuptools = "*" sphinx = "*" recommonmark = "*" sphinx-rtd-theme = "*" +twine = "*" [packages] dataclasses = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 663654b1..64a9bae2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8ac4f9e5cd266406a861a283b321b9eee0ca469638f838e93467403ef2f0594d" + "sha256": "5a1618caef76ff53b66c5e8674d8e639d25f75068f7026ad799e217d307628fc" }, "pipfile-spec": 6, "requires": { @@ -64,11 +64,11 @@ }, "django": { "hashes": [ - "sha256:7c3543e4fb070d14e10926189a7fcf42ba919263b7473dceaefce34d54e8a119", - "sha256:a2814bffd1f007805b19194eb0b9a331933b82bd5da1c3ba3d7b7ba16e06dc4b" + "sha256:6fcc3cbd55b16f9a01f37de8bcbe286e0ea22e87096557f1511051780338eaea", + "sha256:bb407d0bb46395ca1241f829f5bd03f7e482f97f7d1936e26e98dacb201ed4ec" ], "index": "pypi", - "version": "==2.2" + "version": "==2.2.1" }, "django-extensions": { "hashes": [ @@ -203,11 +203,11 @@ }, "youtube-dl": { "hashes": [ - "sha256:46f6e30c673ba71de84748dad4c264d1b6fb30beebf1ef834846a651b4524a78", - "sha256:b20d110e1bed8d16f5771bb938ab6e5da67f08af62b599af65301cca290f2e15" + "sha256:31844229a4f4d7003e03ab309ff2caff1b16ce0acbd3cfb7a13276058af13056", + "sha256:a751bd293e2d7ee963910de14b3eb95b88837021899be488fade0b8abe815650" ], "index": "pypi", - "version": "==2019.4.24" + "version": "==2019.4.30" } }, "develop": { @@ -240,6 +240,13 @@ ], "version": "==0.1.0" }, + "bleach": { + "hashes": [ + "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16", + "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa" + ], + "version": "==3.1.0" + }, "certifi": { "hashes": [ "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", @@ -256,10 +263,10 @@ }, "commonmark": { "hashes": [ - "sha256:9f6dda7876b2bb88dd784440166f4bc8e56cb2b2551264051123bacb0b6c1d8a", - "sha256:abcbc854e0eae5deaf52ae5e328501b78b4a0758bf98ac8bb792fce993006084" + "sha256:14c3df31e8c9c463377e287b2a1eefaa6019ab97b22dad36e2f32be59d61d68d", + "sha256:867fc5db078ede373ab811e16b6789e9d033b15ccd7296f370ca52d1ee792ce0" ], - "version": "==0.8.1" + "version": "==0.9.0" }, "decorator": { "hashes": [ @@ -449,6 +456,13 @@ ], "version": "==0.7.5" }, + "pkginfo": { + "hashes": [ + "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb", + "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32" + ], + "version": "==1.5.0.1" + }, "prompt-toolkit": { "hashes": [ "sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780", @@ -499,6 +513,13 @@ ], "version": "==2019.1" }, + "readme-renderer": { + "hashes": [ + "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f", + "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d" + ], + "version": "==24.0" + }, "recommonmark": { "hashes": [ "sha256:a520b8d25071a51ae23a27cf6252f2fe387f51bdc913390d83b2b50617f5bb48", @@ -514,6 +535,13 @@ ], "version": "==2.21.0" }, + "requests-toolbelt": { + "hashes": [ + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" + ], + "version": "==0.9.1" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", @@ -586,6 +614,13 @@ ], "version": "==1.1.3" }, + "tqdm": { + "hashes": [ + "sha256:d385c95361699e5cf7622485d9b9eae2d4864b21cd5a2374a9c381ffed701021", + "sha256:e22977e3ebe961f72362f6ddfb9197cc531c9737aaf5f607ef09740c849ecd05" + ], + "version": "==4.31.1" + }, "traitlets": { "hashes": [ "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835", @@ -593,30 +628,37 @@ ], "version": "==4.3.2" }, + "twine": { + "hashes": [ + "sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446", + "sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc" + ], + "index": "pypi", + "version": "==1.13.0" + }, "typed-ast": { "hashes": [ - "sha256:04894d268ba6eab7e093d43107869ad49e7b5ef40d1a94243ea49b352061b200", - "sha256:16616ece19daddc586e499a3d2f560302c11f122b9c692bc216e821ae32aa0d0", - "sha256:252fdae740964b2d3cdfb3f84dcb4d6247a48a6abe2579e8029ab3be3cdc026c", - "sha256:2af80a373af123d0b9f44941a46df67ef0ff7a60f95872412a145f4500a7fc99", - "sha256:2c88d0a913229a06282b285f42a31e063c3bf9071ff65c5ea4c12acb6977c6a7", - "sha256:2ea99c029ebd4b5a308d915cc7fb95b8e1201d60b065450d5d26deb65d3f2bc1", - "sha256:3d2e3ab175fc097d2a51c7a0d3fda442f35ebcc93bb1d7bd9b95ad893e44c04d", - "sha256:4766dd695548a15ee766927bf883fb90c6ac8321be5a60c141f18628fb7f8da8", - "sha256:56b6978798502ef66625a2e0f80cf923da64e328da8bbe16c1ff928c70c873de", - "sha256:5cddb6f8bce14325b2863f9d5ac5c51e07b71b462361fd815d1d7706d3a9d682", - "sha256:644ee788222d81555af543b70a1098f2025db38eaa99226f3a75a6854924d4db", - "sha256:64cf762049fc4775efe6b27161467e76d0ba145862802a65eefc8879086fc6f8", - "sha256:68c362848d9fb71d3c3e5f43c09974a0ae319144634e7a47db62f0f2a54a7fa7", - "sha256:6c1f3c6f6635e611d58e467bf4371883568f0de9ccc4606f17048142dec14a1f", - "sha256:b213d4a02eec4ddf622f4d2fbc539f062af3788d1f332f028a2e19c42da53f15", - "sha256:bb27d4e7805a7de0e35bd0cb1411bc85f807968b2b0539597a49a23b00a622ae", - "sha256:c9d414512eaa417aadae7758bc118868cd2396b0e6138c1dd4fda96679c079d3", - "sha256:f0937165d1e25477b01081c4763d2d9cdc3b18af69cb259dd4f640c9b900fe5e", - "sha256:fb96a6e2c11059ecf84e6741a319f93f683e440e341d4489c9b161eca251cf2a", - "sha256:fc71d2d6ae56a091a8d94f33ec9d0f2001d1cb1db423d8b4355debfe9ce689b7" + "sha256:132eae51d6ef3ff4a8c47c393a4ef5ebf0d1aecc96880eb5d6c8ceab7017cc9b", + "sha256:18141c1484ab8784006c839be8b985cfc82a2e9725837b0ecfa0203f71c4e39d", + "sha256:2baf617f5bbbfe73fd8846463f5aeafc912b5ee247f410700245d68525ec584a", + "sha256:3d90063f2cbbe39177e9b4d888e45777012652d6110156845b828908c51ae462", + "sha256:4304b2218b842d610aa1a1d87e1dc9559597969acc62ce717ee4dfeaa44d7eee", + "sha256:4983ede548ffc3541bae49a82675996497348e55bafd1554dc4e4a5d6eda541a", + "sha256:5315f4509c1476718a4825f45a203b82d7fdf2a6f5f0c8f166435975b1c9f7d4", + "sha256:6cdfb1b49d5345f7c2b90d638822d16ba62dc82f7616e9b4caa10b72f3f16649", + "sha256:7b325f12635598c604690efd7a0197d0b94b7d7778498e76e0710cd582fd1c7a", + "sha256:8d3b0e3b8626615826f9a626548057c5275a9733512b137984a68ba1598d3d2f", + "sha256:8f8631160c79f53081bd23446525db0bc4c5616f78d04021e6e434b286493fd7", + "sha256:912de10965f3dc89da23936f1cc4ed60764f712e5fa603a09dd904f88c996760", + "sha256:b010c07b975fe853c65d7bbe9d4ac62f1c69086750a574f6292597763781ba18", + "sha256:c908c10505904c48081a5415a1e295d8403e353e0c14c42b6d67f8f97fae6616", + "sha256:c94dd3807c0c0610f7c76f078119f4ea48235a953512752b9175f9f98f5ae2bd", + "sha256:ce65dee7594a84c466e79d7fb7d3303e7295d16a83c22c7c4037071b059e2c21", + "sha256:eaa9cfcb221a8a4c2889be6f93da141ac777eb8819f077e1d09fb12d00a09a93", + "sha256:f3376bc31bad66d46d44b4e6522c5c21976bf9bca4ef5987bb2bf727f4506cbb", + "sha256:f9202fa138544e13a4ec1a6792c35834250a85958fde1251b6a22e07d1260ae7" ], - "version": "==1.3.4" + "version": "==1.3.5" }, "typing-extensions": { "hashes": [ @@ -628,10 +670,10 @@ }, "urllib3": { "hashes": [ - "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", - "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" + "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", + "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" ], - "version": "==1.24.2" + "version": "==1.24.3" }, "wcwidth": { "hashes": [ @@ -639,6 +681,13 @@ "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" ], "version": "==0.1.7" + }, + "webencodings": { + "hashes": [ + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + ], + "version": "==0.5.1" } } } From ef77a6d43f69a60d4d29cc74e61833a3fd7b39c4 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 May 2019 15:39:55 -0400 Subject: [PATCH 04/12] add some missing fields to setup.py --- setup.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 310c9691..32809c76 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,15 @@ import os import setuptools -with open("README.md", "r") as fh: - long_description = fh.read() +BASE_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) +PYTHON_DIR = os.path.join(BASE_DIR, 'archivebox') +with open('README.md', "r") as f: + README = f.read() -script_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) +with open(os.path.join(PYTHON_DIR, 'VERSION'), 'r') as f: + VERSION = f.read().strip() -VERSION = open(os.path.join(script_dir, 'archivebox', 'VERSION'), 'r').read().strip() setuptools.setup( name="archivebox", @@ -15,9 +17,10 @@ setuptools.setup( author="Nick Sweeting", author_email="git@nicksweeting.com", description="The self-hosted internet archive.", - long_description=long_description, + long_description=README, long_description_content_type="text/markdown", url="https://github.com/pirate/ArchiveBox", + license='MIT', project_urls={ 'Documentation': 'https://github.com/pirate/ArchiveBox/Wiki', 'Community': 'https://github.com/pirate/ArchiveBox/wiki/Web-Archiving-Community', @@ -27,7 +30,7 @@ setuptools.setup( 'Changelog': 'https://github.com/pirate/ArchiveBox/wiki/Changelog', 'Patreon': 'https://github.com/pirate/ArchiveBox/wiki/Donations', }, - packages=setuptools.find_packages(), + packages=['archivebox',], python_requires='>=3.6', install_requires=[ "dataclasses==0.6", From ba21ff46f3b65809f47a5b37920cc8dbe402355d Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 May 2019 16:10:34 -0400 Subject: [PATCH 05/12] reverse the url order --- setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 32809c76..723aeaa1 100644 --- a/setup.py +++ b/setup.py @@ -22,15 +22,15 @@ setuptools.setup( url="https://github.com/pirate/ArchiveBox", license='MIT', project_urls={ - 'Documentation': 'https://github.com/pirate/ArchiveBox/Wiki', - 'Community': 'https://github.com/pirate/ArchiveBox/wiki/Web-Archiving-Community', - 'Source': 'https://github.com/pirate/ArchiveBox', - 'Bug Tracker': 'https://github.com/pirate/ArchiveBox/issues', - 'Roadmap': 'https://github.com/pirate/ArchiveBox/wiki/Roadmap', + 'Donate': 'https://github.com/pirate/ArchiveBox/wiki/Donations', 'Changelog': 'https://github.com/pirate/ArchiveBox/wiki/Changelog', - 'Patreon': 'https://github.com/pirate/ArchiveBox/wiki/Donations', + 'Roadmap': 'https://github.com/pirate/ArchiveBox/wiki/Roadmap', + 'Bug Tracker': 'https://github.com/pirate/ArchiveBox/issues', + 'Source': 'https://github.com/pirate/ArchiveBox', + 'Community': 'https://github.com/pirate/ArchiveBox/wiki/Web-Archiving-Community', + 'Documentation': 'https://github.com/pirate/ArchiveBox/Wiki', }, - packages=['archivebox',], + packages=setuptools.find_packages(), python_requires='>=3.6', install_requires=[ "dataclasses==0.6", From e0489d77e71a60f2a66d2ede9f774d8fa0eea632 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 May 2019 17:39:45 -0400 Subject: [PATCH 06/12] bump the version --- archivebox/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/VERSION b/archivebox/VERSION index 1d0ba9ea..267577d4 100644 --- a/archivebox/VERSION +++ b/archivebox/VERSION @@ -1 +1 @@ -0.4.0 +0.4.1 From d398bd59b017a28fd3911e32550c5d20ec1f09c2 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 May 2019 17:51:21 -0400 Subject: [PATCH 07/12] switch to pure Manifest-based package includes --- MANIFEST.in | 9 ++------- setup.py | 14 +------------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 9100b772..a73ef711 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,4 @@ include LICENSE +include README.md include archivebox/VERSION -graft archivebox/themes -graft archivebox/themes/static -graft archivebox/themes/admin -graft archivebox/themes/default -graft archivebox/themes/default/static -graft archivebox/themes/legacy -graft archivebox/themes/legacy/static +recursive-include archivebox/themes * diff --git a/setup.py b/setup.py index 723aeaa1..f23ae7b5 100644 --- a/setup.py +++ b/setup.py @@ -55,19 +55,7 @@ setuptools.setup( 'archivebox = archivebox.__main__:main', ], }, - package_data={ - 'archivebox': [ - # Manifest.ini must correspond 1:1 with this list - 'VERSION', - 'themes/*', - 'themes/static/*', - 'themes/admin/*' - 'themes/default/*' - 'themes/default/static/*' - 'themes/legacy/*', - 'themes/legacy/static/*', - ], - }, + include_package_data=True, classifiers=[ "Development Status :: 4 - Beta", From d016f1efb50a6ba13b84a05754fd92e631e90346 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 May 2019 17:52:43 -0400 Subject: [PATCH 08/12] bump version --- archivebox/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/VERSION b/archivebox/VERSION index 267577d4..2b7c5ae0 100644 --- a/archivebox/VERSION +++ b/archivebox/VERSION @@ -1 +1 @@ -0.4.1 +0.4.2 From 3c3b2ee62167c499f7f2a047b9d635a28a58544a Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 2 May 2019 19:15:16 -0400 Subject: [PATCH 09/12] expose more django server config options --- archivebox/config/__init__.py | 29 ++++++++++++--- archivebox/config/stubs.py | 9 ++++- archivebox/core/settings.py | 43 ++++++++++------------- archivebox/core/urls.py | 6 ++++ archivebox/core/views.py | 18 +++++++++- archivebox/main.py | 6 ++-- archivebox/themes/default/main_index.html | 2 +- etc/ArchiveBox.conf.default | 13 +++++-- 8 files changed, 89 insertions(+), 37 deletions(-) diff --git a/archivebox/config/__init__.py b/archivebox/config/__init__.py index 72baec64..04b8515c 100644 --- a/archivebox/config/__init__.py +++ b/archivebox/config/__init__.py @@ -44,10 +44,19 @@ CONFIG_DEFAULTS: Dict[str, ConfigDefaultDict] = { 'TIMEOUT': {'type': int, 'default': 60}, 'MEDIA_TIMEOUT': {'type': int, 'default': 3600}, 'OUTPUT_PERMISSIONS': {'type': str, 'default': '755'}, - 'FOOTER_INFO': {'type': str, 'default': 'Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.'}, 'URL_BLACKLIST': {'type': str, 'default': None}, }, + 'SERVER_CONFIG': { + 'SECRET_KEY': {'type': str, 'default': None}, + 'ALLOWED_HOSTS': {'type': str, 'default': '*'}, + 'DEBUG': {'type': bool, 'default': False}, + 'PUBLIC_INDEX': {'type': bool, 'default': True}, + 'PUBLIC_SNAPSHOTS': {'type': bool, 'default': True}, + 'FOOTER_INFO': {'type': str, 'default': 'Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.'}, + 'ACTIVE_THEME': {'type': str, 'default': 'default'}, + }, + 'ARCHIVE_METHOD_TOGGLES': { 'SAVE_TITLE': {'type': bool, 'default': True, 'aliases': ('FETCH_TITLE',)}, 'SAVE_FAVICON': {'type': bool, 'default': True, 'aliases': ('FETCH_FAVICON',)}, @@ -313,9 +322,6 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict: with open(config_path, 'w+') as f: f.write(CONFIG_HEADER) - if not config: - return {} - config_file = ConfigParser() config_file.optionxform = str config_file.read(config_path) @@ -336,6 +342,21 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict: config_file[section] = {**existing_config, key: val} + # always make sure there's a SECRET_KEY defined for Django + existing_secret_key = None + if 'SERVER_CONFIG' in config_file and 'SECRET_KEY' in config_file['SERVER_CONFIG']: + existing_secret_key = config_file['SERVER_CONFIG']['SECRET_KEY'] + + if (not existing_secret_key) or ('not a valid secret' in existing_secret_key): + from django.utils.crypto import get_random_string + chars = 'abcdefghijklmnopqrstuvwxyz0123456789-_+!.' + random_secret_key = get_random_string(50, chars) + if 'SERVER_CONFIG' in config_file: + config_file['SERVER_CONFIG']['SECRET_KEY'] = random_secret_key + else: + config_file['SERVER_CONFIG'] = {'SECRET_KEY': random_secret_key} + + f.write(CONFIG_HEADER) config_file.write(f) try: diff --git a/archivebox/config/stubs.py b/archivebox/config/stubs.py index f7d5059a..7d3925dd 100644 --- a/archivebox/config/stubs.py +++ b/archivebox/config/stubs.py @@ -22,9 +22,16 @@ class ConfigDict(BaseConfig, total=False): TIMEOUT: int MEDIA_TIMEOUT: int OUTPUT_PERMISSIONS: str - FOOTER_INFO: str URL_BLACKLIST: Optional[str] + SECRET_KEY: str + ALLOWED_HOSTS: str + DEBUG: bool + PUBLIC_INDEX: bool + PUBLIC_SNAPSHOTS: bool + FOOTER_INFO: str + ACTIVE_THEME: str + SAVE_TITLE: bool SAVE_FAVICON: bool SAVE_WGET: bool diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index e128f8d0..463a7079 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -3,26 +3,25 @@ __package__ = 'archivebox.core' import os import sys -SECRET_KEY = '---------------- not a valid secret key ! ----------------' -DEBUG = os.getenv('DEBUG', 'False').lower() == 'true' -ALLOWED_HOSTS = ['*'] -REPO_DIR = os.path.abspath(os.path.join(os.path.abspath(__file__), os.path.pardir, os.path.pardir)) -OUTPUT_DIR = os.path.abspath(os.getenv('OUTPUT_DIR', os.curdir)) -ARCHIVE_DIR = os.path.join(OUTPUT_DIR, 'archive') -DATABASE_FILE = os.path.join(OUTPUT_DIR, 'index.sqlite3') +from ..config import ( + OUTPUT_DIR, + SECRET_KEY, + DEBUG, + ALLOWED_HOSTS, + PYTHON_DIR, + ACTIVE_THEME, + SQL_INDEX_FILENAME, +) -ACTIVE_THEME = 'default' +ALLOWED_HOSTS = ALLOWED_HOSTS.split(',') IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3] -APPEND_SLASH = True - INSTALLED_APPS = [ 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', - # 'django.contrib.sites', 'django.contrib.messages', 'django.contrib.admin', 'django.contrib.staticfiles', @@ -40,17 +39,17 @@ MIDDLEWARE = [ 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', - # 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] ROOT_URLCONF = 'core.urls' +APPEND_SLASH = True TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', 'DIRS': [ - os.path.join(REPO_DIR, 'themes', ACTIVE_THEME), - os.path.join(REPO_DIR, 'themes', 'default'), - os.path.join(REPO_DIR, 'themes'), + os.path.join(PYTHON_DIR, 'themes', ACTIVE_THEME), + os.path.join(PYTHON_DIR, 'themes', 'default'), + os.path.join(PYTHON_DIR, 'themes'), ], 'APP_DIRS': True, 'OPTIONS': { @@ -69,7 +68,7 @@ WSGI_APPLICATION = 'core.wsgi.application' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': DATABASE_FILE, + 'NAME': os.path.join(OUTPUT_DIR, SQL_INDEX_FILENAME), } } @@ -104,7 +103,7 @@ SHELL_PLUS_PRINT_SQL = False IPYTHON_ARGUMENTS = ['--no-confirm-exit', '--no-banner'] IPYTHON_KERNEL_DISPLAY_NAME = 'ArchiveBox Django Shell' if IS_SHELL: - os.environ['PYTHONSTARTUP'] = os.path.join(REPO_DIR, 'core', 'welcome_message.py') + os.environ['PYTHONSTARTUP'] = os.path.join(PYTHON_DIR, 'core', 'welcome_message.py') LANGUAGE_CODE = 'en-us' @@ -118,11 +117,7 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' STATIC_URL = '/static/' STATICFILES_DIRS = [ - os.path.join(REPO_DIR, 'themes', ACTIVE_THEME, 'static'), - os.path.join(REPO_DIR, 'themes', 'default', 'static'), - os.path.join(REPO_DIR, 'themes', 'static'), + os.path.join(PYTHON_DIR, 'themes', ACTIVE_THEME, 'static'), + os.path.join(PYTHON_DIR, 'themes', 'default', 'static'), + os.path.join(PYTHON_DIR, 'themes', 'static'), ] - -SERVE_STATIC = True - - diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py index 2a001f6b..9b4af5a5 100644 --- a/archivebox/core/urls.py +++ b/archivebox/core/urls.py @@ -22,8 +22,14 @@ urlpatterns = [ path('add/', AddLinks.as_view(), name='AddLinks'), path('static/', views.serve), + + path('accounts/login/', RedirectView.as_view(url='/admin/login/')), + path('accounts/logout/', RedirectView.as_view(url='/admin/logout/')), + path('accounts/', include('django.contrib.auth.urls')), path('admin/', admin.site.urls), + + path('', MainIndex.as_view(), name='Home'), ] diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 2c140d58..7fee7408 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -4,11 +4,18 @@ from django.shortcuts import render, redirect from django.http import HttpResponse from django.views import View, static +from django.conf import settings from core.models import Snapshot from ..index import load_main_index, load_main_index_meta -from ..config import OUTPUT_DIR, VERSION, FOOTER_INFO +from ..config import ( + OUTPUT_DIR, + VERSION, + FOOTER_INFO, + PUBLIC_INDEX, + PUBLIC_SNAPSHOTS, +) from ..util import base_url @@ -16,6 +23,9 @@ class MainIndex(View): template = 'main_index.html' def get(self, request): + if not request.user.is_authenticated and not PUBLIC_INDEX: + return redirect(f'/admin/login/?next={request.path}') + all_links = load_main_index(out_dir=OUTPUT_DIR) meta_info = load_main_index_meta(out_dir=OUTPUT_DIR) @@ -34,6 +44,9 @@ class AddLinks(View): template = 'add_links.html' def get(self, request): + if not request.user.is_authenticated and not PUBLIC_INDEX: + return redirect(f'/admin/login/?next={request.path}') + context = {} return render(template_name=self.template, request=request, context=context) @@ -54,6 +67,9 @@ class LinkDetails(View): if '/' not in path: return redirect(f'{path}/index.html') + if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS: + return redirect(f'/admin/login/?next={request.path}') + try: slug, archivefile = path.split('/', 1) except (IndexError, ValueError): diff --git a/archivebox/main.py b/archivebox/main.py index 00529743..80e4b77b 100644 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -292,14 +292,14 @@ def init(force: bool=False, out_dir: str=OUTPUT_DIR) -> None: setup_django(out_dir, check_db=False) from django.conf import settings - assert settings.DATABASE_FILE == os.path.join(out_dir, SQL_INDEX_FILENAME) - print(f' √ {settings.DATABASE_FILE}') + DATABASE_FILE = os.path.join(out_dir, SQL_INDEX_FILENAME) + print(f' √ {DATABASE_FILE}') print() for migration_line in apply_migrations(out_dir): print(f' {migration_line}') - assert os.path.exists(settings.DATABASE_FILE) + assert os.path.exists(DATABASE_FILE) # from django.contrib.auth.models import User # if IS_TTY and not User.objects.filter(is_superuser=True).exists(): diff --git a/archivebox/themes/default/main_index.html b/archivebox/themes/default/main_index.html index 4ad00be7..925c4fa4 100644 --- a/archivebox/themes/default/main_index.html +++ b/archivebox/themes/default/main_index.html @@ -190,7 +190,7 @@
Add Links   |   - Admin   |   + Admin   |   Docs
diff --git a/etc/ArchiveBox.conf.default b/etc/ArchiveBox.conf.default index 31017ce4..df9abe22 100644 --- a/etc/ArchiveBox.conf.default +++ b/etc/ArchiveBox.conf.default @@ -1,6 +1,6 @@ # This is the example default configiration file for ArchiveBox. # -# Copy example config from here into your project's ArchiveBox.conf file, +# Copy lines from here into your project's ArchiveBox.conf file and uncomment, # DO NOT EDIT THIS FILE DIRECTLY! # # See the list of all the possible options. documentation, and examples here: @@ -11,10 +11,17 @@ # ONLY_NEW = False # TIMEOUT = 60 # MEDIA_TIMEOUT = 3600 -# ACTIVE_THEME = default -# FOOTER_INFO = Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests. # URL_BLACKLIST = (://(.*\.)?facebook\.com)|(://(.*\.)?ebay\.com)|(.*\.exe$) +[SERVER_CONFIG] +# SECRET_KEY = ---------------- not a valid secret key ! ---------------- +# DEBUG = False +# PUBLIC_INDEX = True +# PUBLIC_SNAPSHOTS = True +# FOOTER_INFO = Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests. +# ACTIVE_THEME = default + + [ARCHIVE_METHOD_TOGGLES] # SAVE_TITLE = True # SAVE_FAVICON = True From ca9c9ef956e9ecc7e4f3b07ed6ea74cd434ebb68 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 6 May 2019 17:16:20 -0400 Subject: [PATCH 10/12] add warning about running manage.py directly --- archivebox/manage.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/archivebox/manage.py b/archivebox/manage.py index 52c21895..3976c2c2 100755 --- a/archivebox/manage.py +++ b/archivebox/manage.py @@ -3,6 +3,20 @@ import os import sys if __name__ == '__main__': + # if you're a developer working on archivebox, still prefer the archivebox + # versions of ./manage.py commands whenever possible. When that's not possible + # (e.g. makemigrations), you can comment out this check temporarily + + print("[X] Don't run ./manage.py directly, use the archivebox CLI instead e.g.:") + print(' archivebox manage createsuperuser') + print() + print(' Hint: Use these archivebox commands instead of the ./manage.py equivalents:') + print(' archivebox init (migrates the databse to latest version)') + print(' archivebox server (runs the Django web server)') + print(' archivebox shell (opens an iPython Django shell with all models imported)') + print(' archivebox manage [cmd] (any other management commands)') + raise SystemExit(2) + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') try: from django.core.management import execute_from_command_line From 0bb216ce026e15a592a21d6eaa7e46a95aef7bc5 Mon Sep 17 00:00:00 2001 From: Mashiat Sarker Shakkhar Date: Mon, 26 Aug 2019 17:25:22 -0400 Subject: [PATCH 11/12] util.py: Use dateparser to parse date strings. --- archivebox/util.py | 46 ++-------------------------------------------- 1 file changed, 2 insertions(+), 44 deletions(-) diff --git a/archivebox/util.py b/archivebox/util.py index 8b606db1..5a4ec88c 100644 --- a/archivebox/util.py +++ b/archivebox/util.py @@ -10,6 +10,7 @@ from urllib.request import Request, urlopen from urllib.parse import urlparse, quote, unquote from html import escape, unescape from datetime import datetime +from dateutil import parser as dateparser from base32_crockford import encode as base32_encode # type: ignore import json as pyjson @@ -140,51 +141,8 @@ def parse_date(date: Any) -> Optional[datetime]: date = str(date) if isinstance(date, str): - if date.replace('.', '').isdigit(): - # this is a brittle attempt at unix timestamp parsing (which is - # notoriously hard to do). It may lead to dates being off by - # anything from hours to decades, depending on which app, OS, - # and sytem time configuration was used for the original timestamp - # more info: https://github.com/pirate/ArchiveBox/issues/119 + return dateparser.parse(date) - # Note: always always always store the original timestamp string - # somewhere indepentendly of the parsed datetime, so that later - # bugs dont repeatedly misparse and rewrite increasingly worse dates. - # the correct date can always be re-derived from the timestamp str - timestamp = float(date) - - EARLIEST_POSSIBLE = 473403600.0 # 1985 - LATEST_POSSIBLE = 1735707600.0 # 2025 - - if EARLIEST_POSSIBLE < timestamp < LATEST_POSSIBLE: - # number is seconds - return datetime.fromtimestamp(timestamp) - - elif EARLIEST_POSSIBLE * 1000 < timestamp < LATEST_POSSIBLE * 1000: - # number is milliseconds - return datetime.fromtimestamp(timestamp / 1000) - - elif EARLIEST_POSSIBLE * 1000*1000 < timestamp < LATEST_POSSIBLE * 1000*1000: - # number is microseconds - return datetime.fromtimestamp(timestamp / (1000*1000)) - - else: - # continue to the end and raise a parsing failed error. - # we dont want to even attempt parsing timestamp strings that - # arent within these ranges - pass - - if '-' in date: - # 2019-04-07T05:44:39.227520 - try: - return datetime.fromisoformat(date) - except Exception: - pass - try: - return datetime.strptime(date, '%Y-%m-%d %H:%M') - except Exception: - pass - raise ValueError('Tried to parse invalid date! {}'.format(date)) From 2ba17136e098f9d321356a758742f2c7b00ab592 Mon Sep 17 00:00:00 2001 From: Mashiat Sarker Shakkhar Date: Mon, 26 Aug 2019 17:33:58 -0400 Subject: [PATCH 12/12] index/schema.py: Make cmd_version actually optional. During migration, we may have to import json index file which was produced with an older version. If the index file is missing cmd_version, migration will fail. --- archivebox/index/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/archivebox/index/schema.py b/archivebox/index/schema.py index 66baa9d9..194878f7 100644 --- a/archivebox/index/schema.py +++ b/archivebox/index/schema.py @@ -59,6 +59,7 @@ class ArchiveResult: } info['start_ts'] = parse_date(info['start_ts']) info['end_ts'] = parse_date(info['end_ts']) + info['cmd_version'] = info.get('cmd_version') return cls(**info) def to_dict(self, *keys) -> dict: