diff --git a/archivebox/__init__.py b/archivebox/__init__.py index b0c00b61..52f40d83 100755 --- a/archivebox/__init__.py +++ b/archivebox/__init__.py @@ -1 +1,7 @@ __package__ = 'archivebox' + + +# monkey patch django timezone to add back utc (it was removed in Django 5.0) +import datetime +from django.utils import timezone +timezone.utc = datetime.timezone.utc diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py index 3f4dcd7c..f7c616e7 100644 --- a/archivebox/core/admin.py +++ b/archivebox/core/admin.py @@ -14,6 +14,8 @@ from django.shortcuts import render, redirect from django.contrib.auth import get_user_model from django import forms +from signal_webhooks.admin import WebhookAdmin, WebhookModel + from ..util import htmldecode, urldecode, ansi_to_html from core.models import Snapshot, ArchiveResult, Tag @@ -102,6 +104,7 @@ class ArchiveBoxAdmin(admin.AdminSite): archivebox_admin = ArchiveBoxAdmin() archivebox_admin.register(get_user_model()) archivebox_admin.register(APIToken) +archivebox_admin.register(WebhookModel, WebhookAdmin) archivebox_admin.disable_action('delete_selected') class ArchiveResultInline(admin.TabularInline): diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 923ae62c..5c1183fd 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -18,6 +18,7 @@ from ..config import ( CUSTOM_TEMPLATES_DIR, SQL_INDEX_FILENAME, OUTPUT_DIR, + ARCHIVE_DIR, LOGS_DIR, TIMEZONE, @@ -63,6 +64,7 @@ INSTALLED_APPS = [ 'core', 'api', + 'signal_webhooks', 'django_extensions', ] @@ -253,6 +255,29 @@ CACHES = { EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, + "archive": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + "OPTIONS": { + "base_url": "/archive/", + "location": ARCHIVE_DIR, + }, + }, + # "personas": { + # "BACKEND": "django.core.files.storage.FileSystemStorage", + # "OPTIONS": { + # "base_url": "/personas/", + # "location": PERSONAS_DIR, + # }, + # }, +} + ################################################################################ ### Security Settings ################################################################################ @@ -379,3 +404,15 @@ LOGGING = { } }, } + + +# Add default webhook configuration to the User model +SIGNAL_WEBHOOKS = { + "HOOKS": { + "django.contrib.auth.models.User": ..., + "core.models.Snapshot": ..., + "core.models.ArchiveResult": ..., + "core.models.Tag": ..., + "api.models.APIToken": ..., + }, +} diff --git a/archivebox/index/schema.py b/archivebox/index/schema.py index 0a9b39c5..8aa4e1c3 100644 --- a/archivebox/index/schema.py +++ b/archivebox/index/schema.py @@ -4,6 +4,7 @@ WARNING: THIS FILE IS ALL LEGACY CODE TO BE REMOVED. DO NOT ADD ANY NEW FEATURES TO THIS FILE, NEW CODE GOES HERE: core/models.py +These are the old types we used to use before ArchiveBox v0.4 (before we switched to Django). """ __package__ = 'archivebox.index' diff --git a/archivebox/main.py b/archivebox/main.py index 1f3324d2..6f745c0b 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -230,7 +230,7 @@ def version(quiet: bool=False, p = platform.uname() print( 'ArchiveBox v{}'.format(get_version(CONFIG)), - *((f'COMMIT_HASH={COMMIT_HASH[:7]}',) if COMMIT_HASH else ()), + f'COMMIT_HASH={COMMIT_HASH[:7] if COMMIT_HASH else 'unknown'}', f'BUILD_TIME={BUILD_TIME}', ) print( @@ -1356,7 +1356,7 @@ def manage(args: Optional[List[str]]=None, out_dir: Path=OUTPUT_DIR) -> None: if (args and "createsuperuser" in args) and (IN_DOCKER and not IS_TTY): stderr('[!] Warning: you need to pass -it to use interactive commands in docker', color='lightyellow') stderr(' docker run -it archivebox manage {}'.format(' '.join(args or ['...'])), color='lightyellow') - stderr() + stderr('') execute_from_command_line([f'{ARCHIVEBOX_BINARY} manage', *(args or ['help'])]) diff --git a/archivebox/manage.py b/archivebox/manage.py index 1a9b2975..413a4cfb 100755 --- a/archivebox/manage.py +++ b/archivebox/manage.py @@ -7,7 +7,7 @@ if __name__ == '__main__': # versions of ./manage.py commands whenever possible. When that's not possible # (e.g. makemigrations), you can comment out this check temporarily - if not ('makemigrations' in sys.argv or 'migrate' in sys.argv): + if not ('makemigrations' in sys.argv or 'migrate' in sys.argv or 'startapp' in sys.argv): print("[X] Don't run ./manage.py directly (unless you are a developer running makemigrations):") print() print(' Hint: Use these archivebox CLI commands instead of the ./manage.py equivalents:') diff --git a/package-lock.json b/package-lock.json index 4496a88b..f8225db7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,9 +25,9 @@ } }, "node_modules/@babel/runtime-corejs2": { - "version": "7.24.4", - "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.24.4.tgz", - "integrity": "sha512-ZCKqyUKt/Coimg+3Kafu43yNetgYnTXzNbEGAgxc81J5sI0qFNbQ613w7PNny+SmijAmGVroL0GDvx5rG/JI5Q==", + "version": "7.24.5", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.24.5.tgz", + "integrity": "sha512-cC9jiO6s/IN+xwCHYy1AGrcFJ4bwgIwb8HX1KaoEpRsznLlO4x9eBP6AX7RIeMSWlQqEj2WHox637OS8cDq6Ew==", "dependencies": { "core-js": "^2.6.12", "regenerator-runtime": "^0.14.0" @@ -203,9 +203,9 @@ "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" }, "node_modules/@types/node": { - "version": "20.12.7", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.7.tgz", - "integrity": "sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==", + "version": "20.12.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.8.tgz", + "integrity": "sha512-NU0rJLJnshZWdE/097cdCBbyW1h4hEg0xpovcoAQYHl8dnEyp/NAOiE45pvc+Bd1Dt+2r94v2eGFpQJ4R7g+2w==", "optional": true, "dependencies": { "undici-types": "~5.26.4" @@ -713,9 +713,9 @@ "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==" }, "node_modules/dompurify": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.0.tgz", - "integrity": "sha512-yoU4rhgPKCo+p5UrWWWNKiIq+ToGqmVVhk0PmMYBK4kRsR3/qhemNFL8f6CFmBd4gMwm3F4T7HBoydP5uY07fA==" + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.2.tgz", + "integrity": "sha512-hLGGBI1tw5N8qTELr3blKjAML/LY4ANxksbS612UiJyDfyf/2D092Pvm+S7pmeTGJRqvlJkFzBoHBQKgQlOQVg==" }, "node_modules/domutils": { "version": "1.5.1", @@ -1655,6 +1655,26 @@ "node": ">=18" } }, + "node_modules/puppeteer-core/node_modules/ws": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz", + "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/qs": { "version": "6.5.3", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz", @@ -2071,9 +2091,9 @@ } }, "node_modules/tough-cookie": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz", - "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==", + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz", + "integrity": "sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==", "dependencies": { "psl": "^1.1.33", "punycode": "^2.1.1", @@ -2276,9 +2296,9 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/ws": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz", - "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==", + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", + "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", "engines": { "node": ">=10.0.0" }, diff --git a/pyproject.toml b/pyproject.toml index 2c8b3fe2..8f009769 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ # - gallery-dl # - scihubdl # - See Github issues for more... + "django-signal-webhooks>=0.3.0", ] homepage = "https://github.com/ArchiveBox/ArchiveBox" diff --git a/requirements.txt b/requirements.txt index f19aff82..9632140d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,26 +2,33 @@ # Please do not edit it manually. annotated-types==0.6.0 +anyio==4.3.0 asgiref==3.8.1 asttokens==2.4.1 brotli==1.1.0; implementation_name == "cpython" brotlicffi==1.1.0.0; implementation_name != "cpython" certifi==2024.2.2 -cffi==1.16.0; implementation_name != "cpython" +cffi==1.16.0; platform_python_implementation != "PyPy" or implementation_name != "cpython" charset-normalizer==3.3.2 colorama==0.4.6; sys_platform == "win32" croniter==2.0.5 +cryptography==42.0.6 dateparser==1.2.0 decorator==5.1.1 django==5.0.4 django-auth-ldap==4.8.0 django-extensions==3.2.3 django-ninja==1.1.0 +django-settings-holder==0.1.2 +django-signal-webhooks==0.3.0 exceptiongroup==1.2.1; python_version < "3.11" executing==2.0.1 feedparser==6.0.11 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 idna==3.7 -ipython==8.23.0 +ipython==8.24.0 jedi==0.19.1 matplotlib-inline==0.1.7 mutagen==1.47.0 @@ -33,20 +40,21 @@ ptyprocess==0.7.0; sys_platform != "win32" and sys_platform != "emscripten" pure-eval==0.2.2 pyasn1==0.6.0 pyasn1-modules==0.4.0 -pycparser==2.22; implementation_name != "cpython" +pycparser==2.22; platform_python_implementation != "PyPy" or implementation_name != "cpython" pycryptodomex==3.20.0 pydantic==2.7.1 pydantic-core==2.18.2 -pygments==2.17.2 +pygments==2.18.0 python-crontab==3.0.0 python-dateutil==2.9.0.post0 python-ldap==3.4.4 pytz==2024.1 -regex==2024.4.16 +regex==2024.4.28 requests==2.31.0 setuptools==69.5.1 sgmllib3k==1.0.0 six==1.16.0 +sniffio==1.3.1 sonic-client==1.0.0 sqlparse==0.5.0 stack-data==0.6.3