mirror/archivebox
1
0
Fork 0
mirror of https://github.com/pirate/ArchiveBox.git synced 2025-08-10 08:34:08 +02:00
Code Issues Releases Wiki Activity

fix imports and deps

Browse Source
This commit is contained in:
Nick Sweeting
2024-11-18 18:07:34 -08:00
parent 6b83b4c995
commit 0acd388c02
7 changed files with 614 additions and 695 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
archivebox/cli/archivebox_add.py
View File

@@ -21,7 +21,6 @@ from archivebox.misc.checks import check_data_folder
from archivebox.parsers import PARSERS
from archivebox.logging_util import SmartFormatter, accept_stdin, stderr
from abid_utils.models import get_or_create_system_user_pk
if TYPE_CHECKING:
from core.models import Snapshot
@@ -55,6 +54,8 @@ def add(urls: str | list[str],
from seeds.models import Seed
from crawls.models import Crawl
from actors.orchestrator import Orchestrator
from abid_utils.models import get_or_create_system_user_pk
created_by_id = created_by_id or get_or_create_system_user_pk()

2
archivebox/core/models.py
View File

@@ -449,6 +449,8 @@ class Snapshot(ModelWithOutputDir, ModelWithStateMachine, ABIDModel):
for extractor in EXTRACTORS:
if not extractor:
continue
if ArchiveResult.objects.filter(snapshot=self, extractor=extractor).exists():
continue
archiveresult, created = ArchiveResult.objects.get_or_create(
snapshot=self,
extractor=extractor,

8
archivebox/pkgs/abx-spec-config/abx_spec_config/__init__.py
View File

@@ -43,7 +43,7 @@ class ConfigPluginSpec:
@staticmethod
@abx.hookspec(firstresult=True)
@abx.hookimpl
def get_CONFIGS() -> dict[abx.PluginId, BaseConfigSet]:
def get_CONFIGS() -> dict[abx.PluginId, 'BaseConfigSet | ConstantsDict']:
"""Get the config for all plugins by plugin_id -> {plugin_abc: PluginABCConfigSet(), plugin_xyz: PluginXYZConfigSet(), ...}"""
return abx.as_dict(pm.hook.get_CONFIG())
@@ -117,7 +117,7 @@ class ConfigPluginSpec:
return benedict({
key: value
for configset in pm.hook.get_CONFIGS().values()
for key, value in configset.from_collection().items()
for key, value in (configset.from_collection().items() if isinstance(configset, BaseConfigSet) else {})
}) if collection == ... else collection
@staticmethod
@@ -129,7 +129,7 @@ class ConfigPluginSpec:
return benedict({
key: value
for configset in pm.hook.get_CONFIGS().values()
for key, value in configset.from_environment().items()
for key, value in (configset.from_environment().items() if isinstance(configset, BaseConfigSet) else ())
}) if environment == ... else environment
@staticmethod
@@ -151,7 +151,7 @@ class ConfigPluginSpec:
return benedict({
key: value
for configset in pm.hook.get_CONFIGS().values()
for key, value in configset.from_defaults().items()
for key, value in (configset.from_defaults().items() if isinstance(configset, BaseConfigSet) else configset.items())
}) if default == ... else default

6
archivebox/pkgs/abx/abx.py
View File

@@ -5,7 +5,7 @@ __author__ = 'Nick Sweeting'
__homepage__ = 'https://github.com/ArchiveBox/ArchiveBox'
__order__ = 0
import sys
import inspect
import importlib
import itertools
@@ -446,9 +446,11 @@ def load_plugins(plugins: Iterable[PluginId | ModuleType | Type] | Dict[PluginId
PLUGINS_TO_LOAD = sorted(PLUGINS_TO_LOAD, key=lambda x: x['order'])
for plugin_info in PLUGINS_TO_LOAD:
if '--version' not in sys.argv and '--help' not in sys.argv:
print(f'🧩 Loading plugin: {plugin_info["id"]}...', end='\r', flush=True, file=sys.stderr)
pm.register(plugin_info['module'])
LOADED_PLUGINS[plugin_info['id']] = plugin_info
print(f' √ Loaded plugin: {plugin_info["id"]}')
# print('\x1b[2K', end='\r', flush=True, file=sys.stderr)
return benedict(LOADED_PLUGINS)
@cache

8
pyproject.toml
View File

@@ -143,6 +143,9 @@ all = [
]
[tool.uv]
environments = ["sys_platform == 'darwin'", "sys_platform == 'linux'"]
package = true
# compile-bytecode = true
dev-dependencies = [
### BUILD
"uv>=0.4.26",
@@ -219,6 +222,11 @@ abx-plugin-htmltotext = { workspace = true }
members = ["archivebox/pkgs/*"]
exclude = ["archivebox/pkgs/__pycache__"]
[tool.uv.pip]
all-extras = true
python-version = "3.10"
# compile-bytecode = true
[build-system]
requires = ["pdm-backend"]
build-backend = "pdm.backend"

88
requirements.txt
View File

@@ -1,6 +1,6 @@
# This file was autogenerated by uv via the following command:
# uv pip compile pyproject.toml --all-extras -o requirements.txt
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx
# via
# archivebox (pyproject.toml)
# abx-plugin-archivedotorg
@@ -30,59 +30,74 @@
# abx-spec-django
# abx-spec-extractor
# abx-spec-searchbackend
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-archivedotorg
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-archivedotorg
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-chrome
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-chrome
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-curl
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-curl
# via
# archivebox (pyproject.toml)
# abx-plugin-archivedotorg
# abx-plugin-favicon
# abx-plugin-title
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-default-binproviders
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-default-binproviders
# via
# archivebox (pyproject.toml)
# abx-plugin-git
# abx-plugin-npm
# abx-plugin-pip
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-favicon
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-favicon
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-git
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-git
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-htmltotext
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-htmltotext
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-ldap-auth
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-ldap-auth
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-mercury
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-mercury
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-npm
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-npm
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-pip
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-pip
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-playwright
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-playwright
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-puppeteer
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-puppeteer
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-readability
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-readability
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-ripgrep-search
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-ripgrep-search
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-singlefile
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-singlefile
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-sonic-search
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-sonic-search
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-sqlitefts-search
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-sqlitefts-search
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-title
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-title
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-wget
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-wget
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-ytdlp
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-ytdlp
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-archivebox
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-abx-pkg
# via
# archivebox (pyproject.toml)
# abx-plugin-chrome
# abx-plugin-curl
# abx-plugin-default-binproviders
# abx-plugin-git
# abx-plugin-npm
# abx-plugin-pip
# abx-plugin-playwright
# abx-plugin-puppeteer
# abx-plugin-singlefile
# abx-plugin-sonic-search
# abx-plugin-wget
# abx-plugin-ytdlp
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-archivebox
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-config
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-config
# via
# archivebox (pyproject.toml)
# abx-plugin-archivedotorg
@@ -105,28 +120,13 @@
# abx-plugin-title
# abx-plugin-wget
# abx-plugin-ytdlp
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-django
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-django
# via
# archivebox (pyproject.toml)
# abx-plugin-ldap-auth
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-extractor
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-extractor
# via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-abx-pkg
# via
# archivebox (pyproject.toml)
# abx-plugin-chrome
# abx-plugin-curl
# abx-plugin-default-binproviders
# abx-plugin-git
# abx-plugin-npm
# abx-plugin-pip
# abx-plugin-playwright
# abx-plugin-puppeteer
# abx-plugin-singlefile
# abx-plugin-sonic-search
# abx-plugin-wget
# abx-plugin-ytdlp
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-searchbackend
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-searchbackend
# via
# archivebox (pyproject.toml)
# abx-plugin-ripgrep-search
@@ -308,7 +308,7 @@ ipython==8.29.0
# ipdb
jedi==0.19.2
# via ipython
libcst==1.5.0
libcst==1.5.1
# via django-autotyping
mailchecker==6.0.11
# via python-benedict
@@ -515,7 +515,7 @@ xlrd==2.0.1
# via python-benedict
xmltodict==0.14.2
# via python-benedict
yt-dlp==2024.11.4
yt-dlp==2024.11.18
# via archivebox (pyproject.toml)
zope-interface==7.1.1
# via twisted

1194
uv.lock generated
View File

File diff suppressed because it is too large Load Diff