From e50f8cb3b6f868c97fa96c6012c64154655e2647 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 18 Nov 2024 18:30:28 -0800 Subject: [PATCH] fix abx handling of obj, module, and class based plugins, fix archivebox version cmd --- .../abx_spec_archivebox/reads.py | 33 --------------- .../abx_spec_config/__init__.py | 41 +++++++++++-------- archivebox/pkgs/abx/abx.py | 28 +++++++++---- 3 files changed, 44 insertions(+), 58 deletions(-) delete mode 100644 archivebox/pkgs/abx-spec-archivebox/abx_spec_archivebox/reads.py diff --git a/archivebox/pkgs/abx-spec-archivebox/abx_spec_archivebox/reads.py b/archivebox/pkgs/abx-spec-archivebox/abx_spec_archivebox/reads.py deleted file mode 100644 index 30d6667d..00000000 --- a/archivebox/pkgs/abx-spec-archivebox/abx_spec_archivebox/reads.py +++ /dev/null @@ -1,33 +0,0 @@ -__package__ = 'abx.archivebox' - - -from benedict import benedict - - -def get_scope_config(defaults: benedict | None = None, persona=None, seed=None, crawl=None, snapshot=None, archiveresult=None, extra_config=None): - """Get all the relevant config for the given scope, in correct precedence order""" - - from django.conf import settings - default_config: benedict = defaults or settings.CONFIG - - snapshot = snapshot or (archiveresult and archiveresult.snapshot) - crawl = crawl or (snapshot and snapshot.crawl) - seed = seed or (crawl and crawl.seed) - persona = persona or (crawl and crawl.persona) - - persona_config = persona.config if persona else {} - seed_config = seed.config if seed else {} - crawl_config = crawl.config if crawl else {} - snapshot_config = snapshot.config if snapshot else {} - archiveresult_config = archiveresult.config if archiveresult else {} - extra_config = extra_config or {} - - return benedict({ - **default_config, # defaults / config file / environment variables - **persona_config, # lowest precedence - **seed_config, - **crawl_config, - **snapshot_config, - **archiveresult_config, - **extra_config, # highest precedence - }) diff --git a/archivebox/pkgs/abx-spec-config/abx_spec_config/__init__.py b/archivebox/pkgs/abx-spec-config/abx_spec_config/__init__.py index 5e05c185..2c705201 100644 --- a/archivebox/pkgs/abx-spec-config/abx_spec_config/__init__.py +++ b/archivebox/pkgs/abx-spec-config/abx_spec_config/__init__.py @@ -62,32 +62,35 @@ class ConfigPluginSpec: @staticmethod @abx.hookspec(firstresult=True) @abx.hookimpl - def get_SCOPE_CONFIG(extra=None, archiveresult=None, snapshot=None, crawl=None, user=None, collection=..., environment=..., machine=..., default=...) -> dict[ConfigKeyStr, Any]: + def get_SCOPE_CONFIG(extra=None, archiveresult=None, snapshot=None, crawl=None, user=None, request=None, collection=..., environment=..., machine=..., default=...) -> dict[ConfigKeyStr, Any]: """Get the config as it applies to you right now, based on the current context""" return benedict({ **pm.hook.get_default_config(default=default), - # **pm.hook.get_machine_config(machine), + **pm.hook.get_machine_config(machine=machine), **pm.hook.get_environment_config(environment=environment), **pm.hook.get_collection_config(collection=collection), **pm.hook.get_user_config(user=user), **pm.hook.get_crawl_config(crawl=crawl), **pm.hook.get_snapshot_config(snapshot=snapshot), **pm.hook.get_archiveresult_config(archiveresult=archiveresult), - # **pm.hook.get_request_config(request=request), + **pm.hook.get_request_config(request=request), **(extra or {}), }) @staticmethod - # @abx.hookspec(firstresult=True) - # @abx.hookimpl - # def get_request_config(request) -> dict: - # session = getattr(request, 'session', None) - # return getattr(session, 'config', None) or {} + @abx.hookspec(firstresult=True) + @abx.hookimpl + def get_request_config(request=None) -> dict: + if not request: + return {} + return request.session.get('config', None) or {} @staticmethod @abx.hookspec(firstresult=True) @abx.hookimpl - def get_archiveresult_config(archiveresult) -> dict[ConfigKeyStr, Any]: + def get_archiveresult_config(archiveresult=None) -> dict[ConfigKeyStr, Any]: + if not archiveresult: + return {} return getattr(archiveresult, 'config', None) or {} @staticmethod @@ -99,7 +102,9 @@ class ConfigPluginSpec: @staticmethod @abx.hookspec(firstresult=True) @abx.hookimpl - def get_crawl_config(crawl) -> dict[ConfigKeyStr, Any]: + def get_crawl_config(crawl=None) -> dict[ConfigKeyStr, Any]: + if not crawl: + return {} return getattr(crawl, 'config', None) or {} @staticmethod @@ -133,14 +138,14 @@ class ConfigPluginSpec: }) if environment == ... else environment @staticmethod - # @abx.hookspec(firstresult=True) - # @abx.hookimpl - # def get_machine_config(machine=...) -> dict: - # # ... = ellipsis, means automatically get the machine config from the currently executing machine - # # {} = empty dict, override to ignore the machine config - # if machine == ...: - # machine = Machine.objects.get_current() - # return getattr(machine, 'config', None) or {} + @abx.hookspec(firstresult=True) + @abx.hookimpl + def get_machine_config(machine=...) -> dict: + # ... = ellipsis, means automatically get the machine config from the currently executing machine + # {} = empty dict, override to ignore the machine config + # if machine == ...: + # machine = Machine.objects.get_current() + return getattr(machine, 'config', None) or {} @staticmethod @abx.hookspec(firstresult=True) diff --git a/archivebox/pkgs/abx/abx.py b/archivebox/pkgs/abx/abx.py index eb5567ac..481bb0ac 100644 --- a/archivebox/pkgs/abx/abx.py +++ b/archivebox/pkgs/abx/abx.py @@ -284,7 +284,7 @@ def get_plugin(plugin: PluginId | ModuleType | Type) -> PluginInfo: # raise ValueError(f'Invalid plugin, must be a module, class, or plugin ID (package name): {plugin}') - assert module + assert module and hasattr(module, '__package__') plugin_file = Path(inspect.getfile(module)) plugin_package = module.__package__ or module.__name__ @@ -356,10 +356,22 @@ def get_all_hook_specs() -> Dict[str, Dict[str, Any]]: for hook_name in get_all_hook_names(): for plugin_module in pm.get_plugins(): - if hasattr(plugin_module, hook_name): - hookspecopts = pm.parse_hookspec_opts(plugin_module, hook_name) + if inspect.ismodule(plugin_module): + plugin = plugin_module + plugin_module = plugin_module + elif inspect.isclass(plugin_module): + plugin = plugin_module + plugin_module = inspect.getmodule(plugin) + else: + plugin = type(plugin_module) + plugin_module = inspect.getmodule(plugin) + + assert plugin and plugin_module and hasattr(plugin_module, '__package__') + + if hasattr(plugin, hook_name): + hookspecopts = pm.parse_hookspec_opts(plugin, hook_name) if hookspecopts: - method = getattr(plugin_module, hook_name) + method = getattr(plugin, hook_name) signature = inspect.signature(method) return_type = signature.return_annotation if signature.return_annotation != inspect._empty else None @@ -381,9 +393,10 @@ def get_all_hook_specs() -> Dict[str, Dict[str, Any]]: 'signature': call_signature, 'hookspec_opts': hookspecopts, 'hookspec_signature': signature, - 'hookspec_plugin': plugin_module.__package__, + 'hookspec_plugin': method.__package__, } - return hook_specs + + return benedict(hook_specs) @@ -466,7 +479,8 @@ def get_plugin_hooks(plugin: PluginId | ModuleType | Type | None) -> Dict[AttrNa elif inspect.ismodule(plugin) or inspect.isclass(plugin): plugin_module = plugin else: - raise ValueError(f'Invalid plugin, cannot get hooks: {plugin}') + plugin_module = type(plugin) + # raise ValueError(f'Invalid plugin, cannot get hooks: {plugin}') for attr_name in dir(plugin_module): if attr_name.startswith('_'):