mirror/archivebox
1
0
Fork 0
mirror of https://github.com/pirate/ArchiveBox.git synced 2025-08-29 01:00:11 +02:00
Code Issues Releases Wiki Activity

deduplicate method history when merging links

Browse Source
This commit is contained in:
Nick Sweeting
2019-04-11 03:41:25 -04:00
parent 0d2f7eb58e
commit 0272c9b8c0
1 changed files with 11 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
archivebox/legacy/index.py
View File

@@ -28,6 +28,7 @@ from .util import (
TimedProgress, TimedProgress,
copy_and_overwrite, copy_and_overwrite,
atomic_write, atomic_write,
ExtendedEncoder,
) )
from .parse import parse_links from .parse import parse_links
from .logs import ( from .logs import (
@@ -93,6 +94,16 @@ def merge_links(a: Link, b: Link) -> Link:
method: (a.history.get(method) or []) + (b.history.get(method) or []) method: (a.history.get(method) or []) + (b.history.get(method) or [])
for method in all_methods for method in all_methods
} }
for method in all_methods:
deduped_jsons = {
json.dumps(result, sort_keys=True, cls=ExtendedEncoder)
for result in history[method]
}
history[method] = list(reversed(sorted(
(ArchiveResult.from_json(json.loads(result)) for result in deduped_jsons),
key=lambda result: result.start_ts,
)))
return Link( return Link(
url=url, url=url,