mirror/archivebox
1
0
Fork 0
mirror of https://github.com/pirate/ArchiveBox.git synced 2025-08-15 02:57:05 +02:00
Code Issues Releases Wiki Activity

Merge pull request #350 from cdvv7788/title-encoding-issue

Browse Source 
Title encoding issue
This commit is contained in:
Nick Sweeting
2020-07-02 13:21:15 -04:00
committed by GitHub
parent 0c48449aa6 528fc8f1f6
commit b4d1fe8b40
1 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
archivebox/util.py
View File

@@ -160,6 +160,15 @@ def download_url(url: str, timeout: int=None) -> str:
verify=CHECK_SSL_VALIDITY,
timeout=timeout,
)
if response.headers.get('Content-Type') == 'application/rss+xml':
# Based on https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
_TEMPLATE = r'''%s\s*=\s*["']?\s*%s\s*["']?'''
_XML_ENCODING_RE = _TEMPLATE % ('encoding', r'(?P<xmlcharset>[\w-]+)')
_BODY_ENCODING_PATTERN = r'<\s*(\?xml\s[^>]+%s)' % (_XML_ENCODING_RE)
_BODY_ENCODING_STR_RE = re.compile(_BODY_ENCODING_PATTERN, re.I | re.VERBOSE)
match = _BODY_ENCODING_STR_RE.search(response.text[:1024])
if match:
response.encoding = match.group('xmlcharset')
return response.text