From c1f0f7517a2da95e0d2077ece27164448fa9683d Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 2 Feb 2018 12:08:06 -0800 Subject: [PATCH 01/89] indexsource: add new XmlQueryIndexSource - support outbackcdx (tinycdxserver) and OpenWayback xmlquery interface (ukwa/ukwa-pywb#2) - convert xml to cdx iter for exact match - support prefix match (eg. for fuzzy matching) via chaining prefix query, and lazy urlquery in iterator --- pywb/warcserver/index/indexsource.py | 111 ++++++++++++++++++++++++++- pywb/warcserver/warcserver.py | 3 + 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/pywb/warcserver/index/indexsource.py b/pywb/warcserver/index/indexsource.py index 09101e18c..8807e1a54 100644 --- a/pywb/warcserver/index/indexsource.py +++ b/pywb/warcserver/index/indexsource.py @@ -7,6 +7,11 @@ from pywb.utils.binsearch import iter_range from pywb.utils.canonicalize import canonicalize +from pywb.utils.wbexception import NotFoundException, BadRequestException + +from warcio.timeutils import timestamp_to_http_date, http_date_to_timestamp +from warcio.timeutils import timestamp_now, pad_timestamp, PAD_14_DOWN + from pywb.utils.format import res_template from pywb.utils.io import no_except_close from pywb.utils.memento import MementoUtils @@ -14,6 +19,25 @@ from pywb.warcserver.http import DefaultAdapters from pywb.warcserver.index.cdxobject import CDXObject +from pywb.utils.format import ParamFormatter, res_template +from pywb.utils.memento import MementoUtils + +from pywb.warcserver.index.cdxops import cdx_sort_closest + +from six.moves.urllib.parse import quote_plus + +try: + from lxml import etree +except: + import xml.etree.ElementTree as etree + +import redis + +import requests + +import re +import logging + #============================================================================= class BaseIndexSource(object): @@ -190,7 +214,92 @@ def init_from_config(cls, config): return cls(config['api_url'], config['replay_url']) -#============================================================================= +# ============================================================================= +class XmlQueryIndexSource(BaseIndexSource): + EXACT_SUFFIX = '?q=type:urlquery+url:{url}' + PREFIX_SUFFIX = '?q=type:prefixquery+url:{url}' + + def __init__(self, query_api_url): + self.query_api_url = query_api_url + self.session = requests.session() + + def load_index(self, params): + closest = params.get('closest') + + url = params.get('url', '') + + matchType = params.get('matchType', 'exact') + + if matchType == 'exact': + query_url = self.query_api_url + self.EXACT_SUFFIX + elif matchType == 'prefix': + query_url = self.query_api_url + self.PREFIX_SUFFIX + else: + raise BadRequestException('matchType={0} is not supported'.format(matchType=matchType)) + + query_url = query_url.format(url=quote_plus(url)) + + try: + response = self.session.get(query_url) + response.raise_for_status() + + results = etree.fromstring(response.text) + + items = results.find('results').findall('result') + + if matchType == 'exact': + cdx_iter = [self.convert_to_cdx(item) for item in items] + if closest: + cdx_iter = cdx_sort_closest(closest, cdx_iter, limit=10000) + + else: + cdx_iter = self.prefix_query_iter(items) + + except Exception: + if self.logger.getEffectiveLevel() == logging.DEBUG: + import traceback + traceback.print_exc() + + raise NotFoundException('url {0} not found'.format(url)) + + return cdx_iter + + def prefix_query_iter(self, items): + for item in items: + url = self.gettext(item, 'originalurl') + if not url: + continue + + cdx_iter = self.load_index({'url': url}) + for cdx in cdx_iter: + yield cdx + + def convert_to_cdx(self, item): + cdx = CDXObject() + cdx['urlkey'] = self.gettext(item, 'urlkey') + cdx['timestamp'] = self.gettext(item, 'capturedate')[:14] + cdx['url'] = self.gettext(item, 'url') + cdx['mime'] = self.gettext(item, 'mimetype') + cdx['status'] = self.gettext(item, 'httpresponsecode') + cdx['digest'] = self.gettext(item, 'digest') + cdx['offset'] = self.gettext(item, 'compressedoffset') + cdx['filename'] = self.gettext(item, 'file') + return cdx + + def gettext(self, item, name): + elem = item.find(name) + if elem is not None: + return elem.text + else: + return '' + + @classmethod + def init_from_string(cls, value): + if value.startswith('xmlquery+'): + return cls(value[9:]) + + +# ============================================================================= class LiveIndexSource(BaseIndexSource): def __init__(self): self._init_sesh(DefaultAdapters.live_adapter) diff --git a/pywb/warcserver/warcserver.py b/pywb/warcserver/warcserver.py index 9f9233d19..c3ed1ef8f 100644 --- a/pywb/warcserver/warcserver.py +++ b/pywb/warcserver/warcserver.py @@ -10,6 +10,8 @@ from pywb.warcserver.index.indexsource import FileIndexSource, RemoteIndexSource from pywb.warcserver.index.indexsource import MementoIndexSource, RedisIndexSource from pywb.warcserver.index.indexsource import LiveIndexSource, WBMementoIndexSource +from pywb.warcserver.index.indexsource import XmlQueryIndexSource + from pywb.warcserver.index.zipnum import ZipNumIndexSource from pywb import DEFAULT_CONFIG @@ -20,6 +22,7 @@ SOURCE_LIST = [LiveIndexSource, + XmlQueryIndexSource, WBMementoIndexSource, RedisMultiKeyIndexSource, MementoIndexSource, From 94eb4ad20617dca613cb1210a5e9cc5f197e3492 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 2 Feb 2018 12:47:49 -0800 Subject: [PATCH 02/89] loaders: add WebHDFSLoader loader to support handling 'webhdfs://' scheme to load over http from WebHDFS (ukwa/ukwa-pywb#3) tests: add basic test for WebHFDSLoader api format --- pywb/utils/loaders.py | 22 ++++++++++++++++++++++ pywb/utils/test/test_loaders.py | 17 +++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/pywb/utils/loaders.py b/pywb/utils/loaders.py index d3fc7be70..da4d8116d 100644 --- a/pywb/utils/loaders.py +++ b/pywb/utils/loaders.py @@ -225,6 +225,7 @@ def init_default_loaders(): BlockLoader.loaders['s3'] = S3Loader BlockLoader.loaders['file'] = LocalFileLoader BlockLoader.loaders['pkg'] = PackageLoader + BlockLoader.loaders['webhdfs'] = WebHDFSLoader @staticmethod def set_profile_loader(src): @@ -401,6 +402,27 @@ def s3_load(anon=False): return obj['Body'] +# ================================================================= +class WebHDFSLoader(HttpLoader): + HTTP_URL = 'http://{host}/webhdfs/v1{path}?op=OPEN&offset={offset}' + LENGTH_PARAM = '&length={length}' + + def load(self, url, offset, length): + parts = urlsplit(url) + + http_url = self.HTTP_URL + + if length > 0: + http_url += self.LENGTH_PARAM + + full_url = http_url.format(host=parts.netloc, + path=parts.path, + offset=offset, + length=length) + + return super(WebHDFSLoader, self).load(full_url, 0, -1) + + # ================================================================= # Signed Cookie-Maker # ================================================================= diff --git a/pywb/utils/test/test_loaders.py b/pywb/utils/test/test_loaders.py index 4a2176165..eef7217a5 100644 --- a/pywb/utils/test/test_loaders.py +++ b/pywb/utils/test/test_loaders.py @@ -85,6 +85,8 @@ from pywb.utils.loaders import extract_client_cookie from pywb.utils.loaders import read_last_line +from mock import patch + from warcio.bufferedreaders import DecompressingBufferedReader from pywb import get_test_dir @@ -117,6 +119,21 @@ def test_s3_read_2(): reader = DecompressingBufferedReader(BytesIO(buff)) assert reader.readline() == b'\n' +def test_mock_webhdfs_load(): + def mock_load(expected): + def mock(self, url, offset, length): + assert url == expected + assert offset == 0 + assert length == -1 + return None + + return mock + + with patch('pywb.utils.loaders.HttpLoader.load', mock_load('http://remote-host:1234/webhdfs/v1/some/file.warc.gz?op=OPEN&offset=10&length=50')): + res = BlockLoader().load('webhdfs://remote-host:1234/some/file.warc.gz', 10, 50) + + with patch('pywb.utils.loaders.HttpLoader.load', mock_load('http://remote-host/webhdfs/v1/some/file.warc.gz?op=OPEN&offset=10')): + res = BlockLoader().load('webhdfs://remote-host/some/file.warc.gz', 10, -1) # Error From ec88e962b3d6040ffba5ed946fca9aab4af53dd9 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Mon, 5 Feb 2018 13:35:54 -0800 Subject: [PATCH 03/89] indexsource: add tests for XmlQueryIndexSource, add missing init_from_config() (ukwa/ukwa-pywb#2) --- pywb/warcserver/index/indexsource.py | 8 + .../index/test/test_xmlquery_indexsource.py | 162 ++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 pywb/warcserver/index/test/test_xmlquery_indexsource.py diff --git a/pywb/warcserver/index/indexsource.py b/pywb/warcserver/index/indexsource.py index 8807e1a54..efd5e1c56 100644 --- a/pywb/warcserver/index/indexsource.py +++ b/pywb/warcserver/index/indexsource.py @@ -299,6 +299,14 @@ def init_from_string(cls, value): return cls(value[9:]) + @classmethod + def init_from_config(cls, config): + if config['type'] != 'xmlquery': + return + + return cls(config['api_url']) + + # ============================================================================= class LiveIndexSource(BaseIndexSource): def __init__(self): diff --git a/pywb/warcserver/index/test/test_xmlquery_indexsource.py b/pywb/warcserver/index/test/test_xmlquery_indexsource.py new file mode 100644 index 000000000..fc5d45b22 --- /dev/null +++ b/pywb/warcserver/index/test/test_xmlquery_indexsource.py @@ -0,0 +1,162 @@ +from pywb.warcserver.test.testutils import BaseTestClass, key_ts_res + +from pywb.warcserver.index.indexsource import XmlQueryIndexSource +from pywb.warcserver.index.aggregator import SimpleAggregator + +from mock import patch + + +# ============================================================================ +def mock_get(self, url): + string = '' + if 'type:urlquery' in url: + if 'http%3A%2F%2Fexample.com%2Fsome%2Fpath' in url: + string = URL_RESPONSE_2 + + elif 'http%3A%2F%2Fexample.com%2F' in url: + string = URL_RESPONSE_1 + + elif 'type:prefixquery' in url: + string = PREFIX_QUERY + + class MockResponse(object): + def __init__(self, string): + self.string = string + + @property + def text(self): + return self.string + + def raise_for_status(self): + pass + + + return MockResponse(string) + + +# ============================================================================ +class TestXmlQueryIndexSource(BaseTestClass): + @classmethod + def setup_class(cls): + super(TestXmlQueryIndexSource, cls).setup_class() + + def do_query(self, params): + return SimpleAggregator({'source': XmlQueryIndexSource('http://localhost:8080/path')})(params) + + @patch('pywb.warcserver.index.indexsource.requests.sessions.Session.get', mock_get) + def test_exact_query(self): + res, errs = self.do_query({'url': 'http://example.com/'}) + expected = """\ +com,example)/ 20180112200243 example.warc.gz +com,example)/ 20180216200300 example.warc.gz""" + assert(key_ts_res(res) == expected) + assert(errs == {}) + + + @patch('pywb.warcserver.index.indexsource.requests.sessions.Session.get', mock_get) + def test_exact_query_2(self): + res, errs = self.do_query({'url': 'http://example.com/some/path'}) + expected = """\ +com,example)/some/path 20180112200243 example.warc.gz +com,example)/some/path 20180216200300 example.warc.gz""" + assert(key_ts_res(res) == expected) + assert(errs == {}) + + + @patch('pywb.warcserver.index.indexsource.requests.sessions.Session.get', mock_get) + def test_prefix_query(self): + res, errs = self.do_query({'url': 'http://example.com/', 'matchType': 'prefix'}) + expected = """\ +com,example)/ 20180112200243 example.warc.gz +com,example)/ 20180216200300 example.warc.gz +com,example)/some/path 20180112200243 example.warc.gz +com,example)/some/path 20180216200300 example.warc.gz""" + assert(key_ts_res(res) == expected) + assert(errs == {}) + + +# ============================================================================ +URL_RESPONSE_1 = """ + + + + 10 + text/html + example.warc.gz + - + com,example)/ + 7NZ7K6ZTRC4SOJODXH3S4AGZV7QSBWLF + 200 + - + http://example.ccom/ + 20180112200243 + + + 29570 + text/html + example.warc.gz + - + com,example)/ + LCKPKJJU5VPEN6HUJZ6JUYRGTPFD7ZC3 + 200 + - + http://example.com/ + 20180216200300 + + + +""" + +URL_RESPONSE_2 = """ + + + + 10 + text/html + example.warc.gz + - + com,example)/some/path + 7NZ7K6ZTRC4SOJODXH3S4AGZV7QSBWLF + 200 + - + http://example.com/some/path + 20180112200243 + + + 29570 + text/html + example.warc.gz + - + com,example)/some/path + LCKPKJJU5VPEN6HUJZ6JUYRGTPFD7ZC3 + 200 + - + http://example.com/some/path + 20180216200300 + + + +""" + +PREFIX_QUERY = """ + + + + com,example)/ + http://example.com/ + 2 + 2 + 20180112200243 + 20180216200300 + + + com,example)/some/path + http://example.com/some/path + 2 + 2 + 20180112200243 + 20180216200300 + + + +""" From 959481fd48fbbce4da303f6c7b7c10f94570f4e0 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Wed, 7 Feb 2018 23:22:47 -0800 Subject: [PATCH 04/89] loaders: webhdfs loader: support optional '&user.name=' param from WEBHDFS_USER env var or '&delegation=' from WEBHDFS_TOKEN env var (fixes ukwa/ukwa-pywb#5) --- pywb/utils/loaders.py | 25 ++++++++++++++-------- pywb/utils/test/test_loaders.py | 38 ++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/pywb/utils/loaders.py b/pywb/utils/loaders.py index da4d8116d..6d721de37 100644 --- a/pywb/utils/loaders.py +++ b/pywb/utils/loaders.py @@ -404,23 +404,30 @@ def s3_load(anon=False): # ================================================================= class WebHDFSLoader(HttpLoader): - HTTP_URL = 'http://{host}/webhdfs/v1{path}?op=OPEN&offset={offset}' - LENGTH_PARAM = '&length={length}' + HTTP_URL = 'http://{host}/webhdfs/v1{path}?' def load(self, url, offset, length): parts = urlsplit(url) - http_url = self.HTTP_URL + http_url = self.HTTP_URL.format(host=parts.netloc, + path=parts.path) + + params = {'op': 'OPEN', + 'offset': str(offset) + } if length > 0: - http_url += self.LENGTH_PARAM + params['length'] = str(length) + + if os.environ.get('WEBHDFS_USER'): + params['user.name'] = os.environ.get('WEBHDFS_USER') + + if os.environ.get('WEBHDFS_TOKEN'): + params['delegation'] = os.environ.get('WEBHDFS_TOKEN') - full_url = http_url.format(host=parts.netloc, - path=parts.path, - offset=offset, - length=length) + http_url += urlencode(params) - return super(WebHDFSLoader, self).load(full_url, 0, -1) + return super(WebHDFSLoader, self).load(http_url, 0, -1) # ================================================================= diff --git a/pywb/utils/test/test_loaders.py b/pywb/utils/test/test_loaders.py index eef7217a5..819390b33 100644 --- a/pywb/utils/test/test_loaders.py +++ b/pywb/utils/test/test_loaders.py @@ -85,6 +85,8 @@ from pywb.utils.loaders import extract_client_cookie from pywb.utils.loaders import read_last_line +from pywb.utils.canonicalize import canonicalize + from mock import patch from warcio.bufferedreaders import DecompressingBufferedReader @@ -119,20 +121,36 @@ def test_s3_read_2(): reader = DecompressingBufferedReader(BytesIO(buff)) assert reader.readline() == b'\n' -def test_mock_webhdfs_load(): - def mock_load(expected): - def mock(self, url, offset, length): - assert url == expected - assert offset == 0 - assert length == -1 - return None +def mock_load(expected): + def mock(self, url, offset, length): + assert canonicalize(url) == canonicalize(expected) + assert offset == 0 + assert length == -1 + return None - return mock + return mock - with patch('pywb.utils.loaders.HttpLoader.load', mock_load('http://remote-host:1234/webhdfs/v1/some/file.warc.gz?op=OPEN&offset=10&length=50')): +def test_mock_webhdfs_load_1(): + expected = 'http://remote-host:1234/webhdfs/v1/some/file.warc.gz?op=OPEN&offset=10&length=50' + with patch('pywb.utils.loaders.HttpLoader.load', mock_load(expected)): res = BlockLoader().load('webhdfs://remote-host:1234/some/file.warc.gz', 10, 50) - with patch('pywb.utils.loaders.HttpLoader.load', mock_load('http://remote-host/webhdfs/v1/some/file.warc.gz?op=OPEN&offset=10')): +def test_mock_webhdfs_load_2(): + expected = 'http://remote-host/webhdfs/v1/some/file.warc.gz?op=OPEN&offset=10' + with patch('pywb.utils.loaders.HttpLoader.load', mock_load(expected)): + res = BlockLoader().load('webhdfs://remote-host/some/file.warc.gz', 10, -1) + +def test_mock_webhdfs_load_3_username(): + os.environ['WEBHDFS_USER'] = 'someuser' + expected = 'http://remote-host/webhdfs/v1/some/file.warc.gz?op=OPEN&offset=10&user.name=someuser' + with patch('pywb.utils.loaders.HttpLoader.load', mock_load(expected)): + res = BlockLoader().load('webhdfs://remote-host/some/file.warc.gz', 10, -1) + +def test_mock_webhdfs_load_4_token(): + os.environ['WEBHDFS_USER'] = '' + os.environ['WEBHDFS_TOKEN'] = 'ATOKEN' + expected = 'http://remote-host/webhdfs/v1/some/file.warc.gz?op=OPEN&offset=10&delegation=ATOKEN' + with patch('pywb.utils.loaders.HttpLoader.load', mock_load(expected)): res = BlockLoader().load('webhdfs://remote-host/some/file.warc.gz', 10, -1) From b38cfb8d675f959a600cd61febd0cd2d3a70382b Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 9 Feb 2018 17:16:46 -0800 Subject: [PATCH 05/89] apps: frontendapp customizations (to support ukwa/ukwa-pywb#6) - support extending with custom rewriterapp by setting REWRITER_APP_CLASS - correctly default to 'config.yaml' if no config file specified --- pywb/apps/frontendapp.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pywb/apps/frontendapp.py b/pywb/apps/frontendapp.py index d3614a372..41f5232f9 100644 --- a/pywb/apps/frontendapp.py +++ b/pywb/apps/frontendapp.py @@ -57,13 +57,16 @@ class FrontEndApp(object): PROXY_CA_PATH = os.path.join('proxy-certs', 'pywb-ca.pem') + REWRITER_APP_CLS = RewriterApp + ALL_DIGITS = re.compile(r'^\d+$') - def __init__(self, config_file='./config.yaml', custom_config=None): + def __init__(self, config_file=None, custom_config=None): """ :param str config_file: Path to the config file :param dict custom_config: Dictionary containing additional configuration information """ + config_file = config_file or './config.yaml' self.handler = self.handle_request self.warcserver = WarcServer(config_file=config_file, custom_config=custom_config) @@ -92,9 +95,9 @@ def __init__(self, config_file='./config.yaml', custom_config=None): upstream_paths = self.get_upstream_paths(self.warcserver_server.port) framed_replay = config.get('framed_replay', True) - self.rewriterapp = RewriterApp(framed_replay, - config=config, - paths=upstream_paths) + self.rewriterapp = self.REWRITER_APP_CLS(framed_replay, + config=config, + paths=upstream_paths) self.templates_dir = config.get('templates_dir', 'templates') self.static_dir = config.get('static_dir', 'static') From 5b7ca18e0f95beb5f117b0a067d02d051ded98e2 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Sat, 10 Feb 2018 09:17:15 -0800 Subject: [PATCH 06/89] rewriting: try more granular modifers to distinguish embeds: (in part for ukwa/ukwa-pywb#6) - 'ba_' - for rewriting - 'je_' - 'javascript-embed' default for client-side rewriting in wombat better modifiers for css rewriting (server and client): - 'ce_' - 'css-embed' for any url() embeds in CSS - 'cs_' - for css stylesheet @import rewriting/other .css --- pywb/rewrite/html_rewriter.py | 2 +- pywb/rewrite/test/test_html_rewriter.py | 24 +++++++++++------------ pywb/rewrite/test/test_regex_rewriters.py | 24 +++++++++++------------ 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/pywb/rewrite/html_rewriter.py b/pywb/rewrite/html_rewriter.py index bf28bf254..92c4e8e0f 100644 --- a/pywb/rewrite/html_rewriter.py +++ b/pywb/rewrite/html_rewriter.py @@ -56,7 +56,7 @@ def _init_rewrite_tags(defmod): 'archive': 'oe_'}, 'area': {'href': defmod}, 'audio': {'src': 'oe_'}, - 'base': {'href': defmod}, + 'base': {'href': 'ba_'}, 'blockquote': {'cite': defmod}, 'body': {'background': 'im_'}, 'button': {'formaction': defmod}, diff --git a/pywb/rewrite/test/test_html_rewriter.py b/pywb/rewrite/test/test_html_rewriter.py index 65bbfe287..26f12248f 100644 --- a/pywb/rewrite/test/test_html_rewriter.py +++ b/pywb/rewrite/test/test_html_rewriter.py @@ -25,23 +25,23 @@ # Base Tests -- w/ rewrite (default) >>> parse('') - + # Full Path >>> parse('', urlrewriter=full_path_urlrewriter) - + # Full Path Scheme Rel Base >>> parse('', urlrewriter=full_path_urlrewriter) - + # Rel Base >>> parse('', urlrewriter=full_path_urlrewriter) - + # Rel Base + example >>> parse('', urlrewriter=full_path_urlrewriter) - + # Rel Base >>> parse('', urlrewriter=full_path_urlrewriter) @@ -53,7 +53,7 @@ # ensure trailing slash added >>> parse('') - + # Base Tests -- no rewrite >>> parse('', urlrewriter=no_base_canon_rewriter) @@ -244,29 +244,29 @@
>>> parse('
') -
+
>>> parse('') - + >>> parse('') - + >>> parse('') - + >>> parse('') >>> parse("") - + #>>> parse('') # Style >>> parse('') - + # Unterminated style tag, handle and auto-terminate >>> parse('') - + # Unterminated style tag, handle and auto-terminate >>> parse(' - - +{% block body %}

- Collection Search Page + Collection {{ coll }} Search Page

@@ -33,12 +21,9 @@

-
{% endif %} - - - - - +{% endblock %} diff --git a/tests/test_auto_colls.py b/tests/test_auto_colls.py index ed4a04251..b734907ca 100644 --- a/tests/test_auto_colls.py +++ b/tests/test_auto_colls.py @@ -341,14 +341,14 @@ def test_add_modify_home_template(self): with open(filename, 'r+b') as fh: buf = fh.read() - buf = buf.replace(b'', b'Custom Test Homepage') + buf = buf.replace(b'Pywb Wayback Machine', b'Custom Test Homepage') fh.seek(0) fh.write(buf) resp = self.testapp.get('/') resp.charset = 'utf-8' assert resp.content_type == 'text/html' - assert 'Custom Test Homepage' in resp.text, resp.text + assert 'Custom Test Homepage' in resp.text, resp.text @patch('pywb.manager.manager.get_input', lambda x: 'y') def test_add_template_input_yes(self): From 7ac9a37bb445cfb940c4e06c80955bb59bee49ca Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 8 Mar 2019 10:10:02 -0800 Subject: [PATCH 39/89] acl: support for exact acl rules via '###' suffix - ex: rule 'com,example)/###' matches http://example.com/ only - wb-manager acl add/remove --exact-match adds/remove exact match rules - tests: add tests for exact match queries, acl --- pywb/manager/aclmanager.py | 28 +++++++++++++++++++--------- pywb/warcserver/access_checker.py | 22 ++++++++++++++++++++-- pywb/warcserver/test/test_access.py | 14 ++++++++++++++ sample_archive/access/pywb.aclj | 1 + tests/test_acl.py | 11 +++++++++-- tests/test_acl_manager.py | 17 +++++++++++++++++ 6 files changed, 80 insertions(+), 13 deletions(-) diff --git a/pywb/manager/aclmanager.py b/pywb/manager/aclmanager.py index d5d8f37f5..248f91e6c 100644 --- a/pywb/manager/aclmanager.py +++ b/pywb/manager/aclmanager.py @@ -101,13 +101,19 @@ def save_acl(self, r=None): except Exception as e: print('Error Saving ACL Rules: ' + str(e)) - def to_key(self, url_or_surt): + def to_key(self, url_or_surt, exact_match=False): """ If 'url_or_surt' already a SURT, use as is + If exact match, add the exact match suffix """ if self.SURT_RX.search(url_or_surt): - return url_or_surt + result = url_or_surt else: - return canonicalize(url_or_surt) + result = canonicalize(url_or_surt) + + if exact_match: + result += AccessChecker.EXACT_SUFFIX + + return result def validate_access(self, access): if access not in self.VALID_ACCESS: @@ -118,14 +124,14 @@ def validate_access(self, access): return True def add_rule(self, r): - return self._add_rule(r.url, r.access) + return self._add_rule(r.url, r.access, r.exact_match) - def _add_rule(self, url, access): + def _add_rule(self, url, access, exact_match=False): if not self.validate_access(access): return acl = CDXObject() - acl['urlkey'] = self.to_key(url) + acl['urlkey'] = self.to_key(url, exact_match) acl['timestamp'] = '-' acl['access'] = access acl['url'] = url @@ -183,7 +189,7 @@ def validate(self, log=False): def remove_rule(self, r): i = 0 - urlkey = self.to_key(r.url) + urlkey = self.to_key(r.url, r.exact_match) for rule in self.rules: if urlkey == rule['urlkey']:# and r.timestamp == rule['timestamp']: acl = self.rules.pop(i) @@ -251,10 +257,14 @@ def command(name, *args, **kwargs): op.add_argument(arg, nargs='?', default='allow') else: op.add_argument(arg) + + if kwargs.get('exact_opt'): + op.add_argument('-e', '--exact-match', action='store_true', default=False) + op.set_defaults(acl_func=kwargs['func']) - command('add', 'coll_name', 'url', 'access', func=cls.add_rule) - command('remove', 'coll_name', 'url', func=cls.remove_rule) + command('add', 'coll_name', 'url', 'access', func=cls.add_rule, exact_opt=True) + command('remove', 'coll_name', 'url', func=cls.remove_rule, exact_opt=True) command('list', 'coll_name', func=cls.list_rules) command('validate', 'coll_name', func=cls.validate_save) command('match', 'coll_name', 'url', 'default_access', func=cls.find_match) diff --git a/pywb/warcserver/access_checker.py b/pywb/warcserver/access_checker.py index c648e4f95..a0eb4abfb 100644 --- a/pywb/warcserver/access_checker.py +++ b/pywb/warcserver/access_checker.py @@ -16,7 +16,12 @@ def rev_cmp(a, b): return (a < b) - (a > b) def _do_iter(self, fh, params): - for line in search(fh, params['key'], prev_size=1, compare_func=self.rev_cmp): + exact_suffix = params.get('exact_match_suffix') + key = params['key'] + if exact_suffix: + key += exact_suffix + + for line in search(fh, key, prev_size=1, compare_func=self.rev_cmp): yield line @@ -43,6 +48,9 @@ class CacheDirectoryAccessSource(CacheDirectoryMixin, DirectoryAccessSource): # ============================================================================ class AccessChecker(object): + EXACT_SUFFIX = '###' + EXACT_SUFFIX_B = b'###' + def __init__(self, access_source, default_access='allow'): if isinstance(access_source, str): self.aggregator = self.create_access_aggregator([access_source]) @@ -76,22 +84,32 @@ def create_access_source(self, filename): raise Exception('Invalid Access Source: ' + filename) def find_access_rule(self, url, ts=None, urlkey=None): - params = {'url': url, 'urlkey': urlkey, 'nosource': 'true'} + params = {'url': url, + 'urlkey': urlkey, + 'nosource': 'true', + 'exact_match_suffix': self.EXACT_SUFFIX_B + } + acl_iter, errs = self.aggregator(params) if errs: print(errs) key = params['key'] + key_exact = key + self.EXACT_SUFFIX_B tld = key.split(b',')[0] for acl in acl_iter: + # skip empty/invalid lines if not acl: continue acl_key = acl.split(b' ')[0] + if key_exact == acl_key: + return CDXObject(acl) + if key.startswith(acl_key): return CDXObject(acl) diff --git a/pywb/warcserver/test/test_access.py b/pywb/warcserver/test/test_access.py index db2bcab13..41a8a11db 100644 --- a/pywb/warcserver/test/test_access.py +++ b/pywb/warcserver/test/test_access.py @@ -114,3 +114,17 @@ def test_excludes_dir(self): assert edx['urlkey'] == 'net,example)/abc/path' assert edx['access'] == 'block' + # exact-only matchc + edx = access.find_access_rule('https://www.iana.org/') + assert edx['urlkey'] == 'org,iana)/###' + assert edx['access'] == 'allow' + + edx = access.find_access_rule('https://www.iana.org/any/other') + assert edx['urlkey'] == 'org,iana)/' + assert edx['access'] == 'exclude' + + edx = access.find_access_rule('https://www.iana.org/x') + assert edx['urlkey'] == 'org,iana)/' + assert edx['access'] == 'exclude' + + diff --git a/sample_archive/access/pywb.aclj b/sample_archive/access/pywb.aclj index 4808fb45d..c06ba1891 100644 --- a/sample_archive/access/pywb.aclj +++ b/sample_archive/access/pywb.aclj @@ -1,5 +1,6 @@ org,iana)/about - {"access": "block"} org,iana)/_css/2013.1/fonts/opensans-semibold.ttf - {"access": "allow"} org,iana)/_css - {"access": "exclude"} +org,iana)/### - {"access": "allow"} org,iana)/ - {"access": "exclude"} org,example)/?example=1 - {"access": "block"} diff --git a/tests/test_acl.py b/tests/test_acl.py index 7c87c6d34..2554d2e51 100644 --- a/tests/test_acl.py +++ b/tests/test_acl.py @@ -18,11 +18,18 @@ def query(self, url, coll='pywb'): return self.testapp.get('/{coll}/cdx?'.format(coll=coll) + urlencode(params, doseq=1)) def test_excluded_url(self): - resp = self.query('http://www.iana.org/') + resp = self.query('http://www.iana.org/domains/root') assert len(resp.text.splitlines()) == 0 - self.testapp.get('/pywb/mp_/http://www.iana.org/', status=404) + self.testapp.get('/pywb/mp_/http://www.iana.org/domains/root', status=404) + + def test_allowed_exact_url(self): + resp = self.query('http://www.iana.org/') + + assert len(resp.text.splitlines()) == 3 + + self.testapp.get('/pywb/mp_/http://www.iana.org/', status=200) def test_blocked_url(self): resp = self.query('http://www.iana.org/about/') diff --git a/tests/test_acl_manager.py b/tests/test_acl_manager.py index 945c4beca..16f2239dc 100644 --- a/tests/test_acl_manager.py +++ b/tests/test_acl_manager.py @@ -79,6 +79,23 @@ def test_remove_acl(self): with open(self.acl_filename, 'rt') as fh: assert fh.read() == """\ com,example)/ - {"access": "allow", "url": "http://example.com/"} +""" + + def test_acl_add_exact(self): + wb_manager(['acl', 'add', '--exact-match', self.acl_filename, 'example.com', 'block']) + + with open(self.acl_filename, 'rt') as fh: + assert fh.read() == """\ +com,example)/### - {"access": "block", "url": "example.com"} +com,example)/ - {"access": "allow", "url": "http://example.com/"} +""" + + def test_remove_acl_exact(self): + wb_manager(['acl', 'remove', '-e', self.acl_filename, 'https://example.com/']) + + with open(self.acl_filename, 'rt') as fh: + assert fh.read() == """\ +com,example)/ - {"access": "allow", "url": "http://example.com/"} """ def test_validate_and_sort_acl(self): From e04adea7a8dac81b64174130506cb630d0f86fa7 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Sat, 9 Mar 2019 15:48:45 -0800 Subject: [PATCH 40/89] transclusions/augmentations: add new video/audio translcusions script - enabled with 'transclusions: 2' (default) config option - legacy flash-supporting transclusions script (still working) available via 'transclusions: 1' or enable_flash_video_rewrite option - add transclusions.js with support for poster image - legacy vidrw: don't add undefined url as source - locatization: wrap text in not_found.html to be translatable --- pywb/static/transclusions.js | 88 +++++++++++++++++++++++++++++++++ pywb/static/vidrw.js | 3 ++ pywb/templates/head_insert.html | 6 +-- pywb/templates/not_found.html | 4 +- tests/test_integration.py | 1 + 5 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 pywb/static/transclusions.js diff --git a/pywb/static/transclusions.js b/pywb/static/transclusions.js new file mode 100644 index 000000000..8238caa4c --- /dev/null +++ b/pywb/static/transclusions.js @@ -0,0 +1,88 @@ +(function() { + var loaded = false; + + document.addEventListener("readystatechange", function() { + if (document.readyState === "complete") { + if (!loaded) { + loadTransclusions(); + loaded = true; + } + } + }); + + function loadTransclusions() { + var viUrl = window.location.href.replace("mp_", "vi_"); + + window.fetch(viUrl) + .then(function(response) { + return response.json(); + }) + .then(function(json) { + addTransclusions(json); + }) + .catch(function(err) { + }); + } + + function addTransclusions(json) { + var selector = json.selector || "object, embed"; + var result = document.querySelector(selector); + if (!result) { + console.warn("No target to add video/audio transclusions"); + return; + } + + var parentElem = result.parentElement; + + if (!json.formats) { + console.warn("No formats to add!"); + return; + } + + var isAudio = false; + + try { + isAudio = json.formats.reduce(function(accum, curr) { + return accum && (curr.skip_as_source || (curr && curr.mime && curr.mime.startsWith("audio/"))); + }, true); + } catch (e) { + isAudio = false; + } + + var media = document.createElement(!isAudio ? "video" : "audio"); + media.setAttribute("controls", "true"); + media.setAttribute("style", "width: 100%; height: 100%"); + //media.setAttribute("autoplay", "true"); + //media.setAttribute("muted", true); + + media.oncanplaythrough = function() { + if (!media.hasStarted) { + //media.muted = true; + media.hasStarted = true; + } + //media.play(); + } + + json.formats.forEach(function(data) { + if (data.skip_as_source) { + return; + } + + if (data.name === "png_poster") { + media.setAttribute("poster", data.url); + return; + } + + var source = document.createElement("source"); + source.src = data.url; + if (data.mime) { + source.type = data.mime; + } + media.appendChild(source); + }); + + parentElem.replaceChild(media, result); + } + +})(); + diff --git a/pywb/static/vidrw.js b/pywb/static/vidrw.js index 73f4c1b40..3296ea08a 100644 --- a/pywb/static/vidrw.js +++ b/pywb/static/vidrw.js @@ -594,6 +594,9 @@ __wbvidrw = (function() { if (i < 0) { url = info.url; + if (!url) { + continue; + } format = get_format_ext(info); } else { if (!info.formats[i]._wb_canPlay) { diff --git a/pywb/templates/head_insert.html b/pywb/templates/head_insert.html index d42d0a354..2527d89c6 100644 --- a/pywb/templates/head_insert.html +++ b/pywb/templates/head_insert.html @@ -51,12 +51,12 @@ {% endif %} -{% if config.enable_flash_video_rewrite %} +{% if config.enable_flash_video_rewrite or config.transclusions_version == 1 %} -{% endif %} -{% if config.enable_transclusions %} +{% elif config.transclusions_version | default(2) == 2 %} + {% endif %} {{ banner_html }} diff --git a/pywb/templates/not_found.html b/pywb/templates/not_found.html index 4cfc47285..ac5cf5fc2 100644 --- a/pywb/templates/not_found.html +++ b/pywb/templates/not_found.html @@ -5,10 +5,10 @@ {% block body %}
-

URL Not Found

+

{% trans %}URL Not Found{% endtrans %}

- The url {{ url }} could not be found in this collection. + {% trans %}The url {{ url }} could not be found in this collection.{% endtrans %}

{% if wbrequest and wbrequest.env.pywb_proxy_magic and url %}

diff --git a/tests/test_integration.py b/tests/test_integration.py index 485c2eadf..211d19a84 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -103,6 +103,7 @@ def test_replay_content(self, fmod): assert '"20140127171238"' in resp.text, resp.text assert 'wombat.js' in resp.text + assert 'transclusions.js' in resp.text assert '_WBWombatInit' in resp.text, resp.text assert 'wbinfo.enable_auto_fetch = false;' in resp.text assert '/pywb/20140127171238{0}/http://www.iana.org/time-zones"'.format(fmod) in resp.text From ce10d9af7c4ed2a6bcdb50c0479440ea0745675d Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Wed, 3 Apr 2019 12:32:09 -0700 Subject: [PATCH 41/89] docstrings: add docstrings, remove duplicate call, cleanup ACLManager init --- pywb/manager/aclmanager.py | 10 ++++++++++ pywb/manager/manager.py | 1 + 2 files changed, 11 insertions(+) diff --git a/pywb/manager/aclmanager.py b/pywb/manager/aclmanager.py index 248f91e6c..4ce46c091 100644 --- a/pywb/manager/aclmanager.py +++ b/pywb/manager/aclmanager.py @@ -22,6 +22,9 @@ class ACLManager(CollectionsManager): DEFAULT_FILE = 'access-rules.aclj' def __init__(self, r): + """ + :param r: Parsed result from ArgumentParser + """ self.rules = [] coll_name = r.coll_name @@ -32,6 +35,13 @@ def __init__(self, r): super(ACLManager, self).__init__(coll_name, must_exist=False) + def process(self, r): + """ + Process acl command + :param r: Parsed result from ArgumentParser + :return: + """ + # if target exists as a file, use that if os.path.isfile(self.target): self.acl_file = self.target diff --git a/pywb/manager/manager.py b/pywb/manager/manager.py index d86b12d70..dc36be64d 100644 --- a/pywb/manager/manager.py +++ b/pywb/manager/manager.py @@ -434,6 +434,7 @@ def do_migrate(r): from pywb.manager.aclmanager import ACLManager def do_acl(r): acl = ACLManager(r) + acl.process(r) acl_help = 'Configure Access Control Lists (ACL) for a collection' acl = subparsers.add_parser('acl', help=acl_help) From 1a7fdd0d70395f42207c00dadb4a175053919a06 Mon Sep 17 00:00:00 2001 From: John Berlin Date: Wed, 3 Apr 2019 18:02:25 -0400 Subject: [PATCH 42/89] documented and cleaned up the aclmanager.py --- pywb/manager/aclmanager.py | 122 ++++++++++++++++++++++++++++++------- 1 file changed, 99 insertions(+), 23 deletions(-) diff --git a/pywb/manager/aclmanager.py b/pywb/manager/aclmanager.py index 4ce46c091..be18ee4cf 100644 --- a/pywb/manager/aclmanager.py +++ b/pywb/manager/aclmanager.py @@ -1,16 +1,11 @@ import os -import sys -import json import re - -from argparse import ArgumentParser, RawTextHelpFormatter -from collections import OrderedDict +import sys from pywb.manager.manager import CollectionsManager -from pywb.warcserver.index.cdxobject import CDXObject from pywb.utils.canonicalize import canonicalize - from pywb.warcserver.access_checker import AccessChecker +from pywb.warcserver.index.cdxobject import CDXObject # ============================================================================ @@ -23,7 +18,8 @@ class ACLManager(CollectionsManager): def __init__(self, r): """ - :param r: Parsed result from ArgumentParser + :param argparse.Namespace r: Parsed result from ArgumentParser + :rtype: None """ self.rules = [] @@ -35,11 +31,14 @@ def __init__(self, r): super(ACLManager, self).__init__(coll_name, must_exist=False) + self.acl_file = None + def process(self, r): """ Process acl command - :param r: Parsed result from ArgumentParser - :return: + + :param argparse.Namespace r: Parsed result from ArgumentParser + :rtype: None """ # if target exists as a file, use that @@ -71,6 +70,13 @@ def process(self, r): r.acl_func(self, r) def is_valid_auto_coll(self, coll_name): + """Returns T/F indicating if the supplied collection name + is a valid collection + + :param coll_name: The collection name to check + :return: T/F indicating a valid collection + :rtype: bool + """ if not self.COLL_RX.match(coll_name): return False @@ -80,6 +86,12 @@ def is_valid_auto_coll(self, coll_name): return True def load_acl(self, must_exist=True): + """Loads the access control list + + :param bool must_exist: Does the acl file have to exist + :return: T/F indicating load success + :rtype: bool + """ try: with open(self.acl_file, 'rb') as fh: for line in fh: @@ -98,6 +110,12 @@ def load_acl(self, must_exist=True): return False def save_acl(self, r=None): + """Save the contents of the rules as cdxj entries to + the access control list file + + :param argparse.Namespace|None r: Not used + :rtype: None + """ try: os.makedirs(os.path.dirname(self.acl_file)) except OSError: @@ -114,6 +132,10 @@ def save_acl(self, r=None): def to_key(self, url_or_surt, exact_match=False): """ If 'url_or_surt' already a SURT, use as is If exact match, add the exact match suffix + + :param str url_or_surt: The url or surt to be converted to an acl key + :param bool exact_match: Should the exact match suffix be added to key + :rtype: str """ if self.SURT_RX.search(url_or_surt): result = url_or_surt @@ -126,17 +148,35 @@ def to_key(self, url_or_surt, exact_match=False): return result def validate_access(self, access): + """Returns true if the supplied access value is valid + otherwise the terminates the process + + :param str access: The access value to be validated + :return: True if valid + :rtype: bool + """ if access not in self.VALID_ACCESS: print('Valid access values are: ' + ', '.join(self.VALID_ACCESS)) sys.exit(1) - return False return True def add_rule(self, r): + """Adds a rule the ACL manager + + :param argparse.Namespace r: The argparse namespace representing the rule to be added + :rtype: None + """ return self._add_rule(r.url, r.access, r.exact_match) def _add_rule(self, url, access, exact_match=False): + """Adds an rule to the acl file + + :param str url: The URL for the rule + :param str access: The access value for the rule + :param bool exact_match: Is the rule an absolute value + :rtype: None + """ if not self.validate_access(access): return @@ -172,11 +212,22 @@ def _add_rule(self, url, access, exact_match=False): self.save_acl() - def validate_save(self, r=None): - if self.validate(True): - self.save_acl() + def validate_save(self, r=None, log=False): + """Validates the acl rules and saves the file - def validate(self, log=False): + :param argparse.Namespace|None r: Not used + :param bool log: Should a report be printed to stdout + :rtype: None + """ + self.validate(log=log, correct=True) + + def validate(self, log=False, correct=False): + """Validates the acl rules returning T/F if the list should be saved + + :param bool log: Should the results of validating be logged to stdout + :param bool correct: Should invalid results be corrected and saved + :rtype: None + """ last_rule = None out_of_order = False for rule in self.rules: @@ -189,19 +240,22 @@ def validate(self, log=False): if out_of_order: if log: print('Rules out of order, resorting') - self.rules.sort(reverse=True) - return True - else: - if log: - print('Rules in order') - - return False + if correct: + self.rules.sort(reverse=True) + self.save_acl() + elif log: + print('Rules in order') def remove_rule(self, r): + """Removes a rule from the acl file + + :param argparse.Namespace r: Parsed result from ArgumentParser + :rtype: None + """ i = 0 urlkey = self.to_key(r.url, r.exact_match) for rule in self.rules: - if urlkey == rule['urlkey']:# and r.timestamp == rule['timestamp']: + if urlkey == rule['urlkey']: acl = self.rules.pop(i) print('Removed Rule:') self.print_rule(acl) @@ -213,6 +267,11 @@ def remove_rule(self, r): print('Rule to remove not found!') def list_rules(self, r): + """Print the acl rules to the stdout + + :param argparse.Namespace|None r: Not used + :rtype: None + """ print('Rules for {0} from {1}:'.format(self.target, self.acl_file)) print('') for rule in self.rules: @@ -220,6 +279,11 @@ def list_rules(self, r): print('') def find_match(self, r): + """Finds a matching acl rule + + :param argparse.Namespace r: Parsed result from ArgumentParser + :rtype: None + """ access_checker = AccessChecker(self.acl_file, '') rule = access_checker.find_access_rule(r.url) @@ -234,6 +298,8 @@ def find_match(self, r): def add_excludes(self, r): """ Import old-style excludes, in url-per-line format + + :param argparse.Namespace r: Parsed result from ArgumentParser """ if not self.validate_access(r.access): return @@ -253,10 +319,20 @@ def add_excludes(self, r): sys.exit(1) def print_rule(self, rule): + """Prints the supplied rule to the std out + + :param CDXObject rule: The rule to be printed + :rtype: None + """ print(' ' + rule.to_cdxj()) @classmethod def init_parser(cls, parser): + """Initializes an argument parser for acl commands + + :param argparse.ArgumentParser parser: The parser to be initialized + :rtype: None + """ subparsers = parser.add_subparsers(dest='op') subparsers.required = True From 41c37129c0842467761ef3413126db2cffa71638 Mon Sep 17 00:00:00 2001 From: John Berlin Date: Wed, 3 Apr 2019 18:06:57 -0400 Subject: [PATCH 43/89] documented and cleaned up the aclmanager.py2 --- pywb/manager/aclmanager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pywb/manager/aclmanager.py b/pywb/manager/aclmanager.py index be18ee4cf..d4ce218bb 100644 --- a/pywb/manager/aclmanager.py +++ b/pywb/manager/aclmanager.py @@ -149,7 +149,7 @@ def to_key(self, url_or_surt, exact_match=False): def validate_access(self, access): """Returns true if the supplied access value is valid - otherwise the terminates the process + otherwise terminates the process :param str access: The access value to be validated :return: True if valid @@ -174,7 +174,7 @@ def _add_rule(self, url, access, exact_match=False): :param str url: The URL for the rule :param str access: The access value for the rule - :param bool exact_match: Is the rule an absolute value + :param bool exact_match: Is the rule to be added an exact match :rtype: None """ if not self.validate_access(access): From 9a40d29ac3c3cfe5a5704c990242cdc352b2af7a Mon Sep 17 00:00:00 2001 From: John Berlin Date: Wed, 3 Apr 2019 18:35:30 -0400 Subject: [PATCH 44/89] added lxml requirments entry to extra_requirments.txt and documented pywb.warcserver.index.indexsource.XmlQueryIndexSource --- extra_requirements.txt | 1 + pywb/warcserver/index/indexsource.py | 66 ++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/extra_requirements.txt b/extra_requirements.txt index 2ec3ac8d5..aa65d62ae 100644 --- a/extra_requirements.txt +++ b/extra_requirements.txt @@ -4,3 +4,4 @@ boto3 uwsgi git+https://github.com/esnme/ultrajson.git pysocks +lxml diff --git a/pywb/warcserver/index/indexsource.py b/pywb/warcserver/index/indexsource.py index 40b8e750f..9aa8ebe3a 100644 --- a/pywb/warcserver/index/indexsource.py +++ b/pywb/warcserver/index/indexsource.py @@ -18,14 +18,8 @@ from pywb.utils.wbexception import NotFoundException from pywb.warcserver.http import DefaultAdapters from pywb.warcserver.index.cdxobject import CDXObject - -from pywb.utils.format import ParamFormatter, res_template -from pywb.utils.memento import MementoUtils - from pywb.warcserver.index.cdxops import cdx_sort_closest -from six.moves.urllib.parse import quote_plus - try: from lxml import etree except: @@ -222,14 +216,28 @@ def init_from_config(cls, config): # ============================================================================= class XmlQueryIndexSource(BaseIndexSource): - EXACT_QUERY = 'type:urlquery url:' - PREFIX_QUERY = 'type:prefixquery url:' + """An index source class for XML files""" + + EXACT_QUERY = 'type:urlquery url:' # type: str + PREFIX_QUERY = 'type:prefixquery url:' # type: str def __init__(self, query_api_url): - self.query_api_url = query_api_url - self.session = requests.session() + """Initialize the XmlQueryIndexSource instance + + :param str query_api_url: The query api URL + """ + self.query_api_url = query_api_url # type: str + self.session = requests.session() # type: requests.Session def load_index(self, params): + """Loads the xml query index based on the supplied params + + :param dict[str, str] params: The query params + :return: A list or generator of cdx objects + :raises NotFoundException: If the query url is not found + or the results of the query returns no cdx entries + :raises BadRequestException: If the match type is not exact or prefix + """ closest = params.get('closest') url = params.get('url', '') @@ -244,8 +252,8 @@ def load_index(self, params): raise BadRequestException('matchType={0} is not supported'.format(matchType=matchType)) try: - #OpenSearch API requires double-escaping - #TODO: add option to not double escape if needed + # OpenSearch API requires double-escaping + # TODO: add option to not double escape if needed query_url = self.query_api_url + '?q=' + quote_plus(query + quote_plus(url)) self.logger.debug("Running query: %s" % query_url) response = self.session.get(query_url) @@ -278,6 +286,11 @@ def load_index(self, params): return cdx_iter def prefix_query_iter(self, items): + """Returns an iterator yielding the results of performing a prefix query + + :param items: The xml entry elements representing an query + :return: An iterator yielding the results of the query + """ for item in items: url = self.gettext(item, 'originalurl') if not url: @@ -288,6 +301,12 @@ def prefix_query_iter(self, items): yield cdx def convert_to_cdx(self, item): + """Converts the etree element to an CDX object + + :param item: The etree element to be converted + :return: The CDXObject representing the supplied etree element object + :rtype: CDXObject + """ cdx = CDXObject() cdx['urlkey'] = self.gettext(item, 'urlkey') cdx['timestamp'] = self.gettext(item, 'capturedate')[:14] @@ -300,6 +319,13 @@ def convert_to_cdx(self, item): return cdx def gettext(self, item, name): + """Returns the value of the supplied name + + :param item: The etree element to be converted + :param name: The name of the field to get its value for + :return: The value of the field + :rtype: str + """ elem = item.find(name) if elem is not None: return elem.text @@ -308,12 +334,25 @@ def gettext(self, item, name): @classmethod def init_from_string(cls, value): + """Creates and initializes a new instance of XmlQueryIndexSource + IFF the supplied value starts with xmlquery+ + + :param str value: The string by which to initialize the XmlQueryIndexSource + :return: The initialized XmlQueryIndexSource or None + :rtype: XmlQueryIndexSource|None + """ if value.startswith('xmlquery+'): return cls(value[9:]) - @classmethod def init_from_config(cls, config): + """Creates and initializes a new instance of XmlQueryIndexSource + IFF the supplied dictionary contains the type key equal to xmlquery + + :param dict[str, str] config: + :return: The initialized XmlQueryIndexSource or None + :rtype: XmlQueryIndexSource|None + """ if config['type'] != 'xmlquery': return @@ -565,6 +604,7 @@ def handle_timemap(self, params): timeout=params.get('_timeout')) res.raise_for_status() + assert(res.text) except Exception as e: no_except_close(res) From 8d98b9111eaf8b5a5232e6bf0c38340602acaebd Mon Sep 17 00:00:00 2001 From: John Berlin Date: Wed, 10 Apr 2019 14:00:53 -0400 Subject: [PATCH 45/89] added additional code documentation in order to meet the documentation requirements of pywb --- pywb/apps/frontendapp.py | 27 ++++++++-- pywb/apps/rewriterapp.py | 26 ++++++++++ pywb/utils/loaders.py | 14 +++++ pywb/utils/memento.py | 20 ++++++++ pywb/utils/wbexception.py | 70 ++++++++++++++++++++++++- pywb/warcserver/access_checker.py | 85 +++++++++++++++++++++++++++++-- 6 files changed, 231 insertions(+), 11 deletions(-) diff --git a/pywb/apps/frontendapp.py b/pywb/apps/frontendapp.py index cba2dff67..124eeae53 100644 --- a/pywb/apps/frontendapp.py +++ b/pywb/apps/frontendapp.py @@ -1,12 +1,10 @@ from gevent.monkey import patch_all; patch_all() -#from bottle import run, Bottle, request, response, debug from werkzeug.routing import Map, Rule from werkzeug.exceptions import HTTPException, NotFound from werkzeug.wsgi import pop_path_info from six.moves.urllib.parse import urljoin from six import iteritems -from warcio.statusandheaders import StatusAndHeaders from warcio.utils import to_native_str from warcio.timeutils import iso_date_to_timestamp from wsgiprox.wsgiprox import WSGIProxMiddleware @@ -17,14 +15,14 @@ from pywb.utils.loaders import load_yaml_config from pywb.utils.geventserver import GeventServer from pywb.utils.io import StreamIter -from pywb.utils.wbexception import NotFoundException, WbException, AppPageNotFound +from pywb.utils.wbexception import WbException, AppPageNotFound from pywb.warcserver.warcserver import WarcServer from pywb.rewrite.templateview import BaseInsertView from pywb.apps.static_handler import StaticHandler -from pywb.apps.rewriterapp import RewriterApp, UpstreamException +from pywb.apps.rewriterapp import RewriterApp from pywb.apps.wbrequestresponse import WbResponse import os @@ -71,6 +69,8 @@ def __init__(self, config_file=None, custom_config=None): self.handler = self.handle_request self.warcserver = WarcServer(config_file=config_file, custom_config=custom_config) + self.recorder = None + self.recorder_path = None config = self.warcserver.config @@ -151,7 +151,11 @@ def get_upstream_paths(self, port): return base_paths def init_recorder(self, recorder_config): - """Initialize the recording functionality of pywb. If recording_config is None this function is a no op""" + """Initialize the recording functionality of pywb. If recording_config is None this function is a no op + + :param str|dict|None recorder_config: The configuration for the recorder app + :rtype: None + """ if not recorder_config: self.recorder = None self.recorder_path = None @@ -204,6 +208,12 @@ def init_autoindex(self, auto_interval): indexer.start() def is_proxy_enabled(self, environ): + """Returns T/F indicating if proxy mode is enabled + + :param dict environ: The WSGI environment dictionary for the request + :return: T/F indicating if proxy mode is enabled + :rtype: bool + """ return self.proxy_prefix is not None and 'wsgiprox.proxy_host' in environ def serve_home(self, environ): @@ -485,6 +495,13 @@ def _check_refer_redirect(self, environ): return WbResponse.redir_response(full_url, '307 Redirect') def __call__(self, environ, start_response): + """Handles a request + + :param dict environ: The WSGI environment dictionary for the request + :param start_response: + :return: The WbResponse for the request + :rtype: WbResponse + """ return self.handler(environ, start_response) def handle_request(self, environ, start_response): diff --git a/pywb/apps/rewriterapp.py b/pywb/apps/rewriterapp.py index c7f313e6e..c60d068eb 100644 --- a/pywb/apps/rewriterapp.py +++ b/pywb/apps/rewriterapp.py @@ -147,6 +147,17 @@ def _check_accept_dt(self, wb_url, environ): return is_timegate def _get_prefer_mod(self, wb_url, environ, content_rw, is_proxy): + """Returns the default rewrite modifier and rewrite modifier based on the + value of the Prefer HTTP header if it is present + + :param WbUrl wb_url: The WbUrl for the URL being rewritten + :param dict environ: The WSGI environment dictionary for the request + :param content_rw: The content rewriter instance + :param bool is_proxy: Is the rewrite operating in proxy mode + :return: A tuple containing the default rewrite modifier and rewrite modifier based + on the value of the Prefer HTTP header if it is present + :rtype: tuple[str|None, str|None] + """ if not self.enable_prefer: return None, None @@ -516,6 +527,21 @@ def format_response(self, response, wb_url, full_prefix, is_timegate, is_proxy): def _add_memento_links(self, url, full_prefix, memento_dt, memento_ts, status_headers, is_timegate, is_proxy, coll=None, pref_applied=None, mod=None, is_memento=True): + """ + + :param str url: The URI-R being rewritten + :param str full_prefix: The replay prefix + :param str|None memento_dt: The memento datetime for the URI-R being rewritten + :param str memento_ts: The memento timestamp + :param warcio.StatusAndHeaders status_headers: + :param bool is_timegate: Are we returning a response for a timegate + :param bool is_proxy: Are we operating in proxy mode + :param str|None coll: The collection the URI-R is from + :param str|None pref_applied: + :param str|None mod: The rewrite modifier + :param bool is_memento: + :rtype: None + """ replay_mod = mod or self.replay_mod diff --git a/pywb/utils/loaders.py b/pywb/utils/loaders.py index 6ca5b6132..e2933811b 100644 --- a/pywb/utils/loaders.py +++ b/pywb/utils/loaders.py @@ -35,6 +35,11 @@ # ============================================================================ def init_yaml_env_vars(): + """Initializes the yaml parser to be able to set + the value of fields from environment variables + + :rtype: None + """ env_rx = re.compile(r'\$\{[^}]+\}') yaml.add_implicit_resolver('!envvar', env_rx) @@ -421,9 +426,18 @@ def s3_load(anon=False): # ================================================================= class WebHDFSLoader(HttpLoader): + """Loader class specifically for loading webhdfs content""" + HTTP_URL = 'http://{host}/webhdfs/v1{path}?' def load(self, url, offset, length): + """Loads the supplied web hdfs content + + :param str url: The URL to the web hdfs content to be loaded + :param int|float|double offset: The offset of the content to be loaded + :param int|float|double length: The length of the content to be loaded + :return: The raw response content + """ parts = urlsplit(url) http_url = self.HTTP_URL.format(host=parts.netloc, diff --git a/pywb/utils/memento.py b/pywb/utils/memento.py index f55c2dc6a..a20319cf2 100644 --- a/pywb/utils/memento.py +++ b/pywb/utils/memento.py @@ -66,6 +66,16 @@ def parse_links(cls, link_header, def_name='timemap'): @classmethod def make_timemap_memento_link(cls, cdx, datetime=None, rel='memento', end=',\n', memento_format=None): + """Creates a memento link string for a timemap + + :param dict cdx: The cdx object + :param str|None datetime: The datetime + :param str rel: The rel type + :param str end: Optional string appended to the end of the created link string + :param str|None memento_format: Optional string used to format the URL + :return: A memento link string + :rtype: str + """ url = cdx.get('url') if not url: url = 'file://{0}:{1}:{2}'.format(cdx.get('filename'), cdx.get('offset'), cdx.get('length')) @@ -113,6 +123,16 @@ def make_link(cls, url, type): @classmethod def make_memento_link(cls, url, type, dt, coll=None, memento_format=None): + """Creates a memento link string + + :param str url: A URL + :param str type: The rel type + :param str dt: The datetime of the URL + :param str|None coll: Optional name of a collection + :param str|None memento_format: Optional string used to format the supplied URL + :return: A memento link string + :rtype: str + """ if memento_format: memento_format = memento_format.format(url=url, timestamp=http_date_to_timestamp(dt)) diff --git a/pywb/utils/wbexception.py b/pywb/utils/wbexception.py index 228df9f24..9d986dd11 100644 --- a/pywb/utils/wbexception.py +++ b/pywb/utils/wbexception.py @@ -3,16 +3,34 @@ #================================================================= class WbException(Exception): + """Base class for exceptions raised by Pywb""" + def __init__(self, msg=None, url=None): - Exception.__init__(self, msg) + """Initialize a new WbException + + :param str|None msg: The message for the error response + :param str|None url: The URL that caused the error + :rtype: None + """ + super(WbException, self).__init__(msg) self.msg = msg self.url = url @property def status_code(self): + """Returns the status code to be used for the error response + + :return: The status code for the error response (500) + :rtype: int + """ return 500 def status(self): + """Returns the HTTP status line for the error response + + :return: The HTTP status line for the error response + :rtype: str + """ return str(self.status_code) + ' ' + HTTP_STATUS_CODES.get(self.status_code, 'Unknown Error') def __repr__(self): @@ -25,46 +43,96 @@ def __repr__(self): #================================================================= class AccessException(WbException): + """An Exception used to indicate an access control violation""" + @property def status_code(self): + """Returns the status code to be used for the error response + + :return: The status code for the error response (451) + :rtype: int + """ return 451 #================================================================= class BadRequestException(WbException): + """An Exception used to indicate that request was bad""" + @property def status_code(self): + """Returns the status code to be used for the error response + + :return: The status code for the error response (400) + :rtype: int + """ return 400 #================================================================= class NotFoundException(WbException): + """An Exception used to indicate that a resource was not found""" + @property def status_code(self): + """Returns the status code to be used for the error response + + :return: The status code for the error response (404) + :rtype: int + """ return 404 #================================================================= class LiveResourceException(WbException): + """An Exception used to indicate that an error was encountered during the + retrial of a live web resource""" + @property def status_code(self): + """Returns the status code to be used for the error response + + :return: The status code for the error response (400) + :rtype: int + """ return 400 # ============================================================================ class UpstreamException(WbException): + """An Exception used to indicate that an error was encountered from an upstream endpoint""" + def __init__(self, status_code, url, details): + """Initialize a new UpstreamException + + :param int status_code: The status code for the error response + :param str url: The URL that caused the error + :param str details: The details of the error encountered + :rtype: None + """ super(UpstreamException, self).__init__(url=url, msg=details) self._status_code = status_code @property def status_code(self): + """Returns the status code to be used for the error response + + :return: The status code for the error response + :rtype: int + """ return self._status_code # ============================================================================ class AppPageNotFound(WbException): + """An Exception used to indicate that a page was not found""" + @property def status_code(self): + """Returns the status code to be used for the error response + + :return: The status code for the error response (400) + :rtype: int + """ return 404 diff --git a/pywb/warcserver/access_checker.py b/pywb/warcserver/access_checker.py index a0eb4abfb..9cd2790e7 100644 --- a/pywb/warcserver/access_checker.py +++ b/pywb/warcserver/access_checker.py @@ -11,11 +11,28 @@ # ============================================================================ class FileAccessIndexSource(FileIndexSource): + """An Index Source class specific to access control lists""" + @staticmethod def rev_cmp(a, b): + """Performs a comparison between two items using the + algorithm of the removed builtin cmp + + :param a: A value to be compared + :param b: A value to be compared + :return: The result of the comparison + :rtype: int + """ return (a < b) - (a > b) def _do_iter(self, fh, params): + """Iterates over the supplied file handle to an access control list + yielding the results of the search for the params key + + :param TextIO fh: The file handle to an access control list + :param dict params: The params of the + :return: A generator yielding the results of the param search + """ exact_suffix = params.get('exact_match_suffix') key = params['key'] if exact_suffix: @@ -27,31 +44,47 @@ def _do_iter(self, fh, params): # ============================================================================ class ReverseMergeMixin(object): + """A mixin that provides revered merge functionality""" + def _merge(self, iter_list): + """Merges the supplied list of iterators in reverse + + :param iter_list: The list of iterators to be merged + :return: An iterator that yields the results of the reverse merge + """ return merge(*(iter_list), reverse=True) # ============================================================================ class AccessRulesAggregator(ReverseMergeMixin, SimpleAggregator): - pass + """An Aggregator specific to access control""" # ============================================================================ class DirectoryAccessSource(ReverseMergeMixin, DirectoryIndexSource): - INDEX_SOURCES = [('.aclj', FileAccessIndexSource)] + """An directory index source specific to access control""" + + INDEX_SOURCES = [('.aclj', FileAccessIndexSource)] # type: list[tuple] # ============================================================================ class CacheDirectoryAccessSource(CacheDirectoryMixin, DirectoryAccessSource): - pass + """An cache directory index source specific to access control""" # ============================================================================ class AccessChecker(object): - EXACT_SUFFIX = '###' - EXACT_SUFFIX_B = b'###' + """An access checker class""" + + EXACT_SUFFIX = '###' # type: str + EXACT_SUFFIX_B = b'###' # type: bytes def __init__(self, access_source, default_access='allow'): + """Initialize a new AccessChecker + + :param str|list[str]|AccessRulesAggregator access_source: An access source + :param str default_access: The default access action (allow) + """ if isinstance(access_source, str): self.aggregator = self.create_access_aggregator([access_source]) elif isinstance(access_source, list): @@ -66,6 +99,13 @@ def __init__(self, access_source, default_access='allow'): self.default_rule['default'] = 'true' def create_access_aggregator(self, source_files): + """Creates a new AccessRulesAggregator using the supplied list + of access control file names + + :param list[str] source_files: The list of access control file names + :return: The created AccessRulesAggregator + :rtype: AccessRulesAggregator + """ sources = {} for filename in source_files: sources[filename] = self.create_access_source(filename) @@ -74,6 +114,17 @@ def create_access_aggregator(self, source_files): return aggregator def create_access_source(self, filename): + """Creates a new access source for the supplied filename. + + If the filename is for a directory an CacheDirectoryAccessSource + instance is returned otherwise an FileAccessIndexSource instance + + :param str filename: The name of an file/directory + :return: An instance of CacheDirectoryAccessSource or FileAccessIndexSource + depending on if the supplied filename is for a directory or file + :rtype: CacheDirectoryAccessSource|FileAccessIndexSource + :raises Exception: Indicates an invalid access source was supplied + """ if os.path.isdir(filename): return CacheDirectoryAccessSource(filename) @@ -84,6 +135,16 @@ def create_access_source(self, filename): raise Exception('Invalid Access Source: ' + filename) def find_access_rule(self, url, ts=None, urlkey=None): + """Attempts to find the access control rule for the + supplied URL otherwise returns the default rule + + :param str url: The URL for the rule to be found + :param str|None ts: A timestamp (not used) + :param str|None urlkey: The access control url key + :return: The access control rule for the supplied URL + if one exists otherwise the default rule + :rtype: CDXObject + """ params = {'url': url, 'urlkey': urlkey, 'nosource': 'true', @@ -121,10 +182,24 @@ def find_access_rule(self, url, ts=None, urlkey=None): return self.default_rule def __call__(self, res): + """Wraps the cdx iter in the supplied tuple returning a + the wrapped cdx iter and the other members of the supplied + tuple in same order + + :param tuple res: The result tuple + :return: An tuple + """ cdx_iter, errs = res return self.wrap_iter(cdx_iter), errs def wrap_iter(self, cdx_iter): + """Wraps the supplied cdx iter and yields cdx objects + that contain the access control results for the cdx object + being yielded + + :param cdx_iter: The cdx object iterator to be wrapped + :return: The wrapped cdx object iterator + """ last_rule = None last_url = None From 61b6ff21e1a1c8c9b28e122bbb6f23cce6022b60 Mon Sep 17 00:00:00 2001 From: John Berlin Date: Wed, 4 Sep 2019 13:41:56 -0400 Subject: [PATCH 46/89] added missing comma to setup.py's tests_require list removed package.json from project as it is no longer required removed npm install command from .travis/install.sh --- .travis/install.sh | 1 - package.json | 34 ---------------------------------- setup.py | 26 +++++++++++++------------- 3 files changed, 13 insertions(+), 48 deletions(-) delete mode 100644 package.json diff --git a/.travis/install.sh b/.travis/install.sh index a8ea2733a..439039bd1 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -6,7 +6,6 @@ python setup.py -q install pip install -r extra_requirements.txt pip install coverage pytest-cov coveralls pip install codecov -npm install if [ "$WR_TEST" = "yes" ]; then git clone https://github.com/webrecorder/webrecorder-tests.git diff --git a/package.json b/package.json deleted file mode 100644 index 241a71baf..000000000 --- a/package.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "pywb", - "version": "1.0.0", - "description": "Web archival replay tools", - "main": "index.js", - "directories": { - "doc": "doc", - "test": "tests" - }, - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/ikreymer/pywb.git" - }, - "author": "", - "license": "GPL-3.0", - "bugs": { - "url": "https://github.com/ikreymer/pywb/issues" - }, - "homepage": "https://github.com/ikreymer/pywb#readme", - "devDependencies": { - "chai": "^3.4.1", - "karma": "^0.13.15", - "karma-chai": "^0.1.0", - "karma-chrome-launcher": "^0.2.1", - "karma-firefox-launcher": "^0.1.7", - "karma-html2js-preprocessor": "^0.1.0", - "karma-mocha": "^0.2.1", - "karma-sauce-launcher": "^0.3.0", - "mocha": "^2.3.4" - } -} diff --git a/setup.py b/setup.py index 19c973c37..bf4b5569b 100755 --- a/setup.py +++ b/setup.py @@ -18,12 +18,14 @@ def get_ldecription(): class PyTest(TestCommand): user_options = [] + def finalize_options(self): TestCommand.finalize_options(self) self.test_suite = ' ' def run_tests(self): - from gevent.monkey import patch_all; patch_all() + from gevent.monkey import patch_all + patch_all() import pytest import os @@ -36,7 +38,6 @@ def run_tests(self): sys.exit(errcode) - def get_git_short_hash(): import subprocess try: @@ -45,19 +46,19 @@ def get_git_short_hash(): hash_id = hash_id.decode('utf-8') return hash_id - except: + except Exception: return '' + def generate_git_hash_py(pkg, filename='git_hash.py'): try: git_hash = get_git_short_hash() with open(os.path.join(pkg, filename), 'wt') as fh: fh.write('git_hash = "{0}"\n'.format(git_hash)) - except: + except Exception: pass - def load_requirements(filename): with open(filename, 'rt') as fh: requirements = fh.read().rstrip().split('\n') @@ -67,6 +68,7 @@ def load_requirements(filename): requirements.append("pyAMF") return requirements + def get_package_data(): pkgs = ['static/*.*', 'templates/*', @@ -79,10 +81,8 @@ def get_package_data(): return pkgs - generate_git_hash_py('pywb') - setup( name='pywb', version=__version__, @@ -96,7 +96,7 @@ def get_package_data(): zip_safe=True, package_data={ 'pywb': get_package_data(), - }, + }, data_files=[ ('sample_archive/cdx', glob.glob('sample_archive/cdx/*')), ('sample_archive/cdxj', glob.glob('sample_archive/cdxj/*')), @@ -104,8 +104,8 @@ def get_package_data(): ('sample_archive/zipcdx', glob.glob('sample_archive/zipcdx/*')), ('sample_archive/warcs', glob.glob('sample_archive/warcs/*')), ('sample_archive/text_content', - glob.glob('sample_archive/text_content/*')), - ], + glob.glob('sample_archive/text_content/*')), + ], install_requires=load_requirements('requirements.txt'), tests_require=[ 'pytest', @@ -116,9 +116,9 @@ def get_package_data(): 'urllib3', 'werkzeug', 'httpbin==0.5.0', - 'ujson' - 'lxml', - ], + 'ujson', + 'lxml' + ], cmdclass={'test': PyTest}, test_suite='', entry_points=""" From e34606cecbb4160efbf55db40b85b19358a96b97 Mon Sep 17 00:00:00 2001 From: John Berlin Date: Wed, 4 Sep 2019 14:28:54 -0400 Subject: [PATCH 47/89] static files: - formatted them according to project - query.js: ensured correct timestamp to date function is used templates: - head_insert.html: is_framed check is no longer a string it is a boolean, corrected redirect check tests: - test_html_rewriter.py: added missing rewrite modifier test checking i.style containing a background image html encoded warcserver: - added missing quote_plus import and cleaned up imports --- pywb/rewrite/test/test_html_rewriter.py | 2 +- pywb/static/query.js | 13 ++++++++----- pywb/templates/head_insert.html | 2 +- pywb/warcserver/index/indexsource.py | 13 ++----------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/pywb/rewrite/test/test_html_rewriter.py b/pywb/rewrite/test/test_html_rewriter.py index cd7ab24be..151523b5e 100644 --- a/pywb/rewrite/test/test_html_rewriter.py +++ b/pywb/rewrite/test/test_html_rewriter.py @@ -256,7 +256,7 @@ >>> parse('') - + >>> parse("") diff --git a/pywb/static/query.js b/pywb/static/query.js index 6bba8fb09..d78cda40a 100644 --- a/pywb/static/query.js +++ b/pywb/static/query.js @@ -249,14 +249,12 @@ RenderCalendar.prototype.makeCDXRequest = function() { var queryWorker = new window.Worker(this.staticPrefix + '/queryWorker.js'); var cdxRecordMsg = 'cdxRecord'; var done = 'finished'; - var months = this.text.months; queryWorker.onmessage = function(msg) { var data = msg.data; var terminate = false; if (data.type === cdxRecordMsg) { - data.timeInfo.month = months[data.timeInfo.month]; // render the results sent to us from the worker @@ -586,9 +584,14 @@ RenderCalendar.prototype.renderDateCalPart = function( * Updates the advanced view with the supplied cdx information * @param {Object} cdxObj - CDX object for this capture */ -RenderCalendar.prototype.renderAdvancedSearchPart = function (cdxObj) { +RenderCalendar.prototype.renderAdvancedSearchPart = function(cdxObj) { // display the URL of the result - var displayedInfo = [{ tag: 'small', innerText: this.text.dateTime + this.ts_to_date(cdxObj.timestamp) }]; + var displayedInfo = [ + { + tag: 'small', + innerText: this.text.dateTime + this.tsToDate(cdxObj.timestamp) + } + ]; // display additional information about the result under the URL if (cdxObj.mime) { displayedInfo.push({ @@ -1024,7 +1027,7 @@ RenderCalendar.prototype.dateOrdinal = function(d) { * @param {boolean} [is_gmt] - Should the timestamp be converted to a gmt string * @returns {string} */ -RenderCalendar.prototype.tsToDate = function ts_to_date(ts, is_gmt) { +RenderCalendar.prototype.tsToDate = function(ts, is_gmt) { if (ts.length < 14) return ts; var datestr = ts.substring(0, 4) + diff --git a/pywb/templates/head_insert.html b/pywb/templates/head_insert.html index 2527d89c6..e6deac41e 100644 --- a/pywb/templates/head_insert.html +++ b/pywb/templates/head_insert.html @@ -3,7 +3,7 @@ {% set urlsplit = cdx.url | urlsplit %} wbinfo = {}; wbinfo.top_url = "{{ top_url }}"; -{% if is_framed == 'true' %} +{% if is_framed %} // Fast Top-Frame Redirect if (window == window.top && wbinfo.top_url) { var loc = window.location.href.replace(window.location.hash, ""); diff --git a/pywb/warcserver/index/indexsource.py b/pywb/warcserver/index/indexsource.py index 9aa8ebe3a..afd9dce6d 100644 --- a/pywb/warcserver/index/indexsource.py +++ b/pywb/warcserver/index/indexsource.py @@ -1,21 +1,12 @@ -import logging -import re - -import redis -import requests +from six.moves.urllib.parse import quote_plus from warcio.timeutils import PAD_14_DOWN, http_date_to_timestamp, pad_timestamp, timestamp_now, timestamp_to_http_date from pywb.utils.binsearch import iter_range from pywb.utils.canonicalize import canonicalize -from pywb.utils.wbexception import NotFoundException, BadRequestException - -from warcio.timeutils import timestamp_to_http_date, http_date_to_timestamp -from warcio.timeutils import timestamp_now, pad_timestamp, PAD_14_DOWN - from pywb.utils.format import res_template from pywb.utils.io import no_except_close from pywb.utils.memento import MementoUtils -from pywb.utils.wbexception import NotFoundException +from pywb.utils.wbexception import BadRequestException, NotFoundException from pywb.warcserver.http import DefaultAdapters from pywb.warcserver.index.cdxobject import CDXObject from pywb.warcserver.index.cdxops import cdx_sort_closest From ae78a955de15f037b30ee45ebe781f3d1abf6ea3 Mon Sep 17 00:00:00 2001 From: John Berlin Date: Wed, 4 Sep 2019 14:57:09 -0400 Subject: [PATCH 48/89] templates - base.html: removed including the query pages query.css in every page - query.html: include query.css in head block --- pywb/templates/base.html | 1 - pywb/templates/query.html | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pywb/templates/base.html b/pywb/templates/base.html index f9bf5b027..8980ea30f 100644 --- a/pywb/templates/base.html +++ b/pywb/templates/base.html @@ -7,7 +7,6 @@ - diff --git a/pywb/templates/query.html b/pywb/templates/query.html index bd50df439..e40388311 100644 --- a/pywb/templates/query.html +++ b/pywb/templates/query.html @@ -6,6 +6,7 @@ {% block head %} {{ super() }} + {% endblock %} From 69f7f0200623aa9c81bfb69784989bd9baf3f21c Mon Sep 17 00:00:00 2001 From: John Berlin Date: Wed, 4 Sep 2019 14:59:50 -0400 Subject: [PATCH 49/89] static files: - re-formatted: default_banner.js, queryWorker.js, search.js, wb_frame.js --- pywb/static/default_banner.js | 377 +++++++++++++++++----------------- pywb/static/queryWorker.js | 23 ++- pywb/static/search.js | 328 +++++++++++++++-------------- pywb/static/wb_frame.js | 324 ++++++++++++++--------------- 4 files changed, 542 insertions(+), 510 deletions(-) diff --git a/pywb/static/default_banner.js b/pywb/static/default_banner.js index 5def7ff24..58b43c2a5 100644 --- a/pywb/static/default_banner.js +++ b/pywb/static/default_banner.js @@ -20,200 +20,211 @@ This file is part of pywb, https://github.com/webrecorder/pywb // Creates the default pywb banner. -(function () { - if (window.top !== window) { - return; +(function() { + if (window.top !== window) { + return; + } + + /** + * The default banner class + */ + function DefaultBanner() { + if (!(this instanceof DefaultBanner)) return new DefaultBanner(); + this.banner = null; + this.captureInfo = null; + this.last_state = {}; + this.state = null; + this.title = ''; + this.loadingId = 'bannerLoading'; + this.onMessage = this.onMessage.bind(this); + } + + // Functions required to be exposed by all banners + + /** + * @desc Initialize (display) the banner + */ + DefaultBanner.prototype.init = function() { + if (window.wbinfo) { + this.createBanner('_wb_plain_banner'); + this.set_banner( + window.wbinfo.url, + window.wbinfo.timestamp, + window.wbinfo.is_live, + window.wbinfo.is_framed ? '' : document.title + ); + } else { + this.createBanner('_wb_frame_top_banner'); } - - /** - * The default banner class - */ - function DefaultBanner() { - if (!(this instanceof DefaultBanner)) return new DefaultBanner(); - this.banner = null; - this.captureInfo = null; - this.last_state = {}; - this.state = null; - this.title = ""; - this.loadingId = 'bannerLoading'; - this.onMessage = this.onMessage.bind(this); + }; + + /** + * @desc Called by ContentFrame to detect if the banner is still showing + * that the page is loading + * @returns {boolean} + */ + DefaultBanner.prototype.stillIndicatesLoading = function() { + return document.getElementById(this.loadingId) != null; + }; + + /** + * @param {string} url - The URL of the replayed page + * @param {?string} ts - The timestamp of the replayed page. + * If we are in live mode this is undefined/empty string + * @param {boolean} is_live - A bool indicating if we are operating in live mode + */ + DefaultBanner.prototype.updateCaptureInfo = function(url, ts, is_live) { + if (is_live && !ts) { + ts = new Date().toISOString().replace(/[-T:.Z]/g, ''); + } + this.set_banner(url, ts, is_live, null); + }; + + /** + * @desc Called by ContentFrame when a message is received from the replay iframe + * @param {MessageEvent} event - The message event containing the message received + * from the replayed page + */ + DefaultBanner.prototype.onMessage = function(event) { + var type = event.data.wb_type; + + if (type === 'load' || type === 'replace-url') { + this.state = event.data; + this.last_state = this.state; + this.title = event.data.title || this.title; + } else if (type === 'title') { + this.state = this.last_state; + this.title = event.data.title; + } else { + return; } - // Functions required to be exposed by all banners - - /** - * @desc Initialize (display) the banner - */ - DefaultBanner.prototype.init = function () { - if (window.wbinfo) { - this.createBanner('_wb_plain_banner'); - this.set_banner( - window.wbinfo.url, - window.wbinfo.timestamp, - window.wbinfo.is_live, - window.wbinfo.is_framed ? "" : document.title - ); - } else { - this.createBanner('_wb_frame_top_banner'); - } - }; - - /** - * @desc Called by ContentFrame to detect if the banner is still showing - * that the page is loading - * @returns {boolean} - */ - DefaultBanner.prototype.stillIndicatesLoading = function () { - return document.getElementById(this.loadingId) != null; - }; - - /** - * @param {string} url - The URL of the replayed page - * @param {?string} ts - The timestamp of the replayed page. - * If we are in live mode this is undefined/empty string - * @param {boolean} is_live - A bool indicating if we are operating in live mode - */ - DefaultBanner.prototype.updateCaptureInfo = function (url, ts, is_live) { - if (is_live && !ts) { - ts = new Date().toISOString().replace(/[-T:.Z]/g, '') - } - this.set_banner(url, ts, is_live, null); - }; - - /** - * @desc Called by ContentFrame when a message is received from the replay iframe - * @param {MessageEvent} event - The message event containing the message received - * from the replayed page - */ - DefaultBanner.prototype.onMessage = function (event) { - var type = event.data.wb_type; - - if (type === "load" || type === "replace-url") { - this.state = event.data; - this.last_state = this.state; - this.title = event.data.title || this.title; - } else if (type === "title") { - this.state = this.last_state; - this.title = event.data.title; - } else { - return; - } - - // favicon update - if (type === 'load') { - var head = document.querySelector('head'); - var oldLink = document.querySelectorAll("link[rel*='icon']"); - var i = 0; - for (; i < oldLink.length; i++) { - head.removeChild(oldLink[i]); - } - - if (this.state.icons) { - for (i = 0; i < this.state.icons.length; i++) { - var icon = this.state.icons[i]; - var link = document.createElement('link'); - link.rel = icon.rel; - link.href = icon.href; - head.appendChild(link); - } - } - } - - this.set_banner(this.state.url, this.state.ts, this.state.is_live, this.title); - }; - - // Functions internal to the default banner - - /** - * @desc Creates the underlying HTML elements comprising the banner - * @param {string} bid - The id for the banner - */ - DefaultBanner.prototype.createBanner = function (bid) { - this.banner = document.createElement("wb_div", true); - this.banner.setAttribute("id", bid); - this.banner.setAttribute("lang", "en"); - this.captureInfo = document.createElement('span'); - this.captureInfo.innerHTML = 'Loading...'; - this.captureInfo.id = '_wb_capture_info'; - this.banner.appendChild(this.captureInfo); - document.body.insertBefore(this.banner, document.body.firstChild); - }; - - /** - * @desc Converts a timestamp to a date string. If is_gmt is truthy then - * the returned data string will be the results of date.toGMTString otherwise - * its date.toLocaleString() - * @param {?string} ts - The timestamp to receive the correct date string for - * @param {boolean} is_gmt - Is the returned date string to be in GMT time - * @returns {string} - */ - DefaultBanner.prototype.ts_to_date = function (ts, is_gmt) { - if (!ts) { - return ""; - } - - if (ts.length < 14) { - ts += "00000000000000".substr(ts.length); + // favicon update + if (type === 'load') { + var head = document.querySelector('head'); + var oldLink = document.querySelectorAll("link[rel*='icon']"); + var i = 0; + for (; i < oldLink.length; i++) { + head.removeChild(oldLink[i]); + } + + if (this.state.icons) { + for (i = 0; i < this.state.icons.length; i++) { + var icon = this.state.icons[i]; + var link = document.createElement('link'); + link.rel = icon.rel; + link.href = icon.href; + head.appendChild(link); } + } + } - var datestr = (ts.substring(0, 4) + "-" + - ts.substring(4, 6) + "-" + - ts.substring(6, 8) + "T" + - ts.substring(8, 10) + ":" + - ts.substring(10, 12) + ":" + - ts.substring(12, 14) + "-00:00"); + this.set_banner( + this.state.url, + this.state.ts, + this.state.is_live, + this.title + ); + }; + + // Functions internal to the default banner + + /** + * @desc Creates the underlying HTML elements comprising the banner + * @param {string} bid - The id for the banner + */ + DefaultBanner.prototype.createBanner = function(bid) { + this.banner = document.createElement('wb_div', true); + this.banner.setAttribute('id', bid); + this.banner.setAttribute('lang', 'en'); + this.captureInfo = document.createElement('span'); + this.captureInfo.innerHTML = + 'Loading...'; + this.captureInfo.id = '_wb_capture_info'; + this.banner.appendChild(this.captureInfo); + document.body.insertBefore(this.banner, document.body.firstChild); + }; + + /** + * @desc Converts a timestamp to a date string. If is_gmt is truthy then + * the returned data string will be the results of date.toGMTString otherwise + * its date.toLocaleString() + * @param {?string} ts - The timestamp to receive the correct date string for + * @param {boolean} is_gmt - Is the returned date string to be in GMT time + * @returns {string} + */ + DefaultBanner.prototype.ts_to_date = function(ts, is_gmt) { + if (!ts) { + return ''; + } - var date = new Date(datestr); + if (ts.length < 14) { + ts += '00000000000000'.substr(ts.length); + } - if (is_gmt) { - return date.toGMTString(); - } else { - return date.toLocaleString(); - } - }; - - /** - * @desc Updates the contents displayed by the banner - * @param {?string} url - The URL of the replayed page to be displayed in the banner - * @param {?string} ts - A timestamp to be displayed in the banner - * @param {boolean} is_live - Are we in live mode - * @param {?string} title - The title of the replayed page to be displayed in the banner - */ - DefaultBanner.prototype.set_banner = function (url, ts, is_live, title) { - var capture_str; - var title_str; - - if (!ts) { - return; - } + var datestr = + ts.substring(0, 4) + + '-' + + ts.substring(4, 6) + + '-' + + ts.substring(6, 8) + + 'T' + + ts.substring(8, 10) + + ':' + + ts.substring(10, 12) + + ':' + + ts.substring(12, 14) + + '-00:00'; + + var date = new Date(datestr); + + if (is_gmt) { + return date.toGMTString(); + } else { + return date.toLocaleString(); + } + }; + + /** + * @desc Updates the contents displayed by the banner + * @param {?string} url - The URL of the replayed page to be displayed in the banner + * @param {?string} ts - A timestamp to be displayed in the banner + * @param {boolean} is_live - Are we in live mode + * @param {?string} title - The title of the replayed page to be displayed in the banner + */ + DefaultBanner.prototype.set_banner = function(url, ts, is_live, title) { + var capture_str; + var title_str; + + if (!ts) { + return; + } - var date_str = this.ts_to_date(ts, true); + var date_str = this.ts_to_date(ts, true); - if (title) { - capture_str = title; - } else { - capture_str = url; - } + if (title) { + capture_str = title; + } else { + capture_str = url; + } - title_str = capture_str; - capture_str = "" + capture_str + ""; + title_str = capture_str; + capture_str = "" + capture_str + ''; - if (is_live) { - title_str = " pywb Live: " + title_str; - capture_str += "Live on "; - } else { - title_str += "pywb Archived: " + title_str; - capture_str += "Archived on "; - } + if (is_live) { + title_str = ' pywb Live: ' + title_str; + capture_str += 'Live on '; + } else { + title_str += 'pywb Archived: ' + title_str; + capture_str += 'Archived on '; + } - title_str += " (" + date_str + ")"; - capture_str += date_str; - this.captureInfo.innerHTML = capture_str; - window.document.title = title_str; - }; + title_str += ' (' + date_str + ')'; + capture_str += date_str; + this.captureInfo.innerHTML = capture_str; + window.document.title = title_str; + }; - // all banners will expose themselves by adding themselves as WBBanner on window - window.WBBanner = new DefaultBanner(); + // all banners will expose themselves by adding themselves as WBBanner on window + window.WBBanner = new DefaultBanner(); })(); - - diff --git a/pywb/static/queryWorker.js b/pywb/static/queryWorker.js index 464390d00..4e5cc04c7 100644 --- a/pywb/static/queryWorker.js +++ b/pywb/static/queryWorker.js @@ -18,7 +18,7 @@ var decoder = new TextDecoder('utf-8'); */ var bufferedPreviousChunk = null; -self.onmessage = function (event) { +self.onmessage = function(event) { var data = event.data; if (data.type === 'query') { fetch(data.queryURL) @@ -42,7 +42,8 @@ function defaultErrorCatcher(error) { */ function consumeResponseBodyAsStream(response) { var reader = response.body.getReader(); - reader.read() + reader + .read() .then(function consumeStream(result) { if (result.done) { if (bufferedPreviousChunk) { @@ -58,7 +59,10 @@ function consumeResponseBodyAsStream(response) { return; } transformChunk(result.value); - reader.read().then(consumeStream).catch(defaultErrorCatcher); + reader + .read() + .then(consumeStream) + .catch(defaultErrorCatcher); }) .catch(defaultErrorCatcher); } @@ -154,8 +158,11 @@ function handleCDXRecord(binaryCDXRecord) { year: ts.substring(0, 4), month: ts.substring(4, 6), day: day.charAt(0) === '0' ? day.charAt(1) : day, - time: ts.substring(8, 10) + colon + - ts.substring(10, 12) + colon + + time: + ts.substring(8, 10) + + colon + + ts.substring(10, 12) + + colon + ts.substring(12, 14) }, wasError: false, @@ -163,9 +170,3 @@ function handleCDXRecord(binaryCDXRecord) { recordCountFormatted: recordCount.toLocaleString() }); } - - - - - - diff --git a/pywb/static/search.js b/pywb/static/search.js index 99f7381c5..92bae6995 100644 --- a/pywb/static/search.js +++ b/pywb/static/search.js @@ -1,175 +1,191 @@ - var dtRE = /^\d{4,14}$/; - var didSetWasValidated = false; - var showBadDateTimeClass = 'show-optional-bad-input'; - var filterMods = { - '=': 'Contains', - '==': 'Matches Exactly', - '=~': 'Matches Regex', - '=!': 'Does Not Contains', - '=!=': 'Is Not', - '=!~': 'Does Not Begins With' - }; - - var elemIds = { - filtering: { - by: 'filter-by', - modifier: 'filter-modifier', - expression: 'filter-expression', - list: 'filter-list', - nothing: 'filtering-nothing', - add: 'add-filter', - clear: 'clear-filters' - }, - dateTime: { - from: 'dt-from', - fromBad: 'dt-from-bad', - to: 'dt-to', - toBad: 'dt-to-bad' - }, - match: 'match-type-select', - url: 'search-url', - form: 'search-form', - resultsNewWindow: 'open-results-new-window' - }; +var dtRE = /^\d{4,14}$/; +var didSetWasValidated = false; +var showBadDateTimeClass = 'show-optional-bad-input'; +var filterMods = { + '=': 'Contains', + '==': 'Matches Exactly', + '=~': 'Matches Regex', + '=!': 'Does Not Contains', + '=!=': 'Is Not', + '=!~': 'Does Not Begins With' +}; +var elemIds = { + filtering: { + by: 'filter-by', + modifier: 'filter-modifier', + expression: 'filter-expression', + list: 'filter-list', + nothing: 'filtering-nothing', + add: 'add-filter', + clear: 'clear-filters' + }, + dateTime: { + from: 'dt-from', + fromBad: 'dt-from-bad', + to: 'dt-to', + toBad: 'dt-to-bad' + }, + match: 'match-type-select', + url: 'search-url', + form: 'search-form', + resultsNewWindow: 'open-results-new-window' +}; - function makeCheckDateRangeChecker(dtInputId, dtBadNotice) { - var dtInput = document.getElementById(dtInputId); - dtInput.onblur = function () { - if (dtInput.validity.valid && dtBadNotice.classList.contains(showBadDateTimeClass)) { - return dtBadNotice.classList.remove(showBadDateTimeClass); - } - if (dtInput.validity.valueMissing) { - if (dtBadNotice.classList.contains(showBadDateTimeClass)) { - dtBadNotice.classList.remove(showBadDateTimeClass); - } - return; - } - if (dtInput.validity.badInput) { - if (!dtBadNotice.classList.contains(showBadDateTimeClass)) { - dtBadNotice.classList.add(showBadDateTimeClass); - } - return; - } - var validInput = dtRE.test(dtInput.value); - if (validInput && dtBadNotice.classList.contains(showBadDateTimeClass)) { +function makeCheckDateRangeChecker(dtInputId, dtBadNotice) { + var dtInput = document.getElementById(dtInputId); + dtInput.onblur = function() { + if ( + dtInput.validity.valid && + dtBadNotice.classList.contains(showBadDateTimeClass) + ) { + return dtBadNotice.classList.remove(showBadDateTimeClass); + } + if (dtInput.validity.valueMissing) { + if (dtBadNotice.classList.contains(showBadDateTimeClass)) { dtBadNotice.classList.remove(showBadDateTimeClass); - } else if (!validInput) { + } + return; + } + if (dtInput.validity.badInput) { + if (!dtBadNotice.classList.contains(showBadDateTimeClass)) { dtBadNotice.classList.add(showBadDateTimeClass); } - }; - } + return; + } + var validInput = dtRE.test(dtInput.value); + if (validInput && dtBadNotice.classList.contains(showBadDateTimeClass)) { + dtBadNotice.classList.remove(showBadDateTimeClass); + } else if (!validInput) { + dtBadNotice.classList.add(showBadDateTimeClass); + } + }; +} - function createAndAddNoFilter(filterList) { - var nothing = document.createElement('li'); - nothing.innerText = 'No Filter'; - nothing.id = elemIds.filtering.nothing; - filterList.appendChild(nothing); - } +function createAndAddNoFilter(filterList) { + var nothing = document.createElement('li'); + nothing.innerText = 'No Filter'; + nothing.id = elemIds.filtering.nothing; + filterList.appendChild(nothing); +} - function addFilter(event) { - var by = document.getElementById(elemIds.filtering.by).value; - if (!by) return; - var modifier = document.getElementById(elemIds.filtering.modifier).value; - var expr = document.getElementById(elemIds.filtering.expression).value; - if (!expr) return; - var filterExpr = 'filter' + modifier + by + ':' + expr; - var filterList = document.getElementById(elemIds.filtering.list); - var filterNothing = document.getElementById(elemIds.filtering.nothing); - if (filterNothing) { - filterList.removeChild(filterNothing); - } - var li = document.createElement('li'); - li.innerText = 'By ' + by[0].toUpperCase() + by.substr(1) + ' ' + filterMods[modifier] + ' ' + expr; - li.dataset.filter = filterExpr; - var nukeButton = document.createElement('button'); - nukeButton.type = 'button'; - nukeButton.role = 'button'; - nukeButton.className = 'btn btn-outline-danger close'; - nukeButton.setAttribute('aria-label', 'Remove Filter'); - var buttonX = document.createElement('span'); - buttonX.className = 'px-2'; - buttonX.innerHTML = '×'; - buttonX.setAttribute('aria-hidden', 'true'); - nukeButton.appendChild(buttonX); - nukeButton.onclick = function () { - filterList.removeChild(li); - if (filterList.children.length === 0) { - createAndAddNoFilter(filterList); - } - }; - li.appendChild(nukeButton); - filterList.appendChild(li); +function addFilter(event) { + var by = document.getElementById(elemIds.filtering.by).value; + if (!by) return; + var modifier = document.getElementById(elemIds.filtering.modifier).value; + var expr = document.getElementById(elemIds.filtering.expression).value; + if (!expr) return; + var filterExpr = 'filter' + modifier + by + ':' + expr; + var filterList = document.getElementById(elemIds.filtering.list); + var filterNothing = document.getElementById(elemIds.filtering.nothing); + if (filterNothing) { + filterList.removeChild(filterNothing); } - - function clearFilters(event) { - if (document.getElementById(elemIds.filtering.nothing)) return; - var filterList = document.getElementById(elemIds.filtering.list); - while (filterList.firstElementChild) { - filterList.firstElementChild.onclick = null; - filterList.removeChild(filterList.firstElementChild); + var li = document.createElement('li'); + li.innerText = + 'By ' + + by[0].toUpperCase() + + by.substr(1) + + ' ' + + filterMods[modifier] + + ' ' + + expr; + li.dataset.filter = filterExpr; + var nukeButton = document.createElement('button'); + nukeButton.type = 'button'; + nukeButton.role = 'button'; + nukeButton.className = 'btn btn-outline-danger close'; + nukeButton.setAttribute('aria-label', 'Remove Filter'); + var buttonX = document.createElement('span'); + buttonX.className = 'px-2'; + buttonX.innerHTML = '×'; + buttonX.setAttribute('aria-hidden', 'true'); + nukeButton.appendChild(buttonX); + nukeButton.onclick = function() { + filterList.removeChild(li); + if (filterList.children.length === 0) { + createAndAddNoFilter(filterList); } - createAndAddNoFilter(filterList); + }; + li.appendChild(nukeButton); + filterList.appendChild(li); +} + +function clearFilters(event) { + if (document.getElementById(elemIds.filtering.nothing)) return; + var filterList = document.getElementById(elemIds.filtering.list); + while (filterList.firstElementChild) { + filterList.firstElementChild.onclick = null; + filterList.removeChild(filterList.firstElementChild); } + createAndAddNoFilter(filterList); +} - function performQuery(url) { - var query = [window.wb_prefix + '*?url=' + url]; - var filterExpressions = document.getElementById(elemIds.filtering.list).children; - if (filterExpressions.length) { - for (var i = 0; i < filterExpressions.length; ++i) { - var fexpr = filterExpressions[i]; - if (fexpr.dataset && fexpr.dataset.filter) { - query.push(fexpr.dataset.filter.trim()); - } +function performQuery(url) { + var query = [window.wb_prefix + '*?url=' + url]; + var filterExpressions = document.getElementById(elemIds.filtering.list) + .children; + if (filterExpressions.length) { + for (var i = 0; i < filterExpressions.length; ++i) { + var fexpr = filterExpressions[i]; + if (fexpr.dataset && fexpr.dataset.filter) { + query.push(fexpr.dataset.filter.trim()); } } - var matchType = document.getElementById(elemIds.match).value; - if (matchType) { - query.push('matchType=' + matchType.trim()); - } - var fromT = document.getElementById(elemIds.dateTime.from).value; - if (fromT) { - query.push('from=' + fromT.trim()); - } - var toT = document.getElementById(elemIds.dateTime.to).value; - if (toT) { - query.push('to=' + toT.trim()); - } - var builtQuery = query.join('&'); - if (document.getElementById(elemIds.resultsNewWindow).checked) { - try { - var win = window.open(builtQuery); - win.focus(); - } catch (e) { - document.location.href = builtQuery; - } - } else { + } + var matchType = document.getElementById(elemIds.match).value; + if (matchType) { + query.push('matchType=' + matchType.trim()); + } + var fromT = document.getElementById(elemIds.dateTime.from).value; + if (fromT) { + query.push('from=' + fromT.trim()); + } + var toT = document.getElementById(elemIds.dateTime.to).value; + if (toT) { + query.push('to=' + toT.trim()); + } + var builtQuery = query.join('&'); + if (document.getElementById(elemIds.resultsNewWindow).checked) { + try { + var win = window.open(builtQuery); + win.focus(); + } catch (e) { document.location.href = builtQuery; } + } else { + document.location.href = builtQuery; } +} - $(document).ready(function () { - $('[data-toggle="tooltip"]').tooltip({ - container: 'body', - delay: { show: 1000 } - }); - makeCheckDateRangeChecker(elemIds.dateTime.from, document.getElementById(elemIds.dateTime.fromBad)); - makeCheckDateRangeChecker(elemIds.dateTime.to, document.getElementById(elemIds.dateTime.toBad)); - document.getElementById(elemIds.filtering.add).onclick = addFilter; - document.getElementById(elemIds.filtering.clear).onclick = clearFilters; - var searchURLInput = document.getElementById(elemIds.url); - var form = document.getElementById(elemIds.form); - form.addEventListener('submit', function (event) { - event.preventDefault(); - event.stopPropagation(); - var url = searchURLInput.value; - if (!url) { - if (!didSetWasValidated) { - form.classList.add('was-validated'); - didSetWasValidated = true; - } - return; +$(document).ready(function() { + $('[data-toggle="tooltip"]').tooltip({ + container: 'body', + delay: { show: 1000 } + }); + makeCheckDateRangeChecker( + elemIds.dateTime.from, + document.getElementById(elemIds.dateTime.fromBad) + ); + makeCheckDateRangeChecker( + elemIds.dateTime.to, + document.getElementById(elemIds.dateTime.toBad) + ); + document.getElementById(elemIds.filtering.add).onclick = addFilter; + document.getElementById(elemIds.filtering.clear).onclick = clearFilters; + var searchURLInput = document.getElementById(elemIds.url); + var form = document.getElementById(elemIds.form); + form.addEventListener('submit', function(event) { + event.preventDefault(); + event.stopPropagation(); + var url = searchURLInput.value; + if (!url) { + if (!didSetWasValidated) { + form.classList.add('was-validated'); + didSetWasValidated = true; } - performQuery(url); - }); + return; + } + performQuery(url); }); +}); diff --git a/pywb/static/wb_frame.js b/pywb/static/wb_frame.js index e72f2c26e..8a9f92da7 100644 --- a/pywb/static/wb_frame.js +++ b/pywb/static/wb_frame.js @@ -17,88 +17,90 @@ This file is part of pywb, https://github.com/webrecorder/pywb along with pywb. If not, see . */ - /** * @param {Object} content_info - Information about the contents to be replayed */ function ContentFrame(content_info) { - if (!(this instanceof ContentFrame)) return new ContentFrame(content_info); - this.last_inner_hash = window.location.hash; - this.last_url = content_info.url; - this.last_ts = content_info.request_ts; - this.content_info = content_info; - // bind event callbacks - this.outer_hash_changed = this.outer_hash_changed.bind(this); - this.handle_event = this.handle_event.bind(this); - this.wbBanner = null; - this.checkBannerToId = null; - - window.addEventListener('hashchange', this.outer_hash_changed, false); - window.addEventListener('message', this.handle_event); - - if (document.readyState === 'complete') { - this.init_iframe(); - } else { - document.addEventListener('DOMContentLoaded', this.init_iframe.bind(this), {once: true}); - } - - window.__WB_pmw = function (win) { - this.pm_source = win; - return this; - }; + if (!(this instanceof ContentFrame)) return new ContentFrame(content_info); + this.last_inner_hash = window.location.hash; + this.last_url = content_info.url; + this.last_ts = content_info.request_ts; + this.content_info = content_info; + // bind event callbacks + this.outer_hash_changed = this.outer_hash_changed.bind(this); + this.handle_event = this.handle_event.bind(this); + this.wbBanner = null; + this.checkBannerToId = null; + + window.addEventListener('hashchange', this.outer_hash_changed, false); + window.addEventListener('message', this.handle_event); + + if (document.readyState === 'complete') { + this.init_iframe(); + } else { + document.addEventListener('DOMContentLoaded', this.init_iframe.bind(this), { + once: true + }); + } + + window.__WB_pmw = function(win) { + this.pm_source = win; + return this; + }; } /** * @desc Initializes the replay iframe. If a banner exists (exposed on window as WBBanner) * then the init function of the banner is called. */ -ContentFrame.prototype.init_iframe = function () { - if (typeof (this.content_info.iframe) === 'string') { - this.iframe = document.querySelector(this.content_info.iframe); - } else { - this.iframe = this.content_info.iframe; - } - - if (!this.iframe) { - console.warn('no iframe found ' + this.content_info.iframe + ' found'); - return; - } - - this.extract_prefix(); - if (window.WBBanner) { - this.wbBanner = window.WBBanner; - this.wbBanner.init(); - } - this.load_url(this.content_info.url, this.content_info.request_ts); +ContentFrame.prototype.init_iframe = function() { + if (typeof this.content_info.iframe === 'string') { + this.iframe = document.querySelector(this.content_info.iframe); + } else { + this.iframe = this.content_info.iframe; + } + + if (!this.iframe) { + console.warn('no iframe found ' + this.content_info.iframe + ' found'); + return; + } + + this.extract_prefix(); + if (window.WBBanner) { + this.wbBanner = window.WBBanner; + this.wbBanner.init(); + } + this.load_url(this.content_info.url, this.content_info.request_ts); }; /** * @desc Initializes the prefixes used to load the pages to be replayed */ -ContentFrame.prototype.extract_prefix = function () { - this.app_prefix = this.content_info.app_prefix || this.content_info.prefix; - this.content_prefix = this.content_info.content_prefix || this.content_info.prefix; - - if (this.app_prefix && this.content_prefix) { - return; - } - - var inx = window.location.href.indexOf(this.content_info.url); - - if (inx < 0) { - inx = window.location.href.indexOf('/http') + 1; - if (inx <= 0) { - inx = window.location.href.indexOf('///') + 1; - if (inx <= 0) { - console.warn('No Prefix Found!'); - } - } +ContentFrame.prototype.extract_prefix = function() { + this.app_prefix = this.content_info.app_prefix || this.content_info.prefix; + this.content_prefix = + this.content_info.content_prefix || this.content_info.prefix; + + if (this.app_prefix && this.content_prefix) { + return; + } + + var inx = window.location.href.indexOf(this.content_info.url); + + if (inx < 0) { + inx = window.location.href.indexOf('/http') + 1; + if (inx <= 0) { + inx = window.location.href.indexOf('///') + 1; + if (inx <= 0) { + console.warn('No Prefix Found!'); + } } + } - this.prefix = window.location.href.substr(0, inx); + this.prefix = window.location.href.substr(0, inx); - this.app_prefix = this.app_prefix || this.prefix; - this.content_prefix = this.content_prefix || this.prefix; + this.app_prefix = this.app_prefix || this.prefix; + this.content_prefix = this.content_prefix || this.prefix; }; /** @@ -109,46 +111,46 @@ ContentFrame.prototype.extract_prefix = function () { * @param {?boolean} content_url - Is the abs URL to be constructed using the content_prefix or app_prefix * @returns {string} */ -ContentFrame.prototype.make_url = function (url, ts, content_url) { - var mod, prefix; - - if (content_url) { - mod = 'mp_'; - prefix = this.content_prefix; - } else { - mod = ''; - prefix = this.app_prefix; - } - - if (ts || mod) { - mod += '/'; - } - - if (ts) { - return prefix + ts + mod + url; - } else { - return prefix + mod + url; - } +ContentFrame.prototype.make_url = function(url, ts, content_url) { + var mod, prefix; + + if (content_url) { + mod = 'mp_'; + prefix = this.content_prefix; + } else { + mod = ''; + prefix = this.app_prefix; + } + + if (ts || mod) { + mod += '/'; + } + + if (ts) { + return prefix + ts + mod + url; + } else { + return prefix + mod + url; + } }; /** * @desc Handles and routes all messages received from the replay iframe. * @param {MessageEvent} event - A message event potentially containing a message from the replay iframe */ -ContentFrame.prototype.handle_event = function (event) { - var frame_win = this.iframe.contentWindow; - if (event.source === window.parent) { - // Pass to replay frame - frame_win.postMessage(event.data, '*'); - } else if (event.source === frame_win) { - // Check if iframe url change message - if (typeof (event.data) === 'object' && event.data['wb_type']) { - this.handle_message(event); - } else { - // Pass to parent - window.parent.postMessage(event.data, '*'); - } +ContentFrame.prototype.handle_event = function(event) { + var frame_win = this.iframe.contentWindow; + if (event.source === window.parent) { + // Pass to replay frame + frame_win.postMessage(event.data, '*'); + } else if (event.source === frame_win) { + // Check if iframe url change message + if (typeof event.data === 'object' && event.data['wb_type']) { + this.handle_message(event); + } else { + // Pass to parent + window.parent.postMessage(event.data, '*'); } + } }; /** @@ -156,33 +158,36 @@ ContentFrame.prototype.handle_event = function (event) { * is exposed, calls the onMessage function of the exposed banner. * @param {MessageEvent} event - The message event containing a message from the replay iframe */ -ContentFrame.prototype.handle_message = function (event) { - if (this.wbBanner) { - this.wbBanner.onMessage(event); - } - var state = event.data; - var type = state.wb_type; - - if (type === 'load' || type === 'replace-url') { - this.set_url(state); - } else if (type === 'hashchange') { - this.inner_hash_changed(state); - } +ContentFrame.prototype.handle_message = function(event) { + if (this.wbBanner) { + this.wbBanner.onMessage(event); + } + var state = event.data; + var type = state.wb_type; + + if (type === 'load' || type === 'replace-url') { + this.set_url(state); + } else if (type === 'hashchange') { + this.inner_hash_changed(state); + } }; /** * @desc Updates the URL of the top frame * @param {Object} state - The contents of a message rreceived from the replay iframe */ -ContentFrame.prototype.set_url = function (state) { - if (state.url && (state.url !== this.last_url || state.request_ts !== this.last_ts)) { - var new_url = this.make_url(state.url, state.request_ts, false); - - window.history.replaceState(state, '', new_url); - - this.last_url = state.url; - this.last_ts = state.request_ts; - } +ContentFrame.prototype.set_url = function(state) { + if ( + state.url && + (state.url !== this.last_url || state.request_ts !== this.last_ts) + ) { + var new_url = this.make_url(state.url, state.request_ts, false); + + window.history.replaceState(state, '', new_url); + + this.last_url = state.url; + this.last_ts = state.request_ts; + } }; /** @@ -194,29 +199,28 @@ ContentFrame.prototype.set_url = function (state) { * @param {?string} newTs - The new timestamp of the replay iframe. Is falsy if * operating in live mode */ -ContentFrame.prototype.initBannerUpdateCheck = function (newUrl, newTs) { - if (!this.wbBanner) return; - var contentFrame = this; - var replayIframeLoaded = function () { - contentFrame.iframe.removeEventListener('load', replayIframeLoaded); - contentFrame.checkBannerToId = setTimeout(function () { - contentFrame.checkBannerToId = null; - if (contentFrame.wbBanner.stillIndicatesLoading()) { - contentFrame.wbBanner.updateCaptureInfo( - newUrl, - newTs, - contentFrame.content_prefix.indexOf('/live') !== -1 - ); - } - }, 2000); - }; - if (this.checkBannerToId) { - clearTimeout(this.checkBannerToId); - } - this.iframe.addEventListener('load', replayIframeLoaded); +ContentFrame.prototype.initBannerUpdateCheck = function(newUrl, newTs) { + if (!this.wbBanner) return; + var contentFrame = this; + var replayIframeLoaded = function() { + contentFrame.iframe.removeEventListener('load', replayIframeLoaded); + contentFrame.checkBannerToId = setTimeout(function() { + contentFrame.checkBannerToId = null; + if (contentFrame.wbBanner.stillIndicatesLoading()) { + contentFrame.wbBanner.updateCaptureInfo( + newUrl, + newTs, + contentFrame.content_prefix.indexOf('/live') !== -1 + ); + } + }, 2000); + }; + if (this.checkBannerToId) { + clearTimeout(this.checkBannerToId); + } + this.iframe.addEventListener('load', replayIframeLoaded); }; - /** * @desc Navigates the replay iframe to a newURL and if a banner is exposed * the initBannerUpdateCheck function is called. @@ -224,44 +228,44 @@ ContentFrame.prototype.initBannerUpdateCheck = function (newUrl, newTs) { * @param {?string} newTs - The new timestamp of the replay iframe. Is falsy if * operating in live mode */ -ContentFrame.prototype.load_url = function (newUrl, newTs) { - this.iframe.src = this.make_url(newUrl, newTs, true); - if (this.wbBanner) { - this.initBannerUpdateCheck(newUrl, newTs); - } +ContentFrame.prototype.load_url = function(newUrl, newTs) { + this.iframe.src = this.make_url(newUrl, newTs, true); + if (this.wbBanner) { + this.initBannerUpdateCheck(newUrl, newTs); + } }; /** * @desc Updates this frames hash to the one inside the replay iframe * @param {Object} state - The contents of message received from the replay iframe */ -ContentFrame.prototype.inner_hash_changed = function (state) { - if (window.location.hash !== state.hash) { - window.location.hash = state.hash; - } - this.last_inner_hash = state.hash; +ContentFrame.prototype.inner_hash_changed = function(state) { + if (window.location.hash !== state.hash) { + window.location.hash = state.hash; + } + this.last_inner_hash = state.hash; }; /** * @desc Updates the hash of the replay iframe on a hash change in this frame * @param event */ -ContentFrame.prototype.outer_hash_changed = function (event) { - if (window.location.hash === this.last_inner_hash) { - return; - } +ContentFrame.prototype.outer_hash_changed = function(event) { + if (window.location.hash === this.last_inner_hash) { + return; + } - if (this.iframe) { - var message = {'wb_type': 'outer_hashchange', 'hash': window.location.hash}; + if (this.iframe) { + var message = { wb_type: 'outer_hashchange', hash: window.location.hash }; - this.iframe.contentWindow.postMessage(message, '*', undefined, true); - } + this.iframe.contentWindow.postMessage(message, '*', undefined, true); + } }; /** * @desc Cleans up any event listeners added by the content frame */ -ContentFrame.prototype.close = function () { - window.removeEventListener('hashchange', this.outer_hash_changed); - window.removeEventListener('message', this.handle_event); +ContentFrame.prototype.close = function() { + window.removeEventListener('hashchange', this.outer_hash_changed); + window.removeEventListener('message', this.handle_event); }; From 5ab97a41c2397b962a1f92770dfba14f7089e71c Mon Sep 17 00:00:00 2001 From: John Berlin Date: Wed, 4 Sep 2019 15:39:47 -0400 Subject: [PATCH 50/89] templates: - not_found.html: removed un-needed closing div --- pywb/templates/not_found.html | 1 - 1 file changed, 1 deletion(-) diff --git a/pywb/templates/not_found.html b/pywb/templates/not_found.html index ac5cf5fc2..727cbdf71 100644 --- a/pywb/templates/not_found.html +++ b/pywb/templates/not_found.html @@ -17,7 +17,6 @@

{% trans %}URL Not Found{% endtrans %}

{% endif %} -
{% endblock %} From d6ab31d5294ebc9adce70a37423317dcd389f996 Mon Sep 17 00:00:00 2001 From: John Berlin Date: Thu, 5 Sep 2019 16:41:14 -0400 Subject: [PATCH 51/89] templates: - migrated proxy templates to use new template setup --- pywb/templates/proxy_cert_download.html | 7 +++++++ pywb/templates/proxy_select.html | 10 ++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pywb/templates/proxy_cert_download.html b/pywb/templates/proxy_cert_download.html index b3f51a144..d204f4856 100644 --- a/pywb/templates/proxy_cert_download.html +++ b/pywb/templates/proxy_cert_download.html @@ -1,3 +1,9 @@ +{% extends "base.html" %} + +{% block title %}Download HTTPS Certificate For PyWb Web Archive Replay{% endblock %} + +{% block body %} +

HTTPS Certificate For PyWb Web Archive Replay

{% if not available %}

Sorry, HTTPS support is not configured for this proxy. However, the proxy should work in HTTP mode.

@@ -11,3 +17,4 @@

HTTPS Certificate For PyWb Web Archive Replay

Download for Windows platforms (except if using Firefox. For Firefox, use the above download, even on Windows):

Download Certificate (Window Only)

{% endif %} +{% endblock %} diff --git a/pywb/templates/proxy_select.html b/pywb/templates/proxy_select.html index 0e831ed62..e949b839e 100644 --- a/pywb/templates/proxy_select.html +++ b/pywb/templates/proxy_select.html @@ -1,5 +1,8 @@ - - +{% extends "base.html" %} + +{% block title %}Pywb Proxy Collection Selector{% endblock %} + +{% block body %}

Pywb Proxy Collection Selector

{% if coll %}

@@ -21,5 +24,4 @@

Pywb Proxy Collection Selector

(Once selected, you will not be prompted again, however you can return to this page to switch collections.)

- - +{% endblock %} From 379f7de1ba68ee6cdc9f76e67354f2648763a736 Mon Sep 17 00:00:00 2001 From: John Berlin Date: Thu, 5 Sep 2019 18:13:12 -0400 Subject: [PATCH 52/89] manual - split out the ui customization documentation into its own file ui-customization.rst - added initial documentation covering the new template setup to the ui-customization.rst --- docs/index.rst | 1 + docs/manual/configuring.rst | 89 ------------------------ docs/manual/ui-customization.rst | 115 +++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 89 deletions(-) create mode 100644 docs/manual/ui-customization.rst diff --git a/docs/index.rst b/docs/index.rst index 58e0cb7e5..2d90b93bc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ A subset of features provides the basic functionality of a "Wayback Machine". manual/usage manual/configuring + manual/ui-customization manual/architecture manual/apis code/pywb diff --git a/docs/manual/configuring.rst b/docs/manual/configuring.rst index 807c56b2b..8d568436f 100644 --- a/docs/manual/configuring.rst +++ b/docs/manual/configuring.rst @@ -105,95 +105,6 @@ When resolving a ``example.warc.gz``, pywb will then check (in order): * Then, ``http://remote-backup.example.com/collections//example.warc.gz`` (if first lookup unsuccessful) -UI Customizations ------------------ - -pywb supports UI customizations, either for an entire archive, -or per-collection. - -Static Files -^^^^^^^^^^^^ - -The replay server will automatically support static files placed under the following directories: - -* Files under the root ``static`` directory can be accessed via ``http://my-archive.example.com/static/`` - -* Files under the per-collection ``./collections//static`` directory can be accessed via ``http://my-archive.example.com/static/_//`` - -Templates -^^^^^^^^^ - -pywb users Jinja2 templates to render HTML to render the HTML for all aspects of the application. -A version placed in the ``templates`` directory, either in the root or per collection, will override that template. - -To copy the default pywb template to the template directory run: - -``wb-manager template --add search_html`` - -The following templates are available: - - * ``home.html`` -- Home Page Template, used for ``http://my-archive.example.com/`` - - * ``search.html`` -- Collection Template, used for each collection page ``http://my-archive.example.com//`` - - * ``query.html`` -- Capture Query Page for a given url, used for ``http://my-archive.example.com/`` - -Error Pages: - - * ``not_found.html`` -- Page to show when a url is not found in the archive - - * ``error.html`` -- Generic Error Page for any error (except not found) - -Replay and Banner templates: - - * ``frame_insert.html`` -- Top-frame for framed replay mode (not used with frameless mode) - - * ``head_insert.html`` -- Rewriting code injected into ```` of each replayed page. - This template includes the banner template and itself should generally not need to be modified. - - * ``banner.html`` -- The banner used for frameless replay. Can be set to blank to disable the banner. - - -Custom Outer Replay Frame -^^^^^^^^^^^^^^^^^^^^^^^^^ - -The top-frame used for framed replay can be replaced or augmented -by modifying the ``frame_insert.html``. - -To start with modifying the default outer page, you can add it to the current -templates directory by running ``wb-manager template --add frame_insert_html`` - -To initialize the replay, the outer page should include ``wb_frame.js``, -create an `` diff --git a/pywb/version.py b/pywb/version.py index 86f56f5bc..9cd4096bb 100644 --- a/pywb/version.py +++ b/pywb/version.py @@ -1,4 +1,4 @@ -__version__ = '2.4.0' +__version__ = '2.4.0rc0' if __name__ == '__main__': print(__version__) diff --git a/pywb/warcserver/http.py b/pywb/warcserver/http.py index 4c2aa8c36..68e8711a2 100644 --- a/pywb/warcserver/http.py +++ b/pywb/warcserver/http.py @@ -4,6 +4,7 @@ import six.moves.http_client from requests.adapters import DEFAULT_POOLBLOCK, HTTPAdapter from urllib3.poolmanager import PoolManager +from urllib3.util.retry import Retry six.moves.http_client._MAXHEADERS = 10000 six.moves.http_client._MAXLINE = 131072 @@ -41,8 +42,8 @@ def proxy_manager_for(self, proxy, **proxy_kwargs): # ============================================================================= class DefaultAdapters(object): - live_adapter = PywbHttpAdapter(max_retries=3) - remote_adapter = PywbHttpAdapter(max_retries=3) + live_adapter = PywbHttpAdapter(max_retries=Retry(3)) + remote_adapter = PywbHttpAdapter(max_retries=Retry(3)) requests.packages.urllib3.disable_warnings() diff --git a/pywb/warcserver/index/fuzzymatcher.py b/pywb/warcserver/index/fuzzymatcher.py index d47d00ab2..618b64f2d 100644 --- a/pywb/warcserver/index/fuzzymatcher.py +++ b/pywb/warcserver/index/fuzzymatcher.py @@ -194,6 +194,11 @@ def match_general_fuzzy_query(self, url, urlkey, cdx, rx_cache): check_query = False url_no_query, ext = self.get_ext(url) + # don't fuzzy match to 204 + if cdx.get('status') == '204': + if '__pywb_method=options' in cdx['urlkey']: + return False + # check ext if ext and ext not in self.default_filters['not_exts']: check_query = True diff --git a/tests/test_auto_colls.py b/tests/test_auto_colls.py index b734907ca..61efc7643 100644 --- a/tests/test_auto_colls.py +++ b/tests/test_auto_colls.py @@ -283,6 +283,7 @@ def test_add_custom_banner(self): with open(banner_file, 'w+b') as fh: fh.write(b'
Custom Banner Here!
') + fh.write(b'\n{{ metadata | tojson }}') def test_add_custom_banner_replay(self, fmod): resp = self.get('/test/20140103030321/http://example.com/?example=1', fmod) @@ -314,6 +315,13 @@ def test_more_custom_templates(self): assert 'overriden search page: ' in resp.text assert '"some":"value"' in resp.text + def test_replay_banner_metadata(self, fmod): + """ Test adding metadata in replay banner (both framed and non-frame) + """ + resp = self.get('/test/20140103030321{0}/http://example.com/?example=1', fmod) + assert '
Custom Banner Here!
' in resp.text + assert '"some":"value"' in resp.text + def test_more_custom_templates_replay(self, fmod): resp = self.get('/test/20140103030321{0}/http://example.com/?example=1', fmod) assert resp.status_int == 200 From fed3263ac6d443b7c39aac07714268b09d5c7a99 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Thu, 31 Oct 2019 16:56:36 -0700 Subject: [PATCH 60/89] Docs: Fix access controls and ui customizations docs links (#513) * docs: ensure docs added to access controls, fix typos * begin changelist for 2.4.0 --- CHANGES.rst | 7 +++++++ README.rst | 6 ++++-- docs/index.rst | 1 + docs/manual/configuring.rst | 17 +++++++++++++++-- docs/manual/usage.rst | 6 +++++- 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 87b508738..ffc45d67d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +pywb 2.4.0 changelist +~~~~~~~~~~~~~~~~~~~~~ + +* Access Control System + + + pywb 2.3.5 changelist ~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.rst b/README.rst index e150bc98e..05dd8f968 100644 --- a/README.rst +++ b/README.rst @@ -39,9 +39,11 @@ The 2.x release included a major overhaul of pywb and introduces many new featur * Standalone, modular `client-side rewriting system (wombat.js) `_ to handle most modern web sites. -* Improved 'calendar' query UI, grouping results by year and month, and updated replay banner. +* Improved 'calendar' query UI with incremental loading, grouping results by year and month, and updated replay banner. -* New with 2.4: An extensinble access control system. +* New in 2.4: Extensible UI customizations system for modifying all aspects of the UI. + +* New in 2.4: Robust access control system for blocking or excluding URLs, by prefix or by exact match. Please see the `full documentation `_ for more detailed info on all these features. diff --git a/docs/index.rst b/docs/index.rst index 2d90b93bc..d184a832d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ A subset of features provides the basic functionality of a "Wayback Machine". manual/usage manual/configuring + manual/access-control manual/ui-customization manual/architecture manual/apis diff --git a/docs/manual/configuring.rst b/docs/manual/configuring.rst index 0fb47f54a..216f78665 100644 --- a/docs/manual/configuring.rst +++ b/docs/manual/configuring.rst @@ -60,6 +60,11 @@ The default directory structure for a web archive is as follows:: +-- indexes | | | +-- (CDXJ index files here) + | + | + +-- acl + | | + | +-- (.aclj access control files) | +-- templates | | @@ -105,10 +110,18 @@ When resolving a ``example.warc.gz``, pywb will then check (in order): * Then, ``http://remote-backup.example.com/collections//example.warc.gz`` (if first lookup unsuccessful) +Access Controls +^^^^^^^^^^^^^^^ + +With pywb 2.4, pywb includes an extensible :ref:`access-control` system. +By default, the access control files are stored in ``acl`` directory of each collection. + + UI Customizations ------------------ +^^^^^^^^^^^^^^^^^ -See :ref:`ui-customization` for more details on how to customize the UI. +The ``templates`` directory supports custom Jinja templates to allow customizing the UI. +See :ref:`ui-customizations` for more details on available options. Special and Custom Collections diff --git a/docs/manual/usage.rst b/docs/manual/usage.rst index 124a9ea2f..fdb0172d3 100644 --- a/docs/manual/usage.rst +++ b/docs/manual/usage.rst @@ -22,7 +22,11 @@ and introduces many new features, including: * Significantly improved :ref:`wombat` to handle most modern web sites. -* Improved 'calendar' query UI, grouping results by year and month, and updated replay banner. +* Improved 'calendar' query UI with incremental loading, grouping results by year and month, and updated replay banner. + +* New in 2.4: Extensible :ref:`ui-customizations` for modifying all aspects of the UI. + +* New in 2.4: Robust :ref:`access-control` system for blocking or excluding URLs, by prefix or by exact match. Getting Started From 8baa8cbdb754b77b2bf9c63f3e525b00d2b9cc07 Mon Sep 17 00:00:00 2001 From: Yvan Date: Fri, 1 Nov 2019 01:09:25 +0100 Subject: [PATCH 61/89] docs: fix doc typo in BaseWarcServer example (#507) --- docs/manual/warcserver.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/manual/warcserver.rst b/docs/manual/warcserver.rst index 4c47eba79..72dbd14ca 100644 --- a/docs/manual/warcserver.rst +++ b/docs/manual/warcserver.rst @@ -320,12 +320,12 @@ which does not use a YAML config .. code-block:: python - server = BaseWarcServer() + app = BaseWarcServer() # /live endpoint live_agg = SimpleAggregator({'live': LiveIndexSource()}) - server.add_route('/live', DefaultResourceHandler(live_agg)) + app.add_route('/live', DefaultResourceHandler(live_agg)) # /memento endpoint From 02cc7035e865f4026a6918a6ea23eea06b3cbde5 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Thu, 31 Oct 2019 17:09:42 -0700 Subject: [PATCH 62/89] query: fix query for IE11, don't use ES6 syntax, add URL polyfill (#514) --- pywb/static/query.js | 47 ++++++++++++++++++++++++-------------- pywb/templates/query.html | 1 + static/url-polyfill.min.js | 2 ++ 3 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 static/url-polyfill.min.js diff --git a/pywb/static/query.js b/pywb/static/query.js index 55ab84432..f736b0029 100644 --- a/pywb/static/query.js +++ b/pywb/static/query.js @@ -291,7 +291,9 @@ RenderCalendar.prototype.makeCDXRequest = function() { var spinner = document.getElementById( renderCal.domStructureIds.updatesSpinner ); - if (spinner) spinner.remove(); + if (spinner && spinner.parentNode) { + spinner.parentNode.removeChild(spinner); + } } }; queryWorker.postMessage({ @@ -330,7 +332,9 @@ RenderCalendar.prototype.makeCDXRequest = function() { var spinner = document.getElementById( renderCal.domStructureIds.updatesSpinner ); - if (spinner) spinner.remove(); + if (spinner && spinner.parentNode) { + spinner.parentNode.removeChild(spinner); + } } }); }; @@ -545,24 +549,32 @@ RenderCalendar.prototype.renderDateCalPart = function( if (memoizedYMDT[timeInfo.year] == null) { // we have not seen this year month day before // create the year month day structure (occurs once per result year) - memoizedYMDT[timeInfo.year] = { - [timeInfo.month]: { - [timeInfo.day]: { - [timeInfo.time]: 1 - } - } - }; + + var timeVal = {}; + timeVal[timeInfo.time] = 1; + + var dayVal = {}; + dayVal[timeInfo.day] = timeVal; + + var monthVal = {}; + monthVal[timeInfo.month] = dayVal; + + memoizedYMDT[timeInfo.year] = monthVal; + this.addRegYearListItem(timeInfo, active); this.addRegYearMonthListItem(timeInfo, active); return this.addRegYearMonthDayListItem(cdxObj, timeInfo, 1, active); } else if (memoizedYMDT[timeInfo.year][timeInfo.month] == null) { // we have seen the year before but not the month and day // create the month day structure (occurs for every new month encountered for an existing year) - memoizedYMDT[timeInfo.year][timeInfo.month] = { - [timeInfo.day]: { - [timeInfo.time]: 1 - } - }; + var timeVal = {}; + timeVal[timeInfo.time] = 1; + + var dayVal = {}; + dayVal[timeInfo.day] = timeVal; + + memoizedYMDT[timeInfo.year][timeInfo.month] = dayVal; + this.addRegYearMonthListItem(timeInfo, active); return this.addRegYearMonthDayListItem(cdxObj, timeInfo, 1, active); } @@ -571,9 +583,10 @@ RenderCalendar.prototype.renderDateCalPart = function( var month = memoizedYMDT[timeInfo.year][timeInfo.month]; if (month[timeInfo.day] == null) { // never seen this day (case 1) - month[timeInfo.day] = { - [timeInfo.time]: count - }; + var timeVal = {}; + timeVal[timeInfo.time] = count; + + month[timeInfo.day] = timeVal; } else if (month[timeInfo.day][timeInfo.time] == null) { // we have seen this day before but not at this time (case 2) month[timeInfo.day][timeInfo.time] = count; diff --git a/pywb/templates/query.html b/pywb/templates/query.html index e40388311..6a710fbff 100644 --- a/pywb/templates/query.html +++ b/pywb/templates/query.html @@ -7,6 +7,7 @@ {% block head %} {{ super() }} + {% endblock %} diff --git a/static/url-polyfill.min.js b/static/url-polyfill.min.js new file mode 100644 index 000000000..0e01abcb0 --- /dev/null +++ b/static/url-polyfill.min.js @@ -0,0 +1,2 @@ +(function(t){var e=function(){try{return!!Symbol.iterator}catch(e){return false}};var r=e();var n=function(t){var e={next:function(){var e=t.shift();return{done:e===void 0,value:e}}};if(r){e[Symbol.iterator]=function(){return e}}return e};var i=function(e){return encodeURIComponent(e).replace(/%20/g,"+")};var o=function(e){return decodeURIComponent(String(e).replace(/\+/g," "))};var a=function(){var a=function(e){Object.defineProperty(this,"_entries",{writable:true,value:{}});var t=typeof e;if(t==="undefined"){}else if(t==="string"){if(e!==""){this._fromString(e)}}else if(e instanceof a){var r=this;e.forEach(function(e,t){r.append(t,e)})}else if(e!==null&&t==="object"){if(Object.prototype.toString.call(e)==="[object Array]"){for(var n=0;nt[0]){return+1}else{return 0}});if(r._entries){r._entries={}}for(var e=0;e1?o(i[1]):"")}}})}})(typeof global!=="undefined"?global:typeof window!=="undefined"?window:typeof self!=="undefined"?self:this);(function(h){var e=function(){try{var e=new h.URL("b","http://a");e.pathname="c%20d";return e.href==="http://a/c%20d"&&e.searchParams}catch(e){return false}};var t=function(){var t=h.URL;var e=function(e,t){if(typeof e!=="string")e=String(e);var r=document,n;if(t&&(h.location===void 0||t!==h.location.href)){r=document.implementation.createHTMLDocument("");n=r.createElement("base");n.href=t;r.head.appendChild(n);try{if(n.href.indexOf(t)!==0)throw new Error(n.href)}catch(e){throw new Error("URL unable to set base "+t+" due to "+e)}}var i=r.createElement("a");i.href=e;if(n){r.body.appendChild(i);i.href=i.href}if(i.protocol===":"||!/:/.test(i.href)){throw new TypeError("Invalid URL")}Object.defineProperty(this,"_anchorElement",{value:i});var o=new h.URLSearchParams(this.search);var a=true;var s=true;var f=this;["append","delete","set"].forEach(function(e){var t=o[e];o[e]=function(){t.apply(o,arguments);if(a){s=false;f.search=o.toString();s=true}}});Object.defineProperty(this,"searchParams",{value:o,enumerable:true});var c=void 0;Object.defineProperty(this,"_updateSearchParams",{enumerable:false,configurable:false,writable:false,value:function(){if(this.search!==c){c=this.search;if(s){a=false;this.searchParams._fromString(this.search);a=true}}}})};var r=e.prototype;var n=function(t){Object.defineProperty(r,t,{get:function(){return this._anchorElement[t]},set:function(e){this._anchorElement[t]=e},enumerable:true})};["hash","host","hostname","port","protocol"].forEach(function(e){n(e)});Object.defineProperty(r,"search",{get:function(){return this._anchorElement["search"]},set:function(e){this._anchorElement["search"]=e;this._updateSearchParams()},enumerable:true});Object.defineProperties(r,{toString:{get:function(){var e=this;return function(){return e.href}}},href:{get:function(){return this._anchorElement.href.replace(/\?$/,"")},set:function(e){this._anchorElement.href=e;this._updateSearchParams()},enumerable:true},pathname:{get:function(){return this._anchorElement.pathname.replace(/(^\/?)/,"/")},set:function(e){this._anchorElement.pathname=e},enumerable:true},origin:{get:function(){var e={"http:":80,"https:":443,"ftp:":21}[this._anchorElement.protocol];var t=this._anchorElement.port!=e&&this._anchorElement.port!=="";return this._anchorElement.protocol+"//"+this._anchorElement.hostname+(t?":"+this._anchorElement.port:"")},enumerable:true},password:{get:function(){return""},set:function(e){},enumerable:true},username:{get:function(){return""},set:function(e){},enumerable:true}});e.createObjectURL=function(e){return t.createObjectURL.apply(t,arguments)};e.revokeObjectURL=function(e){return t.revokeObjectURL.apply(t,arguments)};h.URL=e};if(!e()){t()}if(h.location!==void 0&&!("origin"in h.location)){var r=function(){return h.location.protocol+"//"+h.location.hostname+(h.location.port?":"+h.location.port:"")};try{Object.defineProperty(h.location,"origin",{get:r,enumerable:true})}catch(e){setInterval(function(){h.location.origin=r()},100)}}})(typeof global!=="undefined"?global:typeof window!=="undefined"?window:typeof self!=="undefined"?self:this); + From 44dcd39c025b84f432dd52d8f93909a09608af6d Mon Sep 17 00:00:00 2001 From: mark f beasley Date: Fri, 1 Nov 2019 18:30:23 -0400 Subject: [PATCH 63/89] UI: tweak query page to be responsive (#515) --- pywb/static/query.js | 6 +++--- pywb/templates/base.html | 1 + pywb/templates/query.html | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pywb/static/query.js b/pywb/static/query.js index f736b0029..c4a86421d 100644 --- a/pywb/static/query.js +++ b/pywb/static/query.js @@ -391,7 +391,7 @@ RenderCalendar.prototype.createContainers = function() { // years column and initial display structure { tag: 'div', - className: 'col-2 pr-1 pl-1 h-100', + className: 'col-12 col-sm-2 pr-1 pl-1 h-100', child: { tag: 'div', className: 'list-group h-100 auto-overflow', @@ -405,7 +405,7 @@ RenderCalendar.prototype.createContainers = function() { // months initial structure { tag: 'div', - className: 'col-2 pr-1 pl-1 h-100', + className: 'col-12 mt-2 col-sm-2 mt-sm-0 pr-1 pl-1 h-100', child: { tag: 'div', className: 'tab-content h-100', @@ -418,7 +418,7 @@ RenderCalendar.prototype.createContainers = function() { // days initial structure { tag: 'div', - className: 'col-8 pl-1 h-100', + className: 'col-12 mt-3 mb-3 pr-1 mt-sm-0 mb-sm-0 pr-sm-0 col-sm-8 pl-1 h-100', child: { tag: 'div', className: 'tab-content h-100', diff --git a/pywb/templates/base.html b/pywb/templates/base.html index 8980ea30f..716ff388e 100644 --- a/pywb/templates/base.html +++ b/pywb/templates/base.html @@ -2,6 +2,7 @@ + {% block title %}{% endblock %} diff --git a/pywb/templates/query.html b/pywb/templates/query.html index 6a710fbff..55b66f45c 100644 --- a/pywb/templates/query.html +++ b/pywb/templates/query.html @@ -15,11 +15,11 @@ {% block body %}
-

{{ _('Search Results') }}

+

{{ _('Search Results') }}

-
+
+ + + {% endif %} diff --git a/pywb/templates/index.html b/pywb/templates/index.html index ecd1c3809..9157368bc 100644 --- a/pywb/templates/index.html +++ b/pywb/templates/index.html @@ -2,7 +2,7 @@ {% block body %}
-

Pywb Wayback Machine

+

{{ _('Pywb Wayback Machine') }}

This archive contains the following collections:

diff --git a/requirements.txt b/requirements.txt index be83a8f20..d666d6d05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ portalocker wsgiprox>=1.5.1 fakeredis<1.0 tldextract +babel diff --git a/tests/config_test_loc.yaml b/tests/config_test_loc.yaml new file mode 100644 index 000000000..95178ccae --- /dev/null +++ b/tests/config_test_loc.yaml @@ -0,0 +1,14 @@ +debug: true + +collections: + pywb: + index_paths: ./sample_archive/cdx/ + archive_paths: ./sample_archive/warcs/ + +# i18n +locales_root_dir: ./tests/i18n-data/ +locales: + - en + - 'l337' + + diff --git a/tests/i18n-data/.gitignore b/tests/i18n-data/.gitignore new file mode 100644 index 000000000..4d8f59eae --- /dev/null +++ b/tests/i18n-data/.gitignore @@ -0,0 +1 @@ +#allow .mo diff --git a/tests/i18n-data/l337/LC_MESSAGES/messages.mo b/tests/i18n-data/l337/LC_MESSAGES/messages.mo new file mode 100644 index 0000000000000000000000000000000000000000..294b72a250a24f30bbb8bea592fe38cb34007ed2 GIT binary patch literal 604 zcmZvZ&u-K(5XQ})3ppZAh{JH8q6cT~YzkXRiY%=8s&Yr?8~R1rrZieW-GhN2`po9c zTdLYvN7JS#H}teE8oDw9;&eJsPNuWjGQPO*7N1u1%qr>dKb=XZ1O$v9QqC!Rhp7k{ zWh2HAq)T;E_I3Wx9qhP7Hjb{W)J-Lwu>#INCkcFx2RexD|&A1bt6F6ceaU!N3ctyxB3ldQN&?y5Bik7jyQ*7V1PLKNfh)? zcNiOU2Di664jO#OME1ma2X=QQ1xQb4|smxS`t2*$99!Wxd*oMWh-}HL09^kUW QE(>wlXx3rA<$6f|0O?7gRR910 literal 0 HcmV?d00001 diff --git a/tests/i18n-data/l337/LC_MESSAGES/messages.po b/tests/i18n-data/l337/LC_MESSAGES/messages.po new file mode 100644 index 000000000..09a310acf --- /dev/null +++ b/tests/i18n-data/l337/LC_MESSAGES/messages.po @@ -0,0 +1,31 @@ +# Lithuanian translations for pywb. +# Copyright (C) 2019 pywb +# This file is distributed under the same license as the pywb project. +# FIRST AUTHOR , 2019. +# +msgid "" +msgstr "" +"Project-Id-Version: pywb 2.4.0rc0\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2019-11-06 20:20-0800\n" +"PO-Revision-Date: 2019-11-06 20:25-0800\n" +"Last-Translator: FULL NAME \n" +"Language: lt\n" +"Language-Team: lt \n" +"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && " +"(n%100<10 || n%100>=20) ? 1 : 2)\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.5.3\n" + +#: pywb/templates/banner.html:14 +msgid "Language:" +msgstr "L4n9u4g3:" + + +#: pywb/templates/index.html:5 +msgid "Pywb Wayback Machine" +msgstr "Py\\/\\/b W4yb4ck /\\/\\4ch1n3" + + diff --git a/tests/test_locales.py b/tests/test_locales.py new file mode 100644 index 000000000..76773b647 --- /dev/null +++ b/tests/test_locales.py @@ -0,0 +1,31 @@ +from .base_config_test import BaseConfigTest + + +# ============================================================================ +class TestLocales(BaseConfigTest): + @classmethod + def setup_class(cls): + super(TestLocales, cls).setup_class('config_test_loc.yaml') + + def test_locale_en_home(self): + res = self.testapp.get('/en/') + + assert 'Pywb Wayback Machine' in res.text, res.text + + def test_locale_l337_home(self): + res = self.testapp.get('/l337/') + + print(res.text) + assert r'Py\/\/b W4yb4ck /\/\4ch1n3' in res.text + + def test_locale_en_replay_banner(self): + res = self.testapp.get('/en/pywb/mp_/https://example.com/') + assert '"en"' in res.text + assert '"Language:"' in res.text + + def test_locale_l337_replay_banner(self): + res = self.testapp.get('/l337/pywb/mp_/https://example.com/') + assert '"l337"' in res.text + assert '"L4n9u4g3:"' in res.text + + From c7fdfe72a7681dcf28c405fe365a4edc51df10f0 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Tue, 12 Nov 2019 12:38:01 -0800 Subject: [PATCH 67/89] Restrict POST query size (#519) * indexing: restrict POST body appended to query to 16384, avoid reading very large POST requests on indexing --- pywb/warcserver/inputrequest.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pywb/warcserver/inputrequest.py b/pywb/warcserver/inputrequest.py index f910d2e3b..f616648e9 100644 --- a/pywb/warcserver/inputrequest.py +++ b/pywb/warcserver/inputrequest.py @@ -181,6 +181,8 @@ def _get_header(self, name): # ============================================================================ class MethodQueryCanonicalizer(object): + MAX_POST_SIZE = 16384 + def __init__(self, method, mime, length, stream, buffered_stream=None, environ=None): @@ -210,7 +212,9 @@ def __init__(self, method, mime, length, stream, if length <= 0: return - query = b'' + # max POST query allowed, for size considerations, only read upto this size + length = min(length, self.MAX_POST_SIZE) + query = [] while length > 0: buff = stream.read(length) @@ -219,7 +223,9 @@ def __init__(self, method, mime, length, stream, if not buff: break - query += buff + query.append(buff) + + query = b''.join(query) if buffered_stream: buffered_stream.write(query) From 30680803e8ad9bf35655973b0bc39596b0c7fedc Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Tue, 12 Nov 2019 12:41:04 -0800 Subject: [PATCH 68/89] proxy mode: replay improvements for content not captured via proxy mode (#520) - if preflight OPTIONS request, respond directly (don't attempt OPTIONS capture lookup) - if preflight CORS request, ensure response has appropriate CORS headers, even if not captured - wombat: update to latest wombat with updated Date() fixed timezone in proxy mode - bump version to 2.4.0rc3 --- pywb/apps/rewriterapp.py | 20 ++++++++++++++++++++ pywb/apps/wbrequestresponse.py | 8 ++++---- pywb/static/wombatProxyMode.js | 2 +- pywb/version.py | 2 +- tests/test_proxy.py | 14 ++++++++++++++ wombat | 2 +- 6 files changed, 41 insertions(+), 7 deletions(-) diff --git a/pywb/apps/rewriterapp.py b/pywb/apps/rewriterapp.py index 155e4f710..9032588a8 100644 --- a/pywb/apps/rewriterapp.py +++ b/pywb/apps/rewriterapp.py @@ -326,6 +326,10 @@ def render_content(self, wb_url, kwargs, environ): 'pywb.static_prefix', '/static/') is_proxy = ('wsgiprox.proxy_host' in environ) + # if OPTIONS in proxy mode, just generate the proxy responss + if is_proxy and self.is_preflight(environ): + return WbResponse.options_response(environ) + environ['pywb.host_prefix'] = host_prefix if self.use_js_obj_proxy: @@ -551,6 +555,9 @@ def render_content(self, wb_url, kwargs, environ): response = WbResponse(status_headers, gen) + if is_proxy and environ.get('HTTP_ORIGIN'): + response.add_access_control_headers(environ) + return response def format_response(self, response, wb_url, full_prefix, is_timegate, is_proxy): @@ -817,6 +824,19 @@ def is_ajax(self, environ): return False + def is_preflight(self, environ): + if environ.get('REQUEST_METHOD') != 'OPTIONS': + return False + + if not environ.get('HTTP_ORIGIN'): + return False + + if not environ.get('HTTP_ACCESS_CONTROL_REQUEST_METHOD') and not environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'): + return False + + return True + + def get_base_url(self, wb_url, kwargs): type_ = kwargs.get('type') return self.paths[type_].format(**kwargs) diff --git a/pywb/apps/wbrequestresponse.py b/pywb/apps/wbrequestresponse.py index b06045822..b0f0f8d99 100644 --- a/pywb/apps/wbrequestresponse.py +++ b/pywb/apps/wbrequestresponse.py @@ -186,14 +186,14 @@ def add_access_control_headers(self, env=None): allowed_methods = allowed_methods + ', ' + r_method acr_headers = env.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS') if acr_headers is not None: - self.status_headers.add_header('Access-Control-Allow-Headers', acr_headers) + self.status_headers.replace_header('Access-Control-Allow-Headers', acr_headers) allowed_origin = env.get('HTTP_ORIGIN', env.get('HTTP_REFERER', allowed_origin)) if allowed_origin is None: allowed_origin = '*' self.status_headers.replace_header('Access-Control-Allow-Origin', allowed_origin) - self.status_headers.add_header('Access-Control-Allow-Methods', allowed_methods) - self.status_headers.add_header('Access-Control-Allow-Credentials', 'true') - self.status_headers.add_header('Access-Control-Max-Age', '1800') + self.status_headers.replace_header('Access-Control-Allow-Methods', allowed_methods) + self.status_headers.replace_header('Access-Control-Allow-Credentials', 'true') + self.status_headers.replace_header('Access-Control-Max-Age', '1800') return self def __repr__(self): diff --git a/pywb/static/wombatProxyMode.js b/pywb/static/wombatProxyMode.js index c6171acdb..462ed7d9a 100644 --- a/pywb/static/wombatProxyMode.js +++ b/pywb/static/wombatProxyMode.js @@ -16,4 +16,4 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with pywb. If not, see . */ -(function(){function autobind(clazz){for(var prop,propValue,proto=clazz.__proto__||clazz.constructor.prototype||clazz.prototype,clazzProps=Object.getOwnPropertyNames(proto),len=clazzProps.length,i=0;i source[srcset], picture > source[data-srcset], picture > source[data-src], video > source[srcset], video > source[data-srcset], video > source[data-src], audio > source[srcset], audio > source[data-srcset], audio > source[data-src]",autobind(this),this._init(config,true)):new AutoFetcherProxyMode(wombat,config)}function WombatLite($wbwindow,wbinfo){return this instanceof WombatLite?void(this.wb_info=wbinfo,this.$wbwindow=$wbwindow,this.wb_info.top_host=this.wb_info.top_host||"*",this.wb_info.wombat_opts=this.wb_info.wombat_opts||{},this.WBAutoFetchWorker=null,this.historyCB=null):new WombatLite($wbwindow,wbinfo)}AutoFetcherProxyMode.prototype._init=function(config,first){var afwpm=this,wombat=this.wombat;if(document.readyState==="complete")return this.styleTag=document.createElement("style"),this.styleTag.id="$wrStyleParser$",this.styleTag.disabled=true,document.head.appendChild(this.styleTag),void(config.isTop?fetch(config.workerURL).then(function(res){res.text().then(function(text){var blob=new Blob([text],{type:"text/javascript"});afwpm.worker=new wombat.$wbwindow.Worker(URL.createObjectURL(blob),{type:"classic",credentials:"include"}),afwpm.startChecking()}).catch(error=>{console.error("Could not create the backing worker for AutoFetchWorkerProxyMode"),console.error(error)})}):(this.worker={postMessage:function(msg){msg.wb_type||(msg={wb_type:"aaworker",msg:msg}),wombat.$wbwindow.top.postMessage(msg,"*")},terminate:function(){}},this.startChecking()));if(first)var i=setInterval(function(){document.readyState==="complete"&&(afwpm._init(config),clearInterval(i))},1e3)},AutoFetcherProxyMode.prototype.startChecking=function(){for(;this.worker&&this.msgQ.length;)this.postMessage(this.msgQ.shift());this.extractFromLocalDoc(),this.mutationObz=new MutationObserver(this.mutationCB),this.mutationObz.observe(document.documentElement,{characterData:false,characterDataOldValue:false,attributes:true,attributeOldValue:true,subtree:true,childList:true,attributeFilter:["src","srcset"]})},AutoFetcherProxyMode.prototype.terminate=function(){this.worker&&this.worker.terminate()},AutoFetcherProxyMode.prototype.justFetch=function(urls){this.postMessage({type:"fetch-all",values:urls})},AutoFetcherProxyMode.prototype.fetchAsPage=function(url,title){if(url){var headers={"X-Wombat-History-Page":url};if(title){var encodedTitle=encodeURIComponent(title.trim());title&&(headers["X-Wombat-History-Title"]=encodedTitle)}var fetchData={url:url,options:{headers:headers,cache:"no-store"}};this.justFetch([fetchData])}},AutoFetcherProxyMode.prototype.postMessage=function(msg){this.worker?this.worker.postMessage(msg):this.msgQ.push(msg)},AutoFetcherProxyMode.prototype.handleMutatedStyleElem=function(elem,accum,text){var checkNode,baseURI=document.baseURI;if(text){if(!elem.parentNode||elem.parentNode.localName!=="style")return;checkNode=elem.parentNode}else checkNode=elem;try{var extractedMedia=this.extractMediaRules(checkNode.sheet,baseURI);if(extractedMedia.length)return void(accum.media=accum.media.concat(extractedMedia))}catch(e){}!text&&checkNode.href&&accum.deferred.push(this.fetchCSSAndExtract(checkNode.href))},AutoFetcherProxyMode.prototype.handleMutatedElem=function(elem,accum){var baseURI=document.baseURI;if(elem.nodeType===Node.TEXT_NODE)return this.handleMutatedStyleElem(elem,accum,true);switch(elem.localName){case"img":case"video":case"audio":case"source":return this.handleDomElement(elem,baseURI,accum);case"style":return this.handleMutatedStyleElem(elem,accum);case"link":if(elem.rel==="stylesheet"||elem.rel==="preload"&&elem.as==="style")return this.handleMutatedStyleElem(elem,accum);}return this.extractSrcSrcsetFrom(elem,baseURI,accum)},AutoFetcherProxyMode.prototype.mutationCB=function(mutationList,observer){for(var accum={type:"values",srcset:[],src:[],media:[],deferred:[]},i=0;i source[srcset], picture > source[data-srcset], picture > source[data-src], video > source[srcset], video > source[data-srcset], video > source[data-src], audio > source[srcset], audio > source[data-srcset], audio > source[data-src]",autobind(this),this._init(config,true)):new AutoFetcherProxyMode(wombat,config)}function WombatLite($wbwindow,wbinfo){return this instanceof WombatLite?void(this.wb_info=wbinfo,this.$wbwindow=$wbwindow,this.wb_info.top_host=this.wb_info.top_host||"*",this.wb_info.wombat_opts=this.wb_info.wombat_opts||{},this.WBAutoFetchWorker=null,this.historyCB=null):new WombatLite($wbwindow,wbinfo)}AutoFetcherProxyMode.prototype._init=function(config,first){var afwpm=this,wombat=this.wombat;if(document.readyState==="complete")return this.styleTag=document.createElement("style"),this.styleTag.id="$wrStyleParser$",this.styleTag.disabled=true,document.head.appendChild(this.styleTag),void(config.isTop?fetch(config.workerURL).then(function(res){res.text().then(function(text){var blob=new Blob([text],{type:"text/javascript"});afwpm.worker=new wombat.$wbwindow.Worker(URL.createObjectURL(blob),{type:"classic",credentials:"include"}),afwpm.startChecking()}).catch(error=>{console.error("Could not create the backing worker for AutoFetchWorkerProxyMode"),console.error(error)})}):(this.worker={postMessage:function(msg){msg.wb_type||(msg={wb_type:"aaworker",msg:msg}),wombat.$wbwindow.top.postMessage(msg,"*")},terminate:function(){}},this.startChecking()));if(first)var i=setInterval(function(){document.readyState==="complete"&&(afwpm._init(config),clearInterval(i))},1e3)},AutoFetcherProxyMode.prototype.startChecking=function(){for(;this.worker&&this.msgQ.length;)this.postMessage(this.msgQ.shift());this.extractFromLocalDoc(),this.mutationObz=new MutationObserver(this.mutationCB),this.mutationObz.observe(document.documentElement,{characterData:false,characterDataOldValue:false,attributes:true,attributeOldValue:true,subtree:true,childList:true,attributeFilter:["src","srcset"]})},AutoFetcherProxyMode.prototype.terminate=function(){this.worker&&this.worker.terminate()},AutoFetcherProxyMode.prototype.justFetch=function(urls){this.postMessage({type:"fetch-all",values:urls})},AutoFetcherProxyMode.prototype.fetchAsPage=function(url,title){if(url){var headers={"X-Wombat-History-Page":url};if(title){var encodedTitle=encodeURIComponent(title.trim());title&&(headers["X-Wombat-History-Title"]=encodedTitle)}var fetchData={url:url,options:{headers:headers,cache:"no-store"}};this.justFetch([fetchData])}},AutoFetcherProxyMode.prototype.postMessage=function(msg){this.worker?this.worker.postMessage(msg):this.msgQ.push(msg)},AutoFetcherProxyMode.prototype.handleMutatedStyleElem=function(elem,accum,text){var checkNode,baseURI=document.baseURI;if(text){if(!elem.parentNode||elem.parentNode.localName!=="style")return;checkNode=elem.parentNode}else checkNode=elem;try{var extractedMedia=this.extractMediaRules(checkNode.sheet,baseURI);if(extractedMedia.length)return void(accum.media=accum.media.concat(extractedMedia))}catch(e){}!text&&checkNode.href&&accum.deferred.push(this.fetchCSSAndExtract(checkNode.href))},AutoFetcherProxyMode.prototype.handleMutatedElem=function(elem,accum){var baseURI=document.baseURI;if(elem.nodeType===Node.TEXT_NODE)return this.handleMutatedStyleElem(elem,accum,true);switch(elem.localName){case"img":case"video":case"audio":case"source":return this.handleDomElement(elem,baseURI,accum);case"style":return this.handleMutatedStyleElem(elem,accum);case"link":if(elem.rel==="stylesheet"||elem.rel==="preload"&&elem.as==="style")return this.handleMutatedStyleElem(elem,accum);}return this.extractSrcSrcsetFrom(elem,baseURI,accum)},AutoFetcherProxyMode.prototype.mutationCB=function(mutationList,observer){for(var accum={type:"values",srcset:[],src:[],media:[],deferred:[]},i=0;i Date: Fri, 22 Nov 2019 12:25:18 -0800 Subject: [PATCH 69/89] index query limit: ensure 'limit' is correctly applied to XmlQueryIndexSource, fixes ukwa/ukwa-pywb#49 (#523) --- pywb/warcserver/index/indexsource.py | 4 ++++ .../index/test/test_xmlquery_indexsource.py | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pywb/warcserver/index/indexsource.py b/pywb/warcserver/index/indexsource.py index afd9dce6d..cd12a5f4c 100644 --- a/pywb/warcserver/index/indexsource.py +++ b/pywb/warcserver/index/indexsource.py @@ -243,6 +243,10 @@ def load_index(self, params): raise BadRequestException('matchType={0} is not supported'.format(matchType=matchType)) try: + limit = params.get('limit') + if limit: + query = 'limit: {0} '.format(limit) + query + # OpenSearch API requires double-escaping # TODO: add option to not double escape if needed query_url = self.query_api_url + '?q=' + quote_plus(query + quote_plus(url)) diff --git a/pywb/warcserver/index/test/test_xmlquery_indexsource.py b/pywb/warcserver/index/test/test_xmlquery_indexsource.py index 77a058236..6861aff89 100644 --- a/pywb/warcserver/index/test/test_xmlquery_indexsource.py +++ b/pywb/warcserver/index/test/test_xmlquery_indexsource.py @@ -9,9 +9,14 @@ import pytest +query_url = None + + # ============================================================================ def mock_get(self, url): string = '' + global query_url + query_url = url if quote_plus(XmlQueryIndexSource.EXACT_QUERY) in url: if quote_plus(quote_plus('http://example.com/some/path')) in url: string = URL_RESPONSE_2 @@ -65,12 +70,14 @@ def do_query(self, params): @patch('pywb.warcserver.index.indexsource.requests.sessions.Session.get', mock_get) def test_exact_query(self): - res, errs = self.do_query({'url': 'http://example.com/'}) + res, errs = self.do_query({'url': 'http://example.com/', 'limit': 100}) + expected = """\ com,example)/ 20180112200243 example.warc.gz com,example)/ 20180216200300 example.warc.gz""" assert(key_ts_res(res) == expected) assert(errs == {}) + assert query_url == 'http://localhost:8080/path?q=limit%3A+100+type%3Aurlquery+url%3Ahttp%253A%252F%252Fexample.com%252F' @patch('pywb.warcserver.index.indexsource.requests.sessions.Session.get', mock_get) @@ -82,6 +89,8 @@ def test_exact_query_2(self): assert(key_ts_res(res) == expected) assert(errs == {}) + assert query_url == 'http://localhost:8080/path?q=type%3Aurlquery+url%3Ahttp%253A%252F%252Fexample.com%252Fsome%252Fpath' + @patch('pywb.warcserver.index.indexsource.requests.sessions.Session.get', mock_get) def test_prefix_query(self): From 523e35d9735bf5b4aa205a2f8ccdf1a73d66a665 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 20 Dec 2019 17:20:45 -0800 Subject: [PATCH 70/89] fuzzy matching: apply fuzzy match if url prefix and regex match, even if no groups are captured by the regex (#524) --- pywb/warcserver/index/fuzzymatcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pywb/warcserver/index/fuzzymatcher.py b/pywb/warcserver/index/fuzzymatcher.py index 618b64f2d..178c2ce43 100644 --- a/pywb/warcserver/index/fuzzymatcher.py +++ b/pywb/warcserver/index/fuzzymatcher.py @@ -84,7 +84,7 @@ def get_fuzzy_match(self, urlkey, url, params): m = rule.regex.search(urlkey) groups = m and m.groups() - if not groups: + if groups is None: continue matched_rule = rule @@ -99,7 +99,7 @@ def get_fuzzy_match(self, urlkey, url, params): # support matching w/o query if no additional filters # don't include trailing '?' if no filters and replace_after '?' - no_filters = (filters == {'urlkey:'}) and (matched_rule.replace_after == '?') + no_filters = (not filters or filters == {'urlkey:'}) and (matched_rule.replace_after == '?') inx = url.find(matched_rule.replace_after) if inx > 0: From f0b9d5b8e8dbedeeac99608bf9fb3bb2a39f8a44 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Sat, 11 Jan 2020 10:44:49 -0800 Subject: [PATCH 71/89] Rewriting fix for DASH FB and document.write (#529) * rewrite fixes: - dash rewrite fix for fb: when rewriting, match quoted '"dash_prefetched_representation_ids"' as well as w/o quotes, update tests to ensure rewriting both old and new formats - wombat update to fix #527: ensure document.write() doesn't accidentally remove end-tag if end-tag was not lowercase (see webrecorder/wombat#21) * tests: fix recorder cookie filtering test, use https://www.google.com/ for testing * appveyor: fix appveyor builds --- appveyor.yml | 1 + pywb/recorder/test/test_recorder.py | 9 +++++---- pywb/rewrite/rewrite_dash.py | 19 ++++++++++++++----- pywb/rewrite/test/test_content_rewriter.py | 19 +++++++++++++++++++ pywb/rules.yaml | 2 +- pywb/static/wombat.js | 2 +- setup.py | 2 +- wombat | 2 +- 8 files changed, 43 insertions(+), 13 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 394e9b568..7bcf28d8e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,6 +23,7 @@ install: - "pip install pypiwin32" - "pip install certauth boto3 youtube-dl pysocks" - "pip install codecov" + - "pip install wheel" build_script: - "python setup.py install" diff --git a/pywb/recorder/test/test_recorder.py b/pywb/recorder/test/test_recorder.py index 70af7f2fa..e6b9c494e 100644 --- a/pywb/recorder/test/test_recorder.py +++ b/pywb/recorder/test/test_recorder.py @@ -71,8 +71,8 @@ def _get_dedup_index(self, dupe_policy=WriteRevisitDupePolicy(), user=True): return dedup_index - def _test_warc_write(self, recorder_app, host, path, other_params='', link_url=''): - url = 'http://' + host + path + def _test_warc_write(self, recorder_app, host, path, other_params='', link_url='', protocol='http'): + url = protocol + '://' + host + path req_url = '/live/resource/postreq?url=' + url + other_params testapp = webtest.TestApp(recorder_app) resp = testapp.post(req_url, general_req_data.format(host=host, path=path).encode('utf-8')) @@ -231,8 +231,9 @@ def test_record_skip_http_only_cookies_header(self): PerRecordWARCWriter(warc_path, header_filter=header_filter), accept_colls='live') - resp = self._test_warc_write(recorder_app, 'www.google.com', '/') - assert b'HTTP/1.1 302' in resp.body + resp = self._test_warc_write(recorder_app, 'www.google.com', '/', protocol='https') + print(resp.body.decode('utf-8')) + #assert b'HTTP/1.1 302' in resp.body buff = BytesIO(resp.body) record = ArcWarcRecordLoader().parse_record_stream(buff) diff --git a/pywb/rewrite/rewrite_dash.py b/pywb/rewrite/rewrite_dash.py index 595f93b56..7a5111039 100644 --- a/pywb/rewrite/rewrite_dash.py +++ b/pywb/rewrite/rewrite_dash.py @@ -59,21 +59,30 @@ def rewrite_dash(self, stream, rwinfo): # ============================================================================ def rewrite_fb_dash(string, *args): - DASH_SPLIT = r'\n",dash_prefetched_representation_ids:' - inx = string.find(DASH_SPLIT) + DASH_SPLITS = [r'\n",dash_prefetched_representation_ids:', r'\n","dash_prefetched_representation_ids":'] + + inx = -1 + split = None + for split in DASH_SPLITS: + inx = string.find(split) + if inx >= 0: + break + if inx < 0: - return string + return string = string[:inx] buff = string.encode('utf-8').decode('unicode-escape') + buff = buff.replace('\\/', '/') buff = buff.encode('utf-8') io = BytesIO(buff) io, best_ids = RewriteDASH().rewrite_dash(io, None) - string = json.dumps(io.read().decode('utf-8')) + buff = io.read().decode('utf-8') + string = json.dumps(buff) string = string[1:-1].replace('<', r'\x3C') - string += DASH_SPLIT + string += split string += json.dumps(best_ids) return string diff --git a/pywb/rewrite/test/test_content_rewriter.py b/pywb/rewrite/test/test_content_rewriter.py index 8e3d52511..174a222bc 100644 --- a/pywb/rewrite/test/test_content_rewriter.py +++ b/pywb/rewrite/test/test_content_rewriter.py @@ -718,6 +718,25 @@ def test_dash_fb_in_js(self): assert 'dash_prefetched_representation_ids:["1", "7"]' in result assert rep_ids not in result + def test_dash_fb_in_js_2(self): + headers = {'Content-Type': 'text/javascript'} + with open(os.path.join(get_test_dir(), 'text_content', 'sample_dash.mpd'), 'rt') as fh: + content = 'dash_manifest:"' + fh.read().encode('unicode-escape').decode('utf-8') + + rep_ids = r'\n","dash_prefetched_representation_ids":["4","5"]' + content += rep_ids + + headers, gen, is_rw = self.rewrite_record(headers, content, ts='201701js_', + url='http://facebook.com/example/dash/manifest.mpd') + + assert headers.headers == [('Content-Type', 'text/javascript')] + + result = b''.join(gen).decode('utf-8') + + # 4, 5 representations removed, replaced with default 1, 7 + assert '"dash_prefetched_representation_ids":["1", "7"]' in result + assert rep_ids not in result + def test_dash_custom_max_resolution(self): headers = {'Content-Type': 'application/dash+xml'} with open(os.path.join(get_test_dir(), 'text_content', 'sample_dash.mpd'), 'rt') as fh: diff --git a/pywb/rules.yaml b/pywb/rules.yaml index 79fe23fbc..13e3eb3c8 100644 --- a/pywb/rules.yaml +++ b/pywb/rules.yaml @@ -166,7 +166,7 @@ rules: - match: 'Bootloader\.configurePage.*?;' replace: '/* {0} */' - - match: 'dash_manifest:"(.*",dash_prefetched_representation_ids:.*?])' + - match: 'dash_manifest"?:"(.*","?dash_prefetched_representation_ids"?:.*?])' group: 1 function: 'pywb.rewrite.rewrite_dash:rewrite_fb_dash' diff --git a/pywb/static/wombat.js b/pywb/static/wombat.js index 62875bd56..2c9989837 100644 --- a/pywb/static/wombat.js +++ b/pywb/static/wombat.js @@ -16,4 +16,4 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with pywb. If not, see . */ -(function(){function FuncMap(){this._map=[]}function ensureNumber(maybeNumber){try{switch(typeof maybeNumber){case"number":case"bigint":return maybeNumber;}var converted=Number(maybeNumber);return isNaN(converted)?null:converted}catch(e){}return null}function addToStringTagToClass(clazz,tag){typeof self.Symbol!=="undefined"&&typeof self.Symbol.toStringTag!=="undefined"&&Object.defineProperty(clazz.prototype,self.Symbol.toStringTag,{value:tag,enumerable:false})}function autobind(clazz){for(var prop,propValue,proto=clazz.__proto__||clazz.constructor.prototype||clazz.prototype,clazzProps=Object.getOwnPropertyNames(proto),len=clazzProps.length,i=0;i=0){var fnMapping=this._map.splice(idx,1);return fnMapping[0][1]}return null},FuncMap.prototype.map=function(param){for(var i=0;i0&&afw.preserveMedia(media)})},AutoFetcher.prototype.terminate=function(){this.worker.terminate()},AutoFetcher.prototype.justFetch=function(urls){this.worker.postMessage({type:"fetch-all",values:urls})},AutoFetcher.prototype.fetchAsPage=function(url,originalUrl,title){if(url){var headers={"X-Wombat-History-Page":originalUrl};if(title){var encodedTitle=encodeURIComponent(title.trim());title&&(headers["X-Wombat-History-Title"]=encodedTitle)}var fetchData={url:url,options:{headers:headers,cache:"no-store"}};this.justFetch([fetchData])}},AutoFetcher.prototype.postMessage=function(msg,deferred){if(deferred){var afWorker=this;return void Promise.resolve().then(function(){afWorker.worker.postMessage(msg)})}this.worker.postMessage(msg)},AutoFetcher.prototype.preserveSrcset=function(srcset,mod){this.postMessage({type:"values",srcset:{value:srcset,mod:mod,presplit:true}},true)},AutoFetcher.prototype.preserveDataSrcset=function(elem){this.postMessage({type:"values",srcset:{value:elem.dataset.srcset,mod:this.rwMod(elem),presplit:false}},true)},AutoFetcher.prototype.preserveMedia=function(media){this.postMessage({type:"values",media:media},true)},AutoFetcher.prototype.getSrcset=function(elem){return this.wombat.wb_getAttribute?this.wombat.wb_getAttribute.call(elem,"srcset"):elem.getAttribute("srcset")},AutoFetcher.prototype.rwMod=function(elem){switch(elem.tagName){case"SOURCE":return elem.parentElement&&elem.parentElement.tagName==="PICTURE"?"im_":"oe_";case"IMG":return"im_";}return"oe_"},AutoFetcher.prototype.extractFromLocalDoc=function(){var afw=this;Promise.resolve().then(function(){for(var msg={type:"values",context:{docBaseURI:document.baseURI}},media=[],i=0,sheets=document.styleSheets;i=0},Wombat.prototype.isString=function(arg){return arg!=null&&Object.getPrototypeOf(arg)===String.prototype},Wombat.prototype.isSavedSrcSrcset=function(elem){switch(elem.tagName){case"IMG":case"VIDEO":case"AUDIO":return true;case"SOURCE":if(!elem.parentElement)return false;switch(elem.parentElement.tagName){case"PICTURE":case"VIDEO":case"AUDIO":return true;default:return false;}default:return false;}},Wombat.prototype.isSavedDataSrcSrcset=function(elem){return!!(elem.dataset&&elem.dataset.srcset!=null)&&this.isSavedSrcSrcset(elem)},Wombat.prototype.isHostUrl=function(str){if(str.indexOf("www.")===0)return true;var matches=str.match(this.hostnamePortRe);return!!(matches&&matches[0].length<64)||(matches=str.match(this.ipPortRe),!!matches&&matches[0].length<64)},Wombat.prototype.isArgumentsObj=function(maybeArgumentsObj){if(!maybeArgumentsObj||typeof maybeArgumentsObj.toString!=="function")return false;try{return this.utilFns.objToString.call(maybeArgumentsObj)==="[object Arguments]"}catch(e){return false}},Wombat.prototype.deproxyArrayHandlingArgumentsObj=function(maybeArgumentsObj){if(!maybeArgumentsObj||maybeArgumentsObj instanceof NodeList||!maybeArgumentsObj.length)return maybeArgumentsObj;for(var args=this.isArgumentsObj(maybeArgumentsObj)?new Array(maybeArgumentsObj.length):maybeArgumentsObj,i=0;i=0)||scriptType.indexOf("text/template")>=0)},Wombat.prototype.skipWrapScriptTextBasedOnText=function(text){if(!text||text.indexOf(this.WB_ASSIGN_FUNC)>=0||text.indexOf("<")===0)return true;for(var override_props=["window","self","document","location","top","parent","frames","opener"],i=0;i=0)return false;return true},Wombat.prototype.nodeHasChildren=function(node){if(!node)return false;if(typeof node.hasChildNodes==="function")return node.hasChildNodes();var kids=node.children||node.childNodes;return!!kids&&kids.length>0},Wombat.prototype.rwModForElement=function(elem,attrName){if(!elem)return undefined;var mod="mp_";if(!(elem.tagName==="LINK"&&attrName==="href")){var maybeMod=this.tagToMod[elem.tagName];maybeMod!=null&&(mod=maybeMod[attrName])}else if(elem.rel){var relV=elem.rel.trim().toLowerCase(),asV=this.wb_getAttribute.call(elem,"as");if(asV&&this.linkTagMods.linkRelToAs[relV]!=null){var asMods=this.linkTagMods.linkRelToAs[relV];mod=asMods[asV.toLowerCase()]}else this.linkTagMods[relV]!=null&&(mod=this.linkTagMods[relV])}return mod},Wombat.prototype.removeWBOSRC=function(elem){elem.tagName!=="SCRIPT"||elem.__$removedWBOSRC$__||(elem.hasAttribute("__wb_orig_src")&&elem.removeAttribute("__wb_orig_src"),elem.__$removedWBOSRC$__=true)},Wombat.prototype.retrieveWBOSRC=function(elem){if(elem.tagName==="SCRIPT"&&!elem.__$removedWBOSRC$__){var maybeWBOSRC;return maybeWBOSRC=this.wb_getAttribute?this.wb_getAttribute.call(elem,"__wb_orig_src"):elem.getAttribute("__wb_orig_src"),maybeWBOSRC==null&&(elem.__$removedWBOSRC$__=true),maybeWBOSRC}return undefined},Wombat.prototype.wrapScriptTextJsProxy=function(scriptText){return"if (!self.__WB_pmw) { self.__WB_pmw = function(obj) { this.__WB_source = obj; return this; } }\n{\nlet window = _____WB$wombat$assign$function_____(\"window\");\nlet self = _____WB$wombat$assign$function_____(\"self\");\nlet document = _____WB$wombat$assign$function_____(\"document\");\nlet location = _____WB$wombat$assign$function_____(\"location\");\nlet top = _____WB$wombat$assign$function_____(\"top\");\nlet parent = _____WB$wombat$assign$function_____(\"parent\");\nlet frames = _____WB$wombat$assign$function_____(\"frames\");\nlet opener = _____WB$wombat$assign$function_____(\"opener\");\n"+scriptText.replace(this.DotPostMessageRe,".__WB_pmw(self.window)$1")+"\n\n}"},Wombat.prototype.watchElem=function(elem,func){if(!this.$wbwindow.MutationObserver)return false;var m=new this.$wbwindow.MutationObserver(function(records,observer){for(var r,i=0;i"},Wombat.prototype.getFinalUrl=function(useRel,mod,url){var prefix=useRel?this.wb_rel_prefix:this.wb_abs_prefix;return mod==null&&(mod=this.wb_info.mod),this.wb_info.is_live||(prefix+=this.wb_info.wombat_ts),prefix+=mod,prefix[prefix.length-1]!=="/"&&(prefix+="/"),prefix+url},Wombat.prototype.resolveRelUrl=function(url,doc){var docObj=doc||this.$wbwindow.document,parser=this.makeParser(docObj.baseURI,docObj),hash=parser.href.lastIndexOf("#"),href=hash>=0?parser.href.substring(0,hash):parser.href,lastslash=href.lastIndexOf("/");return parser.href=lastslash>=0&&lastslash!==href.length-1?href.substring(0,lastslash+1)+url:href+url,parser.href},Wombat.prototype.extractOriginalURL=function(rewrittenUrl){if(!rewrittenUrl)return"";if(this.wb_is_proxy)return rewrittenUrl;var rwURLString=rewrittenUrl.toString(),url=rwURLString;if(this.startsWithOneOf(url,this.IGNORE_PREFIXES))return url;var start;start=this.startsWith(url,this.wb_abs_prefix)?this.wb_abs_prefix.length:this.wb_rel_prefix&&this.startsWith(url,this.wb_rel_prefix)?this.wb_rel_prefix.length:this.wb_rel_prefix?1:0;var index=url.indexOf("/http",start);return index<0&&(index=url.indexOf("///",start)),index<0&&(index=url.indexOf("/blob:",start)),index>=0?url=url.substr(index+1):(index=url.indexOf(this.wb_replay_prefix),index>=0&&(url=url.substr(index+this.wb_replay_prefix.length)),url.length>4&&url.charAt(2)==="_"&&url.charAt(3)==="/"&&(url=url.substr(4)),url!==rwURLString&&!this.startsWithOneOf(url,this.VALID_PREFIXES)&&!this.startsWith(url,"blob:")&&(url=this.wb_orig_scheme+url)),rwURLString.charAt(0)==="/"&&rwURLString.charAt(1)!=="/"&&this.startsWith(url,this.wb_orig_origin)&&(url=url.substr(this.wb_orig_origin.length)),this.startsWith(url,this.REL_PREFIX)?this.wb_info.wombat_scheme+":"+url:url},Wombat.prototype.makeParser=function(maybeRewrittenURL,doc){var originalURL=this.extractOriginalURL(maybeRewrittenURL),docElem=doc;return doc||(this.$wbwindow.location.href==="about:blank"&&this.$wbwindow.opener?docElem=this.$wbwindow.opener.document:docElem=this.$wbwindow.document),this._makeURLParser(originalURL,docElem)},Wombat.prototype._makeURLParser=function(url,docElem){try{return new this.$wbwindow.URL(url,docElem.baseURI)}catch(e){}var p=docElem.createElement("a");return p._no_rewrite=true,p.href=url,p},Wombat.prototype.defProp=function(obj,prop,setFunc,getFunc,enumerable){var existingDescriptor=Object.getOwnPropertyDescriptor(obj,prop);if(existingDescriptor&&!existingDescriptor.configurable)return false;if(!getFunc)return false;var descriptor={configurable:true,enumerable:enumerable||false,get:getFunc};setFunc&&(descriptor.set=setFunc);try{return Object.defineProperty(obj,prop,descriptor),true}catch(e){return console.warn("Failed to redefine property %s",prop,e.message),false}},Wombat.prototype.defGetterProp=function(obj,prop,getFunc,enumerable){var existingDescriptor=Object.getOwnPropertyDescriptor(obj,prop);if(existingDescriptor&&!existingDescriptor.configurable)return false;if(!getFunc)return false;try{return Object.defineProperty(obj,prop,{configurable:true,enumerable:enumerable||false,get:getFunc}),true}catch(e){return console.warn("Failed to redefine property %s",prop,e.message),false}},Wombat.prototype.getOrigGetter=function(obj,prop){var orig_getter;if(obj.__lookupGetter__&&(orig_getter=obj.__lookupGetter__(prop)),!orig_getter&&Object.getOwnPropertyDescriptor){var props=Object.getOwnPropertyDescriptor(obj,prop);props&&(orig_getter=props.get)}return orig_getter},Wombat.prototype.getOrigSetter=function(obj,prop){var orig_setter;if(obj.__lookupSetter__&&(orig_setter=obj.__lookupSetter__(prop)),!orig_setter&&Object.getOwnPropertyDescriptor){var props=Object.getOwnPropertyDescriptor(obj,prop);props&&(orig_setter=props.set)}return orig_setter},Wombat.prototype.getAllOwnProps=function(obj){for(var ownProps=[],props=Object.getOwnPropertyNames(obj),i=0;i "+final_href),actualLocation.href=final_href}}},Wombat.prototype.checkLocationChange=function(wombatLoc,isTop){var locType=typeof wombatLoc,actual_location=isTop?this.$wbwindow.__WB_replay_top.location:this.$wbwindow.location;locType==="string"?this.updateLocation(wombatLoc,actual_location.href,actual_location):locType==="object"&&this.updateLocation(wombatLoc.href,wombatLoc._orig_href,actual_location)},Wombat.prototype.checkAllLocations=function(){return!this.wb_wombat_updating&&void(this.wb_wombat_updating=true,this.checkLocationChange(this.$wbwindow.WB_wombat_location,false),this.$wbwindow.WB_wombat_location!=this.$wbwindow.__WB_replay_top.WB_wombat_location&&this.checkLocationChange(this.$wbwindow.__WB_replay_top.WB_wombat_location,true),this.wb_wombat_updating=false)},Wombat.prototype.proxyToObj=function(source){if(source)try{var proxyRealObj=source.__WBProxyRealObj__;if(proxyRealObj)return proxyRealObj}catch(e){}return source},Wombat.prototype.objToProxy=function(obj){if(obj)try{var maybeWbProxy=obj._WB_wombat_obj_proxy;if(maybeWbProxy)return maybeWbProxy}catch(e){}return obj},Wombat.prototype.defaultProxyGet=function(obj,prop,ownProps,fnCache){switch(prop){case"__WBProxyRealObj__":return obj;case"location":case"WB_wombat_location":return obj.WB_wombat_location;case"_WB_wombat_obj_proxy":return obj._WB_wombat_obj_proxy;case"__WB_pmw":case"WB_wombat_eval":case this.WB_ASSIGN_FUNC:case this.WB_CHECK_THIS_FUNC:return obj[prop];case"constructor":if(obj.constructor===Window)return obj.constructor;}var retVal=obj[prop],type=typeof retVal;if(type==="function"&&ownProps.indexOf(prop)!==-1){switch(prop){case"requestAnimationFrame":case"cancelAnimationFrame":{if(!this.isNativeFunction(retVal))return retVal;break}}var cachedFN=fnCache[prop];return cachedFN&&cachedFN.original===retVal||(cachedFN={original:retVal,boundFn:retVal.bind(obj)},fnCache[prop]=cachedFN),cachedFN.boundFn}return type==="object"&&retVal&&retVal._WB_wombat_obj_proxy?(retVal instanceof Window&&this.initNewWindowWombat(retVal),retVal._WB_wombat_obj_proxy):retVal},Wombat.prototype.setLoc=function(loc,originalURL){var parser=this.makeParser(originalURL,loc.ownerDocument);loc._orig_href=originalURL,loc._parser=parser;var href=parser.href;loc._hash=parser.hash,loc._href=href,loc._host=parser.host,loc._hostname=parser.hostname,loc._origin=parser.origin?parser.host?parser.origin:"null":parser.protocol+"//"+parser.hostname+(parser.port?":"+parser.port:""),loc._pathname=parser.pathname,loc._port=parser.port,loc._protocol=parser.protocol,loc._search=parser.search,Object.defineProperty||(loc.href=href,loc.hash=parser.hash,loc.host=loc._host,loc.hostname=loc._hostname,loc.origin=loc._origin,loc.pathname=loc._pathname,loc.port=loc._port,loc.protocol=loc._protocol,loc.search=loc._search)},Wombat.prototype.makeGetLocProp=function(prop,origGetter){var wombat=this;return function newGetLocProp(){if(this._no_rewrite)return origGetter.call(this,prop);var curr_orig_href=origGetter.call(this,"href");return prop==="href"?wombat.extractOriginalURL(curr_orig_href):(this._orig_href!==curr_orig_href&&wombat.setLoc(this,curr_orig_href),this["_"+prop])}},Wombat.prototype.makeSetLocProp=function(prop,origSetter,origGetter){var wombat=this;return function newSetLocProp(value){if(this._no_rewrite)return origSetter.call(this,prop,value);if(this["_"+prop]!==value){if(this["_"+prop]=value,!this._parser){var href=origGetter.call(this);this._parser=wombat.makeParser(href,this.ownerDocument)}var rel=false;prop==="href"&&typeof value==="string"&&value&&(value[0]==="."?value=wombat.resolveRelUrl(value,this.ownerDocument):value[0]==="/"&&(value.length<=1||value[1]!=="/")&&(rel=true,value=WB_wombat_location.origin+value));try{this._parser[prop]=value}catch(e){console.log("Error setting "+prop+" = "+value)}prop==="hash"?(value=this._parser[prop],origSetter.call(this,"hash",value)):(rel=rel||value===this._parser.pathname,value=wombat.rewriteUrl(this._parser.href,rel),origSetter.call(this,"href",value))}}},Wombat.prototype.styleReplacer=function(match,n1,n2,n3,offset,string){return n1+this.rewriteUrl(n2)+n3},Wombat.prototype.domConstructorErrorChecker=function(thisObj,what,args,numRequiredArgs){var erorMsg,needArgs=typeof numRequiredArgs==="number"?numRequiredArgs:1;if(thisObj instanceof Window?erorMsg="Failed to construct '"+what+"': Please use the 'new' operator, this DOM object constructor cannot be called as a function.":args&&args.length=0)return url;if(url.indexOf(this.wb_rel_prefix)===0&&url.indexOf("http")>1){var scheme_sep=url.indexOf(":/");return scheme_sep>0&&url[scheme_sep+2]!=="/"?url.substring(0,scheme_sep+2)+"/"+url.substring(scheme_sep+2):url}return this.getFinalUrl(true,mod,this.wb_orig_origin+url)}url.charAt(0)==="."&&(url=this.resolveRelUrl(url,doc));var prefix=this.startsWithOneOf(url.toLowerCase(),this.VALID_PREFIXES);if(prefix){var orig_host=this.$wbwindow.__WB_replay_top.location.host,orig_protocol=this.$wbwindow.__WB_replay_top.location.protocol,prefix_host=prefix+orig_host+"/";if(this.startsWith(url,prefix_host)){if(this.startsWith(url,this.wb_replay_prefix))return url;var curr_scheme=orig_protocol+"//",path=url.substring(prefix_host.length),rebuild=false;return path.indexOf(this.wb_rel_prefix)<0&&url.indexOf("/static/")<0&&(path=this.getFinalUrl(true,mod,WB_wombat_location.origin+"/"+path),rebuild=true),prefix!==curr_scheme&&prefix!==this.REL_PREFIX&&(rebuild=true),rebuild&&(url=useRel?"":curr_scheme+orig_host,path&&path[0]!=="/"&&(url+="/"),url+=path),url}return this.getFinalUrl(useRel,mod,url)}return prefix=this.startsWithOneOf(url,this.BAD_PREFIXES),prefix?this.getFinalUrl(useRel,mod,this.extractOriginalURL(url)):this.isHostUrl(url)&&!this.startsWith(url,originalLoc.host+"/")?this.getFinalUrl(useRel,mod,this.wb_orig_scheme+url):url},Wombat.prototype.rewriteUrl=function(url,useRel,mod,doc){var rewritten=this.rewriteUrl_(url,useRel,mod,doc);return this.debug_rw&&(url===rewritten?console.log("NOT REWRITTEN "+url):console.log("REWRITE: "+url+" -> "+rewritten)),rewritten},Wombat.prototype.performAttributeRewrite=function(elem,name,value,absUrlOnly){switch(name){case"innerHTML":case"outerHTML":return this.rewriteHtml(value);case"filter":return this.rewriteInlineStyle(value);case"style":return this.rewriteStyle(value);case"srcset":return this.rewriteSrcset(value,elem);}if(absUrlOnly&&!this.startsWithOneOf(value,this.VALID_PREFIXES))return value;var mod=this.rwModForElement(elem,name);return this.wbUseAFWorker&&this.WBAutoFetchWorker&&this.isSavedDataSrcSrcset(elem)&&this.WBAutoFetchWorker.preserveDataSrcset(elem),this.rewriteUrl(value,false,mod,elem.ownerDocument)},Wombat.prototype.rewriteAttr=function(elem,name,absUrlOnly){var changed=false;if(!elem||!elem.getAttribute||elem._no_rewrite||elem["_"+name])return changed;var value=this.wb_getAttribute.call(elem,name);if(!value||this.startsWith(value,"javascript:"))return changed;var new_value=this.performAttributeRewrite(elem,name,value,absUrlOnly);return new_value!==value&&(this.removeWBOSRC(elem),this.wb_setAttribute.call(elem,name,new_value),changed=true),changed},Wombat.prototype.noExceptRewriteStyle=function(style){try{return this.rewriteStyle(style)}catch(e){return style}},Wombat.prototype.rewriteStyle=function(style){if(!style)return style;var value=style;return typeof style==="object"&&(value=style.toString()),typeof value==="string"?value.replace(this.STYLE_REGEX,this.styleReplacer).replace(this.IMPORT_REGEX,this.styleReplacer).replace(this.no_wombatRe,""):value},Wombat.prototype.rewriteSrcset=function(value,elem){if(!value)return"";for(var split=value.split(this.srcsetRe),values=[],mod=this.rwModForElement(elem,"srcset"),i=0;i=0){var JS="javascript:";new_value="javascript:window.parent._wb_wombat.initNewWindowWombat(window);"+value.substr(11)}return new_value||(new_value=this.rewriteUrl(value,false,this.rwModForElement(elem,attrName))),new_value!==value&&(this.wb_setAttribute.call(elem,attrName,new_value),true)},Wombat.prototype.rewriteScript=function(elem){if(elem.hasAttribute("src")||!elem.textContent||!this.$wbwindow.Proxy)return this.rewriteAttr(elem,"src");if(this.skipWrapScriptBasedOnType(elem.type))return false;var text=elem.textContent.trim();return!this.skipWrapScriptTextBasedOnText(text)&&(elem.textContent=this.wrapScriptTextJsProxy(text),true)},Wombat.prototype.rewriteSVGElem=function(elem){var changed=this.rewriteAttr(elem,"filter");return changed=this.rewriteAttr(elem,"style")||changed,changed=this.rewriteAttr(elem,"xlink:href")||changed,changed=this.rewriteAttr(elem,"href")||changed,changed=this.rewriteAttr(elem,"src")||changed,changed},Wombat.prototype.rewriteElem=function(elem){var changed=false;if(!elem)return changed;if(elem instanceof SVGElement)changed=this.rewriteSVGElem(elem);else switch(elem.tagName){case"META":var maybeCSP=this.wb_getAttribute.call(elem,"http-equiv");maybeCSP&&maybeCSP.toLowerCase()==="content-security-policy"&&(this.wb_setAttribute.call(elem,"http-equiv","_"+maybeCSP),changed=true);break;case"STYLE":var new_content=this.rewriteStyle(elem.textContent);elem.textContent!==new_content&&(elem.textContent=new_content,changed=true,this.wbUseAFWorker&&this.WBAutoFetchWorker&&elem.sheet!=null&&this.WBAutoFetchWorker.deferredSheetExtraction(elem.sheet));break;case"LINK":changed=this.rewriteAttr(elem,"href"),this.wbUseAFWorker&&elem.rel==="stylesheet"&&this._addEventListener(elem,"load",this.utilFns.wbSheetMediaQChecker);break;case"IMG":changed=this.rewriteAttr(elem,"src"),changed=this.rewriteAttr(elem,"srcset")||changed,changed=this.rewriteAttr(elem,"style")||changed,this.wbUseAFWorker&&this.WBAutoFetchWorker&&elem.dataset.srcset&&this.WBAutoFetchWorker.preserveDataSrcset(elem);break;case"OBJECT":changed=this.rewriteAttr(elem,"data",true),changed=this.rewriteAttr(elem,"style")||changed;break;case"FORM":changed=this.rewriteAttr(elem,"poster"),changed=this.rewriteAttr(elem,"action")||changed,changed=this.rewriteAttr(elem,"style")||changed;break;case"IFRAME":case"FRAME":changed=this.rewriteFrameSrc(elem,"src"),changed=this.rewriteAttr(elem,"style")||changed;break;case"SCRIPT":changed=this.rewriteScript(elem);break;default:{changed=this.rewriteAttr(elem,"src"),changed=this.rewriteAttr(elem,"srcset")||changed,changed=this.rewriteAttr(elem,"href")||changed,changed=this.rewriteAttr(elem,"style")||changed,changed=this.rewriteAttr(elem,"poster")||changed;break}}return elem.hasAttribute&&elem.removeAttribute&&(elem.hasAttribute("crossorigin")&&(elem.removeAttribute("crossorigin"),changed=true),elem.hasAttribute("integrity")&&(elem.removeAttribute("integrity"),changed=true)),changed},Wombat.prototype.recurseRewriteElem=function(curr){if(!this.nodeHasChildren(curr))return false;for(var changed=false,rewriteQ=[curr.children||curr.childNodes];rewriteQ.length>0;)for(var child,children=rewriteQ.shift(),i=0;i"+rwString+"","text/html");if(!inner_doc||!this.nodeHasChildren(inner_doc.head)||!inner_doc.head.children[0].content)return rwString;var template=inner_doc.head.children[0];if(template._no_rewrite=true,this.recurseRewriteElem(template.content)){var new_html=template.innerHTML;if(checkEndTag){var first_elem=template.content.children&&template.content.children[0];if(first_elem){var end_tag="";this.endsWith(new_html,end_tag)&&!this.endsWith(rwString,end_tag)&&(new_html=new_html.substring(0,new_html.length-end_tag.length))}else if(rwString[0]!=="<"||rwString[rwString.length-1]!==">")return this.write_buff+=rwString,undefined}return new_html}return rwString},Wombat.prototype.rewriteHtmlFull=function(string,checkEndTag){var inner_doc=new DOMParser().parseFromString(string,"text/html");if(!inner_doc)return string;for(var changed=false,i=0;i=0)inner_doc.documentElement._no_rewrite=true,new_html=this.reconstructDocType(inner_doc.doctype)+inner_doc.documentElement.outerHTML;else{inner_doc.head._no_rewrite=true,inner_doc.body._no_rewrite=true;var headHasKids=this.nodeHasChildren(inner_doc.head),bodyHasKids=this.nodeHasChildren(inner_doc.body);if(new_html=(headHasKids?inner_doc.head.outerHTML:"")+(bodyHasKids?inner_doc.body.outerHTML:""),checkEndTag)if(inner_doc.all.length>3){var end_tag="";this.endsWith(new_html,end_tag)&&!this.endsWith(string,end_tag)&&(new_html=new_html.substring(0,new_html.length-end_tag.length))}else if(string[0]!=="<"||string[string.length-1]!==">")return void(this.write_buff+=string);new_html=this.reconstructDocType(inner_doc.doctype)+new_html}return new_html}return string},Wombat.prototype.rewriteInlineStyle=function(orig){var decoded;try{decoded=decodeURIComponent(orig)}catch(e){decoded=orig}if(decoded!==orig){var parts=this.rewriteStyle(decoded).split(",",2);return parts[0]+","+encodeURIComponent(parts[1])}return this.rewriteStyle(orig)},Wombat.prototype.rewriteCookie=function(cookie){var wombat=this,rwCookie=cookie.replace(this.wb_abs_prefix,"").replace(this.wb_rel_prefix,"");return rwCookie=rwCookie.replace(this.cookie_domain_regex,function(m,m1){var message={domain:m1,cookie:rwCookie,wb_type:"cookie"};return wombat.sendTopMessage(message,true),wombat.$wbwindow.location.hostname.indexOf(".")>=0&&!wombat.IP_RX.test(wombat.$wbwindow.location.hostname)?"Domain=."+wombat.$wbwindow.location.hostname:""}).replace(this.cookie_path_regex,function(m,m1){var rewritten=wombat.rewriteUrl(m1);return rewritten.indexOf(wombat.wb_curr_host)===0&&(rewritten=rewritten.substring(wombat.wb_curr_host.length)),"Path="+rewritten}),wombat.$wbwindow.location.protocol!=="https:"&&(rwCookie=rwCookie.replace("secure","")),rwCookie.replace(",|",",")},Wombat.prototype.rewriteWorker=function(workerUrl){if(!workerUrl)return workerUrl;var isBlob=workerUrl.indexOf("blob:")===0,isJS=workerUrl.indexOf("javascript:")===0;if(!isBlob&&!isJS){if(!this.startsWithOneOf(workerUrl,this.VALID_PREFIXES)&&!this.startsWith(workerUrl,"/")&&!this.startsWithOneOf(workerUrl,this.BAD_PREFIXES)){var rurl=this.resolveRelUrl(workerUrl,this.$wbwindow.document);return this.rewriteUrl(rurl,false,"wkr_",this.$wbwindow.document)}return this.rewriteUrl(workerUrl,false,"wkr_",this.$wbwindow.document)}var workerCode=isJS?workerUrl.replace("javascript:",""):null;if(isBlob){var x=new XMLHttpRequest;this.utilFns.XHRopen.call(x,"GET",workerUrl,false),x.send(),workerCode=x.responseText.replace(this.workerBlobRe,"").replace(this.rmCheckThisInjectRe,"this")}if(this.wb_info.static_prefix||this.wb_info.ww_rw_script){var originalURL=this.$wbwindow.document.baseURI,ww_rw=this.wb_info.ww_rw_script||this.wb_info.static_prefix+"wombatWorkers.js",rw="(function() { self.importScripts('"+ww_rw+"'); new WBWombat({'prefix': '"+this.wb_abs_prefix+"', 'prefixMod': '"+this.wb_abs_prefix+"wkrf_/', 'originalURL': '"+originalURL+"'}); })();";workerCode=rw+workerCode}var blob=new Blob([workerCode],{type:"application/javascript"});return URL.createObjectURL(blob)},Wombat.prototype.rewriteTextNodeFn=function(fnThis,originalFn,argsObj){var args,deproxiedThis=this.proxyToObj(fnThis);if(argsObj.length>0&&deproxiedThis.parentElement&&deproxiedThis.parentElement.tagName==="STYLE"){args=new Array(argsObj.length);var dataIndex=argsObj.length-1;dataIndex===2?(args[0]=argsObj[0],args[1]=argsObj[1]):dataIndex===1&&(args[0]=argsObj[0]),args[dataIndex]=this.rewriteStyle(argsObj[dataIndex])}else args=argsObj;return originalFn.__WB_orig_apply?originalFn.__WB_orig_apply(deproxiedThis,args):originalFn.apply(deproxiedThis,args)},Wombat.prototype.rewriteDocWriteWriteln=function(fnThis,originalFn,argsObj){var string,thisObj=this.proxyToObj(fnThis),argLen=argsObj.length;if(argLen===0)return originalFn.call(thisObj);string=argLen===1?argsObj[0]:Array.prototype.join.call(argsObj,"");var new_buff=this.rewriteHtml(string,true),res=originalFn.call(thisObj,new_buff);return this.initNewWindowWombat(thisObj.defaultView),res},Wombat.prototype.rewriteChildNodeFn=function(fnThis,originalFn,argsObj){var thisObj=this.proxyToObj(fnThis);if(argsObj.length===0)return originalFn.call(thisObj);var newArgs=this.rewriteElementsInArguments(argsObj);return originalFn.__WB_orig_apply?originalFn.__WB_orig_apply(thisObj,newArgs):originalFn.apply(thisObj,newArgs)},Wombat.prototype.rewriteInsertAdjHTMLOrElemArgs=function(fnThis,originalFn,position,textOrElem,rwHTML){var fnThisObj=this.proxyToObj(fnThis);return fnThisObj._no_rewrite?originalFn.call(fnThisObj,position,textOrElem):rwHTML?originalFn.call(fnThisObj,position,this.rewriteHtml(textOrElem)):(this.rewriteElemComplete(textOrElem),originalFn.call(fnThisObj,position,textOrElem))},Wombat.prototype.rewriteSetTimeoutInterval=function(fnThis,originalFn,argsObj){var rw=this.isString(argsObj[0]),args=rw?new Array(argsObj.length):argsObj;if(rw){args[0]=this.$wbwindow.Proxy?this.wrapScriptTextJsProxy(argsObj[0]):argsObj[0].replace(/\blocation\b/g,"WB_wombat_$&");for(var i=1;i0&&cssStyleValueOverride(this.$wbwindow.CSSStyleValue,"parse"),this.$wbwindow.CSSStyleValue.parseAll&&this.$wbwindow.CSSStyleValue.parseAll.toString().indexOf("[native code]")>0&&cssStyleValueOverride(this.$wbwindow.CSSStyleValue,"parseAll")}if(this.$wbwindow.CSSKeywordValue&&this.$wbwindow.CSSKeywordValue.prototype){var oCSSKV=this.$wbwindow.CSSKeywordValue;this.$wbwindow.CSSKeywordValue=function(CSSKeywordValue_){return function CSSKeywordValue(cssValue){return wombat.domConstructorErrorChecker(this,"CSSKeywordValue",arguments),new CSSKeywordValue_(wombat.rewriteStyle(cssValue))}}(this.$wbwindow.CSSKeywordValue),this.$wbwindow.CSSKeywordValue.prototype=oCSSKV.prototype,Object.defineProperty(this.$wbwindow.CSSKeywordValue.prototype,"constructor",{value:this.$wbwindow.CSSKeywordValue}),addToStringTagToClass(this.$wbwindow.CSSKeywordValue,"CSSKeywordValue")}if(this.$wbwindow.StylePropertyMap&&this.$wbwindow.StylePropertyMap.prototype){var originalSet=this.$wbwindow.StylePropertyMap.prototype.set;this.$wbwindow.StylePropertyMap.prototype.set=function set(){if(arguments.length<=1)return originalSet.__WB_orig_apply?originalSet.__WB_orig_apply(this,arguments):originalSet.apply(this,arguments);var newArgs=new Array(arguments.length);newArgs[0]=arguments[0];for(var i=1;i=0||name==="href"?wombat.extractOriginalURL(value):value},svgImgProto.getAttributeNS=function getAttributeNS(ns,name){var value=orig_getAttrNS.call(this,ns,name);return name.indexOf("xlink:href")>=0||name==="href"?wombat.extractOriginalURL(value):value},svgImgProto.setAttribute=function setAttribute(name,value){var rwValue=value;return(name.indexOf("xlink:href")>=0||name==="href")&&(rwValue=wombat.rewriteUrl(value)),orig_setAttr.call(this,name,rwValue)},svgImgProto.setAttributeNS=function setAttributeNS(ns,name,value){var rwValue=value;return(name.indexOf("xlink:href")>=0||name==="href")&&(rwValue=wombat.rewriteUrl(value)),orig_setAttrNS.call(this,ns,name,rwValue)}}},Wombat.prototype.initCreateElementNSFix=function(){if(this.$wbwindow.document.createElementNS&&this.$wbwindow.Document.prototype.createElementNS){var orig_createElementNS=this.$wbwindow.document.createElementNS,wombat=this,createElementNS=function createElementNS(namespaceURI,qualifiedName){return orig_createElementNS.call(wombat.proxyToObj(this),wombat.extractOriginalURL(namespaceURI),qualifiedName)};this.$wbwindow.Document.prototype.createElementNS=createElementNS,this.$wbwindow.document.createElementNS=createElementNS}},Wombat.prototype.initInsertAdjacentElementHTMLOverrides=function(){var Element=this.$wbwindow.Element;if(Element&&Element.prototype){var elementProto=Element.prototype,rewriteFn=this.rewriteInsertAdjHTMLOrElemArgs;if(elementProto.insertAdjacentHTML){var origInsertAdjacentHTML=elementProto.insertAdjacentHTML;elementProto.insertAdjacentHTML=function insertAdjacentHTML(position,text){return rewriteFn(this,origInsertAdjacentHTML,position,text,true)}}if(elementProto.insertAdjacentElement){var origIAdjElem=elementProto.insertAdjacentElement;elementProto.insertAdjacentElement=function insertAdjacentElement(position,element){return rewriteFn(this,origIAdjElem,position,element,false)}}}},Wombat.prototype.initDomOverride=function(){var Node=this.$wbwindow.Node;if(Node&&Node.prototype){var rewriteFn=this.rewriteNodeFuncArgs;if(Node.prototype.appendChild){var originalAppendChild=Node.prototype.appendChild;Node.prototype.appendChild=function appendChild(newNode,oldNode){return rewriteFn(this,originalAppendChild,newNode,oldNode)}}if(Node.prototype.insertBefore){var originalInsertBefore=Node.prototype.insertBefore;Node.prototype.insertBefore=function insertBefore(newNode,oldNode){return rewriteFn(this,originalInsertBefore,newNode,oldNode)}}if(Node.prototype.replaceChild){var originalReplaceChild=Node.prototype.replaceChild;Node.prototype.replaceChild=function replaceChild(newNode,oldNode){return rewriteFn(this,originalReplaceChild,newNode,oldNode)}}this.overridePropToProxy(Node.prototype,"ownerDocument"),this.overridePropToProxy(this.$wbwindow.HTMLHtmlElement.prototype,"parentNode"),this.overridePropToProxy(this.$wbwindow.Event.prototype,"target")}this.$wbwindow.Element&&this.$wbwindow.Element.prototype&&(this.overrideParentNodeAppendPrepend(this.$wbwindow.Element),this.overrideChildNodeInterface(this.$wbwindow.Element,false)),this.$wbwindow.DocumentFragment&&this.$wbwindow.DocumentFragment.prototype&&this.overrideParentNodeAppendPrepend(this.$wbwindow.DocumentFragment)},Wombat.prototype.initDocOverrides=function($document){if(Object.defineProperty){this.overrideReferrer($document),this.defGetterProp($document,"origin",function origin(){return this.WB_wombat_location.origin}),this.defGetterProp(this.$wbwindow,"origin",function origin(){return this.WB_wombat_location.origin});var wombat=this,domain_setter=function domain(val){var loc=this.WB_wombat_location;loc&&wombat.endsWith(loc.hostname,val)&&(this.__wb_domain=val)},domain_getter=function domain(){return this.__wb_domain||this.WB_wombat_location.hostname};this.defProp($document,"domain",domain_setter,domain_getter)}},Wombat.prototype.initDocWriteOpenCloseOverride=function(){if(this.$wbwindow.DOMParser){var DocumentProto=this.$wbwindow.Document.prototype,$wbDocument=this.$wbwindow.document,docWriteWritelnRWFn=this.rewriteDocWriteWriteln,orig_doc_write=$wbDocument.write,new_write=function write(){return docWriteWritelnRWFn(this,orig_doc_write,arguments)};$wbDocument.write=new_write,DocumentProto.write=new_write;var orig_doc_writeln=$wbDocument.writeln,new_writeln=function writeln(){return docWriteWritelnRWFn(this,orig_doc_writeln,arguments)};$wbDocument.writeln=new_writeln,DocumentProto.writeln=new_writeln;var wombat=this,orig_doc_open=$wbDocument.open,new_open=function open(){var res,thisObj=wombat.proxyToObj(this);if(arguments.length===3){var rwUrl=wombat.rewriteUrl(arguments[0],false,"mp_");res=orig_doc_open.call(thisObj,rwUrl,arguments[1],arguments[2]),wombat.initNewWindowWombat(res,rwUrl)}else res=orig_doc_open.call(thisObj),wombat.initNewWindowWombat(thisObj.defaultView);return res};$wbDocument.open=new_open,DocumentProto.open=new_open;var originalClose=$wbDocument.close,newClose=function close(){var thisObj=wombat.proxyToObj(this);return wombat.initNewWindowWombat(thisObj.defaultView),originalClose.__WB_orig_apply?originalClose.__WB_orig_apply(thisObj,arguments):originalClose.apply(thisObj,arguments)};$wbDocument.close=newClose,DocumentProto.close=newClose;var oBodyGetter=this.getOrigGetter(DocumentProto,"body"),oBodySetter=this.getOrigSetter(DocumentProto,"body");oBodyGetter&&oBodySetter&&this.defProp(DocumentProto,"body",function body(newBody){return newBody&&(newBody instanceof HTMLBodyElement||newBody instanceof HTMLFrameSetElement)&&wombat.rewriteElemComplete(newBody),oBodySetter.call(wombat.proxyToObj(this),newBody)},oBodyGetter)}},Wombat.prototype.initIframeWombat=function(iframe){var win;win=iframe._get_contentWindow?iframe._get_contentWindow.call(iframe):iframe.contentWindow;try{if(!win||win===this.$wbwindow||win._skip_wombat||win._wb_wombat)return}catch(e){return}var src=this.wb_getAttribute.call(iframe,"src");this.initNewWindowWombat(win,src)},Wombat.prototype.initNewWindowWombat=function(win,src){if(win&&!win._wb_wombat)if(!src||src===""||this.startsWith(src,"about:")||src.indexOf("javascript:")>=0){var wombat=new Wombat(win,this.wb_info);win._wb_wombat=wombat.wombatInit()}else this.initProtoPmOrigin(win),this.initPostMessageOverride(win),this.initMessageEventOverride(win),this.initCheckThisFunc(win)},Wombat.prototype.initTimeoutIntervalOverrides=function(){var rewriteFn=this.rewriteSetTimeoutInterval;if(this.$wbwindow.setTimeout&&!this.$wbwindow.setTimeout.__$wbpatched$__){var originalSetTimeout=this.$wbwindow.setTimeout;this.$wbwindow.setTimeout=function setTimeout(){return rewriteFn(this,originalSetTimeout,arguments)},this.$wbwindow.setTimeout.__$wbpatched$__=true}if(this.$wbwindow.setInterval&&!this.$wbwindow.setInterval.__$wbpatched$__){var originalSetInterval=this.$wbwindow.setInterval;this.$wbwindow.setInterval=function setInterval(){return rewriteFn(this,originalSetInterval,arguments)},this.$wbwindow.setInterval.__$wbpatched$__=true}},Wombat.prototype.initWorkerOverrides=function(){var wombat=this;if(this.$wbwindow.Worker&&!this.$wbwindow.Worker._wb_worker_overriden){var orig_worker=this.$wbwindow.Worker;this.$wbwindow.Worker=function(Worker_){return function Worker(url,options){return wombat.domConstructorErrorChecker(this,"Worker",arguments),new Worker_(wombat.rewriteWorker(url),options)}}(orig_worker),this.$wbwindow.Worker.prototype=orig_worker.prototype,Object.defineProperty(this.$wbwindow.Worker.prototype,"constructor",{value:this.$wbwindow.Worker}),this.$wbwindow.Worker._wb_worker_overriden=true}if(this.$wbwindow.SharedWorker&&!this.$wbwindow.SharedWorker.__wb_sharedWorker_overriden){var oSharedWorker=this.$wbwindow.SharedWorker;this.$wbwindow.SharedWorker=function(SharedWorker_){return function SharedWorker(url,options){return wombat.domConstructorErrorChecker(this,"SharedWorker",arguments),new SharedWorker_(wombat.rewriteWorker(url),options)}}(oSharedWorker),this.$wbwindow.SharedWorker.prototype=oSharedWorker.prototype,Object.defineProperty(this.$wbwindow.SharedWorker.prototype,"constructor",{value:this.$wbwindow.SharedWorker}),this.$wbwindow.SharedWorker.__wb_sharedWorker_overriden=true}if(this.$wbwindow.ServiceWorkerContainer&&this.$wbwindow.ServiceWorkerContainer.prototype&&this.$wbwindow.ServiceWorkerContainer.prototype.register){var orig_register=this.$wbwindow.ServiceWorkerContainer.prototype.register;this.$wbwindow.ServiceWorkerContainer.prototype.register=function register(scriptURL,options){var newScriptURL=new URL(scriptURL,wombat.$wbwindow.document.baseURI).href,mod=wombat.getPageUnderModifier();return options&&options.scope?options.scope=wombat.rewriteUrl(options.scope,false,mod):options={scope:wombat.rewriteUrl("/",false,mod)},orig_register.call(this,wombat.rewriteUrl(newScriptURL,false,"sw_"),options)}}if(this.$wbwindow.Worklet&&this.$wbwindow.Worklet.prototype&&this.$wbwindow.Worklet.prototype.addModule&&!this.$wbwindow.Worklet.__wb_workerlet_overriden){var oAddModule=this.$wbwindow.Worklet.prototype.addModule;this.$wbwindow.Worklet.prototype.addModule=function addModule(moduleURL,options){var rwModuleURL=wombat.rewriteUrl(moduleURL,false,"js_");return oAddModule.call(this,rwModuleURL,options)},this.$wbwindow.Worklet.__wb_workerlet_overriden=true}},Wombat.prototype.initLocOverride=function(loc,oSetter,oGetter){if(Object.defineProperty)for(var prop,i=0;i=0){var fnMapping=this._map.splice(idx,1);return fnMapping[0][1]}return null},FuncMap.prototype.map=function(param){for(var i=0;i0&&afw.preserveMedia(media)})},AutoFetcher.prototype.terminate=function(){this.worker.terminate()},AutoFetcher.prototype.justFetch=function(urls){this.worker.postMessage({type:"fetch-all",values:urls})},AutoFetcher.prototype.fetchAsPage=function(url,originalUrl,title){if(url){var headers={"X-Wombat-History-Page":originalUrl};if(title){var encodedTitle=encodeURIComponent(title.trim());title&&(headers["X-Wombat-History-Title"]=encodedTitle)}var fetchData={url:url,options:{headers:headers,cache:"no-store"}};this.justFetch([fetchData])}},AutoFetcher.prototype.postMessage=function(msg,deferred){if(deferred){var afWorker=this;return void Promise.resolve().then(function(){afWorker.worker.postMessage(msg)})}this.worker.postMessage(msg)},AutoFetcher.prototype.preserveSrcset=function(srcset,mod){this.postMessage({type:"values",srcset:{value:srcset,mod:mod,presplit:true}},true)},AutoFetcher.prototype.preserveDataSrcset=function(elem){this.postMessage({type:"values",srcset:{value:elem.dataset.srcset,mod:this.rwMod(elem),presplit:false}},true)},AutoFetcher.prototype.preserveMedia=function(media){this.postMessage({type:"values",media:media},true)},AutoFetcher.prototype.getSrcset=function(elem){return this.wombat.wb_getAttribute?this.wombat.wb_getAttribute.call(elem,"srcset"):elem.getAttribute("srcset")},AutoFetcher.prototype.rwMod=function(elem){switch(elem.tagName){case"SOURCE":return elem.parentElement&&elem.parentElement.tagName==="PICTURE"?"im_":"oe_";case"IMG":return"im_";}return"oe_"},AutoFetcher.prototype.extractFromLocalDoc=function(){var afw=this;Promise.resolve().then(function(){for(var msg={type:"values",context:{docBaseURI:document.baseURI}},media=[],i=0,sheets=document.styleSheets;i=0},Wombat.prototype.isString=function(arg){return arg!=null&&Object.getPrototypeOf(arg)===String.prototype},Wombat.prototype.isSavedSrcSrcset=function(elem){switch(elem.tagName){case"IMG":case"VIDEO":case"AUDIO":return true;case"SOURCE":if(!elem.parentElement)return false;switch(elem.parentElement.tagName){case"PICTURE":case"VIDEO":case"AUDIO":return true;default:return false;}default:return false;}},Wombat.prototype.isSavedDataSrcSrcset=function(elem){return!!(elem.dataset&&elem.dataset.srcset!=null)&&this.isSavedSrcSrcset(elem)},Wombat.prototype.isHostUrl=function(str){if(str.indexOf("www.")===0)return true;var matches=str.match(this.hostnamePortRe);return!!(matches&&matches[0].length<64)||(matches=str.match(this.ipPortRe),!!matches&&matches[0].length<64)},Wombat.prototype.isArgumentsObj=function(maybeArgumentsObj){if(!maybeArgumentsObj||typeof maybeArgumentsObj.toString!=="function")return false;try{return this.utilFns.objToString.call(maybeArgumentsObj)==="[object Arguments]"}catch(e){return false}},Wombat.prototype.deproxyArrayHandlingArgumentsObj=function(maybeArgumentsObj){if(!maybeArgumentsObj||maybeArgumentsObj instanceof NodeList||!maybeArgumentsObj.length)return maybeArgumentsObj;for(var args=this.isArgumentsObj(maybeArgumentsObj)?new Array(maybeArgumentsObj.length):maybeArgumentsObj,i=0;i=0)||scriptType.indexOf("text/template")>=0)},Wombat.prototype.skipWrapScriptTextBasedOnText=function(text){if(!text||text.indexOf(this.WB_ASSIGN_FUNC)>=0||text.indexOf("<")===0)return true;for(var override_props=["window","self","document","location","top","parent","frames","opener"],i=0;i=0)return false;return true},Wombat.prototype.nodeHasChildren=function(node){if(!node)return false;if(typeof node.hasChildNodes==="function")return node.hasChildNodes();var kids=node.children||node.childNodes;return!!kids&&kids.length>0},Wombat.prototype.rwModForElement=function(elem,attrName){if(!elem)return undefined;var mod="mp_";if(!(elem.tagName==="LINK"&&attrName==="href")){var maybeMod=this.tagToMod[elem.tagName];maybeMod!=null&&(mod=maybeMod[attrName])}else if(elem.rel){var relV=elem.rel.trim().toLowerCase(),asV=this.wb_getAttribute.call(elem,"as");if(asV&&this.linkTagMods.linkRelToAs[relV]!=null){var asMods=this.linkTagMods.linkRelToAs[relV];mod=asMods[asV.toLowerCase()]}else this.linkTagMods[relV]!=null&&(mod=this.linkTagMods[relV])}return mod},Wombat.prototype.removeWBOSRC=function(elem){elem.tagName!=="SCRIPT"||elem.__$removedWBOSRC$__||(elem.hasAttribute("__wb_orig_src")&&elem.removeAttribute("__wb_orig_src"),elem.__$removedWBOSRC$__=true)},Wombat.prototype.retrieveWBOSRC=function(elem){if(elem.tagName==="SCRIPT"&&!elem.__$removedWBOSRC$__){var maybeWBOSRC;return maybeWBOSRC=this.wb_getAttribute?this.wb_getAttribute.call(elem,"__wb_orig_src"):elem.getAttribute("__wb_orig_src"),maybeWBOSRC==null&&(elem.__$removedWBOSRC$__=true),maybeWBOSRC}return undefined},Wombat.prototype.wrapScriptTextJsProxy=function(scriptText){return"if (!self.__WB_pmw) { self.__WB_pmw = function(obj) { this.__WB_source = obj; return this; } }\n{\nlet window = _____WB$wombat$assign$function_____(\"window\");\nlet self = _____WB$wombat$assign$function_____(\"self\");\nlet document = _____WB$wombat$assign$function_____(\"document\");\nlet location = _____WB$wombat$assign$function_____(\"location\");\nlet top = _____WB$wombat$assign$function_____(\"top\");\nlet parent = _____WB$wombat$assign$function_____(\"parent\");\nlet frames = _____WB$wombat$assign$function_____(\"frames\");\nlet opener = _____WB$wombat$assign$function_____(\"opener\");\n"+scriptText.replace(this.DotPostMessageRe,".__WB_pmw(self.window)$1")+"\n\n}"},Wombat.prototype.watchElem=function(elem,func){if(!this.$wbwindow.MutationObserver)return false;var m=new this.$wbwindow.MutationObserver(function(records,observer){for(var r,i=0;i"},Wombat.prototype.getFinalUrl=function(useRel,mod,url){var prefix=useRel?this.wb_rel_prefix:this.wb_abs_prefix;return mod==null&&(mod=this.wb_info.mod),this.wb_info.is_live||(prefix+=this.wb_info.wombat_ts),prefix+=mod,prefix[prefix.length-1]!=="/"&&(prefix+="/"),prefix+url},Wombat.prototype.resolveRelUrl=function(url,doc){var docObj=doc||this.$wbwindow.document,parser=this.makeParser(docObj.baseURI,docObj),hash=parser.href.lastIndexOf("#"),href=hash>=0?parser.href.substring(0,hash):parser.href,lastslash=href.lastIndexOf("/");return parser.href=lastslash>=0&&lastslash!==href.length-1?href.substring(0,lastslash+1)+url:href+url,parser.href},Wombat.prototype.extractOriginalURL=function(rewrittenUrl){if(!rewrittenUrl)return"";if(this.wb_is_proxy)return rewrittenUrl;var rwURLString=rewrittenUrl.toString(),url=rwURLString;if(this.startsWithOneOf(url,this.IGNORE_PREFIXES))return url;var start;start=this.startsWith(url,this.wb_abs_prefix)?this.wb_abs_prefix.length:this.wb_rel_prefix&&this.startsWith(url,this.wb_rel_prefix)?this.wb_rel_prefix.length:this.wb_rel_prefix?1:0;var index=url.indexOf("/http",start);return index<0&&(index=url.indexOf("///",start)),index<0&&(index=url.indexOf("/blob:",start)),index>=0?url=url.substr(index+1):(index=url.indexOf(this.wb_replay_prefix),index>=0&&(url=url.substr(index+this.wb_replay_prefix.length)),url.length>4&&url.charAt(2)==="_"&&url.charAt(3)==="/"&&(url=url.substr(4)),url!==rwURLString&&!this.startsWithOneOf(url,this.VALID_PREFIXES)&&!this.startsWith(url,"blob:")&&(url=this.wb_orig_scheme+url)),rwURLString.charAt(0)==="/"&&rwURLString.charAt(1)!=="/"&&this.startsWith(url,this.wb_orig_origin)&&(url=url.substr(this.wb_orig_origin.length)),this.startsWith(url,this.REL_PREFIX)?this.wb_info.wombat_scheme+":"+url:url},Wombat.prototype.makeParser=function(maybeRewrittenURL,doc){var originalURL=this.extractOriginalURL(maybeRewrittenURL),docElem=doc;return doc||(this.$wbwindow.location.href==="about:blank"&&this.$wbwindow.opener?docElem=this.$wbwindow.opener.document:docElem=this.$wbwindow.document),this._makeURLParser(originalURL,docElem)},Wombat.prototype._makeURLParser=function(url,docElem){try{return new this.$wbwindow.URL(url,docElem.baseURI)}catch(e){}var p=docElem.createElement("a");return p._no_rewrite=true,p.href=url,p},Wombat.prototype.defProp=function(obj,prop,setFunc,getFunc,enumerable){var existingDescriptor=Object.getOwnPropertyDescriptor(obj,prop);if(existingDescriptor&&!existingDescriptor.configurable)return false;if(!getFunc)return false;var descriptor={configurable:true,enumerable:enumerable||false,get:getFunc};setFunc&&(descriptor.set=setFunc);try{return Object.defineProperty(obj,prop,descriptor),true}catch(e){return console.warn("Failed to redefine property %s",prop,e.message),false}},Wombat.prototype.defGetterProp=function(obj,prop,getFunc,enumerable){var existingDescriptor=Object.getOwnPropertyDescriptor(obj,prop);if(existingDescriptor&&!existingDescriptor.configurable)return false;if(!getFunc)return false;try{return Object.defineProperty(obj,prop,{configurable:true,enumerable:enumerable||false,get:getFunc}),true}catch(e){return console.warn("Failed to redefine property %s",prop,e.message),false}},Wombat.prototype.getOrigGetter=function(obj,prop){var orig_getter;if(obj.__lookupGetter__&&(orig_getter=obj.__lookupGetter__(prop)),!orig_getter&&Object.getOwnPropertyDescriptor){var props=Object.getOwnPropertyDescriptor(obj,prop);props&&(orig_getter=props.get)}return orig_getter},Wombat.prototype.getOrigSetter=function(obj,prop){var orig_setter;if(obj.__lookupSetter__&&(orig_setter=obj.__lookupSetter__(prop)),!orig_setter&&Object.getOwnPropertyDescriptor){var props=Object.getOwnPropertyDescriptor(obj,prop);props&&(orig_setter=props.set)}return orig_setter},Wombat.prototype.getAllOwnProps=function(obj){for(var ownProps=[],props=Object.getOwnPropertyNames(obj),i=0;i "+final_href),actualLocation.href=final_href}}},Wombat.prototype.checkLocationChange=function(wombatLoc,isTop){var locType=typeof wombatLoc,actual_location=isTop?this.$wbwindow.__WB_replay_top.location:this.$wbwindow.location;locType==="string"?this.updateLocation(wombatLoc,actual_location.href,actual_location):locType==="object"&&this.updateLocation(wombatLoc.href,wombatLoc._orig_href,actual_location)},Wombat.prototype.checkAllLocations=function(){return!this.wb_wombat_updating&&void(this.wb_wombat_updating=true,this.checkLocationChange(this.$wbwindow.WB_wombat_location,false),this.$wbwindow.WB_wombat_location!=this.$wbwindow.__WB_replay_top.WB_wombat_location&&this.checkLocationChange(this.$wbwindow.__WB_replay_top.WB_wombat_location,true),this.wb_wombat_updating=false)},Wombat.prototype.proxyToObj=function(source){if(source)try{var proxyRealObj=source.__WBProxyRealObj__;if(proxyRealObj)return proxyRealObj}catch(e){}return source},Wombat.prototype.objToProxy=function(obj){if(obj)try{var maybeWbProxy=obj._WB_wombat_obj_proxy;if(maybeWbProxy)return maybeWbProxy}catch(e){}return obj},Wombat.prototype.defaultProxyGet=function(obj,prop,ownProps,fnCache){switch(prop){case"__WBProxyRealObj__":return obj;case"location":case"WB_wombat_location":return obj.WB_wombat_location;case"_WB_wombat_obj_proxy":return obj._WB_wombat_obj_proxy;case"__WB_pmw":case"WB_wombat_eval":case this.WB_ASSIGN_FUNC:case this.WB_CHECK_THIS_FUNC:return obj[prop];case"constructor":if(obj.constructor===Window)return obj.constructor;}var retVal=obj[prop],type=typeof retVal;if(type==="function"&&ownProps.indexOf(prop)!==-1){switch(prop){case"requestAnimationFrame":case"cancelAnimationFrame":{if(!this.isNativeFunction(retVal))return retVal;break}}var cachedFN=fnCache[prop];return cachedFN&&cachedFN.original===retVal||(cachedFN={original:retVal,boundFn:retVal.bind(obj)},fnCache[prop]=cachedFN),cachedFN.boundFn}return type==="object"&&retVal&&retVal._WB_wombat_obj_proxy?(retVal instanceof Window&&this.initNewWindowWombat(retVal),retVal._WB_wombat_obj_proxy):retVal},Wombat.prototype.setLoc=function(loc,originalURL){var parser=this.makeParser(originalURL,loc.ownerDocument);loc._orig_href=originalURL,loc._parser=parser;var href=parser.href;loc._hash=parser.hash,loc._href=href,loc._host=parser.host,loc._hostname=parser.hostname,loc._origin=parser.origin?parser.host?parser.origin:"null":parser.protocol+"//"+parser.hostname+(parser.port?":"+parser.port:""),loc._pathname=parser.pathname,loc._port=parser.port,loc._protocol=parser.protocol,loc._search=parser.search,Object.defineProperty||(loc.href=href,loc.hash=parser.hash,loc.host=loc._host,loc.hostname=loc._hostname,loc.origin=loc._origin,loc.pathname=loc._pathname,loc.port=loc._port,loc.protocol=loc._protocol,loc.search=loc._search)},Wombat.prototype.makeGetLocProp=function(prop,origGetter){var wombat=this;return function newGetLocProp(){if(this._no_rewrite)return origGetter.call(this,prop);var curr_orig_href=origGetter.call(this,"href");return prop==="href"?wombat.extractOriginalURL(curr_orig_href):(this._orig_href!==curr_orig_href&&wombat.setLoc(this,curr_orig_href),this["_"+prop])}},Wombat.prototype.makeSetLocProp=function(prop,origSetter,origGetter){var wombat=this;return function newSetLocProp(value){if(this._no_rewrite)return origSetter.call(this,prop,value);if(this["_"+prop]!==value){if(this["_"+prop]=value,!this._parser){var href=origGetter.call(this);this._parser=wombat.makeParser(href,this.ownerDocument)}var rel=false;prop==="href"&&typeof value==="string"&&value&&(value[0]==="."?value=wombat.resolveRelUrl(value,this.ownerDocument):value[0]==="/"&&(value.length<=1||value[1]!=="/")&&(rel=true,value=WB_wombat_location.origin+value));try{this._parser[prop]=value}catch(e){console.log("Error setting "+prop+" = "+value)}prop==="hash"?(value=this._parser[prop],origSetter.call(this,"hash",value)):(rel=rel||value===this._parser.pathname,value=wombat.rewriteUrl(this._parser.href,rel),origSetter.call(this,"href",value))}}},Wombat.prototype.styleReplacer=function(match,n1,n2,n3,offset,string){return n1+this.rewriteUrl(n2)+n3},Wombat.prototype.domConstructorErrorChecker=function(thisObj,what,args,numRequiredArgs){var erorMsg,needArgs=typeof numRequiredArgs==="number"?numRequiredArgs:1;if(thisObj instanceof Window?erorMsg="Failed to construct '"+what+"': Please use the 'new' operator, this DOM object constructor cannot be called as a function.":args&&args.length=0)return url;if(url.indexOf(this.wb_rel_prefix)===0&&url.indexOf("http")>1){var scheme_sep=url.indexOf(":/");return scheme_sep>0&&url[scheme_sep+2]!=="/"?url.substring(0,scheme_sep+2)+"/"+url.substring(scheme_sep+2):url}return this.getFinalUrl(true,mod,this.wb_orig_origin+url)}url.charAt(0)==="."&&(url=this.resolveRelUrl(url,doc));var prefix=this.startsWithOneOf(url.toLowerCase(),this.VALID_PREFIXES);if(prefix){var orig_host=this.$wbwindow.__WB_replay_top.location.host,orig_protocol=this.$wbwindow.__WB_replay_top.location.protocol,prefix_host=prefix+orig_host+"/";if(this.startsWith(url,prefix_host)){if(this.startsWith(url,this.wb_replay_prefix))return url;var curr_scheme=orig_protocol+"//",path=url.substring(prefix_host.length),rebuild=false;return path.indexOf(this.wb_rel_prefix)<0&&url.indexOf("/static/")<0&&(path=this.getFinalUrl(true,mod,WB_wombat_location.origin+"/"+path),rebuild=true),prefix!==curr_scheme&&prefix!==this.REL_PREFIX&&(rebuild=true),rebuild&&(url=useRel?"":curr_scheme+orig_host,path&&path[0]!=="/"&&(url+="/"),url+=path),url}return this.getFinalUrl(useRel,mod,url)}return prefix=this.startsWithOneOf(url,this.BAD_PREFIXES),prefix?this.getFinalUrl(useRel,mod,this.extractOriginalURL(url)):this.isHostUrl(url)&&!this.startsWith(url,originalLoc.host+"/")?this.getFinalUrl(useRel,mod,this.wb_orig_scheme+url):url},Wombat.prototype.rewriteUrl=function(url,useRel,mod,doc){var rewritten=this.rewriteUrl_(url,useRel,mod,doc);return this.debug_rw&&(url===rewritten?console.log("NOT REWRITTEN "+url):console.log("REWRITE: "+url+" -> "+rewritten)),rewritten},Wombat.prototype.performAttributeRewrite=function(elem,name,value,absUrlOnly){switch(name){case"innerHTML":case"outerHTML":return this.rewriteHtml(value);case"filter":return this.rewriteInlineStyle(value);case"style":return this.rewriteStyle(value);case"srcset":return this.rewriteSrcset(value,elem);}if(absUrlOnly&&!this.startsWithOneOf(value,this.VALID_PREFIXES))return value;var mod=this.rwModForElement(elem,name);return this.wbUseAFWorker&&this.WBAutoFetchWorker&&this.isSavedDataSrcSrcset(elem)&&this.WBAutoFetchWorker.preserveDataSrcset(elem),this.rewriteUrl(value,false,mod,elem.ownerDocument)},Wombat.prototype.rewriteAttr=function(elem,name,absUrlOnly){var changed=false;if(!elem||!elem.getAttribute||elem._no_rewrite||elem["_"+name])return changed;var value=this.wb_getAttribute.call(elem,name);if(!value||this.startsWith(value,"javascript:"))return changed;var new_value=this.performAttributeRewrite(elem,name,value,absUrlOnly);return new_value!==value&&(this.removeWBOSRC(elem),this.wb_setAttribute.call(elem,name,new_value),changed=true),changed},Wombat.prototype.noExceptRewriteStyle=function(style){try{return this.rewriteStyle(style)}catch(e){return style}},Wombat.prototype.rewriteStyle=function(style){if(!style)return style;var value=style;return typeof style==="object"&&(value=style.toString()),typeof value==="string"?value.replace(this.STYLE_REGEX,this.styleReplacer).replace(this.IMPORT_REGEX,this.styleReplacer).replace(this.no_wombatRe,""):value},Wombat.prototype.rewriteSrcset=function(value,elem){if(!value)return"";for(var split=value.split(this.srcsetRe),values=[],mod=this.rwModForElement(elem,"srcset"),i=0;i=0){var JS="javascript:";new_value="javascript:window.parent._wb_wombat.initNewWindowWombat(window);"+value.substr(11)}return new_value||(new_value=this.rewriteUrl(value,false,this.rwModForElement(elem,attrName))),new_value!==value&&(this.wb_setAttribute.call(elem,attrName,new_value),true)},Wombat.prototype.rewriteScript=function(elem){if(elem.hasAttribute("src")||!elem.textContent||!this.$wbwindow.Proxy)return this.rewriteAttr(elem,"src");if(this.skipWrapScriptBasedOnType(elem.type))return false;var text=elem.textContent.trim();return!this.skipWrapScriptTextBasedOnText(text)&&(elem.textContent=this.wrapScriptTextJsProxy(text),true)},Wombat.prototype.rewriteSVGElem=function(elem){var changed=this.rewriteAttr(elem,"filter");return changed=this.rewriteAttr(elem,"style")||changed,changed=this.rewriteAttr(elem,"xlink:href")||changed,changed=this.rewriteAttr(elem,"href")||changed,changed=this.rewriteAttr(elem,"src")||changed,changed},Wombat.prototype.rewriteElem=function(elem){var changed=false;if(!elem)return changed;if(elem instanceof SVGElement)changed=this.rewriteSVGElem(elem);else switch(elem.tagName){case"META":var maybeCSP=this.wb_getAttribute.call(elem,"http-equiv");maybeCSP&&maybeCSP.toLowerCase()==="content-security-policy"&&(this.wb_setAttribute.call(elem,"http-equiv","_"+maybeCSP),changed=true);break;case"STYLE":var new_content=this.rewriteStyle(elem.textContent);elem.textContent!==new_content&&(elem.textContent=new_content,changed=true,this.wbUseAFWorker&&this.WBAutoFetchWorker&&elem.sheet!=null&&this.WBAutoFetchWorker.deferredSheetExtraction(elem.sheet));break;case"LINK":changed=this.rewriteAttr(elem,"href"),this.wbUseAFWorker&&elem.rel==="stylesheet"&&this._addEventListener(elem,"load",this.utilFns.wbSheetMediaQChecker);break;case"IMG":changed=this.rewriteAttr(elem,"src"),changed=this.rewriteAttr(elem,"srcset")||changed,changed=this.rewriteAttr(elem,"style")||changed,this.wbUseAFWorker&&this.WBAutoFetchWorker&&elem.dataset.srcset&&this.WBAutoFetchWorker.preserveDataSrcset(elem);break;case"OBJECT":changed=this.rewriteAttr(elem,"data",true),changed=this.rewriteAttr(elem,"style")||changed;break;case"FORM":changed=this.rewriteAttr(elem,"poster"),changed=this.rewriteAttr(elem,"action")||changed,changed=this.rewriteAttr(elem,"style")||changed;break;case"IFRAME":case"FRAME":changed=this.rewriteFrameSrc(elem,"src"),changed=this.rewriteAttr(elem,"style")||changed;break;case"SCRIPT":changed=this.rewriteScript(elem);break;default:{changed=this.rewriteAttr(elem,"src"),changed=this.rewriteAttr(elem,"srcset")||changed,changed=this.rewriteAttr(elem,"href")||changed,changed=this.rewriteAttr(elem,"style")||changed,changed=this.rewriteAttr(elem,"poster")||changed;break}}return elem.hasAttribute&&elem.removeAttribute&&(elem.hasAttribute("crossorigin")&&(elem.removeAttribute("crossorigin"),changed=true),elem.hasAttribute("integrity")&&(elem.removeAttribute("integrity"),changed=true)),changed},Wombat.prototype.recurseRewriteElem=function(curr){if(!this.nodeHasChildren(curr))return false;for(var changed=false,rewriteQ=[curr.children||curr.childNodes];rewriteQ.length>0;)for(var child,children=rewriteQ.shift(),i=0;i"+rwString+"","text/html");if(!inner_doc||!this.nodeHasChildren(inner_doc.head)||!inner_doc.head.children[0].content)return rwString;var template=inner_doc.head.children[0];if(template._no_rewrite=true,this.recurseRewriteElem(template.content)){var new_html=template.innerHTML;if(checkEndTag){var first_elem=template.content.children&&template.content.children[0];if(first_elem){var end_tag="";this.endsWith(new_html,end_tag)&&!this.endsWith(rwString.toLowerCase(),end_tag)&&(new_html=new_html.substring(0,new_html.length-end_tag.length))}else if(rwString[0]!=="<"||rwString[rwString.length-1]!==">")return this.write_buff+=rwString,undefined}return new_html}return rwString},Wombat.prototype.rewriteHtmlFull=function(string,checkEndTag){var inner_doc=new DOMParser().parseFromString(string,"text/html");if(!inner_doc)return string;for(var changed=false,i=0;i=0)inner_doc.documentElement._no_rewrite=true,new_html=this.reconstructDocType(inner_doc.doctype)+inner_doc.documentElement.outerHTML;else{inner_doc.head._no_rewrite=true,inner_doc.body._no_rewrite=true;var headHasKids=this.nodeHasChildren(inner_doc.head),bodyHasKids=this.nodeHasChildren(inner_doc.body);if(new_html=(headHasKids?inner_doc.head.outerHTML:"")+(bodyHasKids?inner_doc.body.outerHTML:""),checkEndTag)if(inner_doc.all.length>3){var end_tag="";this.endsWith(new_html,end_tag)&&!this.endsWith(string.toLowerCase(),end_tag)&&(new_html=new_html.substring(0,new_html.length-end_tag.length))}else if(string[0]!=="<"||string[string.length-1]!==">")return void(this.write_buff+=string);new_html=this.reconstructDocType(inner_doc.doctype)+new_html}return new_html}return string},Wombat.prototype.rewriteInlineStyle=function(orig){var decoded;try{decoded=decodeURIComponent(orig)}catch(e){decoded=orig}if(decoded!==orig){var parts=this.rewriteStyle(decoded).split(",",2);return parts[0]+","+encodeURIComponent(parts[1])}return this.rewriteStyle(orig)},Wombat.prototype.rewriteCookie=function(cookie){var wombat=this,rwCookie=cookie.replace(this.wb_abs_prefix,"").replace(this.wb_rel_prefix,"");return rwCookie=rwCookie.replace(this.cookie_domain_regex,function(m,m1){var message={domain:m1,cookie:rwCookie,wb_type:"cookie"};return wombat.sendTopMessage(message,true),wombat.$wbwindow.location.hostname.indexOf(".")>=0&&!wombat.IP_RX.test(wombat.$wbwindow.location.hostname)?"Domain=."+wombat.$wbwindow.location.hostname:""}).replace(this.cookie_path_regex,function(m,m1){var rewritten=wombat.rewriteUrl(m1);return rewritten.indexOf(wombat.wb_curr_host)===0&&(rewritten=rewritten.substring(wombat.wb_curr_host.length)),"Path="+rewritten}),wombat.$wbwindow.location.protocol!=="https:"&&(rwCookie=rwCookie.replace("secure","")),rwCookie.replace(",|",",")},Wombat.prototype.rewriteWorker=function(workerUrl){if(!workerUrl)return workerUrl;var isBlob=workerUrl.indexOf("blob:")===0,isJS=workerUrl.indexOf("javascript:")===0;if(!isBlob&&!isJS){if(!this.startsWithOneOf(workerUrl,this.VALID_PREFIXES)&&!this.startsWith(workerUrl,"/")&&!this.startsWithOneOf(workerUrl,this.BAD_PREFIXES)){var rurl=this.resolveRelUrl(workerUrl,this.$wbwindow.document);return this.rewriteUrl(rurl,false,"wkr_",this.$wbwindow.document)}return this.rewriteUrl(workerUrl,false,"wkr_",this.$wbwindow.document)}var workerCode=isJS?workerUrl.replace("javascript:",""):null;if(isBlob){var x=new XMLHttpRequest;this.utilFns.XHRopen.call(x,"GET",workerUrl,false),x.send(),workerCode=x.responseText.replace(this.workerBlobRe,"").replace(this.rmCheckThisInjectRe,"this")}if(this.wb_info.static_prefix||this.wb_info.ww_rw_script){var originalURL=this.$wbwindow.document.baseURI,ww_rw=this.wb_info.ww_rw_script||this.wb_info.static_prefix+"wombatWorkers.js",rw="(function() { self.importScripts('"+ww_rw+"'); new WBWombat({'prefix': '"+this.wb_abs_prefix+"', 'prefixMod': '"+this.wb_abs_prefix+"wkrf_/', 'originalURL': '"+originalURL+"'}); })();";workerCode=rw+workerCode}var blob=new Blob([workerCode],{type:"application/javascript"});return URL.createObjectURL(blob)},Wombat.prototype.rewriteTextNodeFn=function(fnThis,originalFn,argsObj){var args,deproxiedThis=this.proxyToObj(fnThis);if(argsObj.length>0&&deproxiedThis.parentElement&&deproxiedThis.parentElement.tagName==="STYLE"){args=new Array(argsObj.length);var dataIndex=argsObj.length-1;dataIndex===2?(args[0]=argsObj[0],args[1]=argsObj[1]):dataIndex===1&&(args[0]=argsObj[0]),args[dataIndex]=this.rewriteStyle(argsObj[dataIndex])}else args=argsObj;return originalFn.__WB_orig_apply?originalFn.__WB_orig_apply(deproxiedThis,args):originalFn.apply(deproxiedThis,args)},Wombat.prototype.rewriteDocWriteWriteln=function(fnThis,originalFn,argsObj){var string,thisObj=this.proxyToObj(fnThis),argLen=argsObj.length;if(argLen===0)return originalFn.call(thisObj);string=argLen===1?argsObj[0]:Array.prototype.join.call(argsObj,"");var new_buff=this.rewriteHtml(string,true),res=originalFn.call(thisObj,new_buff);return this.initNewWindowWombat(thisObj.defaultView),res},Wombat.prototype.rewriteChildNodeFn=function(fnThis,originalFn,argsObj){var thisObj=this.proxyToObj(fnThis);if(argsObj.length===0)return originalFn.call(thisObj);var newArgs=this.rewriteElementsInArguments(argsObj);return originalFn.__WB_orig_apply?originalFn.__WB_orig_apply(thisObj,newArgs):originalFn.apply(thisObj,newArgs)},Wombat.prototype.rewriteInsertAdjHTMLOrElemArgs=function(fnThis,originalFn,position,textOrElem,rwHTML){var fnThisObj=this.proxyToObj(fnThis);return fnThisObj._no_rewrite?originalFn.call(fnThisObj,position,textOrElem):rwHTML?originalFn.call(fnThisObj,position,this.rewriteHtml(textOrElem)):(this.rewriteElemComplete(textOrElem),originalFn.call(fnThisObj,position,textOrElem))},Wombat.prototype.rewriteSetTimeoutInterval=function(fnThis,originalFn,argsObj){var rw=this.isString(argsObj[0]),args=rw?new Array(argsObj.length):argsObj;if(rw){args[0]=this.$wbwindow.Proxy?this.wrapScriptTextJsProxy(argsObj[0]):argsObj[0].replace(/\blocation\b/g,"WB_wombat_$&");for(var i=1;i0&&cssStyleValueOverride(this.$wbwindow.CSSStyleValue,"parse"),this.$wbwindow.CSSStyleValue.parseAll&&this.$wbwindow.CSSStyleValue.parseAll.toString().indexOf("[native code]")>0&&cssStyleValueOverride(this.$wbwindow.CSSStyleValue,"parseAll")}if(this.$wbwindow.CSSKeywordValue&&this.$wbwindow.CSSKeywordValue.prototype){var oCSSKV=this.$wbwindow.CSSKeywordValue;this.$wbwindow.CSSKeywordValue=function(CSSKeywordValue_){return function CSSKeywordValue(cssValue){return wombat.domConstructorErrorChecker(this,"CSSKeywordValue",arguments),new CSSKeywordValue_(wombat.rewriteStyle(cssValue))}}(this.$wbwindow.CSSKeywordValue),this.$wbwindow.CSSKeywordValue.prototype=oCSSKV.prototype,Object.defineProperty(this.$wbwindow.CSSKeywordValue.prototype,"constructor",{value:this.$wbwindow.CSSKeywordValue}),addToStringTagToClass(this.$wbwindow.CSSKeywordValue,"CSSKeywordValue")}if(this.$wbwindow.StylePropertyMap&&this.$wbwindow.StylePropertyMap.prototype){var originalSet=this.$wbwindow.StylePropertyMap.prototype.set;this.$wbwindow.StylePropertyMap.prototype.set=function set(){if(arguments.length<=1)return originalSet.__WB_orig_apply?originalSet.__WB_orig_apply(this,arguments):originalSet.apply(this,arguments);var newArgs=new Array(arguments.length);newArgs[0]=arguments[0];for(var i=1;i=0||name==="href"?wombat.extractOriginalURL(value):value},svgImgProto.getAttributeNS=function getAttributeNS(ns,name){var value=orig_getAttrNS.call(this,ns,name);return name.indexOf("xlink:href")>=0||name==="href"?wombat.extractOriginalURL(value):value},svgImgProto.setAttribute=function setAttribute(name,value){var rwValue=value;return(name.indexOf("xlink:href")>=0||name==="href")&&(rwValue=wombat.rewriteUrl(value)),orig_setAttr.call(this,name,rwValue)},svgImgProto.setAttributeNS=function setAttributeNS(ns,name,value){var rwValue=value;return(name.indexOf("xlink:href")>=0||name==="href")&&(rwValue=wombat.rewriteUrl(value)),orig_setAttrNS.call(this,ns,name,rwValue)}}},Wombat.prototype.initCreateElementNSFix=function(){if(this.$wbwindow.document.createElementNS&&this.$wbwindow.Document.prototype.createElementNS){var orig_createElementNS=this.$wbwindow.document.createElementNS,wombat=this,createElementNS=function createElementNS(namespaceURI,qualifiedName){return orig_createElementNS.call(wombat.proxyToObj(this),wombat.extractOriginalURL(namespaceURI),qualifiedName)};this.$wbwindow.Document.prototype.createElementNS=createElementNS,this.$wbwindow.document.createElementNS=createElementNS}},Wombat.prototype.initInsertAdjacentElementHTMLOverrides=function(){var Element=this.$wbwindow.Element;if(Element&&Element.prototype){var elementProto=Element.prototype,rewriteFn=this.rewriteInsertAdjHTMLOrElemArgs;if(elementProto.insertAdjacentHTML){var origInsertAdjacentHTML=elementProto.insertAdjacentHTML;elementProto.insertAdjacentHTML=function insertAdjacentHTML(position,text){return rewriteFn(this,origInsertAdjacentHTML,position,text,true)}}if(elementProto.insertAdjacentElement){var origIAdjElem=elementProto.insertAdjacentElement;elementProto.insertAdjacentElement=function insertAdjacentElement(position,element){return rewriteFn(this,origIAdjElem,position,element,false)}}}},Wombat.prototype.initDomOverride=function(){var Node=this.$wbwindow.Node;if(Node&&Node.prototype){var rewriteFn=this.rewriteNodeFuncArgs;if(Node.prototype.appendChild){var originalAppendChild=Node.prototype.appendChild;Node.prototype.appendChild=function appendChild(newNode,oldNode){return rewriteFn(this,originalAppendChild,newNode,oldNode)}}if(Node.prototype.insertBefore){var originalInsertBefore=Node.prototype.insertBefore;Node.prototype.insertBefore=function insertBefore(newNode,oldNode){return rewriteFn(this,originalInsertBefore,newNode,oldNode)}}if(Node.prototype.replaceChild){var originalReplaceChild=Node.prototype.replaceChild;Node.prototype.replaceChild=function replaceChild(newNode,oldNode){return rewriteFn(this,originalReplaceChild,newNode,oldNode)}}this.overridePropToProxy(Node.prototype,"ownerDocument"),this.overridePropToProxy(this.$wbwindow.HTMLHtmlElement.prototype,"parentNode"),this.overridePropToProxy(this.$wbwindow.Event.prototype,"target")}this.$wbwindow.Element&&this.$wbwindow.Element.prototype&&(this.overrideParentNodeAppendPrepend(this.$wbwindow.Element),this.overrideChildNodeInterface(this.$wbwindow.Element,false)),this.$wbwindow.DocumentFragment&&this.$wbwindow.DocumentFragment.prototype&&this.overrideParentNodeAppendPrepend(this.$wbwindow.DocumentFragment)},Wombat.prototype.initDocOverrides=function($document){if(Object.defineProperty){this.overrideReferrer($document),this.defGetterProp($document,"origin",function origin(){return this.WB_wombat_location.origin}),this.defGetterProp(this.$wbwindow,"origin",function origin(){return this.WB_wombat_location.origin});var wombat=this,domain_setter=function domain(val){var loc=this.WB_wombat_location;loc&&wombat.endsWith(loc.hostname,val)&&(this.__wb_domain=val)},domain_getter=function domain(){return this.__wb_domain||this.WB_wombat_location.hostname};this.defProp($document,"domain",domain_setter,domain_getter)}},Wombat.prototype.initDocWriteOpenCloseOverride=function(){if(this.$wbwindow.DOMParser){var DocumentProto=this.$wbwindow.Document.prototype,$wbDocument=this.$wbwindow.document,docWriteWritelnRWFn=this.rewriteDocWriteWriteln,orig_doc_write=$wbDocument.write,new_write=function write(){return docWriteWritelnRWFn(this,orig_doc_write,arguments)};$wbDocument.write=new_write,DocumentProto.write=new_write;var orig_doc_writeln=$wbDocument.writeln,new_writeln=function writeln(){return docWriteWritelnRWFn(this,orig_doc_writeln,arguments)};$wbDocument.writeln=new_writeln,DocumentProto.writeln=new_writeln;var wombat=this,orig_doc_open=$wbDocument.open,new_open=function open(){var res,thisObj=wombat.proxyToObj(this);if(arguments.length===3){var rwUrl=wombat.rewriteUrl(arguments[0],false,"mp_");res=orig_doc_open.call(thisObj,rwUrl,arguments[1],arguments[2]),wombat.initNewWindowWombat(res,rwUrl)}else res=orig_doc_open.call(thisObj),wombat.initNewWindowWombat(thisObj.defaultView);return res};$wbDocument.open=new_open,DocumentProto.open=new_open;var originalClose=$wbDocument.close,newClose=function close(){var thisObj=wombat.proxyToObj(this);return wombat.initNewWindowWombat(thisObj.defaultView),originalClose.__WB_orig_apply?originalClose.__WB_orig_apply(thisObj,arguments):originalClose.apply(thisObj,arguments)};$wbDocument.close=newClose,DocumentProto.close=newClose;var oBodyGetter=this.getOrigGetter(DocumentProto,"body"),oBodySetter=this.getOrigSetter(DocumentProto,"body");oBodyGetter&&oBodySetter&&this.defProp(DocumentProto,"body",function body(newBody){return newBody&&(newBody instanceof HTMLBodyElement||newBody instanceof HTMLFrameSetElement)&&wombat.rewriteElemComplete(newBody),oBodySetter.call(wombat.proxyToObj(this),newBody)},oBodyGetter)}},Wombat.prototype.initIframeWombat=function(iframe){var win;win=iframe._get_contentWindow?iframe._get_contentWindow.call(iframe):iframe.contentWindow;try{if(!win||win===this.$wbwindow||win._skip_wombat||win._wb_wombat)return}catch(e){return}var src=this.wb_getAttribute.call(iframe,"src");this.initNewWindowWombat(win,src)},Wombat.prototype.initNewWindowWombat=function(win,src){if(win&&!win._wb_wombat)if(!src||src===""||this.startsWith(src,"about:")||src.indexOf("javascript:")>=0){var wombat=new Wombat(win,this.wb_info);win._wb_wombat=wombat.wombatInit()}else this.initProtoPmOrigin(win),this.initPostMessageOverride(win),this.initMessageEventOverride(win),this.initCheckThisFunc(win)},Wombat.prototype.initTimeoutIntervalOverrides=function(){var rewriteFn=this.rewriteSetTimeoutInterval;if(this.$wbwindow.setTimeout&&!this.$wbwindow.setTimeout.__$wbpatched$__){var originalSetTimeout=this.$wbwindow.setTimeout;this.$wbwindow.setTimeout=function setTimeout(){return rewriteFn(this,originalSetTimeout,arguments)},this.$wbwindow.setTimeout.__$wbpatched$__=true}if(this.$wbwindow.setInterval&&!this.$wbwindow.setInterval.__$wbpatched$__){var originalSetInterval=this.$wbwindow.setInterval;this.$wbwindow.setInterval=function setInterval(){return rewriteFn(this,originalSetInterval,arguments)},this.$wbwindow.setInterval.__$wbpatched$__=true}},Wombat.prototype.initWorkerOverrides=function(){var wombat=this;if(this.$wbwindow.Worker&&!this.$wbwindow.Worker._wb_worker_overriden){var orig_worker=this.$wbwindow.Worker;this.$wbwindow.Worker=function(Worker_){return function Worker(url,options){return wombat.domConstructorErrorChecker(this,"Worker",arguments),new Worker_(wombat.rewriteWorker(url),options)}}(orig_worker),this.$wbwindow.Worker.prototype=orig_worker.prototype,Object.defineProperty(this.$wbwindow.Worker.prototype,"constructor",{value:this.$wbwindow.Worker}),this.$wbwindow.Worker._wb_worker_overriden=true}if(this.$wbwindow.SharedWorker&&!this.$wbwindow.SharedWorker.__wb_sharedWorker_overriden){var oSharedWorker=this.$wbwindow.SharedWorker;this.$wbwindow.SharedWorker=function(SharedWorker_){return function SharedWorker(url,options){return wombat.domConstructorErrorChecker(this,"SharedWorker",arguments),new SharedWorker_(wombat.rewriteWorker(url),options)}}(oSharedWorker),this.$wbwindow.SharedWorker.prototype=oSharedWorker.prototype,Object.defineProperty(this.$wbwindow.SharedWorker.prototype,"constructor",{value:this.$wbwindow.SharedWorker}),this.$wbwindow.SharedWorker.__wb_sharedWorker_overriden=true}if(this.$wbwindow.ServiceWorkerContainer&&this.$wbwindow.ServiceWorkerContainer.prototype&&this.$wbwindow.ServiceWorkerContainer.prototype.register){var orig_register=this.$wbwindow.ServiceWorkerContainer.prototype.register;this.$wbwindow.ServiceWorkerContainer.prototype.register=function register(scriptURL,options){var newScriptURL=new URL(scriptURL,wombat.$wbwindow.document.baseURI).href,mod=wombat.getPageUnderModifier();return options&&options.scope?options.scope=wombat.rewriteUrl(options.scope,false,mod):options={scope:wombat.rewriteUrl("/",false,mod)},orig_register.call(this,wombat.rewriteUrl(newScriptURL,false,"sw_"),options)}}if(this.$wbwindow.Worklet&&this.$wbwindow.Worklet.prototype&&this.$wbwindow.Worklet.prototype.addModule&&!this.$wbwindow.Worklet.__wb_workerlet_overriden){var oAddModule=this.$wbwindow.Worklet.prototype.addModule;this.$wbwindow.Worklet.prototype.addModule=function addModule(moduleURL,options){var rwModuleURL=wombat.rewriteUrl(moduleURL,false,"js_");return oAddModule.call(this,rwModuleURL,options)},this.$wbwindow.Worklet.__wb_workerlet_overriden=true}},Wombat.prototype.initLocOverride=function(loc,oSetter,oGetter){if(Object.defineProperty)for(var prop,i=0;i Date: Sat, 11 Jan 2020 11:12:31 -0800 Subject: [PATCH 72/89] revisit lookup fix (possible fix for ukwa/ukwa-pywb#53) (#530) - if a revisit record has empty hash, don't attempt to lookup an original, simply use with empty payload --- pywb/warcserver/resource/resolvingloader.py | 11 ++++-- tests/test_redirects.py | 37 +++++++++++++-------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/pywb/warcserver/resource/resolvingloader.py b/pywb/warcserver/resource/resolvingloader.py index aa584f5ed..72d8c5ad3 100644 --- a/pywb/warcserver/resource/resolvingloader.py +++ b/pywb/warcserver/resource/resolvingloader.py @@ -11,6 +11,8 @@ class ResolvingLoader(object): MISSING_REVISIT_MSG = 'Original for revisit record could not be loaded' + EMPTY_DIGEST = '3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ' + def __init__(self, path_resolvers, record_loader=None, no_record_parse=False): self.path_resolvers = path_resolvers self.record_loader = record_loader if record_loader is not None else BlockArcWarcRecordLoader() @@ -163,6 +165,13 @@ def _load_different_url_payload(self, cdx, headers_record, Raise exception if no matches found. """ + digest = cdx.get('digest', '-') + + # if the digest is the empty record digest, don't attempt to look up the payload record! + # the payload is simply empty, so use empty payload of existing record + if digest == self.EMPTY_DIGEST: + return headers_record + ref_target_uri = (headers_record.rec_headers. get_header('WARC-Refers-To-Target-URI')) @@ -180,8 +189,6 @@ def _load_different_url_payload(self, cdx, headers_record, else: ref_target_date = iso_date_to_timestamp(ref_target_date) - digest = cdx.get('digest', '-') - try: orig_cdx_lines = self.load_cdx_for_dupe(ref_target_uri, ref_target_date, diff --git a/tests/test_redirects.py b/tests/test_redirects.py index 6ec904ebf..79ce367cd 100644 --- a/tests/test_redirects.py +++ b/tests/test_redirects.py @@ -57,7 +57,7 @@ def create_response_record(self, url, timestamp, text): self.writer.write_record(rec) return rec - def create_revisit_record(self, original, url, redirect_url, timestamp): + def create_revisit_record(self, url, timestamp, redirect_url, original_dt): warc_headers = {} warc_headers['WARC-Date'] = timestamp_to_iso_date(timestamp) @@ -67,9 +67,9 @@ def create_revisit_record(self, original, url, redirect_url, timestamp): http_headers = StatusAndHeaders('302 Temp Redirect', headers_list, protocol='HTTP/1.0') rec = self.writer.create_revisit_record(url, - digest=original.rec_headers['WARC-Payload-Digest'], + digest='3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ', refers_to_uri=url, - refers_to_date=original.rec_headers['WARC-Date'], + refers_to_date=original_dt, warc_headers_dict=warc_headers, http_headers=http_headers) @@ -80,9 +80,12 @@ def test_init_1(self): with open(filename, 'wb') as fh: self.writer = WARCWriter(fh, gzip=True) - redirect = self.create_redirect_record('http://example.com/', 'https://example.com/', '201806026101112') - redirect = self.create_redirect_record('https://example.com/', 'https://www.example.com/', '201806026101112') - response = self.create_response_record('https://www.example.com/', '201806026101112', 'Some Text') + redirect = self.create_redirect_record('http://example.com/', 'https://example.com/', '20180626101112') + redirect = self.create_redirect_record('https://example.com/', 'https://www.example.com/', '20180626101112') + response = self.create_response_record('https://www.example.com/', '20180626101112', 'Some Text') + + revisit = self.create_revisit_record('https://example.com/path', '20190626101112', 'https://example.com/abc', response.rec_headers['WARC-Date']) + revisit = self.create_revisit_record('https://www.example.com/', '20190626101112', 'https://www.example.com/', response.rec_headers['WARC-Date']) wb_manager(['init', 'redir']) @@ -91,7 +94,7 @@ def test_init_1(self): assert os.path.isfile(os.path.join(self.root_dir, self.COLLS_DIR, 'redir', 'indexes', 'index.cdxj')) def test_self_redir_1(self, fmod): - res = self.get('/redir/201806026101112{0}/https://example.com/', fmod) + res = self.get('/redir/20180626101112{0}/https://example.com/', fmod, status=200) assert res.status_code == 200 @@ -102,16 +105,16 @@ def test_redir_init_slash(self): with open(filename, 'wb') as fh: self.writer = WARCWriter(fh, gzip=True) - response = self.create_response_record('https://www.example.com/sub/path/', '201806026101112', 'Sub Path Data') + response = self.create_response_record('https://www.example.com/sub/path/', '20180626101112', 'Sub Path Data') - response = self.create_response_record('https://www.example.com/sub/path/?foo=bar', '201806026101112', 'Sub Path Data Q') + response = self.create_response_record('https://www.example.com/sub/path/?foo=bar', '20180626101112', 'Sub Path Data Q') wb_manager(['add', 'redir', filename]) def test_redir_slash(self, fmod): - res = self.get('/redir/201806026101112{0}/https://example.com/sub/path', fmod, status=307) + res = self.get('/redir/20180626101112{0}/https://example.com/sub/path', fmod, status=307) - assert res.headers['Location'].endswith('/redir/201806026101112{0}/https://example.com/sub/path/'.format(fmod)) + assert res.headers['Location'].endswith('/redir/20180626101112{0}/https://example.com/sub/path/'.format(fmod)) res = res.follow() assert res.status_code == 200 @@ -119,14 +122,22 @@ def test_redir_slash(self, fmod): assert res.text == 'Sub Path Data' def test_redir_slash_with_query(self, fmod): - res = self.get('/redir/201806026101112{0}/https://example.com/sub/path?foo=bar', fmod, status=307) + res = self.get('/redir/20180626101112{0}/https://example.com/sub/path?foo=bar', fmod, status=307) - assert res.headers['Location'].endswith('/redir/201806026101112{0}/https://example.com/sub/path/?foo=bar'.format(fmod)) + assert res.headers['Location'].endswith('/redir/20180626101112{0}/https://example.com/sub/path/?foo=bar'.format(fmod)) res = res.follow() assert res.status_code == 200 assert res.text == 'Sub Path Data Q' + def test_revisit_redirect_302(self, fmod): + res = self.get('/redir/20170626101112{0}/https://example.com/path', fmod, status=302) + assert res.headers['Location'].endswith('/redir/20170626101112{0}/https://example.com/abc'.format(fmod)) + assert res.text == '' + + def test_revisit_redirect_skip_self_redir(self, fmod): + res = self.get('/redir/20190626101112{0}/http://www.example.com/', fmod, status=200) + assert res.text == 'Some Text' From 93ce4f6f7a164d3a045d8e57699905f68233be14 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Sat, 11 Jan 2020 13:05:28 -0800 Subject: [PATCH 73/89] Banner fix (#531) * banner: fix banner display for non-framed and proxy mode replay, ensure new 'View All Captures' ancillary section is also shown * bump version to 2.4.0rc4 --- pywb/static/default_banner.css | 34 ++++++++++++++++++---------------- pywb/static/default_banner.js | 17 ++++++++++++++--- pywb/templates/banner.html | 9 +++++---- pywb/version.py | 2 +- 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/pywb/static/default_banner.css b/pywb/static/default_banner.css index 3a9565f1c..1f9b03d72 100644 --- a/pywb/static/default_banner.css +++ b/pywb/static/default_banner.css @@ -11,19 +11,7 @@ color: white !important; z-index: 2147483643 !important; line-height: normal !important; -} -#title_or_url -{ - display: block !important; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - max-width: 100%; -} - -#_wb_frame_top_banner -{ position: absolute !important; border: 0px; height: 44px !important; @@ -44,6 +32,17 @@ -ms-flex-align: center; } +#title_or_url +{ + display: block !important; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 100%; +} + + + #_wb_frame_top_banner ._wb_linked_logo { display: block; @@ -110,11 +109,14 @@ { font-size: 12px; color: #FFF; - margin-right: 15px; text-align: right; - flex-shrink: 1 0; - -webkit-flex-shrink: 1 0; - -moz-flex-shrink: 1 0; + margin: 0px 15px 0px 0px; + padding: inherit; + background-color: inherit; + width: initial; + flex-shrink: 1; + -webkit-flex-shrink: 1; + -moz-flex-shrink: 1; -ms-flex: 0 0 115px; } #_wb_frame_top_banner #_wb_ancillary_links a:link, diff --git a/pywb/static/default_banner.js b/pywb/static/default_banner.js index 5fd71e018..34fca130c 100644 --- a/pywb/static/default_banner.js +++ b/pywb/static/default_banner.js @@ -45,16 +45,15 @@ This file is part of pywb, https://github.com/webrecorder/pywb * @desc Initialize (display) the banner */ DefaultBanner.prototype.init = function() { + this.createBanner('_wb_frame_top_banner'); + if (window.wbinfo) { - this.createBanner('_wb_plain_banner'); this.set_banner( window.wbinfo.url, window.wbinfo.timestamp, window.wbinfo.is_live, window.wbinfo.is_framed ? '' : document.title ); - } else { - this.createBanner('_wb_frame_top_banner'); } }; @@ -306,4 +305,16 @@ This file is part of pywb, https://github.com/webrecorder/pywb // all banners will expose themselves by adding themselves as WBBanner on window window.WBBanner = new DefaultBanner(); + + // if in replay frame, init immediately + if (window.wbinfo) { + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", function() { + window.WBBanner.init(); + }); + } else { + window.WBBanner.init(); + } + } + })(); diff --git a/pywb/templates/banner.html b/pywb/templates/banner.html index f458db79b..d629dd2b7 100644 --- a/pywb/templates/banner.html +++ b/pywb/templates/banner.html @@ -1,8 +1,4 @@ {% if not env.pywb_proxy_magic or config.proxy.enable_banner | default(true) %} - - - - + + + + + {% endif %} diff --git a/pywb/version.py b/pywb/version.py index cce7cd754..23a4653c7 100644 --- a/pywb/version.py +++ b/pywb/version.py @@ -1,4 +1,4 @@ -__version__ = '2.4.0rc3' +__version__ = '2.4.0rc4' if __name__ == '__main__': print(__version__) From fa021eebab2b2cbf940e2da5df13504d9422a070 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 17 Jan 2020 17:38:08 -0800 Subject: [PATCH 74/89] Misc Fixes for RC5 (#534) * misc fixes (rc 5): - banner: only auto init banner if not in top-frame (check for no-frame mode and replay url is set) - index: 'cdx+' fix for use as internal index: if cdx has a warc filename and offset, don't attempt default live web load - improved self-redirect: avoid www2 -> www redirect altogether, not just for second redirect - tests: update tests for improved self-redirect checking - bump version to pywb-2.4.0-rc5 --- pywb/static/default_banner.js | 4 ++-- pywb/version.py | 2 +- pywb/warcserver/resource/responseloader.py | 14 +++++++----- pywb/warcserver/test/test_handlers.py | 4 ++-- tests/test_redirects.py | 26 ++++++++++++++++++++-- 5 files changed, 38 insertions(+), 12 deletions(-) diff --git a/pywb/static/default_banner.js b/pywb/static/default_banner.js index 34fca130c..c1251dc85 100644 --- a/pywb/static/default_banner.js +++ b/pywb/static/default_banner.js @@ -306,8 +306,8 @@ This file is part of pywb, https://github.com/webrecorder/pywb // all banners will expose themselves by adding themselves as WBBanner on window window.WBBanner = new DefaultBanner(); - // if in replay frame, init immediately - if (window.wbinfo) { + // if wbinfo.url is set and not-framed, init banner in content frame + if (window.wbinfo && window.wbinfo.url && !window.wbinfo.is_framed) { if (document.readyState === "loading") { document.addEventListener("DOMContentLoaded", function() { window.WBBanner.init(); diff --git a/pywb/version.py b/pywb/version.py index 23a4653c7..8042daa5e 100644 --- a/pywb/version.py +++ b/pywb/version.py @@ -1,4 +1,4 @@ -__version__ = '2.4.0rc4' +__version__ = '2.4.0-rc5' if __name__ == '__main__': print(__version__) diff --git a/pywb/warcserver/resource/responseloader.py b/pywb/warcserver/resource/responseloader.py index 7f4aa297e..defe906eb 100644 --- a/pywb/warcserver/resource/responseloader.py +++ b/pywb/warcserver/resource/responseloader.py @@ -139,18 +139,19 @@ def raise_on_self_redirect(self, params, cdx, status_code, location_url): request_url = request_url.split('://', 1)[-1].rstrip('/') self_redir = False + orig_key = params.get('sr-urlkey') or cdx['urlkey'] if request_url == location_url: self_redir = True - elif params.get('sr-urlkey'): - # if new location canonicalized matches old key, also self-redirect - if canonicalize(location_url) == params.get('sr-urlkey'): - self_redir = True + + # if new location canonicalized matches old key, also self-redirect + elif canonicalize(location_url) == orig_key: + self_redir = True if self_redir: msg = 'Self Redirect {0} -> {1}' msg = msg.format(request_url, location_url) - params['sr-urlkey'] = cdx['urlkey'] + params['sr-urlkey'] = orig_key raise LiveResourceException(msg) @staticmethod @@ -267,6 +268,9 @@ def __init__(self, forward_proxy_prefix=None, adapter=None): self.socks_proxy = None def load_resource(self, cdx, params): + if cdx.get('filename') and cdx.get('offset') is not None: + return None + load_url = cdx.get('load_url') if not load_url: return None diff --git a/pywb/warcserver/test/test_handlers.py b/pywb/warcserver/test/test_handlers.py index 52bf520dd..74a4a0a94 100644 --- a/pywb/warcserver/test/test_handlers.py +++ b/pywb/warcserver/test/test_handlers.py @@ -220,8 +220,8 @@ def test_agg_select_mem_unrewrite_headers(self): buff = BytesIO(resp.body) record = ArcWarcRecordLoader().parse_record_stream(buff, no_record_parse=False) print(record.http_headers) - assert record.http_headers.get_statuscode() == '302' - assert record.http_headers.get_header('Location') == 'https://www.iana.org/' + assert record.http_headers.get_statuscode() == '200' + #assert record.http_headers.get_header('Location') == 'https://www.iana.org/' @patch('pywb.warcserver.index.indexsource.MementoIndexSource.get_timegate_links', MementoOverrideTests.mock_link_header('select_live')) def test_agg_select_live(self): diff --git a/tests/test_redirects.py b/tests/test_redirects.py index 79ce367cd..187163732 100644 --- a/tests/test_redirects.py +++ b/tests/test_redirects.py @@ -15,7 +15,7 @@ class TestRedirects(CollsDirMixin, BaseConfigTest): def setup_class(cls): super(TestRedirects, cls).setup_class('config_test.yaml') - def create_redirect_record(self, url, redirect_url, timestamp): + def create_redirect_record(self, url, redirect_url, timestamp, status='301'): warc_headers = {} warc_headers['WARC-Date'] = timestamp_to_iso_date(timestamp) @@ -26,7 +26,7 @@ def create_redirect_record(self, url, redirect_url, timestamp): ('Location', redirect_url) ] - http_headers = StatusAndHeaders('301 Permanent Redirect', headers_list, protocol='HTTP/1.0') + http_headers = StatusAndHeaders(status + ' Redirect', headers_list, protocol='HTTP/1.0') rec = self.writer.create_warc_record(url, 'response', payload=BytesIO(payload), @@ -140,4 +140,26 @@ def test_revisit_redirect_skip_self_redir(self, fmod): res = self.get('/redir/20190626101112{0}/http://www.example.com/', fmod, status=200) assert res.text == 'Some Text' + def test_init_2(self): + filename = os.path.join(self.root_dir, 'redir2.warc.gz') + with open(filename, 'wb') as fh: + self.writer = WARCWriter(fh, gzip=True) + + redirect = self.create_redirect_record('http://www.example.com/path', 'https://www.example.com/path/', '20191003115920') + redirect = self.create_redirect_record('https://www.example.com/path/', 'https://www2.example.com/path', '20191003115927', status='302') + response = self.create_response_record('https://www2.example.com/path', '20191024125646', 'Some Text') + revisit = self.create_revisit_record('https://www2.example.com/path', '20191024125648', 'https://www2.example.com/path', response.rec_headers['WARC-Date']) + + wb_manager(['init', 'redir2']) + + wb_manager(['add', 'redir2', filename]) + + assert os.path.isfile(os.path.join(self.root_dir, self.COLLS_DIR, 'redir2', 'indexes', 'index.cdxj')) + + def test_revisit_redirect_skip_self_redir_2(self, fmod): + res = self.get('/redir2/20191024125648{0}/http://www2.example.com/path', fmod, status=200) + assert res.text == 'Some Text' + + res = self.get('/redir2/20191024125648{0}/https://www.example.com/path', fmod, status=200) + assert res.text == 'Some Text' From 92e459bda52a2b03f33a4b0b8094ed424248d2a5 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Thu, 20 Feb 2020 21:53:00 -0800 Subject: [PATCH 75/89] R6 - Various Fixes (#540) * fixes for RC6: - blockrecordloader: ensure record stream is closed after parsing one record - wrap HttpLoader streams in StreamClosingReader() which should close the connection even if stream not fully consumed - simplify no_except_close may help with ukwa/ukwa-pywb#53 - iframe: add allow fullscreen, autoplay - wombat: update to latest, filter out custom wombat props from getOwnPropertyNames - rules: add rule for vimeo * cdx formatting: fix output=text to return plain text / non-cdxj output * auto fetch fix: - update to latest wombat to fix auto-fetch in rewriting mode - fix /proxy-fetch/ endpoint for proxy mode recording, switch proxy-fetch to run in recording mode - don't use global to allow repeated checks * rewriter html check: peek 1024 bytes to determine if page is html instead of 128 * fix jinja2 dependency for py2 --- pywb/apps/frontendapp.py | 25 +++++++++++++--------- pywb/rewrite/content_rewriter.py | 2 +- pywb/rewrite/regex_rewriters.py | 2 +- pywb/rewrite/test/test_regex_rewriters.py | 3 +++ pywb/rules.yaml | 2 +- pywb/static/autoFetchWorker.js | 17 +++++++++------ pywb/static/wombat.js | 2 +- pywb/templates/frame_insert.html | 2 +- pywb/utils/io.py | 26 +++++++++++++++++------ pywb/utils/loaders.py | 4 ++-- pywb/version.py | 2 +- pywb/warcserver/index/cdxobject.py | 7 ++++-- pywb/warcserver/index/test/test_cdxops.py | 1 + pywb/warcserver/resource/pathresolvers.py | 2 +- requirements.txt | 2 +- tests/test_cdx_server_app.py | 13 ++++++++++++ tests/test_proxy.py | 8 +++++-- wombat | 2 +- 18 files changed, 84 insertions(+), 38 deletions(-) diff --git a/pywb/apps/frontendapp.py b/pywb/apps/frontendapp.py index 00ceb7d3d..eab2e67f9 100644 --- a/pywb/apps/frontendapp.py +++ b/pywb/apps/frontendapp.py @@ -82,6 +82,7 @@ def __init__(self, config_file=None, custom_config=None): self.proxy_prefix = None # the URL prefix to be used for the collection with proxy mode (e.g. /coll/id_/) self.proxy_coll = None # the name of the collection that has proxy mode enabled + self.proxy_record = False # indicate if proxy recording self.init_proxy(config) self.init_recorder(config.get('recorder')) @@ -627,17 +628,21 @@ def init_proxy(self, config): if proxy_coll in self.warcserver.list_fixed_routes(): raise Exception('Can not record into fixed collection') - proxy_coll += self.RECORD_ROUTE + proxy_route = proxy_coll + self.RECORD_ROUTE if not config.get('recorder'): config['recorder'] = 'live' + self.proxy_record = True + else: logging.info('Proxy enabled for collection "{0}"'.format(proxy_coll)) + self.proxy_record = False + proxy_route = proxy_coll if proxy_config.get('enable_content_rewrite', True): - self.proxy_prefix = '/{0}/bn_/'.format(proxy_coll) + self.proxy_prefix = '/{0}/bn_/'.format(proxy_route) else: - self.proxy_prefix = '/{0}/id_/'.format(proxy_coll) + self.proxy_prefix = '/{0}/id_/'.format(proxy_route) self.proxy_default_timestamp = proxy_config.get('default_timestamp') if self.proxy_default_timestamp: @@ -686,14 +691,14 @@ def proxy_fetch(self, env, url): return WbResponse.options_response(env) # ensure full URL - request_url = env['REQUEST_URI'] - # replace with /id_ so we do not get rewritten - url = request_url.replace('/proxy-fetch', '/id_') - # update WSGI environment object - env['REQUEST_URI'] = self.proxy_coll + url - env['PATH_INFO'] = env['PATH_INFO'].replace('/proxy-fetch', self.proxy_coll + '/id_') + url = env['REQUEST_URI'].split('/proxy-fetch/', 1)[-1] + + env['REQUEST_URI'] = self.proxy_prefix + url + env['PATH_INFO'] = self.proxy_prefix + env['PATH_INFO'].split('/proxy-fetch/', 1)[-1] + # make request using normal serve_content - response = self.serve_content(env, self.proxy_coll, url) + response = self.serve_content(env, self.proxy_coll, url, record=self.proxy_record) + # for WR if isinstance(response, WbResponse): response.add_access_control_headers(env=env) diff --git a/pywb/rewrite/content_rewriter.py b/pywb/rewrite/content_rewriter.py index b5fc0b75b..ce7d7c749 100644 --- a/pywb/rewrite/content_rewriter.py +++ b/pywb/rewrite/content_rewriter.py @@ -488,7 +488,7 @@ def _resolve_text_type(self, text_type): else: return text_type - buff = self.read_and_keep(128) + buff = self.read_and_keep(1024) # check if doesn't start with a tag, then likely not html if self.TAG_REGEX.match(buff): diff --git a/pywb/rewrite/regex_rewriters.py b/pywb/rewrite/regex_rewriters.py index a76df7e9f..0764d8dd8 100644 --- a/pywb/rewrite/regex_rewriters.py +++ b/pywb/rewrite/regex_rewriters.py @@ -113,7 +113,7 @@ def __init__(self): # rewriting 'this.' special properties access, not on new line (no ;) (r'(?>> _test_js_obj_proxy('return this.foo') 'return this.foo' +>>> _test_js_obj_proxy('{foo: bar, this: other}') +'{foo: bar, this: other}' + >>> _test_js_obj_proxy(r'this.$location = http://example.com/') 'this.$location = http://example.com/' diff --git a/pywb/rules.yaml b/pywb/rules.yaml index 13e3eb3c8..b189b460f 100644 --- a/pywb/rules.yaml +++ b/pywb/rules.yaml @@ -344,7 +344,7 @@ rules: - videoFileId - signature - - url_prefix: 'net,akamaized,gcs-vimeo)/' + - url_prefix: ['net,akamaized,gcs-vimeo)/', 'net,akamaized,vod)/'] fuzzy_lookup: match: '([/\d]+\.mp4)$' diff --git a/pywb/static/autoFetchWorker.js b/pywb/static/autoFetchWorker.js index f2ab2dd3f..bb0f63573 100644 --- a/pywb/static/autoFetchWorker.js +++ b/pywb/static/autoFetchWorker.js @@ -23,7 +23,7 @@ var config = { rwRe: null, defaultFetchOptions: { cache: 'force-cache', - mode: null + mode: 'cors' } }; @@ -53,7 +53,7 @@ if (!config.haveFetch) { xhr.onreadystatechange = function() { if (xhr.readyState === 4) { if (!config.havePromise) { - fetchDoneOrErrored(); + fetchDone(); } resolve(); } @@ -78,7 +78,7 @@ if (location.search.indexOf('init') !== -1) { config.prefix = init.prefix; config.mod = init.mod; config.prefixMod = init.prefix + init.mod; - config.rwRe = new RegExp(init.rwRe, 'g'); + config.rwRe = new RegExp(init.rwRe); config.relative = init.prefix.split(location.origin)[1]; config.schemeless = '/' + config.relative; })(); @@ -101,11 +101,16 @@ self.onmessage = function(event) { function noop() {} -function fetchDoneOrErrored() { +function fetchDone() { runningFetches -= 1; fetchFromQ(); } +function fetchErrored(err) { + console.warn("Fetch Failed: " + err); + fetchDone(); +} + /** * Fetches the supplied URL and increments the {@link runningFetches} variable * to represent an inflight request. @@ -130,8 +135,8 @@ function fetchURL(toBeFetched) { } fetch(url, options) - .then(fetchDoneOrErrored) - .catch(fetchDoneOrErrored); + .then(fetchDone) + .catch(fetchErrored); } function queueOrFetch(toBeFetched) { diff --git a/pywb/static/wombat.js b/pywb/static/wombat.js index 2c9989837..2ff8d6056 100644 --- a/pywb/static/wombat.js +++ b/pywb/static/wombat.js @@ -16,4 +16,4 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with pywb. If not, see . */ -(function(){function FuncMap(){this._map=[]}function ensureNumber(maybeNumber){try{switch(typeof maybeNumber){case"number":case"bigint":return maybeNumber;}var converted=Number(maybeNumber);return isNaN(converted)?null:converted}catch(e){}return null}function addToStringTagToClass(clazz,tag){typeof self.Symbol!=="undefined"&&typeof self.Symbol.toStringTag!=="undefined"&&Object.defineProperty(clazz.prototype,self.Symbol.toStringTag,{value:tag,enumerable:false})}function autobind(clazz){for(var prop,propValue,proto=clazz.__proto__||clazz.constructor.prototype||clazz.prototype,clazzProps=Object.getOwnPropertyNames(proto),len=clazzProps.length,i=0;i=0){var fnMapping=this._map.splice(idx,1);return fnMapping[0][1]}return null},FuncMap.prototype.map=function(param){for(var i=0;i0&&afw.preserveMedia(media)})},AutoFetcher.prototype.terminate=function(){this.worker.terminate()},AutoFetcher.prototype.justFetch=function(urls){this.worker.postMessage({type:"fetch-all",values:urls})},AutoFetcher.prototype.fetchAsPage=function(url,originalUrl,title){if(url){var headers={"X-Wombat-History-Page":originalUrl};if(title){var encodedTitle=encodeURIComponent(title.trim());title&&(headers["X-Wombat-History-Title"]=encodedTitle)}var fetchData={url:url,options:{headers:headers,cache:"no-store"}};this.justFetch([fetchData])}},AutoFetcher.prototype.postMessage=function(msg,deferred){if(deferred){var afWorker=this;return void Promise.resolve().then(function(){afWorker.worker.postMessage(msg)})}this.worker.postMessage(msg)},AutoFetcher.prototype.preserveSrcset=function(srcset,mod){this.postMessage({type:"values",srcset:{value:srcset,mod:mod,presplit:true}},true)},AutoFetcher.prototype.preserveDataSrcset=function(elem){this.postMessage({type:"values",srcset:{value:elem.dataset.srcset,mod:this.rwMod(elem),presplit:false}},true)},AutoFetcher.prototype.preserveMedia=function(media){this.postMessage({type:"values",media:media},true)},AutoFetcher.prototype.getSrcset=function(elem){return this.wombat.wb_getAttribute?this.wombat.wb_getAttribute.call(elem,"srcset"):elem.getAttribute("srcset")},AutoFetcher.prototype.rwMod=function(elem){switch(elem.tagName){case"SOURCE":return elem.parentElement&&elem.parentElement.tagName==="PICTURE"?"im_":"oe_";case"IMG":return"im_";}return"oe_"},AutoFetcher.prototype.extractFromLocalDoc=function(){var afw=this;Promise.resolve().then(function(){for(var msg={type:"values",context:{docBaseURI:document.baseURI}},media=[],i=0,sheets=document.styleSheets;i=0},Wombat.prototype.isString=function(arg){return arg!=null&&Object.getPrototypeOf(arg)===String.prototype},Wombat.prototype.isSavedSrcSrcset=function(elem){switch(elem.tagName){case"IMG":case"VIDEO":case"AUDIO":return true;case"SOURCE":if(!elem.parentElement)return false;switch(elem.parentElement.tagName){case"PICTURE":case"VIDEO":case"AUDIO":return true;default:return false;}default:return false;}},Wombat.prototype.isSavedDataSrcSrcset=function(elem){return!!(elem.dataset&&elem.dataset.srcset!=null)&&this.isSavedSrcSrcset(elem)},Wombat.prototype.isHostUrl=function(str){if(str.indexOf("www.")===0)return true;var matches=str.match(this.hostnamePortRe);return!!(matches&&matches[0].length<64)||(matches=str.match(this.ipPortRe),!!matches&&matches[0].length<64)},Wombat.prototype.isArgumentsObj=function(maybeArgumentsObj){if(!maybeArgumentsObj||typeof maybeArgumentsObj.toString!=="function")return false;try{return this.utilFns.objToString.call(maybeArgumentsObj)==="[object Arguments]"}catch(e){return false}},Wombat.prototype.deproxyArrayHandlingArgumentsObj=function(maybeArgumentsObj){if(!maybeArgumentsObj||maybeArgumentsObj instanceof NodeList||!maybeArgumentsObj.length)return maybeArgumentsObj;for(var args=this.isArgumentsObj(maybeArgumentsObj)?new Array(maybeArgumentsObj.length):maybeArgumentsObj,i=0;i=0)||scriptType.indexOf("text/template")>=0)},Wombat.prototype.skipWrapScriptTextBasedOnText=function(text){if(!text||text.indexOf(this.WB_ASSIGN_FUNC)>=0||text.indexOf("<")===0)return true;for(var override_props=["window","self","document","location","top","parent","frames","opener"],i=0;i=0)return false;return true},Wombat.prototype.nodeHasChildren=function(node){if(!node)return false;if(typeof node.hasChildNodes==="function")return node.hasChildNodes();var kids=node.children||node.childNodes;return!!kids&&kids.length>0},Wombat.prototype.rwModForElement=function(elem,attrName){if(!elem)return undefined;var mod="mp_";if(!(elem.tagName==="LINK"&&attrName==="href")){var maybeMod=this.tagToMod[elem.tagName];maybeMod!=null&&(mod=maybeMod[attrName])}else if(elem.rel){var relV=elem.rel.trim().toLowerCase(),asV=this.wb_getAttribute.call(elem,"as");if(asV&&this.linkTagMods.linkRelToAs[relV]!=null){var asMods=this.linkTagMods.linkRelToAs[relV];mod=asMods[asV.toLowerCase()]}else this.linkTagMods[relV]!=null&&(mod=this.linkTagMods[relV])}return mod},Wombat.prototype.removeWBOSRC=function(elem){elem.tagName!=="SCRIPT"||elem.__$removedWBOSRC$__||(elem.hasAttribute("__wb_orig_src")&&elem.removeAttribute("__wb_orig_src"),elem.__$removedWBOSRC$__=true)},Wombat.prototype.retrieveWBOSRC=function(elem){if(elem.tagName==="SCRIPT"&&!elem.__$removedWBOSRC$__){var maybeWBOSRC;return maybeWBOSRC=this.wb_getAttribute?this.wb_getAttribute.call(elem,"__wb_orig_src"):elem.getAttribute("__wb_orig_src"),maybeWBOSRC==null&&(elem.__$removedWBOSRC$__=true),maybeWBOSRC}return undefined},Wombat.prototype.wrapScriptTextJsProxy=function(scriptText){return"if (!self.__WB_pmw) { self.__WB_pmw = function(obj) { this.__WB_source = obj; return this; } }\n{\nlet window = _____WB$wombat$assign$function_____(\"window\");\nlet self = _____WB$wombat$assign$function_____(\"self\");\nlet document = _____WB$wombat$assign$function_____(\"document\");\nlet location = _____WB$wombat$assign$function_____(\"location\");\nlet top = _____WB$wombat$assign$function_____(\"top\");\nlet parent = _____WB$wombat$assign$function_____(\"parent\");\nlet frames = _____WB$wombat$assign$function_____(\"frames\");\nlet opener = _____WB$wombat$assign$function_____(\"opener\");\n"+scriptText.replace(this.DotPostMessageRe,".__WB_pmw(self.window)$1")+"\n\n}"},Wombat.prototype.watchElem=function(elem,func){if(!this.$wbwindow.MutationObserver)return false;var m=new this.$wbwindow.MutationObserver(function(records,observer){for(var r,i=0;i"},Wombat.prototype.getFinalUrl=function(useRel,mod,url){var prefix=useRel?this.wb_rel_prefix:this.wb_abs_prefix;return mod==null&&(mod=this.wb_info.mod),this.wb_info.is_live||(prefix+=this.wb_info.wombat_ts),prefix+=mod,prefix[prefix.length-1]!=="/"&&(prefix+="/"),prefix+url},Wombat.prototype.resolveRelUrl=function(url,doc){var docObj=doc||this.$wbwindow.document,parser=this.makeParser(docObj.baseURI,docObj),hash=parser.href.lastIndexOf("#"),href=hash>=0?parser.href.substring(0,hash):parser.href,lastslash=href.lastIndexOf("/");return parser.href=lastslash>=0&&lastslash!==href.length-1?href.substring(0,lastslash+1)+url:href+url,parser.href},Wombat.prototype.extractOriginalURL=function(rewrittenUrl){if(!rewrittenUrl)return"";if(this.wb_is_proxy)return rewrittenUrl;var rwURLString=rewrittenUrl.toString(),url=rwURLString;if(this.startsWithOneOf(url,this.IGNORE_PREFIXES))return url;var start;start=this.startsWith(url,this.wb_abs_prefix)?this.wb_abs_prefix.length:this.wb_rel_prefix&&this.startsWith(url,this.wb_rel_prefix)?this.wb_rel_prefix.length:this.wb_rel_prefix?1:0;var index=url.indexOf("/http",start);return index<0&&(index=url.indexOf("///",start)),index<0&&(index=url.indexOf("/blob:",start)),index>=0?url=url.substr(index+1):(index=url.indexOf(this.wb_replay_prefix),index>=0&&(url=url.substr(index+this.wb_replay_prefix.length)),url.length>4&&url.charAt(2)==="_"&&url.charAt(3)==="/"&&(url=url.substr(4)),url!==rwURLString&&!this.startsWithOneOf(url,this.VALID_PREFIXES)&&!this.startsWith(url,"blob:")&&(url=this.wb_orig_scheme+url)),rwURLString.charAt(0)==="/"&&rwURLString.charAt(1)!=="/"&&this.startsWith(url,this.wb_orig_origin)&&(url=url.substr(this.wb_orig_origin.length)),this.startsWith(url,this.REL_PREFIX)?this.wb_info.wombat_scheme+":"+url:url},Wombat.prototype.makeParser=function(maybeRewrittenURL,doc){var originalURL=this.extractOriginalURL(maybeRewrittenURL),docElem=doc;return doc||(this.$wbwindow.location.href==="about:blank"&&this.$wbwindow.opener?docElem=this.$wbwindow.opener.document:docElem=this.$wbwindow.document),this._makeURLParser(originalURL,docElem)},Wombat.prototype._makeURLParser=function(url,docElem){try{return new this.$wbwindow.URL(url,docElem.baseURI)}catch(e){}var p=docElem.createElement("a");return p._no_rewrite=true,p.href=url,p},Wombat.prototype.defProp=function(obj,prop,setFunc,getFunc,enumerable){var existingDescriptor=Object.getOwnPropertyDescriptor(obj,prop);if(existingDescriptor&&!existingDescriptor.configurable)return false;if(!getFunc)return false;var descriptor={configurable:true,enumerable:enumerable||false,get:getFunc};setFunc&&(descriptor.set=setFunc);try{return Object.defineProperty(obj,prop,descriptor),true}catch(e){return console.warn("Failed to redefine property %s",prop,e.message),false}},Wombat.prototype.defGetterProp=function(obj,prop,getFunc,enumerable){var existingDescriptor=Object.getOwnPropertyDescriptor(obj,prop);if(existingDescriptor&&!existingDescriptor.configurable)return false;if(!getFunc)return false;try{return Object.defineProperty(obj,prop,{configurable:true,enumerable:enumerable||false,get:getFunc}),true}catch(e){return console.warn("Failed to redefine property %s",prop,e.message),false}},Wombat.prototype.getOrigGetter=function(obj,prop){var orig_getter;if(obj.__lookupGetter__&&(orig_getter=obj.__lookupGetter__(prop)),!orig_getter&&Object.getOwnPropertyDescriptor){var props=Object.getOwnPropertyDescriptor(obj,prop);props&&(orig_getter=props.get)}return orig_getter},Wombat.prototype.getOrigSetter=function(obj,prop){var orig_setter;if(obj.__lookupSetter__&&(orig_setter=obj.__lookupSetter__(prop)),!orig_setter&&Object.getOwnPropertyDescriptor){var props=Object.getOwnPropertyDescriptor(obj,prop);props&&(orig_setter=props.set)}return orig_setter},Wombat.prototype.getAllOwnProps=function(obj){for(var ownProps=[],props=Object.getOwnPropertyNames(obj),i=0;i "+final_href),actualLocation.href=final_href}}},Wombat.prototype.checkLocationChange=function(wombatLoc,isTop){var locType=typeof wombatLoc,actual_location=isTop?this.$wbwindow.__WB_replay_top.location:this.$wbwindow.location;locType==="string"?this.updateLocation(wombatLoc,actual_location.href,actual_location):locType==="object"&&this.updateLocation(wombatLoc.href,wombatLoc._orig_href,actual_location)},Wombat.prototype.checkAllLocations=function(){return!this.wb_wombat_updating&&void(this.wb_wombat_updating=true,this.checkLocationChange(this.$wbwindow.WB_wombat_location,false),this.$wbwindow.WB_wombat_location!=this.$wbwindow.__WB_replay_top.WB_wombat_location&&this.checkLocationChange(this.$wbwindow.__WB_replay_top.WB_wombat_location,true),this.wb_wombat_updating=false)},Wombat.prototype.proxyToObj=function(source){if(source)try{var proxyRealObj=source.__WBProxyRealObj__;if(proxyRealObj)return proxyRealObj}catch(e){}return source},Wombat.prototype.objToProxy=function(obj){if(obj)try{var maybeWbProxy=obj._WB_wombat_obj_proxy;if(maybeWbProxy)return maybeWbProxy}catch(e){}return obj},Wombat.prototype.defaultProxyGet=function(obj,prop,ownProps,fnCache){switch(prop){case"__WBProxyRealObj__":return obj;case"location":case"WB_wombat_location":return obj.WB_wombat_location;case"_WB_wombat_obj_proxy":return obj._WB_wombat_obj_proxy;case"__WB_pmw":case"WB_wombat_eval":case this.WB_ASSIGN_FUNC:case this.WB_CHECK_THIS_FUNC:return obj[prop];case"constructor":if(obj.constructor===Window)return obj.constructor;}var retVal=obj[prop],type=typeof retVal;if(type==="function"&&ownProps.indexOf(prop)!==-1){switch(prop){case"requestAnimationFrame":case"cancelAnimationFrame":{if(!this.isNativeFunction(retVal))return retVal;break}}var cachedFN=fnCache[prop];return cachedFN&&cachedFN.original===retVal||(cachedFN={original:retVal,boundFn:retVal.bind(obj)},fnCache[prop]=cachedFN),cachedFN.boundFn}return type==="object"&&retVal&&retVal._WB_wombat_obj_proxy?(retVal instanceof Window&&this.initNewWindowWombat(retVal),retVal._WB_wombat_obj_proxy):retVal},Wombat.prototype.setLoc=function(loc,originalURL){var parser=this.makeParser(originalURL,loc.ownerDocument);loc._orig_href=originalURL,loc._parser=parser;var href=parser.href;loc._hash=parser.hash,loc._href=href,loc._host=parser.host,loc._hostname=parser.hostname,loc._origin=parser.origin?parser.host?parser.origin:"null":parser.protocol+"//"+parser.hostname+(parser.port?":"+parser.port:""),loc._pathname=parser.pathname,loc._port=parser.port,loc._protocol=parser.protocol,loc._search=parser.search,Object.defineProperty||(loc.href=href,loc.hash=parser.hash,loc.host=loc._host,loc.hostname=loc._hostname,loc.origin=loc._origin,loc.pathname=loc._pathname,loc.port=loc._port,loc.protocol=loc._protocol,loc.search=loc._search)},Wombat.prototype.makeGetLocProp=function(prop,origGetter){var wombat=this;return function newGetLocProp(){if(this._no_rewrite)return origGetter.call(this,prop);var curr_orig_href=origGetter.call(this,"href");return prop==="href"?wombat.extractOriginalURL(curr_orig_href):(this._orig_href!==curr_orig_href&&wombat.setLoc(this,curr_orig_href),this["_"+prop])}},Wombat.prototype.makeSetLocProp=function(prop,origSetter,origGetter){var wombat=this;return function newSetLocProp(value){if(this._no_rewrite)return origSetter.call(this,prop,value);if(this["_"+prop]!==value){if(this["_"+prop]=value,!this._parser){var href=origGetter.call(this);this._parser=wombat.makeParser(href,this.ownerDocument)}var rel=false;prop==="href"&&typeof value==="string"&&value&&(value[0]==="."?value=wombat.resolveRelUrl(value,this.ownerDocument):value[0]==="/"&&(value.length<=1||value[1]!=="/")&&(rel=true,value=WB_wombat_location.origin+value));try{this._parser[prop]=value}catch(e){console.log("Error setting "+prop+" = "+value)}prop==="hash"?(value=this._parser[prop],origSetter.call(this,"hash",value)):(rel=rel||value===this._parser.pathname,value=wombat.rewriteUrl(this._parser.href,rel),origSetter.call(this,"href",value))}}},Wombat.prototype.styleReplacer=function(match,n1,n2,n3,offset,string){return n1+this.rewriteUrl(n2)+n3},Wombat.prototype.domConstructorErrorChecker=function(thisObj,what,args,numRequiredArgs){var erorMsg,needArgs=typeof numRequiredArgs==="number"?numRequiredArgs:1;if(thisObj instanceof Window?erorMsg="Failed to construct '"+what+"': Please use the 'new' operator, this DOM object constructor cannot be called as a function.":args&&args.length=0)return url;if(url.indexOf(this.wb_rel_prefix)===0&&url.indexOf("http")>1){var scheme_sep=url.indexOf(":/");return scheme_sep>0&&url[scheme_sep+2]!=="/"?url.substring(0,scheme_sep+2)+"/"+url.substring(scheme_sep+2):url}return this.getFinalUrl(true,mod,this.wb_orig_origin+url)}url.charAt(0)==="."&&(url=this.resolveRelUrl(url,doc));var prefix=this.startsWithOneOf(url.toLowerCase(),this.VALID_PREFIXES);if(prefix){var orig_host=this.$wbwindow.__WB_replay_top.location.host,orig_protocol=this.$wbwindow.__WB_replay_top.location.protocol,prefix_host=prefix+orig_host+"/";if(this.startsWith(url,prefix_host)){if(this.startsWith(url,this.wb_replay_prefix))return url;var curr_scheme=orig_protocol+"//",path=url.substring(prefix_host.length),rebuild=false;return path.indexOf(this.wb_rel_prefix)<0&&url.indexOf("/static/")<0&&(path=this.getFinalUrl(true,mod,WB_wombat_location.origin+"/"+path),rebuild=true),prefix!==curr_scheme&&prefix!==this.REL_PREFIX&&(rebuild=true),rebuild&&(url=useRel?"":curr_scheme+orig_host,path&&path[0]!=="/"&&(url+="/"),url+=path),url}return this.getFinalUrl(useRel,mod,url)}return prefix=this.startsWithOneOf(url,this.BAD_PREFIXES),prefix?this.getFinalUrl(useRel,mod,this.extractOriginalURL(url)):this.isHostUrl(url)&&!this.startsWith(url,originalLoc.host+"/")?this.getFinalUrl(useRel,mod,this.wb_orig_scheme+url):url},Wombat.prototype.rewriteUrl=function(url,useRel,mod,doc){var rewritten=this.rewriteUrl_(url,useRel,mod,doc);return this.debug_rw&&(url===rewritten?console.log("NOT REWRITTEN "+url):console.log("REWRITE: "+url+" -> "+rewritten)),rewritten},Wombat.prototype.performAttributeRewrite=function(elem,name,value,absUrlOnly){switch(name){case"innerHTML":case"outerHTML":return this.rewriteHtml(value);case"filter":return this.rewriteInlineStyle(value);case"style":return this.rewriteStyle(value);case"srcset":return this.rewriteSrcset(value,elem);}if(absUrlOnly&&!this.startsWithOneOf(value,this.VALID_PREFIXES))return value;var mod=this.rwModForElement(elem,name);return this.wbUseAFWorker&&this.WBAutoFetchWorker&&this.isSavedDataSrcSrcset(elem)&&this.WBAutoFetchWorker.preserveDataSrcset(elem),this.rewriteUrl(value,false,mod,elem.ownerDocument)},Wombat.prototype.rewriteAttr=function(elem,name,absUrlOnly){var changed=false;if(!elem||!elem.getAttribute||elem._no_rewrite||elem["_"+name])return changed;var value=this.wb_getAttribute.call(elem,name);if(!value||this.startsWith(value,"javascript:"))return changed;var new_value=this.performAttributeRewrite(elem,name,value,absUrlOnly);return new_value!==value&&(this.removeWBOSRC(elem),this.wb_setAttribute.call(elem,name,new_value),changed=true),changed},Wombat.prototype.noExceptRewriteStyle=function(style){try{return this.rewriteStyle(style)}catch(e){return style}},Wombat.prototype.rewriteStyle=function(style){if(!style)return style;var value=style;return typeof style==="object"&&(value=style.toString()),typeof value==="string"?value.replace(this.STYLE_REGEX,this.styleReplacer).replace(this.IMPORT_REGEX,this.styleReplacer).replace(this.no_wombatRe,""):value},Wombat.prototype.rewriteSrcset=function(value,elem){if(!value)return"";for(var split=value.split(this.srcsetRe),values=[],mod=this.rwModForElement(elem,"srcset"),i=0;i=0){var JS="javascript:";new_value="javascript:window.parent._wb_wombat.initNewWindowWombat(window);"+value.substr(11)}return new_value||(new_value=this.rewriteUrl(value,false,this.rwModForElement(elem,attrName))),new_value!==value&&(this.wb_setAttribute.call(elem,attrName,new_value),true)},Wombat.prototype.rewriteScript=function(elem){if(elem.hasAttribute("src")||!elem.textContent||!this.$wbwindow.Proxy)return this.rewriteAttr(elem,"src");if(this.skipWrapScriptBasedOnType(elem.type))return false;var text=elem.textContent.trim();return!this.skipWrapScriptTextBasedOnText(text)&&(elem.textContent=this.wrapScriptTextJsProxy(text),true)},Wombat.prototype.rewriteSVGElem=function(elem){var changed=this.rewriteAttr(elem,"filter");return changed=this.rewriteAttr(elem,"style")||changed,changed=this.rewriteAttr(elem,"xlink:href")||changed,changed=this.rewriteAttr(elem,"href")||changed,changed=this.rewriteAttr(elem,"src")||changed,changed},Wombat.prototype.rewriteElem=function(elem){var changed=false;if(!elem)return changed;if(elem instanceof SVGElement)changed=this.rewriteSVGElem(elem);else switch(elem.tagName){case"META":var maybeCSP=this.wb_getAttribute.call(elem,"http-equiv");maybeCSP&&maybeCSP.toLowerCase()==="content-security-policy"&&(this.wb_setAttribute.call(elem,"http-equiv","_"+maybeCSP),changed=true);break;case"STYLE":var new_content=this.rewriteStyle(elem.textContent);elem.textContent!==new_content&&(elem.textContent=new_content,changed=true,this.wbUseAFWorker&&this.WBAutoFetchWorker&&elem.sheet!=null&&this.WBAutoFetchWorker.deferredSheetExtraction(elem.sheet));break;case"LINK":changed=this.rewriteAttr(elem,"href"),this.wbUseAFWorker&&elem.rel==="stylesheet"&&this._addEventListener(elem,"load",this.utilFns.wbSheetMediaQChecker);break;case"IMG":changed=this.rewriteAttr(elem,"src"),changed=this.rewriteAttr(elem,"srcset")||changed,changed=this.rewriteAttr(elem,"style")||changed,this.wbUseAFWorker&&this.WBAutoFetchWorker&&elem.dataset.srcset&&this.WBAutoFetchWorker.preserveDataSrcset(elem);break;case"OBJECT":changed=this.rewriteAttr(elem,"data",true),changed=this.rewriteAttr(elem,"style")||changed;break;case"FORM":changed=this.rewriteAttr(elem,"poster"),changed=this.rewriteAttr(elem,"action")||changed,changed=this.rewriteAttr(elem,"style")||changed;break;case"IFRAME":case"FRAME":changed=this.rewriteFrameSrc(elem,"src"),changed=this.rewriteAttr(elem,"style")||changed;break;case"SCRIPT":changed=this.rewriteScript(elem);break;default:{changed=this.rewriteAttr(elem,"src"),changed=this.rewriteAttr(elem,"srcset")||changed,changed=this.rewriteAttr(elem,"href")||changed,changed=this.rewriteAttr(elem,"style")||changed,changed=this.rewriteAttr(elem,"poster")||changed;break}}return elem.hasAttribute&&elem.removeAttribute&&(elem.hasAttribute("crossorigin")&&(elem.removeAttribute("crossorigin"),changed=true),elem.hasAttribute("integrity")&&(elem.removeAttribute("integrity"),changed=true)),changed},Wombat.prototype.recurseRewriteElem=function(curr){if(!this.nodeHasChildren(curr))return false;for(var changed=false,rewriteQ=[curr.children||curr.childNodes];rewriteQ.length>0;)for(var child,children=rewriteQ.shift(),i=0;i"+rwString+"","text/html");if(!inner_doc||!this.nodeHasChildren(inner_doc.head)||!inner_doc.head.children[0].content)return rwString;var template=inner_doc.head.children[0];if(template._no_rewrite=true,this.recurseRewriteElem(template.content)){var new_html=template.innerHTML;if(checkEndTag){var first_elem=template.content.children&&template.content.children[0];if(first_elem){var end_tag="";this.endsWith(new_html,end_tag)&&!this.endsWith(rwString.toLowerCase(),end_tag)&&(new_html=new_html.substring(0,new_html.length-end_tag.length))}else if(rwString[0]!=="<"||rwString[rwString.length-1]!==">")return this.write_buff+=rwString,undefined}return new_html}return rwString},Wombat.prototype.rewriteHtmlFull=function(string,checkEndTag){var inner_doc=new DOMParser().parseFromString(string,"text/html");if(!inner_doc)return string;for(var changed=false,i=0;i=0)inner_doc.documentElement._no_rewrite=true,new_html=this.reconstructDocType(inner_doc.doctype)+inner_doc.documentElement.outerHTML;else{inner_doc.head._no_rewrite=true,inner_doc.body._no_rewrite=true;var headHasKids=this.nodeHasChildren(inner_doc.head),bodyHasKids=this.nodeHasChildren(inner_doc.body);if(new_html=(headHasKids?inner_doc.head.outerHTML:"")+(bodyHasKids?inner_doc.body.outerHTML:""),checkEndTag)if(inner_doc.all.length>3){var end_tag="";this.endsWith(new_html,end_tag)&&!this.endsWith(string.toLowerCase(),end_tag)&&(new_html=new_html.substring(0,new_html.length-end_tag.length))}else if(string[0]!=="<"||string[string.length-1]!==">")return void(this.write_buff+=string);new_html=this.reconstructDocType(inner_doc.doctype)+new_html}return new_html}return string},Wombat.prototype.rewriteInlineStyle=function(orig){var decoded;try{decoded=decodeURIComponent(orig)}catch(e){decoded=orig}if(decoded!==orig){var parts=this.rewriteStyle(decoded).split(",",2);return parts[0]+","+encodeURIComponent(parts[1])}return this.rewriteStyle(orig)},Wombat.prototype.rewriteCookie=function(cookie){var wombat=this,rwCookie=cookie.replace(this.wb_abs_prefix,"").replace(this.wb_rel_prefix,"");return rwCookie=rwCookie.replace(this.cookie_domain_regex,function(m,m1){var message={domain:m1,cookie:rwCookie,wb_type:"cookie"};return wombat.sendTopMessage(message,true),wombat.$wbwindow.location.hostname.indexOf(".")>=0&&!wombat.IP_RX.test(wombat.$wbwindow.location.hostname)?"Domain=."+wombat.$wbwindow.location.hostname:""}).replace(this.cookie_path_regex,function(m,m1){var rewritten=wombat.rewriteUrl(m1);return rewritten.indexOf(wombat.wb_curr_host)===0&&(rewritten=rewritten.substring(wombat.wb_curr_host.length)),"Path="+rewritten}),wombat.$wbwindow.location.protocol!=="https:"&&(rwCookie=rwCookie.replace("secure","")),rwCookie.replace(",|",",")},Wombat.prototype.rewriteWorker=function(workerUrl){if(!workerUrl)return workerUrl;var isBlob=workerUrl.indexOf("blob:")===0,isJS=workerUrl.indexOf("javascript:")===0;if(!isBlob&&!isJS){if(!this.startsWithOneOf(workerUrl,this.VALID_PREFIXES)&&!this.startsWith(workerUrl,"/")&&!this.startsWithOneOf(workerUrl,this.BAD_PREFIXES)){var rurl=this.resolveRelUrl(workerUrl,this.$wbwindow.document);return this.rewriteUrl(rurl,false,"wkr_",this.$wbwindow.document)}return this.rewriteUrl(workerUrl,false,"wkr_",this.$wbwindow.document)}var workerCode=isJS?workerUrl.replace("javascript:",""):null;if(isBlob){var x=new XMLHttpRequest;this.utilFns.XHRopen.call(x,"GET",workerUrl,false),x.send(),workerCode=x.responseText.replace(this.workerBlobRe,"").replace(this.rmCheckThisInjectRe,"this")}if(this.wb_info.static_prefix||this.wb_info.ww_rw_script){var originalURL=this.$wbwindow.document.baseURI,ww_rw=this.wb_info.ww_rw_script||this.wb_info.static_prefix+"wombatWorkers.js",rw="(function() { self.importScripts('"+ww_rw+"'); new WBWombat({'prefix': '"+this.wb_abs_prefix+"', 'prefixMod': '"+this.wb_abs_prefix+"wkrf_/', 'originalURL': '"+originalURL+"'}); })();";workerCode=rw+workerCode}var blob=new Blob([workerCode],{type:"application/javascript"});return URL.createObjectURL(blob)},Wombat.prototype.rewriteTextNodeFn=function(fnThis,originalFn,argsObj){var args,deproxiedThis=this.proxyToObj(fnThis);if(argsObj.length>0&&deproxiedThis.parentElement&&deproxiedThis.parentElement.tagName==="STYLE"){args=new Array(argsObj.length);var dataIndex=argsObj.length-1;dataIndex===2?(args[0]=argsObj[0],args[1]=argsObj[1]):dataIndex===1&&(args[0]=argsObj[0]),args[dataIndex]=this.rewriteStyle(argsObj[dataIndex])}else args=argsObj;return originalFn.__WB_orig_apply?originalFn.__WB_orig_apply(deproxiedThis,args):originalFn.apply(deproxiedThis,args)},Wombat.prototype.rewriteDocWriteWriteln=function(fnThis,originalFn,argsObj){var string,thisObj=this.proxyToObj(fnThis),argLen=argsObj.length;if(argLen===0)return originalFn.call(thisObj);string=argLen===1?argsObj[0]:Array.prototype.join.call(argsObj,"");var new_buff=this.rewriteHtml(string,true),res=originalFn.call(thisObj,new_buff);return this.initNewWindowWombat(thisObj.defaultView),res},Wombat.prototype.rewriteChildNodeFn=function(fnThis,originalFn,argsObj){var thisObj=this.proxyToObj(fnThis);if(argsObj.length===0)return originalFn.call(thisObj);var newArgs=this.rewriteElementsInArguments(argsObj);return originalFn.__WB_orig_apply?originalFn.__WB_orig_apply(thisObj,newArgs):originalFn.apply(thisObj,newArgs)},Wombat.prototype.rewriteInsertAdjHTMLOrElemArgs=function(fnThis,originalFn,position,textOrElem,rwHTML){var fnThisObj=this.proxyToObj(fnThis);return fnThisObj._no_rewrite?originalFn.call(fnThisObj,position,textOrElem):rwHTML?originalFn.call(fnThisObj,position,this.rewriteHtml(textOrElem)):(this.rewriteElemComplete(textOrElem),originalFn.call(fnThisObj,position,textOrElem))},Wombat.prototype.rewriteSetTimeoutInterval=function(fnThis,originalFn,argsObj){var rw=this.isString(argsObj[0]),args=rw?new Array(argsObj.length):argsObj;if(rw){args[0]=this.$wbwindow.Proxy?this.wrapScriptTextJsProxy(argsObj[0]):argsObj[0].replace(/\blocation\b/g,"WB_wombat_$&");for(var i=1;i0&&cssStyleValueOverride(this.$wbwindow.CSSStyleValue,"parse"),this.$wbwindow.CSSStyleValue.parseAll&&this.$wbwindow.CSSStyleValue.parseAll.toString().indexOf("[native code]")>0&&cssStyleValueOverride(this.$wbwindow.CSSStyleValue,"parseAll")}if(this.$wbwindow.CSSKeywordValue&&this.$wbwindow.CSSKeywordValue.prototype){var oCSSKV=this.$wbwindow.CSSKeywordValue;this.$wbwindow.CSSKeywordValue=function(CSSKeywordValue_){return function CSSKeywordValue(cssValue){return wombat.domConstructorErrorChecker(this,"CSSKeywordValue",arguments),new CSSKeywordValue_(wombat.rewriteStyle(cssValue))}}(this.$wbwindow.CSSKeywordValue),this.$wbwindow.CSSKeywordValue.prototype=oCSSKV.prototype,Object.defineProperty(this.$wbwindow.CSSKeywordValue.prototype,"constructor",{value:this.$wbwindow.CSSKeywordValue}),addToStringTagToClass(this.$wbwindow.CSSKeywordValue,"CSSKeywordValue")}if(this.$wbwindow.StylePropertyMap&&this.$wbwindow.StylePropertyMap.prototype){var originalSet=this.$wbwindow.StylePropertyMap.prototype.set;this.$wbwindow.StylePropertyMap.prototype.set=function set(){if(arguments.length<=1)return originalSet.__WB_orig_apply?originalSet.__WB_orig_apply(this,arguments):originalSet.apply(this,arguments);var newArgs=new Array(arguments.length);newArgs[0]=arguments[0];for(var i=1;i=0||name==="href"?wombat.extractOriginalURL(value):value},svgImgProto.getAttributeNS=function getAttributeNS(ns,name){var value=orig_getAttrNS.call(this,ns,name);return name.indexOf("xlink:href")>=0||name==="href"?wombat.extractOriginalURL(value):value},svgImgProto.setAttribute=function setAttribute(name,value){var rwValue=value;return(name.indexOf("xlink:href")>=0||name==="href")&&(rwValue=wombat.rewriteUrl(value)),orig_setAttr.call(this,name,rwValue)},svgImgProto.setAttributeNS=function setAttributeNS(ns,name,value){var rwValue=value;return(name.indexOf("xlink:href")>=0||name==="href")&&(rwValue=wombat.rewriteUrl(value)),orig_setAttrNS.call(this,ns,name,rwValue)}}},Wombat.prototype.initCreateElementNSFix=function(){if(this.$wbwindow.document.createElementNS&&this.$wbwindow.Document.prototype.createElementNS){var orig_createElementNS=this.$wbwindow.document.createElementNS,wombat=this,createElementNS=function createElementNS(namespaceURI,qualifiedName){return orig_createElementNS.call(wombat.proxyToObj(this),wombat.extractOriginalURL(namespaceURI),qualifiedName)};this.$wbwindow.Document.prototype.createElementNS=createElementNS,this.$wbwindow.document.createElementNS=createElementNS}},Wombat.prototype.initInsertAdjacentElementHTMLOverrides=function(){var Element=this.$wbwindow.Element;if(Element&&Element.prototype){var elementProto=Element.prototype,rewriteFn=this.rewriteInsertAdjHTMLOrElemArgs;if(elementProto.insertAdjacentHTML){var origInsertAdjacentHTML=elementProto.insertAdjacentHTML;elementProto.insertAdjacentHTML=function insertAdjacentHTML(position,text){return rewriteFn(this,origInsertAdjacentHTML,position,text,true)}}if(elementProto.insertAdjacentElement){var origIAdjElem=elementProto.insertAdjacentElement;elementProto.insertAdjacentElement=function insertAdjacentElement(position,element){return rewriteFn(this,origIAdjElem,position,element,false)}}}},Wombat.prototype.initDomOverride=function(){var Node=this.$wbwindow.Node;if(Node&&Node.prototype){var rewriteFn=this.rewriteNodeFuncArgs;if(Node.prototype.appendChild){var originalAppendChild=Node.prototype.appendChild;Node.prototype.appendChild=function appendChild(newNode,oldNode){return rewriteFn(this,originalAppendChild,newNode,oldNode)}}if(Node.prototype.insertBefore){var originalInsertBefore=Node.prototype.insertBefore;Node.prototype.insertBefore=function insertBefore(newNode,oldNode){return rewriteFn(this,originalInsertBefore,newNode,oldNode)}}if(Node.prototype.replaceChild){var originalReplaceChild=Node.prototype.replaceChild;Node.prototype.replaceChild=function replaceChild(newNode,oldNode){return rewriteFn(this,originalReplaceChild,newNode,oldNode)}}this.overridePropToProxy(Node.prototype,"ownerDocument"),this.overridePropToProxy(this.$wbwindow.HTMLHtmlElement.prototype,"parentNode"),this.overridePropToProxy(this.$wbwindow.Event.prototype,"target")}this.$wbwindow.Element&&this.$wbwindow.Element.prototype&&(this.overrideParentNodeAppendPrepend(this.$wbwindow.Element),this.overrideChildNodeInterface(this.$wbwindow.Element,false)),this.$wbwindow.DocumentFragment&&this.$wbwindow.DocumentFragment.prototype&&this.overrideParentNodeAppendPrepend(this.$wbwindow.DocumentFragment)},Wombat.prototype.initDocOverrides=function($document){if(Object.defineProperty){this.overrideReferrer($document),this.defGetterProp($document,"origin",function origin(){return this.WB_wombat_location.origin}),this.defGetterProp(this.$wbwindow,"origin",function origin(){return this.WB_wombat_location.origin});var wombat=this,domain_setter=function domain(val){var loc=this.WB_wombat_location;loc&&wombat.endsWith(loc.hostname,val)&&(this.__wb_domain=val)},domain_getter=function domain(){return this.__wb_domain||this.WB_wombat_location.hostname};this.defProp($document,"domain",domain_setter,domain_getter)}},Wombat.prototype.initDocWriteOpenCloseOverride=function(){if(this.$wbwindow.DOMParser){var DocumentProto=this.$wbwindow.Document.prototype,$wbDocument=this.$wbwindow.document,docWriteWritelnRWFn=this.rewriteDocWriteWriteln,orig_doc_write=$wbDocument.write,new_write=function write(){return docWriteWritelnRWFn(this,orig_doc_write,arguments)};$wbDocument.write=new_write,DocumentProto.write=new_write;var orig_doc_writeln=$wbDocument.writeln,new_writeln=function writeln(){return docWriteWritelnRWFn(this,orig_doc_writeln,arguments)};$wbDocument.writeln=new_writeln,DocumentProto.writeln=new_writeln;var wombat=this,orig_doc_open=$wbDocument.open,new_open=function open(){var res,thisObj=wombat.proxyToObj(this);if(arguments.length===3){var rwUrl=wombat.rewriteUrl(arguments[0],false,"mp_");res=orig_doc_open.call(thisObj,rwUrl,arguments[1],arguments[2]),wombat.initNewWindowWombat(res,rwUrl)}else res=orig_doc_open.call(thisObj),wombat.initNewWindowWombat(thisObj.defaultView);return res};$wbDocument.open=new_open,DocumentProto.open=new_open;var originalClose=$wbDocument.close,newClose=function close(){var thisObj=wombat.proxyToObj(this);return wombat.initNewWindowWombat(thisObj.defaultView),originalClose.__WB_orig_apply?originalClose.__WB_orig_apply(thisObj,arguments):originalClose.apply(thisObj,arguments)};$wbDocument.close=newClose,DocumentProto.close=newClose;var oBodyGetter=this.getOrigGetter(DocumentProto,"body"),oBodySetter=this.getOrigSetter(DocumentProto,"body");oBodyGetter&&oBodySetter&&this.defProp(DocumentProto,"body",function body(newBody){return newBody&&(newBody instanceof HTMLBodyElement||newBody instanceof HTMLFrameSetElement)&&wombat.rewriteElemComplete(newBody),oBodySetter.call(wombat.proxyToObj(this),newBody)},oBodyGetter)}},Wombat.prototype.initIframeWombat=function(iframe){var win;win=iframe._get_contentWindow?iframe._get_contentWindow.call(iframe):iframe.contentWindow;try{if(!win||win===this.$wbwindow||win._skip_wombat||win._wb_wombat)return}catch(e){return}var src=this.wb_getAttribute.call(iframe,"src");this.initNewWindowWombat(win,src)},Wombat.prototype.initNewWindowWombat=function(win,src){if(win&&!win._wb_wombat)if(!src||src===""||this.startsWith(src,"about:")||src.indexOf("javascript:")>=0){var wombat=new Wombat(win,this.wb_info);win._wb_wombat=wombat.wombatInit()}else this.initProtoPmOrigin(win),this.initPostMessageOverride(win),this.initMessageEventOverride(win),this.initCheckThisFunc(win)},Wombat.prototype.initTimeoutIntervalOverrides=function(){var rewriteFn=this.rewriteSetTimeoutInterval;if(this.$wbwindow.setTimeout&&!this.$wbwindow.setTimeout.__$wbpatched$__){var originalSetTimeout=this.$wbwindow.setTimeout;this.$wbwindow.setTimeout=function setTimeout(){return rewriteFn(this,originalSetTimeout,arguments)},this.$wbwindow.setTimeout.__$wbpatched$__=true}if(this.$wbwindow.setInterval&&!this.$wbwindow.setInterval.__$wbpatched$__){var originalSetInterval=this.$wbwindow.setInterval;this.$wbwindow.setInterval=function setInterval(){return rewriteFn(this,originalSetInterval,arguments)},this.$wbwindow.setInterval.__$wbpatched$__=true}},Wombat.prototype.initWorkerOverrides=function(){var wombat=this;if(this.$wbwindow.Worker&&!this.$wbwindow.Worker._wb_worker_overriden){var orig_worker=this.$wbwindow.Worker;this.$wbwindow.Worker=function(Worker_){return function Worker(url,options){return wombat.domConstructorErrorChecker(this,"Worker",arguments),new Worker_(wombat.rewriteWorker(url),options)}}(orig_worker),this.$wbwindow.Worker.prototype=orig_worker.prototype,Object.defineProperty(this.$wbwindow.Worker.prototype,"constructor",{value:this.$wbwindow.Worker}),this.$wbwindow.Worker._wb_worker_overriden=true}if(this.$wbwindow.SharedWorker&&!this.$wbwindow.SharedWorker.__wb_sharedWorker_overriden){var oSharedWorker=this.$wbwindow.SharedWorker;this.$wbwindow.SharedWorker=function(SharedWorker_){return function SharedWorker(url,options){return wombat.domConstructorErrorChecker(this,"SharedWorker",arguments),new SharedWorker_(wombat.rewriteWorker(url),options)}}(oSharedWorker),this.$wbwindow.SharedWorker.prototype=oSharedWorker.prototype,Object.defineProperty(this.$wbwindow.SharedWorker.prototype,"constructor",{value:this.$wbwindow.SharedWorker}),this.$wbwindow.SharedWorker.__wb_sharedWorker_overriden=true}if(this.$wbwindow.ServiceWorkerContainer&&this.$wbwindow.ServiceWorkerContainer.prototype&&this.$wbwindow.ServiceWorkerContainer.prototype.register){var orig_register=this.$wbwindow.ServiceWorkerContainer.prototype.register;this.$wbwindow.ServiceWorkerContainer.prototype.register=function register(scriptURL,options){var newScriptURL=new URL(scriptURL,wombat.$wbwindow.document.baseURI).href,mod=wombat.getPageUnderModifier();return options&&options.scope?options.scope=wombat.rewriteUrl(options.scope,false,mod):options={scope:wombat.rewriteUrl("/",false,mod)},orig_register.call(this,wombat.rewriteUrl(newScriptURL,false,"sw_"),options)}}if(this.$wbwindow.Worklet&&this.$wbwindow.Worklet.prototype&&this.$wbwindow.Worklet.prototype.addModule&&!this.$wbwindow.Worklet.__wb_workerlet_overriden){var oAddModule=this.$wbwindow.Worklet.prototype.addModule;this.$wbwindow.Worklet.prototype.addModule=function addModule(moduleURL,options){var rwModuleURL=wombat.rewriteUrl(moduleURL,false,"js_");return oAddModule.call(this,rwModuleURL,options)},this.$wbwindow.Worklet.__wb_workerlet_overriden=true}},Wombat.prototype.initLocOverride=function(loc,oSetter,oGetter){if(Object.defineProperty)for(var prop,i=0;i=0){var fnMapping=this._map.splice(idx,1);return fnMapping[0][1]}return null},FuncMap.prototype.map=function(param){for(var i=0;i0&&afw.preserveMedia(media)})},AutoFetcher.prototype.terminate=function(){this.worker.terminate()},AutoFetcher.prototype.justFetch=function(urls){this.worker.postMessage({type:"fetch-all",values:urls})},AutoFetcher.prototype.fetchAsPage=function(url,originalUrl,title){if(url){var headers={"X-Wombat-History-Page":originalUrl};if(title){var encodedTitle=encodeURIComponent(title.trim());title&&(headers["X-Wombat-History-Title"]=encodedTitle)}var fetchData={url:url,options:{headers:headers,cache:"no-store"}};this.justFetch([fetchData])}},AutoFetcher.prototype.postMessage=function(msg,deferred){if(deferred){var afWorker=this;return void Promise.resolve().then(function(){afWorker.worker.postMessage(msg)})}this.worker.postMessage(msg)},AutoFetcher.prototype.preserveSrcset=function(srcset,mod){this.postMessage({type:"values",srcset:{value:srcset,mod:mod,presplit:true}},true)},AutoFetcher.prototype.preserveDataSrcset=function(elem){this.postMessage({type:"values",srcset:{value:elem.dataset.srcset,mod:this.rwMod(elem),presplit:false}},true)},AutoFetcher.prototype.preserveMedia=function(media){this.postMessage({type:"values",media:media},true)},AutoFetcher.prototype.getSrcset=function(elem){return this.wombat.wb_getAttribute?this.wombat.wb_getAttribute.call(elem,"srcset"):elem.getAttribute("srcset")},AutoFetcher.prototype.rwMod=function(elem){switch(elem.tagName){case"SOURCE":return elem.parentElement&&elem.parentElement.tagName==="PICTURE"?"im_":"oe_";case"IMG":return"im_";}return"oe_"},AutoFetcher.prototype.extractFromLocalDoc=function(){var afw=this;Promise.resolve().then(function(){for(var msg={type:"values",context:{docBaseURI:document.baseURI}},media=[],i=0,sheets=document.styleSheets;i=0},Wombat.prototype.isString=function(arg){return arg!=null&&Object.getPrototypeOf(arg)===String.prototype},Wombat.prototype.isSavedSrcSrcset=function(elem){switch(elem.tagName){case"IMG":case"VIDEO":case"AUDIO":return true;case"SOURCE":if(!elem.parentElement)return false;switch(elem.parentElement.tagName){case"PICTURE":case"VIDEO":case"AUDIO":return true;default:return false;}default:return false;}},Wombat.prototype.isSavedDataSrcSrcset=function(elem){return!!(elem.dataset&&elem.dataset.srcset!=null)&&this.isSavedSrcSrcset(elem)},Wombat.prototype.isHostUrl=function(str){if(str.indexOf("www.")===0)return true;var matches=str.match(this.hostnamePortRe);return!!(matches&&matches[0].length<64)||(matches=str.match(this.ipPortRe),!!matches&&matches[0].length<64)},Wombat.prototype.isArgumentsObj=function(maybeArgumentsObj){if(!maybeArgumentsObj||typeof maybeArgumentsObj.toString!=="function")return false;try{return this.utilFns.objToString.call(maybeArgumentsObj)==="[object Arguments]"}catch(e){return false}},Wombat.prototype.deproxyArrayHandlingArgumentsObj=function(maybeArgumentsObj){if(!maybeArgumentsObj||maybeArgumentsObj instanceof NodeList||!maybeArgumentsObj.length)return maybeArgumentsObj;for(var args=this.isArgumentsObj(maybeArgumentsObj)?new Array(maybeArgumentsObj.length):maybeArgumentsObj,i=0;i=0)||scriptType.indexOf("text/template")>=0)},Wombat.prototype.skipWrapScriptTextBasedOnText=function(text){if(!text||text.indexOf(this.WB_ASSIGN_FUNC)>=0||text.indexOf("<")===0)return true;for(var override_props=["window","self","document","location","top","parent","frames","opener"],i=0;i=0)return false;return true},Wombat.prototype.nodeHasChildren=function(node){if(!node)return false;if(typeof node.hasChildNodes==="function")return node.hasChildNodes();var kids=node.children||node.childNodes;return!!kids&&kids.length>0},Wombat.prototype.rwModForElement=function(elem,attrName){if(!elem)return undefined;var mod="mp_";if(!(elem.tagName==="LINK"&&attrName==="href")){var maybeMod=this.tagToMod[elem.tagName];maybeMod!=null&&(mod=maybeMod[attrName])}else if(elem.rel){var relV=elem.rel.trim().toLowerCase(),asV=this.wb_getAttribute.call(elem,"as");if(asV&&this.linkTagMods.linkRelToAs[relV]!=null){var asMods=this.linkTagMods.linkRelToAs[relV];mod=asMods[asV.toLowerCase()]}else this.linkTagMods[relV]!=null&&(mod=this.linkTagMods[relV])}return mod},Wombat.prototype.removeWBOSRC=function(elem){elem.tagName!=="SCRIPT"||elem.__$removedWBOSRC$__||(elem.hasAttribute("__wb_orig_src")&&elem.removeAttribute("__wb_orig_src"),elem.__$removedWBOSRC$__=true)},Wombat.prototype.retrieveWBOSRC=function(elem){if(elem.tagName==="SCRIPT"&&!elem.__$removedWBOSRC$__){var maybeWBOSRC;return maybeWBOSRC=this.wb_getAttribute?this.wb_getAttribute.call(elem,"__wb_orig_src"):elem.getAttribute("__wb_orig_src"),maybeWBOSRC==null&&(elem.__$removedWBOSRC$__=true),maybeWBOSRC}return undefined},Wombat.prototype.wrapScriptTextJsProxy=function(scriptText){return"if (!self.__WB_pmw) { self.__WB_pmw = function(obj) { this.__WB_source = obj; return this; } }\n{\nlet window = _____WB$wombat$assign$function_____(\"window\");\nlet self = _____WB$wombat$assign$function_____(\"self\");\nlet document = _____WB$wombat$assign$function_____(\"document\");\nlet location = _____WB$wombat$assign$function_____(\"location\");\nlet top = _____WB$wombat$assign$function_____(\"top\");\nlet parent = _____WB$wombat$assign$function_____(\"parent\");\nlet frames = _____WB$wombat$assign$function_____(\"frames\");\nlet opener = _____WB$wombat$assign$function_____(\"opener\");\n"+scriptText.replace(this.DotPostMessageRe,".__WB_pmw(self.window)$1")+"\n\n}"},Wombat.prototype.watchElem=function(elem,func){if(!this.$wbwindow.MutationObserver)return false;var m=new this.$wbwindow.MutationObserver(function(records,observer){for(var r,i=0;i"},Wombat.prototype.getFinalUrl=function(useRel,mod,url){var prefix=useRel?this.wb_rel_prefix:this.wb_abs_prefix;return mod==null&&(mod=this.wb_info.mod),this.wb_info.is_live||(prefix+=this.wb_info.wombat_ts),prefix+=mod,prefix[prefix.length-1]!=="/"&&(prefix+="/"),prefix+url},Wombat.prototype.resolveRelUrl=function(url,doc){var docObj=doc||this.$wbwindow.document,parser=this.makeParser(docObj.baseURI,docObj),hash=parser.href.lastIndexOf("#"),href=hash>=0?parser.href.substring(0,hash):parser.href,lastslash=href.lastIndexOf("/");return parser.href=lastslash>=0&&lastslash!==href.length-1?href.substring(0,lastslash+1)+url:href+url,parser.href},Wombat.prototype.extractOriginalURL=function(rewrittenUrl){if(!rewrittenUrl)return"";if(this.wb_is_proxy)return rewrittenUrl;var rwURLString=rewrittenUrl.toString(),url=rwURLString;if(this.startsWithOneOf(url,this.IGNORE_PREFIXES))return url;var start;start=this.startsWith(url,this.wb_abs_prefix)?this.wb_abs_prefix.length:this.wb_rel_prefix&&this.startsWith(url,this.wb_rel_prefix)?this.wb_rel_prefix.length:this.wb_rel_prefix?1:0;var index=url.indexOf("/http",start);return index<0&&(index=url.indexOf("///",start)),index<0&&(index=url.indexOf("/blob:",start)),index>=0?url=url.substr(index+1):(index=url.indexOf(this.wb_replay_prefix),index>=0&&(url=url.substr(index+this.wb_replay_prefix.length)),url.length>4&&url.charAt(2)==="_"&&url.charAt(3)==="/"&&(url=url.substr(4)),url!==rwURLString&&!this.startsWithOneOf(url,this.VALID_PREFIXES)&&!this.startsWith(url,"blob:")&&(url=this.wb_orig_scheme+url)),rwURLString.charAt(0)==="/"&&rwURLString.charAt(1)!=="/"&&this.startsWith(url,this.wb_orig_origin)&&(url=url.substr(this.wb_orig_origin.length)),this.startsWith(url,this.REL_PREFIX)?this.wb_info.wombat_scheme+":"+url:url},Wombat.prototype.makeParser=function(maybeRewrittenURL,doc){var originalURL=this.extractOriginalURL(maybeRewrittenURL),docElem=doc;return doc||(this.$wbwindow.location.href==="about:blank"&&this.$wbwindow.opener?docElem=this.$wbwindow.opener.document:docElem=this.$wbwindow.document),this._makeURLParser(originalURL,docElem)},Wombat.prototype._makeURLParser=function(url,docElem){try{return new this.$wbwindow.URL(url,docElem.baseURI)}catch(e){}var p=docElem.createElement("a");return p._no_rewrite=true,p.href=url,p},Wombat.prototype.defProp=function(obj,prop,setFunc,getFunc,enumerable){var existingDescriptor=Object.getOwnPropertyDescriptor(obj,prop);if(existingDescriptor&&!existingDescriptor.configurable)return false;if(!getFunc)return false;var descriptor={configurable:true,enumerable:enumerable||false,get:getFunc};setFunc&&(descriptor.set=setFunc);try{return Object.defineProperty(obj,prop,descriptor),true}catch(e){return console.warn("Failed to redefine property %s",prop,e.message),false}},Wombat.prototype.defGetterProp=function(obj,prop,getFunc,enumerable){var existingDescriptor=Object.getOwnPropertyDescriptor(obj,prop);if(existingDescriptor&&!existingDescriptor.configurable)return false;if(!getFunc)return false;try{return Object.defineProperty(obj,prop,{configurable:true,enumerable:enumerable||false,get:getFunc}),true}catch(e){return console.warn("Failed to redefine property %s",prop,e.message),false}},Wombat.prototype.getOrigGetter=function(obj,prop){var orig_getter;if(obj.__lookupGetter__&&(orig_getter=obj.__lookupGetter__(prop)),!orig_getter&&Object.getOwnPropertyDescriptor){var props=Object.getOwnPropertyDescriptor(obj,prop);props&&(orig_getter=props.get)}return orig_getter},Wombat.prototype.getOrigSetter=function(obj,prop){var orig_setter;if(obj.__lookupSetter__&&(orig_setter=obj.__lookupSetter__(prop)),!orig_setter&&Object.getOwnPropertyDescriptor){var props=Object.getOwnPropertyDescriptor(obj,prop);props&&(orig_setter=props.set)}return orig_setter},Wombat.prototype.getAllOwnProps=function(obj){for(var ownProps=[],props=Object.getOwnPropertyNames(obj),i=0;i "+final_href),actualLocation.href=final_href}}},Wombat.prototype.checkLocationChange=function(wombatLoc,isTop){var locType=typeof wombatLoc,actual_location=isTop?this.$wbwindow.__WB_replay_top.location:this.$wbwindow.location;locType==="string"?this.updateLocation(wombatLoc,actual_location.href,actual_location):locType==="object"&&this.updateLocation(wombatLoc.href,wombatLoc._orig_href,actual_location)},Wombat.prototype.checkAllLocations=function(){return!this.wb_wombat_updating&&void(this.wb_wombat_updating=true,this.checkLocationChange(this.$wbwindow.WB_wombat_location,false),this.$wbwindow.WB_wombat_location!=this.$wbwindow.__WB_replay_top.WB_wombat_location&&this.checkLocationChange(this.$wbwindow.__WB_replay_top.WB_wombat_location,true),this.wb_wombat_updating=false)},Wombat.prototype.proxyToObj=function(source){if(source)try{var proxyRealObj=source.__WBProxyRealObj__;if(proxyRealObj)return proxyRealObj}catch(e){}return source},Wombat.prototype.objToProxy=function(obj){if(obj)try{var maybeWbProxy=obj._WB_wombat_obj_proxy;if(maybeWbProxy)return maybeWbProxy}catch(e){}return obj},Wombat.prototype.defaultProxyGet=function(obj,prop,ownProps,fnCache){switch(prop){case"__WBProxyRealObj__":return obj;case"location":case"WB_wombat_location":return obj.WB_wombat_location;case"_WB_wombat_obj_proxy":return obj._WB_wombat_obj_proxy;case"__WB_pmw":case"WB_wombat_eval":case this.WB_ASSIGN_FUNC:case this.WB_CHECK_THIS_FUNC:return obj[prop];case"constructor":if(obj.constructor===Window)return obj.constructor;}var retVal=obj[prop],type=typeof retVal;if(type==="function"&&ownProps.indexOf(prop)!==-1){switch(prop){case"requestAnimationFrame":case"cancelAnimationFrame":{if(!this.isNativeFunction(retVal))return retVal;break}}var cachedFN=fnCache[prop];return cachedFN&&cachedFN.original===retVal||(cachedFN={original:retVal,boundFn:retVal.bind(obj)},fnCache[prop]=cachedFN),cachedFN.boundFn}return type==="object"&&retVal&&retVal._WB_wombat_obj_proxy?(retVal instanceof Window&&this.initNewWindowWombat(retVal),retVal._WB_wombat_obj_proxy):retVal},Wombat.prototype.setLoc=function(loc,originalURL){var parser=this.makeParser(originalURL,loc.ownerDocument);loc._orig_href=originalURL,loc._parser=parser;var href=parser.href;loc._hash=parser.hash,loc._href=href,loc._host=parser.host,loc._hostname=parser.hostname,loc._origin=parser.origin?parser.host?parser.origin:"null":parser.protocol+"//"+parser.hostname+(parser.port?":"+parser.port:""),loc._pathname=parser.pathname,loc._port=parser.port,loc._protocol=parser.protocol,loc._search=parser.search,Object.defineProperty||(loc.href=href,loc.hash=parser.hash,loc.host=loc._host,loc.hostname=loc._hostname,loc.origin=loc._origin,loc.pathname=loc._pathname,loc.port=loc._port,loc.protocol=loc._protocol,loc.search=loc._search)},Wombat.prototype.makeGetLocProp=function(prop,origGetter){var wombat=this;return function newGetLocProp(){if(this._no_rewrite)return origGetter.call(this,prop);var curr_orig_href=origGetter.call(this,"href");return prop==="href"?wombat.extractOriginalURL(curr_orig_href):(this._orig_href!==curr_orig_href&&wombat.setLoc(this,curr_orig_href),this["_"+prop])}},Wombat.prototype.makeSetLocProp=function(prop,origSetter,origGetter){var wombat=this;return function newSetLocProp(value){if(this._no_rewrite)return origSetter.call(this,prop,value);if(this["_"+prop]!==value){if(this["_"+prop]=value,!this._parser){var href=origGetter.call(this);this._parser=wombat.makeParser(href,this.ownerDocument)}var rel=false;prop==="href"&&typeof value==="string"&&value&&(value[0]==="."?value=wombat.resolveRelUrl(value,this.ownerDocument):value[0]==="/"&&(value.length<=1||value[1]!=="/")&&(rel=true,value=WB_wombat_location.origin+value));try{this._parser[prop]=value}catch(e){console.log("Error setting "+prop+" = "+value)}prop==="hash"?(value=this._parser[prop],origSetter.call(this,"hash",value)):(rel=rel||value===this._parser.pathname,value=wombat.rewriteUrl(this._parser.href,rel),origSetter.call(this,"href",value))}}},Wombat.prototype.styleReplacer=function(match,n1,n2,n3,offset,string){return n1+this.rewriteUrl(n2)+n3},Wombat.prototype.domConstructorErrorChecker=function(thisObj,what,args,numRequiredArgs){var erorMsg,needArgs=typeof numRequiredArgs==="number"?numRequiredArgs:1;if(thisObj instanceof Window?erorMsg="Failed to construct '"+what+"': Please use the 'new' operator, this DOM object constructor cannot be called as a function.":args&&args.length=0)return url;if(url.indexOf(this.wb_rel_prefix)===0&&url.indexOf("http")>1){var scheme_sep=url.indexOf(":/");return scheme_sep>0&&url[scheme_sep+2]!=="/"?url.substring(0,scheme_sep+2)+"/"+url.substring(scheme_sep+2):url}return this.getFinalUrl(true,mod,this.wb_orig_origin+url)}url.charAt(0)==="."&&(url=this.resolveRelUrl(url,doc));var prefix=this.startsWithOneOf(url.toLowerCase(),this.VALID_PREFIXES);if(prefix){var orig_host=this.$wbwindow.__WB_replay_top.location.host,orig_protocol=this.$wbwindow.__WB_replay_top.location.protocol,prefix_host=prefix+orig_host+"/";if(this.startsWith(url,prefix_host)){if(this.startsWith(url,this.wb_replay_prefix))return url;var curr_scheme=orig_protocol+"//",path=url.substring(prefix_host.length),rebuild=false;return path.indexOf(this.wb_rel_prefix)<0&&url.indexOf("/static/")<0&&(path=this.getFinalUrl(true,mod,WB_wombat_location.origin+"/"+path),rebuild=true),prefix!==curr_scheme&&prefix!==this.REL_PREFIX&&(rebuild=true),rebuild&&(url=useRel?"":curr_scheme+orig_host,path&&path[0]!=="/"&&(url+="/"),url+=path),url}return this.getFinalUrl(useRel,mod,url)}return prefix=this.startsWithOneOf(url,this.BAD_PREFIXES),prefix?this.getFinalUrl(useRel,mod,this.extractOriginalURL(url)):this.isHostUrl(url)&&!this.startsWith(url,originalLoc.host+"/")?this.getFinalUrl(useRel,mod,this.wb_orig_scheme+url):url},Wombat.prototype.rewriteUrl=function(url,useRel,mod,doc){var rewritten=this.rewriteUrl_(url,useRel,mod,doc);return this.debug_rw&&(url===rewritten?console.log("NOT REWRITTEN "+url):console.log("REWRITE: "+url+" -> "+rewritten)),rewritten},Wombat.prototype.performAttributeRewrite=function(elem,name,value,absUrlOnly){switch(name){case"innerHTML":case"outerHTML":return this.rewriteHtml(value);case"filter":return this.rewriteInlineStyle(value);case"style":return this.rewriteStyle(value);case"srcset":return this.rewriteSrcset(value,elem);}if(absUrlOnly&&!this.startsWithOneOf(value,this.VALID_PREFIXES))return value;var mod=this.rwModForElement(elem,name);return this.wbUseAFWorker&&this.WBAutoFetchWorker&&this.isSavedDataSrcSrcset(elem)&&this.WBAutoFetchWorker.preserveDataSrcset(elem),this.rewriteUrl(value,false,mod,elem.ownerDocument)},Wombat.prototype.rewriteAttr=function(elem,name,absUrlOnly){var changed=false;if(!elem||!elem.getAttribute||elem._no_rewrite||elem["_"+name])return changed;var value=this.wb_getAttribute.call(elem,name);if(!value||this.startsWith(value,"javascript:"))return changed;var new_value=this.performAttributeRewrite(elem,name,value,absUrlOnly);return new_value!==value&&(this.removeWBOSRC(elem),this.wb_setAttribute.call(elem,name,new_value),changed=true),changed},Wombat.prototype.noExceptRewriteStyle=function(style){try{return this.rewriteStyle(style)}catch(e){return style}},Wombat.prototype.rewriteStyle=function(style){if(!style)return style;var value=style;return typeof style==="object"&&(value=style.toString()),typeof value==="string"?value.replace(this.STYLE_REGEX,this.styleReplacer).replace(this.IMPORT_REGEX,this.styleReplacer).replace(this.no_wombatRe,""):value},Wombat.prototype.rewriteSrcset=function(value,elem){if(!value)return"";for(var split=value.split(this.srcsetRe),values=[],mod=this.rwModForElement(elem,"srcset"),i=0;i=0){var JS="javascript:";new_value="javascript:window.parent._wb_wombat.initNewWindowWombat(window);"+value.substr(11)}return new_value||(new_value=this.rewriteUrl(value,false,this.rwModForElement(elem,attrName))),new_value!==value&&(this.wb_setAttribute.call(elem,attrName,new_value),true)},Wombat.prototype.rewriteScript=function(elem){if(elem.hasAttribute("src")||!elem.textContent||!this.$wbwindow.Proxy)return this.rewriteAttr(elem,"src");if(this.skipWrapScriptBasedOnType(elem.type))return false;var text=elem.textContent.trim();return!this.skipWrapScriptTextBasedOnText(text)&&(elem.textContent=this.wrapScriptTextJsProxy(text),true)},Wombat.prototype.rewriteSVGElem=function(elem){var changed=this.rewriteAttr(elem,"filter");return changed=this.rewriteAttr(elem,"style")||changed,changed=this.rewriteAttr(elem,"xlink:href")||changed,changed=this.rewriteAttr(elem,"href")||changed,changed=this.rewriteAttr(elem,"src")||changed,changed},Wombat.prototype.rewriteElem=function(elem){var changed=false;if(!elem)return changed;if(elem instanceof SVGElement)changed=this.rewriteSVGElem(elem);else switch(elem.tagName){case"META":var maybeCSP=this.wb_getAttribute.call(elem,"http-equiv");maybeCSP&&maybeCSP.toLowerCase()==="content-security-policy"&&(this.wb_setAttribute.call(elem,"http-equiv","_"+maybeCSP),changed=true);break;case"STYLE":var new_content=this.rewriteStyle(elem.textContent);elem.textContent!==new_content&&(elem.textContent=new_content,changed=true,this.wbUseAFWorker&&this.WBAutoFetchWorker&&elem.sheet!=null&&this.WBAutoFetchWorker.deferredSheetExtraction(elem.sheet));break;case"LINK":changed=this.rewriteAttr(elem,"href"),this.wbUseAFWorker&&elem.rel==="stylesheet"&&this._addEventListener(elem,"load",this.utilFns.wbSheetMediaQChecker);break;case"IMG":changed=this.rewriteAttr(elem,"src"),changed=this.rewriteAttr(elem,"srcset")||changed,changed=this.rewriteAttr(elem,"style")||changed,this.wbUseAFWorker&&this.WBAutoFetchWorker&&elem.dataset.srcset&&this.WBAutoFetchWorker.preserveDataSrcset(elem);break;case"OBJECT":changed=this.rewriteAttr(elem,"data",true),changed=this.rewriteAttr(elem,"style")||changed;break;case"FORM":changed=this.rewriteAttr(elem,"poster"),changed=this.rewriteAttr(elem,"action")||changed,changed=this.rewriteAttr(elem,"style")||changed;break;case"IFRAME":case"FRAME":changed=this.rewriteFrameSrc(elem,"src"),changed=this.rewriteAttr(elem,"style")||changed;break;case"SCRIPT":changed=this.rewriteScript(elem);break;default:{changed=this.rewriteAttr(elem,"src"),changed=this.rewriteAttr(elem,"srcset")||changed,changed=this.rewriteAttr(elem,"href")||changed,changed=this.rewriteAttr(elem,"style")||changed,changed=this.rewriteAttr(elem,"poster")||changed;break}}return elem.hasAttribute&&elem.removeAttribute&&(elem.hasAttribute("crossorigin")&&(elem.removeAttribute("crossorigin"),changed=true),elem.hasAttribute("integrity")&&(elem.removeAttribute("integrity"),changed=true)),changed},Wombat.prototype.recurseRewriteElem=function(curr){if(!this.nodeHasChildren(curr))return false;for(var changed=false,rewriteQ=[curr.children||curr.childNodes];rewriteQ.length>0;)for(var child,children=rewriteQ.shift(),i=0;i"+rwString+"","text/html");if(!inner_doc||!this.nodeHasChildren(inner_doc.head)||!inner_doc.head.children[0].content)return rwString;var template=inner_doc.head.children[0];if(template._no_rewrite=true,this.recurseRewriteElem(template.content)){var new_html=template.innerHTML;if(checkEndTag){var first_elem=template.content.children&&template.content.children[0];if(first_elem){var end_tag="";this.endsWith(new_html,end_tag)&&!this.endsWith(rwString.toLowerCase(),end_tag)&&(new_html=new_html.substring(0,new_html.length-end_tag.length))}else if(rwString[0]!=="<"||rwString[rwString.length-1]!==">")return this.write_buff+=rwString,undefined}return new_html}return rwString},Wombat.prototype.rewriteHtmlFull=function(string,checkEndTag){var inner_doc=new DOMParser().parseFromString(string,"text/html");if(!inner_doc)return string;for(var changed=false,i=0;i=0)inner_doc.documentElement._no_rewrite=true,new_html=this.reconstructDocType(inner_doc.doctype)+inner_doc.documentElement.outerHTML;else{inner_doc.head._no_rewrite=true,inner_doc.body._no_rewrite=true;var headHasKids=this.nodeHasChildren(inner_doc.head),bodyHasKids=this.nodeHasChildren(inner_doc.body);if(new_html=(headHasKids?inner_doc.head.outerHTML:"")+(bodyHasKids?inner_doc.body.outerHTML:""),checkEndTag)if(inner_doc.all.length>3){var end_tag="";this.endsWith(new_html,end_tag)&&!this.endsWith(string.toLowerCase(),end_tag)&&(new_html=new_html.substring(0,new_html.length-end_tag.length))}else if(string[0]!=="<"||string[string.length-1]!==">")return void(this.write_buff+=string);new_html=this.reconstructDocType(inner_doc.doctype)+new_html}return new_html}return string},Wombat.prototype.rewriteInlineStyle=function(orig){var decoded;try{decoded=decodeURIComponent(orig)}catch(e){decoded=orig}if(decoded!==orig){var parts=this.rewriteStyle(decoded).split(",",2);return parts[0]+","+encodeURIComponent(parts[1])}return this.rewriteStyle(orig)},Wombat.prototype.rewriteCookie=function(cookie){var wombat=this,rwCookie=cookie.replace(this.wb_abs_prefix,"").replace(this.wb_rel_prefix,"");return rwCookie=rwCookie.replace(this.cookie_domain_regex,function(m,m1){var message={domain:m1,cookie:rwCookie,wb_type:"cookie"};return wombat.sendTopMessage(message,true),wombat.$wbwindow.location.hostname.indexOf(".")>=0&&!wombat.IP_RX.test(wombat.$wbwindow.location.hostname)?"Domain=."+wombat.$wbwindow.location.hostname:""}).replace(this.cookie_path_regex,function(m,m1){var rewritten=wombat.rewriteUrl(m1);return rewritten.indexOf(wombat.wb_curr_host)===0&&(rewritten=rewritten.substring(wombat.wb_curr_host.length)),"Path="+rewritten}),wombat.$wbwindow.location.protocol!=="https:"&&(rwCookie=rwCookie.replace("secure","")),rwCookie.replace(",|",",")},Wombat.prototype.rewriteWorker=function(workerUrl){if(!workerUrl)return workerUrl;var isBlob=workerUrl.indexOf("blob:")===0,isJS=workerUrl.indexOf("javascript:")===0;if(!isBlob&&!isJS){if(!this.startsWithOneOf(workerUrl,this.VALID_PREFIXES)&&!this.startsWith(workerUrl,"/")&&!this.startsWithOneOf(workerUrl,this.BAD_PREFIXES)){var rurl=this.resolveRelUrl(workerUrl,this.$wbwindow.document);return this.rewriteUrl(rurl,false,"wkr_",this.$wbwindow.document)}return this.rewriteUrl(workerUrl,false,"wkr_",this.$wbwindow.document)}var workerCode=isJS?workerUrl.replace("javascript:",""):null;if(isBlob){var x=new XMLHttpRequest;this.utilFns.XHRopen.call(x,"GET",workerUrl,false),x.send(),workerCode=x.responseText.replace(this.workerBlobRe,"").replace(this.rmCheckThisInjectRe,"this")}if(this.wb_info.static_prefix||this.wb_info.ww_rw_script){var originalURL=this.$wbwindow.document.baseURI,ww_rw=this.wb_info.ww_rw_script||this.wb_info.static_prefix+"wombatWorkers.js",rw="(function() { self.importScripts('"+ww_rw+"'); new WBWombat({'prefix': '"+this.wb_abs_prefix+"', 'prefixMod': '"+this.wb_abs_prefix+"wkrf_/', 'originalURL': '"+originalURL+"'}); })();";workerCode=rw+workerCode}var blob=new Blob([workerCode],{type:"application/javascript"});return URL.createObjectURL(blob)},Wombat.prototype.rewriteTextNodeFn=function(fnThis,originalFn,argsObj){var args,deproxiedThis=this.proxyToObj(fnThis);if(argsObj.length>0&&deproxiedThis.parentElement&&deproxiedThis.parentElement.tagName==="STYLE"){args=new Array(argsObj.length);var dataIndex=argsObj.length-1;dataIndex===2?(args[0]=argsObj[0],args[1]=argsObj[1]):dataIndex===1&&(args[0]=argsObj[0]),args[dataIndex]=this.rewriteStyle(argsObj[dataIndex])}else args=argsObj;return originalFn.__WB_orig_apply?originalFn.__WB_orig_apply(deproxiedThis,args):originalFn.apply(deproxiedThis,args)},Wombat.prototype.rewriteDocWriteWriteln=function(fnThis,originalFn,argsObj){var string,thisObj=this.proxyToObj(fnThis),argLen=argsObj.length;if(argLen===0)return originalFn.call(thisObj);string=argLen===1?argsObj[0]:Array.prototype.join.call(argsObj,"");var new_buff=this.rewriteHtml(string,true),res=originalFn.call(thisObj,new_buff);return this.initNewWindowWombat(thisObj.defaultView),res},Wombat.prototype.rewriteChildNodeFn=function(fnThis,originalFn,argsObj){var thisObj=this.proxyToObj(fnThis);if(argsObj.length===0)return originalFn.call(thisObj);var newArgs=this.rewriteElementsInArguments(argsObj);return originalFn.__WB_orig_apply?originalFn.__WB_orig_apply(thisObj,newArgs):originalFn.apply(thisObj,newArgs)},Wombat.prototype.rewriteInsertAdjHTMLOrElemArgs=function(fnThis,originalFn,position,textOrElem,rwHTML){var fnThisObj=this.proxyToObj(fnThis);return fnThisObj._no_rewrite?originalFn.call(fnThisObj,position,textOrElem):rwHTML?originalFn.call(fnThisObj,position,this.rewriteHtml(textOrElem)):(this.rewriteElemComplete(textOrElem),originalFn.call(fnThisObj,position,textOrElem))},Wombat.prototype.rewriteSetTimeoutInterval=function(fnThis,originalFn,argsObj){var rw=this.isString(argsObj[0]),args=rw?new Array(argsObj.length):argsObj;if(rw){args[0]=this.$wbwindow.Proxy?this.wrapScriptTextJsProxy(argsObj[0]):argsObj[0].replace(/\blocation\b/g,"WB_wombat_$&");for(var i=1;i0&&cssStyleValueOverride(this.$wbwindow.CSSStyleValue,"parse"),this.$wbwindow.CSSStyleValue.parseAll&&this.$wbwindow.CSSStyleValue.parseAll.toString().indexOf("[native code]")>0&&cssStyleValueOverride(this.$wbwindow.CSSStyleValue,"parseAll")}if(this.$wbwindow.CSSKeywordValue&&this.$wbwindow.CSSKeywordValue.prototype){var oCSSKV=this.$wbwindow.CSSKeywordValue;this.$wbwindow.CSSKeywordValue=function(CSSKeywordValue_){return function CSSKeywordValue(cssValue){return wombat.domConstructorErrorChecker(this,"CSSKeywordValue",arguments),new CSSKeywordValue_(wombat.rewriteStyle(cssValue))}}(this.$wbwindow.CSSKeywordValue),this.$wbwindow.CSSKeywordValue.prototype=oCSSKV.prototype,Object.defineProperty(this.$wbwindow.CSSKeywordValue.prototype,"constructor",{value:this.$wbwindow.CSSKeywordValue}),addToStringTagToClass(this.$wbwindow.CSSKeywordValue,"CSSKeywordValue")}if(this.$wbwindow.StylePropertyMap&&this.$wbwindow.StylePropertyMap.prototype){var originalSet=this.$wbwindow.StylePropertyMap.prototype.set;this.$wbwindow.StylePropertyMap.prototype.set=function set(){if(arguments.length<=1)return originalSet.__WB_orig_apply?originalSet.__WB_orig_apply(this,arguments):originalSet.apply(this,arguments);var newArgs=new Array(arguments.length);newArgs[0]=arguments[0];for(var i=1;i=0||name==="href"?wombat.extractOriginalURL(value):value},svgImgProto.getAttributeNS=function getAttributeNS(ns,name){var value=orig_getAttrNS.call(this,ns,name);return name.indexOf("xlink:href")>=0||name==="href"?wombat.extractOriginalURL(value):value},svgImgProto.setAttribute=function setAttribute(name,value){var rwValue=value;return(name.indexOf("xlink:href")>=0||name==="href")&&(rwValue=wombat.rewriteUrl(value)),orig_setAttr.call(this,name,rwValue)},svgImgProto.setAttributeNS=function setAttributeNS(ns,name,value){var rwValue=value;return(name.indexOf("xlink:href")>=0||name==="href")&&(rwValue=wombat.rewriteUrl(value)),orig_setAttrNS.call(this,ns,name,rwValue)}}},Wombat.prototype.initCreateElementNSFix=function(){if(this.$wbwindow.document.createElementNS&&this.$wbwindow.Document.prototype.createElementNS){var orig_createElementNS=this.$wbwindow.document.createElementNS,wombat=this,createElementNS=function createElementNS(namespaceURI,qualifiedName){return orig_createElementNS.call(wombat.proxyToObj(this),wombat.extractOriginalURL(namespaceURI),qualifiedName)};this.$wbwindow.Document.prototype.createElementNS=createElementNS,this.$wbwindow.document.createElementNS=createElementNS}},Wombat.prototype.initInsertAdjacentElementHTMLOverrides=function(){var Element=this.$wbwindow.Element;if(Element&&Element.prototype){var elementProto=Element.prototype,rewriteFn=this.rewriteInsertAdjHTMLOrElemArgs;if(elementProto.insertAdjacentHTML){var origInsertAdjacentHTML=elementProto.insertAdjacentHTML;elementProto.insertAdjacentHTML=function insertAdjacentHTML(position,text){return rewriteFn(this,origInsertAdjacentHTML,position,text,true)}}if(elementProto.insertAdjacentElement){var origIAdjElem=elementProto.insertAdjacentElement;elementProto.insertAdjacentElement=function insertAdjacentElement(position,element){return rewriteFn(this,origIAdjElem,position,element,false)}}}},Wombat.prototype.initDomOverride=function(){var Node=this.$wbwindow.Node;if(Node&&Node.prototype){var rewriteFn=this.rewriteNodeFuncArgs;if(Node.prototype.appendChild){var originalAppendChild=Node.prototype.appendChild;Node.prototype.appendChild=function appendChild(newNode,oldNode){return rewriteFn(this,originalAppendChild,newNode,oldNode)}}if(Node.prototype.insertBefore){var originalInsertBefore=Node.prototype.insertBefore;Node.prototype.insertBefore=function insertBefore(newNode,oldNode){return rewriteFn(this,originalInsertBefore,newNode,oldNode)}}if(Node.prototype.replaceChild){var originalReplaceChild=Node.prototype.replaceChild;Node.prototype.replaceChild=function replaceChild(newNode,oldNode){return rewriteFn(this,originalReplaceChild,newNode,oldNode)}}this.overridePropToProxy(Node.prototype,"ownerDocument"),this.overridePropToProxy(this.$wbwindow.HTMLHtmlElement.prototype,"parentNode"),this.overridePropToProxy(this.$wbwindow.Event.prototype,"target")}this.$wbwindow.Element&&this.$wbwindow.Element.prototype&&(this.overrideParentNodeAppendPrepend(this.$wbwindow.Element),this.overrideChildNodeInterface(this.$wbwindow.Element,false)),this.$wbwindow.DocumentFragment&&this.$wbwindow.DocumentFragment.prototype&&this.overrideParentNodeAppendPrepend(this.$wbwindow.DocumentFragment)},Wombat.prototype.initDocOverrides=function($document){if(Object.defineProperty){this.overrideReferrer($document),this.defGetterProp($document,"origin",function origin(){return this.WB_wombat_location.origin}),this.defGetterProp(this.$wbwindow,"origin",function origin(){return this.WB_wombat_location.origin});var wombat=this,domain_setter=function domain(val){var loc=this.WB_wombat_location;loc&&wombat.endsWith(loc.hostname,val)&&(this.__wb_domain=val)},domain_getter=function domain(){return this.__wb_domain||this.WB_wombat_location.hostname};this.defProp($document,"domain",domain_setter,domain_getter)}},Wombat.prototype.initDocWriteOpenCloseOverride=function(){if(this.$wbwindow.DOMParser){var DocumentProto=this.$wbwindow.Document.prototype,$wbDocument=this.$wbwindow.document,docWriteWritelnRWFn=this.rewriteDocWriteWriteln,orig_doc_write=$wbDocument.write,new_write=function write(){return docWriteWritelnRWFn(this,orig_doc_write,arguments)};$wbDocument.write=new_write,DocumentProto.write=new_write;var orig_doc_writeln=$wbDocument.writeln,new_writeln=function writeln(){return docWriteWritelnRWFn(this,orig_doc_writeln,arguments)};$wbDocument.writeln=new_writeln,DocumentProto.writeln=new_writeln;var wombat=this,orig_doc_open=$wbDocument.open,new_open=function open(){var res,thisObj=wombat.proxyToObj(this);if(arguments.length===3){var rwUrl=wombat.rewriteUrl(arguments[0],false,"mp_");res=orig_doc_open.call(thisObj,rwUrl,arguments[1],arguments[2]),wombat.initNewWindowWombat(res,rwUrl)}else res=orig_doc_open.call(thisObj),wombat.initNewWindowWombat(thisObj.defaultView);return res};$wbDocument.open=new_open,DocumentProto.open=new_open;var originalClose=$wbDocument.close,newClose=function close(){var thisObj=wombat.proxyToObj(this);return wombat.initNewWindowWombat(thisObj.defaultView),originalClose.__WB_orig_apply?originalClose.__WB_orig_apply(thisObj,arguments):originalClose.apply(thisObj,arguments)};$wbDocument.close=newClose,DocumentProto.close=newClose;var oBodyGetter=this.getOrigGetter(DocumentProto,"body"),oBodySetter=this.getOrigSetter(DocumentProto,"body");oBodyGetter&&oBodySetter&&this.defProp(DocumentProto,"body",function body(newBody){return newBody&&(newBody instanceof HTMLBodyElement||newBody instanceof HTMLFrameSetElement)&&wombat.rewriteElemComplete(newBody),oBodySetter.call(wombat.proxyToObj(this),newBody)},oBodyGetter)}},Wombat.prototype.initIframeWombat=function(iframe){var win;win=iframe._get_contentWindow?iframe._get_contentWindow.call(iframe):iframe.contentWindow;try{if(!win||win===this.$wbwindow||win._skip_wombat||win._wb_wombat)return}catch(e){return}var src=this.wb_getAttribute.call(iframe,"src");this.initNewWindowWombat(win,src)},Wombat.prototype.initNewWindowWombat=function(win,src){if(win&&!win._wb_wombat)if(!src||src===""||this.startsWith(src,"about:")||src.indexOf("javascript:")>=0){var wombat=new Wombat(win,this.wb_info);win._wb_wombat=wombat.wombatInit()}else this.initProtoPmOrigin(win),this.initPostMessageOverride(win),this.initMessageEventOverride(win),this.initCheckThisFunc(win)},Wombat.prototype.initTimeoutIntervalOverrides=function(){var rewriteFn=this.rewriteSetTimeoutInterval;if(this.$wbwindow.setTimeout&&!this.$wbwindow.setTimeout.__$wbpatched$__){var originalSetTimeout=this.$wbwindow.setTimeout;this.$wbwindow.setTimeout=function setTimeout(){return rewriteFn(this,originalSetTimeout,arguments)},this.$wbwindow.setTimeout.__$wbpatched$__=true}if(this.$wbwindow.setInterval&&!this.$wbwindow.setInterval.__$wbpatched$__){var originalSetInterval=this.$wbwindow.setInterval;this.$wbwindow.setInterval=function setInterval(){return rewriteFn(this,originalSetInterval,arguments)},this.$wbwindow.setInterval.__$wbpatched$__=true}},Wombat.prototype.initWorkerOverrides=function(){var wombat=this;if(this.$wbwindow.Worker&&!this.$wbwindow.Worker._wb_worker_overriden){var orig_worker=this.$wbwindow.Worker;this.$wbwindow.Worker=function(Worker_){return function Worker(url,options){return wombat.domConstructorErrorChecker(this,"Worker",arguments),new Worker_(wombat.rewriteWorker(url),options)}}(orig_worker),this.$wbwindow.Worker.prototype=orig_worker.prototype,Object.defineProperty(this.$wbwindow.Worker.prototype,"constructor",{value:this.$wbwindow.Worker}),this.$wbwindow.Worker._wb_worker_overriden=true}if(this.$wbwindow.SharedWorker&&!this.$wbwindow.SharedWorker.__wb_sharedWorker_overriden){var oSharedWorker=this.$wbwindow.SharedWorker;this.$wbwindow.SharedWorker=function(SharedWorker_){return function SharedWorker(url,options){return wombat.domConstructorErrorChecker(this,"SharedWorker",arguments),new SharedWorker_(wombat.rewriteWorker(url),options)}}(oSharedWorker),this.$wbwindow.SharedWorker.prototype=oSharedWorker.prototype,Object.defineProperty(this.$wbwindow.SharedWorker.prototype,"constructor",{value:this.$wbwindow.SharedWorker}),this.$wbwindow.SharedWorker.__wb_sharedWorker_overriden=true}if(this.$wbwindow.ServiceWorkerContainer&&this.$wbwindow.ServiceWorkerContainer.prototype&&this.$wbwindow.ServiceWorkerContainer.prototype.register){var orig_register=this.$wbwindow.ServiceWorkerContainer.prototype.register;this.$wbwindow.ServiceWorkerContainer.prototype.register=function register(scriptURL,options){var newScriptURL=new URL(scriptURL,wombat.$wbwindow.document.baseURI).href,mod=wombat.getPageUnderModifier();return options&&options.scope?options.scope=wombat.rewriteUrl(options.scope,false,mod):options={scope:wombat.rewriteUrl("/",false,mod)},orig_register.call(this,wombat.rewriteUrl(newScriptURL,false,"sw_"),options)}}if(this.$wbwindow.Worklet&&this.$wbwindow.Worklet.prototype&&this.$wbwindow.Worklet.prototype.addModule&&!this.$wbwindow.Worklet.__wb_workerlet_overriden){var oAddModule=this.$wbwindow.Worklet.prototype.addModule;this.$wbwindow.Worklet.prototype.addModule=function addModule(moduleURL,options){var rwModuleURL=wombat.rewriteUrl(moduleURL,false,"js_");return oAddModule.call(this,rwModuleURL,options)},this.$wbwindow.Worklet.__wb_workerlet_overriden=true}},Wombat.prototype.initLocOverride=function(loc,oSetter,oGetter){if(Object.defineProperty)for(var prop,i=0;i=0&&props.splice(foundInx,1);return props}})}catch(e){console.log(e)}},Wombat.prototype.initHashChange=function(){if(this.$wbwindow.__WB_top_frame){var wombat=this,receive_hash_change=function receive_hash_change(event){if(event.data&&event.data.from_top){var message=event.data.message;message.wb_type&&(message.wb_type!=="outer_hashchange"||wombat.$wbwindow.location.hash==message.hash||(wombat.$wbwindow.location.hash=message.hash))}},send_hash_change=function send_hash_change(){var message={wb_type:"hashchange",hash:wombat.$wbwindow.location.hash};wombat.sendTopMessage(message)};this.$wbwindow.addEventListener("message",receive_hash_change),this.$wbwindow.addEventListener("hashchange",send_hash_change)}},Wombat.prototype.initPostMessageOverride=function($wbwindow){if($wbwindow.postMessage&&!$wbwindow.__orig_postMessage){var orig=$wbwindow.postMessage,wombat=this;$wbwindow.__orig_postMessage=orig;var postmessage_rewritten=function postMessage(message,targetOrigin,transfer,from_top){var from,src_id,this_obj=wombat.proxyToObj(this);if(this_obj.__WB_source&&this_obj.__WB_source.WB_wombat_location){var source=this_obj.__WB_source;if(from=source.WB_wombat_location.origin,this_obj.__WB_win_id||(this_obj.__WB_win_id={},this_obj.__WB_counter=0),!source.__WB_id){var id=this_obj.__WB_counter;source.__WB_id=id+source.WB_wombat_location.href,this_obj.__WB_counter+=1}this_obj.__WB_win_id[source.__WB_id]=source,src_id=source.__WB_id,this_obj.__WB_source=undefined}else from=window.WB_wombat_location.origin;var to_origin=targetOrigin;to_origin===this_obj.location.origin&&(to_origin=from);var new_message={from:from,to_origin:to_origin,src_id:src_id,message:message,from_top:from_top};if(targetOrigin!=="*"){if(this_obj.location.origin==="null"||this_obj.location.origin==="")return;targetOrigin=this_obj.location.origin}return orig.call(this_obj,new_message,targetOrigin,transfer)};$wbwindow.postMessage=postmessage_rewritten,$wbwindow.Window.prototype.postMessage=postmessage_rewritten;var eventTarget=null;eventTarget=$wbwindow.EventTarget&&$wbwindow.EventTarget.prototype?$wbwindow.EventTarget.prototype:$wbwindow;var _oAddEventListener=eventTarget.addEventListener;eventTarget.addEventListener=function addEventListener(type,listener,useCapture){var rwListener,obj=wombat.proxyToObj(this);if(type==="message"?rwListener=wombat.message_listeners.add_or_get(listener,function(){return wrapEventListener(listener,obj,wombat)}):type==="storage"?wombat.storage_listeners.add_or_get(listener,function(){return wrapSameOriginEventListener(listener,obj)}):rwListener=listener,rwListener)return _oAddEventListener.call(obj,type,rwListener,useCapture)};var _oRemoveEventListener=eventTarget.removeEventListener;eventTarget.removeEventListener=function removeEventListener(type,listener,useCapture){var rwListener,obj=wombat.proxyToObj(this);if(type==="message"?rwListener=wombat.message_listeners.remove(listener):type==="storage"?wombat.storage_listeners.remove(listener):rwListener=listener,rwListener)return _oRemoveEventListener.call(obj,type,rwListener,useCapture)};var override_on_prop=function(onevent,wrapperFN){var orig_setter=wombat.getOrigSetter($wbwindow,onevent),setter=function(value){this["__orig_"+onevent]=value;var obj=wombat.proxyToObj(this),listener=value?wrapperFN(value,obj,wombat):value;return orig_setter.call(obj,listener)},getter=function(){return this["__orig_"+onevent]};wombat.defProp($wbwindow,onevent,setter,getter)};override_on_prop("onmessage",wrapEventListener),override_on_prop("onstorage",wrapSameOriginEventListener)}},Wombat.prototype.initMessageEventOverride=function($wbwindow){!$wbwindow.MessageEvent||$wbwindow.MessageEvent.prototype.__extended||(this.addEventOverride("target"),this.addEventOverride("srcElement"),this.addEventOverride("currentTarget"),this.addEventOverride("eventPhase"),this.addEventOverride("path"),this.overridePropToProxy($wbwindow.MessageEvent.prototype,"source"),$wbwindow.MessageEvent.prototype.__extended=true)},Wombat.prototype.initUIEventsOverrides=function(){this.overrideAnUIEvent("UIEvent"),this.overrideAnUIEvent("MouseEvent"),this.overrideAnUIEvent("TouchEvent"),this.overrideAnUIEvent("FocusEvent"),this.overrideAnUIEvent("KeyboardEvent"),this.overrideAnUIEvent("WheelEvent"),this.overrideAnUIEvent("InputEvent"),this.overrideAnUIEvent("CompositionEvent")},Wombat.prototype.initOpenOverride=function(){var orig=this.$wbwindow.open;this.$wbwindow.Window.prototype.open&&(orig=this.$wbwindow.Window.prototype.open);var wombat=this,open_rewritten=function open(strUrl,strWindowName,strWindowFeatures){var rwStrUrl=wombat.rewriteUrl(strUrl,false,""),res=orig.call(wombat.proxyToObj(this),rwStrUrl,strWindowName,strWindowFeatures);return wombat.initNewWindowWombat(res,rwStrUrl),res};this.$wbwindow.open=open_rewritten,this.$wbwindow.Window.prototype.open&&(this.$wbwindow.Window.prototype.open=open_rewritten);for(var i=0;i
- +