From 77fc10b6ee3c444eb9df2ca908f7457b4c5901ed Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Mon, 6 Aug 2018 13:15:55 +0800 Subject: [PATCH 01/10] chg: test: remove BukuHTMLParser --- tests/test_BukuHTMLParser.py | 66 ------------------------------------ 1 file changed, 66 deletions(-) delete mode 100644 tests/test_BukuHTMLParser.py diff --git a/tests/test_BukuHTMLParser.py b/tests/test_BukuHTMLParser.py deleted file mode 100644 index 15446758..00000000 --- a/tests/test_BukuHTMLParser.py +++ /dev/null @@ -1,66 +0,0 @@ -"""test module.""" -from itertools import product -from unittest import mock - -import pytest - - -def test_init(): - """test method.""" - from buku import BukuHTMLParser - obj = BukuHTMLParser() - assert not obj.in_title_tag - assert not obj.data - assert obj.prev_tag is None - assert obj.parsed_title is None - - -@pytest.mark.parametrize('tag', ['', 'title']) -def test_handle_starttag(tag): - """test method.""" - attrs = mock.Mock() - from buku import BukuHTMLParser - obj = BukuHTMLParser() - obj.handle_starttag(tag, attrs) - if tag == 'title': - assert obj.in_title_tag - assert obj.prev_tag == tag - else: - assert not obj.in_title_tag - - -@pytest.mark.parametrize('tag, data', product(['', 'title'], [None, 'data'])) -def test_handle_endtag(tag, data): - """test method.""" - from buku import BukuHTMLParser - obj = BukuHTMLParser() - obj.data = data - obj.reset = mock.Mock() - obj.handle_endtag(tag) - # test - if tag == 'title': - assert not obj.in_title_tag - if tag == 'title' and data != '': - assert obj.parsed_title == data - obj.reset.assert_called_once_with() - - -@pytest.mark.parametrize('prev_tag, in_title_tag', product(['', 'title'], [None, 'data'])) -def test_handle_data(prev_tag, in_title_tag): - """test method.""" - new_data = 'new_data' - from buku import BukuHTMLParser - obj = BukuHTMLParser() - obj.prev_tag = prev_tag - obj.data = '' - obj.in_title_tag = in_title_tag - obj.handle_data(new_data) - if obj.prev_tag == 'title' and in_title_tag: - assert obj.data == new_data - - -def test_error(): - """test method.""" - from buku import BukuHTMLParser - obj = BukuHTMLParser() - obj.error(message=mock.Mock()) From 098d0d5e71abbad06b8b4f77dcc5a2e32fe1bacb Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Mon, 6 Aug 2018 13:18:13 +0800 Subject: [PATCH 02/10] chg: test: remove test_get_page_title --- tests/test_buku.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/test_buku.py b/tests/test_buku.py index 4796898c..080f2202 100644 --- a/tests/test_buku.py +++ b/tests/test_buku.py @@ -49,17 +49,6 @@ def test_is_ignored_mime(url, exp_res): assert exp_res == buku.is_ignored_mime(url) -def test_get_page_title(): - """test func.""" - resp = mock.Mock() - parser = mock.Mock() - parser.parsed_title = 'doubled whitespace' - with mock.patch('buku.BukuHTMLParser', return_value=parser): - import buku - res = buku.get_page_title(resp) - assert res == 'doubled whitespace' - - def test_gen_headers(): """test func.""" import buku From f80e5b241a65c68e0774134e03cb4ef518ee357f Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Mon, 6 Aug 2018 13:58:44 +0800 Subject: [PATCH 03/10] fix: test: test_to_temp_file_content --- tests/test_buku.py | 52 +++++++++++++++++----------------------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/tests/test_buku.py b/tests/test_buku.py index 080f2202..1d91e169 100644 --- a/tests/test_buku.py +++ b/tests/test_buku.py @@ -393,43 +393,29 @@ def test_is_editor_valid(editor, exp_res): def test_to_temp_file_content(url, title_in, tags_in, desc): """test func.""" import buku - res = buku.to_temp_file_content(url, title_in, tags_in, desc) - lines = [ - '# Lines beginning with "#" will be stripped.', - '# Add URL in next line (single line).', - '# Add TITLE in next line (single line). Leave blank to web fetch, "-" for no title.', - '# Add comma-separated TAGS in next line (single line).', - '# Add COMMENTS in next line(s).', - ] - idx_offset = 0 - # url - if url is not None: - lines.insert(2, url) - idx_offset += 1 + if desc is None: + desc_text = '\n' + elif desc == '': + desc_text = '-' + else: + desc_text = desc + title_in = '' if title_in is None: title_in = '' elif title_in == '': title_in = '-' - else: - pass - - # title - lines.insert(idx_offset + 3, title_in) - idx_offset += 1 - - # tags - lines.insert(idx_offset + 4, tags_in.strip(buku.DELIM)) - idx_offset += 1 - - # description - if desc is not None and desc != '': - pass - else: - desc = '' - lines.insert(idx_offset + 5, desc) - - for idx, res_line in enumerate(res.splitlines()): - assert lines[idx] == res_line + res = buku.to_temp_file_content(url, title_in, tags_in, desc) + lines = """# Lines beginning with "#" will be stripped. +# Add URL in next line (single line).{} +# Add TITLE in next line (single line). Leave blank to web fetch, "-" for no title.{} +# Add comma-separated TAGS in next line (single line).{} +# Add COMMENTS in next line(s). Leave blank to web fetch, "-" for no comments.{}""".format( + ''.join(['\n', url]) if url is not None else '', + ''.join(['\n', title_in]) if title_in else '\n', + ''.join(['\n', ','.join([x for x in tags_in.split(',') if x])]) if tags_in else '\n', + ''.join(['\n', desc_text]) + ) + assert res == lines @pytest.mark.parametrize( From 8e6c53f404a7fcf5ef489967280dd52288c7d50e Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Mon, 6 Aug 2018 14:19:21 +0800 Subject: [PATCH 04/10] chg: test: more test on to_temp_file_content func --- tests/test_buku.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/test_buku.py b/tests/test_buku.py index 1d91e169..81e8b6ef 100644 --- a/tests/test_buku.py +++ b/tests/test_buku.py @@ -386,8 +386,8 @@ def test_is_editor_valid(editor, exp_res): product( [None, 'example.com'], [None, '', 'title'], - ['', 'tag1,tag2', ',tag1,tag2,'], - [None, '', 'description'], + [None, '', '-', 'tag1,tag2', ',tag1,tag2,', ',,,,,'], + [None, '', '-', 'description'], ) ) def test_to_temp_file_content(url, title_in, tags_in, desc): @@ -399,11 +399,16 @@ def test_to_temp_file_content(url, title_in, tags_in, desc): desc_text = '-' else: desc_text = desc - title_in = '' if title_in is None: - title_in = '' + title_text = '' elif title_in == '': - title_in = '-' + title_text = '-' + else: + title_text = title_in + if tags_in is None: + with pytest.raises(AttributeError): + res = buku.to_temp_file_content(url, title_in, tags_in, desc) + return res = buku.to_temp_file_content(url, title_in, tags_in, desc) lines = """# Lines beginning with "#" will be stripped. # Add URL in next line (single line).{} @@ -411,7 +416,7 @@ def test_to_temp_file_content(url, title_in, tags_in, desc): # Add comma-separated TAGS in next line (single line).{} # Add COMMENTS in next line(s). Leave blank to web fetch, "-" for no comments.{}""".format( ''.join(['\n', url]) if url is not None else '', - ''.join(['\n', title_in]) if title_in else '\n', + ''.join(['\n', title_text]), ''.join(['\n', ','.join([x for x in tags_in.split(',') if x])]) if tags_in else '\n', ''.join(['\n', desc_text]) ) From 3327cf0e7ad56a885a5d89b260db38cf5855c6aa Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Mon, 6 Aug 2018 14:25:22 +0800 Subject: [PATCH 05/10] chg: doc: error on to_temp_file_content --- buku.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/buku.py b/buku.py index 7e1511ba..da278c12 100755 --- a/buku.py +++ b/buku.py @@ -4071,6 +4071,11 @@ def to_temp_file_content(url, title_in, tags_in, desc): ------- str Lines as newline separated string. + + Raises + ------ + AttributeError + when tags_in is None. """ strings = [('# Lines beginning with "#" will be stripped.\n' From c0af32544af67c54477a6e945d5e907cada0fbc1 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Mon, 6 Aug 2018 14:37:28 +0800 Subject: [PATCH 06/10] chg: test: use http for example.com --- tests/test_bukuDb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_bukuDb.py b/tests/test_bukuDb.py index 264f591c..61a8b319 100644 --- a/tests/test_bukuDb.py +++ b/tests/test_bukuDb.py @@ -670,7 +670,6 @@ def refreshdb_fixture(): os.environ['XDG_DATA_HOME'] = TEST_TEMP_DIR_PATH -# @pytest.mark.slowtest @pytest.mark.parametrize( "title_in, exp_res", [ @@ -682,7 +681,7 @@ def refreshdb_fixture(): ) def test_refreshdb(refreshdb_fixture, title_in, exp_res): bdb = refreshdb_fixture - args = ["https://example.com"] + args = ["http://example.com"] if title_in: args.append(title_in) bdb.add_rec(*args) From 551bf5a53d72adf06488a7f2d39050de2b4cd0be Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Mon, 6 Aug 2018 15:05:02 +0800 Subject: [PATCH 07/10] chg: dev: add html5lib package --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b8ebc7bf..3820dbd1 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,13 @@ license='GPLv3', platforms=['any'], py_modules=['buku'], - install_requires=['certifi', 'urllib3>=1.13.1', 'beautifulsoup4>=4.4.1', 'cryptography>=1.2.3'], + install_requires=[ + 'beautifulsoup4>=4.4.1', + 'certifi', + 'cryptography>=1.2.3', + 'urllib3>=1.13.1', + 'html5lib>=1.0.1', + ], packages=find_packages(exclude=['tests']), include_package_data=True, entry_points={ From 41b8785a4e4dede9f12cf64dcb3808324fd8490b Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Mon, 6 Aug 2018 15:42:46 +0800 Subject: [PATCH 08/10] fix: test: test_network_handler_with_url --- tests/test_buku.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/tests/test_buku.py b/tests/test_buku.py index 81e8b6ef..d5289071 100644 --- a/tests/test_buku.py +++ b/tests/test_buku.py @@ -530,15 +530,20 @@ def test_sigint_handler(capsys): @pytest.mark.parametrize( 'url, exp_res', [ - ['http://example.com.', ('', 0, 1)], - ['http://example.com', ('Example Domain', 0, 0)], - ['http://example.com/page1.txt', (('', 1, 0))], - ['about:new_page', (('', 0, 1))], - ['chrome://version/', (('', 0, 1))], - ['chrome://version/', (('', 0, 1))], + ['http://example.com.', (None, None, None, 0, 1)], + ['http://example.com', ('Example Domain', None, None, 0, 0)], + ['http://example.com/page1.txt', (('', '', '', 1, 0))], + ['about:new_page', ((None, None, None, 0, 1))], + ['chrome://version/', ((None, None, None, 0, 1))], + ['chrome://version/', ((None, None, None, 0, 1))], [ 'http://4pda.ru/forum/index.php?showtopic=182463&st=1640#entry6044923', - ('Samsung GT-I5800 Galaxy 580 - Обсуждение - 4PDA', 0, 0) + ( + 'Samsung GT-I5800 Galaxy 580 - Обсуждение - 4PDA', + 'Samsung GT-I5800 Galaxy 580 - Обсуждение - 4PDA', + 'Samsung GT-I5800 Galaxy 580 - Обсуждение - 4PDA', + 0, 0 + ) ], [ 'https://www.google.ru/search?' @@ -547,18 +552,25 @@ def test_sigint_handler(capsys): 'gs_l=serp.3..33i21.28976559.28977886.0.' '28978017.6.6.0.0.0.0.167.668.0j5.5.0....0...1c.1.64.' 'serp..1.2.311.06cSKPTLo18', - ('xkbcomp alt gr', 0, 0) + ('xkbcomp alt gr', None, None, 0, 0) ], [ 'http://www.vim.org/scripts/script.php?script_id=4641', ( 'mlessnau_case - "in-case" selection, deletion and substitution ' - 'for underscore, camel, mixed case : vim online', 0, 0 + 'for underscore, camel, mixed case : vim online', + None, None, 0, 0 ) ], [ 'http://www.kadrof.ru/cat_exchange.shtml', - ('Все биржи фриланса и удаленной работы - больше 110 сайтов | Kadrof.ru', 0, 0) + ( + 'Все биржи фриланса и удаленной работы - больше 110 сайтов | Kadrof.ru', + 'Здесь собраны самые популярные биржи удаленной работы и фриланса для новичков и опытных специалистов. ' + 'Более 110 ресурсов по видам:', + 'биржи удаленной работы,биржи фриланс', + 0, 0 + ) ], ] ) @@ -570,7 +582,9 @@ def test_network_handler_with_url(url, exp_res): buku.myproxy = None res = buku.network_handler(url) if urlparse(url).netloc == 'www.google.ru': - res = (res[0].split(" - ")[0], res[1], res[2]) + temp_res = [res[0].split(" - ")[0], ] + temp_res.extend(res[1:]) + res = tuple(temp_res) assert res == exp_res From 9d2b41438448f0f8aa0805890f9c9dfcffca7c3f Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Mon, 6 Aug 2018 15:47:38 +0800 Subject: [PATCH 09/10] fix: test: test_add_rec_add_invalid_url --- tests/test_bukuDb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bukuDb.py b/tests/test_bukuDb.py index 61a8b319..baf341ee 100644 --- a/tests/test_bukuDb.py +++ b/tests/test_bukuDb.py @@ -1007,7 +1007,7 @@ def test_add_rec_add_invalid_url(caplog, url): ], [ {'url': 'http://example.com', 'tags_in': 'tag1'}, - ('http://example.com', 'Example Domain', ',tag1', '', 0), + ('http://example.com', 'Example Domain', ',tag1,', '', 0), ], [ {'url': 'http://example.com', 'tags_in': ',tag1'}, From f823a6aecc808e035aa6ece853e1df29d3a4b72c Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Mon, 6 Aug 2018 15:51:36 +0800 Subject: [PATCH 10/10] chg: test: simplify test_get_PoolManager --- tests/test_buku.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/test_buku.py b/tests/test_buku.py index d5289071..c82a3a45 100644 --- a/tests/test_buku.py +++ b/tests/test_buku.py @@ -67,18 +67,10 @@ def test_gen_headers(): @pytest.mark.parametrize('m_myproxy', [None, mock.Mock()]) def test_get_PoolManager(m_myproxy): """test func.""" - with mock.patch('buku.urllib3') as m_ul3: + with mock.patch('buku.urllib3'): import buku buku.myproxy = m_myproxy - res = buku.get_PoolManager() - if m_myproxy: - m_ul3.ProxyManager.assert_called_once_with( - m_myproxy, num_pools=1, timeout=15, headers=buku.myheaders) - assert res == m_ul3.ProxyManager.return_value - else: - m_ul3.PoolManager.assert_called_once_with( - num_pools=1, timeout=15, headers=buku.myheaders) - assert res == m_ul3.PoolManager.return_value + assert buku.get_PoolManager() @pytest.mark.parametrize(