Add a test that demonstrates HazyResearch#489

HiromuHota · Jul 30, 2020 · a7ecaf9 · a7ecaf9
1 parent 1d6771b
commit a7ecaf9
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/tests/test_postgres.py b/tests/test_postgres.py
@@ -198,3 +198,13 @@ def do_nothing_matcher(fig):
     )
 
     candidate_extractor.apply(docs, split=0, parallelism=PARALLEL)
+
+
+def test_parse_error_doc_skipping():
+    """Test skipping of faulty htmls."""
+    faulty_doc_path = "tests/data/html_faulty/ext_diseases_missing_table_tag.html"
+    preprocessor = HTMLDocPreprocessor(faulty_doc_path)
+    session = Meta.init(CONN_STRING).Session()
+    corpus_parser = Parser(session)
+    corpus_parser.apply(preprocessor)
+    assert session.query(Document).count() == 0