Merge pull request #618 from flairNLP/fix-the-namibian

Remove `<br>` tags in html

Author: addie9800

src/fundus/publishers/na/the_namibian.py

@@ -2,13 +2,17 @@
 from datetime import datetime
 from typing import List, Optional, Pattern
 
+import lxml.html
 from lxml.etree import XPath
 
-from fundus.parser import ArticleBody, BaseParser, ParserProxy, attribute
+from fundus.parser import ArticleBody, BaseParser, ParserProxy, attribute, function
+from fundus.parser.base_parser import Precomputed
 from fundus.parser.utility import (
     extract_article_body_with_selector,
     generic_author_parsing,
     generic_date_parsing,
+    get_ld_content,
+    get_meta_content,
 )
 
 
@@ -44,5 +48,15 @@ def authors(self) -> List[str]:
 
     class V1_1(V1):
         VALID_UNTIL = datetime.today().date()
-        _paragraph_selector = XPath("//div[contains(@class, 'entry-content')]/p[position()>1]")
-        _summary_selector = XPath("//div[contains(@class, 'entry-content')]/p[position()=1]")
+        _paragraph_selector = XPath("//div[contains(@class, 'entry-content')]/p[(text() or strong) and position()>1]")
+        _summary_selector = XPath("//div[contains(@class, 'entry-content')]/p[(text() or strong) and position()=1]")
+
+        @attribute
+        def body(self) -> ArticleBody:
+            html = re.sub(r"(<br>)+", "<p>", self.precomputed.html)
+            doc = lxml.html.document_fromstring(html)
+            return extract_article_body_with_selector(
+                doc,
+                paragraph_selector=self._paragraph_selector,
+                summary_selector=self._summary_selector,
+            )

tests/resources/parser/test_data/na/TheNamibian.json

@@ -86,7 +86,8 @@
             "He said he, Mbumba and Geingob planned to settle at Swakopmund and to sometimes walk barefoot by the seaside and have coffee at a nearby café.",
             "GOING HOME TO DIE",
             "Political analyst Henning Melber questions the circumstances surrounding Geingob’s departure, suggesting that if an inner circle was aware of his imminent passing, his trip to the US wouldn’t have been for medical treatment.",
-            "Melber speculates that Geingob may have realised his life was ending during his time in the US and chose to return home to die.However, Melber questions why this should be a contentious issue.",
+            "Melber speculates that Geingob may have realised his life was ending during his time in the US and chose to return home to die.",
+            "However, Melber questions why this should be a contentious issue.",
             "“This means that as of his arrival back in Namibia people may have become aware that not much time was left. But why make an issue of it?",
             "“If one could provide evidence that it was exploited for dubious arrangements, it would be another matter. But I do not see any indications pointing reliably into such a direction,” he says."
           ]