|
| 1 | +import datetime |
| 2 | +import re |
| 3 | +from typing import List, Optional |
| 4 | + |
| 5 | +from lxml.cssselect import CSSSelector |
| 6 | +from lxml.etree import XPath |
| 7 | + |
| 8 | +from fundus.parser import ArticleBody, BaseParser, Image, ParserProxy, attribute |
| 9 | +from fundus.parser.utility import ( |
| 10 | + apply_substitution_pattern_over_list, |
| 11 | + extract_article_body_with_selector, |
| 12 | + generic_author_parsing, |
| 13 | + generic_date_parsing, |
| 14 | + generic_topic_parsing, |
| 15 | + image_extraction, |
| 16 | +) |
| 17 | + |
| 18 | + |
| 19 | +class TokyoChunichiShimbunParser(ParserProxy): |
| 20 | + class V1(BaseParser): |
| 21 | + _paragraph_selector = XPath("//main//div[@class='block' and not(descendant::div or descendant::h2)]") |
| 22 | + _subheadline_selector = XPath("//main//div[@class='block']//h2") |
| 23 | + |
| 24 | + _author_bloat_pattern = re.compile(r"記者") |
| 25 | + _topic_bloat_pattern = re.compile(r"話題・|話題") |
| 26 | + |
| 27 | + @attribute |
| 28 | + def body(self) -> Optional[ArticleBody]: |
| 29 | + return extract_article_body_with_selector( |
| 30 | + self.precomputed.doc, |
| 31 | + paragraph_selector=self._paragraph_selector, |
| 32 | + subheadline_selector=self._subheadline_selector, |
| 33 | + ) |
| 34 | + |
| 35 | + @attribute |
| 36 | + def title(self) -> Optional[str]: |
| 37 | + return self.precomputed.ld.bf_search("headline") |
| 38 | + |
| 39 | + @attribute |
| 40 | + def publishing_date(self) -> Optional[datetime.datetime]: |
| 41 | + return generic_date_parsing(self.precomputed.ld.bf_search("datePublished")) |
| 42 | + |
| 43 | + @attribute |
| 44 | + def authors(self) -> List[str]: |
| 45 | + return apply_substitution_pattern_over_list( |
| 46 | + generic_author_parsing(self.precomputed.ld.bf_search("author")), self._author_bloat_pattern |
| 47 | + ) |
| 48 | + |
| 49 | + @attribute |
| 50 | + def topics(self) -> List[str]: |
| 51 | + if topics := apply_substitution_pattern_over_list( |
| 52 | + generic_topic_parsing(self.precomputed.ld.bf_search("articleSection")), self._topic_bloat_pattern |
| 53 | + ): |
| 54 | + return [topic for topic in topics if "ニュース" not in topic] |
| 55 | + return [] |
| 56 | + |
| 57 | + @attribute |
| 58 | + def images(self) -> List[Image]: |
| 59 | + return image_extraction( |
| 60 | + doc=self.precomputed.doc, |
| 61 | + paragraph_selector=self._paragraph_selector, |
| 62 | + image_selector=CSSSelector("main div.image img, main div.thumb img"), |
| 63 | + caption_selector=XPath( |
| 64 | + "./ancestor::div[@class='wrap']//p[@class='caption'] | " |
| 65 | + "./ancestor::div[@class='thumb']//p[@class='thumb-caption']" |
| 66 | + ), |
| 67 | + author_selector=re.compile(r"（(?P<credits>[^）]*?)(撮影)?）\s*$"), |
| 68 | + relative_urls=True, |
| 69 | + ) |
0 commit comments