Merge branch 'unbatch-fundus' into ci_test_matrix

# Conflicts:
#	.github/workflows/tests.yml
#	setup.cfg

Author: MaxDall

.github/workflows/publisher_coverage.yaml

@@ -0,0 +1,78 @@
+name: Publisher Coverage
+
+on:
+  schedule:
+    - cron: '0 1 * * *'  # Runs at 01:00
+
+  workflow_dispatch:
+
+jobs:
+  validate_crawlers:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up Git repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install Fundus
+        run: pip install -e .
+
+      - name: Validate Crawlers
+        run: |
+          set -o pipefail
+          exec python scripts/publisher_coverage.py | tee publisher_coverage.txt
+
+      - name: Upload Coverage Report
+        if: success() || failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Publisher Coverage
+          path: publisher_coverage.txt
+
+
+  create_badge:
+    runs-on: ubuntu-latest
+    needs: validate_crawlers
+    if: success() || failure()
+
+    steps:
+      - name: Set up Git repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Download Coverage Report
+        uses: actions/download-artifact@v3
+        with:
+          name: Publisher Coverage
+
+      - name: Get Success Rate
+        run: echo "SUCCESS_RATE=$(tail -n 1 publisher_coverage.txt | grep -P -o '\d+\/\d+')" >> $GITHUB_ENV
+
+      - name: Get Coverage Bounds
+        run: |
+          echo "TOTAL_PUBLISHERS=$(echo ${{ env.SUCCESS_RATE }} | grep -P -o '\d+' | tail -1)" >> $GITHUB_ENV
+          echo "PASSED_PUBLISHERS=$(echo ${{ env.SUCCESS_RATE }} | grep -P -o '\d+' | head -1)" >> $GITHUB_ENV
+
+      - name: Get Red Threshold
+        # We set the badge colour to red when at least one publisher failed the tests.
+        run: echo "RED_THRESHOLD=$(( ${{ env.TOTAL_PUBLISHERS }} - 1 ))" >> $GITHUB_ENV
+
+      - name: Create Badge
+        uses: schneegans/[email protected]
+        with:
+          auth: ${{ secrets.DOBBERSC_GIST_SECRET }}
+          gistID: ca0ae056b05cbfeaf30fa42f84ddf458
+          filename: fundus_publisher_coverage.json
+          label: Publisher Coverage
+          message: ${{ env.SUCCESS_RATE }}
+          valColorRange: ${{ env.PASSED_PUBLISHERS }}
+          maxColorRange: ${{ env.TOTAL_PUBLISHERS }}
+          minColorRange: ${{ env.RED_THRESHOLD }}

.github/workflows/tests.yml

@@ -31,8 +31,7 @@ jobs:
         id: cache
         with:
           path: ${{ env.pythonLocation }}
-          key: ${{ matrix.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('setup.cfg') }}
-
+          key: ${{ matrix.os }}-{{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
       - name: Install dependencies
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
@@ -62,8 +61,7 @@ jobs:
         id: cache
         with:
           path: ${{ env.pythonLocation }}
-          key: ${{ matrix.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('setup.cfg') }}
-
+          key: ${{ matrix.os }}-{{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
       - name: Install dependencies
         if: steps.cache.outputs.cache-hit != 'true'
         run: |

.gitignore

@@ -16,6 +16,7 @@ __pycache__/
 *.so
 
 # Distribution / packaging
+.pypirc
 .Python
 build/
 develop-eggs/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Max Dallabetta, Conrad Dobberstein, Aaron Wey
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,50 +1,71 @@
-<img alt="alt text" src="resources/fundus_logo.png" width="180"/>
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/flairNLP/fundus/blob/master/resources/logo/svg/logo_darkmode_with_font_and_clear_space.svg">
+    <source media="(prefers-color-scheme: light)" srcset="https://github.com/flairNLP/fundus/blob/master/resources/logo/svg/logo_lightmode_with_font_and_clear_space.svg">
+    <img src="https://github.com/flairNLP/fundus/blob/master/resources/logo/svg/logo_lightmode_with_font_and_clear_space.svg" alt="Logo" width="50%" height="50%">
+  </picture>
+</p>
+
+<p align="center">A very simple <b>news crawler</b> in Python.
+Developed at <a href="https://www.informatik.hu-berlin.de/en/forschung-en/gebiete/ml-en/">Humboldt University of Berlin</a>.
+</p>
+<p align="center">
+<img alt="version" src="https://img.shields.io/badge/version-0.1-green">
+<img alt="python" src="https://img.shields.io/badge/python-3.8-blue">
+<img alt="Static Badge" src="https://img.shields.io/badge/license-MIT-green">
+<img alt="Publisher Coverage" src="https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/dobbersc/ca0ae056b05cbfeaf30fa42f84ddf458/raw/fundus_publisher_coverage.json">
+</p>
+<div align="center">
+<hr>
+
+[Quick Start](#quick-start) |  [Tutorials](#tutorials)  | [News Sources](/docs/supported_publishers.md)
+
+</div>
 
-[![PyPI version](https://badge.fury.io/py/fundus.svg)](https://badge.fury.io/py/fundus)
-[![GitHub Issues](https://img.shields.io/github/issues/flairNLP/fundus.svg)](https://github.com/flairNLP/fundus/issues)
-[![Contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg)](docs/how_to_contribute.md)
-[![License: MIT](https://img.shields.io/badge/License-MIT-brightgreen.svg)](https://opensource.org/licenses/MIT)
-
-A very simple **news crawler**.
-Developed at [Humboldt University of Berlin](https://www.informatik.hu-berlin.de/en/forschung-en/gebiete/ml-en/).
 
 ---
 
 Fundus is:
 
-* A crawler for news ...
+* **A static news crawler.** 
+  Fundus lets you crawl online news articles with only a few lines of Python code!
+  Be it from live websites or the CC-NEWS dataset.
 
-* A Python ...
+* **An open-source Python package.**
+  Fundus is built on the idea of building something together. 
+  We welcome your contribution to  help Fundus [grow](docs/how_to_contribute.md)!
 
-## Quick Start
+<hr>
 
-### Requirements and Installation
+## Quick Start
 
-In your favorite virtual environment, simply do:
+To install from pip, simply do:
 
 ```
 pip install fundus
 ```
 
 Fundus requires Python 3.8+.
 
-### Example 1: Crawl a bunch of German-language news articles
 
-Let's use Fundus to crawl 2 articles of English-language news publishers based in the US.
+## Example 1: Crawl a bunch of English-language news articles
 
-```python
+Let's use Fundus to crawl 2 articles from publishers based in the US.
 
+```python
 from fundus import PublisherCollection, Crawler
 
-# initialize the crawler for news publisher based in the us
+# initialize the crawler for news publishers based in the US
 crawler = Crawler(PublisherCollection.us)
 
 # crawl 2 articles and print
 for article in crawler.crawl(max_articles=2):
     print(article)
 ```
 
-This should print something like:
+That's already it!
+
+If you run this code, it should print out something like this:
 
 ```console
 Fundus-Article:
@@ -53,6 +74,7 @@ Fundus-Article:
           through committee votes on Thursday thanks to a last-minute [...]"
 - URL:    https://freebeacon.com/politics/feinsteins-return-not-enough-for-confirmation-of-controversial-new-hampshire-judicial-nominee/
 - From:   FreeBeacon (2023-05-11 18:41)
+
 Fundus-Article:
 - Title: "Northwestern student government freezes College Republicans funding over [...]"
 - Text:  "Student government at Northwestern University in Illinois "indefinitely" froze
@@ -61,50 +83,76 @@ Fundus-Article:
 - From:   FoxNews (2023-05-09 14:37)
 ```
 
-This means that you crawled 2 articles from different US publishers.
+This printout tells you that you successfully crawled two articles!
 
-### Example 2: Crawl specific news source
+For each article, the printout details:
+- the "Title" of the article, i.e. its headline 
+- the "Text", i.e. the main article body text
+- the "URL" from which it was crawled
+- the news source it is "From"
 
-Maybe you want to crawl a specific news source instead. Let's crawl news articles form Washington Times only:
 
-```python
+## Example 2: Crawl a specific news source
 
+Maybe you want to crawl a specific news source instead. Let's crawl news articles from Washington Times only:
+
+```python
 from fundus import PublisherCollection, Crawler
 
 # initialize the crawler for Washington Times
-crawler = Crawler(PublisherCollection.us.WashingtonTimes)
+crawler = Crawler(PublisherCollection.us.TheNewYorker)
 
-# crawl 5 articles and print
-for article in crawler.crawl(max_articles=5):
+# crawl 2 articles and print
+for article in crawler.crawl(max_articles=2):
     print(article)
 ```
 
+## Example 3: Crawl articles from CC-NEWS
+
+If you're not familiar with CC-NEWS, check out their [paper](https://paperswithcode.com/dataset/cc-news).
+
+````python
+from fundus import PublisherCollection, CCNewsCrawler
+
+# initialize the crawler for news publishers based in the US
+crawler = CCNewsCrawler(*PublisherCollection.us)
+
+# crawl 2 articles and print
+for article in crawler.crawl(max_articles=2):
+  print(article)
+````
+
+
 ## Tutorials
 
 We provide **quick tutorials** to get you started with the library:
 
-1. [**Tutorial 1: How to crawl news with Fundus**](docs/...)
-2. [**Tutorial 2: The Article Class**](docs/...)
-3. [**Tutorial 3: How to add a new news-source**](docs/how_to_contribute.md)
+1. [**Tutorial 1: How to crawl news with Fundus**](docs/1_getting_started.md)
+2. [**Tutorial 2: How to crawl articles from CC-NEWS**](docs/2_crawl_from_cc_news.md)
+3. [**Tutorial 3: The Article Class**](docs/3_the_article_class.md)
+4. [**Tutorial 4: How to filter articles**](docs/4_how_to_filter_articles.md)
+5. [**Tutorial 5: How to search for publishers**](docs/5_how_to_search_for_publishers.md)
 
-The tutorials explain how ...
+If you wish to contribute check out these tutorials:
+1. [**How to contribute**](docs/how_to_contribute.md)
+2. [**How to add a publisher**](docs/how_to_add_a_publisher.md)
 
 ## Currently Supported News Sources
 
 You can find the publishers currently supported [**here**](/docs/supported_publishers.md).
 
-Also: **Adding a new source is easy - consider contributing to the project!**
+Also: **Adding a new publisher is easy - consider contributing to the project!**
 
 ## Contact
 
-Please email your questions or comments to ...
+Please email your questions or comments to [**Max Dallabetta**](mailto:[email protected]?subject=[GitHub]%20Fundus)
 
 ## Contributing
 
 Thanks for your interest in contributing! There are many ways to get involved;
 start with our [contributor guidelines](docs/how_to_contribute.md) and then
 check these [open issues](https://github.com/flairNLP/fundus/issues) for specific tasks.
 
-## [License](/LICENSE)
+## License
 
-?
+[MIT](LICENSE)