From 05383e604d02c029d42d50d50240cfcad83fa3b0 Mon Sep 17 00:00:00 2001
From: Adrian Breiding <ad123br@gmail.com>
Date: Sun, 18 Feb 2024 18:57:01 +0100
Subject: [PATCH 01/12] Add free_access attribute

---
 docs/attribute_guidelines.md                   |  8 ++++++++
 docs/how_to_add_a_publisher.md                 | 18 ++++++++++++++++++
 src/fundus/parser/base_parser.py               | 11 ++++++++++-
 src/fundus/publishers/de/bild.py               |  8 ++++++++
 .../publishers/de/braunschweiger_zeitung.py    |  4 ----
 5 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/docs/attribute_guidelines.md b/docs/attribute_guidelines.md
index 01b906cbc..304ecab25 100644
--- a/docs/attribute_guidelines.md
+++ b/docs/attribute_guidelines.md
@@ -58,4 +58,12 @@ Those attributes will be validated with unit tests when used.
         <td><code>List[str]</code></td>
         <td><code>generic_topic_parsing</code></td>
     </tr>
+    <tr>
+        <td>free_access</td>
+        <td>A boolean which is set to be True, if the article is restricted to users with a subscription. This usually indicates
+        that the article cannot be crawled completely.
+        <i>This attribute is implemented by default</i></td>
+        <td><code>bool</code></td>
+        <td><code></code></td>
+    </tr>
 </table>
diff --git a/docs/how_to_add_a_publisher.md b/docs/how_to_add_a_publisher.md
index 0c9cf14cb..91a00b4fd 100644
--- a/docs/how_to_add_a_publisher.md
+++ b/docs/how_to_add_a_publisher.md
@@ -469,6 +469,24 @@ Instead, we recommend referring to [this](https://devhints.io/xpath) documentati
 Make sure to examine other parsers and consult the [attribute guidelines](attribute_guidelines.md) for specifics on attribute implementation. 
 We strongly encourage utilizing these utility functions, especially when parsing the `ArticleBody`.
 
+### Checking the free_access attribute
+
+In case your new publisher does not have a subscription model, you can go ahead and skip this step. If it does,
+please verify that there is a tag `isAccessibleForFree` within the `<script type="application/ld+json">` blocks in the
+source code of premium articles that is set to either `false` or `False`. It doesn't matter if the tag is missing in the
+freely accessible articles. If this is the case, you can continue with the next step. If not, please overwrite the
+existing function by adding the following snippet to your parser:
+
+```python
+@attribute
+def free_access(self) -> bool:
+    # Your personalized logic goes here
+    pass
+```
+
+Usually you can identify a premium article by an indicator within the URL or by using XPath or CSSSelector and selecting
+the element asking to to purchase a subscription to view the article.
+
 ### Finishing the Parser
 
 Bringing all the above together, the Los Angeles Times now looks like this.
diff --git a/src/fundus/parser/base_parser.py b/src/fundus/parser/base_parser.py
index 066991c83..f01f30ec6 100644
--- a/src/fundus/parser/base_parser.py
+++ b/src/fundus/parser/base_parser.py
@@ -172,7 +172,7 @@ def _search_members(cls, obj_type: type) -> List[Tuple[str, Any]]:
     @classmethod
     def attributes(cls) -> AttributeCollection:
         attrs: List[Attribute] = [
-            func for _, func in cls._search_members(Attribute) if func.__name__ not in ["__ld", "__meta"]
+            func for _, func in cls._search_members(Attribute) if func.__name__ not in ["__ld", "__meta", "free_access"]
         ]
         return AttributeCollection(*attrs)
 
@@ -233,6 +233,15 @@ def __meta(self) -> Dict[str, Any]:
     def __ld(self) -> Optional[LinkedDataMapping]:
         return self.precomputed.ld
 
+    @attribute
+    def free_access(self) -> bool:
+        if (isAccessibleForFree := self.precomputed.ld.bf_search("isAccessibleForFree")) is None:
+            return True
+        elif not isAccessibleForFree or isAccessibleForFree == "false" or isAccessibleForFree == "False":
+            return False
+        else:
+            return True
+
 
 class _ParserCache:
     def __init__(self, factory: Type[BaseParser]):
diff --git a/src/fundus/publishers/de/bild.py b/src/fundus/publishers/de/bild.py
index 1fa1753b1..d895f5d0b 100644
--- a/src/fundus/publishers/de/bild.py
+++ b/src/fundus/publishers/de/bild.py
@@ -1,4 +1,5 @@
 import datetime
+import re
 from typing import List, Optional
 
 from lxml.etree import XPath
@@ -42,3 +43,10 @@ def title(self) -> Optional[str]:
         @attribute
         def topics(self) -> List[str]:
             return generic_topic_parsing(self.precomputed.meta.get("keywords"))
+
+        @attribute
+        def free_access(self) -> bool:
+            if (url := self.precomputed.meta.get("og:url")) is not None:
+                return re.search(r"/bild-plus/", url) is not None
+            else:
+                return True
diff --git a/src/fundus/publishers/de/braunschweiger_zeitung.py b/src/fundus/publishers/de/braunschweiger_zeitung.py
index a80cc6aeb..00cf837ff 100644
--- a/src/fundus/publishers/de/braunschweiger_zeitung.py
+++ b/src/fundus/publishers/de/braunschweiger_zeitung.py
@@ -56,7 +56,3 @@ def authors(self) -> List[str]:
         @attribute
         def publishing_date(self) -> Optional[datetime.datetime]:
             return generic_date_parsing(self.precomputed.ld.bf_search("datePublished"))
-
-        @attribute(validate=False)
-        def free_access(self) -> bool:
-            return self.precomputed.ld.bf_search("isAccessibleForFree") == "True"

From fa5e3ae0df1d57e693fa836367137152b6f1eb9f Mon Sep 17 00:00:00 2001
From: Adrian Breiding <ad123br@gmail.com>
Date: Sun, 18 Feb 2024 19:37:30 +0100
Subject: [PATCH 02/12] update extraction filter ignoring free_access

---
 src/fundus/scraping/pipeline.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/fundus/scraping/pipeline.py b/src/fundus/scraping/pipeline.py
index 568fb97a9..a31c6a2b7 100644
--- a/src/fundus/scraping/pipeline.py
+++ b/src/fundus/scraping/pipeline.py
@@ -95,7 +95,8 @@ def build_extraction_filter() -> Optional[ExtractionFilter]:
                     None
                     if only_complete is False
                     else lambda extracted: not all(
-                        bool(v) if not isinstance(v, Exception) else False for _, v in extracted.items()
+                        bool(v) if not isinstance(v, Exception) else False
+                        for v in [value for key, value in extracted.items() if not (isinstance(value, bool))]
                     )
                 )
             else:

From e31fa75923425113bc4ad048eb2831f6770ed187 Mon Sep 17 00:00:00 2001
From: Adrian Breiding <ad123br@gmail.com>
Date: Sun, 18 Feb 2024 19:45:58 +0100
Subject: [PATCH 03/12] update free_access attribute in Bild

---
 src/fundus/publishers/de/bild.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fundus/publishers/de/bild.py b/src/fundus/publishers/de/bild.py
index d895f5d0b..ad452c758 100644
--- a/src/fundus/publishers/de/bild.py
+++ b/src/fundus/publishers/de/bild.py
@@ -47,6 +47,6 @@ def topics(self) -> List[str]:
         @attribute
         def free_access(self) -> bool:
             if (url := self.precomputed.meta.get("og:url")) is not None:
-                return re.search(r"/bild-plus/", url) is not None
+                return re.search(r"/bild-plus/", url) is None
             else:
                 return True

From 59daea62515c4e3fbc21471bac96baad5a8f14cd Mon Sep 17 00:00:00 2001
From: addie9800 <addie9800@users.noreply.github.com>
Date: Tue, 20 Feb 2024 21:14:03 +0000
Subject: [PATCH 04/12] Update documentation from @
 395a0ebd0cc8e15ee2964929a7d7d3ec6d17679b (#362)

---
 docs/supported_publishers.md | 133 ++++++++++++++++++++++++++---------
 1 file changed, 101 insertions(+), 32 deletions(-)

diff --git a/docs/supported_publishers.md b/docs/supported_publishers.md
index 78bcdfda8..5327163a2 100644
--- a/docs/supported_publishers.md
+++ b/docs/supported_publishers.md
@@ -27,6 +27,7 @@
         </a>
       </td>
       <td>
+        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -60,7 +61,9 @@
           <span>www.berliner-zeitung.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -75,7 +78,9 @@
           <span>www.bild.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -90,10 +95,10 @@
           <span>www.braunschweiger-zeitung.de</span>
         </a>
       </td>
-      <td>&#160;</td>
       <td>
         <code>free_access</code>
       </td>
+      <td>&#160;</td>
     </tr>
     <tr>
       <td>
@@ -107,7 +112,9 @@
           <span>www.businessinsider.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -122,7 +129,9 @@
           <span>www.dw.com</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -137,7 +146,9 @@
           <span>www.taz.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -152,7 +163,9 @@
           <span>www.welt.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -167,7 +180,9 @@
           <span>www.zeit.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -182,7 +197,9 @@
           <span>www.focus.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -197,7 +214,9 @@
           <span>www.faz.net</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -212,7 +231,9 @@
           <span>www.mdr.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -228,6 +249,7 @@
         </a>
       </td>
       <td>
+        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -244,7 +266,9 @@
           <span>www.ntv.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -259,7 +283,9 @@
           <span>www.ndr.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -274,7 +300,9 @@
           <span>www.spiegel.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -289,7 +317,9 @@
           <span>www.stern.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -304,7 +334,9 @@
           <span>www.sueddeutsche.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -319,7 +351,9 @@
           <span>www.tagesschau.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -334,7 +368,9 @@
           <span>www.waz.de</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
   </tbody>
@@ -366,7 +402,9 @@
           <span>www.lemonde.fr</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
   </tbody>
@@ -399,6 +437,7 @@
         </a>
       </td>
       <td>
+        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -432,7 +471,9 @@
           <span>www.theguardian.com</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -447,7 +488,9 @@
           <span>www.independent.co.uk</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -462,7 +505,9 @@
           <span>www.telegraph.co.uk</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -477,7 +522,9 @@
           <span>inews.co.uk</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
   </tbody>
@@ -509,7 +556,9 @@
           <span>www.apnews.com</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -524,7 +573,9 @@
           <span>www.cnbc.com</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>
         <code>key_points</code>
       </td>
@@ -541,7 +592,9 @@
           <span>www.foxnews.com</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -557,6 +610,7 @@
         </a>
       </td>
       <td>
+        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -573,7 +627,9 @@
           <span>occupydemocrats.com</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>
         <code>description</code>
       </td>
@@ -590,7 +646,9 @@
           <span>www.reuters.com</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -606,6 +664,7 @@
         </a>
       </td>
       <td>
+        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -622,7 +681,9 @@
           <span>www.theintercept.com</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -637,7 +698,9 @@
           <span>www.thenation.com</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -652,7 +715,9 @@
           <span>www.newyorker.com</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>
         <code>alternative_description</code>
         <code>alternative_title</code>
@@ -672,7 +737,9 @@
           <span>www.freebeacon.com</span>
         </a>
       </td>
-      <td>&#160;</td>
+      <td>
+        <code>free_access</code>
+      </td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -688,6 +755,7 @@
         </a>
       </td>
       <td>
+        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -706,6 +774,7 @@
       </td>
       <td>
         <code>authors</code>
+        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>

From 1b14d1bbb9d9bf8788ce778bd244da4150813156 Mon Sep 17 00:00:00 2001
From: Adrian Breiding <ad123br@gmail.com>
Date: Tue, 20 Feb 2024 22:15:50 +0100
Subject: [PATCH 05/12] Fix Typo in Documentation

---
 docs/attribute_guidelines.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/attribute_guidelines.md b/docs/attribute_guidelines.md
index 304ecab25..3321f5dcd 100644
--- a/docs/attribute_guidelines.md
+++ b/docs/attribute_guidelines.md
@@ -60,7 +60,7 @@ Those attributes will be validated with unit tests when used.
     </tr>
     <tr>
         <td>free_access</td>
-        <td>A boolean which is set to be True, if the article is restricted to users with a subscription. This usually indicates
+        <td>A boolean which is set to be False, if the article is restricted to users with a subscription. This usually indicates
         that the article cannot be crawled completely.
         <i>This attribute is implemented by default</i></td>
         <td><code>bool</code></td>

From 89f6b7f3a76484363c15cbaa6b87cec88f57816b Mon Sep 17 00:00:00 2001
From: Adrian Breiding <ad123br@gmail.com>
Date: Wed, 21 Feb 2024 19:27:59 +0100
Subject: [PATCH 06/12] Update docs/how_to_add_a_publisher.md

Co-authored-by: Max Dallabetta <max.dallabetta@googlemail.com>
---
 docs/how_to_add_a_publisher.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/how_to_add_a_publisher.md b/docs/how_to_add_a_publisher.md
index 91a00b4fd..3599fce49 100644
--- a/docs/how_to_add_a_publisher.md
+++ b/docs/how_to_add_a_publisher.md
@@ -481,7 +481,7 @@ existing function by adding the following snippet to your parser:
 @attribute
 def free_access(self) -> bool:
     # Your personalized logic goes here
-    pass
+    ...
 ```
 
 Usually you can identify a premium article by an indicator within the URL or by using XPath or CSSSelector and selecting

From ffa233272d35913395d78e73252d2b39f518876e Mon Sep 17 00:00:00 2001
From: Adrian Breiding <ad123br@gmail.com>
Date: Wed, 21 Feb 2024 19:54:12 +0100
Subject: [PATCH 07/12] update scraper and documentation

---
 docs/how_to_add_a_publisher.md | 10 ++++++----
 src/fundus/scraping/scraper.py |  1 -
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/docs/how_to_add_a_publisher.md b/docs/how_to_add_a_publisher.md
index 91a00b4fd..2f2472acc 100644
--- a/docs/how_to_add_a_publisher.md
+++ b/docs/how_to_add_a_publisher.md
@@ -16,6 +16,7 @@
     * [Working with `lxml`](#working-with-lxml)
     * [CSS-Select](#css-select)
     * [XPath](#xpath)
+  * [Checking the free_access attribute](#checking-the-free_access-attribute)
   * [Finishing the Parser](#finishing-the-parser)
 * [6. Generate unit tests](#6-generate-unit-tests)
 * [7. Opening a Pull Request](#7-opening-a-pull-request)
@@ -472,10 +473,11 @@ We strongly encourage utilizing these utility functions, especially when parsing
 ### Checking the free_access attribute
 
 In case your new publisher does not have a subscription model, you can go ahead and skip this step. If it does,
-please verify that there is a tag `isAccessibleForFree` within the `<script type="application/ld+json">` blocks in the
-source code of premium articles that is set to either `false` or `False`. It doesn't matter if the tag is missing in the
-freely accessible articles. If this is the case, you can continue with the next step. If not, please overwrite the
-existing function by adding the following snippet to your parser:
+please verify that there is a tag `isAccessibleForFree` within the HTMLs `ld+json` elements (refer to the section
+[Extracting attributes from Precomputed](#extracting-attributes-from-precomputed) for details) in the source code of
+premium articles that is set to either `false` or `False`. It doesn't matter if the tag is missing in the freely
+accessible articles. If this is the case, you can continue with the next step. If not, please overwrite the existing
+function by adding the following snippet to your parser:
 
 ```python
 @attribute
diff --git a/src/fundus/scraping/scraper.py b/src/fundus/scraping/scraper.py
index f48754831..0522bce88 100644
--- a/src/fundus/scraping/scraper.py
+++ b/src/fundus/scraping/scraper.py
@@ -34,7 +34,6 @@ async def scrape(
             supported_attributes = set(
                 more_itertools.flatten(collection.names for collection in self.parser.attribute_mapping.values())
             )
-            supported_attributes.add("free_access")
             if missing_attributes := extraction_filter.required_attributes - supported_attributes:
                 if len(missing_attributes) == 1:
                     basic_logger.warning(

From 8656c88a490173a938969ddf2dadd07438ddef5d Mon Sep 17 00:00:00 2001
From: Adrian Breiding <ad123br@gmail.com>
Date: Thu, 22 Feb 2024 22:11:46 +0100
Subject: [PATCH 08/12] update test_parser.py to accept free_access as
 attribute with BaseParser

---
 src/fundus/parser/base_parser.py |  2 +-
 tests/test_parser.py             | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/fundus/parser/base_parser.py b/src/fundus/parser/base_parser.py
index f01f30ec6..3bc2eb99e 100644
--- a/src/fundus/parser/base_parser.py
+++ b/src/fundus/parser/base_parser.py
@@ -172,7 +172,7 @@ def _search_members(cls, obj_type: type) -> List[Tuple[str, Any]]:
     @classmethod
     def attributes(cls) -> AttributeCollection:
         attrs: List[Attribute] = [
-            func for _, func in cls._search_members(Attribute) if func.__name__ not in ["__ld", "__meta", "free_access"]
+            func for _, func in cls._search_members(Attribute) if func.__name__ not in ["__ld", "__meta"]
         ]
         return AttributeCollection(*attrs)
 
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 6b519b10a..c1c77c93d 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -47,10 +47,10 @@ def test_functions_iter(self, parser_with_function_test, parser_with_static_meth
         assert parser_with_function_test.functions().names == ["test"]
 
     def test_attributes_iter(self, parser_with_attr_title, parser_with_static_method):
-        assert len(BaseParser.attributes()) == 0
-        assert len(parser_with_static_method.attributes()) == 0
-        assert len(parser_with_attr_title.attributes()) == 1
-        assert parser_with_attr_title.attributes().names == ["title"]
+        assert len(BaseParser.attributes()) == 1
+        assert len(parser_with_static_method.attributes()) == 1
+        assert len(parser_with_attr_title.attributes()) == 2
+        assert parser_with_attr_title.attributes().names == ["free_access", "title"]
 
     def test_supported_unsupported(self):
         class ParserWithValidatedAndUnvalidated(BaseParser):
@@ -63,12 +63,12 @@ def unvalidated(self) -> str:
                 return "unsupported"
 
         parser = ParserWithValidatedAndUnvalidated()
-        assert len(parser.attributes()) == 2
+        assert len(parser.attributes()) == 3
 
         assert (validated := parser.attributes().validated)
         assert isinstance(validated, AttributeCollection)
         assert (funcs := list(validated)) != [parser.validated]
-        assert funcs[0].__func__ == parser.validated.__func__
+        assert funcs[1].__func__ == parser.validated.__func__
 
         assert (unvalidated := parser.attributes().unvalidated)
         assert isinstance(validated, AttributeCollection)

From 099a5e8f1ea29ed72b7f35528e265b04d6b5ff9e Mon Sep 17 00:00:00 2001
From: addie9800 <addie9800@users.noreply.github.com>
Date: Thu, 22 Feb 2024 21:12:27 +0000
Subject: [PATCH 09/12] Update documentation from @
 8656c88a490173a938969ddf2dadd07438ddef5d (#362)

---
 docs/supported_publishers.md | 135 +++++++++--------------------------
 1 file changed, 32 insertions(+), 103 deletions(-)

diff --git a/docs/supported_publishers.md b/docs/supported_publishers.md
index 5327163a2..8c6893897 100644
--- a/docs/supported_publishers.md
+++ b/docs/supported_publishers.md
@@ -27,7 +27,6 @@
         </a>
       </td>
       <td>
-        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -61,9 +60,7 @@
           <span>www.berliner-zeitung.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -78,9 +75,7 @@
           <span>www.bild.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -95,9 +90,7 @@
           <span>www.braunschweiger-zeitung.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -112,9 +105,7 @@
           <span>www.businessinsider.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -129,9 +120,7 @@
           <span>www.dw.com</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -146,9 +135,7 @@
           <span>www.taz.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -163,9 +150,7 @@
           <span>www.welt.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -180,9 +165,7 @@
           <span>www.zeit.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -197,9 +180,7 @@
           <span>www.focus.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -214,9 +195,7 @@
           <span>www.faz.net</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -231,9 +210,7 @@
           <span>www.mdr.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -249,7 +226,6 @@
         </a>
       </td>
       <td>
-        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -266,9 +242,7 @@
           <span>www.ntv.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -283,9 +257,7 @@
           <span>www.ndr.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -300,9 +272,7 @@
           <span>www.spiegel.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -317,9 +287,7 @@
           <span>www.stern.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -334,9 +302,7 @@
           <span>www.sueddeutsche.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -351,9 +317,7 @@
           <span>www.tagesschau.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -368,9 +332,7 @@
           <span>www.waz.de</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
   </tbody>
@@ -402,9 +364,7 @@
           <span>www.lemonde.fr</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
   </tbody>
@@ -437,7 +397,6 @@
         </a>
       </td>
       <td>
-        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -471,9 +430,7 @@
           <span>www.theguardian.com</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -488,9 +445,7 @@
           <span>www.independent.co.uk</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -505,9 +460,7 @@
           <span>www.telegraph.co.uk</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -522,9 +475,7 @@
           <span>inews.co.uk</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
   </tbody>
@@ -556,9 +507,7 @@
           <span>www.apnews.com</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -573,9 +522,7 @@
           <span>www.cnbc.com</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>
         <code>key_points</code>
       </td>
@@ -592,9 +539,7 @@
           <span>www.foxnews.com</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -610,7 +555,6 @@
         </a>
       </td>
       <td>
-        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -627,9 +571,7 @@
           <span>occupydemocrats.com</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>
         <code>description</code>
       </td>
@@ -646,9 +588,7 @@
           <span>www.reuters.com</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -664,7 +604,6 @@
         </a>
       </td>
       <td>
-        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -681,9 +620,7 @@
           <span>www.theintercept.com</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -698,9 +635,7 @@
           <span>www.thenation.com</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -715,9 +650,7 @@
           <span>www.newyorker.com</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>
         <code>alternative_description</code>
         <code>alternative_title</code>
@@ -737,9 +670,7 @@
           <span>www.freebeacon.com</span>
         </a>
       </td>
-      <td>
-        <code>free_access</code>
-      </td>
+      <td>&#160;</td>
       <td>&#160;</td>
     </tr>
     <tr>
@@ -755,7 +686,6 @@
         </a>
       </td>
       <td>
-        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>
@@ -774,7 +704,6 @@
       </td>
       <td>
         <code>authors</code>
-        <code>free_access</code>
         <code>topics</code>
       </td>
       <td>&#160;</td>

From fe398fb2d6d020e74c7dffc89148bb4a1dcfc7b3 Mon Sep 17 00:00:00 2001
From: Adrian Breiding <ad123br@gmail.com>
Date: Fri, 23 Feb 2024 15:30:27 +0100
Subject: [PATCH 10/12] reformat documentation edits

---
 docs/how_to_add_a_publisher.md | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/docs/how_to_add_a_publisher.md b/docs/how_to_add_a_publisher.md
index 70f023119..73b7d17ca 100644
--- a/docs/how_to_add_a_publisher.md
+++ b/docs/how_to_add_a_publisher.md
@@ -472,12 +472,10 @@ We strongly encourage utilizing these utility functions, especially when parsing
 
 ### Checking the free_access attribute
 
-In case your new publisher does not have a subscription model, you can go ahead and skip this step. If it does,
-please verify that there is a tag `isAccessibleForFree` within the HTMLs `ld+json` elements (refer to the section
-[Extracting attributes from Precomputed](#extracting-attributes-from-precomputed) for details) in the source code of
-premium articles that is set to either `false` or `False`. It doesn't matter if the tag is missing in the freely
-accessible articles. If this is the case, you can continue with the next step. If not, please overwrite the existing
-function by adding the following snippet to your parser:
+In case your new publisher does not have a subscription model, you can go ahead and skip this step.
+If it does, please verify that there is a tag `isAccessibleForFree` within the HTMLs `ld+json` elements (refer to the section [Extracting attributes from Precomputed](#extracting-attributes-from-precomputed) for details) in the source code of premium articles that is set to either `false` or `False`.
+It doesn't matter if the tag is missing in the freely accessible articles.
+If this is the case, you can continue with the next step. If not, please overwrite the existing function by adding the following snippet to your parser:
 
 ```python
 @attribute

From 89ec240c29b15d7c3ab031ebea6b7a33672eb59d Mon Sep 17 00:00:00 2001
From: Adrian Breiding <ad123br@gmail.com>
Date: Sat, 24 Feb 2024 17:10:41 +0100
Subject: [PATCH 11/12] Update docs/attribute_guidelines.md

Co-authored-by: Max Dallabetta <max.dallabetta@googlemail.com>
---
 docs/attribute_guidelines.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/attribute_guidelines.md b/docs/attribute_guidelines.md
index 3321f5dcd..7359cf625 100644
--- a/docs/attribute_guidelines.md
+++ b/docs/attribute_guidelines.md
@@ -62,7 +62,7 @@ Those attributes will be validated with unit tests when used.
         <td>free_access</td>
         <td>A boolean which is set to be False, if the article is restricted to users with a subscription. This usually indicates
         that the article cannot be crawled completely.
-        <i>This attribute is implemented by default</i></td>
+        <i><b>This attribute is implemented by default</b></i></td>
         <td><code>bool</code></td>
         <td><code></code></td>
     </tr>

From 7bf65530d8c62c8efe83c19bb8c94fb63c4de630 Mon Sep 17 00:00:00 2001
From: Adrian Breiding <ad123br@gmail.com>
Date: Sat, 24 Feb 2024 17:10:50 +0100
Subject: [PATCH 12/12] Update docs/how_to_add_a_publisher.md

Co-authored-by: Max Dallabetta <max.dallabetta@googlemail.com>
---
 docs/how_to_add_a_publisher.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/how_to_add_a_publisher.md b/docs/how_to_add_a_publisher.md
index 73b7d17ca..ddece3b3d 100644
--- a/docs/how_to_add_a_publisher.md
+++ b/docs/how_to_add_a_publisher.md
@@ -473,7 +473,7 @@ We strongly encourage utilizing these utility functions, especially when parsing
 ### Checking the free_access attribute
 
 In case your new publisher does not have a subscription model, you can go ahead and skip this step.
-If it does, please verify that there is a tag `isAccessibleForFree` within the HTMLs `ld+json` elements (refer to the section [Extracting attributes from Precomputed](#extracting-attributes-from-precomputed) for details) in the source code of premium articles that is set to either `false` or `False`.
+If it does, please verify that there is a tag `isAccessibleForFree` within the HTMLs `ld+json` elements (refer to the section [Extracting attributes from Precomputed](#extracting-attributes-from-precomputed) for details) in the source code of premium articles that is set to either `false` or `False`,  `true`/`True` respectively.
 It doesn't matter if the tag is missing in the freely accessible articles.
 If this is the case, you can continue with the next step. If not, please overwrite the existing function by adding the following snippet to your parser: