Merge pull request #64 from johnb30/utf

Handle UTF errors with invalid bytes.
openeventdata · Jul 18, 2014 · 272df2e · 272df2e
2 parents dc500ca + 1f250c6
commit 272df2e
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/pages_scrape.py b/pages_scrape.py
@@ -32,7 +32,11 @@ def scrape(url, extractor):
 
         page = requests.get(url, headers=headers)
         try:
-            article = extractor.extract(raw_html=page.content)
+            try:
+                article = extractor.extract(raw_html=page.content)
+            except UnicodeDecodeError:
+                article = extractor.extract(raw_html=page.content.decode('utf-8',
+                                                                         errors='replace'))
             text = article.cleaned_text
             meta = article.meta_description
             return text, meta