Fixed bug in exclusion filtering

flathunter.py

@@ -42,11 +42,11 @@ def launch_flat_hunt(config):
     id_watch = IdMaintainer('%s/processed_ids.db' % os.path.dirname(os.path.abspath(__file__)))
 
     hunter = Hunter(config)
-    hunter.hunt_flats(config, searchers, id_watch)
+    hunter.hunt_flats(searchers, id_watch)
 
     while config.get('loop', dict()).get('active', False):
         time.sleep(config.get('loop', dict()).get('sleeping_time', 60 * 10))
-        hunter.hunt_flats(config, searchers, id_watch)
+        hunter.hunt_flats(searchers, id_watch)
 
 
 def main():

flathunter/hunter.py

@@ -15,16 +15,16 @@ class Hunter:
     GM_MODE_BICYCLE = 'bicycling'
     GM_MODE_DRIVING = 'driving'
 
-    def __init__(self, cfg):
-        self.cfg = cfg
-        self.excluded_titles = self.cfg.get('excluded_titles', list())
+    def __init__(self, config):
+        self.config = config
+        self.excluded_titles = self.config.get('excluded_titles', list())
 
-    def hunt_flats(self, config, searchers, id_watch):
-        sender = SenderTelegram(config)
-        new_links = 0
+    def hunt_flats(self, searchers, id_watch):
+        sender = SenderTelegram(self.config)
+        new_exposes = []
         processed = id_watch.get()
 
-        for url in config.get('urls', list()):
+        for url in self.config.get('urls', list()):
             self.__log__.debug('Processing URL: ' + url)
 
             try:
@@ -59,7 +59,7 @@ def hunt_flats(self, config, searchers, id_watch):
                             break
 
                 # calculdate durations
-                message = config.get('message', "").format(
+                message = self.config.get('message', "").format(
                     title=expose['title'],
                     rooms=expose['rooms'],
                     size=expose['size'],
@@ -71,10 +71,10 @@ def hunt_flats(self, config, searchers, id_watch):
                 # durations=self.get_formatted_durations(config, address)).strip()
 
                 # if no excludes, send messages
-                if self.excluded_titles is None:
+                if len(self.excluded_titles) == 0:
                     # send message to all receivers
                     sender.send_msg(message)
-                    new_links += 1
+                    new_exposes.append(expose)
                     id_watch.add(expose['id'])
                     continue
 
@@ -85,10 +85,11 @@ def hunt_flats(self, config, searchers, id_watch):
                 if not found_objects:
                     # send message to all receivers
                     sender.send_msg(message)
-                    new_links += 1
+                    new_exposes.append(expose)
                     id_watch.add(expose['id'])
 
-        self.__log__.info(str(new_links) + ' new offer found')
+        self.__log__.info(str(len(new_exposes)) + ' new offers found')
+        return new_exposes
 
     def get_formatted_durations(self, config, address):
         out = ""

test/test_hunter.py

@@ -0,0 +1,21 @@
+import unittest
+import yaml
+from flathunter.crawl_immowelt import CrawlImmowelt
+from flathunter.hunter import Hunter 
+from flathunter.idmaintainer import IdMaintainer
+
+class HunterTest(unittest.TestCase):
+
+    DUMMY_CONFIG = """
+urls:
+  - https://www.immowelt.de/liste/berlin/wohnungen/mieten?roomi=2&prima=1500&wflmi=70&sort=createdate%2Bdesc
+    """
+
+    def setUp(self):
+        self.hunter = Hunter(yaml.safe_load(self.DUMMY_CONFIG))
+        self.searchers = [CrawlImmowelt()]
+        self.id_watch = IdMaintainer(":memory:")
+
+    def test_hunt_flats(self):
+        exposes = self.hunter.hunt_flats(self.searchers, self.id_watch)
+        self.assertTrue(len(exposes) > 0, "Expected to find exposes")