Pagination and some other changes bunched up

mopidy_dirble/actor.py

@@ -29,54 +29,75 @@ class DirbleLibrary(backend.LibraryProvider):
 
     # TODO: add countries when there is a lookup for countries with stations
     def browse(self, uri):
-        result = []
-        variant, identifier = translator.parse_uri(uri)
+        user_countries = []
+        geographic = []
+        categories = []
+        tracks = []
+
+        variant, identifier, page = translator.parse_uri(uri)
+
+        limit = 20
+        offset = (page or 0) * limit
+        next_offset = None
+        next_name = None
 
         if variant == 'root':
             for category in self.backend.dirble.categories():
-                result.append(translator.category_to_ref(category))
+                categories.append(translator.category_to_ref(category))
             for continent in self.backend.dirble.continents():
-                result.append(translator.continent_to_ref(continent))
+                geographic.append(translator.continent_to_ref(continent))
         elif variant == 'category' and identifier:
-            for category in self.backend.dirble.subcategories(identifier):
-                result.append(translator.category_to_ref(category))
-            for station in self.backend.dirble.stations(category=identifier):
-                result.append(translator.station_to_ref(station))
+            if not page:
+                for category in self.backend.dirble.subcategories(identifier):
+                    categories.append(translator.category_to_ref(category))
+            next_name = self.backend.dirble.category(identifier)['title']
+            stations, next_offset = self.backend.dirble.stations(
+                category=identifier, offset=offset, limit=limit)
+            for station in stations:
+                tracks.append(translator.station_to_ref(station))
         elif variant == 'continent' and identifier:
             for country in self.backend.dirble.countries(continent=identifier):
-                result.append(translator.country_to_ref(country))
+                geographic.append(translator.country_to_ref(country))
         elif variant == 'country' and identifier:
-            for station in self.backend.dirble.stations(country=identifier):
-                result.append(
+            next_name = self.backend.dirble.country(identifier)['name']
+            stations, next_offset = self.backend.dirble.stations(
+                country=identifier, offset=offset, limit=limit)
+            for station in stations:
+                tracks.append(
                     translator.station_to_ref(station, show_country=False))
         else:
             logger.debug('Unknown URI: %s', uri)
             return []
 
-        result.sort(key=lambda ref: ref.name)
-
-        # Handle this case after the general ones as we want the user defined
-        # countries be the first entries, and retain their config sort order.
         if variant == 'root':
-            user_countries = []
             for country_code in self.backend.countries:
                 country = self.backend.dirble.country(country_code)
                 if country:
                     user_countries.append(translator.country_to_ref(country))
                 else:
                     logger.debug('Unknown country: %s', country_code)
-            result = user_countries + result
+
+        categories.sort(key=lambda ref: ref.name)
+        geographic.sort(key=lambda ref: ref.name)
+
+        result = user_countries + geographic + categories + tracks
 
         if not result:
             logger.debug('Did not find any browse results for: %s', uri)
 
+        if next_offset:
+            next_page = int(next_offset / limit)
+            next_uri = translator.unparse_uri(variant, identifier, next_page)
+            next_name += ' page %d' % (next_page + 1)
+            result.append(Ref.directory(uri=next_uri, name=next_name))
+
         return result
 
     def refresh(self, uri=None):
         self.backend.dirble.flush()
 
     def lookup(self, uri):
-        variant, identifier = translator.parse_uri(uri)
+        variant, identifier, _ = translator.parse_uri(uri)
         if variant != 'station':
             return []
         station = self.backend.dirble.station(identifier)
@@ -88,29 +109,20 @@ def search(self, query=None, uris=None, exact=False):
         if not query.get('any'):
             return None
 
-        categories = set()
-        countries = []
-
+        filters = {}
         for uri in uris or []:
-            variant, identifier = translator.parse_uri(uri)
+            variant, identifier, _ = translator.parse_uri(uri)
             if variant == 'country':
-                countries.append(identifier.lower())
+                filters['country'] = identifier
             elif variant == 'continent':
-                countries.extend(self.backend.dirble.countries(identifier))
+                pass
             elif variant == 'category':
-                pending = [self.backend.dirble.category(identifier)]
-                while pending:
-                    c = pending.pop(0)
-                    categories.add(c['id'])
-                    pending.extend(c['children'])
+                filters['category'] = identifier
 
         tracks = []
-        for station in self.backend.dirble.search(' '.join(query['any'])):
-            if countries and station['country'].lower() not in countries:
-                continue
-            station_categories = {c['id'] for c in station['categories']}
-            if categories and not station_categories.intersection(categories):
-                continue
+        query = ' '.join(query['any'])
+        stations, _ = self.backend.dirble.search(query, limit=20, **filters)
+        for station in stations:
             tracks.append(translator.station_to_track(station))
 
         return SearchResult(tracks=tracks)
@@ -120,7 +132,7 @@ def get_images(self, uris):
         for uri in uris:
             result[uri] = []
 
-            variant, identifier = translator.parse_uri(uri)
+            variant, identifier, _ = translator.parse_uri(uri)
             if variant != 'station' or not identifier:
                 continue
 
@@ -138,7 +150,7 @@ def get_images(self, uris):
 class DirblePlayback(backend.PlaybackProvider):
 
     def translate_uri(self, uri):
-        variant, identifier = translator.parse_uri(uri)
+        variant, identifier, _ = translator.parse_uri(uri)
         if variant != 'station':
             return None
 

mopidy_dirble/client.py

@@ -1,12 +1,13 @@
 from __future__ import unicode_literals
 
 import logging
+import math
+import os.path
 import time
-import urllib
 
 from mopidy import __version__ as mopidy_version
 
-from requests import Session, exceptions
+from requests import Request, Session, exceptions
 from requests.adapters import HTTPAdapter
 
 from mopidy_dirble import __version__ as dirble_version
@@ -18,6 +19,22 @@ def _normalize_keys(data):
     return {k.lower(): v for k, v in data.items()}
 
 
+def _repaginate(fetch, offset, limit, page_size):
+    result = []
+    page = int(math.floor(offset / float(page_size)))
+    discard = offset - page * page_size
+
+    while len(result) < limit + 1:
+        fetched = fetch(page)
+        result.extend(fetched[discard:])
+        if len(fetched) < page_size:
+            break
+        page += 1
+        discard = 0
+
+    return result[:limit], offset + limit if len(result) > limit else None
+
+
 class Dirble(object):
     """Light wrapper for Dirble API lookup.
 
@@ -42,7 +59,7 @@ def __init__(self, api_key, timeout):
         self._backoff_max = 60
         self._backoff = 1
 
-        self._base_uri = 'http://api.dirble.com/v2/'
+        self._base_uri = 'https://api.dirble.com/v2/'
 
         self._session = Session()
         self._session.params = {'token': api_key}
@@ -75,18 +92,21 @@ def subcategories(self, identifier):
         category = self.category(identifier)
         return (category or {}).get('children', [])
 
-    def stations(self, category=None, country=None):
+    def stations(self, category=None, country=None, offset=0, limit=20):
         if category and not country:
             path = 'category/%s/stations' % category
         elif country and not category:
-            path = 'countries/%s/stations?all=1' % country.lower()
+            path = 'countries/%s/stations' % country.lower()
         else:
             return []
 
-        stations = self._fetch(path, [])
+        def fetch(page):
+            return self._fetch(path, [], {'page': page, 'per_page': 30})
+
+        stations, next_offset = _repaginate(fetch, offset, limit, 30)
         for station in stations:
             self._stations.setdefault(station['id'], station)
-        return stations
+        return stations, next_offset
 
     def station(self, identifier):
         identifier = int(identifier)  # Ensure we are consistent for cache key.
@@ -114,38 +134,49 @@ def country(self, country_code):
                 self._countries[c['country_code'].lower()] = c
         return self._countries.get(country_code.lower())
 
-    def search(self, query):
-        quoted_query = urllib.quote(query.encode('utf-8'))
-        stations = self._fetch('search/%s' % quoted_query, [])
+    def search(self, query, category=None, country=None, offset=0, limit=20):
+        params = {'query': query, 'per_page': 30}
+        if category is not None:
+            params['category'] = category
+        if country is not None:
+            params['country'] = country.upper()
+
+        def fetch(page):
+            params['page'] = page
+            return self._fetch('search', [], params, 'POST')
+
+        stations, next_offset = _repaginate(fetch, offset, limit, 30)
         for station in stations:
             self._stations.setdefault(station['id'], station)
-        return stations
+        return stations, next_offset
 
-    def _fetch(self, path, default):
+    def _fetch(self, path, default, params=None, method='GET'):
         # Give up right away if we know the token is bad.
         if self._invalid_token:
             return default
 
-        uri = self._base_uri + path
+        request = Request(
+            method, os.path.join(self._base_uri, path), params=params)
+        prepared = self._session.prepare_request(request)
 
         # Try and serve request from our cache.
-        if uri in self._cache:
-            logger.debug('Cache hit: %s', uri)
-            return self._cache[uri]
+        if prepared.url in self._cache:
+            logger.debug('Cache hit: %s', prepared.url)
+            return self._cache[prepared.url]
 
         # Check if we should back of sending queries.
         if time.time() < self._backoff_until:
-            logger.debug('Back off fallback used: %s', uri)
+            logger.debug('Back off fallback used: %s', prepared.url)
             return default
 
         try:
-            logger.debug('Fetching: %s', uri)
-            resp = self._session.get(uri, timeout=self._timeout)
+            logger.debug('Fetching: %s', prepared.url)
+            resp = self._session.send(prepared, timeout=self._timeout)
 
             # Get succeeded, convert JSON, normalize and return.
             if resp.status_code == 200:
                 data = resp.json(object_hook=_normalize_keys)
-                self._cache[uri] = data
+                self._cache[prepared.url] = data
                 self._backoff = 1
                 return data
 

mopidy_dirble/translator.py

@@ -1,26 +1,59 @@
 from __future__ import unicode_literals
 
-import re
+import collections
 
 from mopidy.models import Ref, Track
 
 
-def unparse_uri(variant, identifier):
-    return b'dirble:%s:%s' % (variant, identifier)
+DirbleURI = collections.namedtuple(
+    'DirbleURI', ['variant', 'identifier', 'page'])
+
+
+def unparse_uri(variant, identifier, page=None):
+    uri = b'dirble:%s:%s' % (variant, identifier)
+    if page is not None:
+        uri += b':%s' % page
+    return uri
 
 
 def parse_uri(uri):
-    result = re.findall(r'^dirble:([a-z]+)(?::(\d+|[a-z]{2}))?$', uri)
-    if result:
-        return result[0]
-    return None, None
+    parts = uri.split(':')
+    none = DirbleURI(None, None, None)
+
+    if tuple(parts) == ('dirble', 'root'):
+        return DirbleURI(parts[1], None, None)
+
+    if len(parts) not in (3, 4):
+        return none
+
+    if parts[0] != 'dirble':
+        return none
+
+    if parts[1] in ('station', 'category', 'continent'):
+        if not parts[2].isdigit():
+            return none
+    elif parts[1] in ('country'):
+        if len(parts[2]) != 2 or not parts[2].isalpha():
+            return none
+    else:
+        return none
+
+    offset = None
+    if len(parts) == 4:
+        if parts[1] not in ('category', 'country'):
+            return none
+        if not parts[3].isdigit():
+            return none
+        offset = int(parts[3])
+
+    return DirbleURI(parts[1], parts[2], offset)
 
 
 def station_to_ref(station, show_country=True):
     name = station.get('name').strip()  # TODO: fallback to streams URI?
-    if show_country:
+    if show_country and 'country' in station:
         # TODO: make this a setting so users can set '$name [$country]' etc?
-        name = '%s [%s]' % (name, station.get('country', '??'))
+        name = '%s [%s]' % (name, station['country'])
     uri = unparse_uri('station', station['id'])
     return Ref.track(uri=uri, name=name)