Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add optional redis caching #26

Merged
merged 12 commits into from
Feb 19, 2024
Prev Previous commit
Next Next commit
Refactor
  • Loading branch information
syeopite committed Feb 17, 2024
commit 21c8ed3d1d8fce6446018bfb25ec8705eb59c48d
30 changes: 18 additions & 12 deletions src/cache/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,13 @@
import orjson

class AccessCache(abc.ABC):
@abc.abstractmethod
def get_key(self) -> typing.Tuple[str, str]:
"""Creates a key to get/store an item within the cache"""
pass
def __init__(self, ctx, prefix, cache_ttl, continuation=None, **kwargs):
self.ctx = ctx
self.prefix = prefix
self.cache_ttl = cache_ttl

self.continuation = continuation
self.kwargs = kwargs

@abc.abstractmethod
def fetch(self) -> typing.Dict[str, typing.Any]:
Expand All @@ -19,17 +22,20 @@ def parse(self, initial_results):
"""Parses the initial JSON response from Tumblr"""
pass

@property
@abc.abstractmethod
def prefix(self) -> str:
"""The first segment of the key to get/store an item within the cache"""
def build_key(self) -> typing.Tuple[str, str]:
"""Creates a key to get/store an item within the cache"""
pass

@property
@abc.abstractmethod
def cache_ttl(self) -> int:
"""How long the keep the item within the cache"""
pass
def get_key(self):
base_key = self.build_key()

if self.continuation:
full_key_with_continuation = f"{base_key}:{self.continuation}"
else:
full_key_with_continuation = base_key

return base_key, full_key_with_continuation

async def parse_and_cache(self, base_key, full_key_with_continuation, initial_results):
"""Inserts the given results into the cache within the given key
Expand Down
56 changes: 19 additions & 37 deletions src/cache/blogs.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,15 @@

class BlogPostsCache(AccessCache):
def __init__(self, ctx, blog, continuation, **kwargs):
self.ctx = ctx
self.blog = blog
self.continuation = continuation
self.kwargs = kwargs

@property
def prefix(self):
return f"blog:{self.blog}"
super().__init__(
ctx=ctx,
prefix=f"blog:{blog}",
cache_ttl=ctx.PRIVIBLUR_CONFIG.cache.cache_blog_feed_for,
continuation=continuation,
**kwargs
)

@property
def cache_ttl(self):
return self.ctx.PRIVIBLUR_CONFIG.cache.cache_blog_feed_for
self.blog = blog

async def fetch(self):
"""Fetches blog posts from Tumblr"""
Expand All @@ -26,57 +23,42 @@ async def fetch(self):
def parse(self, initial_results):
return priviblur_extractor.parse_blog_timeline(initial_results)

def get_key(self):
def build_key(self):
# blog:<blog_name>:<kwargs>:<continuation>
path_to_cached_results = [self.prefix, ]
for k,v in self.kwargs.items():
if v:
path_to_cached_results.append(f"{k}:{v}")

base_key = ':'.join(path_to_cached_results)

if self.continuation:
full_key_with_continuation = f"{base_key}:{self.continuation}"
else:
full_key_with_continuation = base_key

return base_key, full_key_with_continuation
return ':'.join(path_to_cached_results)


class BlogPostCache(AccessCache):
def __init__(self, ctx, blog, post_id, **kwargs):
self.ctx = ctx
super().__init__(
ctx=ctx,
prefix=f"blog:{blog}:post:{post_id}",
cache_ttl=ctx.PRIVIBLUR_CONFIG.cache.cache_blog_post_for,
**kwargs
)

self.blog = blog
self.post_id = post_id

self.kwargs = kwargs

self.continuation = None

@property
def prefix(self):
return f"blog:{self.blog}:post:{self.post_id}"

@property
def cache_ttl(self):
return self.ctx.PRIVIBLUR_CONFIG.cache.cache_blog_post_for

async def fetch(self):
return await self.ctx.TumblrAPI.blog_post(self.blog, self.post_id, **self.kwargs)

def parse(self, initial_results):
return priviblur_extractor.parse_timeline(initial_results)

def get_key(self):
def build_key(self):
# blog:<blog_name>:post:<post_id>:<kwargs>
path_to_cached_results = [self.prefix, ]
for k,v in self.kwargs.items():
if v:
path_to_cached_results.append(f"{k}:{v}")

base_key = ':'.join(path_to_cached_results)

return base_key, base_key
return ':'.join(path_to_cached_results)


async def get_blog_posts(ctx, blog, continuation=None, **kwargs):
Expand Down
31 changes: 9 additions & 22 deletions src/cache/explore.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,15 @@

class ExploreCache(AccessCache):
def __init__(self, ctx, type_, continuation, fetch_function, **kwargs):
self.ctx = ctx
self.type_ = type_
self.continuation = continuation
super().__init__(
ctx=ctx,
prefix=f"explore:{type_}",
cache_ttl=ctx.PRIVIBLUR_CONFIG.cache.cache_feed_for,
continuation=continuation,
**kwargs
)

self.fetch_function = fetch_function
self.kwargs = kwargs

@property
def prefix(self):
return f"explore:{self.type_}"

@property
def cache_ttl(self):
return self.ctx.PRIVIBLUR_CONFIG.cache.cache_feed_for

async def fetch(self):
"""Fetches search results from Tumblr"""
Expand All @@ -31,16 +26,8 @@ async def fetch(self):
def parse(self, initial_results):
return priviblur_extractor.parse_timeline(initial_results)

def get_key(self):
# explore:<type>:<continuation>
base_key = self.prefix

if self.continuation:
full_key_with_continuation = f"{base_key}:{self.continuation}"
else:
full_key_with_continuation = base_key

return base_key, full_key_with_continuation
def build_key(self):
return self.prefix


async def get_explore_results(ctx, fetch_function, type_, continuation, **kwargs):
Expand Down
32 changes: 12 additions & 20 deletions src/cache/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,15 @@

class SearchCache(AccessCache):
def __init__(self, ctx, query, continuation, **kwargs):
self.ctx = ctx
self.query = query
self.continuation = continuation
self.kwargs = kwargs

@property
def prefix(self):
return "search"
super().__init__(
ctx=ctx,
prefix=f"search",
cache_ttl=ctx.PRIVIBLUR_CONFIG.cache.cache_feed_for,
continuation=continuation,
**kwargs
)

@property
def cache_ttl(self):
return self.ctx.PRIVIBLUR_CONFIG.cache.cache_feed_for
self.query = query

async def fetch(self):
"""Fetches search results from Tumblr"""
Expand All @@ -31,26 +28,21 @@ async def fetch(self):
def parse(self, initial_results):
return priviblur_extractor.parse_timeline(initial_results)

def get_key(self):
def build_key(self):
# search:<query>:<latest>:<post_filter>:<time_filter>:<continuation>
path_to_cached_results = [self.query, ]

if self.kwargs.get("latest") is True:
path_to_cached_results.append("latest")

if post_filter := self.kwargs.get("post_type_filter"):
path_to_cached_results.append(post_filter.name.lower())

if days := self.kwargs.get("days"):
path_to_cached_results.append(days)

base_key = f"{self.prefix}:{':'.join(path_to_cached_results)}"

if self.continuation:
full_key_with_continuation = f"{base_key}:{self.continuation}"
else:
full_key_with_continuation = base_key
return f"{self.prefix}:{':'.join(path_to_cached_results)}"

return base_key, full_key_with_continuation


async def get_search_results(ctx, query, continuation=None, **kwargs):
search_cache = SearchCache(ctx, query, continuation, **kwargs)
Expand Down
37 changes: 14 additions & 23 deletions src/cache/tagged.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,43 +5,34 @@


class TagBrowseCache(AccessCache):
def __init__(self, ctx, tag, latest, continuation):
self.ctx = ctx
def __init__(self, ctx, tag, latest, continuation, **kwargs):
super().__init__(
ctx=ctx,
prefix=f"tagged",
cache_ttl=ctx.PRIVIBLUR_CONFIG.cache.cache_feed_for,
continuation=continuation,
**kwargs
)

self.tag = tag
self.latest = latest
self.continuation = continuation

@property
def prefix(self):
return "tagged"

@property
def cache_ttl(self):
return self.ctx.PRIVIBLUR_CONFIG.cache.cache_feed_for

async def fetch(self):
"""Fetches search results from Tumblr"""
"""Fetches posts from Tumblr with the given tag"""
return await self.ctx.TumblrAPI.hubs_timeline(self.tag, latest=self.latest, continuation=self.continuation)

def parse(self, initial_results):
return priviblur_extractor.parse_timeline(initial_results)

def get_key(self):
# search:<query>:<latest>:<post_filter>:<time_filter>:<continuation>
path_to_cached_results = [self.tag]
def build_key(self):
# tagged:<tag>:<latest>:<continuation>
path_to_cached_results = [self.prefix, self.tag]

if self.latest is True:
path_to_cached_results.append("latest")

base_key = f"{self.prefix}:{':'.join(path_to_cached_results)}"

if self.continuation:
full_key_with_continuation = f"{base_key}:{self.continuation}"
else:
full_key_with_continuation = base_key
return ':'.join(path_to_cached_results)

return base_key, full_key_with_continuation


async def get_tag_browse_results(ctx, tag, latest=False, continuation=None):
tag_browse_cache = TagBrowseCache(ctx, tag, latest, continuation)
Expand Down