From 330ecf723ea8f716f436039e7ba77e9310df0fcd Mon Sep 17 00:00:00 2001 From: David Vega Lichacz <7826728+realdavidvega@users.noreply.github.com> Date: Tue, 1 Oct 2024 13:17:54 +0200 Subject: [PATCH 1/5] feat: add the ability of removing all expired entries when one is expired --- .../xef/llm/assistants/CachedTool.kt | 79 ++++++++++++------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt index ddb1e3352..9f97b1d09 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt @@ -9,10 +9,28 @@ data class CachedToolKey(val value: K, val seed: String) data class CachedToolValue(val value: V, val timestamp: Long) +data class CachedToolConfig( + val timeCachePolicy: Duration, + val cacheExpirationPolicy: CacheExpirationPolicy +) { + enum class CacheExpirationPolicy { + /** Removes the expired entry when found */ + REMOVE_EXPIRED_ENTRY, + /** Removes all expired entries when one expired entry found */ + REMOVE_ALL_EXPIRED_ENTRIES + } + companion object { + val Default = CachedToolConfig( + timeCachePolicy = 1.days, + cacheExpirationPolicy = CacheExpirationPolicy.REMOVE_ALL_EXPIRED_ENTRIES + ) + } +} + abstract class CachedTool( private val cache: Atomic, CachedToolValue>>, private val seed: String, - private val timeCachePolicy: Duration = 1.days + private val config: CachedToolConfig = CachedToolConfig.Default ) : Tool { /** @@ -49,44 +67,51 @@ abstract class CachedTool( else onCacheMissed(input) /** - * Exposes the cache as a [Map] of [Input] to [Output] filtered by instance [seed] and - * [timeCachePolicy]. Removes expired cache entries. + * Exposes the cache as a [Map] of [Input] to [Output] filtered by instance [seed]. Removes expired cache entries. * * @return the map of input to output. */ suspend fun getCache(): Map { - val lastTimeInCache = timeInMillis() - timeCachePolicy.inWholeMilliseconds val withoutExpired = cache.modify { cachedToolInfo -> - // Filter entries belonging to the current seed and have not expired - val validEntries = - cachedToolInfo - .filter { (key, value) -> - if (key.seed == seed) lastTimeInCache <= value.timestamp else true - } - .toMutableMap() - // Remove expired entries for the current seed only - cachedToolInfo.keys.removeAll { key -> key.seed == seed && !validEntries.containsKey(key) } + // Filter entries not expired + val validEntries = cachedToolInfo.filterExpired() + // Filter entries belonging to the current seed + val withoutExpired = validEntries.filter { (key, _) -> key.seed == seed } // Modifies state A, and returns state B - Pair(cachedToolInfo, validEntries) + Pair(validEntries, withoutExpired) } return withoutExpired.map { it.key.value to it.value.value }.toMap() } - private suspend fun cache(input: CachedToolKey, block: suspend () -> Output): Output { - val cachedToolInfo = cache.get()[input] - if (cachedToolInfo != null) { - val lastTimeInCache = timeInMillis() - timeCachePolicy.inWholeMilliseconds - if (lastTimeInCache > cachedToolInfo.timestamp) { - cache.get().remove(input) - } else { - return cachedToolInfo.value + private suspend fun cache(input: CachedToolKey, block: suspend () -> Output): Output = + cache.modify { cachedToolInfo -> + cachedToolInfo[input]?.let { output -> + if (output.isExpired()) { + val updatedCache = when (config.cacheExpirationPolicy) { + CachedToolConfig.CacheExpirationPolicy.REMOVE_EXPIRED_ENTRY -> cachedToolInfo.apply { remove(input) } + CachedToolConfig.CacheExpirationPolicy.REMOVE_ALL_EXPIRED_ENTRIES -> cachedToolInfo.filterExpired() + } + Pair(updatedCache, null) + } + else Pair(cachedToolInfo, output.value) + } ?: Pair(cachedToolInfo, null) + } ?: run { + val response = block() + if (shouldCacheOutput(input.value, response)) { + cache.update { cachedToolInfo -> + cachedToolInfo[input] = CachedToolValue(response, timeInMillis()) + cachedToolInfo + } } + response } - val response = block() - if (shouldCacheOutput(input.value, response)) { - cache.get()[input] = CachedToolValue(response, timeInMillis()) - } - return response + + private fun MutableMap, CachedToolValue>.filterExpired() = + this.filter { (_, value) -> !value.isExpired() }.toMutableMap() + + private fun CachedToolValue.isExpired(): Boolean { + val lastTimeInCache = timeInMillis() - config.timeCachePolicy.inWholeMilliseconds + return lastTimeInCache > timestamp } } From 9bed278ab57c72f6697017ed22db392267488b0e Mon Sep 17 00:00:00 2001 From: David Vega Lichacz <7826728+realdavidvega@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:59:35 +0200 Subject: [PATCH 2/5] feat: add cached tool expiration policies --- .../xef/llm/assistants/CachedTool.kt | 73 +++++++++++-------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt index 9f97b1d09..82dd27bc7 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt @@ -15,24 +15,35 @@ data class CachedToolConfig( ) { enum class CacheExpirationPolicy { /** Removes the expired entry when found */ - REMOVE_EXPIRED_ENTRY, + REMOVE_SINGLE_EXPIRED, /** Removes all expired entries when one expired entry found */ - REMOVE_ALL_EXPIRED_ENTRIES + REMOVE_ALL_EXPIRED } + companion object { - val Default = CachedToolConfig( - timeCachePolicy = 1.days, - cacheExpirationPolicy = CacheExpirationPolicy.REMOVE_ALL_EXPIRED_ENTRIES - ) + val Default = + CachedToolConfig( + timeCachePolicy = 1.days, + cacheExpirationPolicy = CacheExpirationPolicy.REMOVE_ALL_EXPIRED + ) } } +/** + * Tool that caches the result of the execution of [onCacheMissed] if [shouldUseCache] returns true. + * Otherwise, returns the result of [onCacheMissed]. This output is added to the cache when + * [shouldCacheOutput] returns true. + * + * Cache is stored in a [Map] of [CachedToolKey] to [CachedToolValue]. + * + * Supports expiration policies using [CachedToolConfig]. Expiration happens during access to the + * cache. + */ abstract class CachedTool( private val cache: Atomic, CachedToolValue>>, private val seed: String, private val config: CachedToolConfig = CachedToolConfig.Default ) : Tool { - /** * Logic to be executed when the cache is missed. * @@ -67,45 +78,45 @@ abstract class CachedTool( else onCacheMissed(input) /** - * Exposes the cache as a [Map] of [Input] to [Output] filtered by instance [seed]. Removes expired cache entries. + * Returns a snapshot of the cache as a [Map] of [Input] to [Output] filtered by instance [seed] + * and removing expired cache entries. Does not modify the cache. * * @return the map of input to output. */ - suspend fun getCache(): Map { - val withoutExpired = + suspend fun getValidCacheSnapshot(): Map { + val validEntries = cache.modify { cachedToolInfo -> - // Filter entries not expired - val validEntries = cachedToolInfo.filterExpired() - // Filter entries belonging to the current seed - val withoutExpired = validEntries.filter { (key, _) -> key.seed == seed } - // Modifies state A, and returns state B - Pair(validEntries, withoutExpired) + val validEntries = cachedToolInfo.filterExpired().filter { (key, _) -> key.seed == seed } + Pair(cachedToolInfo, validEntries) } - return withoutExpired.map { it.key.value to it.value.value }.toMap() + return validEntries.map { it.key.value to it.value.value }.toMap() } private suspend fun cache(input: CachedToolKey, block: suspend () -> Output): Output = cache.modify { cachedToolInfo -> cachedToolInfo[input]?.let { output -> if (output.isExpired()) { - val updatedCache = when (config.cacheExpirationPolicy) { - CachedToolConfig.CacheExpirationPolicy.REMOVE_EXPIRED_ENTRY -> cachedToolInfo.apply { remove(input) } - CachedToolConfig.CacheExpirationPolicy.REMOVE_ALL_EXPIRED_ENTRIES -> cachedToolInfo.filterExpired() - } + val updatedCache = + when (config.cacheExpirationPolicy) { + CachedToolConfig.CacheExpirationPolicy.REMOVE_SINGLE_EXPIRED -> + cachedToolInfo.apply { remove(input) } + CachedToolConfig.CacheExpirationPolicy.REMOVE_ALL_EXPIRED -> + cachedToolInfo.filterExpired() + } Pair(updatedCache, null) - } - else Pair(cachedToolInfo, output.value) + } else Pair(cachedToolInfo, output.value) } ?: Pair(cachedToolInfo, null) - } ?: run { - val response = block() - if (shouldCacheOutput(input.value, response)) { - cache.update { cachedToolInfo -> - cachedToolInfo[input] = CachedToolValue(response, timeInMillis()) - cachedToolInfo + } + ?: run { + val response = block() + if (shouldCacheOutput(input.value, response)) { + cache.update { cachedToolInfo -> + cachedToolInfo[input] = CachedToolValue(response, timeInMillis()) + cachedToolInfo + } } + response } - response - } private fun MutableMap, CachedToolValue>.filterExpired() = this.filter { (_, value) -> !value.isExpired() }.toMutableMap() From fdb5b67ec19593161d10422e0ddfd60279c0a99f Mon Sep 17 00:00:00 2001 From: David Vega Lichacz <7826728+realdavidvega@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:16:44 +0200 Subject: [PATCH 3/5] feat: eviction and expiration policies --- .../xef/llm/assistants/CachedTool.kt | 60 ++++++++++++++----- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt index 82dd27bc7..bb013696d 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt @@ -7,24 +7,44 @@ import kotlin.time.Duration.Companion.days data class CachedToolKey(val value: K, val seed: String) -data class CachedToolValue(val value: V, val timestamp: Long) +data class CachedToolValue(val value: V, val accessTimestamp: Long, val writeTimestamp: Long) { + fun withAccessTimestamp() = copy(accessTimestamp = timeInMillis()) + + companion object { + fun withActualResponse(response: V): CachedToolValue = + CachedToolValue( + value = response, + accessTimestamp = timeInMillis(), + writeTimestamp = timeInMillis() + ) + } +} data class CachedToolConfig( val timeCachePolicy: Duration, - val cacheExpirationPolicy: CacheExpirationPolicy + val cacheExpirationPolicy: CacheExpirationPolicy, + val cacheEvictionPolicy: CacheEvictionPolicy ) { enum class CacheExpirationPolicy { + /** Last access time is used to determine expiration */ + ACCESS, + /** Last write time is used to determine expiration */ + WRITE + } + + enum class CacheEvictionPolicy { /** Removes the expired entry when found */ - REMOVE_SINGLE_EXPIRED, + EVICT_SINGLE_EXPIRED, /** Removes all expired entries when one expired entry found */ - REMOVE_ALL_EXPIRED + EVICT_ALL_EXPIRED } companion object { val Default = CachedToolConfig( timeCachePolicy = 1.days, - cacheExpirationPolicy = CacheExpirationPolicy.REMOVE_ALL_EXPIRED + cacheEvictionPolicy = CacheEvictionPolicy.EVICT_ALL_EXPIRED, + cacheExpirationPolicy = CacheExpirationPolicy.WRITE ) } } @@ -79,7 +99,7 @@ abstract class CachedTool( /** * Returns a snapshot of the cache as a [Map] of [Input] to [Output] filtered by instance [seed] - * and removing expired cache entries. Does not modify the cache. + * and removing expired cache entries with the given [config] policies. Does not modify the cache. * * @return the map of input to output. */ @@ -97,21 +117,24 @@ abstract class CachedTool( cachedToolInfo[input]?.let { output -> if (output.isExpired()) { val updatedCache = - when (config.cacheExpirationPolicy) { - CachedToolConfig.CacheExpirationPolicy.REMOVE_SINGLE_EXPIRED -> + when (config.cacheEvictionPolicy) { + CachedToolConfig.CacheEvictionPolicy.EVICT_SINGLE_EXPIRED -> cachedToolInfo.apply { remove(input) } - CachedToolConfig.CacheExpirationPolicy.REMOVE_ALL_EXPIRED -> + CachedToolConfig.CacheEvictionPolicy.EVICT_ALL_EXPIRED -> cachedToolInfo.filterExpired() } Pair(updatedCache, null) - } else Pair(cachedToolInfo, output.value) + } else { + val updatedOutput = output.withAccessTimestamp() + Pair(cachedToolInfo, updatedOutput.value) + } } ?: Pair(cachedToolInfo, null) } ?: run { val response = block() if (shouldCacheOutput(input.value, response)) { cache.update { cachedToolInfo -> - cachedToolInfo[input] = CachedToolValue(response, timeInMillis()) + cachedToolInfo[input] = CachedToolValue.withActualResponse(response) cachedToolInfo } } @@ -121,8 +144,15 @@ abstract class CachedTool( private fun MutableMap, CachedToolValue>.filterExpired() = this.filter { (_, value) -> !value.isExpired() }.toMutableMap() - private fun CachedToolValue.isExpired(): Boolean { - val lastTimeInCache = timeInMillis() - config.timeCachePolicy.inWholeMilliseconds - return lastTimeInCache > timestamp - } + private fun CachedToolValue.isExpired(): Boolean = + when (config.cacheExpirationPolicy) { + CachedToolConfig.CacheExpirationPolicy.ACCESS -> { + val lastTimeInCache = timeInMillis() - accessTimestamp + lastTimeInCache > config.timeCachePolicy.inWholeMilliseconds + } + CachedToolConfig.CacheExpirationPolicy.WRITE -> { + val lastTimeInCache = timeInMillis() - writeTimestamp + lastTimeInCache > config.timeCachePolicy.inWholeMilliseconds + } + } } From dc7b0efa936b88c4eb7640d89e6a72c7273b769f Mon Sep 17 00:00:00 2001 From: David Vega Lichacz <7826728+realdavidvega@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:42:44 +0200 Subject: [PATCH 4/5] docs: update comments --- .../com/xebia/functional/xef/llm/assistants/CachedTool.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt index bb013696d..64ad07e1f 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt @@ -56,8 +56,7 @@ data class CachedToolConfig( * * Cache is stored in a [Map] of [CachedToolKey] to [CachedToolValue]. * - * Supports expiration policies using [CachedToolConfig]. Expiration happens during access to the - * cache. + * Supports expiration policies using [CachedToolConfig]. */ abstract class CachedTool( private val cache: Atomic, CachedToolValue>>, From e23a7a4000f4de69c51311dca34e3b2a2bb8aea0 Mon Sep 17 00:00:00 2001 From: David Vega Lichacz <7826728+realdavidvega@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:47:20 +0200 Subject: [PATCH 5/5] refactor: change naming --- .../functional/xef/llm/assistants/CachedTool.kt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt index 64ad07e1f..61cfdee57 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/assistants/CachedTool.kt @@ -25,6 +25,8 @@ data class CachedToolConfig( val cacheExpirationPolicy: CacheExpirationPolicy, val cacheEvictionPolicy: CacheEvictionPolicy ) { + + /** Policy to expire the entries in the cache, based on last access or last write time. */ enum class CacheExpirationPolicy { /** Last access time is used to determine expiration */ ACCESS, @@ -32,18 +34,19 @@ data class CachedToolConfig( WRITE } + /** Policy to evict the expired entries from the cache, based on one or all expired entries. */ enum class CacheEvictionPolicy { /** Removes the expired entry when found */ - EVICT_SINGLE_EXPIRED, + SINGLE, /** Removes all expired entries when one expired entry found */ - EVICT_ALL_EXPIRED + ALL } companion object { val Default = CachedToolConfig( timeCachePolicy = 1.days, - cacheEvictionPolicy = CacheEvictionPolicy.EVICT_ALL_EXPIRED, + cacheEvictionPolicy = CacheEvictionPolicy.ALL, cacheExpirationPolicy = CacheExpirationPolicy.WRITE ) } @@ -117,10 +120,8 @@ abstract class CachedTool( if (output.isExpired()) { val updatedCache = when (config.cacheEvictionPolicy) { - CachedToolConfig.CacheEvictionPolicy.EVICT_SINGLE_EXPIRED -> - cachedToolInfo.apply { remove(input) } - CachedToolConfig.CacheEvictionPolicy.EVICT_ALL_EXPIRED -> - cachedToolInfo.filterExpired() + CachedToolConfig.CacheEvictionPolicy.SINGLE -> cachedToolInfo.apply { remove(input) } + CachedToolConfig.CacheEvictionPolicy.ALL -> cachedToolInfo.filterExpired() } Pair(updatedCache, null) } else {