Improve FHIR Search Count Performance

docs/performance/fhir-search.md

@@ -49,12 +49,12 @@ curl -s "http://localhost:8080/fhir/Observation?code=http://loinc.org|$CODE&_sum
 
 | System | Dataset | Code    | # Hits | Time (s) | StdDev | T/1M ¹ |
 |--------|---------|---------|-------:|---------:|-------:|-------:|
-| LEA47  | 100k    | 8310-5  |  115 k |     0.06 |  0.001 |   0.55 |
-| LEA47  | 100k    | 55758-7 |  1.0 M |     0.50 |  0.012 |   0.49 |
-| LEA47  | 100k    | 72514-3 |  2.7 M |     1.35 |  0.056 |   0.49 |
-| LEA58  | 100k    | 8310-5  |  115 k |     0.06 |  0.001 |   0.55 |
-| LEA58  | 100k    | 55758-7 |  1.0 M |     0.52 |  0.011 |   0.51 |
-| LEA58  | 100k    | 72514-3 |  2.7 M |     1.39 |  0.027 |   0.50 |
+| LEA47  | 100k    | 8310-5  |  115 k |     0.06 |  0.001 |   0.54 |
+| LEA47  | 100k    | 55758-7 |  1.0 M |     0.50 |  0.010 |   0.49 |
+| LEA47  | 100k    | 72514-3 |  2.7 M |     1.34 |  0.026 |   0.48 |
+| LEA58  | 100k    | 8310-5  |  115 k |     0.07 |  0.002 |   0.57 |
+| LEA58  | 100k    | 55758-7 |  1.0 M |     0.54 |  0.020 |   0.53 |
+| LEA58  | 100k    | 72514-3 |  2.7 M |     1.43 |  0.045 |   0.52 |
 | LEA47  | 1M      | 8310-5  |  1.1 M |     0.59 |  0.006 |   0.50 |
 | LEA47  | 1M      | 55758-7 | 10.1 M |     5.45 |  0.182 |   0.53 |
 | LEA47  | 1M      | 72514-3 | 27.3 M |    14.03 |  0.135 |   0.51 |
@@ -140,16 +140,19 @@ curl -s "http://localhost:8080/fhir/Observation?code=http://loinc.org|$CODE&valu
 
 | System | Dataset | Code    | Value | # Hits | Time (s) | StdDev | T/1M ¹ |
 |--------|---------|---------|------:|-------:|---------:|-------:|-------:|
-| LEA47  | 100k    | 29463-7 |  26.8 |  158 k |    10.47 |  0.038 |  66.21 |
-| LEA47  | 100k    | 29463-7 |  79.5 |  790 k |    10.77 |  0.040 |  13.64 |
-| LEA47  | 100k    | 29463-7 |   183 |  1.6 M |    10.73 |  0.028 |   6.77 |
-| LEA58  | 100k    | 29463-7 |  26.8 |  158 k |    10.33 |  0.055 |  65.33 |
-| LEA58  | 100k    | 29463-7 |  79.5 |  790 k |    10.58 |  0.045 |  13.39 |
-| LEA58  | 100k    | 29463-7 |   183 |  1.6 M |    10.95 |  0.037 |   6.91 |
+| LEA47  | 100k    | 29463-7 |  26.8 |  158 k |     0.85 |  0.006 |   5.38 |
+| LEA47  | 100k    | 29463-7 |  79.5 |  790 k |     0.84 |  0.005 |   1.06 |
+| LEA47  | 100k    | 29463-7 |   183 |  1.6 M |     0.85 |  0.007 |   0.54 |
+| LEA58  | 100k    | 29463-7 |  26.8 |  158 k |     0.87 |  0.010 |   5.53 |
+| LEA58  | 100k    | 29463-7 |  79.5 |  790 k |     0.87 |  0.014 |   1.10 |
+| LEA58  | 100k    | 29463-7 |   183 |  1.6 M |     0.89 |  0.005 |   0.55 |
+| LEA58  | 1M      | 29463-7 |  26.8 |  1.6 M |    17.89 |  0.397 |  11.45 |
+| LEA58  | 1M      | 29463-7 |  79.5 |  7.8 M |    17.69 |  0.167 |   2.25 |
+| LEA58  | 1M      | 29463-7 |   183 | 15.9 M |    17.53 |  0.110 |   1.10 |
 
 ¹ time in seconds per 1 million resources
 
-The measurements show that the time Blaze needs to count resources with two search params (code and value-quantity) is constant. In fact it depends only on the number of resources which qualify for the first search parameter which can be seen on the fixed time of 56 seconds.
+The measurements show that the time Blaze needs to count resources with two search params (code and value-quantity) is constant. In fact it depends only on the number of resources which qualify for the first search parameter which can be seen on the fixed time of 4 seconds.
 
 ### Download of Resources
 
@@ -209,12 +212,12 @@ curl -s "http://localhost:8080/fhir/Observation?date=$YEAR&_summary=count"
 
 | System | Dataset | Year | # Hits | Time (s) | StdDev | T/1M ¹ |
 |--------|---------|------|-------:|---------:|-------:|-------:|
-| LEA47  | 100k    | 2013 |  3.1 M |     2.24 |  0.036 |   0.71 |
-| LEA47  | 100k    | 2019 |  6.0 M |     4.17 |  0.136 |   0.69 |
-| LEA58  | 100k    | 2013 |  3.1 M |     2.35 |  0.058 |   0.75 |
-| LEA58  | 100k    | 2019 |  6.0 M |     4.19 |  0.031 |   0.70 |
-| LEA47  | 1M      | 2013 | 31.1 M |    23.31 |  0.206 |   0.75 |
-| LEA47  | 1M      | 2019 | 60.0 M |    45.98 |  0.582 |   0.76 |
+| LEA47  | 100k    | 2013 |  3.1 M |     2.13 |  0.068 |   0.68 |
+| LEA47  | 100k    | 2019 |  6.0 M |     4.17 |  0.175 |   0.69 |
+| LEA58  | 100k    | 2013 |  3.1 M |     2.22 |  0.071 |   0.71 |
+| LEA58  | 100k    | 2019 |  6.0 M |     4.19 |  0.056 |   0.70 |
+| LEA47  | 1M      | 2013 | 31.1 M |    21.28 |  0.439 |   0.68 |
+| LEA47  | 1M      | 2019 | 60.0 M |    42.58 |  1.502 |   0.70 |
 
 ¹ time in seconds per 1 million resources
 

modules/db-protocols/src/blaze/db/impl/protocols.clj

@@ -83,6 +83,8 @@
 
 (defprotocol SearchParam
   (-compile-value [search-param modifier value] "Can return an anomaly.")
+  (-chunked-resource-handles
+    [search-param context tid modifier compiled-value])
   (-resource-handles
     [search-param context tid modifier compiled-value]
     [search-param context tid modifier compiled-value start-id]
@@ -91,10 +93,6 @@
     [search-param context tid direction]
     [search-param context tid direction start-id]
     "Returns a reducible collection.")
-  (-count-resource-handles
-    [search-param context tid modifier compiled-value]
-    "Returns a CompletableFuture that will complete with the count of the
-    matching resource handles.")
   (-compartment-keys [search-param context compartment tid compiled-value])
   (-matcher [_ context modifier values])
   (-compartment-ids [_ resolver resource])

modules/db/src/blaze/db/impl/index.clj

@@ -1,6 +1,6 @@
 (ns blaze.db.impl.index
   (:require
-   [blaze.async.comp :as ac]
+   [blaze.async.comp :as ac :refer [do-sync]]
    [blaze.byte-string :as bs]
    [blaze.coll.core :as coll]
    [blaze.db.impl.codec :as codec]
@@ -54,19 +54,40 @@
        (resource-handles
         search-param context tid modifier values start-id)))))
 
+(defn- resource-handle-chunk-counter [context other-clauses chunk]
+  (transduce
+   (other-clauses-filter context other-clauses)
+   (completing (fn [sum _] (inc sum)))
+   0
+   chunk))
+
+(defn- resource-handle-chunk-counter-mapper [context other-clauses]
+  (map #(ac/supply-async (fn [] (resource-handle-chunk-counter context other-clauses %)))))
+
+(defn- count-resource-handles [xform chunked-resource-handles]
+  (let [futures (into [] xform chunked-resource-handles)]
+    (do-sync [_ (ac/all-of futures)]
+      (transduce (map ac/join) + futures))))
+
+(defn- chunked-resource-handles
+  [search-param context tid modifier values]
+  (condp = modifier
+    "asc" [(search-param/sorted-resource-handles search-param context tid :asc)]
+    "desc" [(search-param/sorted-resource-handles search-param context tid :desc)]
+    (search-param/chunked-resource-handles search-param context tid modifier values)))
+
 (defn type-query-total
   "Returns a CompletableFuture that will complete with the count of the
   matching resource handles."
   [context tid clauses]
   (let [[[search-param modifier _ values] & other-clauses] clauses]
     (if (seq other-clauses)
-      (ac/completed-future
-       (count
-        (coll/eduction
-         (other-clauses-filter context other-clauses)
-         (resource-handles search-param context tid modifier values))))
-      (search-param/count-resource-handles
-       search-param context tid modifier values))))
+      (count-resource-handles
+       (resource-handle-chunk-counter-mapper context other-clauses)
+       (chunked-resource-handles search-param context tid modifier values))
+      (count-resource-handles
+       (map #(ac/supply-async (fn [] (reduce (fn [sum _] (inc sum)) 0 %))))
+       (chunked-resource-handles search-param context tid modifier values)))))
 
 (defn system-query [_ _]
   ;; TODO: implement

modules/db/src/blaze/db/impl/search_param.clj

@@ -1,7 +1,6 @@
 (ns blaze.db.impl.search-param
   (:require
    [blaze.anomaly :as ba :refer [when-ok]]
-   [blaze.async.comp :as ac]
    [blaze.coll.core :as coll]
    [blaze.db.impl.codec :as codec]
    [blaze.db.impl.index.compartment.search-param-value-resource :as c-sp-vr]
@@ -66,19 +65,19 @@
   ([search-param context tid direction start-id]
    (p/-sorted-resource-handles search-param context tid direction start-id)))
 
-(defn count-resource-handles
-  "Returns a CompletableFuture that will complete with the count of the
-  matching resource handles."
+(defn chunked-resource-handles
+  "Returns an reducible collection of chunks of resource handles.
+
+  Each chunk is a CompletableFuture that will complete with reducible
+  collection of matching resource handles."
   [search-param context tid modifier values]
   (if (= 1 (count values))
-    (p/-count-resource-handles search-param context tid modifier (first values))
-    (ac/completed-future
-     (count
-      (coll/eduction
-       (comp
-        (mapcat (partial p/-resource-handles search-param context tid modifier))
-        (distinct))
-       values)))))
+    (p/-chunked-resource-handles search-param context tid modifier (first values))
+    [(coll/eduction
+      (comp
+       (mapcat (partial p/-resource-handles search-param context tid modifier))
+       (distinct))
+      values)]))
 
 (defn- compartment-keys
   "Returns a reducible collection of `[prefix id hash-prefix]` triples."

modules/db/src/blaze/db/impl/search_param/all.clj

@@ -15,6 +15,9 @@
   (reify p/SearchParam
     (-compile-value [_ _ _])
 
+    (-chunked-resource-handles [search-param context tid modifier value]
+      [(p/-resource-handles search-param context tid modifier value)])
+
     (-resource-handles [_ context tid _ _]
       (rao/type-list context tid))
 

modules/db/src/blaze/db/impl/search_param/chained.clj

@@ -1,7 +1,6 @@
 (ns blaze.db.impl.search-param.chained
   (:require
    [blaze.anomaly :as ba :refer [when-ok]]
-   [blaze.async.comp :as ac]
    [blaze.coll.core :as coll]
    [blaze.db.impl.codec :as codec]
    [blaze.db.impl.index :as index]
@@ -27,6 +26,9 @@
   (-compile-value [_ modifier value]
     (p/-compile-value search-param modifier value))
 
+  (-chunked-resource-handles [search-param context tid modifier value]
+    [(p/-resource-handles search-param context tid modifier value)])
+
   (-resource-handles [_ context tid modifier compiled-value]
     (coll/eduction
      (comp (map #(p/-compile-value ref-search-param ref-modifier (rh/reference %)))
@@ -40,10 +42,6 @@
        (drop-while #(not= start-id (rh/id %)))
        (p/-resource-handles this context tid modifier compiled-value))))
 
-  (-count-resource-handles [this context tid modifier compiled-value]
-    (ac/completed-future
-     (count (p/-resource-handles this context tid modifier compiled-value))))
-
   (-matcher [_ context modifier values]
     (filter
      (fn [resource-handle]

modules/db/src/blaze/db/impl/search_param/composite/token_quantity.clj

@@ -45,6 +45,11 @@
                      code (::spq/unsupported-prefix %)))
              %)))))
 
+  (-chunked-resource-handles [_ context tid _ value]
+    (coll/eduction
+     (u/resource-handle-chunk-mapper context tid)
+     (spq/resource-keys context c-hash tid prefix-length value)))
+
   (-resource-handles [_ context tid _ value]
     (coll/eduction
      (u/resource-handle-mapper context tid)
@@ -55,11 +60,6 @@
      (u/resource-handle-mapper context tid)
      (spq/resource-keys context c-hash tid prefix-length value start-id)))
 
-  (-count-resource-handles [_ context tid _ value]
-    (u/count-resource-handles
-     context tid
-     (spq/resource-keys context c-hash tid prefix-length value)))
-
   (-matcher [_ context _ values]
     (spq/matcher context c-hash prefix-length values))
 

modules/db/src/blaze/db/impl/search_param/composite/token_token.clj

@@ -19,6 +19,11 @@
             v2 (cc/compile-component-value c1 v2)]
         (bs/concat v1 v2))))
 
+  (-chunked-resource-handles [_ context tid _ value]
+    (coll/eduction
+     (u/resource-handle-chunk-mapper context tid)
+     (spt/resource-keys context c-hash tid value)))
+
   (-resource-handles [_ context tid _ value]
     (coll/eduction
      (u/resource-handle-mapper context tid)
@@ -29,11 +34,6 @@
      (u/resource-handle-mapper context tid)
      (spt/resource-keys context c-hash tid value start-id)))
 
-  (-count-resource-handles [_ context tid _ value]
-    (u/count-resource-handles
-     context tid
-     (spt/resource-keys context c-hash tid value)))
-
   (-matcher [_ context _ values]
     (r-sp-v/value-prefix-filter (:snapshot context) c-hash values))
 

modules/db/src/blaze/db/impl/search_param/date.clj

@@ -346,6 +346,11 @@
           (ba/unsupported (u/unsupported-prefix-msg code op)))
         #(assoc % ::anom/message (invalid-date-time-value-msg code value)))))
 
+  (-chunked-resource-handles [_ context tid _ value]
+    (coll/eduction
+     (u/resource-handle-chunk-mapper context tid)
+     (resource-keys context c-hash tid value)))
+
   (-resource-handles [_ context tid _ value]
     (coll/eduction
      (u/resource-handle-mapper context tid)
@@ -356,11 +361,6 @@
      (u/resource-handle-mapper context tid)
      (resource-keys context c-hash tid value start-id)))
 
-  (-count-resource-handles [_ context tid _ value]
-    (u/count-resource-handles
-     context tid
-     (resource-keys context c-hash tid value)))
-
   (-sorted-resource-handles [_ context tid direction]
     (coll/eduction
      (comp drop-value

modules/db/src/blaze/db/impl/search_param/has.clj

@@ -2,7 +2,6 @@
   "https://www.hl7.org/fhir/search.html#has"
   (:require
    [blaze.anomaly :as ba :refer [when-ok]]
-   [blaze.async.comp :as ac]
    [blaze.byte-string :as bs]
    [blaze.coll.core :as coll]
    [blaze.db.impl.codec :as codec]
@@ -115,6 +114,9 @@
        (codec/tid type)
        (p/-compile-value search-param nil value)]))
 
+  (-chunked-resource-handles [search-param context tid modifier value]
+    [(p/-resource-handles search-param context tid modifier value)])
+
   (-resource-handles [_ context tid _ value]
     (resource-handles context tid value))
 
@@ -123,10 +125,6 @@
      (drop-lesser-ids (codec/id-string start-id))
      (resource-handles context tid value)))
 
-  (-count-resource-handles [search-param context tid modifier value]
-    (ac/completed-future
-     (count (p/-resource-handles search-param context tid modifier value))))
-
   (-matcher [_ context _ values]
     (filter #(matches? context % values)))
 

modules/db/src/blaze/db/impl/search_param/list.clj

@@ -1,7 +1,6 @@
 (ns blaze.db.impl.search-param.list
   "https://www.hl7.org/fhir/search.html#list"
   (:require
-   [blaze.async.comp :as ac]
    [blaze.byte-string :as bs]
    [blaze.coll.core :as coll]
    [blaze.db.impl.codec :as codec]
@@ -40,6 +39,9 @@
   (-compile-value [_ _ value]
     (codec/id-byte-string value))
 
+  (-chunked-resource-handles [search-param context tid modifier value]
+    [(p/-resource-handles search-param context tid modifier value)])
+
   (-resource-handles [_ context tid _ list-id]
     (when-let [{:keys [hash]} (u/non-deleted-resource-handle
                                context list-tid list-id)]
@@ -50,10 +52,6 @@
                                context list-tid list-id)]
       (referenced-resource-handles context list-id hash tid start-id)))
 
-  (-count-resource-handles [search-param context tid modifier list-id]
-    (ac/completed-future
-     (count (p/-resource-handles search-param context tid modifier list-id))))
-
   (-index-values [_ _ _]
     []))
 

modules/db/src/blaze/db/impl/search_param/number.clj

@@ -63,6 +63,11 @@
                 ::category ::invalid-decimal-value
                 ::anom/message (u/invalid-decimal-value-msg code value)))))
 
+  (-chunked-resource-handles [_ context tid _ value]
+    (coll/eduction
+     (u/resource-handle-chunk-mapper context tid)
+     (spq/resource-keys context c-hash tid 0 value)))
+
   (-resource-handles [_ context tid _ value]
     (coll/eduction
      (u/resource-handle-mapper context tid)
@@ -73,11 +78,6 @@
      (u/resource-handle-mapper context tid)
      (spq/resource-keys context c-hash tid 0 value start-id)))
 
-  (-count-resource-handles [_ context tid _ value]
-    (u/count-resource-handles
-     context tid
-     (spq/resource-keys context c-hash tid 0 value)))
-
   (-matcher [_ context _ values]
     (spq/matcher context c-hash 0 values))
 

modules/db/src/blaze/db/impl/search_param/quantity.clj

@@ -223,6 +223,11 @@
                 ::category ::invalid-decimal-value
                 ::anom/message (u/invalid-decimal-value-msg code value)))))
 
+  (-chunked-resource-handles [_ context tid _ value]
+    (coll/eduction
+     (u/resource-handle-chunk-mapper context tid)
+     (resource-keys context c-hash tid codec/v-hash-size value)))
+
   (-resource-handles [_ context tid _ value]
     (coll/eduction
      (u/resource-handle-mapper context tid)
@@ -233,11 +238,6 @@
      (u/resource-handle-mapper context tid)
      (resource-keys context c-hash tid codec/v-hash-size value start-id)))
 
-  (-count-resource-handles [_ context tid _ value]
-    (u/count-resource-handles
-     context tid
-     (resource-keys context c-hash tid codec/v-hash-size value)))
-
   (-matcher [_ context _ values]
     (matcher context c-hash codec/v-hash-size values))