[WIP] CQL Cache

samply · Aug 7, 2023 · 39118dc · 39118dc
1 parent 41327e6
commit 39118dc
Show file tree

Hide file tree

Showing 171 changed files with 4,354 additions and 1,541 deletions.
diff --git a/.github/scripts/check-patient-as-of-index-missing.sh b/.github/scripts/check-patient-as-of-index-missing.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+. "$SCRIPT_DIR/util.sh"
+
+BASE="http://localhost:8080/fhir"
+curl -s "$BASE/__admin/rocksdb/index/column-families" | jq -r '."column-families"[]' | grep -q "patient-as-of-index"
+
+test "exit code" "$?" "1"
diff --git a/.github/scripts/check-patient-as-of-index-state.sh b/.github/scripts/check-patient-as-of-index-state.sh
@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+. "$SCRIPT_DIR/util.sh"
+
+BASE="http://localhost:8080/fhir"
+STATE="$(curl -s "$BASE/__admin/db/index/column-families/patient-as-of-index/state" | jq -r .type)"
+
+test "state" "$STATE" "$1"
diff --git a/.github/scripts/test-metrics.sh b/.github/scripts/test-metrics.sh
@@ -14,7 +14,3 @@ num-metrics() {
 test "blaze_rocksdb_block_cache_data_miss index" "$(num-metrics "blaze_rocksdb_block_cache_data_miss" "name=\"index\"")" "1"
 test "blaze_rocksdb_block_cache_data_miss transaction" "$(num-metrics "blaze_rocksdb_block_cache_data_miss" "name=\"transaction\"")" "1"
 test "blaze_rocksdb_block_cache_data_miss resource" "$(num-metrics "blaze_rocksdb_block_cache_data_miss" "name=\"resource\"")" "1"
-
-test "blaze_rocksdb_table_reader_usage_bytes index" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"index\"")" "14"
-test "blaze_rocksdb_table_reader_usage_bytes transaction" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"transaction\"")" "1"
-test "blaze_rocksdb_table_reader_usage_bytes resource" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"resource\"")" "1"
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -39,6 +39,7 @@ jobs:
         - anomaly
         - async
         - byte-buffer
+        - cache-collector
         - cassandra
         - coll
         - cql
@@ -1137,6 +1138,50 @@ jobs:
     - name: Fetch Patient Expecting an Error
       run: .github/scripts/fetch-resource-0-with-missing-resource-content.sh
 
+  build-patient-as-of-index-test:
+    needs: build
+    runs-on: ubuntu-22.04
+
+    steps:
+    - name: Check out Git repository
+      uses: actions/checkout@v3
+
+    - name: Install Blazectl
+      run: .github/scripts/install-blazectl.sh
+
+    - name: Download Blaze Image
+      uses: actions/download-artifact@v3
+      with:
+        name: blaze-image
+        path: /tmp
+
+    - name: Load Blaze Image
+      run: docker load --input /tmp/blaze.tar
+
+    - name: Run Blaze v0.22
+      run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_FRONTEND=true -p 8080:8080 -v blaze-data:/app/data samply/blaze:0.22
+
+    - name: Wait for Blaze
+      run: .github/scripts/wait-for-url.sh  http://localhost:8080/health
+
+    - name: Load Data
+      run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea
+
+    - name: Ensure that the PatientAsOf Index does not exist
+      run: .github/scripts/check-patient-as-of-index-missing.sh
+
+    - name: Shut down Blaze
+      run: docker stop blaze && docker rm blaze
+
+    - name: Run Latest Blaze
+      run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_FRONTEND=true -e LOG_LEVEL=debug -p 8080:8080 -v blaze-data:/app/data blaze:latest
+
+    - name: Wait for Blaze
+      run: .github/scripts/wait-for-url.sh  http://localhost:8080/health
+
+    - name: Ensure that the State of PatientAsOf Index is Current
+      run: .github/scripts/check-patient-as-of-index-state.sh current
+
   distributed-test:
     needs: build
     runs-on: ubuntu-22.04
@@ -1623,13 +1668,14 @@ jobs:
     - bundle-with-references-test
     - jepsen-test
     - openid-auth-test
+    - custom-search-parameters-test
     - doc-copy-data-test
     - big-binary-test
     - frontend-test
     - missing-resource-content-test
+    - build-patient-as-of-index-test
     - distributed-test
     - jepsen-distributed-test
-    - custom-search-parameters-test
     runs-on: ubuntu-22.04
     permissions:
       packages: write

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM eclipse-temurin:17.0.7_7-jre-jammy
+FROM eclipse-temurin:17.0.8_7-jre-jammy
 
 RUN apt-get update && apt-get upgrade -y && \
     apt-get install libjemalloc2 -y && \

diff --git a/dev/blaze/dev.clj b/dev/blaze/dev.clj
@@ -1,9 +1,9 @@
 (ns blaze.dev
   (:require
     [blaze.byte-string :as bs]
+    [blaze.cache-collector.protocols :as ccp]
     [blaze.db.api :as d]
     [blaze.db.api-spec]
-    [blaze.db.cache-collector.protocols :as ccp]
     [blaze.db.resource-cache :as resource-cache]
     [blaze.db.resource-store :as rs]
     [blaze.db.tx-log :as tx-log]
@@ -66,6 +66,11 @@
   (resource-cache/invalidate-all! (:blaze.db/resource-cache system))
   )
 
+;; CQL Expression Cache
+(comment
+  (str (ccp/-stats (:blaze.fhir.operation.evaluate-measure/expr-cache system)))
+  )
+
 ;; RocksDB Stats
 (comment
   (.reset (system [:blaze.db.kv.rocksdb/stats :blaze.db.index-kv-store/stats]))

diff --git a/docs/implementation/database.md b/docs/implementation/database.md
@@ -38,16 +38,17 @@ There are two different sets of indices, ones which depend on the database value
 
 ### Indices depending on t
 
-| Name         | Key Parts | Value                         |
-|--------------|-----------|-------------------------------|
-| ResourceAsOf | type id t | content-hash, num-changes, op |
-| TypeAsOf     | type t id | content-hash, num-changes, op |
-| SystemAsOf   | t type id | content-hash, num-changes, op |
-| TxSuccess    | t         | instant                       |
-| TxError      | t         | anomaly                       |
-| TByInstant   | instant   | t                             |
-| TypeStats    | type t    | total, num-changes            |
-| SystemStats  | t         | total, num-changes            |
+| Name         | Key Parts        | Value                         |
+|--------------|------------------|-------------------------------|
+| ResourceAsOf | type id t        | content-hash, num-changes, op |
+| TypeAsOf     | type t id        | content-hash, num-changes, op |
+| SystemAsOf   | t type id        | content-hash, num-changes, op |
+| PatientAsOf  | pat-id t type id | content-hash, num-changes, op |
+| TxSuccess    | t                | instant                       |
+| TxError      | t                | anomaly                       |
+| TByInstant   | instant          | t                             |
+| TypeStats    | type t           | total, num-changes            |
+| SystemStats  | t                | total, num-changes            |
 
 #### ResourceAsOf
 
@@ -83,12 +84,16 @@ In addition to direct resource lookup, the `ResourceAsOf` index is used for list
 
 #### TypeAsOf
 
-The `TypeAsOf` index contains the same information as the `ResourceAsOf` index with the difference that the components of the key are ordered `type`,  `t` and  `id` instead of `type`, `id` and `t`. The index is used for listing all versions of all resources of a particular type. Such history listings start with the `t` of the database value going into the past. This is done by not only choosing the resource version with the latest `t` less or equal the database values `t` but instead using all older versions. Such versions even include deleted versions because in FHIR it is allowed to bring back a resource to a new life after it was already deleted. The listing is done by simply scanning through the index in reverse. Because the key is ordered by `type`,  `t` and  `id`, the entries will be first ordered by time, newest first, and second by resource identifier.
+The `TypeAsOf` index contains the same information as the `ResourceAsOf` index with the difference that the components of the key are ordered `type`,  `t` and `id` instead of `type`, `id` and `t`. The index is used for listing all versions of all resources of a particular type. Such history listings start with the `t` of the database value going into the past. This is done by not only choosing the resource version with the latest `t` less or equal the database values `t` but instead using all older versions. Such versions even include deleted versions because in FHIR it is allowed to bring back a resource to a new life after it was already deleted. The listing is done by simply scanning through the index in reverse. Because the key is ordered by `type`,  `t` and  `id`, the entries will be first ordered by time, newest first, and second by resource identifier.
 
 #### SystemAsOf
 
 In the same way the `TypeAsOf` index uses a different key ordering in comparison to the `ResourceAsOf` index, the `SystemAsOf` index will use the key order `t`, `type` and `id` in order to provide a global time axis order by resource type and by identifier secondarily.
 
+#### PatientAsOf
+
+The `PatientAsOf` index works like the `SystemAsOf` index but for each Patient individually. It contains all changes to resources in the compartment of a particular Patient on reverse chronological order. Using the `PatientAsOf` index it's possible to create a history of all changes in the Patient compartment or detect the `t` of the last change. The CQL cache uses this index to invalidate cached results of expressions in the Patient context. 
+
 #### TxSuccess
 
 The `TxSuccess` index contains the real point in time, as `java.time.Instant`, successful transactions happened. In other words, this index maps each `t` which is just a monotonically increasing number to a real point in time. 
@@ -115,23 +120,24 @@ The `SystemStats` index keeps track of the total number of resources, and the nu
 
 The indices not depending on `t` directly point to the resource versions by their content hash. 
 
-| Name                                | Key Parts                                                        | Value |
-|-------------------------------------|------------------------------------------------------------------|-------|
-| SearchParamValueResource            | search-param, type, value, id, content-hash                      | -     |
-| ResourceSearchParamValue            | type, id, content-hash, search-param, value                      | -     |
-| CompartmentSearchParamValueResource | co-c-hash, co-res-id, search-param, type, value, id, hash-prefix | -     |
-| CompartmentResource                 | co-c-hash, co-res-id, tid, id                                    | -     |
-| SearchParam                         | code, tid                                                        | id    |
-| ActiveSearchParams                  | id                                                               | -     |
+| Name                                | Key Parts                                                      | Value |
+|-------------------------------------|----------------------------------------------------------------|-------|
+| SearchParamValueResource            | search-param, type, value, id, hash-prefix                     | -     |
+| ResourceSearchParamValue            | type, id, content-hash, search-param, value                    | -     |
+| CompartmentSearchParamValueResource | comp-code, comp-id, search-param, type, value, id, hash-prefix | -     |
+| CompartmentResource                 | comp-code, comp-id, type, id                                   | -     |
+| SearchParam                         | code, type                                                     | id    |
+| ActiveSearchParams                  | id                                                             | -     |
 
 #### SearchParamValueResource
 
 The `SearchParamValueResource` index contains all values from resources that are reachable from search parameters. The components of its key are:
-* `search-param` - a 4-byte hash of the search parameters code used to identify the search parameter
-* `type` - a 4-byte hash of the resource type
-* `value` - the encoded value of the resource reachable by the search parameters FHIRPath expression. The encoding depends on the search parameters type.
-* `id` - the logical id of the resource
-* `content-hash` - a 4-byte prefix of the content-hash of the resource version
+
+ * `search-param` - a 4-byte hash of the search parameters code used to identify the search parameter
+ * `type` - a 4-byte hash of the resource type
+ * `value` - the encoded value of the resource reachable by the search parameters FHIRPath expression. The encoding depends on the search parameters type.
+ * `id` - the logical id of the resource
+ * `content-hash` - a 4-byte prefix of the content-hash of the resource version
 
 The way the `SearchParamValueResource` index is used, depends on the type of the search parameter. The following sections will explain this in detail for each type:
 
@@ -207,11 +213,20 @@ That tuples are further processed against the `ResourceAsOf` index in order to c
 
 **TODO: continue...**
 
+#### CompartmentResource
+
+The `CompartmentResource` index contains all resources that belong to a certain compartment. The components of its key are:
+
+ * `comp-code` - a 4-byte hash of the compartment code, ex. `Patient`
+ * `comp-id` - the logical id of the compartment, ex. the logical id of the Patient
+ * `type` - a 4-byte hash of the resource type of the resource that belongs to the compartment, ex. `Observation`
+ * `id` - the logical id of the resource that belongs to the compartment, ex. the logical id of the Observation
+
 ## Transaction Handling
 
-* a transaction bundle is POST'ed to one arbitrary node
-* this node submits the transaction commands to the central transaction log
-* all nodes (inkl. the transaction submitter) receive the transaction commands from the central transaction log
+ * a transaction bundle is POST'ed to one arbitrary node
+ * this node submits the transaction commands to the central transaction log
+ * all nodes (inkl. the transaction submitter) receive the transaction commands from the central transaction log
 
 ### Transaction Commands
 

diff --git a/docs/monitoring/blaze.json b/docs/monitoring/blaze.json
@@ -447,7 +447,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "blaze_db_cache_estimated_size{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}",
+          "expr": "blaze_cache_estimated_size{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}",
           "hide": false,
           "interval": "",
           "legendFormat": "",
@@ -546,7 +546,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "rate(blaze_db_cache_hits_total{job=\"$job\",instance=\"$instance\", name=\"resource-cache\"}[1m]) / (rate(blaze_db_cache_hits_total[1m]) + rate(blaze_db_cache_misses_total[1m]))",
+          "expr": "rate(blaze_cache_hits_total{job=\"$job\",instance=\"$instance\", name=\"resource-cache\"}[1m]) / (rate(blaze_cache_hits_total[1m]) + rate(blaze_cache_misses_total[1m]))",
           "hide": false,
           "interval": "",
           "legendFormat": "",
@@ -644,7 +644,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "rate(blaze_db_cache_load_successes_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])",
+          "expr": "rate(blaze_cache_load_successes_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])",
           "hide": false,
           "interval": "",
           "legendFormat": "",
@@ -742,7 +742,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "rate(blaze_db_cache_evictions_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])",
+          "expr": "rate(blaze_cache_evictions_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])",
           "hide": false,
           "interval": "",
           "legendFormat": "",
@@ -7985,15 +7985,15 @@
           "type": "prometheus",
           "uid": "${DS_PROMETHEUS}"
         },
-        "definition": "label_values(blaze_db_cache_hits_total, job)",
+        "definition": "label_values(blaze_cache_hits_total, job)",
         "hide": 0,
         "includeAll": false,
         "label": "Job",
         "multi": false,
         "name": "job",
         "options": [],
         "query": {
-          "query": "label_values(blaze_db_cache_hits_total, job)",
+          "query": "label_values(blaze_cache_hits_total, job)",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -8012,14 +8012,14 @@
           "type": "prometheus",
           "uid": "${DS_PROMETHEUS}"
         },
-        "definition": "label_values(blaze_db_cache_estimated_size{job=\"$job\"}, instance)",
+        "definition": "label_values(blaze_cache_estimated_size{job=\"$job\"}, instance)",
         "hide": 0,
         "includeAll": false,
         "multi": false,
         "name": "instance",
         "options": [],
         "query": {
-          "query": "label_values(blaze_db_cache_estimated_size{job=\"$job\"}, instance)",
+          "query": "label_values(blaze_cache_estimated_size{job=\"$job\"}, instance)",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,