Cache Results of the Exists CQL Expression

Closes: #1051
samply · Jun 17, 2024 · 7f06c4d · 7f06c4d
1 parent d6ac14e
commit 7f06c4d
Show file tree

Hide file tree

Showing 208 changed files with 11,309 additions and 4,459 deletions.
diff --git a/.github/scripts/check-bloom-filter.sh b/.github/scripts/check-bloom-filter.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+. "$SCRIPT_DIR/util.sh"
+
+BASE="http://localhost:8080/fhir"
+HASH="$1"
+PATIENT_COUNT="$2"
+
+test "patient count" "$(curl -s "$BASE/__admin/cql/bloom-filters" | jq -r ".[] | select(.hash == \"$HASH\") | .patientCount")" "$PATIENT_COUNT"
diff --git a/.github/scripts/check-patient-last-change-index-missing.sh b/.github/scripts/check-patient-last-change-index-missing.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+. "$SCRIPT_DIR/util.sh"
+
+BASE="http://localhost:8080/fhir"
+curl -s "$BASE/__admin/dbs/index/column-families" | jq -r '.[].name' | grep -q "patient-last-change-index"
+
+test "exit code" "$?" "1"
diff --git a/.github/scripts/check-patient-last-change-index-state.sh b/.github/scripts/check-patient-last-change-index-state.sh
@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+. "$SCRIPT_DIR/util.sh"
+
+BASE="http://localhost:8080/fhir"
+STATE="$(curl -s "$BASE/__admin/dbs/index/column-families/patient-last-change-index/state" | jq -r .type)"
+
+test "state" "$STATE" "$1"
diff --git a/.github/scripts/test-cql-expr-cache-metrics.sh b/.github/scripts/test-cql-expr-cache-metrics.sh
@@ -0,0 +1,19 @@
+#!/bin/bash -e
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+. "$SCRIPT_DIR/util.sh"
+
+URL="http://localhost:8081/metrics"
+
+num-metrics() {
+  NAME="$1"
+  FILTER="$2"
+  curl -s "$URL" | grep "$NAME" | grep -c "$FILTER"
+}
+
+# CQL expression cache is available
+test "blaze_cache_estimated_size cql-expr-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"cql-expr-cache\"")" "1"
+
+# other caches are still available
+test "blaze_cache_estimated_size tx-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"tx-cache\"")" "1"
+test "blaze_cache_estimated_size resource-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"resource-cache\"")" "1"
diff --git a/.github/scripts/test-metrics.sh b/.github/scripts/test-metrics.sh
@@ -15,6 +15,5 @@ test "blaze_rocksdb_block_cache_data_miss index" "$(num-metrics "blaze_rocksdb_b
 test "blaze_rocksdb_block_cache_data_miss transaction" "$(num-metrics "blaze_rocksdb_block_cache_data_miss" "name=\"transaction\"")" "1"
 test "blaze_rocksdb_block_cache_data_miss resource" "$(num-metrics "blaze_rocksdb_block_cache_data_miss" "name=\"resource\"")" "1"
 
-test "blaze_rocksdb_table_reader_usage_bytes index" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"index\"")" "14"
-test "blaze_rocksdb_table_reader_usage_bytes transaction" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"transaction\"")" "1"
-test "blaze_rocksdb_table_reader_usage_bytes resource" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"resource\"")" "1"
+test "blaze_cache_estimated_size tx-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"tx-cache\"")" "1"
+test "blaze_cache_estimated_size resource-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"resource-cache\"")" "1"
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -8,7 +8,7 @@ on:
     - master
     - develop
     tags:
-      - 'v*.*.*'
+    - 'v*.*.*'
   pull_request:
     branches:
     - master
@@ -83,6 +83,7 @@ jobs:
         - async
         - byte-buffer
         - byte-string
+        - cache-collector
         - cassandra
         - coll
         - cql
@@ -333,6 +334,158 @@ jobs:
       with:
         sarif_file: trivy-results.sarif
 
+  cql-expr-cache-test:
+    needs: build
+    runs-on: ubuntu-22.04
+
+    steps:
+    - name: Check out Git repository
+      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+
+    - name: Install Blazectl
+      run: .github/scripts/install-blazectl.sh
+
+    - name: Download Blaze Image
+      uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4
+      with:
+        name: blaze-image
+        path: /tmp
+
+    - name: Load Blaze Image
+      run: docker load --input /tmp/blaze.tar
+
+    - name: Run Blaze
+      run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_ADMIN_API=true -e CQL_EXPR_CACHE_SIZE=1000 -p 8080:8080 -p 8081:8081 -v blaze-data:/app/data blaze:latest
+
+    - name: Wait for Blaze
+      run: .github/scripts/wait-for-url.sh  http://localhost:8080/health
+
+    - name: Docker Logs
+      run: docker logs blaze
+
+    - name: Check Capability Statement
+      run: .github/scripts/check-capability-statement.sh
+
+    - name: Ensure that the State of PatientLastChange Index is Current
+      run: .github/scripts/check-patient-last-change-index-state.sh current
+
+    - name: Load Data
+      run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea
+
+    - name: Prometheus Metrics
+      run: .github/scripts/test-cql-expr-cache-metrics.sh
+
+    - name: Check Total-Number of Resources are 92114
+      run: .github/scripts/check-total-number-of-resources.sh 92114
+
+    - name: Evaluate CQL Query 1
+      run: .github/scripts/evaluate-measure.sh q1 56
+
+    - name: Evaluate CQL Query 1 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl.sh q1 56
+
+    - name: Evaluate CQL Query 1 - Subject List
+      run: .github/scripts/evaluate-measure-subject-list.sh q1 56
+
+    - name: Evaluate CQL Query 1 on Individual Patients
+      run: .github/scripts/evaluate-patient-q1-measure.sh
+
+    - name: Evaluate CQL Query 2
+      run: .github/scripts/evaluate-measure.sh q2 42
+
+    - name: Evaluate CQL Query 2 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl.sh q2 42
+
+    - name: Evaluate CQL Query 2 - Subject List
+      run: .github/scripts/evaluate-measure-subject-list.sh q2 42
+
+    - name: Evaluate CQL Query 4
+      run: .github/scripts/evaluate-measure.sh q4 0
+
+    - name: Check Bloom Filter
+      run: .github/scripts/check-bloom-filter.sh 484CC96511A406D40EE8EEB4DE6A49B05B3766DDEF8EB05AC57C6139D774EDDC 0
+
+    - name: Evaluate CQL Query 4 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl.sh q4 0
+
+    - name: Evaluate CQL Query 4 - Subject List
+      run: .github/scripts/evaluate-measure-subject-list.sh q4 0
+
+    - name: Evaluate CQL Query 7
+      run: .github/scripts/evaluate-measure.sh q7 81
+
+    - name: Evaluate CQL Query 7 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl.sh q7 81
+
+    - name: Evaluate CQL Query 7 - Subject List
+      run: .github/scripts/evaluate-measure-subject-list.sh q7 81
+
+    - name: Evaluate CQL Query 14
+      run: .github/scripts/evaluate-measure.sh q14 96
+
+    - name: Check Bloom Filter
+      run: .github/scripts/check-bloom-filter.sh FAB7FC40F75F294787D8B2090788A291CDAF71640D6DCFE5441C00403225A256 96
+
+    - name: Evaluate CQL Query 14 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl.sh q14 96
+
+    - name: Evaluate CQL Query 14 - Subject List
+      run: .github/scripts/evaluate-measure-subject-list.sh q14 96
+
+    - name: Evaluate CQL Query 17
+      run: .github/scripts/evaluate-measure.sh q17 120
+
+    - name: Evaluate CQL Query 17 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl.sh q17 120
+
+    - name: Evaluate CQL Query 17 - Subject List
+      run: .github/scripts/evaluate-measure-subject-list.sh q17 120
+
+    - name: Evaluate CQL Query 20 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q20-stratifier-city 120
+
+    - name: Evaluate CQL Query 21 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q21-stratifier-city-of-only-women 64
+
+    - name: Evaluate CQL Query 26 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q26-stratifier-bmi 120
+
+    - name: Evaluate CQL Query 27 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q27-stratifier-calculated-bmi 120
+
+    - name: Evaluate CQL Query 32 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q32-stratifier-underweight 120
+
+    - name: Evaluate CQL Query 36
+      run: .github/scripts/evaluate-measure.sh q36-parameter 86
+
+    - name: Check Bloom Filter
+      run: .github/scripts/check-bloom-filter.sh D4FC6CDE1636852F9E362A68CA7BE027A66BF7CB38EBFF9C256C3EB2179C2639 86
+
+    - name: Evaluate CQL Query 36 - Subject List
+      run: .github/scripts/evaluate-measure-subject-list.sh q36-parameter 86
+
+    - name: Evaluate CQL Query 37
+      run: .github/scripts/evaluate-measure.sh q37-overlaps 24
+
+    - name: Check Bloom Filter
+      run: .github/scripts/check-bloom-filter.sh 8A572962E0540DE1BB4F4BF5C0101FF06BE3AE69F4CD489509A6CF8D1646D1E1 24
+
+    - name: Evaluate CQL Query 37 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl.sh q37-overlaps 24
+
+    - name: Evaluate CQL Query 37 - Subject List
+      run: .github/scripts/evaluate-measure-subject-list.sh q37-overlaps 24
+
+    - name: Evaluate CQL Query 46
+      run: .github/scripts/evaluate-measure.sh q46-between-date 19
+
+    - name: Check Bloom Filter
+      run: .github/scripts/check-bloom-filter.sh 5A1FE6D1B996AED4783F0EE04E6E74927D7DDCCC407B0818A8A800769458020B 19
+
+    - name: Evaluate CQL Query 46 using Blazectl
+      run: .github/scripts/evaluate-measure-blazectl.sh q46-between-date 19
+
   integration-test:
     needs: build
     runs-on: ubuntu-22.04
@@ -736,7 +889,7 @@ jobs:
     - name: Delete
       run: .github/scripts/delete.sh
 
-    - name: Delete Violating Referential Integrity 
+    - name: Delete Violating Referential Integrity
       run: .github/scripts/check-referential-integrity-for-delete.sh 409
 
     - name: Batch
@@ -751,7 +904,7 @@ jobs:
     - name: Transaction Read/Write
       run: .github/scripts/transaction-rw.sh
 
-    - name: Transactional Delete Preserving Referential Integrity 
+    - name: Transactional Delete Preserving Referential Integrity
       run: .github/scripts/transactional-delete.sh
 
     - name: Transaction with Invalid (Null) Resource
@@ -1548,6 +1701,50 @@ jobs:
     - name: Fetch Patient Expecting an Error
       run: .github/scripts/fetch-resource-0-with-missing-resource-content.sh
 
+  build-patient-last-change-index-test:
+    needs: build
+    runs-on: ubuntu-22.04
+
+    steps:
+    - name: Check out Git repository
+      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+
+    - name: Install Blazectl
+      run: .github/scripts/install-blazectl.sh
+
+    - name: Download Blaze Image
+      uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4
+      with:
+        name: blaze-image
+        path: /tmp
+
+    - name: Load Blaze Image
+      run: docker load --input /tmp/blaze.tar
+
+    - name: Run Blaze v0.27
+      run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_ADMIN_API=true -p 8080:8080 -v blaze-data:/app/data samply/blaze:0.27
+
+    - name: Wait for Blaze
+      run: .github/scripts/wait-for-url.sh  http://localhost:8080/health
+
+    - name: Load Data
+      run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea
+
+    - name: Ensure that the PatientLastChange Index does not exist
+      run: .github/scripts/check-patient-last-change-index-missing.sh
+
+    - name: Shut down Blaze
+      run: docker stop blaze && docker rm blaze
+
+    - name: Run Latest Blaze
+      run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_ADMIN_API=true -e LOG_LEVEL=debug -p 8080:8080 -v blaze-data:/app/data blaze:latest
+
+    - name: Wait for Blaze
+      run: .github/scripts/wait-for-url.sh  http://localhost:8080/health
+
+    - name: Ensure that the State of PatientLastChange Index is Current
+      run: .github/scripts/check-patient-last-change-index-state.sh current
+
   jepsen-distributed-test:
     needs: build
     runs-on: ubuntu-22.04
@@ -1628,6 +1825,7 @@ jobs:
     if: github.event_name != 'pull_request' || (github.event.pull_request.base.repo.full_name == github.event.pull_request.head.repo.full_name)
     needs:
     - image-scan
+    - cql-expr-cache-test
     - integration-test
     - integration-test-synthea-1000
     - not-enforcing-referential-integrity-test
@@ -1641,11 +1839,12 @@ jobs:
     - jepsen-test
     - jepsen-distributed-test
     - openid-auth-test
+    - custom-search-parameters-test
     - doc-copy-data-test
     - big-binary-test
     - frontend-test
     - missing-resource-content-test
-    - custom-search-parameters-test
+    - build-patient-last-change-index-test
     runs-on: ubuntu-22.04
     permissions:
       packages: write

diff --git a/Makefile b/Makefile
@@ -4,7 +4,7 @@ $(MODULES):
 	$(MAKE) -C $@ $(MAKECMDGOALS)
 
 fmt-root:
-	cljfmt check
+	cljfmt check resources src test deps.edn
 
 fmt: $(MODULES) fmt-root
 

diff --git a/cljfmt.edn b/cljfmt.edn
@@ -12,4 +12,5 @@
   deftests [[:block 1]]
   satisfies-prop [[:block 1]]
   clojure.test.check.properties/for-all [[:block 1]]
-  blaze.metrics.core/collector [[:block 0]]}}
+  blaze.metrics.core/collector [[:block 0]]
+  reify-expr [[:inner 0] [:inner 1]]}}
diff --git a/dev/blaze/dev.clj b/dev/blaze/dev.clj
@@ -1,12 +1,13 @@
 (ns blaze.dev
   (:require
     [blaze.byte-string :as bs]
+    [blaze.cache-collector.protocols :as ccp]
     [blaze.db.api :as d]
     [blaze.db.api-spec]
-    [blaze.db.cache-collector.protocols :as ccp]
     [blaze.db.resource-cache :as resource-cache]
     [blaze.db.resource-store :as rs]
     [blaze.db.tx-log :as tx-log]
+    [blaze.elm.expression :as-alias expr]
     [blaze.spec]
     [blaze.system :as system]
     [blaze.system-spec]
@@ -58,6 +59,11 @@
   (resource-cache/invalidate-all! (:blaze.db/resource-cache system))
   )
 
+;; CQL Expression Cache
+(comment
+  (str (ccp/-stats (::expr/cache system)))
+  )
+
 ;; RocksDB Stats
 (comment
   (.reset (system [:blaze.db.kv.rocksdb/stats :blaze.db.index-kv-store/stats]))

diff --git a/docs/api/admin.md b/docs/api/admin.md
@@ -0,0 +1,7 @@
+# Admin API
+
+## OpenAPI Spec
+
+```sh
+curl http://localhost:8080/fhir/__admin/openapi.json
+```
diff --git a/docs/deployment/environment-variables.md b/docs/deployment/environment-variables.md
@@ -110,6 +110,9 @@ More information about distributed deployment are available [here](distributed-b
 | DB_SYNC_TIMEOUT                         | 10000                      | v0.15  | —       | Timeout in milliseconds for all reading FHIR interactions acquiring the newest database state. |
 | DB_SEARCH_PARAM_BUNDLE                  | —                          | v0.21  | —       | Name of a custom search parameter bundle file.                                                 |
 | ENABLE_ADMIN_API                        | —                          | v0.26  | —       | Set to `true` if the optional Admin API should be enabled. Needed by the frontend.             |
+| CQL_EXPR_CACHE_SIZE                     | 128 (128 MiB)              | v0.28  | —       | Size of the CQL expression cache. Will be disabled if not given.                               |
+| CQL_EXPR_CACHE_REFRESH                  | PT24H                      | v0.28  | —       | The duration after which a Bloom filter of the CQL expression cache will be refreshed.         |
+| CQL_EXPR_CACHE_THREADS                  | 4                          | v0.28  | —       | The maximum number of parallel Bloom filter calculations for the CQL expression cache.         |
 
 ¹ Deprecated