Merge pull request #1262 from samply/fix/concurrent-db-iterator-usage

Fix Concurrent Use of KV Iterators
samply · Nov 9, 2023 · 2625bf3 · 2625bf3
2 parents c975baa + ae85e62
commit 2625bf3
Show file tree

Hide file tree

Showing 40 changed files with 685 additions and 509 deletions.
diff --git a/.github/scripts/count-patients.sh b/.github/scripts/count-patients.sh
@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+. "$SCRIPT_DIR/util.sh"
+
+BASE="http://localhost:8080/fhir"
+TYPE=$1
+QUERY=$2
+EXPECTED_SIZE=$3
+ACTUAL_SIZE=$(blazectl --server "$BASE" download "$TYPE" -q "$QUERY" 2>/dev/null | jq -r .subject.reference | sort -u | wc -l | xargs)
+
+test "number of patients" "$ACTUAL_SIZE" "$EXPECTED_SIZE"
diff --git a/.github/scripts/cql/inpatient-stress.cql b/.github/scripts/cql/inpatient-stress.cql
@@ -0,0 +1,19 @@
+library "inpatient-stress"
+using FHIR version '4.0.0'
+include FHIRHelpers version '4.0.0'
+
+codesystem sct: 'http://snomed.info/sct'
+codesystem act: 'http://terminology.hl7.org/CodeSystem/v3-ActCode'
+
+context Patient
+
+define Stress:
+  [Condition: Code '73595000' from sct]
+
+define "Inpatient Encounter":
+  [Encounter: class in Code 'IMP' from act]
+
+define InInitialPopulation:
+  exists Stress C
+    with "Inpatient Encounter" E
+    such that C.encounter.reference = 'Encounter/' + E.id
diff --git a/.github/scripts/cql/inpatient-stress.yml b/.github/scripts/cql/inpatient-stress.yml
@@ -0,0 +1,5 @@
+library: .github/scripts/cql/inpatient-stress.cql
+group:
+- type: Patient
+  population:
+  - expression: InInitialPopulation
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -670,6 +670,62 @@ jobs:
     - name: Docker Stats
       run: docker stats --no-stream
 
+  integration-test-synthea-1000:
+    needs: build
+    runs-on: ubuntu-22.04
+
+    steps:
+    - name: Check out Git repository
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+
+    - name: Install Blazectl
+      run: .github/scripts/install-blazectl.sh
+
+    - name: Download Blaze Image
+      uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
+      with:
+        name: blaze-image
+        path: /tmp
+
+    - name: Load Blaze Image
+      run: docker load --input /tmp/blaze.tar
+
+    - name: Run Blaze
+      run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e DB_BLOCK_CACHE_SIZE=2048 -p 8080:8080 -v blaze-data:/app/data blaze:latest
+
+    - name: Wait for Blaze
+      run: .github/scripts/wait-for-url.sh  http://localhost:8080/health
+
+    - name: Check Capability Statement
+      run: .github/scripts/check-capability-statement.sh
+
+    - name: Check Referential Integrity Enforced
+      run: .github/scripts/check-referential-integrity-enforced.sh
+
+    - name: Download Synthea Test Data
+      run: wget https://speicherwolke.uni-leipzig.de/index.php/s/kDsa2ifeMFdqK35/download/synthea-1000.tar
+
+    - name: Create Synthea Test Data Dir
+      run: mkdir test-data-synthea-1000
+
+    - name: Unpack Synthea Test Data
+      run: tar -C test-data-synthea-1000 -xf synthea-1000.tar
+
+    - name: Load Data
+      run: blazectl --no-progress --server http://localhost:8080/fhir upload test-data-synthea-1000
+
+    - name: Check Total-Number of Resources are 1099594
+      run: .github/scripts/check-total-number-of-resources.sh 1099594
+
+    - name: Count the number of Patients with Stress Conditions at Inpatient Encounters using FHIR Search
+      run: .github/scripts/count-patients.sh Condition "code=http://snomed.info/sct|73595000&encounter.class=http://terminology.hl7.org/CodeSystem/v3-ActCode|IMP" 13
+
+    - name: Count the number of Patients with Stress Conditions at Inpatient Encounters using CQL
+      run: .github/scripts/evaluate-measure-blazectl.sh inpatient-stress 13
+
+    - name: Docker Stats
+      run: docker stats --no-stream
+
   not-enforcing-referential-integrity-test:
     needs: build
     runs-on: ubuntu-22.04
@@ -1268,6 +1324,7 @@ jobs:
     - build
     - image-scan
     - integration-test
+    - integration-test-synthea-1000
     - not-enforcing-referential-integrity-test
     - small-transactions-test
     - big-transaction-test

diff --git a/docs/performance/cql.md b/docs/performance/cql.md
@@ -55,9 +55,9 @@ cql/search.sh observation-72514-3
 | LEA47  | 1M      | 17861-6 |   25 k |     0.93 |  0.003 |   1.1 M | 
 | LEA47  | 1M      | 8310-5  |  603 k |     1.24 |  0.009 | 808.6 k | 
 | LEA47  | 1M      | 72514-3 |  998 k |     1.43 |  0.007 | 698.1 k |
-| LEA58  | 1M      | 17861-6 |   25 k |     0.76 |  0.007 |   1.3 M |
-| LEA58  | 1M      | 8310-5  |  603 k |     0.96 |  0.006 |   1.0 M | 
-| LEA58  | 1M      | 72514-3 |  998 k |     1.07 |  0.006 | 937.4 k |
+| LEA58  | 1M      | 17861-6 |   25 k |     0.86 |  0.005 |   1.1 M |
+| LEA58  | 1M      | 8310-5  |  603 k |     1.08 |  0.008 | 925.0 k | 
+| LEA58  | 1M      | 72514-3 |  998 k |     1.18 |  0.004 | 845.0 k |
 
 The evaluation of patient based measures doesn't depend on the number of hits (patients). The time needed to evaluate a CQL expression over all patients only depends on the total number of patients. The measurements show that Blaze can evaluate about 350 k Patients per second.
 
@@ -95,9 +95,9 @@ cql/search.sh observation-body-weight-100
 | LEA47  | 1M      | 29463-7 | 13.6 kg |   99 k |    91.49 |  1.195 |  10.9 k | 
 | LEA47  | 1M      | 29463-7 | 75.3 kg |  500 k |    10.66 |  0.851 |  93.8 k | 
 | LEA47  | 1M      | 29463-7 |  185 kg |  998 k |     1.50 |  0.010 | 665.1 k |
-| LEA58  | 1M      | 29463-7 | 13.6 kg |   99 k |     5.06 |  0.051 | 197.7 k | 
-| LEA58  | 1M      | 29463-7 | 75.3 kg |  500 k |     3.48 |  0.022 | 287.6 k | 
-| LEA58  | 1M      | 29463-7 |  185 kg |  998 k |     1.13 |  0.005 | 886.2 k |
+| LEA58  | 1M      | 29463-7 | 13.6 kg |   99 k |     5.08 |  0.006 | 196.7 k |  
+| LEA58  | 1M      | 29463-7 | 75.3 kg |  500 k |     3.66 |  0.016 | 273.5 k | 
+| LEA58  | 1M      | 29463-7 |  185 kg |  998 k |     1.36 |  0.004 | 737.2 k |
 
 ## Code, Date and Age Search
 
@@ -131,8 +131,8 @@ cql/search.sh calcium-date-age
 | LEA47  | 100k    | calcium    |   20 k |     1.50 |  0.035 |  66.6 k |
 | LEA47  | 1M      | hemoglobin |  200 k |     2.99 |  0.026 | 334.6 k |
 | LEA47  | 1M      | calcium    |  199 k |   120.68 |  1.678 |   8.3 k |
-| LEA58  | 1M      | hemoglobin |  200 k |     1.60 |  0.006 | 623.2 k |
-| LEA58  | 1M      | calcium    |  199 k |     6.44 |  0.030 | 155.2 k |
+| LEA58  | 1M      | hemoglobin |  200 k |     1.82 |  0.012 | 550.4 k |
+| LEA58  | 1M      | calcium    |  199 k |     6.78 |  0.028 | 147.6 k |
 
 ## Double Code Search
 
@@ -161,7 +161,7 @@ cql/search.sh condition-two
 | System | Dataset | # Hits | Time (s) | StdDev |  Pat./s |
 |--------|---------|-------:|---------:|-------:|--------:|
 | LEA47  | 1M      |   87 k |     1.09 |  0.005 | 918.4 k |
-| LEA58  | 1M      |   87 k |     0.87 |  0.003 |   1.1 M |
+| LEA58  | 1M      |   87 k |     1.13 |  0.006 | 885.9 k |
 
 ## Ten Frequent Code Search
 
@@ -194,7 +194,7 @@ cql/search.sh condition-ten-frequent
 | System | Dataset | # Hits | Time (s) | StdDev |  Pat./s |
 |--------|---------|-------:|---------:|-------:|--------:|
 | LEA47  | 1M      |  954 k |     1.80 |  0.008 | 554.5 k |
-| LEA58  | 1M      |  954 k |     1.34 |  0.003 | 747.8 k |
+| LEA58  | 1M      |  954 k |     1.81 |  0.012 | 551.7 k |
 
 ## Ten Rare Code Search
 
@@ -227,7 +227,7 @@ cql/search.sh condition-ten-rare
 | System | Dataset | # Hits | Time (s) | StdDev |  Pat./s |
 |--------|---------|-------:|---------:|-------:|--------:|
 | LEA47  | 1M      |    4 k |     2.65 |  0.014 | 377.9 k |
-| LEA58  | 1M      |    4 k |     1.88 |  0.006 | 530.9 k |
+| LEA58  | 1M      |    4 k |     3.36 |  0.015 | 297.8 k |
 
 ## 50 Rare Code Search
 
@@ -238,7 +238,7 @@ cql/search.sh condition-50-rare
 | System | Dataset | # Hits | Time (s) | StdDev |  Pat./s |
 |--------|---------|-------:|---------:|-------:|--------:|
 | LEA47  | 1M      |  155 k |     9.35 |  0.047 | 106.9 k |
-| LEA58  | 1M      |  155 k |     6.45 |  0.034 | 155.0 k |
+| LEA58  | 1M      |  155 k |    13.31 |  0.061 |  75.1 k |
 
 ## All Code Search
 
@@ -249,7 +249,7 @@ cql/search.sh condition-all
 | System | Dataset | # Hits | Time (s) | StdDev |  Pat./s |
 |--------|---------|-------:|---------:|-------:|--------:|
 | LEA47  | 1M      |  995 k |     4.75 |  0.014 | 210.5 k |
-| LEA58  | 1M      |  995 k |     3.36 |  0.020 | 297.2 k |
+| LEA58  | 1M      |  995 k |     6.00 |  0.017 | 166.7 k |
 
 ## Inpatient Stress Search
 
@@ -259,5 +259,4 @@ cql/search.sh inpatient-stress
 
 | System | Dataset | # Hits | Time (s) | StdDev |  Pat./s |
 |--------|---------|-------:|---------:|-------:|--------:|
-| LEA47  | 100k    |    334 |     0.26 |  0.005 | 381.9 k |
-| LEA58  | 1M      |    3 k |     3.01 |  0.017 | 331.8 k |
+| LEA58  | 1M      |   16 k |    11.77 |  0.037 |  84.9 k |
diff --git a/docs/performance/fhir-search.md b/docs/performance/fhir-search.md
@@ -122,9 +122,9 @@ curl -s "http://localhost:8080/fhir/Observation?code=http://loinc.org|$CODE&valu
 
 | System | Dataset | Code    | Value | # Hits | Time (s) | StdDev | T/1M ¹ |
 |--------|---------|---------|------:|-------:|---------:|-------:|-------:|
-| LEA47  | 100k    | 29463-7 |  26.8 |  158 k |    11.43 |  0.048 |  72.32 |
-| LEA47  | 100k    | 29463-7 |  79.5 |  790 k |    11.66 |  0.060 |  14.76 |
-| LEA47  | 100k    | 29463-7 |   183 |  1.6 M |    11.36 |  0.041 |   7.17 |
+| LEA47  | 100k    | 29463-7 |  26.8 |  158 k |    14.24 |  0.073 |  90.05 |
+| LEA47  | 100k    | 29463-7 |  79.5 |  790 k |    14.68 |  0.152 |  18.59 |
+| LEA47  | 100k    | 29463-7 |   183 |  1.6 M |    14.61 |  0.120 |   9.23 |
 | CCX42  | 100k    | 29463-7 |  26.8 |  158 k |    56.45 |  0.149 | 357.08 |
 | CCX42  | 100k    | 29463-7 |  79.5 |  790 k |    56.72 |  0.174 |  71.84 |
 | CCX42  | 100k    | 29463-7 |   183 |  1.6 M |    56.77 |  0.135 |  35.87 |
@@ -148,9 +148,9 @@ blazectl download --server http://localhost:8080/fhir Observation -q "code=http:
 
 | System | Dataset | Code    | Value | # Hits | Time (s) | StdDev | T/1M ¹ |
 |--------|---------|---------|------:|-------:|---------:|-------:|-------:|
-| LEA47  | 100k    | 29463-7 |  26.8 |  158 k |    13.73 |  0.017 |  86.83 |
-| LEA47  | 100k    | 29463-7 |  79.5 |  790 k |    22.93 |  0.231 |  29.04 |
-| LEA47  | 100k    | 29463-7 |   183 |  1.6 M |    32.80 |  0.281 |  20.72 |
+| LEA47  | 100k    | 29463-7 |  26.8 |  158 k |    16.77 |  0.045 | 106.06 |
+| LEA47  | 100k    | 29463-7 |  79.5 |  790 k |    20.84 |  0.193 |  26.39 |
+| LEA47  | 100k    | 29463-7 |   183 |  1.6 M |    35.42 |  0.298 |  22.38 |
 | CCX42  | 100k    | 29463-7 |  26.8 |  158 k |    59.19 |  0.060 | 374.44 |
 | CCX42  | 100k    | 29463-7 |  79.5 |  790 k |    70.26 |  0.142 |  88.98 |
 | CCX42  | 100k    | 29463-7 |   183 |  1.6 M |    83.82 |  0.076 |  52.97 |
@@ -174,9 +174,9 @@ blazectl download --server http://localhost:8080/fhir Observation -q "code=http:
 
 | System | Dataset | Code    | Value | # Hits | Time (s) | StdDev | T/1M ¹ |
 |--------|---------|---------|------:|-------:|---------:|-------:|-------:|
-| LEA47  | 100k    | 29463-7 |  26.8 |  158 k |    12.90 |  0.008 |  81.60 |
-| LEA47  | 100k    | 29463-7 |  79.5 |  790 k |    18.70 |  0.364 |  23.67 |
-| LEA47  | 100k    | 29463-7 |   183 |  1.6 M |    25.26 |  0.123 |  15.96 |
+| LEA47  | 100k    | 29463-7 |  26.8 |  158 k |    15.82 |  0.045 | 100.09 |
+| LEA47  | 100k    | 29463-7 |  79.5 |  790 k |    25.50 |  0.148 |  32.29 |
+| LEA47  | 100k    | 29463-7 |   183 |  1.6 M |    26.93 |  0.132 |  17.02 |
 | CCX42  | 100k    | 29463-7 |  26.8 |  158 k |    58.07 |  0.028 | 367.36 |
 | CCX42  | 100k    | 29463-7 |  79.5 |  790 k |    65.31 |  0.197 |  82.71 |
 | CCX42  | 100k    | 29463-7 |   183 |  1.6 M |    74.40 |  0.183 |  47.01 |
@@ -239,8 +239,8 @@ curl -s "http://localhost:8080/fhir/Observation?date=$YEAR&_summary=count"
 
 | System | Dataset | Year | # Hits | Time (s) | StdDev | T/1M ¹ |
 |--------|---------|------|-------:|---------:|-------:|-------:|
-| LEA47  | 100k    | 2013 |  3.1 M |     2.51 |  0.038 |   0.80 |
-| LEA47  | 100k    | 2019 |  6.0 M |     4.67 |  0.031 |   0.78 |
+| LEA47  | 100k    | 2013 |  3.1 M |     2.56 |  0.024 |   0.81 |
+| LEA47  | 100k    | 2019 |  6.0 M |     4.82 |  0.138 |   0.80 |
 | CCX42  | 100k    | 2013 |  3.1 M |     2.00 |  0.028 |   0.63 |
 | CCX42  | 100k    | 2019 |  6.0 M |     3.91 |  0.142 |   0.65 |
 | LEA47  | 1M      | 2013 | 31.1 M |    23.31 |  0.206 |   0.75 |
@@ -260,8 +260,8 @@ blazectl download --server http://localhost:8080/fhir Observation -q "date=$YEAR
 
 | System | Dataset | Year | # Hits | Time (s) | StdDev |  T/1M ¹ |
 |--------|---------|------|-------:|---------:|-------:|--------:|
-| LEA47  | 100k    | 2013 |  3.1 M |    58.40 |  1.051 |   18.68 |
-| LEA47  | 100k    | 2019 |  6.0 M |   110.71 |  0.404 |   18.51 |
+| LEA47  | 100k    | 2013 |  3.1 M |    56.35 |  1.001 |   18.02 |
+| LEA47  | 100k    | 2019 |  6.0 M |   103.39 |  0.877 |   17.29 |
 | CCX42  | 100k    | 2013 |  3.1 M |   128.16 |  0.406 |   41.00 |
 | CCX42  | 100k    | 2019 |  6.0 M |   276.13 |  2.020 | 46.18 ² |
 | LEA47  | 1M      | 2013 | 31.1 M |   991.28 | 12.329 | 31.90 ² |
@@ -283,8 +283,8 @@ blazectl download --server http://localhost:8080/fhir Observation -q "date=$YEAR
 
 | System | Dataset | Year | # Hits | Time (s) | StdDev |  T/1M ¹ |
 |--------|---------|------|-------:|---------:|-------:|--------:|
-| LEA47  | 100k    | 2013 |  3.1 M |    36.81 |  0.165 |   11.77 |
-| LEA47  | 100k    | 2019 |  6.0 M |    70.11 |  1.357 |   11.72 |
+| LEA47  | 100k    | 2013 |  3.1 M |    35.15 |  0.688 |   11.24 |
+| LEA47  | 100k    | 2019 |  6.0 M |    66.72 |  0.521 |   11.16 |
 | CCX42  | 100k    | 2013 |  3.1 M |    85.59 |  0.293 |   27.38 |
 | CCX42  | 100k    | 2019 |  6.0 M |   179.29 |  1.786 | 29.99 ² |
 | LEA47  | 1M      | 2013 | 31.1 M |   673.36 | 10.199 | 21.67 ² |

diff --git a/modules/db-protocols/src/blaze/db/impl/protocols.clj b/modules/db-protocols/src/blaze/db/impl/protocols.clj
@@ -88,10 +88,7 @@
   (-resource-handles
     [search-param context tid modifier compiled-value]
     [search-param context tid modifier compiled-value start-id]
-    "Returns a reducible collection.
-
-    Changes the state of `context`. Consuming the collection requires exclusive
-    access to `context`.")
+    "Returns a reducible collection.")
   (-sorted-resource-handles
     [search-param context tid direction]
     [search-param context tid direction start-id]