From 9fecaf8f06a0c54ab24b0ac6903f6e15aed1428c Mon Sep 17 00:00:00 2001 From: Alexander Kiel Date: Tue, 22 Aug 2023 10:20:14 +0200 Subject: [PATCH] [WIP] CQL Cache --- .github/distributed-test/docker-compose.yml | 2 + .../check-patient-as-of-index-missing.sh | 9 + .../check-patient-as-of-index-state.sh | 9 + .../scripts/test-cql-expr-cache-metrics.sh | 19 + .github/scripts/test-metrics.sh | 5 +- .github/workflows/build.yml | 194 +++++++- dev/blaze/dev.clj | 7 +- docs/deployment/environment-variables.md | 1 + docs/implementation/cql.md | 40 ++ docs/implementation/database.md | 69 +-- docs/monitoring/blaze.json | 424 +++++++++++------- docs/performance/cql.md | 144 +++++- docs/performance/cql/calcium-date-age.cql | 4 +- docs/performance/cql/code-date-age-search.sh | 22 - docs/performance/cql/code-value-search.sh | 19 - docs/performance/cql/condition-all.cql | 228 ++++++++++ docs/performance/cql/condition-all.yml | 5 + .../cql/condition-ten-frequent.cql | 19 + .../cql/condition-ten-frequent.yml | 5 + docs/performance/cql/condition-ten-rare.cql | 19 + docs/performance/cql/condition-ten-rare.yml | 5 + docs/performance/cql/condition-two.cql | 13 + docs/performance/cql/condition-two.yml | 5 + docs/performance/cql/hemoglobin-date-age.cql | 4 +- docs/performance/cql/search.sh | 19 + docs/performance/cql/simple-code-search.sh | 19 - docs/performance/cql/util.sh | 5 +- docs/performance/synthea/README.md | 4 +- modules/admin-api/.clj-kondo/config.edn | 3 +- modules/admin-api/deps.edn | 5 +- modules/admin-api/src/blaze/admin_api.clj | 85 +++- .../admin-api/test/blaze/admin_api_test.clj | 50 ++- modules/byte-buffer/src/blaze/byte_buffer.clj | 13 +- modules/cache-collector/.clj-kondo/config.edn | 20 + modules/cache-collector/Makefile | 22 + modules/cache-collector/deps.edn | 45 ++ .../src/blaze}/cache_collector.clj | 35 +- .../src/blaze}/cache_collector/protocols.clj | 2 +- .../src/blaze/cache_collector/spec.clj | 8 + .../test/blaze}/cache_collector_test.clj | 58 +-- modules/cache-collector/tests.edn | 5 + modules/cql/.clj-kondo/config.edn | 5 +- modules/cql/src/blaze/elm/code.clj | 2 + modules/cql/src/blaze/elm/compiler.clj | 4 + .../elm/compiler/arithmetic_operators.clj | 26 +- .../blaze/elm/compiler/clinical_operators.clj | 2 + .../elm/compiler/comparison_operators.clj | 5 + .../elm/compiler/conditional_operators.clj | 101 +++-- modules/cql/src/blaze/elm/compiler/core.clj | 24 +- .../elm/compiler/date_time_operators.clj | 97 ++-- .../cql/src/blaze/elm/compiler/expr_cache.clj | 61 +++ .../src/blaze/elm/compiler/external_data.clj | 44 +- .../cql/src/blaze/elm/compiler/function.clj | 3 + .../blaze/elm/compiler/interval_operators.clj | 29 ++ .../cql/src/blaze/elm/compiler/library.clj | 5 +- .../src/blaze/elm/compiler/library_spec.clj | 6 +- .../src/blaze/elm/compiler/list_operators.clj | 68 ++- .../blaze/elm/compiler/logical_operators.clj | 92 ++-- modules/cql/src/blaze/elm/compiler/macros.clj | 374 +++++++++++---- .../cql/src/blaze/elm/compiler/queries.clj | 131 +++--- .../src/blaze/elm/compiler/reusing_logic.clj | 18 + modules/cql/src/blaze/elm/compiler/spec.clj | 12 +- .../blaze/elm/compiler/string_operators.clj | 228 ++++++---- .../blaze/elm/compiler/structured_values.clj | 53 ++- .../src/blaze/elm/compiler/type_operators.clj | 24 +- modules/cql/src/blaze/elm/compiler_spec.clj | 7 + modules/cql/src/blaze/elm/expression.clj | 8 +- modules/cql/src/blaze/elm/expression/spec.clj | 12 +- modules/cql/src/blaze/elm/expression_spec.clj | 5 + .../compiler/conditional_operators_test.clj | 30 +- .../blaze/elm/compiler/expr_cache_spec.clj | 12 + .../blaze/elm/compiler/external_data_test.clj | 9 +- .../test/blaze/elm/compiler/library_test.clj | 21 +- .../elm/compiler/list_operators_test.clj | 91 +++- .../test/blaze/elm/compiler/queries_test.clj | 66 +-- .../cql/test/blaze/elm/compiler/test_util.clj | 3 +- .../src/blaze/db/impl/protocols.clj | 4 +- modules/db-stub/src/blaze/db/api_stub.clj | 6 +- modules/db/.clj-kondo/config.edn | 1 + modules/db/deps.edn | 6 + modules/db/src/blaze/db/api.clj | 7 + .../db/src/blaze/db/cache_collector/spec.clj | 8 - modules/db/src/blaze/db/impl/batch_db.clj | 18 +- modules/db/src/blaze/db/impl/db.clj | 10 + .../src/blaze/db/impl/index/patient_as_of.clj | 87 ++++ modules/db/src/blaze/db/node.clj | 58 ++- .../src/blaze/db/node/patient_as_of_index.clj | 14 + modules/db/src/blaze/db/node/tx_indexer.clj | 4 +- .../src/blaze/db/node/tx_indexer/verify.clj | 59 ++- modules/db/src/blaze/db/node/version.clj | 9 +- modules/db/src/blaze/db/resource_cache.clj | 2 +- .../db/src/blaze/db/search_param_registry.clj | 41 +- .../blaze/db/search_param_registry_spec.clj | 3 +- .../db/test-perf/blaze/db/api_test_perf.clj | 1 + modules/db/test/blaze/db/api_test.clj | 36 ++ .../db/test/blaze/db/impl/batch_db_spec.clj | 1 + modules/db/test/blaze/db/impl/db_spec.clj | 1 + .../db/impl/index/patient_as_of_spec.clj | 26 ++ .../db/impl/index/patient_as_of_test_util.clj | 16 + .../db/node/patient_as_of_index_spec.clj | 15 + .../db/node/patient_as_of_index_test.clj | 49 ++ .../blaze/db/node/tx_indexer/verify_spec.clj | 6 +- .../blaze/db/node/tx_indexer/verify_test.clj | 158 ++++++- .../db/test/blaze/db/node/tx_indexer_spec.clj | 5 +- modules/db/test/blaze/db/node_test.clj | 12 + .../db/test/blaze/db/resource_cache_test.clj | 2 +- .../blaze/db/search_param_registry_test.clj | 23 +- modules/db/test/blaze/db/test_util.clj | 6 +- modules/executor/src/blaze/executors.clj | 23 +- .../.clj-kondo/config.edn | 1 + .../blaze/fhir/operation/evaluate_measure.clj | 43 ++ .../operation/evaluate_measure/measure.clj | 43 +- .../fhir/operation/evaluate_measure/spec.clj | 4 + .../operation/evaluate_measure/cql/spec.clj | 17 +- .../operation/evaluate_measure/cql_test.clj | 2 + .../measure/stratifier_test.clj | 1 + .../evaluate_measure/measure_spec.clj | 20 +- .../evaluate_measure/measure_test.clj | 8 +- .../fhir/operation/evaluate_measure_test.clj | 23 + .../src/blaze/rest_api/middleware/metrics.clj | 6 +- modules/scheduler/src/blaze/scheduler.clj | 19 +- .../src/blaze/scheduler/protocol.clj | 1 + profiling/blaze/profiling.clj | 14 +- resources/blaze.edn | 40 +- src/blaze/system.clj | 2 +- 125 files changed, 3326 insertions(+), 982 deletions(-) create mode 100755 .github/scripts/check-patient-as-of-index-missing.sh create mode 100755 .github/scripts/check-patient-as-of-index-state.sh create mode 100755 .github/scripts/test-cql-expr-cache-metrics.sh create mode 100644 docs/implementation/cql.md delete mode 100755 docs/performance/cql/code-date-age-search.sh delete mode 100755 docs/performance/cql/code-value-search.sh create mode 100644 docs/performance/cql/condition-all.cql create mode 100644 docs/performance/cql/condition-all.yml create mode 100644 docs/performance/cql/condition-ten-frequent.cql create mode 100644 docs/performance/cql/condition-ten-frequent.yml create mode 100644 docs/performance/cql/condition-ten-rare.cql create mode 100644 docs/performance/cql/condition-ten-rare.yml create mode 100644 docs/performance/cql/condition-two.cql create mode 100644 docs/performance/cql/condition-two.yml create mode 100755 docs/performance/cql/search.sh delete mode 100755 docs/performance/cql/simple-code-search.sh create mode 100644 modules/cache-collector/.clj-kondo/config.edn create mode 100644 modules/cache-collector/Makefile create mode 100644 modules/cache-collector/deps.edn rename modules/{db/src/blaze/db => cache-collector/src/blaze}/cache_collector.clj (75%) rename modules/{db/src/blaze/db => cache-collector/src/blaze}/cache_collector/protocols.clj (62%) create mode 100644 modules/cache-collector/src/blaze/cache_collector/spec.clj rename modules/{db/test/blaze/db => cache-collector/test/blaze}/cache_collector_test.clj (63%) create mode 100644 modules/cache-collector/tests.edn create mode 100644 modules/cql/src/blaze/elm/compiler/expr_cache.clj create mode 100644 modules/cql/test/blaze/elm/compiler/expr_cache_spec.clj delete mode 100644 modules/db/src/blaze/db/cache_collector/spec.clj create mode 100644 modules/db/src/blaze/db/impl/index/patient_as_of.clj create mode 100644 modules/db/src/blaze/db/node/patient_as_of_index.clj create mode 100644 modules/db/test/blaze/db/impl/index/patient_as_of_spec.clj create mode 100644 modules/db/test/blaze/db/impl/index/patient_as_of_test_util.clj create mode 100644 modules/db/test/blaze/db/node/patient_as_of_index_spec.clj create mode 100644 modules/db/test/blaze/db/node/patient_as_of_index_test.clj diff --git a/.github/distributed-test/docker-compose.yml b/.github/distributed-test/docker-compose.yml index 2791601d0..4201ceb43 100644 --- a/.github/distributed-test/docker-compose.yml +++ b/.github/distributed-test/docker-compose.yml @@ -81,6 +81,7 @@ services: DB_CASSANDRA_MAX_CONCURRENT_REQUESTS: "128" DB_RESOURCE_CACHE_SIZE: "100000" LOG_LEVEL: "debug" + ENABLE_FRONTEND: "true" ports: - "8081:8081" volumes: @@ -111,6 +112,7 @@ services: DB_CASSANDRA_MAX_CONCURRENT_REQUESTS: "128" DB_RESOURCE_CACHE_SIZE: "100000" LOG_LEVEL: "debug" + ENABLE_FRONTEND: "true" ports: - "8082:8081" volumes: diff --git a/.github/scripts/check-patient-as-of-index-missing.sh b/.github/scripts/check-patient-as-of-index-missing.sh new file mode 100755 index 000000000..fdb175dfb --- /dev/null +++ b/.github/scripts/check-patient-as-of-index-missing.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +. "$SCRIPT_DIR/util.sh" + +BASE="http://localhost:8080/fhir" +curl -s "$BASE/__admin/rocksdb/index/column-families" | jq -r '."column-families"[]' | grep -q "patient-as-of-index" + +test "exit code" "$?" "1" diff --git a/.github/scripts/check-patient-as-of-index-state.sh b/.github/scripts/check-patient-as-of-index-state.sh new file mode 100755 index 000000000..77968e9c7 --- /dev/null +++ b/.github/scripts/check-patient-as-of-index-state.sh @@ -0,0 +1,9 @@ +#!/bin/bash -e + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +. "$SCRIPT_DIR/util.sh" + +BASE="http://localhost:8080/fhir" +STATE="$(curl -s "$BASE/__admin/db/index/column-families/patient-as-of-index/state" | jq -r .type)" + +test "state" "$STATE" "$1" diff --git a/.github/scripts/test-cql-expr-cache-metrics.sh b/.github/scripts/test-cql-expr-cache-metrics.sh new file mode 100755 index 000000000..f3f723891 --- /dev/null +++ b/.github/scripts/test-cql-expr-cache-metrics.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +. "$SCRIPT_DIR/util.sh" + +URL="http://localhost:8081/metrics" + +num-metrics() { + NAME="$1" + FILTER="$2" + curl -s "$URL" | grep "$NAME" | grep -c "$FILTER" +} + +# CQL expression cache is available +test "blaze_cache_estimated_size cql-expr-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"cql-expr-cache\"")" "1" + +# other caches are still available +test "blaze_cache_estimated_size tx-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"tx-cache\"")" "1" +test "blaze_cache_estimated_size resource-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"resource-cache\"")" "1" diff --git a/.github/scripts/test-metrics.sh b/.github/scripts/test-metrics.sh index a739a5c2e..b57741b8a 100755 --- a/.github/scripts/test-metrics.sh +++ b/.github/scripts/test-metrics.sh @@ -15,6 +15,5 @@ test "blaze_rocksdb_block_cache_data_miss index" "$(num-metrics "blaze_rocksdb_b test "blaze_rocksdb_block_cache_data_miss transaction" "$(num-metrics "blaze_rocksdb_block_cache_data_miss" "name=\"transaction\"")" "1" test "blaze_rocksdb_block_cache_data_miss resource" "$(num-metrics "blaze_rocksdb_block_cache_data_miss" "name=\"resource\"")" "1" -test "blaze_rocksdb_table_reader_usage_bytes index" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"index\"")" "14" -test "blaze_rocksdb_table_reader_usage_bytes transaction" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"transaction\"")" "1" -test "blaze_rocksdb_table_reader_usage_bytes resource" "$(num-metrics "blaze_rocksdb_table_reader_usage_bytes" "name=\"resource\"")" "1" +test "blaze_cache_estimated_size tx-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"tx-cache\"")" "1" +test "blaze_cache_estimated_size resource-cache" "$(num-metrics "blaze_cache_estimated_size" "name=\"resource-cache\"")" "1" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee45276d2..7daa46aad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,6 +39,7 @@ jobs: - anomaly - async - byte-buffer + - cache-collector - cassandra - coll - cql @@ -259,6 +260,143 @@ jobs: with: sarif_file: trivy-results.sarif + cql-expr-cache-test: + needs: build + runs-on: ubuntu-22.04 + + steps: + - name: Check out Git repository + uses: actions/checkout@v3 + + - name: Install Blazectl + run: .github/scripts/install-blazectl.sh + + - name: Download Blaze Image + uses: actions/download-artifact@v3 + with: + name: blaze-image + path: /tmp + + - name: Load Blaze Image + run: docker load --input /tmp/blaze.tar + + - name: Run Blaze + run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_FRONTEND=true -e CQL_EXPR_CACHE_SIZE=1000 -p 8080:8080 -p 8081:8081 -v blaze-data:/app/data blaze:latest + + - name: Wait for Blaze + run: .github/scripts/wait-for-url.sh http://localhost:8080/health + + - name: Docker Logs + run: docker logs blaze + + - name: Check Capability Statement + run: .github/scripts/check-capability-statement.sh + + - name: Ensure that the State of PatientAsOf Index is Current + run: .github/scripts/check-patient-as-of-index-state.sh current + + - name: Load Data + run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea + + - name: Prometheus Metrics + run: .github/scripts/test-cql-expr-cache-metrics.sh + + - name: Check Total-Number of Resources are 92114 + run: .github/scripts/check-total-number-of-resources.sh 92114 + + - name: Evaluate CQL Query 1 + run: .github/scripts/evaluate-measure.sh q1 56 + + - name: Evaluate CQL Query 1 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q1 56 + + - name: Evaluate CQL Query 1 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q1 56 + + - name: Evaluate CQL Query 1 on Individual Patients + run: .github/scripts/evaluate-patient-q1-measure.sh + + - name: Evaluate CQL Query 2 + run: .github/scripts/evaluate-measure.sh q2 42 + + - name: Evaluate CQL Query 2 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q2 42 + + - name: Evaluate CQL Query 2 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q2 42 + + - name: Evaluate CQL Query 4 + run: .github/scripts/evaluate-measure.sh q4 0 + + - name: Evaluate CQL Query 4 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q4 0 + + - name: Evaluate CQL Query 4 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q4 0 + + - name: Evaluate CQL Query 7 + run: .github/scripts/evaluate-measure.sh q7 81 + + - name: Evaluate CQL Query 7 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q7 81 + + - name: Evaluate CQL Query 7 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q7 81 + + - name: Evaluate CQL Query 14 + run: .github/scripts/evaluate-measure.sh q14 96 + + - name: Evaluate CQL Query 14 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q14 96 + + - name: Evaluate CQL Query 14 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q14 96 + + - name: Evaluate CQL Query 17 + run: .github/scripts/evaluate-measure.sh q17 120 + + - name: Evaluate CQL Query 17 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q17 120 + + - name: Evaluate CQL Query 17 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q17 120 + + - name: Evaluate CQL Query 20 using Blazectl + run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q20-stratifier-city 120 + + - name: Evaluate CQL Query 21 using Blazectl + run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q21-stratifier-city-of-only-women 64 + + - name: Evaluate CQL Query 26 using Blazectl + run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q26-stratifier-bmi 120 + + - name: Evaluate CQL Query 27 using Blazectl + run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q27-stratifier-calculated-bmi 120 + + - name: Evaluate CQL Query 32 using Blazectl + run: .github/scripts/evaluate-measure-blazectl-stratifier.sh q32-stratifier-underweight 120 + + - name: Evaluate CQL Query 36 + run: .github/scripts/evaluate-measure.sh q36-parameter 86 + + - name: Evaluate CQL Query 36 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q36-parameter 86 + + - name: Evaluate CQL Query 34 + run: .github/scripts/evaluate-measure.sh q37-overlaps 24 + + - name: Evaluate CQL Query 34 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q37-overlaps 24 + + - name: Evaluate CQL Query 34 - Subject List + run: .github/scripts/evaluate-measure-subject-list.sh q37-overlaps 24 + + - name: Evaluate CQL Query 46 + run: .github/scripts/evaluate-measure.sh q46-between-date 19 + + - name: Evaluate CQL Query 46 using Blazectl + run: .github/scripts/evaluate-measure-blazectl.sh q46-between-date 19 + integration-test: needs: build runs-on: ubuntu-22.04 @@ -280,7 +418,7 @@ jobs: run: docker load --input /tmp/blaze.tar - name: Run Blaze - run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -p 8080:8080 -p 8081:8081 -v blaze-data:/app/data blaze:latest + run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_FRONTEND=true -p 8080:8080 -p 8081:8081 -v blaze-data:/app/data blaze:latest - name: Wait for Blaze run: .github/scripts/wait-for-url.sh http://localhost:8080/health @@ -294,6 +432,9 @@ jobs: - name: Check Referential Integrity Enforced run: .github/scripts/check-referential-integrity-enforced.sh + - name: Ensure that the State of PatientAsOf Index is Current + run: .github/scripts/check-patient-as-of-index-state.sh current + - name: Load Data run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea @@ -1137,6 +1278,50 @@ jobs: - name: Fetch Patient Expecting an Error run: .github/scripts/fetch-resource-0-with-missing-resource-content.sh + build-patient-as-of-index-test: + needs: build + runs-on: ubuntu-22.04 + + steps: + - name: Check out Git repository + uses: actions/checkout@v3 + + - name: Install Blazectl + run: .github/scripts/install-blazectl.sh + + - name: Download Blaze Image + uses: actions/download-artifact@v3 + with: + name: blaze-image + path: /tmp + + - name: Load Blaze Image + run: docker load --input /tmp/blaze.tar + + - name: Run Blaze v0.22 + run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_FRONTEND=true -p 8080:8080 -v blaze-data:/app/data samply/blaze:0.22 + + - name: Wait for Blaze + run: .github/scripts/wait-for-url.sh http://localhost:8080/health + + - name: Load Data + run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea + + - name: Ensure that the PatientAsOf Index does not exist + run: .github/scripts/check-patient-as-of-index-missing.sh + + - name: Shut down Blaze + run: docker stop blaze && docker rm blaze + + - name: Run Latest Blaze + run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -e ENABLE_FRONTEND=true -e LOG_LEVEL=debug -p 8080:8080 -v blaze-data:/app/data blaze:latest + + - name: Wait for Blaze + run: .github/scripts/wait-for-url.sh http://localhost:8080/health + + - name: Ensure that the State of PatientAsOf Index is Current + run: .github/scripts/check-patient-as-of-index-state.sh current + distributed-test: needs: build runs-on: ubuntu-22.04 @@ -1193,6 +1378,9 @@ jobs: - name: Check Referential Integrity Enforced run: .github/scripts/check-referential-integrity-enforced.sh + - name: Ensure that the State of PatientAsOf Index is Current + run: .github/scripts/check-patient-as-of-index-state.sh current + - name: Load Data run: blazectl --no-progress --server http://localhost:8080/fhir upload .github/test-data/synthea @@ -1613,6 +1801,7 @@ jobs: needs: - build - image-scan + - cql-expr-cache-test - integration-test - not-enforcing-referential-integrity-test - small-transactions-test @@ -1623,13 +1812,14 @@ jobs: - bundle-with-references-test - jepsen-test - openid-auth-test + - custom-search-parameters-test - doc-copy-data-test - big-binary-test - frontend-test - missing-resource-content-test + - build-patient-as-of-index-test - distributed-test - jepsen-distributed-test - - custom-search-parameters-test runs-on: ubuntu-22.04 permissions: packages: write diff --git a/dev/blaze/dev.clj b/dev/blaze/dev.clj index 98f3aa7e6..3e1e2b96a 100644 --- a/dev/blaze/dev.clj +++ b/dev/blaze/dev.clj @@ -1,9 +1,9 @@ (ns blaze.dev (:require [blaze.byte-string :as bs] + [blaze.cache-collector.protocols :as ccp] [blaze.db.api :as d] [blaze.db.api-spec] - [blaze.db.cache-collector.protocols :as ccp] [blaze.db.resource-cache :as resource-cache] [blaze.db.resource-store :as rs] [blaze.db.tx-log :as tx-log] @@ -66,6 +66,11 @@ (resource-cache/invalidate-all! (:blaze.db/resource-cache system)) ) +;; CQL Expression Cache +(comment + (str (ccp/-stats (:blaze.fhir.operation.evaluate-measure/expr-cache system))) + ) + ;; RocksDB Stats (comment (.reset (system [:blaze.db.kv.rocksdb/stats :blaze.db.index-kv-store/stats])) diff --git a/docs/deployment/environment-variables.md b/docs/deployment/environment-variables.md index bd00448e7..fe309e27e 100644 --- a/docs/deployment/environment-variables.md +++ b/docs/deployment/environment-variables.md @@ -91,6 +91,7 @@ More information about distributed deployment are available [here](distributed.m | ENFORCE_REFERENTIAL_INTEGRITY | true | v0.14 | — | Enforce referential integrity on resource create, update and delete. | | DB_SYNC_TIMEOUT | 10000 | v0.15 | — | Timeout in milliseconds for all reading FHIR interactions acquiring the newest database state. | | DB_SEARCH_PARAM_BUNDLE | — | v0.21 | — | Name of a custom search parameter bundle file. | +| CQL_EXPR_CACHE_SIZE | — | v0.23 | — | Size of the CQL expression cache. Will be disabled if not given. | ¹ Deprecated diff --git a/docs/implementation/cql.md b/docs/implementation/cql.md new file mode 100644 index 000000000..ef0370d74 --- /dev/null +++ b/docs/implementation/cql.md @@ -0,0 +1,40 @@ +# CQL + +## Expression Cache + +* bloom filter + * the set we like to build is the set of expressions returning true + * if the bloom filter returns false, we can be sure that the expression is not in the set of expressions returning true, so it will certainly return false + * the number of expressions returning true is far less then the number of expressions returning false + * we'll fill the Bloom filter with the expressions that returned true + * the problem is the following + * if we don't have filled the filter with all expression, the answer we get has no value + * we don't know whether the expression isn't in the set because we didn't put it there or because if returned false + * but we could use two filters + * one for all expressions returning true and one for all returning false + * if the expression isn't in both filters, it is new and so we have the evaluate it + * we could use a bloom filter for each expression + * then we would see whether we have a bloom filter or not + * we would insert Patients for which the expression returns true + * the first query evaluation is only used for insertion + * after that we mark the filter as ready for query + * now we can determine whether a expression is not true for a certain Patient + * + +* we'll use one Bloom filter per expression +* that Bloom filters will be stored in a Caffeine cache by expression hash +* each Bloom filter will be assigned the t of its creation +* the Bloom filters of all expressions of a query will be collected at the start of the query evaluation +* if a Bloom filter isn't found, its calculation will be queued and carried out asynchronously +* existing Bloom filters are immutable and will be used in query evaluation + * the Patient ID will be used to test whether this Patient isn't in the Bloom filter + * if the Patient ID wasn't found, the expression will return false + * if the Patient ID is found, the expression will be evaluated normally + +### Bloom Filter Calculation + +* the Bloom filter will be calculated for a particular exists expression +* it will be calculated based on a database with a particular t +* that t will be assigned to the Bloom filter +* the calculation will evaluate the expression for each patent of the database +* the ID's of Patients for which the expression returns true will be put into the Bloom filter diff --git a/docs/implementation/database.md b/docs/implementation/database.md index 6da5c8b8d..966bc65ca 100644 --- a/docs/implementation/database.md +++ b/docs/implementation/database.md @@ -38,16 +38,17 @@ There are two different sets of indices, ones which depend on the database value ### Indices depending on t -| Name | Key Parts | Value | -|--------------|-----------|-------------------------------| -| ResourceAsOf | type id t | content-hash, num-changes, op | -| TypeAsOf | type t id | content-hash, num-changes, op | -| SystemAsOf | t type id | content-hash, num-changes, op | -| TxSuccess | t | instant | -| TxError | t | anomaly | -| TByInstant | instant | t | -| TypeStats | type t | total, num-changes | -| SystemStats | t | total, num-changes | +| Name | Key Parts | Value | +|--------------|------------------|-------------------------------| +| ResourceAsOf | type id t | content-hash, num-changes, op | +| TypeAsOf | type t id | content-hash, num-changes, op | +| SystemAsOf | t type id | content-hash, num-changes, op | +| PatientAsOf | pat-id t type id | content-hash, num-changes, op | +| TxSuccess | t | instant | +| TxError | t | anomaly | +| TByInstant | instant | t | +| TypeStats | type t | total, num-changes | +| SystemStats | t | total, num-changes | #### ResourceAsOf @@ -83,12 +84,16 @@ In addition to direct resource lookup, the `ResourceAsOf` index is used for list #### TypeAsOf -The `TypeAsOf` index contains the same information as the `ResourceAsOf` index with the difference that the components of the key are ordered `type`, `t` and `id` instead of `type`, `id` and `t`. The index is used for listing all versions of all resources of a particular type. Such history listings start with the `t` of the database value going into the past. This is done by not only choosing the resource version with the latest `t` less or equal the database values `t` but instead using all older versions. Such versions even include deleted versions because in FHIR it is allowed to bring back a resource to a new life after it was already deleted. The listing is done by simply scanning through the index in reverse. Because the key is ordered by `type`, `t` and `id`, the entries will be first ordered by time, newest first, and second by resource identifier. +The `TypeAsOf` index contains the same information as the `ResourceAsOf` index with the difference that the components of the key are ordered `type`, `t` and `id` instead of `type`, `id` and `t`. The index is used for listing all versions of all resources of a particular type. Such history listings start with the `t` of the database value going into the past. This is done by not only choosing the resource version with the latest `t` less or equal the database values `t` but instead using all older versions. Such versions even include deleted versions because in FHIR it is allowed to bring back a resource to a new life after it was already deleted. The listing is done by simply scanning through the index in reverse. Because the key is ordered by `type`, `t` and `id`, the entries will be first ordered by time, newest first, and second by resource identifier. #### SystemAsOf In the same way the `TypeAsOf` index uses a different key ordering in comparison to the `ResourceAsOf` index, the `SystemAsOf` index will use the key order `t`, `type` and `id` in order to provide a global time axis order by resource type and by identifier secondarily. +#### PatientAsOf + +The `PatientAsOf` index works like the `SystemAsOf` index but for each Patient individually. It contains all changes to resources in the compartment of a particular Patient on reverse chronological order. Using the `PatientAsOf` index it's possible to create a history of all changes in the Patient compartment or detect the `t` of the last change. The CQL cache uses this index to invalidate cached results of expressions in the Patient context. + #### TxSuccess The `TxSuccess` index contains the real point in time, as `java.time.Instant`, successful transactions happened. In other words, this index maps each `t` which is just a monotonically increasing number to a real point in time. @@ -115,23 +120,24 @@ The `SystemStats` index keeps track of the total number of resources, and the nu The indices not depending on `t` directly point to the resource versions by their content hash. -| Name | Key Parts | Value | -|-------------------------------------|------------------------------------------------------------------|-------| -| SearchParamValueResource | search-param, type, value, id, content-hash | - | -| ResourceSearchParamValue | type, id, content-hash, search-param, value | - | -| CompartmentSearchParamValueResource | co-c-hash, co-res-id, search-param, type, value, id, hash-prefix | - | -| CompartmentResource | co-c-hash, co-res-id, tid, id | - | -| SearchParam | code, tid | id | -| ActiveSearchParams | id | - | +| Name | Key Parts | Value | +|-------------------------------------|----------------------------------------------------------------|-------| +| SearchParamValueResource | search-param, type, value, id, hash-prefix | - | +| ResourceSearchParamValue | type, id, content-hash, search-param, value | - | +| CompartmentSearchParamValueResource | comp-code, comp-id, search-param, type, value, id, hash-prefix | - | +| CompartmentResource | comp-code, comp-id, type, id | - | +| SearchParam | code, type | id | +| ActiveSearchParams | id | - | #### SearchParamValueResource The `SearchParamValueResource` index contains all values from resources that are reachable from search parameters. The components of its key are: -* `search-param` - a 4-byte hash of the search parameters code used to identify the search parameter -* `type` - a 4-byte hash of the resource type -* `value` - the encoded value of the resource reachable by the search parameters FHIRPath expression. The encoding depends on the search parameters type. -* `id` - the logical id of the resource -* `content-hash` - a 4-byte prefix of the content-hash of the resource version + + * `search-param` - a 4-byte hash of the search parameters code used to identify the search parameter + * `type` - a 4-byte hash of the resource type + * `value` - the encoded value of the resource reachable by the search parameters FHIRPath expression. The encoding depends on the search parameters type. + * `id` - the logical id of the resource + * `content-hash` - a 4-byte prefix of the content-hash of the resource version The way the `SearchParamValueResource` index is used, depends on the type of the search parameter. The following sections will explain this in detail for each type: @@ -207,11 +213,20 @@ That tuples are further processed against the `ResourceAsOf` index in order to c **TODO: continue...** +#### CompartmentResource + +The `CompartmentResource` index contains all resources that belong to a certain compartment. The components of its key are: + + * `comp-code` - a 4-byte hash of the compartment code, ex. `Patient` + * `comp-id` - the logical id of the compartment, ex. the logical id of the Patient + * `type` - a 4-byte hash of the resource type of the resource that belongs to the compartment, ex. `Observation` + * `id` - the logical id of the resource that belongs to the compartment, ex. the logical id of the Observation + ## Transaction Handling -* a transaction bundle is POST'ed to one arbitrary node -* this node submits the transaction commands to the central transaction log -* all nodes (inkl. the transaction submitter) receive the transaction commands from the central transaction log + * a transaction bundle is POST'ed to one arbitrary node + * this node submits the transaction commands to the central transaction log + * all nodes (inkl. the transaction submitter) receive the transaction commands from the central transaction log ### Transaction Commands diff --git a/docs/monitoring/blaze.json b/docs/monitoring/blaze.json index 33d5b5886..0cd8aaac3 100644 --- a/docs/monitoring/blaze.json +++ b/docs/monitoring/blaze.json @@ -24,7 +24,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 74, + "id": 79, "links": [ { "asDropdown": false, @@ -447,7 +447,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "blaze_db_cache_estimated_size{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}", + "expr": "blaze_cache_estimated_size{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}", "hide": false, "interval": "", "legendFormat": "", @@ -546,7 +546,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(blaze_db_cache_hits_total{job=\"$job\",instance=\"$instance\", name=\"resource-cache\"}[1m]) / (rate(blaze_db_cache_hits_total[1m]) + rate(blaze_db_cache_misses_total[1m]))", + "expr": "rate(blaze_cache_hits_total{job=\"$job\",instance=\"$instance\", name=\"resource-cache\"}[1m]) / (rate(blaze_cache_hits_total[1m]) + rate(blaze_cache_misses_total[1m]))", "hide": false, "interval": "", "legendFormat": "", @@ -644,7 +644,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(blaze_db_cache_load_successes_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])", + "expr": "rate(blaze_cache_load_successes_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])", "hide": false, "interval": "", "legendFormat": "", @@ -742,7 +742,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(blaze_db_cache_evictions_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])", + "expr": "rate(blaze_cache_evictions_total{job=\"$job\",instance=\"$instance\",name=\"resource-cache\"}[1m])", "hide": false, "interval": "", "legendFormat": "", @@ -759,7 +759,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 14 }, "id": 160, "panels": [], @@ -831,7 +831,7 @@ "h": 6, "w": 8, "x": 0, - "y": 16 + "y": 15 }, "id": 162, "links": [], @@ -898,7 +898,7 @@ "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 21 }, "id": 68, "panels": [], @@ -981,7 +981,7 @@ "h": 6, "w": 8, "x": 0, - "y": 23 + "y": 22 }, "id": 63, "options": { @@ -1105,7 +1105,7 @@ "h": 6, "w": 8, "x": 8, - "y": 23 + "y": 22 }, "id": 119, "options": { @@ -1205,7 +1205,7 @@ "h": 6, "w": 8, "x": 16, - "y": 23 + "y": 22 }, "id": 121, "options": { @@ -1305,7 +1305,7 @@ "h": 6, "w": 8, "x": 0, - "y": 29 + "y": 28 }, "id": 69, "options": { @@ -1429,7 +1429,7 @@ "h": 6, "w": 8, "x": 8, - "y": 29 + "y": 28 }, "id": 79, "options": { @@ -1529,7 +1529,7 @@ "h": 6, "w": 8, "x": 16, - "y": 29 + "y": 28 }, "id": 80, "options": { @@ -1628,7 +1628,7 @@ "h": 6, "w": 8, "x": 0, - "y": 35 + "y": 34 }, "id": 64, "options": { @@ -1752,7 +1752,7 @@ "h": 6, "w": 8, "x": 8, - "y": 35 + "y": 34 }, "id": 65, "options": { @@ -1876,7 +1876,7 @@ "h": 6, "w": 8, "x": 16, - "y": 35 + "y": 34 }, "id": 95, "options": { @@ -1930,6 +1930,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Open Iterators. Should be not growing unbounded. If not please file an issue.", "fieldConfig": { "defaults": { "color": { @@ -1989,15 +1990,12 @@ "h": 6, "w": 8, "x": 0, - "y": 41 + "y": 40 }, - "id": 113, - "links": [], + "id": 266, "options": { "legend": { - "calcs": [ - "mean" - ], + "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -2016,18 +2014,15 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(blaze_rocksdb_compaction_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "format": "time_series", + "expr": "blaze_rocksdb_iterators_created_total{job=\"$job\",instance=\"$instance\", name=\"$database\"} - blaze_rocksdb_iterators_deleted_total", + "hide": false, "interval": "", - "intervalFactor": 2, - "legendFormat": " {{name}}", - "metric": "process_cpu_seconds_total", + "legendFormat": "created", "range": true, - "refId": "A", - "step": 10 + "refId": "A" } ], - "title": "Compaction Seconds", + "title": "Iterators Open", "type": "timeseries" }, { @@ -2094,7 +2089,7 @@ "h": 6, "w": 8, "x": 8, - "y": 41 + "y": 40 }, "id": 114, "links": [], @@ -2199,7 +2194,7 @@ "h": 6, "w": 8, "x": 16, - "y": 41 + "y": 40 }, "id": 115, "links": [], @@ -2296,7 +2291,7 @@ } ] }, - "unit": "ops" + "unit": "short" }, "overrides": [] }, @@ -2304,12 +2299,15 @@ "h": 6, "w": 8, "x": 0, - "y": 47 + "y": 46 }, - "id": 155, + "id": 113, + "links": [], "options": { "legend": { - "calcs": [], + "calcs": [ + "mean" + ], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -2328,15 +2326,18 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(blaze_rocksdb_wal_syncs_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", - "hide": false, + "expr": "rate(blaze_rocksdb_compaction_seconds_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "format": "time_series", "interval": "", - "legendFormat": "{{name}}", + "intervalFactor": 2, + "legendFormat": " {{name}}", + "metric": "process_cpu_seconds_total", "range": true, - "refId": "A" + "refId": "A", + "step": 10 } ], - "title": "WAL Sync", + "title": "Compaction Seconds", "type": "timeseries" }, { @@ -2404,7 +2405,7 @@ "h": 6, "w": 8, "x": 8, - "y": 47 + "y": 46 }, "id": 94, "options": { @@ -2504,7 +2505,7 @@ "h": 6, "w": 8, "x": 16, - "y": 47 + "y": 46 }, "id": 97, "options": { @@ -2544,7 +2545,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "The total number of writes ending up with a timeout.", "fieldConfig": { "defaults": { "color": { @@ -2574,7 +2574,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -2604,9 +2604,9 @@ "h": 6, "w": 8, "x": 0, - "y": 53 + "y": 52 }, - "id": 98, + "id": 155, "options": { "legend": { "calcs": [], @@ -2628,7 +2628,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(blaze_rocksdb_write_timeout_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "expr": "rate(blaze_rocksdb_wal_syncs_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{name}}", @@ -2636,7 +2636,7 @@ "refId": "A" } ], - "title": "Write Timeouts", + "title": "WAL Sync", "type": "timeseries" }, { @@ -2703,7 +2703,7 @@ "h": 6, "w": 8, "x": 8, - "y": 53 + "y": 52 }, "id": 112, "links": [], @@ -2808,7 +2808,7 @@ "h": 6, "w": 8, "x": 16, - "y": 53 + "y": 52 }, "id": 154, "options": { @@ -2847,7 +2847,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "The total number of writes ending up with a timeout.", "fieldConfig": { "defaults": { "color": { @@ -2901,40 +2901,15 @@ }, "unit": "ops" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "index" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, - "y": 59 + "y": 58 }, - "id": 156, + "id": 98, "options": { "legend": { "calcs": [], @@ -2956,7 +2931,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(blaze_rocksdb_blocks_compressed_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "expr": "rate(blaze_rocksdb_write_timeout_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{name}}", @@ -2964,7 +2939,7 @@ "refId": "A" } ], - "title": "Blocks Compressed", + "title": "Write Timeouts", "type": "timeseries" }, { @@ -3057,7 +3032,7 @@ "h": 6, "w": 8, "x": 8, - "y": 59 + "y": 58 }, "id": 157, "options": { @@ -3156,7 +3131,7 @@ "h": 6, "w": 8, "x": 16, - "y": 59 + "y": 58 }, "id": 234, "options": { @@ -3221,7 +3196,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Values are per Column Family and stacked.", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3248,7 +3223,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": true, + "spanNulls": false, "stacking": { "group": "A", "mode": "normal" @@ -3273,24 +3248,48 @@ } ] }, - "unit": "bytes" + "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "index" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 6, - "w": 16, + "w": 8, "x": 0, - "y": 65 + "y": 64 }, - "id": 186, - "links": [], + "id": 156, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "right", - "showLegend": true + "placement": "bottom", + "showLegend": false }, "tooltip": { "mode": "multi", @@ -3305,19 +3304,16 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "exemplar": false, - "expr": "blaze_rocksdb_table_reader_usage_bytes{job=\"$job\",instance=\"$instance\", name=\"$database\"}", - "format": "time_series", + "exemplar": true, + "expr": "rate(blaze_rocksdb_blocks_compressed_total{job=\"$job\",instance=\"$instance\", name=\"$database\"}[1m])", + "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{column_family}}", - "metric": "process_cpu_seconds_total", + "legendFormat": "{{name}}", "range": true, - "refId": "A", - "step": 10 + "refId": "A" } ], - "title": "Table Reader Memory Usage", + "title": "Blocks Compressed", "type": "timeseries" }, { @@ -3409,8 +3405,8 @@ "gridPos": { "h": 6, "w": 8, - "x": 16, - "y": 65 + "x": 8, + "y": 64 }, "id": 158, "options": { @@ -3445,6 +3441,110 @@ "title": "Blocks Not Compressed", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Values are per Column Family and stacked.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 64 + }, + "id": 186, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "blaze_rocksdb_table_reader_usage_bytes{job=\"$job\",instance=\"$instance\", name=\"$database\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{column_family}}", + "metric": "process_cpu_seconds_total", + "range": true, + "refId": "A", + "step": 10 + } + ], + "title": "Table Reader Memory Usage", + "type": "timeseries" + }, { "collapsed": false, "datasource": { @@ -3455,7 +3555,7 @@ "h": 1, "w": 24, "x": 0, - "y": 71 + "y": 70 }, "id": 59, "panels": [], @@ -3535,7 +3635,7 @@ "h": 6, "w": 24, "x": 0, - "y": 72 + "y": 71 }, "id": 61, "links": [], @@ -3635,7 +3735,7 @@ "h": 6, "w": 8, "x": 0, - "y": 78 + "y": 77 }, "id": 62, "options": { @@ -3677,7 +3777,7 @@ "h": 1, "w": 24, "x": 0, - "y": 84 + "y": 83 }, "id": 261, "panels": [], @@ -3732,8 +3832,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3749,7 +3848,7 @@ "h": 6, "w": 8, "x": 0, - "y": 85 + "y": 84 }, "id": 259, "options": { @@ -3795,7 +3894,7 @@ "h": 1, "w": 24, "x": 0, - "y": 91 + "y": 90 }, "id": 46, "panels": [ @@ -4098,7 +4197,7 @@ "h": 1, "w": 24, "x": 0, - "y": 92 + "y": 91 }, "id": 86, "panels": [], @@ -4163,8 +4262,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4180,7 +4278,7 @@ "h": 6, "w": 8, "x": 0, - "y": 93 + "y": 92 }, "id": 22, "options": { @@ -4259,8 +4357,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4301,7 +4398,7 @@ "h": 6, "w": 8, "x": 8, - "y": 93 + "y": 92 }, "id": 87, "options": { @@ -4380,8 +4477,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4397,7 +4493,7 @@ "h": 6, "w": 8, "x": 16, - "y": 93 + "y": 92 }, "id": 88, "options": { @@ -4477,8 +4573,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4494,7 +4589,7 @@ "h": 6, "w": 8, "x": 0, - "y": 99 + "y": 98 }, "id": 90, "options": { @@ -4576,8 +4671,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4593,7 +4687,7 @@ "h": 6, "w": 8, "x": 8, - "y": 99 + "y": 98 }, "id": 91, "options": { @@ -4675,8 +4769,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4692,7 +4785,7 @@ "h": 6, "w": 8, "x": 16, - "y": 99 + "y": 98 }, "id": 92, "options": { @@ -4774,8 +4867,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4791,7 +4883,7 @@ "h": 6, "w": 8, "x": 0, - "y": 105 + "y": 104 }, "id": 93, "options": { @@ -4835,7 +4927,7 @@ "h": 1, "w": 24, "x": 0, - "y": 111 + "y": 110 }, "id": 103, "panels": [ @@ -5438,7 +5530,7 @@ "h": 1, "w": 24, "x": 0, - "y": 112 + "y": 111 }, "id": 31, "panels": [ @@ -6271,7 +6363,7 @@ "h": 1, "w": 24, "x": 0, - "y": 113 + "y": 112 }, "id": 24, "panels": [ @@ -6670,7 +6762,7 @@ "h": 1, "w": 24, "x": 0, - "y": 114 + "y": 113 }, "id": 37, "panels": [ @@ -7175,7 +7267,7 @@ "h": 1, "w": 24, "x": 0, - "y": 115 + "y": 114 }, "id": 34, "panels": [], @@ -7238,8 +7330,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7255,7 +7346,7 @@ "h": 7, "w": 8, "x": 0, - "y": 116 + "y": 115 }, "id": 4, "options": { @@ -7332,8 +7423,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7349,7 +7439,7 @@ "h": 7, "w": 8, "x": 8, - "y": 116 + "y": 115 }, "id": 42, "options": { @@ -7427,8 +7517,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7444,7 +7533,7 @@ "h": 7, "w": 8, "x": 16, - "y": 116 + "y": 115 }, "id": 83, "options": { @@ -7522,8 +7611,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7539,7 +7627,7 @@ "h": 7, "w": 8, "x": 0, - "y": 123 + "y": 122 }, "id": 2, "options": { @@ -7616,8 +7704,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7633,7 +7720,7 @@ "h": 7, "w": 8, "x": 8, - "y": 123 + "y": 122 }, "id": 43, "options": { @@ -7710,8 +7797,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7727,7 +7813,7 @@ "h": 7, "w": 8, "x": 16, - "y": 123 + "y": 122 }, "id": 82, "options": { @@ -7805,8 +7891,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7822,7 +7907,7 @@ "h": 7, "w": 8, "x": 0, - "y": 130 + "y": 129 }, "id": 122, "options": { @@ -7900,8 +7985,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7917,7 +8001,7 @@ "h": 7, "w": 8, "x": 8, - "y": 130 + "y": 129 }, "id": 210, "options": { @@ -7985,7 +8069,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(blaze_db_cache_hits_total, job)", + "definition": "label_values(blaze_cache_hits_total, job)", "hide": 0, "includeAll": false, "label": "Job", @@ -7993,7 +8077,7 @@ "name": "job", "options": [], "query": { - "query": "label_values(blaze_db_cache_hits_total, job)", + "query": "label_values(blaze_cache_hits_total, job)", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -8012,14 +8096,14 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(blaze_db_cache_estimated_size{job=\"$job\"}, instance)", + "definition": "label_values(blaze_cache_estimated_size{job=\"$job\"}, instance)", "hide": 0, "includeAll": false, "multi": false, "name": "instance", "options": [], "query": { - "query": "label_values(blaze_db_cache_estimated_size{job=\"$job\"}, instance)", + "query": "label_values(blaze_cache_estimated_size{job=\"$job\"}, instance)", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -8129,6 +8213,6 @@ "timezone": "", "title": "Blaze", "uid": "Q-h9isMWk", - "version": 1, + "version": 2, "weekStart": "" } diff --git a/docs/performance/cql.md b/docs/performance/cql.md index b2926fb8a..f72b49729 100644 --- a/docs/performance/cql.md +++ b/docs/performance/cql.md @@ -42,17 +42,19 @@ define InInitialPopulation: The CQL query is executed with the following `blazectl` command: ```sh -blazectl evaluate-measure "cql/observation-$CODE.yml" --server http://localhost:8080/fhir | jq -rf cql/result.jq +cql/search.sh observation-17861-6 +cql/search.sh observation-8310-5 +cql/search.sh observation-72514-3 ``` | System | Dataset | Code | # Hits | Time (s) | StdDev | Pat./s | |--------|---------|---------|-------:|---------:|-------:|--------:| -| LEA47 | 100k | 17861-6 | 2 k | 0.26 | 0.158 | 384.5 k | -| LEA47 | 100k | 8310-5 | 60 k | 0.28 | 0.142 | 351.4 k | -| LEA47 | 100k | 72514-3 | 100 k | 0.27 | 0.128 | 367.0 k | -| LEA47 | 1M | 17861-6 | 25 k | 2.61 | 0.208 | 383.1 k | -| LEA47 | 1M | 8310-5 | 603 k | 2.68 | 0.201 | 372.8 k | -| LEA47 | 1M | 72514-3 | 998 k | 2.82 | 0.192 | 354.9 k | +| LEA47 | 100k | 17861-6 | 2 k | 0.09 | 0.001 | 1.1 M | +| LEA47 | 100k | 8310-5 | 60 k | 0.10 | 0.001 | 1.0 M | +| LEA47 | 100k | 72514-3 | 100 k | 0.10 | 0.002 | 1.0 M | +| LEA47 | 1M | 17861-6 | 25 k | 0.96 | 0.006 | 1.0 M | +| LEA47 | 1M | 8310-5 | 603 k | 0.99 | 0.008 | 1.0 M | +| LEA47 | 1M | 72514-3 | 998 k | 1.02 | 0.005 | 980.5 k | | LEA58 | 1M | 17861-6 | 25 k | 2.87 | 0.291 | 348.5 k | | LEA58 | 1M | 8310-5 | 603 k | 3.02 | 0.257 | 330.8 k | | LEA58 | 1M | 72514-3 | 998 k | 3.06 | 0.426 | 326.7 k | @@ -80,22 +82,24 @@ define InInitialPopulation: The CQL query is executed with the following `blazectl` command: ```sh -blazectl evaluate-measure "cql/observation-$CODE-$VALUE.yml" --server http://localhost:8080/fhir | jq -rf cql/result.jq +cql/search.sh observation-body-weight-10 +cql/search.sh observation-body-weight-50 +cql/search.sh observation-body-weight-100 ``` | System | Dataset | Code | Value | # Hits | Time (s) | StdDev | Pat./s | |--------|---------|---------|--------:|-------:|---------:|-------:|--------:| -| LEA47 | 100k | 29463-7 | 13.6 kg | 10 k | 0.68 | 0.031 | 146.9 k | -| LEA47 | 100k | 29463-7 | 75.3 kg | 50 k | 0.51 | 0.033 | 197.1 k | -| LEA47 | 100k | 29463-7 | 185 kg | 100 k | 0.30 | 0.106 | 331.6 k | -| LEA47 | 1M | 29463-7 | 13.6 kg | 99 k | 151.05 | 4.674 | 6.6 k | -| LEA47 | 1M | 29463-7 | 75.3 kg | 500 k | 104.68 | 2.022 | 9.6 k | -| LEA47 | 1M | 29463-7 | 185 kg | 998 k | 3.19 | 0.176 | 313.8 k | +| LEA47 | 100k | 29463-7 | 13.6 kg | 10 k | 0.09 | 0.001 | 1.1 M | +| LEA47 | 100k | 29463-7 | 75.3 kg | 50 k | 0.10 | 0.001 | 1.0 M | +| LEA47 | 100k | 29463-7 | 185 kg | 100 k | 0.10 | 0.001 | 1.0 M | +| LEA47 | 1M | 29463-7 | 13.6 kg | 99 k | 0.98 | 0.013 | 1.0 M | +| LEA47 | 1M | 29463-7 | 75.3 kg | 500 k | 1.03 | 0.014 | 966.4 k | +| LEA47 | 1M | 29463-7 | 185 kg | 998 k | 1.03 | 0.007 | 970.5 k | | LEA58 | 1M | 29463-7 | 13.6 kg | 99 k | 8.24 | 0.072 | 121.4 k | | LEA58 | 1M | 29463-7 | 75.3 kg | 500 k | 6.59 | 0.140 | 151.8 k | | LEA58 | 1M | 29463-7 | 185 kg | 998 k | 3.04 | 0.209 | 329.0 k | -## Code and Value Search +## Code, Date and Age Search In this section, CQL Queries for selecting Patients which have Observation resources with code 718-7 (Hemoglobin), date between 2015 and 2019 and age of patient at observation date below 18. @@ -110,21 +114,117 @@ context Patient define InInitialPopulation: exists [Observation: Code '718-7' from loinc] O - where year from O.effective between 2015 and 2019 - and AgeInYearsAt(O.effective) < 18 + where year from (O.effective as dateTime) between 2015 and 2019 + and AgeInYearsAt(O.effective as dateTime) < 18 ``` The CQL query is executed with the following `blazectl` command: ```sh -blazectl evaluate-measure "cql/hemoglobin-date-age.yml" --server http://localhost:8080/fhir | jq -rf cql/result.jq +cql/search.sh hemoglobin-date-age +cql/search.sh calcium-date-age ``` | System | Dataset | Code | # Hits | Time (s) | StdDev | Pat./s | |--------|---------|------------|-------:|---------:|-------:|--------:| -| LEA47 | 100k | hemoglobin | 20 k | 0.35 | 0.034 | 286.5 k | -| LEA47 | 100k | calcium | 20 k | 1.50 | 0.035 | 66.6 k | -| LEA47 | 1M | hemoglobin | 200 k | 4.79 | 0.119 | 208.8 k | -| LEA47 | 1M | calcium | 199 k | 182.90 | 3.900 | 5.5 k | +| LEA47 | 100k | hemoglobin | 20 k | 0.09 | 0.001 | 1.1 M | +| LEA47 | 100k | calcium | 20 k | 0.09 | 0.001 | 1.1 M | +| LEA47 | 1M | hemoglobin | 200 k | 1.01 | 0.018 | 989.7 k | +| LEA47 | 1M | calcium | 199 k | 0.99 | 0.015 | 1.0 M | | LEA58 | 1M | hemoglobin | 200 k | 3.55 | 0.038 | 281.8 k | | LEA58 | 1M | calcium | 199 k | 10.10 | 0.033 | 99.0 k | + +## Double Code Search + +In this section, CQL Queries for selecting Patients which have Condition resources with one of two codes used. + +```text +library "condition-two" +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem sct: 'http://snomed.info/sct' +code fever: '386661006' from sct +code cough: '49727002' from sct + +context Patient + +define InInitialPopulation: + exists [Condition: fever] or + exists [Condition: cough] +``` + +```sh +cql/search.sh condition-two +``` + +| System | Dataset | # Hits | Time (s) | StdDev | Pat./s | +|--------|---------|-------:|---------:|-------:|--------:| +| LEA47 | 100k | 9 k | 0.10 | 0.002 | 988.5 k | +| LEA47 | 1M | 87 k | 1.01 | 0.010 | 988.8 k | + +## Ten Frequent Code Search + +```text +library "condition-ten-frequent" +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem sct: 'http://snomed.info/sct' + +context Patient + +define InInitialPopulation: + exists [Condition: Code '444814009' from sct] or + exists [Condition: Code '840544004' from sct] or + exists [Condition: Code '840539006' from sct] or + exists [Condition: Code '386661006' from sct] or + exists [Condition: Code '195662009' from sct] or + exists [Condition: Code '49727002' from sct] or + exists [Condition: Code '10509002' from sct] or + exists [Condition: Code '72892002' from sct] or + exists [Condition: Code '36955009' from sct] or + exists [Condition: Code '162864005' from sct] +``` + +```sh +cql/search.sh condition-ten-frequent +``` + +| System | Dataset | # Hits | Time (s) | StdDev | Pat./s | +|--------|---------|-------:|---------:|-------:|--------:| +| LEA47 | 100k | 95 k | 0.11 | 0.003 | 899.0 k | +| LEA47 | 1M | 954 k | 1.14 | 0.014 | 880.7 k | + +## Ten Rare Code Search + +```text +library "condition-ten-rare" +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem sct: 'http://snomed.info/sct' + +context Patient + +define InInitialPopulation: + exists [Condition: Code '62718007' from sct] or + exists [Condition: Code '234466008' from sct] or + exists [Condition: Code '288959006' from sct] or + exists [Condition: Code '47505003' from sct] or + exists [Condition: Code '698754002' from sct] or + exists [Condition: Code '157265008' from sct] or + exists [Condition: Code '15802004' from sct] or + exists [Condition: Code '14760008' from sct] or + exists [Condition: Code '36923009' from sct] or + exists [Condition: Code '45816000' from sct] +``` + +```sh +cql/search.sh condition-ten-rare +``` + +| System | Dataset | # Hits | Time (s) | StdDev | Pat./s | +|--------|---------|-------:|---------:|-------:|--------:| +| LEA47 | 100k | 0 k | 0.14 | 0.002 | 726.0 k | +| LEA47 | 1M | 4 k | 1.59 | 0.016 | 627.1 k | diff --git a/docs/performance/cql/calcium-date-age.cql b/docs/performance/cql/calcium-date-age.cql index 30fb87733..4dcc668a7 100644 --- a/docs/performance/cql/calcium-date-age.cql +++ b/docs/performance/cql/calcium-date-age.cql @@ -8,5 +8,5 @@ context Patient define InInitialPopulation: exists [Observation: Code '49765-1' from loinc] O - where year from O.effective between 2015 and 2019 - and AgeInYearsAt(O.effective) < 59 + where year from (O.effective as dateTime as dateTime) between 2015 and 2019 + and AgeInYearsAt(O.effective as dateTime) < 59 diff --git a/docs/performance/cql/code-date-age-search.sh b/docs/performance/cql/code-date-age-search.sh deleted file mode 100755 index 0470f1017..000000000 --- a/docs/performance/cql/code-date-age-search.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -e - -SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" -. "$SCRIPT_DIR/util.sh" - -BASE="http://localhost:8080/fhir" -START_EPOCH="$(date +"%s")" -PATIENT_TOTAL="$(curl -sH 'Accept: application/fhir+json' "$BASE/Patient?_summary=count" | jq -r .total)" -CODE="$1" - -echo "Counting Patients with Observations with code $CODE, date between 2015 and 2019 and age of patient at observation date below 18..." - -MEASURE_FILE="$SCRIPT_DIR/$CODE-date-age.yml" -TIMES_FILE="$START_EPOCH-$CODE-date-age.times" -COUNT="$(blazectl --server "$BASE" evaluate-measure "$MEASURE_FILE" 2> /dev/null | jq -r '.group[0].population[0].count')" - -for i in {0..6} -do - blazectl --server "$BASE" evaluate-measure "$MEASURE_FILE" 2> /dev/null |\ - jq -rf "$SCRIPT_DIR/duration.jq" >> "$TIMES_FILE" -done -calc-cql-print-stats "$TIMES_FILE" "$PATIENT_TOTAL" "$COUNT" diff --git a/docs/performance/cql/code-value-search.sh b/docs/performance/cql/code-value-search.sh deleted file mode 100755 index 2126191c3..000000000 --- a/docs/performance/cql/code-value-search.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e - -SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" -. "$SCRIPT_DIR/util.sh" - -BASE="http://localhost:8080/fhir" -START_EPOCH="$(date +"%s")" -PATIENT_TOTAL="$(curl -sH 'Accept: application/fhir+json' "$BASE/Patient?_summary=count" | jq -r .total)" -CODE="$1" -VALUE="$2" - -echo "Counting Patients with Observations with code $CODE and value $VALUE..." -COUNT="$(blazectl --server "$BASE" evaluate-measure "$SCRIPT_DIR/observation-$CODE-$VALUE.yml" 2> /dev/null | jq -r '.group[0].population[0].count')" -for i in {0..6} -do - blazectl evaluate-measure "cql/observation-$CODE-$VALUE.yml" --server "$BASE" 2> /dev/null |\ - jq -rf cql/duration.jq >> "$START_EPOCH-$CODE-$VALUE.times" -done -calc-cql-print-stats "$START_EPOCH-$CODE-$VALUE.times" "$PATIENT_TOTAL" "$COUNT" diff --git a/docs/performance/cql/condition-all.cql b/docs/performance/cql/condition-all.cql new file mode 100644 index 000000000..3763238e0 --- /dev/null +++ b/docs/performance/cql/condition-all.cql @@ -0,0 +1,228 @@ +library "condition-all" +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem sct: 'http://snomed.info/sct' + +context Patient + +define InInitialPopulation: + exists [Condition: Code '234466008' from sct] or + exists [Condition: Code '65275009' from sct] or + exists [Condition: Code '241929008' from sct] or + exists [Condition: Code '75498004' from sct] or + exists [Condition: Code '10509002' from sct] or + exists [Condition: Code '132281000119108' from sct] or + exists [Condition: Code '706870000' from sct] or + exists [Condition: Code '67782005' from sct] or + exists [Condition: Code '65710008' from sct] or + exists [Condition: Code '195662009' from sct] or + exists [Condition: Code '7200002' from sct] or + exists [Condition: Code '300916003' from sct] or + exists [Condition: Code '26929004' from sct] or + exists [Condition: Code '271737000' from sct] or + exists [Condition: Code '198992004' from sct] or + exists [Condition: Code '74400008' from sct] or + exists [Condition: Code '195967001' from sct] or + exists [Condition: Code '225444004' from sct] or + exists [Condition: Code '24079001' from sct] or + exists [Condition: Code '49436004' from sct] or + exists [Condition: Code '287185009' from sct] or + exists [Condition: Code '87628006' from sct] or + exists [Condition: Code '6072007' from sct] or + exists [Condition: Code '35999006' from sct] or + exists [Condition: Code '60951000119105' from sct] or + exists [Condition: Code '162864005' from sct] or + exists [Condition: Code '408512008' from sct] or + exists [Condition: Code '275272006' from sct] or + exists [Condition: Code '262574004' from sct] or + exists [Condition: Code '840539006' from sct] or + exists [Condition: Code '92691004' from sct] or + exists [Condition: Code '410429000' from sct] or + exists [Condition: Code '128188000' from sct] or + exists [Condition: Code '373587001' from sct] or + exists [Condition: Code '192127007' from sct] or + exists [Condition: Code '233678006' from sct] or + exists [Condition: Code '43724002' from sct] or + exists [Condition: Code '235919008' from sct] or + exists [Condition: Code '88805009' from sct] or + exists [Condition: Code '124171000119105' from sct] or + exists [Condition: Code '431855005' from sct] or + exists [Condition: Code '431856006' from sct] or + exists [Condition: Code '433144002' from sct] or + exists [Condition: Code '278860009' from sct] or + exists [Condition: Code '1121000119107' from sct] or + exists [Condition: Code '185086009' from sct] or + exists [Condition: Code '82423001' from sct] or + exists [Condition: Code '698754002' from sct] or + exists [Condition: Code '40055000' from sct] or + exists [Condition: Code '359817006' from sct] or + exists [Condition: Code '110030002' from sct] or + exists [Condition: Code '62564004' from sct] or + exists [Condition: Code '62106007' from sct] or + exists [Condition: Code '302297009' from sct] or + exists [Condition: Code '14760008' from sct] or + exists [Condition: Code '40275004' from sct] or + exists [Condition: Code '53741008' from sct] or + exists [Condition: Code '49727002' from sct] or + exists [Condition: Code '190905008' from sct] or + exists [Condition: Code '38822007' from sct] or + exists [Condition: Code '44054006' from sct] or + exists [Condition: Code '427089005' from sct] or + exists [Condition: Code '127013003' from sct] or + exists [Condition: Code '422034002' from sct] or + exists [Condition: Code '267060006' from sct] or + exists [Condition: Code '157265008' from sct] or + exists [Condition: Code '62718007' from sct] or + exists [Condition: Code '55680006' from sct] or + exists [Condition: Code '55680006' from sct] or + exists [Condition: Code '267036007' from sct] or + exists [Condition: Code '15802004' from sct] or + exists [Condition: Code '84757009' from sct] or + exists [Condition: Code '84757009' from sct] or + exists [Condition: Code '301011002' from sct] or + exists [Condition: Code '53827007' from sct] or + exists [Condition: Code '370247008' from sct] or + exists [Condition: Code '230265002' from sct] or + exists [Condition: Code '84229001' from sct] or + exists [Condition: Code '707577004' from sct] or + exists [Condition: Code '156073000' from sct] or + exists [Condition: Code '386661006' from sct] or + exists [Condition: Code '203082005' from sct] or + exists [Condition: Code '403190006' from sct] or + exists [Condition: Code '16114001' from sct] or + exists [Condition: Code '58150001' from sct] or + exists [Condition: Code '65966004' from sct] or + exists [Condition: Code '33737001' from sct] or + exists [Condition: Code '1734006' from sct] or + exists [Condition: Code '15724005' from sct] or + exists [Condition: Code '263102004' from sct] or + exists [Condition: Code '235595009' from sct] or + exists [Condition: Code '90560007' from sct] or + exists [Condition: Code '25064002' from sct] or + exists [Condition: Code '84114007' from sct] or + exists [Condition: Code '66857006' from sct] or + exists [Condition: Code '429280009' from sct] or + exists [Condition: Code '428251008' from sct] or + exists [Condition: Code '429007001' from sct] or + exists [Condition: Code '161622006' from sct] or + exists [Condition: Code '399211009' from sct] or + exists [Condition: Code '703151001' from sct] or + exists [Condition: Code '230745008' from sct] or + exists [Condition: Code '80394007' from sct] or + exists [Condition: Code '55822004' from sct] or + exists [Condition: Code '59621000' from sct] or + exists [Condition: Code '302870006' from sct] or + exists [Condition: Code '389087006' from sct] or + exists [Condition: Code '83664006' from sct] or + exists [Condition: Code '196416002' from sct] or + exists [Condition: Code '11218009' from sct] or + exists [Condition: Code '406602003' from sct] or + exists [Condition: Code '444470001' from sct] or + exists [Condition: Code '86175003' from sct] or + exists [Condition: Code '40095003' from sct] or + exists [Condition: Code '444448004' from sct] or + exists [Condition: Code '307731004' from sct] or + exists [Condition: Code '110359009' from sct] or + exists [Condition: Code '57676002' from sct] or + exists [Condition: Code '414564002' from sct] or + exists [Condition: Code '284551006' from sct] or + exists [Condition: Code '283371005' from sct] or + exists [Condition: Code '284549007' from sct] or + exists [Condition: Code '283385000' from sct] or + exists [Condition: Code '201834006' from sct] or + exists [Condition: Code '36955009' from sct] or + exists [Condition: Code '200936003' from sct] or + exists [Condition: Code '97331000119101' from sct] or + exists [Condition: Code '370143000' from sct] or + exists [Condition: Code '36923009' from sct] or + exists [Condition: Code '427089005' from sct] or + exists [Condition: Code '254837009' from sct] or + exists [Condition: Code '363406005' from sct] or + exists [Condition: Code '171131006' from sct] or + exists [Condition: Code '414667000' from sct] or + exists [Condition: Code '237602007' from sct] or + exists [Condition: Code '314994000' from sct] or + exists [Condition: Code '90781000119102' from sct] or + exists [Condition: Code '19169002' from sct] or + exists [Condition: Code '68962001' from sct] or + exists [Condition: Code '22298006' from sct] or + exists [Condition: Code '68235000' from sct] or + exists [Condition: Code '422587007' from sct] or + exists [Condition: Code '126906006' from sct] or + exists [Condition: Code '368581000119106' from sct] or + exists [Condition: Code '47200007' from sct] or + exists [Condition: Code '424132000' from sct] or + exists [Condition: Code '254637007' from sct] or + exists [Condition: Code '1551000119108' from sct] or + exists [Condition: Code '72892002' from sct] or + exists [Condition: Code '5602001' from sct] or + exists [Condition: Code '239872002' from sct] or + exists [Condition: Code '239873007' from sct] or + exists [Condition: Code '64859006' from sct] or + exists [Condition: Code '65363002' from sct] or + exists [Condition: Code '109838007' from sct] or + exists [Condition: Code '22253000' from sct] or + exists [Condition: Code '246677007' from sct] or + exists [Condition: Code '443165006' from sct] or + exists [Condition: Code '446096008' from sct] or + exists [Condition: Code '232353008' from sct] or + exists [Condition: Code '233604007' from sct] or + exists [Condition: Code '233604007' from sct] or + exists [Condition: Code '68496003' from sct] or + exists [Condition: Code '398152000' from sct] or + exists [Condition: Code '47505003' from sct] or + exists [Condition: Code '15777000' from sct] or + exists [Condition: Code '398254007' from sct] or + exists [Condition: Code '399912005' from sct] or + exists [Condition: Code '95417003' from sct] or + exists [Condition: Code '93761005' from sct] or + exists [Condition: Code '67811000119102' from sct] or + exists [Condition: Code '1501000119109' from sct] or + exists [Condition: Code '157141000119108' from sct] or + exists [Condition: Code '236077008' from sct] or + exists [Condition: Code '87433001' from sct] or + exists [Condition: Code '45816000' from sct] or + exists [Condition: Code '713197008' from sct] or + exists [Condition: Code '197927001' from sct] or + exists [Condition: Code '271825005' from sct] or + exists [Condition: Code '267064002' from sct] or + exists [Condition: Code '69896004' from sct] or + exists [Condition: Code '47693006' from sct] or + exists [Condition: Code '30832001' from sct] or + exists [Condition: Code '298382003' from sct] or + exists [Condition: Code '367498001' from sct] or + exists [Condition: Code '403191005' from sct] or + exists [Condition: Code '94260004' from sct] or + exists [Condition: Code '128613002' from sct] or + exists [Condition: Code '91302008' from sct] or + exists [Condition: Code '448813005' from sct] or + exists [Condition: Code '448417001' from sct] or + exists [Condition: Code '770349000' from sct] or + exists [Condition: Code '76571007' from sct] or + exists [Condition: Code '27942005' from sct] or + exists [Condition: Code '36971009' from sct] or + exists [Condition: Code '254632001' from sct] or + exists [Condition: Code '449868002' from sct] or + exists [Condition: Code '267102003' from sct] or + exists [Condition: Code '221360009' from sct] or + exists [Condition: Code '76916001' from sct] or + exists [Condition: Code '44465007' from sct] or + exists [Condition: Code '70704007' from sct] or + exists [Condition: Code '248595008' from sct] or + exists [Condition: Code '43878008' from sct] or + exists [Condition: Code '230690007' from sct] or + exists [Condition: Code '86849004' from sct] or + exists [Condition: Code '840544004' from sct] or + exists [Condition: Code '162573006' from sct] or + exists [Condition: Code '239720000' from sct] or + exists [Condition: Code '403192003' from sct] or + exists [Condition: Code '427419006' from sct] or + exists [Condition: Code '127295002' from sct] or + exists [Condition: Code '79586000' from sct] or + exists [Condition: Code '288959006' from sct] or + exists [Condition: Code '68566005' from sct] or + exists [Condition: Code '444814009' from sct] or + exists [Condition: Code '249497008' from sct] or + exists [Condition: Code '56018004' from sct] or + exists [Condition: Code '39848009' from sct] diff --git a/docs/performance/cql/condition-all.yml b/docs/performance/cql/condition-all.yml new file mode 100644 index 000000000..c05260376 --- /dev/null +++ b/docs/performance/cql/condition-all.yml @@ -0,0 +1,5 @@ +library: cql/condition-all.cql +group: +- type: Patient + population: + - expression: InInitialPopulation diff --git a/docs/performance/cql/condition-ten-frequent.cql b/docs/performance/cql/condition-ten-frequent.cql new file mode 100644 index 000000000..21f54cc03 --- /dev/null +++ b/docs/performance/cql/condition-ten-frequent.cql @@ -0,0 +1,19 @@ +library "condition-ten-frequent" +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem sct: 'http://snomed.info/sct' + +context Patient + +define InInitialPopulation: + exists [Condition: Code '444814009' from sct] or + exists [Condition: Code '840544004' from sct] or + exists [Condition: Code '840539006' from sct] or + exists [Condition: Code '386661006' from sct] or + exists [Condition: Code '195662009' from sct] or + exists [Condition: Code '49727002' from sct] or + exists [Condition: Code '10509002' from sct] or + exists [Condition: Code '72892002' from sct] or + exists [Condition: Code '36955009' from sct] or + exists [Condition: Code '162864005' from sct] diff --git a/docs/performance/cql/condition-ten-frequent.yml b/docs/performance/cql/condition-ten-frequent.yml new file mode 100644 index 000000000..c98e4b241 --- /dev/null +++ b/docs/performance/cql/condition-ten-frequent.yml @@ -0,0 +1,5 @@ +library: cql/condition-ten-frequent.cql +group: +- type: Patient + population: + - expression: InInitialPopulation diff --git a/docs/performance/cql/condition-ten-rare.cql b/docs/performance/cql/condition-ten-rare.cql new file mode 100644 index 000000000..4c94ed068 --- /dev/null +++ b/docs/performance/cql/condition-ten-rare.cql @@ -0,0 +1,19 @@ +library "condition-ten-rare" +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem sct: 'http://snomed.info/sct' + +context Patient + +define InInitialPopulation: + exists [Condition: Code '62718007' from sct] or + exists [Condition: Code '234466008' from sct] or + exists [Condition: Code '288959006' from sct] or + exists [Condition: Code '47505003' from sct] or + exists [Condition: Code '698754002' from sct] or + exists [Condition: Code '157265008' from sct] or + exists [Condition: Code '15802004' from sct] or + exists [Condition: Code '14760008' from sct] or + exists [Condition: Code '36923009' from sct] or + exists [Condition: Code '45816000' from sct] diff --git a/docs/performance/cql/condition-ten-rare.yml b/docs/performance/cql/condition-ten-rare.yml new file mode 100644 index 000000000..09ce1c85d --- /dev/null +++ b/docs/performance/cql/condition-ten-rare.yml @@ -0,0 +1,5 @@ +library: cql/condition-ten-rare.cql +group: +- type: Patient + population: + - expression: InInitialPopulation diff --git a/docs/performance/cql/condition-two.cql b/docs/performance/cql/condition-two.cql new file mode 100644 index 000000000..36630e504 --- /dev/null +++ b/docs/performance/cql/condition-two.cql @@ -0,0 +1,13 @@ +library "condition-two" +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +codesystem sct: 'http://snomed.info/sct' +code fever: '386661006' from sct +code cough: '49727002' from sct + +context Patient + +define InInitialPopulation: + exists [Condition: fever] or + exists [Condition: cough] diff --git a/docs/performance/cql/condition-two.yml b/docs/performance/cql/condition-two.yml new file mode 100644 index 000000000..46c786903 --- /dev/null +++ b/docs/performance/cql/condition-two.yml @@ -0,0 +1,5 @@ +library: cql/condition-two.cql +group: +- type: Patient + population: + - expression: InInitialPopulation diff --git a/docs/performance/cql/hemoglobin-date-age.cql b/docs/performance/cql/hemoglobin-date-age.cql index 81c25492a..554add059 100644 --- a/docs/performance/cql/hemoglobin-date-age.cql +++ b/docs/performance/cql/hemoglobin-date-age.cql @@ -8,5 +8,5 @@ context Patient define InInitialPopulation: exists [Observation: Code '718-7' from loinc] O - where year from O.effective between 2015 and 2019 - and AgeInYearsAt(O.effective) < 18 + where year from (O.effective as dateTime as dateTime) between 2015 and 2019 + and AgeInYearsAt(O.effective as dateTime) < 18 diff --git a/docs/performance/cql/search.sh b/docs/performance/cql/search.sh new file mode 100755 index 000000000..4648a95e8 --- /dev/null +++ b/docs/performance/cql/search.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +. "$SCRIPT_DIR/util.sh" + +BASE="http://localhost:8080/fhir" +START_EPOCH="$(date +"%s")" +PATIENT_TOTAL="$(curl -sH 'Accept: application/fhir+json' "$BASE/Patient?_summary=count" | jq -r .total)" +FILE="$1" + +echo "Counting Patients with criteria from $FILE..." +COUNT="$(blazectl --server "$BASE" evaluate-measure "$SCRIPT_DIR/$FILE.yml" 2> /dev/null | jq -r '.group[0].population[0].count')" +for i in {0..6} +do + blazectl --server "$BASE" evaluate-measure "$SCRIPT_DIR/$FILE.yml" 2> /dev/null |\ + jq -rf "$SCRIPT_DIR/duration.jq" >> "$START_EPOCH-$FILE.times" +done + +calc-cql-print-stats "$START_EPOCH-$FILE.times" "$PATIENT_TOTAL" "$COUNT" diff --git a/docs/performance/cql/simple-code-search.sh b/docs/performance/cql/simple-code-search.sh deleted file mode 100755 index e59cea0e6..000000000 --- a/docs/performance/cql/simple-code-search.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e - -SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" -. "$SCRIPT_DIR/util.sh" - -BASE="http://localhost:8080/fhir" -START_EPOCH="$(date +"%s")" -PATIENT_TOTAL="$(curl -sH 'Accept: application/fhir+json' "$BASE/Patient?_summary=count" | jq -r .total)" -CODE="$1" - -echo "Counting Patients with Observations with code $CODE..." -COUNT="$(blazectl --server "$BASE" evaluate-measure "$SCRIPT_DIR/observation-$CODE.yml" 2> /dev/null | jq -r '.group[0].population[0].count')" -for i in {0..6} -do - blazectl --server "$BASE" evaluate-measure "$SCRIPT_DIR/observation-$CODE.yml" 2> /dev/null |\ - jq -rf "$SCRIPT_DIR/duration.jq" >> "$START_EPOCH-$CODE.times" -done - -calc-cql-print-stats "$START_EPOCH-$CODE.times" "$PATIENT_TOTAL" "$COUNT" diff --git a/docs/performance/cql/util.sh b/docs/performance/cql/util.sh index 6f84918b9..3a9f2825e 100644 --- a/docs/performance/cql/util.sh +++ b/docs/performance/cql/util.sh @@ -24,7 +24,10 @@ calc-cql-print-stats() { fi # shorten the patients per second - if (( $(echo "$PATIENTS_PER_SEC > 1000" | bc) )); then + if (( $(echo "$PATIENTS_PER_SEC > 1000000" | bc) )); then + PATIENTS_PER_SEC=$(echo "scale=2; $PATIENTS_PER_SEC / 1000000" | bc) + PATIENTS_PER_SEC_FORMAT="%4.1f M" + elif (( $(echo "$PATIENTS_PER_SEC > 1000" | bc) )); then PATIENTS_PER_SEC=$(echo "scale=2; $PATIENTS_PER_SEC / 1000" | bc) PATIENTS_PER_SEC_FORMAT="%4.1f k" else diff --git a/docs/performance/synthea/README.md b/docs/performance/synthea/README.md index cc5c421c0..01699ca52 100644 --- a/docs/performance/synthea/README.md +++ b/docs/performance/synthea/README.md @@ -14,8 +14,10 @@ docker run -v "$(pwd)/output:/gen/output" synthea 100 ## Post-Process Bundles +While Synthea is running, it's important to run alrady the post-process loop. Otherwise the amount of data Synthea generates will likely fill your disk with uncompressed FHIR bundles. + ```sh -./post-process-bundles.sh output/fhir +./post-process-bundle-loop.sh output/fhir ``` ## Upload Bundles diff --git a/modules/admin-api/.clj-kondo/config.edn b/modules/admin-api/.clj-kondo/config.edn index b588017ea..b25e046ab 100644 --- a/modules/admin-api/.clj-kondo/config.edn +++ b/modules/admin-api/.clj-kondo/config.edn @@ -1,5 +1,6 @@ {:lint-as - {blaze.admin-api-test/with-handler clojure.core/fn} + {blaze.admin-api-test/with-handler clojure.core/fn + blaze.anomaly/if-ok clojure.core/let} :linters {:unsorted-required-namespaces diff --git a/modules/admin-api/deps.edn b/modules/admin-api/deps.edn index e20aadb66..8c780c24b 100644 --- a/modules/admin-api/deps.edn +++ b/modules/admin-api/deps.edn @@ -12,7 +12,10 @@ {:local/root "../rocksdb"} blaze/spec - {:local/root "../spec"}} + {:local/root "../spec"} + + fi.metosin/reitit-openapi + {:mvn/version "0.7.0-alpha5"}} :aliases {:test diff --git a/modules/admin-api/src/blaze/admin_api.clj b/modules/admin-api/src/blaze/admin_api.clj index f99e365c3..d82519491 100644 --- a/modules/admin-api/src/blaze/admin_api.clj +++ b/modules/admin-api/src/blaze/admin_api.clj @@ -1,10 +1,13 @@ (ns blaze.admin-api (:require + [blaze.anomaly :refer [if-ok]] [blaze.async.comp :as ac] + [blaze.db.impl.index.patient-as-of :as pao] [blaze.db.kv.rocksdb :as rocksdb] [blaze.spec] [clojure.spec.alpha :as s] [integrant.core :as ig] + [reitit.openapi :as openapi] [reitit.ring] [reitit.ring.spec] [ring.util.response :as ring] @@ -18,26 +21,86 @@ (ac/completed-future)))) -(defn- rocksdb-table-handler [index-kv-store] +(defn- cf-state-handler [index-kv-store] (fn [{{:keys [column-family]} :path-params}] - (-> (ring/response {:tables (rocksdb/table-properties index-kv-store (keyword column-family))}) + (-> (if (= "patient-as-of-index" column-family) + (ring/response (pao/state index-kv-store)) + (ring/not-found {:msg (format "The column family `%s` has no state." column-family)})) (ac/completed-future)))) +(defn- cf-rocksdb-table-handler [index-kv-store] + (fn [{{:keys [column-family]} :path-params}] + (-> (if-ok [tables (rocksdb/table-properties index-kv-store (keyword column-family))] + (ring/response {:tables tables}) + (fn [_] (ring/not-found {:msg (format "The column family `%s` was not found." column-family)}))) + (ac/completed-future)))) + + +(def ^:private openapi-handler + (let [handler (openapi/create-openapi-handler)] + (fn [request] + (ac/completed-future (handler request))))) + + (defn- router [{:keys [context-path index-kv-store] :or {context-path ""}}] (reitit.ring/router - ["/rocksdb" - {} - ["/index" + ["" + {:openapi {:id :admin-api} + :middleware [openapi/openapi-feature]} + ["/openapi.json" + {:get {:handler openapi-handler + :openapi {:info {:title "Blaze Admin API" :version "0.22"}} + :no-doc true}}] + ["/db" {} - ["/column-families" + ["/index" {} - ["" - {:get (rocksdb-column-families-handler index-kv-store)}] - ["/{column-family}" + ["/column-families" {} - ["/tables" - {:get (rocksdb-table-handler index-kv-store)}]]]]] + ["" + {:get + {:handler (rocksdb-column-families-handler index-kv-store) + :summary "Fetch a list of all column families." + :openapi + {:responses + {200 + {:content + {"application/json" + {:schema + {:type "object" + :properties + [:column-families + {:type "array" + :items {:type "string"}}]}}}}}}}}] + ["/{column-family}" + {} + ["/state" + {:get + {:handler (cf-state-handler index-kv-store)}}] + ["/rocksdb-tables" + {:get + {:handler (cf-rocksdb-table-handler index-kv-store) + :summary "Fetch a list of all tables of a column family." + :openapi + {:parameters + [{:name "column-family" + :in "path" + :required true}] + :responses + {200 + {:content + {"application/json" + {:schema + {:type "object" + :properties + [:tables + {:type "array" + :items + {:type "object" + :properties + [:data-size {:type "integer"} + :total-raw-key-size {:type "integer"}]}}]}}}}}}}}]]]]]] {:path (str context-path "/__admin") :syntax :bracket})) diff --git a/modules/admin-api/test/blaze/admin_api_test.clj b/modules/admin-api/test/blaze/admin_api_test.clj index f538d8207..383770d65 100644 --- a/modules/admin-api/test/blaze/admin_api_test.clj +++ b/modules/admin-api/test/blaze/admin_api_test.clj @@ -1,6 +1,8 @@ (ns blaze.admin-api-test (:require [blaze.admin-api] + [blaze.anomaly :as ba] + [blaze.db.impl.index.patient-as-of :as pao] [blaze.db.kv.rocksdb.protocols :as p] [blaze.module.test-util :refer [with-system]] [blaze.test-util :as tu :refer [given-thrown]] @@ -47,9 +49,10 @@ (keys column-families)) (-table-properties [_ column-family] - (when (column-families column-family) + (if (column-families column-family) [{:name (str (name column-family) "/table-1") - :data-size 193338}])))) + :data-size 193338}] + (ba/not-found ""))))) (def config @@ -69,28 +72,55 @@ ~@body))) -(deftest rocksdb-column-families-test +(deftest handler-not-found-test + (with-handler [handler] + (given @(handler + {:request-method :get + :uri "/fhir/__admin/foo"}) + :status := 404))) + + +(deftest column-families-test (with-handler [handler] (testing "success" (given @(handler {:request-method :get - :uri "/fhir/__admin/rocksdb/index/column-families"}) + :uri "/fhir/__admin/db/index/column-families"}) :status := 200 [:body :column-families] := ["column-family-1" "column-family-2"])))) -(deftest rocksdb-tables-test +(deftest column-family-state-test (with-handler [handler] - (testing "not found" + (testing "patient-as-of-index" + (with-redefs [pao/state (fn [_] ::state)] + (given @(handler + {:request-method :get + :uri "/fhir/__admin/db/index/column-families/patient-as-of-index/state"}) + :status := 200 + :body := ::state))) + + (testing "other column-family" (given @(handler {:request-method :get - :uri "/fhir/__admin/foo"}) - :status := 404)) + :uri "/fhir/__admin/db/index/column-families/column-family-1/state"}) + :status := 404 + [:body :msg] := "The column family `column-family-1` has no state.")))) + +(deftest rocksdb-tables-test + (with-handler [handler] (testing "success" (given @(handler {:request-method :get - :uri "/fhir/__admin/rocksdb/index/column-families/column-family-1/tables"}) + :uri "/fhir/__admin/db/index/column-families/column-family-1/rocksdb-tables"}) :status := 200 [:body :tables 0 :name] := "column-family-1/table-1" - [:body :tables 0 :data-size] := 193338)))) + [:body :tables 0 :data-size] := 193338)) + + (testing "not-found" + (given @(handler + {:request-method :get + :uri "/fhir/__admin/db/index/column-families/column-family-3/rocksdb-tables"}) + :status := 404 + [:body :msg] := "The column family `column-family-3` was not found.")))) diff --git a/modules/byte-buffer/src/blaze/byte_buffer.clj b/modules/byte-buffer/src/blaze/byte_buffer.clj index b4e6b2348..de8f18577 100644 --- a/modules/byte-buffer/src/blaze/byte_buffer.clj +++ b/modules/byte-buffer/src/blaze/byte_buffer.clj @@ -225,10 +225,15 @@ (defn get-long! {:inline - (fn [byte-buffer] - `(.getLong ~(vary-meta byte-buffer assoc :tag `ByteBuffer)))} - [byte-buffer] - (.getLong ^ByteBuffer byte-buffer)) + (fn + ([byte-buffer] + `(.getLong ~(vary-meta byte-buffer assoc :tag `ByteBuffer))) + ([byte-buffer index] + `(.getLong ~(vary-meta byte-buffer assoc :tag `ByteBuffer) (int ~index))))} + ([byte-buffer] + (.getLong ^ByteBuffer byte-buffer)) + ([byte-buffer index] + (.getLong ^ByteBuffer byte-buffer (int index)))) (defn copy-into-byte-array! diff --git a/modules/cache-collector/.clj-kondo/config.edn b/modules/cache-collector/.clj-kondo/config.edn new file mode 100644 index 000000000..a25d19101 --- /dev/null +++ b/modules/cache-collector/.clj-kondo/config.edn @@ -0,0 +1,20 @@ +{:lint-as + {blaze.module.test-util/with-system clojure.core/with-open} + + :linters + {:unsorted-required-namespaces + {:level :error} + + :single-key-in + {:level :warning} + + :keyword-binding + {:level :error} + + :reduce-without-init + {:level :warning} + + :warn-on-reflection + {:level :warning :warn-only-on-interop true}} + + :skip-comments true} diff --git a/modules/cache-collector/Makefile b/modules/cache-collector/Makefile new file mode 100644 index 000000000..7510b665c --- /dev/null +++ b/modules/cache-collector/Makefile @@ -0,0 +1,22 @@ +lint: + clj-kondo --lint src test deps.edn + +prep: + clojure -X:deps prep + +test: prep + clojure -M:test:kaocha --profile :ci + +test-coverage: prep + clojure -M:test:coverage + +deps-tree: + clojure -X:deps tree + +deps-list: + clojure -X:deps list + +clean: + rm -rf .clj-kondo/.cache .cpcache target + +.PHONY: lint prep test test-coverage deps-tree deps-list clean diff --git a/modules/cache-collector/deps.edn b/modules/cache-collector/deps.edn new file mode 100644 index 000000000..cdb76985b --- /dev/null +++ b/modules/cache-collector/deps.edn @@ -0,0 +1,45 @@ +{:deps + {blaze/metrics + {:local/root "../metrics"} + + blaze/module-base + {:local/root "../module-base"} + + com.github.ben-manes.caffeine/caffeine + {:mvn/version "3.1.8"}} + + :aliases + {:test + {:extra-paths ["test"] + + :extra-deps + {blaze/module-test-util + {:local/root "../module-test-util"}}} + + :kaocha + {:extra-deps + {lambdaisland/kaocha + {:mvn/version "1.85.1342"}} + + :main-opts ["-m" "kaocha.runner"]} + + :test-perf + {:extra-paths ["test-perf"] + + :extra-deps + {blaze/fhir-test-util + {:local/root "../fhir-test-util"} + + criterium/criterium + {:mvn/version "0.4.6"} + + org.openjdk.jol/jol-core + {:mvn/version "0.17"}}} + + :coverage + {:extra-deps + {cloverage/cloverage + {:mvn/version "1.2.4"}} + + :main-opts ["-m" "cloverage.coverage" "--codecov" "-p" "src" "-s" "test" + "-e" ".+spec"]}}} diff --git a/modules/db/src/blaze/db/cache_collector.clj b/modules/cache-collector/src/blaze/cache_collector.clj similarity index 75% rename from modules/db/src/blaze/db/cache_collector.clj rename to modules/cache-collector/src/blaze/cache_collector.clj index 599684c94..62b6bcb6f 100644 --- a/modules/db/src/blaze/db/cache_collector.clj +++ b/modules/cache-collector/src/blaze/cache_collector.clj @@ -1,12 +1,12 @@ -(ns blaze.db.cache-collector +(ns blaze.cache-collector (:require - [blaze.db.cache-collector.protocols :as p] - [blaze.db.cache-collector.spec] + [blaze.cache-collector.protocols :as p] + [blaze.cache-collector.spec] [blaze.metrics.core :as metrics] [clojure.spec.alpha :as s] [integrant.core :as ig]) (:import - [com.github.benmanes.caffeine.cache Cache] + [com.github.benmanes.caffeine.cache Cache AsyncCache] [com.github.benmanes.caffeine.cache.stats CacheStats])) @@ -18,7 +18,12 @@ (-stats [cache] (.stats cache)) (-estimated-size [cache] - (.estimatedSize cache))) + (.estimatedSize cache)) + AsyncCache + (-stats [cache] + (.stats (.synchronous cache))) + (-estimated-size [cache] + (.estimatedSize (.synchronous cache)))) (defn- sample-xf [f] @@ -42,49 +47,49 @@ [name (p/-stats cache) (p/-estimated-size cache)])))) -(defmethod ig/pre-init-spec :blaze.db/cache-collector [_] +(defmethod ig/pre-init-spec :blaze/cache-collector [_] (s/keys :req-un [::caches])) -(defmethod ig/init-key :blaze.db/cache-collector +(defmethod ig/init-key :blaze/cache-collector [_ {:keys [caches]}] (metrics/collector (let [stats (into [] mapper caches)] [(counter-metric - "blaze_db_cache_hits_total" + "blaze_cache_hits_total" "Returns the number of times Cache lookup methods have returned a cached value." (fn [stats _] (.hitCount ^CacheStats stats)) stats) (counter-metric - "blaze_db_cache_misses_total" + "blaze_cache_misses_total" "Returns the number of times Cache lookup methods have returned an uncached (newly loaded) value, or null." (fn [stats _] (.missCount ^CacheStats stats)) stats) (counter-metric - "blaze_db_cache_load_successes_total" + "blaze_cache_load_successes_total" "Returns the number of times Cache lookup methods have successfully loaded a new value." (fn [stats _] (.loadSuccessCount ^CacheStats stats)) stats) (counter-metric - "blaze_db_cache_load_failures_total" + "blaze_cache_load_failures_total" "Returns the number of times Cache lookup methods failed to load a new value, either because no value was found or an exception was thrown while loading." (fn [stats _] (.loadFailureCount ^CacheStats stats)) stats) (counter-metric - "blaze_db_cache_load_seconds_total" + "blaze_cache_load_seconds_total" "Returns the total number of seconds the cache has spent loading new values." (fn [stats _] (/ (double (.totalLoadTime ^CacheStats stats)) 1e9)) stats) (counter-metric - "blaze_db_cache_evictions_total" + "blaze_cache_evictions_total" "Returns the number of times an entry has been evicted." (fn [stats _] (.evictionCount ^CacheStats stats)) stats) (gauge-metric - "blaze_db_cache_estimated_size" + "blaze_cache_estimated_size" "Returns the approximate number of entries in this cache." (fn [_ estimated-size] estimated-size) stats)]))) -(derive :blaze.db/cache-collector :blaze.metrics/collector) +(derive :blaze/cache-collector :blaze.metrics/collector) diff --git a/modules/db/src/blaze/db/cache_collector/protocols.clj b/modules/cache-collector/src/blaze/cache_collector/protocols.clj similarity index 62% rename from modules/db/src/blaze/db/cache_collector/protocols.clj rename to modules/cache-collector/src/blaze/cache_collector/protocols.clj index b8224f17f..6b211f0ae 100644 --- a/modules/db/src/blaze/db/cache_collector/protocols.clj +++ b/modules/cache-collector/src/blaze/cache_collector/protocols.clj @@ -1,4 +1,4 @@ -(ns blaze.db.cache-collector.protocols) +(ns blaze.cache-collector.protocols) (defprotocol StatsCache diff --git a/modules/cache-collector/src/blaze/cache_collector/spec.clj b/modules/cache-collector/src/blaze/cache_collector/spec.clj new file mode 100644 index 000000000..7e4378a17 --- /dev/null +++ b/modules/cache-collector/src/blaze/cache_collector/spec.clj @@ -0,0 +1,8 @@ +(ns blaze.cache-collector.spec + (:require + [blaze.cache-collector.protocols :as p] + [clojure.spec.alpha :as s])) + + +(s/def :blaze.cache-collector/caches + (s/map-of string? (s/nilable #(satisfies? p/StatsCache %)))) diff --git a/modules/db/test/blaze/db/cache_collector_test.clj b/modules/cache-collector/test/blaze/cache_collector_test.clj similarity index 63% rename from modules/db/test/blaze/db/cache_collector_test.clj rename to modules/cache-collector/test/blaze/cache_collector_test.clj index 8b0f7d826..7ea7be27b 100644 --- a/modules/db/test/blaze/db/cache_collector_test.clj +++ b/modules/cache-collector/test/blaze/cache_collector_test.clj @@ -1,6 +1,6 @@ -(ns blaze.db.cache-collector-test +(ns blaze.cache-collector-test (:require - [blaze.db.cache-collector] + [blaze.cache-collector] [blaze.metrics.core :as metrics] [blaze.module.test-util :refer [with-system]] [blaze.test-util :as tu :refer [given-thrown]] @@ -25,55 +25,55 @@ (def config - {:blaze.db/cache-collector + {:blaze/cache-collector {:caches {"name-135224" cache "name-093214" nil}}}) (deftest init-test (testing "nil config" - (given-thrown (ig/init {:blaze.db/cache-collector nil}) - :key := :blaze.db/cache-collector + (given-thrown (ig/init {:blaze/cache-collector nil}) + :key := :blaze/cache-collector :reason := ::ig/build-failed-spec [:explain ::s/problems 0 :pred] := `map?)) (testing "missing config" - (given-thrown (ig/init {:blaze.db/cache-collector {}}) - :key := :blaze.db/cache-collector + (given-thrown (ig/init {:blaze/cache-collector {}}) + :key := :blaze/cache-collector :reason := ::ig/build-failed-spec [:explain ::s/problems 0 :pred] := `(fn ~'[%] (contains? ~'% :caches)))) (testing "invalid caches" - (given-thrown (ig/init {:blaze.db/cache-collector {:caches ::invalid}}) - :key := :blaze.db/cache-collector + (given-thrown (ig/init {:blaze/cache-collector {:caches ::invalid}}) + :key := :blaze/cache-collector :reason := ::ig/build-failed-spec [:explain ::s/problems 0 :pred] := `map? [:explain ::s/problems 0 :val] := ::invalid))) (deftest cache-collector-test - (with-system [{collector :blaze.db/cache-collector} config] + (with-system [{collector :blaze/cache-collector} config] (testing "all zero on fresh cache" (given (metrics/collect collector) - [0 :name] := "blaze_db_cache_hits" + [0 :name] := "blaze_cache_hits" [0 :type] := :counter [0 :samples 0 :value] := 0.0 - [1 :name] := "blaze_db_cache_misses" + [1 :name] := "blaze_cache_misses" [1 :type] := :counter [1 :samples 0 :value] := 0.0 - [2 :name] := "blaze_db_cache_load_successes" + [2 :name] := "blaze_cache_load_successes" [2 :type] := :counter [2 :samples 0 :value] := 0.0 - [3 :name] := "blaze_db_cache_load_failures" + [3 :name] := "blaze_cache_load_failures" [3 :type] := :counter [3 :samples 0 :value] := 0.0 - [4 :name] := "blaze_db_cache_load_seconds" + [4 :name] := "blaze_cache_load_seconds" [4 :type] := :counter [4 :samples 0 :value] := 0.0 - [5 :name] := "blaze_db_cache_evictions" + [5 :name] := "blaze_cache_evictions" [5 :type] := :counter [5 :samples 0 :value] := 0.0 - [6 :name] := "blaze_db_cache_estimated_size" + [6 :name] := "blaze_cache_estimated_size" [6 :type] := :gauge [6 :samples 0 :value] := 0.0)) @@ -82,17 +82,17 @@ (Thread/sleep 100) (given (metrics/collect collector) - [0 :name] := "blaze_db_cache_hits" + [0 :name] := "blaze_cache_hits" [0 :samples 0 :value] := 0.0 - [1 :name] := "blaze_db_cache_misses" + [1 :name] := "blaze_cache_misses" [1 :samples 0 :value] := 1.0 - [2 :name] := "blaze_db_cache_load_successes" + [2 :name] := "blaze_cache_load_successes" [2 :samples 0 :value] := 1.0 - [3 :name] := "blaze_db_cache_load_failures" + [3 :name] := "blaze_cache_load_failures" [3 :samples 0 :value] := 0.0 - [5 :name] := "blaze_db_cache_evictions" + [5 :name] := "blaze_cache_evictions" [5 :samples 0 :value] := 0.0 - [6 :name] := "blaze_db_cache_estimated_size" + [6 :name] := "blaze_cache_estimated_size" [6 :samples 0 :value] := 1.0)) (testing "one loads and one hit" @@ -100,15 +100,15 @@ (Thread/sleep 100) (given (metrics/collect collector) - [0 :name] := "blaze_db_cache_hits" + [0 :name] := "blaze_cache_hits" [0 :samples 0 :value] := 1.0 - [1 :name] := "blaze_db_cache_misses" + [1 :name] := "blaze_cache_misses" [1 :samples 0 :value] := 1.0 - [2 :name] := "blaze_db_cache_load_successes" + [2 :name] := "blaze_cache_load_successes" [2 :samples 0 :value] := 1.0 - [3 :name] := "blaze_db_cache_load_failures" + [3 :name] := "blaze_cache_load_failures" [3 :samples 0 :value] := 0.0 - [5 :name] := "blaze_db_cache_evictions" + [5 :name] := "blaze_cache_evictions" [5 :samples 0 :value] := 0.0 - [6 :name] := "blaze_db_cache_estimated_size" + [6 :name] := "blaze_cache_estimated_size" [6 :samples 0 :value] := 1.0)))) diff --git a/modules/cache-collector/tests.edn b/modules/cache-collector/tests.edn new file mode 100644 index 000000000..94fe5636c --- /dev/null +++ b/modules/cache-collector/tests.edn @@ -0,0 +1,5 @@ +#kaocha/v1 + #merge + [{} + #profile {:ci {:reporter kaocha.report/documentation + :color? false}}] diff --git a/modules/cql/.clj-kondo/config.edn b/modules/cql/.clj-kondo/config.edn index 7172e6ab1..378f20272 100644 --- a/modules/cql/.clj-kondo/config.edn +++ b/modules/cql/.clj-kondo/config.edn @@ -9,7 +9,9 @@ blaze.elm.compiler.macros/defnaryop clojure.core/defn blaze.elm.compiler.macros/defaggop clojure.core/defn blaze.elm.compiler.macros/defbinopp clojure.core/defn - blaze.elm.compiler.macros/defunopp clojure.core/defn} + blaze.elm.compiler.macros/defunopp clojure.core/defn + prometheus.alpha/defcounter clojure.core/def + prometheus.alpha/defhistogram clojure.core/def} :linters {;; because of macros in modules/cql/src/blaze/elm/compiler.clj @@ -43,6 +45,7 @@ blaze.cql-translator t blaze.db.api d blaze.elm.compiler c + blaze.elm.compiler.expr-cache ec blaze.elm.compiler.external-data ed blaze.elm.expression expr cognitect.anomalies anom diff --git a/modules/cql/src/blaze/elm/code.clj b/modules/cql/src/blaze/elm/code.clj index bce654696..8347c26cc 100644 --- a/modules/cql/src/blaze/elm/code.clj +++ b/modules/cql/src/blaze/elm/code.clj @@ -27,6 +27,8 @@ core/Expression (-static [_] true) + (-attach-cache [expr _] + expr) (-eval [this _ _ _] this) (-form [_] diff --git a/modules/cql/src/blaze/elm/compiler.clj b/modules/cql/src/blaze/elm/compiler.clj index 5f3679dce..093ec8c3f 100644 --- a/modules/cql/src/blaze/elm/compiler.clj +++ b/modules/cql/src/blaze/elm/compiler.clj @@ -44,5 +44,9 @@ (core/compile* context expression)) +(defn attach-cache [context expression] + (core/-attach-cache expression context)) + + (defn form [expression] (core/-form expression)) diff --git a/modules/cql/src/blaze/elm/compiler/arithmetic_operators.clj b/modules/cql/src/blaze/elm/compiler/arithmetic_operators.clj index 7fd8a573e..c9c7b4f34 100644 --- a/modules/cql/src/blaze/elm/compiler/arithmetic_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/arithmetic_operators.clj @@ -138,22 +138,28 @@ ;; 16.19. Round +(defn round-op [operand precision] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (round-op (core/-attach-cache operand context) + (core/-attach-cache precision context))) + (-eval [_ context resource scope] + (p/round (core/-eval operand context resource scope) + (core/-eval precision context resource scope))) + (-form [_] + (->> (some-> (core/-form precision) list) + (cons (core/-form operand)) + (cons 'round))))) + (defmethod core/compile* :elm.compiler.type/round [context {:keys [operand precision]}] (let [operand (core/compile* context operand) precision (some->> precision (core/compile* context))] (if (and (core/static? operand) (core/static? precision)) (p/round operand precision) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (p/round (core/-eval operand context resource scope) - (core/-eval precision context resource scope))) - (-form [_] - (->> (some-> (core/-form precision) list) - (cons (core/-form operand)) - (cons 'round))))))) + (round-op operand precision)))) ;; 16.20. Subtract diff --git a/modules/cql/src/blaze/elm/compiler/clinical_operators.clj b/modules/cql/src/blaze/elm/compiler/clinical_operators.clj index 49f5f031a..88509667f 100644 --- a/modules/cql/src/blaze/elm/compiler/clinical_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/clinical_operators.clj @@ -22,6 +22,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ context resource scope] (p/duration-between (core/-eval birth-date context resource scope) diff --git a/modules/cql/src/blaze/elm/compiler/comparison_operators.clj b/modules/cql/src/blaze/elm/compiler/comparison_operators.clj index 2afa94c57..5a9e376fa 100644 --- a/modules/cql/src/blaze/elm/compiler/comparison_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/comparison_operators.clj @@ -13,6 +13,11 @@ (defbinop equal [operand-1 operand-2] (p/equal operand-1 operand-2)) +(comment + (macroexpand-1 + '(defbinop equal [operand-1 operand-2] + (p/equal operand-1 operand-2)))) + ;; 12.2. Equivalent (defbinop equivalent [operand-1 operand-2] diff --git a/modules/cql/src/blaze/elm/compiler/conditional_operators.clj b/modules/cql/src/blaze/elm/compiler/conditional_operators.clj index 7b2ccc72b..6efbdb99f 100644 --- a/modules/cql/src/blaze/elm/compiler/conditional_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/conditional_operators.clj @@ -9,6 +9,51 @@ ;; 15.1. Case +(defn- attach-cache [items context] + (map + (fn [[when then]] + [(core/-attach-cache when context) + (core/-attach-cache then context)]) + items)) + +(defn- comparand-case-op [comparand items else] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (comparand-case-op + (core/-attach-cache comparand context) + (attach-cache items context) + (core/-attach-cache else context))) + (-eval [_ context resource scope] + (let [comparand (core/-eval comparand context resource scope)] + (loop [[[when then] & next-items] items] + (if (p/equal comparand (core/-eval when context resource scope)) + (core/-eval then context resource scope) + (if (empty? next-items) + (core/-eval else context resource scope) + (recur next-items)))))) + (-form [_] + `(~'case ~(core/-form comparand) ~@(map core/-form (flatten items)) ~(core/-form else))))) + +(defn- multi-conditional-case-op [items else] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (multi-conditional-case-op + (attach-cache items context) + (core/-attach-cache else context))) + (-eval [_ context resource scope] + (loop [[[when then] & next-items] items] + (if (core/-eval when context resource scope) + (core/-eval then context resource scope) + (if (empty? next-items) + (core/-eval else context resource scope) + (recur next-items))))) + (-form [_] + `(~'case ~@(map core/-form (flatten items)) ~(core/-form else))))) + (defmethod core/compile* :elm.compiler.type/case [context {:keys [comparand else] items :caseItem}] (let [comparand (some->> comparand (core/compile* context)) @@ -20,42 +65,28 @@ items) else (core/compile* context else)] (if comparand - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (let [comparand (core/-eval comparand context resource scope)] - (loop [[[when then] & next-items] items] - (if (p/equal comparand (core/-eval when context resource scope)) - (core/-eval then context resource scope) - (if (empty? next-items) - (core/-eval else context resource scope) - (recur next-items)))))) - (-form [_] - `(~'case ~(core/-form comparand) ~@(map core/-form (flatten items)) ~else))) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (loop [[[when then] & next-items] items] - (if (core/-eval when context resource scope) - (core/-eval then context resource scope) - (if (empty? next-items) - (core/-eval else context resource scope) - (recur next-items))))) - (-form [_] - `(~'case ~@(map core/-form (flatten items)) ~else)))))) + (comparand-case-op comparand items else) + (multi-conditional-case-op items else)))) ;; 15.2. If -(defrecord IfExpression [condition then else] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (if (core/-eval condition context resource scope) - (core/-eval then context resource scope) - (core/-eval else context resource scope)))) +(defn- if-op [condition then else] + (reify + core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (if-op + (core/-attach-cache condition context) + (core/-attach-cache then context) + (core/-attach-cache else context))) + (-eval [_ context resource scope] + (if (core/-eval condition context resource scope) + (core/-eval then context resource scope) + (core/-eval else context resource scope))) + (-form [_] + (list 'if (core/-form condition) (core/-form then) + (core/-form else))))) (defmethod core/compile* :elm.compiler.type/if @@ -64,5 +95,5 @@ (cond (true? condition) (core/compile* context then) (or (false? condition) (nil? condition)) (core/compile* context else) - :else (->IfExpression condition (core/compile* context then) - (core/compile* context else))))) + :else (if-op condition (core/compile* context then) + (core/compile* context else))))) diff --git a/modules/cql/src/blaze/elm/compiler/core.clj b/modules/cql/src/blaze/elm/compiler/core.clj index 813200722..0d2b54219 100644 --- a/modules/cql/src/blaze/elm/compiler/core.clj +++ b/modules/cql/src/blaze/elm/compiler/core.clj @@ -5,6 +5,7 @@ [clojure.string :as str] [cuerdas.core :as c-str]) (:import + [clojure.lang PersistentVector] [java.time.temporal ChronoUnit])) @@ -13,6 +14,7 @@ (defprotocol Expression (-static [expression]) + (-attach-cache [expression context]) (-eval [expression context resource scope] "Evaluates `expression` on `resource` using `context` and optional `scope` for scoped expressions inside queries.") @@ -23,10 +25,16 @@ (satisfies? Expression x)) +(defn static? [x] + (-static x)) + + (extend-protocol Expression nil (-static [_] true) + (-attach-cache [expr _] + expr) (-eval [expr _ _ _] expr) (-form [_] @@ -35,14 +43,22 @@ Object (-static [_] true) + (-attach-cache [expr _] + expr) (-eval [expr _ _ _] expr) (-form [expr] - expr)) - + expr) -(defn static? [x] - (-static x)) + PersistentVector + (-static [_] + true) + (-attach-cache [expr _] + expr) + (-eval [expr _ _ _] + expr) + (-form [expr] + (mapv -form expr))) (defmulti compile* diff --git a/modules/cql/src/blaze/elm/compiler/date_time_operators.clj b/modules/cql/src/blaze/elm/compiler/date_time_operators.clj index be0c08f03..b81677d9f 100644 --- a/modules/cql/src/blaze/elm/compiler/date_time_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/date_time_operators.clj @@ -29,6 +29,61 @@ ;; 18.6. Date +(defn date-op [year month day] + (reify + system/SystemType + (-type [_] :system/date) + core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (date-op + (core/-attach-cache year context) + (core/-attach-cache month context) + (core/-attach-cache day context))) + (-eval [_ context resource scope] + (when-let [year (core/-eval year context resource scope)] + (if-let [month (core/-eval month context resource scope)] + (if-let [day (core/-eval day context resource scope)] + (system/date year month day) + (system/date year month)) + (system/date year)))) + (-form [_] + (list 'date (core/-form year) (core/-form month) (core/-form day))))) + +(defn year-month-op [year month] + (reify + system/SystemType + (-type [_] :system/date) + core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (year-month-op + (core/-attach-cache year context) + (core/-attach-cache month context))) + (-eval [_ context resource scope] + (when-let [year (core/-eval year context resource scope)] + (if-let [month (core/-eval month context resource scope)] + (system/date year month) + (system/date year)))) + (-form [_] + (list 'date (core/-form year) (core/-form month))))) + +(defn year-op [year] + (reify + system/SystemType + (-type [_] :system/date) + core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (year-op (core/-attach-cache year context))) + (-eval [_ context resource scope] + (some-> (core/-eval year context resource scope) system/date)) + (-form [_] + (list 'date (core/-form year))))) + (defmethod core/compile* :elm.compiler.type/date [context {:keys [year month day]}] (let [year (some->> year (core/compile* context)) @@ -39,55 +94,19 @@ (system/date year month day) (some? day) - (reify - system/SystemType - (-type [_] :system/date) - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [year (core/-eval year context resource scope)] - (if-let [month (core/-eval month context resource scope)] - (if-let [day (core/-eval day context resource scope)] - (system/date year month day) - (system/date year month)) - (system/date year)))) - (-form [_] - (list 'date (core/-form year) (core/-form month) (core/-form day)))) + (date-op year month day) (and (int? month) (int? year)) (system/date year month) (some? month) - (reify - system/SystemType - (-type [_] :system/date) - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [year (core/-eval year context resource scope)] - (if-let [month (core/-eval month context resource scope)] - (system/date year month) - (system/date year)))) - (-form [_] - (list 'date (core/-form year) (core/-form month)))) + (year-month-op year month) (int? year) (system/date year) :else - (when year - (reify - system/SystemType - (-type [_] :system/date) - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (some-> (core/-eval year context resource scope) system/date)) - (-form [_] - (list 'date (core/-form year)))))))) + (some-> year year-op)))) ;; 18.7. DateFrom diff --git a/modules/cql/src/blaze/elm/compiler/expr_cache.clj b/modules/cql/src/blaze/elm/compiler/expr_cache.clj new file mode 100644 index 000000000..5e75cb34f --- /dev/null +++ b/modules/cql/src/blaze/elm/compiler/expr_cache.clj @@ -0,0 +1,61 @@ +(ns blaze.elm.compiler.expr-cache + (:require + [blaze.elm.compiler.external-data] + [prometheus.alpha :as prom :refer [defhistogram]]) + (:import + [blaze.elm.compiler.external_data Resource] + [com.google.common.hash BloomFilter Funnel Funnels] + [java.nio.charset StandardCharsets])) + + +(set! *warn-on-reflection* true) + + +(defhistogram bloom-filter-bytes + "Bloom filter sizes in bytes." + {:namespace "blaze" + :subsystem "cql_expr_cache"} + (take 12 (iterate #(* 4 %) 1))) + + +(defprotocol Cache + (-might-contain [_ resource]) + (-mem-size [_])) + + +(defn cache? [x] + (satisfies? Cache x)) + + +(defn might-contain? [cache resource] + (-might-contain cache resource)) + + +(defn mem-size + "Returns the size of the cache in memory in bytes." + [cache] + (-mem-size cache)) + + +(def ^:private ^Funnel id-funnel + (Funnels/stringFunnel StandardCharsets/ISO_8859_1)) + + +(defn- calc-mem-size [n p] + (long (/ (* (- n) (Math/log p)) (* (Math/log 2) (Math/log 2)) 8))) + + +(defn cache [t resource-ids] + (let [n (count resource-ids) + p (double 0.01) + filter (BloomFilter/create id-funnel n p) + mem-size (calc-mem-size n p)] + (prom/observe! bloom-filter-bytes mem-size) + (run! #(.put filter %) resource-ids) + (reify + Cache + (-might-contain [_ resource] + (or (< t (.lastChangeT ^Resource resource)) + (.mightContain filter (.id ^Resource resource)))) + (-mem-size [_] + mem-size)))) diff --git a/modules/cql/src/blaze/elm/compiler/external_data.clj b/modules/cql/src/blaze/elm/compiler/external_data.clj index 2daf1fd0f..fbf85252f 100644 --- a/modules/cql/src/blaze/elm/compiler/external_data.clj +++ b/modules/cql/src/blaze/elm/compiler/external_data.clj @@ -23,9 +23,18 @@ (set! *warn-on-reflection* true) +(definterface Resource + (^String id []) + (^long lastChangeT [])) + + ;; A resource that is a wrapper of a resource-handle that will lazily pull the ;; resource content if some property other than :id is accessed. -(deftype Resource [db handle content] +(deftype ResourceImpl [db ^String id handle ^long last-change-t content] + Resource + (id [_] id) + (lastChangeT [_] last-change-t) + p/FhirType (-type [_] (p/-type handle)) @@ -35,7 +44,7 @@ (.valAt r key nil)) (valAt [_ key not-found] (case key - :id (rh/id handle) + :id id (-> (or @content (vreset! content @(d/pull-content db handle))) (get key not-found)))) @@ -48,8 +57,21 @@ (instance? Resource x)) +(defn- patient-last-change-t [db handle] + (max (or (d/patient-compartment-last-change-t db (rh/id handle)) (d/t db)) + (rh/t handle))) + + +(defn- last-change-t [db handle] + (case (p/-type handle) + :fhir/Patient + (patient-last-change-t db handle) + (d/t db))) + + (defn mk-resource [db handle] - (Resource. db handle (volatile! nil))) + (ResourceImpl. db (rh/id handle) handle (last-change-t db handle) + (volatile! nil))) (defn resource-mapper [db] @@ -91,6 +113,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ {:keys [db]} {:keys [id]} _] (coll/eduction (resource-mapper db) (d/execute-query db query id))) (-form [_] @@ -107,6 +131,8 @@ core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ {:keys [db]} resource _] (let [{{:keys [reference]} :subject} resource] (when reference @@ -135,6 +161,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ {:keys [db]} {:keys [id]} _] (coll/eduction (resource-mapper db) @@ -147,6 +175,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ _ resource _] [resource]) (-form [_] @@ -165,6 +195,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ context resource scope] (when-let [context-resource (core/-eval related-context-expr context resource scope)] (core/-eval @@ -187,6 +219,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ {:keys [db] :as context} resource scope] (when-let [{:keys [id]} (core/-eval context-expr context resource scope)] (when (string? id) @@ -206,6 +240,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ {:keys [db]} _ _] (coll/eduction (resource-mapper db) (d/type-list db data-type))) (-form [_] @@ -215,6 +251,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ {:keys [db]} _ _] (coll/eduction (resource-mapper db) (d/execute-query db query))) (-form [_] diff --git a/modules/cql/src/blaze/elm/compiler/function.clj b/modules/cql/src/blaze/elm/compiler/function.clj index e9cefd97c..c8996f1cc 100644 --- a/modules/cql/src/blaze/elm/compiler/function.clj +++ b/modules/cql/src/blaze/elm/compiler/function.clj @@ -7,6 +7,9 @@ (reify core/Expression (-static [_] false) + (-attach-cache [_ context] + (arity-n name (core/-attach-cache fn-expr context) operand-names + (map #(core/-attach-cache % context) operands))) (-eval [_ context resource scope] (let [values (map #(core/-eval % context resource scope) operands)] (core/-eval fn-expr context resource (merge scope (zipmap operand-names values))))) diff --git a/modules/cql/src/blaze/elm/compiler/interval_operators.clj b/modules/cql/src/blaze/elm/compiler/interval_operators.clj index 2aa313a7f..cf7e046a5 100644 --- a/modules/cql/src/blaze/elm/compiler/interval_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/interval_operators.clj @@ -24,6 +24,15 @@ core/Expression (-static [_] false) + (-attach-cache [_ context] + (->IntervalExpression + type + (core/-attach-cache low context) + (core/-attach-cache high context) + (core/-attach-cache low-closed-expression context) + (core/-attach-cache high-closed-expression context) + low-closed + high-closed)) (-eval [_ context resource scope] (let [low (core/-eval low context resource scope) high (core/-eval high context resource scope) @@ -87,6 +96,11 @@ (p/after operand-1 operand-2 precision)) +(comment + (macroexpand-1 + '(defbinopp after [operand-1 operand-2 precision] + (p/after operand-1 operand-2 precision)))) + ;; 19.3. Before (defbinopp before [operand-1 operand-2 precision] (p/before operand-1 operand-2 precision)) @@ -197,6 +211,15 @@ (throw (ex-info (core/append-locator "Invalid non-unit interval in `PointFrom` expression at" locator) {:expression expression}))))) +(comment + (macroexpand-1 + '(defunop point-from [interval {{:keys [locator]} :operand :as expression}] + (when interval + (if (p/equal (:start interval) (:end interval)) + (:start interval) + (throw (ex-info (core/append-locator "Invalid non-unit interval in `PointFrom` expression at" locator) + {:expression expression}))))))) + ;; 19.24. ProperContains (defbinopp proper-contains [list-or-interval x precision] @@ -225,6 +248,12 @@ start) +(comment + (macroexpand-1 + '(defunop start [{:keys [start]}] + start)) + ) + ;; 19.30. Starts (defbinopp starts [x y _] (and (p/equal (:start x) (:start y)) diff --git a/modules/cql/src/blaze/elm/compiler/library.clj b/modules/cql/src/blaze/elm/compiler/library.clj index 966e98024..c1aa9dec1 100644 --- a/modules/cql/src/blaze/elm/compiler/library.clj +++ b/modules/cql/src/blaze/elm/compiler/library.clj @@ -60,11 +60,12 @@ itself is returned. Returns an anomaly on errors." - {:arglists '([context expression-def])} + {:arglists '([context parameter-def])} [context {:keys [default] :as parameter-def}] (if (some? default) (-> (ba/try-anomaly - (assoc parameter-def :default (c/compile context default))) + (let [context (assoc context :eval-context "Unfiltered")] + (assoc parameter-def :default (c/compile context default)))) (ba/exceptionally #(assoc % :context context :elm/expression default))) parameter-def)) diff --git a/modules/cql/src/blaze/elm/compiler/library_spec.clj b/modules/cql/src/blaze/elm/compiler/library_spec.clj index 5ad47d211..1508e9aa1 100644 --- a/modules/cql/src/blaze/elm/compiler/library_spec.clj +++ b/modules/cql/src/blaze/elm/compiler/library_spec.clj @@ -36,6 +36,10 @@ (s/keys :req-un [::c/expression-defs ::c/parameter-default-values])) +(s/def ::c/options + map?) + + (s/fdef library/compile-library - :args (s/cat :node :blaze.db/node :library :elm/library :opts map?) + :args (s/cat :node :blaze.db/node :library :elm/library :opts ::c/options) :ret (s/or :library ::c/library :anomaly ::anom/anomaly)) diff --git a/modules/cql/src/blaze/elm/compiler/list_operators.clj b/modules/cql/src/blaze/elm/compiler/list_operators.clj index 2f0c00c74..a0476670c 100644 --- a/modules/cql/src/blaze/elm/compiler/list_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/list_operators.clj @@ -19,18 +19,21 @@ ;; 20.1. List +(defn list-op [elements] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (list-op (mapv #(core/-attach-cache % context) elements))) + (-eval [_ context resource scope] + (mapv #(core/-eval % context resource scope) elements)) + (-form [_] + `(~'list ~@(map core/-form elements))))) + (defmethod core/compile* :elm.compiler.type/list [context {elements :element}] (let [elements (mapv #(core/compile* context %) elements)] - (if (every? core/static? elements) - elements - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (mapv #(core/-eval % context resource scope) elements)) - (-form [_] - `(~'list ~@(map core/-form elements))))))) + (cond-> elements (some (comp not core/static?) elements) list-op))) ;; 20.3. Current @@ -40,6 +43,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ _ _ scopes] (get scopes scope)) (-form [_] @@ -47,6 +52,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ _ _ scope] scope) (-form [_] @@ -66,14 +73,36 @@ [] list))) +(comment + (macroexpand-1 + '(defunop distinct [list] + (when list + (reduce + (fn [result x] + (if (p/contains result x nil) + result + (conj result x))) + [] + list))))) + ;; 20.8. Exists (defunop exists - {:optimizations #{:first :non-distinct}} + {:optimizations #{:first :non-distinct} + :cache true} [list] (not (coll/empty? list))) +(comment + (macroexpand-1 + '(defunop exists + {:optimizations #{:first :non-distinct} + :cache true} + [list] + (not (coll/empty? list))))) + + ;; 20.9. Filter (defmethod core/compile* :elm.compiler.type/filter [context {:keys [source condition scope]}] @@ -104,18 +133,23 @@ ;; 20.10. First ;; ;; TODO: orderBy +(defn first-op [source] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (first-op (core/-attach-cache source context))) + (-eval [_ context resource scopes] + (coll/first (core/-eval source context resource scopes))) + (-form [_] + (list 'first (core/-form source))))) + (defmethod core/compile* :elm.compiler.type/first [context {:keys [source]}] (let [source (core/compile* (assoc context :optimizations #{:first :non-distinct}) source)] (if (core/static? source) (first source) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scopes] - (coll/first (core/-eval source context resource scopes))) - (-form [_] - (list 'first (core/-form source))))))) + (first-op source)))) ;; 20.11. Flatten diff --git a/modules/cql/src/blaze/elm/compiler/logical_operators.clj b/modules/cql/src/blaze/elm/compiler/logical_operators.clj index 1d0829ba4..39afc0a62 100644 --- a/modules/cql/src/blaze/elm/compiler/logical_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/logical_operators.clj @@ -9,10 +9,12 @@ ;; 13.1. And -(defn- nil-and-expr [x] +(defn- and-nil-op [x] (reify core/Expression (-static [_] false) + (-attach-cache [_ context] + (and-nil-op (core/-attach-cache x context))) (-eval [_ context resource scope] (when (false? (core/-eval x context resource scope)) false)) @@ -27,7 +29,25 @@ true nil false false nil nil - (nil-and-expr x))) + (and-nil-op x))) + + +(defn and-op [a b] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (and-op (core/-attach-cache a context) (core/-attach-cache b context))) + (-eval [_ context resource scope] + (let [a (core/-eval a context resource scope)] + (if (false? a) + false + (let [b (core/-eval b context resource scope)] + (cond + (false? b) false + (and (true? a) (true? b)) true))))) + (-form [_] + (list 'and (core/-form a) (core/-form b))))) (defn- dynamic-and @@ -37,20 +57,8 @@ (condp identical? b true a false false - nil (nil-and-expr a) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (let [a (core/-eval a context resource scope)] - (if (false? a) - false - (let [b (core/-eval b context resource scope)] - (cond - (false? b) false - (and (true? a) (true? b)) true))))) - (-form [_] - (list 'and (core/-form a) (core/-form b)))))) + nil (and-nil-op a) + (and-op a b))) (defmethod core/compile* :elm.compiler.type/and @@ -70,16 +78,20 @@ ;; 13.3 Not +(declare not-op) + (defunop not [operand] (when (some? operand) (not operand))) ;; 13.4. Or -(defn- nil-or-expr [x] +(defn- or-nil-op [x] (reify core/Expression (-static [_] false) + (-attach-cache [_ context] + (or-nil-op (core/-attach-cache x context))) (-eval [_ context resource scope] (when (true? (core/-eval x context resource scope)) true)) @@ -94,7 +106,25 @@ true true false nil nil nil - (nil-or-expr x))) + (or-nil-op x))) + + +(defn or-op [a b] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (or-op (core/-attach-cache a context) (core/-attach-cache b context))) + (-eval [_ context resource scope] + (let [a (core/-eval a context resource scope)] + (if (true? a) + true + (let [b (core/-eval b context resource scope)] + (cond + (true? b) true + (and (false? a) (false? b)) false))))) + (-form [_] + (list 'or (core/-form a) (core/-form b))))) (defn- dynamic-or @@ -104,20 +134,8 @@ (condp identical? b true true false a - nil (nil-or-expr a) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (let [a (core/-eval a context resource scope)] - (if (true? a) - true - (let [b (core/-eval b context resource scope)] - (cond - (true? b) true - (and (false? a) (false? b)) false))))) - (-form [_] - (list 'or (core/-form a) (core/-form b)))))) + nil (or-nil-op a) + (or-op a b))) (defmethod core/compile* :elm.compiler.type/or @@ -137,15 +155,7 @@ [a b] (condp identical? b true - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (let [a (core/-eval a context resource scope)] - (when (some? a) - (not a)))) - (-form [_] - (list 'not (core/-form a)))) + (not-op a) false a nil nil (reify core/Expression diff --git a/modules/cql/src/blaze/elm/compiler/macros.clj b/modules/cql/src/blaze/elm/compiler/macros.clj index 617bccd24..a040bdd99 100644 --- a/modules/cql/src/blaze/elm/compiler/macros.clj +++ b/modules/cql/src/blaze/elm/compiler/macros.clj @@ -1,38 +1,82 @@ (ns blaze.elm.compiler.macros (:require + [blaze.async.comp :as ac] + [blaze.db.api :as d] [blaze.elm.compiler.core :as core] - [blaze.elm.expression.cache :as-alias expr-cache])) + [blaze.elm.compiler.expr-cache :as ec] + [blaze.elm.compiler.external-data :as ed] + [blaze.elm.expression :as expr] + [clojure.spec.alpha :as s] + [prometheus.alpha :as prom :refer [defcounter defhistogram]] + [taoensso.timbre :as log]) + (:import + [com.github.benmanes.caffeine.cache AsyncCache] + [java.util.function BiFunction])) (set! *warn-on-reflection* true) -(defn- compile-kw [name] - (keyword "elm.compiler.type" (clojure.core/name name))) +(defhistogram bloom-filter-creation-duration-seconds + "Durations in Cassandra resource store." + {:namespace "blaze" + :subsystem "cql_expr_cache"} + (take 14 (iterate #(* 2 %) 0.1))) + + +(defcounter bloom-filter-useful-total + "Number of times bloom filter has avoided expression evaluation." + {:namespace "blaze" + :subsystem "cql_expr_cache"}) + + +(defcounter bloom-filter-not-useful-total + "Number of times bloom filter has not avoided expression evaluation." + {:namespace "blaze" + :subsystem "cql_expr_cache"}) -(defn generate-binding-vector +(defcounter bloom-filter-false-positive-total + "Number of false positives reported by Bloom all filters." + {:namespace "blaze" + :subsystem "cql_expr_cache"}) + + +(defn- generate-binding-vector "Creates a binding vector of at least `[operand-binding operand]` and - optionally `[operand-binding operand expr-binding expr-sym]` if `expr-binding` - is given." - [operand-binding operand expr-binding expr-sym] - (cond-> [operand-binding operand] expr-binding (conj expr-binding expr-sym))) - - -(defn generate-unop [name operand-sym operand-binding expr-binding expr-sym body] - (let [context-sym (gensym "context") - resource-sym (gensym "resource") - scope-sym (gensym "scope")] - `(reify core/Expression - (~'-static [~'_] - false) - (~'-eval [~'_ ~context-sym ~resource-sym ~scope-sym] - (let ~(generate-binding-vector - operand-binding `(core/-eval ~operand-sym ~context-sym ~resource-sym ~scope-sym) - expr-binding expr-sym) - ~@body)) - (~'-form [~'_] - (list (quote ~name) (core/-form ~operand-sym)))))) + optionally `[operand-binding operand elm-elm-expr-binding elm-expr]` if + `elm-elm-expr-binding` is given." + [operand-binding operand elm-expr-binding elm-expr] + (cond-> [operand-binding operand] elm-expr-binding (conj elm-expr-binding elm-expr))) + + +(defn get-cache [{::expr/keys [cache] :as context} eval-context expr] + (let [cache (.get ^AsyncCache cache (expr/hash expr) + (reify BiFunction + (apply [_ _ executor] + (ac/supply-async + #(let [db (:db context)] + (log/debug "create Bloom filter for expression" + (core/-form expr) + "evaluating it for all" + eval-context + "resources") + (with-open [_ (prom/timer bloom-filter-creation-duration-seconds)] + (ec/cache + (d/t db) + (into + [] + (comp (map (partial ed/mk-resource db)) + (filter (partial expr/eval context expr)) + (map :id)) + (d/type-list db eval-context))))) + executor))))] + (when (.isDone cache) + (.get cache)))) + + +(defn- compile-kw [name] + (keyword "elm.compiler.type" (clojure.core/name name))) (defmacro defunop @@ -40,18 +84,142 @@ [name & more] (let [attr-map (when (map? (first more)) (first more)) more (if (map? (first more)) (next more) more) - [[operand-binding expr-binding] & body] more - context-sym (gensym "context") - operand-sym (gensym "operand") - expr-sym (gensym "expr")] - `(defmethod core/compile* ~(compile-kw name) - [~context-sym ~expr-sym] - (let [~operand-sym (core/compile* (merge ~context-sym ~(dissoc attr-map :cache)) (:operand ~expr-sym))] - (if (core/static? ~operand-sym) - (let [~operand-binding ~operand-sym - ~(or expr-binding '_) ~expr-sym] - ~@body) - ~(generate-unop name operand-sym operand-binding expr-binding expr-sym body)))))) + [[operand-binding elm-expr-binding] & body] more + elm-expr (gensym "elm-expr") + operand (gensym "operand") + context (gensym "context") + eval-context (gensym "eval-context") + resource (gensym "resource") + scope (gensym "scope") + cache (gensym "cache") + expr (gensym "expr")] + `(do + ~(if (:cache attr-map) + `(do + (declare ~(symbol (str name "-caching-op"))) + + (s/fdef ~(symbol (str name "-cache-op")) + :args ~(if elm-expr-binding + `(s/cat :operand core/expr? + :eval-context string? + :cache ec/cache? + :expr :elm/expression) + `(s/cat :operand core/expr? + :eval-context string? + :cache ec/cache?)) + :ret core/expr?) + + (defn ~(symbol (str name "-cache-op")) + ~(cond-> [operand eval-context cache] elm-expr-binding (conj elm-expr)) + (reify core/Expression + (~'-static [~'_] + false) + (~'-attach-cache [~expr ~context] + (if-let [~cache (get-cache ~context ~eval-context ~expr)] + ~(if elm-expr-binding + `(~(symbol (str name "-cache-op")) + (core/-attach-cache ~operand ~context) + ~eval-context ~cache ~elm-expr) + `(~(symbol (str name "-cache-op")) + (core/-attach-cache ~operand ~context) + ~eval-context ~cache)) + ~(if elm-expr-binding + `(~(symbol (str name "-caching-op")) + (core/-attach-cache ~operand ~context) + ~eval-context ~elm-expr) + `(~(symbol (str name "-caching-op")) + (core/-attach-cache ~operand ~context) + ~eval-context)))) + (~'-eval [~'_ ~context ~resource ~scope] + (if (ec/might-contain? ~cache ~resource) + (let [res# (let ~(generate-binding-vector + operand-binding `(core/-eval ~operand + ~context + ~resource + ~scope) + elm-expr-binding elm-expr) + ~@body)] + (prom/inc! bloom-filter-not-useful-total) + (when-not res# + (prom/inc! bloom-filter-false-positive-total)) + res#) + (do (prom/inc! bloom-filter-useful-total) + false))) + (~'-form [~'_] + (list (quote ~name) (core/-form ~operand))))) + + (s/fdef ~(symbol (str name "-caching-op")) + :args ~(if elm-expr-binding + `(s/cat :operand core/expr? + :eval-context string? + :expr :elm/expression) + `(s/cat :operand core/expr? :eval-context string?)) + :ret core/expr?) + + (defn ~(symbol (str name "-caching-op")) + ~(cond-> [operand eval-context] elm-expr-binding (conj elm-expr)) + (reify core/Expression + (~'-static [~'_] + false) + (~'-attach-cache [~expr ~context] + (if-let [~cache (get-cache ~context ~eval-context ~expr)] + ~(if elm-expr-binding + `(~(symbol (str name "-cache-op")) + (core/-attach-cache ~operand ~context) + ~eval-context ~cache ~elm-expr) + `(~(symbol (str name "-cache-op")) + (core/-attach-cache ~operand ~context) + ~eval-context ~cache)) + ~(if elm-expr-binding + `(~(symbol (str name "-caching-op")) + (core/-attach-cache ~operand ~context) + ~eval-context ~elm-expr) + `(~(symbol (str name "-caching-op")) + (core/-attach-cache ~operand ~context) + ~eval-context)))) + (~'-eval [~'_ ~context ~resource ~scope] + (let ~(generate-binding-vector + operand-binding `(core/-eval ~operand ~context ~resource ~scope) + elm-expr-binding elm-expr) + ~@body)) + (~'-form [~'_] + (list (quote ~name) (core/-form ~operand)))))) + + `(defn ~(symbol (str name "-op")) + ~(cond-> [operand] elm-expr-binding (conj elm-expr)) + (reify core/Expression + (~'-static [~'_] + false) + (~'-attach-cache [~'_ ~context] + ~(if elm-expr-binding + `(~(symbol (str name "-op")) + (core/-attach-cache ~operand ~context) ~elm-expr) + `(~(symbol (str name "-op")) + (core/-attach-cache ~operand ~context)))) + (~'-eval [~'_ ~context ~resource ~scope] + (let ~(generate-binding-vector + operand-binding `(core/-eval ~operand ~context ~resource ~scope) + elm-expr-binding elm-expr) + ~@body)) + (~'-form [~'_] + (list (quote ~name) (core/-form ~operand)))))) + + (defmethod core/compile* ~(compile-kw name) + [{~eval-context :eval-context :as ~context} + {~operand :operand :as ~elm-expr}] + (let [~operand (core/compile* (merge ~context ~(dissoc attr-map :cache)) ~operand)] + (if (core/static? ~operand) + (let ~(generate-binding-vector + operand-binding operand + elm-expr-binding elm-expr) + ~@body) + ~(if (:cache attr-map) + (if elm-expr-binding + `(~(symbol (str name "-caching-op")) ~operand ~eval-context ~elm-expr) + `(~(symbol (str name "-caching-op")) ~operand ~eval-context)) + (if elm-expr-binding + `(~(symbol (str name "-op")) ~operand ~elm-expr) + `(~(symbol (str name "-op")) ~operand))))))))) (defmacro defbinop @@ -59,25 +227,35 @@ [name & more] (let [attr-map (when (map? (first more)) (first more)) more (if (map? (first more)) (next more) more) - [[op-1-binding op-2-binding] & body] more] - `(defmethod core/compile* ~(compile-kw name) - [context# {[operand-1# operand-2#] :operand}] - (let [context# (merge context# ~attr-map) - operand-1# (core/compile* context# operand-1#) - operand-2# (core/compile* context# operand-2#)] - (if (and (core/static? operand-1#) (core/static? operand-2#)) - (let [~op-1-binding operand-1# - ~op-2-binding operand-2#] - ~@body) - (reify core/Expression - (~'-static [~'_] - false) - (~'-eval [~'_ context# resource# scope#] - (let [~op-1-binding (core/-eval operand-1# context# resource# scope#) - ~op-2-binding (core/-eval operand-2# context# resource# scope#)] - ~@body)) - (~'-form [~'_] - (list (quote ~name) (core/-form operand-1#) (core/-form operand-2#))))))))) + [[op-1-binding op-2-binding] & body] more + context (gensym "context") + op-1 (gensym "op-1") + op-2 (gensym "op-2")] + `(do + (defn ~(symbol (str name "-op")) [~op-1 ~op-2] + (reify core/Expression + (~'-static [~'_] + false) + (~'-attach-cache [~'_ ~'cache] + (~(symbol (str name "-op")) + (core/-attach-cache ~op-1 ~'cache) + (core/-attach-cache ~op-2 ~'cache))) + (~'-eval [~'_ context# resource# scope#] + (let [~op-1-binding (core/-eval ~op-1 context# resource# scope#) + ~op-2-binding (core/-eval ~op-2 context# resource# scope#)] + ~@body)) + (~'-form [~'_] + (list (quote ~name) (core/-form ~op-1) (core/-form ~op-2))))) + (defmethod core/compile* ~(compile-kw name) + [~context {[~op-1 ~op-2] :operand}] + (let [context# ~(if attr-map `(merge ~context ~attr-map) context) + ~op-1 (core/compile* context# ~op-1) + ~op-2 (core/compile* context# ~op-2)] + (if (and (core/static? ~op-1) (core/static? ~op-2)) + (let [~op-1-binding ~op-1 + ~op-2-binding ~op-2] + ~@body) + (~(symbol (str name "-op")) ~op-1 ~op-2))))))) (defmacro defternop @@ -104,33 +282,42 @@ (defmacro defnaryop {:arglists '([name bindings & body])} [name [operands-binding] & body] - `(defmethod core/compile* ~(compile-kw name) - [context# {operands# :operand}] - (let [operands# (mapv #(core/compile* context# %) operands#)] + `(do + (defn ~(symbol (str name "-op")) [~operands-binding] (reify core/Expression (~'-static [~'_] false) + (~'-attach-cache [~'_ ~'cache] + (~(symbol (str name "-op")) + (mapv #(core/-attach-cache % ~'cache) ~operands-binding))) (~'-eval [~'_ context# resource# scope#] - (let [~operands-binding (mapv #(core/-eval % context# resource# scope#) operands#)] + (let [~operands-binding (mapv #(core/-eval % context# resource# scope#) ~operands-binding)] ~@body)) (~'-form [~'_] - (cons (quote ~name) (map core/-form operands#))))))) + (cons (quote ~name) (map core/-form ~operands-binding))))) + (defmethod core/compile* ~(compile-kw name) + [context# {operands# :operand}] + (~(symbol (str name "-op")) (mapv #(core/compile* context# %) operands#))))) (defmacro defaggop {:arglists '([name bindings & body])} [name [source-binding] & body] - `(defmethod core/compile* ~(compile-kw name) - [context# {source# :source}] - (let [source# (core/compile* context# source#)] + `(do + (defn ~(symbol (str name "-op")) [~source-binding] (reify core/Expression (~'-static [~'_] false) + (~'-attach-cache [~'_ ~'cache] + (~(symbol (str name "-op")) (core/-attach-cache ~source-binding ~'cache))) (~'-eval [~'_ context# resource# scope#] - (let [~source-binding (core/-eval source# context# resource# scope#)] + (let [~source-binding (core/-eval ~source-binding context# resource# scope#)] ~@body)) (~'-form [~'_] - (list (quote ~name) (core/-form source#))))))) + (list (quote ~name) (core/-form ~source-binding))))) + (defmethod core/compile* ~(compile-kw name) + [context# {source# :source}] + (~(symbol (str name "-op")) (core/compile* context# source#))))) (defmacro defunopp @@ -156,26 +343,37 @@ [name & more] (let [attr-map (when (map? (first more)) (first more)) more (if (map? (first more)) (next more) more) - [[op-1-binding op-2-binding precision-binding] & body] more] - `(defmethod core/compile* ~(compile-kw name) - [context# {[operand-1# operand-2#] :operand precision# :precision}] - (let [context# (merge context# ~attr-map) - operand-1# (core/compile* context# operand-1#) - operand-2# (core/compile* context# operand-2#) - chrono-precision# (some-> precision# core/to-chrono-unit)] - (if (and (core/static? operand-1#) (core/static? operand-2#)) - (let [~op-1-binding operand-1# - ~op-2-binding operand-2# - ~precision-binding chrono-precision#] - ~@body) - (reify core/Expression - (~'-static [~'_] - false) - (~'-eval [~'_ context# resource# scope#] - (let [~op-1-binding (core/-eval operand-1# context# resource# scope#) - ~op-2-binding (core/-eval operand-2# context# resource# scope#) - ~precision-binding chrono-precision#] - ~@body)) - (~'-form [~'_] - (list (quote ~name) (core/-form operand-1#) (core/-form operand-2#) - precision#)))))))) + [[op-1-binding op-2-binding precision-binding] & body] more + context (gensym "context") + op-1 (gensym "op-1") + op-2 (gensym "op-2") + precision (gensym "precision")] + `(do + (defn ~(symbol (str name "-op")) [~op-1 ~op-2 ~precision-binding ~precision] + (reify core/Expression + (~'-static [~'_] + false) + (~'-attach-cache [~'_ ~'context] + (~(symbol (str name "-op")) + (core/-attach-cache ~op-1 ~'context) + (core/-attach-cache ~op-2 ~'context) + ~precision-binding ~precision)) + (~'-eval [~'_ context# resource# scope#] + (let [~op-1-binding (core/-eval ~op-1 context# resource# scope#) + ~op-2-binding (core/-eval ~op-2 context# resource# scope#)] + ~@body)) + (~'-form [~'_] + (list (quote ~name) (core/-form ~op-1) (core/-form ~op-2) + ~precision)))) + (defmethod core/compile* ~(compile-kw name) + [~context {[~op-1 ~op-2] :operand ~precision :precision}] + (let [context# ~(if attr-map `(merge ~context ~attr-map) context) + ~op-1 (core/compile* context# ~op-1) + ~op-2 (core/compile* context# ~op-2) + ~precision-binding (some-> ~precision core/to-chrono-unit)] + (if (and (core/static? ~op-1) (core/static? ~op-2)) + (let [~op-1-binding ~op-1 + ~op-2-binding ~op-2] + ~@body) + (~(symbol (str name "-op")) ~op-1 ~op-2 + ~precision-binding ~precision))))))) diff --git a/modules/cql/src/blaze/elm/compiler/queries.clj b/modules/cql/src/blaze/elm/compiler/queries.clj index b05f6a819..a28453524 100644 --- a/modules/cql/src/blaze/elm/compiler/queries.clj +++ b/modules/cql/src/blaze/elm/compiler/queries.clj @@ -135,37 +135,35 @@ return-xform-factory)))) -(defrecord EductionQueryExpression [xform-factory source] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (coll/eduction - (-create xform-factory context resource scope) - (core/-eval source context resource scope))) - (-form [_] - `(~'eduction-query ~(-form xform-factory) ~(core/-form source)))) - - (defn- eduction-expr [xform-factory source] - (->EductionQueryExpression xform-factory source)) - - -(defrecord IntoVectorQueryExpression [xform-factory source] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (into - [] - (-create xform-factory context resource scope) - (core/-eval source context resource scope))) - (-form [_] - `(~'vector-query ~(-form xform-factory) ~(core/-form source)))) + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + ;;TODO: attach cache to xform-factory + (eduction-expr xform-factory (core/-attach-cache source context))) + (-eval [_ context resource scope] + (coll/eduction + (-create xform-factory context resource scope) + (core/-eval source context resource scope))) + (-form [_] + `(~'eduction-query ~(-form xform-factory) ~(core/-form source))))) (defn- into-vector-expr [xform-factory source] - (->IntoVectorQueryExpression xform-factory source)) + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + ;;TODO: attach cache to xform-factory + (into-vector-expr xform-factory (core/-attach-cache source context))) + (-eval [_ context resource scope] + (into + [] + (-create xform-factory context resource scope) + (core/-eval source context resource scope))) + (-form [_] + `(~'vector-query ~(-form xform-factory) ~(core/-form source))))) (deftype AscComparator [] @@ -202,45 +200,56 @@ (if (#{"desc" "descending"} direction) desc-comparator asc-comparator)) -(defrecord SortQueryExpression [source sort-by-item] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - ;; TODO: build a comparator of all sort by items - (->> (vec (core/-eval source context resource scope)) - (sort-by - (if-let [expr (:expression sort-by-item)] - #(core/-eval expr context resource %) - identity) - (comparator (:direction sort-by-item))) - (vec)))) +(defn sort-by-item-form [sort-by-item] + (if-let [expr (:expression sort-by-item)] + [(symbol (:direction sort-by-item)) (core/-form expr)] + (symbol (:direction sort-by-item)))) (defn sort-expr [source sort-by-item] - (->SortQueryExpression source sort-by-item)) - - -(defrecord XformSortQueryExpression [xform-factory source sort-by-item] - core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - ;; TODO: build a comparator of all sort by items - (->> (into - [] - (-create xform-factory context resource scope) - (core/-eval source context resource scope)) - (sort-by - (if-let [expr (:expression sort-by-item)] - #(core/-eval expr context resource %) - identity) - (comparator (:direction sort-by-item))) - (vec)))) + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (sort-expr (core/-attach-cache source context) + (update sort-by-item :expression core/-attach-cache context))) + (-eval [_ context resource scope] + ;; TODO: build a comparator of all sort by items + (->> (vec (core/-eval source context resource scope)) + (sort-by + (if-let [expr (:expression sort-by-item)] + #(core/-eval expr context resource %) + identity) + (comparator (:direction sort-by-item))) + (vec))) + (-form [_] + `(~'sorted-vector-query ~(core/-form source) + ~(sort-by-item-form sort-by-item))))) (defn xform-sort-expr [xform-factory source sort-by-item] - (->XformSortQueryExpression xform-factory source sort-by-item)) + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + ;;TODO: attach cache to xform-factory + (xform-sort-expr xform-factory (core/-attach-cache source context) + (update sort-by-item :expression core/-attach-cache context))) + (-eval [_ context resource scope] + ;; TODO: build a comparator of all sort by items + (->> (into + [] + (-create xform-factory context resource scope) + (core/-eval source context resource scope)) + (sort-by + (if-let [expr (:expression sort-by-item)] + #(core/-eval expr context resource %) + identity) + (comparator (:direction sort-by-item))) + (vec))) + (-form [_] + `(~'sorted-vector-query ~(-form xform-factory) ~(core/-form source) + ~(sort-by-item-form sort-by-item))))) (declare compile-with-equiv-clause) @@ -312,6 +321,8 @@ core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ _ _ scopes] (get scopes key)) (-form [_] diff --git a/modules/cql/src/blaze/elm/compiler/reusing_logic.clj b/modules/cql/src/blaze/elm/compiler/reusing_logic.clj index 0534253d6..e55a9663c 100644 --- a/modules/cql/src/blaze/elm/compiler/reusing_logic.clj +++ b/modules/cql/src/blaze/elm/compiler/reusing_logic.clj @@ -33,6 +33,8 @@ core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ {:keys [expression-defs] :as context} resource _] (if-let [{:keys [expression]} (get expression-defs name)] (core/-eval expression context resource nil) @@ -74,6 +76,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ {:keys [db expression-defs] :as context} _ _] (if-some [{:keys [expression]} (get expression-defs name)] (mapv @@ -110,6 +114,8 @@ core/Expression (-static [_] false) + (-attach-cache [_ context] + (->ToQuantityFunctionExpression (core/-attach-cache operand context))) (-eval [_ context resource scope] (-to-quantity (core/-eval operand context resource scope))) (-form [_] @@ -120,6 +126,8 @@ core/Expression (-static [_] false) + (-attach-cache [_ context] + (->ToCodeFunctionExpression (core/-attach-cache operand context))) (-eval [_ context resource scope] (let [{:keys [system version code]} (core/-eval operand context resource scope)] (code/to-code (type/value system) (type/value version) (type/value code)))) @@ -131,6 +139,8 @@ core/Expression (-static [_] false) + (-attach-cache [_ context] + (->ToDateFunctionExpression (core/-attach-cache operand context))) (-eval [_ context resource scope] (type/value (core/-eval operand context resource scope))) (-form [_] @@ -141,6 +151,8 @@ core/Expression (-static [_] false) + (-attach-cache [_ context] + (->ToDateTimeFunctionExpression (core/-attach-cache operand context))) (-eval [_ {:keys [now] :as context} resource scope] (p/to-date-time (type/value (core/-eval operand context resource scope)) now)) (-form [_] @@ -151,6 +163,8 @@ core/Expression (-static [_] false) + (-attach-cache [_ context] + (->ToStringFunctionExpression (core/-attach-cache operand context))) (-eval [_ context resource scope] (some-> (type/value (core/-eval operand context resource scope)) str)) (-form [_] @@ -176,6 +190,8 @@ core/Expression (-static [_] false) + (-attach-cache [_ context] + (->ToIntervalFunctionExpression (core/-attach-cache operand context))) (-eval [_ context resource scope] (-to-interval (core/-eval operand context resource scope) context)) (-form [_] @@ -230,6 +246,8 @@ (reify core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ _ _ scope] (scope name)) (-form [_] diff --git a/modules/cql/src/blaze/elm/compiler/spec.clj b/modules/cql/src/blaze/elm/compiler/spec.clj index 896730d14..23915e68a 100644 --- a/modules/cql/src/blaze/elm/compiler/spec.clj +++ b/modules/cql/src/blaze/elm/compiler/spec.clj @@ -1,18 +1,24 @@ (ns blaze.elm.compiler.spec (:require [blaze.db.spec] + [blaze.elm.compiler :as-alias c] [blaze.elm.compiler.core :as core] [blaze.elm.spec] + [blaze.fhir.spec.spec] [clojure.spec.alpha :as s])) -(s/def :blaze.elm.compiler/expression +(s/def ::c/expression core/expr?) -(s/def :blaze.elm.compiler/function +(s/def ::c/function fn?) +(s/def ::c/eval-context + (s/or :unfiltered #{"Unfiltered"} :resource-type :fhir.resource/type)) + + (s/def :elm/compile-context - (s/keys :req-un [:elm/library :blaze.db/node])) + (s/keys :req-un [:elm/library ::c/eval-context :blaze.db/node])) diff --git a/modules/cql/src/blaze/elm/compiler/string_operators.clj b/modules/cql/src/blaze/elm/compiler/string_operators.clj index 7ee4c1d17..74b4e8646 100644 --- a/modules/cql/src/blaze/elm/compiler/string_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/string_operators.clj @@ -15,28 +15,40 @@ ;; 17.1. Combine +(defn combine-op + ([source] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (combine-op (core/-attach-cache source context))) + (-eval [_ context resource scope] + (when-let [source (core/-eval source context resource scope)] + (string/combine source))) + (-form [_] + (list 'combine (core/-form source))))) + ([source separator] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (combine-op (core/-attach-cache source context) + (core/-attach-cache separator context))) + (-eval [_ context resource scope] + (when-let [source (core/-eval source context resource scope)] + (string/combine (core/-eval separator context resource scope) + source))) + (-form [_] + (list 'combine (core/-form source) (core/-form separator)))))) + + (defmethod core/compile* :elm.compiler.type/combine [context {:keys [source separator]}] (let [source (core/compile* context source) separator (some->> separator (core/compile* context))] (if separator - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [source (core/-eval source context resource scope)] - (string/combine (core/-eval separator context resource scope) - source))) - (-form [_] - (list 'combine (core/-form source) (core/-form separator)))) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [source (core/-eval source context resource scope)] - (string/combine source))) - (-form [_] - (list 'combine (core/-form source))))))) + (combine-op source separator) + (combine-op source)))) ;; 17.2. Concatenate @@ -56,19 +68,22 @@ ;; 17.7. LastPositionOf +(defn last-position-of-op [pattern string] + (reify core/Expression + (-static [_] + false) + (-eval [_ context resource scope] + (when-let [^String pattern (core/-eval pattern context resource scope)] + (when-let [^String string (core/-eval string context resource scope)] + (.lastIndexOf string pattern)))) + (-form [_] + (list 'last-position-of (core/-form pattern) (core/-form string))))) + + (defmethod core/compile* :elm.compiler.type/last-position-of [context {:keys [pattern string]}] - (let [pattern (core/compile* context pattern) - string (core/compile* context string)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [^String pattern (core/-eval pattern context resource scope)] - (when-let [^String string (core/-eval string context resource scope)] - (.lastIndexOf string pattern)))) - (-form [_] - (list 'last-position-of (core/-form pattern) (core/-form string)))))) + (last-position-of-op (core/compile* context pattern) + (core/compile* context string))) ;; 17.8. Length @@ -88,19 +103,22 @@ ;; 17.12. PositionOf +(defn position-of-op [pattern string] + (reify core/Expression + (-static [_] + false) + (-eval [_ context resource scope] + (when-let [^String pattern (core/-eval pattern context resource scope)] + (when-let [^String string (core/-eval string context resource scope)] + (.indexOf string pattern)))) + (-form [_] + (list 'position-of (core/-form pattern) (core/-form string))))) + + (defmethod core/compile* :elm.compiler.type/position-of [context {:keys [pattern string]}] - (let [pattern (core/compile* context pattern) - string (core/compile* context string)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [^String pattern (core/-eval pattern context resource scope)] - (when-let [^String string (core/-eval string context resource scope)] - (.indexOf string pattern)))) - (-form [_] - (list 'position-of (core/-form pattern) (core/-form string)))))) + (position-of-op (core/compile* context pattern) + (core/compile* context string))) ;; 17.13. ReplaceMatches @@ -110,46 +128,58 @@ ;; 17.14. Split +(defn- split-op + ([string] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (split-op (core/-attach-cache string context))) + (-eval [_ context resource scope] + (when-let [string (core/-eval string context resource scope)] + [string])) + (-form [_] + (list 'split (core/-form string))))) + ([string separator] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (split-op (core/-attach-cache string context) + (core/-attach-cache separator context))) + (-eval [_ context resource scope] + (when-let [string (core/-eval string context resource scope)] + (if (= "" string) + [string] + (if-let [separator (core/-eval separator context resource scope)] + (case (count separator) + 0 + [string] + 1 + (loop [[char & more] string + result [] + acc (StringBuilder.)] + (if (= (str char) separator) + (if more + (recur more (conj result (str acc)) (StringBuilder.)) + (conj result (str acc))) + (if more + (recur more result (.append acc char)) + (conj result (str (.append acc char)))))) + ;; TODO: implement split with more than one char. + (throw (Exception. "TODO: implement split with separators longer than one char."))) + [string])))) + (-form [_] + (list 'split (core/-form string) (core/-form separator)))))) + + (defmethod core/compile* :elm.compiler.type/split [context {string :stringToSplit :keys [separator]}] (let [string (core/compile* context string) separator (some->> separator (core/compile* context))] (if separator - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [string (core/-eval string context resource scope)] - (if (= "" string) - [string] - (if-let [separator (core/-eval separator context resource scope)] - (case (count separator) - 0 - [string] - 1 - (loop [[char & more] string - result [] - acc (StringBuilder.)] - (if (= (str char) separator) - (if more - (recur more (conj result (str acc)) (StringBuilder.)) - (conj result (str acc))) - (if more - (recur more result (.append acc char)) - (conj result (str (.append acc char)))))) - ;; TODO: implement split with more than one char. - (throw (Exception. "TODO: implement split with separators longer than one char."))) - [string])))) - (-form [_] - (list 'split (core/-form string) (core/-form separator)))) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [string (core/-eval string context resource scope)] - [string])) - (-form [_] - (list 'split (core/-form string))))))) + (split-op string separator) + (split-op string)))) ;; 17.16. StartsWith @@ -159,34 +189,40 @@ ;; 17.17. Substring +(defn substring-op + ([string start-index] + (reify core/Expression + (-static [_] + false) + (-eval [_ context resource scope] + (when-let [^String string (core/-eval string context resource scope)] + (when-let [start-index (core/-eval start-index context resource scope)] + (when (and (<= 0 start-index) (< start-index (count string))) + (subs string start-index))))) + (-form [_] + (list 'substring (core/-form string) (core/-form start-index))))) + ([string start-index length] + (reify core/Expression + (-static [_] + false) + (-eval [_ context resource scope] + (when-let [^String string (core/-eval string context resource scope)] + (when-let [start-index (core/-eval start-index context resource scope)] + (when (and (<= 0 start-index) (< start-index (count string))) + (subs string start-index (min (+ start-index length) + (count string))))))) + (-form [_] + (list 'substring (core/-form string) (core/-form start-index) + (core/-form length)))))) + (defmethod core/compile* :elm.compiler.type/substring [context {string :stringToSub start-index :startIndex :keys [length]}] (let [string (core/compile* context string) start-index (core/compile* context start-index) length (some->> length (core/compile* context))] (if length - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [^String string (core/-eval string context resource scope)] - (when-let [start-index (core/-eval start-index context resource scope)] - (when (and (<= 0 start-index) (< start-index (count string))) - (subs string start-index (min (+ start-index length) - (count string))))))) - (-form [_] - (list 'substring (core/-form string) (core/-form start-index) - (core/-form length)))) - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (when-let [^String string (core/-eval string context resource scope)] - (when-let [start-index (core/-eval start-index context resource scope)] - (when (and (<= 0 start-index) (< start-index (count string))) - (subs string start-index))))) - (-form [_] - (list 'substring (core/-form string) (core/-form start-index))))))) + (substring-op string start-index length) + (substring-op string start-index)))) ;; 17.18. Upper diff --git a/modules/cql/src/blaze/elm/compiler/structured_values.clj b/modules/cql/src/blaze/elm/compiler/structured_values.clj index 0b649a208..4f81c3491 100644 --- a/modules/cql/src/blaze/elm/compiler/structured_values.clj +++ b/modules/cql/src/blaze/elm/compiler/structured_values.clj @@ -47,26 +47,35 @@ elements)) +(defn tuple [elements] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (tuple + (reduce-kv + (fn [r key value] + (assoc r key (core/-attach-cache value context))) + {} + elements))) + (-eval [_ context resource scope] + (reduce-kv + (fn [r key value] + (assoc r key (core/-eval value context resource scope))) + {} + elements)) + (-form [_] + (reduce-kv + (fn [r key value] + (assoc r key (core/-form value))) + {} + elements)))) + + (defmethod core/compile* :elm.compiler.type/tuple [context {elements :element}] (let [elements (compile-elements context elements)] - (if (every? core/static? (vals elements)) - elements - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (reduce-kv - (fn [r key value] - (assoc r key (core/-eval value context resource scope))) - {} - elements)) - (-form [_] - (reduce-kv - (fn [r key value] - (assoc r key (core/-form value))) - {} - elements)))))) + (cond-> elements (some (comp not core/static?) (vals elements)) tuple))) ;; 2.2. Instance @@ -85,6 +94,8 @@ core/Expression (-static [_] false) + (-attach-cache [_ context] + (->SourcePropertyExpression (core/-attach-cache source context) key)) (-eval [_ context resource scope] (p/get (core/-eval source context resource scope) key)) (-form [_] @@ -95,6 +106,8 @@ core/Expression (-static [_] false) + (-attach-cache [_ context] + (->SourcePropertyValueExpression (core/-attach-cache source context) key)) (-eval [_ context resource scope] (type/value (p/get (core/-eval source context resource scope) key))) (-form [_] @@ -105,6 +118,8 @@ core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ _ _ value] (p/get value key)) (-form [_] @@ -115,6 +130,8 @@ core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ _ _ scope] (p/get (get scope scope-key) key)) (-form [_] @@ -125,6 +142,8 @@ core/Expression (-static [_] false) + (-attach-cache [expr _] + expr) (-eval [_ _ _ scope] (type/value (p/get (get scope scope-key) key))) (-form [_] diff --git a/modules/cql/src/blaze/elm/compiler/type_operators.clj b/modules/cql/src/blaze/elm/compiler/type_operators.clj index 2ddbae60c..9365fc504 100644 --- a/modules/cql/src/blaze/elm/compiler/type_operators.clj +++ b/modules/cql/src/blaze/elm/compiler/type_operators.clj @@ -78,19 +78,25 @@ :expression expression)))) +(defn as-op [type pred operand] + (reify core/Expression + (-static [_] + false) + (-attach-cache [_ context] + (as-op type pred (core/-attach-cache operand context))) + (-eval [_ context resource scope] + (let [value (core/-eval operand context resource scope)] + (when (pred value) + value))) + (-form [_] + `(~'as ~type ~(core/-form operand))))) + + (defmethod core/compile* :elm.compiler.type/as [context {:keys [operand] :as expression}] (when-some [operand (core/compile* context operand)] (let [[type pred] (matches-type-fn expression)] - (reify core/Expression - (-static [_] - false) - (-eval [_ context resource scope] - (let [value (core/-eval operand context resource scope)] - (when (pred value) - value))) - (-form [_] - `(~'as ~type ~(core/-form operand))))))) + (as-op type pred operand)))) ;; TODO 22.2. CanConvert diff --git a/modules/cql/src/blaze/elm/compiler_spec.clj b/modules/cql/src/blaze/elm/compiler_spec.clj index fe1ce022d..bcf3bf183 100644 --- a/modules/cql/src/blaze/elm/compiler_spec.clj +++ b/modules/cql/src/blaze/elm/compiler_spec.clj @@ -3,6 +3,8 @@ [blaze.elm.compiler :as c] [blaze.elm.compiler.core :as core] [blaze.elm.compiler.spec] + [blaze.elm.expression :as-alias expr] + [blaze.elm.expression.spec] [blaze.elm.spec] [blaze.fhir.spec-spec] [clojure.spec.alpha :as s])) @@ -11,3 +13,8 @@ (s/fdef c/compile :args (s/cat :context :elm/compile-context :expression :elm/expression) :ret core/expr?) + + +(s/fdef c/attach-cache + :args (s/cat :context ::expr/attach-cache-context :expression core/expr?) + :ret core/expr?) diff --git a/modules/cql/src/blaze/elm/expression.clj b/modules/cql/src/blaze/elm/expression.clj index 5c769b643..912a980c2 100644 --- a/modules/cql/src/blaze/elm/expression.clj +++ b/modules/cql/src/blaze/elm/expression.clj @@ -1,5 +1,5 @@ (ns blaze.elm.expression - (:refer-clojure :exclude [eval]) + (:refer-clojure :exclude [eval hash]) (:require [blaze.elm.compiler.core :as core])) @@ -10,3 +10,9 @@ Throws an Exception on errors." [context expression resource] (core/-eval expression context resource nil)) + + +(defn hash + "Hashes `expression`." + [expression] + (clojure.core/hash (core/-form expression))) diff --git a/modules/cql/src/blaze/elm/expression/spec.clj b/modules/cql/src/blaze/elm/expression/spec.clj index 0683e3afc..5571b33b4 100644 --- a/modules/cql/src/blaze/elm/expression/spec.clj +++ b/modules/cql/src/blaze/elm/expression/spec.clj @@ -5,13 +5,19 @@ [blaze.elm.expression :as-alias expr] [blaze.elm.spec] [clojure.spec.alpha :as s] - [java-time.api :as time])) + [java-time.api :as time]) + (:import + [com.github.benmanes.caffeine.cache AsyncCache])) (s/def ::now time/offset-date-time?) +(s/def ::expr/cache + #(instance? AsyncCache %)) + + (s/def ::parameters (s/map-of :elm/name ::c/expression)) @@ -19,3 +25,7 @@ (s/def ::expr/context (s/keys :req-un [:blaze.db/db ::now] :opt-un [::c/expression-defs ::parameters])) + + +(s/def ::expr/attach-cache-context + (s/merge ::expr/context (s/keys :req [::expr/cache]))) diff --git a/modules/cql/src/blaze/elm/expression_spec.clj b/modules/cql/src/blaze/elm/expression_spec.clj index 307666857..4b50396bb 100644 --- a/modules/cql/src/blaze/elm/expression_spec.clj +++ b/modules/cql/src/blaze/elm/expression_spec.clj @@ -15,3 +15,8 @@ :args (s/cat :context ::expr/context :expression ::c/expression :resource (s/nilable ed/resource?))) + + +(s/fdef expr/hash + :args (s/cat :expression ::c/expression) + :ret int?) diff --git a/modules/cql/test/blaze/elm/compiler/conditional_operators_test.clj b/modules/cql/test/blaze/elm/compiler/conditional_operators_test.clj index f4ce27b4c..6846e1f14 100644 --- a/modules/cql/test/blaze/elm/compiler/conditional_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/conditional_operators_test.clj @@ -94,20 +94,20 @@ (let [expr (tu/dynamic-compile {:type "Case" :caseItem - [{:when #elm/parameter-ref "true" - :then #elm/integer "1"}] - :else #elm/integer "2"})] - (has-form expr '(case (param-ref "true") 1 2)))) + [{:when #elm/parameter-ref "x" + :then #elm/parameter-ref "y"}] + :else #elm/parameter-ref "z"})] + (has-form expr '(case (param-ref "x") (param-ref "y") (param-ref "z"))))) (testing "comparand-based" (let [expr (tu/dynamic-compile {:type "Case" :comparand #elm/parameter-ref "a" :caseItem - [{:when #elm/parameter-ref "b" - :then #elm/integer "1"}] - :else #elm/integer "2"})] - (has-form expr '(case (param-ref "a") (param-ref "b") 1 2)))))) + [{:when #elm/parameter-ref "x" + :then #elm/parameter-ref "y"}] + :else #elm/parameter-ref "z"})] + (has-form expr '(case (param-ref "a") (param-ref "x") (param-ref "y") (param-ref "z"))))))) (testing "expression is dynamic" (testing "multi-conditional" @@ -149,4 +149,16 @@ (testing "expression is dynamic" (is (false? (core/-static (tu/dynamic-compile #elm/if [#elm/parameter-ref "x" #elm/parameter-ref "y" - #elm/parameter-ref "z"])))))) + #elm/parameter-ref "z"]))))) + + (testing "form" + (let [expr (c/compile {} #elm/if [#elm/boolean "true" #elm/integer "1" #elm/integer "2"])] + (has-form expr 1)) + + (let [expr (c/compile {} #elm/if [#elm/boolean "false" #elm/integer "1" #elm/integer "2"])] + (has-form expr 2)) + + (let [expr (tu/dynamic-compile #elm/if [#elm/parameter-ref "x" + #elm/parameter-ref "y" + #elm/parameter-ref "z"])] + (has-form expr '(if (param-ref "x") (param-ref "y") (param-ref "z")))))) diff --git a/modules/cql/test/blaze/elm/compiler/expr_cache_spec.clj b/modules/cql/test/blaze/elm/compiler/expr_cache_spec.clj new file mode 100644 index 000000000..58c529db8 --- /dev/null +++ b/modules/cql/test/blaze/elm/compiler/expr_cache_spec.clj @@ -0,0 +1,12 @@ +(ns blaze.elm.compiler.expr-cache-spec + (:require + [blaze.db.tx-log.spec] + [blaze.elm.compiler.expr-cache :as ec] + [blaze.fhir.spec.spec] + [clojure.spec.alpha :as s])) + + +(s/fdef ec/cache + :args (s/cat :t :blaze.db/t + :resource-ids (s/coll-of :blaze.resource/id :kind vector?)) + :ret ec/cache?) diff --git a/modules/cql/test/blaze/elm/compiler/external_data_test.clj b/modules/cql/test/blaze/elm/compiler/external_data_test.clj index 1c96b7e4b..13785a0ab 100644 --- a/modules/cql/test/blaze/elm/compiler/external_data_test.clj +++ b/modules/cql/test/blaze/elm/compiler/external_data_test.clj @@ -16,7 +16,6 @@ [blaze.elm.compiler.test-util :as tu :refer [has-form]] [blaze.elm.expression :as expr] [blaze.elm.expression-spec] - [blaze.elm.expression.cache :as-alias expr-cache] [blaze.fhir.spec :as fhir-spec] [blaze.fhir.spec.type] [blaze.module.test-util :refer [with-system]] @@ -401,9 +400,7 @@ define InInitialPopulation: [\"name-133756\" -> Observation] ") - compile-context {::expr-cache/enabled? false} - {:keys [expression-defs]} (library/compile-library - node library compile-context) + {:keys [expression-defs]} (library/compile-library node library {}) db (d/db node) patient (resource db "Patient" "0") eval-context (assoc (eval-context db) :expression-defs expression-defs) @@ -448,9 +445,7 @@ define InInitialPopulation: [\"name-133730\" -> Observation: Code 'code-133657' from sys] ") - compile-context {::expr-cache/enabled? false} - {:keys [expression-defs]} (library/compile-library - node library compile-context) + {:keys [expression-defs]} (library/compile-library node library {}) db (d/db node) patient (resource db "Patient" "0") eval-context (assoc (eval-context db) :expression-defs expression-defs) diff --git a/modules/cql/test/blaze/elm/compiler/library_test.clj b/modules/cql/test/blaze/elm/compiler/library_test.clj index 9186742c9..cec899581 100644 --- a/modules/cql/test/blaze/elm/compiler/library_test.clj +++ b/modules/cql/test/blaze/elm/compiler/library_test.clj @@ -20,6 +20,9 @@ (test/use-fixtures :each tu/fixture) +(def default-opts {}) + + ;; 5.1. Library ;; ;; 1. The identifier element defines a unique identifier for this library, and @@ -60,13 +63,13 @@ (testing "empty library" (let [library (t/translate "library Test")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library {}) + (given (library/compile-library node library default-opts) :expression-defs := {})))) (testing "one static expression" (let [library (t/translate "library Test define Foo: true")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library {}) + (given (library/compile-library node library default-opts) [:expression-defs "Foo" :context] := "Patient" [:expression-defs "Foo" :expression] := true)))) @@ -76,7 +79,7 @@ context Patient define Gender: Patient.gender")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library {}) + (given (library/compile-library node library default-opts) [:expression-defs "Gender" :context] := "Patient" [:expression-defs "Gender" :expression c/form] := '(:gender (expr-ref "Patient")))))) @@ -87,7 +90,7 @@ define function Gender(P Patient): P.gender define InInitialPopulation: Gender(Patient)")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library {}) + (given (library/compile-library node library default-opts) [:expression-defs "InInitialPopulation" :context] := "Patient" [:expression-defs "InInitialPopulation" :resultTypeName] := "{http://hl7.org/fhir}AdministrativeGender" [:expression-defs "InInitialPopulation" :expression c/form] := '(call "Gender" (expr-ref "Patient")) @@ -103,7 +106,7 @@ define function Inc2(i System.Integer): Inc(i) + 1 define InInitialPopulation: Inc2(1)")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library {}) + (given (library/compile-library node library default-opts) [:expression-defs "InInitialPopulation" :context] := "Patient" [:expression-defs "InInitialPopulation" :expression c/form] := '(call "Inc2" 1))))) @@ -112,7 +115,7 @@ (let [library (t/translate "library Test define function Error(): singleton from {1, 2}")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library {}) + (given (library/compile-library node library default-opts) ::anom/category := ::anom/conflict ::anom/message := "More than one element in `SingletonFrom` expression.")))) @@ -120,7 +123,7 @@ (let [library (t/translate "library Test define Error: singleton from {1, 2}")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library {}) + (given (library/compile-library node library default-opts) ::anom/category := ::anom/conflict ::anom/message := "More than one element in `SingletonFrom` expression."))))) @@ -128,7 +131,7 @@ (let [library (t/translate "library Test parameter \"Measurement Period\" Interval default Interval[@2020-01-01, @2020-12-31]")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library {}) + (given (library/compile-library node library default-opts) [:parameter-default-values "Measurement Period" :start] := #system/date"2020-01-01" [:parameter-default-values "Measurement Period" :end] := #system/date"2020-12-31")))) @@ -136,6 +139,6 @@ (let [library (t/translate "library Test parameter \"Measurement Start\" Integer default singleton from {1, 2}")] (with-system [{:blaze.db/keys [node]} mem-node-config] - (given (library/compile-library node library {}) + (given (library/compile-library node library default-opts) ::anom/category := ::anom/conflict ::anom/message "More than one element in `SingletonFrom` expression."))))) diff --git a/modules/cql/test/blaze/elm/compiler/list_operators_test.clj b/modules/cql/test/blaze/elm/compiler/list_operators_test.clj index 39a530015..59ae88ed2 100644 --- a/modules/cql/test/blaze/elm/compiler/list_operators_test.clj +++ b/modules/cql/test/blaze/elm/compiler/list_operators_test.clj @@ -5,14 +5,18 @@ https://cql.hl7.org/04-logicalspecification.html." (:require [blaze.anomaly-spec] + [blaze.db.api :as d] + [blaze.db.api-stub :refer [mem-node-config with-system-data]] [blaze.elm.compiler :as c] [blaze.elm.compiler-spec] [blaze.elm.compiler.core :as core] [blaze.elm.compiler.core-spec] + [blaze.elm.compiler.expr-cache-spec] + [blaze.elm.compiler.external-data :as ed] [blaze.elm.compiler.list-operators] [blaze.elm.compiler.test-util :as tu :refer [has-form]] + [blaze.elm.expression :as expr] [blaze.elm.expression-spec] - [blaze.elm.expression.cache :as-alias expr-cache] [blaze.elm.literal :as elm] [blaze.elm.literal-spec] [blaze.elm.quantity :as quantity] @@ -20,9 +24,13 @@ [clojure.spec.alpha :as s] [clojure.spec.test.alpha :as st] [clojure.test :as test :refer [are deftest is testing]] - [clojure.test.check.properties :as prop])) + [clojure.test.check.properties :as prop]) + (:import + [com.github.benmanes.caffeine.cache Caffeine] + [java.time OffsetDateTime])) +(set! *warn-on-reflection* true) (st/instrument) (tu/instrument-compile) @@ -153,7 +161,7 @@ ;; 20.8. Exists ;; -;; The Exists operator returns true if the list contains any elements. +;; The Exists operator returns true if the list contains any non-null elements. ;; ;; If the argument is null, the result is false. (deftest compile-exists-test @@ -175,7 +183,82 @@ {:type "Null"} false) - (tu/testing-unary-form elm/exists))) + (testing "with caching expressions" + (with-system-data [{:blaze.db/keys [node]} mem-node-config] + [[[:put {:fhir/type :fhir/Patient :id "0"}]]] + + (let [db (d/db node) + patient (ed/mk-resource db (d/resource-handle db "Patient" "0")) + elm #elm/exists #elm/retrieve{:type "Observation"} + compile-context + {:node node + :eval-context "Patient" + :library {}} + expr (c/compile compile-context elm) + attach-cache-context + {:db db + ::expr/cache (.buildAsync (Caffeine/newBuilder)) + :now (OffsetDateTime/now)} + eval-context + {:db db + :now (OffsetDateTime/now)}] + + (testing "has no Observation at the beginning" + (let [expr (c/attach-cache attach-cache-context expr)] + (is (false? (expr/eval eval-context expr patient))))) + + (Thread/sleep 100) + + (testing "has still no Observation after the Bloom filter is filled" + (let [expr (c/attach-cache attach-cache-context expr)] + (is (false? (expr/eval eval-context expr patient))))) + + (let [tx-op [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}] + db-after @(d/transact node [tx-op])] + + (testing "has an Observation after transaction" + (let [expr (c/attach-cache attach-cache-context expr) + patient (ed/mk-resource db-after (d/resource-handle db "Patient" "0"))] + (is (true? (expr/eval (assoc eval-context :db db-after) expr patient))))) + + (testing "has still no Observation at the old database" + (let [expr (c/attach-cache attach-cache-context expr)] + (is (false? (expr/eval eval-context expr patient))))))))) + + (testing "without caching expressions" + (with-system-data [{:blaze.db/keys [node]} mem-node-config] + [[[:put {:fhir/type :fhir/Patient :id "0"}]]] + + (let [db (d/db node) + patient (ed/mk-resource db (d/resource-handle db "Patient" "0")) + elm #elm/exists #elm/retrieve{:type "Observation"} + compile-context + {:node node + :eval-context "Patient" + :library {}} + expr (c/compile compile-context elm) + eval-context + {:db db + :now (OffsetDateTime/now)}] + + (testing "has no Observation at the beginning" + (is (false? (expr/eval eval-context expr patient)))) + + (let [tx-op [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}] + db-after @(d/transact node [tx-op])] + + (testing "has an Observation after transaction" + (let [patient (ed/mk-resource db-after (d/resource-handle db "Patient" "0"))] + (is (true? (expr/eval (assoc eval-context :db db-after) expr patient))))) + + (testing "has still no Observation at the old database" + (is (false? (expr/eval eval-context expr patient))))))))) + + (tu/testing-unary-dynamic elm/exists) + + (tu/testing-unary-form elm/exists)) ;; 20.9. Filter diff --git a/modules/cql/test/blaze/elm/compiler/queries_test.clj b/modules/cql/test/blaze/elm/compiler/queries_test.clj index 57bf10167..4027d6dfb 100644 --- a/modules/cql/test/blaze/elm/compiler/queries_test.clj +++ b/modules/cql/test/blaze/elm/compiler/queries_test.clj @@ -48,34 +48,48 @@ (testing "Non-retrieve queries" (testing "Sort" (testing "ByDirection" - (are [query res] (= res (core/-eval (c/compile {} query) {} nil nil)) - {:type "Query" - :source - [{:alias "S" - :expression #elm/list [#elm/integer "2" #elm/integer "1" #elm/integer "1"]}] - :sort {:by [{:type "ByDirection" :direction "asc"}]}} - [1 2])) + (let [elm {:type "Query" + :source + [{:alias "S" + :expression #elm/list [#elm/integer "2" #elm/integer "1" #elm/integer "1"]}] + :sort {:by [{:type "ByDirection" :direction "asc"}]}} + expr (c/compile {} elm)] + + (testing "eval" + (is (= [1 2] (core/-eval expr {} nil nil)))) + + (testing "form" + (has-form expr '(sorted-vector-query distinct [2 1 1] asc))))) (testing "ByExpression" - (are [query res] (= res (core/-eval (c/compile {} query) {} nil nil)) - {:type "Query" - :source - [{:alias "S" - :expression - #elm/list - [#elm/quantity [2 "m"] - #elm/quantity [1 "m"] - #elm/quantity [1 "m"]]}] - :sort - {:by - [{:type "ByExpression" - :direction "asc" - :expression - {:type "Property" - :path "value" - :scope "S" - :resultTypeName "{urn:hl7-org:elm-types:r1}decimal"}}]}} - [(quantity/quantity 1 "m") (quantity/quantity 2 "m")]) + (let [elm {:type "Query" + :source + [{:alias "S" + :expression + #elm/list + [#elm/quantity [2 "m"] + #elm/quantity [1 "m"] + #elm/quantity [1 "m"]]}] + :sort + {:by + [{:type "ByExpression" + :direction "asc" + :expression + {:type "Property" + :path "value" + :scope "S" + :resultTypeName "{urn:hl7-org:elm-types:r1}decimal"}}]}} + expr (c/compile {} elm)] + + (testing "eval" + (is (= [(quantity/quantity 1 "m") (quantity/quantity 2 "m")] (core/-eval expr {} nil nil)))) + + (testing "form" + (has-form expr '(sorted-vector-query distinct + [(quantity 2 "m") + (quantity 1 "m") + (quantity 1 "m")] + [asc (:value S)])))) (testing "with IdentifierRef" (are [query res] (= res (core/-eval (c/compile {} query) {} nil nil)) diff --git a/modules/cql/test/blaze/elm/compiler/test_util.clj b/modules/cql/test/blaze/elm/compiler/test_util.clj index 6e39cbc1f..156e36abc 100644 --- a/modules/cql/test/blaze/elm/compiler/test_util.clj +++ b/modules/cql/test/blaze/elm/compiler/test_util.clj @@ -48,7 +48,8 @@ (def dynamic-compile-ctx - {:library + {:eval-context "Patient" + :library {:parameters {:def [{:name "true"} diff --git a/modules/db-protocols/src/blaze/db/impl/protocols.clj b/modules/db-protocols/src/blaze/db/impl/protocols.clj index 982e57ce9..9a1ef5ca5 100644 --- a/modules/db-protocols/src/blaze/db/impl/protocols.clj +++ b/modules/db-protocols/src/blaze/db/impl/protocols.clj @@ -24,6 +24,8 @@ [db compartment tid] [db compartment tid start-id]) + (-patient-compartment-last-change-t [db patient-id]) + (-count-query [db query] "Returns a CompletableFuture that will complete with the count of the matching resource handles.") @@ -109,4 +111,4 @@ (-list-by-type [_ type]) (-list-by-target [_ target]) (-linked-compartments [_ resource]) - (-compartment-resources [_ type])) + (-compartment-resources [_ compartment-type] [_ compartment-type type])) diff --git a/modules/db-stub/src/blaze/db/api_stub.clj b/modules/db-stub/src/blaze/db/api_stub.clj index 9c4123355..561de85f2 100644 --- a/modules/db-stub/src/blaze/db/api_stub.clj +++ b/modules/db-stub/src/blaze/db/api_stub.clj @@ -29,6 +29,7 @@ :kv-store (ig/ref :blaze.db/index-kv-store) :resource-indexer (ig/ref :blaze.db.node/resource-indexer) :search-param-registry (ig/ref :blaze.db/search-param-registry) + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} node-config) @@ -57,6 +58,7 @@ :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil + :patient-as-of-index nil :type-stats-index nil :system-stats-index nil}} @@ -76,7 +78,9 @@ :blaze.db.node.resource-indexer/executor {} :blaze.db/search-param-registry - {:structure-definition-repo structure-definition-repo}}) + {:structure-definition-repo structure-definition-repo} + + :blaze/scheduler {}}) (def mem-node-config diff --git a/modules/db/.clj-kondo/config.edn b/modules/db/.clj-kondo/config.edn index 3d510f5ed..f57f4bffc 100644 --- a/modules/db/.clj-kondo/config.edn +++ b/modules/db/.clj-kondo/config.edn @@ -38,6 +38,7 @@ blaze.db.impl.protocols p blaze.db.search-param-registry sr blaze.executors ex + blaze.scheduler sched buddy.auth auth cognitect.anomalies anom clojure.java.io io diff --git a/modules/db/deps.edn b/modules/db/deps.edn index 8023f0d6d..90e89975d 100644 --- a/modules/db/deps.edn +++ b/modules/db/deps.edn @@ -10,6 +10,9 @@ blaze/byte-string {:local/root "../byte-string"} + blaze/cache-collector + {:local/root "../cache-collector"} + blaze/coll {:local/root "../coll"} @@ -34,6 +37,9 @@ blaze/db-resource-store {:local/root "../db-resource-store"} + blaze/scheduler + {:local/root "../scheduler"} + blaze/spec {:local/root "../spec"} diff --git a/modules/db/src/blaze/db/api.clj b/modules/db/src/blaze/db/api.clj index cc921e3bc..1d0138327 100644 --- a/modules/db/src/blaze/db/api.clj +++ b/modules/db/src/blaze/db/api.clj @@ -290,6 +290,13 @@ +;; ---- Patient-Compartment-Level Functions ----------------------------------- + +(defn patient-compartment-last-change-t [db patient-id] + (p/-patient-compartment-last-change-t db (codec/id-byte-string patient-id))) + + + ;; ---- Common Query Functions ------------------------------------------------ (defn count-query diff --git a/modules/db/src/blaze/db/cache_collector/spec.clj b/modules/db/src/blaze/db/cache_collector/spec.clj deleted file mode 100644 index 1f515be65..000000000 --- a/modules/db/src/blaze/db/cache_collector/spec.clj +++ /dev/null @@ -1,8 +0,0 @@ -(ns blaze.db.cache-collector.spec - (:require - [blaze.db.cache-collector.protocols :as p] - [clojure.spec.alpha :as s])) - - -(s/def :blaze.db.cache-collector/caches - (s/map-of string? (s/nilable #(satisfies? p/StatsCache %)))) diff --git a/modules/db/src/blaze/db/impl/batch_db.clj b/modules/db/src/blaze/db/impl/batch_db.clj index 89a087c8f..ebe707bfe 100644 --- a/modules/db/src/blaze/db/impl/batch_db.clj +++ b/modules/db/src/blaze/db/impl/batch_db.clj @@ -9,6 +9,7 @@ [blaze.db.impl.codec :as codec] [blaze.db.impl.index :as index] [blaze.db.impl.index.compartment.resource :as cr] + [blaze.db.impl.index.patient-as-of :as pao] [blaze.db.impl.index.resource-as-of :as rao] [blaze.db.impl.index.resource-handle :as rh] [blaze.db.impl.index.search-param-value-resource :as sp-vr] @@ -37,7 +38,7 @@ (:total (type-stats/get! iter tid t) 0))) -(defrecord BatchDb [node basis-t context] +(defrecord BatchDb [node basis-t t context] p/Db (-node [_] node) @@ -45,6 +46,9 @@ (-basis-t [_] basis-t) + (-as-of-t [_] + (when (not= basis-t t) t)) + ;; ---- Instance-Level Functions -------------------------------------------- @@ -92,6 +96,13 @@ + ;; ---- Patient-Compartment-Level Functions --------------------------------- + + (-patient-compartment-last-change-t [_ patient-id] + (pao/last-change-t (:paoi context) patient-id t)) + + + ;; ---- Common Query Functions ---------------------------------------------- (-count-query [_ query] @@ -236,8 +247,9 @@ AutoCloseable (close [_] - (let [{:keys [snapshot raoi svri rsvi cri csvri]} context] + (let [{:keys [snapshot raoi paoi svri rsvi cri csvri]} context] (.close ^AutoCloseable raoi) + (.close ^AutoCloseable paoi) (.close ^AutoCloseable svri) (.close ^AutoCloseable rsvi) (.close ^AutoCloseable cri) @@ -321,9 +333,11 @@ (->BatchDb node basis-t + t (let [raoi (kv/new-iterator snapshot :resource-as-of-index)] {:snapshot snapshot :raoi raoi + :paoi (kv/new-iterator snapshot :patient-as-of-index) :resource-handle (rao/resource-handle raoi t) :svri (kv/new-iterator snapshot :search-param-value-index) :rsvi (kv/new-iterator snapshot :resource-value-index) diff --git a/modules/db/src/blaze/db/impl/db.clj b/modules/db/src/blaze/db/impl/db.clj index b2a30c0eb..3201a18bc 100644 --- a/modules/db/src/blaze/db/impl/db.clj +++ b/modules/db/src/blaze/db/impl/db.clj @@ -3,6 +3,7 @@ (:require [blaze.async.comp :as ac] [blaze.db.impl.batch-db :as batch-db] + [blaze.db.impl.index.patient-as-of :as pao] [blaze.db.impl.index.resource-as-of :as rao] [blaze.db.impl.macros :refer [with-open-coll]] [blaze.db.impl.protocols :as p] @@ -85,6 +86,15 @@ + ;; ---- Patient-Compartment-Level Functions --------------------------------- + + (-patient-compartment-last-change-t [_ patient-id] + (with-open [snapshot (kv/new-snapshot kv-store) + paoi (kv/new-iterator snapshot :patient-as-of-index)] + (pao/last-change-t paoi patient-id t))) + + + ;; ---- Common Query Functions ---------------------------------------------- (-count-query [_ query] diff --git a/modules/db/src/blaze/db/impl/index/patient_as_of.clj b/modules/db/src/blaze/db/impl/index/patient_as_of.clj new file mode 100644 index 000000000..b249b0109 --- /dev/null +++ b/modules/db/src/blaze/db/impl/index/patient_as_of.clj @@ -0,0 +1,87 @@ +(ns blaze.db.impl.index.patient-as-of + "Functions for accessing the PatientAsOf index." + (:require + [blaze.byte-buffer :as bb] + [blaze.byte-string :as bs] + [blaze.db.impl.codec :as codec] + [blaze.db.impl.index.rts-as-of :as rts] + [blaze.db.kv :as kv]) + (:import + [java.nio.charset StandardCharsets])) + + +(set! *warn-on-reflection* true) +(set! *unchecked-math* :warn-on-boxed) + + +(def ^:private ^:const ^long t-tid-size + (+ codec/t-size codec/tid-size)) + + +(defn- encode-key + "Encodes the key of the PatientAsOf index from `patient-id`, `t`, `tid` and + `id`." + [patient-id t tid id] + (-> (bb/allocate (-> (unchecked-add-int (bs/size patient-id) (bs/size id)) + (unchecked-add-int t-tid-size))) + (bb/put-byte-string! patient-id) + (bb/put-long! (codec/descending-long ^long t)) + (bb/put-int! tid) + (bb/put-byte-string! id) + bb/array)) + + +(defn index-entry [patient-id tid id t hash num-changes op] + [:patient-as-of-index (encode-key patient-id t tid id) + (rts/encode-value hash num-changes op)]) + + +(defn- encode-patient-id-t [patient-id t] + (-> (bb/allocate (unchecked-add-int (bs/size patient-id) codec/t-size)) + (bb/put-byte-string! patient-id) + (bb/put-long! (codec/descending-long ^long t)) + bb/array)) + + +(defn last-change-t + "Returns the `t` of last change of any resource in the patient compartment not + newer than `t`." + [paoi patient-id t] + (kv/seek! paoi (encode-patient-id-t patient-id t)) + (when (kv/valid? paoi) + (-> (bb/get-long! (bb/wrap (kv/key paoi)) (bs/size patient-id)) + (codec/descending-long)))) + + +(def ^:private state-key + (.getBytes "patient-as-of-state" StandardCharsets/ISO_8859_1)) + + +(defn- encode-state [{:keys [type t]}] + (if (identical? :current type) + (byte-array [0]) + (-> (bb/allocate (inc Long/BYTES)) + (bb/put-byte! 1) + (bb/put-long! t) + bb/array))) + + +(defn decode-state [bytes] + (let [buf (bb/wrap bytes)] + (if (zero? (bb/get-byte! buf)) + {:type :current} + {:type :building + :t (bb/get-long! buf)}))) + + +(defn state + "Returns the state of the PatientAsOf index. + + The state is on of :current or last-t." + [kv-store] + (or (some-> (kv/get kv-store state-key) decode-state) + {:type :building :t 0})) + + +(defn state-index-entry [state] + [state-key (encode-state state)]) diff --git a/modules/db/src/blaze/db/node.clj b/modules/db/src/blaze/db/node.clj index 8bcc308bb..110fa6d04 100644 --- a/modules/db/src/blaze/db/node.clj +++ b/modules/db/src/blaze/db/node.clj @@ -6,6 +6,7 @@ [blaze.db.impl.batch-db :as batch-db] [blaze.db.impl.codec :as codec] [blaze.db.impl.db :as db] + [blaze.db.impl.index.patient-as-of :as pao] [blaze.db.impl.index.resource-handle :as rh] [blaze.db.impl.index.t-by-instant :as t-by-instant] [blaze.db.impl.index.tx-error :as tx-error] @@ -15,6 +16,7 @@ [blaze.db.impl.search-param.all :as search-param-all] [blaze.db.impl.search-param.chained :as spc] [blaze.db.kv :as kv] + [blaze.db.node.patient-as-of-index :as node-pao] [blaze.db.node.protocols :as np] [blaze.db.node.resource-indexer :as resource-indexer] [blaze.db.node.resource-indexer.spec] @@ -31,6 +33,7 @@ [blaze.fhir.spec :as fhir-spec] [blaze.fhir.spec.type :as type] [blaze.module :refer [reg-collector]] + [blaze.scheduler :as sched] [blaze.spec] [blaze.util :refer [conj-vec]] [clojure.spec.alpha :as s] @@ -167,9 +170,9 @@ future)) -(defn- index-tx [db-before tx-data] +(defn- index-tx [search-param-registry db-before tx-data] (with-open [_ (prom/timer duration-seconds "index-transactions")] - (tx-indexer/index-tx db-before tx-data))) + (tx-indexer/index-tx search-param-registry db-before tx-data))) (defn- advance-t! [state t] @@ -222,13 +225,13 @@ "This is the main transaction handling function. If indexes resources and transaction data and commits either success or error." - [{:keys [resource-indexer kv-store] :as node} + [{:keys [resource-indexer kv-store search-param-registry] :as node} {:keys [t instant tx-cmds] :as tx-data}] (log/trace "index transaction with t =" t "and" (count tx-cmds) "command(s)") (prom/observe! transaction-sizes (count tx-cmds)) (let [timer (prom/timer duration-seconds "index-resources") future (resource-indexer/index-resources resource-indexer tx-data) - result (index-tx (np/-db node) tx-data)] + result (index-tx search-param-registry (np/-db node) tx-data)] (if (ba/anomaly? result) (commit-error! node t result) (do @@ -242,7 +245,11 @@ (tx-log/poll! queue poll-timeout))) -(defn- poll-and-index! [node queue poll-timeout] +(defn- poll-and-index! + "Polls `queue` once and indexes the resulting transaction data. + + Waits up to `poll-timeout` for the transaction data to become available." + [node queue poll-timeout] (log/trace "poll transaction queue") (run! (partial index-tx-data! node) (poll-tx-queue! queue poll-timeout))) @@ -482,7 +489,8 @@ :blaze.db/kv-store ::resource-indexer :blaze.db/resource-store - :blaze.db/search-param-registry] + :blaze.db/search-param-registry + :blaze/scheduler] :opt-un [:blaze.db/enforce-referential-integrity])) @@ -530,9 +538,43 @@ (ac/completed-future (db/db node (:t @(.-state node))))))) +(defn- index-patient-as-of-index! + [{:keys [kv-store] :as node} current-t {:keys [t] :as tx-data}] + (log/debug "Build PatientAsOf index with t =" t) + (when-ok [entries (node-pao/index-entries node tx-data)] + (store-tx-entries! kv-store entries)) + (vreset! current-t t)) + + +(defn- poll-and-index-patient-as-of-index! [node queue current-t poll-timeout] + (run! (partial index-patient-as-of-index! node current-t) (poll-tx-queue! queue poll-timeout))) + + +(defn build-patient-as-of-index + [{:keys [tx-log kv-store run? state poll-timeout] :as node}] + (let [{:keys [type t]} (pao/state kv-store)] + (when (identical? :building type) + (let [start-t (inc t) + end-t (:t @state) + current-t (volatile! start-t)] + (log/info "Building PatientAsOf index starting at t =" start-t) + (with-open [queue (tx-log/new-queue tx-log start-t)] + (while (and @run? (< @current-t end-t)) + (try + (poll-and-index-patient-as-of-index! node queue current-t poll-timeout) + (catch Exception e + (log/error "Error while building the PatientAsOf index." e))))) + (if (>= @current-t end-t) + (do + (store-tx-entries! kv-store [(pao/state-index-entry {:type :current})]) + (log/info "Finished building PatientAsOf index.")) + (log/info "Partially build PatientAsOf index up to t =" @current-t + "at a goal of t =" end-t "Will continue at next start.")))))) + + (defmethod ig/init-key :blaze.db/node [_ {:keys [storage tx-log tx-cache indexer-executor kv-store resource-indexer - resource-store search-param-registry poll-timeout] + resource-store search-param-registry scheduler poll-timeout] :or {poll-timeout (time/seconds 1)} :as config}] (init-msg config) @@ -543,6 +585,8 @@ (volatile! true) poll-timeout (ac/future))] + (when (= :building (:type (pao/state kv-store))) + (sched/submit scheduler #(build-patient-as-of-index node))) (execute node indexer-executor) node)) diff --git a/modules/db/src/blaze/db/node/patient_as_of_index.clj b/modules/db/src/blaze/db/node/patient_as_of_index.clj new file mode 100644 index 000000000..2b2614fbb --- /dev/null +++ b/modules/db/src/blaze/db/node/patient_as_of_index.clj @@ -0,0 +1,14 @@ +(ns blaze.db.node.patient-as-of-index + (:require + [blaze.anomaly :refer [when-ok]] + [blaze.db.impl.db :as db] + [blaze.db.impl.index.patient-as-of :as pao] + [blaze.db.node.tx-indexer :as tx-indexer])) + + +(defn index-entries + {:arglists '([node tx-data])} + [{:keys [search-param-registry] :as node} {:keys [t] :as tx-data}] + (when-ok [entries (tx-indexer/index-tx search-param-registry (db/db node (dec t)) tx-data)] + (-> (filterv (comp #{:patient-as-of-index} first) entries) + (conj (pao/state-index-entry {:type :building :t t}))))) diff --git a/modules/db/src/blaze/db/node/tx_indexer.clj b/modules/db/src/blaze/db/node/tx_indexer.clj index c06510004..7c18d30f7 100644 --- a/modules/db/src/blaze/db/node/tx_indexer.clj +++ b/modules/db/src/blaze/db/node/tx_indexer.clj @@ -6,7 +6,7 @@ (defn index-tx - [db-before {:keys [t tx-cmds]}] + [search-param-registry db-before {:keys [t tx-cmds]}] (log/trace "verify transaction commands with t =" t "based on db with t =" (d/basis-t db-before)) - (verify/verify-tx-cmds db-before t tx-cmds)) + (verify/verify-tx-cmds search-param-registry db-before t tx-cmds)) diff --git a/modules/db/src/blaze/db/node/tx_indexer/verify.clj b/modules/db/src/blaze/db/node/tx_indexer/verify.clj index b3d72994a..c3d2e7fc9 100644 --- a/modules/db/src/blaze/db/node/tx_indexer/verify.clj +++ b/modules/db/src/blaze/db/node/tx_indexer/verify.clj @@ -3,11 +3,13 @@ [blaze.anomaly :as ba :refer [throw-anom]] [blaze.db.api :as d] [blaze.db.impl.codec :as codec] + [blaze.db.impl.index.patient-as-of :as pao] [blaze.db.impl.index.resource-handle :as rh] [blaze.db.impl.index.rts-as-of :as rts] [blaze.db.impl.index.system-stats :as system-stats] [blaze.db.impl.index.type-stats :as type-stats] [blaze.db.kv.spec] + [blaze.db.search-param-registry :as sr] [blaze.fhir.hash :as hash] [blaze.util :as u] [clojure.string :as str] @@ -115,8 +117,8 @@ statistics of the transaction outcome. Throws an anomaly on conflicts." - {:arglists '([db-before t res cmd])} - (fn [_db-before _t _res {:keys [op]}] op)) + {:arglists '([search-param-registry db-before t res cmd])} + (fn [_search-param-registry _db-before _t _res {:keys [op]}] op)) (defn- verify-tx-cmd-create-msg [type id] @@ -132,20 +134,27 @@ (throw-anom (ba/conflict (id-collision-msg type id))))) -(defn- index-entries [tid id t hash num-changes op] - (rts/index-entries tid (codec/id-byte-string id) t hash num-changes op)) +(defn- index-entries [tid id t hash num-changes op refs] + (let [id (codec/id-byte-string id)] + (into + (rts/index-entries tid id t hash num-changes op) + (keep (fn [[ref-type ref-id]] + (when (= "Patient" ref-type) + (pao/index-entry (codec/id-byte-string ref-id) tid id t hash + num-changes op)))) + refs))) (def ^:private inc-0 (fnil inc 0)) (defmethod verify-tx-cmd "create" - [db-before t res {:keys [type id hash]}] + [_search-param-registry db-before t res {:keys [type id hash refs]}] (log/trace (verify-tx-cmd-create-msg type id)) (with-open [_ (prom/timer duration-seconds "verify-create")] (check-id-collision! db-before type id) (let [tid (codec/tid type)] - (-> (update res :entries into (index-entries tid id t hash 1 :create)) + (-> (update res :entries into (index-entries tid id t hash 1 :create refs)) (update :new-resources conj [type id]) (update-in [:stats tid :num-changes] inc-0) (update-in [:stats tid :total] inc-0))))) @@ -191,7 +200,8 @@ (defmethod verify-tx-cmd "put" - [db-before t res {:keys [type id hash if-match if-none-match] :as tx-cmd}] + [_search-param-registry db-before t res + {:keys [type id hash if-match if-none-match refs] :as tx-cmd}] (log/trace (verify-tx-cmd-put-msg type id (u/to-seq if-match) if-none-match)) (with-open [_ (prom/timer duration-seconds "verify-put")] (let [tid (codec/tid type) @@ -213,7 +223,7 @@ :else (cond-> - (-> (update res :entries into (index-entries tid id t hash (inc num-changes) :put)) + (-> (update res :entries into (index-entries tid id t hash (inc num-changes) :put refs)) (update :new-resources conj [type id]) (update-in [:stats tid :num-changes] inc-0)) (or (nil? old-t) (identical? :delete op)) @@ -227,7 +237,7 @@ (defmethod verify-tx-cmd "keep" - [db-before _ res {:keys [type id hash if-match] :as tx-cmd}] + [_search-param-registry db-before _ res {:keys [type id hash if-match] :as tx-cmd}] (log/trace (verify-tx-cmd-keep-msg type id (u/to-seq if-match))) (with-open [_ (prom/timer duration-seconds "verify-keep")] (let [if-match (u/to-seq if-match) @@ -245,15 +255,24 @@ res)))) +(defn- patient-refs [search-param-registry db type resource-handle] + (into + [] + (comp (mapcat #(d/include db resource-handle % "Patient")) + (map (fn [{:keys [id]}] ["Patient" id]))) + (sr/compartment-resources search-param-registry "Patient" type))) + + (defmethod verify-tx-cmd "delete" - [db-before t res {:keys [type id]}] + [search-param-registry db-before t res {:keys [type id]}] (log/trace "verify-tx-cmd :delete" (str type "/" id)) (with-open [_ (prom/timer duration-seconds "verify-delete")] (let [tid (codec/tid type) - {:keys [num-changes op] :or {num-changes 0}} - (d/resource-handle db-before type id)] + {:keys [num-changes op] :or {num-changes 0} :as old-resource-handle} + (d/resource-handle db-before type id) + refs (some->> old-resource-handle (patient-refs search-param-registry db-before type))] (cond-> - (-> (update res :entries into (index-entries tid id t hash/deleted-hash (inc num-changes) :delete)) + (-> (update res :entries into (index-entries tid id t hash/deleted-hash (inc num-changes) :delete refs)) (update :del-resources conj [type id]) (update-in [:stats tid :num-changes] inc-0)) (and op (not (identical? :delete op))) @@ -261,13 +280,13 @@ (defmethod verify-tx-cmd :default - [_db-before _t res _tx-cmd] + [_search-param-registry _db-before _t res _tx-cmd] res) -(defn- verify-tx-cmds** [db-before t tx-cmds] +(defn- verify-tx-cmds** [search-param-registry db-before t tx-cmds] (reduce - (partial verify-tx-cmd db-before t) + (partial verify-tx-cmd search-param-registry db-before t) {:entries [] :new-resources #{} :del-resources #{}} @@ -346,11 +365,11 @@ cmds)) -(defn- verify-tx-cmds* [db-before t cmds] +(defn- verify-tx-cmds* [search-param-registry db-before t cmds] (ba/try-anomaly (let [cmds (resolve-ids db-before cmds)] (detect-duplicate-commands! cmds) - (let [res (verify-tx-cmds** db-before t cmds)] + (let [res (verify-tx-cmds** search-param-registry db-before t cmds)] (check-referential-integrity! db-before res cmds) (post-process-res db-before t res))))) @@ -360,7 +379,7 @@ outcome if it is successful or an anomaly if it fails. The `t` is for the new transaction to commit." - [db-before t cmds] + [search-param-registry db-before t cmds] (with-open [_ (prom/timer duration-seconds "verify-tx-cmds") batch-db-before (d/new-batch-db db-before)] - (verify-tx-cmds* batch-db-before t cmds))) + (verify-tx-cmds* search-param-registry batch-db-before t cmds))) diff --git a/modules/db/src/blaze/db/node/version.clj b/modules/db/src/blaze/db/node/version.clj index 001532381..93c7d9d32 100644 --- a/modules/db/src/blaze/db/node/version.clj +++ b/modules/db/src/blaze/db/node/version.clj @@ -1,8 +1,7 @@ (ns blaze.db.node.version (:refer-clojure :exclude [key]) - (:require - [blaze.byte-buffer :as bb]) (:import + [com.google.common.primitives Longs] [java.nio.charset StandardCharsets])) @@ -14,10 +13,8 @@ (defn encode-value [version] - (-> (bb/allocate Integer/BYTES) - (bb/put-int! version) - (bb/array))) + (Longs/toByteArray version)) (defn decode-value [bytes] - (bb/get-int! (bb/wrap bytes))) + (Longs/fromByteArray bytes)) diff --git a/modules/db/src/blaze/db/resource_cache.clj b/modules/db/src/blaze/db/resource_cache.clj index e9e2c1c6b..d6a580816 100644 --- a/modules/db/src/blaze/db/resource_cache.clj +++ b/modules/db/src/blaze/db/resource_cache.clj @@ -4,7 +4,7 @@ Caffeine is used because it have better performance characteristics as a ConcurrentHashMap." (:require - [blaze.db.cache-collector.protocols :as ccp] + [blaze.cache-collector.protocols :as ccp] [blaze.db.resource-cache.spec] [blaze.db.resource-store :as rs] [blaze.db.resource-store.spec] diff --git a/modules/db/src/blaze/db/search_param_registry.clj b/modules/db/src/blaze/db/search_param_registry.clj index 9bdebac11..a976ede0b 100644 --- a/modules/db/src/blaze/db/search_param_registry.clj +++ b/modules/db/src/blaze/db/search_param_registry.clj @@ -47,16 +47,21 @@ (defn compartment-resources - "Returns a seq of [type code] tuples of resources in compartment of `type`. + "Returns a seq of [type code] tuples of resources in compartment of + `compartment-type` or a list of codes if the optional `type` is given. Example: - * [\"Observation\" \"subject\"] and others for \"Patient\"" - [search-param-registry type] - (p/-compartment-resources search-param-registry type)) + * [\"Observation\" \"subject\"] and others for \"Patient\" + * [\"subject\"] and others for \"Patient\" and \"Observation\"" + ([search-param-registry compartment-type] + (p/-compartment-resources search-param-registry compartment-type)) + ([search-param-registry compartment-type type] + (p/-compartment-resources search-param-registry compartment-type type))) (deftype MemSearchParamRegistry [index target-index compartment-index - compartment-resource-index] + compartment-resource-index + compartment-resource-index-by-type] p/SearchParamRegistry (-get [_ code] (get-in index ["Resource" code])) @@ -87,8 +92,11 @@ #{} (compartment-index (name (fhir-spec/fhir-type resource))))) - (-compartment-resources [_ type] - (compartment-resource-index type []))) + (-compartment-resources [_ compartment-type] + (compartment-resource-index compartment-type [])) + + (-compartment-resources [_ compartment-type type] + (get-in compartment-resource-index-by-type [compartment-type type] []))) (def ^:private object-mapper @@ -161,6 +169,15 @@ resource-defs)}) +(defn- index-compartment-resources-by-type [{def-code :code resource-defs :resource}] + {def-code + (reduce + (fn [res {res-type :code param-codes :param}] + (cond-> res param-codes (assoc res-type param-codes))) + {} + resource-defs)}) + + (def ^:private list-search-param {:type "special" :name "_list"}) @@ -258,7 +275,9 @@ patient-compartment (read-classpath-json-resource "blaze/db/compartment/patient.json")] (when-ok [url-index (build-url-index entries) index (build-index url-index entries)] - (->MemSearchParamRegistry (add-special index) - (build-target-index url-index entries) - (index-compartment-def index patient-compartment) - (index-compartment-resources patient-compartment))))) + (->MemSearchParamRegistry + (add-special index) + (build-target-index url-index entries) + (index-compartment-def index patient-compartment) + (index-compartment-resources patient-compartment) + (index-compartment-resources-by-type patient-compartment))))) diff --git a/modules/db/src/blaze/db/search_param_registry_spec.clj b/modules/db/src/blaze/db/search_param_registry_spec.clj index 859f19a2f..553a5cf8b 100644 --- a/modules/db/src/blaze/db/search_param_registry_spec.clj +++ b/modules/db/src/blaze/db/search_param_registry_spec.clj @@ -34,5 +34,6 @@ (s/fdef sr/compartment-resources :args (s/cat :search-param-registry :blaze.db/search-param-registry - :type :fhir.resource/type) + :compartment-type :fhir.resource/type + :type (s/? :fhir.resource/type)) :ret (s/coll-of (s/tuple :fhir.resource/type string?))) diff --git a/modules/db/test-perf/blaze/db/api_test_perf.clj b/modules/db/test-perf/blaze/db/api_test_perf.clj index 7bed864b9..dd0f4fd63 100644 --- a/modules/db/test-perf/blaze/db/api_test_perf.clj +++ b/modules/db/test-perf/blaze/db/api_test_perf.clj @@ -60,6 +60,7 @@ :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil + :patient-as-of-index nil :type-stats-index nil :system-stats-index nil}} diff --git a/modules/db/test/blaze/db/api_test.clj b/modules/db/test/blaze/db/api_test.clj index ef95872ea..cca7cf046 100644 --- a/modules/db/test/blaze/db/api_test.clj +++ b/modules/db/test/blaze/db/api_test.clj @@ -4738,6 +4738,42 @@ [0 :id] := "0"))))) +(deftest patient-compartment-last-change-t-test + (testing "non-existing patient" + (with-system [{:blaze.db/keys [node]} config] + + (testing "just returns nil" + (is (nil? (d/patient-compartment-last-change-t (d/db node) "0")))))) + + (testing "single patient" + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put {:fhir/type :fhir/Patient :id "0"}]]] + + (testing "has no resources in its compartment" + (is (nil? (d/patient-compartment-last-change-t (d/db node) "0")))))) + + + (testing "observation created in same transaction as patient" + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put {:fhir/type :fhir/Patient :id "0"}] + [:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}]]] + + (is (= 1 (d/patient-compartment-last-change-t (d/db node) "0"))))) + + (testing "observation created after the patient" + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put {:fhir/type :fhir/Patient :id "0"}]] + [[:put {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}]]] + + (testing "the last change comes from the second transaction" + (is (= 2 (d/patient-compartment-last-change-t (d/db node) "0")))) + + (testing "at t=1 there was no change" + (is (nil? (d/patient-compartment-last-change-t (d/as-of (d/db node) 1) "0"))))))) + + (defmethod ig/init-key ::defective-resource-store [_ {:keys [hashes-to-store]}] (let [store (atom {})] (reify diff --git a/modules/db/test/blaze/db/impl/batch_db_spec.clj b/modules/db/test/blaze/db/impl/batch_db_spec.clj index dfef7c8a6..08100df1c 100644 --- a/modules/db/test/blaze/db/impl/batch_db_spec.clj +++ b/modules/db/test/blaze/db/impl/batch_db_spec.clj @@ -3,6 +3,7 @@ [blaze.byte-string-spec] [blaze.db.impl.batch-db :as batch-db] [blaze.db.impl.index.compartment.resource-spec] + [blaze.db.impl.index.patient-as-of-spec] [blaze.db.impl.index.resource-as-of-spec] [blaze.db.impl.index.system-as-of-spec] [blaze.db.impl.index.type-as-of-spec] diff --git a/modules/db/test/blaze/db/impl/db_spec.clj b/modules/db/test/blaze/db/impl/db_spec.clj index 1efc82cd5..a4e5577c3 100644 --- a/modules/db/test/blaze/db/impl/db_spec.clj +++ b/modules/db/test/blaze/db/impl/db_spec.clj @@ -5,6 +5,7 @@ [blaze.db.impl.codec-spec] [blaze.db.impl.db :as db] [blaze.db.impl.index-spec] + [blaze.db.impl.index.patient-as-of-spec] [blaze.db.impl.index.system-stats-spec] [blaze.db.impl.index.type-stats-spec] [blaze.db.impl.search-param-spec] diff --git a/modules/db/test/blaze/db/impl/index/patient_as_of_spec.clj b/modules/db/test/blaze/db/impl/index/patient_as_of_spec.clj new file mode 100644 index 000000000..64c10ffe1 --- /dev/null +++ b/modules/db/test/blaze/db/impl/index/patient_as_of_spec.clj @@ -0,0 +1,26 @@ +(ns blaze.db.impl.index.patient-as-of-spec + (:require + [blaze.db.impl.codec-spec] + [blaze.db.impl.index.patient-as-of :as pao] + [blaze.db.kv-spec] + [blaze.db.tx-log.spec] + [blaze.fhir.hash.spec] + [clojure.spec.alpha :as s])) + + +(s/fdef pao/index-entry + :args (s/cat :patient-id :blaze.db/id-byte-string + :tid :blaze.db/tid + :id :blaze.db/id-byte-string + :t :blaze.db/t + :hash :blaze.resource/hash + :num-changes nat-int? + :op keyword?) + :ret :blaze.db.kv/put-entry-w-cf) + + +(s/fdef pao/last-change-t + :args (s/cat :paoi :blaze.db/kv-iterator + :patient-id :blaze.db/id-byte-string + :t :blaze.db/t) + :ret (s/nilable :blaze.db/t)) diff --git a/modules/db/test/blaze/db/impl/index/patient_as_of_test_util.clj b/modules/db/test/blaze/db/impl/index/patient_as_of_test_util.clj new file mode 100644 index 000000000..cc35f6710 --- /dev/null +++ b/modules/db/test/blaze/db/impl/index/patient_as_of_test_util.clj @@ -0,0 +1,16 @@ +(ns blaze.db.impl.index.patient-as-of-test-util + (:require + [blaze.byte-buffer :as bb] + [blaze.byte-string :as bs] + [blaze.db.impl.codec :as codec])) + + +(set! *unchecked-math* :warn-on-boxed) + + +(defn decode-key [patient-id-len byte-array] + (let [buf (bb/wrap byte-array)] + {:patient-id (codec/id-string (bs/from-byte-buffer! buf patient-id-len)) + :t (codec/descending-long (bb/get-long! buf)) + :type (codec/tid->type (bb/get-int! buf)) + :id (codec/id-string (bs/from-byte-buffer! buf (bb/remaining buf)))})) diff --git a/modules/db/test/blaze/db/node/patient_as_of_index_spec.clj b/modules/db/test/blaze/db/node/patient_as_of_index_spec.clj new file mode 100644 index 000000000..9b3595a1f --- /dev/null +++ b/modules/db/test/blaze/db/node/patient_as_of_index_spec.clj @@ -0,0 +1,15 @@ +(ns blaze.db.node.patient-as-of-index-spec + (:require + [blaze.db.kv.spec] + [blaze.db.node.patient-as-of-index :as node-pao] + [blaze.db.node.tx-indexer-spec] + [blaze.db.spec] + [blaze.db.tx-log.spec] + [clojure.spec.alpha :as s] + [cognitect.anomalies :as anom])) + + +(s/fdef node-pao/index-entries + :args (s/cat :node :blaze.db/node + :tx-data :blaze.db/tx-data) + :ret (s/or :entries (s/coll-of :blaze.db.kv/put-entry) :anomaly ::anom/anomaly)) diff --git a/modules/db/test/blaze/db/node/patient_as_of_index_test.clj b/modules/db/test/blaze/db/node/patient_as_of_index_test.clj new file mode 100644 index 000000000..c39a8ab67 --- /dev/null +++ b/modules/db/test/blaze/db/node/patient_as_of_index_test.clj @@ -0,0 +1,49 @@ +(ns blaze.db.node.patient-as-of-index-test + (:require + [blaze.db.impl.index.patient-as-of :as pao] + [blaze.db.impl.index.patient-as-of-test-util :as pao-tu] + [blaze.db.impl.index.rts-as-of-test-util :as rts-tu] + [blaze.db.node.patient-as-of-index :as node-pao] + [blaze.db.node.patient-as-of-index-spec] + [blaze.db.test-util :refer [config with-system-data]] + [blaze.fhir.hash :as hash] + [blaze.test-util :as tu] + [clojure.spec.test.alpha :as st] + [clojure.test :as test :refer [deftest]] + [juxt.iota :refer [given]] + [taoensso.timbre :as log]) + (:import + [java.nio.charset StandardCharsets] + [java.time Instant])) + + +(st/instrument) +(log/set-level! :trace) + + +(test/use-fixtures :each tu/fixture) + + +(def patient-0 {:fhir/type :fhir/Patient :id "0"}) +(def observation-0 {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}) +(def hash-observation-0 (hash/generate observation-0)) + + +(deftest patient-as-of-index-entries-test + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put patient-0]]] + + (given (node-pao/index-entries + node + {:t 2 + :instant Instant/EPOCH + :tx-cmds [{:op "put" :type "Observation" :id "0" + :hash hash-observation-0 :refs [["Patient" "0"]]}]}) + count := 2 + [0 0] := :patient-as-of-index + [0 1 (partial pao-tu/decode-key 1)] := {:patient-id "0" :t 2 :type "Observation" :id "0"} + [0 2 rts-tu/decode-val] := {:hash hash-observation-0 :num-changes 1 :op :put} + + [1 0 #(String. ^bytes % StandardCharsets/ISO_8859_1)] := "patient-as-of-state" + [1 1 pao/decode-state] := {:type :building :t 2}))) diff --git a/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj b/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj index 6e1bc81f4..78d026428 100644 --- a/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj +++ b/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj @@ -8,6 +8,7 @@ [blaze.db.impl.index.type-stats-spec] [blaze.db.kv.spec] [blaze.db.node.tx-indexer.verify :as verify] + [blaze.db.search-param-registry.spec] [blaze.db.spec] [blaze.db.tx-log.spec] [clojure.spec.alpha :as s] @@ -15,6 +16,9 @@ (s/fdef verify/verify-tx-cmds - :args (s/cat :db-before :blaze.db/db :t :blaze.db/t :cmds :blaze.db/tx-cmds) + :args (s/cat :search-param-registry :blaze.db/search-param-registry + :db-before :blaze.db/db + :t :blaze.db/t + :cmds :blaze.db/tx-cmds) :ret (s/or :entries (s/coll-of :blaze.db.kv/put-entry) :anomaly ::anom/anomaly)) diff --git a/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj b/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj index 7bb1bc8a8..4e89e05d7 100644 --- a/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj +++ b/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj @@ -2,6 +2,7 @@ (:require [blaze.db.api :as d] [blaze.db.impl.codec :as codec] + [blaze.db.impl.index.patient-as-of-test-util :as pao-tu] [blaze.db.impl.index.resource-as-of-test-util :as rao-tu] [blaze.db.impl.index.rts-as-of-test-util :as rts-tu] [blaze.db.impl.index.system-as-of-test-util :as sao-tu] @@ -14,7 +15,7 @@ [blaze.db.node.tx-indexer.verify :as verify] [blaze.db.node.tx-indexer.verify-spec] [blaze.db.search-param-registry] - [blaze.db.test-util :refer [config with-system-data]] + [blaze.db.test-util :refer [config search-param-registry with-system-data]] [blaze.db.tx-cache] [blaze.db.tx-log.local] [blaze.fhir.hash :as hash] @@ -44,6 +45,10 @@ (def patient-1 {:fhir/type :fhir/Patient :id "1"}) (def patient-2 {:fhir/type :fhir/Patient :id "2" :identifier [#fhir/Identifier{:value "120426"}]}) +(def observation-0 {:fhir/type :fhir/Observation :id "0" + :subject #fhir/Reference{:reference "Patient/0"}}) +(def allergy-intolerance-0 {:fhir/type :fhir/AllergyIntolerance :id "0" + :patient #fhir/Reference{:reference "Patient/0"}}) (deftest verify-tx-cmds-test @@ -53,6 +58,7 @@ if-none-match [nil "*"]] (with-system [{:blaze.db/keys [node]} config] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 1 [(cond-> {:op (name op) :type "Patient" :id "0" :hash hash} if-none-match @@ -80,13 +86,55 @@ [4 1 ss-tu/decode-key] := {:t 1} [4 2 ss-tu/decode-val] := {:total 1 :num-changes 1}))))) - (testing "adding a second version of a patient to a store containing it already" + (testing "adding one observation to a store containing its Patient already" + (let [hash (hash/generate observation-0)] + (doseq [op [:create :put] + if-none-match [nil "*"]] + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put patient-0]]] + + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 2 + [(cond-> {:op (name op) :type "Observation" :id "0" + :hash hash :refs [["Patient" "0"]]} + if-none-match + (assoc :if-none-match if-none-match))]) + + count := 6 + + [0 0] := :resource-as-of-index + [0 1 rao-tu/decode-key] := {:type "Observation" :id "0" :t 2} + [0 2 rts-tu/decode-val] := {:hash hash :num-changes 1 :op op} + + [1 0] := :type-as-of-index + [1 1 tao-tu/decode-key] := {:type "Observation" :t 2 :id "0"} + [1 2 rts-tu/decode-val] := {:hash hash :num-changes 1 :op op} + + [2 0] := :system-as-of-index + [2 1 sao-tu/decode-key] := {:t 2 :type "Observation" :id "0"} + [2 2 rts-tu/decode-val] := {:hash hash :num-changes 1 :op op} + + [3 0] := :patient-as-of-index + [3 1 (partial pao-tu/decode-key 1)] := {:patient-id "0" :t 2 :type "Observation" :id "0"} + [3 2 rts-tu/decode-val] := {:hash hash :num-changes 1 :op op} + + [4 0] := :type-stats-index + [4 1 ts-tu/decode-key] := {:type "Observation" :t 2} + [4 2 ts-tu/decode-val] := {:total 1 :num-changes 1} + + [5 0] := :system-stats-index + [5 1 ss-tu/decode-key] := {:t 2} + [5 2 ss-tu/decode-val] := {:total 2 :num-changes 2}))))) + + (testing "adding a second version of a Patient to a store containing it already" (let [hash (hash/generate patient-0-v2)] (doseq [if-match [nil 1 [1] [1 2]]] (with-system-data [{:blaze.db/keys [node]} config] [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [(cond-> {:op "put" :type "Patient" :id "0" :hash hash} if-match @@ -122,6 +170,7 @@ [[:delete "Patient" "0"]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 3 [(cond-> {:op "put" :type "Patient" :id "0" :hash hash} if-match @@ -154,6 +203,7 @@ [[[:put patient-0]]] (is (empty? (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0)}]))))) @@ -162,7 +212,10 @@ (with-system [{:blaze.db/keys [node]} config] (let [tx-cmd {:op "keep" :type "Patient" :id "0" :hash (hash/generate patient-0)}] - (given (verify/verify-tx-cmds (d/db node) 1 [tx-cmd]) + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 1 + [tx-cmd]) ::anom/category := ::anom/conflict ::anom/message := "Keep failed on `Patient/0`." :blaze.db/tx-cmd := tx-cmd)))) @@ -173,7 +226,10 @@ [[:put patient-0-v2]]] (let [tx-cmd {:op "keep" :type "Patient" :id "0" :hash (hash/generate patient-0)}] - (given (verify/verify-tx-cmds (d/db node) 1 [tx-cmd]) + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 1 + [tx-cmd]) ::anom/category := ::anom/conflict ::anom/message := "Keep failed on `Patient/0`." :blaze.db/tx-cmd := tx-cmd)))) @@ -189,6 +245,7 @@ :hash (hash/generate patient-0-v2) :if-match if-match}] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 1 [tx-cmd]) ::anom/category := ::anom/conflict @@ -196,7 +253,7 @@ :http/status := 412 :blaze.db/tx-cmd := tx-cmd)))))) - (testing "keeping a non-matching hash and non-matching if-match patient fails" + (testing "keeping a non-matching hash and non-matching if-match Patient fails" (with-system-data [{:blaze.db/keys [node]} config] [[[:put patient-0]] [[:put patient-0-v2]]] @@ -206,7 +263,10 @@ (let [tx-cmd {:op "keep" :type "Patient" :id "0" :hash (hash/generate patient-0) :if-match if-match}] - (given (verify/verify-tx-cmds (d/db node) 1 [tx-cmd]) + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 1 + [tx-cmd]) ::anom/category := ::anom/conflict ::anom/message := "Precondition `W/\"3\"` failed on `Patient/0`." :http/status := 412 @@ -219,6 +279,7 @@ (testing "with different if-matches" (doseq [if-match [nil 1 [1] [1 2]]] (is (empty? (verify/verify-tx-cmds + search-param-registry (d/db node) 1 [(cond-> {:op "keep" :type "Patient" :id "0" @@ -229,6 +290,7 @@ (testing "deleting a Patient from an empty store" (with-system [{:blaze.db/keys [node]} config] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 1 [{:op "delete" :type "Patient" :id "0"}]) @@ -259,6 +321,7 @@ [[[:delete "Patient" "0"]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "delete" :type "Patient" :id "0"}]) @@ -289,6 +352,7 @@ [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "delete" :type "Patient" :id "0"}]) @@ -314,12 +378,85 @@ [4 1 ss-tu/decode-key] := {:t 2} [4 2 ss-tu/decode-val] := {:total 0 :num-changes 2}))) - (testing "adding a second patient to a store containing already one" + (testing "deleting an existing Observation" + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put patient-0] + [:put observation-0]]] + + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 2 + [{:op "delete" :type "Observation" :id "0"}]) + + count := 6 + + [0 0] := :resource-as-of-index + [0 1 rao-tu/decode-key] := {:type "Observation" :id "0" :t 2} + [0 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [1 0] := :type-as-of-index + [1 1 tao-tu/decode-key] := {:type "Observation" :t 2 :id "0"} + [1 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [2 0] := :system-as-of-index + [2 1 sao-tu/decode-key] := {:t 2 :type "Observation" :id "0"} + [2 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [3 0] := :patient-as-of-index + [3 1 (partial pao-tu/decode-key 1)] := {:patient-id "0" :t 2 :type "Observation" :id "0"} + [3 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [4 0] := :type-stats-index + [4 1 ts-tu/decode-key] := {:type "Observation" :t 2} + [4 2 ts-tu/decode-val] := {:total 0 :num-changes 2} + + [5 0] := :system-stats-index + [5 1 ss-tu/decode-key] := {:t 2} + [5 2 ss-tu/decode-val] := {:total 1 :num-changes 3}))) + + (testing "deleting an existing AllergyIntolerance" + (with-system-data [{:blaze.db/keys [node]} config] + [[[:put patient-0] + [:put allergy-intolerance-0]]] + + (given (verify/verify-tx-cmds + search-param-registry + (d/db node) 2 + [{:op "delete" :type "AllergyIntolerance" :id "0"}]) + + count := 6 + + [0 0] := :resource-as-of-index + [0 1 rao-tu/decode-key] := {:type "AllergyIntolerance" :id "0" :t 2} + [0 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [1 0] := :type-as-of-index + [1 1 tao-tu/decode-key] := {:type "AllergyIntolerance" :t 2 :id "0"} + [1 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [2 0] := :system-as-of-index + [2 1 sao-tu/decode-key] := {:t 2 :type "AllergyIntolerance" :id "0"} + [2 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [3 0] := :patient-as-of-index + [3 1 (partial pao-tu/decode-key 1)] := {:patient-id "0" :t 2 :type "AllergyIntolerance" :id "0"} + [3 2 rts-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete} + + [4 0] := :type-stats-index + [4 1 ts-tu/decode-key] := {:type "AllergyIntolerance" :t 2} + [4 2 ts-tu/decode-val] := {:total 0 :num-changes 2} + + [5 0] := :system-stats-index + [5 1 ss-tu/decode-key] := {:t 2} + [5 2 ss-tu/decode-val] := {:total 1 :num-changes 3}))) + + (testing "adding a second Patient to a store containing already one" (let [hash (hash/generate patient-1)] (with-system-data [{:blaze.db/keys [node]} config] [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "put" :type "Patient" :id "1" :hash hash}]) @@ -351,6 +488,7 @@ [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0) @@ -364,6 +502,7 @@ [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0) @@ -377,6 +516,7 @@ [[[:put patient-0]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0) @@ -394,6 +534,7 @@ :birthDate #fhir/date"2020"}]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "create" :type "Patient" :id "foo" :hash (hash/generate patient-0) @@ -409,6 +550,7 @@ (is (empty? (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "create" :type "Patient" :id "0" :hash (hash/generate patient-0) @@ -420,6 +562,7 @@ (given (verify/verify-tx-cmds + search-param-registry (d/db node) 2 [{:op "delete" :type "Patient" :id "2"} {:op "create" :type "Patient" :id "0" @@ -435,6 +578,7 @@ [[:delete "Patient" "0"]]] (given (verify/verify-tx-cmds + search-param-registry (d/db node) 3 [{:op "put" :type "Patient" :id "0" :hash hash}]) diff --git a/modules/db/test/blaze/db/node/tx_indexer_spec.clj b/modules/db/test/blaze/db/node/tx_indexer_spec.clj index 923a95260..f68083076 100644 --- a/modules/db/test/blaze/db/node/tx_indexer_spec.clj +++ b/modules/db/test/blaze/db/node/tx_indexer_spec.clj @@ -3,6 +3,7 @@ [blaze.db.kv.spec] [blaze.db.node.tx-indexer :as tx-indexer] [blaze.db.node.tx-indexer.verify-spec] + [blaze.db.search-param-registry.spec] [blaze.db.spec] [blaze.db.tx-log.spec] [clojure.spec.alpha :as s] @@ -10,5 +11,7 @@ (s/fdef tx-indexer/index-tx - :args (s/cat :db-before :blaze.db/db :tx-data :blaze.db/tx-data) + :args (s/cat :search-param-registry :blaze.db/search-param-registry + :db-before :blaze.db/db + :tx-data :blaze.db/tx-data) :ret (s/or :entries (s/coll-of :blaze.db.kv/put-entry) :anomaly ::anom/anomaly)) diff --git a/modules/db/test/blaze/db/node_test.clj b/modules/db/test/blaze/db/node_test.clj index 0ff5da515..99fc4b702 100644 --- a/modules/db/test/blaze/db/node_test.clj +++ b/modules/db/test/blaze/db/node_test.clj @@ -6,11 +6,13 @@ [blaze.db.api :as d] [blaze.db.api-spec] [blaze.db.impl.db-spec] + [blaze.db.impl.index.patient-as-of :as pao] [blaze.db.impl.index.tx-success :as tx-success] [blaze.db.kv :as kv] [blaze.db.kv.mem-spec] [blaze.db.node :as node] [blaze.db.node-spec] + [blaze.db.node.patient-as-of-index-spec] [blaze.db.node.resource-indexer :as resource-indexer] [blaze.db.node.tx-indexer :as-alias tx-indexer] [blaze.db.node.version :as version] @@ -281,3 +283,13 @@ (deftest existing-data-with-compatible-version (with-system [{:blaze.db/keys [node]} (with-index-store-version config 0)] (is node))) + + +(deftest patient-as-of-index-state-test + (testing "the state is set to current on a fresh start of the node" + (with-system [{:blaze.db/keys [node]} config] + ;; Wait for index building finished + (Thread/sleep 100) + + (given (pao/state (:kv-store node)) + :type := :current)))) diff --git a/modules/db/test/blaze/db/resource_cache_test.clj b/modules/db/test/blaze/db/resource_cache_test.clj index 821e9da6e..08a868aab 100644 --- a/modules/db/test/blaze/db/resource_cache_test.clj +++ b/modules/db/test/blaze/db/resource_cache_test.clj @@ -1,6 +1,6 @@ (ns blaze.db.resource-cache-test (:require - [blaze.db.cache-collector.protocols :as ccp] + [blaze.cache-collector.protocols :as ccp] [blaze.db.kv :as kv] [blaze.db.kv.mem] [blaze.db.resource-cache :as resource-cache] diff --git a/modules/db/test/blaze/db/search_param_registry_test.clj b/modules/db/test/blaze/db/search_param_registry_test.clj index 59dfa68b6..fb8bc8031 100644 --- a/modules/db/test/blaze/db/search_param_registry_test.clj +++ b/modules/db/test/blaze/db/search_param_registry_test.clj @@ -194,12 +194,17 @@ (deftest compartment-resources-test (testing "Patient" (with-system [{:blaze.db/keys [search-param-registry]} config] - (given (sr/compartment-resources search-param-registry "Patient") - count := 100 - [0] := ["Account" "subject"] - [1] := ["AdverseEvent" "subject"] - [2] := ["AllergyIntolerance" "patient"] - [3] := ["AllergyIntolerance" "recorder"] - [4] := ["AllergyIntolerance" "asserter"] - [5] := ["Appointment" "actor"] - [99] := ["VisionPrescription" "patient"])))) + (testing "all resource types" + (given (sr/compartment-resources search-param-registry "Patient") + count := 100 + [0] := ["Account" "subject"] + [1] := ["AdverseEvent" "subject"] + [2] := ["AllergyIntolerance" "patient"] + [3] := ["AllergyIntolerance" "recorder"] + [4] := ["AllergyIntolerance" "asserter"] + [5] := ["Appointment" "actor"] + [99] := ["VisionPrescription" "patient"])) + + (testing "only Observation codes" + (is (= (sr/compartment-resources search-param-registry "Patient" "Observation") + ["subject" "performer"])))))) diff --git a/modules/db/test/blaze/db/test_util.clj b/modules/db/test/blaze/db/test_util.clj index ea7ca5516..38f383087 100644 --- a/modules/db/test/blaze/db/test_util.clj +++ b/modules/db/test/blaze/db/test_util.clj @@ -30,6 +30,7 @@ :kv-store (ig/ref :blaze.db/index-kv-store) :resource-indexer (ig/ref :blaze.db.node/resource-indexer) :search-param-registry search-param-registry + :scheduler (ig/ref :blaze/scheduler) :poll-timeout (time/millis 10)} ::tx-log/local @@ -60,6 +61,7 @@ :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil + :patient-as-of-index nil :type-stats-index nil :system-stats-index nil}} @@ -78,7 +80,9 @@ :search-param-registry search-param-registry :executor (ig/ref :blaze.db.node.resource-indexer/executor)} - :blaze.db.node.resource-indexer/executor {}}) + :blaze.db.node.resource-indexer/executor {} + + :blaze/scheduler {}}) (defmacro with-system-data diff --git a/modules/executor/src/blaze/executors.clj b/modules/executor/src/blaze/executors.clj index f8c4695f9..63c943834 100644 --- a/modules/executor/src/blaze/executors.clj +++ b/modules/executor/src/blaze/executors.clj @@ -50,6 +50,12 @@ (format name-template (swap! thread-counter inc))) +(defn- thread-factory [counter name-template] + (reify ThreadFactory + (newThread [_ r] + (Thread. ^Runnable r ^String (thread-name! counter name-template))))) + + (defn cpu-bound-pool "Returns a thread pool with a fixed number of threads which is the number of available processors." @@ -57,10 +63,7 @@ (let [thread-counter (atom 0)] (Executors/newFixedThreadPool (.availableProcessors (Runtime/getRuntime)) - (reify ThreadFactory - (newThread [_ r] - (Thread. ^Runnable r ^String (thread-name! thread-counter - name-template))))))) + (thread-factory thread-counter name-template)))) (defn io-pool @@ -70,10 +73,14 @@ (let [thread-counter (atom 0)] (Executors/newFixedThreadPool n - (reify ThreadFactory - (newThread [_ r] - (Thread. ^Runnable r ^String (thread-name! thread-counter - name-template))))))) + (thread-factory thread-counter name-template)))) + + +(defn scheduled-pool [n name-template] + (let [thread-counter (atom 0)] + (Executors/newScheduledThreadPool + n + (thread-factory thread-counter name-template)))) (defn single-thread-executor diff --git a/modules/operation-measure-evaluate-measure/.clj-kondo/config.edn b/modules/operation-measure-evaluate-measure/.clj-kondo/config.edn index 2acbb21c8..f493241e1 100644 --- a/modules/operation-measure-evaluate-measure/.clj-kondo/config.edn +++ b/modules/operation-measure-evaluate-measure/.clj-kondo/config.edn @@ -26,6 +26,7 @@ :consistent-alias {:aliases {blaze.db.api d + blaze.elm.compiler.expr-cache ec blaze.elm.compiler.external-data ed blaze.elm.expression expr cognitect.anomalies anom diff --git a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure.clj b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure.clj index f10a60281..2e4256aa5 100644 --- a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure.clj +++ b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure.clj @@ -5,6 +5,10 @@ [blaze.async.comp :as ac] [blaze.coll.core :as coll] [blaze.db.api :as d] + [blaze.elm.compiler.expr-cache :as ec] + [blaze.elm.compiler.macros] + [blaze.elm.expression :as-alias expr] + [blaze.elm.expression.spec] [blaze.executors :as ex] [blaze.fhir.operation.evaluate-measure.measure :as measure] [blaze.fhir.operation.evaluate-measure.measure.spec] @@ -24,6 +28,7 @@ [ring.util.response :as ring] [taoensso.timbre :as log]) (:import + [com.github.benmanes.caffeine.cache Caffeine Weigher] [java.util.concurrent TimeUnit])) @@ -130,6 +135,7 @@ (defmethod ig/pre-init-spec ::handler [_] (s/keys :req-un [:blaze.db/node ::executor :blaze/clock :blaze/rng-fn] + :opt [::expr/cache] :opt-un [::timeout])) @@ -138,6 +144,23 @@ (wrap-coerce-params (handler context))) +(defmethod ig/pre-init-spec ::expr-cache [_] + (s/keys :opt-un [::max-size])) + + +(defmethod ig/init-key ::expr-cache + [_ {:keys [max-size] :or {max-size 0}}] + (log/info "Create CQL expression cache with a size of" max-size "expressions") + (-> (Caffeine/newBuilder) + (.weigher + (reify Weigher + (weigh [_ _ cache] + (ec/mem-size cache)))) + (.maximumWeight max-size) + (.recordStats) + (.buildAsync))) + + (defmethod ig/pre-init-spec ::timeout [_] (s/keys :req-un [:blaze.fhir.operation.evaluate-measure.timeout/millis])) @@ -179,3 +202,23 @@ (reg-collector ::evaluate-duration-seconds measure/evaluate-duration-seconds) + + +(reg-collector ::bloom-filter-creation-duration-seconds + blaze.elm.compiler.macros/bloom-filter-creation-duration-seconds) + + +(reg-collector ::bloom-filter-useful-total + blaze.elm.compiler.macros/bloom-filter-useful-total) + + +(reg-collector ::bloom-filter-not-useful-total + blaze.elm.compiler.macros/bloom-filter-not-useful-total) + + +(reg-collector ::bloom-filter-false-positive-total + blaze.elm.compiler.macros/bloom-filter-false-positive-total) + + +(reg-collector ::bloom-filter-bytes + ec/bloom-filter-bytes) diff --git a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure.clj b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure.clj index 62cea1e41..c56c5db7a 100644 --- a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure.clj +++ b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/measure.clj @@ -4,8 +4,10 @@ [blaze.coll.core :as coll] [blaze.cql-translator :as cql-translator] [blaze.db.api :as d] + [blaze.elm.compiler :as c] [blaze.elm.compiler.external-data :as ed] [blaze.elm.compiler.library :as library] + [blaze.elm.expression :as-alias expr] [blaze.fhir.operation.evaluate-measure.measure.population :as population] [blaze.fhir.operation.evaluate-measure.measure.stratifier :as stratifier] [blaze.fhir.operation.evaluate-measure.measure.util :as u] @@ -83,10 +85,10 @@ using `node`. Returns an anomaly on errors." - [node library] + [node library opts] (when-ok [cql-code (extract-cql-code library) library (translate cql-code)] - (library/compile-library node library {}))) + (library/compile-library node library opts))) (defn- compile-library @@ -94,12 +96,12 @@ using `node`. Returns an anomaly on errors." - {:arglists '([node library])} - [node {:keys [id] :as library}] + {:arglists '([node library opts])} + [node {:keys [id] :as library} opts] (log/debug (format "Start compiling Library with ID `%s`..." id)) (let [timer (prom/timer compile-duration-seconds)] (try - (compile-library* node library) + (compile-library* node library opts) (finally (let [duration (prom/observe-duration! timer)] (log/debug @@ -135,10 +137,10 @@ compilation. Returns an anomaly on errors." - [db measure] + [db measure opts] (if-let [library-ref (-> measure :library first type/value)] (if-let [library (find-library db library-ref)] - (compile-library (d/node db) library) + (compile-library (d/node db) library opts) (ba/incorrect (format "The Library resource with canonical URI `%s` was not found." library-ref) :fhir/issue "value" @@ -335,13 +337,25 @@ (subject-handle* db subject-type subject-ref))) +(defn- attach-cache* [context [name expr]] + [name (update expr :expression (partial c/attach-cache context))]) + + +(defn- attach-cache [context expression-defs] + (into {} (map (partial attach-cache* context)) expression-defs)) + + (defn- enhance-context - [{:keys [clock db timeout] :as context :or {timeout (time/hours 1)}} measure + [{:keys [clock db timeout] + ::expr/keys [cache] + :or {timeout (time/hours 1)} + :as context} measure {:keys [report-type subject-ref]}] (let [subject-type (subject-type measure) now (now clock) timeout-instant (time/instant (time/plus now timeout))] - (when-ok [{:keys [expression-defs function-defs parameter-default-values]} (compile-primary-library db measure) + (when-ok [{:keys [expression-defs function-defs parameter-default-values]} + (compile-primary-library db measure {}) subject-handle (some->> subject-ref (subject-handle db subject-type))] (cond-> (assoc context @@ -349,7 +363,16 @@ :now now :timeout-eclipsed? #(not (.isBefore (.instant ^Clock clock) timeout-instant)) :timeout timeout - :expression-defs expression-defs + :expression-defs + (if cache + (attach-cache + (assoc context + :db db + :now now + :expression-defs expression-defs + :parameters parameter-default-values) + expression-defs) + expression-defs) :function-defs function-defs :parameters parameter-default-values :subject-type subject-type diff --git a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/spec.clj b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/spec.clj index d8f3c7a3f..09e536443 100644 --- a/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/spec.clj +++ b/modules/operation-measure-evaluate-measure/src/blaze/fhir/operation/evaluate_measure/spec.clj @@ -14,6 +14,10 @@ pos-int?) +(s/def ::measure/max-size + nat-int?) + + (s/def ::measure/timeout time/duration?) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql/spec.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql/spec.clj index 8f7561a60..3a459c190 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql/spec.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql/spec.clj @@ -1,15 +1,12 @@ (ns blaze.fhir.operation.evaluate-measure.cql.spec (:require [blaze.elm.compiler :as-alias compiler] + [blaze.elm.expression :as-alias expr] [blaze.fhir.operation.evaluate-measure.cql :as-alias cql] [clojure.spec.alpha :as s] [java-time.api :as time])) -(s/def ::cql/now - time/offset-date-time?) - - (s/def ::cql/timeout-eclipsed? ifn?) @@ -19,8 +16,10 @@ (s/def ::cql/context - (s/keys :req-un [:blaze.db/db ::cql/now ::cql/timeout-eclipsed? ::cql/timeout - ::compiler/expression-defs])) + (s/merge + ::expr/context + (s/keys :req-un [::cql/timeout-eclipsed? ::cql/timeout + ::compiler/expression-defs]))) (s/def ::cql/return-handles? @@ -31,9 +30,5 @@ (s/merge ::cql/context (s/keys :opt-un [::cql/return-handles?]))) -(s/def ::cql/parameters - (s/map-of string? any?)) - - (s/def ::cql/evaluate-individual-expression-context - (s/merge ::cql/context (s/keys :opt-un [::cql/parameters]))) + ::cql/context) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_test.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_test.clj index feba245fb..ac1e588b0 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_test.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/cql_test.clj @@ -21,6 +21,7 @@ [juxt.iota :refer [given]] [taoensso.timbre :as log]) (:import + [com.github.benmanes.caffeine.cache Caffeine] [java.time Clock OffsetDateTime])) @@ -104,6 +105,7 @@ (let [{:keys [expression-defs function-defs]} (compile-library node library)] {:db (d/db node) :now (now fixed-clock) + ::expr/cache (.buildAsync (Caffeine/newBuilder)) :timeout-eclipsed? (constantly false) :timeout (time/seconds 42) :expression-defs expression-defs diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/stratifier_test.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/stratifier_test.clj index bedf1dc60..4c8d5191c 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/stratifier_test.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure/stratifier_test.clj @@ -6,6 +6,7 @@ [blaze.db.api :as d] [blaze.db.api-stub :refer [mem-node-config with-system-data]] [blaze.elm.compiler.library :as library] + [blaze.elm.expression :as-alias expr] [blaze.fhir.operation.evaluate-measure.measure.stratifier :as stratifier] [blaze.fhir.operation.evaluate-measure.measure.stratifier-spec] [blaze.fhir.operation.evaluate-measure.test-util :as em-tu] diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_spec.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_spec.clj index f048a0485..4a3999e54 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_spec.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_spec.clj @@ -2,6 +2,8 @@ (:require [blaze.cql-translator-spec] [blaze.db.spec] + [blaze.elm.expression :as-alias expr] + [blaze.elm.expression.spec] [blaze.fhir.operation.evaluate-measure.cql-spec] [blaze.fhir.operation.evaluate-measure.measure :as measure] [blaze.fhir.operation.evaluate-measure.measure.spec] @@ -15,6 +17,12 @@ [java.time.temporal Temporal])) +(s/def ::context + (s/keys :req [:blaze/base-url ::reitit/router] + :opt [::expr/cache] + :req-un [:blaze/clock :blaze/rng-fn :blaze.db/db])) + + (defn- temporal? [x] (instance? Temporal x)) @@ -33,12 +41,6 @@ (s/fdef measure/evaluate-measure - :args - (s/cat - :context (s/keys :req [:blaze/base-url ::reitit/router] - :req-un [:blaze/clock :blaze/rng-fn :blaze.db/db]) - :measure :blaze/resource - :params ::params) - :ret - (s/or :result (s/keys :req-un [:blaze/resource] :opt-un [:blaze.db/tx-ops]) - :anomaly ::anom/anomaly)) + :args (s/cat :context ::context :measure :blaze/resource :params ::params) + :ret (s/or :result (s/keys :req-un [:blaze/resource] :opt-un [:blaze.db/tx-ops]) + :anomaly ::anom/anomaly)) diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_test.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_test.clj index b8e6490c5..b12e5c6e9 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_test.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure/measure_test.clj @@ -3,6 +3,7 @@ [blaze.anomaly :as ba] [blaze.db.api :as d] [blaze.db.api-stub :refer [mem-node-config with-system-data]] + [blaze.elm.expression :as-alias expr] [blaze.fhir.operation.evaluate-measure.measure :as measure] [blaze.fhir.operation.evaluate-measure.measure-spec] [blaze.fhir.operation.evaluate-measure.measure.population-spec] @@ -22,6 +23,7 @@ [reitit.core :as reitit] [taoensso.timbre :as log]) (:import + [com.github.benmanes.caffeine.cache Caffeine] [java.nio.charset StandardCharsets] [java.util Base64])) @@ -97,10 +99,11 @@ (let [db (d/db node) context {:clock fixed-clock :rng-fn fixed-rng-fn :db db + ::expr/cache (.buildAsync (Caffeine/newBuilder)) :blaze/base-url "" ::reitit/router router} + measure @(d/pull node (d/resource-handle db "Measure" "0")) period [#system/date"2000" #system/date"2020"]] - (measure/evaluate-measure context - @(d/pull node (d/resource-handle db "Measure" "0")) + (measure/evaluate-measure context measure {:period period :report-type report-type}))))) @@ -740,6 +743,7 @@ [1 :population 0 :count] := 2) (given (first-stratifier-strata (evaluate "q29-stratifier-sample-material-type")) + count := 2 [0 :value :text type/value] := "liquid" [0 :population 0 :count] := 1 [1 :value :text type/value] := "tissue" diff --git a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure_test.clj b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure_test.clj index 30b741b5b..b8fb5c152 100644 --- a/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure_test.clj +++ b/modules/operation-measure-evaluate-measure/test/blaze/fhir/operation/evaluate_measure_test.clj @@ -4,6 +4,7 @@ [blaze.async.comp :as ac] [blaze.db.api-stub :as api-stub :refer [with-system-data]] [blaze.db.resource-store :as rs] + [blaze.elm.expression :as-alias expr] [blaze.executors :as ex] [blaze.fhir.operation.evaluate-measure :as evaluate-measure] [blaze.fhir.operation.evaluate-measure.test-util :refer [wrap-error]] @@ -99,6 +100,26 @@ [:explain ::s/problems 3 :val] := ::invalid))) +(deftest expr-cache-init-test + (testing "nil config" + (given-thrown (ig/init {::evaluate-measure/expr-cache nil}) + :key := ::evaluate-measure/expr-cache + :reason := ::ig/build-failed-spec + [:explain ::s/problems 0 :pred] := `map?)) + + (testing "invalid millis" + (given-thrown (ig/init {::evaluate-measure/expr-cache {:max-size ::invalid}}) + :key := ::evaluate-measure/expr-cache + :reason := ::ig/build-failed-spec + [:explain ::s/problems 0 :pred] := `nat-int? + [:explain ::s/problems 0 :val] := ::invalid)) + + (testing "init" + (with-system [{::evaluate-measure/keys [expr-cache]} + {::evaluate-measure/expr-cache {:max-size 125509}}] + (is (s/valid? ::expr/cache expr-cache))))) + + (deftest timeout-init-test (testing "nil config" (given-thrown (ig/init {::evaluate-measure/timeout nil}) @@ -161,9 +182,11 @@ (assoc api-stub/mem-node-config ::evaluate-measure/handler {:node (ig/ref :blaze.db/node) + ::expr/cache (ig/ref ::evaluate-measure/expr-cache) :executor (ig/ref :blaze.test/executor) :clock (ig/ref :blaze.test/fixed-clock) :rng-fn (ig/ref :blaze.test/fixed-rng-fn)} + ::evaluate-measure/expr-cache {} :blaze.test/executor {} :blaze.test/fixed-rng-fn {})) diff --git a/modules/rest-api/src/blaze/rest_api/middleware/metrics.clj b/modules/rest-api/src/blaze/rest_api/middleware/metrics.clj index 5c4fa53fe..c592e88eb 100644 --- a/modules/rest-api/src/blaze/rest_api/middleware/metrics.clj +++ b/modules/rest-api/src/blaze/rest_api/middleware/metrics.clj @@ -2,13 +2,13 @@ (:require [blaze.util :as u] [clojure.string :as str] - [prometheus.alpha :as prom])) + [prometheus.alpha :as prom :refer [defcounter defhistogram]])) (set! *warn-on-reflection* true) -(prom/defcounter requests-total +(defcounter requests-total "Number of requests to this service. Distinguishes between the returned status code, the handler being used to process the request together with the http method." @@ -17,7 +17,7 @@ "code" "interaction" "method") -(prom/defhistogram request-duration-seconds +(defhistogram request-duration-seconds "The HTTP request latencies in seconds." {:namespace "http" :subsystem "fhir"} diff --git a/modules/scheduler/src/blaze/scheduler.clj b/modules/scheduler/src/blaze/scheduler.clj index 0a5aa8fae..7497856fd 100644 --- a/modules/scheduler/src/blaze/scheduler.clj +++ b/modules/scheduler/src/blaze/scheduler.clj @@ -7,13 +7,20 @@ [java-time.api :as time] [taoensso.timbre :as log]) (:import - [java.util.concurrent Executors Future ScheduledExecutorService TimeUnit])) + [java.util.concurrent Future ScheduledExecutorService TimeUnit])) (set! *warn-on-reflection* true) -(defn schedule-at-fixed-rate [scheduler f initial-delay period] +(defn submit [scheduler f] + (p/-submit scheduler f)) + + +(defn schedule-at-fixed-rate + "Schedules the function `f` to be called at a rate of `period` with an + `initial-delay`." + [scheduler f initial-delay period] (p/-schedule-at-fixed-rate scheduler f initial-delay period)) @@ -23,6 +30,9 @@ (extend-protocol p/Scheduler ScheduledExecutorService + (-submit [scheduler f] + (.submit scheduler ^Runnable f)) + (-schedule-at-fixed-rate [scheduler f initial-delay period] (.scheduleAtFixedRate scheduler @@ -35,7 +45,7 @@ (defmethod ig/init-key :blaze/scheduler [_ _] (log/info "Start scheduler") - (Executors/newSingleThreadScheduledExecutor)) + (ex/scheduled-pool 4 "scheduler-%d")) (defmethod ig/halt-key! :blaze/scheduler @@ -45,3 +55,6 @@ (if (ex/await-termination scheduler 10 TimeUnit/SECONDS) (log/info "Scheduler was stopped successfully") (log/warn "Got timeout while stopping the scheduler"))) + + +(derive :blaze/scheduler :blaze.metrics/thread-pool-executor) diff --git a/modules/scheduler/src/blaze/scheduler/protocol.clj b/modules/scheduler/src/blaze/scheduler/protocol.clj index 108dd768f..7a25c46ad 100644 --- a/modules/scheduler/src/blaze/scheduler/protocol.clj +++ b/modules/scheduler/src/blaze/scheduler/protocol.clj @@ -2,4 +2,5 @@ (defprotocol Scheduler + (-submit [scheduler f]) (-schedule-at-fixed-rate [scheduler f initial-delay period])) diff --git a/profiling/blaze/profiling.clj b/profiling/blaze/profiling.clj index 241b8305e..b39b1f3e8 100644 --- a/profiling/blaze/profiling.clj +++ b/profiling/blaze/profiling.clj @@ -1,10 +1,10 @@ (ns blaze.profiling "Profiling namespace without test dependencies." (:require - [blaze.system :as system] - [blaze.db.cache-collector :as cc] + [blaze.cache-collector.protocols :as ccp] [blaze.db.kv.rocksdb :as rocksdb] [blaze.db.resource-cache :as resource-cache] + [blaze.system :as system] [clojure.tools.namespace.repl :refer [refresh]] [taoensso.timbre :as log])) @@ -43,16 +43,21 @@ ;; Transaction Cache (comment - (str (cc/-stats (:blaze.db/tx-cache system))) + (str (ccp/-stats (:blaze.db/tx-cache system))) (resource-cache/invalidate-all! (:blaze.db/tx-cache system)) ) ;; Resource Cache (comment - (str (cc/-stats (:blaze.db/resource-cache system))) + (str (ccp/-stats (:blaze.db/resource-cache system))) (resource-cache/invalidate-all! (:blaze.db/resource-cache system)) ) +;; CQL Expression Cache +(comment + (str (ccp/-stats (:blaze.fhir.operation.evaluate-measure/expr-cache system))) + ) + ;; DB (comment (str (system [:blaze.db.kv.rocksdb/stats :blaze.db.index-kv-store/stats])) @@ -69,6 +74,7 @@ (rocksdb/get-property index-db :resource-as-of-index "rocksdb.stats") (rocksdb/get-property index-db :type-as-of-index "rocksdb.stats") (rocksdb/get-property index-db :system-as-of-index "rocksdb.stats") + (rocksdb/get-property index-db :patient-as-of-index "rocksdb.stats") (rocksdb/get-property index-db :type-stats-index "rocksdb.stats") (rocksdb/get-property index-db :system-stats-index "rocksdb.stats") diff --git a/resources/blaze.edn b/resources/blaze.edn index 112e895f9..bbc5ed48f 100644 --- a/resources/blaze.edn +++ b/resources/blaze.edn @@ -162,6 +162,11 @@ :blaze.fhir.operation.evaluate-measure/compile-duration-seconds {} :blaze.fhir.operation.evaluate-measure/evaluate-duration-seconds {} + :blaze.fhir.operation.evaluate-measure/bloom-filter-creation-duration-seconds {} + :blaze.fhir.operation.evaluate-measure/bloom-filter-useful-total {} + :blaze.fhir.operation.evaluate-measure/bloom-filter-not-useful-total {} + :blaze.fhir.operation.evaluate-measure/bloom-filter-false-positive-total {} + :blaze.fhir.operation.evaluate-measure/bloom-filter-bytes {} ;; ;; FHIR Operation GraphQL @@ -211,6 +216,7 @@ :kv-store #blaze/ref :blaze.db/index-kv-store :resource-indexer #blaze/ref :blaze.db.node/resource-indexer :search-param-registry #blaze/ref :blaze.db/search-param-registry + :scheduler #blaze/ref :blaze/scheduler :enforce-referential-integrity #blaze/cfg ["ENFORCE_REFERENTIAL_INTEGRITY" boolean? true]} :blaze.db.node/indexer-executor {} @@ -232,7 +238,7 @@ :blaze.db.node.tx-indexer/duration-seconds {} - :blaze.db/cache-collector + :blaze/cache-collector {:caches {"tx-cache" #blaze/ref :blaze.db/tx-cache "resource-cache" #blaze/ref :blaze.db/resource-cache}} @@ -274,7 +280,9 @@ ;; :blaze.db/search-param-registry {:structure-definition-repo #blaze/ref :blaze.fhir/structure-definition-repo - :extra-bundle-file #blaze/cfg ["DB_SEARCH_PARAM_BUNDLE" string?]}} + :extra-bundle-file #blaze/cfg ["DB_SEARCH_PARAM_BUNDLE" string?]} + + :blaze/scheduler {}} :storage {:in-memory @@ -299,6 +307,7 @@ :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil + :patient-as-of-index nil :type-stats-index nil :system-stats-index nil}} @@ -441,6 +450,12 @@ :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :patient-as-of-index + {:write-buffer-size-in-mb 8 + :max-bytes-for-level-base-in-mb 32 + :target-file-size-base-in-mb 8 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :type-stats-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 @@ -644,6 +659,12 @@ :target-file-size-base-in-mb 8 :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :patient-as-of-index + {:write-buffer-size-in-mb 8 + :max-bytes-for-level-base-in-mb 32 + :target-file-size-base-in-mb 8 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :type-stats-index {:write-buffer-size-in-mb 2 :max-bytes-for-level-base-in-mb 8 @@ -770,6 +791,17 @@ :scheduler #blaze/ref :blaze/scheduler :provider-url #blaze/cfg ["OPENID_PROVIDER_URL" string?]} - :blaze/http-client {} + :blaze/http-client {}}} + + {:name "CQL Expression Cache" + :toggle "CQL_EXPR_CACHE_SIZE" + :config + {:blaze.fhir.operation.evaluate-measure/handler + {:blaze.elm.expression/cache #blaze/ref :blaze.fhir.operation.evaluate-measure/expr-cache} + + :blaze.fhir.operation.evaluate-measure/expr-cache + {:max-size #blaze/cfg ["CQL_EXPR_CACHE_SIZE" nat-int?]} - :blaze/scheduler {}}}]} + :blaze/cache-collector + {:caches + {"cql-expr-cache" #blaze/ref :blaze.fhir.operation.evaluate-measure/expr-cache}}}}]} diff --git a/src/blaze/system.clj b/src/blaze/system.clj index 0205c8917..39450a766 100644 --- a/src/blaze/system.clj +++ b/src/blaze/system.clj @@ -169,7 +169,7 @@ (let [enabled? (feature-enabled? env feature)] (log/info "Feature" name (if enabled? "enabled" "disabled")) (if enabled? - (merge-with merge res config) + (merge-with (partial merge-with merge) res config) res))) base-config features))