merge reimplementation_end_2023

.Rbuildignore

@@ -14,8 +14,9 @@ man-roxygen
 ^revdep$
 ^LICENSE\.md$
 ^private$
-.DS_Store
-.lintr
+\.DS_Store
+\.lintr
 ^CRAN-RELEASE$
 ^CRAN-SUBMISSION$
 ^docs
+\.vscode

.github/workflows/check-standard.yml

@@ -30,46 +30,32 @@ jobs:
 
     steps:
 
-      # nodbi on linux:
+      # databases:
 
+      # https://github.com/ankane/setup-postgres#setup-postgres
+      # Works on Linux, Mac, and Windows
       - name: PostgreSQL
         uses: ankane/setup-postgres@v1
         with:
-          postgres-version: 14
           database: test
-
+
+      # https://github.com/ankane/setup-mongodb#setup-mongodb
+      # Works on Linux, Mac, and Windows
       - name: MongoDB
-        if: runner.os == 'Linux'
-        uses: supercharge/[email protected]
+        uses: ankane/setup-mongodb@v1
 
-      - name: Prepare ElasticSearch
-        if: runner.os == 'Linux'
-        run: |
-          sudo swapoff -a
-          sudo sysctl -w vm.swappiness=1
-          sudo sysctl -w fs.file-max=262144
-          sudo sysctl -w vm.max_map_count=262144
-
-      - name: Start ElasticSearch
-        if: runner.os == 'Linux'
-        uses: getong/[email protected]
-        with:
-          elasticsearch version: '7.6.1'
-          host port: 9200
-          container port: 9200
-          host node port: 9300
-          node port: 9300
-          discovery type: 'single-node'
+      # https://github.com/ankane/setup-elasticsearch#setup-elasticsearch
+      # Works on Linux, Mac, and Windows
+      - name: ElasticSearch
+        uses: ankane/setup-elasticsearch@v1
 
+      # https://github.com/iamssen/couchdb-github-action
       - name: CouchDB
         if: runner.os == 'Linux'
         uses: iamssen/couchdb-github-action@master
         with:
           couchdb-version: '3.1'
 
-      - name: Install development libraries
-        if: runner.os == 'Linux'
-        run: sudo apt-get install -y libjq-dev
 
       # generic workflow:
 
@@ -83,15 +69,17 @@ jobs:
           http-user-agent: ${{ matrix.config.http-user-agent }}
           use-public-rspm: true
 
-      - uses: r-lib/actions/setup-r-dependencies@v2
-        with:
-          cache-version: 2
-          extra-packages: rcmdcheck, covr, sofa, elastic, mongolite, DBI, duckdb, RSQLite, webfakes, remotes, stringi, jsonlite, uuid, jqr, RPostgres, testthat, withr, callr
 
       # nodbi:
 
-      - name: Install any other deps and duckdb extension library
+      - name: Install development libraries
+        if: runner.os == 'Linux'
+        run: sudo apt-get install -y libjq-dev
+
+      - name: Install deps and duckdb extension library
         run: |
+          install.packages(c("remotes", "covr", "rcmdcheck"))
+          remotes::install_github("r-dbi/RSQLite")
           remotes::install_github("ropensci/nodbi")
           remotes::install_deps(pkgdir = ".", dependencies = TRUE)
           library(duckdb)
@@ -102,13 +90,15 @@ jobs:
           duckdb_shutdown(drv)
         shell: Rscript {0}
 
+
       # generic workflow:
 
       - uses: r-lib/actions/check-r-package@v2
         with:
           upload-snapshots: true
 
-      # also test coverage:
+
+      # test coverage:
 
       - name: Test coverage
         if: matrix.config.os == 'ubuntu-latest' && matrix.config.r == 'release'

DESCRIPTION

@@ -4,7 +4,7 @@ Description: Simplified document database access and manipulation,
      providing a common API across supported 'NoSQL' databases 
      'Elasticsearch', 'CouchDB', 'MongoDB' as well as 
      'SQLite/JSON1', 'PostgreSQL', and 'DuckDB'.
-Version: 0.9.8.9000
+Version: 0.10.0
 Authors@R: c(
     person(given = "Ralf",
            family = "Herold",
@@ -31,19 +31,21 @@ Imports:
     jsonlite,
     uuid,
     jqr,
-    DBI
+    DBI,
+    V8,
+    R.utils
 Suggests:
     sofa (>= 0.3.0),
     elastic (>= 1.0.0),
     mongolite (>= 1.6),
-    RSQLite (>= 2.2.4),
+    RSQLite (>= 2.3.5),
     duckdb (>= 0.6.0),
     RPostgres,
     testthat,
     withr,
     callr,
     webfakes (>= 1.2.0)
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.0
 X-schema.org-applicationCategory: Databases
 X-schema.org-keywords: database, MongoDB, Elasticsearch, CouchDB, SQLite, PostgreSQL, DuckDB, NoSQL, JSON, documents
 X-schema.org-isPartOf: https://ropensci.org

NAMESPACE

@@ -63,3 +63,7 @@ export(src_elastic)
 export(src_mongo)
 export(src_postgres)
 export(src_sqlite)
+importFrom(R.utils,countLines)
+importFrom(V8,JS)
+importFrom(V8,v8)
+importFrom(stringi,stri_match_all_regex)

NEWS.md

@@ -1,6 +1,34 @@
-# nodbi 0.9.8.9000
+# nodbi 0.10.0
 
-* new dev version
+## Deprecated
+* empty parameter `query` now triggers a warning as it should be a valid JSON string; change `query = ""` into `query = "{}"` 
+
+## Changes
+* adapted to use new, faster `JSONB` functions in `SQLite` 3.45.0 (`RSQLite` >= 2.3.4.9005)
+* refactored parts of `docdb_create()` to speed up handling large data frames and lists
+* made Elasticsearch to immediately refresh index after `docdb_create()` and other functions
+* `docdb_update()` now reports which records failed to update and then continues
+* `docdb_delete()` now returns harmonised success logical value across backends
+
+## Potentially breaking change
+`docdb_query()` reimplementation to have the same functionality across all databases (DuckDB, SQLite, PostgreSQL, MongoDB, Elasticsearch, CouchDB); even though the API and unit tests remained, user provisions may break e.g. to handle return values of databases that previously were incompletely implemented (in particular Elasticsearch and CouchDB). Details: 
+
+* `query` can now be complex (nested, various operators) and is digested with a Javascript helper
+* `fields` can now be nested fields (e.g., `friends.name`) to directly return values lifted from the nested field 
+* `listfields` parameter newly implemented to return dot paths for all fields of all or selected documents in collection 
+* expanded use of `jq` via `jqr` for mangling parameters, selecting documents, filtering fields and lifting nested field values
+* if no data are found, returns `NULL` (previously some backends returned an empty data frame)
+* `docdb_query(src, key, query = "{}", fields = "{}")` now delegates to `docdb_get(src, key)`
+* `_id` is always returned, unless specified with `"_id": 0` in parameter `fields`
+* for `scr_postgres`, only fewer than 50 fields if any can be specified in `fields`
+* for `src_sqlite`, minimise the use of the time-costly `json_tree`
+* workaround for path collisions of MongoDB
+* some acceleration of `docdb_query()`
+* factored out common code 
+* expanded testing
+* updated docs
+
+## Bug fixes
 
 # nodbi 0.9.8
 
@@ -10,7 +38,7 @@
 # nodbi 0.9.7
 
 ## Changes
-* changed `docdb_update()` to directly use ndjson from file for duckdb
+* changed `docdb_update()` to directly use NDJSON from file for duckdb
 * cleaned up unnecessary code in `docdb_create()`
 * no more using transactions with `src_duckdb()`