Add a "sqlite-unbundled" feature that dynamically links to system libsqlite3.so library (launchbadge#3507)

* Add a "sqlite-unbundled" feature that dynamically links to system libsqlite3.so library

* update README abouot the newly-added `sqlite-unbundled` feature

* Update README.md to make it clear with bulleted list

Co-authored-by: Austin Bonander <[email protected]>

* more cfg feature updates

Co-authored-by: Austin Bonander <[email protected]>

* update documentation in sqlx-sqlx/src/lib.rs too

and also mention possible build time increasement.

* cargo fmt

* Add "sqlite-unbundled" feature to sqlx-cli

* Add sqlite-unbundled to gituhb actions tests

* cfg(feature = "sqlite") => cfg(any(feature = "sqlite", feature = "sqlite-unbundled"))

* fix

* CI: make sqlite-unbundled tests workaround required-features

by duplicating the relevant test section

* use an internal "_sqlite" feature to do the conditional compilation

---------

Co-authored-by: Austin Bonander <[email protected]>

Author: 2 people

.github/workflows/sqlx.yml

@@ -117,6 +117,7 @@ jobs:
     strategy:
       matrix:
         runtime: [async-std, tokio]
+        linking: [sqlite, sqlite-unbundled]
     needs: check
     steps:
       - uses: actions/checkout@v4
@@ -125,7 +126,11 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
         with:
-          key: "${{ runner.os }}-sqlite-${{ matrix.runtime }}-${{ matrix.tls }}"
+          key: "${{ runner.os }}-${{ matrix.linking }}-${{ matrix.runtime }}-${{ matrix.tls }}"
+
+      - name: Install system sqlite library
+        if: ${{ matrix.linking == 'sqlite-unbundled' }}
+        run: sudo apt-get install -y libsqlite3-dev
 
       - run: echo "using ${DATABASE_URL}"
 
@@ -135,7 +140,7 @@ jobs:
       - run: >
           cargo test
           --no-default-features
-          --features any,macros,sqlite,_unstable-all-types,runtime-${{ matrix.runtime }}
+          --features any,macros,${{ matrix.linking }},_unstable-all-types,runtime-${{ matrix.runtime }}
           --
           --test-threads=1
         env:
@@ -151,8 +156,8 @@ jobs:
       - run: >
           cargo build
           --no-default-features
-          --test sqlite-macros
-          --features any,macros,sqlite,_unstable-all-types,runtime-${{ matrix.runtime }}
+          --test ${{ matrix.linking }}-macros
+          --features any,macros,${{ matrix.linking }},_unstable-all-types,runtime-${{ matrix.runtime }}
         env:
           SQLX_OFFLINE: true
           SQLX_OFFLINE_DIR: .sqlx
@@ -163,8 +168,8 @@ jobs:
       - run: >
           cargo test
           --no-default-features
-          --test sqlite-macros
-          --features any,macros,sqlite,_unstable-all-types,runtime-${{ matrix.runtime }}
+          --test ${{ matrix.linking }}-macros
+          --features any,macros,${{ matrix.linking }},_unstable-all-types,runtime-${{ matrix.runtime }}
         env:
           DATABASE_URL: sqlite://tests/sqlite/sqlite.db
           SQLX_OFFLINE: true

Cargo.toml

@@ -98,12 +98,14 @@ runtime-tokio-rustls = ["runtime-tokio", "tls-rustls-ring"]
 # for conditional compilation
 _rt-async-std = []
 _rt-tokio = []
+_sqlite = []
 
 # database
 any = ["sqlx-core/any", "sqlx-mysql?/any", "sqlx-postgres?/any", "sqlx-sqlite?/any"]
 postgres = ["sqlx-postgres", "sqlx-macros?/postgres"]
 mysql = ["sqlx-mysql", "sqlx-macros?/mysql"]
-sqlite = ["sqlx-sqlite", "sqlx-macros?/sqlite"]
+sqlite = ["_sqlite", "sqlx-sqlite/bundled", "sqlx-macros?/sqlite"]
+sqlite-unbundled = ["_sqlite", "sqlx-sqlite/unbundled", "sqlx-macros?/sqlite-unbundled"]
 
 # types
 json = ["sqlx-macros?/json", "sqlx-mysql?/json", "sqlx-postgres?/json", "sqlx-sqlite?/json"]
@@ -250,6 +252,11 @@ name = "sqlite-macros"
 path = "tests/sqlite/macros.rs"
 required-features = ["sqlite", "macros"]
 
+[[test]]
+name = "sqlite-unbundled-macros"
+path = "tests/sqlite/macros.rs"
+required-features = ["sqlite-unbundled", "macros"]
+
 [[test]]
 name = "sqlite-derives"
 path = "tests/sqlite/derives.rs"

README.md

@@ -183,7 +183,14 @@ be removed in the future.
 
 -   `mssql`: Add support for the MSSQL database server.
 
--   `sqlite`: Add support for the self-contained [SQLite](https://sqlite.org/) database engine.
+-   `sqlite`: Add support for the self-contained [SQLite](https://sqlite.org/) database engine with SQLite bundled and statically-linked.
+
+-   `sqlite-unbundled`: The same as above (`sqlite`), but link SQLite from the system instead of the bundled version.
+    * Allows updating SQLite independently of SQLx or using forked versions.
+    * You must have SQLite installed on the system or provide a path to the library at build time.
+       See [the `rusqlite` README](https://github.com/rusqlite/rusqlite?tab=readme-ov-file#notes-on-building-rusqlite-and-libsqlite3-sys) for details.
+    * May result in link errors if the SQLite version is too old. Version `3.20.0` or newer is recommended.
+    * Can increase build time due to the use of bindgen.
 
 -   `any`: Add support for the `Any` database driver, which can proxy to a database driver at runtime.
 

sqlx-cli/Cargo.toml

@@ -57,6 +57,7 @@ native-tls = ["sqlx/runtime-tokio-native-tls"]
 mysql = ["sqlx/mysql"]
 postgres = ["sqlx/postgres"]
 sqlite = ["sqlx/sqlite"]
+sqlite-unbundled = ["sqlx/sqlite-unbundled"]
 
 # workaround for musl + openssl issues
 openssl-vendored = ["openssl/vendored"]

sqlx-cli/README.md

@@ -19,6 +19,9 @@ $ cargo install sqlx-cli --features openssl-vendored
 
 # use Rustls rather than OpenSSL (be sure to add the features for the databases you intend to use!)
 $ cargo install sqlx-cli --no-default-features --features rustls
+
+# only for sqlite and use the system sqlite library
+$ cargo install sqlx-cli --no-default-features --features sqlite-unbundled
 ```
 
 ## Usage

sqlx-cli/src/database.rs

@@ -11,7 +11,7 @@ pub async fn create(connect_opts: &ConnectOpts) -> anyhow::Result<()> {
     let exists = crate::retry_connect_errors(connect_opts, Any::database_exists).await?;
 
     if !exists {
-        #[cfg(feature = "sqlite")]
+        #[cfg(feature = "_sqlite")]
         sqlx::sqlite::CREATE_DB_WAL.store(
             connect_opts.sqlite_create_db_wal,
             std::sync::atomic::Ordering::Release,

sqlx-cli/src/opt.rs

@@ -258,7 +258,7 @@ pub struct ConnectOpts {
     /// However, if your application sets a `journal_mode` on `SqliteConnectOptions` to something
     /// other than `Wal`, then it will have to take the database file out of WAL mode on connecting,
     /// which requires an exclusive lock and may return a `database is locked` (`SQLITE_BUSY`) error.
-    #[cfg(feature = "sqlite")]
+    #[cfg(feature = "_sqlite")]
     #[clap(long, action = clap::ArgAction::Set, default_value = "true")]
     pub sqlite_create_db_wal: bool,
 }

sqlx-core/src/any/kind.rs

@@ -16,7 +16,7 @@ pub enum AnyKind {
     #[cfg(feature = "mysql")]
     MySql,
 
-    #[cfg(feature = "sqlite")]
+    #[cfg(feature = "_sqlite")]
     Sqlite,
 
     #[cfg(feature = "mssql")]
@@ -48,12 +48,12 @@ impl FromStr for AnyKind {
                 Err(Error::Configuration("database URL has the scheme of a MySQL database but the `mysql` feature is not enabled".into()))
             }
 
-            #[cfg(feature = "sqlite")]
+            #[cfg(feature = "_sqlite")]
             _ if url.starts_with("sqlite:") => {
                 Ok(AnyKind::Sqlite)
             }
 
-            #[cfg(not(feature = "sqlite"))]
+            #[cfg(not(feature = "_sqlite"))]
             _ if url.starts_with("sqlite:") => {
                 Err(Error::Configuration("database URL has the scheme of a SQLite database but the `sqlite` feature is not enabled".into()))
             }

sqlx-macros-core/Cargo.toml

@@ -18,6 +18,8 @@ _tls-native-tls = ["sqlx-core/_tls-native-tls"]
 _tls-rustls-aws-lc-rs = ["sqlx-core/_tls-rustls-aws-lc-rs"]
 _tls-rustls-ring = ["sqlx-core/_tls-rustls-ring"]
 
+_sqlite = []
+
 # SQLx features
 derive = []
 macros = []
@@ -26,7 +28,8 @@ migrate = ["sqlx-core/migrate"]
 # database
 mysql = ["sqlx-mysql"]
 postgres = ["sqlx-postgres"]
-sqlite = ["sqlx-sqlite"]
+sqlite = ["_sqlite", "sqlx-sqlite/bundled"]
+sqlite-unbundled = ["_sqlite", "sqlx-sqlite/unbundled"]
 
 # type integrations
 json = ["sqlx-core/json", "sqlx-mysql?/json", "sqlx-postgres?/json", "sqlx-sqlite?/json"]

sqlx-macros-core/src/database/impls.rs

@@ -44,7 +44,7 @@ mod sqlx {
     #[cfg(feature = "postgres")]
     pub use sqlx_postgres as postgres;
 
-    #[cfg(feature = "sqlite")]
+    #[cfg(feature = "_sqlite")]
     pub use sqlx_sqlite as sqlite;
 }
 
@@ -61,7 +61,7 @@ impl_database_ext! {
     row: sqlx::postgres::PgRow,
 }
 
-#[cfg(feature = "sqlite")]
+#[cfg(feature = "_sqlite")]
 impl_database_ext! {
     sqlx::sqlite::Sqlite,
     row: sqlx::sqlite::SqliteRow,

sqlx-macros-core/src/database/mod.rs

@@ -10,7 +10,7 @@ use sqlx_core::describe::Describe;
 use sqlx_core::executor::Executor;
 use sqlx_core::type_checking::TypeChecking;
 
-#[cfg(any(feature = "postgres", feature = "mysql", feature = "sqlite"))]
+#[cfg(any(feature = "postgres", feature = "mysql", feature = "_sqlite"))]
 mod impls;
 
 pub trait DatabaseExt: Database + TypeChecking {

sqlx-macros-core/src/derives/decode.rs

@@ -225,7 +225,7 @@ fn expand_derive_decode_strong_enum(
         ));
     }
 
-    if cfg!(feature = "sqlite") {
+    if cfg!(feature = "_sqlite") {
         tts.extend(quote!(
             #[automatically_derived]
             impl<'r> ::sqlx::decode::Decode<'r, ::sqlx::sqlite::Sqlite> for #ident {

sqlx-macros-core/src/derives/type.rs

@@ -211,7 +211,7 @@ fn expand_derive_has_sql_type_strong_enum(
         }
     }
 
-    if cfg!(feature = "sqlite") {
+    if cfg!(feature = "_sqlite") {
         tts.extend(quote!(
             #[automatically_derived]
             impl sqlx::Type<::sqlx::Sqlite> for #ident {

sqlx-macros-core/src/lib.rs

@@ -47,7 +47,7 @@ pub const FOSS_DRIVERS: &[QueryDriver] = &[
     QueryDriver::new::<sqlx_mysql::MySql>(),
     #[cfg(feature = "postgres")]
     QueryDriver::new::<sqlx_postgres::Postgres>(),
-    #[cfg(feature = "sqlite")]
+    #[cfg(feature = "_sqlite")]
     QueryDriver::new::<sqlx_sqlite::Sqlite>(),
 ];
 

sqlx-macros/Cargo.toml

@@ -30,6 +30,7 @@ migrate = ["sqlx-macros-core/migrate"]
 mysql = ["sqlx-macros-core/mysql"]
 postgres = ["sqlx-macros-core/postgres"]
 sqlite = ["sqlx-macros-core/sqlite"]
+sqlite-unbundled = ["sqlx-macros-core/sqlite-unbundled"]
 
 # type
 bigdecimal = ["sqlx-macros-core/bigdecimal"]

sqlx-sqlite/Cargo.toml

@@ -23,6 +23,9 @@ uuid = ["dep:uuid", "sqlx-core/uuid"]
 
 regexp = ["dep:regex"]
 
+bundled = ["libsqlite3-sys/bundled"]
+unbundled = ["libsqlite3-sys/buildtime_bindgen"]
+
 [dependencies]
 futures-core = { version = "0.3.19", default-features = false }
 futures-channel = { version = "0.3.19", default-features = false, features = ["sink", "alloc", "std"] }
@@ -55,7 +58,6 @@ default-features = false
 features = [
     "pkg-config",
     "vcpkg",
-    "bundled",
     "unlock_notify"
 ]
 

sqlx-sqlite/src/lib.rs

@@ -2,7 +2,8 @@
 //!
 //! ### Note: linkage is semver-exempt.
 //! This driver uses the `libsqlite3-sys` crate which links the native library for SQLite 3.
-//! For portability, we enable the `bundled` feature which builds and links SQLite from source.
+//! With the "sqlite" feature, we enable the `bundled` feature which builds and links SQLite from
+//! source.
 //!
 //! We reserve the right to upgrade the version of `libsqlite3-sys` as necessary to pick up new
 //! `3.x.y` versions of SQLite.
@@ -20,6 +21,18 @@
 //! ```
 //!
 //! and then upgrade these crates in lockstep when necessary.
+//!
+//! ### Dynamic linking
+//! To dynamically link to a system SQLite library, the "sqlite-unbundled" feature can be used
+//! instead.
+//!
+//! This allows updating SQLite independently of SQLx or using forked versions, but you must have
+//! SQLite installed on the system or provide a path to the library at build time (See
+//! [the `rusqlite` README](https://github.com/rusqlite/rusqlite?tab=readme-ov-file#notes-on-building-rusqlite-and-libsqlite3-sys)
+//! for details).
+//!
+//! It may result in link errors if the SQLite version is too old. Version `3.20.0` or newer is
+//! recommended. It can increase build time due to the use of bindgen.
 
 // SQLite is a C library. All interactions require FFI which is unsafe.
 // All unsafe blocks should have comments pointing to SQLite docs and ensuring that we maintain

src/any/mod.rs

@@ -41,7 +41,7 @@ pub fn install_default_drivers() {
             sqlx_mysql::any::DRIVER,
             #[cfg(feature = "postgres")]
             sqlx_postgres::any::DRIVER,
-            #[cfg(feature = "sqlite")]
+            #[cfg(feature = "_sqlite")]
             sqlx_sqlite::any::DRIVER,
         ])
         .expect("non-default drivers already installed")

src/lib.rs

@@ -44,8 +44,8 @@ pub use sqlx_mysql::{self as mysql, MySql, MySqlConnection, MySqlExecutor, MySql
 #[doc(inline)]
 pub use sqlx_postgres::{self as postgres, PgConnection, PgExecutor, PgPool, Postgres};
 
-#[cfg(feature = "sqlite")]
-#[cfg_attr(docsrs, doc(cfg(feature = "sqlite")))]
+#[cfg(feature = "_sqlite")]
+#[cfg_attr(docsrs, doc(cfg(feature = "_sqlite")))]
 #[doc(inline)]
 pub use sqlx_sqlite::{self as sqlite, Sqlite, SqliteConnection, SqliteExecutor, SqlitePool};
 

tests/ui-tests.rs

@@ -31,7 +31,7 @@ fn ui_tests() {
         }
     }
 
-    if cfg!(feature = "sqlite") {
+    if cfg!(feature = "_sqlite") {
         if dotenvy::var("DATABASE_URL").map_or(true, |v| {
             Path::is_relative(v.trim_start_matches("sqlite://").as_ref())
         }) {