From 5b8bb3b28b0faf9b86aed7e45011c8e7dbca065b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BE=9D=E4=BA=91?= Date: Thu, 3 Oct 2024 02:55:21 +0800 Subject: [PATCH] Add a "sqlite-unbundled" feature that dynamically links to system libsqlite3.so library (#3507) * Add a "sqlite-unbundled" feature that dynamically links to system libsqlite3.so library * update README abouot the newly-added `sqlite-unbundled` feature * Update README.md to make it clear with bulleted list Co-authored-by: Austin Bonander * more cfg feature updates Co-authored-by: Austin Bonander * update documentation in sqlx-sqlx/src/lib.rs too and also mention possible build time increasement. * cargo fmt * Add "sqlite-unbundled" feature to sqlx-cli * Add sqlite-unbundled to gituhb actions tests * cfg(feature = "sqlite") => cfg(any(feature = "sqlite", feature = "sqlite-unbundled")) * fix * CI: make sqlite-unbundled tests workaround required-features by duplicating the relevant test section * use an internal "_sqlite" feature to do the conditional compilation --------- Co-authored-by: Austin Bonander --- .github/workflows/sqlx.yml | 17 +++++++++++------ Cargo.toml | 9 ++++++++- README.md | 9 ++++++++- sqlx-cli/Cargo.toml | 1 + sqlx-cli/README.md | 3 +++ sqlx-cli/src/database.rs | 2 +- sqlx-cli/src/opt.rs | 2 +- sqlx-core/src/any/kind.rs | 6 +++--- sqlx-macros-core/Cargo.toml | 5 ++++- sqlx-macros-core/src/database/impls.rs | 4 ++-- sqlx-macros-core/src/database/mod.rs | 2 +- sqlx-macros-core/src/derives/decode.rs | 2 +- sqlx-macros-core/src/derives/type.rs | 2 +- sqlx-macros-core/src/lib.rs | 2 +- sqlx-macros/Cargo.toml | 1 + sqlx-sqlite/Cargo.toml | 4 +++- sqlx-sqlite/src/lib.rs | 15 ++++++++++++++- src/any/mod.rs | 2 +- src/lib.rs | 4 ++-- tests/ui-tests.rs | 2 +- 20 files changed, 68 insertions(+), 26 deletions(-) diff --git a/.github/workflows/sqlx.yml b/.github/workflows/sqlx.yml index 508f036eba..5a7673f26b 100644 --- a/.github/workflows/sqlx.yml +++ b/.github/workflows/sqlx.yml @@ -117,6 +117,7 @@ jobs: strategy: matrix: runtime: [async-std, tokio] + linking: [sqlite, sqlite-unbundled] needs: check steps: - uses: actions/checkout@v4 @@ -125,7 +126,11 @@ jobs: - uses: Swatinem/rust-cache@v2 with: - key: "${{ runner.os }}-sqlite-${{ matrix.runtime }}-${{ matrix.tls }}" + key: "${{ runner.os }}-${{ matrix.linking }}-${{ matrix.runtime }}-${{ matrix.tls }}" + + - name: Install system sqlite library + if: ${{ matrix.linking == 'sqlite-unbundled' }} + run: sudo apt-get install -y libsqlite3-dev - run: echo "using ${DATABASE_URL}" @@ -135,7 +140,7 @@ jobs: - run: > cargo test --no-default-features - --features any,macros,sqlite,_unstable-all-types,runtime-${{ matrix.runtime }} + --features any,macros,${{ matrix.linking }},_unstable-all-types,runtime-${{ matrix.runtime }} -- --test-threads=1 env: @@ -151,8 +156,8 @@ jobs: - run: > cargo build --no-default-features - --test sqlite-macros - --features any,macros,sqlite,_unstable-all-types,runtime-${{ matrix.runtime }} + --test ${{ matrix.linking }}-macros + --features any,macros,${{ matrix.linking }},_unstable-all-types,runtime-${{ matrix.runtime }} env: SQLX_OFFLINE: true SQLX_OFFLINE_DIR: .sqlx @@ -163,8 +168,8 @@ jobs: - run: > cargo test --no-default-features - --test sqlite-macros - --features any,macros,sqlite,_unstable-all-types,runtime-${{ matrix.runtime }} + --test ${{ matrix.linking }}-macros + --features any,macros,${{ matrix.linking }},_unstable-all-types,runtime-${{ matrix.runtime }} env: DATABASE_URL: sqlite://tests/sqlite/sqlite.db SQLX_OFFLINE: true diff --git a/Cargo.toml b/Cargo.toml index cfbdeff210..bf0a867e1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,12 +98,14 @@ runtime-tokio-rustls = ["runtime-tokio", "tls-rustls-ring"] # for conditional compilation _rt-async-std = [] _rt-tokio = [] +_sqlite = [] # database any = ["sqlx-core/any", "sqlx-mysql?/any", "sqlx-postgres?/any", "sqlx-sqlite?/any"] postgres = ["sqlx-postgres", "sqlx-macros?/postgres"] mysql = ["sqlx-mysql", "sqlx-macros?/mysql"] -sqlite = ["sqlx-sqlite", "sqlx-macros?/sqlite"] +sqlite = ["_sqlite", "sqlx-sqlite/bundled", "sqlx-macros?/sqlite"] +sqlite-unbundled = ["_sqlite", "sqlx-sqlite/unbundled", "sqlx-macros?/sqlite-unbundled"] # types json = ["sqlx-macros?/json", "sqlx-mysql?/json", "sqlx-postgres?/json", "sqlx-sqlite?/json"] @@ -250,6 +252,11 @@ name = "sqlite-macros" path = "tests/sqlite/macros.rs" required-features = ["sqlite", "macros"] +[[test]] +name = "sqlite-unbundled-macros" +path = "tests/sqlite/macros.rs" +required-features = ["sqlite-unbundled", "macros"] + [[test]] name = "sqlite-derives" path = "tests/sqlite/derives.rs" diff --git a/README.md b/README.md index 34ef7dd0dc..ad8dff4a73 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,14 @@ be removed in the future. - `mssql`: Add support for the MSSQL database server. -- `sqlite`: Add support for the self-contained [SQLite](https://sqlite.org/) database engine. +- `sqlite`: Add support for the self-contained [SQLite](https://sqlite.org/) database engine with SQLite bundled and statically-linked. + +- `sqlite-unbundled`: The same as above (`sqlite`), but link SQLite from the system instead of the bundled version. + * Allows updating SQLite independently of SQLx or using forked versions. + * You must have SQLite installed on the system or provide a path to the library at build time. + See [the `rusqlite` README](https://github.com/rusqlite/rusqlite?tab=readme-ov-file#notes-on-building-rusqlite-and-libsqlite3-sys) for details. + * May result in link errors if the SQLite version is too old. Version `3.20.0` or newer is recommended. + * Can increase build time due to the use of bindgen. - `any`: Add support for the `Any` database driver, which can proxy to a database driver at runtime. diff --git a/sqlx-cli/Cargo.toml b/sqlx-cli/Cargo.toml index 7bd4f9143b..0b047ab136 100644 --- a/sqlx-cli/Cargo.toml +++ b/sqlx-cli/Cargo.toml @@ -57,6 +57,7 @@ native-tls = ["sqlx/runtime-tokio-native-tls"] mysql = ["sqlx/mysql"] postgres = ["sqlx/postgres"] sqlite = ["sqlx/sqlite"] +sqlite-unbundled = ["sqlx/sqlite-unbundled"] # workaround for musl + openssl issues openssl-vendored = ["openssl/vendored"] diff --git a/sqlx-cli/README.md b/sqlx-cli/README.md index 118bb1a00f..b20461b8fd 100644 --- a/sqlx-cli/README.md +++ b/sqlx-cli/README.md @@ -19,6 +19,9 @@ $ cargo install sqlx-cli --features openssl-vendored # use Rustls rather than OpenSSL (be sure to add the features for the databases you intend to use!) $ cargo install sqlx-cli --no-default-features --features rustls + +# only for sqlite and use the system sqlite library +$ cargo install sqlx-cli --no-default-features --features sqlite-unbundled ``` ## Usage diff --git a/sqlx-cli/src/database.rs b/sqlx-cli/src/database.rs index 82dcfcfed9..7a2056ab35 100644 --- a/sqlx-cli/src/database.rs +++ b/sqlx-cli/src/database.rs @@ -11,7 +11,7 @@ pub async fn create(connect_opts: &ConnectOpts) -> anyhow::Result<()> { let exists = crate::retry_connect_errors(connect_opts, Any::database_exists).await?; if !exists { - #[cfg(feature = "sqlite")] + #[cfg(feature = "_sqlite")] sqlx::sqlite::CREATE_DB_WAL.store( connect_opts.sqlite_create_db_wal, std::sync::atomic::Ordering::Release, diff --git a/sqlx-cli/src/opt.rs b/sqlx-cli/src/opt.rs index b040117cd2..d5fe315234 100644 --- a/sqlx-cli/src/opt.rs +++ b/sqlx-cli/src/opt.rs @@ -258,7 +258,7 @@ pub struct ConnectOpts { /// However, if your application sets a `journal_mode` on `SqliteConnectOptions` to something /// other than `Wal`, then it will have to take the database file out of WAL mode on connecting, /// which requires an exclusive lock and may return a `database is locked` (`SQLITE_BUSY`) error. - #[cfg(feature = "sqlite")] + #[cfg(feature = "_sqlite")] #[clap(long, action = clap::ArgAction::Set, default_value = "true")] pub sqlite_create_db_wal: bool, } diff --git a/sqlx-core/src/any/kind.rs b/sqlx-core/src/any/kind.rs index f3a77789b0..60397c094c 100644 --- a/sqlx-core/src/any/kind.rs +++ b/sqlx-core/src/any/kind.rs @@ -16,7 +16,7 @@ pub enum AnyKind { #[cfg(feature = "mysql")] MySql, - #[cfg(feature = "sqlite")] + #[cfg(feature = "_sqlite")] Sqlite, #[cfg(feature = "mssql")] @@ -48,12 +48,12 @@ impl FromStr for AnyKind { Err(Error::Configuration("database URL has the scheme of a MySQL database but the `mysql` feature is not enabled".into())) } - #[cfg(feature = "sqlite")] + #[cfg(feature = "_sqlite")] _ if url.starts_with("sqlite:") => { Ok(AnyKind::Sqlite) } - #[cfg(not(feature = "sqlite"))] + #[cfg(not(feature = "_sqlite"))] _ if url.starts_with("sqlite:") => { Err(Error::Configuration("database URL has the scheme of a SQLite database but the `sqlite` feature is not enabled".into())) } diff --git a/sqlx-macros-core/Cargo.toml b/sqlx-macros-core/Cargo.toml index 7a7ba1a30e..d473dc1f40 100644 --- a/sqlx-macros-core/Cargo.toml +++ b/sqlx-macros-core/Cargo.toml @@ -18,6 +18,8 @@ _tls-native-tls = ["sqlx-core/_tls-native-tls"] _tls-rustls-aws-lc-rs = ["sqlx-core/_tls-rustls-aws-lc-rs"] _tls-rustls-ring = ["sqlx-core/_tls-rustls-ring"] +_sqlite = [] + # SQLx features derive = [] macros = [] @@ -26,7 +28,8 @@ migrate = ["sqlx-core/migrate"] # database mysql = ["sqlx-mysql"] postgres = ["sqlx-postgres"] -sqlite = ["sqlx-sqlite"] +sqlite = ["_sqlite", "sqlx-sqlite/bundled"] +sqlite-unbundled = ["_sqlite", "sqlx-sqlite/unbundled"] # type integrations json = ["sqlx-core/json", "sqlx-mysql?/json", "sqlx-postgres?/json", "sqlx-sqlite?/json"] diff --git a/sqlx-macros-core/src/database/impls.rs b/sqlx-macros-core/src/database/impls.rs index 35dc9393bc..499406907f 100644 --- a/sqlx-macros-core/src/database/impls.rs +++ b/sqlx-macros-core/src/database/impls.rs @@ -44,7 +44,7 @@ mod sqlx { #[cfg(feature = "postgres")] pub use sqlx_postgres as postgres; - #[cfg(feature = "sqlite")] + #[cfg(feature = "_sqlite")] pub use sqlx_sqlite as sqlite; } @@ -61,7 +61,7 @@ impl_database_ext! { row: sqlx::postgres::PgRow, } -#[cfg(feature = "sqlite")] +#[cfg(feature = "_sqlite")] impl_database_ext! { sqlx::sqlite::Sqlite, row: sqlx::sqlite::SqliteRow, diff --git a/sqlx-macros-core/src/database/mod.rs b/sqlx-macros-core/src/database/mod.rs index 0ba8a9384f..a2d0a1fa0d 100644 --- a/sqlx-macros-core/src/database/mod.rs +++ b/sqlx-macros-core/src/database/mod.rs @@ -10,7 +10,7 @@ use sqlx_core::describe::Describe; use sqlx_core::executor::Executor; use sqlx_core::type_checking::TypeChecking; -#[cfg(any(feature = "postgres", feature = "mysql", feature = "sqlite"))] +#[cfg(any(feature = "postgres", feature = "mysql", feature = "_sqlite"))] mod impls; pub trait DatabaseExt: Database + TypeChecking { diff --git a/sqlx-macros-core/src/derives/decode.rs b/sqlx-macros-core/src/derives/decode.rs index fced945256..7af9643d93 100644 --- a/sqlx-macros-core/src/derives/decode.rs +++ b/sqlx-macros-core/src/derives/decode.rs @@ -225,7 +225,7 @@ fn expand_derive_decode_strong_enum( )); } - if cfg!(feature = "sqlite") { + if cfg!(feature = "_sqlite") { tts.extend(quote!( #[automatically_derived] impl<'r> ::sqlx::decode::Decode<'r, ::sqlx::sqlite::Sqlite> for #ident { diff --git a/sqlx-macros-core/src/derives/type.rs b/sqlx-macros-core/src/derives/type.rs index 66579e5b6c..8ecd55230c 100644 --- a/sqlx-macros-core/src/derives/type.rs +++ b/sqlx-macros-core/src/derives/type.rs @@ -211,7 +211,7 @@ fn expand_derive_has_sql_type_strong_enum( } } - if cfg!(feature = "sqlite") { + if cfg!(feature = "_sqlite") { tts.extend(quote!( #[automatically_derived] impl sqlx::Type<::sqlx::Sqlite> for #ident { diff --git a/sqlx-macros-core/src/lib.rs b/sqlx-macros-core/src/lib.rs index d2a40aa831..e8804f57fe 100644 --- a/sqlx-macros-core/src/lib.rs +++ b/sqlx-macros-core/src/lib.rs @@ -47,7 +47,7 @@ pub const FOSS_DRIVERS: &[QueryDriver] = &[ QueryDriver::new::(), #[cfg(feature = "postgres")] QueryDriver::new::(), - #[cfg(feature = "sqlite")] + #[cfg(feature = "_sqlite")] QueryDriver::new::(), ]; diff --git a/sqlx-macros/Cargo.toml b/sqlx-macros/Cargo.toml index b34c812309..768db71d8d 100644 --- a/sqlx-macros/Cargo.toml +++ b/sqlx-macros/Cargo.toml @@ -30,6 +30,7 @@ migrate = ["sqlx-macros-core/migrate"] mysql = ["sqlx-macros-core/mysql"] postgres = ["sqlx-macros-core/postgres"] sqlite = ["sqlx-macros-core/sqlite"] +sqlite-unbundled = ["sqlx-macros-core/sqlite-unbundled"] # type bigdecimal = ["sqlx-macros-core/bigdecimal"] diff --git a/sqlx-sqlite/Cargo.toml b/sqlx-sqlite/Cargo.toml index 1ad87de102..4027f68e40 100644 --- a/sqlx-sqlite/Cargo.toml +++ b/sqlx-sqlite/Cargo.toml @@ -23,6 +23,9 @@ uuid = ["dep:uuid", "sqlx-core/uuid"] regexp = ["dep:regex"] +bundled = ["libsqlite3-sys/bundled"] +unbundled = ["libsqlite3-sys/buildtime_bindgen"] + [dependencies] futures-core = { version = "0.3.19", default-features = false } futures-channel = { version = "0.3.19", default-features = false, features = ["sink", "alloc", "std"] } @@ -55,7 +58,6 @@ default-features = false features = [ "pkg-config", "vcpkg", - "bundled", "unlock_notify" ] diff --git a/sqlx-sqlite/src/lib.rs b/sqlx-sqlite/src/lib.rs index 0951084bc9..f8f5534879 100644 --- a/sqlx-sqlite/src/lib.rs +++ b/sqlx-sqlite/src/lib.rs @@ -2,7 +2,8 @@ //! //! ### Note: linkage is semver-exempt. //! This driver uses the `libsqlite3-sys` crate which links the native library for SQLite 3. -//! For portability, we enable the `bundled` feature which builds and links SQLite from source. +//! With the "sqlite" feature, we enable the `bundled` feature which builds and links SQLite from +//! source. //! //! We reserve the right to upgrade the version of `libsqlite3-sys` as necessary to pick up new //! `3.x.y` versions of SQLite. @@ -20,6 +21,18 @@ //! ``` //! //! and then upgrade these crates in lockstep when necessary. +//! +//! ### Dynamic linking +//! To dynamically link to a system SQLite library, the "sqlite-unbundled" feature can be used +//! instead. +//! +//! This allows updating SQLite independently of SQLx or using forked versions, but you must have +//! SQLite installed on the system or provide a path to the library at build time (See +//! [the `rusqlite` README](https://github.com/rusqlite/rusqlite?tab=readme-ov-file#notes-on-building-rusqlite-and-libsqlite3-sys) +//! for details). +//! +//! It may result in link errors if the SQLite version is too old. Version `3.20.0` or newer is +//! recommended. It can increase build time due to the use of bindgen. // SQLite is a C library. All interactions require FFI which is unsafe. // All unsafe blocks should have comments pointing to SQLite docs and ensuring that we maintain diff --git a/src/any/mod.rs b/src/any/mod.rs index fe303cd22d..cc1366af2c 100644 --- a/src/any/mod.rs +++ b/src/any/mod.rs @@ -41,7 +41,7 @@ pub fn install_default_drivers() { sqlx_mysql::any::DRIVER, #[cfg(feature = "postgres")] sqlx_postgres::any::DRIVER, - #[cfg(feature = "sqlite")] + #[cfg(feature = "_sqlite")] sqlx_sqlite::any::DRIVER, ]) .expect("non-default drivers already installed") diff --git a/src/lib.rs b/src/lib.rs index d675fa11c3..a9e90c071d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,8 +44,8 @@ pub use sqlx_mysql::{self as mysql, MySql, MySqlConnection, MySqlExecutor, MySql #[doc(inline)] pub use sqlx_postgres::{self as postgres, PgConnection, PgExecutor, PgPool, Postgres}; -#[cfg(feature = "sqlite")] -#[cfg_attr(docsrs, doc(cfg(feature = "sqlite")))] +#[cfg(feature = "_sqlite")] +#[cfg_attr(docsrs, doc(cfg(feature = "_sqlite")))] #[doc(inline)] pub use sqlx_sqlite::{self as sqlite, Sqlite, SqliteConnection, SqliteExecutor, SqlitePool}; diff --git a/tests/ui-tests.rs b/tests/ui-tests.rs index ad089bcfa7..f74694b870 100644 --- a/tests/ui-tests.rs +++ b/tests/ui-tests.rs @@ -31,7 +31,7 @@ fn ui_tests() { } } - if cfg!(feature = "sqlite") { + if cfg!(feature = "_sqlite") { if dotenvy::var("DATABASE_URL").map_or(true, |v| { Path::is_relative(v.trim_start_matches("sqlite://").as_ref()) }) {