diff --git a/CHANGELOG.md b/CHANGELOG.md index b391c2c3..325669b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ - Date format: YYYY-MM-dd +## 2.6.0 / 2026-xx-xx + +### sqllin-dsl + +* Fix: the default `lookasideSlotSize` and `lookasideSlotCount` of `DSLDBConfiguration` disabled SQLite's lookaside memory allocator on Kotlin/Native and on Android 9 and higher, as those of `DatabaseConfiguration` did; they are now -1, see sqllin-driver +* Fix: on the JVM and Kotlin/Native, a blocking `database { }` scope on another thread could take the TEMP table and triggers that count the changes of observed queries with it, as its transaction can be open while they are created, the connection being locked for each statement rather than for a transaction: its rollback removed them, so the next write failed with "no such table", and once another table was observed, the changes of the table they belonged to were no longer counted. The tracker now creates them again where they are gone, and counts a change of their tables. A failure to tell the observed queries of a change no longer takes the place of the failure of the statement that made it. The guide to observed queries and the documentation of `Database#invoke` and `Database#observe` now say to write through `suspendedScope` while queries are observed, as they run on another thread, and that a blocking scope isn't serialized with them +* Fix documentation: the example of the guide to concurrency safety called `database suspendScope { }`, which doesn't exist, in a function that wasn't a `suspend` one; it is `database suspendedScope { }`, in a `suspend` function + +### sqllin-driver + +* Fix: the default `lookasideSlotSize` and `lookasideSlotCount` of `DatabaseConfiguration`, 0, which its documentation said left SQLite's default configuration, disabled SQLite's lookaside memory allocator on Kotlin/Native and on Android 9 and higher, as `sqlite3_db_config` and `setLookasideConfig` take 0 to disable it, while the JVM and lower versions of Android kept SQLite's default configuration. Their default is now -1, which leaves SQLite's default configuration on every platform, and 0 still disables the allocator. The guides to getting started now say so, and that sqlite-jdbc doesn't support `sqlite3_db_config()`, rather than `sqlite3_config()` +* Fix: on Android, `busyTimeout` didn't take effect, so SQLite waited 2500 ms, as the framework sets it, for a lock that another connection held. On Android 9 and higher, it was given to `SQLiteDatabase.OpenParams.Builder#setIdleConnectionTimeout`, which is deprecated and means another thing, how long a connection may be idle before the framework closes it, and which `SQLiteOpenHelper` drops anyway. `busyTimeout` is now set with `PRAGMA busy_timeout`: from Android 13 on, on every connection of the database, as one in WAL mode reads through several, and below, on its only connection. The guides to getting started listed `busyTimeout`, and in Chinese `journalMode`, among the parameters that only work on Android 9 and higher; only `inMemory`, `lookasideSlotSize` and `lookasideSlotCount` do +* Fix: transactions began differently on each platform, DEFERRED on the JVM and Kotlin/Native and EXCLUSIVE on Android, so on the JVM and Kotlin/Native, a transaction that read before it wrote failed at once with "database is locked" while another connection wrote, as SQLite doesn't wait for `busyTimeout` there, where waiting could deadlock. Transactions now begin IMMEDIATE on every platform, taking the write lock at once, for which SQLite waits as long as `busyTimeout`. On Android, it is what EXCLUSIVE did in WAL mode, and in DELETE mode, other connections can now read until the transaction commits. On the JVM and Kotlin/Native, a read-only connection begins a deferred transaction, as it can't take the write lock. The JVM driver begins and ends transactions with SQL rather than by leaving and entering auto-commit mode, as sqlite-jdbc then begins another transaction of the configured kind at once, which an IMMEDIATE one couldn't while another connection wrote +* Fix: when SQLite rolled a transaction back itself, as it does for a constraint with `ON CONFLICT ROLLBACK`, which `onConflict = OnConflict.ROLLBACK` declares, for a trigger's `RAISE(ROLLBACK)` and for some I/O errors, ending the transaction failed too, as no transaction was active, and that failure took the place of the one that made SQLite roll it back, in `withTransaction` and so in the DSL's `transaction { }`. The failure that made SQLite roll it back is now thrown, with the failure to end the transaction added to it as a suppressed exception + ## 2.5.0 / 2026-10-11 ### All diff --git a/ROADMAP.md b/ROADMAP.md index 30a695ef..5f8b78b0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -11,7 +11,8 @@ done. A new item takes the next number. * F8: Support WASM platform DSL * F12: Support type converters: store a property of any type through a serializer that encodes it to a type SQLite supports, with type-safe WHERE and SET on its column, e.g. to store instances of kotlinx.datetime -* F13: Provide a `PagingSource` for androidx.paging built on observable queries, like Room's `LimitOffsetPagingSource`, so that paging a query doesn't need a hand-written bridge +* F66: Provide table version tokens, which tell a reader when the tables it read change, such as a `PagingSource` that has to invalidate itself: `versioned { }` runs a read-only scope and returns its results with an opaque token of the versions of the tables its SELECTs read, captured under the lock that the read runs under, after the tables are tracked; tokens combine with `+`, which takes the earlier version of a table and rejects the tokens of another `Database`; `awaitChange(token)` suspends until one of their tables changes, and `onChange(token) { }` registers a one-shot callback, which runs after the writing scope releases the lock, and returns null instead when the token is already stale, as it registers before it checks, so that no change between the read and the registration is missed +* F13: Provide a `PagingSource` for androidx.paging built on the table version tokens of F66, like Room's `LimitOffsetPagingSource`, so that paging a query doesn't need a hand-written bridge: it registers with `onChange` from its first load, holding its callback strongly while the registry holds it weakly, so that a source that Pager abandons doesn't stay registered, keeps the previous key of an invalidated offset-keyed source, reads `anchorPosition` relative to the presented list without placeholders, and keeps offsets on a grid of the page size * F14: Support constraints in `ALTER_ADD_COLUMN`, which adds a column with its type only: DEFAULT, NOT NULL with DEFAULT, CHECK, COLLATE NOCASE and REFERENCES * F15: Support WITHOUT ROWID tables, which save space and a lookup when the primary key isn't an INTEGER * F16: Support STRICT tables, which reject values of other types than their columns' (SQLite 3.37.0, Android API 34) @@ -20,7 +21,7 @@ done. A new item takes the next number. * F23: Support the other conflict resolutions: INSERT OR ABORT, OR FAIL and OR ROLLBACK, and UPDATE OR IGNORE, OR REPLACE and the others * F25: Support ORDER BY and LIMIT in UPDATE and DELETE, which need SQLite's `SQLITE_ENABLE_UPDATE_DELETE_LIMIT` option, which Android's lacks * F26: Support INDEXED BY and NOT INDEXED -* F27: Support choosing how a transaction begins, DEFERRED, IMMEDIATE or EXCLUSIVE, which differs between the platforms now +* F27: Support choosing how a transaction begins, DEFERRED or EXCLUSIVE, besides IMMEDIATE, which every platform uses since 2.6.0 * F29: Support EXPLAIN QUERY PLAN, to see which indexes a query uses * F30: Support more PRAGMAs, such as integrity_check, quick_check, table_info, index_list, wal_checkpoint, defer_foreign_keys and case_sensitive_like diff --git a/sqllin-driver/README.md b/sqllin-driver/README.md index 213b0ea5..2562e5a7 100644 --- a/sqllin-driver/README.md +++ b/sqllin-driver/README.md @@ -73,8 +73,8 @@ val databaseConnection = openDatabase( journalMode = JournalMode.WAL, synchronousMode = SynchronousMode.NORMAL, busyTimeout = 5000, - lookasideSlotSize = 0, - lookasideSlotCount = 0, + lookasideSlotSize = -1, + lookasideSlotCount = -1, create = { it.execSQL("create table person (id integer primary key autoincrement, name text, age integer)") }, diff --git a/sqllin-driver/README_CN.md b/sqllin-driver/README_CN.md index 0a4bd11d..8b1030c4 100644 --- a/sqllin-driver/README_CN.md +++ b/sqllin-driver/README_CN.md @@ -63,8 +63,8 @@ val databaseConnection = openDatabase( journalMode = JournalMode.WAL, synchronousMode = SynchronousMode.NORMAL, busyTimeout = 5000, - lookasideSlotSize = 0, - lookasideSlotCount = 0, + lookasideSlotSize = -1, + lookasideSlotCount = -1, create = { it.execSQL("create table person (id integer primary key autoincrement, name text, age integer)") }, diff --git a/sqllin-driver/src/androidHostTest/kotlin/com/ctrip/sqllin/driver/test/AndroidTest.kt b/sqllin-driver/src/androidHostTest/kotlin/com/ctrip/sqllin/driver/test/AndroidTest.kt index b716587c..c5ca7198 100644 --- a/sqllin-driver/src/androidHostTest/kotlin/com/ctrip/sqllin/driver/test/AndroidTest.kt +++ b/sqllin-driver/src/androidHostTest/kotlin/com/ctrip/sqllin/driver/test/AndroidTest.kt @@ -18,12 +18,15 @@ package com.ctrip.sqllin.driver.test import android.content.Context import androidx.test.core.app.ApplicationProvider +import com.ctrip.sqllin.driver.DatabaseConfiguration +import com.ctrip.sqllin.driver.toAndroidOpenParams import com.ctrip.sqllin.driver.toDatabasePath import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner import org.robolectric.annotation.Config import kotlin.test.AfterTest import kotlin.test.Test +import kotlin.test.assertEquals /** * Android unit test that runs on the JVM via Robolectric. The `sdk` levels cover both @@ -57,9 +60,32 @@ class AndroidTest { @Test fun testFailedCommit() = commonTest.testFailedCommit() + @Test + fun testRolledBackBySQLite() = commonTest.testRolledBackBySQLite() + + @Test + fun testImmediateTransaction() = commonTest.testImmediateTransaction() + + @Test + fun testBusyTimeout() = commonTest.testBusyTimeout() + @Test fun testConcurrency() = commonTest.testConcurrency() + /** + * The lookaside configuration is only given to the framework when one is set, as `setLookasideConfig(0, 0)` + * disables the allocator. + */ + @Test + @Config(sdk = [37]) + fun testLookasideConfig() { + val config = DatabaseConfiguration(name = SQL.DATABASE_NAME, path = context.toDatabasePath(), version = 1) + fun lookaside(config: DatabaseConfiguration) = config.toAndroidOpenParams().run { lookasideSlotSize to lookasideSlotCount } + assertEquals(-1 to -1, lookaside(config)) + assertEquals(0 to 0, lookaside(config.copy(lookasideSlotSize = 0, lookasideSlotCount = 0))) + assertEquals(256 to 32, lookaside(config.copy(lookasideSlotSize = 256, lookasideSlotCount = 32))) + } + @AfterTest fun setDown() { context.deleteDatabase(SQL.DATABASE_NAME) diff --git a/sqllin-driver/src/androidMain/kotlin/com/ctrip/sqllin/driver/AndroidDatabaseConnection.kt b/sqllin-driver/src/androidMain/kotlin/com/ctrip/sqllin/driver/AndroidDatabaseConnection.kt index a2f010df..fadc2ae8 100644 --- a/sqllin-driver/src/androidMain/kotlin/com/ctrip/sqllin/driver/AndroidDatabaseConnection.kt +++ b/sqllin-driver/src/androidMain/kotlin/com/ctrip/sqllin/driver/AndroidDatabaseConnection.kt @@ -96,7 +96,8 @@ internal class AndroidDatabaseConnection(private val database: SQLiteDatabase) : } } - override fun beginTransaction() = database.beginTransaction() + // BEGIN IMMEDIATE, as on every platform, which beginTransaction's EXCLUSIVE only is in WAL mode + override fun beginTransaction() = database.beginTransactionNonExclusive() override fun endTransaction() { try { database.endTransaction() diff --git a/sqllin-driver/src/androidMain/kotlin/com/ctrip/sqllin/driver/ExtensionAndroid.kt b/sqllin-driver/src/androidMain/kotlin/com/ctrip/sqllin/driver/ExtensionAndroid.kt index bd76ecb3..8d7f27e4 100644 --- a/sqllin-driver/src/androidMain/kotlin/com/ctrip/sqllin/driver/ExtensionAndroid.kt +++ b/sqllin-driver/src/androidMain/kotlin/com/ctrip/sqllin/driver/ExtensionAndroid.kt @@ -37,7 +37,7 @@ internal value class AndroidDatabasePath(val context: Context) : DatabasePath public actual fun openDatabase(config: DatabaseConfiguration): DatabaseConnection { val isEqualsOrHigherThanAndroidP = Build.VERSION.SDK_INT >= Build.VERSION_CODES.P if (isEqualsOrHigherThanAndroidP && config.inMemory) - return AndroidDatabaseConnection(createInMemory(config.toAndroidOpenParams())) + return AndroidDatabaseConnection(createInMemory(config.toAndroidOpenParams()).apply { setBusyTimeout(config.busyTimeout) }) val helper = if (isEqualsOrHigherThanAndroidP) AndroidDBHelper(config) else @@ -61,6 +61,8 @@ private class OldAndroidDBHelper( private val config: DatabaseConfiguration, ) : SQLiteOpenHelper((config.path as AndroidDatabasePath).context, config.name, null, config.version) { + override fun onConfigure(db: SQLiteDatabase) = db.setBusyTimeout(config.busyTimeout) + override fun onCreate(db: SQLiteDatabase) = config.create(AndroidDatabaseConnection(db)) @@ -76,6 +78,8 @@ private class AndroidDBHelper( private val config: DatabaseConfiguration, ) : SQLiteOpenHelper((config.path as AndroidDatabasePath).context, config.name, config.version, config.toAndroidOpenParams()) { + override fun onConfigure(db: SQLiteDatabase) = db.setBusyTimeout(config.busyTimeout) + override fun onCreate(db: SQLiteDatabase) = config.create(AndroidDatabaseConnection(db)) @@ -85,17 +89,36 @@ private class AndroidDBHelper( /** * Converts [DatabaseConfiguration] to Android's OpenParams for API level 28+. + * + * The lookaside configuration is only set when both of its values are 0 or more, as `setLookasideConfig(0, 0)` + * disables the allocator, and leaving it unset keeps SQLite's default configuration. */ @RequiresApi(Build.VERSION_CODES.P) -@Suppress("DEPRECATION") -private fun DatabaseConfiguration.toAndroidOpenParams(): OpenParams = OpenParams +internal fun DatabaseConfiguration.toAndroidOpenParams(): OpenParams = OpenParams .Builder() .setJournalMode(journalMode.name) .setSynchronousMode(synchronousMode.name) - .setIdleConnectionTimeout(busyTimeout.toLong()) - .setLookasideConfig(lookasideSlotSize, lookasideSlotCount) + .apply { + if (lookasideSlotSize >= 0 && lookasideSlotCount >= 0) + setLookasideConfig(lookasideSlotSize, lookasideSlotCount) + } .build() +/** + * Sets how long SQLite waits for a lock that another connection holds, which the framework sets to 2500 ms. + * + * From Android 13 on, a database in WAL mode reads through several connections, so the PRAGMA is run on each of them, + * and on those opened later; the framework only runs a PRAGMA that returns a row, as this one does, from Android 13 on. + * Before, a database that SQLlin opens has a single connection. + */ +private fun SQLiteDatabase.setBusyTimeout(milliseconds: Int) { + val sql = "PRAGMA busy_timeout = $milliseconds" + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) + execPerConnectionSQL(sql, null) + else + rawQuery(sql, null).use { it.moveToFirst() } +} + public actual fun deleteDatabase(path: DatabasePath, name: String): Boolean = (path as? AndroidDatabasePath)?.context?.deleteDatabase(name) ?: throw IllegalArgumentException("Please use the `Context.toDatabasePath()` to get the DatabasePath") \ No newline at end of file diff --git a/sqllin-driver/src/commonMain/kotlin/com/ctrip/sqllin/driver/DatabaseConfiguration.kt b/sqllin-driver/src/commonMain/kotlin/com/ctrip/sqllin/driver/DatabaseConfiguration.kt index f3b025ee..532cd621 100644 --- a/sqllin-driver/src/commonMain/kotlin/com/ctrip/sqllin/driver/DatabaseConfiguration.kt +++ b/sqllin-driver/src/commonMain/kotlin/com/ctrip/sqllin/driver/DatabaseConfiguration.kt @@ -27,8 +27,12 @@ package com.ctrip.sqllin.driver * @property journalMode The SQLite journal mode. Default: [JournalMode.WAL] * @property synchronousMode The SQLite synchronous mode. Default: [SynchronousMode.NORMAL] * @property busyTimeout Timeout in milliseconds for database lock waits. Default: 5000ms - * @property lookasideSlotSize Size of each lookaside memory slot. Default: 0 (use SQLite default) - * @property lookasideSlotCount Number of lookaside memory slots. Default: 0 (use SQLite default) + * @property lookasideSlotSize The size in bytes of each slot of SQLite's lookaside memory allocator, set together with + * [lookasideSlotCount] when both are 0 or more. Default: -1, which leaves SQLite's default configuration; 0 disables the + * allocator. Android 9 and higher take it when both are positive or both 0, and lower versions don't take it; on the + * JVM, where sqlite-jdbc can't set it, it has no effect + * @property lookasideSlotCount The number of slots of SQLite's lookaside memory allocator of each connection, set + * together with [lookasideSlotSize]. Default: -1, which leaves SQLite's default configuration; 0 disables the allocator * @property create Callback invoked when creating a new database * @property upgrade Callback invoked when upgrading the database schema * @@ -43,8 +47,8 @@ public data class DatabaseConfiguration( val journalMode: JournalMode = JournalMode.WAL, val synchronousMode: SynchronousMode = SynchronousMode.NORMAL, val busyTimeout: Int = 5000, - val lookasideSlotSize: Int = 0, - val lookasideSlotCount: Int = 0, + val lookasideSlotSize: Int = -1, + val lookasideSlotCount: Int = -1, val create: (databaseConnection: DatabaseConnection) -> Unit = {}, val upgrade: (databaseConnection: DatabaseConnection, oldVersion: Int, newVersion: Int) -> Unit = { _, _, _ -> }, ) \ No newline at end of file diff --git a/sqllin-driver/src/commonMain/kotlin/com/ctrip/sqllin/driver/Extension.kt b/sqllin-driver/src/commonMain/kotlin/com/ctrip/sqllin/driver/Extension.kt index 3d784ad8..a483ccbe 100644 --- a/sqllin-driver/src/commonMain/kotlin/com/ctrip/sqllin/driver/Extension.kt +++ b/sqllin-driver/src/commonMain/kotlin/com/ctrip/sqllin/driver/Extension.kt @@ -64,17 +64,29 @@ public inline fun openDatabase(config: DatabaseConfiguration, block: (Databa * The transaction is committed if the block completes successfully, * or rolled back if an exception is thrown. * + * When the block throws, a failure to end the transaction is added to its exception, rather than taking its place: + * SQLite rolls some transactions back itself, as for a constraint with ON CONFLICT ROLLBACK or a trigger's + * RAISE(ROLLBACK), and then the ROLLBACK fails, as no transaction is active. + * * @param block The block to execute within the transaction * @return The result of the block */ public inline fun DatabaseConnection.withTransaction(block: (DatabaseConnection) -> T): T { beginTransaction() + var failure: Throwable? = null try { val result = block(this) setTransactionSuccessful() return result + } catch (e: Throwable) { + failure = e + throw e } finally { - endTransaction() + try { + endTransaction() + } catch (endFailure: Throwable) { + (failure ?: throw endFailure).addSuppressed(endFailure) + } } } diff --git a/sqllin-driver/src/commonTest/kotlin/com/ctrip/sqllin/driver/test/CommonBasicTest.kt b/sqllin-driver/src/commonTest/kotlin/com/ctrip/sqllin/driver/test/CommonBasicTest.kt index c338b8df..80b7ccde 100644 --- a/sqllin-driver/src/commonTest/kotlin/com/ctrip/sqllin/driver/test/CommonBasicTest.kt +++ b/sqllin-driver/src/commonTest/kotlin/com/ctrip/sqllin/driver/test/CommonBasicTest.kt @@ -23,6 +23,7 @@ import com.ctrip.sqllin.driver.withQuery import com.ctrip.sqllin.driver.withTransaction import kotlinx.coroutines.* import kotlinx.coroutines.test.runTest +import kotlin.test.assertContains import kotlin.test.assertEquals import kotlin.test.assertFails @@ -224,6 +225,59 @@ class CommonBasicTest(private val path: DatabasePath) { } } + /** + * SQLite rolls a transaction back itself for a constraint with ON CONFLICT ROLLBACK, or a trigger's RAISE(ROLLBACK), + * so that ending it fails too, which mustn't hide why it was rolled back; and the next transaction can begin. + */ + fun testRolledBackBySQLite() { + openDatabase(getDefaultDBConfig(false)) { + it.execSQL("CREATE TABLE rolled_back(x INT UNIQUE ON CONFLICT ROLLBACK)") + it.execSQL("CREATE TRIGGER no_negative BEFORE INSERT ON rolled_back WHEN new.x < 0 BEGIN SELECT RAISE(ROLLBACK, 'negative'); END") + it.executeInsert("INSERT INTO rolled_back VALUES (?)", arrayOf(1L)) + fun failure(value: Long): String = assertFails { + it.withTransaction { connection -> + connection.executeInsert("INSERT INTO rolled_back VALUES (?)", arrayOf(2L)) + connection.executeInsert("INSERT INTO rolled_back VALUES (?)", arrayOf(value)) + } + }.message ?: "" + assertContains(failure(1L), "UNIQUE constraint failed") + assertContains(failure(-1L), "negative") + it.withTransaction { connection -> + connection.executeInsert("INSERT INTO rolled_back VALUES (?)", arrayOf(3L)) + } + val values = it.withQuery("SELECT x FROM rolled_back ORDER BY x") { cursor -> + buildList { cursor.forEachRow { add(cursor.getLong(0)) } } + } + assertEquals(listOf(1L, 3L), values) + } + } + + /** + * A transaction takes the write lock when it begins, as BEGIN IMMEDIATE does, on every platform: another connection + * can't write until it ends. + */ + fun testImmediateTransaction() { + openDatabase(getDefaultDBConfig(false)) { + openDatabase(getDefaultDBConfig(false).copy(busyTimeout = 0)) { other -> + it.withTransaction { + assertFails { + other.executeInsert(SQL.INSERT_BOOK, arrayOf("The Da Vinci Code", "Dan Brown", 454, 16.96, byteArrayOf())) + } + } + other.executeInsert(SQL.INSERT_BOOK, arrayOf("The Da Vinci Code", "Dan Brown", 454, 16.96, byteArrayOf())) + } + } + } + + /** + * SQLite waits as long as `busyTimeout` for a lock that another connection holds. + */ + fun testBusyTimeout() { + openDatabase(getDefaultDBConfig(false).copy(busyTimeout = 1234)) { + assertEquals(1234L, it.withQuery("PRAGMA busy_timeout") { cursor -> cursor.next(); cursor.getLong(0) }) + } + } + @OptIn(DelicateCoroutinesApi::class, ExperimentalCoroutinesApi::class) fun testConcurrency() = runTest { val readWriteConfig = getDefaultDBConfig(false) diff --git a/sqllin-driver/src/jvmMain/kotlin/com/ctrip/sqllin/driver/JdbcDatabaseConnection.kt b/sqllin-driver/src/jvmMain/kotlin/com/ctrip/sqllin/driver/JdbcDatabaseConnection.kt index b5326d54..9bc2d59b 100644 --- a/sqllin-driver/src/jvmMain/kotlin/com/ctrip/sqllin/driver/JdbcDatabaseConnection.kt +++ b/sqllin-driver/src/jvmMain/kotlin/com/ctrip/sqllin/driver/JdbcDatabaseConnection.kt @@ -56,10 +56,18 @@ internal class JdbcDatabaseConnection(private val connection: Connection) : Abst private val isTransactionSuccess = AtomicBoolean(false) + /** + * Begins an IMMEDIATE transaction, which takes the write lock at once, as on every platform, or a deferred one on a + * read-only connection, which can't take it. + * + * It is begun with SQL, in auto-commit mode, rather than by leaving auto-commit mode: sqlite-jdbc begins another + * transaction of its configured kind as soon as one is committed or rolled back, which an IMMEDIATE one couldn't + * do while another connection writes, so that a COMMIT that succeeded would fail. + */ override fun beginTransaction() { if (isTransactionSuccess.get()) isTransactionSuccess.set(false) - connection.autoCommit = false + execSQL(if (connection.isReadOnly) "BEGIN" else "BEGIN IMMEDIATE", null) } override fun setTransactionSuccessful() { @@ -69,17 +77,16 @@ internal class JdbcDatabaseConnection(private val connection: Connection) : Abst override fun endTransaction() = try { if (isTransactionSuccess.get()) { try { - connection.commit() + execSQL("COMMIT", null) } catch (e: Exception) { // A COMMIT that fails, as one does while a deferred foreign key isn't satisfied, leaves the // transaction open, so that the statements after it would run in it - runCatching { connection.rollback() } + runCatching { execSQL("ROLLBACK", null) } throw e } } else - connection.rollback() + execSQL("ROLLBACK", null) } finally { - connection.autoCommit = true isTransactionSuccess.set(false) } diff --git a/sqllin-driver/src/jvmTest/kotlin/com/ctrip/sqllin/driver/test/JvmTest.kt b/sqllin-driver/src/jvmTest/kotlin/com/ctrip/sqllin/driver/test/JvmTest.kt index 6dac3f4f..736ebc58 100644 --- a/sqllin-driver/src/jvmTest/kotlin/com/ctrip/sqllin/driver/test/JvmTest.kt +++ b/sqllin-driver/src/jvmTest/kotlin/com/ctrip/sqllin/driver/test/JvmTest.kt @@ -49,6 +49,15 @@ class JvmTest { @Test fun testFailedCommit() = commonTest.testFailedCommit() + @Test + fun testRolledBackBySQLite() = commonTest.testRolledBackBySQLite() + + @Test + fun testImmediateTransaction() = commonTest.testImmediateTransaction() + + @Test + fun testBusyTimeout() = commonTest.testBusyTimeout() + @Test fun testConcurrency() = commonTest.testConcurrency() diff --git a/sqllin-driver/src/nativeMain/kotlin/com/ctrip/sqllin/driver/RealDatabaseConnection.kt b/sqllin-driver/src/nativeMain/kotlin/com/ctrip/sqllin/driver/RealDatabaseConnection.kt index 4d60f72c..ef1b8838 100644 --- a/sqllin-driver/src/nativeMain/kotlin/com/ctrip/sqllin/driver/RealDatabaseConnection.kt +++ b/sqllin-driver/src/nativeMain/kotlin/com/ctrip/sqllin/driver/RealDatabaseConnection.kt @@ -72,8 +72,9 @@ internal class RealDatabaseConnection( override fun query(sql: String, bindParams: Array?): CommonCursor = bindParamsToSQL(sql, bindParams).query() + // IMMEDIATE takes the write lock at once, as on every platform; a read-only connection can't take it override fun beginTransaction() = transactionLock.withLock { - database.rawExecSql("BEGIN;") + database.rawExecSql(if (database.isActualReadOnly) "BEGIN;" else "BEGIN IMMEDIATE;") transaction.value = Transaction(isSuccessful = false) } diff --git a/sqllin-driver/src/nativeMain/kotlin/com/ctrip/sqllin/driver/cinterop/NativeDatabase.kt b/sqllin-driver/src/nativeMain/kotlin/com/ctrip/sqllin/driver/cinterop/NativeDatabase.kt index f6435d39..43fbe5f7 100644 --- a/sqllin-driver/src/nativeMain/kotlin/com/ctrip/sqllin/driver/cinterop/NativeDatabase.kt +++ b/sqllin-driver/src/nativeMain/kotlin/com/ctrip/sqllin/driver/cinterop/NativeDatabase.kt @@ -59,7 +59,7 @@ internal class NativeDatabase private constructor(val dbPointer: CPointer): Int = memScoped { + sqlite3_exec(db, "CREATE TABLE t(x); INSERT INTO t VALUES (1); SELECT * FROM t;", null, null, null) + val current = alloc() + val highwater = alloc() + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, current.ptr, highwater.ptr, 0) + highwater.value + } + fun lookasideUsed(config: DatabaseConfiguration): Int { + val database = NativeDatabase.openNativeDatabase(config, ":memory:") + try { + return lookasideUsed(database.dbPointer) + } finally { + database.close() + } + } + val sqliteDefault = memScoped { + val db = alloc>() + sqlite3_open_v2(":memory:", db.ptr, SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE, null) + try { + lookasideUsed(db.value!!) + } finally { + sqlite3_close_v2(db.value) + } + } + val config = DatabaseConfiguration(name = SQL.DATABASE_NAME, path = path, version = 1) + assertEquals(sqliteDefault, lookasideUsed(config)) + assertEquals(0, lookasideUsed(config.copy(lookasideSlotSize = 0, lookasideSlotCount = 0))) + assertTrue(lookasideUsed(config.copy(lookasideSlotSize = 256, lookasideSlotCount = 32)) > 0) + } + @AfterTest fun setDown() { deleteDatabase(path, SQL.DATABASE_NAME) diff --git a/sqllin-dsl-test/src/androidHostTest/kotlin/com/ctrip/sqllin/dsl/test/AndroidTest.kt b/sqllin-dsl-test/src/androidHostTest/kotlin/com/ctrip/sqllin/dsl/test/AndroidTest.kt index d5e14c47..e4586cba 100644 --- a/sqllin-dsl-test/src/androidHostTest/kotlin/com/ctrip/sqllin/dsl/test/AndroidTest.kt +++ b/sqllin-dsl-test/src/androidHostTest/kotlin/com/ctrip/sqllin/dsl/test/AndroidTest.kt @@ -54,6 +54,9 @@ class AndroidTest { @Test fun testObserve() = commonTest.testObserve() + @Test + fun testObserveRestoresTracking() = commonTest.testObserveRestoresTracking() + @Test fun testObserveForeignKeyAction() = commonTest.testObserveForeignKeyAction() diff --git a/sqllin-dsl-test/src/commonMain/kotlin/com/ctrip/sqllin/dsl/test/CommonBasicTest.kt b/sqllin-dsl-test/src/commonMain/kotlin/com/ctrip/sqllin/dsl/test/CommonBasicTest.kt index 8483ed38..f47dcf59 100644 --- a/sqllin-dsl-test/src/commonMain/kotlin/com/ctrip/sqllin/dsl/test/CommonBasicTest.kt +++ b/sqllin-dsl-test/src/commonMain/kotlin/com/ctrip/sqllin/dsl/test/CommonBasicTest.kt @@ -593,6 +593,38 @@ class CommonBasicTest(private val path: DatabasePath) { } } + /** + * Covers the tracker of observed queries creating its TEMP table and triggers again where they are gone, as a + * rollback on another thread can take them on the JVM and Kotlin/Native. Removing the row of a table from the TEMP + * table, and dropping it, stand for that rollback. + */ + @OptIn(ExperimentalCoroutinesApi::class, ExperimentalDSLDatabaseAPI::class) + fun testObserveRestoresTracking() = Database(getNewAPIDBConfig()).databaseAutoClose { database -> + runTest { + val books = Channel>(Channel.UNLIMITED) + backgroundScope.launch { + database.observe(EmptyCoroutineContext) { BookTable SELECT X }.collect { books.send(it) } + } + fun newBooks(): List? { + runCurrent() + return books.tryReceive().getOrNull() + } + assertEquals(emptyList(), books.receive()) + + val first = Book(name = "Kotlin Cookbook", author = "Ken Kousen", price = 37.72, pages = 251) + database { ModificationLogTable DELETE WHERE(ModificationLogTable.table_name EQ "book") } + assertEquals(null, newBooks()) + database { BookTable INSERT first } + assertEquals(listOf(first), newBooks()) + + val second = Book(name = "The Lost Symbol", author = "Dan Brown", price = 19.95, pages = 510) + database { DROP(ModificationLogTable) } + assertEquals(null, newBooks()) + database { BookTable INSERT second } + assertEquals(listOf(first, second), newBooks()) + } + } + /** * Covers an observed query of a table that only a foreign key action changes: SQLite reports the rows that * ON DELETE CASCADE deletes, though no statement names their table. diff --git a/sqllin-dsl-test/src/commonMain/kotlin/com/ctrip/sqllin/dsl/test/Entities.kt b/sqllin-dsl-test/src/commonMain/kotlin/com/ctrip/sqllin/dsl/test/Entities.kt index 2996f930..2fe90739 100644 --- a/sqllin-dsl-test/src/commonMain/kotlin/com/ctrip/sqllin/dsl/test/Entities.kt +++ b/sqllin-dsl-test/src/commonMain/kotlin/com/ctrip/sqllin/dsl/test/Entities.kt @@ -1019,3 +1019,12 @@ data class PersonBookTitle(val age: Age, val title: String?) */ @Serializable data class AuthorCode(val age: Age, val code: Code) + +/** + * The TEMP table in which SQLlin counts the changes of the tables of observed queries, which a test removes rows of, + * and drops, as a rollback on another thread can. + */ +@Suppress("PropertyName") +@DBRow("sqllin_modification_log") +@Serializable +data class ModificationLog(val table_name: String, val version: Long) diff --git a/sqllin-dsl-test/src/jvmTest/kotlin/com/ctrip/sqllin/dsl/test/JvmTest.kt b/sqllin-dsl-test/src/jvmTest/kotlin/com/ctrip/sqllin/dsl/test/JvmTest.kt index 3d277003..50053aa7 100644 --- a/sqllin-dsl-test/src/jvmTest/kotlin/com/ctrip/sqllin/dsl/test/JvmTest.kt +++ b/sqllin-dsl-test/src/jvmTest/kotlin/com/ctrip/sqllin/dsl/test/JvmTest.kt @@ -46,6 +46,9 @@ class JvmTest { @Test fun testObserve() = commonTest.testObserve() + @Test + fun testObserveRestoresTracking() = commonTest.testObserveRestoresTracking() + @Test fun testObserveForeignKeyAction() = commonTest.testObserveForeignKeyAction() diff --git a/sqllin-dsl-test/src/nativeTest/kotlin/com/ctrip/sqllin/dsl/test/NativeTest.kt b/sqllin-dsl-test/src/nativeTest/kotlin/com/ctrip/sqllin/dsl/test/NativeTest.kt index ef0ee1d6..92e1121d 100644 --- a/sqllin-dsl-test/src/nativeTest/kotlin/com/ctrip/sqllin/dsl/test/NativeTest.kt +++ b/sqllin-dsl-test/src/nativeTest/kotlin/com/ctrip/sqllin/dsl/test/NativeTest.kt @@ -62,6 +62,9 @@ class NativeTest { @Test fun testObserve() = commonTest.testObserve() + @Test + fun testObserveRestoresTracking() = commonTest.testObserveRestoresTracking() + @Test fun testObserveForeignKeyAction() = commonTest.testObserveForeignKeyAction() diff --git a/sqllin-dsl/doc/advanced-query-cn.md b/sqllin-dsl/doc/advanced-query-cn.md index d55cd7f3..bcc59370 100644 --- a/sqllin-dsl/doc/advanced-query-cn.md +++ b/sqllin-dsl/doc/advanced-query-cn.md @@ -400,6 +400,10 @@ _DELETE_ 也不会。 只有经由同一个 `Database` 实例的修改才能被感知:同一个文件上的其他实例或连接(比如用 _sqllin-driver_ 打开的连接),以及其他 进程所做的修改,都感知不到。 +查询在写入线程以外的线程上执行,所以数据库是在多个线程间共享的:请像[并发安全](concurrency-safety-cn.md)中说的那样,用 +`suspendedScope` 写入。阻塞式的 `database { }` 不会和这些查询串行执行:在 JVM 和 Kotlin/Native 上,连接是按语句而不是按事务加锁 +的,所以在它的事务开着时,这些查询可能在它的事务里执行。 + 要组合几条 _SELECT_ 的结果,可以用 kotlinx.coroutines 的 `combine` 组合观察它们的几个 Flow。由于每个 Flow 各自执行查询,一次 同时影响两边的表的修改,可能会短暂地呈现一边的新结果和另一边的旧结果。如果结果来自几张表,用一条带 Join 的 _SELECT_ 可以一次 得到。 diff --git a/sqllin-dsl/doc/advanced-query.md b/sqllin-dsl/doc/advanced-query.md index e346551a..ef4a377f 100644 --- a/sqllin-dsl/doc/advanced-query.md +++ b/sqllin-dsl/doc/advanced-query.md @@ -428,6 +428,11 @@ that matches no rows. Only the changes made through the same `Database` instance are seen: not those of another instance or connection on the same file, such as one opened with _sqllin-driver_, nor those of another process. +The queries run on another thread than the one that writes, so the database is shared between threads: write through +`suspendedScope`, as [Concurrency Safety](concurrency-safety.md) says. A blocking `database { }` isn't serialized with +the queries: on the JVM and Kotlin/Native, where the connection is locked for each statement rather than for a +transaction, they can run in its transaction while it is open. + To combine the results of several _SELECT_ statements, combine the flows that observe them, with `combine` from kotlinx.coroutines. As each runs its query on its own, a change to the tables of both may briefly show the new results of one with the old results of the other. When the results come from several tables, a single _SELECT_ with a join gives diff --git a/sqllin-dsl/doc/concurrency-safety-cn.md b/sqllin-dsl/doc/concurrency-safety-cn.md index 8aff7434..2aaa80de 100644 --- a/sqllin-dsl/doc/concurrency-safety-cn.md +++ b/sqllin-dsl/doc/concurrency-safety-cn.md @@ -29,8 +29,8 @@ fun sample() { 我们使用新 API `Database#suspendedScope` 来代替旧 `database {}` 后,将会是这样: ```kotlin -fun sample() { - database suspendScope { +suspend fun sample() { + database suspendedScope { PersonTable { table -> table INSERT Person(age = 4, name = "Tom") table INSERT listOf( diff --git a/sqllin-dsl/doc/concurrency-safety.md b/sqllin-dsl/doc/concurrency-safety.md index 9d8c270c..5885004d 100644 --- a/sqllin-dsl/doc/concurrency-safety.md +++ b/sqllin-dsl/doc/concurrency-safety.md @@ -34,8 +34,8 @@ fun sample() { We use the new API `Database#suspendedScope` to replace the `database {}`, it will be like that: ```kotlin -fun sample() { - database suspendScope { +suspend fun sample() { + database suspendedScope { PersonTable { table -> table INSERT Person(age = 4, name = "Tom") table INSERT listOf( diff --git a/sqllin-dsl/doc/getting-start-cn.md b/sqllin-dsl/doc/getting-start-cn.md index 22c3b643..0cbc7d65 100644 --- a/sqllin-dsl/doc/getting-start-cn.md +++ b/sqllin-dsl/doc/getting-start-cn.md @@ -124,8 +124,8 @@ val database = Database( journalMode = JournalMode.WAL, synchronousMode = SynchronousMode.NORMAL, busyTimeout = 5000, - lookasideSlotSize = 0, - lookasideSlotCount = 0, + lookasideSlotSize = -1, + lookasideSlotCount = -1, create = { it.execSQL("create table person (id integer primary key autoincrement, name text, age integer)") }, @@ -133,8 +133,10 @@ val database = Database( ) ) ``` -注意,由于 Android Framework 的限制,`inMemory`、`journalMode`、`lookasideSlotSize`、`lookasideSlotCount` 这些参数仅在 Android 9 及以上版本生效。 并且,由于 -[sqlite-jdbc](https://github.com/xerial/sqlite-jdbc)(SQLlin 在 JVM 上基于它)不支持 `sqlite3_config()`,`lookasideSlotSize` 和 `lookasideSlotCount` 两个属性在 JVM 平台不生效。 +`lookasideSlotSize` 和 `lookasideSlotCount` 用来配置 SQLite 的 lookaside 内存分配器:默认值 -1 表示保持 SQLite 的默认配置,0 表示关闭它。 + +注意,由于 Android Framework 的限制,`inMemory`、`lookasideSlotSize`、`lookasideSlotCount` 这些参数仅在 Android 9 及以上版本生效。 并且,由于 +[sqlite-jdbc](https://github.com/xerial/sqlite-jdbc)(SQLlin 在 JVM 上基于它)不支持 `sqlite3_db_config()`,`lookasideSlotSize` 和 `lookasideSlotCount` 两个属性在 JVM 平台不生效。 ### 使用 DSLDBConfiguration 进行类型安全的模式管理 @@ -154,8 +156,8 @@ val database = Database( journalMode = JournalMode.WAL, synchronousMode = SynchronousMode.NORMAL, busyTimeout = 5000, - lookasideSlotSize = 0, - lookasideSlotCount = 0, + lookasideSlotSize = -1, + lookasideSlotCount = -1, create = { // Use type-safe DSL instead of raw SQL CREATE(PersonTable) diff --git a/sqllin-dsl/doc/getting-start.md b/sqllin-dsl/doc/getting-start.md index a81105bf..bc561d4d 100644 --- a/sqllin-dsl/doc/getting-start.md +++ b/sqllin-dsl/doc/getting-start.md @@ -131,8 +131,8 @@ val database = Database( journalMode = JournalMode.WAL, synchronousMode = SynchronousMode.NORMAL, busyTimeout = 5000, - lookasideSlotSize = 0, - lookasideSlotCount = 0, + lookasideSlotSize = -1, + lookasideSlotCount = -1, create = { it.execSQL("create table person (id integer primary key autoincrement, name text, age integer)") }, @@ -141,9 +141,12 @@ val database = Database( ) ``` -Note, because of limitation by Android Framework, the `inMemory`, `busyTimeout`, `lookasideSlotSize`, `lookasideSlotCount` +`lookasideSlotSize` and `lookasideSlotCount` configure SQLite's lookaside memory allocator: -1, their default, leaves +SQLite's default configuration, and 0 disables the allocator. + +Note, because of limitation by Android Framework, the `inMemory`, `lookasideSlotSize`, `lookasideSlotCount` only work on Android 9 and higher. And, because [sqlite-jdbc](https://github.com/xerial/sqlite-jdbc)(SQLlin is based on it on JVM) doesn't support -`sqlite3_config()`, the `lookasideSlotSize` and `lookasideSlotCount` don't work on JVM target. +`sqlite3_db_config()`, the `lookasideSlotSize` and `lookasideSlotCount` don't work on JVM target. ### Using DSLDBConfiguration for Type-Safe Schema Management @@ -163,8 +166,8 @@ val database = Database( journalMode = JournalMode.WAL, synchronousMode = SynchronousMode.NORMAL, busyTimeout = 5000, - lookasideSlotSize = 0, - lookasideSlotCount = 0, + lookasideSlotSize = -1, + lookasideSlotCount = -1, create = { // Use type-safe DSL instead of raw SQL CREATE(PersonTable) diff --git a/sqllin-dsl/doc/modify-database-and-transaction-cn.md b/sqllin-dsl/doc/modify-database-and-transaction-cn.md index c94807cd..0aa20dbb 100644 --- a/sqllin-dsl/doc/modify-database-and-transaction-cn.md +++ b/sqllin-dsl/doc/modify-database-and-transaction-cn.md @@ -605,6 +605,8 @@ fun sample() { `transaction {...}` 是 `Database` 的成员函数,将它写在 `TABLE(databaseName) {...}` 函数的内部或外部没有特别限制。 +在所有平台上,事务都以 `BEGIN IMMEDIATE` 开始:它一开始就拿到写锁,所以要写入的其他连接或者其他事务,最多会按 `busyTimeout` 等待它结束。 + ## VACUUM 删除行并不会让数据库文件变小,因为 SQLite 会留着空闲页以便复用。`VACUUM()` 会重建数据库文件,把它压缩到最小的磁盘空间; diff --git a/sqllin-dsl/doc/modify-database-and-transaction.md b/sqllin-dsl/doc/modify-database-and-transaction.md index 318ec34b..90e6c361 100644 --- a/sqllin-dsl/doc/modify-database-and-transaction.md +++ b/sqllin-dsl/doc/modify-database-and-transaction.md @@ -642,6 +642,9 @@ fun sample() { The `transaction {...}` is a member function in `Database`, it is inside or outside of `TABLE(databaseName) {...}` doesn't matter. +A transaction begins with `BEGIN IMMEDIATE` on every platform: it takes the write lock at once, so another connection +that writes, or another transaction, waits for it as long as its `busyTimeout`. + ## VACUUM Deleting rows doesn't shrink the database file, as SQLite keeps the free pages to reuse them. `VACUUM()` rebuilds the diff --git a/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/DSLDBConfiguration.kt b/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/DSLDBConfiguration.kt index d7cbb207..adecb427 100644 --- a/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/DSLDBConfiguration.kt +++ b/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/DSLDBConfiguration.kt @@ -35,8 +35,12 @@ import com.ctrip.sqllin.driver.SynchronousMode * @property journalMode The SQLite journal mode. Default: [JournalMode.WAL] * @property synchronousMode The SQLite synchronous mode. Default: [SynchronousMode.NORMAL] * @property busyTimeout Timeout in milliseconds for lock waits. Default: 5000ms - * @property lookasideSlotSize Size of lookaside memory slots. Default: 0 - * @property lookasideSlotCount Number of lookaside memory slots. Default: 0 + * @property lookasideSlotSize The size in bytes of each slot of SQLite's lookaside memory allocator, set together with + * [lookasideSlotCount] when both are 0 or more. Default: -1, which leaves SQLite's default configuration; 0 disables the + * allocator. Android 9 and higher take it when both are positive or both 0, and lower versions don't take it; on the + * JVM, where sqlite-jdbc can't set it, it has no effect + * @property lookasideSlotCount The number of slots of SQLite's lookaside memory allocator of each connection, set + * together with [lookasideSlotSize]. Default: -1, which leaves SQLite's default configuration; 0 disables the allocator * @property create Callback invoked when creating a new database, executed within a [DatabaseScope] * @property upgrade Callback invoked when upgrading the schema, executed within a [DatabaseScope] * @@ -51,8 +55,8 @@ public data class DSLDBConfiguration( val journalMode: JournalMode = JournalMode.WAL, val synchronousMode: SynchronousMode = SynchronousMode.NORMAL, val busyTimeout: Int = 5000, - val lookasideSlotSize: Int = 0, - val lookasideSlotCount: Int = 0, + val lookasideSlotSize: Int = -1, + val lookasideSlotCount: Int = -1, val create: DatabaseScope.() -> Unit = {}, val upgrade: DatabaseScope.(oldVersion: Int, newVersion: Int) -> Unit = { _, _ -> }, ) { diff --git a/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/Database.kt b/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/Database.kt index 8ddbfccb..14753c88 100644 --- a/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/Database.kt +++ b/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/Database.kt @@ -67,6 +67,11 @@ public class Database internal constructor( * All SQL operations within the block are executed when the scope exits. * Statements are batched and executed in order. * + * Unlike [suspendedScope], it isn't serialized with the scopes of other threads, nor with the queries of + * [observe], which run on `Dispatchers.IO` by default: on the JVM and Kotlin/Native, where the connection is locked + * for each statement rather than for a transaction, their statements can run in its transaction. To share the + * database between threads, use [suspendedScope]. + * * @param block The DSL block containing SQL operations * @return The result of the block */ @@ -91,16 +96,19 @@ public class Database internal constructor( /** * Runs the statements of [databaseScope], then tells the observed queries what they changed. They are told even - * when a statement fails, as the ones before it may have changed something. + * when a statement fails, as the ones before it may have changed something, and then a failure to tell them is + * added to the statement's, rather than taking its place. */ private fun execute(databaseScope: DatabaseScope) { val changes = databaseScope.changes val writtenTables = databaseScope.writtenTables try { databaseScope.executeAllStatements() - } finally { - invalidationTracker?.refresh(changes, writtenTables) + } catch (e: Throwable) { + runCatching { invalidationTracker?.refresh(changes, writtenTables) }.exceptionOrNull()?.let(e::addSuppressed) + throw e } + invalidationTracker?.refresh(changes, writtenTables) } private val executiveMutex by lazy { Mutex() } @@ -140,7 +148,8 @@ public class Database internal constructor( * UPDATE or DELETE that matches no rows. Several changes in a row may lead to a single query. * * Only changes made through this database are seen, not those of another database instance or connection on the - * same file, or of another process. To combine the results of several SELECTs, combine the flows that observe them; + * same file, or of another process. The queries run in [context], on another thread than the one that writes, so + * write through [suspendedScope], which they are serialized with, rather than [invoke], which they aren't. To combine the results of several SELECTs, combine the flows that observe them; * as each runs its query on its own, a change to the tables of both may briefly show the new results of one with the * old results of the other. * diff --git a/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/InvalidationTracker.kt b/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/InvalidationTracker.kt index 48fbed3f..2f56a5bd 100644 --- a/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/InvalidationTracker.kt +++ b/sqllin-dsl/src/commonMain/kotlin/com/ctrip/sqllin/dsl/InvalidationTracker.kt @@ -43,7 +43,10 @@ import kotlinx.coroutines.flow.update * only means the queries run once more. * * TEMP tables and triggers belong to the connection, so changes made through another connection, or by another - * process, aren't seen. A table stays tracked until the connection closes. + * process, aren't seen. A table stays tracked until the connection closes. They are also rolled back with the + * transaction they are created in: on the JVM and Kotlin/Native, the connection is locked for each statement rather + * than for a transaction, so a blocking database scope on another thread can have one open while [track] creates them, + * and its rollback takes them with it. [refresh] creates them again then. * * @author Yuang Qiao */ @@ -158,7 +161,10 @@ internal class InvalidationTracker(private val connection: DatabaseConnection) { val virtual = tables intersect virtualTables() virtualTables.value = virtual triggeredTables.value = if (virtual.isEmpty()) emptySet() else triggeredTables() - for (table in tables - virtual) + } + restoreLostObjects(tables) + if (changes == Changes.SCHEMA) { + for (table in tables - virtualTables.value) createTriggers(table) connection.execSQL(COUNT_ALL) } else { @@ -182,6 +188,42 @@ internal class InvalidationTracker(private val connection: DatabaseConnection) { } } + /** + * Creates the TEMP table and the triggers of the tracked [tables] again where they are gone, as a rollback can take + * them, and counts a change of each table whose row or triggers were gone, as its changes may have been missed. + */ + private fun restoreLostObjects(tables: Set) { + // On Android, a query outside a transaction may run on a reader connection, which has no TEMP objects + val (loggedTables, triggerCounts) = connection.withTransaction { + val logExists = it.withQuery(LOG_EXISTS) { cursor -> cursor.next() && cursor.getLong(0) > 0 } + val logged = if (logExists) it.withQuery(SELECT_LOGGED_TABLES) { cursor -> + buildSet { + cursor.forEachRow { add(cursor.getString(0)!!) } + } + } else null + val triggers = it.withQuery(COUNT_TRIGGERS) { cursor -> + buildMap { + cursor.forEachRow { put(cursor.getString(0)!!, cursor.getLong(1)) } + } + } + logged to triggers + } + val virtual = virtualTables.value + val lostTables = tables.filter { table -> + loggedTables == null || table !in loggedTables || (table !in virtual && (triggerCounts[table] ?: 0L) < OPERATIONS.size) + } + if (lostTables.isEmpty()) + return + connection.execSQL(CREATE_LOG) + for (table in lostTables) { + // Above the version last read, so that the queries that read the table run again + connection.execSQL(INSERT_LOG_VERSION, arrayOf(table, _versions.value[table] ?: 0L)) + connection.execSQL(COUNT_TABLE, arrayOf(table)) + if (table !in virtual) + createTriggers(table) + } + } + /** * Creates the triggers of [table] unless they exist. A table that doesn't exist, as after DROP, gets them once it * is created again. @@ -211,8 +253,13 @@ internal class InvalidationTracker(private val connection: DatabaseConnection) { const val LOG = "sqllin_modification_log" const val CREATE_LOG = "CREATE TEMP TABLE IF NOT EXISTS $LOG(table_name TEXT PRIMARY KEY NOT NULL, version INTEGER NOT NULL DEFAULT 0)" const val INSERT_LOG_ROW = "INSERT OR IGNORE INTO $LOG(table_name) VALUES(?)" + const val INSERT_LOG_VERSION = "INSERT OR IGNORE INTO $LOG(table_name, version) VALUES(?, ?)" const val COUNT_ALL = "UPDATE $LOG SET version = version + 1" + const val COUNT_TABLE = "$COUNT_ALL WHERE table_name = ?" const val SELECT_LOG = "SELECT table_name, version FROM $LOG" + const val SELECT_LOGGED_TABLES = "SELECT table_name FROM $LOG" + const val LOG_EXISTS = "SELECT count(*) FROM sqlite_temp_master WHERE type = 'table' AND name = '$LOG'" + const val COUNT_TRIGGERS = "SELECT tbl_name, count(*) FROM sqlite_temp_master WHERE type = 'trigger' AND name LIKE 'sqllin\\_modification\\_trigger\\_%' ESCAPE '\\' GROUP BY tbl_name" val OPERATIONS = listOf("INSERT", "UPDATE", "DELETE") fun identifier(name: String): String = "\"${name.replace("\"", "\"\"")}\""