From 700642d247df680a07548640846721cc9221ed22 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 6 Jan 2025 23:48:18 +0100 Subject: [PATCH 01/10] Initial support for encryption on the web --- sqlite3/assets/wasm/CMakeLists.txt | 8 +++++--- sqlite3/assets/wasm/helpers.c | 6 +++++- sqlite3/assets/wasm/os_web.c | 11 ++++++++++- sqlite3/example/web/main.dart | 5 +++-- sqlite3/lib/src/vfs.dart | 18 +++++++++++++++--- sqlite3/lib/src/wasm/bindings.dart | 14 ++++++++++++++ sqlite3/lib/src/wasm/wasm_interop.dart | 23 ++++++++++++++++++++--- 7 files changed, 72 insertions(+), 13 deletions(-) diff --git a/sqlite3/assets/wasm/CMakeLists.txt b/sqlite3/assets/wasm/CMakeLists.txt index 71f5a747..3cdb4717 100644 --- a/sqlite3/assets/wasm/CMakeLists.txt +++ b/sqlite3/assets/wasm/CMakeLists.txt @@ -12,7 +12,7 @@ include(FetchContent) FetchContent_Declare( sqlite3 # NOTE: When changing this, also update `test/wasm/sqlite3_test.dart` - URL https://sqlite.org/2024/sqlite-autoconf-3470200.tar.gz + URL https://github.com/utelle/SQLite3MultipleCiphers/releases/download/v1.9.2/sqlite3mc-1.9.2-sqlite-3.47.2-amalgamation.zip DOWNLOAD_EXTRACT_TIMESTAMP NEW ) @@ -38,13 +38,13 @@ macro(base_sqlite3_target name debug) set(sources ${CMAKE_CURRENT_SOURCE_DIR}/os_web.c ${CMAKE_CURRENT_SOURCE_DIR}/helpers.c - ${sqlite3_SOURCE_DIR}/sqlite3.c + ${sqlite3_SOURCE_DIR}/sqlite3mc_amalgamation.c ) set(flags -Wall -Wextra -Wno-unused-parameter -Wno-unused-function) if(${debug}) list(APPEND sources "${CMAKE_BINARY_DIR}/vfstrace.c") - list(APPEND flags "-g" "-DDEBUG") + list(APPEND flags "-g" "-DDEBUG" "-O1") else() list(APPEND flags "-Oz" "-DNDEBUG" "-flto") endif() @@ -56,6 +56,8 @@ macro(base_sqlite3_target name debug) -o ${clang_output} -I ${PROJECT_SOURCE_DIR} -I ${sqlite3_SOURCE_DIR} -D_HAVE_SQLITE_CONFIG_H + -DSQLITE_OMIT_AUTOINIT + -D__WASM__ -mcpu=generic -mexec-model=reactor -fno-stack-protector -fno-stack-clash-protection diff --git a/sqlite3/assets/wasm/helpers.c b/sqlite3/assets/wasm/helpers.c index 380104ca..90b4e992 100644 --- a/sqlite3/assets/wasm/helpers.c +++ b/sqlite3/assets/wasm/helpers.c @@ -187,7 +187,11 @@ SQLITE_API sqlite3_vfs *dart_sqlite3_register_vfs(const char *name, int dartId, vfstrace_register(traceName, name, &dartvfs_trace_log1, NULL, makeDefault); #else // Just register the VFS as is. - sqlite3_vfs_register(vfs, makeDefault); + int rc = sqlite3_vfs_register(vfs, makeDefault); + if (rc) { + free(vfs); + return NULL; + } #endif return vfs; } diff --git a/sqlite3/assets/wasm/os_web.c b/sqlite3/assets/wasm/os_web.c index 98004432..35e6db42 100644 --- a/sqlite3/assets/wasm/os_web.c +++ b/sqlite3/assets/wasm/os_web.c @@ -5,6 +5,15 @@ #include "bridge.h" #include "sqlite3.h" -int sqlite3_os_init(void) { return SQLITE_OK; } +extern int sqlite3_powersync_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi); + +int getentropy(void* buf, size_t n) { + return xRandomness(-1, (int) n, (char*) buf); +} + +int sqlite3_os_init(void) { + return SQLITE_OK; +} int sqlite3_os_end(void) { return SQLITE_OK; } diff --git a/sqlite3/example/web/main.dart b/sqlite3/example/web/main.dart index f7bf0bfc..9adb623c 100644 --- a/sqlite3/example/web/main.dart +++ b/sqlite3/example/web/main.dart @@ -15,11 +15,12 @@ Future main() async { print(sqlite3.version); sqlite3.registerVirtualFileSystem( - await IndexedDbFileSystem.open(dbName: 'sqlite3-example'), + InMemoryFileSystem(), makeDefault: true, ); - sqlite3.open('/database') + sqlite3.open('/database', vfs: 'multipleciphers-dart-memory') + ..execute("pragma key = 'test';") ..execute('pragma user_version = 1') ..execute('CREATE TABLE foo (bar INTEGER NOT NULL);') ..execute('INSERT INTO foo (bar) VALUES (?)', [3]) diff --git a/sqlite3/lib/src/vfs.dart b/sqlite3/lib/src/vfs.dart index 7cad5883..88395584 100644 --- a/sqlite3/lib/src/vfs.dart +++ b/sqlite3/lib/src/vfs.dart @@ -132,18 +132,30 @@ abstract base class BaseVirtualFileSystem extends VirtualFileSystem { final Random random; BaseVirtualFileSystem({Random? random, required String name}) - : random = random ?? Random.secure(), + : random = random ?? _fallbackRandom, super(name); @override void xRandomness(Uint8List target) { + generateRandomness(target, random); + } + + @override + DateTime xCurrentTime() => DateTime.now(); + + /// Fills [target] with random bytes. + /// + /// An optional [random] source can be provided, otherwise a default instance + /// of [Random.secure] will be used. + static void generateRandomness(Uint8List target, [Random? random]) { + random ??= _fallbackRandom; + for (var i = 0; i < target.length; i++) { target[i] = random.nextInt(1 << 8); } } - @override - DateTime xCurrentTime() => DateTime.now(); + static final Random _fallbackRandom = Random.secure(); } /// A [VirtualFileSystemFile] base class that implements [xRead] to zero-fill diff --git a/sqlite3/lib/src/wasm/bindings.dart b/sqlite3/lib/src/wasm/bindings.dart index f6245d2d..bf210817 100644 --- a/sqlite3/lib/src/wasm/bindings.dart +++ b/sqlite3/lib/src/wasm/bindings.dart @@ -5,6 +5,7 @@ import 'dart:typed_data'; import 'package:sqlite3/src/vfs.dart'; import '../constants.dart'; +import '../exception.dart'; import '../functions.dart'; import '../implementation/bindings.dart'; import 'wasm_interop.dart' as wasm; @@ -55,6 +56,8 @@ final class WasmSqliteBindings extends RawSqliteBindings { @override SqliteResult sqlite3_open_v2( String name, int flags, String? zVfs) { + sqlite3_initialize(); + final namePtr = bindings.allocateZeroTerminated(name); final outDb = bindings.malloc(wasm.WasmBindings.pointerSize); final vfsPtr = zVfs == null ? 0 : bindings.allocateZeroTerminated(zVfs); @@ -76,12 +79,23 @@ final class WasmSqliteBindings extends RawSqliteBindings { return bindings.memory.readString(bindings.sqlite3_sourceid()); } + void sqlite3_initialize() { + final rc = bindings.sqlite3_initialize(); + if (rc != 0) { + throw SqliteException(rc, 'sqlite3_initialize call failed'); + } + } + @override void registerVirtualFileSystem(VirtualFileSystem vfs, int makeDefault) { final name = bindings.allocateZeroTerminated(vfs.name); final id = bindings.callbacks.registerVfs(vfs); final ptr = bindings.dart_sqlite3_register_vfs(name, id, makeDefault); + if (ptr == 0) { + throw StateError('could not register vfs'); + } + sqlite3_initialize(); DartCallbacks.sqliteVfsPointer[vfs] = ptr; } diff --git a/sqlite3/lib/src/wasm/wasm_interop.dart b/sqlite3/lib/src/wasm/wasm_interop.dart index 75a1ca94..1f9d1d68 100644 --- a/sqlite3/lib/src/wasm/wasm_interop.dart +++ b/sqlite3/lib/src/wasm/wasm_interop.dart @@ -83,7 +83,7 @@ class WasmBindings { _sqlite3_stmt_readonly, _sqlite3_stmt_isexplain; - final JSFunction? _sqlite3_db_config; + final JSFunction? _sqlite3_db_config, _sqlite3_initialize; final Global _sqlite3_temp_directory; @@ -160,7 +160,9 @@ class WasmBindings { _sqlite3_stmt_isexplain = instance.functions['sqlite3_stmt_isexplain']!, _sqlite3_stmt_readonly = instance.functions['sqlite3_stmt_readonly']!, _sqlite3_db_config = instance.functions['dart_sqlite3_db_config_int'], + _sqlite3_initialize = instance.functions['sqlite3_initialize'], _sqlite3_temp_directory = instance.globals['sqlite3_temp_directory']! + // Note when adding new fields: We remove functions from the wasm module that // aren't referenced in Dart. We consider a symbol used when it appears in a // string literal in an initializer of this constructor (`tool/wasm_dce.dart`). @@ -210,6 +212,13 @@ class WasmBindings { void sqlite3_free(Pointer ptr) => _sqlite3_free.callReturningVoid(ptr.toJS); + int sqlite3_initialize() { + return switch (_sqlite3_initialize) { + final fun? => fun.callReturningInt0(), + null => 0, + }; + } + int create_scalar_function( Pointer db, Pointer functionName, int nArg, int eTextRep, int id) { return _create_scalar.callReturningInt5( @@ -601,10 +610,18 @@ class _InjectedValues { }); }).toJS, 'xRandomness': ((int vfsId, int nByte, Pointer zOut) { - final vfs = callbacks.registeredVfs[vfsId]!; + final vfs = callbacks.registeredVfs[vfsId]; return _runVfs(() { - vfs.xRandomness(memory.buffer.toDart.asUint8List(zOut, nByte)); + final target = memory.buffer.toDart.asUint8List(zOut, nByte); + + if (vfs != null) { + vfs.xRandomness(target); + } else { + // Fall back to a default random source. We're using this to + // implement `getentropy` in C which is used by sqlite3mc. + return BaseVirtualFileSystem.generateRandomness(target); + } }); }).toJS, 'xSleep': ((int vfsId, int micros) { From 132b821b9bba1b951b1e47c6a7ba4f48f887e0b6 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 10 Jan 2025 22:51:18 +0100 Subject: [PATCH 02/10] Keep compiling regular sqlite3 --- sqlite3/assets/wasm/CMakeLists.txt | 29 ++++++++++++++++++++++------- sqlite3/assets/wasm/getentropy.c | 9 +++++++++ sqlite3/assets/wasm/os_web.c | 12 +----------- 3 files changed, 32 insertions(+), 18 deletions(-) create mode 100644 sqlite3/assets/wasm/getentropy.c diff --git a/sqlite3/assets/wasm/CMakeLists.txt b/sqlite3/assets/wasm/CMakeLists.txt index 3cdb4717..869256e3 100644 --- a/sqlite3/assets/wasm/CMakeLists.txt +++ b/sqlite3/assets/wasm/CMakeLists.txt @@ -12,11 +12,18 @@ include(FetchContent) FetchContent_Declare( sqlite3 # NOTE: When changing this, also update `test/wasm/sqlite3_test.dart` + URL https://sqlite.org/2024/sqlite-autoconf-3470200.tar.gz + DOWNLOAD_EXTRACT_TIMESTAMP NEW +) + +FetchContent_Declare( + sqlite3mc URL https://github.com/utelle/SQLite3MultipleCiphers/releases/download/v1.9.2/sqlite3mc-1.9.2-sqlite-3.47.2-amalgamation.zip DOWNLOAD_EXTRACT_TIMESTAMP NEW ) FetchContent_MakeAvailable(sqlite3) +FetchContent_MakeAvailable(sqlite3mc) file(DOWNLOAD https://raw.githubusercontent.com/sqlite/sqlite/master/src/test_vfstrace.c "${CMAKE_BINARY_DIR}/vfstrace.c") @@ -30,7 +37,7 @@ add_custom_command( ) add_custom_target(required_symbols DEPENDS required_symbols.txt) -macro(base_sqlite3_target name debug) +macro(base_sqlite3_target name debug crypto) set(clang_output ${name}.clang.wasm) set(init_output ${name}.init.wasm) set(output ${init_output}) @@ -38,13 +45,20 @@ macro(base_sqlite3_target name debug) set(sources ${CMAKE_CURRENT_SOURCE_DIR}/os_web.c ${CMAKE_CURRENT_SOURCE_DIR}/helpers.c - ${sqlite3_SOURCE_DIR}/sqlite3mc_amalgamation.c ) set(flags -Wall -Wextra -Wno-unused-parameter -Wno-unused-function) + if(${crypto}) + list(APPEND sources "${sqlite3mc_SOURCE_DIR}/sqlite3mc_amalgamation.c") + list(APPEND sources "${CMAKE_CURRENT_SOURCE_DIR}/getentropy.c") + list(APPEND flags "-DSQLITE_OMIT_AUTOINIT") + else() + list(APPEND sources "${sqlite3_SOURCE_DIR}/sqlite3.c") + endif() + if(${debug}) list(APPEND sources "${CMAKE_BINARY_DIR}/vfstrace.c") - list(APPEND flags "-g" "-DDEBUG" "-O1") + list(APPEND flags "-g" "-DDEBUG") else() list(APPEND flags "-Oz" "-DNDEBUG" "-flto") endif() @@ -56,7 +70,6 @@ macro(base_sqlite3_target name debug) -o ${clang_output} -I ${PROJECT_SOURCE_DIR} -I ${sqlite3_SOURCE_DIR} -D_HAVE_SQLITE_CONFIG_H - -DSQLITE_OMIT_AUTOINIT -D__WASM__ -mcpu=generic -mexec-model=reactor @@ -90,10 +103,12 @@ macro(base_sqlite3_target name debug) add_custom_target(${name} DEPENDS ${output}) endmacro() -base_sqlite3_target(sqlite3_debug true) -base_sqlite3_target(sqlite3_opt false) +base_sqlite3_target(sqlite3_debug true false) +base_sqlite3_target(sqlite3_opt false false) +base_sqlite3_target(sqlite3mc false true) add_custom_target(output) add_custom_command(TARGET output COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/sqlite3_opt.wasm ${PROJECT_SOURCE_DIR}/../../example/web/sqlite3.wasm) add_custom_command(TARGET output COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/sqlite3_debug.init.wasm ${PROJECT_SOURCE_DIR}/../../example/web/sqlite3.debug.wasm) -add_dependencies(output sqlite3_debug sqlite3_opt) +add_custom_command(TARGET output COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/sqlite3mc.wasm ${PROJECT_SOURCE_DIR}/../../example/web/sqlite3mc.wasm) +add_dependencies(output sqlite3_debug sqlite3_opt sqlite3mc) diff --git a/sqlite3/assets/wasm/getentropy.c b/sqlite3/assets/wasm/getentropy.c new file mode 100644 index 00000000..b73a9bf2 --- /dev/null +++ b/sqlite3/assets/wasm/getentropy.c @@ -0,0 +1,9 @@ +#include +#include "bridge.h" + +// sqlite3mc calls getentropy on initialization. That call pulls a bunch of WASI imports in when +// using the default WASI libc, which we're trying to avoid here. +// Instead, we use a local implementation backed by `Random.secure()` in Dart. +int getentropy(void* buf, size_t n) { + return xRandomness(-1, (int) n, (char*) buf); +} diff --git a/sqlite3/assets/wasm/os_web.c b/sqlite3/assets/wasm/os_web.c index 35e6db42..04f2e97e 100644 --- a/sqlite3/assets/wasm/os_web.c +++ b/sqlite3/assets/wasm/os_web.c @@ -2,18 +2,8 @@ #include #include -#include "bridge.h" #include "sqlite3.h" -extern int sqlite3_powersync_init(sqlite3 *db, char **pzErrMsg, - const sqlite3_api_routines *pApi); - -int getentropy(void* buf, size_t n) { - return xRandomness(-1, (int) n, (char*) buf); -} - -int sqlite3_os_init(void) { - return SQLITE_OK; -} +int sqlite3_os_init(void) { return SQLITE_OK; } int sqlite3_os_end(void) { return SQLITE_OK; } From 66c51933277b8d19a82160ef392a7fe3d1a0cfbb Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 10 Jan 2025 23:03:10 +0100 Subject: [PATCH 03/10] Reformat native code --- sqlite3/assets/wasm/getentropy.c | 10 ++++++---- sqlite3/tool/format_native.dart | 11 ----------- sqlite3/tool/format_native.sh | 2 ++ 3 files changed, 8 insertions(+), 15 deletions(-) delete mode 100644 sqlite3/tool/format_native.dart create mode 100755 sqlite3/tool/format_native.sh diff --git a/sqlite3/assets/wasm/getentropy.c b/sqlite3/assets/wasm/getentropy.c index b73a9bf2..8189f06b 100644 --- a/sqlite3/assets/wasm/getentropy.c +++ b/sqlite3/assets/wasm/getentropy.c @@ -1,9 +1,11 @@ #include + #include "bridge.h" -// sqlite3mc calls getentropy on initialization. That call pulls a bunch of WASI imports in when -// using the default WASI libc, which we're trying to avoid here. -// Instead, we use a local implementation backed by `Random.secure()` in Dart. +// sqlite3mc calls getentropy on initialization. That call pulls a bunch of WASI +// imports in when using the default WASI libc, which we're trying to avoid +// here. Instead, we use a local implementation backed by `Random.secure()` in +// Dart. int getentropy(void* buf, size_t n) { - return xRandomness(-1, (int) n, (char*) buf); + return xRandomness(-1, (int)n, (char*)buf); } diff --git a/sqlite3/tool/format_native.dart b/sqlite3/tool/format_native.dart deleted file mode 100644 index 7e054980..00000000 --- a/sqlite3/tool/format_native.dart +++ /dev/null @@ -1,11 +0,0 @@ -import 'dart:io'; - -Future main() async { - final process = await Process.start( - 'clang-format', - ['--style=google', '-i', 'assets/sqlite3.h'], - mode: ProcessStartMode.inheritStdio, - ); - - exitCode = await process.exitCode; -} diff --git a/sqlite3/tool/format_native.sh b/sqlite3/tool/format_native.sh new file mode 100755 index 00000000..236ae0e1 --- /dev/null +++ b/sqlite3/tool/format_native.sh @@ -0,0 +1,2 @@ +#!/bin/sh +clang-format --style=google -i assets/sqlite3.h assets/wasm/*.{c,h} test/**/*.c From d250ac8fbea0852a88053c6d1d0b81773553f3f4 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 10 Jan 2025 23:10:25 +0100 Subject: [PATCH 04/10] Make encryption example a separate button --- sqlite3/example/web/index.html | 1 + sqlite3/example/web/main.dart | 30 ++++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/sqlite3/example/web/index.html b/sqlite3/example/web/index.html index 0170e6c5..c84a5097 100644 --- a/sqlite3/example/web/index.html +++ b/sqlite3/example/web/index.html @@ -22,6 +22,7 @@

sqlite3 web demo

  • With an Origin-Private FileSystem (OPFS)-based storage implementation. Note that this requires two workers and a special header that is not available with build_runner serve, but it will work when launching this website with dart run tool/example_server.dart
    • +
  • With an in-memory test (using SQLite3 Multiple Ciphers for encryption):
    • After launching the example, you can check the console and the code in main.dart (or worker.dart for the OPFS example) diff --git a/sqlite3/example/web/main.dart b/sqlite3/example/web/main.dart index 9adb623c..9587d524 100644 --- a/sqlite3/example/web/main.dart +++ b/sqlite3/example/web/main.dart @@ -5,22 +5,21 @@ import 'package:sqlite3/wasm.dart'; Future main() async { final startIndexedDb = document.getElementById('start-idb')!; final startOpfs = document.getElementById('start-opfs')!; + final startEncryption = document.getElementById('start-encryption')!; startIndexedDb.onClick.listen((_) async { startIndexedDb.remove(); - final sqlite3 = await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3.debug.wasm')); print(sqlite3.version); sqlite3.registerVirtualFileSystem( - InMemoryFileSystem(), + await IndexedDbFileSystem.open(dbName: 'sqlite3-example'), makeDefault: true, ); - sqlite3.open('/database', vfs: 'multipleciphers-dart-memory') - ..execute("pragma key = 'test';") + sqlite3.open('/database') ..execute('pragma user_version = 1') ..execute('CREATE TABLE foo (bar INTEGER NOT NULL);') ..execute('INSERT INTO foo (bar) VALUES (?)', [3]) @@ -36,4 +35,27 @@ Future main() async { final worker = Worker('worker.dart.js'); worker.postMessage('start'); }); + + startEncryption.onClick.listen((_) async { + startEncryption.remove(); + final sqlite3 = await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3mc.wasm')); + + sqlite3.registerVirtualFileSystem(InMemoryFileSystem(), makeDefault: true); + + sqlite3.open('/database') + ..execute("pragma key = 'test';") + ..execute('pragma user_version = 1') + ..execute('CREATE TABLE foo (bar INTEGER NOT NULL);') + ..execute('INSERT INTO foo (bar) VALUES (?)', [3]) + ..dispose(); + + final db = sqlite3.open('/database'); + try { + db.select('SELECT * FROM foo'); + } on SqliteException { + print('database call failed (expected due to missing key)'); + } + db.execute("pragma key = 'test';"); + print(db.select('SELECT * FROM foo')); + }); } From fd1a9824a802e3af9b4afce92214a6f34a458f32 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 10 Jan 2025 23:16:26 +0100 Subject: [PATCH 05/10] Update web test expectations --- sqlite3/dart_test.yaml | 1 + sqlite3/test/wasm/sqlite3_test.dart | 2 +- sqlite3/test/wasm/utils.dart | 9 +-------- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/sqlite3/dart_test.yaml b/sqlite3/dart_test.yaml index 6f2a0f90..53172fda 100644 --- a/sqlite3/dart_test.yaml +++ b/sqlite3/dart_test.yaml @@ -22,6 +22,7 @@ override_platforms: presets: full: platforms: [vm, chrome, firefox] + compilers: [dart2js, dart2wasm] on_os: windows: platforms: [vm, chrome, firefox, edge] diff --git a/sqlite3/test/wasm/sqlite3_test.dart b/sqlite3/test/wasm/sqlite3_test.dart index 468bf474..6b1d8f96 100644 --- a/sqlite3/test/wasm/sqlite3_test.dart +++ b/sqlite3/test/wasm/sqlite3_test.dart @@ -48,7 +48,7 @@ void main() { expect( version, isA() - .having((e) => e.libVersion, 'libVersion', startsWith('3.46')), + .having((e) => e.libVersion, 'libVersion', startsWith('3.47')), ); }); diff --git a/sqlite3/test/wasm/utils.dart b/sqlite3/test/wasm/utils.dart index a2d9987d..dabb8ae4 100644 --- a/sqlite3/test/wasm/utils.dart +++ b/sqlite3/test/wasm/utils.dart @@ -1,5 +1,3 @@ -import 'dart:math'; - import 'package:sqlite3/wasm.dart'; import 'package:test/scaffolding.dart'; @@ -16,12 +14,7 @@ Future loadSqlite3WithoutVfs() async { Future loadSqlite3([VirtualFileSystem? defaultVfs]) async { final sqlite3 = await loadSqlite3WithoutVfs(); sqlite3.registerVirtualFileSystem( - defaultVfs ?? - InMemoryFileSystem( - // Not using the default Random.secure() because it's not supported - // by dart2wasm - random: Random(), - ), + defaultVfs ?? InMemoryFileSystem(), makeDefault: true, ); return sqlite3; From 28c89b1c57194573ef80d9569c06857bfec8af01 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 10 Jan 2025 23:16:32 +0100 Subject: [PATCH 06/10] Test sqlite3_test package --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 00cc9fe5..0155e866 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,7 +84,7 @@ jobs: needs: [compile_sqlite3] strategy: matrix: - package: [sqlite3] + package: [sqlite3, sqlite3_test] dart: [stable] name: Analyze on Dart ${{ matrix.dart }} @@ -182,8 +182,8 @@ jobs: - name: Test sqlite3_test package run: | dart pub get - dart test -P ci - working-directory: sqlite3/ + dart test + working-directory: sqlite3_test/ - name: Web tests run: | From 935088be1cd116056368893f464c42a4a31c64ef Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 10 Jan 2025 23:32:59 +0100 Subject: [PATCH 07/10] Add basic test for encryption --- .github/workflows/main.yml | 3 ++- sqlite3/test/wasm/encryption_test.dart | 30 ++++++++++++++++++++++++++ sqlite3/test/wasm/sqlite3_test.dart | 5 +---- sqlite3/test/wasm/utils.dart | 6 +++--- 4 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 sqlite3/test/wasm/encryption_test.dart diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0155e866..dff343ea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -187,7 +187,8 @@ jobs: - name: Web tests run: | - curl https://storage.googleapis.com/simon-public-euw3/assets/sqlite3/wasm/2.4.6/sqlite3.wasm -o example/web/sqlite3.wasm + curl https://simon-public.fsn1.your-objectstorage.com/assets/sqlite3/2.6.0/sqlite3.wasm -o example/web/sqlite3.wasm + curl https://simon-public.fsn1.your-objectstorage.com/assets/sqlite3/2.6.0/sqlite3mc.wasm -o example/web/sqlite3.wasm dart test -P web -r expanded # If browsers behave differently on different platforms, surely that's not our fault... # So, only run browser tests on Linux to be faster. diff --git a/sqlite3/test/wasm/encryption_test.dart b/sqlite3/test/wasm/encryption_test.dart new file mode 100644 index 00000000..94227672 --- /dev/null +++ b/sqlite3/test/wasm/encryption_test.dart @@ -0,0 +1,30 @@ +@Tags(['wasm']) +library; + +import 'package:sqlite3/wasm.dart'; +import 'package:test/test.dart'; + +import 'utils.dart'; + +void main() { + test('can open databases with sqlite3mc', () async { + final sqlite3 = await loadSqlite3WithoutVfs(encryption: true); + sqlite3.registerVirtualFileSystem(InMemoryFileSystem(), makeDefault: true); + + sqlite3.open('/test') + ..execute('pragma key = "key"') + ..execute('CREATE TABLE foo (bar TEXT) STRICT;') + ..execute('INSERT INTO foo VALUES (?)', ['test']) + ..dispose(); + + final database = sqlite3.open('/test'); + expect( + () => database.select('SELECT * FROM foo'), + throwsA(isA() + .having((e) => e.message, 'message', contains('not a database'))), + ); + + database.execute('pragma key = "key"'); + expect(database.select('SELECT * FROM foo'), isNotEmpty); + }); +} diff --git a/sqlite3/test/wasm/sqlite3_test.dart b/sqlite3/test/wasm/sqlite3_test.dart index 6b1d8f96..6f091db3 100644 --- a/sqlite3/test/wasm/sqlite3_test.dart +++ b/sqlite3/test/wasm/sqlite3_test.dart @@ -3,7 +3,6 @@ library; import 'dart:js_interop'; import 'dart:js_interop_unsafe'; -import 'dart:math'; import 'package:http/http.dart' as http; import 'package:sqlite3/wasm.dart'; @@ -35,9 +34,7 @@ void main() { sqlite3 = await WasmSqlite3.load(response.bodyBytes); sqlite3.registerVirtualFileSystem( - // Not using the default Random.secure() because it's not supported - // by dart2wasm - InMemoryFileSystem(random: Random()), + InMemoryFileSystem(), makeDefault: true, ); } diff --git a/sqlite3/test/wasm/utils.dart b/sqlite3/test/wasm/utils.dart index dabb8ae4..39dccdb0 100644 --- a/sqlite3/test/wasm/utils.dart +++ b/sqlite3/test/wasm/utils.dart @@ -1,12 +1,12 @@ import 'package:sqlite3/wasm.dart'; import 'package:test/scaffolding.dart'; -Future loadSqlite3WithoutVfs() async { +Future loadSqlite3WithoutVfs({bool encryption = false}) async { final channel = spawnHybridUri('/test/wasm/asset_server.dart'); final port = (await channel.stream.first as double).toInt(); - final sqliteWasm = - Uri.parse('http://localhost:$port/example/web/sqlite3.wasm'); + final filename = encryption ? 'sqlite3mc.wasm' : 'sqlite3.wasm'; + final sqliteWasm = Uri.parse('http://localhost:$port/example/web/$filename'); return await WasmSqlite3.loadFromUrl(sqliteWasm); } From ee5bdd9f0f520fb715ebe23f9ae6467ee8ab3252 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 10 Jan 2025 23:49:47 +0100 Subject: [PATCH 08/10] Add changelog entry --- sqlite3/CHANGELOG.md | 3 +++ sqlite3/README.md | 24 +++++++++++++++++-- .../sqlite3_flutter_libs/Package.resolved | 4 ++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/sqlite3/CHANGELOG.md b/sqlite3/CHANGELOG.md index 25465aa4..04dc5322 100644 --- a/sqlite3/CHANGELOG.md +++ b/sqlite3/CHANGELOG.md @@ -4,6 +4,9 @@ systems. - Add `jsonb`, a Dart `Codec` converting Dart object from and to SQLite `JSONB` values. +- __Experimentally__ support encryption on the web through SQLite Multiple + Ciphers. The readme provides more information on how to use encryption on the + web. ## 2.5.0 diff --git a/sqlite3/README.md b/sqlite3/README.md index 9ee18f7e..0e770881 100644 --- a/sqlite3/README.md +++ b/sqlite3/README.md @@ -96,7 +96,7 @@ Emscripten or any JavaScript glue code. Please note that stable web support for `package:sqlite3` is restricted to Dart being compiled to JavaScript. Support for `dart2wasm` is experimental. The API -is identical, but the implementation [is severly limited](https://github.com/simolus3/sqlite3.dart/issues/230). +is identical, but the implementation [is severely limited](https://github.com/simolus3/sqlite3.dart/issues/230). ### Setup @@ -130,7 +130,7 @@ in `package:sqlite3/sqlite3.dart`, databases can be opened in similar ways. An example for such web folder is in `example/web/` of this repo. To view the example, copy a compiled `sqlite3.wasm` file to `web/sqlite3.wasm` in this directory. -Then, run `dart run build_runner serve example:8080` and visit `http://localhost:8080/web/` in a browser. +Then, run `dart run build_runner serve example:8080` and visit `http://localhost:8080/web/` in a browser. Another `example/multiplatform/` uses common interface to `sqlite3` on web and native platforms. To run this example, merge its files into a Flutter app. @@ -143,6 +143,26 @@ version in `package:sqlite3/wasm.dart`. By having shared code depend on the common interfaces, it can be used for both native and web apps. +### Web encryption + +Starting from version 2.6.0, `package:sqlite3/wasm.dart` supports loading a compiled version of +[SQLite Multiple Ciphers](https://utelle.github.io/SQLite3MultipleCiphers/) providing encryption +support for the web. +Please note that this variant is not currently tested as well as the regular SQLite version. +For this reason, using SQLite Multiple Ciphers with `package:sqlite3/wasm.dart` should be considered +experimental for the time being. + +To test the encryption integration, download `sqlite3mc.wasm` from the [releases](https://github.com/simolus3/sqlite3.dart/releases) +of this package and use that as a URL to load sqlite3 on the web: + +```dart +final sqlite3 = await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3mc.wasm')); +sqlite3.registerVirtualFileSystem(InMemoryFileSystem(), makeDefault: true); + +final database = sqlite3.open('/database') + ..execute("pragma key = 'test';"); // TODO: Replace key +``` + ### Testing To run the tests of this package with wasm, either download the `sqlite3.wasm` file from the diff --git a/sqlite3_flutter_libs/darwin/sqlite3_flutter_libs/Package.resolved b/sqlite3_flutter_libs/darwin/sqlite3_flutter_libs/Package.resolved index 81f49377..82d9b552 100644 --- a/sqlite3_flutter_libs/darwin/sqlite3_flutter_libs/Package.resolved +++ b/sqlite3_flutter_libs/darwin/sqlite3_flutter_libs/Package.resolved @@ -5,8 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/sbooth/CSQLite.git", "state" : { - "revision" : "f9bc82fd757667a5d1819db4fbb073c97488eb85", - "version" : "3.47.1" + "revision" : "c10dbeae1ea2bee3acd571c47509d6aaed1e9b92", + "version" : "3.47.2" } } ], From 7135a78a9ac0c840fc549817b18520672aed5ee0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 11 Jan 2025 00:37:59 +0100 Subject: [PATCH 09/10] Fix sqlite3_test on Windows --- sqlite3_test/CHANGELOG.md | 4 ++++ sqlite3_test/lib/sqlite3_test.dart | 1 + sqlite3_test/pubspec.yaml | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sqlite3_test/CHANGELOG.md b/sqlite3_test/CHANGELOG.md index a0712a79..170b4664 100644 --- a/sqlite3_test/CHANGELOG.md +++ b/sqlite3_test/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.1.1 + +- Fix writes on Windows. + ## 0.1.0 - Initial version. diff --git a/sqlite3_test/lib/sqlite3_test.dart b/sqlite3_test/lib/sqlite3_test.dart index 252877eb..2c0d9e2d 100644 --- a/sqlite3_test/lib/sqlite3_test.dart +++ b/sqlite3_test/lib/sqlite3_test.dart @@ -181,6 +181,7 @@ final class _TestFile implements VirtualFileSystemFile { } _file.unlockSync(); + _lockLevel = SqlFileLockingLevels.SQLITE_LOCK_NONE; if (mode != SqlFileLockingLevels.SQLITE_LOCK_NONE) { return xLock(mode); } diff --git a/sqlite3_test/pubspec.yaml b/sqlite3_test/pubspec.yaml index 12f1517e..af74012f 100644 --- a/sqlite3_test/pubspec.yaml +++ b/sqlite3_test/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite3_test description: Integration of fake clocks and other test utilities for SQLite databases. -version: 0.1.0 +version: 0.1.1 homepage: https://github.com/simolus3/sqlite3.dart/tree/main/sqlite3_test repository: https://github.com/simolus3/sqlite3.dart topics: From 40ac348afd141818fb41318cb5ffa5c198a42414 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 11 Jan 2025 00:50:35 +0100 Subject: [PATCH 10/10] Fix destination for sqlite3mc download --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dff343ea..fea10c30 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -188,7 +188,7 @@ jobs: - name: Web tests run: | curl https://simon-public.fsn1.your-objectstorage.com/assets/sqlite3/2.6.0/sqlite3.wasm -o example/web/sqlite3.wasm - curl https://simon-public.fsn1.your-objectstorage.com/assets/sqlite3/2.6.0/sqlite3mc.wasm -o example/web/sqlite3.wasm + curl https://simon-public.fsn1.your-objectstorage.com/assets/sqlite3/2.6.0/sqlite3mc.wasm -o example/web/sqlite3mc.wasm dart test -P web -r expanded # If browsers behave differently on different platforms, surely that's not our fault... # So, only run browser tests on Linux to be faster.