Initial support for encryption on the web

.github/workflows/main.yml

@@ -84,7 +84,7 @@ jobs:
     needs: [compile_sqlite3]
     strategy:
       matrix:
-        package: [sqlite3]
+        package: [sqlite3, sqlite3_test]
         dart: [stable]
 
     name: Analyze on Dart ${{ matrix.dart }}
@@ -182,12 +182,13 @@ jobs:
     - name: Test sqlite3_test package
       run: |
         dart pub get
-        dart test -P ci
-      working-directory: sqlite3/
+        dart test
+      working-directory: sqlite3_test/
 
     - name: Web tests
       run: |
-        curl https://storage.googleapis.com/simon-public-euw3/assets/sqlite3/wasm/2.4.6/sqlite3.wasm -o example/web/sqlite3.wasm
+        curl https://simon-public.fsn1.your-objectstorage.com/assets/sqlite3/2.6.0/sqlite3.wasm -o example/web/sqlite3.wasm
+        curl https://simon-public.fsn1.your-objectstorage.com/assets/sqlite3/2.6.0/sqlite3mc.wasm -o example/web/sqlite3mc.wasm
         dart test -P web -r expanded
       # If browsers behave differently on different platforms, surely that's not our fault...
       # So, only run browser tests on Linux to be faster.

sqlite3/CHANGELOG.md

@@ -4,6 +4,9 @@
   systems.
 - Add `jsonb`, a Dart `Codec` converting Dart object from and to SQLite
   `JSONB` values.
+- __Experimentally__ support encryption on the web through SQLite Multiple
+  Ciphers. The readme provides more information on how to use encryption on the
+  web.
 
 ## 2.5.0
 

sqlite3/README.md

@@ -96,7 +96,7 @@ Emscripten or any JavaScript glue code.
 
 Please note that stable web support for `package:sqlite3` is restricted to Dart
 being compiled to JavaScript. Support for `dart2wasm` is experimental. The API
-is identical, but the implementation [is severly limited](https://github.com/simolus3/sqlite3.dart/issues/230).
+is identical, but the implementation [is severely limited](https://github.com/simolus3/sqlite3.dart/issues/230).
 
 ### Setup
 
@@ -130,7 +130,7 @@ in `package:sqlite3/sqlite3.dart`, databases can be opened in similar ways.
 
 An example for such web folder is in `example/web/` of this repo.
 To view the example, copy a compiled `sqlite3.wasm` file to `web/sqlite3.wasm` in this directory.
-Then, run `dart run build_runner serve example:8080` and  visit `http://localhost:8080/web/` in a browser.
+Then, run `dart run build_runner serve example:8080` and visit `http://localhost:8080/web/` in a browser.
 
 Another `example/multiplatform/` uses common interface to `sqlite3` on web and native platforms.
 To run this example, merge its files into a Flutter app.
@@ -143,6 +143,26 @@ version in `package:sqlite3/wasm.dart`.
 By having shared code depend on the common interfaces, it can be used for both native and web
 apps.
 
+### Web encryption
+
+Starting from version 2.6.0, `package:sqlite3/wasm.dart` supports loading a compiled version of
+[SQLite Multiple Ciphers](https://utelle.github.io/SQLite3MultipleCiphers/) providing encryption
+support for the web.
+Please note that this variant is not currently tested as well as the regular SQLite version.
+For this reason, using SQLite Multiple Ciphers with `package:sqlite3/wasm.dart` should be considered
+experimental for the time being.
+
+To test the encryption integration, download `sqlite3mc.wasm` from the [releases](https://github.com/simolus3/sqlite3.dart/releases)
+of this package and use that as a URL to load sqlite3 on the web:
+
+```dart
+final sqlite3 = await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3mc.wasm'));
+sqlite3.registerVirtualFileSystem(InMemoryFileSystem(), makeDefault: true);
+
+final database = sqlite3.open('/database')
+  ..execute("pragma key = 'test';"); // TODO: Replace key
+```
+
 ### Testing
 
 To run the tests of this package with wasm, either download the `sqlite3.wasm` file from the

sqlite3/assets/wasm/CMakeLists.txt

@@ -16,7 +16,14 @@ FetchContent_Declare(
     DOWNLOAD_EXTRACT_TIMESTAMP NEW
 )
 
+FetchContent_Declare(
+    sqlite3mc
+    URL https://github.com/utelle/SQLite3MultipleCiphers/releases/download/v1.9.2/sqlite3mc-1.9.2-sqlite-3.47.2-amalgamation.zip
+    DOWNLOAD_EXTRACT_TIMESTAMP NEW
+)
+
 FetchContent_MakeAvailable(sqlite3)
+FetchContent_MakeAvailable(sqlite3mc)
 
 file(DOWNLOAD https://raw.githubusercontent.com/sqlite/sqlite/master/src/test_vfstrace.c "${CMAKE_BINARY_DIR}/vfstrace.c")
 
@@ -30,18 +37,25 @@ add_custom_command(
 )
 add_custom_target(required_symbols DEPENDS required_symbols.txt)
 
-macro(base_sqlite3_target name debug)
+macro(base_sqlite3_target name debug crypto)
   set(clang_output ${name}.clang.wasm)
   set(init_output ${name}.init.wasm)
   set(output ${init_output})
 
   set(sources
     ${CMAKE_CURRENT_SOURCE_DIR}/os_web.c
     ${CMAKE_CURRENT_SOURCE_DIR}/helpers.c
-    ${sqlite3_SOURCE_DIR}/sqlite3.c
   )
   set(flags -Wall -Wextra -Wno-unused-parameter -Wno-unused-function)
 
+  if(${crypto})
+    list(APPEND sources "${sqlite3mc_SOURCE_DIR}/sqlite3mc_amalgamation.c")
+    list(APPEND sources "${CMAKE_CURRENT_SOURCE_DIR}/getentropy.c")
+    list(APPEND flags "-DSQLITE_OMIT_AUTOINIT")
+  else()
+    list(APPEND sources "${sqlite3_SOURCE_DIR}/sqlite3.c")
+  endif()
+
   if(${debug})
     list(APPEND sources "${CMAKE_BINARY_DIR}/vfstrace.c")
     list(APPEND flags "-g" "-DDEBUG")
@@ -56,6 +70,7 @@ macro(base_sqlite3_target name debug)
       -o ${clang_output}
       -I ${PROJECT_SOURCE_DIR} -I ${sqlite3_SOURCE_DIR}
       -D_HAVE_SQLITE_CONFIG_H
+      -D__WASM__
       -mcpu=generic
       -mexec-model=reactor
       -fno-stack-protector -fno-stack-clash-protection
@@ -88,10 +103,12 @@ macro(base_sqlite3_target name debug)
   add_custom_target(${name} DEPENDS ${output})
 endmacro()
 
-base_sqlite3_target(sqlite3_debug true)
-base_sqlite3_target(sqlite3_opt false)
+base_sqlite3_target(sqlite3_debug true false)
+base_sqlite3_target(sqlite3_opt false false)
+base_sqlite3_target(sqlite3mc false true)
 
 add_custom_target(output)
 add_custom_command(TARGET output COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/sqlite3_opt.wasm ${PROJECT_SOURCE_DIR}/../../example/web/sqlite3.wasm)
 add_custom_command(TARGET output COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/sqlite3_debug.init.wasm ${PROJECT_SOURCE_DIR}/../../example/web/sqlite3.debug.wasm)
-add_dependencies(output sqlite3_debug sqlite3_opt)
+add_custom_command(TARGET output COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/sqlite3mc.wasm ${PROJECT_SOURCE_DIR}/../../example/web/sqlite3mc.wasm)
+add_dependencies(output sqlite3_debug sqlite3_opt sqlite3mc)

sqlite3/assets/wasm/getentropy.c

@@ -0,0 +1,11 @@
+#include <stdlib.h>
+
+#include "bridge.h"
+
+// sqlite3mc calls getentropy on initialization. That call pulls a bunch of WASI
+// imports in when using the default WASI libc, which we're trying to avoid
+// here. Instead, we use a local implementation backed by `Random.secure()` in
+// Dart.
+int getentropy(void* buf, size_t n) {
+  return xRandomness(-1, (int)n, (char*)buf);
+}

sqlite3/assets/wasm/helpers.c

@@ -187,7 +187,11 @@ SQLITE_API sqlite3_vfs *dart_sqlite3_register_vfs(const char *name, int dartId,
   vfstrace_register(traceName, name, &dartvfs_trace_log1, NULL, makeDefault);
 #else
   // Just register the VFS as is.
-  sqlite3_vfs_register(vfs, makeDefault);
+  int rc = sqlite3_vfs_register(vfs, makeDefault);
+  if (rc) {
+    free(vfs);
+    return NULL;
+  }
 #endif
   return vfs;
 }

sqlite3/assets/wasm/os_web.c

@@ -2,7 +2,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "bridge.h"
 #include "sqlite3.h"
 
 int sqlite3_os_init(void) { return SQLITE_OK; }

sqlite3/dart_test.yaml

@@ -22,6 +22,7 @@ override_platforms:
 presets:
   full:
     platforms: [vm, chrome, firefox]
+    compilers: [dart2js, dart2wasm]
     on_os:
       windows:
         platforms: [vm, chrome, firefox, edge]

sqlite3/example/web/index.html

@@ -22,6 +22,7 @@ <h1>sqlite3 web demo</h1>
   <li>With an Origin-Private FileSystem (OPFS)-based storage implementation. Note that this requires two workers and a special header
       that is not available with <code>build_runner serve</code>, but it will work when launching this website with <code>dart run tool/example_server.dart</code> <button id="start-opfs">Start demo!</button>
   </li>
+  <li>With an in-memory test (using <a href="https://utelle.github.io/SQLite3MultipleCiphers/">SQLite3 Multiple Ciphers</a> for encryption): <button id="start-encryption">Start encryption!</button></li>
 </ul>
 
 After launching the example, you can check the console and the code in <code>main.dart</code> (or <code>worker.dart</code> for the OPFS example)

sqlite3/example/web/main.dart

@@ -5,10 +5,10 @@ import 'package:sqlite3/wasm.dart';
 Future<void> main() async {
   final startIndexedDb = document.getElementById('start-idb')!;
   final startOpfs = document.getElementById('start-opfs')!;
+  final startEncryption = document.getElementById('start-encryption')!;
 
   startIndexedDb.onClick.listen((_) async {
     startIndexedDb.remove();
-
     final sqlite3 =
         await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3.debug.wasm'));
 
@@ -35,4 +35,27 @@ Future<void> main() async {
     final worker = Worker('worker.dart.js');
     worker.postMessage('start');
   });
+
+  startEncryption.onClick.listen((_) async {
+    startEncryption.remove();
+    final sqlite3 = await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3mc.wasm'));
+
+    sqlite3.registerVirtualFileSystem(InMemoryFileSystem(), makeDefault: true);
+
+    sqlite3.open('/database')
+      ..execute("pragma key = 'test';")
+      ..execute('pragma user_version = 1')
+      ..execute('CREATE TABLE foo (bar INTEGER NOT NULL);')
+      ..execute('INSERT INTO foo (bar) VALUES (?)', [3])
+      ..dispose();
+
+    final db = sqlite3.open('/database');
+    try {
+      db.select('SELECT * FROM foo');
+    } on SqliteException {
+      print('database call failed (expected due to missing key)');
+    }
+    db.execute("pragma key = 'test';");
+    print(db.select('SELECT * FROM foo'));
+  });
 }

sqlite3/lib/src/vfs.dart

@@ -132,18 +132,30 @@ abstract base class BaseVirtualFileSystem extends VirtualFileSystem {
   final Random random;
 
   BaseVirtualFileSystem({Random? random, required String name})
-      : random = random ?? Random.secure(),
+      : random = random ?? _fallbackRandom,
         super(name);
 
   @override
   void xRandomness(Uint8List target) {
+    generateRandomness(target, random);
+  }
+
+  @override
+  DateTime xCurrentTime() => DateTime.now();
+
+  /// Fills [target] with random bytes.
+  ///
+  /// An optional [random] source can be provided, otherwise a default instance
+  /// of [Random.secure] will be used.
+  static void generateRandomness(Uint8List target, [Random? random]) {
+    random ??= _fallbackRandom;
+
     for (var i = 0; i < target.length; i++) {
       target[i] = random.nextInt(1 << 8);
     }
   }
 
-  @override
-  DateTime xCurrentTime() => DateTime.now();
+  static final Random _fallbackRandom = Random.secure();
 }
 
 /// A [VirtualFileSystemFile] base class that implements [xRead] to zero-fill

sqlite3/lib/src/wasm/bindings.dart

@@ -5,6 +5,7 @@ import 'dart:typed_data';
 import 'package:sqlite3/src/vfs.dart';
 
 import '../constants.dart';
+import '../exception.dart';
 import '../functions.dart';
 import '../implementation/bindings.dart';
 import 'wasm_interop.dart' as wasm;
@@ -55,6 +56,8 @@ final class WasmSqliteBindings extends RawSqliteBindings {
   @override
   SqliteResult<RawSqliteDatabase> sqlite3_open_v2(
       String name, int flags, String? zVfs) {
+    sqlite3_initialize();
+
     final namePtr = bindings.allocateZeroTerminated(name);
     final outDb = bindings.malloc(wasm.WasmBindings.pointerSize);
     final vfsPtr = zVfs == null ? 0 : bindings.allocateZeroTerminated(zVfs);
@@ -76,12 +79,23 @@ final class WasmSqliteBindings extends RawSqliteBindings {
     return bindings.memory.readString(bindings.sqlite3_sourceid());
   }
 
+  void sqlite3_initialize() {
+    final rc = bindings.sqlite3_initialize();
+    if (rc != 0) {
+      throw SqliteException(rc, 'sqlite3_initialize call failed');
+    }
+  }
+
   @override
   void registerVirtualFileSystem(VirtualFileSystem vfs, int makeDefault) {
     final name = bindings.allocateZeroTerminated(vfs.name);
     final id = bindings.callbacks.registerVfs(vfs);
 
     final ptr = bindings.dart_sqlite3_register_vfs(name, id, makeDefault);
+    if (ptr == 0) {
+      throw StateError('could not register vfs');
+    }
+    sqlite3_initialize();
     DartCallbacks.sqliteVfsPointer[vfs] = ptr;
   }
 

sqlite3/lib/src/wasm/wasm_interop.dart

@@ -83,7 +83,7 @@ class WasmBindings {
       _sqlite3_stmt_readonly,
       _sqlite3_stmt_isexplain;
 
-  final JSFunction? _sqlite3_db_config;
+  final JSFunction? _sqlite3_db_config, _sqlite3_initialize;
 
   final Global _sqlite3_temp_directory;
 
@@ -160,7 +160,9 @@ class WasmBindings {
         _sqlite3_stmt_isexplain = instance.functions['sqlite3_stmt_isexplain']!,
         _sqlite3_stmt_readonly = instance.functions['sqlite3_stmt_readonly']!,
         _sqlite3_db_config = instance.functions['dart_sqlite3_db_config_int'],
+        _sqlite3_initialize = instance.functions['sqlite3_initialize'],
         _sqlite3_temp_directory = instance.globals['sqlite3_temp_directory']!
+
   // Note when adding new fields: We remove functions from the wasm module that
   // aren't referenced in Dart. We consider a symbol used when it appears in a
   // string literal in an initializer of this constructor (`tool/wasm_dce.dart`).
@@ -210,6 +212,13 @@ class WasmBindings {
 
   void sqlite3_free(Pointer ptr) => _sqlite3_free.callReturningVoid(ptr.toJS);
 
+  int sqlite3_initialize() {
+    return switch (_sqlite3_initialize) {
+      final fun? => fun.callReturningInt0(),
+      null => 0,
+    };
+  }
+
   int create_scalar_function(
       Pointer db, Pointer functionName, int nArg, int eTextRep, int id) {
     return _create_scalar.callReturningInt5(
@@ -601,10 +610,18 @@ class _InjectedValues {
           });
         }).toJS,
         'xRandomness': ((int vfsId, int nByte, Pointer zOut) {
-          final vfs = callbacks.registeredVfs[vfsId]!;
+          final vfs = callbacks.registeredVfs[vfsId];
 
           return _runVfs(() {
-            vfs.xRandomness(memory.buffer.toDart.asUint8List(zOut, nByte));
+            final target = memory.buffer.toDart.asUint8List(zOut, nByte);
+
+            if (vfs != null) {
+              vfs.xRandomness(target);
+            } else {
+              // Fall back to a default random source. We're using this to
+              // implement `getentropy` in C which is used by sqlite3mc.
+              return BaseVirtualFileSystem.generateRandomness(target);
+            }
           });
         }).toJS,
         'xSleep': ((int vfsId, int micros) {

sqlite3/test/wasm/encryption_test.dart

@@ -0,0 +1,30 @@
+@Tags(['wasm'])
+library;
+
+import 'package:sqlite3/wasm.dart';
+import 'package:test/test.dart';
+
+import 'utils.dart';
+
+void main() {
+  test('can open databases with sqlite3mc', () async {
+    final sqlite3 = await loadSqlite3WithoutVfs(encryption: true);
+    sqlite3.registerVirtualFileSystem(InMemoryFileSystem(), makeDefault: true);
+
+    sqlite3.open('/test')
+      ..execute('pragma key = "key"')
+      ..execute('CREATE TABLE foo (bar TEXT) STRICT;')
+      ..execute('INSERT INTO foo VALUES (?)', ['test'])
+      ..dispose();
+
+    final database = sqlite3.open('/test');
+    expect(
+      () => database.select('SELECT * FROM foo'),
+      throwsA(isA<SqliteException>()
+          .having((e) => e.message, 'message', contains('not a database'))),
+    );
+
+    database.execute('pragma key = "key"');
+    expect(database.select('SELECT * FROM foo'), isNotEmpty);
+  });
+}