From 37e5f7b405d750847092994493cf8b0e3338564d Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Fri, 21 Jun 2024 21:26:22 -0300 Subject: [PATCH 01/35] add nix flake, only devShell for now --- .envrc | 1 + .gitignore | 1 + flake.lock | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 29 ++++++++++++++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 .envrc create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 0000000000..3550a30f2d --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore index 12cc04a5dd..44f8a4c394 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /target .vscode/ .idea/ +.direnv/ *.sqld libsql-server/testrollbackrestore/** diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..268eed5bc4 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1718895438, + "narHash": "sha256-k3JqJrkdoYwE3fHE6xGDY676AYmyh4U2Zw+0Bwe5DLU=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "d603719ec6e294f034936c0d0dc06f689d91b6c3", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..263e632aa7 --- /dev/null +++ b/flake.nix @@ -0,0 +1,29 @@ +{ + description = "A very basic flake"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable"; + flake-utils = { + url = "github:numtide/flake-utils"; + }; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + }; + in { + devShells.default = + with pkgs; + mkShell { + nativeBuildInputs = with pkgs; [ + tcl + gnumake + clang + ]; + }; + } + ); +} From 6616400f386cd14465aedc69c06feea23b1a9faf Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 25 Jun 2024 02:12:15 -0300 Subject: [PATCH 02/35] fix(flake): add missing dependencies --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 263e632aa7..43c21ed2ca 100644 --- a/flake.nix +++ b/flake.nix @@ -19,7 +19,9 @@ with pkgs; mkShell { nativeBuildInputs = with pkgs; [ + zig tcl + cmake gnumake clang ]; From ac4193d5da4a4b7eea63ab2b56a92c55a27cb651 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 25 Jun 2024 02:15:12 -0300 Subject: [PATCH 03/35] add: introduce zig build --- libsql-sqlite3/.gitignore | 2 + libsql-sqlite3/build.zig | 336 +++++++++++++++++++++++++++++++++ libsql-sqlite3/src/sqlite.h.in | 8 +- 3 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 libsql-sqlite3/build.zig diff --git a/libsql-sqlite3/.gitignore b/libsql-sqlite3/.gitignore index 121c0ce330..54036a68e5 100644 --- a/libsql-sqlite3/.gitignore +++ b/libsql-sqlite3/.gitignore @@ -62,3 +62,5 @@ libsql /crates/target/ /has_tclsh* /libsql.wasm +zig-out/ +.zig-cache diff --git a/libsql-sqlite3/build.zig b/libsql-sqlite3/build.zig new file mode 100644 index 0000000000..9872c590d2 --- /dev/null +++ b/libsql-sqlite3/build.zig @@ -0,0 +1,336 @@ +const std = @import("std"); + +pub const Debug = struct { + step: std.Build.Step, + path: std.Build.LazyPath, + + pub fn create(b: *std.Build, path: std.Build.LazyPath) *Debug { + const self = b.allocator.create(Debug) catch @panic("OOM"); + self.* = .{ + .step = std.Build.Step.init(.{ + .id = .custom, + .name = "debug path", + .owner = b, + .makeFn = make, + }), + .path = path, + }; + return self; + } + + pub fn make(step: *std.Build.Step, _: std.Progress.Node) !void { + const self: *Debug = @fieldParentPtr("step", step); + + const b = self.step.owner; + + std.debug.print("{s}", .{self.path.getPath(b)}); + } +}; + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const lemon = b.addExecutable(.{ + .name = "lemon", + .root_source_file = null, + .target = target, + .optimize = .ReleaseFast, + }); + lemon.addCSourceFile(.{ .file = b.path("tool/lemon.c") }); + lemon.linkLibC(); + + { + const run = b.addRunArtifact(lemon); + run.addArg("-DSQLITE_ENABLE_MATH_FUNCTIONS"); + const parse = run.addPrefixedOutputDirectoryArg("-d", "parser"); + run.addFileArg(b.path("src/parse.y")); + + const debug = Debug.create(b, parse); + debug.step.dependOn(&run.step); + + const step = b.step("gen-parse", "Generate parse files"); + step.dependOn(&debug.step); + step.dependOn(&run.step); + } + + const sources = .{ + .sqlite3 = &.{ + "src/alter.c", + "src/analyze.c", + "src/attach.c", + "src/auth.c", + "src/backup.c", + "src/bitvec.c", + "src/btmutex.c", + "src/btree.c", + "src/build.c", + "src/callback.c", + "src/complete.c", + "src/ctime.c", + "src/date.c", + "src/dbpage.c", + "src/dbstat.c", + "src/delete.c", + "src/expr.c", + "src/fault.c", + "src/fkey.c", + "src/func.c", + "src/global.c", + "src/hash.c", + "src/insert.c", + "src/json.c", + "src/legacy.c", + "src/loadext.c", + "src/main.c", + "src/malloc.c", + "src/mem0.c", + "src/mem1.c", + "src/mem2.c", + "src/mem3.c", + "src/mem5.c", + "src/memdb.c", + "src/memjournal.c", + "src/mutex.c", + "src/mutex_noop.c", + "src/mutex_unix.c", + "src/mutex_w32.c", + "src/notify.c", + "src/os.c", + "src/os_kv.c", + "src/os_unix.c", + "src/os_win.c", + "src/pager.c", + "src/pcache.c", + "src/pcache1.c", + "src/pragma.c", + "src/prepare.c", + "src/printf.c", + "src/random.c", + "src/resolve.c", + "src/rowset.c", + "src/select.c", + "src/status.c", + "src/table.c", + "src/threads.c", + "src/tokenize.c", + "src/treeview.c", + "src/trigger.c", + "src/utf.c", + "src/update.c", + "src/upsert.c", + "src/util.c", + "src/vacuum.c", + "src/vdbe.c", + "src/vdbeapi.c", + "src/vdbeaux.c", + "src/vdbeblob.c", + "src/vdbemem.c", + "src/vdbesort.c", + "src/vdbetrace.c", + "src/vdbevtab.c", + "src/vtab.c", + "src/wal.c", + "src/walker.c", + "src/where.c", + "src/wherecode.c", + "src/whereexpr.c", + "src/window.c", + }, + .extensions = .{ + .fts3 = &.{ + "ext/fts3/fts3.c", + "ext/fts3/fts3.h", + "ext/fts3/fts3Int.h", + "ext/fts3/fts3_aux.c", + "ext/fts3/fts3_expr.c", + "ext/fts3/fts3_hash.c", + "ext/fts3/fts3_hash.h", + "ext/fts3/fts3_icu.c", + "ext/fts3/fts3_porter.c", + "ext/fts3/fts3_snippet.c", + "ext/fts3/fts3_tokenizer.h", + "ext/fts3/fts3_tokenizer.c", + "ext/fts3/fts3_tokenizer1.c", + "ext/fts3/fts3_tokenize_vtab.c", + "ext/fts3/fts3_unicode.c", + "ext/fts3/fts3_unicode2.c", + "ext/fts3/fts3_write.c", + }, + .icu = &.{ + "ext/icu/sqliteicu.h", + "ext/icu/icu.c", + }, + .rtree = &.{ + "ext/rtree/rtree.h", + "ext/rtree/rtree.c", + "ext/rtree/geopoly.c", + }, + .session = &.{ + "ext/session/sqlite3session.c", + "ext/session/sqlite3session.h", + }, + .auth = &.{ + "ext/userauth/userauth.c", + "ext/userauth/sqlite3userauth.h", + }, + .rbu = &.{ + "ext/rbu/sqlite3rbu.h", + "ext/rbu/sqlite3rbu.c", + }, + .stmt = &.{ + "ext/misc/stmt.c", + }, + .wasm = &.{ + "ext/udf/wasmedge_bindings.c", + }, + }, + }; + + const sqlite_header = b.addConfigHeader( + .{ + .include_path = "sqlite3.h", + .style = .{ .cmake = b.path("src/sqlite.h.in") }, + }, + .{ + .sqlite_version = "3.44.0", + .sqlite_version_number = 3044000, + .sqlite_source_id = "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad8alt1", + .libsql_version = "0.2.3", + }, + ); + + const sqlite_cfg = b.addConfigHeader( + .{ + .include_path = "sqlite_cfg.h", + .style = .{ .autoconf = b.path("sqlite_cfg.h.in") }, + }, + .{ + .HAVE_DLFCN_H = null, + + // Define to 1 if you have the `fdatasync' function. + .HAVE_FDATASYNC = 1, + + // Define to 1 if you have the `gmtime_r' function. + .HAVE_GMTIME_R = 1, + + .STDC_HEADERS = 1, + .HAVE_STDINT_H = 1, + .HAVE_STDLIB_H = 1, + + .HAVE_INT16_T = 1, + .HAVE_INT32_T = 1, + .HAVE_INT64_T = 1, + .HAVE_INT8_T = 1, + .HAVE_INTPTR_T = 1, + .HAVE_INTTYPES_H = 1, + + .HAVE_UINT16_T = 1, + .HAVE_UINT32_T = 1, + .HAVE_UINT64_T = 1, + .HAVE_UINT8_T = 1, + .HAVE_UINTPTR_T = 1, + + // Define to 1 if you have the `isnan' function. + .HAVE_ISNAN = 1, + + // Define to 1 if you have the `localtime_r' function. + .HAVE_LOCALTIME_R = 1, + + // Define to 1 if you have the `localtime_s' function. + .HAVE_LOCALTIME_S = 1, + + // Define to 1 if you have the header file. + .HAVE_MALLOC_H = 1, + + // Define to 1 if you have the `malloc_usable_size' function. + .HAVE_MALLOC_USABLE_SIZE = 1, + + // Define to 1 if you have the header file. + .HAVE_MEMORY_H = 1, + + // Define to 1 if you have the `pread' function. + .HAVE_PREAD = 1, + .HAVE_PREAD64 = 1, + .HAVE_PWRITE = 1, + .HAVE_PWRITE64 = 1, + + // Define to 1 if you have the `strchrnul' function. + .HAVE_STRCHRNUL = 1, + + // Define to 1 if you have the header file. + .HAVE_STRINGS_H = 1, + + // Define to 1 if you have the header file. + .HAVE_STRING_H = 1, + + // Define to 1 if you have the header file. + .HAVE_SYS_STAT_H = 1, + + // Define to 1 if you have the header file. + .HAVE_SYS_TYPES_H = 1, + + + // Define to 1 if you have the header file. + .HAVE_UNISTD_H = 1, + + // Define to 1 if you have the `usleep' function. + .HAVE_USLEEP = 1, + + // Define to 1 if you have the `utime' function. + .HAVE_UTIME = 1, + + // Define to 1 if you have the header file. + .HAVE_ZLIB_H = 1, + + // Define to the sub-directory in which libtool stores uninstalled libraries. + .LT_OBJDIR = null, + + // Define to the address where bug reports for this package should be sent. + .PACKAGE_BUGREPORT = "", + + // Define to the full name of this package. + .PACKAGE_NAME = "", + + // Define to the full name and version of this package. + .PACKAGE_STRING = "", + + // Define to the one symbol short name of this package. + .PACKAGE_TARNAME = "", + + // Define to the home page for this package. + .PACKAGE_URL = "", + + // Define to the version of this package. + .PACKAGE_VERSION = "", + + // Number of bits in a file offset, on hosts where this is settable. + ._FILE_OFFSET_BITS = null, + + // Define for large files, on AIX-style hosts. + ._LARGE_FILES = null, + }, + ); + + const debug = Debug.create(b, sqlite_header.getOutput()); + + const sqlite3 = b.addStaticLibrary(.{ + .name = "sqlite3", + .root_source_file = null, + .target = target, + .optimize = optimize, + }); + sqlite3.addIncludePath(b.path("src/")); + sqlite3.addIncludePath(b.path(".")); + sqlite3.addConfigHeader(sqlite_header); + sqlite3.addConfigHeader(sqlite_cfg); + sqlite3.addCSourceFiles(.{ .files = sources.sqlite3, .flags = &.{ "-g", "-DLIBSQL_ENABLE_WASM_RUNTIME" } }); + sqlite3.linkLibC(); + + debug.step.dependOn(&sqlite_header.step); + + const install = b.addInstallArtifact(sqlite3, .{}); + + b.getInstallStep().dependOn(&install.step); + b.getInstallStep().dependOn(&debug.step); +} diff --git a/libsql-sqlite3/src/sqlite.h.in b/libsql-sqlite3/src/sqlite.h.in index c336d235d3..2818c9cb14 100644 --- a/libsql-sqlite3/src/sqlite.h.in +++ b/libsql-sqlite3/src/sqlite.h.in @@ -149,11 +149,11 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "--VERS--" -#define SQLITE_VERSION_NUMBER --VERSION-NUMBER-- -#define SQLITE_SOURCE_ID "--SOURCE-ID--" +#define SQLITE_VERSION "${sqlite_version}" +#define SQLITE_VERSION_NUMBER ${sqlite_version_number} +#define SQLITE_SOURCE_ID "${sqlite_source_id}" -#define LIBSQL_VERSION "--LIBSQL-VERS--" +#define LIBSQL_VERSION ${libsql_version}" /* ** CAPI3REF: Run-Time Library Version Numbers From 77169c4e1afc7b7d2e9c9d6a2ee232e4283fba23 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 25 Jun 2024 02:19:41 -0300 Subject: [PATCH 04/35] fix(sqlite3): include vdbeInt.h (#1501) --- libsql-sqlite3/src/alter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libsql-sqlite3/src/alter.c b/libsql-sqlite3/src/alter.c index 74fad7c998..92832ed47f 100644 --- a/libsql-sqlite3/src/alter.c +++ b/libsql-sqlite3/src/alter.c @@ -13,6 +13,7 @@ ** that implements the ALTER TABLE command. */ #include "sqliteInt.h" +#include "vdbeInt.h" /* ** The code in this file only exists if we are not omitting the From 0d3c464363a6b33684d93dbcb67878c43013f3ca Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 25 Jun 2024 16:27:53 -0300 Subject: [PATCH 05/35] build(libsql-sqlite3): first iteration of build.zig Now it can compile sqlite3 without any extensions and wasm support --- libsql-sqlite3/build.zig | 326 ++++++++++++++++++++------------- libsql-sqlite3/src/sqlite.h.in | 2 +- 2 files changed, 199 insertions(+), 129 deletions(-) diff --git a/libsql-sqlite3/build.zig b/libsql-sqlite3/build.zig index 9872c590d2..709a831fc1 100644 --- a/libsql-sqlite3/build.zig +++ b/libsql-sqlite3/build.zig @@ -15,6 +15,9 @@ pub const Debug = struct { }), .path = path, }; + + path.addStepDependencies(&self.step); + return self; } @@ -23,37 +26,106 @@ pub const Debug = struct { const b = self.step.owner; - std.debug.print("{s}", .{self.path.getPath(b)}); + std.debug.print("welp {s}\n", .{self.path.getPath(b)}); } }; -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); +pub const Amalgamation = struct { + step: std.Build.Step, + lazy_paths: std.ArrayList(std.Build.LazyPath), + output_list: std.Build.GeneratedFile, + basename: []const u8, - const lemon = b.addExecutable(.{ - .name = "lemon", - .root_source_file = null, - .target = target, - .optimize = .ReleaseFast, - }); - lemon.addCSourceFile(.{ .file = b.path("tool/lemon.c") }); - lemon.linkLibC(); + pub fn create(b: *std.Build, basename: []const u8, lazy_paths: []const std.Build.LazyPath) *Amalgamation { + const self = b.allocator.create(Amalgamation) catch @panic("OOM"); - { - const run = b.addRunArtifact(lemon); - run.addArg("-DSQLITE_ENABLE_MATH_FUNCTIONS"); - const parse = run.addPrefixedOutputDirectoryArg("-d", "parser"); - run.addFileArg(b.path("src/parse.y")); + self.* = .{ + .step = std.Build.Step.init(.{ + .id = .custom, + .name = "amalgamate files", + .owner = b, + .makeFn = make, + }), + .lazy_paths = undefined, + .basename = basename, + .output_list = .{ .step = &self.step }, + }; + + var list = std.ArrayList(std.Build.LazyPath).init(b.allocator); - const debug = Debug.create(b, parse); - debug.step.dependOn(&run.step); + for (lazy_paths) |lp| { + list.append(lp) catch @panic("OOM"); + lp.addStepDependencies(&self.step); + } - const step = b.step("gen-parse", "Generate parse files"); - step.dependOn(&debug.step); - step.dependOn(&run.step); + self.lazy_paths = list; + + return self; } + pub fn getOutput(self: *const Amalgamation) std.Build.LazyPath { + return .{ .generated = .{ .file = &self.output_list } }; + } + + pub fn make(step: *std.Build.Step, prog_node: std.Progress.Node) !void { + _ = prog_node; + const self: *Amalgamation = @fieldParentPtr("step", step); + const b = step.owner; + + var man = b.graph.cache.obtain(); + defer man.deinit(); + + var output = std.ArrayList(u8).init(b.allocator); + defer output.deinit(); + + for (self.lazy_paths.items) |lp| { + const file = try std.fs.cwd().readFileAlloc( + b.allocator, + lp.getPath2(b, step), + 2 * 1024 * 1024, + ); + defer b.allocator.free(file); + + try std.fmt.format(output.writer(), + \\/* amalg:begin {s} */ + \\ + , .{lp.getPath(b)}); + + try output.appendSlice(file); + + try std.fmt.format(output.writer(), + \\/* amalg:end {s} */ + \\ + , .{lp.getPath(b)}); + } + + man.hash.addBytes(output.items); + + if (try step.cacheHit(&man)) { + const digest = man.final(); + self.output_list.path = try b.cache_root.join(b.allocator, &.{ "o", &digest, self.basename }); + return; + } + + const digest = man.final(); + + const sub_path = b.pathJoin(&.{ "o", &digest, self.basename }); + const sub_path_dirname = std.fs.path.dirname(sub_path).?; + + try b.cache_root.handle.makePath(sub_path_dirname); + try b.cache_root.handle.writeFile(.{ + .sub_path = sub_path, + .data = output.items, + }); + self.output_list.path = try b.cache_root.join(b.allocator, &.{sub_path}); + // try man.writeManifest(); + } +}; + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + const sources = .{ .sqlite3 = &.{ "src/alter.c", @@ -157,37 +229,17 @@ pub fn build(b: *std.Build) void { "ext/fts3/fts3_unicode2.c", "ext/fts3/fts3_write.c", }, - .icu = &.{ - "ext/icu/sqliteicu.h", - "ext/icu/icu.c", - }, - .rtree = &.{ - "ext/rtree/rtree.h", - "ext/rtree/rtree.c", - "ext/rtree/geopoly.c", - }, - .session = &.{ - "ext/session/sqlite3session.c", - "ext/session/sqlite3session.h", - }, - .auth = &.{ - "ext/userauth/userauth.c", - "ext/userauth/sqlite3userauth.h", - }, - .rbu = &.{ - "ext/rbu/sqlite3rbu.h", - "ext/rbu/sqlite3rbu.c", - }, - .stmt = &.{ - "ext/misc/stmt.c", - }, - .wasm = &.{ - "ext/udf/wasmedge_bindings.c", - }, + .icu = &.{"ext/icu/icu.c"}, + .rtree = &.{ "ext/rtree/rtree.c", "ext/rtree/geopoly.c" }, + .session = &.{"ext/session/sqlite3session.c"}, + .auth = &.{"ext/userauth/userauth.c"}, + .rbu = &.{"ext/rbu/sqlite3rbu.c"}, + .stmt = &.{"ext/misc/stmt.c"}, + .wasm = &.{"ext/udf/wasmedge_bindings.c"}, }, }; - const sqlite_header = b.addConfigHeader( + const sqlite_header_base = b.addConfigHeader( .{ .include_path = "sqlite3.h", .style = .{ .cmake = b.path("src/sqlite.h.in") }, @@ -207,12 +259,8 @@ pub fn build(b: *std.Build) void { }, .{ .HAVE_DLFCN_H = null, - - // Define to 1 if you have the `fdatasync' function. - .HAVE_FDATASYNC = 1, - - // Define to 1 if you have the `gmtime_r' function. - .HAVE_GMTIME_R = 1, + .HAVE_FDATASYNC = 1, // Define to 1 if you have the `fdatasync' function. + .HAVE_GMTIME_R = 1, // Define to 1 if you have the `gmtime_r' function. .STDC_HEADERS = 1, .HAVE_STDINT_H = 1, @@ -231,88 +279,101 @@ pub fn build(b: *std.Build) void { .HAVE_UINT8_T = 1, .HAVE_UINTPTR_T = 1, - // Define to 1 if you have the `isnan' function. - .HAVE_ISNAN = 1, - - // Define to 1 if you have the `localtime_r' function. - .HAVE_LOCALTIME_R = 1, - - // Define to 1 if you have the `localtime_s' function. - .HAVE_LOCALTIME_S = 1, - - // Define to 1 if you have the header file. - .HAVE_MALLOC_H = 1, - - // Define to 1 if you have the `malloc_usable_size' function. - .HAVE_MALLOC_USABLE_SIZE = 1, - - // Define to 1 if you have the header file. - .HAVE_MEMORY_H = 1, + .HAVE_ISNAN = 1, // Define to 1 if you have the `isnan' function. + .HAVE_LOCALTIME_R = 1, // Define to 1 if you have the `localtime_r' function. + .HAVE_LOCALTIME_S = 1, // Define to 1 if you have the `localtime_s' function. + .HAVE_MALLOC_H = 1, // Define to 1 if you have the header file. + .HAVE_MALLOC_USABLE_SIZE = 1, // Define to 1 if you have the `malloc_usable_size' function. + .HAVE_MEMORY_H = 1, // Define to 1 if you have the header file. - // Define to 1 if you have the `pread' function. - .HAVE_PREAD = 1, + .HAVE_PREAD = 1, // Define to 1 if you have the `pread' function. .HAVE_PREAD64 = 1, .HAVE_PWRITE = 1, .HAVE_PWRITE64 = 1, - // Define to 1 if you have the `strchrnul' function. - .HAVE_STRCHRNUL = 1, - - // Define to 1 if you have the header file. - .HAVE_STRINGS_H = 1, - - // Define to 1 if you have the header file. - .HAVE_STRING_H = 1, - - // Define to 1 if you have the header file. - .HAVE_SYS_STAT_H = 1, - - // Define to 1 if you have the header file. - .HAVE_SYS_TYPES_H = 1, - - - // Define to 1 if you have the header file. - .HAVE_UNISTD_H = 1, - - // Define to 1 if you have the `usleep' function. - .HAVE_USLEEP = 1, - - // Define to 1 if you have the `utime' function. - .HAVE_UTIME = 1, + .HAVE_STRCHRNUL = 1, // Define to 1 if you have the `strchrnul' function. + .HAVE_STRINGS_H = 1, // Define to 1 if you have the header file. + .HAVE_STRING_H = 1, // Define to 1 if you have the header file. + .HAVE_SYS_STAT_H = 1, // Define to 1 if you have the header file. + .HAVE_SYS_TYPES_H = 1, // Define to 1 if you have the header file. + .HAVE_UNISTD_H = 1, // Define to 1 if you have the header file. + .HAVE_USLEEP = 1, // Define to 1 if you have the `usleep' function. + .HAVE_UTIME = 1, // Define to 1 if you have the `utime' function. + .HAVE_ZLIB_H = 1, // Define to 1 if you have the header file. + + .LT_OBJDIR = null, // Define to the sub-directory in which libtool stores uninstalled libraries. + .PACKAGE_BUGREPORT = "", // Define to the address where bug reports for this package should be sent. + .PACKAGE_NAME = "", // Define to the full name of this package. + .PACKAGE_STRING = "", // Define to the full name and version of this package. + .PACKAGE_TARNAME = "", // Define to the one symbol short name of this package. + .PACKAGE_URL = "", // Define to the home page for this package. + .PACKAGE_VERSION = "", // Define to the version of this package. + ._FILE_OFFSET_BITS = null, // Number of bits in a file offset, on hosts where this is settable. + ._LARGE_FILES = null, // Define for large files, on AIX-style hosts. + }, + ); - // Define to 1 if you have the header file. - .HAVE_ZLIB_H = 1, + const lemon = b.addExecutable(.{ + .name = "lemon", + .root_source_file = null, + .target = target, + .optimize = .ReleaseFast, + }); + lemon.addCSourceFile(.{ .file = b.path("tool/lemon.c") }); + lemon.linkLibC(); - // Define to the sub-directory in which libtool stores uninstalled libraries. - .LT_OBJDIR = null, + const mkkeywordhash = b.addExecutable(.{ + .name = "mkkeywordhash", + .root_source_file = null, + .target = target, + .optimize = .ReleaseFast, + }); + mkkeywordhash.addCSourceFile(.{ .file = b.path("tool/mkkeywordhash.c") }); + mkkeywordhash.linkLibC(); - // Define to the address where bug reports for this package should be sent. - .PACKAGE_BUGREPORT = "", + var parser = parser: { + const run = b.addRunArtifact(lemon); + run.addArg("-DSQLITE_ENABLE_MATH_FUNCTIONS"); + const parser = run.addPrefixedOutputDirectoryArg("-d", "."); + run.addArg("-S"); + run.addFileArg(b.path("src/parse.y")); + break :parser parser; + }; - // Define to the full name of this package. - .PACKAGE_NAME = "", + var keywordhash = b.addRunArtifact(mkkeywordhash); - // Define to the full name and version of this package. - .PACKAGE_STRING = "", + const sqlite_header = Amalgamation.create(b, "sqlite3.h", &.{ + sqlite_header_base.getOutput(), + b.path("src/page_header.h"), + b.path("src/wal.h"), + }); - // Define to the one symbol short name of this package. - .PACKAGE_TARNAME = "", + const debug = Debug.create(b, parser); + b.getInstallStep().dependOn(&debug.step); - // Define to the home page for this package. - .PACKAGE_URL = "", + const parser_vdbe = Amalgamation.create(b, "parse_vbde", &.{ + parser.path(b, "parse.h"), + b.path("src/vdbe.c"), + }); - // Define to the version of this package. - .PACKAGE_VERSION = "", + const opcode_h = b.addSystemCommand(&.{"tclsh"}); + opcode_h.addFileArg(b.path("tool/mkopcodeh.tcl")); + opcode_h.setStdIn(.{ .lazy_path = parser_vdbe.getOutput() }); - // Number of bits in a file offset, on hosts where this is settable. - ._FILE_OFFSET_BITS = null, + const opcode_c = b.addSystemCommand(&.{"tclsh"}); + opcode_c.addFileArg(b.path("tool/mkopcodec.tcl")); + opcode_c.addFileArg(opcode_h.captureStdOut()); - // Define for large files, on AIX-style hosts. - ._LARGE_FILES = null, - }, - ); + const opcode = b.addWriteFiles(); + _ = opcode.addCopyFile(keywordhash.captureStdOut(), "keywordhash.h"); + _ = opcode.addCopyFile(opcode_h.captureStdOut(), "opcodes.h"); + _ = opcode.addCopyFile(opcode_c.captureStdOut(), "opcodes.c"); - const debug = Debug.create(b, sqlite_header.getOutput()); + const flags = &.{ + "-g", + // "-DLIBSQL_ENABLE_WASM_RUNTIME", + "-DLIBSQL_OMIT_ALTERTABLE", + }; const sqlite3 = b.addStaticLibrary(.{ .name = "sqlite3", @@ -321,16 +382,25 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }); sqlite3.addIncludePath(b.path("src/")); - sqlite3.addIncludePath(b.path(".")); - sqlite3.addConfigHeader(sqlite_header); + sqlite3.addIncludePath(opcode.getDirectory()); + sqlite3.addIncludePath(parser); + sqlite3.addCSourceFile(.{ + .file = parser.path(b, "parse.c"), + .flags = flags, + }); + sqlite3.addIncludePath(sqlite_header.getOutput().dirname()); sqlite3.addConfigHeader(sqlite_cfg); - sqlite3.addCSourceFiles(.{ .files = sources.sqlite3, .flags = &.{ "-g", "-DLIBSQL_ENABLE_WASM_RUNTIME" } }); + sqlite3.addCSourceFiles(.{ + .files = sources.sqlite3, + .flags = flags, + }); + sqlite3.addCSourceFile(.{ + .file = opcode.getDirectory().path(b, "opcodes.c"), + .flags = flags, + }); sqlite3.linkLibC(); - debug.step.dependOn(&sqlite_header.step); - const install = b.addInstallArtifact(sqlite3, .{}); b.getInstallStep().dependOn(&install.step); - b.getInstallStep().dependOn(&debug.step); } diff --git a/libsql-sqlite3/src/sqlite.h.in b/libsql-sqlite3/src/sqlite.h.in index 2818c9cb14..17deb4c417 100644 --- a/libsql-sqlite3/src/sqlite.h.in +++ b/libsql-sqlite3/src/sqlite.h.in @@ -153,7 +153,7 @@ extern "C" { #define SQLITE_VERSION_NUMBER ${sqlite_version_number} #define SQLITE_SOURCE_ID "${sqlite_source_id}" -#define LIBSQL_VERSION ${libsql_version}" +#define LIBSQL_VERSION "${libsql_version}" /* ** CAPI3REF: Run-Time Library Version Numbers From 118e6b04f9a76c6a600b7568c0c365879a066b67 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Thu, 27 Jun 2024 22:35:28 -0300 Subject: [PATCH 06/35] fix(libsql-sqlite3): add parsing functions to internal API --- libsql-sqlite3/src/sqliteInt.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libsql-sqlite3/src/sqliteInt.h b/libsql-sqlite3/src/sqliteInt.h index 455d221476..947452b04c 100644 --- a/libsql-sqlite3/src/sqliteInt.h +++ b/libsql-sqlite3/src/sqliteInt.h @@ -5095,6 +5095,10 @@ int sqlite3SafetyCheckSickOrOk(sqlite3*); void sqlite3ChangeCookie(Parse*, int); With *sqlite3WithDup(sqlite3 *db, With *p); +/* libsql parse functions */ +void libsql_create_function(Parse *pParse, Token *pName, Token *pLang, Token *pBody, int isBlob, int noErr); +void libsql_drop_function(Parse *pParse, Token *pName, int noErr); + #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); #endif From 6385aae4a2e21049153de0b51ff7fc8f5c0b9602 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Thu, 27 Jun 2024 22:36:35 -0300 Subject: [PATCH 07/35] bump wasmtime-bindings rust-toolchain to lastest stable --- libsql-sqlite3/crates/wasmtime-bindings/rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsql-sqlite3/crates/wasmtime-bindings/rust-toolchain b/libsql-sqlite3/crates/wasmtime-bindings/rust-toolchain index b6148bc0a7..b3a8c61e6a 100644 --- a/libsql-sqlite3/crates/wasmtime-bindings/rust-toolchain +++ b/libsql-sqlite3/crates/wasmtime-bindings/rust-toolchain @@ -1 +1 @@ -1.66.0 +1.79.0 From f5ecc9a14db88d48a189e9a61c8085dd90430b29 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Thu, 27 Jun 2024 22:37:59 -0300 Subject: [PATCH 08/35] temporary solution for build.crab current limitations --- libsql-sqlite3/crates/wasmtime-bindings/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsql-sqlite3/crates/wasmtime-bindings/Cargo.toml b/libsql-sqlite3/crates/wasmtime-bindings/Cargo.toml index e25335cdd8..02c9c1ecaa 100644 --- a/libsql-sqlite3/crates/wasmtime-bindings/Cargo.toml +++ b/libsql-sqlite3/crates/wasmtime-bindings/Cargo.toml @@ -14,7 +14,7 @@ wasmtime-wasi = "9.0" [lib] name = "libsql_wasm" -crate-type = ["lib", "cdylib", "staticlib"] +crate-type = ["staticlib"] doc = false test = false doctest = false From 4594f32bca7176673320b4bf252338aba4f6e537 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Fri, 28 Jun 2024 17:40:31 -0300 Subject: [PATCH 09/35] flake: add icu and pkg-config --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 43c21ed2ca..640584ed69 100644 --- a/flake.nix +++ b/flake.nix @@ -19,12 +19,16 @@ with pkgs; mkShell { nativeBuildInputs = with pkgs; [ + pkg-config zig tcl cmake gnumake clang ]; + buildInputs = with pkgs; [ + icu.dev + ]; }; } ); From 04e71ac9c06c40fb45407c9a557c1ec580e050d8 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Fri, 28 Jun 2024 17:41:02 -0300 Subject: [PATCH 10/35] WIP --- libsql-sqlite3/build.zig | 220 +++++++++++++++-------------------- libsql-sqlite3/build.zig.zon | 41 +++++++ 2 files changed, 134 insertions(+), 127 deletions(-) create mode 100644 libsql-sqlite3/build.zig.zon diff --git a/libsql-sqlite3/build.zig b/libsql-sqlite3/build.zig index 709a831fc1..79116870bd 100644 --- a/libsql-sqlite3/build.zig +++ b/libsql-sqlite3/build.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const crab = @import("build.crab"); pub const Debug = struct { step: std.Build.Step, @@ -128,105 +129,35 @@ pub fn build(b: *std.Build) void { const sources = .{ .sqlite3 = &.{ - "src/alter.c", - "src/analyze.c", - "src/attach.c", - "src/auth.c", - "src/backup.c", - "src/bitvec.c", - "src/btmutex.c", - "src/btree.c", - "src/build.c", - "src/callback.c", - "src/complete.c", - "src/ctime.c", - "src/date.c", - "src/dbpage.c", - "src/dbstat.c", - "src/delete.c", - "src/expr.c", - "src/fault.c", - "src/fkey.c", - "src/func.c", - "src/global.c", - "src/hash.c", - "src/insert.c", - "src/json.c", - "src/legacy.c", - "src/loadext.c", - "src/main.c", - "src/malloc.c", - "src/mem0.c", - "src/mem1.c", - "src/mem2.c", - "src/mem3.c", - "src/mem5.c", - "src/memdb.c", - "src/memjournal.c", - "src/mutex.c", - "src/mutex_noop.c", - "src/mutex_unix.c", - "src/mutex_w32.c", - "src/notify.c", - "src/os.c", - "src/os_kv.c", - "src/os_unix.c", - "src/os_win.c", - "src/pager.c", - "src/pcache.c", - "src/pcache1.c", - "src/pragma.c", - "src/prepare.c", - "src/printf.c", - "src/random.c", - "src/resolve.c", - "src/rowset.c", - "src/select.c", - "src/status.c", - "src/table.c", - "src/threads.c", - "src/tokenize.c", - "src/treeview.c", - "src/trigger.c", - "src/utf.c", - "src/update.c", - "src/upsert.c", - "src/util.c", - "src/vacuum.c", - "src/vdbe.c", - "src/vdbeapi.c", - "src/vdbeaux.c", - "src/vdbeblob.c", - "src/vdbemem.c", - "src/vdbesort.c", - "src/vdbetrace.c", - "src/vdbevtab.c", - "src/vtab.c", - "src/wal.c", - "src/walker.c", - "src/where.c", - "src/wherecode.c", - "src/whereexpr.c", - "src/window.c", + "src/alter.c", "src/analyze.c", "src/attach.c", "src/auth.c", + "src/backup.c", "src/bitvec.c", "src/btmutex.c", "src/btree.c", + "src/build.c", "src/callback.c", "src/complete.c", "src/ctime.c", + "src/date.c", "src/dbpage.c", "src/dbstat.c", "src/delete.c", + "src/expr.c", "src/fault.c", "src/fkey.c", "src/func.c", + "src/global.c", "src/hash.c", "src/insert.c", "src/json.c", + "src/legacy.c", "src/loadext.c", "src/main.c", "src/malloc.c", + "src/mem0.c", "src/mem1.c", "src/mem2.c", "src/mem3.c", + "src/mem5.c", "src/memdb.c", "src/memjournal.c", "src/mutex.c", + "src/mutex_noop.c", "src/mutex_unix.c", "src/mutex_w32.c", "src/notify.c", + "src/os.c", "src/os_kv.c", "src/os_unix.c", "src/os_win.c", + "src/pager.c", "src/pcache.c", "src/pcache1.c", "src/pragma.c", + "src/prepare.c", "src/printf.c", "src/random.c", "src/resolve.c", + "src/rowset.c", "src/select.c", "src/status.c", "src/table.c", + "src/threads.c", "src/tokenize.c", "src/treeview.c", "src/trigger.c", + "src/utf.c", "src/update.c", "src/upsert.c", "src/util.c", + "src/vacuum.c", "src/vdbe.c", "src/vdbeapi.c", "src/vdbeaux.c", + "src/vdbeblob.c", "src/vdbemem.c", "src/vdbesort.c", "src/vdbetrace.c", + "src/vdbevtab.c", "src/vtab.c", "src/wal.c", "src/walker.c", + "src/where.c", "src/wherecode.c", "src/whereexpr.c", "src/window.c", }, .extensions = .{ .fts3 = &.{ - "ext/fts3/fts3.c", - "ext/fts3/fts3.h", - "ext/fts3/fts3Int.h", - "ext/fts3/fts3_aux.c", - "ext/fts3/fts3_expr.c", - "ext/fts3/fts3_hash.c", - "ext/fts3/fts3_hash.h", - "ext/fts3/fts3_icu.c", - "ext/fts3/fts3_porter.c", - "ext/fts3/fts3_snippet.c", - "ext/fts3/fts3_tokenizer.h", - "ext/fts3/fts3_tokenizer.c", - "ext/fts3/fts3_tokenizer1.c", - "ext/fts3/fts3_tokenize_vtab.c", - "ext/fts3/fts3_unicode.c", - "ext/fts3/fts3_unicode2.c", + "ext/fts3/fts3.c", "ext/fts3/fts3_aux.c", + "ext/fts3/fts3_expr.c", "ext/fts3/fts3_hash.c", + "ext/fts3/fts3_icu.c", "ext/fts3/fts3_porter.c", + "ext/fts3/fts3_snippet.c", "ext/fts3/fts3_tokenizer.c", + "ext/fts3/fts3_tokenizer1.c", "ext/fts3/fts3_tokenize_vtab.c", + "ext/fts3/fts3_unicode.c", "ext/fts3/fts3_unicode2.c", "ext/fts3/fts3_write.c", }, .icu = &.{"ext/icu/icu.c"}, @@ -245,10 +176,10 @@ pub fn build(b: *std.Build) void { .style = .{ .cmake = b.path("src/sqlite.h.in") }, }, .{ + .libsql_version = "0.2.3", .sqlite_version = "3.44.0", .sqlite_version_number = 3044000, .sqlite_source_id = "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad8alt1", - .libsql_version = "0.2.3", }, ); @@ -333,6 +264,7 @@ pub fn build(b: *std.Build) void { var parser = parser: { const run = b.addRunArtifact(lemon); + run.setCwd(b.path("tool/")); run.addArg("-DSQLITE_ENABLE_MATH_FUNCTIONS"); const parser = run.addPrefixedOutputDirectoryArg("-d", "."); run.addArg("-S"); @@ -340,16 +272,23 @@ pub fn build(b: *std.Build) void { break :parser parser; }; + var wasm_runtime = b.option(bool, "wasm-runtime", "Enable wasm runtime (default: false)") orelse false; + var icu = b.option(bool, "icu", "Enable icu extension (default: false)") orelse false; + var keywordhash = b.addRunArtifact(mkkeywordhash); - const sqlite_header = Amalgamation.create(b, "sqlite3.h", &.{ - sqlite_header_base.getOutput(), - b.path("src/page_header.h"), - b.path("src/wal.h"), - }); + const sqlite_header = Amalgamation.create( + b, + "sqlite3.h", + &.{ + sqlite_header_base.getOutput(), + b.path("src/page_header.h"), + b.path("src/wal.h"), + }, + ); - const debug = Debug.create(b, parser); - b.getInstallStep().dependOn(&debug.step); + // const debug = Debug.create(b, parser); + // b.getInstallStep().dependOn(&debug.step); const parser_vdbe = Amalgamation.create(b, "parse_vbde", &.{ parser.path(b, "parse.h"), @@ -364,15 +303,14 @@ pub fn build(b: *std.Build) void { opcode_c.addFileArg(b.path("tool/mkopcodec.tcl")); opcode_c.addFileArg(opcode_h.captureStdOut()); - const opcode = b.addWriteFiles(); - _ = opcode.addCopyFile(keywordhash.captureStdOut(), "keywordhash.h"); - _ = opcode.addCopyFile(opcode_h.captureStdOut(), "opcodes.h"); - _ = opcode.addCopyFile(opcode_c.captureStdOut(), "opcodes.c"); + const generated = b.addWriteFiles(); + _ = generated.addCopyFile(keywordhash.captureStdOut(), "keywordhash.h"); + _ = generated.addCopyFile(opcode_h.captureStdOut(), "opcodes.h"); + _ = generated.addCopyFile(opcode_c.captureStdOut(), "opcodes.c"); const flags = &.{ "-g", - // "-DLIBSQL_ENABLE_WASM_RUNTIME", - "-DLIBSQL_OMIT_ALTERTABLE", + if (wasm_runtime) "-DLIBSQL_ENABLE_WASM_RUNTIME" else "", }; const sqlite3 = b.addStaticLibrary(.{ @@ -381,24 +319,52 @@ pub fn build(b: *std.Build) void { .target = target, .optimize = optimize, }); - sqlite3.addIncludePath(b.path("src/")); - sqlite3.addIncludePath(opcode.getDirectory()); - sqlite3.addIncludePath(parser); - sqlite3.addCSourceFile(.{ - .file = parser.path(b, "parse.c"), - .flags = flags, - }); - sqlite3.addIncludePath(sqlite_header.getOutput().dirname()); - sqlite3.addConfigHeader(sqlite_cfg); - sqlite3.addCSourceFiles(.{ - .files = sources.sqlite3, - .flags = flags, - }); - sqlite3.addCSourceFile(.{ - .file = opcode.getDirectory().path(b, "opcodes.c"), - .flags = flags, + sqlite3.addObject(object: { + const object = b.addObject(.{ + .name = "sqlite3", + .target = target, + .optimize = optimize, + }); + object.linkLibC(); + object.addIncludePath(b.path("src/")); + object.addIncludePath(generated.getDirectory()); + object.addIncludePath(parser); + object.addIncludePath(sqlite_header.getOutput().dirname()); + object.addConfigHeader(sqlite_cfg); + object.addCSourceFile(.{ + .file = generated.getDirectory().path(b, "opcodes.c"), + .flags = flags, + }); + object.addCSourceFile(.{ .file = parser.path(b, "parse.c"), .flags = flags }); + object.addCSourceFiles(.{ .files = sources.sqlite3, .flags = flags }); + + if (icu) { + } + + if (wasm_runtime) { + _ = generated.addCopyFile(b.path("ext/udf/wasm_bindings.h"), "ext/udf/wasm_bindings.h"); + + const libsql_wasm = crab.addCargoBuildWithUserOptions(b, .{ + .name = "liblibsql_wasm.a", + .manifest_path = b.path("crates/wasmtime-bindings/Cargo.toml"), + .cargo_args = &.{ + "--release", + "--lib", + }, + }, .{ + .target = target, + .optimize = .ReleaseSafe, + }); + + object.addCSourceFile(.{ + .file = b.path(sources.wasm), + .flags = flags ++ &.{"-DSQLITE_CORE"}, + }); + object.addObjectFile(libsql_wasm); + } + + break :object object; }); - sqlite3.linkLibC(); const install = b.addInstallArtifact(sqlite3, .{}); diff --git a/libsql-sqlite3/build.zig.zon b/libsql-sqlite3/build.zig.zon new file mode 100644 index 0000000000..fc57b37792 --- /dev/null +++ b/libsql-sqlite3/build.zig.zon @@ -0,0 +1,41 @@ +.{ + // This is the default name used by packages depending on this one. For + // example, when a user runs `zig fetch --save `, this field is used + // as the key in the `dependencies` table. Although the user can choose a + // different name, most users will stick with this provided value. + // + // It is redundant to include "zig" in this name because it is already + // within the Zig package namespace. + .name = "libsql-sqlite3", + + // This is a [Semantic Version](https://semver.org/). + // In a future version of Zig it will be used for package deduplication. + .version = "0.0.0", + + // This field is optional. + // This is currently advisory only; Zig does not yet do anything + // with this value. + //.minimum_zig_version = "0.11.0", + + // This field is optional. + // Each dependency must either provide a `url` and `hash`, or a `path`. + // `zig build --fetch` can be used to fetch all dependencies of a package, recursively. + // Once all dependencies are fetched, `zig build` no longer requires + // internet connectivity. + .dependencies = .{ + .@"build.crab" = .{ + .url = "https://github.com/akarpovskii/build.crab/archive/refs/tags/v0.1.5.tar.gz", + .hash = "1220c7d071bdde500955b0d5b807cb1a66687b915a020c3e87496623243912c77bed", + }, + }, + .paths = .{ + "build.zig", + "build.zig.zon", + "src", + "ext", + "crate", + // For example... + //"LICENSE", + //"README.md", + }, +} From 2eb6d564c02943e6c2bf6c5c5f90834bdffc9525 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Sat, 29 Jun 2024 02:52:11 -0300 Subject: [PATCH 11/35] add(flake): zlib dev dependency --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 640584ed69..2079ec1416 100644 --- a/flake.nix +++ b/flake.nix @@ -28,6 +28,7 @@ ]; buildInputs = with pkgs; [ icu.dev + zlib.dev ]; }; } From ee33ce31bee3d6068781cbb819cceda8a20d49e5 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Mon, 1 Jul 2024 22:32:06 -0300 Subject: [PATCH 12/35] fix(libsql-sqlite3): definition seems to not match this might be reverted later, needs further inspection --- libsql-sqlite3/src/test8.c | 50 ++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/libsql-sqlite3/src/test8.c b/libsql-sqlite3/src/test8.c index 71f8884c01..e217bd0a52 100644 --- a/libsql-sqlite3/src/test8.c +++ b/libsql-sqlite3/src/test8.c @@ -1337,32 +1337,30 @@ static sqlite3_module echoModule = { }; static sqlite3_module echoModuleV2 = { - 700, /* iVersion */ - echoCreate, - echoConnect, - echoBestIndex, - echoDisconnect, - echoDestroy, - echoOpen, /* xOpen - open a cursor */ - echoClose, /* xClose - close a cursor */ - echoFilter, /* xFilter - configure scan constraints */ - echoNext, /* xNext - advance a cursor */ - echoEof, /* xEof */ - echoColumn, /* xColumn - read data */ - echoRowid, /* xRowid - read data */ - echoUpdate, /* xUpdate - write data */ - echoBegin, /* xBegin - begin transaction */ - echoSync, /* xSync - sync transaction */ - echoCommit, /* xCommit - commit transaction */ - echoRollback, /* xRollback - rollback transaction */ - echoFindFunction, /* xFindFunction - function overloading */ - echoRename, /* xRename - rename the table */ - echoSavepoint, - echoRelease, - echoRollbackTo, - 0, /* xShadowName */ - echoPreparedSql, - 0 /* xIntegrity */ + .iVersion = 700, /* iVersion */ + .xCreate = echoCreate, + .xConnect = echoConnect, + .xBestIndex = echoBestIndex, + .xDisconnect = echoDisconnect, + .xDestroy = echoDestroy, + .xOpen = echoOpen, /* xOpen - open a cursor */ + .xClose = echoClose, /* xClose - close a cursor */ + .xFilter = echoFilter, /* xFilter - configure scan constraints */ + .xNext = echoNext, /* xNext - advance a cursor */ + .xEof = echoEof, /* xEof */ + .xColumn = echoColumn, /* xColumn - read data */ + .xRowid = echoRowid, /* xRowid - read data */ + .xUpdate = echoUpdate, /* xUpdate - write data */ + .xBegin = echoBegin, /* xBegin - begin transaction */ + .xSync = echoSync, /* xSync - sync transaction */ + .xCommit = echoCommit, /* xCommit - commit transaction */ + .xRollback = echoRollback, /* xRollback - rollback transaction */ + .xFindFunction = echoFindFunction, /* xFindFunction - function overloading */ + .xRename = echoRename, /* xRename - rename the table */ + .xSavepoint = echoSavepoint, + .xRelease = echoRelease, + .xRollbackTo = echoRollbackTo, + .xPreparedSql = echoPreparedSql, }; /* From 5573d707fb0ff7da0e2c7452eef0157c7842d9d8 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Mon, 1 Jul 2024 22:33:32 -0300 Subject: [PATCH 13/35] fix(build.zig): mostly working, needs some refactor --- libsql-sqlite3/build.zig | 842 +++++++++++++++++++++++++++-------- libsql-sqlite3/build.zig.zon | 4 + 2 files changed, 658 insertions(+), 188 deletions(-) diff --git a/libsql-sqlite3/build.zig b/libsql-sqlite3/build.zig index 79116870bd..4e7e56ffc5 100644 --- a/libsql-sqlite3/build.zig +++ b/libsql-sqlite3/build.zig @@ -1,11 +1,71 @@ const std = @import("std"); const crab = @import("build.crab"); +const fs = std.fs; +const Build = std.Build; +const LazyPath = Build.LazyPath; + +const sources = .{ + .sqlite3 = &.{ + "src/alter.c", "src/analyze.c", "src/attach.c", "src/auth.c", + "src/backup.c", "src/bitvec.c", "src/btmutex.c", "src/btree.c", + "src/build.c", "src/callback.c", "src/complete.c", "src/ctime.c", + "src/date.c", "src/dbpage.c", "src/dbstat.c", "src/delete.c", + "src/expr.c", "src/fault.c", "src/fkey.c", "src/func.c", + "src/global.c", "src/hash.c", "src/insert.c", "src/json.c", + "src/legacy.c", "src/loadext.c", "src/main.c", "src/malloc.c", + "src/mem0.c", "src/mem1.c", "src/mem2.c", "src/mem3.c", + "src/mem5.c", "src/memdb.c", "src/memjournal.c", "src/mutex.c", + "src/mutex_noop.c", "src/mutex_unix.c", "src/mutex_w32.c", "src/notify.c", + "src/os.c", "src/os_kv.c", "src/os_unix.c", "src/os_win.c", + "src/pager.c", "src/pcache.c", "src/pcache1.c", "src/pragma.c", + "src/prepare.c", "src/printf.c", "src/random.c", "src/resolve.c", + "src/rowset.c", "src/select.c", "src/status.c", "src/table.c", + "src/threads.c", "src/tokenize.c", "src/treeview.c", "src/trigger.c", + "src/utf.c", "src/update.c", "src/upsert.c", "src/util.c", + "src/vacuum.c", "src/vdbe.c", "src/vdbeapi.c", "src/vdbeaux.c", + "src/vdbeblob.c", "src/vdbemem.c", "src/vdbesort.c", "src/vdbetrace.c", + "src/vdbevtab.c", "src/vtab.c", "src/wal.c", "src/walker.c", + "src/where.c", "src/wherecode.c", "src/whereexpr.c", "src/window.c", + }, + .fuzzcheck = &.{ + "test/fuzzcheck.c", + "test/ossfuzz.c", + "test/fuzzinvariants.c", + "ext/recover/dbdata.c", + "ext/recover/sqlite3recover.c", + "test/vt02.c", + }, + .@"test" = &.{ + "src/test1.c", "src/test2.c", + "src/test3.c", "src/test4.c", + "src/test5.c", "src/test6.c", + "src/test8.c", "src/test9.c", + "src/test_autoext.c", "src/test_async.c", + "src/test_backup.c", "src/test_bestindex.c", + "src/test_blob.c", "src/test_btree.c", + "src/test_config.c", "src/test_delete.c", + "src/test_demovfs.c", "src/test_devsym.c", + "src/test_fs.c", "src/test_func.c", + "src/test_hexio.c", "src/test_init.c", + "src/test_intarray.c", "src/test_journal.c", + "src/test_malloc.c", "src/test_md5.c", + "src/test_multiplex.c", "src/test_mutex.c", + "src/test_onefile.c", "src/test_osinst.c", + "src/test_pcache.c", "src/test_quota.c", + "src/test_rtree.c", "src/test_schema.c", + "src/test_superlock.c", "src/test_syscall.c", + "src/test_tclsh.c", "src/test_tclvar.c", + "src/test_thread.c", "src/test_vdbecov.c", + "src/test_vfs.c", "src/test_windirent.c", + "src/test_window.c", "src/test_wsd.c", + }, +}; pub const Debug = struct { step: std.Build.Step, path: std.Build.LazyPath, - pub fn create(b: *std.Build, path: std.Build.LazyPath) *Debug { + pub fn create(b: *Build, path: LazyPath) *Debug { const self = b.allocator.create(Debug) catch @panic("OOM"); self.* = .{ .step = std.Build.Step.init(.{ @@ -22,22 +82,22 @@ pub const Debug = struct { return self; } - pub fn make(step: *std.Build.Step, _: std.Progress.Node) !void { + pub fn make(step: *Build.Step, _: std.Progress.Node) !void { const self: *Debug = @fieldParentPtr("step", step); const b = self.step.owner; - std.debug.print("welp {s}\n", .{self.path.getPath(b)}); + std.debug.print("lazy_path: {s}\n", .{self.path.getPath(b)}); } }; pub const Amalgamation = struct { - step: std.Build.Step, + step: Build.Step, lazy_paths: std.ArrayList(std.Build.LazyPath), - output_list: std.Build.GeneratedFile, + output_list: Build.GeneratedFile, basename: []const u8, - pub fn create(b: *std.Build, basename: []const u8, lazy_paths: []const std.Build.LazyPath) *Amalgamation { + pub fn create(b: *Build, basename: []const u8, lazy_paths: []const LazyPath) *Amalgamation { const self = b.allocator.create(Amalgamation) catch @panic("OOM"); self.* = .{ @@ -52,7 +112,7 @@ pub const Amalgamation = struct { .output_list = .{ .step = &self.step }, }; - var list = std.ArrayList(std.Build.LazyPath).init(b.allocator); + var list = std.ArrayList(LazyPath).init(b.allocator); for (lazy_paths) |lp| { list.append(lp) catch @panic("OOM"); @@ -64,11 +124,11 @@ pub const Amalgamation = struct { return self; } - pub fn getOutput(self: *const Amalgamation) std.Build.LazyPath { + pub fn getOutput(self: *const Amalgamation) LazyPath { return .{ .generated = .{ .file = &self.output_list } }; } - pub fn make(step: *std.Build.Step, prog_node: std.Progress.Node) !void { + pub fn make(step: *Build.Step, prog_node: std.Progress.Node) !void { _ = prog_node; const self: *Amalgamation = @fieldParentPtr("step", step); const b = step.owner; @@ -119,254 +179,660 @@ pub const Amalgamation = struct { .data = output.items, }); self.output_list.path = try b.cache_root.join(b.allocator, &.{sub_path}); - // try man.writeManifest(); + + try man.writeManifest(); } }; -pub fn build(b: *std.Build) void { - const target = b.standardTargetOptions(.{}); - const optimize = b.standardOptimizeOption(.{}); - - const sources = .{ - .sqlite3 = &.{ - "src/alter.c", "src/analyze.c", "src/attach.c", "src/auth.c", - "src/backup.c", "src/bitvec.c", "src/btmutex.c", "src/btree.c", - "src/build.c", "src/callback.c", "src/complete.c", "src/ctime.c", - "src/date.c", "src/dbpage.c", "src/dbstat.c", "src/delete.c", - "src/expr.c", "src/fault.c", "src/fkey.c", "src/func.c", - "src/global.c", "src/hash.c", "src/insert.c", "src/json.c", - "src/legacy.c", "src/loadext.c", "src/main.c", "src/malloc.c", - "src/mem0.c", "src/mem1.c", "src/mem2.c", "src/mem3.c", - "src/mem5.c", "src/memdb.c", "src/memjournal.c", "src/mutex.c", - "src/mutex_noop.c", "src/mutex_unix.c", "src/mutex_w32.c", "src/notify.c", - "src/os.c", "src/os_kv.c", "src/os_unix.c", "src/os_win.c", - "src/pager.c", "src/pcache.c", "src/pcache1.c", "src/pragma.c", - "src/prepare.c", "src/printf.c", "src/random.c", "src/resolve.c", - "src/rowset.c", "src/select.c", "src/status.c", "src/table.c", - "src/threads.c", "src/tokenize.c", "src/treeview.c", "src/trigger.c", - "src/utf.c", "src/update.c", "src/upsert.c", "src/util.c", - "src/vacuum.c", "src/vdbe.c", "src/vdbeapi.c", "src/vdbeaux.c", - "src/vdbeblob.c", "src/vdbemem.c", "src/vdbesort.c", "src/vdbetrace.c", - "src/vdbevtab.c", "src/vtab.c", "src/wal.c", "src/walker.c", - "src/where.c", "src/wherecode.c", "src/whereexpr.c", "src/window.c", - }, - .extensions = .{ - .fts3 = &.{ - "ext/fts3/fts3.c", "ext/fts3/fts3_aux.c", - "ext/fts3/fts3_expr.c", "ext/fts3/fts3_hash.c", - "ext/fts3/fts3_icu.c", "ext/fts3/fts3_porter.c", - "ext/fts3/fts3_snippet.c", "ext/fts3/fts3_tokenizer.c", - "ext/fts3/fts3_tokenizer1.c", "ext/fts3/fts3_tokenize_vtab.c", - "ext/fts3/fts3_unicode.c", "ext/fts3/fts3_unicode2.c", - "ext/fts3/fts3_write.c", - }, - .icu = &.{"ext/icu/icu.c"}, - .rtree = &.{ "ext/rtree/rtree.c", "ext/rtree/geopoly.c" }, - .session = &.{"ext/session/sqlite3session.c"}, - .auth = &.{"ext/userauth/userauth.c"}, - .rbu = &.{"ext/rbu/sqlite3rbu.c"}, - .stmt = &.{"ext/misc/stmt.c"}, - .wasm = &.{"ext/udf/wasmedge_bindings.c"}, +fn cflags(b: *Build, flags: []const []const u8) [][]const u8 { + return std.mem.concat(b.allocator, []const u8, &.{ + &.{ + "-g", + // "-O3", + "-DBUILD_sqlite", + "-DSQLITE_HAVE_ZLIB", + "-DNDEBUG", + "-DSQLITE_ENABLE_MATH_FUNCTIONS", + "-DSQLITE_TEMP_STORE=2", + "-DSQLITE_USE_URI=1", + "-DSQLITE_THREADSAFE=1", + "-D_HAVE_SQLITE_CONFIG", + "-DSQLITE_CORE", + + // FIX: make this configurable + "-DSQLITE_OS_UNIX=1", }, - }; + flags, + }) catch @panic("OOM"); +} - const sqlite_header_base = b.addConfigHeader( - .{ - .include_path = "sqlite3.h", - .style = .{ .cmake = b.path("src/sqlite.h.in") }, - }, - .{ - .libsql_version = "0.2.3", - .sqlite_version = "3.44.0", - .sqlite_version_number = 3044000, - .sqlite_source_id = "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad8alt1", - }, - ); +const Sqlite3Options = struct { + target: Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, + wasm_runtime: bool, + icu: bool, + @"test": bool = false, +}; +const Sqlite3 = struct { + target: Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, + wasm_runtime: bool, + icu: bool, + @"test": bool = false, +}; + +fn addSqlite3(b: *Build, options: Sqlite3Options) struct { + h: LazyPath, + lib: *Build.Step.Compile, +} { const sqlite_cfg = b.addConfigHeader( .{ .include_path = "sqlite_cfg.h", .style = .{ .autoconf = b.path("sqlite_cfg.h.in") }, }, .{ - .HAVE_DLFCN_H = null, .HAVE_FDATASYNC = 1, // Define to 1 if you have the `fdatasync' function. - .HAVE_GMTIME_R = 1, // Define to 1 if you have the `gmtime_r' function. + .HAVE_GMTIME_R = 1, // only use is in src/data.c:1578 - .STDC_HEADERS = 1, + // available in C99, better remove this later and assume C99 compatibility .HAVE_STDINT_H = 1, - .HAVE_STDLIB_H = 1, + // has the same stuff as , with more bloat -- remove + // later in sqliteInt.h + .HAVE_INTTYPES_H = 1, + + // this should be allways available .HAVE_INT16_T = 1, .HAVE_INT32_T = 1, .HAVE_INT64_T = 1, .HAVE_INT8_T = 1, .HAVE_INTPTR_T = 1, - .HAVE_INTTYPES_H = 1, - .HAVE_UINT16_T = 1, .HAVE_UINT32_T = 1, .HAVE_UINT64_T = 1, .HAVE_UINT8_T = 1, .HAVE_UINTPTR_T = 1, - .HAVE_ISNAN = 1, // Define to 1 if you have the `isnan' function. - .HAVE_LOCALTIME_R = 1, // Define to 1 if you have the `localtime_r' function. - .HAVE_LOCALTIME_S = 1, // Define to 1 if you have the `localtime_s' function. - .HAVE_MALLOC_H = 1, // Define to 1 if you have the header file. - .HAVE_MALLOC_USABLE_SIZE = 1, // Define to 1 if you have the `malloc_usable_size' function. - .HAVE_MEMORY_H = 1, // Define to 1 if you have the header file. + // available in C99 + .HAVE_ISNAN = 1, + + // available in C23, previously was a non-POSIX API on Unix-like systems + .HAVE_LOCALTIME_R = 1, + + // available in C11, previously was a Microsoft API + .HAVE_LOCALTIME_S = 1, + + // generaly avaiable, non-POSIX + .HAVE_MALLOC_H = 1, + + // malloc_usable_size is defined in for unix systems, + // MacOS seems to not have it, not sure + // TODO: invetigate the use of this API + .HAVE_MALLOC_USABLE_SIZE = 1, - .HAVE_PREAD = 1, // Define to 1 if you have the `pread' function. - .HAVE_PREAD64 = 1, + // TODO: stage removal, useful but not used + .HAVE_PREAD = 1, .HAVE_PWRITE = 1, - .HAVE_PWRITE64 = 1, - - .HAVE_STRCHRNUL = 1, // Define to 1 if you have the `strchrnul' function. - .HAVE_STRINGS_H = 1, // Define to 1 if you have the header file. - .HAVE_STRING_H = 1, // Define to 1 if you have the header file. - .HAVE_SYS_STAT_H = 1, // Define to 1 if you have the header file. - .HAVE_SYS_TYPES_H = 1, // Define to 1 if you have the header file. - .HAVE_UNISTD_H = 1, // Define to 1 if you have the header file. - .HAVE_USLEEP = 1, // Define to 1 if you have the `usleep' function. - .HAVE_UTIME = 1, // Define to 1 if you have the `utime' function. - .HAVE_ZLIB_H = 1, // Define to 1 if you have the header file. - - .LT_OBJDIR = null, // Define to the sub-directory in which libtool stores uninstalled libraries. - .PACKAGE_BUGREPORT = "", // Define to the address where bug reports for this package should be sent. - .PACKAGE_NAME = "", // Define to the full name of this package. - .PACKAGE_STRING = "", // Define to the full name and version of this package. - .PACKAGE_TARNAME = "", // Define to the one symbol short name of this package. - .PACKAGE_URL = "", // Define to the home page for this package. + + // Legacy API available only on GLibc and deprecated in musl, remove this later + .HAVE_PREAD64 = null, + .HAVE_PWRITE64 = null, + ._FILE_OFFSET_BITS = null, + + // used in src/printf.c:227, seems to be a GNU-ism, but fine + .HAVE_STRCHRNUL = 1, + + .HAVE_USLEEP = 1, // src/os_unix.c *important for performance* + .HAVE_UTIME = 1, // src/os_unix.c src/vxworks.h + .PACKAGE_VERSION = "", // Define to the version of this package. - ._FILE_OFFSET_BITS = null, // Number of bits in a file offset, on hosts where this is settable. - ._LARGE_FILES = null, // Define for large files, on AIX-style hosts. + + // never used + .HAVE_DLFCN_H = null, + .HAVE_MEMORY_H = null, + .HAVE_STDLIB_H = null, + .HAVE_STRINGS_H = null, + .HAVE_STRING_H = null, + .HAVE_SYS_STAT_H = null, + .HAVE_SYS_TYPES_H = null, + .HAVE_UNISTD_H = null, + .HAVE_ZLIB_H = null, + .LT_OBJDIR = null, + .PACKAGE_BUGREPORT = null, + .PACKAGE_NAME = null, + .PACKAGE_STRING = null, + .PACKAGE_TARNAME = null, + .PACKAGE_URL = null, + .STDC_HEADERS = null, + ._LARGE_FILES = null, }, ); const lemon = b.addExecutable(.{ .name = "lemon", .root_source_file = null, - .target = target, + .target = options.target, .optimize = .ReleaseFast, }); lemon.addCSourceFile(.{ .file = b.path("tool/lemon.c") }); lemon.linkLibC(); - const mkkeywordhash = b.addExecutable(.{ - .name = "mkkeywordhash", - .root_source_file = null, - .target = target, - .optimize = .ReleaseFast, - }); - mkkeywordhash.addCSourceFile(.{ .file = b.path("tool/mkkeywordhash.c") }); - mkkeywordhash.linkLibC(); - - var parser = parser: { + var parse = parse: { const run = b.addRunArtifact(lemon); run.setCwd(b.path("tool/")); run.addArg("-DSQLITE_ENABLE_MATH_FUNCTIONS"); - const parser = run.addPrefixedOutputDirectoryArg("-d", "."); + const dir = run.addPrefixedOutputDirectoryArg("-d", "."); run.addArg("-S"); run.addFileArg(b.path("src/parse.y")); - break :parser parser; + + break :parse .{ + .h = dir.path(b, "parse.h"), + .c = dir.path(b, "parse.c"), + }; }; - var wasm_runtime = b.option(bool, "wasm-runtime", "Enable wasm runtime (default: false)") orelse false; - var icu = b.option(bool, "icu", "Enable icu extension (default: false)") orelse false; + const keywordhash = keywordhash: { + const mkkeywordhash = b.addExecutable(.{ + .name = "mkkeywordhash", + .root_source_file = null, + .target = options.target, + .optimize = .ReleaseFast, + }); + mkkeywordhash.addCSourceFile(.{ .file = b.path("tool/mkkeywordhash.c") }); + mkkeywordhash.linkLibC(); + + break :keywordhash .{ + .h = b.addWriteFiles().addCopyFile( + b.addRunArtifact(mkkeywordhash).captureStdOut(), + "keywordhash.h", + ), + }; + }; + + const opcodes: struct { + h: std.Build.LazyPath, + c: std.Build.LazyPath, + } = opcode: { + const h = h: { + const h = b.addSystemCommand(&.{"tclsh"}); + h.addFileArg(b.path("tool/mkopcodeh.tcl")); + h.setStdIn(.{ + .lazy_path = Amalgamation.create(b, "parse_vbde", &.{ + parse.h, + b.path("src/vdbe.c"), + }).getOutput(), + }); + break :h b.addWriteFiles().addCopyFile( + h.captureStdOut(), + "opcodes.h", + ); + }; + + const c = c: { + const c = b.addSystemCommand(&.{"tclsh"}); + c.addFileArg(b.path("tool/mkopcodec.tcl")); + c.addFileArg(h); + break :c b.addWriteFiles().addCopyFile( + c.captureStdOut(), + "opcodes.c", + ); + }; - var keywordhash = b.addRunArtifact(mkkeywordhash); + break :opcode .{ + .h = h, + .c = c, + }; + }; - const sqlite_header = Amalgamation.create( - b, - "sqlite3.h", - &.{ - sqlite_header_base.getOutput(), - b.path("src/page_header.h"), - b.path("src/wal.h"), + const h = Amalgamation.create(b, "sqlite3.h", &.{ + base: { // sqlite.h.in + const base = b.addConfigHeader( + .{ .style = .{ .cmake = b.path("src/sqlite.h.in") } }, + .{ + // TODO: Remove hard coded values + .libsql_version = "0.2.3", + .sqlite_version = "3.44.0", + .sqlite_version_number = 3044000, + .sqlite_source_id = "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad8alt1", + }, + ); + break :base base.getOutput(); }, - ); + b.path("ext/rtree/sqlite3rtree.h"), + b.path("ext/session/sqlite3session.h"), + b.path("ext/fts5/fts5.h"), + b.path("src/page_header.h"), // this must be above wal.h, since it depends on this + filtered: { // wal.h without includes + const filtered = b.addSystemCommand(&.{ "grep", "-v", "#include" }); + filtered.addFileArg(b.path("src/wal.h")); + break :filtered filtered.captureStdOut(); + }, + b.path("ext/udf/wasm_bindings.h"), + // b.path("ext/recover/sqlite3recover.h"), + }).getOutput(); + + const session = session: { + const lib = b.addStaticLibrary(.{ + .name = "session", + .target = options.target, + .optimize = options.optimize, + .link_libc = true, + }); - // const debug = Debug.create(b, parser); - // b.getInstallStep().dependOn(&debug.step); + lib.addIncludePath(h.dirname()); + lib.addIncludePath(b.path("src")); + lib.addIncludePath(b.path("ext/session/")); - const parser_vdbe = Amalgamation.create(b, "parse_vbde", &.{ - parser.path(b, "parse.h"), - b.path("src/vdbe.c"), - }); + lib.addCSourceFiles(.{ + .files = &.{ + "ext/session/sqlite3session.c", + }, + .flags = cflags(b, &.{ + // "-DSQLITE_OMIT_LOAD_EXTENSION", + }), + }); + + if (options.@"test") lib.root_module.addCMacro("SQLITE_TEST", "1"); + // lib.root_module.addCMacro("SQLITE_CORE", "1"); + break :session lib; + }; + + const fts3 = fts3: { + const lib = b.addStaticLibrary(.{ + .name = "fts3", + .target = options.target, + .optimize = options.optimize, + .link_libc = true, + }); + lib.addIncludePath(h.dirname()); + lib.addIncludePath(b.path("src")); + lib.addIncludePath(b.path("ext/fts3/")); + lib.addCSourceFiles(.{ + .files = &.{ + "ext/fts3/fts3.c", + "ext/fts3/fts3_aux.c", + "ext/fts3/fts3_term.c", + "ext/fts3/fts3_expr.c", + "ext/fts3/fts3_hash.c", + "ext/fts3/fts3_icu.c", + "ext/fts3/fts3_porter.c", + "ext/fts3/fts3_snippet.c", + "ext/fts3/fts3_tokenizer.c", + "ext/fts3/fts3_tokenizer1.c", + "ext/fts3/fts3_tokenize_vtab.c", + "ext/fts3/fts3_unicode.c", + "ext/fts3/fts3_unicode2.c", + "ext/fts3/fts3_write.c", + }, + .flags = cflags(b, &.{ + // "-DSQLITE_OMIT_LOAD_EXTENSION", + // "-DSQLITE_ENABLE_FTS3", + }), + }); + + if (options.@"test") { + // lib.root_module.addCMacro("SQLITE_TEST", "1"); + // lib.root_module.addCMacro("USE_TCL_STUBS", ""); + // lib.linkSystemLibrary("tcl"); + } + + // lib.root_module.addCMacro("SQLITE_CORE", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_FTS3", "1"); + break :fts3 lib; + }; - const opcode_h = b.addSystemCommand(&.{"tclsh"}); - opcode_h.addFileArg(b.path("tool/mkopcodeh.tcl")); - opcode_h.setStdIn(.{ .lazy_path = parser_vdbe.getOutput() }); + const rtree = rtree: { + const lib = b.addStaticLibrary(.{ + .name = "rtree", + .target = options.target, + .optimize = options.optimize, + .link_libc = true, + }); + lib.addIncludePath(h.dirname()); + lib.addIncludePath(b.path("src")); + lib.addCSourceFiles(.{ + .files = &.{ + "ext/rtree/rtree.c", + }, + .flags = cflags(b, &.{}), + }); + if (options.@"test") lib.root_module.addCMacro("SQLITE_TEST", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); + break :rtree lib; + }; + + const icu = icu: { + const lib = b.addStaticLibrary(.{ + .name = "rtree", + .target = options.target, + .optimize = options.optimize, + .link_libc = true, + }); + lib.addIncludePath(h.dirname()); + lib.addIncludePath(b.path("src")); + lib.addCSourceFiles(.{ + .files = &.{"ext/icu/icu.c"}, + .flags = cflags(b, &.{}), + }); + if (options.@"test") lib.root_module.addCMacro("SQLITE_TEST", "1"); + // lib.root_module.addCMacro("SQLITE_CORE", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_ICU", "1"); + lib.linkSystemLibrary("icu-io"); + break :icu lib; + }; + + const fts5 = fts5: { + var fts5parse = fts5parse: { + const run = b.addRunArtifact(lemon); + run.setCwd(b.path("tool/")); + run.addArg("-DSQLITE_ENABLE_MATH_FUNCTIONS"); + const dir = run.addPrefixedOutputDirectoryArg("-d", "."); + run.addArg("-S"); + run.addFileArg(b.path("ext/fts5/fts5parse.y")); + + break :fts5parse .{ + .h = dir.path(b, "fts5parse.h"), + .c = dir.path(b, "fts5parse.c"), + }; + }; - const opcode_c = b.addSystemCommand(&.{"tclsh"}); - opcode_c.addFileArg(b.path("tool/mkopcodec.tcl")); - opcode_c.addFileArg(opcode_h.captureStdOut()); + const lib = b.addStaticLibrary(.{ + .name = "fts5", + .target = options.target, + .optimize = options.optimize, + .use_llvm = false, + .link_libc = true, + }); + + lib.addIncludePath(h.dirname()); // includes fts5.h, for some stupid reason + lib.addIncludePath(b.path("src")); + lib.addIncludePath(b.path("ext/fts5/")); + lib.addIncludePath(fts5parse.h.dirname()); + + lib.addCSourceFile(.{ + .file = fts5parse.c, + .flags = cflags(b, &.{}), + }); + lib.addCSourceFiles(.{ + .files = &.{ + "ext/fts5/fts5_aux.c", "ext/fts5/fts5_buffer.c", + "ext/fts5/fts5_config.c", "ext/fts5/fts5_expr.c", + "ext/fts5/fts5_hash.c", "ext/fts5/fts5_index.c", + "ext/fts5/fts5_main.c", "ext/fts5/fts5_storage.c", + "ext/fts5/fts5_tokenize.c", "ext/fts5/fts5_unicode2.c", + "ext/fts5/fts5_varint.c", "ext/fts5/fts5_vocab.c", + }, + .flags = cflags(b, &.{ + // "-DSQLITE_ENABLE_FTS5", + }), + }); + + lib.root_module.addCMacro("SQLITE_CORE", "1"); + lib.root_module.addCMacro("SQLITE_DEBUG", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); - const generated = b.addWriteFiles(); - _ = generated.addCopyFile(keywordhash.captureStdOut(), "keywordhash.h"); - _ = generated.addCopyFile(opcode_h.captureStdOut(), "opcodes.h"); - _ = generated.addCopyFile(opcode_c.captureStdOut(), "opcodes.c"); + if (options.@"test") lib.root_module.addCMacro("SQLITE_TEST", "1"); - const flags = &.{ - "-g", - if (wasm_runtime) "-DLIBSQL_ENABLE_WASM_RUNTIME" else "", + break :fts5 lib; }; - const sqlite3 = b.addStaticLibrary(.{ + const lib = b.addStaticLibrary(.{ .name = "sqlite3", - .root_source_file = null, - .target = target, - .optimize = optimize, + .target = options.target, + .optimize = options.optimize, + .link_libc = true, }); - sqlite3.addObject(object: { - const object = b.addObject(.{ - .name = "sqlite3", - .target = target, - .optimize = optimize, + lib.installHeader(h, "sqlite3.h"); + + if (options.@"test") { + lib.installHeader(keywordhash.h, "keywordhash.h"); + lib.installHeader(opcodes.h, "opcodes.h"); + } + + lib.addIncludePath(h.dirname()); + lib.addIncludePath(b.path("src/")); + + lib.addIncludePath(opcodes.h.dirname()); + lib.addIncludePath(keywordhash.h.dirname()); + lib.addIncludePath(parse.h.dirname()); + lib.addConfigHeader(sqlite_cfg); + + lib.addCSourceFile(.{ .file = opcodes.c, .flags = cflags(b, &.{}) }); + lib.addCSourceFile(.{ .file = parse.c, .flags = cflags(b, &.{}) }); + lib.addCSourceFiles(.{ .files = sources.sqlite3, .flags = cflags(b, &.{}) }); + + lib.linkLibrary(fts3); + lib.linkLibrary(fts5); + lib.linkLibrary(icu); + lib.linkLibrary(rtree); + lib.linkLibrary(session); + + if (options.@"test") { + lib.root_module.addCMacro("SQLITE_DEFAULT_PAGE_SIZE", "1024"); + lib.root_module.addCMacro("SQLITE_TEST", "1"); + lib.root_module.addCMacro("SQLITE_NO_SYNC", "1"); + } + + // lib.root_module.addCMacro("SQLITE_ENABLE_FTS4", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_FTS3", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_ICU", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_DBSTAT_VTAB", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_BYTECODE_VTAB", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_STAT4", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_DESERIALIZE", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_FTS3_PARENTHESIS", "1"); + + // lib.root_module.addCMacro("SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION", "1"); + // lib.root_module.addCMacro("SQLITE_DEBUG", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_STMT_SCANSTATUS", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_OFFSET_SQL_FUNC", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_API_ARMOR", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_EXPLAIN_COMMENTS", "1"); + + if (options.wasm_runtime) { + const libsql_wasm = crab.addCargoBuildWithUserOptions(b, .{ + .name = "liblibsql_wasm.a", + .manifest_path = b.path("crates/wasmtime-bindings/Cargo.toml"), + .cargo_args = &.{ + "--release", + "--lib", + }, + }, .{ + .target = options.target, + .optimize = .ReleaseSafe, }); - object.linkLibC(); - object.addIncludePath(b.path("src/")); - object.addIncludePath(generated.getDirectory()); - object.addIncludePath(parser); - object.addIncludePath(sqlite_header.getOutput().dirname()); - object.addConfigHeader(sqlite_cfg); - object.addCSourceFile(.{ - .file = generated.getDirectory().path(b, "opcodes.c"), - .flags = flags, + + lib.root_module.addCMacro("LIBSQL_ENABLE_WASM_RUNTIME", ""); + lib.addIncludePath(b.path(".")); // to reach "ext/udf/wasm_bindings.h" + lib.addCSourceFile(.{ + .file = b.path("ext/udf/wasmedge_bindings.c"), + .flags = &.{ + "-DSQLITE_CORE", + }, }); - object.addCSourceFile(.{ .file = parser.path(b, "parse.c"), .flags = flags }); - object.addCSourceFiles(.{ .files = sources.sqlite3, .flags = flags }); + lib.addObjectFile(libsql_wasm); + } - if (icu) { - } + if (options.icu) {} - if (wasm_runtime) { - _ = generated.addCopyFile(b.path("ext/udf/wasm_bindings.h"), "ext/udf/wasm_bindings.h"); + return .{ + .h = h, + .lib = lib, + }; +} - const libsql_wasm = crab.addCargoBuildWithUserOptions(b, .{ - .name = "liblibsql_wasm.a", - .manifest_path = b.path("crates/wasmtime-bindings/Cargo.toml"), - .cargo_args = &.{ - "--release", - "--lib", - }, - }, .{ - .target = target, - .optimize = .ReleaseSafe, - }); +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); - object.addCSourceFile(.{ - .file = b.path(sources.wasm), - .flags = flags ++ &.{"-DSQLITE_CORE"}, - }); - object.addObjectFile(libsql_wasm); - } + const zlib = b.dependency("zlib", .{ + .target = target, + .optimize = optimize, + }); + + const wasm_runtime = b.option( + bool, + "wasm-runtime", + "enable wasm runtime (default: false)", + ) orelse false; + + const icu = b.option( + bool, + "icu", + "Enable icu extension (default: false)", + ) orelse false; - break :object object; + const sqlite3 = addSqlite3(b, .{ + .target = target, + .optimize = optimize, + .wasm_runtime = wasm_runtime, + .icu = icu, + }); + + const fuzzcheck = b.addExecutable(.{ + .name = "fuzzcheck", + .root_source_file = null, + .target = target, + .optimize = .ReleaseFast, + }); + fuzzcheck.addIncludePath(b.path("src/")); + fuzzcheck.addIncludePath(b.path("ext/recover")); + fuzzcheck.addCSourceFiles(.{ + .files = sources.fuzzcheck, + .flags = cflags(b, &.{}), }); + fuzzcheck.linkSystemLibrary("m"); + fuzzcheck.linkLibrary(sqlite3.lib); + fuzzcheck.linkLibrary(zlib.artifact("z")); + fuzzcheck.linkLibC(); - const install = b.addInstallArtifact(sqlite3, .{}); + fuzzcheck.root_module.addCMacro("SQLITE_OSS_FUZZ", ""); + fuzzcheck.root_module.addCMacro("SQLITE_NO_SYNC", "1"); + fuzzcheck.root_module.addCMacro("SQLITE_OMIT_LOAD_EXTENSION", "1"); - b.getInstallStep().dependOn(&install.step); + const sqlite3_test = addSqlite3(b, .{ + .target = target, + .optimize = optimize, + .wasm_runtime = wasm_runtime, + .icu = icu, + .@"test" = true, + }); + + const testfixture = b.addExecutable(.{ + .name = "testfixture", + .root_source_file = null, + .target = target, + .optimize = .ReleaseFast, + }); + testfixture.linkLibrary(sqlite3_test.lib); + testfixture.addIncludePath(b.path("src/")); + testfixture.addIncludePath(sqlite3_test.h.dirname()); + testfixture.addCSourceFiles(.{ + .files = sources.@"test", + .flags = cflags(b, &.{}), + }); + testfixture.addCSourceFiles(.{ + .files = &.{ + "src/tclsqlite.c", "ext/rbu/test_rbu.c", + "ext/misc/cksumvfs.c", "ext/misc/stmt.c", + "ext/expert/sqlite3expert.c", "ext/expert/test_expert.c", + "ext/misc/amatch.c", "ext/misc/appendvfs.c", + "ext/misc/basexx.c", "ext/misc/carray.c", + "ext/misc/closure.c", "ext/misc/csv.c", + "ext/misc/decimal.c", "ext/misc/eval.c", + "ext/misc/explain.c", "ext/misc/fileio.c", + "ext/misc/fuzzer.c", "ext/fts5/fts5_tcl.c", + "ext/fts5/fts5_test_mi.c", "ext/fts5/fts5_test_tok.c", + "ext/misc/ieee754.c", "ext/misc/mmapwarm.c", + "ext/misc/nextchar.c", "ext/misc/normalize.c", + "ext/misc/percentile.c", "ext/misc/prefixes.c", + "ext/misc/qpvtab.c", "ext/misc/regexp.c", + "ext/misc/remember.c", "ext/misc/series.c", + "ext/misc/spellfix.c", "ext/misc/totype.c", + "ext/misc/unionvtab.c", "ext/misc/wholenumber.c", + "ext/misc/zipfile.c", "ext/userauth/userauth.c", + "ext/rtree/test_rtreedoc.c", "ext/recover/test_recover.c", + "ext/recover/sqlite3recover.c", "ext/recover/dbdata.c", + }, + .flags = cflags(b, &.{}), + }); + + testfixture.root_module.addCMacro("SQLITE_TEST", "1"); + testfixture.root_module.addCMacro("SQLITE_CRASH_TEST", "1"); + + testfixture.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); + testfixture.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); + testfixture.root_module.addCMacro("SQLITE_ENABLE_BYTECODE_VTAB", "1"); + + testfixture.root_module.addCMacro("SQLITE_DEFAULT_PAGE_SIZE", "1024"); + testfixture.root_module.addCMacro("TCLSH_INIT_PROC", "sqlite3TestInit"); + testfixture.root_module.addCMacro("SQLITE_SERIES_CONSTRAINT_VERIFY", "1"); + testfixture.root_module.addCMacro("SQLITE_CKSUMVFS_STATIC", ""); + + // testfixture.root_module.addCMacro("SQLITE_ENABLE_MATH_FUNCTIONS", "1"); + // testfixture.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); + // testfixture.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); + // testfixture.root_module.addCMacro("SQLITE_ENABLE_SESSION", "1"); + // testfixture.root_module.addCMacro("SQLITE_ENABLE_STAT4", "1"); + // testfixture.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); + // testfixture.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); + // testfixture.root_module.addCMacro("SQLITE_ENABLE_BYTECODE_VTAB", "1"); + // testfixture.root_module.addCMacro("SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION", "1"); + // testfixture.root_module.addCMacro("SQLITE_ENABLE_STMT_SCANSTATUS", "1"); + // testfixture.root_module.addCMacro("SQLITE_ENABLE_OFFSET_SQL_FUNC", "1"); + + testfixture.linkSystemLibrary("z"); + testfixture.linkSystemLibrary("m"); + // TODO: Package TCL + testfixture.linkSystemLibrary("tcl"); + testfixture.linkLibC(); + + // run text + { + const run = b.addRunArtifact(testfixture); + const step = b.step("test", "Run testfixture"); + + if (b.args) |args| { + run.addArgs(args); + } + + step.dependOn(&run.step); + } + + // run fuzzcheck + { + const run = b.addRunArtifact(fuzzcheck); + const step = b.step("fuzzcheck", "Run fuzzcheck (default: test/fuzzdata[1..=8].db)"); + + if (b.args) |args| { + run.addArgs(args); + } else { + inline for (&.{ + "test/fuzzdata1.db", + "test/fuzzdata2.db", + "test/fuzzdata3.db", + "test/fuzzdata4.db", + "test/fuzzdata5.db", + "test/fuzzdata6.db", + "test/fuzzdata7.db", + "test/fuzzdata8.db", + }) |path| { + run.addFileArg(b.path(path)); + } + } + + step.dependOn(&run.step); + } + + // b.getInstallStep().dependOn( + // &b.addInstallArtifact(testfixture, .{}).step, + // ); + b.getInstallStep().dependOn( + &b.addInstallArtifact(sqlite3.lib, .{}).step, + ); } diff --git a/libsql-sqlite3/build.zig.zon b/libsql-sqlite3/build.zig.zon index fc57b37792..fd45e8b960 100644 --- a/libsql-sqlite3/build.zig.zon +++ b/libsql-sqlite3/build.zig.zon @@ -27,6 +27,10 @@ .url = "https://github.com/akarpovskii/build.crab/archive/refs/tags/v0.1.5.tar.gz", .hash = "1220c7d071bdde500955b0d5b807cb1a66687b915a020c3e87496623243912c77bed", }, + .zlib = .{ + .url = "https://github.com/allyourcodebase/zlib/archive/refs/tags/1.3.1.tar.gz", + .hash = "122034ab2a12adf8016ffa76e48b4be3245ffd305193edba4d83058adbcfa749c107", + }, }, .paths = .{ "build.zig", From 9e737d567be7f71576bfc5818d6771f1ab559770 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Mon, 1 Jul 2024 22:35:18 -0300 Subject: [PATCH 14/35] refactor(libsql-sqlite3): remove unnecessary includes with inline definitions --- libsql-sqlite3/src/main.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/libsql-sqlite3/src/main.c b/libsql-sqlite3/src/main.c index d6a6e6ef9d..d74a07d299 100644 --- a/libsql-sqlite3/src/main.c +++ b/libsql-sqlite3/src/main.c @@ -16,16 +16,6 @@ */ #include "sqliteInt.h" -#ifdef SQLITE_ENABLE_FTS3 -# include "fts3.h" -#endif -#ifdef SQLITE_ENABLE_RTREE -# include "rtree.h" -#endif -#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) -# include "sqliteicu.h" -#endif - /* ** This is an extension initializer that is a no-op and always ** succeeds, except that it fails if the fault-simulation is set @@ -41,6 +31,15 @@ static int sqlite3TestExtInit(sqlite3 *db){ ** Forward declarations of external module initializer functions ** for modules that need them. */ +#ifdef SQLITE_ENABLE_FTS3 +int sqlite3Fts3Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_RTREE +int sqlite3RtreeInit(sqlite3*); +#endif +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) +int sqlite3IcuInit(sqlite3*); +#endif #ifdef SQLITE_ENABLE_FTS5 int sqlite3Fts5Init(sqlite3*); #endif From a6424182e0c5a0402f1732d0a740c14d48c6f34b Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Mon, 1 Jul 2024 22:54:02 -0300 Subject: [PATCH 15/35] fix(sqlite3): does not compile without this header Due to the sources being amalgamated, this goes unnoticed. There is no script to generate this file in the current source. Where is this from? --- libsql-sqlite3/ext/fts5/fts5_unicode2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsql-sqlite3/ext/fts5/fts5_unicode2.c b/libsql-sqlite3/ext/fts5/fts5_unicode2.c index 3e97264fa8..e21bccb46a 100644 --- a/libsql-sqlite3/ext/fts5/fts5_unicode2.c +++ b/libsql-sqlite3/ext/fts5/fts5_unicode2.c @@ -15,7 +15,7 @@ ** DO NOT EDIT THIS MACHINE GENERATED FILE. */ - +#include "fts5Int.h" #include From a19ee6006a296c6a12bcaafc211466b0a1a30aae Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Fri, 5 Jul 2024 17:10:01 -0300 Subject: [PATCH 16/35] fix(mksqlite3h): substitute new version patterns --- libsql-sqlite3/tool/mksqlite3h.tcl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libsql-sqlite3/tool/mksqlite3h.tcl b/libsql-sqlite3/tool/mksqlite3h.tcl index 45eed68fb7..4c000125e7 100644 --- a/libsql-sqlite3/tool/mksqlite3h.tcl +++ b/libsql-sqlite3/tool/mksqlite3h.tcl @@ -138,11 +138,11 @@ foreach file $filelist { # line when copying wal.h into sqlite3.h. if {[string match {*#include*[<"]page_header.h[>"]*} $line]} continue - regsub -- --VERS-- $line $zVersion line - regsub -- --VERSION-NUMBER-- $line $nVersion line - regsub -- --SOURCE-ID-- $line "$zSourceId" line + regsub -- {\$\{sqlite_version\}} $line $zVersion line + regsub -- {\$\{sqlite_version_number\}} $line $nVersion line + regsub -- {\$\{sqlite_source_id\}} $line "$zSourceId" line - regsub -- --LIBSQL-VERS-- $line $zLibSQLVersion line + regsub -- {\$\{libsql_version\}} $line $zLibSQLVersion line if {[regexp $varpattern $line] && ![regexp {^ *typedef} $line]} { set line "SQLITE_API $line" From 1b947f94323ed3fd02dc8acf5ec2dd44c2b3b753 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 9 Jul 2024 15:30:02 -0300 Subject: [PATCH 17/35] Revert "temporary solution for build.crab current limitations" This reverts commit f5ecc9a14db88d48a189e9a61c8085dd90430b29. --- libsql-sqlite3/crates/wasmtime-bindings/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsql-sqlite3/crates/wasmtime-bindings/Cargo.toml b/libsql-sqlite3/crates/wasmtime-bindings/Cargo.toml index 02c9c1ecaa..e25335cdd8 100644 --- a/libsql-sqlite3/crates/wasmtime-bindings/Cargo.toml +++ b/libsql-sqlite3/crates/wasmtime-bindings/Cargo.toml @@ -14,7 +14,7 @@ wasmtime-wasi = "9.0" [lib] name = "libsql_wasm" -crate-type = ["staticlib"] +crate-type = ["lib", "cdylib", "staticlib"] doc = false test = false doctest = false From 21f35f508e0b9734e66cfe208e9a88f363be393f Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 9 Jul 2024 15:54:10 -0300 Subject: [PATCH 18/35] fix(flake): enable bindgenHook --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 2079ec1416..1523790b35 100644 --- a/flake.nix +++ b/flake.nix @@ -25,6 +25,7 @@ cmake gnumake clang + rustPlatform.bindgenHook ]; buildInputs = with pkgs; [ icu.dev From ceaa00ee7abb9a96c28b7b758eb7d8f3ce4e67d0 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 9 Jul 2024 15:55:17 -0300 Subject: [PATCH 19/35] update .gitignore --- .gitignore | 3 ++- libsql-sqlite3/.gitignore | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b064cb2bed..f8fee39698 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .idea/ .direnv/ *.sqld +tags libsql-server/testrollbackrestore/** libsql-server/testbackuprestore/** @@ -13,4 +14,4 @@ libsql-sqlite3/**.o.tmp /bindings/c/generated /bindings/c/**.xcframework -/bindings/**/.DS_Store \ No newline at end of file +/bindings/**/.DS_Store diff --git a/libsql-sqlite3/.gitignore b/libsql-sqlite3/.gitignore index 54036a68e5..8148f67d9a 100644 --- a/libsql-sqlite3/.gitignore +++ b/libsql-sqlite3/.gitignore @@ -64,3 +64,6 @@ libsql /libsql.wasm zig-out/ .zig-cache +testdir*/ +testrunner.db +testrunner.log From 63af3e13493327e8747abd16f78300ab4862998e Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 9 Jul 2024 15:57:57 -0300 Subject: [PATCH 20/35] add(libsql-ffi): introduce zig-build --- libsql-ffi/Cargo.toml | 1 + libsql-ffi/build.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/libsql-ffi/Cargo.toml b/libsql-ffi/Cargo.toml index 0ee8c216cd..145aae7b83 100644 --- a/libsql-ffi/Cargo.toml +++ b/libsql-ffi/Cargo.toml @@ -18,6 +18,7 @@ bindgen = "0.66.1" cc = "1.0" [features] +zig-build = [] session = [] wasmtime-bindings = ["dep:libsql-wasmtime-bindings"] multiple-ciphers = [] diff --git a/libsql-ffi/build.rs b/libsql-ffi/build.rs index d56093de86..98dbfa18c9 100644 --- a/libsql-ffi/build.rs +++ b/libsql-ffi/build.rs @@ -10,6 +10,48 @@ const BUNDLED_DIR: &str = "bundled"; const SQLITE_DIR: &str = "../libsql-sqlite3"; fn main() { + if cfg!(feature = "zig-build") && !cfg!(feature = "multiple-ciphers") { + println!("cargo:rerun-if-changed=../libsql-sqlite3/"); + + let out_dir_str = env::var("OUT_DIR").unwrap(); + let out_dir = Path::new(out_dir_str.as_str()); + + Command::new("zig") + .arg("build") + .args([ + "--build-file", + PathBuf::from("../libsql-sqlite3/build.zig") + .canonicalize() + .unwrap() + .to_str() + .unwrap(), + ]) + .args([ + "--cache-dir", + out_dir.join(".zig-cache").to_str().unwrap().as_ref(), + ]) + .args(["--prefix", out_dir.to_str().unwrap().as_ref()]) + .spawn() + .unwrap(); + + bindings::write_to_out_dir( + HeaderLocation::FromPath( + out_dir + .join("include/sqlite3.h") + .to_str() + .unwrap() + .to_string(), + ), + out_dir.join("bindgen.rs").as_path(), + ); + + println!("cargo:rerun-if-changed=../libsql-sqlite3/"); + println!("cargo:rustc-link-search={}", out_dir.join("lib/").to_str().unwrap()); + println!("cargo:rustc-link-lib=static=sqlite3"); + + return; + } + let target = env::var("TARGET").unwrap(); let host = env::var("HOST").unwrap(); From f363f6017fe4b9b42523f647d887a2c66f082f5f Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 9 Jul 2024 15:59:18 -0300 Subject: [PATCH 21/35] minor zig build tweaks --- libsql-sqlite3/build.zig | 250 ++++++++++++++++------------------- libsql-sqlite3/build.zig.zon | 4 +- 2 files changed, 116 insertions(+), 138 deletions(-) diff --git a/libsql-sqlite3/build.zig b/libsql-sqlite3/build.zig index 4e7e56ffc5..e9a08176a6 100644 --- a/libsql-sqlite3/build.zig +++ b/libsql-sqlite3/build.zig @@ -35,29 +35,49 @@ const sources = .{ "ext/recover/sqlite3recover.c", "test/vt02.c", }, - .@"test" = &.{ - "src/test1.c", "src/test2.c", - "src/test3.c", "src/test4.c", - "src/test5.c", "src/test6.c", - "src/test8.c", "src/test9.c", - "src/test_autoext.c", "src/test_async.c", - "src/test_backup.c", "src/test_bestindex.c", - "src/test_blob.c", "src/test_btree.c", - "src/test_config.c", "src/test_delete.c", - "src/test_demovfs.c", "src/test_devsym.c", - "src/test_fs.c", "src/test_func.c", - "src/test_hexio.c", "src/test_init.c", - "src/test_intarray.c", "src/test_journal.c", - "src/test_malloc.c", "src/test_md5.c", - "src/test_multiplex.c", "src/test_mutex.c", - "src/test_onefile.c", "src/test_osinst.c", - "src/test_pcache.c", "src/test_quota.c", - "src/test_rtree.c", "src/test_schema.c", - "src/test_superlock.c", "src/test_syscall.c", - "src/test_tclsh.c", "src/test_tclvar.c", - "src/test_thread.c", "src/test_vdbecov.c", - "src/test_vfs.c", "src/test_windirent.c", - "src/test_window.c", "src/test_wsd.c", + .testfixture = &.{ + "src/test1.c", "src/test2.c", + "src/test3.c", "src/test4.c", + "src/test5.c", "src/test6.c", + "src/test8.c", "src/test9.c", + "src/test_autoext.c", "src/test_async.c", + "src/test_backup.c", "src/test_bestindex.c", + "src/test_blob.c", "src/test_btree.c", + "src/test_config.c", "src/test_delete.c", + "src/test_demovfs.c", "src/test_devsym.c", + "src/test_fs.c", "src/test_func.c", + "src/test_hexio.c", "src/test_init.c", + "src/test_intarray.c", "src/test_journal.c", + "src/test_malloc.c", "src/test_md5.c", + "src/test_multiplex.c", "src/test_mutex.c", + "src/test_onefile.c", "src/test_osinst.c", + "src/test_pcache.c", "src/test_quota.c", + "src/test_rtree.c", "src/test_schema.c", + "src/test_superlock.c", "src/test_syscall.c", + "src/test_tclsh.c", "src/test_tclvar.c", + "src/test_thread.c", "src/test_vdbecov.c", + "src/test_vfs.c", "src/test_windirent.c", + "src/test_window.c", "src/test_wsd.c", + "src/tclsqlite.c", "ext/rbu/test_rbu.c", + "ext/misc/cksumvfs.c", "ext/misc/stmt.c", + "ext/expert/sqlite3expert.c", "ext/expert/test_expert.c", + "ext/misc/amatch.c", "ext/misc/appendvfs.c", + "ext/misc/basexx.c", "ext/misc/carray.c", + "ext/misc/closure.c", "ext/misc/csv.c", + "ext/misc/decimal.c", "ext/misc/eval.c", + "ext/misc/explain.c", "ext/misc/fileio.c", + "ext/misc/fuzzer.c", "ext/fts5/fts5_tcl.c", + "ext/fts5/fts5_test_mi.c", "ext/fts5/fts5_test_tok.c", + "ext/misc/ieee754.c", "ext/misc/mmapwarm.c", + "ext/misc/nextchar.c", "ext/misc/normalize.c", + "ext/misc/percentile.c", "ext/misc/prefixes.c", + "ext/misc/qpvtab.c", "ext/misc/regexp.c", + "ext/misc/remember.c", "ext/misc/series.c", + "ext/misc/spellfix.c", "ext/misc/totype.c", + "ext/misc/unionvtab.c", "ext/misc/wholenumber.c", + "ext/misc/zipfile.c", "ext/userauth/userauth.c", + "ext/rtree/test_rtreedoc.c", "ext/recover/test_recover.c", + "ext/recover/sqlite3recover.c", "ext/recover/dbdata.c", }, }; @@ -190,17 +210,13 @@ fn cflags(b: *Build, flags: []const []const u8) [][]const u8 { "-g", // "-O3", "-DBUILD_sqlite", - "-DSQLITE_HAVE_ZLIB", "-DNDEBUG", "-DSQLITE_ENABLE_MATH_FUNCTIONS", - "-DSQLITE_TEMP_STORE=2", - "-DSQLITE_USE_URI=1", + // "-DSQLITE_TEMP_STORE=2", + // "-DSQLITE_USE_URI=1", "-DSQLITE_THREADSAFE=1", "-D_HAVE_SQLITE_CONFIG", "-DSQLITE_CORE", - - // FIX: make this configurable - "-DSQLITE_OS_UNIX=1", }, flags, }) catch @panic("OOM"); @@ -209,18 +225,22 @@ fn cflags(b: *Build, flags: []const []const u8) [][]const u8 { const Sqlite3Options = struct { target: Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, - wasm_runtime: bool, - icu: bool, + wasm_runtime: bool = false, + icu: bool = false, + fts3: bool = false, + fts5: bool = false, + geopoly: bool = false, + rtree: bool = false, + session: bool = false, + preupdate_hook: bool = false, @"test": bool = false, }; -const Sqlite3 = struct { - target: Build.ResolvedTarget, - optimize: std.builtin.OptimizeMode, - wasm_runtime: bool, - icu: bool, - @"test": bool = false, -}; +fn filterIncludes(b: *Build, lp: LazyPath) LazyPath { + const filtered = b.addSystemCommand(&.{ "grep", "-v", "#include" }); + filtered.addFileArg(lp); + return filtered.captureStdOut(); +} fn addSqlite3(b: *Build, options: Sqlite3Options) struct { h: LazyPath, @@ -399,15 +419,11 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { ); break :base base.getOutput(); }, - b.path("ext/rtree/sqlite3rtree.h"), - b.path("ext/session/sqlite3session.h"), - b.path("ext/fts5/fts5.h"), + filterIncludes(b, b.path("ext/rtree/sqlite3rtree.h")), + filterIncludes(b, b.path("ext/session/sqlite3session.h")), + filterIncludes(b, b.path("ext/fts5/fts5.h")), b.path("src/page_header.h"), // this must be above wal.h, since it depends on this - filtered: { // wal.h without includes - const filtered = b.addSystemCommand(&.{ "grep", "-v", "#include" }); - filtered.addFileArg(b.path("src/wal.h")); - break :filtered filtered.captureStdOut(); - }, + filterIncludes(b, b.path("src/wal.h")), b.path("ext/udf/wasm_bindings.h"), // b.path("ext/recover/sqlite3recover.h"), }).getOutput(); @@ -421,6 +437,7 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { }); lib.addIncludePath(h.dirname()); + lib.addIncludePath(opcodes.h.dirname()); lib.addIncludePath(b.path("src")); lib.addIncludePath(b.path("ext/session/")); @@ -428,13 +445,14 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { .files = &.{ "ext/session/sqlite3session.c", }, - .flags = cflags(b, &.{ - // "-DSQLITE_OMIT_LOAD_EXTENSION", - }), + .flags = cflags(b, &.{}), }); if (options.@"test") lib.root_module.addCMacro("SQLITE_TEST", "1"); - // lib.root_module.addCMacro("SQLITE_CORE", "1"); + + lib.root_module.addCMacro("SQLITE_ENABLE_SESSION", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_PREUPDATE_HOOK", "1"); + break :session lib; }; @@ -465,10 +483,7 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { "ext/fts3/fts3_unicode2.c", "ext/fts3/fts3_write.c", }, - .flags = cflags(b, &.{ - // "-DSQLITE_OMIT_LOAD_EXTENSION", - // "-DSQLITE_ENABLE_FTS3", - }), + .flags = cflags(b, &.{}), }); if (options.@"test") { @@ -477,8 +492,8 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { // lib.linkSystemLibrary("tcl"); } - // lib.root_module.addCMacro("SQLITE_CORE", "1"); lib.root_module.addCMacro("SQLITE_ENABLE_FTS3", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_FTS3_PARENTHESIS", "1"); break :fts3 lib; }; @@ -515,10 +530,11 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { .files = &.{"ext/icu/icu.c"}, .flags = cflags(b, &.{}), }); + if (options.@"test") lib.root_module.addCMacro("SQLITE_TEST", "1"); - // lib.root_module.addCMacro("SQLITE_CORE", "1"); lib.root_module.addCMacro("SQLITE_ENABLE_ICU", "1"); lib.linkSystemLibrary("icu-io"); + break :icu lib; }; @@ -541,7 +557,6 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { .name = "fts5", .target = options.target, .optimize = options.optimize, - .use_llvm = false, .link_libc = true, }); @@ -550,10 +565,7 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { lib.addIncludePath(b.path("ext/fts5/")); lib.addIncludePath(fts5parse.h.dirname()); - lib.addCSourceFile(.{ - .file = fts5parse.c, - .flags = cflags(b, &.{}), - }); + lib.addCSourceFile(.{ .file = fts5parse.c, .flags = cflags(b, &.{}) }); lib.addCSourceFiles(.{ .files = &.{ "ext/fts5/fts5_aux.c", "ext/fts5/fts5_buffer.c", @@ -563,13 +575,10 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { "ext/fts5/fts5_tokenize.c", "ext/fts5/fts5_unicode2.c", "ext/fts5/fts5_varint.c", "ext/fts5/fts5_vocab.c", }, - .flags = cflags(b, &.{ - // "-DSQLITE_ENABLE_FTS5", - }), + .flags = cflags(b, &.{}), }); - lib.root_module.addCMacro("SQLITE_CORE", "1"); - lib.root_module.addCMacro("SQLITE_DEBUG", "1"); + // lib.root_module.addCMacro("SQLITE_DEBUG", "1"); lib.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); if (options.@"test") lib.root_module.addCMacro("SQLITE_TEST", "1"); @@ -602,6 +611,8 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { lib.addCSourceFile(.{ .file = parse.c, .flags = cflags(b, &.{}) }); lib.addCSourceFiles(.{ .files = sources.sqlite3, .flags = cflags(b, &.{}) }); + lib.addCSourceFile(.{ .file = b.path("ext/misc/stmt.c"), .flags = cflags(b, &.{}) }); + lib.linkLibrary(fts3); lib.linkLibrary(fts5); lib.linkLibrary(icu); @@ -614,29 +625,40 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { lib.root_module.addCMacro("SQLITE_NO_SYNC", "1"); } - // lib.root_module.addCMacro("SQLITE_ENABLE_FTS4", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_FTS3", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_ICU", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); lib.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); lib.root_module.addCMacro("SQLITE_ENABLE_DBSTAT_VTAB", "1"); lib.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); lib.root_module.addCMacro("SQLITE_ENABLE_BYTECODE_VTAB", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_STAT4", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_DESERIALIZE", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_COLUMN_METADATA", "1"); + + if (options.target.result.os.tag == .windows) { + lib.root_module.addCMacro("SQLITE_OS_WIN", "1"); + } else { + lib.root_module.addCMacro("SQLITE_OS_UNIX", "1"); + } + + // lib.root_module.addCMacro("SQLITE_DEFAULT_FOREIGN_KEYS", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_API_ARMOR", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_COLUMN_METADATA", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_DBSTAT_VTAB", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_FTS3", "1"); // lib.root_module.addCMacro("SQLITE_ENABLE_FTS3_PARENTHESIS", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_JSON1", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_LOAD_EXTENSION", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_MEMORY_MANAGEMENT", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_STAT2", "1"); + // lib.root_module.addCMacro("SQLITE_ENABLE_STAT4", "1"); + // lib.root_module.addCMacro("SQLITE_SOUNDEX", "1"); + // lib.root_module.addCMacro("SQLITE_THREADSAFE", "1"); + lib.root_module.addCMacro("SQLITE_USE_URI", "1"); + // lib.root_module.addCMacro("HAVE_USLEEP", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION", "1"); - // lib.root_module.addCMacro("SQLITE_DEBUG", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_STMT_SCANSTATUS", "1"); - lib.root_module.addCMacro("SQLITE_ENABLE_OFFSET_SQL_FUNC", "1"); - lib.root_module.addCMacro("SQLITE_ENABLE_API_ARMOR", "1"); lib.root_module.addCMacro("SQLITE_ENABLE_EXPLAIN_COMMENTS", "1"); if (options.wasm_runtime) { - const libsql_wasm = crab.addCargoBuildWithUserOptions(b, .{ - .name = "liblibsql_wasm.a", + const libsql_wasm = crab.addCargoBuild(b, .{ .manifest_path = b.path("crates/wasmtime-bindings/Cargo.toml"), .cargo_args = &.{ "--release", @@ -651,11 +673,10 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { lib.addIncludePath(b.path(".")); // to reach "ext/udf/wasm_bindings.h" lib.addCSourceFile(.{ .file = b.path("ext/udf/wasmedge_bindings.c"), - .flags = &.{ - "-DSQLITE_CORE", - }, + .flags = cflags(b, &.{}), }); - lib.addObjectFile(libsql_wasm); + lib.addLibraryPath(libsql_wasm); + lib.linkSystemLibrary("libsql_wasm"); } if (options.icu) {} @@ -728,42 +749,14 @@ pub fn build(b: *std.Build) void { .root_source_file = null, .target = target, .optimize = .ReleaseFast, + .link_libc = true, }); testfixture.linkLibrary(sqlite3_test.lib); testfixture.addIncludePath(b.path("src/")); - testfixture.addIncludePath(sqlite3_test.h.dirname()); - testfixture.addCSourceFiles(.{ - .files = sources.@"test", - .flags = cflags(b, &.{}), - }); - testfixture.addCSourceFiles(.{ - .files = &.{ - "src/tclsqlite.c", "ext/rbu/test_rbu.c", - "ext/misc/cksumvfs.c", "ext/misc/stmt.c", - "ext/expert/sqlite3expert.c", "ext/expert/test_expert.c", - "ext/misc/amatch.c", "ext/misc/appendvfs.c", - "ext/misc/basexx.c", "ext/misc/carray.c", - "ext/misc/closure.c", "ext/misc/csv.c", - "ext/misc/decimal.c", "ext/misc/eval.c", - "ext/misc/explain.c", "ext/misc/fileio.c", - "ext/misc/fuzzer.c", "ext/fts5/fts5_tcl.c", - "ext/fts5/fts5_test_mi.c", "ext/fts5/fts5_test_tok.c", - "ext/misc/ieee754.c", "ext/misc/mmapwarm.c", - "ext/misc/nextchar.c", "ext/misc/normalize.c", - "ext/misc/percentile.c", "ext/misc/prefixes.c", - "ext/misc/qpvtab.c", "ext/misc/regexp.c", - "ext/misc/remember.c", "ext/misc/series.c", - "ext/misc/spellfix.c", "ext/misc/totype.c", - "ext/misc/unionvtab.c", "ext/misc/wholenumber.c", - "ext/misc/zipfile.c", "ext/userauth/userauth.c", - "ext/rtree/test_rtreedoc.c", "ext/recover/test_recover.c", - "ext/recover/sqlite3recover.c", "ext/recover/dbdata.c", - }, - .flags = cflags(b, &.{}), - }); + testfixture.addCSourceFiles(.{ .files = sources.testfixture, .flags = cflags(b, &.{}) }); + testfixture.root_module.addCMacro("SQLITE_HAVE_ZLIB", "1"); testfixture.root_module.addCMacro("SQLITE_TEST", "1"); - testfixture.root_module.addCMacro("SQLITE_CRASH_TEST", "1"); testfixture.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); testfixture.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); @@ -772,39 +765,24 @@ pub fn build(b: *std.Build) void { testfixture.root_module.addCMacro("SQLITE_DEFAULT_PAGE_SIZE", "1024"); testfixture.root_module.addCMacro("TCLSH_INIT_PROC", "sqlite3TestInit"); testfixture.root_module.addCMacro("SQLITE_SERIES_CONSTRAINT_VERIFY", "1"); - testfixture.root_module.addCMacro("SQLITE_CKSUMVFS_STATIC", ""); - - // testfixture.root_module.addCMacro("SQLITE_ENABLE_MATH_FUNCTIONS", "1"); - // testfixture.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); - // testfixture.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); - // testfixture.root_module.addCMacro("SQLITE_ENABLE_SESSION", "1"); - // testfixture.root_module.addCMacro("SQLITE_ENABLE_STAT4", "1"); - // testfixture.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); - // testfixture.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); - // testfixture.root_module.addCMacro("SQLITE_ENABLE_BYTECODE_VTAB", "1"); - // testfixture.root_module.addCMacro("SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION", "1"); - // testfixture.root_module.addCMacro("SQLITE_ENABLE_STMT_SCANSTATUS", "1"); - // testfixture.root_module.addCMacro("SQLITE_ENABLE_OFFSET_SQL_FUNC", "1"); - - testfixture.linkSystemLibrary("z"); - testfixture.linkSystemLibrary("m"); - // TODO: Package TCL + testfixture.root_module.addCMacro("SQLITE_CKSUMVFS_STATIC", "1"); + + testfixture.linkLibrary(zlib.artifact("z")); testfixture.linkSystemLibrary("tcl"); - testfixture.linkLibC(); - // run text { const run = b.addRunArtifact(testfixture); - const step = b.step("test", "Run testfixture"); + const step = b.step("test", "Run testfixture (default: testrunner.tcl veryquick)"); if (b.args) |args| { run.addArgs(args); + } else { + run.addArgs(&.{ "test/testrunner.tcl", "veryquick" }); } step.dependOn(&run.step); } - // run fuzzcheck { const run = b.addRunArtifact(fuzzcheck); const step = b.step("fuzzcheck", "Run fuzzcheck (default: test/fuzzdata[1..=8].db)"); diff --git a/libsql-sqlite3/build.zig.zon b/libsql-sqlite3/build.zig.zon index fd45e8b960..056238e2a6 100644 --- a/libsql-sqlite3/build.zig.zon +++ b/libsql-sqlite3/build.zig.zon @@ -24,8 +24,8 @@ // internet connectivity. .dependencies = .{ .@"build.crab" = .{ - .url = "https://github.com/akarpovskii/build.crab/archive/refs/tags/v0.1.5.tar.gz", - .hash = "1220c7d071bdde500955b0d5b807cb1a66687b915a020c3e87496623243912c77bed", + .url = "https://github.com/akarpovskii/build.crab/archive/refs/tags/v0.1.6.tar.gz", + .hash = "12204d44b1484cd3ca05c48e859c904f21665199d2e349ed86e4a143900a05c69baf", }, .zlib = .{ .url = "https://github.com/allyourcodebase/zlib/archive/refs/tags/1.3.1.tar.gz", From b4f6fe429b63fdb3368233b06db0fb52a768f1ba Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 9 Jul 2024 16:00:27 -0300 Subject: [PATCH 22/35] bump to latest rust stable --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f92c249df9..d5ce77ece5 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] profile = "default" -channel = "1.78.0" +channel = "1.79.0" From c80a27fcf0581d2325b1ae56f681e7c8e9e3033a Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 9 Jul 2024 16:01:35 -0300 Subject: [PATCH 23/35] fix(libsql): make tests pass --- libsql/src/lib.rs | 2 +- libsql/src/load_extension_guard.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libsql/src/lib.rs b/libsql/src/lib.rs index 3d9260afe9..b2f42bfb36 100644 --- a/libsql/src/lib.rs +++ b/libsql/src/lib.rs @@ -49,7 +49,7 @@ //! # async fn run() { //! use libsql::Builder; //! -//! let db = Builder::new_remote("libsql://my-remote-db.com", "my-auth-token").build().await.unwrap(); +//! let db = Builder::new_remote("libsql://my-remote-db.com".to_string(), "my-auth-token".to_string()).build().await.unwrap(); //! let conn = db.connect().unwrap(); //! conn.execute("CREATE TABLE IF NOT EXISTS users (email TEXT)", ()).await.unwrap(); //! conn.execute("INSERT INTO users (email) VALUES ('alice@example.org')", ()).await.unwrap(); diff --git a/libsql/src/load_extension_guard.rs b/libsql/src/load_extension_guard.rs index 5192716e17..61e9fd1b69 100644 --- a/libsql/src/load_extension_guard.rs +++ b/libsql/src/load_extension_guard.rs @@ -7,7 +7,7 @@ use crate::{Connection, Result}; /// /// # Example /// -/// ```rust,no_run +/// ```rust,ignore /// let _guard = LoadExtensionGuard::new(conn)?; /// conn.load_extension("uuid", None)?; /// ``` From a456f78bed0df5b1963968ab2d0556b0213290fa Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 9 Jul 2024 16:51:16 -0300 Subject: [PATCH 24/35] fix(build): prevent race condition with block --- libsql-ffi/build.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libsql-ffi/build.rs b/libsql-ffi/build.rs index 98dbfa18c9..113a6a07de 100644 --- a/libsql-ffi/build.rs +++ b/libsql-ffi/build.rs @@ -32,6 +32,8 @@ fn main() { ]) .args(["--prefix", out_dir.to_str().unwrap().as_ref()]) .spawn() + .unwrap() + .wait() .unwrap(); bindings::write_to_out_dir( From 77e6a56e8f1b8f91bc35511959ec4d035a9d2961 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 9 Jul 2024 16:56:44 -0300 Subject: [PATCH 25/35] add libsql_vector tests to veryquick --- libsql-sqlite3/test/veryquick.test | 1 + 1 file changed, 1 insertion(+) diff --git a/libsql-sqlite3/test/veryquick.test b/libsql-sqlite3/test/veryquick.test index c8d6ce8f8b..ddf093f827 100644 --- a/libsql-sqlite3/test/veryquick.test +++ b/libsql-sqlite3/test/veryquick.test @@ -13,6 +13,7 @@ set testdir [file dirname $argv0] source $testdir/permutations.test +run_test suite libsql_vector run_test_suite veryquick finish_test From 04baef82750105ad452e3bab037a1b5235eb4d3f Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 9 Jul 2024 16:59:14 -0300 Subject: [PATCH 26/35] feat(build): add vector and rust_suite --- libsql-sqlite3/build.zig | 130 ++++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 62 deletions(-) diff --git a/libsql-sqlite3/build.zig b/libsql-sqlite3/build.zig index e9a08176a6..032137baec 100644 --- a/libsql-sqlite3/build.zig +++ b/libsql-sqlite3/build.zig @@ -6,26 +6,27 @@ const LazyPath = Build.LazyPath; const sources = .{ .sqlite3 = &.{ - "src/alter.c", "src/analyze.c", "src/attach.c", "src/auth.c", - "src/backup.c", "src/bitvec.c", "src/btmutex.c", "src/btree.c", - "src/build.c", "src/callback.c", "src/complete.c", "src/ctime.c", - "src/date.c", "src/dbpage.c", "src/dbstat.c", "src/delete.c", - "src/expr.c", "src/fault.c", "src/fkey.c", "src/func.c", - "src/global.c", "src/hash.c", "src/insert.c", "src/json.c", - "src/legacy.c", "src/loadext.c", "src/main.c", "src/malloc.c", - "src/mem0.c", "src/mem1.c", "src/mem2.c", "src/mem3.c", - "src/mem5.c", "src/memdb.c", "src/memjournal.c", "src/mutex.c", - "src/mutex_noop.c", "src/mutex_unix.c", "src/mutex_w32.c", "src/notify.c", - "src/os.c", "src/os_kv.c", "src/os_unix.c", "src/os_win.c", - "src/pager.c", "src/pcache.c", "src/pcache1.c", "src/pragma.c", - "src/prepare.c", "src/printf.c", "src/random.c", "src/resolve.c", - "src/rowset.c", "src/select.c", "src/status.c", "src/table.c", - "src/threads.c", "src/tokenize.c", "src/treeview.c", "src/trigger.c", - "src/utf.c", "src/update.c", "src/upsert.c", "src/util.c", - "src/vacuum.c", "src/vdbe.c", "src/vdbeapi.c", "src/vdbeaux.c", - "src/vdbeblob.c", "src/vdbemem.c", "src/vdbesort.c", "src/vdbetrace.c", - "src/vdbevtab.c", "src/vtab.c", "src/wal.c", "src/walker.c", - "src/where.c", "src/wherecode.c", "src/whereexpr.c", "src/window.c", + "src/alter.c", "src/analyze.c", "src/attach.c", "src/auth.c", + "src/backup.c", "src/bitvec.c", "src/btmutex.c", "src/btree.c", + "src/build.c", "src/callback.c", "src/complete.c", "src/ctime.c", + "src/date.c", "src/dbpage.c", "src/dbstat.c", "src/delete.c", + "src/expr.c", "src/fault.c", "src/fkey.c", "src/func.c", + "src/global.c", "src/hash.c", "src/insert.c", "src/json.c", + "src/legacy.c", "src/loadext.c", "src/main.c", "src/malloc.c", + "src/mem0.c", "src/mem1.c", "src/mem2.c", "src/mem3.c", + "src/mem5.c", "src/memdb.c", "src/memjournal.c", "src/mutex.c", + "src/mutex_noop.c", "src/mutex_unix.c", "src/mutex_w32.c", "src/notify.c", + "src/os.c", "src/os_kv.c", "src/os_unix.c", "src/os_win.c", + "src/pager.c", "src/pcache.c", "src/pcache1.c", "src/pragma.c", + "src/prepare.c", "src/printf.c", "src/random.c", "src/resolve.c", + "src/rowset.c", "src/select.c", "src/status.c", "src/table.c", + "src/threads.c", "src/tokenize.c", "src/treeview.c", "src/trigger.c", + "src/utf.c", "src/update.c", "src/upsert.c", "src/util.c", + "src/vacuum.c", "src/vdbe.c", "src/vdbeapi.c", "src/vdbeaux.c", + "src/vdbeblob.c", "src/vdbemem.c", "src/vdbesort.c", "src/vdbetrace.c", + "src/vdbevtab.c", "src/vtab.c", "src/wal.c", "src/walker.c", + "src/where.c", "src/wherecode.c", "src/whereexpr.c", "src/window.c", + "src/vector.c", "src/vectorfloat32.c", "src/vectorfloat64.c", }, .fuzzcheck = &.{ "test/fuzzcheck.c", @@ -36,48 +37,48 @@ const sources = .{ "test/vt02.c", }, .testfixture = &.{ - "src/test1.c", "src/test2.c", - "src/test3.c", "src/test4.c", - "src/test5.c", "src/test6.c", - "src/test8.c", "src/test9.c", - "src/test_autoext.c", "src/test_async.c", - "src/test_backup.c", "src/test_bestindex.c", - "src/test_blob.c", "src/test_btree.c", - "src/test_config.c", "src/test_delete.c", - "src/test_demovfs.c", "src/test_devsym.c", - "src/test_fs.c", "src/test_func.c", - "src/test_hexio.c", "src/test_init.c", - "src/test_intarray.c", "src/test_journal.c", - "src/test_malloc.c", "src/test_md5.c", - "src/test_multiplex.c", "src/test_mutex.c", - "src/test_onefile.c", "src/test_osinst.c", - "src/test_pcache.c", "src/test_quota.c", - "src/test_rtree.c", "src/test_schema.c", - "src/test_superlock.c", "src/test_syscall.c", - "src/test_tclsh.c", "src/test_tclvar.c", - "src/test_thread.c", "src/test_vdbecov.c", - "src/test_vfs.c", "src/test_windirent.c", - "src/test_window.c", "src/test_wsd.c", - "src/tclsqlite.c", "ext/rbu/test_rbu.c", - "ext/misc/cksumvfs.c", "ext/misc/stmt.c", - "ext/expert/sqlite3expert.c", "ext/expert/test_expert.c", - "ext/misc/amatch.c", "ext/misc/appendvfs.c", - "ext/misc/basexx.c", "ext/misc/carray.c", - "ext/misc/closure.c", "ext/misc/csv.c", - "ext/misc/decimal.c", "ext/misc/eval.c", - "ext/misc/explain.c", "ext/misc/fileio.c", - "ext/misc/fuzzer.c", "ext/fts5/fts5_tcl.c", - "ext/fts5/fts5_test_mi.c", "ext/fts5/fts5_test_tok.c", - "ext/misc/ieee754.c", "ext/misc/mmapwarm.c", - "ext/misc/nextchar.c", "ext/misc/normalize.c", - "ext/misc/percentile.c", "ext/misc/prefixes.c", - "ext/misc/qpvtab.c", "ext/misc/regexp.c", - "ext/misc/remember.c", "ext/misc/series.c", - "ext/misc/spellfix.c", "ext/misc/totype.c", - "ext/misc/unionvtab.c", "ext/misc/wholenumber.c", - "ext/misc/zipfile.c", "ext/userauth/userauth.c", - "ext/rtree/test_rtreedoc.c", "ext/recover/test_recover.c", - "ext/recover/sqlite3recover.c", "ext/recover/dbdata.c", + "src/test1.c", "src/test2.c", + "src/test3.c", "src/test4.c", + "src/test5.c", "src/test6.c", + "src/test8.c", "src/test9.c", + "src/test_autoext.c", "src/test_async.c", + "src/test_backup.c", "src/test_bestindex.c", + "src/test_blob.c", "src/test_btree.c", + "src/test_config.c", "src/test_delete.c", + "src/test_demovfs.c", "src/test_devsym.c", + "src/test_fs.c", "src/test_func.c", + "src/test_hexio.c", "src/test_init.c", + "src/test_intarray.c", "src/test_journal.c", + "src/test_malloc.c", "src/test_md5.c", + "src/test_multiplex.c", "src/test_mutex.c", + "src/test_onefile.c", "src/test_osinst.c", + "src/test_pcache.c", "src/test_quota.c", + "src/test_rtree.c", "src/test_schema.c", + "src/test_superlock.c", "src/test_syscall.c", + "src/test_tclsh.c", "src/test_tclvar.c", + "src/test_thread.c", "src/test_vdbecov.c", + "src/test_vfs.c", "src/test_windirent.c", + "src/test_window.c", "src/test_wsd.c", + "src/tclsqlite.c", "ext/rbu/test_rbu.c", + "ext/misc/cksumvfs.c", "ext/expert/sqlite3expert.c", + "ext/expert/test_expert.c", "ext/misc/amatch.c", + "ext/misc/appendvfs.c", "ext/misc/basexx.c", + "ext/misc/carray.c", "ext/misc/closure.c", + "ext/misc/csv.c", "ext/misc/decimal.c", + "ext/misc/eval.c", "ext/misc/explain.c", + "ext/misc/fileio.c", "ext/misc/fuzzer.c", + "ext/fts5/fts5_tcl.c", "ext/fts5/fts5_test_mi.c", + "ext/fts5/fts5_test_tok.c", "ext/misc/ieee754.c", + "ext/misc/mmapwarm.c", "ext/misc/nextchar.c", + "ext/misc/normalize.c", "ext/misc/percentile.c", + "ext/misc/prefixes.c", "ext/misc/qpvtab.c", + "ext/misc/regexp.c", "ext/misc/remember.c", + "ext/misc/series.c", "ext/misc/spellfix.c", + "ext/misc/totype.c", "ext/misc/unionvtab.c", + "ext/misc/wholenumber.c", "ext/misc/zipfile.c", + "ext/userauth/userauth.c", "ext/rtree/test_rtreedoc.c", + "ext/recover/test_recover.c", "ext/recover/sqlite3recover.c", + "ext/recover/dbdata.c", }, }; @@ -780,7 +781,12 @@ pub fn build(b: *std.Build) void { run.addArgs(&.{ "test/testrunner.tcl", "veryquick" }); } + const rust_suite = b.addSystemCommand(&.{ "cargo", "test" }); + rust_suite.setCwd(b.path("test/rust_suite")); + + step.dependOn(&run.step); + step.dependOn(&rust_suite.step); } { From 534aed25187f4b6712bfb2f8fab0cc98e0b711ed Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Tue, 9 Jul 2024 17:00:03 -0300 Subject: [PATCH 27/35] feat(libsql-sys): introduce zig build --- libsql-sys/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsql-sys/Cargo.toml b/libsql-sys/Cargo.toml index 26dd091ea9..7eafd1016e 100644 --- a/libsql-sys/Cargo.toml +++ b/libsql-sys/Cargo.toml @@ -12,7 +12,7 @@ categories = ["external-ffi-bindings"] [dependencies] bytes = "1.5.0" -libsql-ffi = { version = "0.3", path = "../libsql-ffi/" } +libsql-ffi = { version = "0.3", path = "../libsql-ffi/", features = ["zig-build"] } once_cell = "1.18.0" rusqlite = { workspace = true, features = ["trace"], optional = true } tracing = "0.1.37" From 778e25226517b422d4e78d386f064d388a8105dd Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Wed, 10 Jul 2024 22:22:47 -0300 Subject: [PATCH 28/35] Revert 1 commits 77e6a56 'add libsql_vector tests to veryquick' --- libsql-sqlite3/test/veryquick.test | 1 - 1 file changed, 1 deletion(-) diff --git a/libsql-sqlite3/test/veryquick.test b/libsql-sqlite3/test/veryquick.test index ddf093f827..c8d6ce8f8b 100644 --- a/libsql-sqlite3/test/veryquick.test +++ b/libsql-sqlite3/test/veryquick.test @@ -13,7 +13,6 @@ set testdir [file dirname $argv0] source $testdir/permutations.test -run_test suite libsql_vector run_test_suite veryquick finish_test From af4e81f5b17ec3f827ffda722d1890e1edfd5ff3 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Fri, 12 Jul 2024 01:39:05 -0300 Subject: [PATCH 29/35] refactor: large refactor of the build and minor changes --- flake.nix | 1 + libsql-sqlite3/build.zig | 427 ++++++++++------------ libsql-sqlite3/test/rust_suite/Cargo.toml | 2 +- 3 files changed, 196 insertions(+), 234 deletions(-) diff --git a/flake.nix b/flake.nix index 1523790b35..8f453294e2 100644 --- a/flake.nix +++ b/flake.nix @@ -26,6 +26,7 @@ gnumake clang rustPlatform.bindgenHook + wasmtime ]; buildInputs = with pkgs; [ icu.dev diff --git a/libsql-sqlite3/build.zig b/libsql-sqlite3/build.zig index 032137baec..196cde7ca5 100644 --- a/libsql-sqlite3/build.zig +++ b/libsql-sqlite3/build.zig @@ -78,38 +78,17 @@ const sources = .{ "ext/misc/wholenumber.c", "ext/misc/zipfile.c", "ext/userauth/userauth.c", "ext/rtree/test_rtreedoc.c", "ext/recover/test_recover.c", "ext/recover/sqlite3recover.c", - "ext/recover/dbdata.c", + "ext/recover/dbdata.c", "ext/session/test_session.c", + "ext/fts3/fts3_test.c", + }, + .fts5 = &.{ + "ext/fts5/fts5_aux.c", "ext/fts5/fts5_buffer.c", + "ext/fts5/fts5_config.c", "ext/fts5/fts5_expr.c", + "ext/fts5/fts5_hash.c", "ext/fts5/fts5_index.c", + "ext/fts5/fts5_main.c", "ext/fts5/fts5_storage.c", + "ext/fts5/fts5_tokenize.c", "ext/fts5/fts5_unicode2.c", + "ext/fts5/fts5_varint.c", "ext/fts5/fts5_vocab.c", }, -}; - -pub const Debug = struct { - step: std.Build.Step, - path: std.Build.LazyPath, - - pub fn create(b: *Build, path: LazyPath) *Debug { - const self = b.allocator.create(Debug) catch @panic("OOM"); - self.* = .{ - .step = std.Build.Step.init(.{ - .id = .custom, - .name = "debug path", - .owner = b, - .makeFn = make, - }), - .path = path, - }; - - path.addStepDependencies(&self.step); - - return self; - } - - pub fn make(step: *Build.Step, _: std.Progress.Node) !void { - const self: *Debug = @fieldParentPtr("step", step); - - const b = self.step.owner; - - std.debug.print("lazy_path: {s}\n", .{self.path.getPath(b)}); - } }; pub const Amalgamation = struct { @@ -226,15 +205,14 @@ fn cflags(b: *Build, flags: []const []const u8) [][]const u8 { const Sqlite3Options = struct { target: Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, - wasm_runtime: bool = false, - icu: bool = false, - fts3: bool = false, - fts5: bool = false, - geopoly: bool = false, - rtree: bool = false, - session: bool = false, - preupdate_hook: bool = false, - @"test": bool = false, + wasm_runtime: bool, + icu: bool, + fts3: bool, + fts5: bool, + geopoly: bool, + rtree: bool, + session: bool, + @"test": bool, }; fn filterIncludes(b: *Build, lp: LazyPath) LazyPath { @@ -243,10 +221,7 @@ fn filterIncludes(b: *Build, lp: LazyPath) LazyPath { return filtered.captureStdOut(); } -fn addSqlite3(b: *Build, options: Sqlite3Options) struct { - h: LazyPath, - lib: *Build.Step.Compile, -} { +fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { const sqlite_cfg = b.addConfigHeader( .{ .include_path = "sqlite_cfg.h", @@ -426,46 +401,64 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { b.path("src/page_header.h"), // this must be above wal.h, since it depends on this filterIncludes(b, b.path("src/wal.h")), b.path("ext/udf/wasm_bindings.h"), - // b.path("ext/recover/sqlite3recover.h"), }).getOutput(); - const session = session: { - const lib = b.addStaticLibrary(.{ - .name = "session", - .target = options.target, - .optimize = options.optimize, - .link_libc = true, - }); + const lib = b.addStaticLibrary(.{ + .name = "sqlite3", + .target = options.target, + .optimize = options.optimize, + .link_libc = true, + }); + lib.installHeader(h, "sqlite3.h"); - lib.addIncludePath(h.dirname()); - lib.addIncludePath(opcodes.h.dirname()); - lib.addIncludePath(b.path("src")); - lib.addIncludePath(b.path("ext/session/")); + lib.addIncludePath(h.dirname()); + lib.addIncludePath(b.path("src/")); + + lib.addIncludePath(opcodes.h.dirname()); + lib.addIncludePath(keywordhash.h.dirname()); + lib.addIncludePath(parse.h.dirname()); + lib.addConfigHeader(sqlite_cfg); + lib.addCSourceFile(.{ .file = opcodes.c, .flags = cflags(b, &.{}) }); + lib.addCSourceFile(.{ .file = parse.c, .flags = cflags(b, &.{}) }); + lib.addCSourceFiles(.{ .files = sources.sqlite3, .flags = cflags(b, &.{}) }); + lib.addCSourceFile(.{ .file = b.path("ext/misc/stmt.c"), .flags = cflags(b, &.{}) }); + + if (options.fts5) { + var fts5parse = fts5parse: { + const run = b.addRunArtifact(lemon); + run.setCwd(b.path("tool/")); + run.addArg("-DSQLITE_ENABLE_MATH_FUNCTIONS"); + const dir = run.addPrefixedOutputDirectoryArg("-d", "."); + run.addArg("-S"); + run.addFileArg(b.path("ext/fts5/fts5parse.y")); + + break :fts5parse .{ + .h = dir.path(b, "fts5parse.h"), + .c = dir.path(b, "fts5parse.c"), + }; + }; + + lib.addIncludePath(b.path("ext/fts5/")); + lib.addIncludePath(fts5parse.h.dirname()); + + lib.addCSourceFile(.{ .file = fts5parse.c, .flags = cflags(b, &.{}) }); lib.addCSourceFiles(.{ .files = &.{ - "ext/session/sqlite3session.c", + "ext/fts5/fts5_aux.c", "ext/fts5/fts5_buffer.c", + "ext/fts5/fts5_config.c", "ext/fts5/fts5_expr.c", + "ext/fts5/fts5_hash.c", "ext/fts5/fts5_index.c", + "ext/fts5/fts5_main.c", "ext/fts5/fts5_storage.c", + "ext/fts5/fts5_tokenize.c", "ext/fts5/fts5_unicode2.c", + "ext/fts5/fts5_varint.c", "ext/fts5/fts5_vocab.c", }, .flags = cflags(b, &.{}), }); - if (options.@"test") lib.root_module.addCMacro("SQLITE_TEST", "1"); - - lib.root_module.addCMacro("SQLITE_ENABLE_SESSION", "1"); - lib.root_module.addCMacro("SQLITE_ENABLE_PREUPDATE_HOOK", "1"); - - break :session lib; - }; + lib.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); + } - const fts3 = fts3: { - const lib = b.addStaticLibrary(.{ - .name = "fts3", - .target = options.target, - .optimize = options.optimize, - .link_libc = true, - }); - lib.addIncludePath(h.dirname()); - lib.addIncludePath(b.path("src")); + if (options.fts3) { lib.addIncludePath(b.path("ext/fts3/")); lib.addCSourceFiles(.{ .files = &.{ @@ -487,150 +480,78 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { .flags = cflags(b, &.{}), }); - if (options.@"test") { - // lib.root_module.addCMacro("SQLITE_TEST", "1"); - // lib.root_module.addCMacro("USE_TCL_STUBS", ""); - // lib.linkSystemLibrary("tcl"); - } - lib.root_module.addCMacro("SQLITE_ENABLE_FTS3", "1"); lib.root_module.addCMacro("SQLITE_ENABLE_FTS3_PARENTHESIS", "1"); - break :fts3 lib; - }; + } - const rtree = rtree: { - const lib = b.addStaticLibrary(.{ - .name = "rtree", + if (options.rtree) { + lib.addCSourceFile(.{ .file = b.path("ext/rtree/rtree.c"), .flags = cflags(b, &.{}) }); + lib.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); + if (options.geopoly) lib.root_module.addCMacro("SQLITE_ENABLE_GEOPOLY", "1"); + } + + if (options.wasm_runtime) { + const libsql_wasm = crab.addCargoBuild(b, .{ + .manifest_path = b.path("crates/wasmtime-bindings/Cargo.toml"), + .cargo_args = &.{ + "--release", + "--lib", + }, + }, .{ .target = options.target, - .optimize = options.optimize, - .link_libc = true, + .optimize = .ReleaseSafe, }); - lib.addIncludePath(h.dirname()); - lib.addIncludePath(b.path("src")); - lib.addCSourceFiles(.{ - .files = &.{ - "ext/rtree/rtree.c", - }, + + lib.root_module.addCMacro("LIBSQL_ENABLE_WASM_RUNTIME", ""); + lib.addIncludePath(b.path(".")); // to reach "ext/udf/wasm_bindings.h" + lib.addCSourceFile(.{ + .file = b.path("ext/udf/wasmedge_bindings.c"), .flags = cflags(b, &.{}), }); - if (options.@"test") lib.root_module.addCMacro("SQLITE_TEST", "1"); - lib.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); - break :rtree lib; - }; + lib.addLibraryPath(libsql_wasm); + lib.linkSystemLibrary("libsql_wasm"); + } - const icu = icu: { - const lib = b.addStaticLibrary(.{ - .name = "rtree", - .target = options.target, - .optimize = options.optimize, - .link_libc = true, - }); - lib.addIncludePath(h.dirname()); - lib.addIncludePath(b.path("src")); + if (options.icu) { lib.addCSourceFiles(.{ .files = &.{"ext/icu/icu.c"}, .flags = cflags(b, &.{}), }); - if (options.@"test") lib.root_module.addCMacro("SQLITE_TEST", "1"); lib.root_module.addCMacro("SQLITE_ENABLE_ICU", "1"); - lib.linkSystemLibrary("icu-io"); - - break :icu lib; - }; - - const fts5 = fts5: { - var fts5parse = fts5parse: { - const run = b.addRunArtifact(lemon); - run.setCwd(b.path("tool/")); - run.addArg("-DSQLITE_ENABLE_MATH_FUNCTIONS"); - const dir = run.addPrefixedOutputDirectoryArg("-d", "."); - run.addArg("-S"); - run.addFileArg(b.path("ext/fts5/fts5parse.y")); - - break :fts5parse .{ - .h = dir.path(b, "fts5parse.h"), - .c = dir.path(b, "fts5parse.c"), - }; - }; - - const lib = b.addStaticLibrary(.{ - .name = "fts5", - .target = options.target, - .optimize = options.optimize, - .link_libc = true, - }); + lib.linkSystemLibrary("icuuc"); + lib.linkSystemLibrary("icuio"); + lib.linkSystemLibrary("icui18n"); + } - lib.addIncludePath(h.dirname()); // includes fts5.h, for some stupid reason - lib.addIncludePath(b.path("src")); - lib.addIncludePath(b.path("ext/fts5/")); - lib.addIncludePath(fts5parse.h.dirname()); + if (options.session) { + lib.addIncludePath(b.path("ext/session/")); - lib.addCSourceFile(.{ .file = fts5parse.c, .flags = cflags(b, &.{}) }); lib.addCSourceFiles(.{ - .files = &.{ - "ext/fts5/fts5_aux.c", "ext/fts5/fts5_buffer.c", - "ext/fts5/fts5_config.c", "ext/fts5/fts5_expr.c", - "ext/fts5/fts5_hash.c", "ext/fts5/fts5_index.c", - "ext/fts5/fts5_main.c", "ext/fts5/fts5_storage.c", - "ext/fts5/fts5_tokenize.c", "ext/fts5/fts5_unicode2.c", - "ext/fts5/fts5_varint.c", "ext/fts5/fts5_vocab.c", - }, + .files = &.{"ext/session/sqlite3session.c"}, .flags = cflags(b, &.{}), }); - // lib.root_module.addCMacro("SQLITE_DEBUG", "1"); - lib.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); - - if (options.@"test") lib.root_module.addCMacro("SQLITE_TEST", "1"); - - break :fts5 lib; - }; - - const lib = b.addStaticLibrary(.{ - .name = "sqlite3", - .target = options.target, - .optimize = options.optimize, - .link_libc = true, - }); - lib.installHeader(h, "sqlite3.h"); + lib.root_module.addCMacro("SQLITE_ENABLE_SESSION", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_PREUPDATE_HOOK", "1"); + } if (options.@"test") { lib.installHeader(keywordhash.h, "keywordhash.h"); lib.installHeader(opcodes.h, "opcodes.h"); - } - lib.addIncludePath(h.dirname()); - lib.addIncludePath(b.path("src/")); - - lib.addIncludePath(opcodes.h.dirname()); - lib.addIncludePath(keywordhash.h.dirname()); - lib.addIncludePath(parse.h.dirname()); - lib.addConfigHeader(sqlite_cfg); - - lib.addCSourceFile(.{ .file = opcodes.c, .flags = cflags(b, &.{}) }); - lib.addCSourceFile(.{ .file = parse.c, .flags = cflags(b, &.{}) }); - lib.addCSourceFiles(.{ .files = sources.sqlite3, .flags = cflags(b, &.{}) }); - - lib.addCSourceFile(.{ .file = b.path("ext/misc/stmt.c"), .flags = cflags(b, &.{}) }); - - lib.linkLibrary(fts3); - lib.linkLibrary(fts5); - lib.linkLibrary(icu); - lib.linkLibrary(rtree); - lib.linkLibrary(session); - - if (options.@"test") { lib.root_module.addCMacro("SQLITE_DEFAULT_PAGE_SIZE", "1024"); lib.root_module.addCMacro("SQLITE_TEST", "1"); lib.root_module.addCMacro("SQLITE_NO_SYNC", "1"); + + lib.linkSystemLibrary("tcl8.6"); } - lib.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); - lib.root_module.addCMacro("SQLITE_ENABLE_DBSTAT_VTAB", "1"); - lib.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); - lib.root_module.addCMacro("SQLITE_ENABLE_BYTECODE_VTAB", "1"); - lib.root_module.addCMacro("SQLITE_ENABLE_COLUMN_METADATA", "1"); + if (options.target.result.os.tag == .wasi) { + lib.root_module.addCMacro("SQLITE_WASI", "1"); + lib.root_module.addCMacro("SQLITE_OMIT_SHARED_MEM", "1"); + lib.root_module.addCMacro("SQLITE_OMIT_SHARED_CACHE", "1"); + } if (options.target.result.os.tag == .windows) { lib.root_module.addCMacro("SQLITE_OS_WIN", "1"); @@ -638,6 +559,14 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { lib.root_module.addCMacro("SQLITE_OS_UNIX", "1"); } + lib.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_DBSTAT_VTAB", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_BYTECODE_VTAB", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_COLUMN_METADATA", "1"); + + lib.root_module.addCMacro("SQLITE_ENABLE_LOAD_EXTENSION", "1"); + // lib.root_module.addCMacro("SQLITE_DEFAULT_FOREIGN_KEYS", "1"); // lib.root_module.addCMacro("SQLITE_ENABLE_API_ARMOR", "1"); // lib.root_module.addCMacro("SQLITE_ENABLE_COLUMN_METADATA", "1"); @@ -646,7 +575,6 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { // lib.root_module.addCMacro("SQLITE_ENABLE_FTS3_PARENTHESIS", "1"); // lib.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); // lib.root_module.addCMacro("SQLITE_ENABLE_JSON1", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_LOAD_EXTENSION", "1"); // lib.root_module.addCMacro("SQLITE_ENABLE_MEMORY_MANAGEMENT", "1"); // lib.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); // lib.root_module.addCMacro("SQLITE_ENABLE_STAT2", "1"); @@ -682,10 +610,7 @@ fn addSqlite3(b: *Build, options: Sqlite3Options) struct { if (options.icu) {} - return .{ - .h = h, - .lib = lib, - }; + return lib; } pub fn build(b: *std.Build) void { @@ -697,24 +622,48 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }); - const wasm_runtime = b.option( - bool, - "wasm-runtime", - "enable wasm runtime (default: false)", - ) orelse false; - - const icu = b.option( - bool, - "icu", - "Enable icu extension (default: false)", - ) orelse false; - - const sqlite3 = addSqlite3(b, .{ + const options: Sqlite3Options = .{ .target = target, .optimize = optimize, - .wasm_runtime = wasm_runtime, - .icu = icu, - }); + .wasm_runtime = b.option( + bool, + "wasm-runtime", + "Enable wasm runtime (default: false)", + ) orelse false, + .icu = b.option( + bool, + "icu", + "Enable icu extension (default: false)", + ) orelse false, + .fts3 = b.option( + bool, + "fts3", + "Enable fts3 extension (default: true)", + ) orelse true, + .fts5 = b.option( + bool, + "fts5", + "Enable fts5 extension (default: true)", + ) orelse true, + .rtree = b.option( + bool, + "rtree", + "Enable rtree extension (default: true)", + ) orelse true, + .session = b.option( + bool, + "session", + "Enable session extension (default: true)", + ) orelse true, + .geopoly = b.option( + bool, + "geopoly", + "Enable geopoly extension (default: true)", + ) orelse true, + .@"test" = false, + }; + + const libsql = addLibsql(b, options); const fuzzcheck = b.addExecutable(.{ .name = "fuzzcheck", @@ -729,7 +678,7 @@ pub fn build(b: *std.Build) void { .flags = cflags(b, &.{}), }); fuzzcheck.linkSystemLibrary("m"); - fuzzcheck.linkLibrary(sqlite3.lib); + fuzzcheck.linkLibrary(libsql); fuzzcheck.linkLibrary(zlib.artifact("z")); fuzzcheck.linkLibC(); @@ -737,44 +686,53 @@ pub fn build(b: *std.Build) void { fuzzcheck.root_module.addCMacro("SQLITE_NO_SYNC", "1"); fuzzcheck.root_module.addCMacro("SQLITE_OMIT_LOAD_EXTENSION", "1"); - const sqlite3_test = addSqlite3(b, .{ - .target = target, - .optimize = optimize, - .wasm_runtime = wasm_runtime, - .icu = icu, - .@"test" = true, - }); - const testfixture = b.addExecutable(.{ .name = "testfixture", .root_source_file = null, .target = target, .optimize = .ReleaseFast, - .link_libc = true, }); - testfixture.linkLibrary(sqlite3_test.lib); + testfixture.linkLibC(); + testfixture.linkLibrary( + addLibsql(b, .{ + .target = options.target, + .optimize = options.optimize, + .rtree = options.rtree, + .session = options.session, + .fts3 = options.fts3, + .fts5 = options.fts5, + .geopoly = options.geopoly, + .icu = options.icu, + .wasm_runtime = options.wasm_runtime, + .@"test" = true, + }), + ); testfixture.addIncludePath(b.path("src/")); testfixture.addCSourceFiles(.{ .files = sources.testfixture, .flags = cflags(b, &.{}) }); testfixture.root_module.addCMacro("SQLITE_HAVE_ZLIB", "1"); testfixture.root_module.addCMacro("SQLITE_TEST", "1"); + testfixture.root_module.addCMacro("TCLSH_INIT_PROC", "sqlite3TestInit"); testfixture.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); testfixture.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); testfixture.root_module.addCMacro("SQLITE_ENABLE_BYTECODE_VTAB", "1"); - - testfixture.root_module.addCMacro("SQLITE_DEFAULT_PAGE_SIZE", "1024"); - testfixture.root_module.addCMacro("TCLSH_INIT_PROC", "sqlite3TestInit"); testfixture.root_module.addCMacro("SQLITE_SERIES_CONSTRAINT_VERIFY", "1"); testfixture.root_module.addCMacro("SQLITE_CKSUMVFS_STATIC", "1"); + testfixture.root_module.addCMacro("SQLITE_DEFAULT_PAGE_SIZE", "1024"); + + if (options.icu) testfixture.root_module.addCMacro("SQLITE_ENABLE_ICU", "1"); + if (options.fts3) testfixture.root_module.addCMacro("SQLITE_ENABLE_FTS3", "1"); + if (options.session) { + testfixture.root_module.addCMacro("SQLITE_ENABLE_SESSION", "1"); + testfixture.root_module.addCMacro("SQLITE_ENABLE_PREUPDATE_HOOK", "1"); + } testfixture.linkLibrary(zlib.artifact("z")); testfixture.linkSystemLibrary("tcl"); { const run = b.addRunArtifact(testfixture); - const step = b.step("test", "Run testfixture (default: testrunner.tcl veryquick)"); - if (b.args) |args| { run.addArgs(args); } else { @@ -783,16 +741,18 @@ pub fn build(b: *std.Build) void { const rust_suite = b.addSystemCommand(&.{ "cargo", "test" }); rust_suite.setCwd(b.path("test/rust_suite")); + rust_suite.step.dependOn(&run.step); - + const step = b.step( + "test", + "Run testfixture (default: testrunner.tcl veryquick)", + ); step.dependOn(&run.step); step.dependOn(&rust_suite.step); } { const run = b.addRunArtifact(fuzzcheck); - const step = b.step("fuzzcheck", "Run fuzzcheck (default: test/fuzzdata[1..=8].db)"); - if (b.args) |args| { run.addArgs(args); } else { @@ -810,13 +770,14 @@ pub fn build(b: *std.Build) void { } } + const step = b.step( + "fuzzcheck", + "Run fuzzcheck (default: test/fuzzdata[1..=8].db)", + ); step.dependOn(&run.step); } - // b.getInstallStep().dependOn( - // &b.addInstallArtifact(testfixture, .{}).step, - // ); - b.getInstallStep().dependOn( - &b.addInstallArtifact(sqlite3.lib, .{}).step, - ); + b.getInstallStep().dependOn(&b.addInstallArtifact(testfixture, .{}).step); + b.getInstallStep().dependOn(&b.addInstallArtifact(fuzzcheck, .{}).step); + b.getInstallStep().dependOn(&b.addInstallArtifact(libsql, .{}).step); } diff --git a/libsql-sqlite3/test/rust_suite/Cargo.toml b/libsql-sqlite3/test/rust_suite/Cargo.toml index 90a8251ebf..7b4fd1cdbf 100644 --- a/libsql-sqlite3/test/rust_suite/Cargo.toml +++ b/libsql-sqlite3/test/rust_suite/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dev-dependencies] -libsql-sys = { path = "../../../libsql-sys", features = ["wal", "wasmtime-bindings", "rusqlite"], default_features = false } +libsql-sys = { path = "../../../libsql-sys", features = ["wal", "wasmtime-bindings", "rusqlite"], default-features = false } itertools = "0.10" tempfile = "3.3" From 1cbbd2503cd57678c0051e1c2b805d0189080fb2 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Mon, 15 Jul 2024 05:04:30 -0300 Subject: [PATCH 30/35] refactor(build): more options and fixes --- libsql-sqlite3/build.zig | 282 ++++++++++++++++++--------------------- 1 file changed, 132 insertions(+), 150 deletions(-) diff --git a/libsql-sqlite3/build.zig b/libsql-sqlite3/build.zig index 196cde7ca5..97b76b8408 100644 --- a/libsql-sqlite3/build.zig +++ b/libsql-sqlite3/build.zig @@ -3,6 +3,7 @@ const crab = @import("build.crab"); const fs = std.fs; const Build = std.Build; const LazyPath = Build.LazyPath; +const assert = std.debug.assert; const sources = .{ .sqlite3 = &.{ @@ -84,10 +85,15 @@ const sources = .{ .fts5 = &.{ "ext/fts5/fts5_aux.c", "ext/fts5/fts5_buffer.c", "ext/fts5/fts5_config.c", "ext/fts5/fts5_expr.c", - "ext/fts5/fts5_hash.c", "ext/fts5/fts5_index.c", - "ext/fts5/fts5_main.c", "ext/fts5/fts5_storage.c", - "ext/fts5/fts5_tokenize.c", "ext/fts5/fts5_unicode2.c", - "ext/fts5/fts5_varint.c", "ext/fts5/fts5_vocab.c", + "ext/fts5/fts5_hash.c", "ext/fts5/fts5_main.c", + "ext/fts5/fts5_storage.c", "ext/fts5/fts5_tokenize.c", + "ext/fts5/fts5_unicode2.c", "ext/fts5/fts5_varint.c", + "ext/fts5/fts5_vocab.c", + + "ext/fts5/fts5_index.c", + // this file was a UB in line 4212 that is catched by + // -fsanitize=undefined -- crashes the fts5secure3.test with a SIGILL if + // -fno-sanitize=signed-integer-overflow is not set }, }; @@ -184,24 +190,6 @@ pub const Amalgamation = struct { } }; -fn cflags(b: *Build, flags: []const []const u8) [][]const u8 { - return std.mem.concat(b.allocator, []const u8, &.{ - &.{ - "-g", - // "-O3", - "-DBUILD_sqlite", - "-DNDEBUG", - "-DSQLITE_ENABLE_MATH_FUNCTIONS", - // "-DSQLITE_TEMP_STORE=2", - // "-DSQLITE_USE_URI=1", - "-DSQLITE_THREADSAFE=1", - "-D_HAVE_SQLITE_CONFIG", - "-DSQLITE_CORE", - }, - flags, - }) catch @panic("OOM"); -} - const Sqlite3Options = struct { target: Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, @@ -212,6 +200,8 @@ const Sqlite3Options = struct { geopoly: bool, rtree: bool, session: bool, + default_foreign_keys: bool, + explain_comments: bool, @"test": bool, }; @@ -221,7 +211,17 @@ fn filterIncludes(b: *Build, lp: LazyPath) LazyPath { return filtered.captureStdOut(); } -fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { +const cflags = [_][]const u8{ + "-fno-sanitize=signed-integer-overflow", + "-DBUILD_sqlite", + "-DSQLITE_ENABLE_MATH_FUNCTIONS", + "-DSQLITE_TEMP_STORE=2", + "-DSQLITE_THREADSAFE=1", + "-D_HAVE_SQLITE_CONFIG", + "-DSQLITE_CORE", +}; + +fn addSqlite(b: *Build, options: Sqlite3Options) *Build.Step.Compile { const sqlite_cfg = b.addConfigHeader( .{ .include_path = "sqlite_cfg.h", @@ -381,16 +381,30 @@ fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { }; }; + const libsql_version = comptime std.mem.trim(u8, @embedFile("LIBSQL_VERSION"), &std.ascii.whitespace); + const sqlite_version = comptime std.mem.trim(u8, @embedFile("VERSION"), &std.ascii.whitespace); + const manifest_uuid = comptime std.mem.trim(u8, @embedFile("manifest.uuid"), &std.ascii.whitespace); + const version = comptime std.SemanticVersion.parse("3.44.0") catch unreachable; + + assert(version.major <= 999); + assert(version.minor <= 999); + assert(version.patch <= 999); + + const version_number = + version.major * 1_000_000 + + version.minor * 1_000 + + version.patch; + const h = Amalgamation.create(b, "sqlite3.h", &.{ base: { // sqlite.h.in const base = b.addConfigHeader( .{ .style = .{ .cmake = b.path("src/sqlite.h.in") } }, .{ - // TODO: Remove hard coded values - .libsql_version = "0.2.3", - .sqlite_version = "3.44.0", - .sqlite_version_number = 3044000, - .sqlite_source_id = "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad8alt1", + .libsql_version = libsql_version, + .sqlite_version = sqlite_version, + .sqlite_version_number = @as(i64, @intCast(version_number)), + // TODO: Remove hard coded date + .sqlite_source_id = "2023-11-01 11:23:50 " ++ manifest_uuid, }, ); break :base base.getOutput(); @@ -419,10 +433,9 @@ fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { lib.addIncludePath(parse.h.dirname()); lib.addConfigHeader(sqlite_cfg); - lib.addCSourceFile(.{ .file = opcodes.c, .flags = cflags(b, &.{}) }); - lib.addCSourceFile(.{ .file = parse.c, .flags = cflags(b, &.{}) }); - lib.addCSourceFiles(.{ .files = sources.sqlite3, .flags = cflags(b, &.{}) }); - lib.addCSourceFile(.{ .file = b.path("ext/misc/stmt.c"), .flags = cflags(b, &.{}) }); + lib.addCSourceFile(.{ .file = opcodes.c, .flags = &cflags }); + lib.addCSourceFile(.{ .file = parse.c, .flags = &cflags }); + lib.addCSourceFiles(.{ .files = sources.sqlite3, .flags = &cflags }); if (options.fts5) { var fts5parse = fts5parse: { @@ -442,18 +455,8 @@ fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { lib.addIncludePath(b.path("ext/fts5/")); lib.addIncludePath(fts5parse.h.dirname()); - lib.addCSourceFile(.{ .file = fts5parse.c, .flags = cflags(b, &.{}) }); - lib.addCSourceFiles(.{ - .files = &.{ - "ext/fts5/fts5_aux.c", "ext/fts5/fts5_buffer.c", - "ext/fts5/fts5_config.c", "ext/fts5/fts5_expr.c", - "ext/fts5/fts5_hash.c", "ext/fts5/fts5_index.c", - "ext/fts5/fts5_main.c", "ext/fts5/fts5_storage.c", - "ext/fts5/fts5_tokenize.c", "ext/fts5/fts5_unicode2.c", - "ext/fts5/fts5_varint.c", "ext/fts5/fts5_vocab.c", - }, - .flags = cflags(b, &.{}), - }); + lib.addCSourceFile(.{ .file = fts5parse.c, .flags = &cflags }); + lib.addCSourceFiles(.{ .files = sources.fts5, .flags = &cflags }); lib.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); } @@ -477,7 +480,7 @@ fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { "ext/fts3/fts3_unicode2.c", "ext/fts3/fts3_write.c", }, - .flags = cflags(b, &.{}), + .flags = &cflags, }); lib.root_module.addCMacro("SQLITE_ENABLE_FTS3", "1"); @@ -485,7 +488,7 @@ fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { } if (options.rtree) { - lib.addCSourceFile(.{ .file = b.path("ext/rtree/rtree.c"), .flags = cflags(b, &.{}) }); + lib.addCSourceFile(.{ .file = b.path("ext/rtree/rtree.c"), .flags = &cflags }); lib.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); if (options.geopoly) lib.root_module.addCMacro("SQLITE_ENABLE_GEOPOLY", "1"); } @@ -506,7 +509,7 @@ fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { lib.addIncludePath(b.path(".")); // to reach "ext/udf/wasm_bindings.h" lib.addCSourceFile(.{ .file = b.path("ext/udf/wasmedge_bindings.c"), - .flags = cflags(b, &.{}), + .flags = &cflags, }); lib.addLibraryPath(libsql_wasm); lib.linkSystemLibrary("libsql_wasm"); @@ -515,7 +518,7 @@ fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { if (options.icu) { lib.addCSourceFiles(.{ .files = &.{"ext/icu/icu.c"}, - .flags = cflags(b, &.{}), + .flags = &cflags, }); lib.root_module.addCMacro("SQLITE_ENABLE_ICU", "1"); @@ -529,7 +532,7 @@ fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { lib.addCSourceFiles(.{ .files = &.{"ext/session/sqlite3session.c"}, - .flags = cflags(b, &.{}), + .flags = &cflags, }); lib.root_module.addCMacro("SQLITE_ENABLE_SESSION", "1"); @@ -551,6 +554,8 @@ fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { lib.root_module.addCMacro("SQLITE_WASI", "1"); lib.root_module.addCMacro("SQLITE_OMIT_SHARED_MEM", "1"); lib.root_module.addCMacro("SQLITE_OMIT_SHARED_CACHE", "1"); + } else { + lib.root_module.addCMacro("SQLITE_THREADSAFE", "1"); } if (options.target.result.os.tag == .windows) { @@ -559,61 +564,35 @@ fn addLibsql(b: *Build, options: Sqlite3Options) *Build.Step.Compile { lib.root_module.addCMacro("SQLITE_OS_UNIX", "1"); } - lib.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); - lib.root_module.addCMacro("SQLITE_ENABLE_DBSTAT_VTAB", "1"); + lib.addCSourceFile(.{ + .file = b.path("ext/misc/stmt.c"), + .flags = &cflags, + }); lib.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); - lib.root_module.addCMacro("SQLITE_ENABLE_BYTECODE_VTAB", "1"); + + lib.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); // src/dbpage.c + lib.root_module.addCMacro("SQLITE_ENABLE_DBSTAT_VTAB", "1"); // src/dbstat.c + lib.root_module.addCMacro("SQLITE_ENABLE_BYTECODE_VTAB", "1"); // src/vdbevtab.c + lib.root_module.addCMacro("SQLITE_ENABLE_COLUMN_METADATA", "1"); + lib.root_module.addCMacro("SQLITE_USE_URI", "1"); // used in sqliteConfig lib.root_module.addCMacro("SQLITE_ENABLE_LOAD_EXTENSION", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_API_ARMOR", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_MEMORY_MANAGEMENT", "1"); + lib.root_module.addCMacro("SQLITE_ENABLE_STAT4", "1"); + lib.root_module.addCMacro("SQLITE_SOUNDEX", "1"); - // lib.root_module.addCMacro("SQLITE_DEFAULT_FOREIGN_KEYS", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_API_ARMOR", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_COLUMN_METADATA", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_DBSTAT_VTAB", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_FTS3", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_FTS3_PARENTHESIS", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_FTS5", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_JSON1", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_MEMORY_MANAGEMENT", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_STAT2", "1"); - // lib.root_module.addCMacro("SQLITE_ENABLE_STAT4", "1"); - // lib.root_module.addCMacro("SQLITE_SOUNDEX", "1"); - // lib.root_module.addCMacro("SQLITE_THREADSAFE", "1"); - lib.root_module.addCMacro("SQLITE_USE_URI", "1"); - // lib.root_module.addCMacro("HAVE_USLEEP", "1"); - - lib.root_module.addCMacro("SQLITE_ENABLE_EXPLAIN_COMMENTS", "1"); + if (options.explain_comments) + lib.root_module.addCMacro("SQLITE_ENABLE_EXPLAIN_COMMENTS", "1"); - if (options.wasm_runtime) { - const libsql_wasm = crab.addCargoBuild(b, .{ - .manifest_path = b.path("crates/wasmtime-bindings/Cargo.toml"), - .cargo_args = &.{ - "--release", - "--lib", - }, - }, .{ - .target = options.target, - .optimize = .ReleaseSafe, - }); - - lib.root_module.addCMacro("LIBSQL_ENABLE_WASM_RUNTIME", ""); - lib.addIncludePath(b.path(".")); // to reach "ext/udf/wasm_bindings.h" - lib.addCSourceFile(.{ - .file = b.path("ext/udf/wasmedge_bindings.c"), - .flags = cflags(b, &.{}), - }); - lib.addLibraryPath(libsql_wasm); - lib.linkSystemLibrary("libsql_wasm"); - } - - if (options.icu) {} + if (options.default_foreign_keys) + lib.root_module.addCMacro("SQLITE_DEFAULT_FOREIGN_KEYS", "1"); return lib; } -pub fn build(b: *std.Build) void { +pub fn build(b: *std.Build) !void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); @@ -660,25 +639,32 @@ pub fn build(b: *std.Build) void { "geopoly", "Enable geopoly extension (default: true)", ) orelse true, + .default_foreign_keys = b.option( + bool, + "default-foreign-keys", + "Enable foreign keys constraints by default (default: false)", + ) orelse false, + .explain_comments = b.option( + bool, + "explain-comments", + "Enable explain comments (default: true)", + ) orelse true, .@"test" = false, }; - const libsql = addLibsql(b, options); + const sqlite = addSqlite(b, options); const fuzzcheck = b.addExecutable(.{ .name = "fuzzcheck", .root_source_file = null, .target = target, - .optimize = .ReleaseFast, + .optimize = optimize, }); fuzzcheck.addIncludePath(b.path("src/")); fuzzcheck.addIncludePath(b.path("ext/recover")); - fuzzcheck.addCSourceFiles(.{ - .files = sources.fuzzcheck, - .flags = cflags(b, &.{}), - }); + fuzzcheck.addCSourceFiles(.{ .files = sources.fuzzcheck, .flags = &cflags }); fuzzcheck.linkSystemLibrary("m"); - fuzzcheck.linkLibrary(libsql); + fuzzcheck.linkLibrary(sqlite); fuzzcheck.linkLibrary(zlib.artifact("z")); fuzzcheck.linkLibC(); @@ -686,69 +672,75 @@ pub fn build(b: *std.Build) void { fuzzcheck.root_module.addCMacro("SQLITE_NO_SYNC", "1"); fuzzcheck.root_module.addCMacro("SQLITE_OMIT_LOAD_EXTENSION", "1"); + try fuzzcheck.root_module.c_macros.appendSlice( + b.allocator, + sqlite.root_module.c_macros.items, + ); + + const sqlite_test = addSqlite(b, .{ + .target = options.target, + .optimize = options.optimize, + .rtree = options.rtree, + .session = options.session, + .fts3 = options.fts3, + .fts5 = options.fts5, + .geopoly = options.geopoly, + .icu = options.icu, + .wasm_runtime = options.wasm_runtime, + .explain_comments = options.explain_comments, + .@"test" = true, + // NOTE: If this is set, tests will break. + .default_foreign_keys = false, + }); + const testfixture = b.addExecutable(.{ .name = "testfixture", .root_source_file = null, .target = target, .optimize = .ReleaseFast, }); + + testfixture.linkLibrary(zlib.artifact("z")); + testfixture.linkLibrary(sqlite_test); + testfixture.linkSystemLibrary("tcl"); testfixture.linkLibC(); - testfixture.linkLibrary( - addLibsql(b, .{ - .target = options.target, - .optimize = options.optimize, - .rtree = options.rtree, - .session = options.session, - .fts3 = options.fts3, - .fts5 = options.fts5, - .geopoly = options.geopoly, - .icu = options.icu, - .wasm_runtime = options.wasm_runtime, - .@"test" = true, - }), - ); + testfixture.addIncludePath(b.path("src/")); - testfixture.addCSourceFiles(.{ .files = sources.testfixture, .flags = cflags(b, &.{}) }); + testfixture.addCSourceFiles(.{ .files = sources.testfixture, .flags = &cflags }); testfixture.root_module.addCMacro("SQLITE_HAVE_ZLIB", "1"); testfixture.root_module.addCMacro("SQLITE_TEST", "1"); - testfixture.root_module.addCMacro("TCLSH_INIT_PROC", "sqlite3TestInit"); - testfixture.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); - testfixture.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); - testfixture.root_module.addCMacro("SQLITE_ENABLE_BYTECODE_VTAB", "1"); - testfixture.root_module.addCMacro("SQLITE_SERIES_CONSTRAINT_VERIFY", "1"); - testfixture.root_module.addCMacro("SQLITE_CKSUMVFS_STATIC", "1"); - testfixture.root_module.addCMacro("SQLITE_DEFAULT_PAGE_SIZE", "1024"); - - if (options.icu) testfixture.root_module.addCMacro("SQLITE_ENABLE_ICU", "1"); - if (options.fts3) testfixture.root_module.addCMacro("SQLITE_ENABLE_FTS3", "1"); - if (options.session) { - testfixture.root_module.addCMacro("SQLITE_ENABLE_SESSION", "1"); - testfixture.root_module.addCMacro("SQLITE_ENABLE_PREUPDATE_HOOK", "1"); - } + testfixture.root_module.addCMacro("TCLSH_INIT_PROC", "sqlite3TestInit"); // src/tclsqlite.c + testfixture.root_module.addCMacro("SQLITE_CKSUMVFS_STATIC", "1"); // ext/mist/cksumvfs.c - testfixture.linkLibrary(zlib.artifact("z")); - testfixture.linkSystemLibrary("tcl"); + try testfixture.root_module.c_macros.appendSlice( + b.allocator, + sqlite_test.root_module.c_macros.items, + ); { const run = b.addRunArtifact(testfixture); + run.addFileArg(b.path("test/testrunner.tcl")); + if (b.args) |args| { run.addArgs(args); } else { - run.addArgs(&.{ "test/testrunner.tcl", "veryquick" }); + run.addArg("veryquick"); } const rust_suite = b.addSystemCommand(&.{ "cargo", "test" }); rust_suite.setCwd(b.path("test/rust_suite")); - rust_suite.step.dependOn(&run.step); + rust_suite.step.dependOn(&run.step); // run after main tests const step = b.step( "test", - "Run testfixture (default: testrunner.tcl veryquick)", + "Run tests (default: veryquick)", ); + step.dependOn(&run.step); step.dependOn(&rust_suite.step); + step.dependOn(&b.addInstallArtifact(testfixture, .{}).step); } { @@ -756,18 +748,9 @@ pub fn build(b: *std.Build) void { if (b.args) |args| { run.addArgs(args); } else { - inline for (&.{ - "test/fuzzdata1.db", - "test/fuzzdata2.db", - "test/fuzzdata3.db", - "test/fuzzdata4.db", - "test/fuzzdata5.db", - "test/fuzzdata6.db", - "test/fuzzdata7.db", - "test/fuzzdata8.db", - }) |path| { - run.addFileArg(b.path(path)); - } + inline for (1..8) |i| run.addFileArg( + b.path(std.fmt.comptimePrint("test/fuzzdata{d}.db", .{i})), + ); } const step = b.step( @@ -775,9 +758,8 @@ pub fn build(b: *std.Build) void { "Run fuzzcheck (default: test/fuzzdata[1..=8].db)", ); step.dependOn(&run.step); + step.dependOn(&b.addInstallArtifact(fuzzcheck, .{}).step); } - b.getInstallStep().dependOn(&b.addInstallArtifact(testfixture, .{}).step); - b.getInstallStep().dependOn(&b.addInstallArtifact(fuzzcheck, .{}).step); - b.getInstallStep().dependOn(&b.addInstallArtifact(libsql, .{}).step); + b.getInstallStep().dependOn(&b.addInstallArtifact(sqlite, .{}).step); } From 387031065366693ac8e3d4b4ec5d97c5d37c92f9 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Mon, 15 Jul 2024 05:20:11 -0300 Subject: [PATCH 31/35] fix(build): add SQLITE_DEBUG on .Debug optimze --- libsql-sqlite3/build.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libsql-sqlite3/build.zig b/libsql-sqlite3/build.zig index 97b76b8408..51c8afc674 100644 --- a/libsql-sqlite3/build.zig +++ b/libsql-sqlite3/build.zig @@ -564,10 +564,7 @@ fn addSqlite(b: *Build, options: Sqlite3Options) *Build.Step.Compile { lib.root_module.addCMacro("SQLITE_OS_UNIX", "1"); } - lib.addCSourceFile(.{ - .file = b.path("ext/misc/stmt.c"), - .flags = &cflags, - }); + lib.addCSourceFile(.{ .file = b.path("ext/misc/stmt.c"), .flags = &cflags }); lib.root_module.addCMacro("SQLITE_ENABLE_STMTVTAB", "1"); lib.root_module.addCMacro("SQLITE_ENABLE_DBPAGE_VTAB", "1"); // src/dbpage.c @@ -583,6 +580,9 @@ fn addSqlite(b: *Build, options: Sqlite3Options) *Build.Step.Compile { lib.root_module.addCMacro("SQLITE_ENABLE_STAT4", "1"); lib.root_module.addCMacro("SQLITE_SOUNDEX", "1"); + if (options.optimize == .Debug) + lib.root_module.addCMacro("SQLITE_DEBUG", "1"); + if (options.explain_comments) lib.root_module.addCMacro("SQLITE_ENABLE_EXPLAIN_COMMENTS", "1"); From 2b3cc21a2599017a32987b1352ac98a9a631f0d3 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Mon, 15 Jul 2024 14:35:29 -0300 Subject: [PATCH 32/35] revert test8.c bug fix --- libsql-sqlite3/build.zig | 1 + libsql-sqlite3/src/test8.c | 50 ++++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/libsql-sqlite3/build.zig b/libsql-sqlite3/build.zig index 51c8afc674..aee9d979b7 100644 --- a/libsql-sqlite3/build.zig +++ b/libsql-sqlite3/build.zig @@ -212,6 +212,7 @@ fn filterIncludes(b: *Build, lp: LazyPath) LazyPath { } const cflags = [_][]const u8{ + "-Wno-incompatible-pointer-types", // to compile with test8.c bug "-fno-sanitize=signed-integer-overflow", "-DBUILD_sqlite", "-DSQLITE_ENABLE_MATH_FUNCTIONS", diff --git a/libsql-sqlite3/src/test8.c b/libsql-sqlite3/src/test8.c index e217bd0a52..71f8884c01 100644 --- a/libsql-sqlite3/src/test8.c +++ b/libsql-sqlite3/src/test8.c @@ -1337,30 +1337,32 @@ static sqlite3_module echoModule = { }; static sqlite3_module echoModuleV2 = { - .iVersion = 700, /* iVersion */ - .xCreate = echoCreate, - .xConnect = echoConnect, - .xBestIndex = echoBestIndex, - .xDisconnect = echoDisconnect, - .xDestroy = echoDestroy, - .xOpen = echoOpen, /* xOpen - open a cursor */ - .xClose = echoClose, /* xClose - close a cursor */ - .xFilter = echoFilter, /* xFilter - configure scan constraints */ - .xNext = echoNext, /* xNext - advance a cursor */ - .xEof = echoEof, /* xEof */ - .xColumn = echoColumn, /* xColumn - read data */ - .xRowid = echoRowid, /* xRowid - read data */ - .xUpdate = echoUpdate, /* xUpdate - write data */ - .xBegin = echoBegin, /* xBegin - begin transaction */ - .xSync = echoSync, /* xSync - sync transaction */ - .xCommit = echoCommit, /* xCommit - commit transaction */ - .xRollback = echoRollback, /* xRollback - rollback transaction */ - .xFindFunction = echoFindFunction, /* xFindFunction - function overloading */ - .xRename = echoRename, /* xRename - rename the table */ - .xSavepoint = echoSavepoint, - .xRelease = echoRelease, - .xRollbackTo = echoRollbackTo, - .xPreparedSql = echoPreparedSql, + 700, /* iVersion */ + echoCreate, + echoConnect, + echoBestIndex, + echoDisconnect, + echoDestroy, + echoOpen, /* xOpen - open a cursor */ + echoClose, /* xClose - close a cursor */ + echoFilter, /* xFilter - configure scan constraints */ + echoNext, /* xNext - advance a cursor */ + echoEof, /* xEof */ + echoColumn, /* xColumn - read data */ + echoRowid, /* xRowid - read data */ + echoUpdate, /* xUpdate - write data */ + echoBegin, /* xBegin - begin transaction */ + echoSync, /* xSync - sync transaction */ + echoCommit, /* xCommit - commit transaction */ + echoRollback, /* xRollback - rollback transaction */ + echoFindFunction, /* xFindFunction - function overloading */ + echoRename, /* xRename - rename the table */ + echoSavepoint, + echoRelease, + echoRollbackTo, + 0, /* xShadowName */ + echoPreparedSql, + 0 /* xIntegrity */ }; /* From db7847a65c8b2c4d637b29164623ad3573b9f933 Mon Sep 17 00:00:00 2001 From: Levy Albuquerque Date: Mon, 15 Jul 2024 14:40:16 -0300 Subject: [PATCH 33/35] revert main.c changes --- libsql-sqlite3/build.zig | 1 + libsql-sqlite3/src/main.c | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/libsql-sqlite3/build.zig b/libsql-sqlite3/build.zig index aee9d979b7..0af00afee5 100644 --- a/libsql-sqlite3/build.zig +++ b/libsql-sqlite3/build.zig @@ -489,6 +489,7 @@ fn addSqlite(b: *Build, options: Sqlite3Options) *Build.Step.Compile { } if (options.rtree) { + lib.addIncludePath(b.path("ext/rtree/")); lib.addCSourceFile(.{ .file = b.path("ext/rtree/rtree.c"), .flags = &cflags }); lib.root_module.addCMacro("SQLITE_ENABLE_RTREE", "1"); if (options.geopoly) lib.root_module.addCMacro("SQLITE_ENABLE_GEOPOLY", "1"); diff --git a/libsql-sqlite3/src/main.c b/libsql-sqlite3/src/main.c index a1d40654f1..8fb842eee7 100644 --- a/libsql-sqlite3/src/main.c +++ b/libsql-sqlite3/src/main.c @@ -16,6 +16,16 @@ */ #include "sqliteInt.h" +#ifdef SQLITE_ENABLE_FTS3 +# include "fts3.h" +#endif +#ifdef SQLITE_ENABLE_RTREE +# include "rtree.h" +#endif +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) +# include "sqliteicu.h" +#endif + /* ** This is an extension initializer that is a no-op and always ** succeeds, except that it fails if the fault-simulation is set @@ -31,15 +41,6 @@ static int sqlite3TestExtInit(sqlite3 *db){ ** Forward declarations of external module initializer functions ** for modules that need them. */ -#ifdef SQLITE_ENABLE_FTS3 -int sqlite3Fts3Init(sqlite3*); -#endif -#ifdef SQLITE_ENABLE_RTREE -int sqlite3RtreeInit(sqlite3*); -#endif -#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) -int sqlite3IcuInit(sqlite3*); -#endif #ifdef SQLITE_ENABLE_FTS5 int sqlite3Fts5Init(sqlite3*); #endif From 0d6c5a1347a65af3faabf5ff814bbb5bebe8f721 Mon Sep 17 00:00:00 2001 From: "Levy A." Date: Wed, 28 Aug 2024 22:29:28 -0300 Subject: [PATCH 34/35] chore: cargo fmt --- libsql-ffi/build.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libsql-ffi/build.rs b/libsql-ffi/build.rs index 113a6a07de..83af249c18 100644 --- a/libsql-ffi/build.rs +++ b/libsql-ffi/build.rs @@ -48,7 +48,10 @@ fn main() { ); println!("cargo:rerun-if-changed=../libsql-sqlite3/"); - println!("cargo:rustc-link-search={}", out_dir.join("lib/").to_str().unwrap()); + println!( + "cargo:rustc-link-search={}", + out_dir.join("lib/").to_str().unwrap() + ); println!("cargo:rustc-link-lib=static=sqlite3"); return; From 3c88ee964f9d4798a35112774d49373e31592895 Mon Sep 17 00:00:00 2001 From: "Levy A." Date: Wed, 28 Aug 2024 22:31:54 -0300 Subject: [PATCH 35/35] chore: update bundle --- libsql-ffi/bundled/src/sqlite3.c | 14 ++++++++++---- libsql-ffi/bundled/src/sqlite3.h | 6 +++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/libsql-ffi/bundled/src/sqlite3.c b/libsql-ffi/bundled/src/sqlite3.c index 69dd8e24a6..9b5819d61e 100644 --- a/libsql-ffi/bundled/src/sqlite3.c +++ b/libsql-ffi/bundled/src/sqlite3.c @@ -29,6 +29,7 @@ ** configure ** configure.ac ** ext/fts5/fts5_tokenize.c +** ext/fts5/fts5_unicode2.c ** ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java ** ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java ** ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java @@ -538,11 +539,11 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.1" +#define SQLITE_VERSION "3.45.1" #define SQLITE_VERSION_NUMBER 3045001 -#define SQLITE_SOURCE_ID "2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257ccalt1" +#define SQLITE_SOURCE_ID "2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257ccalt1" -#define LIBSQL_VERSION "0.2.3" +#define LIBSQL_VERSION "0.2.3" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -21616,6 +21617,10 @@ SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3*); SQLITE_PRIVATE void sqlite3ChangeCookie(Parse*, int); SQLITE_PRIVATE With *sqlite3WithDup(sqlite3 *db, With *p); +/* libsql parse functions */ +void libsql_create_function(Parse *pParse, Token *pName, Token *pLang, Token *pBody, int isBlob, int noErr); +void libsql_drop_function(Parse *pParse, Token *pName, int noErr); + #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) SQLITE_PRIVATE void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); #endif @@ -116950,6 +116955,7 @@ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ ** that implements the ALTER TABLE command. */ /* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ /* ** The code in this file only exists if we are not omitting the @@ -261386,7 +261392,7 @@ static int sqlite3Fts5TokenizerInit(fts5_api *pApi){ ** DO NOT EDIT THIS MACHINE GENERATED FILE. */ - +/* #include "fts5Int.h" */ /* #include */ diff --git a/libsql-ffi/bundled/src/sqlite3.h b/libsql-ffi/bundled/src/sqlite3.h index d526834332..1fb0dfa6da 100644 --- a/libsql-ffi/bundled/src/sqlite3.h +++ b/libsql-ffi/bundled/src/sqlite3.h @@ -149,11 +149,11 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.1" +#define SQLITE_VERSION "3.45.1" #define SQLITE_VERSION_NUMBER 3045001 -#define SQLITE_SOURCE_ID "2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257ccalt1" +#define SQLITE_SOURCE_ID "2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257ccalt1" -#define LIBSQL_VERSION "0.2.3" +#define LIBSQL_VERSION "0.2.3" /* ** CAPI3REF: Run-Time Library Version Numbers