Merge pull request finos#2739 from tomjakubowski/tom-pyodide-exceptions

Emscripten wheel fixes

.github/actions/install-deps/action.yaml

@@ -200,7 +200,7 @@ runs:
         - name: manylinux deps
           shell: bash
           run: |
-              if [ -x "$(command -v dnf)" ]; then 
+              if [ -x "$(command -v dnf)" ]; then
                 dnf install wget -y
               fi
           if: ${{ runner.os == 'Linux' && inputs.cpp == 'true' && inputs.javascript == 'false' }}
@@ -216,6 +216,21 @@ runs:
           run: sudo node tools/perspective-scripts/install_tools.mjs
           if: ${{ runner.os != 'Windows' && inputs.cpp == 'true' && inputs.manylinux  == 'false' }}
 
+        - name: Install Python Pyodide dependencies
+          shell: bash
+          run: python -m pip install -r rust/perspective-python/requirements-pyodide.txt
+          if: ${{ inputs.pyodide == 'true' }}
+
+        - name: Install Pyodide distribution
+          shell: bash
+          run: pnpm run install_pyodide
+          if: ${{ inputs.pyodide == 'true' }}
+
+        - name: Install Python Playwright browsers
+          shell: bash
+          run: python -m playwright install
+          if: ${{ inputs.pyodide == 'true' }}
+
         # - name: Install CCache
         #   shell: bash
         #   run: sudo apt install -y ccache

.github/workflows/build.yaml

@@ -290,7 +290,7 @@ jobs:
     #                               `-'
     build_emscripten_wheel:
         runs-on: ${{ matrix.os }}
-        if: ${{ startsWith(github.ref, 'refs/tags/v') }}
+        # if: ${{ startsWith(github.ref, 'refs/tags/v') }}
         strategy:
             fail-fast: false
             matrix:
@@ -313,21 +313,27 @@ jobs:
               id: init-step
               uses: ./.github/actions/install-deps
               with:
+                  pyodide: "true"
                   javascript: "false"
                   arch: ${{ matrix.arch }}
                   manylinux: "false"
                   skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }}
 
             - name: Python Build Pyodide
               run: pnpm install && pnpm run build
-              if: ${{ runner.os == 'Linux' && matrix.arch == 'x86_64' }}
+              env:
+                  PSP_PYODIDE: 1
+                  PACKAGE: "perspective-python"
+                  CI: 1
+
+            - name: "Test Pyodide"
+              run: pnpm run test
               env:
                   PSP_PYODIDE: 1
                   PACKAGE: "perspective-python"
                   CI: 1
 
             - uses: actions/upload-artifact@v4
-              # if: ${{ runner.os != 'Windows' }}
               with:
                   name: perspective-python-dist-wasm32-emscripten-${{ matrix.python-version }}
                   path: rust/target/wheels/*.whl

.gitignore

@@ -126,6 +126,7 @@ target.vscode
 !.vscode/extensions.json
 !.vscode/settings.default.json
 !.vscode/tasks.json
+!.vscode/perspective.code-snippets
 
 .llvm
 

.vscode/perspective.code-snippets

@@ -0,0 +1,52 @@
+{
+	"perspective authors copyright notice: #-comment": {
+		"scope": "python,toml,yaml",
+		"body": [
+			"#  ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓",
+			"#  ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃",
+			"#  ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃",
+			"#  ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃",
+			"#  ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃",
+			"#  ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫",
+			"#  ┃ Copyright (c) 2017, the Perspective Authors.                              ┃",
+			"#  ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃",
+			"#  ┃ This file is part of the Perspective library, distributed under the terms ┃",
+			"#  ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃",
+			"#  ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛"
+		],
+		"prefix": "# copyright"
+	},
+	"perspective authors copyright notice: //-comment": {
+		"scope": "javascript,cpp,c,typescript,less,rust,proto3",
+		"body": [
+			"// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓",
+			"// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃",
+			"// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃",
+			"// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃",
+			"// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃",
+			"// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫",
+			"// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃",
+			"// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃",
+			"// ┃ This file is part of the Perspective library, distributed under the terms ┃",
+			"// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃",
+			"// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛"
+		],
+		"prefix": "// copyright"
+	},
+	// Place your perspective workspace snippets here. Each snippet is defined under a snippet name and has a scope, prefix, body and
+	// description. Add comma separated ids of the languages where the snippet is applicable in the scope field. If scope
+	// is left empty or omitted, the snippet gets applied to all languages. The prefix is what is
+	// used to trigger the snippet and the body will be expanded and inserted. Possible variables are:
+	// $1, $2 for tab stops, $0 for the final cursor position, and ${1:label}, ${2:another} for placeholders.
+	// Placeholders with the same ids are connected.
+	// Example:
+	// "Print to console": {
+	// 	"scope": "javascript,typescript",
+	// 	"prefix": "log",
+	// 	"body": [
+	// 		"console.log('$1');",
+	// 		"$2"
+	// 	],
+	// 	"description": "Log output to console"
+	// }
+}

cpp/perspective/CMakeLists.txt

@@ -5,8 +5,6 @@ include(CheckCCompilerFlag)
 set(CMAKE_BUILD_TYPE "Release")
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-
-
 # CMAKE POLICIES
 # option() should use new cmake behavior wrt variable clobbering
 cmake_policy(SET CMP0077 NEW)
@@ -28,9 +26,6 @@ if(NOT DEFINED PSP_CMAKE_MODULE_PATH)
     set(PSP_CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake")
 endif()
 
-
-
-
 set(CMAKE_MODULE_PATH "${PSP_CMAKE_MODULE_PATH}/modules" ${CMAKE_MODULE_PATH})
 set(MSVC_RUNTIME_LIBRARY MultiThreaded)
 
@@ -195,38 +190,15 @@ if(PSP_PYTHON_BUILD AND MACOS)
     # don't link against build python
     # https://blog.tim-smith.us/2015/09/python-extension-modules-os-x/
     set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup")
-
-    # # TODO This now needs to be set externally because we cross-compile.
-    # # check_c_compiler_flag("-arch x86_64" x86_64Supported)
-    # check_c_compiler_flag("-arch arm64" arm64Supported)
-
-    # if(x86_64Supported AND arm64Supported)
-    #     set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64" CACHE STRING "Build universal architecture for OSX" FORCE)
-    # elseif(x86_64Supported)
-    #     set(CMAKE_REQUIRED_LINK_OPTIONS "-arch;x86_64")
-    #     set(CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Build x86_64 architecture for OSX" FORCE)
-    # elseif(arm64Supported)
-    #     set(CMAKE_REQUIRED_LINK_OPTIONS "-arch;arm64")
-    #     set(CMAKE_OSX_ARCHITECTURES "arm64" CACHE STRING "Build arm64 architecture for OSX" FORCE)
-    # endif()
 endif()
 
 # ######################
 include_directories("${CMAKE_CURRENT_SOURCE_DIR}/src/include")
 
-# Needs to be set early so that all translation units use it.
-if (NOT PSP_PYODIDE)
-    set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -fexceptions")
-endif()
-
 if(NOT DEFINED PSP_WASM_EXCEPTIONS AND NOT PSP_PYTHON_BUILD)
     set(PSP_WASM_EXCEPTIONS ON)
 endif()
 
-# if(NOT WIN32)
-# set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
-# set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG")
-# endif()
 if(PSP_WASM_BUILD)
     # ###################
     # EMSCRIPTEN BUILD #
@@ -246,6 +218,7 @@ if(PSP_WASM_BUILD)
         ")
 
     if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
+        # Pyodide DEBUG block
         set(OPT_FLAGS " \
             -O0 \
             -g3 \
@@ -262,6 +235,7 @@ if(PSP_WASM_BUILD)
             ")
         endif()
     else()
+        # Pyodide RELEASE block
         set(OPT_FLAGS " \
             -O3 \
             -g0 \
@@ -369,34 +343,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
             # so we use a custom FindPythonHeaders that is the same as the
             # default, but ignores when the python libraries can't be found.
             psp_build_message("${Red}Manylinux build has no python shared libraries${ColorReset}")
-            # find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter)
-            # find_package(PythonHeaders ${PSP_PYTHON_VERSION} EXACT REQUIRED)
-        else()
-            # psp_build_message("${Cyan}Use python shared libraries${ColorReset}")
-            # if(PSP_PYODIDE)
-            #     find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter)
-            # else()
-            #     find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development.Module)
-            # endif()
-
-            # link_directories(${Python_LIBRARY_DIRS})
         endif()
-
-        # psp_build_message("${Cyan}Using Python ${Python_VERSION}, \nPython_INCLUDE_DIRS: ${Python_INCLUDE_DIRS}, \nPython_LIBRARIES: ${Python_LIBRARIES}, \nPython_EXECUTABLE: ${Python_EXECUTABLE} ${ColorReset}")
-        # include_directories(${Python_INCLUDE_DIRS})
-
-        # psp_build_dep("pybind11" "${PSP_CMAKE_MODULE_PATH}/Pybind.txt.in")
-
-        # find_package(NumPy REQUIRED)
-
-        # if(NOT PYTHON_NUMPY_FOUND)
-        #     message(FATAL_ERROR "${Red}Numpy could not be located${ColorReset}")
-        # else()
-        #     psp_build_message("${Cyan}Numpy found: ${PYTHON_NUMPY_INCLUDE_DIR}${ColorReset}")
-        #     include_directories(${PYTHON_NUMPY_INCLUDE_DIR})
-        # endif()
-
-        # ####################
     endif()
 endif()
 
@@ -674,34 +621,30 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
         # Python extra targets #
         # #######################
         if(PSP_WASM_BUILD)
+            # Pyodide
             set(CMAKE_EXECUTABLE_SUFFIX ".wasm")
             set(CMAKE_EXE_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} --pre-js \"${PSP_CPP_SRC}/env.js\" ")
             add_library(psp STATIC ${PYTHON_SOURCE_FILES})
             target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_ENABLE_WASM=1)
+            # support for emscripten exceptions https://emscripten.org/docs/porting/exceptions.html#emscripten-javascript-based-exception-support
+            target_compile_options(psp PUBLIC -fexceptions -fvisibility=hidden)
+            target_compile_options(arrow PUBLIC -fexceptions -fvisibility=hidden)
+            target_compile_options(re2 PUBLIC -fexceptions -fvisibility=hidden)
+            target_compile_options(protos PUBLIC -fexceptions -fvisibility=hidden)
         else()
+            # Cpython
             add_library(psp STATIC ${PYTHON_SOURCE_FILES})
             target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1)
         endif()
-        # add_library(psppy SHARED ${PYTHON_BINDING_SOURCE_FILES})
 
         include_directories(${PSP_PYTHON_SRC}/include)
 
-        # target_compile_definitions(psppy PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1)
-
-        if(WIN32)
-            #target_compile_definitions(psppy PRIVATE WIN32=1)
-            # target_compile_definitions(psppy PRIVATE _WIN32=1)
-
-            # .dll not importable
-            # set_property(TARGET psppy PROPERTY SUFFIX .pyd)
-        elseif(MACOS OR NOT MANYLINUX)
-            # target_compile_options(psppy PRIVATE -Wdeprecated-declarations)
+        if(MACOS OR NOT MANYLINUX)
             set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
-            # set_property(TARGET psppy PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
 
             target_compile_options(psp PRIVATE -fvisibility=hidden)
-            # target_compile_options(psppy PRIVATE -fvisibility=hidden)
         elseif(MANYLINUX)
+            # intentionally blank
         else()
             target_compile_options(psp PRIVATE -fvisibility=hidden)
             # target_compile_options(psppy PRIVATE -Wdeprecated-declarations)

cpp/perspective/build.js

@@ -28,18 +28,36 @@ function bootstrap(file) {
     });
 }
 
+let cmake_flags = "";
+let make_flags = "";
+
+if (!!process.env.PSP_BUILD_VERBOSE) {
+    cmake_flags += "-Wdev --debug-output ";
+    make_flags += "VERBOSE=1 ";
+} else {
+    cmake_flags = "-Wno-dev "; // suppress developer warnings
+}
+
 try {
     execSync(`mkdirp ${cwd}`, { stdio });
     process.env.CLICOLOR_FORCE = 1;
-    execSync(`emcmake cmake ${__dirname} -Wno-dev -DCMAKE_BUILD_TYPE=${env}`, {
-        cwd,
-        stdio,
-    });
+    execSync(
+        `emcmake cmake ${__dirname} ${cmake_flags} -DCMAKE_BUILD_TYPE=${env}`,
+        {
+            cwd,
+            stdio,
+        }
+    );
 
-    execSync(`emmake make -j${process.env.PSP_NUM_CPUS || os.cpus().length}`, {
-        cwd,
-        stdio,
-    });
+    execSync(
+        `emmake make -j${
+            process.env.PSP_NUM_CPUS || os.cpus().length
+        } ${make_flags}`,
+        {
+            cwd,
+            stdio,
+        }
+    );
 
     execSync(`cpy web/**/* ../web`, { cwd, stdio });
     execSync(`cpy node/**/* ../node`, { cwd, stdio });

cpp/perspective/src/cpp/arrow_csv.cpp

@@ -387,7 +387,6 @@ class CustomISO8601Parser : public arrow::TimestampParser {
             if (length == 29) {
                 // YYYY-MM-DD[ T]hh:mm:ss.sssssssss  -- nanos
                 // arrow handles YYYY-MM-DD[ T]hh:mm:ss.sss[+-]HH:MM
-                std::cout << "DDD WOOHOOOOO!\n";
                 arrow_vendored::date::year_month_day ymd;
                 if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
                     return false;
@@ -645,4 +644,4 @@ csvToTable(
     return *maybe_table;
 }
 
-} // namespace perspective::apachearrow
+} // namespace perspective::apachearrow

cpp/perspective/src/cpp/server.cpp

@@ -2262,7 +2262,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, const Request& req) {
         case proto::Request::kServerSystemInfoReq: {
             proto::Response resp;
             auto* sys_info = resp.mutable_server_system_info_resp();
-#ifdef PSP_ENABLE_WASM
+#if defined(PSP_ENABLE_WASM) && !defined(PSP_ENABLE_PYTHON)
             auto heap_size = psp_heap_size();
             sys_info->set_heap_size(heap_size);
 #else

cpp/perspective/src/include/perspective/base.h

@@ -159,7 +159,7 @@ std::is_pod<X>::value && std::is_standard_layout<X>::value , \
 #define LOG_DEBUG(X)
 #endif
 
-#if defined(PSP_ENABLE_WASM)
+#if defined(PSP_ENABLE_WASM) && !defined(PSP_PYODIDE)
 #define ESM_EXPORT(X) __attribute__((import_module("env"), import_name(X)))
 
 PERSPECTIVE_EXPORT ESM_EXPORT("psp_stack_trace") extern "C" const
@@ -170,7 +170,7 @@ PERSPECTIVE_EXPORT ESM_EXPORT("psp_heap_size") extern "C" size_t
 
 #endif
 
-#if defined(PSP_DEBUG) && defined(PSP_ENABLE_WASM)
+#if defined(PSP_DEBUG) && defined(PSP_ENABLE_WASM) && !defined(PSP_PYODIDE)
 #define PSP_COMPLAIN_AND_ABORT(X)                                              \
     {                                                                          \
         std::stringstream __SS__;                                              \

package.json

@@ -15,6 +15,7 @@
     "type": "module",
     "emscripten": "3.1.58",
     "llvm": "17.0.6",
+    "pyodide": "0.26.2",
     "engines": {
         "node": ">=16"
     },
@@ -100,6 +101,7 @@
         "postinstall:playwright": "npx playwright install --with-deps",
         "postinstall:vscode": "cp -n ./.vscode/settings.default.json ./.vscode/settings.json || true",
         "install_llvm": "node tools/perspective-scripts/install_llvm.mjs",
+        "install_pyodide": "node tools/perspective-scripts/install_pyodide.mjs",
         "build,test": "npm run --silent build && npm run --silent test",
         "build_js": "node tools/perspective-scripts/build_js.mjs",
         "build": "node tools/perspective-scripts/build.mjs",