From 3461629a62e96464f25343c4b479273ccd0f74ca Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Wed, 4 Sep 2024 13:22:13 -0700 Subject: [PATCH 01/11] Build with Emscripten exceptions for Pyodide Switch from using CMAKE_CXX_FLAGS, which has confusing and global semantics, to calling target_compile_options on all the targets This confines all the pyodide exception build code to a single branch of the cmake file Remove some commented-out code and dead branches in the build. I think this helps with reading and scanning the Cmake files Signed-off-by: Tom Jakubowski --- cpp/perspective/CMakeLists.txt | 78 ++++---------------------- pnpm-lock.yaml | 2 + rust/perspective-python/build.mjs | 2 +- rust/perspective-python/pyproject.toml | 2 +- rust/perspective-server/build/psp.rs | 1 + 5 files changed, 17 insertions(+), 68 deletions(-) diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index 69783da7da..62f0f54030 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -5,8 +5,6 @@ include(CheckCCompilerFlag) set(CMAKE_BUILD_TYPE "Release") set(CMAKE_CXX_STANDARD 17) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - - # CMAKE POLICIES # option() should use new cmake behavior wrt variable clobbering cmake_policy(SET CMP0077 NEW) @@ -28,9 +26,6 @@ if(NOT DEFINED PSP_CMAKE_MODULE_PATH) set(PSP_CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake") endif() - - - set(CMAKE_MODULE_PATH "${PSP_CMAKE_MODULE_PATH}/modules" ${CMAKE_MODULE_PATH}) set(MSVC_RUNTIME_LIBRARY MultiThreaded) @@ -195,38 +190,15 @@ if(PSP_PYTHON_BUILD AND MACOS) # don't link against build python # https://blog.tim-smith.us/2015/09/python-extension-modules-os-x/ set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup") - - # # TODO This now needs to be set externally because we cross-compile. - # # check_c_compiler_flag("-arch x86_64" x86_64Supported) - # check_c_compiler_flag("-arch arm64" arm64Supported) - - # if(x86_64Supported AND arm64Supported) - # set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64" CACHE STRING "Build universal architecture for OSX" FORCE) - # elseif(x86_64Supported) - # set(CMAKE_REQUIRED_LINK_OPTIONS "-arch;x86_64") - # set(CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Build x86_64 architecture for OSX" FORCE) - # elseif(arm64Supported) - # set(CMAKE_REQUIRED_LINK_OPTIONS "-arch;arm64") - # set(CMAKE_OSX_ARCHITECTURES "arm64" CACHE STRING "Build arm64 architecture for OSX" FORCE) - # endif() endif() # ###################### include_directories("${CMAKE_CURRENT_SOURCE_DIR}/src/include") -# Needs to be set early so that all translation units use it. -if (NOT PSP_PYODIDE) - set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -fexceptions") -endif() - if(NOT DEFINED PSP_WASM_EXCEPTIONS AND NOT PSP_PYTHON_BUILD) set(PSP_WASM_EXCEPTIONS ON) endif() -# if(NOT WIN32) -# set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") -# set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") -# endif() if(PSP_WASM_BUILD) # ################### # EMSCRIPTEN BUILD # @@ -246,6 +218,7 @@ if(PSP_WASM_BUILD) ") if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) + # Pyodide DEBUG block set(OPT_FLAGS " \ -O0 \ -g3 \ @@ -262,6 +235,7 @@ if(PSP_WASM_BUILD) ") endif() else() + # Pyodide RELEASE block set(OPT_FLAGS " \ -O3 \ -g0 \ @@ -369,34 +343,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) # so we use a custom FindPythonHeaders that is the same as the # default, but ignores when the python libraries can't be found. psp_build_message("${Red}Manylinux build has no python shared libraries${ColorReset}") - # find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter) - # find_package(PythonHeaders ${PSP_PYTHON_VERSION} EXACT REQUIRED) - else() - # psp_build_message("${Cyan}Use python shared libraries${ColorReset}") - # if(PSP_PYODIDE) - # find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter) - # else() - # find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development.Module) - # endif() - - # link_directories(${Python_LIBRARY_DIRS}) endif() - - # psp_build_message("${Cyan}Using Python ${Python_VERSION}, \nPython_INCLUDE_DIRS: ${Python_INCLUDE_DIRS}, \nPython_LIBRARIES: ${Python_LIBRARIES}, \nPython_EXECUTABLE: ${Python_EXECUTABLE} ${ColorReset}") - # include_directories(${Python_INCLUDE_DIRS}) - - # psp_build_dep("pybind11" "${PSP_CMAKE_MODULE_PATH}/Pybind.txt.in") - - # find_package(NumPy REQUIRED) - - # if(NOT PYTHON_NUMPY_FOUND) - # message(FATAL_ERROR "${Red}Numpy could not be located${ColorReset}") - # else() - # psp_build_message("${Cyan}Numpy found: ${PYTHON_NUMPY_INCLUDE_DIR}${ColorReset}") - # include_directories(${PYTHON_NUMPY_INCLUDE_DIR}) - # endif() - - # #################### endif() endif() @@ -593,7 +540,7 @@ endif() if(PSP_PYODIDE) set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \ --no-entry \ - -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server \ + -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server,_psp_stack_trace,_psp_heap_size \ -s SIDE_MODULE=2 \ ") else() @@ -674,11 +621,18 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) # Python extra targets # # ####################### if(PSP_WASM_BUILD) + # Pyodide set(CMAKE_EXECUTABLE_SUFFIX ".wasm") set(CMAKE_EXE_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} --pre-js \"${PSP_CPP_SRC}/env.js\" ") add_library(psp STATIC ${PYTHON_SOURCE_FILES}) target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_ENABLE_WASM=1) + # support for emscripten exceptions https://emscripten.org/docs/porting/exceptions.html#emscripten-javascript-based-exception-support + target_compile_options(psp PUBLIC -fexceptions -fvisibility=hidden) + target_compile_options(arrow PUBLIC -fexceptions -fvisibility=hidden) + target_compile_options(re2 PUBLIC -fexceptions -fvisibility=hidden) + target_compile_options(protos PUBLIC -fexceptions -fvisibility=hidden) else() + # Cpython add_library(psp STATIC ${PYTHON_SOURCE_FILES}) target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1) endif() @@ -688,20 +642,12 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) # target_compile_definitions(psppy PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1) - if(WIN32) - #target_compile_definitions(psppy PRIVATE WIN32=1) - # target_compile_definitions(psppy PRIVATE _WIN32=1) - - # .dll not importable - # set_property(TARGET psppy PROPERTY SUFFIX .pyd) - elseif(MACOS OR NOT MANYLINUX) - # target_compile_options(psppy PRIVATE -Wdeprecated-declarations) + if(MACOS OR NOT MANYLINUX) set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path}) - # set_property(TARGET psppy PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path}) target_compile_options(psp PRIVATE -fvisibility=hidden) - # target_compile_options(psppy PRIVATE -fvisibility=hidden) elseif(MANYLINUX) + # intentionally blank else() target_compile_options(psp PRIVATE -fvisibility=hidden) # target_compile_options(psppy PRIVATE -Wdeprecated-declarations) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7d29ba7c33..bfbe0d72c5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -831,6 +831,8 @@ importers: specifier: ^0.1.16 version: 0.1.16 + rust/perspective: {} + rust/perspective-js: dependencies: stoppable: diff --git a/rust/perspective-python/build.mjs b/rust/perspective-python/build.mjs index 16a17cb7bd..33727696e5 100644 --- a/rust/perspective-python/build.mjs +++ b/rust/perspective-python/build.mjs @@ -80,7 +80,7 @@ if (build_wheel) { target = "--target=aarch64-unknown-linux-gnu"; } - cmd.sh(`maturin build ${flags} --features=external-cpp ${target}`); + cmd.sh(`maturin build ${flags} -vv --features=external-cpp ${target}`); } if (build_sdist) { diff --git a/rust/perspective-python/pyproject.toml b/rust/perspective-python/pyproject.toml index 471fb1982e..f8438bceb0 100644 --- a/rust/perspective-python/pyproject.toml +++ b/rust/perspective-python/pyproject.toml @@ -33,7 +33,7 @@ classifiers = [ # "Framework :: Jupyter :: JupyterLab :: Extensions :: Mime Renderers", "Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt", ] -dependencies = ["Jinja2>=2.0,<4", "ipywidgets>=7.5.1,<9"] +dependencies = ["Jinja2>=2.0,<4", "ipywidgets>=7.5.1,<9", "build>=1.2.1"] [tool.maturin] module-name = "perspective" diff --git a/rust/perspective-server/build/psp.rs b/rust/perspective-server/build/psp.rs index 0e739b7bd1..39f40e5bed 100644 --- a/rust/perspective-server/build/psp.rs +++ b/rust/perspective-server/build/psp.rs @@ -110,6 +110,7 @@ pub fn cmake_build() -> Result, std::io::Error> { } println!("cargo:warning=MESSAGE Building cmake {}", profile); + dst.very_verbose(true); let artifact_dir = dst.build(); Ok(Some(artifact_dir)) From 09d72216db69f5c81444a06b10a92f71e7009346 Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Wed, 4 Sep 2024 20:53:50 -0700 Subject: [PATCH 02/11] Hide psp_stack_trace, psp_heap_size from pyodide This fixes a nasty warning on installing the wheel. It could likely be fixed another way, maybe with linker flags, but we decided to cut the feature since it's not of main importance Signed-off-by: Tom Jakubowski --- cpp/perspective/CMakeLists.txt | 2 +- cpp/perspective/src/cpp/server.cpp | 2 +- cpp/perspective/src/include/perspective/base.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index 62f0f54030..5d491f2fc4 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -540,7 +540,7 @@ endif() if(PSP_PYODIDE) set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \ --no-entry \ - -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server,_psp_stack_trace,_psp_heap_size \ + -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server \ -s SIDE_MODULE=2 \ ") else() diff --git a/cpp/perspective/src/cpp/server.cpp b/cpp/perspective/src/cpp/server.cpp index 1670a81536..29208c9188 100644 --- a/cpp/perspective/src/cpp/server.cpp +++ b/cpp/perspective/src/cpp/server.cpp @@ -2262,7 +2262,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, const Request& req) { case proto::Request::kServerSystemInfoReq: { proto::Response resp; auto* sys_info = resp.mutable_server_system_info_resp(); -#ifdef PSP_ENABLE_WASM +#if defined(PSP_ENABLE_WASM) && !defined(PSP_ENABLE_PYTHON) auto heap_size = psp_heap_size(); sys_info->set_heap_size(heap_size); #else diff --git a/cpp/perspective/src/include/perspective/base.h b/cpp/perspective/src/include/perspective/base.h index 2ff2dd8e09..43a112fa3c 100644 --- a/cpp/perspective/src/include/perspective/base.h +++ b/cpp/perspective/src/include/perspective/base.h @@ -159,7 +159,7 @@ std::is_pod::value && std::is_standard_layout::value , \ #define LOG_DEBUG(X) #endif -#if defined(PSP_ENABLE_WASM) +#if defined(PSP_ENABLE_WASM) && !defined(PSP_PYODIDE) #define ESM_EXPORT(X) __attribute__((import_module("env"), import_name(X))) PERSPECTIVE_EXPORT ESM_EXPORT("psp_stack_trace") extern "C" const @@ -170,7 +170,7 @@ PERSPECTIVE_EXPORT ESM_EXPORT("psp_heap_size") extern "C" size_t #endif -#if defined(PSP_DEBUG) && defined(PSP_ENABLE_WASM) +#if defined(PSP_DEBUG) && defined(PSP_ENABLE_WASM) && !defined(PSP_PYODIDE) #define PSP_COMPLAIN_AND_ABORT(X) \ { \ std::stringstream __SS__; \ From 97eb93f8f4c8da1c5fa431a74ad066b3562e58ba Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Thu, 5 Sep 2024 11:34:39 -0700 Subject: [PATCH 03/11] Add option to make builds much more verbose Signed-off-by: Tom Jakubowski --- cpp/perspective/build.js | 34 +++++++++++++++++++++------- rust/perspective-python/build.mjs | 6 ++++- rust/perspective-server/build/psp.rs | 4 +++- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/cpp/perspective/build.js b/cpp/perspective/build.js index 26d856f08e..2092acba68 100644 --- a/cpp/perspective/build.js +++ b/cpp/perspective/build.js @@ -28,18 +28,36 @@ function bootstrap(file) { }); } +let cmake_flags = ""; +let make_flags = ""; + +if (!!process.env.PSP_BUILD_VERBOSE) { + cmake_flags += "-Wdev --debug-output "; + make_flags += "VERBOSE=1 "; +} else { + cmake_flags = "-Wno-dev "; // suppress developer warnings +} + try { execSync(`mkdirp ${cwd}`, { stdio }); process.env.CLICOLOR_FORCE = 1; - execSync(`emcmake cmake ${__dirname} -Wno-dev -DCMAKE_BUILD_TYPE=${env}`, { - cwd, - stdio, - }); + execSync( + `emcmake cmake ${__dirname} ${cmake_flags} -DCMAKE_BUILD_TYPE=${env}`, + { + cwd, + stdio, + } + ); - execSync(`emmake make -j${process.env.PSP_NUM_CPUS || os.cpus().length}`, { - cwd, - stdio, - }); + execSync( + `emmake make -j${ + process.env.PSP_NUM_CPUS || os.cpus().length + } ${make_flags}`, + { + cwd, + stdio, + } + ); execSync(`cpy web/**/* ../web`, { cwd, stdio }); execSync(`cpy node/**/* ../node`, { cwd, stdio }); diff --git a/rust/perspective-python/build.mjs b/rust/perspective-python/build.mjs index 33727696e5..e399569f8e 100644 --- a/rust/perspective-python/build.mjs +++ b/rust/perspective-python/build.mjs @@ -80,7 +80,11 @@ if (build_wheel) { target = "--target=aarch64-unknown-linux-gnu"; } - cmd.sh(`maturin build ${flags} -vv --features=external-cpp ${target}`); + if (!!process.env.PSP_BUILD_VERBOSE) { + flags += " -vv"; + } + + cmd.sh(`maturin build ${flags} --features=external-cpp ${target}`); } if (build_sdist) { diff --git a/rust/perspective-server/build/psp.rs b/rust/perspective-server/build/psp.rs index 39f40e5bed..7f66e6a860 100644 --- a/rust/perspective-server/build/psp.rs +++ b/rust/perspective-server/build/psp.rs @@ -110,7 +110,9 @@ pub fn cmake_build() -> Result, std::io::Error> { } println!("cargo:warning=MESSAGE Building cmake {}", profile); - dst.very_verbose(true); + if std::env::var("PSP_BUILD_VERBOSE").unwrap_or_default() != "" { // checks non-empty env var + dst.very_verbose(true); + } let artifact_dir = dst.build(); Ok(Some(artifact_dir)) From 1e4bd90dab256969837dd66715c41b07e31dc332 Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Thu, 5 Sep 2024 20:04:15 -0700 Subject: [PATCH 04/11] Delete more dead build code Signed-off-by: Tom Jakubowski --- cpp/perspective/CMakeLists.txt | 3 --- rust/perspective-js/build.js | 3 --- 2 files changed, 6 deletions(-) diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index 5d491f2fc4..c172f3b0fe 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -636,12 +636,9 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) add_library(psp STATIC ${PYTHON_SOURCE_FILES}) target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1) endif() - # add_library(psppy SHARED ${PYTHON_BINDING_SOURCE_FILES}) include_directories(${PSP_PYTHON_SRC}/include) - # target_compile_definitions(psppy PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1) - if(MACOS OR NOT MANYLINUX) set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path}) diff --git a/rust/perspective-js/build.js b/rust/perspective-js/build.js index 63e725f59a..63446a22ed 100644 --- a/rust/perspective-js/build.js +++ b/rust/perspective-js/build.js @@ -13,12 +13,9 @@ import { execSync } from "child_process"; import { build } from "@finos/perspective-esbuild-plugin/build.js"; import { PerspectiveEsbuildPlugin } from "@finos/perspective-esbuild-plugin"; -import * as fs from "node:fs"; import { NodeModulesExternal } from "@finos/perspective-esbuild-plugin/external.js"; import cpy from "cpy"; -const cpy_mod = import("cpy"); - const IS_DEBUG = !!process.env.PSP_DEBUG || process.argv.indexOf("--debug") >= 0; From 6078dafd8171590d9c0989483b17d2df298fdca1 Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Thu, 5 Sep 2024 20:27:59 -0700 Subject: [PATCH 05/11] Constrain ipywidgets dep This makes our wheel compatible with widgetsnbextension 4.0.11, which is the version provided by jupyterlite-pyodide-kernel 0.4.1 Signed-off-by: Tom Jakubowski --- rust/perspective-python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/perspective-python/pyproject.toml b/rust/perspective-python/pyproject.toml index f8438bceb0..691a2c2758 100644 --- a/rust/perspective-python/pyproject.toml +++ b/rust/perspective-python/pyproject.toml @@ -33,7 +33,7 @@ classifiers = [ # "Framework :: Jupyter :: JupyterLab :: Extensions :: Mime Renderers", "Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt", ] -dependencies = ["Jinja2>=2.0,<4", "ipywidgets>=7.5.1,<9", "build>=1.2.1"] +dependencies = ["Jinja2>=2.0,<4", "ipywidgets>=7.5.1,<=8.1.3", "build>=1.2.1"] [tool.maturin] module-name = "perspective" From 72212ce9b1583f7413c90ee2de03ee8a086d4e4a Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Fri, 6 Sep 2024 11:14:16 -0700 Subject: [PATCH 06/11] Add pyodide virtual package In setup, this package selects `perspective-python` and sets some environment variables to make the build run for Pyodide. Co-authored-by: Andrew Stein Signed-off-by: Tom Jakubowski --- tools/perspective-scripts/setup.mjs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tools/perspective-scripts/setup.mjs b/tools/perspective-scripts/setup.mjs index a00be21620..a6437a8e87 100644 --- a/tools/perspective-scripts/setup.mjs +++ b/tools/perspective-scripts/setup.mjs @@ -30,6 +30,12 @@ const CONFIG = new Proxy( ).replace(/\|/g, ","); } } + remove(keyname) { + const idx = this.config.find((x) => s.startsWith(keyname)); + if (idx >= 0) { + this.config.splice(idx); + } + } add(new_config) { for (const key in new_config) { const val = new_config[key]; @@ -120,6 +126,11 @@ async function focus_package() { name: "perspective-python", value: "perspective-python", }, + { + key: "q", + name: "perspective-pyodide (Python)", + value: "perspective-pyodide", + }, { key: "r", name: "perspective-rs", @@ -166,6 +177,17 @@ async function focus_package() { if (Array.isArray(new_config.PACKAGE)) { if (new_config.PACKAGE.length > 0) { + let pyodide = new_config.PACKAGE.indexOf("perspective-pyodide"); + if (pyodide >= 0) { + new_config.PACKAGE.splice(pyodide, 1); + new_config.PSP_PYODIDE = 1; + new_config.CI = 1; + new_config.PACKAGE.push("perspective-python"); + } else { + CONFIG.remove("PSP_PYODIDE"); + CONFIG.remove("CI"); + } + new_config.PACKAGE = `${new_config.PACKAGE.join(",")}`; } else { new_config.PACKAGE = undefined; From d1100bddf5aee41c5bba0b33d965dbb79624b72d Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Thu, 5 Sep 2024 22:01:22 -0700 Subject: [PATCH 07/11] Add pytest-pyodide tests Integration/smoke tests for the Pyodide wheel, which run for the perspective-pyodide virtual package and in the Pyodide CI actions. Some changes which support installing the wheels in a pytest-pyodide environment: - Defer some third-party imports to "run time" by moving them deeper than module scope - Add try/except guards to widget re-exports so that wheel can be used without Jupyter being installed - Move Jupyter deps in pyproject.toml to an optional-dependencies group Signed-off-by: Tom Jakubowski --- .github/actions/install-deps/action.yaml | 17 ++++- .github/workflows/build.yaml | 10 ++- package.json | 2 + pnpm-lock.yaml | 3 + rust/perspective-python/package.json | 3 +- .../perspective/__init__.py | 10 ++- .../perspective/widget/__init__.py | 8 +- .../perspective/widget/widget.py | 2 +- .../pyodide-tests/README.md | 5 ++ .../pyodide-tests/tests/__init__.py | 11 +++ .../pyodide-tests/tests/conftest.py | 19 +++++ .../pyodide-tests/tests/test_smoke.py | 74 +++++++++++++++++++ rust/perspective-python/pyproject.toml | 5 +- .../requirements-pyodide.txt | 5 ++ rust/perspective-python/requirements.txt | 2 +- rust/perspective-python/test.mjs | 49 ++++++++++++ tools/perspective-scripts/install_pyodide.mjs | 57 ++++++++++++++ tools/perspective-scripts/pyodide.mjs | 41 ++++++++++ tools/perspective-scripts/workspace.mjs | 53 +++++++++++++ 19 files changed, 364 insertions(+), 12 deletions(-) create mode 100644 rust/perspective-python/pyodide-tests/README.md create mode 100644 rust/perspective-python/pyodide-tests/tests/__init__.py create mode 100644 rust/perspective-python/pyodide-tests/tests/conftest.py create mode 100644 rust/perspective-python/pyodide-tests/tests/test_smoke.py create mode 100644 rust/perspective-python/requirements-pyodide.txt create mode 100644 rust/perspective-python/test.mjs create mode 100644 tools/perspective-scripts/install_pyodide.mjs create mode 100644 tools/perspective-scripts/pyodide.mjs create mode 100644 tools/perspective-scripts/workspace.mjs diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml index f45a21e71c..de7816b83a 100644 --- a/.github/actions/install-deps/action.yaml +++ b/.github/actions/install-deps/action.yaml @@ -200,7 +200,7 @@ runs: - name: manylinux deps shell: bash run: | - if [ -x "$(command -v dnf)" ]; then + if [ -x "$(command -v dnf)" ]; then dnf install wget -y fi if: ${{ runner.os == 'Linux' && inputs.cpp == 'true' && inputs.javascript == 'false' }} @@ -216,6 +216,21 @@ runs: run: sudo node tools/perspective-scripts/install_tools.mjs if: ${{ runner.os != 'Windows' && inputs.cpp == 'true' && inputs.manylinux == 'false' }} + - name: Install Python Pyodide dependencies + shell: bash + run: python -m pip install -r rust/perspective-python/requirements-pyodide.txt + if: ${{ inputs.pyodide == 'true' }} + + - name: Install Pyodide distribution + shell: bash + run: pnpm run install_pyodide + if: ${{ inputs.pyodide == 'true' }} + + - name: Install Python Playwright browsers + shell: bash + run: python -m playwright install + if: ${{ inputs.pyodide == 'true' }} + # - name: Install CCache # shell: bash # run: sudo apt install -y ccache diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e57a950f43..074004abbd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -313,6 +313,7 @@ jobs: id: init-step uses: ./.github/actions/install-deps with: + pyodide: "true" javascript: "false" arch: ${{ matrix.arch }} manylinux: "false" @@ -320,14 +321,19 @@ jobs: - name: Python Build Pyodide run: pnpm install && pnpm run build - if: ${{ runner.os == 'Linux' && matrix.arch == 'x86_64' }} + env: + PSP_PYODIDE: 1 + PACKAGE: "perspective-python" + CI: 1 + + - name: "Test Pyodide" + run: pnpm run test env: PSP_PYODIDE: 1 PACKAGE: "perspective-python" CI: 1 - uses: actions/upload-artifact@v4 - # if: ${{ runner.os != 'Windows' }} with: name: perspective-python-dist-wasm32-emscripten-${{ matrix.python-version }} path: rust/target/wheels/*.whl diff --git a/package.json b/package.json index 5167b6f126..4352da62a2 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "type": "module", "emscripten": "3.1.58", "llvm": "17.0.6", + "pyodide": "0.26.2", "engines": { "node": ">=16" }, @@ -100,6 +101,7 @@ "postinstall:playwright": "npx playwright install --with-deps", "postinstall:vscode": "cp -n ./.vscode/settings.default.json ./.vscode/settings.json || true", "install_llvm": "node tools/perspective-scripts/install_llvm.mjs", + "install_pyodide": "node tools/perspective-scripts/install_pyodide.mjs", "build,test": "npm run --silent build && npm run --silent test", "build_js": "node tools/perspective-scripts/build_js.mjs", "build": "node tools/perspective-scripts/build.mjs", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bfbe0d72c5..c20a0376a5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -876,6 +876,9 @@ importers: specifier: workspace:* version: link:../../packages/perspective-jupyterlab devDependencies: + '@finos/perspective-scripts': + specifier: workspace:* + version: link:../../tools/perspective-scripts cpy: specifier: ^9.0.1 version: 9.0.1 diff --git a/rust/perspective-python/package.json b/rust/perspective-python/package.json index e76c8c0054..2d3c3146eb 100644 --- a/rust/perspective-python/package.json +++ b/rust/perspective-python/package.json @@ -12,9 +12,10 @@ "scripts": { "build": "node build.mjs", "clean": "rimraf perspective.data && rimraf dist && rimraf build", - "test": "python -m pytest perspective/tests" + "test": "node test.mjs" }, "devDependencies": { + "@finos/perspective-scripts": "workspace:*", "cpy": "^9.0.1" }, "dependencies": { diff --git a/rust/perspective-python/perspective/__init__.py b/rust/perspective-python/perspective/__init__.py index 7e64f2824f..e28741bfd0 100644 --- a/rust/perspective-python/perspective/__init__.py +++ b/rust/perspective-python/perspective/__init__.py @@ -33,8 +33,14 @@ PySyncServer, ) -from .widget import PerspectiveWidget -from .viewer import PerspectiveViewer +try: + from .widget import PerspectiveWidget +except ImportError: + ... +try: + from .viewer import PerspectiveViewer +except ImportError: + ... # from .psp_cffi import ServerBase diff --git a/rust/perspective-python/perspective/widget/__init__.py b/rust/perspective-python/perspective/widget/__init__.py index 0eba0dde2f..c539618c4a 100644 --- a/rust/perspective-python/perspective/widget/__init__.py +++ b/rust/perspective-python/perspective/widget/__init__.py @@ -10,7 +10,9 @@ # ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ # ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -from .widget import PerspectiveWidget, set_jupyter_html_export +try: + from .widget import PerspectiveWidget, set_jupyter_html_export - -__all__ = ["PerspectiveWidget", "set_jupyter_html_export"] + __all__ = ["PerspectiveWidget", "set_jupyter_html_export"] +except ImportError: + pass diff --git a/rust/perspective-python/perspective/widget/widget.py b/rust/perspective-python/perspective/widget/widget.py index c0c18dc455..4f762a8125 100644 --- a/rust/perspective-python/perspective/widget/widget.py +++ b/rust/perspective-python/perspective/widget/widget.py @@ -11,7 +11,6 @@ # ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ import base64 -import jinja2 import logging import os import re @@ -233,6 +232,7 @@ def _repr_mimebundle_(self, **kwargs): viewer_attrs = self.save() data = self.table.view().to_arrow() b64_data = base64.encodebytes(data) + import jinja2 jinja_env = jinja2.Environment( loader=jinja2.PackageLoader("perspective"), diff --git a/rust/perspective-python/pyodide-tests/README.md b/rust/perspective-python/pyodide-tests/README.md new file mode 100644 index 0000000000..4a748f230e --- /dev/null +++ b/rust/perspective-python/pyodide-tests/README.md @@ -0,0 +1,5 @@ +# pyodide-tests + +Smoke and integration tests for the perspective-python Pyodide wheel. + +These tests require that a Pyodide wheel has been built to rust/target/wheels diff --git a/rust/perspective-python/pyodide-tests/tests/__init__.py b/rust/perspective-python/pyodide-tests/tests/__init__.py new file mode 100644 index 0000000000..284e70816f --- /dev/null +++ b/rust/perspective-python/pyodide-tests/tests/__init__.py @@ -0,0 +1,11 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ diff --git a/rust/perspective-python/pyodide-tests/tests/conftest.py b/rust/perspective-python/pyodide-tests/tests/conftest.py new file mode 100644 index 0000000000..6998f30656 --- /dev/null +++ b/rust/perspective-python/pyodide-tests/tests/conftest.py @@ -0,0 +1,19 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + + +def pytest_addoption(parser): + parser.addoption( + "--perspective-emscripten-wheel", + action="store", + help="path to emscripten wheel", + ) diff --git a/rust/perspective-python/pyodide-tests/tests/test_smoke.py b/rust/perspective-python/pyodide-tests/tests/test_smoke.py new file mode 100644 index 0000000000..bdec58da15 --- /dev/null +++ b/rust/perspective-python/pyodide-tests/tests/test_smoke.py @@ -0,0 +1,74 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +# The regular Python test suite doesn't work in pytest-pyodide --run-in-pyodide, +# for several reasons. +# One: https://github.com/pyodide/pytest-pyodide/issues/81 + +from pytest_pyodide import run_in_pyodide, spawn_web_server +import pytest + + +@pytest.fixture(scope="session") +def psp_wheel_path(pytestconfig): + wheel = pytestconfig.getoption("--perspective-emscripten-wheel") + if wheel is None: + raise RuntimeError( + "pytest option --perspective-emscripten-wheel= is required but wasn't specified" + ) + return wheel + + +# Based on micropip's test server fixture: +# https://github.com/pyodide/micropip/blob/eb8c4497d742e515d24d532db2b9cc014328265b/tests/conftest.py#L64-L87 +@pytest.fixture() +def psp_wheel_url(psp_wheel_path): + from pathlib import Path + + wheel = Path(psp_wheel_path) + wheels_dir = wheel.parent + with spawn_web_server(wheels_dir) as server: + host, port, _ = server + wheel_url = f"http://{host}:{port}/{wheel.name}" + yield wheel_url + + +@pytest.fixture(autouse=True) +@run_in_pyodide(packages=["micropip"]) +async def psp_installed(selenium, psp_wheel_url): + """Installs perspective wheel from rust/target/wheels dir using micropip""" + # Autoused, so every test has perspective installed without them explicitly listing it as a fixture + import micropip + + await micropip.install(psp_wheel_url) + + +@run_in_pyodide +async def test_parsing_bad_csv_raises_exception(selenium): + import pytest + import perspective + + server = perspective.Server() + client = server.new_local_client() + with pytest.raises(perspective.PerspectiveError) as exc_info: + client.table("a,b,c\n1,2") + assert exc_info.match("CSV parse error") + + +@run_in_pyodide +async def test_parsing_good_csv(selenium): + import perspective + + server = perspective.Server() + client = server.new_local_client() + abc123 = client.table("a,b,c\n1,2,3\n") + assert abc123.columns() == ["a", "b", "c"] diff --git a/rust/perspective-python/pyproject.toml b/rust/perspective-python/pyproject.toml index 691a2c2758..c8ee5b6f4b 100644 --- a/rust/perspective-python/pyproject.toml +++ b/rust/perspective-python/pyproject.toml @@ -33,7 +33,10 @@ classifiers = [ # "Framework :: Jupyter :: JupyterLab :: Extensions :: Mime Renderers", "Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt", ] -dependencies = ["Jinja2>=2.0,<4", "ipywidgets>=7.5.1,<=8.1.3", "build>=1.2.1"] +dependencies = [] +[project.optional-dependencies] +jupyter = ["Jinja2>=2.0,<4", "ipywidgets>=7.5.1,<=8.1.3"] + [tool.maturin] module-name = "perspective" diff --git a/rust/perspective-python/requirements-pyodide.txt b/rust/perspective-python/requirements-pyodide.txt new file mode 100644 index 0000000000..5c2dd70ceb --- /dev/null +++ b/rust/perspective-python/requirements-pyodide.txt @@ -0,0 +1,5 @@ +# Requirements for testing pyodide +pytest==8.2.2 +pytest-playwright==0.5.2 +pytest-pyodide==0.58.3 + diff --git a/rust/perspective-python/requirements.txt b/rust/perspective-python/requirements.txt index 083009f67c..7ec3780b56 100644 --- a/rust/perspective-python/requirements.txt +++ b/rust/perspective-python/requirements.txt @@ -16,4 +16,4 @@ ruff==0.5.0 tornado==6.4.1 traitlets==5.14.3 widgetsnbextension==4.0.11 -wheel==0.43.0 \ No newline at end of file +wheel==0.43.0 diff --git a/rust/perspective-python/test.mjs b/rust/perspective-python/test.mjs new file mode 100644 index 0000000000..146c7b0e89 --- /dev/null +++ b/rust/perspective-python/test.mjs @@ -0,0 +1,49 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import { getPyodideDistDir } from "@finos/perspective-scripts/pyodide.mjs"; +import { getEmscriptenWheelPath } from "@finos/perspective-scripts/workspace.mjs"; + +// avoid executing this script directly, instead run `pnpm run test` from the workspace root + +const execOpts = { stdio: "inherit" }; +if (process.env.PSP_PYODIDE) { + const pyodideDistDir = getPyodideDistDir(); + if (!fs.existsSync(pyodideDistDir)) { + console.error( + `Error: Pyodide distribution not found at ${pyodideDistDir}\n\nRun: pnpm -w run install_pyodide\n\n` + ); + process.exit(1); + } + const emscriptenWheel = getEmscriptenWheelPath(); + if (!fs.existsSync(emscriptenWheel)) { + console.error( + `Error: Emscripten wheel not found at ${emscriptenWheel}\n\nRun: pnpm run build\n\n` + ); + process.exit(1); + } + execFileSync( + "pytest", + [ + "pyodide-tests/", + "--runner=playwright", + "--runtime=chrome", + `--dist-dir=${pyodideDistDir}`, + `--perspective-emscripten-wheel=${emscriptenWheel}`, + ], + execOpts + ); +} else { + execFileSync("pytest", ["perspective/tests"], execOpts); +} diff --git a/tools/perspective-scripts/install_pyodide.mjs b/tools/perspective-scripts/install_pyodide.mjs new file mode 100644 index 0000000000..d21493044e --- /dev/null +++ b/tools/perspective-scripts/install_pyodide.mjs @@ -0,0 +1,57 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +// Download a full Pyodide distribution from github and extract it to rust/target dir for +// use in integration tests + +import { getPyodideVersion, getPyodideDownloadDir } from "./pyodide.mjs"; + +import assert from "node:assert"; +import fs from "node:fs"; +import path from "node:path"; + +import { execFileSync } from "node:child_process"; +const pyodideVersion = getPyodideVersion(); + +function downloadPyodide() { + const pyodideUrl = `https://github.com/pyodide/pyodide/releases/download/${pyodideVersion}/pyodide-${pyodideVersion}.tar.bz2`; + const downloadDir = getPyodideDownloadDir(); // the download dir is versioned + const tarball = path.join(downloadDir, `pyodide-${pyodideVersion}.tar.bz2`); + const buildStamp = path.join(downloadDir, "psp-build-stamp.txt"); + const pyodideLock = path.join(downloadDir, "pyodide", "pyodide-lock.json"); + if (fs.existsSync(buildStamp) && fs.existsSync(pyodideLock)) { + console.log( + `Pyodide ${pyodideVersion} already extracted to ${downloadDir}` + ); + } else { + fs.rmSync(buildStamp, { force: true }); + console.log( + `Downloading Pyodide ${pyodideVersion} from ${pyodideUrl}...` + ); + fs.mkdirSync(downloadDir, { recursive: true }); + execFileSync("wget", ["-O", tarball, pyodideUrl], { + stdio: "inherit", + }); + console.log(`Extracting ${tarball}...`); + execFileSync("tar", ["-xvf", tarball, "-C", downloadDir], { + stdio: "inherit", + }); + console.log(`Removing ${tarball}...`); + fs.rmSync(tarball); + // assert presence of a known file + assert(fs.existsSync(pyodideLock), `${pyodideLock} not found`); + console.log(`Extracted to ${downloadDir}`); + fs.writeFileSync(buildStamp, ""); // prevent re-download/extract + } +} + +downloadPyodide(); diff --git a/tools/perspective-scripts/pyodide.mjs b/tools/perspective-scripts/pyodide.mjs new file mode 100644 index 0000000000..604cb63acc --- /dev/null +++ b/tools/perspective-scripts/pyodide.mjs @@ -0,0 +1,41 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +// Pyodide information: +// - Pyodide version we build against +// - Pyodide dist directory + +import path from "node:path"; + +import { getWorkspacePackageJson, getRustTargetDir } from "./workspace.mjs"; + +export function getPyodideVersion() { + const pyodideVersion = getWorkspacePackageJson().pyodide; + if (!pyodideVersion) { + throw new Error(`"pyodide" not set in package.json`); + } + return pyodideVersion; +} + +/** + * @returns pyodide download directory for the current version + */ +export function getPyodideDownloadDir() { + return path.join(getRustTargetDir(), "pyodide", getPyodideVersion()); +} + +/** + * @returns pyodide dist directory for the current version + */ +export function getPyodideDistDir() { + return path.join(getPyodideDownloadDir(), "pyodide"); +} diff --git a/tools/perspective-scripts/workspace.mjs b/tools/perspective-scripts/workspace.mjs new file mode 100644 index 0000000000..5af914d4b4 --- /dev/null +++ b/tools/perspective-scripts/workspace.mjs @@ -0,0 +1,53 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import fs from "node:fs"; +import url from "node:url"; +import path from "node:path"; + +const __dirname = url.fileURLToPath(new URL(".", import.meta.url)).slice(0, -1); +const workspaceRoot = path.normalize(path.join(__dirname, "..", "..")); +const rustTargetDir = path.join(workspaceRoot, "rust", "target"); +const rustWheelsDir = path.join(rustTargetDir, "wheels"); + +const memoize = (f) => { + let val = undefined; + return () => { + if (typeof val !== "undefined") return val; + val = f(); + return val; + }; +}; + +export function getWorkspaceRoot() { + return workspaceRoot; +} +export function getRustTargetDir() { + return rustTargetDir; +} +export function getRustWheelsDir() { + return rustWheelsDir; +} +export function getEmscriptenWheelPath() { + const pspVersion = getWorkspacePackageJson().version; + const wheeljunk = "cp39-abi3-emscripten_3_1_58_wasm32"; + return path.join( + rustWheelsDir, + `perspective_python-${pspVersion}-${wheeljunk}.whl` + ); +} +/** + * @returns memoized, deserialized contents of workspace package.json + */ +export const getWorkspacePackageJson = memoize(() => + JSON.parse(fs.readFileSync(path.join(workspaceRoot, "package.json"))) +); From 13faa99600cf49678833b801b9be51015021cf1f Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Mon, 9 Sep 2024 16:13:56 -0700 Subject: [PATCH 08/11] Re-enable building emscripten wheel off-tag This makes the emscripten wheel build on normal PRs Signed-off-by: Tom Jakubowski --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 074004abbd..da0f30b13c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -290,7 +290,7 @@ jobs: # `-' build_emscripten_wheel: runs-on: ${{ matrix.os }} - if: ${{ startsWith(github.ref, 'refs/tags/v') }} + # if: ${{ startsWith(github.ref, 'refs/tags/v') }} strategy: fail-fast: false matrix: From 9cc59604824ed57402e17c5b727bd0c001d6d3d2 Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Fri, 6 Sep 2024 13:23:28 -0700 Subject: [PATCH 09/11] Replace old-style import asserts for node compat This `import assert` syntax is experimental, and was removed in newer Node versions in favor of with `import ... with`. some node versions (including Node 18) support both syntaxes Unsure what minimum node version we need to support - if we can restrict to Node 18+, then we could instead replace this with `import ... with` Signed-off-by: Tom Jakubowski --- packages/perspective-jupyterlab/test/jupyter/utils.mjs | 7 ++++++- rust/perspective-python/build.mjs | 4 +++- tools/perspective-scripts/install_emsdk.mjs | 7 +++---- tools/perspective-scripts/install_llvm.mjs | 7 +++---- tools/perspective-scripts/repack_wheel.mjs | 4 +++- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/perspective-jupyterlab/test/jupyter/utils.mjs b/packages/perspective-jupyterlab/test/jupyter/utils.mjs index d314bdd3e0..c315da0d8d 100644 --- a/packages/perspective-jupyterlab/test/jupyter/utils.mjs +++ b/packages/perspective-jupyterlab/test/jupyter/utils.mjs @@ -14,11 +14,16 @@ import { test, expect } from "@playwright/test"; import fs from "fs"; import path from "path"; import rimraf from "rimraf"; -import notebook_template from "./notebook_template.json" assert { type: "json" }; import { fileURLToPath } from "url"; import { dirname } from "path"; const __dirname = dirname(fileURLToPath(import.meta.url)); +const notebook_template = JSON.parse( + fs.readFileSync(__dirname + "/notebook_template.json", { + encoding: "utf-8", + }) +); + const DIST_ROOT = path.join(__dirname, "..", "..", "dist", "esm"); const TEST_CONFIG_ROOT = path.join(__dirname, "..", "config", "jupyter"); diff --git a/rust/perspective-python/build.mjs b/rust/perspective-python/build.mjs index e399569f8e..f0c6e09be4 100644 --- a/rust/perspective-python/build.mjs +++ b/rust/perspective-python/build.mjs @@ -11,11 +11,13 @@ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ import * as fs from "node:fs"; -import pkg from "./package.json" assert { type: "json" }; import sh from "../../tools/perspective-scripts/sh.mjs"; import * as url from "url"; const __dirname = url.fileURLToPath(new URL(".", import.meta.url)).slice(0, -1); +const pkg = JSON.parse( + fs.readFileSync(__dirname + "/package.json", { encoding: "utf-8" }) +); let flags = "--release"; if (!!process.env.PSP_DEBUG) { diff --git a/tools/perspective-scripts/install_emsdk.mjs b/tools/perspective-scripts/install_emsdk.mjs index b20d98612c..74a9be32dc 100644 --- a/tools/perspective-scripts/install_emsdk.mjs +++ b/tools/perspective-scripts/install_emsdk.mjs @@ -15,10 +15,9 @@ import os from "os"; import fs from "fs"; import * as dotenv from "dotenv"; import sh from "./sh.mjs"; -import * as url from "url"; +import { getWorkspaceRoot, getWorkspacePackageJson } from "./workspace.mjs"; -const __dirname = url.fileURLToPath(new URL(".", import.meta.url)).slice(0, -1); -const pkg = JSON.parse(fs.readFileSync(`${__dirname}/../../package.json`)); +const pkg = getWorkspacePackageJson(); const emscripten = pkg.emscripten; @@ -27,7 +26,7 @@ dotenv.config({ }); function base() { - return sh.path`${__dirname}/../../.emsdk`; + return sh.path`${getWorkspaceRoot()}/.emsdk`; } function emsdk_checkout() { diff --git a/tools/perspective-scripts/install_llvm.mjs b/tools/perspective-scripts/install_llvm.mjs index 08ffdf99e4..3b5b58d5d4 100644 --- a/tools/perspective-scripts/install_llvm.mjs +++ b/tools/perspective-scripts/install_llvm.mjs @@ -10,18 +10,17 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -import pkg from "../../package.json" assert { type: "json" }; import os from "os"; import * as fs from "node:fs"; -import * as url from "url"; import { mkdirSync } from "fs"; import { execSync } from "child_process"; import path from "path"; +import { getWorkspaceRoot, getWorkspacePackageJson } from "./workspace.mjs"; -const __dirname = url.fileURLToPath(new URL(".", import.meta.url)).slice(0, -1); +const pkg = getWorkspacePackageJson(); const LLVM_VERSION = pkg.llvm; -const DOWNLOAD_DIR = path.join(`${__dirname}/../../.llvm`, "llvm-toolchain"); +const DOWNLOAD_DIR = path.join(getWorkspaceRoot(), ".llvm", "llvm-toolchain"); function getLLVMPackageName() { const system = os.platform(); diff --git a/tools/perspective-scripts/repack_wheel.mjs b/tools/perspective-scripts/repack_wheel.mjs index 0b0e3df40a..843ee73e3b 100644 --- a/tools/perspective-scripts/repack_wheel.mjs +++ b/tools/perspective-scripts/repack_wheel.mjs @@ -12,7 +12,9 @@ import { execSync } from "node:child_process"; import * as fs from "node:fs"; -import pkg from "../../package.json" assert { type: "json" }; +import { getWorkspacePackageJson } from "./workspace.mjs"; + +const pkg = getWorkspacePackageJson(); const wheel_file = fs.readdirSync(".").filter((x) => x.endsWith(".whl"))[0]; execSync(`wheel unpack ${wheel_file}`); From 25158fd48079e7eb9e522a8fba2da066761326ad Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Fri, 6 Sep 2024 13:04:51 -0700 Subject: [PATCH 10/11] Add vscode snippets for copyright notices Signed-off-by: Tom Jakubowski --- .gitignore | 1 + .vscode/perspective.code-snippets | 52 +++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 .vscode/perspective.code-snippets diff --git a/.gitignore b/.gitignore index 334f190766..924a7fba98 100644 --- a/.gitignore +++ b/.gitignore @@ -126,6 +126,7 @@ target.vscode !.vscode/extensions.json !.vscode/settings.default.json !.vscode/tasks.json +!.vscode/perspective.code-snippets .llvm diff --git a/.vscode/perspective.code-snippets b/.vscode/perspective.code-snippets new file mode 100644 index 0000000000..b9fb383e33 --- /dev/null +++ b/.vscode/perspective.code-snippets @@ -0,0 +1,52 @@ +{ + "perspective authors copyright notice: #-comment": { + "scope": "python,toml,yaml", + "body": [ + "# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓", + "# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃", + "# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃", + "# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃", + "# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃", + "# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫", + "# ┃ Copyright (c) 2017, the Perspective Authors. ┃", + "# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃", + "# ┃ This file is part of the Perspective library, distributed under the terms ┃", + "# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃", + "# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛" + ], + "prefix": "# copyright" + }, + "perspective authors copyright notice: //-comment": { + "scope": "javascript,cpp,c,typescript,less,rust,proto3", + "body": [ + "// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓", + "// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃", + "// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃", + "// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃", + "// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃", + "// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫", + "// ┃ Copyright (c) 2017, the Perspective Authors. ┃", + "// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃", + "// ┃ This file is part of the Perspective library, distributed under the terms ┃", + "// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃", + "// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛" + ], + "prefix": "// copyright" + }, + // Place your perspective workspace snippets here. Each snippet is defined under a snippet name and has a scope, prefix, body and + // description. Add comma separated ids of the languages where the snippet is applicable in the scope field. If scope + // is left empty or omitted, the snippet gets applied to all languages. The prefix is what is + // used to trigger the snippet and the body will be expanded and inserted. Possible variables are: + // $1, $2 for tab stops, $0 for the final cursor position, and ${1:label}, ${2:another} for placeholders. + // Placeholders with the same ids are connected. + // Example: + // "Print to console": { + // "scope": "javascript,typescript", + // "prefix": "log", + // "body": [ + // "console.log('$1');", + // "$2" + // ], + // "description": "Log output to console" + // } +} From 45c8af02f5eb9d039a2a29de11394e536a735c3d Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Fri, 6 Sep 2024 18:46:28 -0700 Subject: [PATCH 11/11] OOOOOOHOOW DDD (Remove accidentally included debug log) Signed-off-by: Tom Jakubowski --- cpp/perspective/src/cpp/arrow_csv.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/perspective/src/cpp/arrow_csv.cpp b/cpp/perspective/src/cpp/arrow_csv.cpp index 81eeadae0c..3a0480badc 100644 --- a/cpp/perspective/src/cpp/arrow_csv.cpp +++ b/cpp/perspective/src/cpp/arrow_csv.cpp @@ -387,7 +387,6 @@ class CustomISO8601Parser : public arrow::TimestampParser { if (length == 29) { // YYYY-MM-DD[ T]hh:mm:ss.sssssssss -- nanos // arrow handles YYYY-MM-DD[ T]hh:mm:ss.sss[+-]HH:MM - std::cout << "DDD WOOHOOOOO!\n"; arrow_vendored::date::year_month_day ymd; if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) { return false; @@ -645,4 +644,4 @@ csvToTable( return *maybe_table; } -} // namespace perspective::apachearrow \ No newline at end of file +} // namespace perspective::apachearrow