From 2a204178fe32bcb2ac2ceb15b943910b30a644e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=9A=93?= Date: Wed, 8 Jan 2025 19:39:48 -0500 Subject: [PATCH] Implement stackful coroutine-based executor for libretro builds This executor has the advantage of being able to work correctly when there are Ruby stack frames underneath C/C++ stack frames in the stack. Still need to implement handling Ruby fibers. --- .github/workflows/autobuild.yml | 2 +- meson.build | 16 ++++- retro/sandbox-bindgen.rb | 98 +++++++++++++++++++------- src/core.cpp | 47 ++++++------ src/sandbox/sandbox.cpp | 14 ++-- src/sandbox/sandbox.h | 34 +++++---- subprojects/boost_any.wrap | 4 ++ subprojects/boost_assert.wrap | 4 ++ subprojects/boost_config.wrap | 4 ++ subprojects/boost_container_hash.wrap | 4 ++ subprojects/boost_core.wrap | 4 ++ subprojects/boost_describe.wrap | 4 ++ subprojects/boost_mp11.wrap | 4 ++ subprojects/boost_static_assert.wrap | 4 ++ subprojects/boost_throw_exception.wrap | 4 ++ subprojects/boost_type_index.wrap | 4 ++ 16 files changed, 177 insertions(+), 74 deletions(-) create mode 100644 subprojects/boost_any.wrap create mode 100644 subprojects/boost_assert.wrap create mode 100644 subprojects/boost_config.wrap create mode 100644 subprojects/boost_container_hash.wrap create mode 100644 subprojects/boost_core.wrap create mode 100644 subprojects/boost_describe.wrap create mode 100644 subprojects/boost_mp11.wrap create mode 100644 subprojects/boost_static_assert.wrap create mode 100644 subprojects/boost_throw_exception.wrap create mode 100644 subprojects/boost_type_index.wrap diff --git a/.github/workflows/autobuild.yml b/.github/workflows/autobuild.yml index ad3d78b10..c48fc9262 100644 --- a/.github/workflows/autobuild.yml +++ b/.github/workflows/autobuild.yml @@ -488,7 +488,7 @@ jobs: PATH="$HOMEBREW_PREFIX/opt/gpatch/libexec/gnubin:$PATH" meson setup build --buildtype release -Db_lto=true -Dretro=true -Dretro_phase1_path=retro/build/retro-phase1 cd build ninja -v - strip libretro-mkxp-z.dylib + strip -x libretro-mkxp-z.dylib mv libretro-mkxp-z.dylib ${{ runner.temp }}/retro-phase2 - uses: actions/upload-artifact@v4 diff --git a/meson.build b/meson.build index d8fa736dc..ad9f4341d 100644 --- a/meson.build +++ b/meson.build @@ -55,8 +55,8 @@ if get_option('retro') == true 'ENABLE_LIB_ONLY': true, }) - lzma_options = cmake.subproject_options() - lzma_options.add_cmake_defines({ + liblzma_options = cmake.subproject_options() + liblzma_options.add_cmake_defines({ 'CMAKE_POSITION_INDEPENDENT_CODE': true, 'BUILD_SHARED_LIBS': false, 'ENABLE_NLS': false, @@ -113,9 +113,19 @@ if get_option('retro') == true 'retro-' + meson.project_name(), dependencies: [ cmake.subproject('boost_asio', options: boost_options).dependency('boost_asio'), + cmake.subproject('boost_mp11', options: boost_options).dependency('boost_mp11'), + cmake.subproject('boost_describe', options: boost_options).dependency('boost_describe'), + cmake.subproject('boost_config', options: boost_options).dependency('boost_config'), + cmake.subproject('boost_assert', options: boost_options).dependency('boost_assert'), + cmake.subproject('boost_static_assert', options: boost_options).dependency('boost_static_assert'), + cmake.subproject('boost_throw_exception', options: boost_options).dependency('boost_throw_exception'), + cmake.subproject('boost_core', options: boost_options).dependency('boost_core'), + cmake.subproject('boost_container_hash', options: boost_options).dependency('boost_container_hash'), + cmake.subproject('boost_type_index', options: boost_options).dependency('boost_type_index'), + cmake.subproject('boost_any', options: boost_options).dependency('boost_any'), cmake.subproject('zlib', options: zlib_options).dependency('zlibstatic'), cmake.subproject('bzip2', options: bzip2_options).dependency('bz2_static'), - cmake.subproject('liblzma', options: lzma_options).dependency('liblzma'), + cmake.subproject('liblzma', options: liblzma_options).dependency('liblzma'), cmake.subproject('zstd', options: zstd_options).dependency('libzstd_static'), cmake.subproject('libzip', options: libzip_options).dependency('zip'), ], diff --git a/retro/sandbox-bindgen.rb b/retro/sandbox-bindgen.rb index f5343de11..302a3824d 100644 --- a/retro/sandbox-bindgen.rb +++ b/retro/sandbox-bindgen.rb @@ -151,11 +151,14 @@ #ifndef MKXP_SANDBOX_BINDGEN_H #define MKXP_SANDBOX_BINDGEN_H - + #include #include #include + #include + #include #include + #include #{MODULE_INCLUDE} #include "src/sandbox/types.h" @@ -168,20 +171,61 @@ typedef wasm_size_t VALUE; typedef wasm_size_t ID; - struct SandboxBind { - private: - wasm_ptr_t next_func_ptr; - std::shared_ptr instance; - wasm_ptr_t sbindgen_malloc(wasm_ptr_t); - wasm_ptr_t sbindgen_create_func_ptr(); + namespace mkxp_sandbox { + struct bindings { + private: + wasm_ptr_t next_func_ptr; + std::shared_ptr instance; + size_t depth; + std::vector stack; + wasm_ptr_t sbindgen_malloc(wasm_ptr_t); + wasm_ptr_t sbindgen_create_func_ptr(); + + public: + bindings(std::shared_ptr); + + template struct stack_frame { + friend struct bindings; + + private: + struct bindings &bindings; + T &inner; + static inline T &init(struct bindings &bindings) { + if (bindings.depth == bindings.stack.size()) { + bindings.stack.push_back(T(bindings)); + } else if (bindings.depth > bindings.stack.size()) { + throw SandboxTrapException(); + } + try { + return boost::any_cast(bindings.stack[bindings.depth++]); + } catch (boost::bad_any_cast &) { + throw SandboxTrapException(); + } + } + stack_frame(struct bindings &b) : bindings(b), inner(init(b)) {} + + public: + ~stack_frame() { + if (inner.is_complete()) { + bindings.stack.pop_back(); + } + --bindings.depth; + } + inline T &operator()() { + return inner; + } + }; - public: - SandboxBind(std::shared_ptr); + template struct stack_frame bind() { + return (struct stack_frame)(*this); + } HEREDOC HEADER_END = <<~HEREDOC + } + #endif // MKXP_SANDBOX_BINDGEN_H HEREDOC @@ -210,7 +254,6 @@ // Autogenerated by sandbox-bindgen.rb. Don't manually modify this file - modify sandbox-bindgen.rb instead! #include - #include #include "mkxp-sandbox-bindgen.h" #if WABT_BIG_ENDIAN @@ -223,10 +266,13 @@ #define SERIALIZE_PTR(value) SERIALIZE_#{MEMORY64 ? '64' : '32'}(value) - SandboxBind::SandboxBind(std::shared_ptr m) : next_func_ptr(-1), instance(m) {} + using namespace mkxp_sandbox; + + + bindings::bindings(std::shared_ptr m) : next_func_ptr(-1), instance(m), depth(0) {} - wasm_ptr_t SandboxBind::sbindgen_malloc(wasm_size_t size) { + wasm_ptr_t bindings::sbindgen_malloc(wasm_size_t size) { wasm_ptr_t buf = w2c_#{MODULE_NAME}_#{MALLOC_FUNC}(instance.get(), size); // Verify that the entire allocated buffer is in valid memory @@ -239,7 +285,7 @@ } - wasm_ptr_t SandboxBind::sbindgen_create_func_ptr() { + wasm_ptr_t bindings::sbindgen_create_func_ptr() { if (next_func_ptr == (wasm_ptr_t)-1) { next_func_ptr = instance->w2c_T0.size; } @@ -426,21 +472,19 @@ coroutine_vars.append("#{coroutine_ret} r") if handler[:primitive] != :void - coroutine_args = ['SandboxBind &bind'] - coroutine_args.append((0...args.length).map do |i| + coroutine_args = (0...args.length).map do |i| args[i] == '...' ? '...' : !ARG_HANDLERS[args[i]][:formatter].nil? ? ARG_HANDLERS[args[i]][:formatter].call("a#{i}") : !ARG_HANDLERS[args[i]][:keep] ? "#{VAR_TYPE_TABLE[ARG_HANDLERS[args[i]][:primitive]]} a#{i}" : "#{args[i]} a#{i}" - end) + end - declaration_args = ['SandboxBind &'] - declaration_args.append((0...args.length).map do |i| + declaration_args = (0...args.length).map do |i| args[i] == '...' ? '...' : !ARG_HANDLERS[args[i]][:formatter].nil? ? ARG_HANDLERS[args[i]][:formatter].call('') : !ARG_HANDLERS[args[i]][:keep] ? "#{VAR_TYPE_TABLE[ARG_HANDLERS[args[i]][:primitive]]}" : "#{args[i]}" - end) + end coroutine_inner = <<~HEREDOC #{handler[:primitive] == :void ? '' : 'r = '}w2c_#{MODULE_NAME}_#{func_name}(#{(['bind.instance.get()'] + (0...args.length).map { |i| args[i] == '...' || transformed_args.include?(i) ? "f#{i}" : "a#{i}" }).join(', ')}); @@ -451,6 +495,7 @@ coroutine_finalizer = (0...buffers.length).map { |i| "w2c_#{MODULE_NAME}_#{FREE_FUNC}(bind.instance.get(), #{buffers[buffers.length - 1 - i]});" } coroutine_definition = <<~HEREDOC + #{func_name}::#{func_name}(bindings &bind) : bind(bind) {} #{coroutine_ret} #{func_name}::operator()(#{coroutine_args.join(', ')}) {#{coroutine_vars.empty? ? '' : (coroutine_vars.map { |var| "\n #{var} = 0;" }.join + "\n")} reenter (this) { #{coroutine_initializer.empty? ? '' : (coroutine_initializer.split("\n").map { |line| " #{line}" }.join("\n") + "\n\n")} for (;;) { @@ -462,8 +507,13 @@ coroutine_declaration = <<~HEREDOC struct #{func_name} : boost::asio::coroutine { + friend struct bindings; + friend struct bindings::stack_frame; #{coroutine_ret} operator()(#{declaration_args.join(', ')}); - #{fields.empty? ? '' : (" private:\n" + fields.map { |field| " #{field};\n" }.join)}}; + private: + #{func_name}(bindings &bind); + bindings &bind; + #{fields.empty? ? '' : fields.map { |field| " #{field};\n" }.join}}; HEREDOC func_names.append(func_name) @@ -474,11 +524,11 @@ File.open('mkxp-sandbox-bindgen.h', 'w') do |file| file.write(HEADER_START) for func_name in func_names - file.write(" friend struct #{func_name};\n") + file.write(" friend struct #{func_name};\n") end - file.write("};\n") + file.write(" };\n") for declaration in declarations - file.write("\n" + declaration) + file.write("\n" + declaration.split("\n").map { |line| " #{line}" }.join("\n").rstrip) end file.write(HEADER_END) end @@ -486,6 +536,6 @@ file.write(PRELUDE) for coroutine in coroutines file.write("\n\n") - file.write(coroutine) + file.write(coroutine.rstrip) end end diff --git a/src/core.cpp b/src/core.cpp index 7c26bbc18..c3a5d511d 100644 --- a/src/core.cpp +++ b/src/core.cpp @@ -24,24 +24,30 @@ #include #include #include -#include -#include -#include "core.h" #include "sandbox/sandbox.h" +#include "core.h" -#define AWAIT(coroutine, ...) do { coroutine(__VA_ARGS__); if (coroutine.is_complete()) break; yield; } while (1) +#define SANDBOX_AWAIT(coroutine, ...) \ + do { \ + { \ + auto frame = sandbox->bindings.bind(); \ + frame()(__VA_ARGS__); \ + if (frame().is_complete()) break; \ + } \ + yield; \ + } while (1) using namespace mkxp_retro; static void fallback_log(enum retro_log_level level, const char *fmt, ...) { - va_list va; + std::va_list va; va_start(va, fmt); - vfprintf(stderr, fmt, va); + std::vfprintf(stderr, fmt, va); va_end(va); } static uint32_t *frame_buf; -static std::unique_ptr sandbox; +static std::unique_ptr sandbox; static const char *game_path = NULL; static VALUE my_cpp_func(w2c_ruby *ruby, int32_t argc, wasm_ptr_t argv, VALUE self) { @@ -50,24 +56,17 @@ static VALUE my_cpp_func(w2c_ruby *ruby, int32_t argc, wasm_ptr_t argv, VALUE se } static bool init_sandbox() { - struct runtime : boost::asio::coroutine { - struct rb_eval_string eval; - struct rb_define_global_function define; - + struct main : boost::asio::coroutine { void operator()() { reenter (this) { - AWAIT(eval, sandbox->bind, "puts 'Hello, World!'"); + SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "puts 'Hello, World!'"); - eval = rb_eval_string(); - AWAIT(eval, sandbox->bind, "require 'zlib'; p Zlib::Deflate::deflate('hello')"); + SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "require 'zlib'; p Zlib::Deflate::deflate('hello')"); - AWAIT(define, sandbox->bind, "my_cpp_func", (VALUE (*)(void *, ANYARGS))my_cpp_func, -1); + SANDBOX_AWAIT(mkxp_sandbox::rb_define_global_function, "my_cpp_func", (VALUE (*)(void *, ANYARGS))my_cpp_func, -1); + SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "my_cpp_func(1, nil, 3, 'this is a string', :symbol, 2)"); - eval = rb_eval_string(); - AWAIT(eval, sandbox->bind, "my_cpp_func(1, nil, 3, 'this is a string', :symbol, 2)"); - - eval = rb_eval_string(); - AWAIT(eval, sandbox->bind, "p Dir.glob '/mkxp-retro-game/*'"); + SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "p Dir.glob '/mkxp-retro-game/*'"); } } }; @@ -75,12 +74,8 @@ static bool init_sandbox() { sandbox.reset(); try { - sandbox.reset(new Sandbox(game_path)); - - struct runtime runtime; - - // TODO: Replace this loop with a stackful executor, otherwise you won't be able to call into the Ruby API from inside of a C/C++ function that is itself called from inside of Ruby. - do runtime(); while (w2c_ruby_mkxp_sandbox_yield(&sandbox->module_instance())); + sandbox.reset(new struct mkxp_sandbox::sandbox(game_path)); + sandbox->run(); } catch (SandboxException) { log_printf(RETRO_LOG_ERROR, "Failed to initialize Ruby\n"); sandbox.reset(); diff --git a/src/sandbox/sandbox.cpp b/src/sandbox/sandbox.cpp index dbe186a41..e28fb0a45 100644 --- a/src/sandbox/sandbox.cpp +++ b/src/sandbox/sandbox.cpp @@ -38,12 +38,14 @@ #define WASM_MEM(address) ((void *)&ruby->w2c_memory.data[address]) #define AWAIT(statement) do statement; while (w2c_ruby_mkxp_sandbox_yield(RB)) +using namespace mkxp_sandbox; + // This function is imported by wasm-rt-impl.c from wasm2c extern "C" void mkxp_sandbox_trap_handler(wasm_rt_trap_t code) { throw SandboxTrapException(); } -usize Sandbox::sandbox_malloc(usize size) { +usize sandbox::sandbox_malloc(usize size) { usize buf = w2c_ruby_mkxp_sandbox_malloc(RB, size); // Verify that the returned pointer is non-null and the entire allocated buffer is in valid memory @@ -55,11 +57,11 @@ usize Sandbox::sandbox_malloc(usize size) { return buf; } -void Sandbox::sandbox_free(usize ptr) { +void sandbox::sandbox_free(usize ptr) { w2c_ruby_mkxp_sandbox_free(RB, ptr); } -Sandbox::Sandbox(const char *game_path) : ruby(new struct w2c_ruby), wasi(new wasi_t(ruby, game_path)), bind(ruby) { +sandbox::sandbox(const char *game_path) : ruby(new struct w2c_ruby), wasi(new wasi_t(ruby, game_path)), bindings(ruby) { try { // Initialize the sandbox wasm_rt_init(); @@ -138,14 +140,10 @@ Sandbox::Sandbox(const char *game_path) : ruby(new struct w2c_ruby), wasi(new wa } } -Sandbox::~Sandbox() { +sandbox::~sandbox() { try { w2c_ruby_mkxp_sandbox_deinit(RB); } catch (SandboxTrapException) {} wasm2c_ruby_free(RB); wasm_rt_free(); } - -w2c_ruby &Sandbox::module_instance() { - return *ruby; -} diff --git a/src/sandbox/sandbox.h b/src/sandbox/sandbox.h index 3bdcc1465..bcd512bfb 100644 --- a/src/sandbox/sandbox.h +++ b/src/sandbox/sandbox.h @@ -26,19 +26,29 @@ #include #include "types.h" -struct Sandbox { - private: - std::shared_ptr ruby; - std::unique_ptr wasi; +namespace mkxp_sandbox { + struct sandbox { + private: + std::shared_ptr ruby; + std::unique_ptr wasi; + usize sandbox_malloc(usize size); + void sandbox_free(usize ptr); - usize sandbox_malloc(usize size); - void sandbox_free(usize ptr); + public: + struct mkxp_sandbox::bindings bindings; + sandbox(const char *game_path); + ~sandbox(); - public: - SandboxBind bind; - Sandbox(const char *game_path); - ~Sandbox(); - struct w2c_ruby &module_instance(); -}; + // TODO: handle Ruby fibers properly instead of crashing whenever Ruby switches to a different fiber than the main one + template inline void run() { + T coroutine = T(); + do { + coroutine(); + w2c_ruby_mkxp_sandbox_yield(ruby.get()); + } while (!coroutine.is_complete()); + + } + }; +} #endif // MKXPZ_SANDBOX_H diff --git a/subprojects/boost_any.wrap b/subprojects/boost_any.wrap new file mode 100644 index 000000000..030e6524a --- /dev/null +++ b/subprojects/boost_any.wrap @@ -0,0 +1,4 @@ +[wrap-git] +url = https://github.com/boostorg/any +revision = boost-1.87.0 +depth = 1 diff --git a/subprojects/boost_assert.wrap b/subprojects/boost_assert.wrap new file mode 100644 index 000000000..37ff42b96 --- /dev/null +++ b/subprojects/boost_assert.wrap @@ -0,0 +1,4 @@ +[wrap-git] +url = https://github.com/boostorg/assert +revision = boost-1.87.0 +depth = 1 diff --git a/subprojects/boost_config.wrap b/subprojects/boost_config.wrap new file mode 100644 index 000000000..73f6d1037 --- /dev/null +++ b/subprojects/boost_config.wrap @@ -0,0 +1,4 @@ +[wrap-git] +url = https://github.com/boostorg/config +revision = boost-1.87.0 +depth = 1 diff --git a/subprojects/boost_container_hash.wrap b/subprojects/boost_container_hash.wrap new file mode 100644 index 000000000..e2b3bab4d --- /dev/null +++ b/subprojects/boost_container_hash.wrap @@ -0,0 +1,4 @@ +[wrap-git] +url = https://github.com/boostorg/container_hash +revision = boost-1.87.0 +depth = 1 diff --git a/subprojects/boost_core.wrap b/subprojects/boost_core.wrap new file mode 100644 index 000000000..c3bacf901 --- /dev/null +++ b/subprojects/boost_core.wrap @@ -0,0 +1,4 @@ +[wrap-git] +url = https://github.com/boostorg/core +revision = boost-1.87.0 +depth = 1 diff --git a/subprojects/boost_describe.wrap b/subprojects/boost_describe.wrap new file mode 100644 index 000000000..8637ad26f --- /dev/null +++ b/subprojects/boost_describe.wrap @@ -0,0 +1,4 @@ +[wrap-git] +url = https://github.com/boostorg/describe +revision = boost-1.87.0 +depth = 1 diff --git a/subprojects/boost_mp11.wrap b/subprojects/boost_mp11.wrap new file mode 100644 index 000000000..b736cf779 --- /dev/null +++ b/subprojects/boost_mp11.wrap @@ -0,0 +1,4 @@ +[wrap-git] +url = https://github.com/boostorg/mp11 +revision = boost-1.87.0 +depth = 1 diff --git a/subprojects/boost_static_assert.wrap b/subprojects/boost_static_assert.wrap new file mode 100644 index 000000000..c05ae570e --- /dev/null +++ b/subprojects/boost_static_assert.wrap @@ -0,0 +1,4 @@ +[wrap-git] +url = https://github.com/boostorg/static_assert +revision = boost-1.87.0 +depth = 1 diff --git a/subprojects/boost_throw_exception.wrap b/subprojects/boost_throw_exception.wrap new file mode 100644 index 000000000..a13ff220b --- /dev/null +++ b/subprojects/boost_throw_exception.wrap @@ -0,0 +1,4 @@ +[wrap-git] +url = https://github.com/boostorg/throw_exception +revision = boost-1.87.0 +depth = 1 diff --git a/subprojects/boost_type_index.wrap b/subprojects/boost_type_index.wrap new file mode 100644 index 000000000..37a778f2c --- /dev/null +++ b/subprojects/boost_type_index.wrap @@ -0,0 +1,4 @@ +[wrap-git] +url = https://github.com/boostorg/type_index +revision = boost-1.87.0 +depth = 1