fix(sandbox): clear sandbox store inside a block (#4420)

node/cli/src/cli.rs

@@ -53,6 +53,13 @@ pub struct RunCmd {
     #[arg(long, default_value_t = SandboxBackend::Wasmer)]
     pub sandbox_backend: SandboxBackend,
 
+    /// Sets a limit at which the underlying sandbox store will be cleared (applies only to the Wasmer sandbox backend),
+    /// potentially altering performance characteristics.
+    ///
+    /// See https://github.com/gear-tech/gear/pull/4420 for more context.
+    #[arg(long, default_value_t = 50)]
+    pub sandbox_store_clear_counter_limit: u32,
+
     /// The upper limit for the amount of gas a validator can burn in one block.
     #[arg(long)]
     pub max_gas: Option<u64>,

node/cli/src/command.rs

@@ -133,10 +133,13 @@ macro_rules! unwrap_client {
 pub fn run() -> sc_cli::Result<()> {
     let cli = Cli::from_args();
 
-    gear_runtime_interface::sandbox_init(match cli.run.sandbox_backend {
-        SandboxBackend::Wasmer => gear_runtime_interface::SandboxBackend::Wasmer,
-        SandboxBackend::Wasmi => gear_runtime_interface::SandboxBackend::Wasmi,
-    });
+    gear_runtime_interface::sandbox_init(
+        match cli.run.sandbox_backend {
+            SandboxBackend::Wasmer => gear_runtime_interface::SandboxBackend::Wasmer,
+            SandboxBackend::Wasmi => gear_runtime_interface::SandboxBackend::Wasmi,
+        },
+        cli.run.sandbox_store_clear_counter_limit.into(),
+    );
 
     let old_base = BasePath::from_project("", "", "gear-node");
     let new_base = BasePath::from_project("", "", &Cli::executable_name());

runtime-interface/sandbox/src/detail.rs

@@ -16,7 +16,10 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use core::{cell::RefCell, sync::atomic::Ordering};
+use core::{
+    cell::RefCell,
+    sync::atomic::{AtomicU32, Ordering},
+};
 
 use codec::{Decode, Encode};
 use gear_sandbox_host::sandbox::{self as sandbox_env, env::Instantiate};
@@ -49,10 +52,9 @@ impl Sandboxes {
     }
 
     // Clears the underlying store if the counter exceeds the limit.
-    #[cfg(feature = "runtime-benchmarks")]
     pub fn clear(&mut self) {
-        BENCH_SANDBOX_RESET_COUNTER.with_borrow_mut(|c| {
-            if *c >= BENCH_SANDBOX_RESET_COUNTER_LIMIT {
+        SANDBOX_STORE_CLEAR_COUNTER.with_borrow_mut(|c| {
+            if *c >= SANDBOX_STORE_CLEAR_COUNTER_LIMIT.load(Ordering::SeqCst) {
                 *c = 0;
                 self.store.clear();
             }
@@ -74,8 +76,13 @@ thread_local! {
 
 /// Sets the global sandbox backend type.
 /// Buy default, it's set to `Wasmer`, so in case of `Wasmer` it's not necessary to call this function.
-pub fn init(sandbox_backend: sandbox_env::SandboxBackend) {
+/// Also sets the store clear counter limit, which is used to clear the store after reaching a certain limit.
+pub fn init(sandbox_backend: sandbox_env::SandboxBackend, store_clear_counter_limit: Option<u32>) {
     SANDBOX_BACKEND_TYPE.store(sandbox_backend, Ordering::SeqCst);
+    SANDBOX_STORE_CLEAR_COUNTER_LIMIT.store(
+        store_clear_counter_limit.unwrap_or(DEFAULT_SANDBOX_STORE_CLEAR_COUNTER_LIMIT),
+        Ordering::SeqCst,
+    );
 }
 
 struct SupervisorContext<'a, 'b> {
@@ -471,12 +478,15 @@ pub fn memory_new(context: &mut dyn FunctionContext, initial: u32, maximum: u32)
 
         let data_ptr: *const _ = caller.data();
         method_result = SANDBOXES.with(|sandboxes| {
-            // The usual method to clear the store doesn't work for benchmarks (see `Sanboxes::get`).
-            // This issue is more prominent in so-called "onetime syscall" benchmarks because
-            // they run with minimal steps and a large number of repeats, leading to significant slowdowns.
-            // Therefore, we have to clear it manually if the `BENCH_SANDBOX_RESET_COUNTER` exceeds the limit.
-            // Otherwise, the store becomes too big and will slow down the benchmarks.
-            #[cfg(feature = "runtime-benchmarks")]
+            // HACK: It was discovered that starting with version 4.0, Wasmer experiences a slowdown
+            // when creating a large number of memory/instances beyond a certain threshold.
+            // The usual method to clear the store doesn't work for benchmarks (see `Sandboxes::get`)
+            // or when too many instances/memories are created **within a single block**, as the store
+            // is only cleared at the start of a new block.
+            // This is a temporary solution to reset the store after reaching a certain limit
+            // (see `SANDBOX_STORE_CLEAR_COUNTER_LIMIT`) for memory/instances.
+            // Otherwise, the store grows too large, leading to performance degradation during
+            // normal node execution and benchmarks.
             sandboxes.borrow_mut().clear();
 
             sandboxes
@@ -603,10 +613,11 @@ pub fn set_global_val(
     method_result
 }
 
-#[cfg(feature = "runtime-benchmarks")]
-const BENCH_SANDBOX_RESET_COUNTER_LIMIT: u32 = 100;
+const DEFAULT_SANDBOX_STORE_CLEAR_COUNTER_LIMIT: u32 = 50;
+
+static SANDBOX_STORE_CLEAR_COUNTER_LIMIT: AtomicU32 =
+    AtomicU32::new(DEFAULT_SANDBOX_STORE_CLEAR_COUNTER_LIMIT);
 
-#[cfg(feature = "runtime-benchmarks")]
 thread_local! {
-    static BENCH_SANDBOX_RESET_COUNTER: RefCell<u32> = const { RefCell::new(0) };
+    static SANDBOX_STORE_CLEAR_COUNTER: RefCell<u32> = const { RefCell::new(0) };
 }

sandbox/host/src/sandbox.rs

@@ -539,15 +539,9 @@ impl<DT: Clone> SandboxComponents<DT> {
 
     /// Clear instance list and memory list.
     pub fn clear(&mut self) {
-        log::trace!(
-            "clear; instances = {}",
-            self.instances.iter().any(|i| i.is_some())
-        );
+        log::trace!("clear; instances cnt = {}", self.instances.len());
         self.instances.clear();
-        log::trace!(
-            "clear; memories = {}",
-            self.memories.iter().any(|m| m.is_some())
-        );
+        log::trace!("clear; memories cnt = {}", self.memories.len());
         self.memories.clear();
 
         match self.backend_context {

utils/gear-replay-cli/src/cmd/mod.rs

@@ -63,7 +63,7 @@ pub enum Command {
 
 impl Command {
     pub async fn run(&self, shared: &SharedParams) -> sc_cli::Result<()> {
-        gear_runtime_interface::sandbox_init(gear_runtime_interface::SandboxBackend::Wasmer);
+        gear_runtime_interface::sandbox_init(gear_runtime_interface::SandboxBackend::Wasmer, None);
 
         match &self {
             Command::ReplayBlock(cmd) => {