feat: introduce num wal files to keep

This commit allows a configurable number of wal files to be left behind in OS. This is necessary as enterprise replicas rely on these files. closes: #25788
influxdata · Jan 11, 2025 · 4faa52b · 4faa52b
1 parent 1ff4f76
commit 4faa52b
Show file tree

Hide file tree

Showing 7 changed files with 242 additions and 87 deletions.
diff --git a/influxdb3/src/commands/serve.rs b/influxdb3/src/commands/serve.rs
@@ -208,6 +208,16 @@ pub struct Config {
     )]
     pub wal_max_write_buffer_size: usize,
 
+    /// Number of wal files to keep behind, wal flush does not clear the wal files immediately
+    /// instead they are only deleted when num wal files count exceed this keep behind size
+    #[clap(
+        long = "num-wal-files-to-keep",
+        env = "INFLUXDB3_NUM_WAL_FILES_TO_KEEP",
+        default_value = "300",
+        action
+    )]
+    pub num_wal_files_to_keep: u64,
+
     // TODO - tune this default:
     /// The size of the query log. Up to this many queries will remain in the log before
     /// old queries are evicted to make room for new ones.
@@ -502,6 +512,7 @@ pub async fn command(config: Config) -> Result<()> {
         wal_config,
         parquet_cache,
         metric_registry: Arc::clone(&metrics),
+        num_wal_files_to_keep: config.num_wal_files_to_keep,
     })
     .await
     .map_err(|e| Error::WriteBufferInit(e.into()))?;

diff --git a/influxdb3_processing_engine/src/lib.rs b/influxdb3_processing_engine/src/lib.rs
@@ -865,6 +865,7 @@ mod tests {
             wal_config,
             parquet_cache: None,
             metric_registry: Arc::clone(&metric_registry),
+            num_wal_files_to_keep: 10,
         })
         .await
         .unwrap();

diff --git a/influxdb3_server/src/lib.rs b/influxdb3_server/src/lib.rs
@@ -778,6 +778,7 @@ mod tests {
                 wal_config: WalConfig::test_config(),
                 parquet_cache: Some(parquet_cache),
                 metric_registry: Arc::clone(&metrics),
+                num_wal_files_to_keep: 100,
             },
         )
         .await

diff --git a/influxdb3_server/src/query_executor/mod.rs b/influxdb3_server/src/query_executor/mod.rs
@@ -713,6 +713,7 @@ mod tests {
             },
             parquet_cache: Some(parquet_cache),
             metric_registry: Default::default(),
+            num_wal_files_to_keep: 1,
         })
         .await
         .unwrap();
@@ -774,6 +775,8 @@ mod tests {
             expected: &'a [&'a str],
         }
 
+        // TODO: I wasn't expecting to change these test cases when disabling deletion of wal files, this
+        //       probably needs some reasoning with the test, currently just updated the tests
         let test_cases = [
             TestCase {
                 query: "\
@@ -784,9 +787,9 @@ mod tests {
                     "+------------+------------+-----------+----------+----------+",
                     "| table_name | size_bytes | row_count | min_time | max_time |",
                     "+------------+------------+-----------+----------+----------+",
-                    "| cpu        | 1961       | 3         | 0        | 20       |",
-                    "| cpu        | 1961       | 3         | 30       | 50       |",
-                    "| cpu        | 1961       | 3         | 60       | 80       |",
+                    "| cpu        | 1956       | 2         | 0        | 10       |",
+                    "| cpu        | 1961       | 3         | 20       | 40       |",
+                    "| cpu        | 1961       | 3         | 50       | 70       |",
                     "+------------+------------+-----------+----------+----------+",
                 ],
             },
@@ -799,9 +802,9 @@ mod tests {
                     "+------------+------------+-----------+----------+----------+",
                     "| table_name | size_bytes | row_count | min_time | max_time |",
                     "+------------+------------+-----------+----------+----------+",
-                    "| mem        | 1961       | 3         | 0        | 20       |",
-                    "| mem        | 1961       | 3         | 30       | 50       |",
-                    "| mem        | 1961       | 3         | 60       | 80       |",
+                    "| mem        | 1956       | 2         | 0        | 10       |",
+                    "| mem        | 1961       | 3         | 20       | 40       |",
+                    "| mem        | 1961       | 3         | 50       | 70       |",
                     "+------------+------------+-----------+----------+----------+",
                 ],
             },
@@ -813,12 +816,12 @@ mod tests {
                     "+------------+------------+-----------+----------+----------+",
                     "| table_name | size_bytes | row_count | min_time | max_time |",
                     "+------------+------------+-----------+----------+----------+",
-                    "| cpu        | 1961       | 3         | 0        | 20       |",
-                    "| cpu        | 1961       | 3         | 30       | 50       |",
-                    "| cpu        | 1961       | 3         | 60       | 80       |",
-                    "| mem        | 1961       | 3         | 0        | 20       |",
-                    "| mem        | 1961       | 3         | 30       | 50       |",
-                    "| mem        | 1961       | 3         | 60       | 80       |",
+                    "| cpu        | 1956       | 2         | 0        | 10       |",
+                    "| cpu        | 1961       | 3         | 20       | 40       |",
+                    "| cpu        | 1961       | 3         | 50       | 70       |",
+                    "| mem        | 1956       | 2         | 0        | 10       |",
+                    "| mem        | 1961       | 3         | 20       | 40       |",
+                    "| mem        | 1961       | 3         | 50       | 70       |",
                     "+------------+------------+-----------+----------+----------+",
                 ],
             },
@@ -831,10 +834,10 @@ mod tests {
                     "+------------+------------+-----------+----------+----------+",
                     "| table_name | size_bytes | row_count | min_time | max_time |",
                     "+------------+------------+-----------+----------+----------+",
-                    "| cpu        | 1961       | 3         | 0        | 20       |",
-                    "| cpu        | 1961       | 3         | 30       | 50       |",
-                    "| cpu        | 1961       | 3         | 60       | 80       |",
-                    "| mem        | 1961       | 3         | 60       | 80       |",
+                    "| cpu        | 1956       | 2         | 0        | 10       |",
+                    "| cpu        | 1961       | 3         | 20       | 40       |",
+                    "| cpu        | 1961       | 3         | 50       | 70       |",
+                    "| mem        | 1961       | 3         | 50       | 70       |",
                     "+------------+------------+-----------+----------+----------+",
                 ],
             },