Added ability to specify additional Spark config (#37)

src/corvus_python/pyspark/utilities/session_utils/local_spark_session.py

@@ -32,6 +32,7 @@ class LocalSparkSessionConfig():
             Azure CLI credentials.
         enable_spark_ui (bool, optional): Whether to enable the Spark UI. Defaults to True. It can be useful to turn
             the UI off for unit tests to avoid port binding issues.
+        additional_spark_config (dict, optional): Additional Spark configuration to set. Defaults to an empty dict.
     """
     workload_name: str
     storage_configuration: StorageConfiguration = LocalFileSystemStorageConfiguration(os.path.join(CWD, "data"))
@@ -42,6 +43,7 @@ class LocalSparkSessionConfig():
     enable_az_cli_auth: bool = False
     enable_spark_ui: bool = True
     extra_packages: list = field(default_factory=lambda: [])
+    additional_spark_config: dict = field(default_factory=lambda: {})
 
 
 @dataclass
@@ -100,6 +102,10 @@ def create_spark_session(self) -> SparkSession:
                 .config("spark.driver.extraJavaOptions", f"-Dderby.system.home={hive_metastore_dir}") \
                 .enableHiveSupport()
 
+        if self.config.additional_spark_config:
+            for k, v in self.config.additional_spark_config.items():
+                builder = builder.config(k, v)
+
         spark = configure_spark_with_delta_pip(builder, extra_packages=extra_packages).getOrCreate()
 
         if self.config.enable_az_cli_auth: