Add new option to allow disabling the Spark UI (#22)

* Add new option to allow disabling the Spark UI

* Prevent publishing PR builds

.github/workflows/build.yml

@@ -59,7 +59,7 @@ jobs:
       netSdkVersion: '8.0.x'
       # additionalNetSdkVersion: ''
       # workflow_dispatch inputs are always strings, the type property is just for the UI
-      forcePublish: ${{ github.event.inputs.forcePublish == 'true' || github.event_name == 'pull_request' }}
+      forcePublish: ${{ github.event.inputs.forcePublish == 'true' }}
       skipCleanup: ${{ github.event.inputs.skipCleanup == 'true' }}
       # testArtifactName: ''
       # testArtifactPath: ''

src/corvus_python/pyspark/utilities/session_utils/local_spark_session.py

@@ -30,6 +30,8 @@ class LocalSparkSessionConfig():
         enable_az_cli_auth (bool, optional): Whether to enable Azure CLI authentication. Defaults to False. If using
             Azure Data Lake Storage Gen 2, this should be set to True to enable authentication using your current
             Azure CLI credentials.
+        enable_spark_ui (bool, optional): Whether to enable the Spark UI. Defaults to True. It can be useful to turn
+            the UI off for unit tests to avoid port binding issues.
     """
     workload_name: str
     storage_configuration: StorageConfiguration = LocalFileSystemStorageConfiguration(os.path.join(CWD, "data"))
@@ -38,6 +40,7 @@ class LocalSparkSessionConfig():
     hive_metastore_dir: str = os.path.join(CWD, "metastore")
     install_hadoop_azure_package: bool = False
     enable_az_cli_auth: bool = False
+    enable_spark_ui: bool = True
 
 
 @dataclass
@@ -64,6 +67,9 @@ def create_spark_session(self) -> SparkSession:
             .config(
                 "spark.sql.catalog.spark_catalog",
                 "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+            .config(
+                "spark.ui.enabled",
+                "true" if self.config.enable_spark_ui else "false")
         )
 
         extra_packages = []