[Fix] Remove single-node validation from interactive clusters (#4222)

## Changes There were reports on databricks/cli#1546 about customers being unable to use cluster policies for single-node job clusters because Terraform was performing this validation. #4216 removed this validation for job clusters. This PR removes this validation for interactive clusters as well to keep them consistent and unblock the use of policy from interactive clusters. Also, the clusters team is improving the API interface to simplify creating single-node clusters, significantly reducing the chances of users misconfiguring them. ## Tests Unit test and manually. Clusters are now successfully created and updated with `num_workers=0`
databricks · Nov 14, 2024 · 6e7ca4c · 6e7ca4c
1 parent 21735c4
commit 6e7ca4c
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 92 deletions.
diff --git a/clusters/resource_cluster.go b/clusters/resource_cluster.go
@@ -26,26 +26,6 @@ var clusterSchema = resourceClusterSchema()
 var clusterSchemaVersion = 4
 
 const (
-	numWorkerErr = `num_workers may be 0 only for single-node clusters. To create a single node
-cluster please include the following configuration in your cluster configuration:
-
-  spark_conf = {
-    "spark.databricks.cluster.profile" : "singleNode"
-    "spark.master" : "local[*]"
-  }
-
-  custom_tags = {
-    "ResourceClass" = "SingleNode"
-  }
-
-Please note that the Databricks Terraform provider cannot detect if the above configuration
-is defined in a policy used by the cluster. Please define this in the cluster configuration
-itself to create a single node cluster.
-
-For more details please see:
-  1. https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/cluster#fixed-size-or-autoscaling-cluster
-  2. https://docs.databricks.com/clusters/single-node.html`
-
 	unsupportedExceptCreateEditClusterSpecErr = "unsupported type %T, must be one of %scompute.CreateCluster, %scompute.ClusterSpec or %scompute.EditCluster. Please report this issue to the GitHub repo"
 )
 
@@ -130,39 +110,6 @@ func ZoneDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
 	return false
 }
 
-func Validate(cluster any) error {
-	var profile, master, resourceClass string
-	switch c := cluster.(type) {
-	case compute.CreateCluster:
-		if c.NumWorkers > 0 || c.Autoscale != nil {
-			return nil
-		}
-		profile = c.SparkConf["spark.databricks.cluster.profile"]
-		master = c.SparkConf["spark.master"]
-		resourceClass = c.CustomTags["ResourceClass"]
-	case compute.EditCluster:
-		if c.NumWorkers > 0 || c.Autoscale != nil {
-			return nil
-		}
-		profile = c.SparkConf["spark.databricks.cluster.profile"]
-		master = c.SparkConf["spark.master"]
-		resourceClass = c.CustomTags["ResourceClass"]
-	case compute.ClusterSpec:
-		if c.NumWorkers > 0 || c.Autoscale != nil {
-			return nil
-		}
-		profile = c.SparkConf["spark.databricks.cluster.profile"]
-		master = c.SparkConf["spark.master"]
-		resourceClass = c.CustomTags["ResourceClass"]
-	default:
-		return fmt.Errorf(unsupportedExceptCreateEditClusterSpecErr, cluster, "", "", "")
-	}
-	if profile == "singleNode" && strings.HasPrefix(master, "local") && resourceClass == "SingleNode" {
-		return nil
-	}
-	return errors.New(numWorkerErr)
-}
-
 // This method is a duplicate of ModifyRequestOnInstancePool() in clusters/clusters_api.go that uses Go SDK.
 // Long term, ModifyRequestOnInstancePool() in clusters_api.go will be removed once all the resources using clusters are migrated to Go SDK.
 func ModifyRequestOnInstancePool(cluster any) error {
@@ -443,9 +390,6 @@ func resourceClusterCreate(ctx context.Context, d *schema.ResourceData, c *commo
 	clusters := w.Clusters
 	var createClusterRequest compute.CreateCluster
 	common.DataToStructPointer(d, clusterSchema, &createClusterRequest)
-	if err := Validate(createClusterRequest); err != nil {
-		return err
-	}
 	if err = ModifyRequestOnInstancePool(&createClusterRequest); err != nil {
 		return err
 	}
@@ -596,9 +540,6 @@ func resourceClusterUpdate(ctx context.Context, d *schema.ResourceData, c *commo
 
 	if hasClusterConfigChanged(d) {
 		log.Printf("[DEBUG] Cluster state has changed!")
-		if err := Validate(cluster); err != nil {
-			return err
-		}
 		if err = ModifyRequestOnInstancePool(&cluster); err != nil {
 			return err
 		}

diff --git a/clusters/resource_cluster_test.go b/clusters/resource_cluster_test.go
@@ -1630,22 +1630,6 @@ func TestResourceClusterCreate_SingleNode(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, 0, d.Get("num_workers"))
 }
-
-func TestResourceClusterCreate_SingleNodeFail(t *testing.T) {
-	_, err := qa.ResourceFixture{
-		Create:   true,
-		Resource: ResourceCluster(),
-		State: map[string]any{
-			"autotermination_minutes": 120,
-			"cluster_name":            "Single Node Cluster",
-			"spark_version":           "7.3.x-scala12",
-			"node_type_id":            "Standard_F4s",
-			"is_pinned":               false,
-		},
-	}.Apply(t)
-	assert.EqualError(t, err, numWorkerErr)
-}
-
 func TestResourceClusterCreate_NegativeNumWorkers(t *testing.T) {
 	_, err := qa.ResourceFixture{
 		Create:   true,
@@ -1662,27 +1646,59 @@ func TestResourceClusterCreate_NegativeNumWorkers(t *testing.T) {
 	require.Equal(t, true, strings.Contains(err.Error(), "expected num_workers to be at least (0)"))
 }
 
-func TestResourceClusterUpdate_FailNumWorkersZero(t *testing.T) {
-	_, err := qa.ResourceFixture{
-		ID:       "abc",
-		Update:   true,
-		Resource: ResourceCluster(),
-		InstanceState: map[string]string{
-			"autotermination_minutes": "15",
-			"cluster_name":            "Shared Autoscaling",
-			"spark_version":           "7.1-scala12",
-			"node_type_id":            "i3.xlarge",
-			"num_workers":             "100",
+func TestResourceClusterCreate_NumWorkersIsZero(t *testing.T) {
+	d, err := qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			nothingPinned,
+			{
+				Method:   "POST",
+				Resource: "/api/2.1/clusters/create",
+				ExpectedRequest: compute.CreateCluster{
+					NumWorkers:             0,
+					ClusterName:            "Zero workers cluster",
+					SparkVersion:           "7.3.x-scala12",
+					NodeTypeId:             "Standard_F4s",
+					AutoterminationMinutes: 120,
+					ForceSendFields:        []string{"NumWorkers"},
+				},
+				Response: compute.ClusterDetails{
+					ClusterId: "abc",
+					State:     compute.StateRunning,
+				},
+			},
+			{
+				Method:       "GET",
+				ReuseRequest: true,
+				Resource:     "/api/2.1/clusters/get?cluster_id=abc",
+				Response: compute.ClusterDetails{
+					ClusterId:              "abc",
+					ClusterName:            "Zero workers cluster",
+					SparkVersion:           "7.3.x-scala12",
+					NodeTypeId:             "Standard_F4s",
+					AutoterminationMinutes: 120,
+					State:                  compute.StateRunning,
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/libraries/cluster-status?cluster_id=abc",
+				Response: compute.ClusterLibraryStatuses{
+					LibraryStatuses: []compute.LibraryFullStatus{},
+				},
+			},
 		},
+		Create:   true,
+		Resource: ResourceCluster(),
 		State: map[string]any{
-			"autotermination_minutes": 15,
-			"cluster_name":            "Shared Autoscaling",
-			"spark_version":           "7.1-scala12",
-			"node_type_id":            "i3.xlarge",
-			"num_workers":             0,
+			"autotermination_minutes": 120,
+			"cluster_name":            "Zero workers cluster",
+			"spark_version":           "7.3.x-scala12",
+			"node_type_id":            "Standard_F4s",
+			"is_pinned":               false,
 		},
 	}.Apply(t)
-	assert.EqualError(t, err, numWorkerErr)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, d.Get("num_workers"))
 }
 
 func TestModifyClusterRequestAws(t *testing.T) {