From 92357dcf9cfb4acd64ace55b00e82e38300ec2ad Mon Sep 17 00:00:00 2001
From: Alex Ott <alexey.ott@databricks.com>
Date: Tue, 29 Oct 2024 06:17:42 -0400
Subject: [PATCH] [Fix] Handle edge case for `effective_properties` in
 `databricks_sql_table` (#4153)

## Changes
<!-- Summary of your changes that are easy to understand -->

It was reported in #4098 that some of the specified options, like,
`multiLine`, `recursiveFileLookup` and potentially more, aren't returned
as `option.multiLine`, etc., but instead are expanded into full names,
like, `spark.sql.dataSourceOptions.multiLine`.

This PR changes lookup logic a bit, and if we can't find
`option.something`, then we're looking for all options ending with
`.something` (only if there are no `.` in the name).

Resolves #4098

## Tests
<!--
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [ ] relevant change in `docs/` folder
- [ ] covered with integration tests in `internal/acceptance`
- [ ] relevant acceptance tests are passing
- [ ] using Go SDK
---
 catalog/resource_sql_table.go      | 29 +++++++++++++++++++++--------
 catalog/resource_sql_table_test.go |  7 +++++--
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/catalog/resource_sql_table.go b/catalog/resource_sql_table.go
index ce9d4dbd7a..710c8a20bf 100644
--- a/catalog/resource_sql_table.go
+++ b/catalog/resource_sql_table.go
@@ -21,6 +21,7 @@ import (
 )
 
 var MaxSqlExecWaitTimeout = 50
+var optionPrefixes = []string{"option.", "spark.sql.dataSourceOptions."}
 
 type SqlColumnInfo struct {
 	Name     string         `json:"name"`
@@ -67,7 +68,6 @@ type SqlTableInfo struct {
 }
 
 func (ti SqlTableInfo) CustomizeSchema(s *common.CustomizableSchema) *common.CustomizableSchema {
-
 	caseInsensitiveFields := []string{"name", "catalog_name", "schema_name"}
 	for _, field := range caseInsensitiveFields {
 		s.SchemaPath(field).SetCustomSuppressDiff(common.EqualFoldDiffSuppress)
@@ -598,18 +598,31 @@ func ResourceSqlTable() common.Resource {
 			// If the user specified a property but the value of that property has changed, that will appear
 			// as a change in the effective property/option. To cause a diff to be detected, we need to
 			// reset the effective property/option to the requested value.
-			userSpecifiedProperties := d.Get("properties").(map[string]interface{})
-			userSpecifiedOptions := d.Get("options").(map[string]interface{})
-			effectiveProperties := d.Get("effective_properties").(map[string]interface{})
-			diff := make(map[string]interface{})
+			userSpecifiedProperties := d.Get("properties").(map[string]any)
+			userSpecifiedOptions := d.Get("options").(map[string]any)
+			effectiveProperties := d.Get("effective_properties").(map[string]any)
+			diff := make(map[string]any)
 			for k, userSpecifiedValue := range userSpecifiedProperties {
 				if effectiveValue, ok := effectiveProperties[k]; !ok || effectiveValue != userSpecifiedValue {
 					diff[k] = userSpecifiedValue
 				}
 			}
-			for k, userSpecifiedValue := range userSpecifiedOptions {
-				if effectiveValue, ok := effectiveProperties["option."+k]; !ok || effectiveValue != userSpecifiedValue {
-					diff["option."+k] = userSpecifiedValue
+			for userOptName, userSpecifiedValue := range userSpecifiedOptions {
+				var found bool
+				var effectiveValue any
+				var effectOptName string
+				// If the option is not found, check if the user specified the option without the prefix
+				// i.e. if user specified `multiLine` for JSON, then backend returns `spark.sql.dataSourceOptions.multiLine`
+				for _, prefix := range optionPrefixes {
+					effectOptName = prefix + userOptName
+					if v, ok := effectiveProperties[effectOptName]; ok {
+						found = true
+						effectiveValue = v
+						break
+					}
+				}
+				if !found || effectiveValue != userSpecifiedValue {
+					diff[effectOptName] = userSpecifiedValue
 				}
 			}
 			if len(diff) > 0 {
diff --git a/catalog/resource_sql_table_test.go b/catalog/resource_sql_table_test.go
index f2f0a6c5e2..b2495480cb 100644
--- a/catalog/resource_sql_table_test.go
+++ b/catalog/resource_sql_table_test.go
@@ -1625,15 +1625,18 @@ func TestResourceSqlTable_Diff_ExistingResource(t *testing.T) {
 			}
 			options = {
 				"myopt" = "myval"
+				"multiLine" = "true"
 			}`,
 			map[string]string{
 				"properties.%":                      "1",
 				"properties.myprop":                 "myval",
-				"options.%":                         "1",
+				"options.%":                         "2",
 				"options.myopt":                     "myval",
-				"effective_properties.%":            "2",
+				"options.multiLine":                 "true",
+				"effective_properties.%":            "3",
 				"effective_properties.myprop":       "myval",
 				"effective_properties.option.myopt": "myval",
+				"effective_properties.spark.sql.dataSourceOptions.multiLine": "true",
 			},
 			nil,
 		},