From a4c66da5e12f20af21546d146ce4be4a1723da4e Mon Sep 17 00:00:00 2001 From: Arpit Jasapara Date: Wed, 15 May 2024 15:48:24 -0700 Subject: [PATCH] Rename lakehouse monitor to quality monitor --- README.md | 2 +- catalog/resource_lakehouse_monitor.go | 18 ++++----- catalog/resource_lakehouse_monitor_test.go | 40 +++++++++---------- catalog/resource_online_table.go | 4 +- docs/resources/lakehouse_monitor.md | 16 ++++---- internal/acceptance/lakehouse_monitor_test.go | 36 ++++++++--------- provider/provider.go | 2 +- 7 files changed, 59 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 5dc0029bbc..0903625638 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ | [databricks_job](docs/resources/job.md) | [databricks_job](docs/data-sources/job.md) data | [databricks_jobs](docs/data-sources/jobs.md) -| [databricks_lakehouse_monitor](docs/resources/lakehouse_monitor.md) +| [databricks_quality_monitor](docs/resources/quality_monitor.md) | [databricks_library](docs/resources/library.md) | [databricks_metastore](docs/resources/metastore.md) | [databricks_metastore_assignment](docs/resources/metastore_assignment.md) diff --git a/catalog/resource_lakehouse_monitor.go b/catalog/resource_lakehouse_monitor.go index ac57786a3d..9e9169fd4e 100644 --- a/catalog/resource_lakehouse_monitor.go +++ b/catalog/resource_lakehouse_monitor.go @@ -12,11 +12,11 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) -const lakehouseMonitorDefaultProvisionTimeout = 15 * time.Minute +const qualityMonitorDefaultProvisionTimeout = 15 * time.Minute func WaitForMonitor(w *databricks.WorkspaceClient, ctx context.Context, monitorName string) error { - return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError { - endpoint, err := w.LakehouseMonitors.GetByTableName(ctx, monitorName) + return retry.RetryContext(ctx, qualityMonitorDefaultProvisionTimeout, func() *retry.RetryError { + endpoint, err := w.QualityMonitors.GetByTableName(ctx, monitorName) if err != nil { return retry.NonRetryableError(err) } @@ -31,7 +31,7 @@ func WaitForMonitor(w *databricks.WorkspaceClient, ctx context.Context, monitorN }) } -func ResourceLakehouseMonitor() common.Resource { +func ResourceQualityMonitor() common.Resource { monitorSchema := common.StructToSchema( catalog.MonitorInfo{}, func(m map[string]*schema.Schema) map[string]*schema.Schema { @@ -68,7 +68,7 @@ func ResourceLakehouseMonitor() common.Resource { common.DataToStructPointer(d, monitorSchema, &create) create.TableName = d.Get("table_name").(string) - endpoint, err := w.LakehouseMonitors.Create(ctx, create) + endpoint, err := w.QualityMonitors.Create(ctx, create) if err != nil { return err } @@ -84,7 +84,7 @@ func ResourceLakehouseMonitor() common.Resource { if err != nil { return err } - endpoint, err := w.LakehouseMonitors.GetByTableName(ctx, d.Id()) + endpoint, err := w.QualityMonitors.GetByTableName(ctx, d.Id()) if err != nil { return err @@ -99,7 +99,7 @@ func ResourceLakehouseMonitor() common.Resource { var update catalog.UpdateMonitor common.DataToStructPointer(d, monitorSchema, &update) update.TableName = d.Get("table_name").(string) - _, err = w.LakehouseMonitors.Update(ctx, update) + _, err = w.QualityMonitors.Update(ctx, update) if err != nil { return err } @@ -110,11 +110,11 @@ func ResourceLakehouseMonitor() common.Resource { if err != nil { return err } - return w.LakehouseMonitors.DeleteByTableName(ctx, d.Id()) + return w.QualityMonitors.DeleteByTableName(ctx, d.Id()) }, Schema: monitorSchema, Timeouts: &schema.ResourceTimeout{ - Create: schema.DefaultTimeout(lakehouseMonitorDefaultProvisionTimeout), + Create: schema.DefaultTimeout(qualityMonitorDefaultProvisionTimeout), }, } } diff --git a/catalog/resource_lakehouse_monitor_test.go b/catalog/resource_lakehouse_monitor_test.go index 652fd7d4b7..e9d062ed1c 100644 --- a/catalog/resource_lakehouse_monitor_test.go +++ b/catalog/resource_lakehouse_monitor_test.go @@ -9,14 +9,14 @@ import ( "github.com/stretchr/testify/mock" ) -func TestLakehouseMonitorCornerCases(t *testing.T) { - qa.ResourceCornerCases(t, ResourceLakehouseMonitor()) +func TestQualityMonitorCornerCases(t *testing.T) { + qa.ResourceCornerCases(t, ResourceQualityMonitor()) } -func TestLakehouseMonitorCreateTimeseries(t *testing.T) { +func TestQualityMonitorCreateTimeseries(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { - e := w.GetMockLakehouseMonitorsAPI().EXPECT() + e := w.GetMockQualityMonitorsAPI().EXPECT() e.Create(mock.Anything, catalog.CreateMonitor{ TableName: "test_table", OutputSchemaName: "output.schema", @@ -40,7 +40,7 @@ func TestLakehouseMonitorCreateTimeseries(t *testing.T) { DriftMetricsTableName: "test_table_drift", }, nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), HCL: ` table_name = "test_table", assets_dir = "sample.dir", @@ -54,10 +54,10 @@ func TestLakehouseMonitorCreateTimeseries(t *testing.T) { }.ApplyNoError(t) } -func TestLakehouseMonitorCreateInference(t *testing.T) { +func TestQualityMonitorCreateInference(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { - e := w.GetMockLakehouseMonitorsAPI().EXPECT() + e := w.GetMockQualityMonitorsAPI().EXPECT() e.Create(mock.Anything, catalog.CreateMonitor{ TableName: "test_table", OutputSchemaName: "output.schema", @@ -89,7 +89,7 @@ func TestLakehouseMonitorCreateInference(t *testing.T) { }, }, nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), HCL: ` table_name = "test_table", assets_dir = "sample.dir", @@ -106,10 +106,10 @@ func TestLakehouseMonitorCreateInference(t *testing.T) { }.ApplyNoError(t) } -func TestLakehouseMonitorCreateSnapshot(t *testing.T) { +func TestQualityMonitorCreateSnapshot(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { - e := w.GetMockLakehouseMonitorsAPI().EXPECT() + e := w.GetMockQualityMonitorsAPI().EXPECT() e.Create(mock.Anything, catalog.CreateMonitor{ TableName: "test_table", OutputSchemaName: "output.schema", @@ -129,7 +129,7 @@ func TestLakehouseMonitorCreateSnapshot(t *testing.T) { Snapshot: &catalog.MonitorSnapshot{}, }, nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), HCL: ` table_name = "test_table", assets_dir = "sample.dir", @@ -140,10 +140,10 @@ func TestLakehouseMonitorCreateSnapshot(t *testing.T) { }.ApplyNoError(t) } -func TestLakehouseMonitorGet(t *testing.T) { +func TestQualityMonitorGet(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { - e := w.GetMockLakehouseMonitorsAPI().EXPECT() + e := w.GetMockQualityMonitorsAPI().EXPECT() e.GetByTableName(mock.Anything, "test_table").Return(&catalog.MonitorInfo{ TableName: "test_table", Status: catalog.MonitorInfoStatusMonitorStatusActive, @@ -157,16 +157,16 @@ func TestLakehouseMonitorGet(t *testing.T) { }, DriftMetricsTableName: "test_table_drift"}, nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), Read: true, ID: "test_table", }.ApplyNoError(t) } -func TestLakehouseMonitorUpdate(t *testing.T) { +func TestQualityMonitorUpdate(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { - e := w.GetMockLakehouseMonitorsAPI().EXPECT() + e := w.GetMockQualityMonitorsAPI().EXPECT() e.Update(mock.Anything, catalog.UpdateMonitor{ TableName: "test_table", OutputSchemaName: "output.schema", @@ -204,7 +204,7 @@ func TestLakehouseMonitorUpdate(t *testing.T) { DriftMetricsTableName: "test_table_drift", }, nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), Update: true, ID: "test_table", InstanceState: map[string]string{ @@ -225,13 +225,13 @@ func TestLakehouseMonitorUpdate(t *testing.T) { }.ApplyNoError(t) } -func TestLakehouseMonitorDelete(t *testing.T) { +func TestQualityMonitorDelete(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { - e := w.GetMockLakehouseMonitorsAPI().EXPECT() + e := w.GetMockQualityMonitorsAPI().EXPECT() e.DeleteByTableName(mock.Anything, "test_table").Return(nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), Delete: true, ID: "test_table", }.ApplyNoError(t) diff --git a/catalog/resource_online_table.go b/catalog/resource_online_table.go index 70425b9237..31f53a8445 100644 --- a/catalog/resource_online_table.go +++ b/catalog/resource_online_table.go @@ -17,7 +17,7 @@ import ( const onlineTableDefaultProvisionTimeout = 45 * time.Minute func waitForOnlineTableCreation(w *databricks.WorkspaceClient, ctx context.Context, onlineTableName string) error { - return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError { + return retry.RetryContext(ctx, qualityMonitorDefaultProvisionTimeout, func() *retry.RetryError { endpoint, err := w.OnlineTables.GetByName(ctx, onlineTableName) if err != nil { return retry.NonRetryableError(err) @@ -40,7 +40,7 @@ func waitForOnlineTableCreation(w *databricks.WorkspaceClient, ctx context.Conte } func waitForOnlineTableDeletion(w *databricks.WorkspaceClient, ctx context.Context, onlineTableName string) error { - return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError { + return retry.RetryContext(ctx, qualityMonitorDefaultProvisionTimeout, func() *retry.RetryError { _, err := w.OnlineTables.GetByName(ctx, onlineTableName) if err == nil { return retry.RetryableError(fmt.Errorf("online table %s is still not deleted", onlineTableName)) diff --git a/docs/resources/lakehouse_monitor.md b/docs/resources/lakehouse_monitor.md index 33cbd56c01..a3292f65d2 100644 --- a/docs/resources/lakehouse_monitor.md +++ b/docs/resources/lakehouse_monitor.md @@ -1,11 +1,11 @@ --- subcategory: "Unity Catalog" --- -# databricks_lakehouse_monitor Resource +# databricks_quality_monitor Resource This resource allows you to manage [Lakehouse Monitors](https://docs.databricks.com/en/lakehouse-monitoring/index.html) in Databricks. -A `databricks_lakehouse_monitor` is attached to a [databricks_sql_table](sql_table.md) and can be of type timeseries, snapshot or inference. +A `databricks_quality_monitor` is attached to a [databricks_sql_table](sql_table.md) and can be of type timeseries, snapshot or inference. ## Example Usage @@ -41,9 +41,9 @@ resource "databricks_sql_table" "myTestTable" { } } -resource "databricks_lakehouse_monitor" "testTimeseriesMonitor" { +resource "databricks_quality_monitor" "testTimeseriesMonitor" { table_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}.${databricks_sql_table.myTestTable.name}" - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myTestTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myTestTable.name}" output_schema_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}" time_series { granularities = ["1 hour"] @@ -55,9 +55,9 @@ resource "databricks_lakehouse_monitor" "testTimeseriesMonitor" { ### Inference Monitor ```hcl -resource "databricks_lakehouse_monitor" "testMonitorInference" { +resource "databricks_quality_monitor" "testMonitorInference" { table_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}.${databricks_table.myTestTable.name}" - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_table.myTestTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_table.myTestTable.name}" output_schema_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}" inference_log { granularities = ["1 hour"] @@ -70,9 +70,9 @@ resource "databricks_lakehouse_monitor" "testMonitorInference" { ``` ### Snapshot Monitor ```hcl -resource "databricks_lakehouse_monitor" "testMonitorInference" { +resource "databricks_quality_monitor" "testMonitorInference" { table_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}.${databricks_table.myTestTable.name}" - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_table.myTestTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_table.myTestTable.name}" output_schema_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}" snapshot {} } diff --git a/internal/acceptance/lakehouse_monitor_test.go b/internal/acceptance/lakehouse_monitor_test.go index 79f449259f..9b9da2c91c 100644 --- a/internal/acceptance/lakehouse_monitor_test.go +++ b/internal/acceptance/lakehouse_monitor_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -var commonPartLakehouseMonitoring = `resource "databricks_catalog" "sandbox" { +var commonPartQualityMonitoring = `resource "databricks_catalog" "sandbox" { name = "sandbox{var.STICKY_RANDOM}" comment = "this catalog is managed by terraform" properties = { @@ -46,16 +46,16 @@ resource "databricks_sql_table" "myInferenceTable" { ` -func TestUcAccLakehouseMonitor(t *testing.T) { +func TestUcAccQualityMonitor(t *testing.T) { if os.Getenv("GOOGLE_CREDENTIALS") != "" { - t.Skipf("databricks_lakehouse_monitor resource is not available on GCP") + t.Skipf("databricks_quality_monitor resource is not available on GCP") } unityWorkspaceLevel(t, step{ - Template: commonPartLakehouseMonitoring + ` + Template: commonPartQualityMonitoring + ` - resource "databricks_lakehouse_monitor" "testMonitorInference" { + resource "databricks_quality_monitor" "testMonitorInference" { table_name = databricks_sql_table.myInferenceTable.id - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myInferenceTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}" output_schema_name = databricks_schema.things.id inference_log { granularities = ["1 day"] @@ -79,9 +79,9 @@ func TestUcAccLakehouseMonitor(t *testing.T) { } } - resource "databricks_lakehouse_monitor" "testMonitorTimeseries" { + resource "databricks_quality_monitor" "testMonitorTimeseries" { table_name = databricks_sql_table.myTimeseries.id - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myTimeseries.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myTimeseries.name}" output_schema_name = databricks_schema.things.id time_series { granularities = ["1 day"] @@ -102,9 +102,9 @@ func TestUcAccLakehouseMonitor(t *testing.T) { } } - resource "databricks_lakehouse_monitor" "testMonitorSnapshot" { + resource "databricks_quality_monitor" "testMonitorSnapshot" { table_name = databricks_sql_table.mySnapshot.id - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myTimeseries.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myTimeseries.name}" output_schema_name = databricks_schema.things.id snapshot { } @@ -113,15 +113,15 @@ func TestUcAccLakehouseMonitor(t *testing.T) { }) } -func TestUcAccUpdateLakehouseMonitor(t *testing.T) { +func TestUcAccUpdateQualityMonitor(t *testing.T) { if os.Getenv("GOOGLE_CREDENTIALS") != "" { - t.Skipf("databricks_lakehouse_monitor resource is not available on GCP") + t.Skipf("databricks_quality_monitor resource is not available on GCP") } unityWorkspaceLevel(t, step{ - Template: commonPartLakehouseMonitoring + ` - resource "databricks_lakehouse_monitor" "testMonitorInference" { + Template: commonPartQualityMonitoring + ` + resource "databricks_quality_monitor" "testMonitorInference" { table_name = databricks_sql_table.myInferenceTable.id - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myInferenceTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}" output_schema_name = databricks_schema.things.id inference_log { granularities = ["1 day"] @@ -133,10 +133,10 @@ func TestUcAccUpdateLakehouseMonitor(t *testing.T) { } `, }, step{ - Template: commonPartLakehouseMonitoring + ` - resource "databricks_lakehouse_monitor" "testMonitorInference" { + Template: commonPartQualityMonitoring + ` + resource "databricks_quality_monitor" "testMonitorInference" { table_name = databricks_sql_table.myInferenceTable.id - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myInferenceTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}" output_schema_name = databricks_schema.things.id inference_log { granularities = ["1 hour"] diff --git a/provider/provider.go b/provider/provider.go index 683f9a819d..a2df2c7104 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -145,7 +145,7 @@ func DatabricksProvider() *schema.Provider { "databricks_instance_profile": aws.ResourceInstanceProfile().ToResource(), "databricks_ip_access_list": access.ResourceIPAccessList().ToResource(), "databricks_job": jobs.ResourceJob().ToResource(), - "databricks_lakehouse_monitor": catalog.ResourceLakehouseMonitor().ToResource(), + "databricks_quality_monitor": catalog.ResourceQualityMonitor().ToResource(), "databricks_library": clusters.ResourceLibrary().ToResource(), "databricks_metastore": catalog.ResourceMetastore().ToResource(), "databricks_metastore_assignment": catalog.ResourceMetastoreAssignment().ToResource(),