Skip to content

Commit

Permalink
Rename lakehouse monitor to quality monitor
Browse files Browse the repository at this point in the history
  • Loading branch information
arpitjasa-db committed May 15, 2024
1 parent bd32bf3 commit a4c66da
Show file tree
Hide file tree
Showing 7 changed files with 59 additions and 59 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
| [databricks_job](docs/resources/job.md)
| [databricks_job](docs/data-sources/job.md) data
| [databricks_jobs](docs/data-sources/jobs.md)
| [databricks_lakehouse_monitor](docs/resources/lakehouse_monitor.md)
| [databricks_quality_monitor](docs/resources/quality_monitor.md)
| [databricks_library](docs/resources/library.md)
| [databricks_metastore](docs/resources/metastore.md)
| [databricks_metastore_assignment](docs/resources/metastore_assignment.md)
Expand Down
18 changes: 9 additions & 9 deletions catalog/resource_lakehouse_monitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ import (
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
)

const lakehouseMonitorDefaultProvisionTimeout = 15 * time.Minute
const qualityMonitorDefaultProvisionTimeout = 15 * time.Minute

func WaitForMonitor(w *databricks.WorkspaceClient, ctx context.Context, monitorName string) error {
return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError {
endpoint, err := w.LakehouseMonitors.GetByTableName(ctx, monitorName)
return retry.RetryContext(ctx, qualityMonitorDefaultProvisionTimeout, func() *retry.RetryError {
endpoint, err := w.QualityMonitors.GetByTableName(ctx, monitorName)

Check failure on line 19 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / tests

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)

Check failure on line 19 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / tests

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)

Check failure on line 19 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / compute_diff

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)
if err != nil {
return retry.NonRetryableError(err)
}
Expand All @@ -31,7 +31,7 @@ func WaitForMonitor(w *databricks.WorkspaceClient, ctx context.Context, monitorN
})
}

func ResourceLakehouseMonitor() common.Resource {
func ResourceQualityMonitor() common.Resource {
monitorSchema := common.StructToSchema(
catalog.MonitorInfo{},
func(m map[string]*schema.Schema) map[string]*schema.Schema {
Expand Down Expand Up @@ -68,7 +68,7 @@ func ResourceLakehouseMonitor() common.Resource {
common.DataToStructPointer(d, monitorSchema, &create)
create.TableName = d.Get("table_name").(string)

endpoint, err := w.LakehouseMonitors.Create(ctx, create)
endpoint, err := w.QualityMonitors.Create(ctx, create)

Check failure on line 71 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / tests

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)

Check failure on line 71 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / tests

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)

Check failure on line 71 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / compute_diff

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)
if err != nil {
return err
}
Expand All @@ -84,7 +84,7 @@ func ResourceLakehouseMonitor() common.Resource {
if err != nil {
return err
}
endpoint, err := w.LakehouseMonitors.GetByTableName(ctx, d.Id())
endpoint, err := w.QualityMonitors.GetByTableName(ctx, d.Id())

Check failure on line 87 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / tests

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)

Check failure on line 87 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / tests

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)

Check failure on line 87 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / compute_diff

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)
if err != nil {
return err

Expand All @@ -99,7 +99,7 @@ func ResourceLakehouseMonitor() common.Resource {
var update catalog.UpdateMonitor
common.DataToStructPointer(d, monitorSchema, &update)
update.TableName = d.Get("table_name").(string)
_, err = w.LakehouseMonitors.Update(ctx, update)
_, err = w.QualityMonitors.Update(ctx, update)

Check failure on line 102 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / tests

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)

Check failure on line 102 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / tests

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)

Check failure on line 102 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / compute_diff

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)
if err != nil {
return err
}
Expand All @@ -110,11 +110,11 @@ func ResourceLakehouseMonitor() common.Resource {
if err != nil {
return err
}
return w.LakehouseMonitors.DeleteByTableName(ctx, d.Id())
return w.QualityMonitors.DeleteByTableName(ctx, d.Id())

Check failure on line 113 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / tests

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors) (compile)

Check failure on line 113 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / tests

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)

Check failure on line 113 in catalog/resource_lakehouse_monitor.go

View workflow job for this annotation

GitHub Actions / compute_diff

w.QualityMonitors undefined (type *databricks.WorkspaceClient has no field or method QualityMonitors)
},
Schema: monitorSchema,
Timeouts: &schema.ResourceTimeout{
Create: schema.DefaultTimeout(lakehouseMonitorDefaultProvisionTimeout),
Create: schema.DefaultTimeout(qualityMonitorDefaultProvisionTimeout),
},
}
}
40 changes: 20 additions & 20 deletions catalog/resource_lakehouse_monitor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ import (
"github.com/stretchr/testify/mock"
)

func TestLakehouseMonitorCornerCases(t *testing.T) {
qa.ResourceCornerCases(t, ResourceLakehouseMonitor())
func TestQualityMonitorCornerCases(t *testing.T) {
qa.ResourceCornerCases(t, ResourceQualityMonitor())
}

func TestLakehouseMonitorCreateTimeseries(t *testing.T) {
func TestQualityMonitorCreateTimeseries(t *testing.T) {
qa.ResourceFixture{
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
e := w.GetMockLakehouseMonitorsAPI().EXPECT()
e := w.GetMockQualityMonitorsAPI().EXPECT()
e.Create(mock.Anything, catalog.CreateMonitor{
TableName: "test_table",
OutputSchemaName: "output.schema",
Expand All @@ -40,7 +40,7 @@ func TestLakehouseMonitorCreateTimeseries(t *testing.T) {
DriftMetricsTableName: "test_table_drift",
}, nil)
},
Resource: ResourceLakehouseMonitor(),
Resource: ResourceQualityMonitor(),
HCL: `
table_name = "test_table",
assets_dir = "sample.dir",
Expand All @@ -54,10 +54,10 @@ func TestLakehouseMonitorCreateTimeseries(t *testing.T) {
}.ApplyNoError(t)
}

func TestLakehouseMonitorCreateInference(t *testing.T) {
func TestQualityMonitorCreateInference(t *testing.T) {
qa.ResourceFixture{
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
e := w.GetMockLakehouseMonitorsAPI().EXPECT()
e := w.GetMockQualityMonitorsAPI().EXPECT()
e.Create(mock.Anything, catalog.CreateMonitor{
TableName: "test_table",
OutputSchemaName: "output.schema",
Expand Down Expand Up @@ -89,7 +89,7 @@ func TestLakehouseMonitorCreateInference(t *testing.T) {
},
}, nil)
},
Resource: ResourceLakehouseMonitor(),
Resource: ResourceQualityMonitor(),
HCL: `
table_name = "test_table",
assets_dir = "sample.dir",
Expand All @@ -106,10 +106,10 @@ func TestLakehouseMonitorCreateInference(t *testing.T) {
}.ApplyNoError(t)
}

func TestLakehouseMonitorCreateSnapshot(t *testing.T) {
func TestQualityMonitorCreateSnapshot(t *testing.T) {
qa.ResourceFixture{
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
e := w.GetMockLakehouseMonitorsAPI().EXPECT()
e := w.GetMockQualityMonitorsAPI().EXPECT()
e.Create(mock.Anything, catalog.CreateMonitor{
TableName: "test_table",
OutputSchemaName: "output.schema",
Expand All @@ -129,7 +129,7 @@ func TestLakehouseMonitorCreateSnapshot(t *testing.T) {
Snapshot: &catalog.MonitorSnapshot{},
}, nil)
},
Resource: ResourceLakehouseMonitor(),
Resource: ResourceQualityMonitor(),
HCL: `
table_name = "test_table",
assets_dir = "sample.dir",
Expand All @@ -140,10 +140,10 @@ func TestLakehouseMonitorCreateSnapshot(t *testing.T) {
}.ApplyNoError(t)
}

func TestLakehouseMonitorGet(t *testing.T) {
func TestQualityMonitorGet(t *testing.T) {
qa.ResourceFixture{
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
e := w.GetMockLakehouseMonitorsAPI().EXPECT()
e := w.GetMockQualityMonitorsAPI().EXPECT()
e.GetByTableName(mock.Anything, "test_table").Return(&catalog.MonitorInfo{
TableName: "test_table",
Status: catalog.MonitorInfoStatusMonitorStatusActive,
Expand All @@ -157,16 +157,16 @@ func TestLakehouseMonitorGet(t *testing.T) {
},
DriftMetricsTableName: "test_table_drift"}, nil)
},
Resource: ResourceLakehouseMonitor(),
Resource: ResourceQualityMonitor(),
Read: true,
ID: "test_table",
}.ApplyNoError(t)
}

func TestLakehouseMonitorUpdate(t *testing.T) {
func TestQualityMonitorUpdate(t *testing.T) {
qa.ResourceFixture{
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
e := w.GetMockLakehouseMonitorsAPI().EXPECT()
e := w.GetMockQualityMonitorsAPI().EXPECT()
e.Update(mock.Anything, catalog.UpdateMonitor{
TableName: "test_table",
OutputSchemaName: "output.schema",
Expand Down Expand Up @@ -204,7 +204,7 @@ func TestLakehouseMonitorUpdate(t *testing.T) {
DriftMetricsTableName: "test_table_drift",
}, nil)
},
Resource: ResourceLakehouseMonitor(),
Resource: ResourceQualityMonitor(),
Update: true,
ID: "test_table",
InstanceState: map[string]string{
Expand All @@ -225,13 +225,13 @@ func TestLakehouseMonitorUpdate(t *testing.T) {
}.ApplyNoError(t)
}

func TestLakehouseMonitorDelete(t *testing.T) {
func TestQualityMonitorDelete(t *testing.T) {
qa.ResourceFixture{
MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) {
e := w.GetMockLakehouseMonitorsAPI().EXPECT()
e := w.GetMockQualityMonitorsAPI().EXPECT()
e.DeleteByTableName(mock.Anything, "test_table").Return(nil)
},
Resource: ResourceLakehouseMonitor(),
Resource: ResourceQualityMonitor(),
Delete: true,
ID: "test_table",
}.ApplyNoError(t)
Expand Down
4 changes: 2 additions & 2 deletions catalog/resource_online_table.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
const onlineTableDefaultProvisionTimeout = 45 * time.Minute

func waitForOnlineTableCreation(w *databricks.WorkspaceClient, ctx context.Context, onlineTableName string) error {
return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError {
return retry.RetryContext(ctx, qualityMonitorDefaultProvisionTimeout, func() *retry.RetryError {
endpoint, err := w.OnlineTables.GetByName(ctx, onlineTableName)
if err != nil {
return retry.NonRetryableError(err)
Expand All @@ -40,7 +40,7 @@ func waitForOnlineTableCreation(w *databricks.WorkspaceClient, ctx context.Conte
}

func waitForOnlineTableDeletion(w *databricks.WorkspaceClient, ctx context.Context, onlineTableName string) error {
return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError {
return retry.RetryContext(ctx, qualityMonitorDefaultProvisionTimeout, func() *retry.RetryError {
_, err := w.OnlineTables.GetByName(ctx, onlineTableName)
if err == nil {
return retry.RetryableError(fmt.Errorf("online table %s is still not deleted", onlineTableName))
Expand Down
16 changes: 8 additions & 8 deletions docs/resources/lakehouse_monitor.md
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
---
subcategory: "Unity Catalog"
---
# databricks_lakehouse_monitor Resource
# databricks_quality_monitor Resource

This resource allows you to manage [Lakehouse Monitors](https://docs.databricks.com/en/lakehouse-monitoring/index.html) in Databricks.

A `databricks_lakehouse_monitor` is attached to a [databricks_sql_table](sql_table.md) and can be of type timeseries, snapshot or inference.
A `databricks_quality_monitor` is attached to a [databricks_sql_table](sql_table.md) and can be of type timeseries, snapshot or inference.

## Example Usage

Expand Down Expand Up @@ -41,9 +41,9 @@ resource "databricks_sql_table" "myTestTable" {
}
}
resource "databricks_lakehouse_monitor" "testTimeseriesMonitor" {
resource "databricks_quality_monitor" "testTimeseriesMonitor" {
table_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}.${databricks_sql_table.myTestTable.name}"
assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myTestTable.name}"
assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myTestTable.name}"
output_schema_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}"
time_series {
granularities = ["1 hour"]
Expand All @@ -55,9 +55,9 @@ resource "databricks_lakehouse_monitor" "testTimeseriesMonitor" {
### Inference Monitor

```hcl
resource "databricks_lakehouse_monitor" "testMonitorInference" {
resource "databricks_quality_monitor" "testMonitorInference" {
table_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}.${databricks_table.myTestTable.name}"
assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_table.myTestTable.name}"
assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_table.myTestTable.name}"
output_schema_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}"
inference_log {
granularities = ["1 hour"]
Expand All @@ -70,9 +70,9 @@ resource "databricks_lakehouse_monitor" "testMonitorInference" {
```
### Snapshot Monitor
```hcl
resource "databricks_lakehouse_monitor" "testMonitorInference" {
resource "databricks_quality_monitor" "testMonitorInference" {
table_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}.${databricks_table.myTestTable.name}"
assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_table.myTestTable.name}"
assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_table.myTestTable.name}"
output_schema_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}"
snapshot {}
}
Expand Down
36 changes: 18 additions & 18 deletions internal/acceptance/lakehouse_monitor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"testing"
)

var commonPartLakehouseMonitoring = `resource "databricks_catalog" "sandbox" {
var commonPartQualityMonitoring = `resource "databricks_catalog" "sandbox" {
name = "sandbox{var.STICKY_RANDOM}"
comment = "this catalog is managed by terraform"
properties = {
Expand Down Expand Up @@ -46,16 +46,16 @@ resource "databricks_sql_table" "myInferenceTable" {
`

func TestUcAccLakehouseMonitor(t *testing.T) {
func TestUcAccQualityMonitor(t *testing.T) {
if os.Getenv("GOOGLE_CREDENTIALS") != "" {
t.Skipf("databricks_lakehouse_monitor resource is not available on GCP")
t.Skipf("databricks_quality_monitor resource is not available on GCP")
}
unityWorkspaceLevel(t, step{
Template: commonPartLakehouseMonitoring + `
Template: commonPartQualityMonitoring + `
resource "databricks_lakehouse_monitor" "testMonitorInference" {
resource "databricks_quality_monitor" "testMonitorInference" {
table_name = databricks_sql_table.myInferenceTable.id
assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myInferenceTable.name}"
assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}"
output_schema_name = databricks_schema.things.id
inference_log {
granularities = ["1 day"]
Expand All @@ -79,9 +79,9 @@ func TestUcAccLakehouseMonitor(t *testing.T) {
}
}
resource "databricks_lakehouse_monitor" "testMonitorTimeseries" {
resource "databricks_quality_monitor" "testMonitorTimeseries" {
table_name = databricks_sql_table.myTimeseries.id
assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myTimeseries.name}"
assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myTimeseries.name}"
output_schema_name = databricks_schema.things.id
time_series {
granularities = ["1 day"]
Expand All @@ -102,9 +102,9 @@ func TestUcAccLakehouseMonitor(t *testing.T) {
}
}
resource "databricks_lakehouse_monitor" "testMonitorSnapshot" {
resource "databricks_quality_monitor" "testMonitorSnapshot" {
table_name = databricks_sql_table.mySnapshot.id
assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myTimeseries.name}"
assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myTimeseries.name}"
output_schema_name = databricks_schema.things.id
snapshot {
}
Expand All @@ -113,15 +113,15 @@ func TestUcAccLakehouseMonitor(t *testing.T) {
})
}

func TestUcAccUpdateLakehouseMonitor(t *testing.T) {
func TestUcAccUpdateQualityMonitor(t *testing.T) {
if os.Getenv("GOOGLE_CREDENTIALS") != "" {
t.Skipf("databricks_lakehouse_monitor resource is not available on GCP")
t.Skipf("databricks_quality_monitor resource is not available on GCP")
}
unityWorkspaceLevel(t, step{
Template: commonPartLakehouseMonitoring + `
resource "databricks_lakehouse_monitor" "testMonitorInference" {
Template: commonPartQualityMonitoring + `
resource "databricks_quality_monitor" "testMonitorInference" {
table_name = databricks_sql_table.myInferenceTable.id
assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myInferenceTable.name}"
assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}"
output_schema_name = databricks_schema.things.id
inference_log {
granularities = ["1 day"]
Expand All @@ -133,10 +133,10 @@ func TestUcAccUpdateLakehouseMonitor(t *testing.T) {
}
`,
}, step{
Template: commonPartLakehouseMonitoring + `
resource "databricks_lakehouse_monitor" "testMonitorInference" {
Template: commonPartQualityMonitoring + `
resource "databricks_quality_monitor" "testMonitorInference" {
table_name = databricks_sql_table.myInferenceTable.id
assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myInferenceTable.name}"
assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}"
output_schema_name = databricks_schema.things.id
inference_log {
granularities = ["1 hour"]
Expand Down
2 changes: 1 addition & 1 deletion provider/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func DatabricksProvider() *schema.Provider {
"databricks_instance_profile": aws.ResourceInstanceProfile().ToResource(),
"databricks_ip_access_list": access.ResourceIPAccessList().ToResource(),
"databricks_job": jobs.ResourceJob().ToResource(),
"databricks_lakehouse_monitor": catalog.ResourceLakehouseMonitor().ToResource(),
"databricks_quality_monitor": catalog.ResourceQualityMonitor().ToResource(),
"databricks_library": clusters.ResourceLibrary().ToResource(),
"databricks_metastore": catalog.ResourceMetastore().ToResource(),
"databricks_metastore_assignment": catalog.ResourceMetastoreAssignment().ToResource(),
Expand Down

0 comments on commit a4c66da

Please sign in to comment.