From f2ecb5cee97c25b33624f1c93ceb6d0085959ba3 Mon Sep 17 00:00:00 2001 From: Arpit Jasapara <87999496+arpitjasa-db@users.noreply.github.com> Date: Thu, 23 May 2024 00:58:17 -0700 Subject: [PATCH] Rename lakehouse monitor to quality monitor (#3584) Deprecate resource instead of breaking rename fix Apply suggestions from code review Co-authored-by: vuong-nguyen <44292934+nkvuong@users.noreply.github.com> --- README.md | 1 + catalog/resource_lakehouse_monitor.go | 115 +-------------- catalog/resource_online_table.go | 4 +- catalog/resource_quality_monitor.go | 120 +++++++++++++++ ...st.go => resource_quality_monitor_test.go} | 28 ++-- docs/resources/lakehouse_monitor.md | 2 + docs/resources/quality_monitor.md | 137 ++++++++++++++++++ ...onitor_test.go => quality_monitor_test.go} | 36 ++--- provider/provider.go | 1 + 9 files changed, 298 insertions(+), 146 deletions(-) create mode 100644 catalog/resource_quality_monitor.go rename catalog/{resource_lakehouse_monitor_test.go => resource_quality_monitor_test.go} (91%) create mode 100644 docs/resources/quality_monitor.md rename internal/acceptance/{lakehouse_monitor_test.go => quality_monitor_test.go} (68%) diff --git a/README.md b/README.md index 5dc0029bbc..7b97782cf4 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ | [databricks_permissions](docs/resources/permissions.md) | [databricks_pipeline](docs/resources/pipeline.md) | [databricks_pipelines](docs/data-sources/pipelines.md) data +| [databricks_quality_monitor](docs/resources/quality_monitor.md) | [databricks_repo](docs/resources/repo.md) | [databricks_schema](docs/resources/schema.md) | [databricks_schemas](docs/data-sources/schema.md) data diff --git a/catalog/resource_lakehouse_monitor.go b/catalog/resource_lakehouse_monitor.go index 8e7c0db778..8e3f6cb25f 100644 --- a/catalog/resource_lakehouse_monitor.go +++ b/catalog/resource_lakehouse_monitor.go @@ -1,120 +1,11 @@ package catalog import ( - "context" - "fmt" - "time" - - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/terraform-provider-databricks/common" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/retry" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) -const lakehouseMonitorDefaultProvisionTimeout = 15 * time.Minute - -func WaitForMonitor(w *databricks.WorkspaceClient, ctx context.Context, monitorName string) error { - return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError { - endpoint, err := w.QualityMonitors.GetByTableName(ctx, monitorName) - if err != nil { - return retry.NonRetryableError(err) - } - - switch endpoint.Status { - case catalog.MonitorInfoStatusMonitorStatusActive: - return nil - case catalog.MonitorInfoStatusMonitorStatusError, catalog.MonitorInfoStatusMonitorStatusFailed: - return retry.NonRetryableError(fmt.Errorf("monitor status retrund %s for monitor: %s", endpoint.Status, monitorName)) - } - return retry.RetryableError(fmt.Errorf("monitor %s is still pending", monitorName)) - }) -} - func ResourceLakehouseMonitor() common.Resource { - monitorSchema := common.StructToSchema( - catalog.MonitorInfo{}, - func(m map[string]*schema.Schema) map[string]*schema.Schema { - common.CustomizeSchemaPath(m, "assets_dir").SetRequired() - common.CustomizeSchemaPath(m, "output_schema_name").SetRequired() - common.CustomizeSchemaPath(m, "table_name").SetRequired() - common.CustomizeSchemaPath(m).AddNewField("skip_builtin_dashboard", &schema.Schema{ - Type: schema.TypeBool, - Optional: true, - Required: false, - }) - common.CustomizeSchemaPath(m).AddNewField("warehouse_id", &schema.Schema{ - Type: schema.TypeString, - Optional: true, - Required: false, - }) - common.CustomizeSchemaPath(m, "monitor_version").SetReadOnly() - common.CustomizeSchemaPath(m, "drift_metrics_table_name").SetReadOnly() - common.CustomizeSchemaPath(m, "profile_metrics_table_name").SetReadOnly() - common.CustomizeSchemaPath(m, "status").SetReadOnly() - common.CustomizeSchemaPath(m, "dashboard_id").SetReadOnly() - return m - }, - ) - - return common.Resource{ - Create: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() - if err != nil { - return err - } - - var create catalog.CreateMonitor - common.DataToStructPointer(d, monitorSchema, &create) - create.TableName = d.Get("table_name").(string) - - endpoint, err := w.QualityMonitors.Create(ctx, create) - if err != nil { - return err - } - err = WaitForMonitor(w, ctx, create.TableName) - if err != nil { - return err - } - d.SetId(endpoint.TableName) - return nil - }, - Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() - if err != nil { - return err - } - endpoint, err := w.QualityMonitors.GetByTableName(ctx, d.Id()) - if err != nil { - return err - - } - return common.StructToData(endpoint, monitorSchema, d) - }, - Update: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() - if err != nil { - return err - } - var update catalog.UpdateMonitor - common.DataToStructPointer(d, monitorSchema, &update) - update.TableName = d.Get("table_name").(string) - _, err = w.QualityMonitors.Update(ctx, update) - if err != nil { - return err - } - return WaitForMonitor(w, ctx, update.TableName) - }, - Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() - if err != nil { - return err - } - return w.QualityMonitors.DeleteByTableName(ctx, d.Id()) - }, - Schema: monitorSchema, - Timeouts: &schema.ResourceTimeout{ - Create: schema.DefaultTimeout(lakehouseMonitorDefaultProvisionTimeout), - }, - } + r := ResourceQualityMonitor() + r.DeprecationMessage = "Use `databricks_quality_monitor` instead." + return r } diff --git a/catalog/resource_online_table.go b/catalog/resource_online_table.go index 70425b9237..0dd75ffe90 100644 --- a/catalog/resource_online_table.go +++ b/catalog/resource_online_table.go @@ -17,7 +17,7 @@ import ( const onlineTableDefaultProvisionTimeout = 45 * time.Minute func waitForOnlineTableCreation(w *databricks.WorkspaceClient, ctx context.Context, onlineTableName string) error { - return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError { + return retry.RetryContext(ctx, onlineTableDefaultProvisionTimeout, func() *retry.RetryError { endpoint, err := w.OnlineTables.GetByName(ctx, onlineTableName) if err != nil { return retry.NonRetryableError(err) @@ -40,7 +40,7 @@ func waitForOnlineTableCreation(w *databricks.WorkspaceClient, ctx context.Conte } func waitForOnlineTableDeletion(w *databricks.WorkspaceClient, ctx context.Context, onlineTableName string) error { - return retry.RetryContext(ctx, lakehouseMonitorDefaultProvisionTimeout, func() *retry.RetryError { + return retry.RetryContext(ctx, onlineTableDefaultProvisionTimeout, func() *retry.RetryError { _, err := w.OnlineTables.GetByName(ctx, onlineTableName) if err == nil { return retry.RetryableError(fmt.Errorf("online table %s is still not deleted", onlineTableName)) diff --git a/catalog/resource_quality_monitor.go b/catalog/resource_quality_monitor.go new file mode 100644 index 0000000000..9e9169fd4e --- /dev/null +++ b/catalog/resource_quality_monitor.go @@ -0,0 +1,120 @@ +package catalog + +import ( + "context" + "fmt" + "time" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/terraform-provider-databricks/common" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/retry" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +const qualityMonitorDefaultProvisionTimeout = 15 * time.Minute + +func WaitForMonitor(w *databricks.WorkspaceClient, ctx context.Context, monitorName string) error { + return retry.RetryContext(ctx, qualityMonitorDefaultProvisionTimeout, func() *retry.RetryError { + endpoint, err := w.QualityMonitors.GetByTableName(ctx, monitorName) + if err != nil { + return retry.NonRetryableError(err) + } + + switch endpoint.Status { + case catalog.MonitorInfoStatusMonitorStatusActive: + return nil + case catalog.MonitorInfoStatusMonitorStatusError, catalog.MonitorInfoStatusMonitorStatusFailed: + return retry.NonRetryableError(fmt.Errorf("monitor status retrund %s for monitor: %s", endpoint.Status, monitorName)) + } + return retry.RetryableError(fmt.Errorf("monitor %s is still pending", monitorName)) + }) +} + +func ResourceQualityMonitor() common.Resource { + monitorSchema := common.StructToSchema( + catalog.MonitorInfo{}, + func(m map[string]*schema.Schema) map[string]*schema.Schema { + common.CustomizeSchemaPath(m, "assets_dir").SetRequired() + common.CustomizeSchemaPath(m, "output_schema_name").SetRequired() + common.CustomizeSchemaPath(m, "table_name").SetRequired() + common.CustomizeSchemaPath(m).AddNewField("skip_builtin_dashboard", &schema.Schema{ + Type: schema.TypeBool, + Optional: true, + Required: false, + }) + common.CustomizeSchemaPath(m).AddNewField("warehouse_id", &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Required: false, + }) + common.CustomizeSchemaPath(m, "monitor_version").SetReadOnly() + common.CustomizeSchemaPath(m, "drift_metrics_table_name").SetReadOnly() + common.CustomizeSchemaPath(m, "profile_metrics_table_name").SetReadOnly() + common.CustomizeSchemaPath(m, "status").SetReadOnly() + common.CustomizeSchemaPath(m, "dashboard_id").SetReadOnly() + return m + }, + ) + + return common.Resource{ + Create: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { + w, err := c.WorkspaceClient() + if err != nil { + return err + } + + var create catalog.CreateMonitor + common.DataToStructPointer(d, monitorSchema, &create) + create.TableName = d.Get("table_name").(string) + + endpoint, err := w.QualityMonitors.Create(ctx, create) + if err != nil { + return err + } + err = WaitForMonitor(w, ctx, create.TableName) + if err != nil { + return err + } + d.SetId(endpoint.TableName) + return nil + }, + Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { + w, err := c.WorkspaceClient() + if err != nil { + return err + } + endpoint, err := w.QualityMonitors.GetByTableName(ctx, d.Id()) + if err != nil { + return err + + } + return common.StructToData(endpoint, monitorSchema, d) + }, + Update: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { + w, err := c.WorkspaceClient() + if err != nil { + return err + } + var update catalog.UpdateMonitor + common.DataToStructPointer(d, monitorSchema, &update) + update.TableName = d.Get("table_name").(string) + _, err = w.QualityMonitors.Update(ctx, update) + if err != nil { + return err + } + return WaitForMonitor(w, ctx, update.TableName) + }, + Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { + w, err := c.WorkspaceClient() + if err != nil { + return err + } + return w.QualityMonitors.DeleteByTableName(ctx, d.Id()) + }, + Schema: monitorSchema, + Timeouts: &schema.ResourceTimeout{ + Create: schema.DefaultTimeout(qualityMonitorDefaultProvisionTimeout), + }, + } +} diff --git a/catalog/resource_lakehouse_monitor_test.go b/catalog/resource_quality_monitor_test.go similarity index 91% rename from catalog/resource_lakehouse_monitor_test.go rename to catalog/resource_quality_monitor_test.go index af1e6d852f..e9d062ed1c 100644 --- a/catalog/resource_lakehouse_monitor_test.go +++ b/catalog/resource_quality_monitor_test.go @@ -9,11 +9,11 @@ import ( "github.com/stretchr/testify/mock" ) -func TestLakehouseMonitorCornerCases(t *testing.T) { - qa.ResourceCornerCases(t, ResourceLakehouseMonitor()) +func TestQualityMonitorCornerCases(t *testing.T) { + qa.ResourceCornerCases(t, ResourceQualityMonitor()) } -func TestLakehouseMonitorCreateTimeseries(t *testing.T) { +func TestQualityMonitorCreateTimeseries(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { e := w.GetMockQualityMonitorsAPI().EXPECT() @@ -40,7 +40,7 @@ func TestLakehouseMonitorCreateTimeseries(t *testing.T) { DriftMetricsTableName: "test_table_drift", }, nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), HCL: ` table_name = "test_table", assets_dir = "sample.dir", @@ -54,7 +54,7 @@ func TestLakehouseMonitorCreateTimeseries(t *testing.T) { }.ApplyNoError(t) } -func TestLakehouseMonitorCreateInference(t *testing.T) { +func TestQualityMonitorCreateInference(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { e := w.GetMockQualityMonitorsAPI().EXPECT() @@ -89,7 +89,7 @@ func TestLakehouseMonitorCreateInference(t *testing.T) { }, }, nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), HCL: ` table_name = "test_table", assets_dir = "sample.dir", @@ -106,7 +106,7 @@ func TestLakehouseMonitorCreateInference(t *testing.T) { }.ApplyNoError(t) } -func TestLakehouseMonitorCreateSnapshot(t *testing.T) { +func TestQualityMonitorCreateSnapshot(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { e := w.GetMockQualityMonitorsAPI().EXPECT() @@ -129,7 +129,7 @@ func TestLakehouseMonitorCreateSnapshot(t *testing.T) { Snapshot: &catalog.MonitorSnapshot{}, }, nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), HCL: ` table_name = "test_table", assets_dir = "sample.dir", @@ -140,7 +140,7 @@ func TestLakehouseMonitorCreateSnapshot(t *testing.T) { }.ApplyNoError(t) } -func TestLakehouseMonitorGet(t *testing.T) { +func TestQualityMonitorGet(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { e := w.GetMockQualityMonitorsAPI().EXPECT() @@ -157,13 +157,13 @@ func TestLakehouseMonitorGet(t *testing.T) { }, DriftMetricsTableName: "test_table_drift"}, nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), Read: true, ID: "test_table", }.ApplyNoError(t) } -func TestLakehouseMonitorUpdate(t *testing.T) { +func TestQualityMonitorUpdate(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { e := w.GetMockQualityMonitorsAPI().EXPECT() @@ -204,7 +204,7 @@ func TestLakehouseMonitorUpdate(t *testing.T) { DriftMetricsTableName: "test_table_drift", }, nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), Update: true, ID: "test_table", InstanceState: map[string]string{ @@ -225,13 +225,13 @@ func TestLakehouseMonitorUpdate(t *testing.T) { }.ApplyNoError(t) } -func TestLakehouseMonitorDelete(t *testing.T) { +func TestQualityMonitorDelete(t *testing.T) { qa.ResourceFixture{ MockWorkspaceClientFunc: func(w *mocks.MockWorkspaceClient) { e := w.GetMockQualityMonitorsAPI().EXPECT() e.DeleteByTableName(mock.Anything, "test_table").Return(nil) }, - Resource: ResourceLakehouseMonitor(), + Resource: ResourceQualityMonitor(), Delete: true, ID: "test_table", }.ApplyNoError(t) diff --git a/docs/resources/lakehouse_monitor.md b/docs/resources/lakehouse_monitor.md index 33cbd56c01..d526f6fbc1 100644 --- a/docs/resources/lakehouse_monitor.md +++ b/docs/resources/lakehouse_monitor.md @@ -3,6 +3,8 @@ subcategory: "Unity Catalog" --- # databricks_lakehouse_monitor Resource +NOTE: This resource has been deprecated and will be removed soon. Please use the [databricks_quality_monitor resource](./quality_monitor.md) instead. + This resource allows you to manage [Lakehouse Monitors](https://docs.databricks.com/en/lakehouse-monitoring/index.html) in Databricks. A `databricks_lakehouse_monitor` is attached to a [databricks_sql_table](sql_table.md) and can be of type timeseries, snapshot or inference. diff --git a/docs/resources/quality_monitor.md b/docs/resources/quality_monitor.md new file mode 100644 index 0000000000..a3292f65d2 --- /dev/null +++ b/docs/resources/quality_monitor.md @@ -0,0 +1,137 @@ +--- +subcategory: "Unity Catalog" +--- +# databricks_quality_monitor Resource + +This resource allows you to manage [Lakehouse Monitors](https://docs.databricks.com/en/lakehouse-monitoring/index.html) in Databricks. + +A `databricks_quality_monitor` is attached to a [databricks_sql_table](sql_table.md) and can be of type timeseries, snapshot or inference. + +## Example Usage + +```hcl +resource "databricks_catalog" "sandbox" { + name = "sandbox" + comment = "this catalog is managed by terraform" + properties = { + purpose = "testing" + } +} + +resource "databricks_schema" "things" { + catalog_name = databricks_catalog.sandbox.id + name = "things" + comment = "this database is managed by terraform" + properties = { + kind = "various" + } +} + +resource "databricks_sql_table" "myTestTable" { + catalog_name = "main" + schema_name = databricks_schema.things.name + name = "bar" + table_type = "MANAGED" + data_source_format = "DELTA" + + column { + name = "timestamp" + position = 1 + type = "int" + } +} + +resource "databricks_quality_monitor" "testTimeseriesMonitor" { + table_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}.${databricks_sql_table.myTestTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myTestTable.name}" + output_schema_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}" + time_series { + granularities = ["1 hour"] + timestamp_col = "timestamp" + } +} +``` + +### Inference Monitor + +```hcl +resource "databricks_quality_monitor" "testMonitorInference" { + table_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}.${databricks_table.myTestTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_table.myTestTable.name}" + output_schema_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}" + inference_log { + granularities = ["1 hour"] + timestamp_col = "timestamp" + prediction_col = "prediction" + model_id_col = "model_id" + problem_type = "PROBLEM_TYPE_REGRESSION" + } +} +``` +### Snapshot Monitor +```hcl +resource "databricks_quality_monitor" "testMonitorInference" { + table_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}.${databricks_table.myTestTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_table.myTestTable.name}" + output_schema_name = "${databricks_catalog.sandbox.name}.${databricks_schema.things.name}" + snapshot {} +} +``` + +## Argument Reference + +The following arguments are supported: + +* `table_name` - (Required) - The full name of the table to attach the monitor too. Its of the format {catalog}.{schema}.{tableName} +* `assets_dir` - (Required) - The directory to store the monitoring assets (Eg. Dashboard and Metric Tables) +* `output_schema_name` - (Required) - Schema where output metric tables are created +* `baseline_table_name` - Name of the baseline table from which drift metrics are computed from.Columns in the monitored table should also be present in the baseline +table. +* `custom_metrics` - Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time windows). + * `definition` - [create metric definition](https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition) + * `input_columns` - Columns on the monitored table to apply the custom metrics to. + * `name` - Name of the custom metric. + * `output_data_type` - The output type of the custom metric. + * `type` - The type of the custom metric. +* `data_classification_config` - The data classification config for the monitor +* `inference_log` - Configuration for the inference log monitor + * `granularities` - List of granularities to use when aggregating data into time windows based on their timestamp. + * `label_col` - Column of the model label + * `model_id_col` - Column of the model id or version + * `prediction_col` - Column of the model prediction + * `prediction_proba_col` - Column of the model prediction probabilities + * `problem_type` - Problem type the model aims to solve. Either `PROBLEM_TYPE_CLASSIFICATION` or `PROBLEM_TYPE_REGRESSION` + * `timestamp_col` - Column of the timestamp of predictions +* `snapshot` - Configuration for monitoring snapshot tables. +* `time_series` - Configuration for monitoring timeseries tables. + * `granularities` - List of granularities to use when aggregating data into time windows based on their timestamp. + * `timestamp_col` - Column of the timestamp of predictions +* `notifications` - The notification settings for the monitor. The following optional blocks are supported, each consisting of the single string array field with name `email_addresses` containing a list of emails to notify: + * `on_failure` - who to send notifications to on monitor failure. + * `on_new_classification_tag_detected` - Who to send notifications to when new data classification tags are detected. +* `schedule` - The schedule for automatically updating and refreshing metric tables. This block consists of following fields: + * `quartz_cron_expression` - string expression that determines when to run the monitor. See [Quartz documentation](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for examples. + * `timezone_id` - string with timezone id (e.g., `PST`) in which to evaluate the Quartz expression. + * `pause_status` - optional string field that indicates whether a schedule is paused (`PAUSED`) or not (`UNPAUSED`). +* `skip_builtin_dashboard` - Whether to skip creating a default dashboard summarizing data quality metrics. +* `slicing_exprs` - List of column expressions to slice data with for targeted analysis. The data is grouped by each expression independently, resulting in a separate slice for each predicate and its complements. For high-cardinality columns, only the top 100 unique values by frequency will generate slices. +* `warehouse_id` - Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used. + +## Attribute Reference + +In addition to all arguments above, the following attributes are exported: + +* `id` - ID of this monitor is the same as the full table name of the format `{catalog}.{schema_name}.{table_name}` +* `monitor_version` - The version of the monitor config (e.g. 1,2,3). If negative, the monitor may be corrupted +* `drift_metrics_table_name` - The full name of the drift metrics table. Format: __catalog_name__.__schema_name__.__table_name__. +* `profile_metrics_table_name` - The full name of the profile metrics table. Format: __catalog_name__.__schema_name__.__table_name__. +* `status` - Status of the Monitor +* `dashboard_id` - The ID of the generated dashboard. + +## Related Resources + +The following resources are often used in the same context: + +* [databricks_catalog](catalog.md) +* [databricks_schema](schema.md) +* [databricks_sql_table](sql_table.md) diff --git a/internal/acceptance/lakehouse_monitor_test.go b/internal/acceptance/quality_monitor_test.go similarity index 68% rename from internal/acceptance/lakehouse_monitor_test.go rename to internal/acceptance/quality_monitor_test.go index 79f449259f..9b9da2c91c 100644 --- a/internal/acceptance/lakehouse_monitor_test.go +++ b/internal/acceptance/quality_monitor_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -var commonPartLakehouseMonitoring = `resource "databricks_catalog" "sandbox" { +var commonPartQualityMonitoring = `resource "databricks_catalog" "sandbox" { name = "sandbox{var.STICKY_RANDOM}" comment = "this catalog is managed by terraform" properties = { @@ -46,16 +46,16 @@ resource "databricks_sql_table" "myInferenceTable" { ` -func TestUcAccLakehouseMonitor(t *testing.T) { +func TestUcAccQualityMonitor(t *testing.T) { if os.Getenv("GOOGLE_CREDENTIALS") != "" { - t.Skipf("databricks_lakehouse_monitor resource is not available on GCP") + t.Skipf("databricks_quality_monitor resource is not available on GCP") } unityWorkspaceLevel(t, step{ - Template: commonPartLakehouseMonitoring + ` + Template: commonPartQualityMonitoring + ` - resource "databricks_lakehouse_monitor" "testMonitorInference" { + resource "databricks_quality_monitor" "testMonitorInference" { table_name = databricks_sql_table.myInferenceTable.id - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myInferenceTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}" output_schema_name = databricks_schema.things.id inference_log { granularities = ["1 day"] @@ -79,9 +79,9 @@ func TestUcAccLakehouseMonitor(t *testing.T) { } } - resource "databricks_lakehouse_monitor" "testMonitorTimeseries" { + resource "databricks_quality_monitor" "testMonitorTimeseries" { table_name = databricks_sql_table.myTimeseries.id - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myTimeseries.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myTimeseries.name}" output_schema_name = databricks_schema.things.id time_series { granularities = ["1 day"] @@ -102,9 +102,9 @@ func TestUcAccLakehouseMonitor(t *testing.T) { } } - resource "databricks_lakehouse_monitor" "testMonitorSnapshot" { + resource "databricks_quality_monitor" "testMonitorSnapshot" { table_name = databricks_sql_table.mySnapshot.id - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myTimeseries.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myTimeseries.name}" output_schema_name = databricks_schema.things.id snapshot { } @@ -113,15 +113,15 @@ func TestUcAccLakehouseMonitor(t *testing.T) { }) } -func TestUcAccUpdateLakehouseMonitor(t *testing.T) { +func TestUcAccUpdateQualityMonitor(t *testing.T) { if os.Getenv("GOOGLE_CREDENTIALS") != "" { - t.Skipf("databricks_lakehouse_monitor resource is not available on GCP") + t.Skipf("databricks_quality_monitor resource is not available on GCP") } unityWorkspaceLevel(t, step{ - Template: commonPartLakehouseMonitoring + ` - resource "databricks_lakehouse_monitor" "testMonitorInference" { + Template: commonPartQualityMonitoring + ` + resource "databricks_quality_monitor" "testMonitorInference" { table_name = databricks_sql_table.myInferenceTable.id - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myInferenceTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}" output_schema_name = databricks_schema.things.id inference_log { granularities = ["1 day"] @@ -133,10 +133,10 @@ func TestUcAccUpdateLakehouseMonitor(t *testing.T) { } `, }, step{ - Template: commonPartLakehouseMonitoring + ` - resource "databricks_lakehouse_monitor" "testMonitorInference" { + Template: commonPartQualityMonitoring + ` + resource "databricks_quality_monitor" "testMonitorInference" { table_name = databricks_sql_table.myInferenceTable.id - assets_dir = "/Shared/provider-test/databricks_lakehouse_monitoring/${databricks_sql_table.myInferenceTable.name}" + assets_dir = "/Shared/provider-test/databricks_quality_monitoring/${databricks_sql_table.myInferenceTable.name}" output_schema_name = databricks_schema.things.id inference_log { granularities = ["1 hour"] diff --git a/provider/provider.go b/provider/provider.go index 683f9a819d..d0d0d024e5 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -174,6 +174,7 @@ func DatabricksProvider() *schema.Provider { "databricks_permissions": permissions.ResourcePermissions().ToResource(), "databricks_pipeline": pipelines.ResourcePipeline().ToResource(), "databricks_provider": sharing.ResourceProvider().ToResource(), + "databricks_quality_monitor": catalog.ResourceQualityMonitor().ToResource(), "databricks_recipient": sharing.ResourceRecipient().ToResource(), "databricks_registered_model": catalog.ResourceRegisteredModel().ToResource(), "databricks_repo": repos.ResourceRepo().ToResource(),