Skip to content

Commit

Permalink
[Exporter] Emit WSFS/UC files from parameters in the jobs (#4318)
Browse files Browse the repository at this point in the history
## Changes
<!-- Summary of your changes that are easy to understand -->

The global `parameters` array may contain references to files on
Workspace File System (WSFS) or UC Volumes. This PR adds emitting of
these files and handling references

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [ ] relevant change in `docs/` folder
- [ ] covered with integration tests in `internal/acceptance`
- [ ] using Go SDK
- [ ] using TF Plugin Framework
  • Loading branch information
alexott authored Dec 17, 2024
1 parent 2b97994 commit d2bb4e5
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 7 deletions.
22 changes: 17 additions & 5 deletions exporter/importables.go
Original file line number Diff line number Diff line change
Expand Up @@ -444,16 +444,23 @@ var resourcesMap map[string]importable = map[string]importable{
{Path: "task.sql_task.dashboard.dashboard_id", Resource: "databricks_sql_dashboard"},
{Path: "task.sql_task.query.query_id", Resource: "databricks_query"},
{Path: "task.sql_task.warehouse_id", Resource: "databricks_sql_endpoint"},
{Path: "task.webhook_notifications.on_duration_warning_threshold_exceeded.id", Resource: "databricks_notification_destination"},
{Path: "task.webhook_notifications.on_duration_warning_threshold_exceeded.id",
Resource: "databricks_notification_destination"},
{Path: "task.webhook_notifications.on_failure.id", Resource: "databricks_notification_destination"},
{Path: "task.webhook_notifications.on_start.id", Resource: "databricks_notification_destination"},
{Path: "task.webhook_notifications.on_success.id", Resource: "databricks_notification_destination"},
{Path: "task.webhook_notifications.on_streaming_backlog_exceeded.id", Resource: "databricks_notification_destination"},
{Path: "parameter.default", Resource: "databricks_workspace_file", Match: "workspace_path"},
{Path: "parameter.default", Resource: "databricks_workspace_file", Match: "path"},
{Path: "parameter.default", Resource: "databricks_file", Match: "path"},
{Path: "task.email_notifications.on_duration_warning_threshold_exceeded", Resource: "databricks_user",
Match: "user_name", MatchType: MatchCaseInsensitive},
{Path: "task.email_notifications.on_failure", Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
{Path: "task.email_notifications.on_start", Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
{Path: "task.email_notifications.on_success", Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
{Path: "task.email_notifications.on_failure", Resource: "databricks_user", Match: "user_name",
MatchType: MatchCaseInsensitive},
{Path: "task.email_notifications.on_start", Resource: "databricks_user", Match: "user_name",
MatchType: MatchCaseInsensitive},
{Path: "task.email_notifications.on_success", Resource: "databricks_user", Match: "user_name",
MatchType: MatchCaseInsensitive},
{Path: "task.email_notifications.on_streaming_backlog_exceeded", Resource: "databricks_user",
Match: "user_name", MatchType: MatchCaseInsensitive},
{Path: "run_as.user_name", Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
Expand Down Expand Up @@ -497,7 +504,8 @@ var resourcesMap map[string]importable = map[string]importable{
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
{Path: "job_cluster.new_cluster.init_scripts.workspace.destination", Resource: "databricks_repo", Match: "workspace_path",
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
{Path: "job_cluster.new_cluster.init_scripts.workspace.destination", Resource: "databricks_repo", Match: "path",
{Path: "job_cluster.new_cluster.init_scripts.workspace.destination", Resource: "databricks_repo", Match: "path"},
{Path: "parameter.default", Resource: "databricks_repo", Match: "workspace_path",
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
},
Import: func(ic *importContext, r *resource) error {
Expand Down Expand Up @@ -669,6 +677,10 @@ var resourcesMap map[string]importable = map[string]importable{
ic.emitJobsDestinationNotifications(job.WebhookNotifications.OnStart)
ic.emitJobsDestinationNotifications(job.WebhookNotifications.OnStreamingBacklogExceeded)
}
for _, param := range job.Parameters {
ic.emitIfWsfsFile(param.Default)
ic.emitIfVolumeFile(param.Default)
}

return ic.importLibraries(r.Data, s)
},
Expand Down
4 changes: 2 additions & 2 deletions exporter/util_workspace.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ func hasWorkspacePrefix(path string) bool {
}

func maybeStripWorkspacePrefix(path string) string {
if hasWorkspacePrefix(path) {
return path[10:]
if strings.HasPrefix(path, "/Workspace/") {
return strings.TrimPrefix(path, "/Workspace")
}
return path
}
Expand Down

0 comments on commit d2bb4e5

Please sign in to comment.