-
Notifications
You must be signed in to change notification settings - Fork 63
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Handle
${workspace.file_path} references in source-linked deploymen…
…ts (#2046) ## Changes 1. Updates `workspace.file_path` during source-linked deployment to address cases like this https://github.com/databricks/bundle-examples/blob/main/default_python/resources/default_python_pipeline.yml#L13 2. Updates `workspace.file_path` in `metadata.json` 3. Prints warning for users when `workspace.file_path` is explicitly set but deploy is running in source-linked mode ## Tests Unit test
- Loading branch information
1 parent
185bbd2
commit 0289bec
Showing
13 changed files
with
301 additions
and
161 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
75 changes: 75 additions & 0 deletions
75
bundle/config/mutator/apply_source_linked_deployment_preset.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| package mutator | ||
|
|
||
| import ( | ||
| "context" | ||
| "strings" | ||
|
|
||
| "github.com/databricks/cli/bundle" | ||
| "github.com/databricks/cli/bundle/config" | ||
| "github.com/databricks/cli/libs/dbr" | ||
| "github.com/databricks/cli/libs/diag" | ||
| "github.com/databricks/cli/libs/dyn" | ||
| ) | ||
|
|
||
| type applySourceLinkedDeploymentPreset struct{} | ||
|
|
||
| // Apply source-linked deployment preset | ||
| func ApplySourceLinkedDeploymentPreset() *applySourceLinkedDeploymentPreset { | ||
| return &applySourceLinkedDeploymentPreset{} | ||
| } | ||
|
|
||
| func (m *applySourceLinkedDeploymentPreset) Name() string { | ||
| return "ApplySourceLinkedDeploymentPreset" | ||
| } | ||
|
|
||
| func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { | ||
| if config.IsExplicitlyDisabled(b.Config.Presets.SourceLinkedDeployment) { | ||
| return nil | ||
| } | ||
|
|
||
| var diags diag.Diagnostics | ||
| isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/") | ||
| target := b.Config.Bundle.Target | ||
|
|
||
| if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) { | ||
| if !isDatabricksWorkspace { | ||
| path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment")) | ||
| diags = diags.Append( | ||
| diag.Diagnostic{ | ||
| Severity: diag.Warning, | ||
| Summary: "source-linked deployment is available only in the Databricks Workspace", | ||
| Paths: []dyn.Path{ | ||
| path, | ||
| }, | ||
| Locations: b.Config.GetLocations(path[2:].String()), | ||
| }, | ||
| ) | ||
|
|
||
| disabled := false | ||
| b.Config.Presets.SourceLinkedDeployment = &disabled | ||
| return diags | ||
| } | ||
| } | ||
|
|
||
| if isDatabricksWorkspace && b.Config.Bundle.Mode == config.Development { | ||
| enabled := true | ||
| b.Config.Presets.SourceLinkedDeployment = &enabled | ||
| } | ||
|
|
||
| if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { | ||
| path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("workspace"), dyn.Key("file_path")) | ||
|
|
||
| diags = diags.Append( | ||
| diag.Diagnostic{ | ||
| Severity: diag.Warning, | ||
| Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", | ||
| Paths: []dyn.Path{ | ||
| path[2:], | ||
| }, | ||
| Locations: b.Config.GetLocations(path[2:].String()), | ||
| }, | ||
| ) | ||
| } | ||
|
|
||
| return diags | ||
| } |
122 changes: 122 additions & 0 deletions
122
bundle/config/mutator/apply_source_linked_deployment_preset_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,122 @@ | ||
| package mutator_test | ||
|
|
||
| import ( | ||
| "context" | ||
| "runtime" | ||
| "testing" | ||
|
|
||
| "github.com/databricks/cli/bundle" | ||
| "github.com/databricks/cli/bundle/config" | ||
| "github.com/databricks/cli/bundle/config/mutator" | ||
| "github.com/databricks/cli/bundle/internal/bundletest" | ||
| "github.com/databricks/cli/libs/dbr" | ||
| "github.com/databricks/cli/libs/dyn" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestApplyPresetsSourceLinkedDeployment(t *testing.T) { | ||
| if runtime.GOOS == "windows" { | ||
| t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace") | ||
| } | ||
|
|
||
| testContext := context.Background() | ||
| enabled := true | ||
| disabled := false | ||
| workspacePath := "/Workspace/user.name@company.com" | ||
|
|
||
| tests := []struct { | ||
| name string | ||
| ctx context.Context | ||
| mutateBundle func(b *bundle.Bundle) | ||
| initialValue *bool | ||
| expectedValue *bool | ||
| expectedWarning string | ||
| }{ | ||
| { | ||
| name: "preset enabled, bundle in Workspace, databricks runtime", | ||
| ctx: dbr.MockRuntime(testContext, true), | ||
| initialValue: &enabled, | ||
| expectedValue: &enabled, | ||
| }, | ||
| { | ||
| name: "preset enabled, bundle not in Workspace, databricks runtime", | ||
| ctx: dbr.MockRuntime(testContext, true), | ||
| mutateBundle: func(b *bundle.Bundle) { | ||
| b.SyncRootPath = "/Users/user.name@company.com" | ||
| }, | ||
| initialValue: &enabled, | ||
| expectedValue: &disabled, | ||
| expectedWarning: "source-linked deployment is available only in the Databricks Workspace", | ||
| }, | ||
| { | ||
| name: "preset enabled, bundle in Workspace, not databricks runtime", | ||
| ctx: dbr.MockRuntime(testContext, false), | ||
| initialValue: &enabled, | ||
| expectedValue: &disabled, | ||
| expectedWarning: "source-linked deployment is available only in the Databricks Workspace", | ||
| }, | ||
| { | ||
| name: "preset disabled, bundle in Workspace, databricks runtime", | ||
| ctx: dbr.MockRuntime(testContext, true), | ||
| initialValue: &disabled, | ||
| expectedValue: &disabled, | ||
| }, | ||
| { | ||
| name: "preset nil, bundle in Workspace, databricks runtime", | ||
| ctx: dbr.MockRuntime(testContext, true), | ||
| initialValue: nil, | ||
| expectedValue: nil, | ||
| }, | ||
| { | ||
| name: "preset nil, dev mode true, bundle in Workspace, databricks runtime", | ||
| ctx: dbr.MockRuntime(testContext, true), | ||
| mutateBundle: func(b *bundle.Bundle) { | ||
| b.Config.Bundle.Mode = config.Development | ||
| }, | ||
| initialValue: nil, | ||
| expectedValue: &enabled, | ||
| }, | ||
| { | ||
| name: "preset enabled, workspace.file_path is defined by user", | ||
| ctx: dbr.MockRuntime(testContext, true), | ||
| mutateBundle: func(b *bundle.Bundle) { | ||
| b.Config.Workspace.FilePath = "file_path" | ||
| }, | ||
| initialValue: &enabled, | ||
| expectedValue: &enabled, | ||
| expectedWarning: "workspace.file_path setting will be ignored in source-linked deployment mode", | ||
| }, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| b := &bundle.Bundle{ | ||
| SyncRootPath: workspacePath, | ||
| Config: config.Root{ | ||
| Presets: config.Presets{ | ||
| SourceLinkedDeployment: tt.initialValue, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| if tt.mutateBundle != nil { | ||
| tt.mutateBundle(b) | ||
| } | ||
|
|
||
| bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}}) | ||
| bundletest.SetLocation(b, "workspace.file_path", []dyn.Location{{File: "databricks.yml"}}) | ||
|
|
||
| diags := bundle.Apply(tt.ctx, b, mutator.ApplySourceLinkedDeploymentPreset()) | ||
| if diags.HasError() { | ||
| t.Fatalf("unexpected error: %v", diags) | ||
| } | ||
|
|
||
| if tt.expectedWarning != "" { | ||
| require.Equal(t, tt.expectedWarning, diags[0].Summary) | ||
| require.NotEmpty(t, diags[0].Locations) | ||
| } | ||
|
|
||
| require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.