Migrate bundle/tests/undefined_resources_test.go to acceptance test (#…

…2106)

Add sort_blocks.py helper to deal with non-determinism.

acceptance/acceptance_test.go

@@ -35,10 +35,16 @@ var Scripts = map[string]bool{
 }
 
 func TestAccept(t *testing.T) {
-	execPath := BuildCLI(t)
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+
+	execPath := BuildCLI(t, cwd)
 	// $CLI is what test scripts are using
 	t.Setenv("CLI", execPath)
 
+	// Make helper scripts available
+	t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
+
 	server := StartServer(t)
 	AddHandlers(server)
 	// Redirect API access to local server:
@@ -199,9 +205,7 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	return strings.Join(prepares, "\n")
 }
 
-func BuildCLI(t *testing.T) string {
-	cwd, err := os.Getwd()
-	require.NoError(t, err)
+func BuildCLI(t *testing.T, cwd string) string {
 	execPath := filepath.Join(cwd, "build", "databricks")
 	if runtime.GOOS == "windows" {
 		execPath += ".exe"

acceptance/bin/sort_blocks.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""
+Helper to sort blocks in text file. A block is a set of lines separated from others by empty line.
+
+This is to workaround non-determinism in the output.
+"""
+import sys
+
+blocks = []
+
+for line in sys.stdin:
+    if not line.strip():
+        if blocks and blocks[-1]:
+            blocks.append('')
+        continue
+    if not blocks:
+        blocks.append('')
+    blocks[-1] += line
+
+blocks.sort()
+print("\n".join(blocks))

acceptance/bundle/undefined_resources/output.txt

@@ -0,0 +1,19 @@
+Error: experiment undefined-experiment is not defined
+  at resources.experiments.undefined-experiment
+  in databricks.yml:11:26
+
+Error: job undefined-job is not defined
+  at resources.jobs.undefined-job
+  in databricks.yml:6:19
+
+Error: pipeline undefined-pipeline is not defined
+  at resources.pipelines.undefined-pipeline
+  in databricks.yml:14:24
+
+Found 3 errors
+
+Name: undefined-job
+Target: default
+
+
+Exit code: 1

acceptance/bundle/undefined_resources/script

@@ -0,0 +1,2 @@
+# We need sort_blocks.py because the order of diagnostics is currently randomized
+$CLI bundle validate 2>&1 | sort_blocks.py