From 43ac922bb315c88112a7e485bcc6e427c50b57bb Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 10 Jan 2025 18:42:32 +0100 Subject: [PATCH 1/5] Move existing path translation tests --- .../bundle/paths/original_fallback}/README.md | 0 .../paths/original_fallback}/databricks.yml | 2 +- .../original_fallback/output.development.json | 125 +++++++++++++++ .../bundle/paths/original_fallback/output.txt | 76 +++++++++ .../paths/original_fallback}/override_job.yml | 0 .../original_fallback}/override_pipeline.yml | 0 .../original_fallback}/resources/my_job.yml | 12 ++ .../resources/my_pipeline.yml | 0 .../bundle/paths/original_fallback/script | 54 +++++++ .../src/dbt_project/.gitkeep | 0 .../paths/original_fallback}/src/file.py | 0 .../paths/original_fallback}/src/file1.py | 0 .../paths/original_fallback}/src/file2.py | 0 .../paths/original_fallback}/src/notebook.py | 0 .../paths/original_fallback}/src/notebook1.py | 0 .../paths/original_fallback}/src/notebook2.py | 0 .../paths/original_fallback}/src/sql.sql | 0 .../bundle/paths/original_nominal}/README.md | 0 .../paths/original_nominal}/databricks.yml | 2 +- .../original_nominal/output.development.json | 147 ++++++++++++++++++ .../bundle/paths/original_nominal/output.txt | 88 +++++++++++ .../paths/original_nominal}/override_job.yml | 0 .../original_nominal}/override_pipeline.yml | 0 .../original_nominal}/resources/my_job.yml | 14 ++ .../resources/my_pipeline.yml | 0 .../bundle/paths/original_nominal/script | 64 ++++++++ .../src/dbt_project/.gitkeep | 0 .../paths/original_nominal}/src/file.py | 0 .../paths/original_nominal}/src/file1.py | 0 .../paths/original_nominal}/src/file2.py | 0 .../paths/original_nominal}/src/notebook.py | 0 .../paths/original_nominal}/src/notebook1.py | 0 .../paths/original_nominal}/src/notebook2.py | 0 .../paths/original_nominal}/src/sql.sql | 0 .../databricks.yml | 0 .../output.default.json | 27 ++++ .../output.override.json | 27 ++++ .../output.txt | 50 ++++++ .../resources/job.yml | 0 .../original_relative_path_translation/script | 12 ++ .../src/file1.py | 0 .../src/file2.py | 0 bundle/tests/path_translation_test.go | 112 ------------- .../tests/relative_path_translation_test.go | 28 ---- 44 files changed, 698 insertions(+), 142 deletions(-) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/README.md (100%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_fallback}/databricks.yml (80%) create mode 100644 acceptance/bundle/paths/original_fallback/output.development.json create mode 100644 acceptance/bundle/paths/original_fallback/output.txt rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/override_job.yml (100%) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/override_pipeline.yml (100%) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/resources/my_job.yml (71%) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/resources/my_pipeline.yml (100%) create mode 100644 acceptance/bundle/paths/original_fallback/script rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/src/dbt_project/.gitkeep (100%) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/src/file.py (100%) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/src/file1.py (100%) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/src/file2.py (100%) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/src/notebook.py (100%) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/src/notebook1.py (100%) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/src/notebook2.py (100%) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_fallback}/src/sql.sql (100%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/README.md (100%) rename {bundle/tests/path_translation/fallback => acceptance/bundle/paths/original_nominal}/databricks.yml (79%) create mode 100644 acceptance/bundle/paths/original_nominal/output.development.json create mode 100644 acceptance/bundle/paths/original_nominal/output.txt rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/override_job.yml (100%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/override_pipeline.yml (100%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/resources/my_job.yml (74%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/resources/my_pipeline.yml (100%) create mode 100644 acceptance/bundle/paths/original_nominal/script rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/src/dbt_project/.gitkeep (100%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/src/file.py (100%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/src/file1.py (100%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/src/file2.py (100%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/src/notebook.py (100%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/src/notebook1.py (100%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/src/notebook2.py (100%) rename {bundle/tests/path_translation/nominal => acceptance/bundle/paths/original_nominal}/src/sql.sql (100%) rename {bundle/tests/relative_path_translation => acceptance/bundle/paths/original_relative_path_translation}/databricks.yml (100%) create mode 100644 acceptance/bundle/paths/original_relative_path_translation/output.default.json create mode 100644 acceptance/bundle/paths/original_relative_path_translation/output.override.json create mode 100644 acceptance/bundle/paths/original_relative_path_translation/output.txt rename {bundle/tests/relative_path_translation => acceptance/bundle/paths/original_relative_path_translation}/resources/job.yml (100%) create mode 100644 acceptance/bundle/paths/original_relative_path_translation/script rename {bundle/tests/relative_path_translation => acceptance/bundle/paths/original_relative_path_translation}/src/file1.py (100%) rename {bundle/tests/relative_path_translation => acceptance/bundle/paths/original_relative_path_translation}/src/file2.py (100%) delete mode 100644 bundle/tests/path_translation_test.go delete mode 100644 bundle/tests/relative_path_translation_test.go diff --git a/bundle/tests/path_translation/fallback/README.md b/acceptance/bundle/paths/original_fallback/README.md similarity index 100% rename from bundle/tests/path_translation/fallback/README.md rename to acceptance/bundle/paths/original_fallback/README.md diff --git a/bundle/tests/path_translation/nominal/databricks.yml b/acceptance/bundle/paths/original_fallback/databricks.yml similarity index 80% rename from bundle/tests/path_translation/nominal/databricks.yml rename to acceptance/bundle/paths/original_fallback/databricks.yml index cd425920de..9608b5332e 100644 --- a/bundle/tests/path_translation/nominal/databricks.yml +++ b/acceptance/bundle/paths/original_fallback/databricks.yml @@ -1,5 +1,5 @@ bundle: - name: path_translation_nominal + name: original_fallback include: - "resources/*.yml" diff --git a/acceptance/bundle/paths/original_fallback/output.development.json b/acceptance/bundle/paths/original_fallback/output.development.json new file mode 100644 index 0000000000..ba5229036d --- /dev/null +++ b/acceptance/bundle/paths/original_fallback/output.development.json @@ -0,0 +1,125 @@ +{ + "jobs": { + "my_job": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "default", + "new_cluster": { + "spark_version": "15.4.x-scala2.12" + } + } + ], + "name": "placeholder", + "permissions": [], + "queue": { + "enabled": true + }, + "tags": {}, + "tasks": [ + { + "job_cluster_key": "default", + "notebook_task": { + "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/notebook" + }, + "task_key": "notebook_example" + }, + { + "job_cluster_key": "default", + "spark_python_task": { + "python_file": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/file.py" + }, + "task_key": "spark_python_example" + }, + { + "dbt_task": { + "commands": [ + "dbt run", + "dbt run" + ], + "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/dbt_project" + }, + "job_cluster_key": "default", + "task_key": "dbt_example" + }, + { + "job_cluster_key": "default", + "sql_task": { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/sql.sql" + }, + "warehouse_id": "cafef00d" + }, + "task_key": "sql_example" + }, + { + "job_cluster_key": "default", + "libraries": [ + { + "whl": "dist/wheel1.whl" + }, + { + "whl": "dist/wheel2.whl" + } + ], + "python_wheel_task": { + "package_name": "my_package" + }, + "task_key": "python_wheel_example" + }, + { + "job_cluster_key": "default", + "libraries": [ + { + "jar": "target/jar1.jar" + }, + { + "jar": "target/jar2.jar" + } + ], + "spark_jar_task": { + "main_class_name": "com.example.Main" + }, + "task_key": "spark_jar_example" + } + ] + } + }, + "pipelines": { + "my_pipeline": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/state/metadata.json" + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/file1.py" + } + }, + { + "notebook": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/notebook1" + } + }, + { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/file2.py" + } + }, + { + "notebook": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/notebook2" + } + } + ], + "name": "placeholder", + "permissions": [] + } + } +} diff --git a/acceptance/bundle/paths/original_fallback/output.txt b/acceptance/bundle/paths/original_fallback/output.txt new file mode 100644 index 0000000000..831a47f390 --- /dev/null +++ b/acceptance/bundle/paths/original_fallback/output.txt @@ -0,0 +1,76 @@ + +>>> $CLI bundle validate -t development -o json + +Exit code: 0 + +>>> $CLI bundle validate -t error +Error: notebook this value is overridden not found. Local notebook references are expected +to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb] + +Name: original_fallback +Target: error +Workspace: + User: tester@databricks.com + Path: /Workspace/Users/tester@databricks.com/.bundle/original_fallback/error + +Found 1 error + +Exit code: 1 + +>>> jq -e +.jobs.my_job.tasks[0].task_key == "notebook_example" and +(.jobs.my_job.tasks[0].notebook_task.notebook_path | endswith("/src/notebook")) + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[1].task_key == "spark_python_example" and +(.jobs.my_job.tasks[1].spark_python_task.python_file | endswith("/src/file.py")) + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[2].task_key == "dbt_example" and +(.jobs.my_job.tasks[2].dbt_task.project_directory | endswith("/src/dbt_project")) + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[3].task_key == "sql_example" and +(.jobs.my_job.tasks[3].sql_task.file.path | endswith("/src/sql.sql")) + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[4].task_key == "python_wheel_example" and +(.jobs.my_job.tasks[4].libraries[0].whl == "dist/wheel1.whl") and +(.jobs.my_job.tasks[4].libraries[1].whl == "dist/wheel2.whl") + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[5].task_key == "spark_jar_example" and +(.jobs.my_job.tasks[5].libraries[0].jar == "target/jar1.jar") and +(.jobs.my_job.tasks[5].libraries[1].jar == "target/jar2.jar") + output.development.json +true + +>>> jq -e +(.pipelines.my_pipeline.libraries[0].file.path | endswith("/src/file1.py")) + output.development.json +true + +>>> jq -e +(.pipelines.my_pipeline.libraries[1].notebook.path | endswith("/src/notebook1")) + output.development.json +true + +>>> jq -e +(.pipelines.my_pipeline.libraries[2].file.path | endswith("/src/file2.py")) + output.development.json +true + +>>> jq -e +(.pipelines.my_pipeline.libraries[3].notebook.path | endswith("/src/notebook2")) + output.development.json +true diff --git a/bundle/tests/path_translation/fallback/override_job.yml b/acceptance/bundle/paths/original_fallback/override_job.yml similarity index 100% rename from bundle/tests/path_translation/fallback/override_job.yml rename to acceptance/bundle/paths/original_fallback/override_job.yml diff --git a/bundle/tests/path_translation/fallback/override_pipeline.yml b/acceptance/bundle/paths/original_fallback/override_pipeline.yml similarity index 100% rename from bundle/tests/path_translation/fallback/override_pipeline.yml rename to acceptance/bundle/paths/original_fallback/override_pipeline.yml diff --git a/bundle/tests/path_translation/fallback/resources/my_job.yml b/acceptance/bundle/paths/original_fallback/resources/my_job.yml similarity index 71% rename from bundle/tests/path_translation/fallback/resources/my_job.yml rename to acceptance/bundle/paths/original_fallback/resources/my_job.yml index 4907df4f04..921ee412b9 100644 --- a/bundle/tests/path_translation/fallback/resources/my_job.yml +++ b/acceptance/bundle/paths/original_fallback/resources/my_job.yml @@ -4,33 +4,45 @@ resources: name: "placeholder" tasks: - task_key: notebook_example + job_cluster_key: default notebook_task: notebook_path: "this value is overridden" - task_key: spark_python_example + job_cluster_key: default spark_python_task: python_file: "this value is overridden" - task_key: dbt_example + job_cluster_key: default dbt_task: project_directory: "this value is overridden" commands: - "dbt run" - task_key: sql_example + job_cluster_key: default sql_task: file: path: "this value is overridden" warehouse_id: cafef00d - task_key: python_wheel_example + job_cluster_key: default python_wheel_task: package_name: my_package libraries: - whl: ../dist/wheel1.whl - task_key: spark_jar_example + job_cluster_key: default spark_jar_task: main_class_name: com.example.Main libraries: - jar: ../target/jar1.jar + + # Include a job cluster for completeness + job_clusters: + - job_cluster_key: default + new_cluster: + spark_version: 15.4.x-scala2.12 diff --git a/bundle/tests/path_translation/fallback/resources/my_pipeline.yml b/acceptance/bundle/paths/original_fallback/resources/my_pipeline.yml similarity index 100% rename from bundle/tests/path_translation/fallback/resources/my_pipeline.yml rename to acceptance/bundle/paths/original_fallback/resources/my_pipeline.yml diff --git a/acceptance/bundle/paths/original_fallback/script b/acceptance/bundle/paths/original_fallback/script new file mode 100644 index 0000000000..bbad0ad01f --- /dev/null +++ b/acceptance/bundle/paths/original_fallback/script @@ -0,0 +1,54 @@ +errcode trace $CLI bundle validate -t development -o json | jq '.resources' > output.development.json +errcode trace $CLI bundle validate -t error + +# Job assertions + +trace jq -e ' +.jobs.my_job.tasks[0].task_key == "notebook_example" and +(.jobs.my_job.tasks[0].notebook_task.notebook_path | endswith("/src/notebook")) +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[1].task_key == "spark_python_example" and +(.jobs.my_job.tasks[1].spark_python_task.python_file | endswith("/src/file.py")) +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[2].task_key == "dbt_example" and +(.jobs.my_job.tasks[2].dbt_task.project_directory | endswith("/src/dbt_project")) +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[3].task_key == "sql_example" and +(.jobs.my_job.tasks[3].sql_task.file.path | endswith("/src/sql.sql")) +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[4].task_key == "python_wheel_example" and +(.jobs.my_job.tasks[4].libraries[0].whl == "dist/wheel1.whl") and +(.jobs.my_job.tasks[4].libraries[1].whl == "dist/wheel2.whl") +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[5].task_key == "spark_jar_example" and +(.jobs.my_job.tasks[5].libraries[0].jar == "target/jar1.jar") and +(.jobs.my_job.tasks[5].libraries[1].jar == "target/jar2.jar") +' output.development.json + +# Pipeline assertions + +trace jq -e ' +(.pipelines.my_pipeline.libraries[0].file.path | endswith("/src/file1.py")) +' output.development.json + +trace jq -e ' +(.pipelines.my_pipeline.libraries[1].notebook.path | endswith("/src/notebook1")) +' output.development.json + +trace jq -e ' +(.pipelines.my_pipeline.libraries[2].file.path | endswith("/src/file2.py")) +' output.development.json + +trace jq -e ' +(.pipelines.my_pipeline.libraries[3].notebook.path | endswith("/src/notebook2")) +' output.development.json diff --git a/bundle/tests/path_translation/fallback/src/dbt_project/.gitkeep b/acceptance/bundle/paths/original_fallback/src/dbt_project/.gitkeep similarity index 100% rename from bundle/tests/path_translation/fallback/src/dbt_project/.gitkeep rename to acceptance/bundle/paths/original_fallback/src/dbt_project/.gitkeep diff --git a/bundle/tests/path_translation/fallback/src/file.py b/acceptance/bundle/paths/original_fallback/src/file.py similarity index 100% rename from bundle/tests/path_translation/fallback/src/file.py rename to acceptance/bundle/paths/original_fallback/src/file.py diff --git a/bundle/tests/path_translation/fallback/src/file1.py b/acceptance/bundle/paths/original_fallback/src/file1.py similarity index 100% rename from bundle/tests/path_translation/fallback/src/file1.py rename to acceptance/bundle/paths/original_fallback/src/file1.py diff --git a/bundle/tests/path_translation/fallback/src/file2.py b/acceptance/bundle/paths/original_fallback/src/file2.py similarity index 100% rename from bundle/tests/path_translation/fallback/src/file2.py rename to acceptance/bundle/paths/original_fallback/src/file2.py diff --git a/bundle/tests/path_translation/fallback/src/notebook.py b/acceptance/bundle/paths/original_fallback/src/notebook.py similarity index 100% rename from bundle/tests/path_translation/fallback/src/notebook.py rename to acceptance/bundle/paths/original_fallback/src/notebook.py diff --git a/bundle/tests/path_translation/fallback/src/notebook1.py b/acceptance/bundle/paths/original_fallback/src/notebook1.py similarity index 100% rename from bundle/tests/path_translation/fallback/src/notebook1.py rename to acceptance/bundle/paths/original_fallback/src/notebook1.py diff --git a/bundle/tests/path_translation/fallback/src/notebook2.py b/acceptance/bundle/paths/original_fallback/src/notebook2.py similarity index 100% rename from bundle/tests/path_translation/fallback/src/notebook2.py rename to acceptance/bundle/paths/original_fallback/src/notebook2.py diff --git a/bundle/tests/path_translation/fallback/src/sql.sql b/acceptance/bundle/paths/original_fallback/src/sql.sql similarity index 100% rename from bundle/tests/path_translation/fallback/src/sql.sql rename to acceptance/bundle/paths/original_fallback/src/sql.sql diff --git a/bundle/tests/path_translation/nominal/README.md b/acceptance/bundle/paths/original_nominal/README.md similarity index 100% rename from bundle/tests/path_translation/nominal/README.md rename to acceptance/bundle/paths/original_nominal/README.md diff --git a/bundle/tests/path_translation/fallback/databricks.yml b/acceptance/bundle/paths/original_nominal/databricks.yml similarity index 79% rename from bundle/tests/path_translation/fallback/databricks.yml rename to acceptance/bundle/paths/original_nominal/databricks.yml index 92be3f9217..8759ec2f90 100644 --- a/bundle/tests/path_translation/fallback/databricks.yml +++ b/acceptance/bundle/paths/original_nominal/databricks.yml @@ -1,5 +1,5 @@ bundle: - name: path_translation_fallback + name: original_nominal include: - "resources/*.yml" diff --git a/acceptance/bundle/paths/original_nominal/output.development.json b/acceptance/bundle/paths/original_nominal/output.development.json new file mode 100644 index 0000000000..93cbf56f97 --- /dev/null +++ b/acceptance/bundle/paths/original_nominal/output.development.json @@ -0,0 +1,147 @@ +{ + "jobs": { + "my_job": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "default", + "new_cluster": { + "spark_version": "15.4.x-scala2.12" + } + } + ], + "name": "placeholder", + "permissions": [], + "queue": { + "enabled": true + }, + "tags": {}, + "tasks": [ + { + "job_cluster_key": "default", + "notebook_task": { + "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/notebook" + }, + "task_key": "notebook_example" + }, + { + "job_cluster_key": "default", + "spark_python_task": { + "python_file": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/file.py" + }, + "task_key": "spark_python_example" + }, + { + "dbt_task": { + "commands": [ + "dbt run", + "dbt run" + ], + "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/dbt_project" + }, + "job_cluster_key": "default", + "task_key": "dbt_example" + }, + { + "job_cluster_key": "default", + "sql_task": { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/sql.sql" + }, + "warehouse_id": "cafef00d" + }, + "task_key": "sql_example" + }, + { + "job_cluster_key": "default", + "libraries": [ + { + "whl": "dist/wheel1.whl" + }, + { + "whl": "dist/wheel2.whl" + } + ], + "python_wheel_task": { + "package_name": "my_package" + }, + "task_key": "python_wheel_example" + }, + { + "job_cluster_key": "default", + "libraries": [ + { + "jar": "target/jar1.jar" + }, + { + "jar": "target/jar2.jar" + } + ], + "spark_jar_task": { + "main_class_name": "com.example.Main" + }, + "task_key": "spark_jar_example" + }, + { + "for_each_task": { + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/notebook" + } + } + }, + "job_cluster_key": "default", + "task_key": "for_each_notebook_example" + }, + { + "for_each_task": { + "task": { + "job_cluster_key": "default", + "spark_python_task": { + "python_file": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/file.py" + } + } + }, + "task_key": "for_each_spark_python_example" + } + ] + } + }, + "pipelines": { + "my_pipeline": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/state/metadata.json" + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/file1.py" + } + }, + { + "notebook": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/notebook1" + } + }, + { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/file2.py" + } + }, + { + "notebook": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/notebook2" + } + } + ], + "name": "placeholder", + "permissions": [] + } + } +} diff --git a/acceptance/bundle/paths/original_nominal/output.txt b/acceptance/bundle/paths/original_nominal/output.txt new file mode 100644 index 0000000000..606179950f --- /dev/null +++ b/acceptance/bundle/paths/original_nominal/output.txt @@ -0,0 +1,88 @@ + +>>> $CLI bundle validate -t development -o json + +Exit code: 0 + +>>> $CLI bundle validate -t error +Error: notebook this value is overridden not found. Local notebook references are expected +to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb] + +Name: original_nominal +Target: error +Workspace: + User: tester@databricks.com + Path: /Workspace/Users/tester@databricks.com/.bundle/original_nominal/error + +Found 1 error + +Exit code: 1 + +>>> jq -e +.jobs.my_job.tasks[0].task_key == "notebook_example" and +(.jobs.my_job.tasks[0].notebook_task.notebook_path | endswith("/src/notebook")) + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[1].task_key == "spark_python_example" and +(.jobs.my_job.tasks[1].spark_python_task.python_file | endswith("/src/file.py")) + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[2].task_key == "dbt_example" and +(.jobs.my_job.tasks[2].dbt_task.project_directory | endswith("/src/dbt_project")) + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[3].task_key == "sql_example" and +(.jobs.my_job.tasks[3].sql_task.file.path | endswith("/src/sql.sql")) + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[4].task_key == "python_wheel_example" and +(.jobs.my_job.tasks[4].libraries[0].whl == "dist/wheel1.whl") and +(.jobs.my_job.tasks[4].libraries[1].whl == "dist/wheel2.whl") + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[5].task_key == "spark_jar_example" and +(.jobs.my_job.tasks[5].libraries[0].jar == "target/jar1.jar") and +(.jobs.my_job.tasks[5].libraries[1].jar == "target/jar2.jar") + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[6].task_key == "for_each_notebook_example" and +(.jobs.my_job.tasks[6].for_each_task.task.notebook_task.notebook_path | endswith("/src/notebook")) + output.development.json +true + +>>> jq -e +.jobs.my_job.tasks[7].task_key == "for_each_spark_python_example" and +(.jobs.my_job.tasks[7].for_each_task.task.spark_python_task.python_file | endswith("/src/file.py")) + output.development.json +true + +>>> jq -e +(.pipelines.my_pipeline.libraries[0].file.path | endswith("/src/file1.py")) + output.development.json +true + +>>> jq -e +(.pipelines.my_pipeline.libraries[1].notebook.path | endswith("/src/notebook1")) + output.development.json +true + +>>> jq -e +(.pipelines.my_pipeline.libraries[2].file.path | endswith("/src/file2.py")) + output.development.json +true + +>>> jq -e +(.pipelines.my_pipeline.libraries[3].notebook.path | endswith("/src/notebook2")) + output.development.json +true diff --git a/bundle/tests/path_translation/nominal/override_job.yml b/acceptance/bundle/paths/original_nominal/override_job.yml similarity index 100% rename from bundle/tests/path_translation/nominal/override_job.yml rename to acceptance/bundle/paths/original_nominal/override_job.yml diff --git a/bundle/tests/path_translation/nominal/override_pipeline.yml b/acceptance/bundle/paths/original_nominal/override_pipeline.yml similarity index 100% rename from bundle/tests/path_translation/nominal/override_pipeline.yml rename to acceptance/bundle/paths/original_nominal/override_pipeline.yml diff --git a/bundle/tests/path_translation/nominal/resources/my_job.yml b/acceptance/bundle/paths/original_nominal/resources/my_job.yml similarity index 74% rename from bundle/tests/path_translation/nominal/resources/my_job.yml rename to acceptance/bundle/paths/original_nominal/resources/my_job.yml index 2020c9dc84..13996a20c0 100644 --- a/bundle/tests/path_translation/nominal/resources/my_job.yml +++ b/acceptance/bundle/paths/original_nominal/resources/my_job.yml @@ -4,38 +4,45 @@ resources: name: "placeholder" tasks: - task_key: notebook_example + job_cluster_key: default notebook_task: notebook_path: "this value is overridden" - task_key: spark_python_example + job_cluster_key: default spark_python_task: python_file: "this value is overridden" - task_key: dbt_example + job_cluster_key: default dbt_task: project_directory: "this value is overridden" commands: - "dbt run" - task_key: sql_example + job_cluster_key: default sql_task: file: path: "this value is overridden" warehouse_id: cafef00d - task_key: python_wheel_example + job_cluster_key: default python_wheel_task: package_name: my_package libraries: - whl: ../dist/wheel1.whl - task_key: spark_jar_example + job_cluster_key: default spark_jar_task: main_class_name: com.example.Main libraries: - jar: ../target/jar1.jar - task_key: for_each_notebook_example + job_cluster_key: default for_each_task: task: notebook_task: @@ -44,5 +51,12 @@ resources: - task_key: for_each_spark_python_example for_each_task: task: + job_cluster_key: default spark_python_task: python_file: "this value is overridden" + + # Include a job cluster for completeness + job_clusters: + - job_cluster_key: default + new_cluster: + spark_version: 15.4.x-scala2.12 diff --git a/bundle/tests/path_translation/nominal/resources/my_pipeline.yml b/acceptance/bundle/paths/original_nominal/resources/my_pipeline.yml similarity index 100% rename from bundle/tests/path_translation/nominal/resources/my_pipeline.yml rename to acceptance/bundle/paths/original_nominal/resources/my_pipeline.yml diff --git a/acceptance/bundle/paths/original_nominal/script b/acceptance/bundle/paths/original_nominal/script new file mode 100644 index 0000000000..1331f6f660 --- /dev/null +++ b/acceptance/bundle/paths/original_nominal/script @@ -0,0 +1,64 @@ +errcode trace $CLI bundle validate -t development -o json | jq '.resources' > output.development.json +errcode trace $CLI bundle validate -t error + +# Job assertions + +trace jq -e ' +.jobs.my_job.tasks[0].task_key == "notebook_example" and +(.jobs.my_job.tasks[0].notebook_task.notebook_path | endswith("/src/notebook")) +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[1].task_key == "spark_python_example" and +(.jobs.my_job.tasks[1].spark_python_task.python_file | endswith("/src/file.py")) +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[2].task_key == "dbt_example" and +(.jobs.my_job.tasks[2].dbt_task.project_directory | endswith("/src/dbt_project")) +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[3].task_key == "sql_example" and +(.jobs.my_job.tasks[3].sql_task.file.path | endswith("/src/sql.sql")) +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[4].task_key == "python_wheel_example" and +(.jobs.my_job.tasks[4].libraries[0].whl == "dist/wheel1.whl") and +(.jobs.my_job.tasks[4].libraries[1].whl == "dist/wheel2.whl") +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[5].task_key == "spark_jar_example" and +(.jobs.my_job.tasks[5].libraries[0].jar == "target/jar1.jar") and +(.jobs.my_job.tasks[5].libraries[1].jar == "target/jar2.jar") +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[6].task_key == "for_each_notebook_example" and +(.jobs.my_job.tasks[6].for_each_task.task.notebook_task.notebook_path | endswith("/src/notebook")) +' output.development.json + +trace jq -e ' +.jobs.my_job.tasks[7].task_key == "for_each_spark_python_example" and +(.jobs.my_job.tasks[7].for_each_task.task.spark_python_task.python_file | endswith("/src/file.py")) +' output.development.json + +# Pipeline assertions + +trace jq -e ' +(.pipelines.my_pipeline.libraries[0].file.path | endswith("/src/file1.py")) +' output.development.json + +trace jq -e ' +(.pipelines.my_pipeline.libraries[1].notebook.path | endswith("/src/notebook1")) +' output.development.json + +trace jq -e ' +(.pipelines.my_pipeline.libraries[2].file.path | endswith("/src/file2.py")) +' output.development.json + +trace jq -e ' +(.pipelines.my_pipeline.libraries[3].notebook.path | endswith("/src/notebook2")) +' output.development.json diff --git a/bundle/tests/path_translation/nominal/src/dbt_project/.gitkeep b/acceptance/bundle/paths/original_nominal/src/dbt_project/.gitkeep similarity index 100% rename from bundle/tests/path_translation/nominal/src/dbt_project/.gitkeep rename to acceptance/bundle/paths/original_nominal/src/dbt_project/.gitkeep diff --git a/bundle/tests/path_translation/nominal/src/file.py b/acceptance/bundle/paths/original_nominal/src/file.py similarity index 100% rename from bundle/tests/path_translation/nominal/src/file.py rename to acceptance/bundle/paths/original_nominal/src/file.py diff --git a/bundle/tests/path_translation/nominal/src/file1.py b/acceptance/bundle/paths/original_nominal/src/file1.py similarity index 100% rename from bundle/tests/path_translation/nominal/src/file1.py rename to acceptance/bundle/paths/original_nominal/src/file1.py diff --git a/bundle/tests/path_translation/nominal/src/file2.py b/acceptance/bundle/paths/original_nominal/src/file2.py similarity index 100% rename from bundle/tests/path_translation/nominal/src/file2.py rename to acceptance/bundle/paths/original_nominal/src/file2.py diff --git a/bundle/tests/path_translation/nominal/src/notebook.py b/acceptance/bundle/paths/original_nominal/src/notebook.py similarity index 100% rename from bundle/tests/path_translation/nominal/src/notebook.py rename to acceptance/bundle/paths/original_nominal/src/notebook.py diff --git a/bundle/tests/path_translation/nominal/src/notebook1.py b/acceptance/bundle/paths/original_nominal/src/notebook1.py similarity index 100% rename from bundle/tests/path_translation/nominal/src/notebook1.py rename to acceptance/bundle/paths/original_nominal/src/notebook1.py diff --git a/bundle/tests/path_translation/nominal/src/notebook2.py b/acceptance/bundle/paths/original_nominal/src/notebook2.py similarity index 100% rename from bundle/tests/path_translation/nominal/src/notebook2.py rename to acceptance/bundle/paths/original_nominal/src/notebook2.py diff --git a/bundle/tests/path_translation/nominal/src/sql.sql b/acceptance/bundle/paths/original_nominal/src/sql.sql similarity index 100% rename from bundle/tests/path_translation/nominal/src/sql.sql rename to acceptance/bundle/paths/original_nominal/src/sql.sql diff --git a/bundle/tests/relative_path_translation/databricks.yml b/acceptance/bundle/paths/original_relative_path_translation/databricks.yml similarity index 100% rename from bundle/tests/relative_path_translation/databricks.yml rename to acceptance/bundle/paths/original_relative_path_translation/databricks.yml diff --git a/acceptance/bundle/paths/original_relative_path_translation/output.default.json b/acceptance/bundle/paths/original_relative_path_translation/output.default.json new file mode 100644 index 0000000000..4cab5e5b3c --- /dev/null +++ b/acceptance/bundle/paths/original_relative_path_translation/output.default.json @@ -0,0 +1,27 @@ +{ + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/relative_path_translation/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "permissions": [], + "queue": { + "enabled": true + }, + "tags": {}, + "tasks": [ + { + "spark_python_task": { + "python_file": "/Workspace/remote/src/file1.py" + }, + "task_key": "local" + }, + { + "spark_python_task": { + "python_file": "/Workspace/remote/src/file1.py" + }, + "task_key": "variable_reference" + } + ] +} diff --git a/acceptance/bundle/paths/original_relative_path_translation/output.override.json b/acceptance/bundle/paths/original_relative_path_translation/output.override.json new file mode 100644 index 0000000000..610ece3a82 --- /dev/null +++ b/acceptance/bundle/paths/original_relative_path_translation/output.override.json @@ -0,0 +1,27 @@ +{ + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/relative_path_translation/override/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "permissions": [], + "queue": { + "enabled": true + }, + "tags": {}, + "tasks": [ + { + "spark_python_task": { + "python_file": "/Workspace/remote/src/file2.py" + }, + "task_key": "local" + }, + { + "spark_python_task": { + "python_file": "/Workspace/remote/src/file2.py" + }, + "task_key": "variable_reference" + } + ] +} diff --git a/acceptance/bundle/paths/original_relative_path_translation/output.txt b/acceptance/bundle/paths/original_relative_path_translation/output.txt new file mode 100644 index 0000000000..72bc1c5e84 --- /dev/null +++ b/acceptance/bundle/paths/original_relative_path_translation/output.txt @@ -0,0 +1,50 @@ + +>>> $CLI bundle validate -t default -o json +Error: Missing required cluster or environment settings + at resources.jobs.job.tasks[0] + in resources/job.yml:5:11 + +Task "local" requires a cluster or an environment to run. +Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster. + +Error: Missing required cluster or environment settings + at resources.jobs.job.tasks[1] + in resources/job.yml:9:11 + +Task "variable_reference" requires a cluster or an environment to run. +Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster. + + +Exit code: 1 + +>>> $CLI bundle validate -t override -o json +Error: Missing required cluster or environment settings + at resources.jobs.job.tasks[0] + in resources/job.yml:5:11 + databricks.yml:27:15 + +Task "local" requires a cluster or an environment to run. +Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster. + +Error: Missing required cluster or environment settings + at resources.jobs.job.tasks[1] + in resources/job.yml:9:11 + databricks.yml:31:15 + +Task "variable_reference" requires a cluster or an environment to run. +Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster. + + +Exit code: 1 + +>>> jq -e +(.tasks[0].spark_python_task.python_file | endswith("/src/file1.py")) and +(.tasks[1].spark_python_task.python_file | endswith("/src/file1.py")) + output.default.json +true + +>>> jq -e +(.tasks[0].spark_python_task.python_file | endswith("/src/file2.py")) and +(.tasks[1].spark_python_task.python_file | endswith("/src/file2.py")) + output.override.json +true diff --git a/bundle/tests/relative_path_translation/resources/job.yml b/acceptance/bundle/paths/original_relative_path_translation/resources/job.yml similarity index 100% rename from bundle/tests/relative_path_translation/resources/job.yml rename to acceptance/bundle/paths/original_relative_path_translation/resources/job.yml diff --git a/acceptance/bundle/paths/original_relative_path_translation/script b/acceptance/bundle/paths/original_relative_path_translation/script new file mode 100644 index 0000000000..dba5c3d35d --- /dev/null +++ b/acceptance/bundle/paths/original_relative_path_translation/script @@ -0,0 +1,12 @@ +errcode trace $CLI bundle validate -t default -o json | jq '.resources.jobs.job' > output.default.json +errcode trace $CLI bundle validate -t override -o json | jq '.resources.jobs.job' > output.override.json + +trace jq -e ' +(.tasks[0].spark_python_task.python_file | endswith("/src/file1.py")) and +(.tasks[1].spark_python_task.python_file | endswith("/src/file1.py")) +' output.default.json + +trace jq -e ' +(.tasks[0].spark_python_task.python_file | endswith("/src/file2.py")) and +(.tasks[1].spark_python_task.python_file | endswith("/src/file2.py")) +' output.override.json diff --git a/bundle/tests/relative_path_translation/src/file1.py b/acceptance/bundle/paths/original_relative_path_translation/src/file1.py similarity index 100% rename from bundle/tests/relative_path_translation/src/file1.py rename to acceptance/bundle/paths/original_relative_path_translation/src/file1.py diff --git a/bundle/tests/relative_path_translation/src/file2.py b/acceptance/bundle/paths/original_relative_path_translation/src/file2.py similarity index 100% rename from bundle/tests/relative_path_translation/src/file2.py rename to acceptance/bundle/paths/original_relative_path_translation/src/file2.py diff --git a/bundle/tests/path_translation_test.go b/bundle/tests/path_translation_test.go deleted file mode 100644 index 05702d2a27..0000000000 --- a/bundle/tests/path_translation_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package config_tests - -import ( - "context" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/mutator" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestPathTranslationFallback(t *testing.T) { - b := loadTarget(t, "./path_translation/fallback", "development") - - m := mutator.TranslatePaths() - diags := bundle.Apply(context.Background(), b, m) - require.NoError(t, diags.Error()) - - j := b.Config.Resources.Jobs["my_job"] - assert.Len(t, j.Tasks, 6) - - assert.Equal(t, "notebook_example", filepath.ToSlash(j.Tasks[0].TaskKey)) - assert.Equal(t, "src/notebook", filepath.ToSlash(j.Tasks[0].NotebookTask.NotebookPath)) - - assert.Equal(t, "spark_python_example", filepath.ToSlash(j.Tasks[1].TaskKey)) - assert.Equal(t, "src/file.py", filepath.ToSlash(j.Tasks[1].SparkPythonTask.PythonFile)) - - assert.Equal(t, "dbt_example", filepath.ToSlash(j.Tasks[2].TaskKey)) - assert.Equal(t, "src/dbt_project", filepath.ToSlash(j.Tasks[2].DbtTask.ProjectDirectory)) - - assert.Equal(t, "sql_example", filepath.ToSlash(j.Tasks[3].TaskKey)) - assert.Equal(t, "src/sql.sql", filepath.ToSlash(j.Tasks[3].SqlTask.File.Path)) - - assert.Equal(t, "python_wheel_example", filepath.ToSlash(j.Tasks[4].TaskKey)) - assert.Equal(t, "dist/wheel1.whl", filepath.ToSlash(j.Tasks[4].Libraries[0].Whl)) - assert.Equal(t, "dist/wheel2.whl", filepath.ToSlash(j.Tasks[4].Libraries[1].Whl)) - - assert.Equal(t, "spark_jar_example", filepath.ToSlash(j.Tasks[5].TaskKey)) - assert.Equal(t, "target/jar1.jar", filepath.ToSlash(j.Tasks[5].Libraries[0].Jar)) - assert.Equal(t, "target/jar2.jar", filepath.ToSlash(j.Tasks[5].Libraries[1].Jar)) - - p := b.Config.Resources.Pipelines["my_pipeline"] - assert.Len(t, p.Libraries, 4) - - assert.Equal(t, "src/file1.py", filepath.ToSlash(p.Libraries[0].File.Path)) - assert.Equal(t, "src/notebook1", filepath.ToSlash(p.Libraries[1].Notebook.Path)) - assert.Equal(t, "src/file2.py", filepath.ToSlash(p.Libraries[2].File.Path)) - assert.Equal(t, "src/notebook2", filepath.ToSlash(p.Libraries[3].Notebook.Path)) -} - -func TestPathTranslationFallbackError(t *testing.T) { - b := loadTarget(t, "./path_translation/fallback", "error") - - m := mutator.TranslatePaths() - diags := bundle.Apply(context.Background(), b, m) - assert.ErrorContains(t, diags.Error(), `notebook this value is overridden not found`) -} - -func TestPathTranslationNominal(t *testing.T) { - b := loadTarget(t, "./path_translation/nominal", "development") - - m := mutator.TranslatePaths() - diags := bundle.Apply(context.Background(), b, m) - assert.NoError(t, diags.Error()) - - j := b.Config.Resources.Jobs["my_job"] - assert.Len(t, j.Tasks, 8) - - assert.Equal(t, "notebook_example", filepath.ToSlash(j.Tasks[0].TaskKey)) - assert.Equal(t, "src/notebook", filepath.ToSlash(j.Tasks[0].NotebookTask.NotebookPath)) - - assert.Equal(t, "spark_python_example", filepath.ToSlash(j.Tasks[1].TaskKey)) - assert.Equal(t, "src/file.py", filepath.ToSlash(j.Tasks[1].SparkPythonTask.PythonFile)) - - assert.Equal(t, "dbt_example", filepath.ToSlash(j.Tasks[2].TaskKey)) - assert.Equal(t, "src/dbt_project", filepath.ToSlash(j.Tasks[2].DbtTask.ProjectDirectory)) - - assert.Equal(t, "sql_example", filepath.ToSlash(j.Tasks[3].TaskKey)) - assert.Equal(t, "src/sql.sql", filepath.ToSlash(j.Tasks[3].SqlTask.File.Path)) - - assert.Equal(t, "python_wheel_example", filepath.ToSlash(j.Tasks[4].TaskKey)) - assert.Equal(t, "dist/wheel1.whl", filepath.ToSlash(j.Tasks[4].Libraries[0].Whl)) - assert.Equal(t, "dist/wheel2.whl", filepath.ToSlash(j.Tasks[4].Libraries[1].Whl)) - - assert.Equal(t, "spark_jar_example", filepath.ToSlash(j.Tasks[5].TaskKey)) - assert.Equal(t, "target/jar1.jar", filepath.ToSlash(j.Tasks[5].Libraries[0].Jar)) - assert.Equal(t, "target/jar2.jar", filepath.ToSlash(j.Tasks[5].Libraries[1].Jar)) - - assert.Equal(t, "for_each_notebook_example", filepath.ToSlash(j.Tasks[6].TaskKey)) - assert.Equal(t, "src/notebook", filepath.ToSlash(j.Tasks[6].ForEachTask.Task.NotebookTask.NotebookPath)) - - assert.Equal(t, "for_each_spark_python_example", filepath.ToSlash(j.Tasks[7].TaskKey)) - assert.Equal(t, "src/file.py", filepath.ToSlash(j.Tasks[7].ForEachTask.Task.SparkPythonTask.PythonFile)) - - p := b.Config.Resources.Pipelines["my_pipeline"] - assert.Len(t, p.Libraries, 4) - - assert.Equal(t, "src/file1.py", filepath.ToSlash(p.Libraries[0].File.Path)) - assert.Equal(t, "src/notebook1", filepath.ToSlash(p.Libraries[1].Notebook.Path)) - assert.Equal(t, "src/file2.py", filepath.ToSlash(p.Libraries[2].File.Path)) - assert.Equal(t, "src/notebook2", filepath.ToSlash(p.Libraries[3].Notebook.Path)) -} - -func TestPathTranslationNominalError(t *testing.T) { - b := loadTarget(t, "./path_translation/nominal", "error") - - m := mutator.TranslatePaths() - diags := bundle.Apply(context.Background(), b, m) - assert.ErrorContains(t, diags.Error(), `notebook this value is overridden not found`) -} diff --git a/bundle/tests/relative_path_translation_test.go b/bundle/tests/relative_path_translation_test.go deleted file mode 100644 index 0f553ac3d3..0000000000 --- a/bundle/tests/relative_path_translation_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package config_tests - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRelativePathTranslationDefault(t *testing.T) { - b, diags := initializeTarget(t, "./relative_path_translation", "default") - require.NoError(t, diags.Error()) - - t0 := b.Config.Resources.Jobs["job"].Tasks[0] - assert.Equal(t, "/Workspace/remote/src/file1.py", t0.SparkPythonTask.PythonFile) - t1 := b.Config.Resources.Jobs["job"].Tasks[1] - assert.Equal(t, "/Workspace/remote/src/file1.py", t1.SparkPythonTask.PythonFile) -} - -func TestRelativePathTranslationOverride(t *testing.T) { - b, diags := initializeTarget(t, "./relative_path_translation", "override") - require.NoError(t, diags.Error()) - - t0 := b.Config.Resources.Jobs["job"].Tasks[0] - assert.Equal(t, "/Workspace/remote/src/file2.py", t0.SparkPythonTask.PythonFile) - t1 := b.Config.Resources.Jobs["job"].Tasks[1] - assert.Equal(t, "/Workspace/remote/src/file2.py", t1.SparkPythonTask.PythonFile) -} From 98235f67b8ac4a1b1d4b7da5617dd695a4c57771 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 10 Jan 2025 18:43:46 +0100 Subject: [PATCH 2/5] Rename --- .../{original_fallback => fallback}/README.md | 0 .../databricks.yml | 2 +- .../output.development.json | 20 ++++++++-------- .../output.txt | 4 ++-- .../override_job.yml | 0 .../override_pipeline.yml | 0 .../resources/my_job.yml | 0 .../resources/my_pipeline.yml | 0 .../{original_fallback => fallback}/script | 0 .../src/dbt_project/.gitkeep | 0 .../src/file.py | 0 .../src/file1.py | 0 .../src/file2.py | 0 .../src/notebook.py | 0 .../src/notebook1.py | 0 .../src/notebook2.py | 0 .../src/sql.sql | 0 .../{original_nominal => nominal}/README.md | 0 .../databricks.yml | 2 +- .../output.development.json | 24 +++++++++---------- .../{original_nominal => nominal}/output.txt | 4 ++-- .../override_job.yml | 0 .../override_pipeline.yml | 0 .../resources/my_job.yml | 0 .../resources/my_pipeline.yml | 0 .../{original_nominal => nominal}/script | 0 .../src/dbt_project/.gitkeep | 0 .../{original_nominal => nominal}/src/file.py | 0 .../src/file1.py | 0 .../src/file2.py | 0 .../src/notebook.py | 0 .../src/notebook1.py | 0 .../src/notebook2.py | 0 .../{original_nominal => nominal}/src/sql.sql | 0 .../databricks.yml | 0 .../output.default.json | 0 .../output.override.json | 0 .../output.txt | 0 .../resources/job.yml | 0 .../script | 0 .../src/file1.py | 0 .../src/file2.py | 0 42 files changed, 28 insertions(+), 28 deletions(-) rename acceptance/bundle/paths/{original_fallback => fallback}/README.md (100%) rename acceptance/bundle/paths/{original_nominal => fallback}/databricks.yml (84%) rename acceptance/bundle/paths/{original_fallback => fallback}/output.development.json (83%) rename acceptance/bundle/paths/{original_fallback => fallback}/output.txt (95%) rename acceptance/bundle/paths/{original_fallback => fallback}/override_job.yml (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/override_pipeline.yml (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/resources/my_job.yml (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/resources/my_pipeline.yml (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/script (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/src/dbt_project/.gitkeep (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/src/file.py (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/src/file1.py (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/src/file2.py (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/src/notebook.py (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/src/notebook1.py (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/src/notebook2.py (100%) rename acceptance/bundle/paths/{original_fallback => fallback}/src/sql.sql (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/README.md (100%) rename acceptance/bundle/paths/{original_fallback => nominal}/databricks.yml (83%) rename acceptance/bundle/paths/{original_nominal => nominal}/output.development.json (83%) rename acceptance/bundle/paths/{original_nominal => nominal}/output.txt (95%) rename acceptance/bundle/paths/{original_nominal => nominal}/override_job.yml (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/override_pipeline.yml (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/resources/my_job.yml (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/resources/my_pipeline.yml (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/script (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/src/dbt_project/.gitkeep (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/src/file.py (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/src/file1.py (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/src/file2.py (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/src/notebook.py (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/src/notebook1.py (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/src/notebook2.py (100%) rename acceptance/bundle/paths/{original_nominal => nominal}/src/sql.sql (100%) rename acceptance/bundle/paths/{original_relative_path_translation => relative_path_translation}/databricks.yml (100%) rename acceptance/bundle/paths/{original_relative_path_translation => relative_path_translation}/output.default.json (100%) rename acceptance/bundle/paths/{original_relative_path_translation => relative_path_translation}/output.override.json (100%) rename acceptance/bundle/paths/{original_relative_path_translation => relative_path_translation}/output.txt (100%) rename acceptance/bundle/paths/{original_relative_path_translation => relative_path_translation}/resources/job.yml (100%) rename acceptance/bundle/paths/{original_relative_path_translation => relative_path_translation}/script (100%) rename acceptance/bundle/paths/{original_relative_path_translation => relative_path_translation}/src/file1.py (100%) rename acceptance/bundle/paths/{original_relative_path_translation => relative_path_translation}/src/file2.py (100%) diff --git a/acceptance/bundle/paths/original_fallback/README.md b/acceptance/bundle/paths/fallback/README.md similarity index 100% rename from acceptance/bundle/paths/original_fallback/README.md rename to acceptance/bundle/paths/fallback/README.md diff --git a/acceptance/bundle/paths/original_nominal/databricks.yml b/acceptance/bundle/paths/fallback/databricks.yml similarity index 84% rename from acceptance/bundle/paths/original_nominal/databricks.yml rename to acceptance/bundle/paths/fallback/databricks.yml index 8759ec2f90..c6d0abe0aa 100644 --- a/acceptance/bundle/paths/original_nominal/databricks.yml +++ b/acceptance/bundle/paths/fallback/databricks.yml @@ -1,5 +1,5 @@ bundle: - name: original_nominal + name: fallback include: - "resources/*.yml" diff --git a/acceptance/bundle/paths/original_fallback/output.development.json b/acceptance/bundle/paths/fallback/output.development.json similarity index 83% rename from acceptance/bundle/paths/original_fallback/output.development.json rename to acceptance/bundle/paths/fallback/output.development.json index ba5229036d..66c3119e85 100644 --- a/acceptance/bundle/paths/original_fallback/output.development.json +++ b/acceptance/bundle/paths/fallback/output.development.json @@ -3,7 +3,7 @@ "my_job": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/state/metadata.json" + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", @@ -25,14 +25,14 @@ { "job_cluster_key": "default", "notebook_task": { - "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/notebook" + "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook" }, "task_key": "notebook_example" }, { "job_cluster_key": "default", "spark_python_task": { - "python_file": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/file.py" + "python_file": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file.py" }, "task_key": "spark_python_example" }, @@ -42,7 +42,7 @@ "dbt run", "dbt run" ], - "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/dbt_project" + "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/dbt_project" }, "job_cluster_key": "default", "task_key": "dbt_example" @@ -51,7 +51,7 @@ "job_cluster_key": "default", "sql_task": { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/sql.sql" + "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/sql.sql" }, "warehouse_id": "cafef00d" }, @@ -94,27 +94,27 @@ "my_pipeline": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/state/metadata.json" + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/state/metadata.json" }, "libraries": [ { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/file1.py" + "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file1.py" } }, { "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/notebook1" + "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook1" } }, { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/file2.py" + "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file2.py" } }, { "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/original_fallback/development/files/src/notebook2" + "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook2" } } ], diff --git a/acceptance/bundle/paths/original_fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt similarity index 95% rename from acceptance/bundle/paths/original_fallback/output.txt rename to acceptance/bundle/paths/fallback/output.txt index 831a47f390..d3bc8414a0 100644 --- a/acceptance/bundle/paths/original_fallback/output.txt +++ b/acceptance/bundle/paths/fallback/output.txt @@ -7,11 +7,11 @@ Exit code: 0 Error: notebook this value is overridden not found. Local notebook references are expected to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb] -Name: original_fallback +Name: fallback Target: error Workspace: User: tester@databricks.com - Path: /Workspace/Users/tester@databricks.com/.bundle/original_fallback/error + Path: /Workspace/Users/tester@databricks.com/.bundle/fallback/error Found 1 error diff --git a/acceptance/bundle/paths/original_fallback/override_job.yml b/acceptance/bundle/paths/fallback/override_job.yml similarity index 100% rename from acceptance/bundle/paths/original_fallback/override_job.yml rename to acceptance/bundle/paths/fallback/override_job.yml diff --git a/acceptance/bundle/paths/original_fallback/override_pipeline.yml b/acceptance/bundle/paths/fallback/override_pipeline.yml similarity index 100% rename from acceptance/bundle/paths/original_fallback/override_pipeline.yml rename to acceptance/bundle/paths/fallback/override_pipeline.yml diff --git a/acceptance/bundle/paths/original_fallback/resources/my_job.yml b/acceptance/bundle/paths/fallback/resources/my_job.yml similarity index 100% rename from acceptance/bundle/paths/original_fallback/resources/my_job.yml rename to acceptance/bundle/paths/fallback/resources/my_job.yml diff --git a/acceptance/bundle/paths/original_fallback/resources/my_pipeline.yml b/acceptance/bundle/paths/fallback/resources/my_pipeline.yml similarity index 100% rename from acceptance/bundle/paths/original_fallback/resources/my_pipeline.yml rename to acceptance/bundle/paths/fallback/resources/my_pipeline.yml diff --git a/acceptance/bundle/paths/original_fallback/script b/acceptance/bundle/paths/fallback/script similarity index 100% rename from acceptance/bundle/paths/original_fallback/script rename to acceptance/bundle/paths/fallback/script diff --git a/acceptance/bundle/paths/original_fallback/src/dbt_project/.gitkeep b/acceptance/bundle/paths/fallback/src/dbt_project/.gitkeep similarity index 100% rename from acceptance/bundle/paths/original_fallback/src/dbt_project/.gitkeep rename to acceptance/bundle/paths/fallback/src/dbt_project/.gitkeep diff --git a/acceptance/bundle/paths/original_fallback/src/file.py b/acceptance/bundle/paths/fallback/src/file.py similarity index 100% rename from acceptance/bundle/paths/original_fallback/src/file.py rename to acceptance/bundle/paths/fallback/src/file.py diff --git a/acceptance/bundle/paths/original_fallback/src/file1.py b/acceptance/bundle/paths/fallback/src/file1.py similarity index 100% rename from acceptance/bundle/paths/original_fallback/src/file1.py rename to acceptance/bundle/paths/fallback/src/file1.py diff --git a/acceptance/bundle/paths/original_fallback/src/file2.py b/acceptance/bundle/paths/fallback/src/file2.py similarity index 100% rename from acceptance/bundle/paths/original_fallback/src/file2.py rename to acceptance/bundle/paths/fallback/src/file2.py diff --git a/acceptance/bundle/paths/original_fallback/src/notebook.py b/acceptance/bundle/paths/fallback/src/notebook.py similarity index 100% rename from acceptance/bundle/paths/original_fallback/src/notebook.py rename to acceptance/bundle/paths/fallback/src/notebook.py diff --git a/acceptance/bundle/paths/original_fallback/src/notebook1.py b/acceptance/bundle/paths/fallback/src/notebook1.py similarity index 100% rename from acceptance/bundle/paths/original_fallback/src/notebook1.py rename to acceptance/bundle/paths/fallback/src/notebook1.py diff --git a/acceptance/bundle/paths/original_fallback/src/notebook2.py b/acceptance/bundle/paths/fallback/src/notebook2.py similarity index 100% rename from acceptance/bundle/paths/original_fallback/src/notebook2.py rename to acceptance/bundle/paths/fallback/src/notebook2.py diff --git a/acceptance/bundle/paths/original_fallback/src/sql.sql b/acceptance/bundle/paths/fallback/src/sql.sql similarity index 100% rename from acceptance/bundle/paths/original_fallback/src/sql.sql rename to acceptance/bundle/paths/fallback/src/sql.sql diff --git a/acceptance/bundle/paths/original_nominal/README.md b/acceptance/bundle/paths/nominal/README.md similarity index 100% rename from acceptance/bundle/paths/original_nominal/README.md rename to acceptance/bundle/paths/nominal/README.md diff --git a/acceptance/bundle/paths/original_fallback/databricks.yml b/acceptance/bundle/paths/nominal/databricks.yml similarity index 83% rename from acceptance/bundle/paths/original_fallback/databricks.yml rename to acceptance/bundle/paths/nominal/databricks.yml index 9608b5332e..5d3c22f918 100644 --- a/acceptance/bundle/paths/original_fallback/databricks.yml +++ b/acceptance/bundle/paths/nominal/databricks.yml @@ -1,5 +1,5 @@ bundle: - name: original_fallback + name: nominal include: - "resources/*.yml" diff --git a/acceptance/bundle/paths/original_nominal/output.development.json b/acceptance/bundle/paths/nominal/output.development.json similarity index 83% rename from acceptance/bundle/paths/original_nominal/output.development.json rename to acceptance/bundle/paths/nominal/output.development.json index 93cbf56f97..361693aebb 100644 --- a/acceptance/bundle/paths/original_nominal/output.development.json +++ b/acceptance/bundle/paths/nominal/output.development.json @@ -3,7 +3,7 @@ "my_job": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/state/metadata.json" + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", @@ -25,14 +25,14 @@ { "job_cluster_key": "default", "notebook_task": { - "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/notebook" + "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook" }, "task_key": "notebook_example" }, { "job_cluster_key": "default", "spark_python_task": { - "python_file": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/file.py" + "python_file": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file.py" }, "task_key": "spark_python_example" }, @@ -42,7 +42,7 @@ "dbt run", "dbt run" ], - "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/dbt_project" + "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/dbt_project" }, "job_cluster_key": "default", "task_key": "dbt_example" @@ -51,7 +51,7 @@ "job_cluster_key": "default", "sql_task": { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/sql.sql" + "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/sql.sql" }, "warehouse_id": "cafef00d" }, @@ -91,7 +91,7 @@ "for_each_task": { "task": { "notebook_task": { - "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/notebook" + "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook" } } }, @@ -103,7 +103,7 @@ "task": { "job_cluster_key": "default", "spark_python_task": { - "python_file": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/file.py" + "python_file": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file.py" } } }, @@ -116,27 +116,27 @@ "my_pipeline": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/state/metadata.json" + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/state/metadata.json" }, "libraries": [ { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/file1.py" + "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file1.py" } }, { "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/notebook1" + "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook1" } }, { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/file2.py" + "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file2.py" } }, { "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/original_nominal/development/files/src/notebook2" + "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook2" } } ], diff --git a/acceptance/bundle/paths/original_nominal/output.txt b/acceptance/bundle/paths/nominal/output.txt similarity index 95% rename from acceptance/bundle/paths/original_nominal/output.txt rename to acceptance/bundle/paths/nominal/output.txt index 606179950f..37efe49099 100644 --- a/acceptance/bundle/paths/original_nominal/output.txt +++ b/acceptance/bundle/paths/nominal/output.txt @@ -7,11 +7,11 @@ Exit code: 0 Error: notebook this value is overridden not found. Local notebook references are expected to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb] -Name: original_nominal +Name: nominal Target: error Workspace: User: tester@databricks.com - Path: /Workspace/Users/tester@databricks.com/.bundle/original_nominal/error + Path: /Workspace/Users/tester@databricks.com/.bundle/nominal/error Found 1 error diff --git a/acceptance/bundle/paths/original_nominal/override_job.yml b/acceptance/bundle/paths/nominal/override_job.yml similarity index 100% rename from acceptance/bundle/paths/original_nominal/override_job.yml rename to acceptance/bundle/paths/nominal/override_job.yml diff --git a/acceptance/bundle/paths/original_nominal/override_pipeline.yml b/acceptance/bundle/paths/nominal/override_pipeline.yml similarity index 100% rename from acceptance/bundle/paths/original_nominal/override_pipeline.yml rename to acceptance/bundle/paths/nominal/override_pipeline.yml diff --git a/acceptance/bundle/paths/original_nominal/resources/my_job.yml b/acceptance/bundle/paths/nominal/resources/my_job.yml similarity index 100% rename from acceptance/bundle/paths/original_nominal/resources/my_job.yml rename to acceptance/bundle/paths/nominal/resources/my_job.yml diff --git a/acceptance/bundle/paths/original_nominal/resources/my_pipeline.yml b/acceptance/bundle/paths/nominal/resources/my_pipeline.yml similarity index 100% rename from acceptance/bundle/paths/original_nominal/resources/my_pipeline.yml rename to acceptance/bundle/paths/nominal/resources/my_pipeline.yml diff --git a/acceptance/bundle/paths/original_nominal/script b/acceptance/bundle/paths/nominal/script similarity index 100% rename from acceptance/bundle/paths/original_nominal/script rename to acceptance/bundle/paths/nominal/script diff --git a/acceptance/bundle/paths/original_nominal/src/dbt_project/.gitkeep b/acceptance/bundle/paths/nominal/src/dbt_project/.gitkeep similarity index 100% rename from acceptance/bundle/paths/original_nominal/src/dbt_project/.gitkeep rename to acceptance/bundle/paths/nominal/src/dbt_project/.gitkeep diff --git a/acceptance/bundle/paths/original_nominal/src/file.py b/acceptance/bundle/paths/nominal/src/file.py similarity index 100% rename from acceptance/bundle/paths/original_nominal/src/file.py rename to acceptance/bundle/paths/nominal/src/file.py diff --git a/acceptance/bundle/paths/original_nominal/src/file1.py b/acceptance/bundle/paths/nominal/src/file1.py similarity index 100% rename from acceptance/bundle/paths/original_nominal/src/file1.py rename to acceptance/bundle/paths/nominal/src/file1.py diff --git a/acceptance/bundle/paths/original_nominal/src/file2.py b/acceptance/bundle/paths/nominal/src/file2.py similarity index 100% rename from acceptance/bundle/paths/original_nominal/src/file2.py rename to acceptance/bundle/paths/nominal/src/file2.py diff --git a/acceptance/bundle/paths/original_nominal/src/notebook.py b/acceptance/bundle/paths/nominal/src/notebook.py similarity index 100% rename from acceptance/bundle/paths/original_nominal/src/notebook.py rename to acceptance/bundle/paths/nominal/src/notebook.py diff --git a/acceptance/bundle/paths/original_nominal/src/notebook1.py b/acceptance/bundle/paths/nominal/src/notebook1.py similarity index 100% rename from acceptance/bundle/paths/original_nominal/src/notebook1.py rename to acceptance/bundle/paths/nominal/src/notebook1.py diff --git a/acceptance/bundle/paths/original_nominal/src/notebook2.py b/acceptance/bundle/paths/nominal/src/notebook2.py similarity index 100% rename from acceptance/bundle/paths/original_nominal/src/notebook2.py rename to acceptance/bundle/paths/nominal/src/notebook2.py diff --git a/acceptance/bundle/paths/original_nominal/src/sql.sql b/acceptance/bundle/paths/nominal/src/sql.sql similarity index 100% rename from acceptance/bundle/paths/original_nominal/src/sql.sql rename to acceptance/bundle/paths/nominal/src/sql.sql diff --git a/acceptance/bundle/paths/original_relative_path_translation/databricks.yml b/acceptance/bundle/paths/relative_path_translation/databricks.yml similarity index 100% rename from acceptance/bundle/paths/original_relative_path_translation/databricks.yml rename to acceptance/bundle/paths/relative_path_translation/databricks.yml diff --git a/acceptance/bundle/paths/original_relative_path_translation/output.default.json b/acceptance/bundle/paths/relative_path_translation/output.default.json similarity index 100% rename from acceptance/bundle/paths/original_relative_path_translation/output.default.json rename to acceptance/bundle/paths/relative_path_translation/output.default.json diff --git a/acceptance/bundle/paths/original_relative_path_translation/output.override.json b/acceptance/bundle/paths/relative_path_translation/output.override.json similarity index 100% rename from acceptance/bundle/paths/original_relative_path_translation/output.override.json rename to acceptance/bundle/paths/relative_path_translation/output.override.json diff --git a/acceptance/bundle/paths/original_relative_path_translation/output.txt b/acceptance/bundle/paths/relative_path_translation/output.txt similarity index 100% rename from acceptance/bundle/paths/original_relative_path_translation/output.txt rename to acceptance/bundle/paths/relative_path_translation/output.txt diff --git a/acceptance/bundle/paths/original_relative_path_translation/resources/job.yml b/acceptance/bundle/paths/relative_path_translation/resources/job.yml similarity index 100% rename from acceptance/bundle/paths/original_relative_path_translation/resources/job.yml rename to acceptance/bundle/paths/relative_path_translation/resources/job.yml diff --git a/acceptance/bundle/paths/original_relative_path_translation/script b/acceptance/bundle/paths/relative_path_translation/script similarity index 100% rename from acceptance/bundle/paths/original_relative_path_translation/script rename to acceptance/bundle/paths/relative_path_translation/script diff --git a/acceptance/bundle/paths/original_relative_path_translation/src/file1.py b/acceptance/bundle/paths/relative_path_translation/src/file1.py similarity index 100% rename from acceptance/bundle/paths/original_relative_path_translation/src/file1.py rename to acceptance/bundle/paths/relative_path_translation/src/file1.py diff --git a/acceptance/bundle/paths/original_relative_path_translation/src/file2.py b/acceptance/bundle/paths/relative_path_translation/src/file2.py similarity index 100% rename from acceptance/bundle/paths/original_relative_path_translation/src/file2.py rename to acceptance/bundle/paths/relative_path_translation/src/file2.py From 1f39e6b9de6e4ef8da904e71bc2e8ed6aaf29300 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 10 Jan 2025 18:46:23 +0100 Subject: [PATCH 3/5] Fix warnings in relative path translation test --- .../output.default.json | 10 ++++++ .../output.override.json | 10 ++++++ .../relative_path_translation/output.txt | 34 ++----------------- .../resources/job.yml | 8 +++++ 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/acceptance/bundle/paths/relative_path_translation/output.default.json b/acceptance/bundle/paths/relative_path_translation/output.default.json index 4cab5e5b3c..e652a42756 100644 --- a/acceptance/bundle/paths/relative_path_translation/output.default.json +++ b/acceptance/bundle/paths/relative_path_translation/output.default.json @@ -5,6 +5,14 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "default", + "new_cluster": { + "spark_version": "15.4.x-scala2.12" + } + } + ], "permissions": [], "queue": { "enabled": true @@ -12,12 +20,14 @@ "tags": {}, "tasks": [ { + "job_cluster_key": "default", "spark_python_task": { "python_file": "/Workspace/remote/src/file1.py" }, "task_key": "local" }, { + "job_cluster_key": "default", "spark_python_task": { "python_file": "/Workspace/remote/src/file1.py" }, diff --git a/acceptance/bundle/paths/relative_path_translation/output.override.json b/acceptance/bundle/paths/relative_path_translation/output.override.json index 610ece3a82..5c81f4e987 100644 --- a/acceptance/bundle/paths/relative_path_translation/output.override.json +++ b/acceptance/bundle/paths/relative_path_translation/output.override.json @@ -5,6 +5,14 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "default", + "new_cluster": { + "spark_version": "15.4.x-scala2.12" + } + } + ], "permissions": [], "queue": { "enabled": true @@ -12,12 +20,14 @@ "tags": {}, "tasks": [ { + "job_cluster_key": "default", "spark_python_task": { "python_file": "/Workspace/remote/src/file2.py" }, "task_key": "local" }, { + "job_cluster_key": "default", "spark_python_task": { "python_file": "/Workspace/remote/src/file2.py" }, diff --git a/acceptance/bundle/paths/relative_path_translation/output.txt b/acceptance/bundle/paths/relative_path_translation/output.txt index 72bc1c5e84..8442be96ce 100644 --- a/acceptance/bundle/paths/relative_path_translation/output.txt +++ b/acceptance/bundle/paths/relative_path_translation/output.txt @@ -1,41 +1,11 @@ >>> $CLI bundle validate -t default -o json -Error: Missing required cluster or environment settings - at resources.jobs.job.tasks[0] - in resources/job.yml:5:11 -Task "local" requires a cluster or an environment to run. -Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster. - -Error: Missing required cluster or environment settings - at resources.jobs.job.tasks[1] - in resources/job.yml:9:11 - -Task "variable_reference" requires a cluster or an environment to run. -Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster. - - -Exit code: 1 +Exit code: 0 >>> $CLI bundle validate -t override -o json -Error: Missing required cluster or environment settings - at resources.jobs.job.tasks[0] - in resources/job.yml:5:11 - databricks.yml:27:15 - -Task "local" requires a cluster or an environment to run. -Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster. - -Error: Missing required cluster or environment settings - at resources.jobs.job.tasks[1] - in resources/job.yml:9:11 - databricks.yml:31:15 - -Task "variable_reference" requires a cluster or an environment to run. -Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster. - -Exit code: 1 +Exit code: 0 >>> jq -e (.tasks[0].spark_python_task.python_file | endswith("/src/file1.py")) and diff --git a/acceptance/bundle/paths/relative_path_translation/resources/job.yml b/acceptance/bundle/paths/relative_path_translation/resources/job.yml index 93f121f253..9540ff1ad8 100644 --- a/acceptance/bundle/paths/relative_path_translation/resources/job.yml +++ b/acceptance/bundle/paths/relative_path_translation/resources/job.yml @@ -3,12 +3,20 @@ resources: job: tasks: - task_key: local + job_cluster_key: default spark_python_task: python_file: ../src/file1.py - task_key: variable_reference + job_cluster_key: default spark_python_task: # Note: this is a pure variable reference yet needs to persist the location # of the reference, not the location of the variable value. # Also see https://github.com/databricks/cli/issues/1330. python_file: ${var.file_path} + + # Include a job cluster for completeness + job_clusters: + - job_cluster_key: default + new_cluster: + spark_version: 15.4.x-scala2.12 From 42f55f405c9bb2fcc3c8503bc4be8948ddd5d315 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 14 Jan 2025 09:54:55 +0100 Subject: [PATCH 4/5] Remove jq assertions --- .../paths/fallback/output.development.json | 125 --------------- .../bundle/paths/fallback/output.job.json | 67 ++++++++ .../paths/fallback/output.pipeline.json | 22 +++ acceptance/bundle/paths/fallback/output.txt | 58 ------- acceptance/bundle/paths/fallback/script | 58 +------ .../bundle/paths/fallback/script.cleanup | 1 + .../paths/nominal/output.development.json | 147 ------------------ .../bundle/paths/nominal/output.job.json | 89 +++++++++++ .../bundle/paths/nominal/output.pipeline.json | 22 +++ acceptance/bundle/paths/nominal/output.txt | 70 --------- acceptance/bundle/paths/nominal/script | 68 +------- .../bundle/paths/nominal/script.cleanup | 1 + .../output.default.json | 37 +---- .../output.override.json | 37 +---- .../relative_path_translation/output.txt | 16 -- .../paths/relative_path_translation/script | 16 +- 16 files changed, 226 insertions(+), 608 deletions(-) delete mode 100644 acceptance/bundle/paths/fallback/output.development.json create mode 100644 acceptance/bundle/paths/fallback/output.job.json create mode 100644 acceptance/bundle/paths/fallback/output.pipeline.json create mode 100644 acceptance/bundle/paths/fallback/script.cleanup delete mode 100644 acceptance/bundle/paths/nominal/output.development.json create mode 100644 acceptance/bundle/paths/nominal/output.job.json create mode 100644 acceptance/bundle/paths/nominal/output.pipeline.json create mode 100644 acceptance/bundle/paths/nominal/script.cleanup diff --git a/acceptance/bundle/paths/fallback/output.development.json b/acceptance/bundle/paths/fallback/output.development.json deleted file mode 100644 index 66c3119e85..0000000000 --- a/acceptance/bundle/paths/fallback/output.development.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "jobs": { - "my_job": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "job_clusters": [ - { - "job_cluster_key": "default", - "new_cluster": { - "spark_version": "15.4.x-scala2.12" - } - } - ], - "name": "placeholder", - "permissions": [], - "queue": { - "enabled": true - }, - "tags": {}, - "tasks": [ - { - "job_cluster_key": "default", - "notebook_task": { - "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook" - }, - "task_key": "notebook_example" - }, - { - "job_cluster_key": "default", - "spark_python_task": { - "python_file": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file.py" - }, - "task_key": "spark_python_example" - }, - { - "dbt_task": { - "commands": [ - "dbt run", - "dbt run" - ], - "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/dbt_project" - }, - "job_cluster_key": "default", - "task_key": "dbt_example" - }, - { - "job_cluster_key": "default", - "sql_task": { - "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/sql.sql" - }, - "warehouse_id": "cafef00d" - }, - "task_key": "sql_example" - }, - { - "job_cluster_key": "default", - "libraries": [ - { - "whl": "dist/wheel1.whl" - }, - { - "whl": "dist/wheel2.whl" - } - ], - "python_wheel_task": { - "package_name": "my_package" - }, - "task_key": "python_wheel_example" - }, - { - "job_cluster_key": "default", - "libraries": [ - { - "jar": "target/jar1.jar" - }, - { - "jar": "target/jar2.jar" - } - ], - "spark_jar_task": { - "main_class_name": "com.example.Main" - }, - "task_key": "spark_jar_example" - } - ] - } - }, - "pipelines": { - "my_pipeline": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/state/metadata.json" - }, - "libraries": [ - { - "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file1.py" - } - }, - { - "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook1" - } - }, - { - "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file2.py" - } - }, - { - "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook2" - } - } - ], - "name": "placeholder", - "permissions": [] - } - } -} diff --git a/acceptance/bundle/paths/fallback/output.job.json b/acceptance/bundle/paths/fallback/output.job.json new file mode 100644 index 0000000000..fa69143f53 --- /dev/null +++ b/acceptance/bundle/paths/fallback/output.job.json @@ -0,0 +1,67 @@ +[ + { + "job_cluster_key": "default", + "notebook_task": { + "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook" + }, + "task_key": "notebook_example" + }, + { + "job_cluster_key": "default", + "spark_python_task": { + "python_file": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file.py" + }, + "task_key": "spark_python_example" + }, + { + "dbt_task": { + "commands": [ + "dbt run", + "dbt run" + ], + "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/dbt_project" + }, + "job_cluster_key": "default", + "task_key": "dbt_example" + }, + { + "job_cluster_key": "default", + "sql_task": { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/sql.sql" + }, + "warehouse_id": "cafef00d" + }, + "task_key": "sql_example" + }, + { + "job_cluster_key": "default", + "libraries": [ + { + "whl": "dist/wheel1.whl" + }, + { + "whl": "dist/wheel2.whl" + } + ], + "python_wheel_task": { + "package_name": "my_package" + }, + "task_key": "python_wheel_example" + }, + { + "job_cluster_key": "default", + "libraries": [ + { + "jar": "target/jar1.jar" + }, + { + "jar": "target/jar2.jar" + } + ], + "spark_jar_task": { + "main_class_name": "com.example.Main" + }, + "task_key": "spark_jar_example" + } +] diff --git a/acceptance/bundle/paths/fallback/output.pipeline.json b/acceptance/bundle/paths/fallback/output.pipeline.json new file mode 100644 index 0000000000..1bfcc45dca --- /dev/null +++ b/acceptance/bundle/paths/fallback/output.pipeline.json @@ -0,0 +1,22 @@ +[ + { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file1.py" + } + }, + { + "notebook": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook1" + } + }, + { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file2.py" + } + }, + { + "notebook": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook2" + } + } +] diff --git a/acceptance/bundle/paths/fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt index d3bc8414a0..a2aff856c5 100644 --- a/acceptance/bundle/paths/fallback/output.txt +++ b/acceptance/bundle/paths/fallback/output.txt @@ -16,61 +16,3 @@ Workspace: Found 1 error Exit code: 1 - ->>> jq -e -.jobs.my_job.tasks[0].task_key == "notebook_example" and -(.jobs.my_job.tasks[0].notebook_task.notebook_path | endswith("/src/notebook")) - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[1].task_key == "spark_python_example" and -(.jobs.my_job.tasks[1].spark_python_task.python_file | endswith("/src/file.py")) - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[2].task_key == "dbt_example" and -(.jobs.my_job.tasks[2].dbt_task.project_directory | endswith("/src/dbt_project")) - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[3].task_key == "sql_example" and -(.jobs.my_job.tasks[3].sql_task.file.path | endswith("/src/sql.sql")) - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[4].task_key == "python_wheel_example" and -(.jobs.my_job.tasks[4].libraries[0].whl == "dist/wheel1.whl") and -(.jobs.my_job.tasks[4].libraries[1].whl == "dist/wheel2.whl") - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[5].task_key == "spark_jar_example" and -(.jobs.my_job.tasks[5].libraries[0].jar == "target/jar1.jar") and -(.jobs.my_job.tasks[5].libraries[1].jar == "target/jar2.jar") - output.development.json -true - ->>> jq -e -(.pipelines.my_pipeline.libraries[0].file.path | endswith("/src/file1.py")) - output.development.json -true - ->>> jq -e -(.pipelines.my_pipeline.libraries[1].notebook.path | endswith("/src/notebook1")) - output.development.json -true - ->>> jq -e -(.pipelines.my_pipeline.libraries[2].file.path | endswith("/src/file2.py")) - output.development.json -true - ->>> jq -e -(.pipelines.my_pipeline.libraries[3].notebook.path | endswith("/src/notebook2")) - output.development.json -true diff --git a/acceptance/bundle/paths/fallback/script b/acceptance/bundle/paths/fallback/script index bbad0ad01f..29aa420c59 100644 --- a/acceptance/bundle/paths/fallback/script +++ b/acceptance/bundle/paths/fallback/script @@ -1,54 +1,10 @@ -errcode trace $CLI bundle validate -t development -o json | jq '.resources' > output.development.json -errcode trace $CLI bundle validate -t error - -# Job assertions - -trace jq -e ' -.jobs.my_job.tasks[0].task_key == "notebook_example" and -(.jobs.my_job.tasks[0].notebook_task.notebook_path | endswith("/src/notebook")) -' output.development.json - -trace jq -e ' -.jobs.my_job.tasks[1].task_key == "spark_python_example" and -(.jobs.my_job.tasks[1].spark_python_task.python_file | endswith("/src/file.py")) -' output.development.json - -trace jq -e ' -.jobs.my_job.tasks[2].task_key == "dbt_example" and -(.jobs.my_job.tasks[2].dbt_task.project_directory | endswith("/src/dbt_project")) -' output.development.json - -trace jq -e ' -.jobs.my_job.tasks[3].task_key == "sql_example" and -(.jobs.my_job.tasks[3].sql_task.file.path | endswith("/src/sql.sql")) -' output.development.json +errcode trace $CLI bundle validate -t development -o json > output.tmp.json -trace jq -e ' -.jobs.my_job.tasks[4].task_key == "python_wheel_example" and -(.jobs.my_job.tasks[4].libraries[0].whl == "dist/wheel1.whl") and -(.jobs.my_job.tasks[4].libraries[1].whl == "dist/wheel2.whl") -' output.development.json +# Capture job tasks +jq '.resources.jobs.my_job.tasks' output.tmp.json > output.job.json -trace jq -e ' -.jobs.my_job.tasks[5].task_key == "spark_jar_example" and -(.jobs.my_job.tasks[5].libraries[0].jar == "target/jar1.jar") and -(.jobs.my_job.tasks[5].libraries[1].jar == "target/jar2.jar") -' output.development.json +# Capture pipeline libraries +jq '.resources.pipelines.my_pipeline.libraries' output.tmp.json > output.pipeline.json -# Pipeline assertions - -trace jq -e ' -(.pipelines.my_pipeline.libraries[0].file.path | endswith("/src/file1.py")) -' output.development.json - -trace jq -e ' -(.pipelines.my_pipeline.libraries[1].notebook.path | endswith("/src/notebook1")) -' output.development.json - -trace jq -e ' -(.pipelines.my_pipeline.libraries[2].file.path | endswith("/src/file2.py")) -' output.development.json - -trace jq -e ' -(.pipelines.my_pipeline.libraries[3].notebook.path | endswith("/src/notebook2")) -' output.development.json +# Expect failure for the "error" target +errcode trace $CLI bundle validate -t error diff --git a/acceptance/bundle/paths/fallback/script.cleanup b/acceptance/bundle/paths/fallback/script.cleanup new file mode 100644 index 0000000000..f93425dffd --- /dev/null +++ b/acceptance/bundle/paths/fallback/script.cleanup @@ -0,0 +1 @@ +rm -f output.tmp.json diff --git a/acceptance/bundle/paths/nominal/output.development.json b/acceptance/bundle/paths/nominal/output.development.json deleted file mode 100644 index 361693aebb..0000000000 --- a/acceptance/bundle/paths/nominal/output.development.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "jobs": { - "my_job": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "job_clusters": [ - { - "job_cluster_key": "default", - "new_cluster": { - "spark_version": "15.4.x-scala2.12" - } - } - ], - "name": "placeholder", - "permissions": [], - "queue": { - "enabled": true - }, - "tags": {}, - "tasks": [ - { - "job_cluster_key": "default", - "notebook_task": { - "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook" - }, - "task_key": "notebook_example" - }, - { - "job_cluster_key": "default", - "spark_python_task": { - "python_file": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file.py" - }, - "task_key": "spark_python_example" - }, - { - "dbt_task": { - "commands": [ - "dbt run", - "dbt run" - ], - "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/dbt_project" - }, - "job_cluster_key": "default", - "task_key": "dbt_example" - }, - { - "job_cluster_key": "default", - "sql_task": { - "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/sql.sql" - }, - "warehouse_id": "cafef00d" - }, - "task_key": "sql_example" - }, - { - "job_cluster_key": "default", - "libraries": [ - { - "whl": "dist/wheel1.whl" - }, - { - "whl": "dist/wheel2.whl" - } - ], - "python_wheel_task": { - "package_name": "my_package" - }, - "task_key": "python_wheel_example" - }, - { - "job_cluster_key": "default", - "libraries": [ - { - "jar": "target/jar1.jar" - }, - { - "jar": "target/jar2.jar" - } - ], - "spark_jar_task": { - "main_class_name": "com.example.Main" - }, - "task_key": "spark_jar_example" - }, - { - "for_each_task": { - "task": { - "notebook_task": { - "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook" - } - } - }, - "job_cluster_key": "default", - "task_key": "for_each_notebook_example" - }, - { - "for_each_task": { - "task": { - "job_cluster_key": "default", - "spark_python_task": { - "python_file": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file.py" - } - } - }, - "task_key": "for_each_spark_python_example" - } - ] - } - }, - "pipelines": { - "my_pipeline": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/state/metadata.json" - }, - "libraries": [ - { - "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file1.py" - } - }, - { - "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook1" - } - }, - { - "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file2.py" - } - }, - { - "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook2" - } - } - ], - "name": "placeholder", - "permissions": [] - } - } -} diff --git a/acceptance/bundle/paths/nominal/output.job.json b/acceptance/bundle/paths/nominal/output.job.json new file mode 100644 index 0000000000..1b2a4a541c --- /dev/null +++ b/acceptance/bundle/paths/nominal/output.job.json @@ -0,0 +1,89 @@ +[ + { + "job_cluster_key": "default", + "notebook_task": { + "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook" + }, + "task_key": "notebook_example" + }, + { + "job_cluster_key": "default", + "spark_python_task": { + "python_file": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file.py" + }, + "task_key": "spark_python_example" + }, + { + "dbt_task": { + "commands": [ + "dbt run", + "dbt run" + ], + "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/dbt_project" + }, + "job_cluster_key": "default", + "task_key": "dbt_example" + }, + { + "job_cluster_key": "default", + "sql_task": { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/sql.sql" + }, + "warehouse_id": "cafef00d" + }, + "task_key": "sql_example" + }, + { + "job_cluster_key": "default", + "libraries": [ + { + "whl": "dist/wheel1.whl" + }, + { + "whl": "dist/wheel2.whl" + } + ], + "python_wheel_task": { + "package_name": "my_package" + }, + "task_key": "python_wheel_example" + }, + { + "job_cluster_key": "default", + "libraries": [ + { + "jar": "target/jar1.jar" + }, + { + "jar": "target/jar2.jar" + } + ], + "spark_jar_task": { + "main_class_name": "com.example.Main" + }, + "task_key": "spark_jar_example" + }, + { + "for_each_task": { + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook" + } + } + }, + "job_cluster_key": "default", + "task_key": "for_each_notebook_example" + }, + { + "for_each_task": { + "task": { + "job_cluster_key": "default", + "spark_python_task": { + "python_file": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file.py" + } + } + }, + "task_key": "for_each_spark_python_example" + } +] diff --git a/acceptance/bundle/paths/nominal/output.pipeline.json b/acceptance/bundle/paths/nominal/output.pipeline.json new file mode 100644 index 0000000000..21b4a6ebd2 --- /dev/null +++ b/acceptance/bundle/paths/nominal/output.pipeline.json @@ -0,0 +1,22 @@ +[ + { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file1.py" + } + }, + { + "notebook": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook1" + } + }, + { + "file": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file2.py" + } + }, + { + "notebook": { + "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook2" + } + } +] diff --git a/acceptance/bundle/paths/nominal/output.txt b/acceptance/bundle/paths/nominal/output.txt index 37efe49099..671386d811 100644 --- a/acceptance/bundle/paths/nominal/output.txt +++ b/acceptance/bundle/paths/nominal/output.txt @@ -16,73 +16,3 @@ Workspace: Found 1 error Exit code: 1 - ->>> jq -e -.jobs.my_job.tasks[0].task_key == "notebook_example" and -(.jobs.my_job.tasks[0].notebook_task.notebook_path | endswith("/src/notebook")) - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[1].task_key == "spark_python_example" and -(.jobs.my_job.tasks[1].spark_python_task.python_file | endswith("/src/file.py")) - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[2].task_key == "dbt_example" and -(.jobs.my_job.tasks[2].dbt_task.project_directory | endswith("/src/dbt_project")) - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[3].task_key == "sql_example" and -(.jobs.my_job.tasks[3].sql_task.file.path | endswith("/src/sql.sql")) - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[4].task_key == "python_wheel_example" and -(.jobs.my_job.tasks[4].libraries[0].whl == "dist/wheel1.whl") and -(.jobs.my_job.tasks[4].libraries[1].whl == "dist/wheel2.whl") - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[5].task_key == "spark_jar_example" and -(.jobs.my_job.tasks[5].libraries[0].jar == "target/jar1.jar") and -(.jobs.my_job.tasks[5].libraries[1].jar == "target/jar2.jar") - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[6].task_key == "for_each_notebook_example" and -(.jobs.my_job.tasks[6].for_each_task.task.notebook_task.notebook_path | endswith("/src/notebook")) - output.development.json -true - ->>> jq -e -.jobs.my_job.tasks[7].task_key == "for_each_spark_python_example" and -(.jobs.my_job.tasks[7].for_each_task.task.spark_python_task.python_file | endswith("/src/file.py")) - output.development.json -true - ->>> jq -e -(.pipelines.my_pipeline.libraries[0].file.path | endswith("/src/file1.py")) - output.development.json -true - ->>> jq -e -(.pipelines.my_pipeline.libraries[1].notebook.path | endswith("/src/notebook1")) - output.development.json -true - ->>> jq -e -(.pipelines.my_pipeline.libraries[2].file.path | endswith("/src/file2.py")) - output.development.json -true - ->>> jq -e -(.pipelines.my_pipeline.libraries[3].notebook.path | endswith("/src/notebook2")) - output.development.json -true diff --git a/acceptance/bundle/paths/nominal/script b/acceptance/bundle/paths/nominal/script index 1331f6f660..29aa420c59 100644 --- a/acceptance/bundle/paths/nominal/script +++ b/acceptance/bundle/paths/nominal/script @@ -1,64 +1,10 @@ -errcode trace $CLI bundle validate -t development -o json | jq '.resources' > output.development.json -errcode trace $CLI bundle validate -t error - -# Job assertions - -trace jq -e ' -.jobs.my_job.tasks[0].task_key == "notebook_example" and -(.jobs.my_job.tasks[0].notebook_task.notebook_path | endswith("/src/notebook")) -' output.development.json - -trace jq -e ' -.jobs.my_job.tasks[1].task_key == "spark_python_example" and -(.jobs.my_job.tasks[1].spark_python_task.python_file | endswith("/src/file.py")) -' output.development.json - -trace jq -e ' -.jobs.my_job.tasks[2].task_key == "dbt_example" and -(.jobs.my_job.tasks[2].dbt_task.project_directory | endswith("/src/dbt_project")) -' output.development.json - -trace jq -e ' -.jobs.my_job.tasks[3].task_key == "sql_example" and -(.jobs.my_job.tasks[3].sql_task.file.path | endswith("/src/sql.sql")) -' output.development.json - -trace jq -e ' -.jobs.my_job.tasks[4].task_key == "python_wheel_example" and -(.jobs.my_job.tasks[4].libraries[0].whl == "dist/wheel1.whl") and -(.jobs.my_job.tasks[4].libraries[1].whl == "dist/wheel2.whl") -' output.development.json +errcode trace $CLI bundle validate -t development -o json > output.tmp.json -trace jq -e ' -.jobs.my_job.tasks[5].task_key == "spark_jar_example" and -(.jobs.my_job.tasks[5].libraries[0].jar == "target/jar1.jar") and -(.jobs.my_job.tasks[5].libraries[1].jar == "target/jar2.jar") -' output.development.json +# Capture job tasks +jq '.resources.jobs.my_job.tasks' output.tmp.json > output.job.json -trace jq -e ' -.jobs.my_job.tasks[6].task_key == "for_each_notebook_example" and -(.jobs.my_job.tasks[6].for_each_task.task.notebook_task.notebook_path | endswith("/src/notebook")) -' output.development.json +# Capture pipeline libraries +jq '.resources.pipelines.my_pipeline.libraries' output.tmp.json > output.pipeline.json -trace jq -e ' -.jobs.my_job.tasks[7].task_key == "for_each_spark_python_example" and -(.jobs.my_job.tasks[7].for_each_task.task.spark_python_task.python_file | endswith("/src/file.py")) -' output.development.json - -# Pipeline assertions - -trace jq -e ' -(.pipelines.my_pipeline.libraries[0].file.path | endswith("/src/file1.py")) -' output.development.json - -trace jq -e ' -(.pipelines.my_pipeline.libraries[1].notebook.path | endswith("/src/notebook1")) -' output.development.json - -trace jq -e ' -(.pipelines.my_pipeline.libraries[2].file.path | endswith("/src/file2.py")) -' output.development.json - -trace jq -e ' -(.pipelines.my_pipeline.libraries[3].notebook.path | endswith("/src/notebook2")) -' output.development.json +# Expect failure for the "error" target +errcode trace $CLI bundle validate -t error diff --git a/acceptance/bundle/paths/nominal/script.cleanup b/acceptance/bundle/paths/nominal/script.cleanup new file mode 100644 index 0000000000..f93425dffd --- /dev/null +++ b/acceptance/bundle/paths/nominal/script.cleanup @@ -0,0 +1 @@ +rm -f output.tmp.json diff --git a/acceptance/bundle/paths/relative_path_translation/output.default.json b/acceptance/bundle/paths/relative_path_translation/output.default.json index e652a42756..e2514b392f 100644 --- a/acceptance/bundle/paths/relative_path_translation/output.default.json +++ b/acceptance/bundle/paths/relative_path_translation/output.default.json @@ -1,37 +1,6 @@ { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/relative_path_translation/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "job_clusters": [ - { - "job_cluster_key": "default", - "new_cluster": { - "spark_version": "15.4.x-scala2.12" - } - } - ], - "permissions": [], - "queue": { - "enabled": true - }, - "tags": {}, - "tasks": [ - { - "job_cluster_key": "default", - "spark_python_task": { - "python_file": "/Workspace/remote/src/file1.py" - }, - "task_key": "local" - }, - { - "job_cluster_key": "default", - "spark_python_task": { - "python_file": "/Workspace/remote/src/file1.py" - }, - "task_key": "variable_reference" - } + "paths": [ + "/Workspace/remote/src/file1.py", + "/Workspace/remote/src/file1.py" ] } diff --git a/acceptance/bundle/paths/relative_path_translation/output.override.json b/acceptance/bundle/paths/relative_path_translation/output.override.json index 5c81f4e987..729d2eaa04 100644 --- a/acceptance/bundle/paths/relative_path_translation/output.override.json +++ b/acceptance/bundle/paths/relative_path_translation/output.override.json @@ -1,37 +1,6 @@ { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/relative_path_translation/override/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "job_clusters": [ - { - "job_cluster_key": "default", - "new_cluster": { - "spark_version": "15.4.x-scala2.12" - } - } - ], - "permissions": [], - "queue": { - "enabled": true - }, - "tags": {}, - "tasks": [ - { - "job_cluster_key": "default", - "spark_python_task": { - "python_file": "/Workspace/remote/src/file2.py" - }, - "task_key": "local" - }, - { - "job_cluster_key": "default", - "spark_python_task": { - "python_file": "/Workspace/remote/src/file2.py" - }, - "task_key": "variable_reference" - } + "paths": [ + "/Workspace/remote/src/file2.py", + "/Workspace/remote/src/file2.py" ] } diff --git a/acceptance/bundle/paths/relative_path_translation/output.txt b/acceptance/bundle/paths/relative_path_translation/output.txt index 8442be96ce..362f2ec7b3 100644 --- a/acceptance/bundle/paths/relative_path_translation/output.txt +++ b/acceptance/bundle/paths/relative_path_translation/output.txt @@ -1,20 +1,4 @@ >>> $CLI bundle validate -t default -o json -Exit code: 0 - >>> $CLI bundle validate -t override -o json - -Exit code: 0 - ->>> jq -e -(.tasks[0].spark_python_task.python_file | endswith("/src/file1.py")) and -(.tasks[1].spark_python_task.python_file | endswith("/src/file1.py")) - output.default.json -true - ->>> jq -e -(.tasks[0].spark_python_task.python_file | endswith("/src/file2.py")) and -(.tasks[1].spark_python_task.python_file | endswith("/src/file2.py")) - output.override.json -true diff --git a/acceptance/bundle/paths/relative_path_translation/script b/acceptance/bundle/paths/relative_path_translation/script index dba5c3d35d..252e9a07f9 100644 --- a/acceptance/bundle/paths/relative_path_translation/script +++ b/acceptance/bundle/paths/relative_path_translation/script @@ -1,12 +1,4 @@ -errcode trace $CLI bundle validate -t default -o json | jq '.resources.jobs.job' > output.default.json -errcode trace $CLI bundle validate -t override -o json | jq '.resources.jobs.job' > output.override.json - -trace jq -e ' -(.tasks[0].spark_python_task.python_file | endswith("/src/file1.py")) and -(.tasks[1].spark_python_task.python_file | endswith("/src/file1.py")) -' output.default.json - -trace jq -e ' -(.tasks[0].spark_python_task.python_file | endswith("/src/file2.py")) and -(.tasks[1].spark_python_task.python_file | endswith("/src/file2.py")) -' output.override.json +trace $CLI bundle validate -t default -o json | \ + jq '{ paths: [.resources.jobs.job.tasks[].spark_python_task.python_file] }' > output.default.json +trace $CLI bundle validate -t override -o json | \ + jq '{ paths: [.resources.jobs.job.tasks[].spark_python_task.python_file] }' > output.override.json From e683c1bf44c75d7add60d989d0bb408100c18387 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 14 Jan 2025 17:16:10 +0100 Subject: [PATCH 5/5] Update output fixtures --- acceptance/bundle/paths/fallback/output.job.json | 8 ++++---- .../bundle/paths/fallback/output.pipeline.json | 8 ++++---- acceptance/bundle/paths/fallback/output.txt | 4 ++-- acceptance/bundle/paths/nominal/output.job.json | 12 ++++++------ acceptance/bundle/paths/nominal/output.pipeline.json | 8 ++++---- acceptance/bundle/paths/nominal/output.txt | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/acceptance/bundle/paths/fallback/output.job.json b/acceptance/bundle/paths/fallback/output.job.json index fa69143f53..fe9e1cf3de 100644 --- a/acceptance/bundle/paths/fallback/output.job.json +++ b/acceptance/bundle/paths/fallback/output.job.json @@ -2,14 +2,14 @@ { "job_cluster_key": "default", "notebook_task": { - "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook" + "notebook_path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook" }, "task_key": "notebook_example" }, { "job_cluster_key": "default", "spark_python_task": { - "python_file": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file.py" + "python_file": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file.py" }, "task_key": "spark_python_example" }, @@ -19,7 +19,7 @@ "dbt run", "dbt run" ], - "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/dbt_project" + "project_directory": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/dbt_project" }, "job_cluster_key": "default", "task_key": "dbt_example" @@ -28,7 +28,7 @@ "job_cluster_key": "default", "sql_task": { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/sql.sql" + "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/sql.sql" }, "warehouse_id": "cafef00d" }, diff --git a/acceptance/bundle/paths/fallback/output.pipeline.json b/acceptance/bundle/paths/fallback/output.pipeline.json index 1bfcc45dca..38521cb228 100644 --- a/acceptance/bundle/paths/fallback/output.pipeline.json +++ b/acceptance/bundle/paths/fallback/output.pipeline.json @@ -1,22 +1,22 @@ [ { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file1.py" + "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file1.py" } }, { "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook1" + "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook1" } }, { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/file2.py" + "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file2.py" } }, { "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/fallback/development/files/src/notebook2" + "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook2" } } ] diff --git a/acceptance/bundle/paths/fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt index a2aff856c5..f694610d2c 100644 --- a/acceptance/bundle/paths/fallback/output.txt +++ b/acceptance/bundle/paths/fallback/output.txt @@ -10,8 +10,8 @@ to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb] Name: fallback Target: error Workspace: - User: tester@databricks.com - Path: /Workspace/Users/tester@databricks.com/.bundle/fallback/error + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/fallback/error Found 1 error diff --git a/acceptance/bundle/paths/nominal/output.job.json b/acceptance/bundle/paths/nominal/output.job.json index 1b2a4a541c..9e1cb4d90c 100644 --- a/acceptance/bundle/paths/nominal/output.job.json +++ b/acceptance/bundle/paths/nominal/output.job.json @@ -2,14 +2,14 @@ { "job_cluster_key": "default", "notebook_task": { - "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook" + "notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook" }, "task_key": "notebook_example" }, { "job_cluster_key": "default", "spark_python_task": { - "python_file": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file.py" + "python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py" }, "task_key": "spark_python_example" }, @@ -19,7 +19,7 @@ "dbt run", "dbt run" ], - "project_directory": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/dbt_project" + "project_directory": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/dbt_project" }, "job_cluster_key": "default", "task_key": "dbt_example" @@ -28,7 +28,7 @@ "job_cluster_key": "default", "sql_task": { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/sql.sql" + "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/sql.sql" }, "warehouse_id": "cafef00d" }, @@ -68,7 +68,7 @@ "for_each_task": { "task": { "notebook_task": { - "notebook_path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook" + "notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook" } } }, @@ -80,7 +80,7 @@ "task": { "job_cluster_key": "default", "spark_python_task": { - "python_file": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file.py" + "python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py" } } }, diff --git a/acceptance/bundle/paths/nominal/output.pipeline.json b/acceptance/bundle/paths/nominal/output.pipeline.json index 21b4a6ebd2..277b0c4a1a 100644 --- a/acceptance/bundle/paths/nominal/output.pipeline.json +++ b/acceptance/bundle/paths/nominal/output.pipeline.json @@ -1,22 +1,22 @@ [ { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file1.py" + "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file1.py" } }, { "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook1" + "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook1" } }, { "file": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/file2.py" + "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file2.py" } }, { "notebook": { - "path": "/Workspace/Users/tester@databricks.com/.bundle/nominal/development/files/src/notebook2" + "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook2" } } ] diff --git a/acceptance/bundle/paths/nominal/output.txt b/acceptance/bundle/paths/nominal/output.txt index 671386d811..1891703356 100644 --- a/acceptance/bundle/paths/nominal/output.txt +++ b/acceptance/bundle/paths/nominal/output.txt @@ -10,8 +10,8 @@ to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb] Name: nominal Target: error Workspace: - User: tester@databricks.com - Path: /Workspace/Users/tester@databricks.com/.bundle/nominal/error + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/nominal/error Found 1 error