merged upstream and added plan modifiers to required fields

.codegen/_openapi_sha

@@ -1 +1 @@
-7016dcbf2e011459416cf408ce21143bcc4b3a25
+779817ed8d63031f5ea761fbd25ee84f38feec0d

.gitattributes

@@ -1,20 +1,40 @@
+internal/service/apps_tf/legacy_model.go linguist-generated=true
 internal/service/apps_tf/model.go linguist-generated=true
+internal/service/billing_tf/legacy_model.go linguist-generated=true
 internal/service/billing_tf/model.go linguist-generated=true
+internal/service/catalog_tf/legacy_model.go linguist-generated=true
 internal/service/catalog_tf/model.go linguist-generated=true
+internal/service/cleanrooms_tf/legacy_model.go linguist-generated=true
 internal/service/cleanrooms_tf/model.go linguist-generated=true
+internal/service/compute_tf/legacy_model.go linguist-generated=true
 internal/service/compute_tf/model.go linguist-generated=true
+internal/service/dashboards_tf/legacy_model.go linguist-generated=true
 internal/service/dashboards_tf/model.go linguist-generated=true
+internal/service/files_tf/legacy_model.go linguist-generated=true
 internal/service/files_tf/model.go linguist-generated=true
+internal/service/iam_tf/legacy_model.go linguist-generated=true
 internal/service/iam_tf/model.go linguist-generated=true
+internal/service/jobs_tf/legacy_model.go linguist-generated=true
 internal/service/jobs_tf/model.go linguist-generated=true
+internal/service/marketplace_tf/legacy_model.go linguist-generated=true
 internal/service/marketplace_tf/model.go linguist-generated=true
+internal/service/ml_tf/legacy_model.go linguist-generated=true
 internal/service/ml_tf/model.go linguist-generated=true
+internal/service/oauth2_tf/legacy_model.go linguist-generated=true
 internal/service/oauth2_tf/model.go linguist-generated=true
+internal/service/pipelines_tf/legacy_model.go linguist-generated=true
 internal/service/pipelines_tf/model.go linguist-generated=true
+internal/service/provisioning_tf/legacy_model.go linguist-generated=true
 internal/service/provisioning_tf/model.go linguist-generated=true
+internal/service/serving_tf/legacy_model.go linguist-generated=true
 internal/service/serving_tf/model.go linguist-generated=true
+internal/service/settings_tf/legacy_model.go linguist-generated=true
 internal/service/settings_tf/model.go linguist-generated=true
+internal/service/sharing_tf/legacy_model.go linguist-generated=true
 internal/service/sharing_tf/model.go linguist-generated=true
+internal/service/sql_tf/legacy_model.go linguist-generated=true
 internal/service/sql_tf/model.go linguist-generated=true
+internal/service/vectorsearch_tf/legacy_model.go linguist-generated=true
 internal/service/vectorsearch_tf/model.go linguist-generated=true
+internal/service/workspace_tf/legacy_model.go linguist-generated=true
 internal/service/workspace_tf/model.go linguist-generated=true

.github/workflows/codeql-analysis.yml

@@ -12,7 +12,10 @@ on:
 jobs:
   analyze:
     name: Analyze
-    runs-on: ubuntu-latest
+
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
 
     strategy:
       fail-fast: false

.github/workflows/external-message.yml

@@ -13,7 +13,10 @@ on:
 
 jobs:
   comment-on-pr:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     permissions:
       pull-requests: write
 
@@ -44,13 +47,13 @@ jobs:
           gh pr comment ${{ github.event.pull_request.number }} --body \
           "<!-- INTEGRATION_TESTS_MANUAL -->
           If integration tests don't run automatically, an authorized user can run them manually by following the instructions below:
-          
+
           Trigger:
           [go/deco-tests-run/terraform](https://go/deco-tests-run/terraform)
 
           Inputs:
           * PR number: ${{github.event.pull_request.number}}
           * Commit SHA: \`${{ env.COMMIT_SHA }}\`
-          
+
           Checks will be approved automatically on success.
           "

.github/workflows/integration-tests.yml

@@ -6,12 +6,16 @@ on:
     types: [opened, synchronize]
 
   merge_group:
-  
+
 
 jobs:
   check-token:
     name: Check secrets access
-    runs-on: ubuntu-latest
+
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     environment: "test-trigger-is"
     outputs:
       has_token: ${{ steps.set-token-status.outputs.has_token }}
@@ -26,14 +30,18 @@ jobs:
               echo "DECO_WORKFLOW_TRIGGER_APP_ID is set. User has access to secrets."
               echo "::set-output name=has_token::true"
             fi
-    
+
   trigger-tests:
     name: Trigger Tests
-    runs-on: ubuntu-latest
+
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     needs: check-token
     if: github.event_name == 'pull_request'  && needs.check-token.outputs.has_token == 'true'
     environment: "test-trigger-is"
-    
+
     steps:
     - uses: actions/checkout@v3
 
@@ -45,28 +53,32 @@ jobs:
         private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
         owner: ${{ secrets.ORG_NAME }}
         repositories: ${{secrets.REPO_NAME}}
-    
+
     - name: Trigger Workflow in Another Repo
       env:
         GH_TOKEN: ${{ steps.generate-token.outputs.token }}
       run: |
         gh workflow run terraform-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \
         --ref main \
         -f pull_request_number=${{ github.event.pull_request.number }} \
-        -f commit_sha=${{ github.event.pull_request.head.sha }} 
+        -f commit_sha=${{ github.event.pull_request.head.sha }}
+
 
-    
 
-  # Statuses and checks apply to specific commits (by hash). 
+  # Statuses and checks apply to specific commits (by hash).
   # Enforcement of required checks is done both at the PR level and the merge queue level.
-  # In case of multiple commits in a single PR, the hash of the squashed commit 
+  # In case of multiple commits in a single PR, the hash of the squashed commit
   # will not match the one for the latest (approved) commit in the PR.
   # We auto approve the check for the merge queue for two reasons:
   # * Queue times out due to duration of tests.
   # * Avoid running integration tests twice, since it was already run at the tip of the branch before squashing.
   auto-approve:
     if: github.event_name == 'merge_group'
-    runs-on: ubuntu-latest
+
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     steps:
       - name: Mark Check
         env:
@@ -77,4 +89,4 @@ jobs:
               -H "X-GitHub-Api-Version: 2022-11-28" \
               /repos/${{ github.repository }}/statuses/${{ github.sha }} \
               -f 'state=success' \
-              -f 'context=Integration Tests Check'
+              -f 'context=Integration Tests Check'

.github/workflows/release.yml

@@ -9,7 +9,10 @@ on:
 
 jobs:
   goreleaser:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/tagging.yml

@@ -0,0 +1,42 @@
+name: tagging
+
+on:
+  workflow_dispatch:
+  # Enable for automatic tagging
+  #schedule:
+  #  - cron: '0 0 * * TUE'
+
+# Ensure that only a single instance of the workflow is running at a time.
+concurrency: 
+  group: "tagging"
+
+
+jobs:
+  tag:
+    environment: "release-is"
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Generate GitHub App Token
+        id: generate-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.DECO_SDK_TAGGING_APP_ID }}
+          private-key: ${{ secrets.DECO_SDK_TAGGING_PRIVATE_KEY }}
+
+      # NOTE: actions/create-github-app-token automatically masks the output token,
+      # so the token is not exposed in the logs even as part of the remote.
+      - name: Set up Git configuration
+        run: |
+          git config user.name "Databricks SDK Release Bot"
+          git remote set-url origin https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/${{ github.repository }}.git
+
+      - name: Run Deco Taggging script
+        run: |
+          python tagging.py

.package.json

@@ -0,0 +1,5 @@
+{
+    "version": {
+        "common/version.go": "version = \"$VERSION\""
+    }
+}

.release_metadata.json

@@ -0,0 +1,3 @@
+{
+    "timestamp": "2025-01-16 13:55:53"
+}

CHANGELOG.md

@@ -1,5 +1,118 @@
 # Version changelog
 
+## Release v1.63.1
+
+### Exporter
+
+ * Refactor jobs implementation to use only Go SDK
+
+### Internal Changes
+
+ * Introduced new automated tagging workflow. At the time of this commit,
+   the workflow must be run manually.
+
+
+## [Release] Release v1.63.0
+
+### New Features and Improvements
+
+ * Add `fallback` to `databricks_external_location` ([#4372](https://github.com/databricks/terraform-provider-databricks/pull/4372)).
+
+
+### Bug Fixes
+
+ * Send only what is required in Update of `databricks_credential` ([#4349](https://github.com/databricks/terraform-provider-databricks/pull/4349)).
+
+
+### Documentation
+
+ * Add `MANAGE` privilege to the relevant UC resources ([#4371](https://github.com/databricks/terraform-provider-databricks/pull/4371)).
+ * Add a note about schema evolution and `databricks_sql_table` ([#4352](https://github.com/databricks/terraform-provider-databricks/pull/4352)).
+ * Add description of `clean_rooms_notebook_task` in `databricks_job` ([#4368](https://github.com/databricks/terraform-provider-databricks/pull/4368)).
+ * Add docs for cluster attributes for clusters assigned to groups ([#4359](https://github.com/databricks/terraform-provider-databricks/pull/4359)).
+ * Add note about use of `azurerm_databricks` for enhanced security settings ([#4393](https://github.com/databricks/terraform-provider-databricks/pull/4393)).
+ * Clarify views sharing in `databricks_share` ([#4378](https://github.com/databricks/terraform-provider-databricks/pull/4378)).
+ * Document job edit_mode ([#4354](https://github.com/databricks/terraform-provider-databricks/pull/4354)).
+
+
+### Internal Changes
+
+ * Auto generated customizable schemas ([#4356](https://github.com/databricks/terraform-provider-databricks/pull/4356)).
+ * Disable `terraform fmt` in exporter to workaround build issues ([#4394](https://github.com/databricks/terraform-provider-databricks/pull/4394)).
+ * Fixed redundant optional owners in share resource and bumped Go SDK ([#4396](https://github.com/databricks/terraform-provider-databricks/pull/4396)).
+ * Load auto-generated resources and data sources ([#4367](https://github.com/databricks/terraform-provider-databricks/pull/4367)).
+ * Migrate workflows that need write access to use hosted runners ([#4377](https://github.com/databricks/terraform-provider-databricks/pull/4377)).
+ * Remove TF annotations on TFSDK models ([#4383](https://github.com/databricks/terraform-provider-databricks/pull/4383)).
+ * Restart Cluster before Library Installation ([#4384](https://github.com/databricks/terraform-provider-databricks/pull/4384)).
+
+
+### Dependency Updates
+
+ * Bump github.com/zclconf/go-cty from 1.15.1 to 1.16.0 ([#4374](https://github.com/databricks/terraform-provider-databricks/pull/4374)).
+ * Bump github.com/zclconf/go-cty from 1.16.0 to 1.16.1 ([#4397](https://github.com/databricks/terraform-provider-databricks/pull/4397)).
+
+
+### Exporter
+
+ * Add exporting of `for_each_task` in jobs ([#4347](https://github.com/databricks/terraform-provider-databricks/pull/4347)).
+ * Add references for missing permissions/grants types ([#4390](https://github.com/databricks/terraform-provider-databricks/pull/4390)).
+ * Support for group assigned clusters ([#4361](https://github.com/databricks/terraform-provider-databricks/pull/4361)).
+ * Use Go SDK to list clusters ([#4385](https://github.com/databricks/terraform-provider-databricks/pull/4385)).
+ * Use `Workspace.Export` from Go SDK for notebooks/files ([#4391](https://github.com/databricks/terraform-provider-databricks/pull/4391)).
+
+
+## [Release] Release v1.62.1
+
+### Bug Fixes
+
+ * Reflect backend updates in state for databricks_app ([#4337](https://github.com/databricks/terraform-provider-databricks/pull/4337)).
+
+
+### Documentation
+
+ * Update `databricks_workspace_conf` documentation ([#4334](https://github.com/databricks/terraform-provider-databricks/pull/4334)).
+ * apply `make fmt-docs` to all docs ([#4344](https://github.com/databricks/terraform-provider-databricks/pull/4344)).
+
+
+### Internal Changes
+
+ * Generate both SdkV2-compatible and Plugin Framework-compatible structures ([#4332](https://github.com/databricks/terraform-provider-databricks/pull/4332)).
+ * Mark TestAccServicePrincipalResourceOnAzure test as flaky ([#4333](https://github.com/databricks/terraform-provider-databricks/pull/4333)).
+ * Retry on 504 when calling the permission API ([#4355](https://github.com/databricks/terraform-provider-databricks/pull/4355)).
+
+
+## [Release] Release v1.62.0
+
+### New Features and Improvements
+
+ * Allow to use GCP SA in `databricks_credential` (storage only) ([#4302](https://github.com/databricks/terraform-provider-databricks/pull/4302)).
+
+
+### Bug Fixes
+
+ * Bump Golang x/crypto to 0.31 ([#4319](https://github.com/databricks/terraform-provider-databricks/pull/4319)).
+
+
+### Internal Changes
+
+ * Generate models with correctly computed annotations ([#4316](https://github.com/databricks/terraform-provider-databricks/pull/4316)).
+ * Update Go SDK to 0.54.0 ([#4328](https://github.com/databricks/terraform-provider-databricks/pull/4328)).
+
+
+### Dependency Updates
+
+ * Bump github.com/hashicorp/terraform-plugin-framework-validators from 0.15.0 to 0.16.0 ([#4320](https://github.com/databricks/terraform-provider-databricks/pull/4320)).
+
+
+### Exporter
+
+ * Correctly handle DB-managed UC objects ([#4323](https://github.com/databricks/terraform-provider-databricks/pull/4323)).
+ * Emit WSFS/UC files from `parameters` in the jobs ([#4318](https://github.com/databricks/terraform-provider-databricks/pull/4318)).
+ * Improve support of `databricks_model_serving` ([#4324](https://github.com/databricks/terraform-provider-databricks/pull/4324)).
+ * add support for `databricks_credential` ([#4292](https://github.com/databricks/terraform-provider-databricks/pull/4292)).
+ * export `databricks_repo` for Git Folders outside of `/Repos` ([#4308](https://github.com/databricks/terraform-provider-databricks/pull/4308)).
+
+
 ## [Release] Release v1.61.0
 
 ### New Features and Improvements

CONTRIBUTING.md

@@ -120,7 +120,10 @@ We are migrating the resource from SDKv2 to Plugin Framework provider and hence
 
 ### Adding a new resource
 1. Check if the directory for this particular resource exists under `internal/providers/pluginfw/products`, if not create the directory eg: `cluster`, `volume` etc... Please note: Resources and Data sources are organized under the same package for that service.
-2. Create a file with resource_resource-name.go and write the CRUD methods, schema for that resource. For reference, please take a look at existing resources eg: `resource_quality_monitor.go`. Make sure to set the user agent in all the CRUD methods. In the `Metadata()`, if the resource is to be used as default, use the method `GetDatabricksProductionName()` else use `GetDatabricksStagingName()` which suffixes the name with `_pluginframework`. 
+2. Create a file with resource_resource-name.go and write the CRUD methods, schema for that resource. For reference, please take a look at existing resources eg: `resource_app.go`.
+  - Make sure to set the user agent in all the CRUD methods.
+  - In the `Metadata()`, use the method `GetDatabricksProductionName()`.
+  - In the `Schema()` method, import the appropriate struct from the `internal/service/{package}_tf` package and use the `ResourceStructToSchema` method to convert the struct to schema. Use the struct that does not have the `_SdkV2` suffix. The schema for the struct is automatically generated and maintained within the `ApplySchemaCustomizations` method of that struct. If you need to customize the schema further, pass in a `CustomizableSchema` to `ResourceStructToSchema` and customize the schema there. If you need to use a manually crafted struct in place of the auto-generated one, you must implement the `ApplySchemaCustomizations` method in a similar way.
 3. Create a file with `resource_resource-name_acc_test.go` and add integration tests here.
 4. Create a file with `resource_resource-name_test.go` and add unit tests here. Note: Please make sure to abstract specific method of the resource so they are unit test friendly and not testing internal part of terraform plugin framework library. You can compare the diagnostics, for example: please take a look at: `data_cluster_test.go` 
 5. Add the resource under `internal/providers/pluginfw/pluginfw.go` in `Resources()` method. Please update the list so that it stays in alphabetically sorted order.
@@ -139,9 +142,9 @@ There must not be any behaviour change or schema change when migrating a resourc
 - Please make sure there are no breaking differences due to changes in schema by running: `make diff-schema`. 
 - Integration tests shouldn't require any major changes.   
 
-By default, `ResourceStructToSchema` will convert a `types.List` field to a `ListAttribute` or `ListNestedAttribute`. For resources or data sources migrated from the SDKv2, `ListNestedBlock` must be used for such fields. To do this, call `cs.ConfigureAsSdkV2Compatible()` in the `ResourceStructToSchema` callback:
+By default, `ResourceStructToSchema` will convert a `types.List` field to a `ListAttribute` or `ListNestedAttribute`. For resources or data sources migrated from the SDKv2, `ListNestedBlock` must be used for such fields. To do this, use the `_SdkV2` variant from the `internal/service/{package}_tf` package when defining the resource schema and when interacting with the plan, config and state. Additionally, in the `Schema()` method, call `cs.ConfigureAsSdkV2Compatible()` in the `ResourceStructToSchema` callback:
 ```go
-resp.Schema = tfschema.ResourceStructToSchema(ctx, Resource{}, func(c tfschema.CustomizableSchema) tfschema.CustomizableSchema {
+resp.Schema = tfschema.ResourceStructToSchema(ctx, Resource_SdkV2{}, func(c tfschema.CustomizableSchema) tfschema.CustomizableSchema {
     cs.ConfigureAsSdkV2Compatible()
     // Add any additional configuration here
     return cs

NEXT_CHANGELOG.md

@@ -0,0 +1,14 @@
+# NEXT CHANGELOG
+
+## Release v1.64.0
+
+### New Features and Improvements
+
+### Bug Fixes
+
+### Documentation
+
+### Exporter
+
+### Internal Changes
+