Custom annotations for bundle-specific JSON schema fields

.codegen.json

@@ -11,7 +11,7 @@
         "required": ["go"],
         "post_generate": [
             "go test -timeout 240s -run TestConsistentDatabricksSdkVersion github.com/databricks/cli/internal/build",
-            "go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json",
+            "make schema",
             "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
             "echo 'go.sum linguist-generated=true' >> ./.gitattributes",
             "echo 'bundle/schema/jsonschema.json linguist-generated=true' >> ./.gitattributes"

.github/workflows/push.yml

@@ -99,14 +99,19 @@ jobs:
       # By default the ajv-cli runs in strict mode which will fail if the schema
       # itself is not valid. Strict mode is more strict than the JSON schema
       # specification. See for details: https://ajv.js.org/options.html#strict-mode-options
+      # The ajv-cli is configured to use the markdownDescription keyword which is not part of the JSON schema specification,
+      # but is used in editors like VSCode to render markdown in the description field
       - name: Validate bundle schema
         run: |
           go run main.go bundle schema > schema.json
 
+          # Add markdownDescription keyword to ajv
+          echo "module.exports=function(a){a.addKeyword('markdownDescription')}" >> keywords.js
+
           for file in ./bundle/internal/schema/testdata/pass/*.yml; do
-            ajv test -s schema.json -d $file --valid
+            ajv test -s schema.json -d $file --valid -c=./keywords.js
           done
 
           for file in ./bundle/internal/schema/testdata/fail/*.yml; do
-            ajv test -s schema.json -d $file --invalid
+            ajv test -s schema.json -d $file --invalid -c=./keywords.js
           done

Makefile

@@ -29,6 +29,10 @@ snapshot:
 vendor:
 	@echo "✓ Filling vendor folder with library code ..."
 	@go mod vendor
+
+schema:
+	@echo "✓ Generating json-schema ..."
+	@go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
 
 INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h
 
@@ -38,4 +42,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck test testonly coverage build snapshot vendor integration integration-short
+.PHONY: lint lintcheck test testonly coverage build snapshot vendor schema integration integration-short

bundle/internal/schema/annotations.go

@@ -0,0 +1,209 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"reflect"
+	"regexp"
+	"strings"
+
+	yaml3 "gopkg.in/yaml.v3"
+
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/dyn/merge"
+	"github.com/databricks/cli/libs/dyn/yamlloader"
+	"github.com/databricks/cli/libs/dyn/yamlsaver"
+	"github.com/databricks/cli/libs/jsonschema"
+)
+
+type annotation struct {
+	Description         string `json:"description,omitempty"`
+	MarkdownDescription string `json:"markdown_description,omitempty"`
+	Title               string `json:"title,omitempty"`
+	Default             any    `json:"default,omitempty"`
+	Enum                []any  `json:"enum,omitempty"`
+}
+
+type annotationHandler struct {
+	// Annotations read from all annotation files including all overrides
+	parsedAnnotations annotationFile
+	// Missing annotations for fields that are found in config that need to be added to the annotation file
+	missingAnnotations annotationFile
+}
+
+/**
+ * Parsed file with annotations, expected format:
+ * github.com/databricks/cli/bundle/config.Bundle:
+ *  	cluster_id:
+ *      description: "Description"
+ */
+type annotationFile map[string]map[string]annotation
+
+const Placeholder = "PLACEHOLDER"
+
+// Adds annotations to the JSON schema reading from the annotation files.
+// More details https://json-schema.org/understanding-json-schema/reference/annotations
+func newAnnotationHandler(sources []string) (*annotationHandler, error) {
+	prev := dyn.NilValue
+	for _, path := range sources {
+		b, err := os.ReadFile(path)
+		if err != nil {
+			return nil, err
+		}
+		generated, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b))
+		if err != nil {
+			return nil, err
+		}
+		prev, err = merge.Merge(prev, generated)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var data annotationFile
+
+	err := convert.ToTyped(&data, prev)
+	if err != nil {
+		return nil, err
+	}
+
+	d := &annotationHandler{}
+	d.parsedAnnotations = data
+	d.missingAnnotations = annotationFile{}
+	return d, nil
+}
+
+func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	refPath := getPath(typ)
+	shouldHandle := strings.HasPrefix(refPath, "github.com")
+	if !shouldHandle {
+		return s
+	}
+
+	annotations := d.parsedAnnotations[refPath]
+	if annotations == nil {
+		annotations = map[string]annotation{}
+	}
+
+	rootTypeAnnotation, ok := annotations[RootTypeKey]
+	if ok {
+		assignAnnotation(&s, rootTypeAnnotation)
+	}
+
+	for k, v := range s.Properties {
+		item := annotations[k]
+		if item.Description == "" {
+			item.Description = Placeholder
+
+			emptyAnnotations := d.missingAnnotations[refPath]
+			if emptyAnnotations == nil {
+				emptyAnnotations = map[string]annotation{}
+				d.missingAnnotations[refPath] = emptyAnnotations
+			}
+			emptyAnnotations[k] = item
+		}
+		assignAnnotation(v, item)
+	}
+	return s
+}
+
+// Writes missing annotations with placeholder values back to the annotation file
+func (d *annotationHandler) syncWithMissingAnnotations(outputPath string) error {
+	existingFile, err := os.ReadFile(outputPath)
+	if err != nil {
+		return err
+	}
+	existing, err := yamlloader.LoadYAML("", bytes.NewBuffer(existingFile))
+	if err != nil {
+		return err
+	}
+	missingAnnotations, err := convert.FromTyped(&d.missingAnnotations, dyn.NilValue)
+	if err != nil {
+		return err
+	}
+
+	output, err := merge.Merge(existing, missingAnnotations)
+	if err != nil {
+		return err
+	}
+
+	err = saveYamlWithStyle(outputPath, output)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func getPath(typ reflect.Type) string {
+	return typ.PkgPath() + "." + typ.Name()
+}
+
+func assignAnnotation(s *jsonschema.Schema, a annotation) {
+	if a.Description != Placeholder {
+		s.Description = a.Description
+	}
+
+	if a.Default != nil {
+		s.Default = a.Default
+	}
+	s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription)
+	s.Title = a.Title
+	s.Enum = a.Enum
+}
+
+func saveYamlWithStyle(outputPath string, input dyn.Value) error {
+	style := map[string]yaml3.Style{}
+	file, _ := input.AsMap()
+	for _, v := range file.Keys() {
+		style[v.MustString()] = yaml3.LiteralStyle
+	}
+
+	saver := yamlsaver.NewSaverWithStyle(style)
+	err := saver.SaveAsYAML(file, outputPath, true)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func convertLinksToAbsoluteUrl(s string) string {
+	if s == "" {
+		return s
+	}
+	base := "https://docs.databricks.com"
+	referencePage := "/dev-tools/bundles/reference.html"
+
+	// Regular expression to match Markdown-style links like [_](link)
+	re := regexp.MustCompile(`\[_\]\(([^)]+)\)`)
+	result := re.ReplaceAllStringFunc(s, func(match string) string {
+		matches := re.FindStringSubmatch(match)
+		if len(matches) < 2 {
+			return match
+		}
+		link := matches[1]
+		var text, absoluteURL string
+
+		if strings.HasPrefix(link, "#") {
+			text = strings.TrimPrefix(link, "#")
+			absoluteURL = fmt.Sprintf("%s%s%s", base, referencePage, link)
+
+			// Handle relative paths like /dev-tools/bundles/resources.html#dashboard
+		} else if strings.HasPrefix(link, "/") {
+			absoluteURL = strings.ReplaceAll(fmt.Sprintf("%s%s", base, link), ".md", ".html")
+			if strings.Contains(link, "#") {
+				parts := strings.Split(link, "#")
+				text = parts[1]
+			} else {
+				text = "link"
+			}
+		} else {
+			return match
+		}
+
+		return fmt.Sprintf("[%s](%s)", text, absoluteURL)
+	})
+
+	return result
+}