Fix never-converging job diff on reordered webhook notifications

acceptance/bundle/resources/jobs/webhook-reorder-remote/databricks.yml

@@ -0,0 +1,20 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      name: webhook reorder
+      webhook_notifications:
+        on_success:
+          - id: alpha
+          - id: beta
+          - id: gamma
+      tasks:
+        - task_key: main
+          run_job_task:
+            job_id: 123
+          webhook_notifications:
+            on_start:
+              - id: delta
+              - id: epsilon

acceptance/bundle/resources/jobs/webhook-reorder-remote/out.plan.direct.json

@@ -0,0 +1,27 @@
+{
+  "email_notifications": {
+    "action": "skip",
+    "reason": "empty",
+    "remote": {}
+  },
+  "tasks[task_key='main'].email_notifications": {
+    "action": "skip",
+    "reason": "empty",
+    "remote": {}
+  },
+  "tasks[task_key='main'].run_if": {
+    "action": "skip",
+    "reason": "backend_default",
+    "remote": "ALL_SUCCESS"
+  },
+  "tasks[task_key='main'].timeout_seconds": {
+    "action": "skip",
+    "reason": "empty",
+    "remote": 0
+  },
+  "timeout_seconds": {
+    "action": "skip",
+    "reason": "empty",
+    "remote": 0
+  }
+}

acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt

@@ -0,0 +1,123 @@
+
+>>> [CLI] bundle plan
+create jobs.my_job
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== Remote returns webhook destinations in a different order
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> print_requests.py //jobs
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/create",
+  "body": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "webhook reorder",
+    "queue": {
+      "enabled": true
+    },
+    "tasks": [
+      {
+        "run_job_task": {
+          "job_id": 123
+        },
+        "task_key": "main",
+        "webhook_notifications": {
+          "on_start": [
+            {
+              "id": "delta"
+            },
+            {
+              "id": "epsilon"
+            }
+          ]
+        }
+      }
+    ],
+    "webhook_notifications": {
+      "on_success": [
+        {
+          "id": "alpha"
+        },
+        {
+          "id": "beta"
+        },
+        {
+          "id": "gamma"
+        }
+      ]
+    }
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/reset",
+  "body": {
+    "job_id": [MY_JOB_ID],
+    "new_settings": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "email_notifications": {},
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "webhook reorder",
+      "queue": {
+        "enabled": true
+      },
+      "tasks": [
+        {
+          "email_notifications": {},
+          "run_if": "ALL_SUCCESS",
+          "run_job_task": {
+            "job_id": 123
+          },
+          "task_key": "main",
+          "timeout_seconds": 0,
+          "webhook_notifications": {
+            "on_start": [
+              {
+                "id": "epsilon"
+              },
+              {
+                "id": "delta"
+              }
+            ]
+          }
+        }
+      ],
+      "timeout_seconds": 0,
+      "webhook_notifications": {
+        "on_success": [
+          {
+            "id": "alpha"
+          },
+          {
+            "id": "gamma"
+          },
+          {
+            "id": "beta"
+          }
+        ]
+      }
+    }
+  }
+}

acceptance/bundle/resources/jobs/webhook-reorder-remote/script

@@ -0,0 +1,19 @@
+trace $CLI bundle plan
+$CLI bundle deploy
+job_id="$(read_id.py my_job)"
+
+trace $CLI bundle plan | contains.py "0 to add, 0 to change, 0 to delete"
+
+title "Remote returns webhook destinations in a different order\n"
+edit_resource.py jobs "$job_id" <<EOF
+s = r["webhook_notifications"]["on_success"]
+s[:] = [s[0], s[2], s[1]]
+t = r["tasks"][0]["webhook_notifications"]["on_start"]
+t[:] = [t[1], t[0]]
+EOF
+
+# The reordered remote must not produce a phantom diff: on_* lists are diffed by id.
+trace $CLI bundle plan
+$CLI bundle plan -o json | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json
+
+trace print_requests.py //jobs

acceptance/bundle/resources/jobs/webhook-reorder-remote/test.toml

@@ -0,0 +1,2 @@
+# The phantom-diff bug and the KeyedSlices fix are specific to the direct engine.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ['direct']

bundle/direct/dresources/job.go

@@ -75,15 +75,42 @@ func getDependsOnTaskKey(x jobs.TaskDependency) (string, string) {
 	return "task_key", x.TaskKey
 }
 
+func getWebhookKey(x jobs.Webhook) (string, string) {
+	return "id", x.Id
+}
+
+// The Jobs API returns webhook_notifications.on_* in an arbitrary order, so
+// diff them by id to avoid a phantom diff that never converges.
+var webhookNotificationEvents = []string{
+	"on_start",
+	"on_success",
+	"on_failure",
+	"on_duration_warning_threshold_exceeded",
+	"on_streaming_backlog_exceeded",
+}
+
+// webhook_notifications appears at the job, task, and for_each task levels.
+var webhookNotificationParents = []string{
+	"webhook_notifications",
+	"tasks[*].webhook_notifications",
+	"tasks[*].for_each_task.task.webhook_notifications",
+}
+
 func (*ResourceJob) KeyedSlices() map[string]any {
-	return map[string]any{
+	result := map[string]any{
 		"tasks":                                  getTaskKey,
 		"parameters":                             getParameterName,
 		"job_clusters":                           getJobClusterKey,
 		"environments":                           getEnvironmentKey,
 		"tasks[*].depends_on":                    getDependsOnTaskKey,
 		"tasks[*].for_each_task.task.depends_on": getDependsOnTaskKey,
 	}
+	for _, parent := range webhookNotificationParents {
+		for _, event := range webhookNotificationEvents {
+			result[parent+"."+event] = getWebhookKey
+		}
+	}
+	return result
 }
 
 func (r *ResourceJob) DoRead(ctx context.Context, id string) (*JobRemote, error) {

bundle/direct/dresources/job_test.go

@@ -4,7 +4,10 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/databricks/cli/libs/structs/structdiff"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // TestJobRemote verifies that all fields from jobs.Job (except Settings and pagination/internal fields)
@@ -17,3 +20,71 @@ func TestJobRemote(t *testing.T) {
 		"NextPageToken":   true, // Pagination field, not relevant for single job read
 	})
 }
+
+func webhooks(ids ...string) []jobs.Webhook {
+	result := make([]jobs.Webhook, len(ids))
+	for i, id := range ids {
+		result[i] = jobs.Webhook{Id: id}
+	}
+	return result
+}
+
+// TestJobWebhookNotificationsOrderInsensitive verifies that a webhook list
+// reordered by the Jobs API produces no diff, but a changed set still does.
+func TestJobWebhookNotificationsOrderInsensitive(t *testing.T) {
+	keys := (&ResourceJob{}).KeyedSlices()
+
+	config := jobs.JobSettings{
+		WebhookNotifications: &jobs.WebhookNotifications{
+			OnSuccess: webhooks("a", "b", "c"),
+		},
+		Tasks: []jobs.Task{
+			{
+				TaskKey: "t1",
+				WebhookNotifications: &jobs.WebhookNotifications{
+					OnFailure: webhooks("a", "b"),
+				},
+				ForEachTask: &jobs.ForEachTask{
+					Task: jobs.Task{
+						TaskKey: "inner",
+						WebhookNotifications: &jobs.WebhookNotifications{
+							OnStart: webhooks("x", "y"),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	remote := jobs.JobSettings{
+		WebhookNotifications: &jobs.WebhookNotifications{
+			OnSuccess: webhooks("a", "c", "b"),
+		},
+		Tasks: []jobs.Task{
+			{
+				TaskKey: "t1",
+				WebhookNotifications: &jobs.WebhookNotifications{
+					OnFailure: webhooks("b", "a"),
+				},
+				ForEachTask: &jobs.ForEachTask{
+					Task: jobs.Task{
+						TaskKey: "inner",
+						WebhookNotifications: &jobs.WebhookNotifications{
+							OnStart: webhooks("y", "x"),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	changes, err := structdiff.GetStructDiff(config, remote, keys)
+	require.NoError(t, err)
+	assert.Empty(t, changes)
+
+	// A genuinely different destination set is still detected.
+	remote.WebhookNotifications.OnSuccess = webhooks("a", "b", "d")
+	changes, err = structdiff.GetStructDiff(config, remote, keys)
+	require.NoError(t, err)
+	assert.NotEmpty(t, changes)
+}