bundle: record and read deployment state via DMS

acceptance/bundle/dms/depends-on/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: dms-depends-on
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  jobs:
+    parent:
+      name: parent
+    child:
+      name: child
+      description: depends on ${resources.jobs.parent.id}

acceptance/bundle/dms/depends-on/output.txt

@@ -0,0 +1,26 @@
+
+=== Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //versions/1/operations --sort --oneline
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.child"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.child", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "description": "depends on [NUMID]", "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "child", "queue": {"enabled": true}}, "depends_on": [{"node": "resources.jobs.parent", "label": "${resources.jobs.parent.id}"}]}, "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.parent"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.parent", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "parent", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}}
+
+=== Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.child
+  delete resources.jobs.parent
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default
+
+Deleting files...
+Destroy complete!
+
+>>> print_requests.py //jobs --oneline
+{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}}
+{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}}

acceptance/bundle/dms/depends-on/script

@@ -0,0 +1,8 @@
+title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself"
+trace $CLI bundle deploy
+trace print_requests.py //versions/1/operations --sort --oneline
+
+title "Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references"
+rm -rf .databricks
+trace $CLI bundle destroy --auto-approve
+trace print_requests.py //jobs --oneline

acceptance/bundle/dms/existing-state/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: dms-existing-state
+
+experimental:
+  record_deployment_history: false
+
+resources:
+  jobs:
+    one:
+      name: one

acceptance/bundle/dms/existing-state/output.txt

@@ -0,0 +1,52 @@
+
+=== Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort --oneline
+
+=== Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time
+>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true
+
+>>> musterr [CLI] bundle deploy
+Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history
+
+
+=== No deployment was created in DMS
+>>> print_requests.py //api/2.0/bundle --sort --oneline
+
+=== Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked
+>>> musterr [CLI] bundle deploy
+Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history
+
+
+>>> print_requests.py //api/2.0/bundle --sort --oneline
+
+=== Destroy clears the tracked resources, so recording can be enabled afterwards
+>>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.one
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default
+
+Deleting files...
+Destroy complete!
+
+>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort --oneline
+{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}}

acceptance/bundle/dms/existing-state/script

@@ -0,0 +1,22 @@
+title "Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --sort --oneline
+
+title "Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time"
+trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true"
+trace musterr $CLI bundle deploy
+
+title "No deployment was created in DMS"
+trace print_requests.py //api/2.0/bundle --sort --oneline
+
+title "Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked"
+rm -rf .databricks
+trace musterr $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --sort --oneline
+
+title "Destroy clears the tracked resources, so recording can be enabled afterwards"
+trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false"
+trace $CLI bundle destroy --auto-approve
+trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --sort --oneline

acceptance/bundle/dms/multiple-resources/databricks.yml

@@ -0,0 +1,18 @@
+bundle:
+  name: dms-multiple-resources
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  jobs:
+    one:
+      name: one
+    two:
+      name: two
+    three:
+      name: three
+    four:
+      name: four
+    five:
+      name: five

acceptance/bundle/dms/multiple-resources/output.txt

@@ -0,0 +1,25 @@
+
+=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it.
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //versions/1/operations --sort --del-body state --oneline
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}}
+
+=== Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort --oneline
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}

acceptance/bundle/dms/multiple-resources/script

@@ -0,0 +1,7 @@
+title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it."
+trace $CLI bundle deploy
+trace print_requests.py //versions/1/operations --sort --del-body state --oneline
+
+title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --sort --oneline

acceptance/bundle/dms/no-resources/databricks.yml

@@ -0,0 +1,5 @@
+bundle:
+  name: dms-no-resources
+
+experimental:
+  record_deployment_history: true

acceptance/bundle/dms/no-resources/output.txt

@@ -0,0 +1,76 @@
+
+=== First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files...
+Deploying resources...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort --get
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "body": {
+    "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state",
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[NUMID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json
+{
+  "object_type": "FILE",
+  "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json"
+}
+
+=== Redeploy: the deployment is resolved from that node, so no second deployment is created
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files...
+Deploying resources...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --sort --get
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[NUMID]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[NUMID]/resources"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[NUMID]/versions",
+  "q": {
+    "version_id": "2"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}

acceptance/bundle/dms/no-resources/script

@@ -0,0 +1,8 @@
+title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --sort --get
+trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}'
+
+title "Redeploy: the deployment is resolved from that node, so no second deployment is created"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --sort --get

acceptance/bundle/dms/record/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: dms-record
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: foo