diff --git a/.nextchanges/bundles/cluster-policies.md b/.nextchanges/bundles/cluster-policies.md new file mode 100644 index 00000000000..ace7f4b426b --- /dev/null +++ b/.nextchanges/bundles/cluster-policies.md @@ -0,0 +1 @@ +Add support for the `cluster_policies` resource type in Declarative Automation Bundles. Cluster policies are only supported in direct deployment mode. diff --git a/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl b/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl new file mode 100644 index 00000000000..aa514b0e0db --- /dev/null +++ b/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + cluster_policies: + foo: + name: test-cluster-policy-$UNIQUE_NAME + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 1289c80af5e..f174ba144c4 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -16,6 +16,9 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] # instance_pools resource is not supported on v0.293.0 EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] +# cluster_policies resource is not supported on v0.293.0 +EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] + # job_runs resource is not supported on v0.293.0 EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"] diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 0ea874aac37..2dcac058e8c 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -8,6 +8,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 0ea874aac37..2dcac058e8c 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -8,6 +8,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index cb927f4a69a..aa24bf58eed 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -14,6 +14,8 @@ EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmp EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] # Instance pools are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] +# Cluster policies are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 0ea874aac37..2dcac058e8c 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -8,6 +8,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index faa2872a3b0..5c63aeef5b6 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 6c2b033fa97..ac9a5e87d5e 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -279,6 +279,39 @@ resources.catalogs.*.grants[*] catalog.PrivilegeAssignment ALL resources.catalogs.*.grants[*].principal string ALL resources.catalogs.*.grants[*].privileges []catalog.Privilege ALL resources.catalogs.*.grants[*].privileges[*] catalog.Privilege ALL +resources.cluster_policies.*.created_at_timestamp int64 REMOTE +resources.cluster_policies.*.creator_user_name string REMOTE +resources.cluster_policies.*.definition any INPUT +resources.cluster_policies.*.definition string ALL +resources.cluster_policies.*.description string ALL +resources.cluster_policies.*.id string INPUT +resources.cluster_policies.*.is_default bool REMOTE +resources.cluster_policies.*.libraries []compute.Library ALL +resources.cluster_policies.*.libraries[*] compute.Library ALL +resources.cluster_policies.*.libraries[*].cran *compute.RCranLibrary ALL +resources.cluster_policies.*.libraries[*].cran.package string ALL +resources.cluster_policies.*.libraries[*].cran.repo string ALL +resources.cluster_policies.*.libraries[*].egg string ALL +resources.cluster_policies.*.libraries[*].jar string ALL +resources.cluster_policies.*.libraries[*].maven *compute.MavenLibrary ALL +resources.cluster_policies.*.libraries[*].maven.coordinates string ALL +resources.cluster_policies.*.libraries[*].maven.exclusions []string ALL +resources.cluster_policies.*.libraries[*].maven.exclusions[*] string ALL +resources.cluster_policies.*.libraries[*].maven.repo string ALL +resources.cluster_policies.*.libraries[*].pypi *compute.PythonPyPiLibrary ALL +resources.cluster_policies.*.libraries[*].pypi.package string ALL +resources.cluster_policies.*.libraries[*].pypi.repo string ALL +resources.cluster_policies.*.libraries[*].requirements string ALL +resources.cluster_policies.*.libraries[*].whl string ALL +resources.cluster_policies.*.lifecycle resources.Lifecycle INPUT +resources.cluster_policies.*.lifecycle.prevent_destroy bool INPUT +resources.cluster_policies.*.max_clusters_per_user int64 ALL +resources.cluster_policies.*.modified_status string INPUT +resources.cluster_policies.*.name string ALL +resources.cluster_policies.*.policy_family_definition_overrides string ALL +resources.cluster_policies.*.policy_family_id string ALL +resources.cluster_policies.*.policy_id string REMOTE +resources.cluster_policies.*.url string INPUT resources.clusters.*.apply_policy_default_values bool ALL resources.clusters.*.autoscale *compute.AutoScale ALL resources.clusters.*.autoscale.max_workers int ALL diff --git a/acceptance/bundle/resources/cluster_policies/basic/databricks.yml b/acceptance/bundle/resources/cluster_policies/basic/databricks.yml new file mode 100644 index 00000000000..5156a2b9b0f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/basic/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test_cluster_policy + +resources: + cluster_policies: + test_cluster_policy: + name: my_cluster_policy + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/basic/out.test.toml b/acceptance/bundle/resources/cluster_policies/basic/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/basic/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/basic/output.txt b/acceptance/bundle/resources/cluster_policies/basic/output.txt new file mode 100644 index 00000000000..3d0358684e4 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/basic/output.txt @@ -0,0 +1,126 @@ + +>>> [CLI] bundle validate +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + +Validation OK! + +>>> [CLI] bundle validate -o json +{ + "test_cluster_policy": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_cluster_policy" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify the create request +>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/create"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_cluster_policy" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy + URL: [DATABRICKS_URL]/compute/policies/[UUID]?w=[NUMID] + +=== Update the cluster policy name +>>> update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify the update request +>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/edit"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/edit", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_cluster_policy_2", + "policy_id": "[UUID]" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy_2 + URL: [DATABRICKS_URL]/compute/policies/[UUID]?w=[NUMID] + +=== Destroy the cluster policy +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.test_cluster_policy + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + +Deleting files... +Destroy complete! + +=== Verify the destroy request +>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/delete"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/delete", + "body": { + "policy_id": "[UUID]" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy_2 + URL: (not deployed) + +>>> [CLI] bundle destroy --auto-approve +No active deployment found to destroy! diff --git a/acceptance/bundle/resources/cluster_policies/basic/script b/acceptance/bundle/resources/cluster_policies/basic/script new file mode 100644 index 00000000000..412bdcf1520 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/basic/script @@ -0,0 +1,33 @@ +trace $CLI bundle validate +trace $CLI bundle validate -o json | jq ".resources.cluster_policies" + +trace $CLI bundle summary + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm out.requests.txt +} +trap cleanup EXIT +trace $CLI bundle deploy + +title "Verify the create request" +trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/create")))' out.requests.txt + +trace $CLI bundle summary + +title "Update the cluster policy name" +trace update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 +trace $CLI bundle deploy + +title "Verify the update request" +trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/edit")))' out.requests.txt + +trace $CLI bundle summary + +title "Destroy the cluster policy" +trace $CLI bundle destroy --auto-approve + +title "Verify the destroy request" +trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/delete")))' out.requests.txt + +trace $CLI bundle summary diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_a/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_a/databricks.yml new file mode 100644 index 00000000000..195681f9c74 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_a/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: cluster_policy_producer + +resources: + cluster_policies: + pol: + name: shared_policy + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_b/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_b/databricks.yml new file mode 100644 index 00000000000..afacb0e118e --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/bundle_b/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: cluster_policy_consumer + +resources: + jobs: + j: + name: consumer_job + tasks: + - task_key: main + new_cluster: + # Replaced at test time with the policy id created by bundle_a. + policy_id: PLACEHOLDER_POLICY_ID + spark_version: 13.3.x-scala2.12 + num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/out.test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/output.txt b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/output.txt new file mode 100644 index 00000000000..84da21ee84c --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/output.txt @@ -0,0 +1,73 @@ + +=== Bundle A creates the cluster policy +>>> withdir bundle_a [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Inject A's policy id into bundle B, then deploy B +>>> update_file.py bundle_b/databricks.yml PLACEHOLDER_POLICY_ID [POL_ID] + +>>> withdir bundle_b [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Policy created by A, then B's job carries that same policy id +>>> print_requests.py //policies/clusters //jobs +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "shared_policy" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "consumer_job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "num_workers": 1, + "policy_id": "[POL_ID]", + "spark_version": "13.3.x-scala2.12" + }, + "task_key": "main" + } + ] + } +} + +=== Cleanup +>>> withdir bundle_b [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default + +Deleting files... +Destroy complete! + +>>> withdir bundle_a [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/script b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/script new file mode 100644 index 00000000000..e2b47428da7 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/script @@ -0,0 +1,17 @@ +title "Bundle A creates the cluster policy" +trace withdir bundle_a $CLI bundle deploy + +# Capture A's server-generated policy id and register [POL_ID]. +pol_id=$(withdir bundle_a read_id.py pol) + +title "Inject A's policy id into bundle B, then deploy B" +trace update_file.py bundle_b/databricks.yml PLACEHOLDER_POLICY_ID "$pol_id" +trace withdir bundle_b $CLI bundle deploy + +title "Policy created by A, then B's job carries that same policy id" +trace print_requests.py //policies/clusters //jobs + +title "Cleanup" +trace withdir bundle_b $CLI bundle destroy --auto-approve +trace withdir bundle_a $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_id/test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/test.toml new file mode 100644 index 00000000000..601384fdf96 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_id/test.toml @@ -0,0 +1 @@ +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml new file mode 100644 index 00000000000..86501c512c3 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_a/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: cluster_policy_producer + +resources: + cluster_policies: + pol: + name: shared_lookup_policy + definition: |- + { + "spark_version": { + "type": "fixed", + "value": "13.3.x-scala2.12" + } + } diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_b/databricks.yml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_b/databricks.yml new file mode 100644 index 00000000000..0789f610f03 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/bundle_b/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: cluster_policy_consumer + +variables: + policy: + description: Resolve the policy created by bundle_a by name. + lookup: + cluster_policy: shared_lookup_policy + +resources: + jobs: + j: + name: consumer_job + tasks: + - task_key: main + new_cluster: + policy_id: ${var.policy} + spark_version: 13.3.x-scala2.12 + num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/out.test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/output.txt b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/output.txt new file mode 100644 index 00000000000..b5061bbc3f3 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/output.txt @@ -0,0 +1,74 @@ + +=== Bundle A creates the cluster policy +>>> withdir bundle_a [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Bundle B resolves the policy by name via lookup +>>> withdir bundle_b [CLI] bundle validate -o json +{ + "policy": { + "description": "Resolve the policy created by bundle_a by name.", + "lookup": { + "cluster_policy": "shared_lookup_policy" + }, + "value": "[POL_ID]" + } +} + +=== Deploy B; its job carries the resolved policy id +>>> withdir bundle_b [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "consumer_job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "num_workers": 1, + "policy_id": "[POL_ID]", + "spark_version": "13.3.x-scala2.12" + }, + "task_key": "main" + } + ] + } +} + +=== Cleanup +>>> withdir bundle_b [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_consumer/default + +Deleting files... +Destroy complete! + +>>> withdir bundle_a [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_producer/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/script b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/script new file mode 100644 index 00000000000..affac0bb5bb --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/script @@ -0,0 +1,17 @@ +title "Bundle A creates the cluster policy" +trace withdir bundle_a $CLI bundle deploy + +# Register [POL_ID] for A's server-generated policy id. +pol_id=$(withdir bundle_a read_id.py pol) + +title "Bundle B resolves the policy by name via lookup" +trace withdir bundle_b $CLI bundle validate -o json | jq '.variables' + +title "Deploy B; its job carries the resolved policy id" +trace withdir bundle_b $CLI bundle deploy +trace print_requests.py //jobs + +title "Cleanup" +trace withdir bundle_b $CLI bundle destroy --auto-approve +trace withdir bundle_a $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/test.toml b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/test.toml new file mode 100644 index 00000000000..601384fdf96 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/cross_bundle_lookup/test.toml @@ -0,0 +1 @@ +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml b/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml new file mode 100644 index 00000000000..395e21aa5d0 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: cluster_policy_definition_multiline + +resources: + cluster_policies: + pol: + name: my_policy + definition: |- + { + "spark_version": { + "type": "fixed", + "value": "13.3.x-scala2.12" + } + } diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml b/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt new file mode 100644 index 00000000000..e6912527df8 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt @@ -0,0 +1,34 @@ + +>>> [CLI] bundle validate -o json +{ + "pol": { + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", + "name": "my_policy" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Create body preserves the block-scalar definition as a newline-escaped string +>>> print_requests.py //policies/clusters +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", + "name": "my_policy" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/script b/acceptance/bundle/resources/cluster_policies/definition_multiline/script new file mode 100644 index 00000000000..c134016b6e7 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/script @@ -0,0 +1,9 @@ +trace $CLI bundle validate -o json | jq ".resources.cluster_policies" + +trace $CLI bundle deploy + +title "Create body preserves the block-scalar definition as a newline-escaped string" +trace print_requests.py //policies/clusters + +trace $CLI bundle destroy --auto-approve +rm out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml b/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml new file mode 100644 index 00000000000..2b5685d1c84 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: cluster_policy_definition_yaml + +resources: + cluster_policies: + pol: + name: my_policy + definition: + spark_version: + type: fixed + value: 13.3.x-scala2.12 diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml b/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt new file mode 100644 index 00000000000..ac4229558e4 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt @@ -0,0 +1,34 @@ + +>>> [CLI] bundle validate -o json +{ + "pol": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_policy" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Native YAML definition serializes to the compact JSON string the API receives +>>> print_requests.py //policies/clusters +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_policy" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/script b/acceptance/bundle/resources/cluster_policies/definition_yaml/script new file mode 100644 index 00000000000..d4579c26c5d --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/script @@ -0,0 +1,9 @@ +trace $CLI bundle validate -o json | jq ".resources.cluster_policies" + +trace $CLI bundle deploy + +title "Native YAML definition serializes to the compact JSON string the API receives" +trace print_requests.py //policies/clusters + +trace $CLI bundle destroy --auto-approve +rm out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml b/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml new file mode 100644 index 00000000000..8b716db56df --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/direct-only/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: cluster_policy_direct_only + +resources: + cluster_policies: + pol: + name: my_policy + definition: + spark_version: + type: fixed + value: 13.3.x-scala2.12 diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/out.test.toml b/acceptance/bundle/resources/cluster_policies/direct-only/out.test.toml new file mode 100644 index 00000000000..d2059b4b5d7 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/direct-only/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/output.txt b/acceptance/bundle/resources/cluster_policies/direct-only/output.txt new file mode 100644 index 00000000000..8656fb8e6f9 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/direct-only/output.txt @@ -0,0 +1,11 @@ + +=== Deploy should fail in terraform mode +>>> [CLI] bundle deploy +Error: Cluster Policy resources are only supported with direct deployment mode + in databricks.yml:6:5 + +Cluster Policy resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use cluster_policy resources. +Learn more at https://docs.databricks.com/dev-tools/bundles/direct + + +Exit code: 1 diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/script b/acceptance/bundle/resources/cluster_policies/direct-only/script new file mode 100644 index 00000000000..db1c9b194ba --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/direct-only/script @@ -0,0 +1,4 @@ +title "Deploy should fail in terraform mode" +trace $CLI bundle deploy 2>&1 | contains.py \ + "Cluster Policy resources are only supported with direct deployment mode" \ + "DATABRICKS_BUNDLE_ENGINE" diff --git a/acceptance/bundle/resources/cluster_policies/direct-only/test.toml b/acceptance/bundle/resources/cluster_policies/direct-only/test.toml new file mode 100644 index 00000000000..554b3c0b60d --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/direct-only/test.toml @@ -0,0 +1,5 @@ +Cloud = false +RecordRequests = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] + +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml b/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml new file mode 100644 index 00000000000..7bc49b780c2 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: cluster_policy_job_ref + +resources: + cluster_policies: + pol: + name: my_policy + definition: |- + { + "spark_version": { + "type": "fixed", + "value": "13.3.x-scala2.12" + } + } + + jobs: + j: + name: my_job + tasks: + - task_key: main + new_cluster: + policy_id: ${resources.cluster_policies.pol.id} + spark_version: 13.3.x-scala2.12 + num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/out.test.toml b/acceptance/bundle/resources/cluster_policies/job_ref/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt new file mode 100644 index 00000000000..00620492ca3 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt @@ -0,0 +1,123 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Deploy requests in dependency order: policy create precedes job create, job carries resolved policy id +>>> print_requests.py //policies/clusters //jobs +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", + "name": "my_policy" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "my_job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "num_workers": 1, + "policy_id": "[POL_ID]", + "spark_version": "13.3.x-scala2.12" + }, + "task_key": "main" + } + ] + } +} + +=== Update both the policy and the job +>>> update_file.py databricks.yml my_policy my_policy_2 + +>>> update_file.py databricks.yml my_job my_job_2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //policies/clusters //jobs +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/edit", + "body": { + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", + "name": "my_policy_2", + "policy_id": "[POL_ID]" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [NUMID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "my_job_2", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "num_workers": 1, + "policy_id": "[POL_ID]", + "spark_version": "13.3.x-scala2.12" + }, + "task_key": "main" + } + ] + } + } +} + +=== Destroy: job delete precedes policy delete +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //policies/clusters //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [NUMID] + } +} +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/delete", + "body": { + "policy_id": "[POL_ID]" + } +} diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/script b/acceptance/bundle/resources/cluster_policies/job_ref/script new file mode 100644 index 00000000000..bd4b09777bb --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/script @@ -0,0 +1,17 @@ +trace $CLI bundle deploy + +# Register [POL_ID] so the resolved reference in the job body is deterministic. +pol_id=`read_id.py pol` + +title "Deploy requests in dependency order: policy create precedes job create, job carries resolved policy id" +trace print_requests.py //policies/clusters //jobs + +title "Update both the policy and the job" +trace update_file.py databricks.yml my_policy my_policy_2 +trace update_file.py databricks.yml my_job my_job_2 +trace $CLI bundle deploy +trace print_requests.py //policies/clusters //jobs + +title "Destroy: job delete precedes policy delete" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //policies/clusters //jobs diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/databricks.yml b/acceptance/bundle/resources/cluster_policies/missing_ref/databricks.yml new file mode 100644 index 00000000000..55419f57f30 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: cluster_policy_missing_ref + +resources: + jobs: + j: + name: my_job + tasks: + - task_key: main + new_cluster: + # References a cluster policy that is not declared in this bundle. + policy_id: ${resources.cluster_policies.missing.id} + spark_version: 13.3.x-scala2.12 + num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/out.plan.txt b/acceptance/bundle/resources/cluster_policies/missing_ref/out.plan.txt new file mode 100644 index 00000000000..e43b42e0e8f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/out.plan.txt @@ -0,0 +1,2 @@ +Error: invalid dependency "${resources.cluster_policies.missing.id}", no such node "resources.cluster_policies.missing" + diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/out.test.toml b/acceptance/bundle/resources/cluster_policies/missing_ref/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/output.txt b/acceptance/bundle/resources/cluster_policies/missing_ref/output.txt new file mode 100644 index 00000000000..d06d4ddc5ef --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/output.txt @@ -0,0 +1,2 @@ + +=== Plan fails: job references an undeclared cluster policy \ No newline at end of file diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/script b/acceptance/bundle/resources/cluster_policies/missing_ref/script new file mode 100644 index 00000000000..2fa06ac7476 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/script @@ -0,0 +1,2 @@ +title "Plan fails: job references an undeclared cluster policy" +musterr $CLI bundle plan &> out.plan.txt diff --git a/acceptance/bundle/resources/cluster_policies/missing_ref/test.toml b/acceptance/bundle/resources/cluster_policies/missing_ref/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/missing_ref/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/acceptance/bundle/resources/cluster_policies/test.toml b/acceptance/bundle/resources/cluster_policies/test.toml new file mode 100644 index 00000000000..3fe510e7c1b --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/test.toml @@ -0,0 +1,6 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index 1cd89cceda4..75591ed7423 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,13 +9,14 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses === test auto-completion handler >>> [CLI] __complete experimental open , alerts apps catalogs +cluster_policies clusters dashboards database_catalogs diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 46262ba8dbf..99ac6759ee8 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -36,6 +36,7 @@ var unsupportedResources = []string{ "vector_search_indexes", "job_runs", "secrets", + "cluster_policies", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 72817d13566..f31866a6fc3 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -321,6 +321,16 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } } + // Cluster Policies: Prefix. The policy name is a user-facing display name + // (unique, 1-100 chars), not the API id (policy_id), so prefixing it in dev + // mode avoids collisions between developers without changing identity. + for _, cp := range r.ClusterPolicies { + if cp == nil { + continue + } + cp.Name = prefix + cp.Name + } + // Vector Search Endpoints: no prefix. The endpoint name is the primary key // (it's what GET/UPDATE/DELETE address by), so prefixing it would change // the resource's identity rather than just its display name. diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 53fdf89f50e..35eed05cc53 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -153,6 +153,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { InstancePools: map[string]*resources.InstancePool{ "instance_pool1": {CreateInstancePool: compute.CreateInstancePool{InstancePoolName: "instance_pool1", NodeTypeId: "i3.xlarge"}}, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "cluster_policy1": {CreatePolicy: compute.CreatePolicy{Name: "cluster_policy1"}}, + }, Dashboards: map[string]*resources.Dashboard{ "dashboard1": { DashboardConfig: resources.DashboardConfig{ diff --git a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go new file mode 100644 index 00000000000..093abb1d330 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go @@ -0,0 +1,65 @@ +package resourcemutator + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const definitionFieldName = "definition" + +type configureClusterPolicyDefinition struct{} + +func ConfigureClusterPolicyDefinition() bundle.Mutator { + return &configureClusterPolicyDefinition{} +} + +func (c configureClusterPolicyDefinition) Name() string { + return "ConfigureClusterPolicyDefinition" +} + +func (c configureClusterPolicyDefinition) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("cluster_policies"), + dyn.AnyKey(), + ) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + def := v.Get(definitionFieldName) + + // Marshal an inline structured definition to a JSON string so both + // config-side and state-side carry the same plain string. Otherwise + // YAML decodes small ints as Go `int` while state JSON round-trip + // decodes them as `float64`, and structdiff reports false drift. + switch def.Kind() { + case dyn.KindInvalid, dyn.KindNil, dyn.KindString: + // KindInvalid means definition is absent; leave it for backend validation. + return v, nil + case dyn.KindMap, dyn.KindSequence: + jsonBytes, err := json.Marshal(def.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline definition: %w", err) + } + return dyn.Set(v, definitionFieldName, dyn.V(string(jsonBytes))) + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("definition must be a string, map, or sequence, got %s", def.Kind()), + Locations: def.Locations(), + }) + return v, nil + } + }) + }) + + diags = diags.Extend(diag.FromErr(err)) + return diags +} diff --git a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go new file mode 100644 index 00000000000..ad909c6a47b --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go @@ -0,0 +1,74 @@ +package resourcemutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfigureClusterPolicyDefinition(t *testing.T) { + tests := []struct { + name string + definition any + wantDefinition any + // wantErr, when non-empty, is a substring expected in the diagnostics. + wantErr string + }{ + { + // Inline maps are marshaled to a compact JSON string with sorted keys + // so config and state hold an identical string and don't drift. + name: "inline map is marshaled to a JSON string", + definition: map[string]any{"spark_version": map[string]any{"type": "fixed", "value": "13.3.x"}}, + wantDefinition: `{"spark_version":{"type":"fixed","value":"13.3.x"}}`, + }, + { + name: "inline sequence is marshaled to a JSON string", + definition: []any{"a", "b"}, + wantDefinition: `["a","b"]`, + }, + { + name: "inline string is left unchanged", + definition: `{"spark_version":{"type":"fixed"}}`, + wantDefinition: `{"spark_version":{"type":"fixed"}}`, + }, + { + name: "absent definition passes through", + wantDefinition: nil, + }, + { + name: "non-structured definition is rejected", + definition: true, + wantErr: "definition must be a string, map, or sequence, got bool", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cp := &resources.ClusterPolicy{Definition: tt.definition} + + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + ClusterPolicies: map[string]*resources.ClusterPolicy{"pol": cp}, + }, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, resourcemutator.ConfigureClusterPolicyDefinition()) + + if tt.wantErr != "" { + require.Error(t, diags.Error()) + assert.ErrorContains(t, diags.Error(), tt.wantErr) + return + } + + require.NoError(t, diags.Error()) + assert.Equal(t, tt.wantDefinition, b.Config.Resources.ClusterPolicies["pol"].Definition) + }) + } +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index e8c33f0c59b..d771c0b3f57 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -208,6 +208,10 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Updates (dynamic): resources.genie_spaces.*.serialized_space ConfigureGenieSpaceSerializedSpace(), + // Reads (dynamic): resources.cluster_policies.*.definition + // Updates (dynamic): resources.cluster_policies.*.definition (inline YAML -> JSON string) + ConfigureClusterPolicyDefinition(), + // Reads (typed): resources.alerts.*.file_path // Updates (typed): resources.alerts.* (loads alert configuration from .dbalert.json file) mutator.LoadDBAlertFiles(), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 87312608de8..be9abfe6836 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -31,41 +31,43 @@ func allResourceTypes(t *testing.T) []string { // Assert the total list of resource supported, as a sanity check that using // the dyn library gives us the correct list of all resources supported. Please // also update this check when adding a new resource - require.Equal(t, []string{ - "alerts", - "apps", - "catalogs", - "clusters", - "dashboards", - "database_catalogs", - "database_instances", - "experiments", - "external_locations", - "genie_spaces", - "instance_pools", - "job_runs", - "jobs", - "model_serving_endpoints", - "models", - "pipelines", - "postgres_branches", - "postgres_catalogs", - "postgres_databases", - "postgres_endpoints", - "postgres_projects", - "postgres_roles", - "postgres_synced_tables", - "quality_monitors", - "registered_models", - "schemas", - "secret_scopes", - "secrets", - "sql_warehouses", - "synced_database_tables", - "vector_search_endpoints", - "vector_search_indexes", - "volumes", - }, + require.Equal( + t, []string{ + "alerts", + "apps", + "catalogs", + "cluster_policies", + "clusters", + "dashboards", + "database_catalogs", + "database_instances", + "experiments", + "external_locations", + "genie_spaces", + "instance_pools", + "job_runs", + "jobs", + "model_serving_endpoints", + "models", + "pipelines", + "postgres_branches", + "postgres_catalogs", + "postgres_databases", + "postgres_endpoints", + "postgres_projects", + "postgres_roles", + "postgres_synced_tables", + "quality_monitors", + "registered_models", + "schemas", + "secret_scopes", + "secrets", + "sql_warehouses", + "synced_database_tables", + "vector_search_endpoints", + "vector_search_indexes", + "volumes", + }, resourceTypes, ) @@ -174,6 +176,7 @@ var allowList = []string{ "alerts", "catalogs", "clusters", + "cluster_policies", "dashboards", "database_catalogs", "database_instances", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index ab12ec9f052..633332a78c0 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -45,6 +45,7 @@ type Resources struct { VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` Secrets map[string]*resources.Secret `json:"secrets,omitempty"` + ClusterPolicies map[string]*resources.ClusterPolicy `json:"cluster_policies,omitempty"` } type ConfigResource interface { @@ -131,6 +132,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), collectResourceMap(descriptions["instance_pools"], r.InstancePools), collectResourceMap(descriptions["secrets"], r.Secrets), + collectResourceMap(descriptions["cluster_policies"], r.ClusterPolicies), } } @@ -195,5 +197,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), "secrets": (&resources.Secret{}).ResourceDescription(), + "cluster_policies": (&resources.ClusterPolicy{}).ResourceDescription(), } } diff --git a/bundle/config/resources/cluster_policy.go b/bundle/config/resources/cluster_policy.go new file mode 100644 index 00000000000..98263d1a676 --- /dev/null +++ b/bundle/config/resources/cluster_policy.go @@ -0,0 +1,63 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type ClusterPolicy struct { + BaseResource + compute.CreatePolicy + + // Shadows the embedded compute.CreatePolicy.Definition (a string). `any` lets the + // definition be authored as inline YAML; ConfigureClusterPolicyDefinition normalizes + // it to a JSON string before deploy. + Definition any `json:"definition,omitempty"` +} + +func (s *ClusterPolicy) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s ClusterPolicy) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s *ClusterPolicy) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.ClusterPolicies.GetByPolicyId(ctx, id) + if err != nil { + log.Debugf(ctx, "cluster policy %s does not exist", id) + return false, err + } + return true, nil +} + +func (*ClusterPolicy) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "cluster_policy", + PluralName: "cluster_policies", + SingularTitle: "Cluster Policy", + PluralTitle: "Cluster Policies", + } +} + +func (s *ClusterPolicy) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + s.URL = workspaceurls.ResourceURL(baseURL, "cluster_policies", s.ID) +} + +func (s *ClusterPolicy) GetName() string { + return s.Name +} + +func (s *ClusterPolicy) GetURL() string { + return s.URL +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index d83f4da59b8..7e56f47a64a 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -204,6 +204,9 @@ func TestResourcesBindSupport(t *testing.T) { InstancePools: map[string]*resources.InstancePool{ "my_instance_pool": {}, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "my_cluster_policy": {}, + }, Dashboards: map[string]*resources.Dashboard{ "my_dashboard": {}, }, @@ -366,6 +369,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockSchemasAPI().EXPECT().GetByFullName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockClustersAPI().EXPECT().GetByClusterId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockInstancePoolsAPI().EXPECT().GetByInstancePoolId(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockClusterPoliciesAPI().EXPECT().GetByPolicyId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockLakeviewAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockGenieAPI().EXPECT().GetSpace(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVolumesAPI().EXPECT().Read(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index f8a5140576f..eaf3040f1c7 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -16,6 +16,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { // Resources that are only supported in direct mode and should not be converted to Terraform ignoredResources := []string{ "catalogs", + "cluster_policies", "external_locations", "genie_spaces", "instance_pools", diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index ad310468da0..2c82df2aab9 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -40,6 +40,7 @@ var SupportedResources = map[string]any{ "vector_search_indexes": (*ResourceVectorSearchIndex)(nil), "instance_pools": (*ResourceInstancePool)(nil), "secrets": (*ResourceSecret)(nil), + "cluster_policies": (*ResourceClusterPolicy)(nil), // Permissions "jobs.permissions": (*ResourcePermissions)(nil), diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 5e61d803183..ec2e3c2519c 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -6,6 +6,8 @@ apps: apps.App catalogs: catalog.CreateCatalog +cluster_policies: compute.CreatePolicy + clusters: compute.ClusterSpec dashboards: dashboards.Dashboard diff --git a/bundle/direct/dresources/cluster_policy.go b/bundle/direct/dresources/cluster_policy.go new file mode 100644 index 00000000000..e422a227616 --- /dev/null +++ b/bundle/direct/dresources/cluster_policy.go @@ -0,0 +1,74 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type ResourceClusterPolicy struct { + client *databricks.WorkspaceClient +} + +func (*ResourceClusterPolicy) New(client *databricks.WorkspaceClient) *ResourceClusterPolicy { + return &ResourceClusterPolicy{client: client} +} + +func (*ResourceClusterPolicy) PrepareState(input *resources.ClusterPolicy) *compute.CreatePolicy { + cp := input.CreatePolicy + // The top-level Definition shadows the embedded string; ConfigureClusterPolicyDefinition + // has already normalized it to a JSON string by this point. + if s, ok := input.Definition.(string); ok { + cp.Definition = s + } + return &cp +} + +// RemapState copies the config fields shared by Policy and CreatePolicy; +// output-only fields (policy_id, created_at_timestamp, creator_user_name, is_default) are not in the state. +func (*ResourceClusterPolicy) RemapState(remote *compute.Policy) *compute.CreatePolicy { + return &compute.CreatePolicy{ + Definition: remote.Definition, + Description: remote.Description, + Libraries: remote.Libraries, + MaxClustersPerUser: remote.MaxClustersPerUser, + Name: remote.Name, + PolicyFamilyDefinitionOverrides: remote.PolicyFamilyDefinitionOverrides, + PolicyFamilyId: remote.PolicyFamilyId, + ForceSendFields: utils.FilterFields[compute.CreatePolicy](remote.ForceSendFields), + } +} + +func (r *ResourceClusterPolicy) DoRead(ctx context.Context, id string) (*compute.Policy, error) { + return r.client.ClusterPolicies.GetByPolicyId(ctx, id) +} + +func (r *ResourceClusterPolicy) DoCreate(ctx context.Context, config *compute.CreatePolicy) (string, *compute.Policy, error) { + resp, err := r.client.ClusterPolicies.Create(ctx, *config) + if err != nil { + return "", nil, err + } + + return resp.PolicyId, nil, nil +} + +func (r *ResourceClusterPolicy) DoUpdate(ctx context.Context, id string, config *compute.CreatePolicy, _ *PlanEntry) (*compute.Policy, error) { + return nil, r.client.ClusterPolicies.Edit(ctx, compute.EditPolicy{ + PolicyId: id, + Name: config.Name, + Definition: config.Definition, + Description: config.Description, + Libraries: config.Libraries, + MaxClustersPerUser: config.MaxClustersPerUser, + PolicyFamilyDefinitionOverrides: config.PolicyFamilyDefinitionOverrides, + PolicyFamilyId: config.PolicyFamilyId, + ForceSendFields: utils.FilterFields[compute.EditPolicy](config.ForceSendFields), + }) +} + +func (r *ResourceClusterPolicy) DoDelete(ctx context.Context, id string, _ *compute.CreatePolicy) error { + return r.client.ClusterPolicies.DeleteByPolicyId(ctx, id) +} diff --git a/bundle/direct/dresources/cluster_policy_test.go b/bundle/direct/dresources/cluster_policy_test.go new file mode 100644 index 00000000000..60377bfd0b3 --- /dev/null +++ b/bundle/direct/dresources/cluster_policy_test.go @@ -0,0 +1,47 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" +) + +func TestClusterPolicyPrepareState(t *testing.T) { + tests := []struct { + name string + definition any + want string + }{ + { + // The normal post-mutator case: definition is already a JSON string. + name: "string definition is copied into state", + definition: `{"spark_version":{"type":"fixed"}}`, + want: `{"spark_version":{"type":"fixed"}}`, + }, + { + // ConfigureClusterPolicyDefinition guarantees a string, so a non-string + // is ignored rather than reaching the API. + name: "non-string definition is ignored", + definition: map[string]any{"spark_version": "fixed"}, + want: "", + }, + { + name: "absent definition leaves state empty", + definition: nil, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input := &resources.ClusterPolicy{Definition: tt.definition} + input.Name = "my_policy" + + got := (*ResourceClusterPolicy)(nil).PrepareState(input) + + assert.Equal(t, tt.want, got.Definition) + assert.Equal(t, "my_policy", got.Name) + }) + } +} diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index d5fb0d6c98d..bf63d30dc95 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -70,6 +70,8 @@ resources: # catalogs: no api field behaviors + # cluster_policies: no api field behaviors + # clusters: no api field behaviors dashboards: diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 60bedb38d16..bd1c02857ad 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -543,6 +543,13 @@ resources: "azure_tenant_id": "description": |- PLACEHOLDER + "cluster_policies": + "description": |- + PLACEHOLDER + "$fields": + "lifecycle": + "description": |- + PLACEHOLDER "clusters": "description": |- The cluster definitions for the bundle, where each key is the name of a cluster. diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 9268be0985f..97fa418097c 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -40,6 +40,10 @@ var RequiredFields = map[string][]string{ "resources.catalogs.*": {"name"}, "resources.catalogs.*.managed_encryption_settings.azure_encryption_settings": {"azure_tenant_id"}, + "resources.cluster_policies.*.libraries[*].cran": {"package"}, + "resources.cluster_policies.*.libraries[*].maven": {"coordinates"}, + "resources.cluster_policies.*.libraries[*].pypi": {"package"}, + "resources.clusters.*.cluster_log_conf.dbfs": {"destination"}, "resources.clusters.*.cluster_log_conf.s3": {"destination"}, "resources.clusters.*.cluster_log_conf.volumes": {"destination"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index b4084979746..85de80de072 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -589,6 +589,51 @@ } ] }, + "resources.ClusterPolicy": { + "oneOf": [ + { + "type": "object", + "properties": { + "definition": { + "description": "Policy definition document expressed in [Databricks Cluster Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).", + "$ref": "#/$defs/interface" + }, + "description": { + "description": "Additional human-readable description of the cluster policy.", + "$ref": "#/$defs/string" + }, + "libraries": { + "description": "A list of libraries to be installed on the next cluster restart that uses this policy. The maximum number of libraries is 500.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.Library" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "max_clusters_per_user": { + "description": "Max number of clusters per user that can be active using this policy. If not present, there is no max limit.", + "$ref": "#/$defs/int64" + }, + "name": { + "description": "Cluster Policy name requested by the user. This has to be unique. Length must be between 1 and 100\ncharacters.", + "$ref": "#/$defs/string" + }, + "policy_family_definition_overrides": { + "description": "Policy definition JSON document expressed in [Databricks Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).\nThe JSON document must be passed as a string and cannot be embedded in the requests.\n\nYou can use this to customize the policy definition inherited from the policy family.\nPolicy rules specified here are merged into the inherited policy definition.", + "$ref": "#/$defs/string" + }, + "policy_family_id": { + "description": "ID of the policy family. The cluster policy's policy definition inherits the policy\nfamily's policy definition.\n\nCannot be used with `definition`. Use `policy_family_definition_overrides` instead to\ncustomize the policy definition.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Dashboard": { "oneOf": [ { @@ -3356,6 +3401,9 @@ "catalogs": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Catalog" }, + "cluster_policies": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ClusterPolicy" + }, "clusters": { "description": "The cluster definitions for the bundle, where each key is the name of a cluster.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster", @@ -14573,6 +14621,20 @@ } ] }, + "resources.ClusterPolicy": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ClusterPolicy" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Dashboard": { "oneOf": [ { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index b706b0770cf..ab39a4917b0 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -59,6 +59,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, "resources.instance_pools.test_instance_pool": {ID: "1"}, "resources.secrets.test_secret": {ID: "main.default.test_secret"}, + "resources.cluster_policies.test_cluster_policy": {ID: "cp-1"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -154,6 +155,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.InstancePools["test_instance_pool"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + assert.Equal(t, "cp-1", config.Resources.ClusterPolicies["test_cluster_policy"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.ClusterPolicies["test_cluster_policy"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Secrets["test_secret"].ModifiedStatus) @@ -402,6 +406,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "test_cluster_policy": { + CreatePolicy: compute.CreatePolicy{ + Name: "test_cluster_policy", + }, + }, + }, }, } @@ -507,6 +518,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.InstancePools["test_instance_pool"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + assert.Empty(t, config.Resources.ClusterPolicies["test_cluster_policy"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ClusterPolicies["test_cluster_policy"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -914,6 +928,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "test_cluster_policy": { + CreatePolicy: compute.CreatePolicy{ + Name: "test_cluster_policy", + }, + }, + "test_cluster_policy_new": { + CreatePolicy: compute.CreatePolicy{ + Name: "test_cluster_policy_new", + }, + }, + }, }, } state := ExportedResourcesMap{ @@ -973,6 +999,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.vector_search_indexes.test_vector_search_index_old": {ID: "vs-index-old"}, "resources.instance_pools.test_instance_pool": {ID: "1"}, "resources.instance_pools.test_instance_pool_old": {ID: "2"}, + "resources.cluster_policies.test_cluster_policy": {ID: "cp-1"}, + "resources.cluster_policies.test_cluster_policy_old": {ID: "cp-2"}, "resources.secrets.test_secret": {ID: "main.default.test_secret"}, "resources.secrets.test_secret_old": {ID: "main.default.test_secret_old"}, } @@ -1177,6 +1205,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.InstancePools["test_instance_pool_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.InstancePools["test_instance_pool_new"].ModifiedStatus) + assert.Equal(t, "cp-1", config.Resources.ClusterPolicies["test_cluster_policy"].ID) + assert.Empty(t, config.Resources.ClusterPolicies["test_cluster_policy"].ModifiedStatus) + assert.Equal(t, "cp-2", config.Resources.ClusterPolicies["test_cluster_policy_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.ClusterPolicies["test_cluster_policy_old"].ModifiedStatus) + assert.Empty(t, config.Resources.ClusterPolicies["test_cluster_policy_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ClusterPolicies["test_cluster_policy_new"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) assert.Empty(t, config.Resources.Secrets["test_secret"].ModifiedStatus) assert.Equal(t, "main.default.test_secret_old", config.Resources.Secrets["test_secret_old"].ID) diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index 71904677781..95502694472 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -110,6 +110,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "alerts", "apps", "catalogs", + "cluster_policies", "clusters", "dashboards", "database_catalogs", @@ -148,7 +149,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go new file mode 100644 index 00000000000..5616a32a00f --- /dev/null +++ b/libs/testserver/cluster_policies.go @@ -0,0 +1,98 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "slices" + + "github.com/databricks/databricks-sdk-go/service/compute" +) + +func (s *FakeWorkspace) ClusterPoliciesCreate(req Request) any { + // Unmarshal into the stored (GET) type directly: CreatePolicy and Policy + // share JSON field names, so every config field is carried over. + var policy compute.Policy + if err := json.Unmarshal(req.Body, &policy); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + id := nextUUID() + policy.PolicyId = id + s.ClusterPolicies[id] = policy + + return Response{Body: compute.CreatePolicyResponse{PolicyId: id}} +} + +func (s *FakeWorkspace) ClusterPoliciesList(req Request) any { + defer s.LockUnlock()() + + ids := make([]string, 0, len(s.ClusterPolicies)) + for id := range s.ClusterPolicies { + ids = append(ids, id) + } + slices.Sort(ids) + + policies := make([]compute.Policy, 0, len(ids)) + for _, id := range ids { + policies = append(policies, s.ClusterPolicies[id]) + } + + return Response{Body: compute.ListPoliciesResponse{Policies: policies}} +} + +func (s *FakeWorkspace) ClusterPoliciesGet(req Request, policyId string) any { + defer s.LockUnlock()() + + policy, ok := s.ClusterPolicies[policyId] + if !ok { + return Response{StatusCode: 404} + } + + return Response{Body: policy} +} + +func (s *FakeWorkspace) ClusterPoliciesEdit(req Request) any { + var request compute.EditPolicy + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + policy, ok := s.ClusterPolicies[request.PolicyId] + if !ok { + return Response{StatusCode: 404} + } + + // Edit is a full replace of the writable fields; server-set fields + // (policy_id, created_at_timestamp, creator_user_name, is_default) are kept as stored. + policy.Name = request.Name + policy.Definition = request.Definition + policy.Description = request.Description + policy.Libraries = request.Libraries + policy.MaxClustersPerUser = request.MaxClustersPerUser + policy.PolicyFamilyDefinitionOverrides = request.PolicyFamilyDefinitionOverrides + policy.PolicyFamilyId = request.PolicyFamilyId + s.ClusterPolicies[request.PolicyId] = policy + + return Response{} +} + +func (s *FakeWorkspace) ClusterPoliciesDelete(req Request) any { + var request compute.DeletePolicy + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + if _, ok := s.ClusterPolicies[request.PolicyId]; !ok { + return Response{StatusCode: 404} + } + + delete(s.ClusterPolicies, request.PolicyId) + + return Response{} +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 445417b3ff1..1982e13658a 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -196,6 +196,7 @@ type FakeWorkspace struct { ModelRegistryModelIDs map[string]string // model name -> numeric ID Clusters map[string]compute.ClusterDetails InstancePools map[string]compute.GetInstancePool + ClusterPolicies map[string]compute.Policy Catalogs map[string]catalog.CatalogInfo ExternalLocations map[string]catalog.ExternalLocationInfo RegisteredModels map[string]catalog.RegisteredModelInfo @@ -428,7 +429,13 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { SingleUserName: TestUser.UserName, }, }, - InstancePools: map[string]compute.GetInstancePool{}, + InstancePools: map[string]compute.GetInstancePool{}, + ClusterPolicies: map[string]compute.Policy{ + // Seeded so the stateful list keeps backing the variable-lookup tests + // (e.g. acceptance/bundle/variables/env_overrides resolves these by name). + "5678": {PolicyId: "5678", Name: "wrong-cluster-policy"}, + "9876": {PolicyId: "9876", Name: "some-test-cluster-policy"}, + }, VectorSearchIndexesPendingDeletion: map[string]int{}, } } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 082b771d1ea..05ac9df9370 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -27,18 +27,7 @@ var TestMetastore = catalog.MetastoreAssignment{ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/policies/clusters/list", func(req Request) any { - return compute.ListPoliciesResponse{ - Policies: []compute.Policy{ - { - PolicyId: "5678", - Name: "wrong-cluster-policy", - }, - { - PolicyId: "9876", - Name: "some-test-cluster-policy", - }, - }, - } + return req.Workspace.ClusterPoliciesList(req) }) server.Handle("GET", "/api/2.0/instance-pools/list", func(req Request) any { @@ -62,6 +51,13 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.InstancePoolsGet(req, req.URL.Query().Get("instance_pool_id")) }) + server.Handle("POST", "/api/2.0/policies/clusters/create", func(req Request) any { return req.Workspace.ClusterPoliciesCreate(req) }) + server.Handle("POST", "/api/2.0/policies/clusters/edit", func(req Request) any { return req.Workspace.ClusterPoliciesEdit(req) }) + server.Handle("POST", "/api/2.0/policies/clusters/delete", func(req Request) any { return req.Workspace.ClusterPoliciesDelete(req) }) + server.Handle("GET", "/api/2.0/policies/clusters/get", func(req Request) any { + return req.Workspace.ClusterPoliciesGet(req, req.URL.Query().Get("policy_id")) + }) + server.Handle("GET", "/api/2.1/clusters/list", func(req Request) any { return compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index 4839c1e4273..61c3f271468 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -11,6 +11,7 @@ var resourceURLPatterns = map[string]string{ "alerts": "sql/alerts-v2/%s", "apps": "apps/%s", "catalogs": "explore/data/%s", + "cluster_policies": "compute/policies/%s", "clusters": "compute/clusters/%s", "dashboards": "dashboardsv3/%s/published", "database_catalogs": "explore/data/%s",