diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/.gitignore b/acceptance/bundle/deploy/immutable-no-artifacts/.gitignore new file mode 100644 index 00000000000..78caa68e38e --- /dev/null +++ b/acceptance/bundle/deploy/immutable-no-artifacts/.gitignore @@ -0,0 +1,2 @@ +*.txt +*.json diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml +++ b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index 3fc999917ab..3eaf0316527 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -8,21 +8,114 @@ Workspace: Validation OK! +>>> [CLI] bundle plan -o json + +>>> jq .plan["resources.jobs.my_job"].new_state.value.tasks plan.json +[ + { + "notebook_task": { + "base_parameters": { + "path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/some_path" + }, + "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook" + }, + "task_key": "notebook_task" + }, + { + "environment_key": "env", + "spark_python_task": { + "python_file": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py" + }, + "task_key": "spark_python_task" + } +] + +>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json +{ + "action": "create", + "new_state": { + "value": { + "remote_root": "/Workspace/Users/[UUID]/.snapshots", + "relative_path": "[UUID]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "bundle_id": "[UUID]", + "acl": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ] + } + } +} + >>> [CLI] bundle deploy -Uploading immutable bundle snapshot... Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] jobs get [NUMID] +>>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py" ->>> [CLI] jobs get [NUMID] +>>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook" ->>> [CLI] jobs get [NUMID] +>>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/some_path" +>>> title Check snapshot plan after deploy: + +=== Check snapshot plan after deploy: +>>> [CLI] bundle plan -o json + +>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json +{ + "action": "skip", + "remote_state": { + "relative_path": "[UUID]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" + }, + "changes": { + "acl": { + "action": "skip", + "reason": "missing_in_remote", + "old": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "new": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "remote": null + }, + "bundle_id": { + "action": "skip", + "reason": "missing_in_remote", + "old": "[UUID]", + "new": "[UUID]", + "remote": "" + }, + "remote_root": { + "action": "skip", + "reason": "missing_in_remote", + "old": "/Workspace/Users/[UUID]/.snapshots", + "new": "/Workspace/Users/[UUID]/.snapshots", + "remote": "" + } + } +} + +>>> [CLI] bundle deploy +Deploying resources... +Updating deployment state... +Deployment complete! +Snapshot did not change as expected + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.my_job diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/script b/acceptance/bundle/deploy/immutable-no-artifacts/script index a7aad718750..6f8f5f97bd6 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/script +++ b/acceptance/bundle/deploy/immutable-no-artifacts/script @@ -6,13 +6,33 @@ cleanup() { trap cleanup EXIT trace $CLI bundle validate -trace $CLI bundle deploy +trace $CLI bundle plan -o json > plan.json +trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json +trace jq '.plan["resources.internal_immutable_snapshots.immutable"]' plan.json +trace $CLI bundle deploy $(readplanarg plan.json) # Get a job and check that task paths point into the snapshot JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id') -trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file' -trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path' -trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.base_parameters.path' +$CLI jobs get $JOB_ID > job.json +trace jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file' +trace jq '.settings.tasks' job.json | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path' +trace jq '.settings.tasks' job.json | jq '.[] | select(.notebook_task != null) | .notebook_task.base_parameters.path' +PYTHON_FILE_1=$(jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') +trace title "Check snapshot plan after deploy: " +trace $CLI bundle plan -o json > plan.json +trace jq '.plan["resources.internal_immutable_snapshots.immutable"]' plan.json + +trace $CLI bundle deploy $(readplanarg plan.json) +rm plan.json +rm job.json + +PYTHON_FILE_2=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') +if [ "$PYTHON_FILE_1" == "$PYTHON_FILE_2" ]; then + echo "Snapshot did not change as expected" +else + echo "Snapshot changed" + exit 1 +fi trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/test.toml b/acceptance/bundle/deploy/immutable-no-artifacts/test.toml index d1aa5d2ddce..202e70986d8 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/test.toml +++ b/acceptance/bundle/deploy/immutable-no-artifacts/test.toml @@ -1,5 +1,6 @@ Cloud = false # Temporary disable cloud tests until the API is fully available RecordRequests = true +EnvMatrix.READPLAN = ["", "1"] # immutable_folder only works with the direct engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] @@ -17,3 +18,9 @@ Ignore = [ [[Repls]] Old = '[0-9a-f]{64}' New = '[SNAPSHOT_HASH]' + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan plan.json' +New = '' diff --git a/acceptance/bundle/deploy/immutable-permissions-change/output.txt b/acceptance/bundle/deploy/immutable-permissions-change/output.txt index 68b79efc831..3b4f5e73600 100644 --- a/acceptance/bundle/deploy/immutable-permissions-change/output.txt +++ b/acceptance/bundle/deploy/immutable-permissions-change/output.txt @@ -1,7 +1,6 @@ === Deploy without permissions >>> [CLI] bundle deploy -Uploading immutable bundle snapshot... Deploying resources... Updating deployment state... Deployment complete! @@ -21,7 +20,6 @@ Consider using a adding a top-level permissions section such as the following: See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. in databricks.yml:22:3 -Uploading immutable bundle snapshot... Deploying resources... Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index dcc78a79103..bc97046230b 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -10,10 +10,12 @@ Validation OK! >>> [CLI] bundle plan -o json Building python_artifact... + +>>> jq .plan["resources.jobs.my_job"].new_state.value.tasks plan.json [ { "notebook_task": { - "notebook_path": "${workspace.snapshot_path}/files/src/notebook" + "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook" }, "task_key": "notebook_task" }, @@ -28,30 +30,145 @@ Building python_artifact... { "environment_key": "env", "spark_python_task": { - "python_file": "${workspace.snapshot_path}/files/src/main.py" + "python_file": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py" }, "task_key": "spark_python_task" } ] +>>> jq .plan["resources.jobs.my_job"].new_state.value.environments plan.json +[ + { + "environment_key": "env", + "spec": { + "dependencies": [ + "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" + ], + "environment_version": "4" + } + } +] + +>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json +{ + "action": "create", + "new_state": { + "value": { + "remote_root": "/Workspace/Users/[UUID]/.snapshots", + "relative_path": "[UUID]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "bundle_id": "[UUID]", + "acl": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ] + } + } +} + >>> [CLI] bundle deploy Building python_artifact... -Uploading immutable bundle snapshot... Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] jobs get [NUMID] +>>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py" ->>> [CLI] jobs get [NUMID] +>>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook" ->>> [CLI] jobs get [NUMID] +>>> jq .settings.environments[0].spec.dependencies job.json [ "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" ] +>>> title Check snapshot plan after deploy + +=== Check snapshot plan after deploy +>>> [CLI] bundle plan -o json +Building python_artifact... + +>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json +{ + "action": "recreate", + "new_state": { + "value": { + "remote_root": "/Workspace/Users/[UUID]/.snapshots", + "relative_path": "[UUID]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "bundle_id": "[UUID]", + "acl": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ] + } + }, + "remote_state": { + "relative_path": "[UUID]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" + }, + "changes": { + "acl": { + "action": "skip", + "reason": "missing_in_remote", + "old": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "new": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "remote": null + }, + "bundle_id": { + "action": "skip", + "reason": "missing_in_remote", + "old": "[UUID]", + "new": "[UUID]", + "remote": "" + }, + "full_path": { + "action": "recreate", + "reason": "immutable", + "old": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "new": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "remote": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" + }, + "relative_path": { + "action": "recreate", + "reason": "immutable", + "old": "[UUID]/[SNAPSHOT_HASH]", + "new": "[UUID]/[SNAPSHOT_HASH]", + "remote": "[UUID]/[SNAPSHOT_HASH]" + }, + "remote_root": { + "action": "skip", + "reason": "missing_in_remote", + "old": "/Workspace/Users/[UUID]/.snapshots", + "new": "/Workspace/Users/[UUID]/.snapshots", + "remote": "" + } + } +} + +>>> [CLI] bundle deploy +Building python_artifact... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Check that job uses a new snapshot: Snapshot changed as expected + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.my_job diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/script index 85e9a908d1d..5babe793a9a 100644 --- a/acceptance/bundle/deploy/immutable/script +++ b/acceptance/bundle/deploy/immutable/script @@ -5,15 +5,37 @@ cleanup() { trap cleanup EXIT trace $CLI bundle validate -trace $CLI bundle plan -o json | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' -trace $CLI bundle deploy +trace $CLI bundle plan -o json > plan.json +trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json +trace jq '.plan["resources.jobs.my_job"].new_state.value.environments' plan.json +trace jq '.plan["resources.internal_immutable_snapshots.immutable"]' plan.json +trace $CLI bundle deploy # Get a job and check that task paths are immutable JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id') -trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file' -trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path' -trace $CLI jobs get $JOB_ID | jq '.settings.environments[0].spec.dependencies' +$CLI jobs get $JOB_ID > job.json +trace jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file' +trace jq '.settings.tasks' job.json | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path' +trace jq '.settings.environments[0].spec.dependencies' job.json +PYTHON_FILE_1=$(jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') + +trace title "Check snapshot plan after deploy" +trace $CLI bundle plan -o json > plan.json +trace jq '.plan["resources.internal_immutable_snapshots.immutable"]' plan.json +rm plan.json +rm job.json + +trace $CLI bundle deploy + +title "Check that job uses a new snapshot: " +PYTHON_FILE_2=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') +if [ "$PYTHON_FILE_1" != "$PYTHON_FILE_2" ]; then + echo "Snapshot changed as expected" +else + echo "Snapshot did not change" + exit 1 +fi # Redirect run output to a log file — the real workspace produces different output than the local test server. $CLI bundle run my_job &> LOG.run diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 6c2b033fa97..068ac635da3 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -836,6 +836,17 @@ resources.instance_pools.*.permissions[*].group_name string ALL resources.instance_pools.*.permissions[*].level iam.PermissionLevel ALL resources.instance_pools.*.permissions[*].service_principal_name string ALL resources.instance_pools.*.permissions[*].user_name string ALL +resources.internal_immutable_snapshots.*.acl []snapshot.ACLEntry INPUT STATE +resources.internal_immutable_snapshots.*.acl[*] snapshot.ACLEntry INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].group_name string INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].permission_level string INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].service_principal_name string INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].user_name string INPUT STATE +resources.internal_immutable_snapshots.*.bundle_id string INPUT STATE +resources.internal_immutable_snapshots.*.full_path string REMOTE STATE +resources.internal_immutable_snapshots.*.relative_path string REMOTE STATE +resources.internal_immutable_snapshots.*.remote_root string INPUT STATE +resources.internal_immutable_snapshots.*.zip_content string INPUT resources.job_runs.*.dbt_commands []string ALL resources.job_runs.*.dbt_commands[*] string ALL resources.job_runs.*.id string INPUT diff --git a/acceptance/bundle/resources/apps/immutable/output.txt b/acceptance/bundle/resources/apps/immutable/output.txt index e7ae9f1ba23..d9df97347e0 100644 --- a/acceptance/bundle/resources/apps/immutable/output.txt +++ b/acceptance/bundle/resources/apps/immutable/output.txt @@ -1,6 +1,5 @@ >>> [CLI] bundle deploy -Uploading immutable bundle snapshot... Deploying resources... Updating deployment state... Deployment complete! @@ -37,6 +36,6 @@ You can access the app at my-immutable-app-123.cloud.databricksapps.com "path": "/api/2.0/apps/my-immutable-app/deployments", "body": { "mode": "SNAPSHOT", - "source_code_path": "${workspace.snapshot_path}/files/app" + "source_code_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files/app" } } diff --git a/acceptance/bundle/validate/immutable_workspace_paths/output.txt b/acceptance/bundle/validate/immutable_workspace_paths/output.txt index fef8083634f..a2f776d66d4 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/output.txt +++ b/acceptance/bundle/validate/immutable_workspace_paths/output.txt @@ -6,14 +6,14 @@ Warning: Pattern user_repls.json does not match any files { "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/artifacts", + "artifact_path": "${resources.internal_immutable_snapshots.immutable.full_path}/artifacts", "current_user": { "domain_friendly_name": "[USERNAME]", "id": "[USERID]", "short_name": "[USERNAME]", "userName": "[USERNAME]" }, - "file_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/files", + "file_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files", "resource_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/resources", "root_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default", "state_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/state" @@ -23,7 +23,7 @@ Warning: Pattern user_repls.json does not match any files "ai_runtime_task": { "deployments": [ { - "command_path": "${workspace.snapshot_path}/files/src/main.py", + "command_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files/src/main.py", "compute": { "accelerator_count": 1, "accelerator_type": "GPU_1xA10" @@ -37,7 +37,7 @@ Warning: Pattern user_repls.json does not match any files { "existing_cluster_id": "0101-120000-aaaaaaaa", "spark_python_task": { - "python_file": "${workspace.snapshot_path}/files/src/main.py" + "python_file": "${resources.internal_immutable_snapshots.immutable.full_path}/files/src/main.py" }, "task_key": "my_task" } diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml b/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml new file mode 100644 index 00000000000..a715ab3b03e --- /dev/null +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: snapshot_resource_not_allowed + +resources: + internal_immutable_snapshots: + immutable: + bundle_id: 123 + zip_content: |- + This is a test zip content. diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/out.test.toml b/acceptance/bundle/validate/snapshot_resource_not_allowed/out.test.toml new file mode 100644 index 00000000000..98ea5040486 --- /dev/null +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt b/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt new file mode 100644 index 00000000000..b390e82ce64 --- /dev/null +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt @@ -0,0 +1,11 @@ + +>>> [CLI] bundle validate +Error: Internal resources cannot be set in bundle configuration + at resources + +Name: snapshot_resource_not_allowed +Target: default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/script b/acceptance/bundle/validate/snapshot_resource_not_allowed/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/acceptance/invariant_test.go b/acceptance/invariant_test.go index 1c8204055ed..ad8e1d14022 100644 --- a/acceptance/invariant_test.go +++ b/acceptance/invariant_test.go @@ -22,7 +22,8 @@ const invariantConfigsDir = "bundle/invariant/configs" // for permissions/grants coverage. Add a config and remove the entry to close a gap; // the test fails if an entry here is actually covered, so the list only shrinks. var LackingInvariantTest = map[string]bool{ - "quality_monitors": true, + "quality_monitors": true, + "internal_immutable_snapshots": true, } // TestInvariantConfigsCoverage ensures that the invariant test configs in diff --git a/bundle/config/mutator/default_workspace_paths.go b/bundle/config/mutator/default_workspace_paths.go index 02a1ddb3b11..33e1531fdd6 100644 --- a/bundle/config/mutator/default_workspace_paths.go +++ b/bundle/config/mutator/default_workspace_paths.go @@ -26,7 +26,11 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl } if b.Config.Workspace.FilePath == "" { - b.Config.Workspace.FilePath = path.Join(root, "files") + if b.IsImmutableFolder() { + b.Config.Workspace.FilePath = "${resources.internal_immutable_snapshots.immutable.full_path}/files" + } else { + b.Config.Workspace.FilePath = path.Join(root, "files") + } } if b.Config.Workspace.ResourcePath == "" { @@ -34,7 +38,11 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl } if b.Config.Workspace.ArtifactPath == "" { - b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts") + if b.IsImmutableFolder() { + b.Config.Workspace.ArtifactPath = "${resources.internal_immutable_snapshots.immutable.full_path}/artifacts" + } else { + b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts") + } } if b.Config.Workspace.StatePath == "" { diff --git a/bundle/config/mutator/reject_internal_resources.go b/bundle/config/mutator/reject_internal_resources.go new file mode 100644 index 00000000000..c377ecb0aaa --- /dev/null +++ b/bundle/config/mutator/reject_internal_resources.go @@ -0,0 +1,34 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type rejectInternalResources struct{} + +// RejectInternalResources returns a mutator that errors when a user has set +// any internal resource field in their bundle configuration. +func RejectInternalResources() bundle.Mutator { + return &rejectInternalResources{} +} + +func (m *rejectInternalResources) Name() string { + return "RejectInternalResources" +} + +func (m *rejectInternalResources) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + if b.Config.Resources.HasInternalResources() { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "Internal resources cannot be set in bundle configuration", + Paths: []dyn.Path{dyn.MustPathFromString("resources")}, + }) + } + + return diags +} diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index 9a03c85d362..8e98bc52092 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -80,24 +80,6 @@ func ResolveVariableReferencesOnlyResources(prefixes ...string) bundle.Mutator { } } -// ResolveVariableReferencesOnlyResourcesExcluding is like ResolveVariableReferencesOnlyResources -// but leaves the listed variable reference paths unresolved. Use this when a workspace path will -// be updated by a later mutator (e.g. snapshot.Upload sets workspace.file_path to the snapshot -// location) and the final value should be substituted at that later point. -func ResolveVariableReferencesOnlyResourcesExcluding(excluded []string, prefixes ...string) bundle.Mutator { - if len(prefixes) == 0 { - prefixes = defaultPrefixes - } - return &resolveVariableReferences{ - prefixes: prefixes, - lookupFn: lookup, - extraRounds: maxResolutionRounds - 1, - pattern: dyn.NewPattern(dyn.Key("resources")), - includeResources: true, - excludePaths: excluded, - } -} - func ResolveVariableReferencesWithoutResources(prefixes ...string) bundle.Mutator { if len(prefixes) == 0 { prefixes = defaultPrefixes diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 46262ba8dbf..29940dd3821 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -35,6 +35,7 @@ var unsupportedResources = []string{ "postgres_synced_tables", "vector_search_indexes", "job_runs", + "internal_immutable_snapshots", "secrets", } diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 53fdf89f50e..3a6eaae4b70 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -320,6 +320,12 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + Snapshots: map[string]*resources.Snapshot{ + "snapshot1": { + BundleID: "bundle1", + ACL: nil, + }, + }, }, }, SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"), diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index 28a7cbefe78..b1f689808f6 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -45,7 +45,7 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes - immutableExcludingResolver(b), + mutator.ResolveVariableReferencesOnlyResources(), mutator.NormalizePaths(), // Translate dashboard paths into paths in the workspace file system diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index e8c33f0c59b..32243687074 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -133,22 +133,6 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { ) } -// immutableExcludingResolver returns a variable reference resolver for the resources -// section. When experimental.immutable_folder is enabled it excludes -// workspace.file_path, workspace.artifact_path, and workspace.snapshot_path from -// resolution: those paths are set by snapshot.Upload() in the Deploy phase, so -// resolving them here would freeze them to the default bundle path instead. -// workspace.snapshot_path is also excluded so it stays as a literal ${...} template -// in the plan output (making the pre-upload intent visible). -func immutableExcludingResolver(b *bundle.Bundle) bundle.Mutator { - if b.IsImmutableFolder() { - return mutator.ResolveVariableReferencesOnlyResourcesExcluding( - []string{"workspace.file_path", "workspace.artifact_path", "workspace.snapshot_path"}, - ) - } - return mutator.ResolveVariableReferencesOnlyResources() -} - // Normalization is applied multiple times if resource is modified during initialization // // If bundle is modified outside of 'resources' section, these changes are discarded. @@ -162,7 +146,7 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes - immutableExcludingResolver(b), + mutator.ResolveVariableReferencesOnlyResources(), // Reads (dynamic): resources.pipelines.*.libraries (checks for notebook.path and file.path fields) // Updates (dynamic): resources.pipelines.*.libraries (expands glob patterns in path fields to multiple library entries) diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 87312608de8..47cfba799a9 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -43,6 +43,7 @@ func allResourceTypes(t *testing.T) []string { "external_locations", "genie_spaces", "instance_pools", + "internal_immutable_snapshots", "job_runs", "jobs", "model_serving_endpoints", @@ -194,6 +195,7 @@ var allowList = []string{ "genie_spaces", "instance_pools", "job_runs", + "internal_immutable_snapshots", "schemas", "secret_scopes", "secrets", diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index c44e91160b9..18fbafc43cc 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -322,21 +322,7 @@ func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.V func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { switch { case b.IsImmutableFolder(): - // Reject an explicit workspace.file_path: immutable bundles control that path - // automatically (it is set to the content-addressed snapshot location after upload). - // A user-supplied value would be silently discarded, so we error early instead. - if loc := b.Config.GetLocation("workspace.file_path"); loc.File != "" { - return diag.Diagnostics{{ - Severity: diag.Error, - Summary: "workspace.file_path cannot be configured when experimental.immutable_folder is true", - Locations: []dyn.Location{loc}, - }} - } - // Use a placeholder referencing workspace.snapshot_path so that paths are stored - // as ${workspace.snapshot_path}/files/ during validate. After - // snapshot.Upload() sets workspace.snapshot_path, a variable-resolution pass - // expands these references to the actual content-addressed paths. - t.remoteRoot = "${workspace.snapshot_path}/files" + t.remoteRoot = "${resources.internal_immutable_snapshots.immutable.full_path}/files" case config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment): t.remoteRoot = t.b.SyncRootPath default: diff --git a/bundle/config/resources.go b/bundle/config/resources.go index ab12ec9f052..891a72333ed 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -45,6 +45,9 @@ type Resources struct { VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` Secrets map[string]*resources.Secret `json:"secrets,omitempty"` + + // Internal resources + Snapshots map[string]*resources.Snapshot `json:"internal_immutable_snapshots,omitempty" bundle:"internal"` } type ConfigResource interface { @@ -130,6 +133,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), collectResourceMap(descriptions["instance_pools"], r.InstancePools), + collectResourceMap(descriptions["internal_immutable_snapshots"], r.Snapshots), collectResourceMap(descriptions["secrets"], r.Secrets), } } @@ -159,41 +163,46 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) return found[0], nil } +func (r *Resources) HasInternalResources() bool { + return len(r.Snapshots) > 0 +} + // SupportedResources returns a map which keys correspond to the resource key in the bundle configuration. func SupportedResources() map[string]resources.ResourceDescription { return map[string]resources.ResourceDescription{ - "jobs": (&resources.Job{}).ResourceDescription(), - "job_runs": (&resources.JobRun{}).ResourceDescription(), - "pipelines": (&resources.Pipeline{}).ResourceDescription(), - "models": (&resources.MlflowModel{}).ResourceDescription(), - "experiments": (&resources.MlflowExperiment{}).ResourceDescription(), - "instance_pools": (&resources.InstancePool{}).ResourceDescription(), - "model_serving_endpoints": (&resources.ModelServingEndpoint{}).ResourceDescription(), - "registered_models": (&resources.RegisteredModel{}).ResourceDescription(), - "quality_monitors": (&resources.QualityMonitor{}).ResourceDescription(), - "catalogs": (&resources.Catalog{}).ResourceDescription(), - "schemas": (&resources.Schema{}).ResourceDescription(), - "external_locations": (&resources.ExternalLocation{}).ResourceDescription(), - "clusters": (&resources.Cluster{}).ResourceDescription(), - "dashboards": (&resources.Dashboard{}).ResourceDescription(), - "genie_spaces": (&resources.GenieSpace{}).ResourceDescription(), - "volumes": (&resources.Volume{}).ResourceDescription(), - "apps": (&resources.App{}).ResourceDescription(), - "secret_scopes": (&resources.SecretScope{}).ResourceDescription(), - "alerts": (&resources.Alert{}).ResourceDescription(), - "sql_warehouses": (&resources.SqlWarehouse{}).ResourceDescription(), - "database_instances": (&resources.DatabaseInstance{}).ResourceDescription(), - "database_catalogs": (&resources.DatabaseCatalog{}).ResourceDescription(), - "synced_database_tables": (&resources.SyncedDatabaseTable{}).ResourceDescription(), - "postgres_projects": (&resources.PostgresProject{}).ResourceDescription(), - "postgres_branches": (&resources.PostgresBranch{}).ResourceDescription(), - "postgres_endpoints": (&resources.PostgresEndpoint{}).ResourceDescription(), - "postgres_catalogs": (&resources.PostgresCatalog{}).ResourceDescription(), - "postgres_databases": (&resources.PostgresDatabase{}).ResourceDescription(), - "postgres_roles": (&resources.PostgresRole{}).ResourceDescription(), - "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), - "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), - "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), - "secrets": (&resources.Secret{}).ResourceDescription(), + "jobs": (&resources.Job{}).ResourceDescription(), + "job_runs": (&resources.JobRun{}).ResourceDescription(), + "pipelines": (&resources.Pipeline{}).ResourceDescription(), + "models": (&resources.MlflowModel{}).ResourceDescription(), + "experiments": (&resources.MlflowExperiment{}).ResourceDescription(), + "instance_pools": (&resources.InstancePool{}).ResourceDescription(), + "model_serving_endpoints": (&resources.ModelServingEndpoint{}).ResourceDescription(), + "registered_models": (&resources.RegisteredModel{}).ResourceDescription(), + "quality_monitors": (&resources.QualityMonitor{}).ResourceDescription(), + "catalogs": (&resources.Catalog{}).ResourceDescription(), + "schemas": (&resources.Schema{}).ResourceDescription(), + "external_locations": (&resources.ExternalLocation{}).ResourceDescription(), + "clusters": (&resources.Cluster{}).ResourceDescription(), + "dashboards": (&resources.Dashboard{}).ResourceDescription(), + "genie_spaces": (&resources.GenieSpace{}).ResourceDescription(), + "volumes": (&resources.Volume{}).ResourceDescription(), + "apps": (&resources.App{}).ResourceDescription(), + "secret_scopes": (&resources.SecretScope{}).ResourceDescription(), + "alerts": (&resources.Alert{}).ResourceDescription(), + "sql_warehouses": (&resources.SqlWarehouse{}).ResourceDescription(), + "database_instances": (&resources.DatabaseInstance{}).ResourceDescription(), + "database_catalogs": (&resources.DatabaseCatalog{}).ResourceDescription(), + "synced_database_tables": (&resources.SyncedDatabaseTable{}).ResourceDescription(), + "postgres_projects": (&resources.PostgresProject{}).ResourceDescription(), + "postgres_branches": (&resources.PostgresBranch{}).ResourceDescription(), + "postgres_endpoints": (&resources.PostgresEndpoint{}).ResourceDescription(), + "postgres_catalogs": (&resources.PostgresCatalog{}).ResourceDescription(), + "postgres_databases": (&resources.PostgresDatabase{}).ResourceDescription(), + "postgres_roles": (&resources.PostgresRole{}).ResourceDescription(), + "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), + "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), + "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), + "internal_immutable_snapshots": (&resources.Snapshot{}).ResourceDescription(), + "secrets": (&resources.Secret{}).ResourceDescription(), } } diff --git a/bundle/config/resources/snapshot.go b/bundle/config/resources/snapshot.go new file mode 100644 index 00000000000..5147609bc8d --- /dev/null +++ b/bundle/config/resources/snapshot.go @@ -0,0 +1,78 @@ +package resources + +import ( + "context" + "net/url" + "path" + + "github.com/databricks/cli/libs/snapshot" + "github.com/databricks/databricks-sdk-go" +) + +// Snapshot is an internal resource that stores the bundle zip as an immutable +// workspace object. It is created by the deploy pipeline and is not intended +// to be declared in user-authored databricks.yml files. +// +// JSON tags are present because the direct-deploy engine serialises the in-memory +// state to a JSON plan file (resources.internal_immutable_snapshots.*). Fields +// that must not leak into the plan file use json:"-". +type Snapshot struct { + // BundleID is the stable UUID that identifies the bundle deployment, used + // as the first path component of the snapshot workspace path. + BundleID string `json:"bundle_id"` + // ACL is the access control list applied to the uploaded snapshot, granting + // CAN_READ to the deploying user and to every principal in bundle.permissions. + ACL []snapshot.ACLEntry `json:"acl"` + // ZipContent holds the raw zip bytes of the bundle source tree. It is + // populated just before upload. The counterpart SnapshotState.ZipContent + // carries json:"-" so the zip bytes never reach the plan file; SyncZipContent + // re-injects them from here when deploying from a plan. + ZipContent string `json:"zip_content"` + // RemoteRoot is the workspace root path returned by the snapshot rootpath + // API (e.g. /Workspace/Users//.snapshots). + RemoteRoot string `json:"remote_root"` + + Lifecycle Lifecycle `json:"-"` +} + +func (s *Snapshot) RelativePath() string { + return path.Join(s.BundleID, snapshot.HashFromContent([]byte(s.ZipContent))) +} + +func (s *Snapshot) FullPath() string { + return path.Join(s.RemoteRoot, s.RelativePath()) +} + +func (s *Snapshot) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.Workspace.GetStatusByPath(ctx, s.FullPath()) + if err != nil { + return false, err + } + return true, nil +} + +func (s *Snapshot) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "internal_immutable_snapshot", + PluralName: "internal_immutable_snapshots", + SingularTitle: "Internal Immutable Snapshot", + PluralTitle: "Internal Immutable Snapshots", + } +} + +func (s *Snapshot) GetName() string { + return s.RelativePath() +} + +func (s *Snapshot) GetURL() string { + // Skipping URL initialization for snapshots + return "" +} + +func (s *Snapshot) InitializeURL(_ url.URL) { + // Secret scopes do not have a URL +} + +func (s *Snapshot) GetLifecycle() LifecycleConfig { + return s.Lifecycle +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index d83f4da59b8..21322b418fb 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -15,6 +15,9 @@ import ( "github.com/databricks/databricks-sdk-go/service/serving" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structtag" + "github.com/databricks/cli/libs/structs/structwalk" "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/apps" @@ -128,13 +131,14 @@ func TestBundleResourcePluralNamesResolveInWorkspaceURLs(t *testing.T) { // A job run does have a workspace URL, but it's addressed by two IDs // (job + run) so it can't be expressed as a single-ID pattern here; it's // built in JobRun.InitializeURL via workspaceurls.JobRunURL instead. - "job_runs": true, - "postgres_branches": true, - "postgres_databases": true, - "postgres_endpoints": true, - "postgres_projects": true, - "postgres_roles": true, - "secret_scopes": true, + "job_runs": true, + "postgres_branches": true, + "postgres_databases": true, + "postgres_endpoints": true, + "postgres_projects": true, + "postgres_roles": true, + "secret_scopes": true, + "internal_immutable_snapshots": true, } supported := SupportedResources() @@ -351,7 +355,8 @@ func TestResourcesBindSupport(t *testing.T) { }, } unbindableResources := map[string]bool{ - "model": true, + "model": true, + "internal_immutable_snapshot": true, } ctx := t.Context() @@ -409,3 +414,35 @@ func TestResourcesBindSupport(t *testing.T) { } } } + +func TestAllInteralResourcesAreMarkedAsInternal(t *testing.T) { + internalResourceKeys := map[string]reflect.Type{} + err := structwalk.WalkType(reflect.TypeFor[Resources](), func(path *structpath.PatternNode, typ reflect.Type, field *reflect.StructField) bool { + if path.Len() > 2 { + return false + } + if field == nil { + return true + } + tag := field.Tag.Get("bundle") + if structtag.BundleTag(tag).Internal() { + internalResourceKeys[field.Name] = typ + } + return true + }) + assert.NoError(t, err) + + for key, typ := range internalResourceKeys { + r := reflect.MakeMap(typ) + r.SetMapIndex(reflect.ValueOf("my_resources"), reflect.New(typ.Elem()).Elem()) + + resources := &Resources{} + res := reflect.ValueOf(resources).Elem() + field := res.FieldByName(key) + if !field.IsValid() && !field.CanSet() { + t.Fatalf("Field %s is not valid", key) + } + field.Set(r) + assert.True(t, resources.HasInternalResources()) + } +} diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index 4ea5c4308ad..e7070b20b02 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -76,7 +76,7 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // We only validate UC Volumes paths right now. - if !libraries.IsVolumesPath(b.Config.Workspace.ArtifactPath) { + if !libraries.IsVolumesPath(b.Config.Workspace.ArtifactPath) || b.IsImmutableFolder() { return nil } diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index f68670b7e86..9e5d2522d0d 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -78,12 +78,6 @@ type Workspace struct { // Remote workspace path for deployment state. // This defaults to "${workspace.root}/state". StatePath string `json:"state_path,omitempty"` - - // SnapshotPath is the workspace path of the immutable snapshot uploaded during - // deployment. Set by snapshot.Upload() and used by the subsequent variable-resolution - // pass to expand ${workspace.snapshot_path} placeholders in resource configs. - // Only populated at runtime for bundles with experimental.immutable_folder = true. - SnapshotPath string `json:"snapshot_path,omitempty" bundle:"internal"` } type User struct { diff --git a/bundle/deploy/snapshot/path.go b/bundle/deploy/snapshot/path.go index 62a4b9f91ab..b82cf42a336 100644 --- a/bundle/deploy/snapshot/path.go +++ b/bundle/deploy/snapshot/path.go @@ -4,8 +4,6 @@ import ( "archive/zip" "bytes" "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "io" @@ -17,6 +15,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/libs/fileset" + "github.com/databricks/cli/libs/snapshot" libsync "github.com/databricks/cli/libs/sync" ) @@ -71,12 +70,12 @@ func BundleZip(ctx context.Context, b *bundle.Bundle) ([]byte, int, error) { // addMetadataToZip writes the snapshot metadata file into the zip so that // any change to the ACL changes the snapshot hash and forces a new snapshot. -func addMetadataToZip(zw *zip.Writer, acl []ACLEntry) error { +func addMetadataToZip(zw *zip.Writer, acl []snapshot.ACLEntry) error { aclJSON, err := json.Marshal(acl) if err != nil { return fmt.Errorf("marshal ACL for permissions hash: %w", err) } - data, err := json.Marshal(snapshotMetadata{PermissionsHash: IDFromContent(aclJSON)}) + data, err := json.Marshal(snapshotMetadata{PermissionsHash: snapshot.HashFromContent(aclJSON)}) if err != nil { return fmt.Errorf("marshal snapshot metadata: %w", err) } @@ -93,23 +92,6 @@ func addMetadataToZip(zw *zip.Writer, acl []ACLEntry) error { return err } -// IDFromContent returns the SHA-256 hex digest of content. -func IDFromContent(content []byte) string { - h := sha256.Sum256(content) - return hex.EncodeToString(h[:]) -} - -// SnapshotID builds the bundle zip and returns its SHA-256 hex digest. -// Called after artifacts are built so that ApplyImmutableWorkspacePaths and -// snapshot.Upload both hash identical content. -func SnapshotID(ctx context.Context, b *bundle.Bundle) (string, error) { - content, _, err := BundleZip(ctx, b) - if err != nil { - return "", err - } - return IDFromContent(content), nil -} - // addSyncRootToZip returns the number of files added from the sync root. func addSyncRootToZip(ctx context.Context, zw *zip.Writer, b *bundle.Bundle) (int, error) { opts, err := files.GetSyncOptions(ctx, b) @@ -168,8 +150,6 @@ func addArtifactsToZip(zw *zip.Writer, b *bundle.Bundle) error { if af.Patched != "" { source = af.Patched } - // ".internal" matches libraries.InternalDirName so that ReplaceWithRemotePath - // produces library paths that resolve correctly inside the snapshot. if err := addLocalFileToZip(zw, source, "artifacts/.internal"); err != nil { return err } diff --git a/bundle/deploy/snapshot/path_test.go b/bundle/deploy/snapshot/path_test.go index 5f567d9f377..54407a98e20 100644 --- a/bundle/deploy/snapshot/path_test.go +++ b/bundle/deploy/snapshot/path_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deploy/snapshot" + libsnapshot "github.com/databricks/cli/libs/snapshot" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" @@ -93,26 +94,6 @@ func TestBundleZipRespectsExcludes(t *testing.T) { assert.Less(t, len(zipExcl), len(zipAll)) } -func TestIDFromContent(t *testing.T) { - id := snapshot.IDFromContent([]byte("hello")) - // SHA-256 of "hello" - assert.Equal(t, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", id) - assert.Len(t, id, 64, "SHA-256 hex must be 64 characters") -} - -func TestSnapshotIDMatchesBundleZipHash(t *testing.T) { - b := makeBundleWithFiles(t, map[string]string{"task.py": "x = 1"}) - - zipContent, _, err := snapshot.BundleZip(t.Context(), b) - require.NoError(t, err) - expectedID := snapshot.IDFromContent(zipContent) - - id, err := snapshot.SnapshotID(t.Context(), b) - require.NoError(t, err) - - assert.Equal(t, expectedID, id) -} - func zipEntryNames(t *testing.T, zipContent []byte) []string { t.Helper() r, err := zip.NewReader(bytes.NewReader(zipContent), int64(len(zipContent))) @@ -166,6 +147,6 @@ func TestBundleZipChangesWithPermissions(t *testing.T) { require.NoError(t, err) assert.NotEqual(t, zipNoPerms, zipWithPerms, "adding top-level permissions must produce a different snapshot zip") - assert.NotEqual(t, snapshot.IDFromContent(zipNoPerms), snapshot.IDFromContent(zipWithPerms), + assert.NotEqual(t, libsnapshot.HashFromContent(zipNoPerms), libsnapshot.HashFromContent(zipWithPerms), "snapshot IDs must differ when top-level permissions change") } diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index 8ec7a215d03..a0dfb1157b9 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -3,27 +3,26 @@ package snapshot import ( "context" "fmt" - "path" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/snapshot" + "github.com/google/uuid" ) // fileLimitWarning is the file count above which immutable folder deployments may fail. const fileLimitWarning = 1000 type snapshotUpload struct { - // uploader allows test injection of a custom SnapshotUploader. - uploader SnapshotUploader + skipZip bool } -// Upload returns a mutator that builds the bundle zip, uploads it via -// /api/2.0/repos/snapshots, and updates workspace.file_path and -// workspace.artifact_path to the content-addressed location returned by the API. -func Upload() bundle.Mutator { - return &snapshotUpload{} +// PlanUpload returns a mutator that builds the bundle zip, uploads it via +// /api/2.0/repos/snapshots, and registers the snapshot as an internal resource. +func PlanUpload(skipZip bool) bundle.Mutator { + return &snapshotUpload{skipZip: skipZip} } func (m *snapshotUpload) Name() string { @@ -31,63 +30,88 @@ func (m *snapshotUpload) Name() string { } func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - uploader := m.uploader - if uploader == nil { - var err error - uploader, err = NewSnapshotUploader(b.WorkspaceClient(ctx)) - if err != nil { - return diag.FromErr(err) - } + uploader, err := snapshot.NewSnapshotClient(b.WorkspaceClient(ctx)) + if err != nil { + return diag.FromErr(err) } - cmdio.LogString(ctx, "Uploading immutable bundle snapshot...") - - zipContent, fileCount, err := BundleZip(ctx, b) + remoteRoot, err := uploader.GetSnapshotRootPath(ctx) if err != nil { - return diag.FromErr(fmt.Errorf("failed to build snapshot zip: %w", err)) + return diag.FromErr(err) } - var diags diag.Diagnostics - if fileCount > fileLimitWarning { - diags = append(diags, diag.Warningf( - "immutable folder deployment may not work correctly: bundle contains %d files (limit is %d)", - fileCount, fileLimitWarning, - )...) + + if b.Config.Resources.Snapshots == nil { + b.Config.Resources.Snapshots = make(map[string]*resources.Snapshot) } - snapshotID := IDFromContent(zipContent) - log.Debugf(ctx, "snapshot.Upload: snapshotID=%s zip=%d bytes", snapshotID, len(zipContent)) - - acl := BuildACL(b) - // Use the deployment lineage UUID as bundle_id so the snapshot directory is - // keyed to this specific deployment (not to the bundle name, which can be - // reused across unrelated deployments). - bundleID := b.DeploymentBundle.StateDB.GetOrInitLineage() - info, err := uploader.Upload(ctx, bundleID, snapshotID, acl, zipContent) - if err != nil { - return diag.FromErr(err) + if _, ok := b.Config.Resources.Snapshots["immutable"]; !ok { + b.Config.Resources.Snapshots["immutable"] = &resources.Snapshot{ + BundleID: BundleID(b), + ACL: BuildACL(b), + RemoteRoot: remoteRoot, + } } - log.Infof(ctx, "Snapshot uploaded to %s", info.Path) + var diags diag.Diagnostics + if !m.skipZip { + zipContent, fileCount, err := BundleZip(ctx, b) + if err != nil { + return diag.FromErr(fmt.Errorf("failed to build snapshot zip: %w", err)) + } + + if fileCount > fileLimitWarning { + diags = append(diags, diag.Warningf( + "immutable folder deployment may not work correctly: bundle contains %d files (limit is %d)", + fileCount, fileLimitWarning, + )...) + } - b.Config.Workspace.SnapshotPath = info.Path - b.Config.Workspace.FilePath = path.Join(info.Path, "files") - // Only set artifact_path when artifacts are present; with no artifacts the - // zip has no "artifacts" directory and a get-status on it would 404. - if len(b.Config.Artifacts) > 0 { - b.Config.Workspace.ArtifactPath = path.Join(info.Path, "artifacts") + b.Config.Resources.Snapshots["immutable"].ZipContent = string(zipContent) } return diags } +// SyncZipContent copies the zip content from b.Config.Resources.Snapshots["immutable"] +// into the in-memory state cache entry for the snapshot resource. This is needed when +// deploying from a plan file: the plan JSON omits ZipContent (json:"-"), so InitForApply +// leaves it empty, causing DoCreate to upload an empty zip and derive a wrong snapshot ID. +func SyncZipContent(b *bundle.Bundle) { + snap := b.Config.Resources.Snapshots["immutable"] + if snap == nil || snap.ZipContent == "" { + return + } + sv, ok := b.DeploymentBundle.StateCache.Load("resources.internal_immutable_snapshots.immutable") + if !ok { + return + } + state, ok := sv.Value.(*dresources.SnapshotState) + if !ok { + return + } + state.ZipContent = snap.ZipContent +} + +// bundleIDNamespace is the UUID namespace used to derive the bundle ID. +var bundleIDNamespace = uuid.MustParse("4b4e4b5a-3c3d-4e4f-8b8c-9d9e9f0a0b0c") + +// BundleID returns a stable UUID that identifies the bundle deployment. +// It is derived deterministically from the bundle name, target, and workspace host +// so that every CLI invocation for the same deployment produces the same value. +// This is used as the path prefix for immutable snapshots in the workspace. +func BundleID(b *bundle.Bundle) string { + key := b.Config.Bundle.Name + "/" + b.Config.Bundle.Target + "/" + b.Config.Workspace.Host + return uuid.NewSHA1(bundleIDNamespace, []byte(key)).String() +} + // BuildACL constructs the access_control_list for the snapshot upload. // It grants CAN_READ to the current user and to every principal listed in the // top-level permissions section of the bundle config. -func BuildACL(b *bundle.Bundle) []ACLEntry { - acl := []ACLEntry{ +func BuildACL(b *bundle.Bundle) []snapshot.ACLEntry { + acl := []snapshot.ACLEntry{ {UserName: b.Config.Workspace.CurrentUser.UserName, PermissionLevel: "CAN_READ"}, } for _, p := range b.Config.Permissions { - acl = append(acl, ACLEntry{ + acl = append(acl, snapshot.ACLEntry{ UserName: p.UserName, GroupName: p.GroupName, ServicePrincipalName: p.ServicePrincipalName, diff --git a/bundle/deploy/snapshot/upload_test.go b/bundle/deploy/snapshot/upload_test.go index 3b53f31ae3c..5bace57e2be 100644 --- a/bundle/deploy/snapshot/upload_test.go +++ b/bundle/deploy/snapshot/upload_test.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deploy/snapshot" + libsnapshot "github.com/databricks/cli/libs/snapshot" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" ) @@ -29,7 +30,7 @@ func TestBuildACLCurrentUserOnly(t *testing.T) { acl := snapshot.BuildACL(b) - assert.Equal(t, []snapshot.ACLEntry{ + assert.Equal(t, []libsnapshot.ACLEntry{ {UserName: "alice@example.com", PermissionLevel: "CAN_READ"}, }, acl) } @@ -44,7 +45,7 @@ func TestBuildACLWithTopLevelPermissions(t *testing.T) { acl := snapshot.BuildACL(b) - assert.Equal(t, []snapshot.ACLEntry{ + assert.Equal(t, []libsnapshot.ACLEntry{ {UserName: "alice@example.com", PermissionLevel: "CAN_READ"}, {UserName: "bob@example.com", PermissionLevel: "CAN_READ"}, {GroupName: "devs", PermissionLevel: "CAN_READ"}, diff --git a/bundle/deploy/snapshot/upload_warning_test.go b/bundle/deploy/snapshot/upload_warning_test.go index ae45c45a87d..778c6b4795c 100644 --- a/bundle/deploy/snapshot/upload_warning_test.go +++ b/bundle/deploy/snapshot/upload_warning_test.go @@ -3,6 +3,7 @@ package snapshot import ( "context" "fmt" + "math" "os" "path/filepath" "testing" @@ -12,16 +13,25 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/testserver" "github.com/databricks/cli/libs/vfs" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type mockUploader struct{ path string } - -func (m *mockUploader) Upload(_ context.Context, _, _ string, _ []ACLEntry, _ []byte) (*SnapshotInfo, error) { - return &SnapshotInfo{Path: m.path}, nil +func setupTestClient(t *testing.T) *databricks.WorkspaceClient { + t.Helper() + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + RateLimitPerSecond: math.MaxInt, + }) + require.NoError(t, err) + return client } func makeBundle(t *testing.T, nFiles int) *bundle.Bundle { @@ -59,19 +69,20 @@ func testContext(t *testing.T) context.Context { func TestUploadWarnsAboveFileLimit(t *testing.T) { b := makeBundle(t, fileLimitWarning+1) - m := &snapshotUpload{uploader: &mockUploader{path: "/snapshots/test"}} + b.SetWorkpaceClient(setupTestClient(t)) + m := &snapshotUpload{} diags := m.Apply(testContext(t), b) require.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Contains(t, diags[0].Summary, fmt.Sprintf("%d files", fileLimitWarning+1)) - assert.Equal(t, "/snapshots/test", b.Config.Workspace.SnapshotPath) } func TestUploadNoWarningBelowFileLimit(t *testing.T) { b := makeBundle(t, 5) - m := &snapshotUpload{uploader: &mockUploader{path: "/snapshots/test"}} + b.SetWorkpaceClient(setupTestClient(t)) + m := &snapshotUpload{} diags := m.Apply(testContext(t), b) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index e44b86509ef..3367c14106a 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -23,6 +23,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { "secrets", "vector_search_endpoints", "vector_search_indexes", + "internal_immutable_snapshots", } for resourceType := range supportedResources { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 21c29d78d33..078a451af37 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -722,12 +722,6 @@ func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode) } func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { - // ${workspace.snapshot_path} is resolved by the mutator pipeline after - // snapshot.Upload() — not by the direct engine. Return errDelayed so the - // template string is preserved in the plan output rather than causing an error. - if path.String() == "workspace.snapshot_path" { - return nil, errDelayed - } targetResourceKey, fieldPath := splitResourcePath(path) targetGroup := config.GetResourceTypeFromKey(targetResourceKey) @@ -785,8 +779,8 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s localConfig := sv.Value - adapter := b.Adapters[targetGroup] - if adapter == nil { + adapter, err := b.getAdapterForKey(targetResourceKey) + if err != nil { return nil, fmt.Errorf("internal error: %s: unknown resource type %q", targetResourceKey, targetGroup) } @@ -961,7 +955,6 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root } slices.Sort(nodes) - for _, node := range nodes { delete(existingKeys, node) @@ -1049,9 +1042,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root targetNodeDP, _ := config.GetNodeAndType(targetPathParsed) targetNode := targetNodeDP.String() - // ${workspace.snapshot_path} is resolved by the mutator pipeline after - // snapshot.Upload(), not by the direct engine — skip it here. - if targetPath == "workspace.snapshot_path" { + if targetNode == "" { continue } diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index ad310468da0..ef285de031f 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -67,6 +67,9 @@ var SupportedResources = map[string]any{ "registered_models.grants": (*ResourceGrants)(nil), "vector_search_indexes.grants": (*ResourceGrants)(nil), "secrets.grants": (*ResourceGrants)(nil), + + // Internal resources + "internal_immutable_snapshots": (*ResourceSnapshot)(nil), } func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) { diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 75a94ffc88c..db41e575c15 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/snapshot" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" @@ -307,6 +308,12 @@ var testConfig map[string]any = map[string]any{ Privileges: []catalog.Privilege{catalog.PrivilegeSelect}, }}, }, + + "internal_immutable_snapshots": &resources.Snapshot{ + RemoteRoot: "/Workspace/Users/" + testserver.TestUserSP.UserName + "/.snapshots", + BundleID: "test-bundle-id", + ACL: []snapshot.ACLEntry{{UserName: "user@example.com", PermissionLevel: "CAN_READ"}}, + }, } type prepareWorkspace func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) @@ -911,6 +918,7 @@ func TestAll(t *testing.T) { // testIgnoreFilter encapsulates the logic for filtering fields based on ignore_remote_changes config. type testIgnoreFilter struct { ignoreFields map[string]bool + adapter *Adapter } // newTestIgnoreFilter creates a filter from the adapter's resource configs. @@ -938,7 +946,7 @@ func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { } return true }) - return &testIgnoreFilter{ignoreFields: ignoreFields} + return &testIgnoreFilter{ignoreFields: ignoreFields, adapter: adapter} } // shouldIgnore returns true if the field at the given path should be ignored. @@ -951,7 +959,17 @@ func (f *testIgnoreFilter) shouldIgnore(path string) bool { if prefix, _, ok := strings.Cut(path, "."); ok { topLevelField = prefix } - return f.ignoreFields[topLevelField] + + parts := strings.Split(topLevelField, "[") + if len(parts) > 1 { + topLevelField = parts[0] + } + + if f.ignoreFields[topLevelField] { + return true + } + + return structaccess.ValidatePath(f.adapter.RemoteType(), structpath.MustParsePath(path)) != nil } // filterChanges returns only the changes that should not be ignored. @@ -1079,6 +1097,10 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W } require.NoError(t, structwalk.Walk(newState, func(path *structpath.PathNode, val any, field *reflect.StructField) { + // Skip fields configured in ignore_remote_changes. + if ignoreFilter.shouldIgnore(path.String()) { + return + } remoteValue, err := structaccess.Get(remappedState, path) if err != nil { t.Errorf("Failed to read %s from remapped remote state %#v", path.String(), remappedState) @@ -1093,10 +1115,6 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W // testserver can set field to backend-generated value return } - // Skip fields configured in ignore_remote_changes. - if ignoreFilter.shouldIgnore(path.String()) { - return - } // t.Logf("Testing %s v=%#v, remoteValue=%#v", path.String(), val, remoteValue) // We expect fields set explicitly to be preserved by testserver, which is true for all resources as of today. // If not true for your resource, add exception here: @@ -1118,6 +1136,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W } deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") + isImmutable := strings.HasSuffix(group, "internal_immutable_snapshots") // Apps DoDelete is fire-and-forget: the API returns success while the app // sits in DELETING state for up to ~20 minutes before the record is removed. // A GET on the DELETING app returns the app, not 404 -- the testserver @@ -1128,6 +1147,9 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W remoteAfterDelete, err := adapter.DoRead(ctx, createdID) switch { + case isImmutable: + require.NoError(t, err) + assert.True(t, adapter.IsGone(remoteAfterDelete)) case deleteIsNoop: require.NoError(t, err) // The resource genuinely still exists, so it must not report as gone. diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index e98a1ac2330..37f93316b48 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -849,3 +849,7 @@ resources: backend_defaults: # The Vector Search API assigns index_subtype when the config omits it - field: index_subtype + + internal_immutable_snapshots: + recreate_on_changes: + - reason: immutable diff --git a/bundle/direct/dresources/snapshot.go b/bundle/direct/dresources/snapshot.go new file mode 100644 index 00000000000..872aa424f62 --- /dev/null +++ b/bundle/direct/dresources/snapshot.go @@ -0,0 +1,100 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/snapshot" + "github.com/databricks/databricks-sdk-go" +) + +type ResourceSnapshot struct { + uploader *snapshot.SnapshotClient +} + +type SnapshotState struct { + RemoteRoot string `json:"remote_root"` + RelativePath string `json:"relative_path"` + FullPath string `json:"full_path"` + BundleID string `json:"bundle_id"` + ACL []snapshot.ACLEntry `json:"acl"` + ZipContent string `json:"-"` +} + +type SnapshotRemote struct { + RelativePath string `json:"relative_path"` + FullPath string `json:"full_path"` +} + +func (s *ResourceSnapshot) New(client *databricks.WorkspaceClient) *ResourceSnapshot { + // Return a zero-value instance when client is nil (e.g. refschema introspection). + if client == nil { + return &ResourceSnapshot{ + uploader: nil, + } + } + + uploader, err := snapshot.NewSnapshotClient(client) + if err != nil { + panic(err) + } + + return &ResourceSnapshot{ + uploader: uploader, + } +} + +func (s *ResourceSnapshot) PrepareState(input *resources.Snapshot) *SnapshotState { + return &SnapshotState{ + RemoteRoot: input.RemoteRoot, + RelativePath: input.RelativePath(), + FullPath: input.FullPath(), + BundleID: input.BundleID, + ACL: input.ACL, + ZipContent: input.ZipContent, + } +} + +func (s *ResourceSnapshot) RemapState(remote *SnapshotRemote) *SnapshotState { + return &SnapshotState{ + RemoteRoot: "", + RelativePath: remote.RelativePath, + FullPath: remote.FullPath, + BundleID: "", + ACL: nil, + ZipContent: "", + } +} + +func (s *ResourceSnapshot) DoRead(ctx context.Context, id string) (*SnapshotRemote, error) { + info, err := s.uploader.Get(ctx, id) + if err != nil { + return nil, err + } + return &SnapshotRemote{ + RelativePath: id, + FullPath: info.Path, + }, nil +} + +func (s *ResourceSnapshot) DoCreate(ctx context.Context, state *SnapshotState) (string, *SnapshotRemote, error) { + path := state.RelativePath + info, err := s.uploader.Upload(ctx, path, state.BundleID, state.ACL, []byte(state.ZipContent)) + if err != nil { + return "", nil, err + } + return path, &SnapshotRemote{RelativePath: path, FullPath: info.Path}, nil +} + +func (s *ResourceSnapshot) DoUpdate(ctx context.Context, id string, newState *SnapshotState, entry *PlanEntry) (*SnapshotRemote, error) { + return nil, nil +} + +func (s *ResourceSnapshot) DoDelete(ctx context.Context, id string, state *SnapshotState) error { + return nil +} + +// IsGone treats a snapshot as already-deleted. The snapshot is immutable, so it can't be deleted. +func (s *ResourceSnapshot) IsGone(remote *SnapshotRemote) bool { + return true +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 109b98c3a86..97168625546 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -56,6 +56,11 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, + "internal_immutable_snapshots": { + "bundle_id", + "acl", + "remote_root", + }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. @@ -93,6 +98,9 @@ var knownMissingInStateType = map[string][]string{ "keyvault_metadata", "name", }, + "internal_immutable_snapshots": { + "zip_content", + }, } // TestInputSubset validates that all fields in InputType exist in StateType. diff --git a/bundle/libraries/filer.go b/bundle/libraries/filer.go index 762732262be..ddd178ae948 100644 --- a/bundle/libraries/filer.go +++ b/bundle/libraries/filer.go @@ -26,6 +26,9 @@ func GetFilerForLibraries(ctx context.Context, b *bundle.Bundle) (filer.Filer, s uploadPath := path.Join(b.Config.Workspace.ArtifactPath, InternalDirName) uploadPath = ensureWorkspaceOrVolumesPrefix(uploadPath) + if b.IsImmutableFolder() { + uploadPath = path.Join("${resources.internal_immutable_snapshots.immutable.full_path}", "artifacts", InternalDirName) + } switch { case IsVolumesPath(artifactPath): @@ -43,6 +46,9 @@ func GetFilerForLibrariesCleanup(ctx context.Context, b *bundle.Bundle) (filer.F } artifactPath = ensureWorkspaceOrVolumesPrefix(artifactPath) + if b.IsImmutableFolder() { + artifactPath = path.Join("${resources.internal_immutable_snapshots.immutable.full_path}", "artifacts") + } switch { case IsVolumesPath(artifactPath): diff --git a/bundle/phases/build.go b/bundle/phases/build.go index 580a18f7ab6..c5e3d8f28c3 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -56,12 +56,6 @@ func Build(ctx context.Context, b *bundle.Bundle) LibLocationMap { return nil } - // For immutable bundles, library remote paths are set in the deploy phase - // after snapshot.Upload() provides the content-addressed workspace.artifact_path. - if b.IsImmutableFolder() { - return nil - } - libs, diags := libraries.ReplaceWithRemotePath(ctx, b) for _, d := range diags { logdiag.LogDiag(ctx, d) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index e585fe51609..d18e025edaf 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/deploy" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" @@ -112,7 +111,8 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st return } - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, statemgmt.Load(state), metadata.Compute(), metadata.Upload(), @@ -137,7 +137,8 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st // It also cleans up the artifacts directory and transforms wheel tasks. // It is called by only "bundle deploy". func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]libraries.LocationToUpdate) { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, artifacts.CleanUp(), libraries.Upload(libs), ) @@ -153,7 +154,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, scripts.Execute(config.ScriptPreDeploy), lock.Acquire(lock.GoalDeploy), ) @@ -174,36 +176,20 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - if immutable { - // Upload all source files and built artifacts as a single immutable snapshot. - // snapshot.Upload() sets workspace.snapshot_path; the variable-resolution - // pass expands ${workspace.snapshot_path} placeholders written by translate_paths. - bundle.ApplySeqContext(ctx, b, - snapshot.Upload(), - mutator.ResolveVariableReferencesOnlyResources("workspace"), - ) - if !logdiag.HasError(ctx) { - _, libDiags := libraries.ReplaceWithRemotePath(ctx, b) - for _, d := range libDiags { - logdiag.LogDiag(ctx, d) - } - } - } else { + if !immutable { uploadLibraries(ctx, b, libs) - } - - if logdiag.HasError(ctx) { - return - } + if logdiag.HasError(ctx) { + return + } - if !immutable { bundle.ApplySeqContext(ctx, b, files.Upload(outputHandler)) if logdiag.HasError(ctx) { return } } - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, deploy.StateUpdate(), deploy.StatePush(), permissions.ApplyWorkspaceRootPermissions(), @@ -215,6 +201,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } + if immutable { + bundle.ApplyContext(ctx, b, snapshot.PlanUpload(false)) + if logdiag.HasError(ctx) { + return + } + } + planFromFile := plan != nil if plan == nil { // State is already open for read by process.go (for direct engine) @@ -243,6 +236,12 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand logdiag.LogError(ctx, err) return } + if immutable { + // The plan JSON omits ZipContent (json:"-"), so InitForApply leaves it + // empty in the state cache. Transfer the zip content built by PlanUpload + // above so DoCreate uploads the correct content and derives the right ID. + snapshot.SyncZipContent(b) + } } // InitForApply receives ctx and could log a diagnostic without returning an @@ -287,7 +286,8 @@ func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *d // b.Select is rejected for the terraform engine in ProcessBundleRet, so it is // never set here. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Plan(terraform.PlanGoal("deploy")), diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index bed07e121a4..11ab1609aad 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" + "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/cmdio" @@ -196,6 +197,12 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { var plan *deployplan.Plan if engine.IsDirect() { + if b.IsImmutableFolder() { + bundle.ApplyContext(ctx, b, snapshot.PlanUpload(true)) + if logdiag.HasError(ctx) { + return + } + } plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), nil) if err != nil { logdiag.LogError(ctx, err) diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index a130da820d6..266c3eac8b6 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -27,6 +27,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: initialize") bundle.ApplySeqContext(ctx, b, + // Reads (dynamic): resources.internal_immutable_snapshots (and other internal keys) + // Warns and removes any internal resource fields set by the user. + mutator.RejectInternalResources(), + // Reads (dynamic): resource.*.* // Checks that none of resources.. is nil. Raises error otherwise. validate.AllResourcesHaveValues(), diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index b706b0770cf..ce1b6b779c3 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -1191,6 +1191,11 @@ func AssertFullResourceCoverage(t *testing.T, config *config.Root) { resources := reflect.ValueOf(config.Resources) for i := range resources.NumField() { field := resources.Field(i) + // Get Tag field from reflect.Value + tag := resources.Type().Field(i).Tag.Get("bundle") + if tag == "internal" { + continue + } if field.Kind() == reflect.Map { assert.True( t, diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index 20df8cb5f0f..5ce0464c0cc 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/cmd/bundle/utils" @@ -61,6 +62,12 @@ It is useful for previewing changes before running 'bundle deploy'.`, } ctx := cmd.Context() + if b.IsImmutableFolder() { + bundle.ApplyContext(ctx, b, snapshot.PlanUpload(false)) + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + } plan := phases.RunPlan(ctx, b, stateDesc.Engine) if logdiag.HasError(ctx) { return root.ErrAlreadyPrinted diff --git a/bundle/deploy/snapshot/client.go b/libs/snapshot/client.go similarity index 66% rename from bundle/deploy/snapshot/client.go rename to libs/snapshot/client.go index 4e5df29e408..10ebd58b8b1 100644 --- a/bundle/deploy/snapshot/client.go +++ b/libs/snapshot/client.go @@ -8,6 +8,7 @@ import ( "mime/multipart" "net/http" "net/textproto" + "path" "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go" @@ -29,18 +30,10 @@ type ACLEntry struct { PermissionLevel string `json:"permission_level"` } -// SnapshotUploader abstracts the /api/2.0/repos/snapshots endpoint. -// snapshotID is the content-addressed key supplied by the caller; the API uses -// it as the final path component so that identical content always resolves to -// the same workspace location. -// This interface exists so the implementation can later be replaced with a Go SDK call. -type SnapshotUploader interface { - Upload(ctx context.Context, bundleID, snapshotID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) -} - -// snapshotAPIClient implements SnapshotUploader against /api/2.0/repos/snapshots. -type snapshotAPIClient struct { - client *databricksclient.DatabricksClient +// SnapshotClient implements the /api/2.0/repos/snapshots endpoint. +type SnapshotClient struct { + workspaceClient *databricks.WorkspaceClient + client *databricksclient.DatabricksClient } // snapshotUploadResponse mirrors the /api/2.0/repos/snapshots response body. @@ -50,23 +43,27 @@ type snapshotUploadResponse struct { } `json:"snapshot"` } -// NewSnapshotUploader creates a SnapshotUploader backed by /api/2.0/repos/snapshots. -func NewSnapshotUploader(w *databricks.WorkspaceClient) (SnapshotUploader, error) { +type snapshotRootPathResponse struct { + Path string `json:"path"` +} + +// NewSnapshotClient creates a SnapshotClient backed by /api/2.0/repos/snapshots. +func NewSnapshotClient(w *databricks.WorkspaceClient) (*SnapshotClient, error) { c, err := databricksclient.New(w.Config) if err != nil { return nil, err } - return &snapshotAPIClient{client: c}, nil + return &SnapshotClient{workspaceClient: w, client: c}, nil } // Upload uploads zipContent as an immutable snapshot identified by snapshotID. // snapshotID is the SHA-256 of the zip and is used by the server as the // content-addressed path component. acl grants CAN_READ to each listed principal. -func (c *snapshotAPIClient) Upload(ctx context.Context, bundleID, snapshotID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) { +func (c *SnapshotClient) Upload(ctx context.Context, path, bundleID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) { var body bytes.Buffer mw := multipart.NewWriter(&body) - if err := mw.WriteField("snapshot_id", snapshotID); err != nil { + if err := mw.WriteField("snapshot_id", HashFromContent(zipContent)); err != nil { return nil, fmt.Errorf("failed to write snapshot_id: %w", err) } if err := mw.WriteField("bundle_id", bundleID); err != nil { @@ -112,3 +109,25 @@ func (c *snapshotAPIClient) Upload(ctx context.Context, bundleID, snapshotID str return &SnapshotInfo{Path: resp.Snapshot.Path}, nil } + +func (c *SnapshotClient) Get(ctx context.Context, snapshotRelativePath string) (*SnapshotInfo, error) { + rootPath, err := c.GetSnapshotRootPath(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get snapshot root path: %w", err) + } + snapshotPath := path.Join(rootPath, snapshotRelativePath) + resp, err := c.workspaceClient.Workspace.GetStatusByPath(ctx, snapshotPath) + if err != nil { + return nil, fmt.Errorf("snapshot get: %w", err) + } + return &SnapshotInfo{Path: resp.Path}, nil +} + +func (c *SnapshotClient) GetSnapshotRootPath(ctx context.Context) (string, error) { + var resp snapshotRootPathResponse + err := c.client.Do(ctx, http.MethodGet, "/api/2.0/repos/snapshots/rootpath", auth.WorkspaceIDHeaders(c.client.Config), nil, nil, &resp) + if err != nil { + return "", fmt.Errorf("snapshot root path get: %w", err) + } + return path.Clean(resp.Path), nil +} diff --git a/libs/snapshot/hash.go b/libs/snapshot/hash.go new file mode 100644 index 00000000000..104858896c3 --- /dev/null +++ b/libs/snapshot/hash.go @@ -0,0 +1,12 @@ +package snapshot + +import ( + "crypto/sha256" + "encoding/hex" +) + +// HashFromContent returns the SHA-256 hex digest of content. +func HashFromContent(content []byte) string { + h := sha256.Sum256(content) + return hex.EncodeToString(h[:]) +} diff --git a/libs/sync/path.go b/libs/sync/path.go index 6b60ef682de..760695acfe8 100644 --- a/libs/sync/path.go +++ b/libs/sync/path.go @@ -26,6 +26,12 @@ func repoPathForPath(me *iam.User, remotePath string) string { // expected base paths and if it is a directory or repository. // If dryRun is set, a missing remote directory is not created. func EnsureRemotePathIsUsable(ctx context.Context, wsc *databricks.WorkspaceClient, remotePath string, me *iam.User, dryRun bool) error { + // If the remote path is a variable, we don't need to check if it's usable. + // This happens for immutable folder paths. Their paths are fully resolved at deployment stage and can't be validated at this point. + if strings.HasPrefix(remotePath, "${") { + return nil + } + var err error // TODO: we should cache CurrentUser.Me at the SDK level diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 082b771d1ea..6d83d08751e 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -638,6 +638,12 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.ReposDelete(req) }) + server.Handle("GET", "/api/2.0/repos/snapshots/rootpath", func(req Request) any { + return map[string]any{ + "path": "/Workspace/Users/" + TestUserSP.UserName + "/.snapshots/", + } + }) + server.Handle("POST", "/api/2.0/repos/snapshots", func(req Request) any { contentType := req.Headers.Get("Content-Type") mediaType, params, err := mime.ParseMediaType(contentType) @@ -670,6 +676,7 @@ func AddDefaultHandlers(server *Server) { // The real API uses the workspace user UUID (not email) in the snapshot path, // matching service-principal identities used in cloud acceptance tests. snapshotPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/%s", TestUserSP.UserName, bundleID, snapshotID) + req.Workspace.WorkspaceMkdirs(workspace.Mkdirs{Path: snapshotPath}) return map[string]any{ "snapshot": map[string]any{ "path": snapshotPath,