From 4e36717024f76d4a07b5a060a50a6fe4f2acc11b Mon Sep 17 00:00:00 2001 From: Qi Tan <16416018+TQJADE@users.noreply.github.com> Date: Sun, 19 Apr 2026 22:19:49 -0700 Subject: [PATCH 1/4] Introduce crds.install to Helm Chart --- .../sparkapplications.spark.apache.org-v1.yaml | 2 ++ .../crds/sparkclusters.spark.apache.org-v1.yaml | 2 ++ .../spark-kubernetes-operator/values.schema.json | 11 +++++++++++ .../helm/spark-kubernetes-operator/values.yaml | 3 +++ spark-operator-api/build.gradle | 14 +++++--------- 5 files changed, 23 insertions(+), 9 deletions(-) rename build-tools/helm/spark-kubernetes-operator/{ => templates}/crds/sparkapplications.spark.apache.org-v1.yaml (99%) rename build-tools/helm/spark-kubernetes-operator/{ => templates}/crds/sparkclusters.spark.apache.org-v1.yaml (99%) diff --git a/build-tools/helm/spark-kubernetes-operator/crds/sparkapplications.spark.apache.org-v1.yaml b/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkapplications.spark.apache.org-v1.yaml similarity index 99% rename from build-tools/helm/spark-kubernetes-operator/crds/sparkapplications.spark.apache.org-v1.yaml rename to build-tools/helm/spark-kubernetes-operator/templates/crds/sparkapplications.spark.apache.org-v1.yaml index 4f3f6c17..2202dd2d 100644 --- a/build-tools/helm/spark-kubernetes-operator/crds/sparkapplications.spark.apache.org-v1.yaml +++ b/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkapplications.spark.apache.org-v1.yaml @@ -13,6 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Values.crds.install }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -26511,3 +26512,4 @@ spec: storage: true subresources: status: {} +{{- end }} diff --git a/build-tools/helm/spark-kubernetes-operator/crds/sparkclusters.spark.apache.org-v1.yaml b/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkclusters.spark.apache.org-v1.yaml similarity index 99% rename from build-tools/helm/spark-kubernetes-operator/crds/sparkclusters.spark.apache.org-v1.yaml rename to build-tools/helm/spark-kubernetes-operator/templates/crds/sparkclusters.spark.apache.org-v1.yaml index f3203379..c8d9e039 100644 --- a/build-tools/helm/spark-kubernetes-operator/crds/sparkclusters.spark.apache.org-v1.yaml +++ b/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkclusters.spark.apache.org-v1.yaml @@ -13,6 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Values.crds.install }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -22119,3 +22120,4 @@ spec: storage: true subresources: status: {} +{{- end }} diff --git a/build-tools/helm/spark-kubernetes-operator/values.schema.json b/build-tools/helm/spark-kubernetes-operator/values.schema.json index 314c661b..62626f3d 100644 --- a/build-tools/helm/spark-kubernetes-operator/values.schema.json +++ b/build-tools/helm/spark-kubernetes-operator/values.schema.json @@ -11,6 +11,17 @@ "type": "string", "description": "Override fully qualified app name" }, + "crds": { + "type": "object", + "description": "CRD installation configuration", + "required": ["install"], + "properties": { + "install": { + "type": "boolean", + "description": "Install CRDs with the chart. Set to false if CRDs are managed separately." + } + } + }, "image": { "type": "object", "description": "Operator image configuration", diff --git a/build-tools/helm/spark-kubernetes-operator/values.yaml b/build-tools/helm/spark-kubernetes-operator/values.yaml index 64c6d13b..fa387839 100644 --- a/build-tools/helm/spark-kubernetes-operator/values.yaml +++ b/build-tools/helm/spark-kubernetes-operator/values.yaml @@ -22,6 +22,9 @@ image: imagePullSecrets: [ ] +crds: + install: true + operatorDeployment: # Replicas must be 1 unless leader election is enabled replicas: 1 diff --git a/spark-operator-api/build.gradle b/spark-operator-api/build.gradle index 4a3416e5..28e6f356 100644 --- a/spark-operator-api/build.gradle +++ b/spark-operator-api/build.gradle @@ -68,7 +68,7 @@ tasks.register("assertGeneratedCRDMatchesHelmChart") { def currentPath = projectDir.absolutePath doLast { def generatedCRDFileBase = "$currentPath/build/resources/main/" - def stagedCRDFileBase = "$currentPath/../build-tools/helm/spark-kubernetes-operator/crds/" + def stagedCRDFileBase = "$currentPath/../build-tools/helm/spark-kubernetes-operator/templates/crds/" def generatedAppCRD = [ "yq", "e", @@ -82,16 +82,12 @@ tasks.register("assertGeneratedCRDMatchesHelmChart") { "${generatedCRDFileBase}sparkclusters.spark.apache.org-v1.yml" ].execute().text.trim() def stagedAppCRD = [ - "yq", - "e", - ".spec.versions[-1]", - "${stagedCRDFileBase}sparkapplications.spark.apache.org-v1.yaml" + "bash", "-c", + "sed '/^{{/d' ${stagedCRDFileBase}sparkapplications.spark.apache.org-v1.yaml | yq e '.spec.versions[-1]' -" ].execute().text.trim() def stagedClusterCRD = [ - "yq", - "e", - ".spec.versions[-1]", - "${stagedCRDFileBase}sparkclusters.spark.apache.org-v1.yaml" + "bash", "-c", + "sed '/^{{/d' ${stagedCRDFileBase}sparkclusters.spark.apache.org-v1.yaml | yq e '.spec.versions[-1]' -" ].execute().text.trim() if (generatedAppCRD != stagedAppCRD || generatedClusterCRD != stagedClusterCRD) { def errorMessage = new StringBuilder("Generated CRD yaml does not match the staged " + From b2197df3032de6b2ffe93997b4f8260a737cd753 Mon Sep 17 00:00:00 2001 From: Qi Tan <16416018+TQJADE@users.noreply.github.com> Date: Thu, 23 Apr 2026 09:24:54 -0700 Subject: [PATCH 2/4] Support crd.annotations --- .../crds/sparkapplications.spark.apache.org-v1.yaml | 4 ++++ .../templates/crds/sparkclusters.spark.apache.org-v1.yaml | 4 ++++ .../helm/spark-kubernetes-operator/values.schema.json | 7 +++++++ build-tools/helm/spark-kubernetes-operator/values.yaml | 2 ++ spark-operator-api/build.gradle | 4 ++-- 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkapplications.spark.apache.org-v1.yaml b/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkapplications.spark.apache.org-v1.yaml index 2202dd2d..73bebaf0 100644 --- a/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkapplications.spark.apache.org-v1.yaml +++ b/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkapplications.spark.apache.org-v1.yaml @@ -18,6 +18,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: sparkapplications.spark.apache.org + {{- with .Values.crds.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: group: spark.apache.org names: diff --git a/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkclusters.spark.apache.org-v1.yaml b/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkclusters.spark.apache.org-v1.yaml index c8d9e039..bfb743cb 100644 --- a/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkclusters.spark.apache.org-v1.yaml +++ b/build-tools/helm/spark-kubernetes-operator/templates/crds/sparkclusters.spark.apache.org-v1.yaml @@ -18,6 +18,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: sparkclusters.spark.apache.org + {{- with .Values.crds.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: group: spark.apache.org names: diff --git a/build-tools/helm/spark-kubernetes-operator/values.schema.json b/build-tools/helm/spark-kubernetes-operator/values.schema.json index 62626f3d..c65db734 100644 --- a/build-tools/helm/spark-kubernetes-operator/values.schema.json +++ b/build-tools/helm/spark-kubernetes-operator/values.schema.json @@ -19,6 +19,13 @@ "install": { "type": "boolean", "description": "Install CRDs with the chart. Set to false if CRDs are managed separately." + }, + "annotations": { + "type": "object", + "description": "Annotations to add to CRD resources. Default includes helm.sh/resource-policy: keep to prevent CRD deletion on helm uninstall.", + "additionalProperties": { + "type": "string" + } } } }, diff --git a/build-tools/helm/spark-kubernetes-operator/values.yaml b/build-tools/helm/spark-kubernetes-operator/values.yaml index fa387839..4d1d74e8 100644 --- a/build-tools/helm/spark-kubernetes-operator/values.yaml +++ b/build-tools/helm/spark-kubernetes-operator/values.yaml @@ -24,6 +24,8 @@ imagePullSecrets: [ ] crds: install: true + annotations: + "helm.sh/resource-policy": keep operatorDeployment: # Replicas must be 1 unless leader election is enabled diff --git a/spark-operator-api/build.gradle b/spark-operator-api/build.gradle index 28e6f356..29713702 100644 --- a/spark-operator-api/build.gradle +++ b/spark-operator-api/build.gradle @@ -83,11 +83,11 @@ tasks.register("assertGeneratedCRDMatchesHelmChart") { ].execute().text.trim() def stagedAppCRD = [ "bash", "-c", - "sed '/^{{/d' ${stagedCRDFileBase}sparkapplications.spark.apache.org-v1.yaml | yq e '.spec.versions[-1]' -" + "sed '/{{/d' ${stagedCRDFileBase}sparkapplications.spark.apache.org-v1.yaml | yq e '.spec.versions[-1]' -" ].execute().text.trim() def stagedClusterCRD = [ "bash", "-c", - "sed '/^{{/d' ${stagedCRDFileBase}sparkclusters.spark.apache.org-v1.yaml | yq e '.spec.versions[-1]' -" + "sed '/{{/d' ${stagedCRDFileBase}sparkclusters.spark.apache.org-v1.yaml | yq e '.spec.versions[-1]' -" ].execute().text.trim() if (generatedAppCRD != stagedAppCRD || generatedClusterCRD != stagedClusterCRD) { def errorMessage = new StringBuilder("Generated CRD yaml does not match the staged " + From 4bfbb74ab9bc5ea8e52c2799df4695334af01875 Mon Sep 17 00:00:00 2001 From: Qi Tan <16416018+TQJADE@users.noreply.github.com> Date: Fri, 24 Apr 2026 20:29:56 -0700 Subject: [PATCH 3/4] Second operator installation should not install crd --- docs/operations.md | 2 +- tests/e2e/helm/dynamic-config-values-2.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/operations.md b/docs/operations.md index 5233352c..02ef8178 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -165,7 +165,7 @@ helm install us-west-1 spark/spark-kubernetes-operator --create-namespace --name ``` ```bash -helm install us-west-2 spark/spark-kubernetes-operator --create-namespace --namespace us-west-2 --set operatorRbac.clusterRole.name=spark-operator-clusterrole-us-west-2 --set operatorRbac.clusterRoleBinding.name=spark-operator-clusterrolebinding-us-west-2 --set workloadResources.clusterRole.name=spark-workload-clusterrole-us-west-2 +helm install us-west-2 spark/spark-kubernetes-operator --create-namespace --namespace us-west-2 --set crds.install=false --set operatorRbac.clusterRole.name=spark-operator-clusterrole-us-west-2 --set operatorRbac.clusterRoleBinding.name=spark-operator-clusterrolebinding-us-west-2 --set workloadResources.clusterRole.name=spark-workload-clusterrole-us-west-2 ``` Check installation. diff --git a/tests/e2e/helm/dynamic-config-values-2.yaml b/tests/e2e/helm/dynamic-config-values-2.yaml index b6410cb1..6724454a 100644 --- a/tests/e2e/helm/dynamic-config-values-2.yaml +++ b/tests/e2e/helm/dynamic-config-values-2.yaml @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +crds: + install: false + workloadResources: namespaces: overrideWatchedNamespaces: false From 382597984e082296ab55182b246e4da84f0602d3 Mon Sep 17 00:00:00 2001 From: Qi Tan <16416018+TQJADE@users.noreply.github.com> Date: Wed, 29 Apr 2026 15:02:17 -0700 Subject: [PATCH 4/4] Update operations doc for helm upgrade changes --- docs/operations.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/operations.md b/docs/operations.md index 02ef8178..73d7a5f3 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -61,6 +61,8 @@ following table: | Parameters | Description | Default value | |------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------| +| crds.install | Whether to install CRDs with the chart. Set to `false` when CRDs are managed externally or when deploying additional operator instances. | true | +| crds.annotations | Annotations to add to CRD resources. The default prevents CRDs from being deleted on `helm uninstall`. | `"helm.sh/resource-policy": keep` | | image.repository | The image repository of spark-kubernetes-operator. | apache/spark-kubernetes-operator | | image.pullPolicy | The image pull policy of spark-kubernetes-operator. | IfNotPresent | | image.tag | The image tag of spark-kubernetes-operator. | 0.9.0-SNAPSHOT | @@ -156,6 +158,21 @@ metadata: "spark.operator/sentinel": "true" ``` +## Upgrading from chart versions that used the `crds/` directory + +Older chart versions installed CRDs via Helm's special `crds/` directory, which does not +track ownership metadata. The current chart manages CRDs as regular templates. Before +running `helm upgrade`, you must label and annotate the existing CRDs so Helm can adopt +them. Replace `` and `` with your Helm release values: + +```bash +kubectl label crd sparkapplications.spark.apache.org app.kubernetes.io/managed-by=Helm +kubectl annotate crd sparkapplications.spark.apache.org meta.helm.sh/release-name= meta.helm.sh/release-namespace= + +kubectl label crd sparkclusters.spark.apache.org app.kubernetes.io/managed-by=Helm +kubectl annotate crd sparkclusters.spark.apache.org meta.helm.sh/release-name= meta.helm.sh/release-namespace= +``` + ## Example Install HelmChart at `us-west-1` and `us-west-2` namespaces.