diff --git a/infra/terraform/envs/sandbox/app-deps.tf b/infra/terraform/envs/sandbox/app-deps.tf index 44db515..5382e57 100644 --- a/infra/terraform/envs/sandbox/app-deps.tf +++ b/infra/terraform/envs/sandbox/app-deps.tf @@ -166,6 +166,13 @@ resource "aws_db_instance" "app" { tags = merge(local.common_tags, { Name = "${local.cluster_name}-postgres" }) + + # auto_minor_version_upgrade is enabled, so RDS bumps the minor version out of + # band (e.g. 17.5 -> 17.9). Ignore engine_version here so Terraform does not + # try to "downgrade" back to the pinned value on every plan. + lifecycle { + ignore_changes = [engine_version] + } } # --------------------------------------------------------------- diff --git a/infra/terraform/envs/sandbox/main.tf b/infra/terraform/envs/sandbox/main.tf index 80ff278..7e503c3 100644 --- a/infra/terraform/envs/sandbox/main.tf +++ b/infra/terraform/envs/sandbox/main.tf @@ -66,5 +66,6 @@ module "eks" { external_secrets_secret_name_prefixes = var.external_secrets_secret_name_prefixes external_secrets_ssm_parameter_prefixes = var.external_secrets_ssm_parameter_prefixes enable_efs_csi_driver = var.enable_efs_csi_driver + enable_prefix_delegation = true common_tags = local.common_tags } diff --git a/infra/terraform/envs/sandbox/node-groups.tf b/infra/terraform/envs/sandbox/node-groups.tf index 23274ff..f2b4e59 100644 --- a/infra/terraform/envs/sandbox/node-groups.tf +++ b/infra/terraform/envs/sandbox/node-groups.tf @@ -1,54 +1,54 @@ -resource "aws_eks_node_group" "system" { - cluster_name = module.eks.cluster_id - node_group_name = "${local.cluster_name}-system" - node_role_arn = module.eks.nodegroup_role_arn - subnet_ids = module.vpc.private_subnet_ids +# SANDBOX runs on a single consolidated node group (no HA — fail-fast by design). +# The launch template raises kubelet max-pods to 110 so one t3.medium can host +# the full workload once VPC CNI prefix delegation is enabled (see main.tf). +resource "aws_launch_template" "node" { + name_prefix = "${local.cluster_name}-node-" - ami_type = "AL2023_x86_64_STANDARD" - capacity_type = "ON_DEMAND" - disk_size = var.system_node_disk_size - instance_types = var.system_node_instance_types + # AL2023 NodeConfig merged by the managed node group bootstrap. + user_data = base64encode(<<-EOT + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="//" - labels = { - env = local.env - nodepool = "system" - } + --// + Content-Type: application/node.eks.aws - scaling_config { - desired_size = var.system_node_desired_size - max_size = var.system_node_max_size - min_size = var.system_node_min_size - } + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + config: + maxPods: 110 + --//-- + EOT + ) - update_config { - max_unavailable = 1 - } + tags = local.common_tags lifecycle { - ignore_changes = [ - scaling_config[0].desired_size - ] + create_before_destroy = true } - - tags = merge(local.common_tags, { - "k8s.io/cluster-autoscaler/${local.cluster_name}" = "owned" - "k8s.io/cluster-autoscaler/enabled" = "true" - }) - - depends_on = [module.vpc] } +# Single node group for SANDBOX. Kept as resource "app" (not renamed) so changing +# the launch template triggers an in-place EKS rolling node update rather than a +# destroy/recreate of the node group. resource "aws_eks_node_group" "app" { cluster_name = module.eks.cluster_id node_group_name = "${local.cluster_name}-app" node_role_arn = module.eks.nodegroup_role_arn subnet_ids = module.vpc.private_subnet_ids - ami_type = "AL2023_x86_64_STANDARD" - capacity_type = "ON_DEMAND" - disk_size = var.app_node_disk_size + ami_type = "AL2023_x86_64_STANDARD" + capacity_type = "ON_DEMAND" + instance_types = var.app_node_instance_types + launch_template { + id = aws_launch_template.node.id + version = aws_launch_template.node.latest_version + } + labels = { env = local.env nodepool = "app" @@ -61,6 +61,7 @@ resource "aws_eks_node_group" "app" { min_size = var.app_node_min_size } + # Rolling node replacement on launch-template changes. update_config { max_unavailable = 1 } @@ -71,10 +72,7 @@ resource "aws_eks_node_group" "app" { ] } - tags = merge(local.common_tags, { - "k8s.io/cluster-autoscaler/${local.cluster_name}" = "owned" - "k8s.io/cluster-autoscaler/enabled" = "true" - }) + tags = local.common_tags depends_on = [module.vpc] } diff --git a/infra/terraform/envs/sandbox/outputs.tf b/infra/terraform/envs/sandbox/outputs.tf index fdd541e..12f357f 100644 --- a/infra/terraform/envs/sandbox/outputs.tf +++ b/infra/terraform/envs/sandbox/outputs.tf @@ -33,13 +33,8 @@ output "public_subnet_ids" { value = module.vpc.public_subnet_ids } -output "system_node_group_name" { - description = "System managed node group name." - value = aws_eks_node_group.system.node_group_name -} - output "app_node_group_name" { - description = "Application managed node group name." + description = "Consolidated managed node group name (single node, no HA)." value = aws_eks_node_group.app.node_group_name } diff --git a/infra/terraform/envs/sandbox/terraform.tfvars b/infra/terraform/envs/sandbox/terraform.tfvars index d096cae..531bb9d 100644 --- a/infra/terraform/envs/sandbox/terraform.tfvars +++ b/infra/terraform/envs/sandbox/terraform.tfvars @@ -44,14 +44,10 @@ db_max_allocated_storage = 100 db_multi_az = false db_backup_retention_days = 7 -system_node_instance_types = ["t3.medium"] -system_node_min_size = 1 -system_node_max_size = 2 -system_node_desired_size = 1 -system_node_disk_size = 30 - +# SANDBOX: single consolidated t3.medium node (no HA — fail-fast by design). +# system_node_* vars are unused now that the system node group is removed. app_node_instance_types = ["t3.medium"] app_node_min_size = 1 -app_node_max_size = 2 +app_node_max_size = 1 app_node_desired_size = 1 app_node_disk_size = 50 diff --git a/infra/terraform/k8s-manifests/sandbox/addons/cluster-autoscaler.yaml b/infra/terraform/k8s-manifests/sandbox/addons/cluster-autoscaler.yaml deleted file mode 100644 index 8caa054..0000000 --- a/infra/terraform/k8s-manifests/sandbox/addons/cluster-autoscaler.yaml +++ /dev/null @@ -1,128 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cluster-autoscaler - namespace: kube-system - annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::996810415034:role/ctdl-xtra-sandbox-cluster-autoscaler-irsa-role ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cluster-autoscaler -rules: - - apiGroups: [""] - resources: ["events", "endpoints"] - verbs: ["create", "patch"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["create", "list", "watch"] - - apiGroups: [""] - resources: ["configmaps"] - resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] - verbs: ["delete", "get", "update", "watch"] - - apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] - - apiGroups: [""] - resources: ["pods/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["endpoints"] - resourceNames: ["cluster-autoscaler"] - verbs: ["get", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["watch", "list", "get", "update"] - - apiGroups: [""] - resources: ["namespaces", "pods", "services", "replicationcontrollers", "persistentvolumeclaims", "persistentvolumes"] - verbs: ["watch", "list", "get"] - - apiGroups: ["extensions"] - resources: ["replicasets", "daemonsets"] - verbs: ["watch", "list", "get"] - - apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] - verbs: ["watch", "list"] - - apiGroups: ["apps"] - resources: ["statefulsets", "replicasets", "daemonsets"] - verbs: ["watch", "list", "get"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "csinodes", "csidrivers", "csistoragecapacities", "volumeattachments"] - verbs: ["watch", "list", "get"] - - apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - resourceNames: ["cluster-autoscaler"] - verbs: ["get", "update"] - - apiGroups: ["resource.k8s.io"] - resources: ["resourceclaims", "resourceslices", "deviceclasses"] - verbs: ["watch", "list", "get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cluster-autoscaler -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-autoscaler -subjects: - - kind: ServiceAccount - name: cluster-autoscaler - namespace: kube-system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cluster-autoscaler - namespace: kube-system - labels: - app: cluster-autoscaler -spec: - replicas: 1 - selector: - matchLabels: - app: cluster-autoscaler - template: - metadata: - labels: - app: cluster-autoscaler - spec: - serviceAccountName: cluster-autoscaler - priorityClassName: system-cluster-critical - nodeSelector: - nodepool: system - containers: - - name: cluster-autoscaler - image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.35.0 - imagePullPolicy: IfNotPresent - command: - - ./cluster-autoscaler - - --v=4 - - --stderrthreshold=info - - --cloud-provider=aws - - --skip-nodes-with-local-storage=false - - --skip-nodes-with-system-pods=false - - --expander=least-waste - - --balance-similar-node-groups - - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/ctdl-xtra-sandbox - resources: - requests: - cpu: 100m - memory: 600Mi - limits: - cpu: 100m - memory: 600Mi - volumeMounts: - - name: ssl-certs - mountPath: /etc/ssl/certs/ca-certificates.crt - readOnly: true - volumes: - - name: ssl-certs - hostPath: - path: /etc/ssl/certs/ca-bundle.crt diff --git a/infra/terraform/k8s-manifests/sandbox/addons/ingress-nginx-values.yaml b/infra/terraform/k8s-manifests/sandbox/addons/ingress-nginx-values.yaml index 33e3d6f..3f10094 100644 --- a/infra/terraform/k8s-manifests/sandbox/addons/ingress-nginx-values.yaml +++ b/infra/terraform/k8s-manifests/sandbox/addons/ingress-nginx-values.yaml @@ -1,7 +1,8 @@ controller: - replicaCount: 2 + # SANDBOX runs on a single consolidated node group (nodepool: app), no HA. + replicaCount: 1 nodeSelector: - nodepool: system + nodepool: app service: externalTrafficPolicy: Local annotations: diff --git a/infra/terraform/k8s-manifests/sandbox/addons/install-foundation.sh b/infra/terraform/k8s-manifests/sandbox/addons/install-foundation.sh index 29a7686..64f39bb 100755 --- a/infra/terraform/k8s-manifests/sandbox/addons/install-foundation.sh +++ b/infra/terraform/k8s-manifests/sandbox/addons/install-foundation.sh @@ -59,8 +59,8 @@ helm upgrade --install metrics-server metrics-server/metrics-server \ --wait \ --timeout 5m -kubectl --context "${CONTEXT}" apply -f "${SCRIPT_DIR}/cluster-autoscaler.yaml" -kubectl --context "${CONTEXT}" -n kube-system rollout status deployment/cluster-autoscaler --timeout=180s +# cluster-autoscaler intentionally not deployed on SANDBOX: the node group is +# fixed at a single node (min=max=desired=1), so there is nothing to scale. kubectl --context "${CONTEXT}" apply -f "${SCRIPT_DIR}/skooner.yaml" kubectl --context "${CONTEXT}" -n kube-system rollout status deployment/skooner --timeout=180s diff --git a/infra/terraform/k8s-manifests/sandbox/addons/metrics-server-values.yaml b/infra/terraform/k8s-manifests/sandbox/addons/metrics-server-values.yaml index 5a43ad5..76f0a00 100644 --- a/infra/terraform/k8s-manifests/sandbox/addons/metrics-server-values.yaml +++ b/infra/terraform/k8s-manifests/sandbox/addons/metrics-server-values.yaml @@ -1,4 +1,5 @@ -replicas: 2 +# SANDBOX runs on a single consolidated node group (nodepool: app), no HA. +replicas: 1 resources: requests: @@ -9,4 +10,4 @@ resources: memory: 400Mi nodeSelector: - nodepool: system + nodepool: app diff --git a/infra/terraform/modules/eks/addons.tf b/infra/terraform/modules/eks/addons.tf index 9e01013..b66d8b5 100644 --- a/infra/terraform/modules/eks/addons.tf +++ b/infra/terraform/modules/eks/addons.tf @@ -4,6 +4,13 @@ resource "aws_eks_addon" "vpc_cni" { resolve_conflicts_on_create = "OVERWRITE" resolve_conflicts_on_update = "OVERWRITE" + configuration_values = var.enable_prefix_delegation ? jsonencode({ + env = { + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + } + }) : null + tags = var.common_tags } diff --git a/infra/terraform/modules/eks/variables.tf b/infra/terraform/modules/eks/variables.tf index e059ed4..35ab97b 100644 --- a/infra/terraform/modules/eks/variables.tf +++ b/infra/terraform/modules/eks/variables.tf @@ -135,3 +135,9 @@ variable "enable_efs_csi_driver" { type = bool default = false } + +variable "enable_prefix_delegation" { + description = "Enable VPC CNI prefix delegation (raises per-node pod capacity)." + type = bool + default = false +}