Jk/cumulus 4773

CHANGELOG.md

@@ -1,9 +1,19 @@
 # CHANGELOG
 
 ## Unreleased
-
+* Update Dockerfile python3 stage to use only node v22, remove all yum installs in favor of dnf
 * Add dynamic_throttled_queues to allow for throttled queues with configurable names to be defined according to the pattern: https://sqs.${data.aws_region.current.name}.amazonaws.com/${data.aws_caller_identity.current.account_id}/${local.prefix}-${q.queue_name}.   This allows for queue configurations to be defined programatically for similar deployments across deployment/account/regions/etc.
 
+## v21.3.2.0
+* Upgrade to [Cumulus v21.3.2](https://github.com/nasa/cumulus/releases/tag/v21.3.2)
+* **NOTE** this version of Cumulus includes core version of CnmResponse task lambda that may conflict with non-core versions. See [v21.3.2 Migration Notes](https://github.com/nasa/cumulus/releases/tag/v21.3.2)
+
+## v21.3.1.0
+* Upgrade to [Cumulus v21.3.1](https://github.com/nasa/cumulus/releases/tag/v21.3.1)
+* Update dockerfile to use Amazon Linux 2023 and Node v22
+* **NOTE** This version of Cumulus requires manual database changes, see [v21.3 Migration Notes](https://github.com/nasa/cumulus/releases/tag/v21.3.0)
+
+## v21.2.0.0
 * Upgrade to [Cumulus v21.2.0](https://github.com/nasa/cumulus/releases/tag/v21.2.0)
 * add variable "archive_records_config" to cumulus/variables.tf
 

Dockerfile

@@ -1,7 +1,7 @@
-FROM public.ecr.aws/lambda/python:3.9 AS python3
+FROM amazonlinux:2023 AS python3
 # This image can be used to do Python 3 & NodeJS development, and
 # includes the AWS CLI and Terraform. It contains:
-
+#
 #   * CLI utilities: git, make, wget, etc
 #   * Python 3
 #   * NodeJS
@@ -10,69 +10,74 @@ FROM public.ecr.aws/lambda/python:3.9 AS python3
 #   * Terraform
 #   * Docker
 
-# Amazon Linux 2 does not support node 18.x or node 20.x glibc=2.27 and >=2.28 is required
-ENV NODE_VERSION="16.x"
 ENV TERRAFORM_VERSION="1.12.2"
 ENV AWS_CLI_VERSION="2.27.43"
+ARG USER
+
+WORKDIR /CIRRUS-core
+# Base packages + Python + Docker + Yarn
+RUN dnf install -y \
+        gcc \
+        gcc-c++ \
+        git \
+        make \
+        unzip \
+        zip \
+        jq \
+        python3 \
+        python3-pip \
+        docker \
+    && dnf clean all
 
-# Add NodeJS and Yarn repos & update package index
-RUN \
-        yum install https://rpm.nodesource.com/pub_${NODE_VERSION}/nodistro/repo/nodesource-release-nodistro-1.noarch.rpm -y && \
-        yum install nodejs -y --setopt=nodesource-nodejs.module_hotfixes=1 && \
-        curl -sL https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo && \
-        yum update -y
-
-# Add Docker
-
-ARG DOCKER_VERSION=25.0.5
-RUN yum install gzip -y && yum install tar -y
-RUN curl -fsSL "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" \
-  | tar -xz -C /usr/local/bin --strip-components=1 docker/docker \
- && docker --version
 ENV DOCKER_CONFIG=/tmp/.docker
 RUN mkdir -p "$DOCKER_CONFIG" && chmod 1777 "$DOCKER_CONFIG"
-# CLI utilities
-RUN yum install -y gcc gcc-c++ git make unzip zip jq
+
+# Remove any preexisting generic/older Node packages, then install Node 22/yarn globally
+RUN dnf remove -y nodejs nodejs18 || true \
+    && dnf install -y nodejs22\
+    && dnf clean all
+
+RUN npm install -g yarn
 
 # Terraform
 RUN \
         curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" -o "terraform.zip" && \
-        unzip terraform.zip && \
+        unzip -q terraform.zip && \
         chmod +x terraform && \
-        mv terraform /usr/local/bin
+        mv terraform /usr/local/bin && \
+        rm -f terraform.zip
 
 # AWS CLI
 RUN \
-        curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-$AWS_CLI_VERSION.zip" -o "awscliv2.zip" && \
-        unzip awscliv2.zip && \
-        ./aws/install
-
-# Node JS
-RUN \
-        yum install -y nodejs yarn
+        curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip" -o "awscliv2.zip" && \
+        unzip -q awscliv2.zip && \
+        ./aws/install && \
+        rm -rf aws awscliv2.zip
 
 # SSM SessionManager plugin
 RUN \
         curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" && \
-        yum install -y session-manager-plugin.rpm
+        dnf install -y ./session-manager-plugin.rpm && \
+        rm -f session-manager-plugin.rpm && \
+        dnf clean all
 
 # Add user for keygen in Makefile
 ARG USER
-RUN \
-        echo "user:x:${USER}:0:root:/:/bin/bash" >> /etc/passwd
+RUN echo "user:x:${USER}:0:root:/:/bin/bash" >> /etc/passwd
 
-RUN python3 -m pip install boto3 setuptools
+RUN python3 -m pip install --no-cache-dir boto3 setuptools
 
-# Uncommenting fixes: `fatal: detected dubious ownership in repository at '/CIRRUS-core'
-# Uncommenting fixes: `fatal: detected dubious ownership in repository at '/CIRRUS-DAAC'
+# Uncommenting fixes:
+#   fatal: detected dubious ownership in repository at '/CIRRUS-core'
+#   fatal: detected dubious ownership in repository at '/CIRRUS-DAAC'
 # COPY .gitconfig /.gitconfig
 
 WORKDIR /CIRRUS-core
 
 # Bypass the bootstrap.sh script that runs in lambda
 ENTRYPOINT []
 
-FROM public.ecr.aws/lambda/python:3.11 AS python3.11
+FROM public.ecr.aws/lambda/python:3.12 AS python3.12
 # This image can be used to do Python 3 & NodeJS development, and
 # includes the AWS CLI and Terraform. It contains:
 
@@ -83,59 +88,48 @@ FROM public.ecr.aws/lambda/python:3.11 AS python3.11
 #   * AWS CLI
 #   * Terraform
 #   * Docker
-
-# Amazon Linux 2 does not support node 18.x or node 20.x glibc=2.27 and >=2.28 is required
-ENV NODE_VERSION="16.x"
 ENV TERRAFORM_VERSION="1.12.2"
 ENV AWS_CLI_VERSION="2.27.43"
 
-# Add NodeJS and Yarn repos & update package index
-RUN \
-        yum install https://rpm.nodesource.com/pub_${NODE_VERSION}/nodistro/repo/nodesource-release-nodistro-1.noarch.rpm -y && \
-        yum install nodejs -y --setopt=nodesource-nodejs.module_hotfixes=1 && \
-        curl -sL https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo && \
-        yum update -y
-
-# CLI utilities
-RUN yum install -y gcc gcc-c++ git make unzip zip jq
-
-# Add Docker
-
-ARG DOCKER_VERSION=25.0.5
-RUN yum install gzip -y && yum install tar -y
-RUN curl -fsSL "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" \
-  | tar -xz -C /usr/local/bin --strip-components=1 docker/docker \
- && docker --version
+# CLI utilities/docker
+RUN dnf install -y gcc gcc-c++ git make unzip zip jq docker
 ENV DOCKER_CONFIG=/tmp/.docker
 RUN mkdir -p "$DOCKER_CONFIG" && chmod 1777 "$DOCKER_CONFIG" 
+
 # Terraform
 RUN \
         curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" -o "terraform.zip" && \
-        unzip terraform.zip && \
+        unzip -q terraform.zip && \
         chmod +x terraform && \
-        mv terraform /usr/local/bin
+        mv terraform /usr/local/bin && \
+        rm -f terraform.zip
 
 # AWS CLI
 RUN \
         curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-$AWS_CLI_VERSION.zip" -o "awscliv2.zip" && \
-        unzip awscliv2.zip && \
-        ./aws/install
+        unzip -q awscliv2.zip && \
+        ./aws/install && \
+        rm -rf aws awscliv2.zip
+
 
 # Node JS
 RUN \
-        yum install -y nodejs yarn
+        dnf remove -y nodejs nodejs18 || true \
+        && dnf install -y nodejs22 pip \
+        && npm install -g yarn
 
 # SSM SessionManager plugin
 RUN \
         curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" && \
-        yum install -y session-manager-plugin.rpm
+        rpm -i session-manager-plugin.rpm && \
+        rm -f session-manager-plugin.rpm
 
 # Add user for keygen in Makefile
 ARG USER
 RUN \
         echo "user:x:${USER}:0:root:/:/bin/bash" >> /etc/passwd
 
-RUN python3 -m pip install boto3 setuptools
+RUN python3 -m pip install --no-cache-dir boto3 setuptools
 
 # Uncommenting fixes: `fatal: detected dubious ownership in repository at '/CIRRUS-core'
 # Uncommenting fixes: `fatal: detected dubious ownership in repository at '/CIRRUS-DAAC'

Makefile

@@ -20,7 +20,7 @@
 #  PYTHON_VER:            python3
 
 # ---------------------------
-DOCKER_TAG := v21.2.0.0
+DOCKER_TAG := v21.3.2.0
 export TF_IN_AUTOMATION="true"
 export TF_VAR_MATURITY=${MATURITY}
 export TF_VAR_DEPLOY_NAME=${DEPLOY_NAME}

cumulus/main.tf

@@ -1,5 +1,5 @@
 module "cumulus" {
-  source = "https://github.com/nasa/cumulus/releases/download/v21.2.0/terraform-aws-cumulus.zip//tf-modules/cumulus"
+  source = "https://github.com/nasa/cumulus/releases/download/v21.3.2/terraform-aws-cumulus.zip//tf-modules/cumulus"
 
   cumulus_message_adapter_lambda_layer_version_arn = data.terraform_remote_state.daac.outputs.cma_layer_arn
 

data-persistence/main.tf

@@ -1,5 +1,5 @@
 module "data_persistence" {
-  source = "https://github.com/nasa/cumulus/releases/download/v21.2.0/terraform-aws-cumulus.zip//tf-modules/data-persistence"
+  source = "https://github.com/nasa/cumulus/releases/download/v21.3.2/terraform-aws-cumulus.zip//tf-modules/data-persistence"
 
   prefix                = local.prefix
   subnet_ids            = data.aws_subnets.subnet_ids.ids