Introduce Wally development containers

.devcontainer/devcontainer.json

@@ -0,0 +1,13 @@
+{
+  "name": "Wally",
+  "image":  "ghcr.io/openhwgroup/wally:latest-devel",
+  "containerEnv": {
+    "WALLY": "${containerWorkspaceFolder}"
+  },
+  "customizations": {
+    "vscode": {
+      "extensions": []
+    }
+  },
+  "postStartCommand": "/etc/wally/container-setup.sh"
+}

.github/workflows/container-release.yml

@@ -0,0 +1,197 @@
+name: Container Release
+on:
+  push:
+    paths:
+      - bin/wally-tool-chain-install.sh
+      - bin/wally-environment-check.sh
+      - bin/wally-package-install.sh
+      - bin/installation/**
+      - containers/wally-devel/**
+      - .github/workflows/container-release.yml # Self-trigger
+  workflow_dispatch:
+    inputs:
+      publish:
+        description: "Publish to the container registry"
+        required: false
+        default: false
+        type: boolean
+
+env:
+  REGISTRY: ghcr.io/doganulus
+  IMAGE_NAME: wally
+  HOSTARCH: amd64
+  PLATFORM: linux-amd64
+  CONTAINER_VERSION: latest
+  CONTAINER_LATEST_VERSION: latest
+  CONTAINERS_ROOT: /home/runner/.local/share/containers
+  TMPDIR: /home/runner/.local/share/containers/tmp
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  buildah-build:
+    name: Build container images
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04, ubuntu-24.04-arm]
+    runs-on: ${{ matrix.os }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Install container tools
+        run: sudo apt-get install podman buildah jq
+
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@v10
+        with:
+          root-reserve-mb: 2048 # Reserve disk space for repository
+          remove-dotnet: "true"
+          remove-android: "true"
+          remove-haskell: "true"
+          remove-codeql: "true"
+          remove-docker-images: "true"
+          build-mount-path: ${{ env.CONTAINERS_ROOT }} # The remaining space only for container build
+
+      - run: mkdir -p $TMPDIR
+
+      - name: Prepare environment variables
+        run: |
+          echo "HOSTARCH=$(podman info --format='{{.Host.Arch}}')" >> $GITHUB_ENV
+          echo "PLATFORM=$(podman info --format='{{.Version.OsArch}}' | sed 's/\//-/g')" >> $GITHUB_ENV
+          echo "CONTAINER_VERSION=$(date +'%Y%m%d')" >> $GITHUB_ENV
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the GitHub Container registry
+        uses: redhat-actions/podman-login@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build builder container image
+        id: build-builder
+        uses: redhat-actions/buildah-build@v2
+        with:
+          context: .
+          image: wally
+          tags: |
+            ${{ env.CONTAINER_VERSION }}-builder
+            ${{ env.CONTAINER_VERSION }}-builder-${{ env.HOSTARCH }}
+          layers: true
+          oci: true
+          build-args: |
+            CONTAINER_VERSION=${{ env.CONTAINER_VERSION }}
+            CONTAINER_IMAGE_REGISTRY_REMOTE=${{ env.REGISTRY }}
+          extra-args: |
+            --target wally-builder
+          containerfiles: |
+            ./containers/wally-devel/Dockerfile
+
+      - name: Build devel container image
+        id: build-devel
+        uses: redhat-actions/buildah-build@v2
+        with:
+          context: .
+          image: wally
+          tags: ${{ env.CONTAINER_VERSION }}-devel ${{ env.CONTAINER_VERSION }}-devel-${{ env.HOSTARCH }}
+          layers: true
+          oci: true
+          build-args: |
+            CONTAINER_VERSION=${{ env.CONTAINER_VERSION }}
+            CONTAINER_IMAGE_REGISTRY_REMOTE=${{ env.REGISTRY }}
+          extra-args: |
+            --target wally-devel
+          containerfiles: |
+            ./containers/wally-devel/Dockerfile
+
+      - name: Push to GitHub Container Repository
+        if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish
+        id: push-builder-ghcr
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          image: ${{ steps.build-builder.outputs.image }}
+          tags: ${{ env.CONTAINER_VERSION }}-builder-${{ env.HOSTARCH }}
+          digestfile: ${{ runner.temp }}/digest-wally-builder-${{ env.CONTAINER_VERSION }}-${{ env.PLATFORM }}
+
+      - name: Push to GitHub Container Repository
+        if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish
+        id: push-devel-ghcr
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          image: ${{ steps.build-devel.outputs.image }}
+          tags: ${{ env.CONTAINER_VERSION }}-devel-${{ env.HOSTARCH }}
+          digestfile: ${{ runner.temp }}/digest-wally-devel-${{ env.CONTAINER_VERSION }}-${{ env.PLATFORM }}
+
+      - name: Upload digests
+        if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish
+        uses: actions/upload-artifact@v4
+        with:
+          name: digest-wally-${{ env.CONTAINER_VERSION }}-${{ env.PLATFORM }}
+          path: ${{ runner.temp }}/digest-*
+          if-no-files-found: error
+          retention-days: 1
+          compression-level: 0 # no compression
+
+  buildah-merge:
+    name: Merge container images
+    runs-on: ubuntu-24.04
+    needs: buildah-build
+    if: always() && github.event_name == 'workflow_dispatch' && github.event.inputs.publish
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digest-*
+          merge-multiple: true
+
+      - name: Log in to the GitHub Container registry
+        uses: redhat-actions/podman-login@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Prepare environment variables
+        run: |
+          if [ "${{ env.CONTAINER_VERSION }}" == "latest" ]; then
+            echo "CONTAINER_VERSION=$(date +'%Y%m%d')" >> $GITHUB_ENV
+          fi
+          echo "CONTAINER_LATEST_VERSION=$(date +'%Y%m%d')" >> $GITHUB_ENV
+
+      - name: Create and push manifest list for wally-builder
+        run: |
+          MANIFEST=wally-builder
+          FULL_IMAGE_NAME=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          buildah manifest create $MANIFEST
+          for digest in ${{ runner.temp }}/digests/digest-wally-builder-*; do
+            echo "Adding $(cat $digest)"
+            buildah manifest add $MANIFEST $FULL_IMAGE_NAME@$(cat $digest)
+          done
+          buildah manifest push --all $MANIFEST docker://$FULL_IMAGE_NAME:${{ env.CONTAINER_VERSION }}-builder
+          if [ "${{ env.CONTAINER_VERSION }}" == "${{ env.CONTAINER_LATEST_VERSION }}" ]; then
+            buildah manifest push --all $MANIFEST docker://$FULL_IMAGE_NAME:latest-builder
+          fi
+
+      - name: Create and push manifest list for wally-devel
+        run: |
+          MANIFEST=wally-devel
+          FULL_IMAGE_NAME=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          buildah manifest create $MANIFEST
+          for digest in ${{ runner.temp }}/digests/digest-wally-devel-*; do
+            echo "Adding $(cat $digest)"
+            buildah manifest add $MANIFEST $FULL_IMAGE_NAME@$(cat $digest)
+          done
+          buildah manifest push --all $MANIFEST docker://$FULL_IMAGE_NAME:${{ env.CONTAINER_VERSION }}-devel
+          if [ "${{ env.CONTAINER_VERSION }}" == "${{ env.CONTAINER_LATEST_VERSION }}" ]; then
+            buildah manifest push --all $MANIFEST docker://$FULL_IMAGE_NAME:latest-devel
+          fi

.github/workflows/container-retention.yml

@@ -0,0 +1,24 @@
+name: Container Retention
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 12 * * 0"
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  clean:
+    runs-on: ubuntu-latest
+    name: Github Container Registry Retention Policy
+    steps:
+      - name: Clean up untagged images
+        uses: snok/[email protected]
+        with:
+          account: openhwgroup
+          token: ${{ secrets.GITHUB_TOKEN }}
+          image-names: "wally"
+          tag-selection: untagged
+          cut-off: 1h
+          dry-run: false

bin/wally-environment-check.sh

@@ -160,12 +160,14 @@ while [[ "$#" -gt 0 ]]; do
     case $1 in
         -c|--clean) clean=true ;;
         --no-buildroot) no_buildroot=true ;;
+        --no-skywater) no_skywater=true ;;
         --packages-only) packages_only=true;;
         -h|--help)
             echo -e "Usage: $0 [\$RISCV] [options]"
             echo -e "${BOLD}Options:${ENDC}"
             echo -e "  -c, --clean          Remove build directories after installation"
             echo -e "  --no-buildroot       Skip installing Buildroot and Linux testvectors"
+            echo -e "  --no-skywater        Skip installing Skywater PDK"
             echo -e "  \$RISCV              Directory to install RISC-V tools (default: /opt/riscv as root, $HOME/riscv otherwise)"
             exit 0 ;;
         *) RISCV="$1" ;;

bin/wally-tool-chain-install.sh

@@ -107,8 +107,11 @@ source "$WALLY"/bin/installation/verilator-install.sh
 
 # OSU Skywater 130 cell library (https://foss-eda-tools.googlesource.com/skywater-pdk/libs/sky130_osu_sc_t12)
 # The OSU Skywater 130 cell library is a standard cell library that is used to synthesize Wally.
-source "$WALLY"/bin/installation/skywater-lib-install.sh
-
+if [ "$no_skywater" = true ]; then
+    echo -e "${OK_COLOR}Skipping OSU Skywater 130 cell library installation.${ENDC}"
+else
+    source "$WALLY"/bin/installation/skywater-lib-install.sh
+fi
 
 # Buildroot and Linux testvectors
 # Buildroot is used to boot a minimal version of Linux on Wally.

containers/Makefile

@@ -0,0 +1,40 @@
+WALLY_CONTAINER_BASE_IMAGE ?= docker.io/almalinux/9-base:9.6
+WALLY_CONTAINER_IMAGE_REGISTRY ?= localhost
+WALLY_CONTAINER_IMAGE_REGISTRY_REMOTE ?= ghcr.io/openhwgroup
+WALLY_CONTAINER_IMAGE_NAME ?= wally
+WALLY_CONTAINER_TARGET_VERSION ?= latest
+WALLY_CONTAINER_CURRENT_VERSION ?= $(shell date +'%Y%m%d')
+
+WALLY_CONTAINER_BUILD_CONTEXT ?= .. # Set to the project root
+
+info:
+	@echo "Container version: ${WALLY_CONTAINER_TARGET_VERSION}"
+	@echo "Container current version: ${WALLY_CONTAINER_CURRENT_VERSION}"
+
+builder:
+	buildah build \
+		-f wally-devel/Dockerfile \
+		--build-arg WALLY_CONTAINER_BASE_IMAGE=${WALLY_CONTAINER_BASE_IMAGE} \
+		--build-arg WALLY_CONTAINER_VERSION=${WALLY_CONTAINER_CURRENT_VERSION} \
+		--format oci \
+		--layers=true \
+		--target wally-builder \
+		--tag ${WALLY_CONTAINER_IMAGE_REGISTRY}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_TARGET_VERSION}-builder \
+		--tag ${WALLY_CONTAINER_IMAGE_REGISTRY}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_CURRENT_VERSION}-builder \
+		--tag ${WALLY_CONTAINER_IMAGE_REGISTRY_REMOTE}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_TARGET_VERSION}-builder \
+		--tag ${WALLY_CONTAINER_IMAGE_REGISTRY_REMOTE}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_CURRENT_VERSION}-builder \
+	${WALLY_CONTAINER_BUILD_CONTEXT}
+
+devel: builder
+	buildah build \
+		-f wally-devel/Dockerfile \
+		--build-arg WALLY_CONTAINER_BASE_IMAGE=${WALLY_CONTAINER_BASE_IMAGE} \
+		--build-arg WALLY_CONTAINER_VERSION=${WALLY_CONTAINER_CURRENT_VERSION} \
+		--format oci \
+		--layers=true \
+		--target wally-devel \
+		--tag ${WALLY_CONTAINER_IMAGE_REGISTRY}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_TARGET_VERSION}-devel \
+		--tag ${WALLY_CONTAINER_IMAGE_REGISTRY}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_CURRENT_VERSION}-devel \
+		--tag ${WALLY_CONTAINER_IMAGE_REGISTRY_REMOTE}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_TARGET_VERSION}-devel \
+		--tag ${WALLY_CONTAINER_IMAGE_REGISTRY_REMOTE}/${WALLY_CONTAINER_IMAGE_NAME}:${WALLY_CONTAINER_CURRENT_VERSION}-devel \
+	${WALLY_CONTAINER_BUILD_CONTEXT}

containers/wally-devel/Dockerfile

@@ -0,0 +1,41 @@
+ARG WALLY_CONTAINER_VERSION=latest
+ARG WALLY_CONTAINER_BASE_IMAGE=docker.io/almalinux/9-base:9.6
+
+ARG WALLY_CONTAINER_BUILDER_IMAGE=wally-builder
+ARG WALLY_CONTAINER_DEVEL_IMAGE=wally-devel
+
+ARG RISCV_GNU_TOOLCHAIN_VERSION=23863c2ca74e6c050f0c97e7af61f5f1776aadd1
+
+FROM ${WALLY_CONTAINER_BASE_IMAGE} as wally-builder
+ARG TARGETARCH TARGETOS TARGETPLATFORM TARGETVARIANT
+
+ARG RISCV_GNU_TOOLCHAIN_VERSION
+ENV RISCV_GNU_TOOLCHAIN_VERSION=${RISCV_GNU_TOOLCHAIN_VERSION}
+ENV RISCV_INSTALL_PREFIX=/opt/riscv
+
+ENV WALLY_HOME=/root
+ENV WALLY_SOURCE_DIR=/workspaces/cvw
+ENV WALLY_PYTHON_VENV_DIR=${WALLY_HOME}/.venv/wally
+ENV WALLY_PYTHON_EXECUTABLE=${WALLY_PYTHON_VENV_DIR}/bin/python
+
+ENV RISCV=${RISCV_INSTALL_PREFIX}
+ENV WALLY=${WALLY_SOURCE_DIR}
+
+RUN --mount=type=bind,source=./bin,target=${WALLY_SOURCE_DIR}/bin \
+    bash "${WALLY_SOURCE_DIR}/bin/wally-package-install.sh" --clean
+
+RUN --mount=type=bind,source=./bin,target=${WALLY_SOURCE_DIR}/bin \
+    bash "${WALLY_SOURCE_DIR}/bin/installation/riscv-gnu-toolchain-install.sh" --clean
+
+RUN --mount=type=bind,source=./bin,target=${WALLY_SOURCE_DIR}/bin \
+    --mount=type=bind,source=./linux,target=${WALLY_SOURCE_DIR}/linux \
+    bash "${WALLY_SOURCE_DIR}/bin/wally-tool-chain-install.sh" --clean --no-buildroot --no-skywater
+
+COPY --chmod=755 ./containers/wally-devel/container-setup.sh /etc/wally/container-setup.sh
+
+ENV WALLY_CORE_DUMP_SIZE=300000
+ENV WALLY_PYTHON_VENV_DIR=${RISCV_INSTALL_PREFIX}/riscv-python
+ENV WALLY_PYTHON_EXECUTABLE=${WALLY_PYTHON_VENV_DIR}/bin/python
+
+FROM ${WALLY_CONTAINER_BUILDER_IMAGE} as wally-devel
+ARG TARGETARCH TARGETOS TARGETPLATFORM TARGETVARIANT

containers/wally-devel/container-setup.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+# Append PATH and source command to ~/.bashrc
+echo 'export PATH="${WALLY}/bin:${RISCV}/bin:$PATH"' >> "$HOME/.bashrc"
+echo 'export CVW_ARCH_VERIF="${WALLY}/addins/cvw-arch-verif"' >> "$HOME/.bashrc"
+
+echo 'source ${WALLY_PYTHON_VENV_DIR}/bin/activate' >> "$HOME/.bashrc"
+
+# Set core dump size (default 300000 KB if not provided)
+ulimit -c "${WALLY_CORE_DUMP_SIZE:=300000}"
+
+# Install pre-commit hook if missing
+if [ ! -e "$WALLY/.git/hooks/pre-commit" ]; then
+    pushd "$WALLY" > /dev/null || exit 1
+    echo "Installing pre-commit hooks"
+    pre-commit install
+    popd > /dev/null || exit 1
+fi