diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 44eb0348a..46321924f 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -61,7 +61,7 @@ jobs: with: path: | vcpkg-archives/ - key: vcpkg-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles('shared/GdalCore.opt', 'osx/vcpkg-makefile') }} + key: vcpkg-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles('shared/GdalCore.opt', 'osx/vcpkg-makefile', 'shared/vcpkg.json', 'shared/vcpkg-configuration.json', 'shared/vcpkg-lock.json') }} restore-keys: | vcpkg-${{ matrix.os }}-${{ matrix.arch }}- @@ -76,10 +76,11 @@ jobs: build-osx/gdal-build/ build-osx/vcpkg/installed/ build-osx/.build-state.json + build-osx/.build-state-vcpkg-*.txt build-osx/.build-times.txt - key: builds-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles('shared/GdalCore.opt', 'osx/gdal-makefile', 'shared/common.mk') }} + key: builds-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles('shared/GdalCore.opt', 'osx/gdal-makefile', 'shared/common.mk', 'osx/vcpkg-makefile', 'shared/vcpkg.json', 'shared/vcpkg-configuration.json', 'shared/vcpkg-lock.json') }} restore-keys: | - builds-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles('shared/GdalCore.opt') }}- + builds-${{ matrix.os }}-${{ matrix.arch }}- # Layer 3: .NET SDK packages - name: Cache .NET packages diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ed2140f8d..1494e1a74 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -118,8 +118,39 @@ jobs: - uses: actions/download-artifact@v6.0.0 with: - pattern: packages-* + name: packages-core path: nuget + merge-multiple: true + + - uses: actions/download-artifact@v6.0.0 + with: + name: packages-unix-x64 + path: nuget + merge-multiple: true + + - uses: actions/download-artifact@v6.0.0 + with: + name: packages-unix-arm64 + path: nuget + merge-multiple: true + + - uses: actions/download-artifact@v6.0.0 + with: + name: packages-win-x64 + path: nuget + merge-multiple: true + + - uses: actions/download-artifact@v6.0.0 + with: + name: packages-osx-x64 + path: nuget + merge-multiple: true + + - uses: actions/download-artifact@v6.0.0 + with: + name: packages-osx-arm64 + path: nuget + merge-multiple: true - name: Remove old formats run: | diff --git a/.github/workflows/unix.yml b/.github/workflows/unix.yml index 39d22543a..aa0ec794e 100644 --- a/.github/workflows/unix.yml +++ b/.github/workflows/unix.yml @@ -56,21 +56,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.11.1 - - name: Cache restore - uses: actions/cache@v4.3.0 - with: - path: | - .dotnet/ - ci/cache/ - key: ${{ matrix.os }}-buildx-${{ matrix.arch }}-${{ github.run_id }} - restore-keys: | - ${{ matrix.os }}-buildx-${{ matrix.arch }}- - - - name: Ensure cache directories - run: | - mkdir -p ci/cache/.dotnet - mkdir -p ci/cache/vcpkg-archives - - name: Log in to the Container registry uses: docker/login-action@v3 with: @@ -105,8 +90,8 @@ jobs: DOTNET_VERSION=${{ inputs.dotnet-version }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha,scope=gdalnetcore-${{ matrix.arch }} - cache-to: type=gha,scope=gdalnetcore-${{ matrix.arch }},mode=max + cache-from: type=gha,scope=gdalnetcore-${{ matrix.arch }}-${{ hashFiles('shared/GdalCore.opt', 'unix/vcpkg-makefile', 'shared/vcpkg.json', 'shared/vcpkg-configuration.json', 'shared/vcpkg-lock.json') }} + cache-to: type=gha,scope=gdalnetcore-${{ matrix.arch }}-${{ hashFiles('shared/GdalCore.opt', 'unix/vcpkg-makefile', 'shared/vcpkg.json', 'shared/vcpkg-configuration.json', 'shared/vcpkg-lock.json') }},mode=max - name: Build and push on local runner uses: docker/build-push-action@v5 @@ -124,6 +109,8 @@ jobs: DOTNET_VERSION=${{ inputs.dotnet-version }} DOTNET_INSTALL_DIR=/build/ci/cache/.dotnet VCPKG_DEFAULT_BINARY_CACHE=/build/ci/cache/vcpkg-archives/ + cache-from: type=gha,scope=gdalnetcore-${{ matrix.arch }}-${{ hashFiles('shared/GdalCore.opt', 'unix/vcpkg-makefile', 'shared/vcpkg.json', 'shared/vcpkg-configuration.json', 'shared/vcpkg-lock.json') }} + cache-to: type=gha,scope=gdalnetcore-${{ matrix.arch }}-${{ hashFiles('shared/GdalCore.opt', 'unix/vcpkg-makefile', 'shared/vcpkg.json', 'shared/vcpkg-configuration.json', 'shared/vcpkg-lock.json') }},mode=max - name: Extract artifacts run: | @@ -311,4 +298,4 @@ jobs: if: ${{ github.event.pull_request.merged == true || github.ref == 'refs/heads/main' || fromJson(inputs.is-version-branch) }} run: | make -f push-packages-makefile PRERELEASE=${{ inputs.is-pre-release }} INCLUDE_CORE=1 \ - BUILD_NUMBER_TAIL=${{ github.run_number }} API_KEY_GITHUB=${{ secrets.API_KEY_GITHUB }} API_KEY_NUGET=${{ secrets.API_KEY_NUGET }} \ No newline at end of file + BUILD_NUMBER_TAIL=${{ github.run_number }} API_KEY_GITHUB=${{ secrets.API_KEY_GITHUB }} API_KEY_NUGET=${{ secrets.API_KEY_NUGET }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index fcb3f4d24..ea665d275 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -52,25 +52,31 @@ jobs: with: path: | vcpkg-archives/ - key: vcpkg-${{ runner.os }}-${{ hashFiles('shared/GdalCore.opt', 'win/install.ps1') }} + key: vcpkg-${{ runner.os }}-${{ hashFiles('shared/GdalCore.opt', 'win/install.ps1', 'win/vcpkg-makefile.vc', 'shared/vcpkg.json', 'shared/vcpkg-configuration.json', 'shared/vcpkg-lock.json') }} restore-keys: | vcpkg-${{ runner.os }}- # Layer 2: Build outputs (PROJ, GDAL, VCPKG installed, SDK) - # Windows caches entire build-win/ because install.ps1 manages all - # subdirectories internally (gdal-build/, proj-build/, vcpkg/, sdk/, downloads/) - # Note: Windows install.ps1 does not use build state tracking. - # Cache hit avoids re-downloading but does not skip compilation. - # Build state integration for Windows is deferred to Phase 2. + # Keep this cache narrow enough that the separate VCPKG archive cache + # is less likely to be evicted. Source clones and downloads are rebuilt + # on demand, but the installed outputs/state stamps stay reusable. - name: Cache build outputs id: cache-builds uses: actions/cache@v4 with: path: | - build-win/ - key: builds-${{ runner.os }}-${{ hashFiles('shared/GdalCore.opt', 'win/install.ps1') }} + build-win/.build-state-*.txt + build-win/gdal-build/ + build-win/proj-build/ + build-win/gdal-cmake-temp/swig/csharp/ + build-win/sdk/release-1930-x64/ + build-win/vcpkg/ + !build-win/vcpkg/buildtrees/ + !build-win/vcpkg/downloads/ + !build-win/vcpkg/packages/ + key: builds-${{ runner.os }}-${{ hashFiles('shared/GdalCore.opt', 'win/install.ps1', 'win/functions.psm1', 'win/partials.psm1', 'win/vcpkg-makefile.vc', 'shared/vcpkg.json', 'shared/vcpkg-configuration.json', 'shared/vcpkg-lock.json') }} restore-keys: | - builds-${{ runner.os }}-${{ hashFiles('shared/GdalCore.opt') }}- + builds-${{ runner.os }}- # Layer 3: .NET SDK packages - name: Cache .NET packages @@ -91,7 +97,18 @@ jobs: id: compile-source run: | git config --system core.longpaths true - ./install.ps1 -buildNumberTail ${{ github.run_number }} -preRelease $${{ inputs.is-pre-release }} + $maxAttempts = 5 + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { + ./install.ps1 -buildNumberTail ${{ github.run_number }} -preRelease $${{ inputs.is-pre-release }} + if ($LASTEXITCODE -eq 0) { + break + } + if ($attempt -ge $maxAttempts) { + exit $LASTEXITCODE + } + Write-Host "Retrying Windows package build after a transient failure..." + Start-Sleep -Seconds 15 + } echo "GDAL_VERSION=$env:GDAL_VERSION" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - name: Store metadata as artifact diff --git a/ci/Dockerfile.unix b/ci/Dockerfile.unix index 6f00d639c..1c58adc66 100644 --- a/ci/Dockerfile.unix +++ b/ci/Dockerfile.unix @@ -30,7 +30,13 @@ RUN echo 'deb http://deb.debian.org/debian bookworm main' > /etc/apt/sources.lis # Install newer autoconf (2.71) from source since Debian 11 has older version RUN cd /tmp && \ - wget https://ftp.gnu.org/gnu/autoconf/autoconf-2.71.tar.gz && \ + for url in \ + https://ftp.gnu.org/gnu/autoconf/autoconf-2.71.tar.gz \ + https://mirrors.kernel.org/gnu/autoconf/autoconf-2.71.tar.gz; do \ + if curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors "$url" -o autoconf-2.71.tar.gz; then \ + break; \ + fi; \ + done && \ tar -xzf autoconf-2.71.tar.gz && \ cd autoconf-2.71 && \ ./configure --prefix=/usr/local && \ @@ -85,6 +91,9 @@ ARG VCPKG_DEFAULT_BINARY_CACHE=/build/ci/cache/vcpkg-archives/ RUN mkdir -p $VCPKG_DEFAULT_BINARY_CACHE COPY --from=base /tmp/gdal-netcore-env /tmp/gdal-netcore-env COPY --from=base /tmp/gdal-netcore-arch /tmp/gdal-netcore-arch +COPY shared/vcpkg.json /build/shared/vcpkg.json +COPY shared/vcpkg-configuration.json /build/shared/vcpkg-configuration.json +COPY shared/vcpkg-lock.json /build/shared/vcpkg-lock.json COPY shared/GdalCore.opt /build/shared/ COPY unix/RID.opt /build/unix/RID.opt COPY unix/vcpkg-makefile /build/unix/ @@ -105,6 +114,17 @@ RUN chmod +x /build/ci/vcpkg-error-handler.sh RUN --mount=type=cache,target=/build/ci/cache/vcpkg-archives \ --mount=type=cache,target=/build/build-unix/vcpkg/buildtrees \ --mount=type=cache,target=/build/build-unix/vcpkg/downloads \ + mkdir -p /build/build-unix/vcpkg/downloads && \ + if [ ! -f /build/build-unix/vcpkg/downloads/thrift-0.22.0.tar.gz ]; then \ + for url in \ + https://downloads.apache.org/thrift/0.22.0/thrift-0.22.0.tar.gz \ + https://dlcdn.apache.org/thrift/0.22.0/thrift-0.22.0.tar.gz \ + https://archive.apache.org/dist/thrift/0.22.0/thrift-0.22.0.tar.gz; do \ + if curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors "$url" -o /build/build-unix/vcpkg/downloads/thrift-0.22.0.tar.gz; then \ + break; \ + fi; \ + done; \ + fi && \ set -a && . /tmp/gdal-netcore-env && set +a; \ make -f vcpkg-makefile $(cat /tmp/gdal-netcore-arch) || { /build/ci/vcpkg-error-handler.sh /build/build-unix /build/vcpkg-error-artifacts; exit 1; } @@ -171,4 +191,4 @@ COPY --from=package-stage /build/tests/gdal-formats /build/tests/gdal-formats COPY --from=package-stage /build/nuget /build/nuget # copy error artifacts if they exist COPY --from=package-stage /build/vcpkg-error-artifacts /build/vcpkg-error-artifacts -ENTRYPOINT ["bash"] \ No newline at end of file +ENTRYPOINT ["bash"] diff --git a/osx/before-install.sh b/osx/before-install.sh index 46bf3be70..e62c2ff10 100755 --- a/osx/before-install.sh +++ b/osx/before-install.sh @@ -10,4 +10,13 @@ brew install make pkg-config autoconf automake \ brew install --ignore-dependencies pipx || python3 -m pip install --user pipx # issue with libtool on macOS https://github.com/Homebrew/homebrew-core/issues/180040 -brew reinstall libtool \ No newline at end of file +brew reinstall libtool + +if [ -n "${GITHUB_PATH:-}" ]; then + for formula in make libtool; do + prefix=$(brew --prefix "$formula" 2>/dev/null || true) + if [ -n "$prefix" ] && [ -d "$prefix/libexec/gnubin" ]; then + echo "$prefix/libexec/gnubin" >> "$GITHUB_PATH" + fi + done +fi diff --git a/osx/gdal-makefile b/osx/gdal-makefile index 5c01306f0..b656f7ad2 100644 --- a/osx/gdal-makefile +++ b/osx/gdal-makefile @@ -11,8 +11,7 @@ TARGETS = hdf proj gdal all: $(TARGETS) @echo "$(LOG_PREFIX) Everything looks good. Linux libraries for GDAL is ready to packaging!" - @echo "$(LOG_PREFIX) Installed libraries (vcpkg static): $(VCPKG_REQUIRE_OSX)" - @echo "$(LOG_PREFIX) Installed libraries (vcpkg dynamic): $(VCPKG_REQUIRE_OSX_DYNAMIC)" + @echo "$(LOG_PREFIX) Installed libraries (vcpkg): shared manifest features" @echo "$(LOG_PREFIX) Compiled libraries: $(TARGETS)" pre_vcpkg: @@ -34,7 +33,7 @@ configure_hdf: @if [[ -d "$(BUILD_ROOT)/hdf-build" ]]; then rm -r "$(BUILD_ROOT)/hdf-build"; fi; -mkdir -p $(HDF_CMAKE_TMP) - @cd $(HDF_CMAKE_TMP) && cmake $(HDF_SOURCE) \ + @cd $(HDF_CMAKE_TMP) && cmake $(HDF_ROOT) \ -DCMAKE_INSTALL_PREFIX=$(BUILD_ROOT)/hdf-build -Wno-dev \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_FLAGS="-fPIC -w" \ diff --git a/osx/vcpkg-makefile b/osx/vcpkg-makefile index e98333bc1..db2b7ba24 100644 --- a/osx/vcpkg-makefile +++ b/osx/vcpkg-makefile @@ -21,34 +21,9 @@ all: install_vcpkg install_requirements force: $(addsuffix -force, $(TARGETS)) @exit 0 -# accepts any params to install/uninstall package +# Manifest mode owns package inventory in shared/vcpkg.json. % : -ifneq ($(filter $(TARGET_CLEAN),$(VCPKG_REQUIRE_OSX)),'') - @echo "'vcpkg make' > trying to make stuff for => $(TARGET_CLEAN)" - @if [[ "$(TARGET_LOWER)" == *"-force"* ]] ; then \ - $(MAKE) -f vcpkg-makefile upgrade; \ - $(VCPKG) remove $(TARGET_CLEAN):$(VCPKG_RID) --recurse || exit 1; \ - $(VCPKG) install $(TARGET_CLEAN):$(VCPKG_RID) --recurse --clean-after-build || { \ - echo "$(TARGET_PREFIX) ERROR: Failed to install $(TARGET_CLEAN):$(VCPKG_RID)" >&2; \ - if [ -f $(ROOT_DIR)/ci/vcpkg-error-handler.sh ]; then \ - chmod +x $(ROOT_DIR)/ci/vcpkg-error-handler.sh; \ - $(ROOT_DIR)/ci/vcpkg-error-handler.sh $(BUILD_ROOT) $(ROOT_DIR)/vcpkg-error-artifacts; \ - else \ - echo "$(TARGET_PREFIX) Searching for issue_body.md for $(TARGET_CLEAN)..." >&2; \ - find $(VCPKG_ROOT)/installed -path "*/$(TARGET_CLEAN)*/issue_body.md" -type f 2>/dev/null | while read -r issue_file; do \ - echo "$(TARGET_PREFIX) Found issue file: $$issue_file" >&2; \ - cat "$$issue_file" >&2; \ - done; \ - fi; \ - exit 1; \ - }; \ - else \ - echo "To rebuild add {key}-force"; \ - $(VCPKG) list; \ - fi; -else - @echo "Can not make $(TARGET_CLEAN)" -endif + @echo "$(TARGET_PREFIX) Manifest mode owns package inventory; use 'make -f vcpkg-makefile' to install declared features." pull: @if [ ! -d "$(VCPKG_ROOT)/.git" ]; then \ @@ -115,55 +90,85 @@ endef export VCPKG_TRIPLET_OSX_DYNAMIC_RELEASE export VCPKG_TRIPLET_OSX_STATIC_RELEASE +VCPKG_MANIFEST_ARGS=--x-manifest-root=$(VCPKG_MANIFEST_ROOT) --x-install-root=$(VCPKG_ROOT)/installed $(VCPKG_TRIPLETS_OVERLAY) --no-print-usage +VCPKG_MANIFEST_LOCK=$(VCPKG_MANIFEST_ROOT)/vcpkg-lock.json +VCPKG_MANIFEST_LOCK_INSTALLED=$(VCPKG_ROOT)/installed/vcpkg/vcpkg-lock.json +VCPKG_STATE_STAMP=$(BUILD_ROOT)/.build-state-vcpkg-$(BUILD_ARCH).txt + patch_vcpkg_triplets: @(cd $(VCPKG_ROOT) && mkdir -p custom-triplets) @echo "$$VCPKG_TRIPLET_OSX_DYNAMIC_RELEASE" > $(VCPKG_ROOT)/custom-triplets/$(VCPKG_RID)-dynamic.cmake @echo "$$VCPKG_TRIPLET_OSX_STATIC_RELEASE" > $(VCPKG_ROOT)/custom-triplets/$(VCPKG_RID).cmake -install_packages: - @for pack in $(VCPKG_REQUIRE_OSX); do \ - echo "$(TARGET_PREFIX) Installing $$pack:$(VCPKG_RID)..."; \ - MAKELEVEL=0 $(VCPKG) install $$pack:$(VCPKG_RID) $(VCPKG_PARAMS_INSTALL) $(VCPKG_ENSURE_INSTALLED) || \ - { \ - echo "$(TARGET_PREFIX) ERROR: Failed to install $$pack:$(VCPKG_RID)" >&2; \ - if [ -f $(ROOT_DIR)/ci/vcpkg-error-handler.sh ]; then \ - chmod +x $(ROOT_DIR)/ci/vcpkg-error-handler.sh; \ - $(ROOT_DIR)/ci/vcpkg-error-handler.sh $(BUILD_ROOT) $(ROOT_DIR)/vcpkg-error-artifacts; \ - else \ - echo "$(TARGET_PREFIX) Searching for issue_body.md for $$pack..." >&2; \ - find $(VCPKG_ROOT)/installed -path "*/$$pack*/issue_body.md" -type f 2>/dev/null | while read -r issue_file; do \ - echo "$(TARGET_PREFIX) Found issue file: $$issue_file" >&2; \ - cat "$$issue_file" >&2; \ - done; \ - echo "$(TARGET_PREFIX) Checking build logs..." >&2; \ - cat $(VCPKG_ROOT)/buildtrees/$$pack/*err.log 2>/dev/null || true; \ - cat $(VCPKG_ROOT)/buildtrees/$$pack/*out.log 2>/dev/null || true; \ - fi; \ - exit 1; \ - }; \ - done - @for pack in $(VCPKG_REQUIRE_OSX_DYNAMIC); do \ - echo "$(TARGET_PREFIX) Installing $$pack:$(VCPKG_RID)-dynamic..."; \ - MAKELEVEL=0 $(VCPKG) install $$pack:$(VCPKG_RID)-dynamic $(VCPKG_PARAMS_INSTALL) $(VCPKG_ENSURE_INSTALLED) || \ - { \ - echo "$(TARGET_PREFIX) ERROR: Failed to install $$pack:$(VCPKG_RID)-dynamic" >&2; \ - if [ -f $(ROOT_DIR)/ci/vcpkg-error-handler.sh ]; then \ - chmod +x $(ROOT_DIR)/ci/vcpkg-error-handler.sh; \ - $(ROOT_DIR)/ci/vcpkg-error-handler.sh $(BUILD_ROOT) $(ROOT_DIR)/vcpkg-error-artifacts; \ - else \ - echo "$(TARGET_PREFIX) Searching for issue_body.md for $$pack..." >&2; \ - find $(VCPKG_ROOT)/installed -path "*/$$pack*/issue_body.md" -type f 2>/dev/null | while read -r issue_file; do \ - echo "$(TARGET_PREFIX) Found issue file: $$issue_file" >&2; \ - cat "$$issue_file" >&2; \ - done; \ - echo "$(TARGET_PREFIX) Checking build logs..." >&2; \ - cat $(VCPKG_ROOT)/buildtrees/$$pack/*err.log 2>/dev/null || true; \ - cat $(VCPKG_ROOT)/buildtrees/$$pack/*out.log 2>/dev/null || true; \ +sync_manifest_lock_to_install_root: + @mkdir -p $(VCPKG_ROOT)/installed/vcpkg + @if [ -f "$(VCPKG_MANIFEST_LOCK)" ]; then \ + cp "$(VCPKG_MANIFEST_LOCK)" "$(VCPKG_MANIFEST_LOCK_INSTALLED)"; \ + fi + +sync_manifest_lock_from_install_root: + @if [ -f "$(VCPKG_MANIFEST_LOCK_INSTALLED)" ]; then \ + cmp -s "$(VCPKG_MANIFEST_LOCK_INSTALLED)" "$(VCPKG_MANIFEST_LOCK)" || cp "$(VCPKG_MANIFEST_LOCK_INSTALLED)" "$(VCPKG_MANIFEST_LOCK)"; \ + fi + +install_packages: sync_manifest_lock_to_install_root + @current_signature="$$( { \ + printf '%s\n' \ + 'component=vcpkg' \ + 'triplet=$(VCPKG_RID)' \ + 'vcpkg=$(VCPKG_COMMIT_VER)' \ + "build_arch=$(BUILD_ARCH)" \ + "xcode=$$(xcodebuild -version 2>/dev/null | tr '\n' ' ')" \ + "clang=$$(clang --version 2>/dev/null | head -n 1)"; \ + shasum -a 256 \ + '$(ROOT_DIR)/shared/GdalCore.opt' \ + '$(ROOT_DIR)/osx/vcpkg-makefile' \ + '$(ROOT_DIR)/shared/vcpkg.json' \ + '$(ROOT_DIR)/shared/vcpkg-configuration.json' \ + '$(ROOT_DIR)/shared/vcpkg-lock.json'; \ + } | shasum -a 256 | awk '{print $$1}' )"; \ + if [ -f '$(VCPKG_STATE_STAMP)' ] && \ + [ -d '$(VCPKG_ROOT)/installed/$(VCPKG_RID)' ] && \ + [ -d '$(VCPKG_ROOT)/installed/$(VCPKG_RID)-dynamic' ] && \ + [ "$$(cat '$(VCPKG_STATE_STAMP)')" = "$$current_signature" ]; then \ + echo "$(TARGET_PREFIX) Cached manifest outputs for $(BUILD_ARCH) are up to date, skipping install."; \ + $(MAKE) -f vcpkg-makefile sync_manifest_lock_from_install_root; \ + exit 0; \ + fi; \ + sanitize_env='env -u PKG_CONFIG_PATH -u CMAKE_PREFIX_PATH -u CPATH -u LIBRARY_PATH -u DYLD_FALLBACK_LIBRARY_PATH'; \ + run_manifest_install() { \ + triplet="$$1"; \ + feature="$$2"; \ + attempt=1; \ + max_attempts=5; \ + echo "$(TARGET_PREFIX) Installing manifest feature $$feature for $$triplet..."; \ + until $$sanitize_env MAKELEVEL=0 $(VCPKG) install $(VCPKG_MANIFEST_ARGS) --triplet "$$triplet" --x-feature="$$feature" --clean-after-build; do \ + status=$$?; \ + if [ $$attempt -ge $$max_attempts ]; then \ + echo "$(TARGET_PREFIX) ERROR: Failed to install manifest feature $$feature for $$triplet" >&2; \ + if [ -f $(ROOT_DIR)/ci/vcpkg-error-handler.sh ]; then \ + chmod +x $(ROOT_DIR)/ci/vcpkg-error-handler.sh; \ + $(ROOT_DIR)/ci/vcpkg-error-handler.sh $(BUILD_ROOT) $(ROOT_DIR)/vcpkg-error-artifacts; \ + else \ + echo "$(TARGET_PREFIX) Searching for issue_body.md for $$feature..." >&2; \ + find $(VCPKG_ROOT)/installed -path "*/issue_body.md" -type f 2>/dev/null | while read -r issue_file; do \ + echo "$(TARGET_PREFIX) Found issue file: $$issue_file" >&2; \ + cat "$$issue_file" >&2; \ + done; \ + echo "$(TARGET_PREFIX) Checking build logs..." >&2; \ + find $(VCPKG_ROOT)/buildtrees -name '*err.log' -o -name '*out.log' | while read -r build_log; do cat "$$build_log" >&2; done; \ + fi; \ + exit $$status; \ fi; \ - exit 1; \ - }; \ - done + attempt=$$((attempt + 1)); \ + echo "$(TARGET_PREFIX) Retrying manifest feature $$feature for $$triplet (attempt $$attempt/$$max_attempts)..." >&2; \ + sleep 15; \ + done; \ + }; \ + run_manifest_install "$(VCPKG_RID)" "osx-static-tools"; \ + run_manifest_install "$(VCPKG_RID)-dynamic" "osx-dynamic"; \ + $(MAKE) -f vcpkg-makefile sync_manifest_lock_from_install_root; \ + printf '%s' "$$current_signature" > '$(VCPKG_STATE_STAMP)' remove_packages: - @$(foreach pack,$(VCPKG_REQUIRE_OSX), $(VCPKG) remove $(pack):$(VCPKG_RID) $(VCPKG_PARAMS_REMOVE);) - @$(foreach pack,$(VCPKG_REQUIRE_OSX_DYNAMIC), $(VCPKG) remove $(pack):$(VCPKG_RID)-dynamic $(VCPKG_PARAMS_REMOVE);) + @echo "$(TARGET_PREFIX) Manifest mode owns package inventory; use build_cleanup to remove installed trees." diff --git a/shared/GdalCore.opt b/shared/GdalCore.opt index 807f7487d..908c40b35 100644 --- a/shared/GdalCore.opt +++ b/shared/GdalCore.opt @@ -9,8 +9,8 @@ BUILD_NUMBER_TAIL=100 ### build (drivers) root BUILD_ROOT=$(ROOT_DIR)/build-$(BASE_RID) -# Mar 20, 2026 -GDAL_VERSION=3.12.3 +# May 4, 2026 +GDAL_VERSION=3.12.4 GDAL_ROOT=$(BUILD_ROOT)/gdal-source GDAL_REPO=https://github.com/OSGeo/gdal.git GDAL_COMMIT_VER=v$(GDAL_VERSION) @@ -24,26 +24,11 @@ PROJ_COMMIT_VER=$(PROJ_VERSION) # ---------------------- VCPKG ---------------------- VCPKG_ROOT=$(BUILD_ROOT)/vcpkg +VCPKG_MANIFEST_DIR=$(ROOT_DIR)/shared +VCPKG_MANIFEST_ROOT=$(VCPKG_MANIFEST_DIR) VCPKG_REPO=https://github.com/microsoft/vcpkg.git VCPKG_COMMIT_VER=2026.03.18 - -# base requirements for all runtimes -VCPKG_REQUIRE=geos "tiff[zstd,zip,jpeg,tools,lzma,cxx,webp]" "curl[tool,openssl]" "poppler[cairo,cms,zlib,glib,curl,private-api]" -VCPKG_REQUIRE_DYNAMIC=unixodbc openssl zlib expat xerces-c zlib libxml2 libpq openjpeg cfitsio "openexr[tools]" libwebp giflib hdf5 pcre freexl libkml libpng "libjxl[tools]" netcdf-c "libgeotiff[tools]" "sqlite3[tool,rtree]" cryptopp blosc arrow[parquet] -# windows runtime now depends on GisInternals SDK -# we have nothing to install except custom geos and proj -VCPKG_REQUIRE_WIN=$(VCPKG_REQUIRE) "sqlite3[tool,rtree]" libiconv lz4 liblzma "libjxl[tools]" blosc cryptopp arrow[parquet] -VCPKG_REQUIRE_WIN_STATIC= - -# the default configuration is dynamic -# libmysql is not available from vcpkg for arm64 -VCPKG_REQUIRE_UNIX= -VCPKG_REQUIRE_UNIX_DYNAMIC=$(VCPKG_REQUIRE) $(VCPKG_REQUIRE_DYNAMIC) - -# osx requires the same libs as linux -# sqlite static is required only to build proj.db -VCPKG_REQUIRE_OSX=$(VCPKG_REQUIRE_UNIX) "sqlite3[tool,rtree]" -VCPKG_REQUIRE_OSX_DYNAMIC=$(VCPKG_REQUIRE_DYNAMIC) $(VCPKG_REQUIRE) netcdf-c libmysql +# Package inventory now lives in shared/vcpkg.json and shared/vcpkg-configuration.json. VCPKG_CLEANUP=buildtrees downloads packages installed @@ -82,6 +67,9 @@ SWIG_INCLUDE=$(SWIG_BASE)/include HDF_BUILD=$(BUILD_ROOT)/hdf-build HDF_CMAKE_TMP=$(BUILD_ROOT)/hdf-cmake-temp HDF_VERSION=4.3.1 +HDF_ROOT=$(BUILD_ROOT)/hdf-source +HDF_REPO=https://github.com/HDFGroup/hdf4.git +HDF_COMMIT_VER=hdf$(HDF_VERSION) ##### PROJ build location PROJ_BUILD=$(BUILD_ROOT)/proj-build diff --git a/shared/common.mk b/shared/common.mk index 80c8bf172..c0fc1e08a 100644 --- a/shared/common.mk +++ b/shared/common.mk @@ -148,27 +148,8 @@ else @echo "$(LOG_PREFIX) Can not make $(TARGET_CLEAN)" endif -# HDF download targets -init_hdf: check_hdf_sources - @echo "$(TARGET_PREFIX) HDF sources restore complete" - -HDF_ZIP=hdf$(HDF_VERSION).zip -HDF_SOURCE=$(BUILD_ROOT)/hdf4-hdf$(HDF_VERSION) - -download_hdf: - @echo "$(TARGET_PREFIX) Downloading HDF source ${HDF_ZIP}..." - $(LIB_PATH_VAR)="" curl -JL "https://github.com/HDFGroup/hdf4/archive/refs/tags/$(HDF_ZIP)" -o "$(BUILD_ROOT)/$(HDF_ZIP)" - @echo "$(TARGET_PREFIX) HDF source downloaded!" - @echo "$(TARGET_PREFIX) Extracting HDF source..." - cd "$(BUILD_ROOT)"; unzip -oq "$(BUILD_ROOT)/$(HDF_ZIP)" -d . - -check_hdf_sources: - @if [[ ! -f "$(BUILD_ROOT)/$(HDF_ZIP)" ]] || [[ ! -d "$(HDF_SOURCE)" ]]; then \ - $(MAKE) -f gdal-makefile download_hdf; \ - fi; - # Convenience target for resetting all repositories -reset: reset_proj reset_gdal +reset: reset_hdf reset_proj reset_gdal @echo "$(TARGET_PREFIX) Reset ALL is complete" endif diff --git a/shared/vcpkg-configuration.json b/shared/vcpkg-configuration.json new file mode 100644 index 000000000..9f52b61fe --- /dev/null +++ b/shared/vcpkg-configuration.json @@ -0,0 +1,7 @@ +{ + "default-registry": { + "kind": "git", + "repository": "https://github.com/microsoft/vcpkg", + "baseline": "c3867e714dd3a51c272826eea77267876517ed99" + } +} diff --git a/shared/vcpkg-lock.json b/shared/vcpkg-lock.json new file mode 100644 index 000000000..3489882d0 --- /dev/null +++ b/shared/vcpkg-lock.json @@ -0,0 +1,5 @@ +{ + "https://github.com/microsoft/vcpkg": { + "HEAD": "b46d9050a9d40d54d24cac3ef8d50402d421f598" + } +} diff --git a/shared/vcpkg.json b/shared/vcpkg.json new file mode 100644 index 000000000..24c1c2eb9 --- /dev/null +++ b/shared/vcpkg.json @@ -0,0 +1,252 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", + "name": "maxrev-gdal-netcore-dependencies", + "version-string": "1.0.0", + "features": { + "linux-dynamic": { + "description": "Linux runtime dependency set installed through the dynamic overlay triplet.", + "dependencies": [ + "geos", + { + "name": "tiff", + "features": [ + "zstd", + "zip", + "jpeg", + "tools", + "lzma", + "cxx", + "webp" + ] + }, + { + "name": "curl", + "features": [ + "tool", + "openssl" + ] + }, + { + "name": "poppler", + "features": [ + "cairo", + "cms", + "zlib", + "glib", + "curl", + "private-api" + ] + }, + "unixodbc", + "openssl", + "zlib", + "expat", + "xerces-c", + "libxml2", + "libpq", + "openjpeg", + "cfitsio", + { + "name": "openexr", + "features": [ + "tools" + ] + }, + "libwebp", + "giflib", + "hdf5", + "pcre", + "freexl", + "libkml", + "libpng", + { + "name": "libjxl", + "features": [ + "tools" + ] + }, + "netcdf-c", + { + "name": "libgeotiff", + "features": [ + "tools" + ] + }, + { + "name": "sqlite3", + "features": [ + "tool", + "rtree" + ] + }, + "cryptopp", + "blosc", + { + "name": "arrow", + "features": [ + "parquet" + ] + } + ] + }, + "osx-static-tools": { + "description": "macOS static sqlite tool pass used to build proj.db.", + "dependencies": [ + { + "name": "sqlite3", + "features": [ + "tool", + "rtree" + ] + } + ] + }, + "osx-dynamic": { + "description": "macOS dynamic runtime dependency set while preserving the separate sqlite tool pass.", + "dependencies": [ + "unixodbc", + "openssl", + "zlib", + "expat", + "xerces-c", + "libxml2", + "libpq", + "openjpeg", + "cfitsio", + { + "name": "openexr", + "features": [ + "tools" + ] + }, + "libwebp", + "giflib", + "hdf5", + "pcre", + "freexl", + "libkml", + "libpng", + { + "name": "libjxl", + "features": [ + "tools" + ] + }, + "netcdf-c", + { + "name": "libgeotiff", + "features": [ + "tools" + ] + }, + { + "name": "sqlite3", + "features": [ + "tool", + "rtree" + ] + }, + "cryptopp", + "blosc", + { + "name": "arrow", + "features": [ + "parquet" + ] + }, + "geos", + { + "name": "tiff", + "features": [ + "zstd", + "zip", + "jpeg", + "tools", + "lzma", + "cxx", + "webp" + ] + }, + { + "name": "curl", + "features": [ + "tool", + "openssl" + ] + }, + { + "name": "poppler", + "features": [ + "cairo", + "cms", + "zlib", + "glib", + "curl", + "private-api" + ] + }, + "libmysql" + ] + }, + "windows-dynamic": { + "description": "Windows runtime dependency set kept authoritative in the shared manifest.", + "dependencies": [ + "geos", + { + "name": "tiff", + "features": [ + "zstd", + "zip", + "jpeg", + "tools", + "lzma", + "cxx", + "webp" + ] + }, + { + "name": "curl", + "features": [ + "tool", + "openssl" + ] + }, + { + "name": "poppler", + "features": [ + "cairo", + "cms", + "zlib", + "glib", + "curl", + "private-api" + ] + }, + { + "name": "sqlite3", + "features": [ + "tool", + "rtree" + ] + }, + "libiconv", + "lz4", + "liblzma", + { + "name": "libjxl", + "features": [ + "tools" + ] + }, + "blosc", + "cryptopp", + { + "name": "arrow", + "features": [ + "parquet" + ] + } + ] + } + } +} diff --git a/unix/gdal-makefile b/unix/gdal-makefile index db5dde446..1d99dabb4 100644 --- a/unix/gdal-makefile +++ b/unix/gdal-makefile @@ -11,7 +11,7 @@ TARGETS = hdf proj gdal all: $(TARGETS) @echo "$(LOG_PREFIX) Everything looks good. Linux libraries for GDAL is ready to packaging!" - @echo "$(LOG_PREFIX) Configured libraries (vcpkg): $(VCPKG_REQUIRE_UNIX)" + @echo "$(LOG_PREFIX) Configured libraries (vcpkg): shared manifest features" @echo "$(LOG_PREFIX) Configured libraries: $(TARGETS)" pre_vcpkg: @@ -33,7 +33,7 @@ configure_hdf: @if [[ -d "$(BUILD_ROOT)/hdf-cmake-temp" ]]; then rm -r "$(BUILD_ROOT)/hdf-cmake-temp" >/dev/null 2>&1 || true; fi; -mkdir -p $(HDF_CMAKE_TMP) - @cd $(HDF_CMAKE_TMP) && cmake $(HDF_SOURCE) \ + @cd $(HDF_CMAKE_TMP) && cmake $(HDF_ROOT) \ -DCMAKE_INSTALL_PREFIX=$(BUILD_ROOT)/hdf-build \ -DCMAKE_MESSAGE_LOG_LEVEL=ERROR \ -DCMAKE_BUILD_TYPE=Release \ diff --git a/unix/publish-makefile b/unix/publish-makefile index 88180a30d..ab10c7b6b 100644 --- a/unix/publish-makefile +++ b/unix/publish-makefile @@ -15,7 +15,7 @@ export MSBUILDSINGLELOADCONTEXT = 1 export LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:${VCPKG_INSTALLED_DYNAMIC}/lib RUNTIME_PROJECT_FINAL=$(RUNTIME_PROJECT_UNIX_FINAL) -GEOS_VERSION=$(shell $(MAKE) -f ../unix/generate-projects-makefile get-version IN_FILE=$(VCPKG_INSTALLED_DYNAMIC)/lib/pkgconfig/geos.pc) +GEOS_VERSION=$(shell $(MAKE) --no-print-directory -s -f ../unix/generate-projects-makefile get-version IN_FILE=$(VCPKG_INSTALLED_DYNAMIC)/lib/pkgconfig/geos.pc) CAT_NAME=unix RUNTIME_PACKAGE_PARTIAL=LinuxRuntime BUILD_ARCH_SUFFIX=.$(BUILD_ARCH) diff --git a/unix/vcpkg-makefile b/unix/vcpkg-makefile index 61e72226e..639d03a59 100644 --- a/unix/vcpkg-makefile +++ b/unix/vcpkg-makefile @@ -24,34 +24,9 @@ force: $(addsuffix -force, $(TARGETS)) help: ## Display this help screen @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -# accepts any params to install/uninstall package +# Manifest mode owns package inventory in shared/vcpkg.json. % : -ifneq ($(filter $(TARGET_CLEAN),$(VCPKG_REQUIRE_UNIX)),'') - @echo "'vcpkg make' > trying to make stuff for => $(TARGET_CLEAN)" - @if [[ "$(TARGET_LOWER)" == *"-force"* ]] ; then \ - $(MAKE) -f vcpkg-makefile upgrade; \ - $(VCPKG) remove $(TARGET_CLEAN):$(VCPKG_RID) --recurse; \ - $(VCPKG) install $(TARGET_CLEAN):$(VCPKG_RID) --recurse --clean-after-build || { \ - echo "$(TARGET_PREFIX) ERROR: Failed to install $(TARGET_CLEAN):$(VCPKG_RID)" >&2; \ - if [ -f $(ROOT_DIR)/ci/vcpkg-error-handler.sh ]; then \ - chmod +x $(ROOT_DIR)/ci/vcpkg-error-handler.sh; \ - $(ROOT_DIR)/ci/vcpkg-error-handler.sh $(BUILD_ROOT) $(ROOT_DIR)/vcpkg-error-artifacts; \ - else \ - echo "$(TARGET_PREFIX) Searching for issue_body.md for $(TARGET_CLEAN)..." >&2; \ - find $(VCPKG_ROOT)/installed -path "*/$(TARGET_CLEAN)*/issue_body.md" -type f 2>/dev/null | while read -r issue_file; do \ - echo "$(TARGET_PREFIX) Found issue file: $$issue_file" >&2; \ - cat "$$issue_file" >&2; \ - done; \ - fi; \ - exit 1; \ - }; \ - else \ - echo "To rebuild add {key}-force"; \ - $(VCPKG) list; \ - fi; -else - @echo "Can not make $(TARGET_CLEAN)" -endif + @echo "$(TARGET_PREFIX) Manifest mode owns package inventory; use 'make -f vcpkg-makefile' to install declared features." pull: ## pull the package @if [ ! -d "$(VCPKG_ROOT)/.git" ]; then \ @@ -112,51 +87,51 @@ endef export VCPKG_TRIPLET_LINUX_DYNAMIC_RELEASE export VCPKG_TRIPLET_LINUX_STATIC_RELEASE +VCPKG_MANIFEST_ARGS=--x-manifest-root=$(VCPKG_MANIFEST_ROOT) --x-install-root=$(VCPKG_ROOT)/installed $(VCPKG_TRIPLETS_OVERLAY) --no-print-usage +VCPKG_MANIFEST_LOCK=$(VCPKG_MANIFEST_ROOT)/vcpkg-lock.json +VCPKG_MANIFEST_LOCK_INSTALLED=$(VCPKG_ROOT)/installed/vcpkg/vcpkg-lock.json + patch_vcpkg_triplets: @mkdir -p $(VCPKG_ROOT)/custom-triplets @echo "$$VCPKG_TRIPLET_LINUX_DYNAMIC_RELEASE" > $(VCPKG_ROOT)/custom-triplets/$(VCPKG_RID)-dynamic.cmake @echo "$$VCPKG_TRIPLET_LINUX_STATIC_RELEASE" > $(VCPKG_ROOT)/custom-triplets/$(VCPKG_RID).cmake -install_packages: - @for pack in $(VCPKG_REQUIRE_UNIX); do \ - echo "$(TARGET_PREFIX) Installing $$pack:$(VCPKG_RID)..."; \ - MAKELEVEL=0 $(VCPKG) install $$pack:$(VCPKG_RID) $(VCPKG_PARAMS_INSTALL) $(VCPKG_ENSURE_INSTALLED) || { \ - echo "$(TARGET_PREFIX) ERROR: Failed to install $$pack:$(VCPKG_RID)" >&2; \ +sync_manifest_lock_to_install_root: + @mkdir -p $(VCPKG_ROOT)/installed/vcpkg + @if [ -f "$(VCPKG_MANIFEST_LOCK)" ]; then \ + cp "$(VCPKG_MANIFEST_LOCK)" "$(VCPKG_MANIFEST_LOCK_INSTALLED)"; \ + fi + +sync_manifest_lock_from_install_root: + @if [ -f "$(VCPKG_MANIFEST_LOCK_INSTALLED)" ]; then \ + cmp -s "$(VCPKG_MANIFEST_LOCK_INSTALLED)" "$(VCPKG_MANIFEST_LOCK)" || cp "$(VCPKG_MANIFEST_LOCK_INSTALLED)" "$(VCPKG_MANIFEST_LOCK)"; \ + fi + +install_packages: sync_manifest_lock_to_install_root + @echo "$(TARGET_PREFIX) Installing manifest feature linux-dynamic for $(VCPKG_RID)-dynamic..." + @attempt=1; \ + max_attempts=5; \ + until MAKELEVEL=0 $(VCPKG) install $(VCPKG_MANIFEST_ARGS) --triplet $(VCPKG_RID)-dynamic --x-feature=linux-dynamic --clean-after-build; do \ + status=$$?; \ + if [ $$attempt -ge $$max_attempts ]; then \ + echo "$(TARGET_PREFIX) ERROR: Failed to install manifest feature linux-dynamic for $(VCPKG_RID)-dynamic" >&2; \ if [ -f $(ROOT_DIR)/ci/vcpkg-error-handler.sh ]; then \ chmod +x $(ROOT_DIR)/ci/vcpkg-error-handler.sh; \ $(ROOT_DIR)/ci/vcpkg-error-handler.sh $(BUILD_ROOT) $(ROOT_DIR)/vcpkg-error-artifacts; \ else \ - echo "$(TARGET_PREFIX) Searching for issue_body.md for $$pack..." >&2; \ - find $(VCPKG_ROOT)/installed -path "*/$$pack*/issue_body.md" -type f 2>/dev/null | while read -r issue_file; do \ + echo "$(TARGET_PREFIX) Searching for issue_body.md for manifest install..." >&2; \ + find $(VCPKG_ROOT)/installed -path "*/issue_body.md" -type f 2>/dev/null | while read -r issue_file; do \ echo "$(TARGET_PREFIX) Found issue file: $$issue_file" >&2; \ cat "$$issue_file" >&2; \ done; \ fi; \ - exit 1; \ - }; \ - done - @for pack in $(VCPKG_REQUIRE_UNIX_DYNAMIC); do \ - echo "$(TARGET_PREFIX) Installing $$pack:$(VCPKG_RID)-dynamic..."; \ - MAKELEVEL=0 $(VCPKG) install $$pack:$(VCPKG_RID)-dynamic $(VCPKG_PARAMS_INSTALL) $(VCPKG_ENSURE_INSTALLED) || { \ - echo "$(TARGET_PREFIX) ERROR: Failed to install $$pack:$(VCPKG_RID)-dynamic" >&2; \ - if [ -f $(ROOT_DIR)/ci/vcpkg-error-handler.sh ]; then \ - chmod +x $(ROOT_DIR)/ci/vcpkg-error-handler.sh; \ - $(ROOT_DIR)/ci/vcpkg-error-handler.sh $(BUILD_ROOT) $(ROOT_DIR)/vcpkg-error-artifacts; \ - else \ - echo "$(TARGET_PREFIX) Searching for issue_body.md for $$pack..." >&2; \ - find $(VCPKG_ROOT)/installed -path "*/$$pack*/issue_body.md" -type f 2>/dev/null | while read -r issue_file; do \ - echo "$(TARGET_PREFIX) Found issue file: $$issue_file" >&2; \ - cat "$$issue_file" >&2; \ - done; \ - fi; \ - exit 1; \ - }; \ + exit $$status; \ + fi; \ + attempt=$$((attempt + 1)); \ + echo "$(TARGET_PREFIX) Retrying manifest feature linux-dynamic for $(VCPKG_RID)-dynamic (attempt $$attempt/$$max_attempts)..." >&2; \ + sleep 15; \ done + @$(MAKE) -f vcpkg-makefile sync_manifest_lock_from_install_root remove_packages: - @for pack in $(VCPKG_REQUIRE_UNIX); do \ - $(VCPKG) remove $$pack:$(VCPKG_RID) $(VCPKG_PARAMS_REMOVE) || exit 1; \ - done - @for pack in $(VCPKG_REQUIRE_UNIX_DYNAMIC); do \ - $(VCPKG) remove $$pack:$(VCPKG_RID)-dynamic $(VCPKG_PARAMS_REMOVE) || exit 1; \ - done + @echo "$(TARGET_PREFIX) Manifest mode owns package inventory; use build_cleanup to remove installed trees." diff --git a/win/functions.psm1 b/win/functions.psm1 index 82beec289..28135fb2e 100644 --- a/win/functions.psm1 +++ b/win/functions.psm1 @@ -275,4 +275,95 @@ function Get-PathRelative { ) $path = $inputPath -replace [regex]::escape($env:BUILD_ROOT), $relativePath ; return $path -replace '\\', '/' -} \ No newline at end of file +} + +function Get-BuildStateStampPath { + param ( + [Parameter(Mandatory = $true)] + [string] $Component, + [string] $BuildRoot = $env:BUILD_ROOT + ) + + if ([string]::IsNullOrWhiteSpace($BuildRoot)) { + throw "BUILD_ROOT is not set" + } + + return Join-Path $BuildRoot ".build-state-$Component.txt" +} + +function Get-FileSignatures { + param ( + [Parameter(Mandatory = $true)] + [string[]] $Paths + ) + + $signatures = @() + foreach ($path in $Paths) { + $resolvedPath = Get-ForceResolvePath $path + if (Test-Path -Path $resolvedPath) { + $hash = (Get-FileHash -Path $resolvedPath -Algorithm SHA256).Hash.ToLowerInvariant() + $signatures += "$resolvedPath=$hash" + } + else { + $signatures += "$resolvedPath=missing" + } + } + + return $signatures +} + +function Get-GitHeadOrDefault { + param ( + [Parameter(Mandatory = $true)] + [string] $RepositoryPath, + [Parameter(Mandatory = $true)] + [string] $DefaultValue + ) + + if (Test-Path -Path (Join-Path $RepositoryPath ".git") -PathType Container) { + $head = & git -C $RepositoryPath rev-parse HEAD 2>$null + if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace($head)) { + return $head.Trim() + } + } + + return $DefaultValue +} + +function Test-BuildStateStamp { + param ( + [Parameter(Mandatory = $true)] + [string] $Component, + [Parameter(Mandatory = $true)] + [string] $Signature, + [string[]] $RequiredPaths = @() + ) + + foreach ($path in $RequiredPaths) { + $resolvedPath = Get-ForceResolvePath $path + if (-Not (Test-Path -Path $resolvedPath)) { + return $false + } + } + + $stampPath = Get-BuildStateStampPath -Component $Component + if (-Not (Test-Path -Path $stampPath -PathType Leaf)) { + return $false + } + + $savedSignature = (Get-Content -Path $stampPath -Raw).Trim() + return $savedSignature -eq $Signature.Trim() +} + +function Save-BuildStateStamp { + param ( + [Parameter(Mandatory = $true)] + [string] $Component, + [Parameter(Mandatory = $true)] + [string] $Signature + ) + + New-FolderIfNotExists $env:BUILD_ROOT + $stampPath = Get-BuildStateStampPath -Component $Component + Set-Content -Path $stampPath -Value $Signature -NoNewline +} diff --git a/win/install.ps1 b/win/install.ps1 index 41887bac9..706258e29 100644 --- a/win/install.ps1 +++ b/win/install.ps1 @@ -35,6 +35,7 @@ try { Get-VcpkgInstallation -bootstrapVcpkg $bootstrapVcpkg + Write-BuildInfo "Using shared vcpkg authority files shared/vcpkg.json, shared/vcpkg-configuration.json, and shared/vcpkg-lock.json" Install-VcpkgPackagesSharedConfig $installVcpkgPackages Get-GdalSdkIsAvailable @@ -61,4 +62,4 @@ catch finally { Pop-Location -StackName "gdal.netcore|root" #Get-Variable | Where-Object Name -notin $existingVariables.Name | Remove-Variable -ErrorAction SilentlyContinue -} \ No newline at end of file +} diff --git a/win/partials.psm1 b/win/partials.psm1 index 6ea7f8ba8..881979e5b 100644 --- a/win/partials.psm1 +++ b/win/partials.psm1 @@ -1,4 +1,15 @@ function Set-GdalVariables { + $sharedConfig = Get-ForceResolvePath "$PSScriptRoot\..\shared\GdalCore.opt" + $gdalVersion = (Select-String -Path $sharedConfig -Pattern '^GDAL_VERSION=(.+)$' | Select-Object -First 1).Matches.Groups[1].Value.Trim() + $projVersion = (Select-String -Path $sharedConfig -Pattern '^PROJ_VERSION=(.+)$' | Select-Object -First 1).Matches.Groups[1].Value.Trim() + $vcpkgCommit = (Select-String -Path $sharedConfig -Pattern '^VCPKG_COMMIT_VER=(.+)$' | Select-Object -First 1).Matches.Groups[1].Value.Trim() + + $env:GDAL_VERSION = $gdalVersion + $env:GDAL_COMMIT_VER = "v$gdalVersion" + $env:PROJ_VERSION = $projVersion + $env:PROJ_COMMIT_VER = $projVersion + $env:VCPKG_COMMIT_VER = $vcpkgCommit + $env:VS_VERSION = "Visual Studio 17 2022" $env:SDK = "release-1930-x64" #2022 x64 $env:SDK_ZIP = "$env:SDK" + "-dev.zip" @@ -26,7 +37,10 @@ function Set-GdalVariables { $env:SDK_LIB = "$env:SDK_PREFIX\lib" $env:SDK_BIN = "$env:SDK_PREFIX\bin" $env:GDAL_INSTALL_DIR = "$env:BUILD_ROOT\gdal-build" - $env:VCPKG_INSTALLED = "$env:BUILD_ROOT\vcpkg\installed\x64-windows" + $env:VCPKG_MANIFEST_ROOT = (Get-ForceResolvePath "$PSScriptRoot\..\shared") + $env:VCPKG_INSTALL_ROOT = "$env:BUILD_ROOT\vcpkg\installed" + $env:VCPKG_LOCKFILE = "$env:VCPKG_MANIFEST_ROOT\vcpkg-lock.json" + $env:VCPKG_INSTALLED = "$env:VCPKG_INSTALL_ROOT\x64-windows" $env:VCPKG_INSTALLED_PKGCONFIG = "$env:VCPKG_INSTALLED\lib\pkgconfig" $env:WEBP_ROOT = Get-ForceResolvePath("$env:BUILD_ROOT\sdk\libwebp*") @@ -37,6 +51,84 @@ function Set-GdalVariables { Add-EnvPath $env:VCPKG_ROOT_GDAL -Prepend } +function Get-WindowsBuildSignature { + param ( + [Parameter(Mandatory = $true)] + [string] $Component + ) + + Set-GdalVariables + + $root = Get-ForceResolvePath "$PSScriptRoot\.." + $commonInputs = Get-FileSignatures -Paths @( + "$root\shared\GdalCore.opt", + "$root\win\install.ps1", + "$root\win\partials.psm1", + "$root\win\vcpkg-makefile.vc", + "$root\shared\vcpkg.json", + "$root\shared\vcpkg-configuration.json", + "$root\shared\vcpkg-lock.json", + "$root\shared\patch\CMakeLists.txt.patch" + ) + + switch ($Component) { + "vcpkg" { + $componentData = @( + "component=vcpkg", + "vcpkgCommit=$env:VCPKG_COMMIT_VER", + "triplet=x64-windows" + ) + } + "proj" { + $componentData = @( + "component=proj", + "projVersion=$env:PROJ_VERSION", + "projSource=$(Get-GitHeadOrDefault -RepositoryPath $env:PROJ_SOURCE -DefaultValue $env:PROJ_COMMIT_VER)" + ) + } + "gdal" { + $componentData = @( + "component=gdal", + "gdalVersion=$env:GDAL_VERSION", + "gdalSource=$(Get-GitHeadOrDefault -RepositoryPath $env:GDAL_SOURCE -DefaultValue $env:GDAL_COMMIT_VER)" + ) + } + default { + throw "Unknown build component: $Component" + } + } + + return (@($componentData) + @($commonInputs)) -join "`n" +} + +function Test-WindowsBuildReuse { + param ( + [Parameter(Mandatory = $true)] + [string] $Component, + [Parameter(Mandatory = $true)] + [string[]] $RequiredPaths, + [Parameter(Mandatory = $true)] + [string] $Message + ) + + $signature = Get-WindowsBuildSignature -Component $Component + if (Test-BuildStateStamp -Component $Component -Signature $signature -RequiredPaths $RequiredPaths) { + Write-BuildStep $Message + return $true + } + + return $false +} + +function Save-WindowsBuildReuse { + param ( + [Parameter(Mandatory = $true)] + [string] $Component + ) + + Save-BuildStateStamp -Component $Component -Signature (Get-WindowsBuildSignature -Component $Component) +} + function Get-7ZipInstallation { Write-BuildStep "Checking for 7z installation" New-FolderIfNotExists $env:7Z_ROOT @@ -103,6 +195,16 @@ function Get-VcpkgInstallation { } New-FolderIfNotExists $env:VCPKG_DEFAULT_BINARY_CACHE + if (Test-WindowsBuildReuse -Component "vcpkg" ` + -RequiredPaths @( + "$env:VCPKG_ROOT_GDAL\vcpkg.exe", + "$env:VCPKG_INSTALLED\include", + "$env:VCPKG_INSTALLED\lib" + ) ` + -Message "Reusing cached VCPKG installation and installed packages") { + return + } + Write-BuildStep "Checking for VCPKG installation" if ($bootstrapVcpkg) { Get-CloneAndCheckoutCleanGitRepo https://github.com/Microsoft/vcpkg.git master $env:VCPKG_ROOT_GDAL @@ -122,7 +224,19 @@ function Install-VcpkgPackagesSharedConfig { ) if ($installVcpkgPackages) { + if (Test-WindowsBuildReuse -Component "vcpkg" ` + -RequiredPaths @( + "$env:VCPKG_ROOT_GDAL\vcpkg.exe", + "$env:VCPKG_INSTALLED\include", + "$env:VCPKG_INSTALLED\lib" + ) ` + -Message "Skipping VCPKG manifest install because cached packages are up to date") { + return + } + + Write-BuildInfo "Installing Windows dependencies from shared manifest $env:VCPKG_MANIFEST_ROOT\vcpkg.json with lock $env:VCPKG_LOCKFILE" exec { & nmake -f ./vcpkg-makefile.vc install_requirements } + Save-WindowsBuildReuse -Component "vcpkg" } } @@ -131,6 +245,16 @@ function Install-Proj { [bool] $cleanProjBuild = $true, [bool] $cleanProjIntermediate = $true ) + + if (Test-WindowsBuildReuse -Component "proj" ` + -RequiredPaths @( + "$env:PROJ_INSTALL_DIR\include\proj.h", + "$env:PROJ_INSTALL_DIR\lib\proj.lib" + ) ` + -Message "Skipping PROJ rebuild because cached outputs are up to date") { + return + } + Write-BuildStep "Building PROJ" if ($cleanProjBuild) { Write-BuildInfo "Cleaning PROJ build folder" @@ -170,6 +294,7 @@ function Install-Proj { exec { cmake --build . -j $env:CMAKE_PARALLEL_JOBS --config Release --target install } Write-BuildStep "Done building PROJ" + Save-WindowsBuildReuse -Component "proj" } function Get-GdalVersion { @@ -211,6 +336,15 @@ function Build-Gdal { $env:TIFF_INCLUDE_DIR = "-DTIFF_INCLUDE_DIR=$env:VCPKG_INSTALLED\include\tiff" $env:TIFF_LIBRARY = "-DTIFF_LIBRARY_RELEASE=$env:VCPKG_INSTALLED\lib\tiff.lib" + if (Test-WindowsBuildReuse -Component "gdal" ` + -RequiredPaths @( + "$env:GDAL_INSTALL_DIR\bin\gdal.dll", + "$env:GdalCmakeBuild\swig\csharp" + ) ` + -Message "Skipping GDAL rebuild because cached outputs are up to date") { + return + } + Write-BuildStep "Configuring GDAL" Set-Location "$env:BUILD_ROOT" @@ -291,6 +425,7 @@ function Build-Gdal { # remove source. this was added by GDAL exec { dotnet nuget remove source local } Write-BuildStep "GDAL was built successfully" + Save-WindowsBuildReuse -Component "gdal" } function Build-GenerateProjectFiles { diff --git a/win/vcpkg-makefile.vc b/win/vcpkg-makefile.vc index 8da46799b..f8bb90130 100644 --- a/win/vcpkg-makefile.vc +++ b/win/vcpkg-makefile.vc @@ -15,6 +15,11 @@ VCPKG_CLEANUP = buildtrees downloads packages installed VCPKG_ROOT = $(BUILD_ROOT)\vcpkg VCPKG = "$(VCPKG_ROOT)\vcpkg.exe" +VCPKG_INSTALL_ROOT = $(VCPKG_ROOT)\installed +VCPKG_MANIFEST_ROOT = $(MAKEDIR)\..\shared +VCPKG_MANIFEST_LOCK = $(VCPKG_MANIFEST_ROOT)\vcpkg-lock.json +VCPKG_MANIFEST_LOCK_INSTALLED = $(VCPKG_INSTALL_ROOT)\vcpkg\vcpkg-lock.json +VCPKG_MANIFEST_ARGS = --x-manifest-root="$(VCPKG_MANIFEST_ROOT)" --x-install-root="$(VCPKG_INSTALL_ROOT)" $(VCPKG_TRIPLETS_OVERLAY) --no-print-usage install_vcpkg: @IF NOT EXIST "$(VCPKG_ROOT)\" git clone $(VCPKG_REPO) $(VCPKG_ROOT) @@ -28,7 +33,7 @@ checkout_ver: bootstrap: call "$(VCPKG_ROOT)/bootstrap-vcpkg.bat"; -install_requirements: install_vcpkg checkout_ver bootstrap patch_vcpkg_triplets install_packages +install_requirements: install_vcpkg checkout_ver bootstrap patch_vcpkg_triplets sync_manifest_lock_to_install_root install_packages sync_manifest_lock_from_install_root VCPKG_WINDOWS_DEFAULT_TRIPLET = "$(VCPKG_CUSTOM_TRIPLETS)/x64-windows.cmake" @@ -41,6 +46,13 @@ patch_vcpkg_triplets: @echo set(VCPKG_LIBRARY_LINKAGE dynamic) >> $(VCPKG_WINDOWS_DEFAULT_TRIPLET) @echo set(VCPKG_BUILD_TYPE release) >> $(VCPKG_WINDOWS_DEFAULT_TRIPLET) +sync_manifest_lock_to_install_root: + @if not exist "$(VCPKG_INSTALL_ROOT)\vcpkg" md "$(VCPKG_INSTALL_ROOT)\vcpkg" + @if exist "$(VCPKG_MANIFEST_LOCK)" copy /Y "$(VCPKG_MANIFEST_LOCK)" "$(VCPKG_MANIFEST_LOCK_INSTALLED)" >nul + +sync_manifest_lock_from_install_root: + @if exist "$(VCPKG_MANIFEST_LOCK_INSTALLED)" copy /Y "$(VCPKG_MANIFEST_LOCK_INSTALLED)" "$(VCPKG_MANIFEST_LOCK)" >nul + upgrade: checkout_ver $(VCPKG) upgrade $(VCPKG_TRIPLETS_OVERLAY) upgrade-force: checkout_ver @@ -54,15 +66,9 @@ $(VCPKG_CLEANUP): build_cleanup: $(VCPKG_CLEANUP) install_packages: - @for %p in ( $(VCPKG_REQUIRE_WIN) ) \ - do $(VCPKG) install %p:$(VCPKG_RID) $(VCPKG_PARAMS_INSTALL) $(VCPKG_ENSURE_INSTALLED) - @for %p in ( $(VCPKG_REQUIRE_WIN_STATIC) ) \ - do $(VCPKG) install %p:$(VCPKG_RID)-static $(VCPKG_PARAMS_INSTALL) $(VCPKG_ENSURE_INSTALLED) + @echo "$(LOG_PREFIX) Installing manifest feature windows-dynamic for $(VCPKG_RID)" + @$(VCPKG) install $(VCPKG_MANIFEST_ARGS) --triplet $(VCPKG_RID) --x-feature=windows-dynamic $(VCPKG_ENSURE_INSTALLED) remove: $(VCPKG) remove $(r):$(VCPKG_RID) $(VCPKG_PARAMS_REMOVE) remove_packages: - @for %p in ( $(VCPKG_REQUIRE_WIN) ) \ - do $(VCPKG) remove %p:$(VCPKG_RID) $(VCPKG_PARAMS_REMOVE) - @for %p in ( $(VCPKG_REQUIRE_WIN_STATIC) ) \ - do $(VCPKG) remove %p:$(VCPKG_RID)-static $(VCPKG_PARAMS_REMOVE) - + @echo "$(LOG_PREFIX) Manifest mode owns package inventory; use build_cleanup to remove installed trees."