From a9a08816db7d82349cad9edc93460aad0648daf2 Mon Sep 17 00:00:00 2001 From: Yueming Hao Date: Mon, 8 Dec 2025 17:10:53 -0800 Subject: [PATCH 1/4] feat: implement on-demand nightly publishing - Add check_new_commits.sh script to detect if there are new commits since last nightly release - Query PyPI API for latest nightly version and extract timestamp - Skip publishing if no new commits, reducing unnecessary releases - Include network retry mechanism (3 retries) with fail-open design --- .github/scripts/check_new_commits.sh | 108 +++++++++++++++++++++++++++ .github/workflows/nightly-pypi.yml | 13 +++- 2 files changed, 119 insertions(+), 2 deletions(-) create mode 100755 .github/scripts/check_new_commits.sh diff --git a/.github/scripts/check_new_commits.sh b/.github/scripts/check_new_commits.sh new file mode 100755 index 0000000..b32fc72 --- /dev/null +++ b/.github/scripts/check_new_commits.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# Check if there are new commits since the last nightly PyPI release. +# This script is used by nightly-pypi.yml to implement on-demand nightly publishing. +# +# Exit codes: +# 0 - Should publish (new commits found or check skipped due to errors) +# 1 - Should skip publish (no new commits) +# +# Output: +# Sets GITHUB_OUTPUT variable: should_publish=true/false + +set -o pipefail + +# Configuration +MAX_RETRIES=3 +RETRY_DELAY=5 +CURL_TIMEOUT=10 +PACKAGE_NAME="${PACKAGE_NAME:-tritonparse}" + +# Function: Fetch latest nightly version from PyPI with retry +fetch_latest_nightly() { + local retries=0 + + while [ $retries -lt $MAX_RETRIES ]; do + local response + response=$(curl -s --max-time $CURL_TIMEOUT \ + "https://pypi.org/pypi/${PACKAGE_NAME}/json" 2>/dev/null) + local curl_exit=$? + + if [ $curl_exit -eq 0 ] && [ -n "$response" ]; then + # Try to parse and extract latest dev version + local latest + latest=$(echo "$response" | \ + jq -r '.releases | keys[] | select(contains(".dev"))' 2>/dev/null | \ + sort -V | tail -1) + + if [ $? -eq 0 ]; then + echo "$latest" + return 0 + fi + fi + + retries=$((retries + 1)) + echo "::warning::PyPI API request failed (attempt $retries/$MAX_RETRIES)" + + if [ $retries -lt $MAX_RETRIES ]; then + echo "Retrying in ${RETRY_DELAY}s..." + sleep $RETRY_DELAY + fi + done + + echo "::warning::All $MAX_RETRIES attempts failed" + return 1 +} + +main() { + echo "Checking for new commits since last nightly release..." + + # Step 1: Fetch latest nightly version (with retry) + LATEST_NIGHTLY=$(fetch_latest_nightly) + FETCH_STATUS=$? + + # Step 2: Network request failed -> skip check, proceed with publish + if [ $FETCH_STATUS -ne 0 ]; then + echo "::warning::Failed to fetch PyPI data after $MAX_RETRIES attempts" + echo "::warning::Skipping commit check, proceeding with publish" + echo "should_publish=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Step 3: No nightly version exists -> first nightly release + if [ -z "$LATEST_NIGHTLY" ]; then + echo "No existing nightly version found, proceeding with publish" + echo "should_publish=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "Latest nightly on PyPI: $LATEST_NIGHTLY" + + # Step 4: Extract timestamp from version (format: X.Y.Z.devYYYYMMDDHHMMSS) + TIMESTAMP=$(echo "$LATEST_NIGHTLY" | grep -oP '\.dev\K[0-9]{14}' || true) + + if [ -z "$TIMESTAMP" ]; then + echo "::warning::Cannot parse timestamp from version, proceeding with publish" + echo "should_publish=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Step 5: Convert to date format for git log + SINCE_DATE="${TIMESTAMP:0:4}-${TIMESTAMP:4:2}-${TIMESTAMP:6:2} ${TIMESTAMP:8:2}:${TIMESTAMP:10:2}:${TIMESTAMP:12:2} UTC" + echo "Last nightly published at: $SINCE_DATE" + + # Step 6: Check for new commits since last nightly + COMMITS_SINCE=$(git log --since="$SINCE_DATE" --oneline | wc -l) + + if [ "$COMMITS_SINCE" -eq 0 ]; then + echo "No new commits since last nightly, skipping publish" + echo "should_publish=false" >> "$GITHUB_OUTPUT" + exit 1 + else + echo "Found $COMMITS_SINCE new commit(s) since last nightly" + echo "should_publish=true" >> "$GITHUB_OUTPUT" + exit 0 + fi +} + +main diff --git a/.github/workflows/nightly-pypi.yml b/.github/workflows/nightly-pypi.yml index 07a9066..5d99e43 100644 --- a/.github/workflows/nightly-pypi.yml +++ b/.github/workflows/nightly-pypi.yml @@ -23,9 +23,16 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.11" + + - name: Check for new commits since last nightly + id: check + if: github.ref_type != 'tag' + run: bash .github/scripts/check_new_commits.sh + continue-on-error: true + - name: Compute nightly version from latest tag (next patch + timestamp) id: ver - if: github.ref_type != 'tag' + if: github.ref_type != 'tag' && steps.check.outputs.should_publish != 'false' run: | # Get latest tag; allow 'v' prefix; fail if none if ! TAG=$(git describe --tags --abbrev=0 2>/dev/null); then @@ -46,6 +53,7 @@ jobs: echo "Computed nightly version: ${NEXT}.dev${DATE}" - name: Build sdist/wheel + if: github.ref_type == 'tag' || steps.check.outputs.should_publish != 'false' run: | python -m pip install --upgrade pip pip install build setuptools-scm @@ -55,12 +63,13 @@ jobs: python -m build - name: Check metadata + if: github.ref_type == 'tag' || steps.check.outputs.should_publish != 'false' run: | pip install twine twine check dist/* - name: Publish to PyPI (API token) - if: github.event_name == 'schedule' || github.ref_type == 'tag' + if: (github.event_name == 'schedule' || github.ref_type == 'tag') && steps.check.outputs.should_publish != 'false' uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ From 8e08eb74906d04b93a62b04b7390b31e7750c301 Mon Sep 17 00:00:00 2001 From: Yueming Hao Date: Mon, 8 Dec 2025 17:22:45 -0800 Subject: [PATCH 2/4] remove extra space Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/scripts/check_new_commits.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/check_new_commits.sh b/.github/scripts/check_new_commits.sh index b32fc72..813405f 100755 --- a/.github/scripts/check_new_commits.sh +++ b/.github/scripts/check_new_commits.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) Meta Platforms, Inc. and affiliates. +# Copyright (c) Meta Platforms, Inc. and affiliates. # Check if there are new commits since the last nightly PyPI release. # This script is used by nightly-pypi.yml to implement on-demand nightly publishing. # From bb609442617bba179a113d2562e6dcd90787a4a9 Mon Sep 17 00:00:00 2001 From: Yueming Hao Date: Mon, 8 Dec 2025 17:32:58 -0800 Subject: [PATCH 3/4] fix: address code review feedback - Add comments explaining why set -e is not used (fail-open design) - Add comments documenting jq/curl dependencies - Explicitly capture jq exit status and check for non-empty result - Replace grep -oP with sed for better portability - Remove unnecessary continue-on-error: true from workflow --- .github/scripts/check_new_commits.sh | 10 ++++++++-- .github/workflows/nightly-pypi.yml | 1 - 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/scripts/check_new_commits.sh b/.github/scripts/check_new_commits.sh index 813405f..ee1926c 100755 --- a/.github/scripts/check_new_commits.sh +++ b/.github/scripts/check_new_commits.sh @@ -11,6 +11,8 @@ # Sets GITHUB_OUTPUT variable: should_publish=true/false set -o pipefail +# Note: set -e is intentionally not used to allow explicit error handling. +# The script implements fail-open behavior where errors should not block publishing. # Configuration MAX_RETRIES=3 @@ -18,6 +20,9 @@ RETRY_DELAY=5 CURL_TIMEOUT=10 PACKAGE_NAME="${PACKAGE_NAME:-tritonparse}" +# Dependencies: This script requires 'jq' and 'curl' to be installed. +# These are pre-installed on ubuntu-latest GitHub Actions runners. + # Function: Fetch latest nightly version from PyPI with retry fetch_latest_nightly() { local retries=0 @@ -34,8 +39,9 @@ fetch_latest_nightly() { latest=$(echo "$response" | \ jq -r '.releases | keys[] | select(contains(".dev"))' 2>/dev/null | \ sort -V | tail -1) + local jq_exit=$? - if [ $? -eq 0 ]; then + if [ $jq_exit -eq 0 ] && [ -n "$latest" ]; then echo "$latest" return 0 fi @@ -79,7 +85,7 @@ main() { echo "Latest nightly on PyPI: $LATEST_NIGHTLY" # Step 4: Extract timestamp from version (format: X.Y.Z.devYYYYMMDDHHMMSS) - TIMESTAMP=$(echo "$LATEST_NIGHTLY" | grep -oP '\.dev\K[0-9]{14}' || true) + TIMESTAMP=$(echo "$LATEST_NIGHTLY" | sed -n 's/.*\.dev\([0-9]\{14\}\).*/\1/p') if [ -z "$TIMESTAMP" ]; then echo "::warning::Cannot parse timestamp from version, proceeding with publish" diff --git a/.github/workflows/nightly-pypi.yml b/.github/workflows/nightly-pypi.yml index 5d99e43..1a04254 100644 --- a/.github/workflows/nightly-pypi.yml +++ b/.github/workflows/nightly-pypi.yml @@ -28,7 +28,6 @@ jobs: id: check if: github.ref_type != 'tag' run: bash .github/scripts/check_new_commits.sh - continue-on-error: true - name: Compute nightly version from latest tag (next patch + timestamp) id: ver From accb11d1ca614c9471e785d0ad0507e824c1f274 Mon Sep 17 00:00:00 2001 From: Yueming Hao Date: Mon, 8 Dec 2025 20:40:33 -0800 Subject: [PATCH 4/4] change return value to 0 Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/scripts/check_new_commits.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/check_new_commits.sh b/.github/scripts/check_new_commits.sh index ee1926c..39b015a 100755 --- a/.github/scripts/check_new_commits.sh +++ b/.github/scripts/check_new_commits.sh @@ -103,7 +103,7 @@ main() { if [ "$COMMITS_SINCE" -eq 0 ]; then echo "No new commits since last nightly, skipping publish" echo "should_publish=false" >> "$GITHUB_OUTPUT" - exit 1 + exit 0 else echo "Found $COMMITS_SINCE new commit(s) since last nightly" echo "should_publish=true" >> "$GITHUB_OUTPUT"