sync.yml

name: Sync upstream releases

# Upstream ships roughly monthly, so this is quiet by design: on most days the
# ingest finds nothing, `data/` is byte-identical, and the job ends without
# opening anything.
on:
  schedule:
    - cron: "17 6 * * *"
  workflow_dispatch:

permissions:
  contents: write
  issues: write

jobs:
  sync:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: oven-sh/setup-bun@v2
        with:
          bun-version: 1.3.14

      - run: bun install --frozen-lockfile

      - name: Ingest new releases
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: bun run ingest

      - name: Detect changes
        id: changed
        # A new release is a new, untracked snapshot file, which `git diff`
        # ignores; intent-to-add makes it show up as a change.
        run: |
          git add --intent-to-add data
          if git diff --quiet -- data; then
            echo "any=false" >> "$GITHUB_OUTPUT"
          else
            echo "any=true" >> "$GITHUB_OUTPUT"
            # `v` only where there is a version to prefix: an untagged source is
            # dated, and `v2026-09-01` names a tag that does not exist.
            echo "versions=$(git diff --name-only -- data | grep '\.json\.gz$' | sed -E 's#^data/##; s#\.json\.gz$##; s#/([0-9]+\.[0-9]+)#  v\1#; s#/#  #' | paste -sd, - | sed 's/,/, /g')" >> "$GITHUB_OUTPUT"
          fi

      # Build first: the MCP worker tests read dist/. The tests assert against
      # real releases, so a new snapshot that breaks them means a normalizer no
      # longer understands upstream; that data must not reach main.
      - name: Build and test
        id: test
        if: steps.changed.outputs.any == 'true'
        run: bun run build && bun run test
        continue-on-error: true

      # Green snapshots land on main directly; the Cloudflare Git integration
      # deploys from there. Red ones go to a branch with an issue, because
      # publishing them would mean publishing wrong history.
      - name: Commit
        if: steps.changed.outputs.any == 'true'
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add data
          git commit -m "data: ingest ${{ steps.changed.outputs.versions }}"

      - name: Push to main
        if: steps.changed.outputs.any == 'true' && steps.test.outcome == 'success'
        run: git push origin HEAD:main

      - name: Push branch
        if: steps.changed.outputs.any == 'true' && steps.test.outcome != 'success'
        run: git push --force origin HEAD:sync/upstream

      - name: Update the sync issue
        if: steps.changed.outputs.any == 'true'
        uses: actions/github-script@v7
        env:
          VERSIONS: ${{ steps.changed.outputs.versions }}
          TESTS: ${{ steps.test.outcome }}
        with:
          script: |
            const versions = process.env.VERSIONS
            const passed = process.env.TESTS === "success"
            const repo = { owner: context.repo.owner, repo: context.repo.repo }
            const run = `${context.serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${context.runId}`
            // One open issue at most, tracking the failing branch until it is fixed.
            const { data: open } = await github.rest.issues.listForRepo({
              ...repo,
              state: "open",
              labels: "upstream-sync",
            })
            const existing = open[0]
            if (passed) {
              if (existing) {
                await github.rest.issues.createComment({ ...repo, issue_number: existing.number, body: `Superseded: ${versions} landed on main. Run: ${run}` })
                await github.rest.issues.update({ ...repo, issue_number: existing.number, state: "closed" })
              }
              return
            }
            const compare = `${context.serverUrl}/${repo.owner}/${repo.repo}/compare/main...sync/upstream`
            await (existing ? github.rest.issues.update : github.rest.issues.create)({
              ...repo,
              ...(existing ? { issue_number: existing.number } : {}),
              title: `Upstream sync blocked: ${versions}`,
              labels: ["upstream-sync"],
              body: [
                `New upstream data is on \`sync/upstream\`: ${compare}`,
                "",
                "**Tests failed against the new snapshots.** Upstream has changed a model, a document layout or the proto grammar in a way a normalizer does not yet understand. Fix the normalizer, then merge.",
                "",
                `Run: ${run}`,
              ].join("\n"),
            })

  # A silent scheduled job is indistinguishable from a broken one, so failure
  # has to surface somewhere a human looks.
  notify:
    needs: sync
    if: failure()
    runs-on: ubuntu-latest
    steps:
      - uses: actions/github-script@v7
        with:
          script: |
            const title = "Upstream sync failed"
            const { data: issues } = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              state: "open",
              labels: "sync-failure",
            })
            if (issues.some((issue) => issue.title === title)) return
            await github.rest.issues.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title,
              labels: ["sync-failure"],
              body: `The scheduled ingest failed: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
            })