diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 4591125d1..d9a36cb44 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -135,41 +135,3 @@ jobs: image: ${{ matrix.image }} custom_tag: ${{ matrix.image == 'dev-go' && needs.envs.outputs.DEV_TAG || '' }} secrets: inherit - - fly-deploy-staging: - needs: detect-changes - if: ${{ github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && needs.detect-changes.outputs.services != '[]' && needs.detect-changes.outputs.services != '' }} - strategy: - fail-fast: false - max-parallel: 2 - matrix: - service: ${{ fromJson(needs.detect-changes.outputs.services) }} - uses: ./.github/workflows/fly-deploy.yaml - with: - service: ${{ matrix.service }} - app: ${{ vars.FLY_ORGANIZATION_STAGING }}-${{ matrix.service }} - organization: ${{ vars.FLY_ORGANIZATION_STAGING }} - version: ${{ needs.detect-changes.outputs.commit_hash }} - environment: "staging" - secrets: inherit - - fly-deploy-prod: - needs: detect-changes - # TODO - # For now, it's ok to deploy if ANY version tag is pushed to (we assume it's the newest version and needs deployment) - # In the future we need to be able to deploy to different, API incompatible, production environments - # Maybe even automate resource creation - if: ${{ github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/v') && needs.detect-changes.outputs.services != '[]' && needs.detect-changes.outputs.services != '' }} - strategy: - fail-fast: false - max-parallel: 2 - matrix: - service: ${{ fromJson(needs.detect-changes.outputs.services) }} - uses: ./.github/workflows/fly-deploy.yaml - with: - service: ${{ matrix.service }} - app: ${{ vars.FLY_ORGANIZATION_PROD }}-${{ matrix.service }} - organization: ${{ vars.FLY_ORGANIZATION_PROD }} - version: ${{ needs.detect-changes.outputs.commit_hash }} - environment: "production" - secrets: inherit diff --git a/.github/workflows/fly-deploy.yaml b/.github/workflows/fly-deploy.yaml deleted file mode 100644 index c0e974f2c..000000000 --- a/.github/workflows/fly-deploy.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Deploys a single service to fly based on inputs provided -# does nothing if no fly.toml exists - -name: 'Deploy to Fly' - -env: - # renovate: datasource=github-releases depName=superfly/flyctl versioning=semver - FLY_CLI_VERSION: '0.3.57' - -on: - workflow_call: - inputs: - service: - description: 'The name of the service (name of the directory).' - type: string - required: true - app: - description: 'The name of the app on Fly.' - type: string - required: true - organization: - description: 'The name of the organization on Fly.' - type: string - required: true - version: - description: "The version of the app." - type: string - required: false - environment: - description: - Which config file to choose. - Currently either "staging" or "production". - If no such fly..toml exists, we fallback to fly.toml. - type: string - required: false - default: "" - -concurrency: - group: ${{ github.workflow }}-${{ inputs.app }} - cancel-in-progress: true - -jobs: - - fly-toml: - runs-on: ubuntu-22.04 - outputs: - fly_toml: ${{ steps.fly-toml.outputs.fly_toml }} - steps: - - uses: actions/checkout@v4 - - name: Does a fly.toml exist? - id: fly-toml - run: | - # fallback to fly.toml if needed - fly_toml=$([ "$suggested_fly_toml" = "fly..toml" ] && echo "fly.toml" || echo "$suggested_fly_toml") - # only set if it exists - echo "fly_toml=$((test -f services/$service/$fly_toml && echo $fly_toml) || (test -f services/$service/fly.toml && echo fly.toml))" >> $GITHUB_OUTPUT - env: - service: ${{ inputs.service }} - suggested_fly_toml: fly.${{ inputs.environment }}.toml - - deploy: - runs-on: ubuntu-22.04 - needs: fly-toml - if: ${{ needs.fly-toml.outputs.fly_toml != '' }} - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - steps: - - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 - - name: Configure Dapr for deployment - run: sed -i "s/helpwave-staging/$organization/g" dapr/config.yaml - env: - organization: ${{ inputs.organization }} - - name: Setup flyctl - uses: superfly/flyctl-actions/setup-flyctl@master - with: - version: ${{ env.FLY_CLI_VERSION }} - - name: Deploy - run: | - flyctl deploy --config services/${{ inputs.service }}/${{ needs.fly-toml.outputs.fly_toml }} --app ${{ inputs.app }} --build-arg VERSION=${{ inputs.version }} --remote-only --wait-timeout=2m - env: - DOCKER_BUILDKIT: 1 diff --git a/.github/workflows/migrations.yaml b/.github/workflows/migrations.yaml deleted file mode 100644 index e6712c0b8..000000000 --- a/.github/workflows/migrations.yaml +++ /dev/null @@ -1,219 +0,0 @@ -name: migrations - -on: - pull_request: - -permissions: - contents: read - pull-requests: read - -env: - # renovate: datasource=github-releases depName=golang-migrate/migrate - MIGRATE_VERSION: '4.18.1' - # renovate: datasource=github-releases depName=superfly/flyctl versioning=semver - FLY_CLI_VERSION: '0.3.57' - # renovate: datasource=github-releases depName=sqlc-dev/sqlc - SQLC_VERSION: '1.27.0' - -jobs: - detect-changes: - runs-on: ubuntu-22.04 - outputs: - # Stringified JSON Array of changed services - services: ${{ steps.changes.outputs.all_changed_files }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # Collect the services, which have changes in their migrations directory. - # Available in ${{ steps.changes.outputs.all_changed_files }} - # as a json array (it's a string) - - name: Detect changes in migrations - id: changes - uses: tj-actions/changed-files@v44 - with: - json: "true" - escape_json: "false" - dir_names: "true" - dir_names_exclude_current_dir: "true" - dir_names_max_depth: 1 - path: "services" - files: ./*/migrations/** - - # Summary for debugging - - name: Summarize - run: | - echo "services: $services" >> $GITHUB_STEP_SUMMARY - env: - services: ${{ steps.changes.outputs.all_changed_files }} - - disallow-modifications: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Detect modifications in migrations - id: changes - uses: tj-actions/changed-files@v44 - with: - json: "true" - escape_json: "false" - path: "services" - files: ./*/migrations/** - - name: Don't modify existing migrations! - if: steps.changes.outputs.modified_files != '[]' - uses: helpwave/pg-fingerprint-action@main - with: - root: "services" - github_token: ${{ secrets.GITHUB_TOKEN }} - files: ${{ steps.changes.outputs.modified_files }} - - migrations: - runs-on: ubuntu-22.04 - needs: detect-changes - if: needs.detect-changes.outputs.services != '[]' - strategy: - matrix: - service: ${{ fromJson(needs.detect-changes.outputs.services) }} - - env: - svc: ${{ matrix.service }} - FLY_DB_APP: ${{ secrets.STAGING_DB_FLY_DB_APP }} - REMOTE_DB: ${{ vars.STAGING_DB_REMOTE_DB_PREFIX }}${{ matrix.service }} - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres # will be overwritten in "Pull staging data" - - services: - postgres: - image: postgres:15.6 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - env: - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - steps: - - name: Setup flyctl - uses: superfly/flyctl-actions/setup-flyctl@master - with: - version: ${{ env.FLY_CLI_VERSION }} - # ubuntu-22.04 ships with psql 14.x, - # whose pg_dump is incompatible with our postgres server (15.x) - - name: Setup postgres client - run: | - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null - sudo apt update - sudo apt remove postgresql-client - sudo apt install postgresql-client-15 -y - /usr/lib/postgresql/15/bin/pg_dump --version - - name: install migrate - run: | - TEMP_DEB="$(mktemp)" - wget -O "$TEMP_DEB" 'https://github.com/golang-migrate/migrate/releases/download/v${{ env.MIGRATE_VERSION }}/migrate.linux-amd64.deb' - sudo dpkg -i "$TEMP_DEB" - rm -f "$TEMP_DEB" - - name: install sqlc - run: wget -qO- https://downloads.sqlc.dev/sqlc_${{ env.SQLC_VERSION }}_linux_amd64.tar.gz | sudo tar xvz -C /usr/bin - - name: Clone Repo - uses: actions/checkout@v4 - - name: Open tunnel to staging db - run: | - flyctl proxy 5431:5432 -a $FLY_DB_APP & - count=0 - max_attempts=10 - - until pg_isready -h localhost -p 5431; do - if [ $count -ge $max_attempts ]; then - echo "PostgreSQL is not ready after $max_attempts attempts. Exiting..." - exit 1 - fi - - echo "Waiting for PostgreSQL to be ready... (Attempt $((count+1))/$max_attempts)" - sleep 3 - - count=$((count+1)) - done - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - - name: Pull staging data - run: | - # service-name -> service_name - REMOTE_DB=$(echo "$REMOTE_DB" | sed "s/-/_/g") - # Build URIs - REMOTE=postgres://$STAGING_DB_USER:$STAGING_DB_PASS@localhost:5431/$REMOTE_DB - LOCAL=postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB - # Remote -> SQL -> Local - /usr/lib/postgresql/15/bin/pg_dump -C -c --if-exists --no-comments -O -x $REMOTE -f dump.sql - /usr/lib/postgresql/15/bin/psql -q $LOCAL -f dump.sql - # Use newly created database from now on - echo "POSTGRES_DB=$REMOTE_DB" >> "$GITHUB_ENV" - env: - STAGING_DB_USER: ${{ secrets.STAGING_DB_USER }} - STAGING_DB_PASS: ${{ secrets.STAGING_DB_PASS }} - - name: Collect current version - id: collect-version - run: | - echo -n "VERSION=" > $GITHUB_OUTPUT - - # for some reason, beyond my comprehension, - # the output of migrate is sent to stderr in the CI, but stdout on local - version_output=$(./migrate.sh $svc version 2>&1 | tail -n1) - - # Check if the version_output is a number - if [[ $version_output =~ ^[0-9]+$ ]]; then - echo $version_output >> $GITHUB_OUTPUT - else - echo "Error: Version is not a number: $version_output" >&2 - echo "0" >> $GITHUB_OUTPUT - fi - - name: Check version - run: | - ./migrate.sh $svc desired - if [ "$current" -ge "$(./migrate.sh $svc desired)" ]; then - echo "Migrations must be newer than the version of staging! You probably lack behind, merge or rebase onto main first!" - exit 1 - fi - env: - current: ${{ steps.collect-version.outputs.VERSION }} - - - name: Run UP migrations (1/2) - run: ./migrate.sh $svc up - - name: Generate schema.sql - run: | - /usr/lib/postgresql/15/bin/pg_dump \ - postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB \ - --schema-only -O > ./services/${{ matrix.service }}/schema.sql - - sed 's/\(-- Dumped from database version [0-9]\+\.[0-9]\+\).*/\1/' -i ./services/${{ matrix.service }}/schema.sql - sed 's/\(-- Dumped by pg_dump version [0-9]\+\.[0-9]\+\).*/\1/' -i ./services/${{ matrix.service }}/schema.sql - - name: Generate models - run: cd services/${{ matrix.service }} && sqlc generate - - name: Verify schema.sql and models were generated properly before - run: git diff --exit-code -- services/${{ matrix.service }} || (echo "You forgot to run ./models.sh before checking in" && exit 1) - - name: Run DOWN migrations (1/2) - run: | - [ "$VERSION" -eq 0 ] && yes | ./migrate.sh $svc down || ./migrate.sh $svc goto $VERSION - env: - VERSION: ${{ steps.collect-version.outputs.VERSION }} - - name: Run UP migrations (2/2) - run: ./migrate.sh $svc up - - name: Run DOWN migrations (2/2) - run: | - [ "$VERSION" -eq 0 ] && yes | ./migrate.sh $svc down || ./migrate.sh $svc goto $VERSION - env: - VERSION: ${{ steps.collect-version.outputs.VERSION }} diff --git a/images/dev-go/Dockerfile b/images/dev-go/Dockerfile index c85f15c48..a5065b972 100644 --- a/images/dev-go/Dockerfile +++ b/images/dev-go/Dockerfile @@ -90,11 +90,6 @@ RUN mkdir "$HOME/dapr" \ && wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - | DAPR_INSTALL_DIR="$HOME/dapr" /bin/bash -s ${DAPR_CLI_VERSION} \ && dapr init -s --runtime-version ${DAPR_RT_VERSION} -# install fly -# https://fly.io/docs/ -RUN curl -L https://fly.io/install.sh | sh -ENV PATH="$PATH:$HOME/.fly/bin" - # install go-migrate # https://github.com/golang-migrate/migrate # renovate: datasource=github-releases depName=golang-migrate/migrate