Spoc 374: Add 'make installcheck' a Spock standard workflow

.github/workflows/installcheck.yml

@@ -0,0 +1,145 @@
+#
+# Installcheck Tests Workflow
+#
+# This workflow runs installcheck tests on node n1 of a three-node Spock cluster.
+# Environment variables are loaded from tests/docker/pgedge.env
+#
+# NOTES:
+# The 'security_label' test has no chance to pass without extra coding. So, just
+# skip it for now. This decision may be revised later if something is changed
+# in the PostgreSQL core.
+# The sad fact is that n2 and n3 can't pass these tests without errors, because
+# the LR is limited and doesn't support many features. That turns their logs
+# into a mess, but we still save them to detect any potential issues.
+#
+
+name: Installcheck Tests (Three-Node Cluster)
+run-name: Running installcheck tests on three-node Spock cluster
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+
+jobs:
+  installcheck:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        pgver: [15, 16, 17, 18]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout spock
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Add permissions
+        run: |
+          sudo chmod -R a+w ${GITHUB_WORKSPACE}
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3
+
+      - name: Set up docker compose
+        uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746 # v1
+        with:
+          version: latest
+
+      - name: Start docker cluster
+        run: |
+          cd ${GITHUB_WORKSPACE}/tests/docker/
+          # To minimize regression tests difference, override pgedge.env with
+          # standard PostgreSQL regression test defaults
+          sed -i 's/^DBUSER=.*/DBUSER=regression/' pgedge.env
+          sed -i 's/^DBNAME=.*/DBNAME=regression/' pgedge.env
+          # Build and start the cluster
+          PGVER=${{ matrix.pgver }} DBUSER=regression DBNAME=regression \
+            docker compose up --build --wait -d
+        timeout-minutes: 20
+
+      - name: Run installcheck on node n1
+        id: installcheck
+        run: |
+          cd ${GITHUB_WORKSPACE}/tests/docker/
+
+          # Run installcheck on n1 and capture output
+          docker compose exec -T pgedge-n1 bash -c '~/run-installcheck.sh' \
+            | tee installcheck_output.txt || true
+
+          # Analyze the output for test failures
+          echo "Analyzing test results..."
+
+          # Check for 'exited with exit code' which indicates a crash or abnormal termination
+          if grep -q "exited with exit code" installcheck_output.txt; then
+            echo "ERROR: Found 'exited with exit code' in output - indicates abnormal termination"
+            echo "test_result=failed" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          # Check for 'not ok' test results, excluding 'security_label'
+          # Extract all 'not ok' lines and filter out security_label
+          FAILED_TESTS=$(grep "not ok" installcheck_output.txt | grep -v "security_label" || true)
+
+          if [ -n "$FAILED_TESTS" ]; then
+            echo "ERROR: Found failing tests (excluding security_label):"
+            echo "$FAILED_TESTS"
+            echo "test_result=failed" >> $GITHUB_OUTPUT
+          else
+            echo "All tests passed (security_label failures are expected and ignored)"
+            echo "test_result=passed" >> $GITHUB_OUTPUT
+          fi
+        timeout-minutes: 30
+
+      - name: Collect installcheck artifacts
+        if: ${{ always() }}
+        run: |
+          cd ${GITHUB_WORKSPACE}/tests/docker/
+          mkdir -p installcheck-logs
+
+          # Copy the output file we captured
+          cp installcheck_output.txt installcheck-logs/
+
+          # Copy regression output from n1 (where tests run)
+          # pg_regress outputs to /home/pgedge/installcheck-output/
+          docker compose cp pgedge-n1:/home/pgedge/installcheck-output/regression.diffs installcheck-logs/
+          docker compose cp pgedge-n1:/home/pgedge/installcheck-output/regression.out installcheck-logs/ || true
+          docker compose cp pgedge-n1:/home/pgedge/installcheck-output/results installcheck-logs/results/ || true
+
+          # Collect PostgreSQL logs from all nodes
+          for node in n1 n2 n3; do
+            echo "Collecting logs from $node..."
+            # PostgreSQL log file (from pg_ctl -l in entrypoint.sh)
+            docker compose cp pgedge-$node:/home/pgedge/logfile.log installcheck-logs/$node-logfile.log || true
+            # Container logs
+            docker compose logs pgedge-$node > installcheck-logs/$node-container.log 2>&1 || true
+          done
+
+      - name: Upload installcheck artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: installcheck-logs-pg${{ matrix.pgver }}
+          path: tests/docker/installcheck-logs/
+          if-no-files-found: error
+          retention-days: 7
+
+      - name: Cleanup docker cluster
+        if: ${{ always() }}
+        run: |
+          cd ${GITHUB_WORKSPACE}/tests/docker/
+          docker compose down
+
+      - name: Check test result
+        run: |
+          if [ "${{ steps.installcheck.outputs.test_result }}" = "failed" ]; then
+            echo "Installcheck tests failed (see artifacts for details)"
+            exit 1
+          fi
+          echo "Installcheck tests passed successfully"

.github/workflows/spockbench.yml

@@ -50,11 +50,84 @@ jobs:
       - name: Build docker container
         run: |
           cd ${GITHUB_WORKSPACE}/
+          # Build docker image once for specific version, needed for this test
           docker build \
             --build-arg PGVER=${{ matrix.pgver }} \
             -t spock -f tests/docker/Dockerfile-step-1.el9 .
 
+      - name: Start docker cluster
+        run: |
+          cd ${GITHUB_WORKSPACE}/tests/docker/
+          echo PG_VER=${{ matrix.pgver }} >> pgedge.env
+          docker compose up --wait -d
+        timeout-minutes: 20
+
+      - name: Run tests on all nodes
+        run: |
+          cd ${GITHUB_WORKSPACE}/tests/docker/
+
+          # Launch tests in background with per-node timeout and capture PIDs
+          docker compose exec -T pgedge-n1 bash -c "~/tests/run-tests.sh" &
+          PID1=$!
+          docker compose exec -T pgedge-n2 bash -c "~/tests/run-tests.sh" &
+          PID2=$!
+          docker compose exec -T pgedge-n3 bash -c "~/tests/run-tests.sh" &
+          PID3=$!
+
+          # Wait for all jobs and capture their exit codes
+          wait $PID1
+          EXIT1=$?
+          wait $PID2
+          EXIT2=$?
+          wait $PID3
+          EXIT3=$?
+
+          # Fail if any node failed
+          if [ $EXIT1 -ne 0 ] || [ $EXIT2 -ne 0 ] || [ $EXIT3 -ne 0 ]; then
+            echo "ERROR: One or more nodes failed"
+            exit 1
+          fi
+
+          echo "All nodes completed successfully"
+        timeout-minutes: 10
+
+      - name: Collect node logs
+        if: ${{ always() }}
+        run: |
+          cd ${GITHUB_WORKSPACE}/tests/docker/
+          mkdir -p node-logs
+
+          # Collect PostgreSQL logs and spockbench output from each node
+          for node in n1 n2 n3; do
+            echo "Collecting logs from $node..."
+            docker compose cp pgedge-$node:/home/pgedge/pgedge/data/pg${{ matrix.pgver }}/log node-logs/$node-pg-log/ || true
+            docker compose cp pgedge-$node:/home/pgedge/spock/spockbench-$node.out node-logs/ || true
+            docker compose logs pgedge-$node > node-logs/$node-container.log 2>&1 || true
+          done
+
+      - name: Upload node logs
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: spockbench-node-logs-${{ matrix.pgver }}
+          path: tests/docker/node-logs/
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - name: Check spockbench output
+        if: ${{ always() }}
+        run: |
+          cd ${GITHUB_WORKSPACE}/tests/docker
+          ./check-outputs.sh
+
+      - name: Cleanup spockbench docker container
+        if: ${{ always() }}
+        run: |
+          cd ${GITHUB_WORKSPACE}/tests/docker/
+          docker compose down
+
       - name: Run regression tests
+        if: ${{ always() }}
         run: |
           REG_CT_NAME="spock-regress-${{ matrix.pgver }}-${{ github.run_id }}-${{ github.run_attempt }}"
           echo "REG_CT_NAME=$REG_CT_NAME" >> "$GITHUB_ENV"
@@ -78,6 +151,7 @@ jobs:
           retention-days: 7
 
       - name: Run TAP tests
+        if: ${{ always() }}
         run: |
           TAP_CT_NAME="spock-tap-${{ matrix.pgver }}-${{ github.run_id }}-${{ github.run_attempt }}"
           echo "TAP_CT_NAME=$TAP_CT_NAME" >> "$GITHUB_ENV"
@@ -102,15 +176,3 @@ jobs:
             tests/logs/**
           if-no-files-found: ignore
           retention-days: 7
-
-      - name: Start docker
-        run: |
-          cd ${GITHUB_WORKSPACE}/tests/docker/
-          echo PG_VER=${{ matrix.pgver }} >> pgedge.env
-          docker compose up
-
-      - name: Check spockbench output
-        run: |
-          cd ${GITHUB_WORKSPACE}/tests/docker
-          ./check-outputs.sh
-

sql/spock--6.0.0-devel.sql

@@ -281,9 +281,18 @@ RETURNS oid CALLED ON NULL INPUT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spoc
 CREATE FUNCTION spock.repset_drop(set_name name, ifexists boolean DEFAULT false)
 RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_drop_replication_set';
 
-CREATE FUNCTION spock.repset_add_table(set_name name, relation regclass, synchronize_data boolean DEFAULT false,
-	columns text[] DEFAULT NULL, row_filter text DEFAULT NULL, include_partitions boolean default true)
-RETURNS boolean CALLED ON NULL INPUT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_replication_set_add_table';
+CREATE FUNCTION spock.repset_add_table(
+  set_name           name,
+  relation           regclass,
+  synchronize_data   boolean DEFAULT false,
+  columns            text[] DEFAULT NULL,
+  row_filter         text DEFAULT NULL,
+  include_partitions boolean default true
+)
+RETURNS boolean
+AS 'MODULE_PATHNAME', 'spock_replication_set_add_table'
+LANGUAGE C CALLED ON NULL INPUT VOLATILE;
+
 CREATE FUNCTION spock.repset_add_all_tables(set_name name, schema_names text[], synchronize_data boolean DEFAULT false)
 RETURNS boolean STRICT VOLATILE LANGUAGE c AS 'MODULE_PATHNAME', 'spock_replication_set_add_all_tables';
 CREATE FUNCTION spock.repset_remove_table(set_name name, relation regclass, include_partitions boolean default true)
@@ -652,9 +661,9 @@ $$ LANGUAGE plpgsql;
 
 -- Set delta_apply security label on specific column
 CREATE FUNCTION spock.delta_apply(
-  rel regclass,
+  rel      regclass,
   att_name name,
-  to_drop boolean DEFAULT false
+  to_drop  boolean DEFAULT false
 ) RETURNS boolean AS $$
 DECLARE
   label     text;

src/spock_apply_heap.c

@@ -928,8 +928,7 @@ spock_apply_heap_insert(SpockRelation *rel, SpockTupleData *newtup)
 									remoteslot, &localslot,
 									true);
 
-	if (check_all_uc_indexes &&
-		!found)
+	if (check_all_uc_indexes && !found)
 	{
 		/*
 		 * Handle the special case of looking through all unique indexes

src/spock_conflict.c

@@ -348,7 +348,6 @@ spock_report_conflict(ConflictType conflict_type,
 	const char *idxname = "(unknown)";
 	const char *qualrelname;
 
-
 	/* Ignore update-update conflict for same origin */
 	if (conflict_type == CT_UPDATE_EXISTS)
 	{

src/spock_relcache.c

@@ -156,6 +156,7 @@ spock_relation_open(uint32 remoteid, LOCKMODE lockmode)
 			 * value of the relreplident field. But it is just a cache ...
 			 */
 			relinfo->ri_RelationDesc->rd_rel->relreplident = REPLICA_IDENTITY_DEFAULT;
+			relinfo->ri_RelationDesc->rd_indexvalid = false;
 			entry->idxoid = RelationGetReplicaIndex(relinfo->ri_RelationDesc);
 			Assert(entry->idxoid != InvalidOid);
 			relinfo->ri_RelationDesc->rd_rel->relreplident = REPLICA_IDENTITY_FULL;

tests/docker/Dockerfile-base.md

@@ -35,27 +35,33 @@ The image includes **all** libraries required to build PostgreSQL with maximum f
 
 | Library | Purpose | Configure Flag |
 |---------|---------|----------------|
-| `zstd-devel` | Zstandard compression | `--with-zstd` |
+| `libzstd-devel` | Zstandard compression | `--with-zstd` |
 | `lz4-devel` | LZ4 compression | `--with-lz4` |
 | `libicu-devel` | Unicode and internationalization | `--with-icu` |
 | `libxml2-devel` | XML support | `--with-libxml` |
 | `libxslt-devel` | XSLT transformations | `--with-libxslt` |
 | `openssl-devel` | SSL/TLS connections | `--with-openssl` |
 | `krb5-devel` | Kerberos authentication | `--with-gssapi` |
+| `cyrus-sasl-gssapi` | SASL GSSAPI support | Related to GSSAPI |
 | `openldap-devel` | LDAP authentication | `--with-ldap` |
 | `pam-devel` | PAM authentication | `--with-pam` |
 | `systemd-devel` | Systemd integration | `--with-systemd` |
 | `python3-devel` | PL/Python language | `--with-python` |
 | `readline-devel` | Enhanced psql CLI | Built-in |
 | `llvm-devel` | JIT compilation | `--with-llvm` |
-| `libuuid-devel` | UUID generation | `--with-uuid=ossp` |
+| `libuuid-devel`, `uuid-devel` | UUID generation | `--with-uuid=ossp` |
+| `libpq`, `libpq-devel` | PostgreSQL client library | Development headers |
+| `jansson-devel` | JSON parsing | For extensions |
+| `zlib-devel` | Compression library | Built-in |
+| `pkgconfig` | Package config tool | Build system helper |
 
 ### 3. Testing Infrastructure
 
-- **Perl Testing Framework**: `perl-IPC-Run`, `Test::More` for PostgreSQL TAP tests
-- **SSH Configuration**: Pre-configured SSH keys for multi-node testing scenarios
+- **Perl Testing Framework**: `perl-IPC-Run`, `perl-Test-Simple` (includes Test::More) for PostgreSQL TAP tests
+- **SSH Configuration**: Pre-configured SSH keys and `openssh-clients`, `openssh-server` for multi-node testing scenarios
 - **Network Tools**: `nc` (netcat), `bind-utils` (dig, nslookup) for connectivity testing
 - **Process Tools**: `procps` for monitoring and debugging
+- **Utility Tools**: `curl`, `unzip` for downloading and extracting archives
 
 ### 4. User Configuration
 
@@ -77,10 +83,7 @@ The image includes **all** libraries required to build PostgreSQL with maximum f
 1. **System Packages** (~500MB compressed):
    - Rocky Linux 9 base system updates
    - Development Tools group install
-   - 40+ development packages and their dependencies
-
-2. **Perl Modules** (via CPAN):
-   - `Test::More` - PostgreSQL TAP test framework
+   - 40+ development packages and their dependencies (all installed via dnf)
 
 ## Image Size and Optimization
 
@@ -96,7 +99,6 @@ This large size is **intentional and appropriate** for a development/testing bas
 ```dockerfile
 dnf clean all
 rm -rf /var/cache/dnf/* /tmp/* /var/tmp/*
-rm -rf /root/.cpanm
 ```
 
 ## Usage Examples