diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0eb7f4502..14454f924 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -161,6 +161,7 @@ jobs: scripts/ci/prepull-images.sh \ mysql:8.0 \ mysql:8.4 \ + postgres:16 \ localstack/localstack:3.0 \ testcontainers/ryuk:0.13.0 - name: "Test" diff --git a/go.mod b/go.mod index 9f21ba6e1..0ea025370 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 github.com/hmarr/codeowners v1.2.1 + github.com/jackc/pgx/v5 v5.10.0 github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17 github.com/planetscale/planetscale-go v0.155.0 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 @@ -35,6 +36,7 @@ require ( github.com/testcontainers/testcontainers-go v0.40.0 github.com/testcontainers/testcontainers-go/modules/localstack v0.40.0 github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 @@ -161,6 +163,9 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/serf v0.10.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.6 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 259c01699..0536139d3 100644 --- a/go.sum +++ b/go.sum @@ -364,6 +364,14 @@ github.com/hmarr/codeowners v1.2.1 h1:+9yndrwG0UVP1GkLBEQMSbSUNeLpbrbL924SRthA/9 github.com/hmarr/codeowners v1.2.1/go.mod h1:KPlR1p/B4owPjwfNIBueWlOP4CmqlQFX9b6nANG6j40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -397,6 +405,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/linkdata/deadlock v0.5.5 h1:d6O+rzEqasSfamGDA8u7bjtaq7hOX8Ha4Zn36Wxrkvo= github.com/linkdata/deadlock v0.5.5/go.mod h1:tXb28stzAD3trzEEK0UJWC+rZKuobCoPktPYzebb1u0= github.com/lmittmann/tint v1.1.3 h1:Hv4EaHWXQr+GTFnOU4VKf8UvAtZgn0VuKT+G0wFlO3I= @@ -420,6 +430,8 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= @@ -625,6 +637,8 @@ github.com/testcontainers/testcontainers-go/modules/localstack v0.40.0 h1:b+lN2C github.com/testcontainers/testcontainers-go/modules/localstack v0.40.0/go.mod h1:8LuTSboTo2MJKFKV5xH6z4ZH1s3jhRJWwvtPJzKogj4= github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0 h1:P9Txfy5Jothx2wFdcus0QoSmX/PKSIXZxrTbZPVJswA= github.com/testcontainers/testcontainers-go/modules/mysql v0.40.0/go.mod h1:oZPHHqJqXG7FD8OB/yWH7gLnDvZUlFHAVJNrGftL+eg= +github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0 h1:s2bIayFXlbDFexo96y+htn7FzuhpXLYJNnIuglNKqOk= +github.com/testcontainers/testcontainers-go/modules/postgres v0.40.0/go.mod h1:h+u/2KoREGTnTl9UwrQ/g+XhasAT8E6dClclAADeXoQ= github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= diff --git a/pkg/schema/postgres/applies.sql b/pkg/schema/postgres/applies.sql new file mode 100644 index 000000000..3d76b00f5 --- /dev/null +++ b/pkg/schema/postgres/applies.sql @@ -0,0 +1,42 @@ +CREATE TABLE applies ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + apply_identifier varchar(255) NOT NULL, + lock_id bigint NOT NULL, + plan_id bigint NOT NULL, + database_name varchar(255) NOT NULL, + database_type varchar(50) NOT NULL, + repository varchar(255) NOT NULL, + pull_request bigint NOT NULL, + environment varchar(50) NOT NULL, + deployment varchar(255) NOT NULL DEFAULT '', + caller varchar(255) NOT NULL DEFAULT '', + installation_id bigint NOT NULL DEFAULT 0, + external_id varchar(255) NOT NULL DEFAULT '', + idempotency_key varchar(255) DEFAULT NULL, + engine varchar(50) NOT NULL, + state varchar(100) NOT NULL, + error_message text, + options jsonb NOT NULL, + attempt integer NOT NULL DEFAULT 0, + lease_owner varchar(255) NOT NULL DEFAULT '', + lease_token varchar(64) NOT NULL DEFAULT '', + lease_acquired_at timestamp DEFAULT NULL, + started_at timestamp DEFAULT NULL, + completed_at timestamp DEFAULT NULL, + revert_skipped_at timestamp DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE UNIQUE INDEX idx_applies_apply_identifier ON applies (apply_identifier); +CREATE UNIQUE INDEX idx_applies_idempotency_key ON applies (idempotency_key); +CREATE INDEX idx_applies_lock_id ON applies (lock_id); +CREATE INDEX idx_applies_plan_id ON applies (plan_id); +CREATE INDEX idx_applies_database_env ON applies (database_name, database_type, environment); +CREATE INDEX idx_applies_database_env_deployment ON applies (database_name, database_type, environment, deployment); +CREATE INDEX idx_applies_repo_pr ON applies (repository, pull_request); +CREATE INDEX idx_applies_created_id ON applies (created_at, id); +CREATE INDEX idx_applies_environment_created_id ON applies (environment, created_at, id); +CREATE INDEX idx_applies_state_created_id ON applies (state, created_at, id); +CREATE INDEX idx_applies_environment_state_created_id ON applies (environment, state, created_at, id); +CREATE INDEX idx_applies_completed_at_state ON applies (completed_at, state); diff --git a/pkg/schema/postgres/apply_comments.sql b/pkg/schema/postgres/apply_comments.sql new file mode 100644 index 000000000..e0c863cce --- /dev/null +++ b/pkg/schema/postgres/apply_comments.sql @@ -0,0 +1,17 @@ +CREATE TABLE apply_comments ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + apply_id bigint NOT NULL, + comment_state varchar(50) NOT NULL, + github_comment_id bigint NOT NULL, + posted_volume integer DEFAULT NULL, + posted_phase varchar(32) DEFAULT NULL, + pending_freeze_github_comment_id bigint DEFAULT NULL, + edit_count integer NOT NULL DEFAULT 0, + last_edited_at timestamp DEFAULT NULL, + superseded_at timestamp DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE UNIQUE INDEX idx_apply_comments_apply_comment_state ON apply_comments (apply_id, comment_state); +CREATE INDEX idx_apply_comments_github_comment ON apply_comments (github_comment_id); diff --git a/pkg/schema/postgres/apply_control_requests.sql b/pkg/schema/postgres/apply_control_requests.sql new file mode 100644 index 000000000..513cfab90 --- /dev/null +++ b/pkg/schema/postgres/apply_control_requests.sql @@ -0,0 +1,15 @@ +CREATE TABLE apply_control_requests ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + apply_id bigint NOT NULL, + operation varchar(50) NOT NULL, + status varchar(50) NOT NULL, + requested_by varchar(255) NOT NULL DEFAULT '', + error_message text, + metadata jsonb NOT NULL, + completed_at timestamp DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE UNIQUE INDEX idx_apply_control_requests_apply_operation ON apply_control_requests (apply_id, operation); +CREATE INDEX idx_apply_control_requests_status ON apply_control_requests (status, created_at); diff --git a/pkg/schema/postgres/apply_logs.sql b/pkg/schema/postgres/apply_logs.sql new file mode 100644 index 000000000..4c77fce84 --- /dev/null +++ b/pkg/schema/postgres/apply_logs.sql @@ -0,0 +1,18 @@ +CREATE TABLE apply_logs ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + apply_id bigint NOT NULL, + task_id bigint DEFAULT NULL, + level varchar(20) NOT NULL DEFAULT 'info', + event_type varchar(100) NOT NULL, + source varchar(100) NOT NULL DEFAULT 'schemabot', + message text NOT NULL, + old_state varchar(100) DEFAULT NULL, + new_state varchar(100) DEFAULT NULL, + metadata jsonb DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE INDEX idx_apply_logs_apply_created ON apply_logs (apply_id, created_at); +CREATE INDEX idx_apply_logs_task_id ON apply_logs (task_id); +CREATE INDEX idx_apply_logs_level ON apply_logs (level); +CREATE INDEX idx_apply_logs_event_type ON apply_logs (event_type); diff --git a/pkg/schema/postgres/apply_operations.sql b/pkg/schema/postgres/apply_operations.sql new file mode 100644 index 000000000..d6d625f14 --- /dev/null +++ b/pkg/schema/postgres/apply_operations.sql @@ -0,0 +1,29 @@ +CREATE TABLE apply_operations ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + apply_id bigint NOT NULL, + deployment varchar(255) NOT NULL, + operation_key varchar(255) NOT NULL DEFAULT '', + operation_kind varchar(32) NOT NULL DEFAULT 'work', + target varchar(255) NOT NULL DEFAULT '', + external_id varchar(255) DEFAULT NULL, + external_operation_id varchar(255) DEFAULT NULL, + engine_resume_context varchar(255) DEFAULT NULL, + engine_resume_metadata jsonb DEFAULT NULL, + state varchar(100) NOT NULL DEFAULT 'pending', + error_message text, + cutover_policy varchar(16) NOT NULL DEFAULT 'rolling', + on_failure varchar(16) NOT NULL DEFAULT 'halt', + attempt integer NOT NULL DEFAULT 0, + lease_owner varchar(255) NOT NULL DEFAULT '', + lease_token varchar(64) NOT NULL DEFAULT '', + lease_acquired_at timestamp DEFAULT NULL, + started_at timestamp DEFAULT NULL, + completed_at timestamp DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE UNIQUE INDEX idx_apply_operations_apply_deployment_key ON apply_operations (apply_id, deployment, operation_key); +CREATE INDEX idx_apply_operations_deployment_state ON apply_operations (deployment, state); +CREATE INDEX idx_apply_operations_state_created_id ON apply_operations (state, created_at, id); +CREATE INDEX idx_apply_operations_apply_created_id ON apply_operations (apply_id, created_at, id); diff --git a/pkg/schema/postgres/apply_target_locks.sql b/pkg/schema/postgres/apply_target_locks.sql new file mode 100644 index 000000000..3aa75dc33 --- /dev/null +++ b/pkg/schema/postgres/apply_target_locks.sql @@ -0,0 +1,12 @@ +CREATE TABLE apply_target_locks ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + database_name varchar(255) NOT NULL, + database_type varchar(50) NOT NULL, + environment varchar(50) NOT NULL, + deployment varchar(255) NOT NULL DEFAULT '', + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE UNIQUE INDEX idx_apply_target_locks_target ON apply_target_locks (database_name, database_type, environment); +CREATE UNIQUE INDEX idx_apply_target_locks_target_v2 ON apply_target_locks (database_name, database_type, environment, deployment); diff --git a/pkg/schema/postgres/checks.sql b/pkg/schema/postgres/checks.sql new file mode 100644 index 000000000..9c0692807 --- /dev/null +++ b/pkg/schema/postgres/checks.sql @@ -0,0 +1,25 @@ +CREATE TABLE checks ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + repository varchar(255) NOT NULL, + pull_request bigint NOT NULL, + head_sha varchar(64) NOT NULL, + environment varchar(50) NOT NULL, + database_type varchar(50) NOT NULL, + database_name varchar(255) NOT NULL, + check_run_id bigint DEFAULT NULL, + apply_id bigint DEFAULT NULL, + has_changes boolean NOT NULL DEFAULT TRUE, + status varchar(255) NOT NULL, + conclusion varchar(255) DEFAULT NULL, + blocking_reason varchar(255) DEFAULT NULL, + error_message text, + change_summary varchar(255) DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE UNIQUE INDEX idx_checks_check_key ON checks (repository, pull_request, environment, database_type, database_name); +CREATE INDEX idx_checks_repo_env_db ON checks (repository, environment, database_type, database_name); +CREATE INDEX idx_checks_repo_pr ON checks (repository, pull_request); +CREATE INDEX idx_checks_check_run ON checks (check_run_id); +CREATE INDEX idx_checks_apply_id ON checks (apply_id); diff --git a/pkg/schema/postgres/locks.sql b/pkg/schema/postgres/locks.sql new file mode 100644 index 000000000..ebb61ae6d --- /dev/null +++ b/pkg/schema/postgres/locks.sql @@ -0,0 +1,14 @@ +CREATE TABLE locks ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + database_name varchar(255) NOT NULL, + database_type varchar(50) NOT NULL, + repository varchar(255) NOT NULL, + pull_request bigint NOT NULL, + owner varchar(255) NOT NULL, + pending_plan_id varchar(255) NOT NULL DEFAULT '', + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE UNIQUE INDEX idx_locks_database ON locks (database_name, database_type); +CREATE INDEX idx_locks_repo_pr ON locks (repository, pull_request); diff --git a/pkg/schema/postgres/plan_comments.sql b/pkg/schema/postgres/plan_comments.sql new file mode 100644 index 000000000..4ea76a094 --- /dev/null +++ b/pkg/schema/postgres/plan_comments.sql @@ -0,0 +1,17 @@ +CREATE TABLE plan_comments ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + repository varchar(255) NOT NULL, + pull_request bigint NOT NULL, + database_name varchar(255) NOT NULL, + database_type varchar(50) NOT NULL, + environment_scope varchar(255) NOT NULL DEFAULT '', + head_sha varchar(64) NOT NULL, + github_comment_id bigint NOT NULL, + github_node_id varchar(255) NOT NULL, + minimized_at timestamp DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE INDEX idx_plan_comments_slot ON plan_comments (repository, pull_request, database_name, database_type, minimized_at); +CREATE INDEX idx_plan_comments_github_comment ON plan_comments (github_comment_id); diff --git a/pkg/schema/postgres/plans.sql b/pkg/schema/postgres/plans.sql new file mode 100644 index 000000000..2b763a619 --- /dev/null +++ b/pkg/schema/postgres/plans.sql @@ -0,0 +1,20 @@ +CREATE TABLE plans ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + plan_identifier varchar(255) NOT NULL, + database_name varchar(255) NOT NULL, + database_type varchar(50) NOT NULL, + deployment varchar(255) NOT NULL DEFAULT '', + target varchar(255) NOT NULL DEFAULT '', + repository varchar(255) NOT NULL, + pull_request bigint NOT NULL, + schema_path varchar(1024) NOT NULL DEFAULT '', + environment varchar(50) NOT NULL DEFAULT '', + schema_files jsonb NOT NULL, + plan_data jsonb NOT NULL, + head_sha varchar(64) NOT NULL DEFAULT '', + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE UNIQUE INDEX idx_plans_plan_identifier ON plans (plan_identifier); +CREATE INDEX idx_plans_repo_pr ON plans (repository, pull_request); +CREATE INDEX idx_plans_database_env ON plans (database_name, database_type, environment); diff --git a/pkg/schema/postgres/settings.sql b/pkg/schema/postgres/settings.sql new file mode 100644 index 000000000..e0c8b10bc --- /dev/null +++ b/pkg/schema/postgres/settings.sql @@ -0,0 +1,9 @@ +CREATE TABLE settings ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + setting_key varchar(255) NOT NULL, + setting_value text NOT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE UNIQUE INDEX idx_settings_setting_key ON settings (setting_key); diff --git a/pkg/schema/postgres/tasks.sql b/pkg/schema/postgres/tasks.sql new file mode 100644 index 000000000..ca2173502 --- /dev/null +++ b/pkg/schema/postgres/tasks.sql @@ -0,0 +1,42 @@ +CREATE TABLE tasks ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + task_identifier varchar(255) NOT NULL, + apply_id bigint NOT NULL, + apply_operation_id bigint DEFAULT NULL, + plan_id bigint NOT NULL, + database_name varchar(255) NOT NULL, + database_type varchar(50) NOT NULL, + namespace varchar(255) NOT NULL DEFAULT '', + table_name varchar(255) DEFAULT NULL, + shard varchar(255) NOT NULL DEFAULT '', + ddl text, + ddl_action varchar(50) DEFAULT NULL, + engine varchar(50) NOT NULL, + repository varchar(255) NOT NULL, + pull_request bigint NOT NULL, + environment varchar(50) NOT NULL, + state varchar(100) NOT NULL, + error_message text, + options jsonb DEFAULT NULL, + attempt integer NOT NULL DEFAULT 0, + rows_copied bigint DEFAULT 0, + rows_total bigint DEFAULT 0, + progress_percent integer DEFAULT 0, + eta_seconds integer DEFAULT NULL, + checksum_rows_checked bigint DEFAULT 0, + checksum_rows_total bigint DEFAULT 0, + cutover_attempts integer NOT NULL DEFAULT 0, + is_instant boolean DEFAULT FALSE, + engine_migration_id varchar(255) DEFAULT NULL, + started_at timestamp DEFAULT NULL, + completed_at timestamp DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE UNIQUE INDEX idx_tasks_task_identifier ON tasks (task_identifier); +CREATE INDEX idx_tasks_apply_id ON tasks (apply_id); +CREATE INDEX idx_tasks_apply_operation_id ON tasks (apply_operation_id); +CREATE INDEX idx_tasks_database ON tasks (database_name); +CREATE INDEX idx_tasks_repo_pr ON tasks (repository, pull_request); +CREATE INDEX idx_tasks_state ON tasks (state); diff --git a/pkg/schema/postgres/webhook_events.sql b/pkg/schema/postgres/webhook_events.sql new file mode 100644 index 000000000..93fd99262 --- /dev/null +++ b/pkg/schema/postgres/webhook_events.sql @@ -0,0 +1,30 @@ +CREATE TABLE webhook_events ( + id bigint GENERATED BY DEFAULT AS IDENTITY, + provider varchar(50) NOT NULL DEFAULT 'github', + delivery_id varchar(64) NOT NULL, + event varchar(100) NOT NULL, + action varchar(100) NOT NULL DEFAULT '', + repository varchar(255) NOT NULL DEFAULT '', + pull_request bigint NOT NULL DEFAULT 0, + head_sha varchar(64) NOT NULL DEFAULT '', + tenant_id varchar(255) NOT NULL DEFAULT '', + payload jsonb NOT NULL, + state varchar(50) NOT NULL, + attempts bigint NOT NULL DEFAULT 0, + lease_owner varchar(255) DEFAULT NULL, + lease_token varchar(64) DEFAULT NULL, + lease_expires_at timestamp DEFAULT NULL, + retry_after timestamp DEFAULT NULL, + last_error text, + received_at timestamp NOT NULL, + started_at timestamp DEFAULT NULL, + completed_at timestamp DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); +CREATE UNIQUE INDEX idx_webhook_events_provider_delivery ON webhook_events (provider, delivery_id); +CREATE INDEX idx_webhook_events_claimable ON webhook_events (state, retry_after, lease_expires_at, created_at); +CREATE INDEX idx_webhook_events_provider_repo_pr ON webhook_events (provider, repository, pull_request); +CREATE INDEX idx_webhook_events_head_sha ON webhook_events (head_sha); +CREATE INDEX idx_webhook_events_received_at ON webhook_events (received_at); diff --git a/pkg/schema/postgres_integration_test.go b/pkg/schema/postgres_integration_test.go new file mode 100644 index 000000000..12de6fe18 --- /dev/null +++ b/pkg/schema/postgres_integration_test.go @@ -0,0 +1,168 @@ +//go:build integration + +package schema + +import ( + "database/sql" + "strings" + "testing" + + "github.com/block/spirit/pkg/statement" + "github.com/block/spirit/pkg/utils" + _ "github.com/jackc/pgx/v5/stdlib" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/postgres" + + "github.com/block/schemabot/pkg/testutil" +) + +// TestPostgresSchemaFilesExecuteAndMirrorMySQL executes every embedded +// postgres schema file against a real PostgreSQL server, then verifies each +// table mirrors its MySQL counterpart column-for-column: same column names, +// same nullability, the declared type mapping, and identical varchar widths. +// The MySQL side is parsed from the embedded files; the PostgreSQL side is +// read back from information_schema after the DDL runs, so the comparison +// covers what the server actually created, not what the file claims. +func TestPostgresSchemaFilesExecuteAndMirrorMySQL(t *testing.T) { + ctx := t.Context() + + container, err := postgres.Run(ctx, + "postgres:16", + postgres.WithDatabase("schemabot_test"), + postgres.WithUsername("schemabot"), + postgres.WithPassword("test"), + postgres.BasicWaitStrategies(), + ) + require.NoError(t, err, "failed to start postgres") + t.Cleanup(func() { + if err := testcontainers.TerminateContainer(container); err != nil { + t.Logf("failed to terminate container: %v", err) + } + }) + + dsn, err := testutil.ContainerConnectionString(ctx, container, "sslmode=disable") + require.NoError(t, err, "failed to get connection string") + + db, err := sql.Open("pgx", dsn) + require.NoError(t, err) + t.Cleanup(func() { utils.CloseAndLog(db) }) + require.NoError(t, db.PingContext(ctx)) + + for name, content := range readSchemaDir(t, "postgres") { + for stmt := range strings.SplitSeq(content, ";") { + stmt = strings.TrimSpace(stmt) + if stmt == "" { + continue + } + _, err := db.ExecContext(ctx, stmt) + require.NoError(t, err, "%s: statement failed:\n%s", name, stmt) + } + } + + for name, content := range readSchemaDir(t, "mysql") { + parsed, err := statement.ParseCreateTable(content) + require.NoError(t, err, "parse mysql schema file %s", name) + + pgColumns := postgresColumns(t, db, parsed.TableName) + require.NotEmpty(t, pgColumns, "table %s missing from postgres schema", parsed.TableName) + + mysqlNames := make([]string, 0, len(parsed.Columns)) + for _, col := range parsed.Columns { + mysqlNames = append(mysqlNames, col.Name) + pgCol, ok := pgColumns[col.Name] + if !assert.True(t, ok, "table %s: column %s missing from postgres schema", parsed.TableName, col.Name) { + continue + } + assert.Equal(t, col.Nullable, pgCol.nullable, + "table %s: column %s nullability differs (mysql nullable=%v)", parsed.TableName, col.Name, col.Nullable) + + wantType, ok := expectedPostgresColumnType(col) + require.True(t, ok, "table %s: column %s has mysql type %q with no declared postgres mapping", parsed.TableName, col.Name, col.Type) + assert.Equal(t, wantType, pgCol.dataType, + "table %s: column %s type differs (mysql type=%q)", parsed.TableName, col.Name, col.Type) + + if col.Type == "varchar" { + require.NotNil(t, col.Length, "table %s: mysql varchar column %s has no length", parsed.TableName, col.Name) + if assert.True(t, pgCol.charMaxLen.Valid, "table %s: postgres column %s has no character maximum length", parsed.TableName, col.Name) { + assert.Equal(t, int64(*col.Length), pgCol.charMaxLen.Int64, + "table %s: column %s varchar width differs", parsed.TableName, col.Name) + } + } + } + pgNames := make([]string, 0, len(pgColumns)) + for colName := range pgColumns { + pgNames = append(pgNames, colName) + } + assert.ElementsMatch(t, mysqlNames, pgNames, "table %s: column sets differ", parsed.TableName) + } +} + +// postgresColumn is one column definition read back from information_schema +// after the DDL ran, so assertions cover what the server actually created. +type postgresColumn struct { + nullable bool + dataType string + charMaxLen sql.NullInt64 +} + +// postgresColumns returns column name → definition for a table in the public +// schema of the connected PostgreSQL database. +func postgresColumns(t *testing.T, db *sql.DB, tableName string) map[string]postgresColumn { + t.Helper() + + rows, err := db.QueryContext(t.Context(), + `SELECT column_name, is_nullable, data_type, character_maximum_length + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = $1`, + tableName, + ) + require.NoError(t, err) + defer utils.CloseAndLog(rows) + + columns := make(map[string]postgresColumn) + for rows.Next() { + var colName, isNullable string + var col postgresColumn + require.NoError(t, rows.Scan(&colName, &isNullable, &col.dataType, &col.charMaxLen)) + col.nullable = isNullable == "YES" + columns[colName] = col + } + require.NoError(t, rows.Err()) + return columns +} + +// expectedPostgresColumnType returns the information_schema data_type the +// postgres translation of a TiDB-parsed mysql column must produce. This is +// the single declaration of the cross-dialect type mapping: identity bigint +// PKs, bigint for unsigned ints (PostgreSQL has no unsigned types and integer +// would halve the range), boolean for tinyint(1), jsonb for json, and +// zone-less timestamp for zone-less datetime. +func expectedPostgresColumnType(col statement.Column) (string, bool) { + unsigned := col.Unsigned != nil && *col.Unsigned + switch col.Type { + case "tinyint": + if col.Length == nil || *col.Length != 1 { + return "", false + } + return "boolean", true + case "int": + if unsigned { + return "bigint", true + } + return "integer", true + case "bigint": + return "bigint", true + case "varchar": + return "character varying", true + case "text": + return "text", true + case "datetime": + return "timestamp without time zone", true + case "json": + return "jsonb", true + default: + return "", false + } +} diff --git a/pkg/schema/postgres_test.go b/pkg/schema/postgres_test.go new file mode 100644 index 000000000..12ff71220 --- /dev/null +++ b/pkg/schema/postgres_test.go @@ -0,0 +1,198 @@ +package schema + +import ( + "fmt" + "regexp" + "strings" + "testing" + + "github.com/block/spirit/pkg/statement" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// postgresMaxIdentifierLength is PostgreSQL's NAMEDATALEN-1 limit: identifiers +// longer than this are silently truncated by the server, so index names must +// stay within it to keep names stable and collision-free. +const postgresMaxIdentifierLength = 63 + +func readSchemaDir(t *testing.T, dir string) map[string]string { + t.Helper() + + var fsys = MySQLFS + if dir == "postgres" { + fsys = PostgresFS + } + entries, err := fsys.ReadDir(dir) + require.NoError(t, err) + require.NotEmpty(t, entries) + + files := make(map[string]string, len(entries)) + for _, entry := range entries { + content, err := fsys.ReadFile(dir + "/" + entry.Name()) + require.NoError(t, err) + files[entry.Name()] = string(content) + } + return files +} + +// TestPostgresFilesMirrorMySQLFiles verifies the postgres schema directory +// contains exactly the same set of table files as the mysql directory, so a +// table added to one dialect cannot be silently missed in the other. +func TestPostgresFilesMirrorMySQLFiles(t *testing.T) { + mysqlFiles := readSchemaDir(t, "mysql") + postgresFiles := readSchemaDir(t, "postgres") + + mysqlNames := make([]string, 0, len(mysqlFiles)) + for name := range mysqlFiles { + mysqlNames = append(mysqlNames, name) + } + postgresNames := make([]string, 0, len(postgresFiles)) + for name := range postgresFiles { + postgresNames = append(postgresNames, name) + } + assert.ElementsMatch(t, mysqlNames, postgresNames) +} + +// TestPostgresFilesContainNoMySQLSyntax lints every postgres schema file for +// MySQL constructs that must not appear in the postgres DDL. Most are syntax +// PostgreSQL rejects outright; charset and collate are policy bans — the +// storage DDL uses the database's default collation, and any collation +// decision (e.g. case-insensitive matching) belongs to the store layer where +// its query semantics can be tested, not hidden in the bootstrap DDL. +func TestPostgresFilesContainNoMySQLSyntax(t *testing.T) { + forbidden := []string{ + "`", + "auto_increment", + "on update current_timestamp", + "engine=", + "unsigned", + "tinyint", + "datetime", + "charset", + "collate", + } + + for name, content := range readSchemaDir(t, "postgres") { + lower := strings.ToLower(content) + for _, token := range forbidden { + assert.NotContains(t, lower, token, "%s contains MySQL-only construct %q", name, token) + } + } +} + +// TestPostgresTableNamesMatchFilenames verifies each postgres file defines +// exactly one table named after the file, matching the mysql convention the +// schema bootstrapper relies on. +func TestPostgresTableNamesMatchFilenames(t *testing.T) { + createTableRe := regexp.MustCompile(`(?m)^CREATE TABLE (\S+) \(`) + + for name, content := range readSchemaDir(t, "postgres") { + matches := createTableRe.FindAllStringSubmatch(content, -1) + require.Len(t, matches, 1, "%s must contain exactly one CREATE TABLE", name) + assert.Equal(t, strings.TrimSuffix(name, ".sql"), matches[0][1], "%s table name must match filename", name) + } +} + +// postgresIndex is one CREATE INDEX statement parsed from a postgres schema file. +type postgresIndex struct { + file string + name string + table string + unique bool + columns []string +} + +var postgresCreateIndexRe = regexp.MustCompile(`(?m)^CREATE (UNIQUE )?INDEX (\S+) ON (\S+) \(([^)]+)\);`) + +// readPostgresIndexes parses every CREATE INDEX statement in the postgres +// schema directory. The parse count per file is cross-checked against literal +// CREATE INDEX occurrences, so an index written in a format the regex does +// not recognize fails loudly instead of silently escaping the schema lints. +func readPostgresIndexes(t *testing.T) []postgresIndex { + t.Helper() + + var indexes []postgresIndex + for name, content := range readSchemaDir(t, "postgres") { + matches := postgresCreateIndexRe.FindAllStringSubmatch(content, -1) + literal := strings.Count(content, "CREATE INDEX ") + strings.Count(content, "CREATE UNIQUE INDEX ") + require.Len(t, matches, literal, "%s: every CREATE INDEX must be a single-line statement of the form CREATE [UNIQUE] INDEX name ON table (cols);", name) + for _, match := range matches { + columns := strings.Split(match[4], ",") + for i, col := range columns { + columns[i] = strings.TrimSpace(col) + } + indexes = append(indexes, postgresIndex{ + file: name, + name: match[2], + table: match[3], + unique: match[1] != "", + columns: columns, + }) + } + } + require.NotEmpty(t, indexes) + return indexes +} + +// TestPostgresIndexNames verifies index naming across the postgres schema. +// Unlike MySQL, PostgreSQL index names share one schema-wide namespace, so +// every index name must be globally unique, carry its table's name as a +// prefix, and fit within the identifier length limit. +func TestPostgresIndexNames(t *testing.T) { + seen := make(map[string]string) + for _, idx := range readPostgresIndexes(t) { + table := strings.TrimSuffix(idx.file, ".sql") + assert.Equal(t, table, idx.table, "%s: index %s must target its own file's table", idx.file, idx.name) + assert.True(t, strings.HasPrefix(idx.name, "idx_"+table+"_"), + "%s: index %s must be prefixed idx_%s_", idx.file, idx.name, table) + assert.LessOrEqual(t, len(idx.name), postgresMaxIdentifierLength, + "%s: index %s exceeds the PostgreSQL identifier length limit", idx.file, idx.name) + if prev, dup := seen[idx.name]; dup { + t.Errorf("index name %s in %s collides with %s (PostgreSQL index names are schema-wide)", idx.name, idx.file, prev) + } + seen[idx.name] = idx.file + } +} + +// indexShape is the dialect-neutral identity of an index: the table it +// covers, its ordered column list, and whether it is unique. Names are +// excluded — they legitimately differ across dialects (postgres names are +// table-prefixed and occasionally renamed for clarity) and are linted by +// TestPostgresIndexNames instead. +func indexShape(table string, unique bool, columns []string) string { + kind := "INDEX" + if unique { + kind = "UNIQUE" + } + return fmt.Sprintf("%s %s (%s)", kind, table, strings.Join(columns, ", ")) +} + +// TestPostgresIndexesMirrorMySQLIndexes verifies every MySQL index has a +// postgres counterpart on the same table with the same ordered column list +// and the same uniqueness, and that postgres defines no extra indexes. The +// PostgreSQL store's ON CONFLICT upserts and duplicate-rejection guarantees +// depend on the unique indexes existing with exactly the MySQL semantics, so +// a dropped or de-uniquified index must fail here rather than surface as a +// runtime error or silent duplicate acceptance. +func TestPostgresIndexesMirrorMySQLIndexes(t *testing.T) { + pgShapes := make([]string, 0) + for _, idx := range readPostgresIndexes(t) { + pgShapes = append(pgShapes, indexShape(idx.table, idx.unique, idx.columns)) + } + + mysqlShapes := make([]string, 0, len(pgShapes)) + for name, content := range readSchemaDir(t, "mysql") { + parsed, err := statement.ParseCreateTable(content) + require.NoError(t, err, "parse mysql schema file %s", name) + for _, idx := range parsed.Indexes { + if idx.Type == "PRIMARY KEY" { + continue + } + mysqlShapes = append(mysqlShapes, indexShape(parsed.TableName, idx.Type == "UNIQUE", idx.Columns)) + } + } + + assert.ElementsMatch(t, mysqlShapes, pgShapes, + "index sets differ between dialects: every mysql index needs a postgres counterpart with the same table, ordered columns, and uniqueness, and vice versa") +} diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index f17dea5ae..7c60a7f4d 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -1,13 +1,6 @@ // Package schema provides embedded SQL schema files and shared schema types -// for SchemaBot storage. -// -// Tables: -// - locks: Deployment locks -// - checks: Schema check tracking -// - settings: Runtime settings -// - plans: Schema change plans -// - tasks: Schema change tasks -// - apply_operations: Per-(apply, deployment) child rows for multi-deployment applies +// for SchemaBot storage. Each storage table has one file per dialect +// directory; see mysql/ and postgres/. package schema import "embed" @@ -18,6 +11,16 @@ import "embed" //go:embed mysql/*.sql var MySQLFS embed.FS +// PostgresFS contains the embedded SQL schema files for SchemaBot's own +// storage tables on PostgreSQL. Each file holds one CREATE TABLE statement +// followed by its CREATE INDEX statements, separated by semicolons. The files +// mirror the MySQL schema table-for-table and column-for-column; updated_at +// stamping is the application's responsibility on PostgreSQL (there is no +// ON UPDATE CURRENT_TIMESTAMP equivalent and no trigger is installed). +// +//go:embed postgres/*.sql +var PostgresFS embed.FS + // SchemaFiles maps namespace names to their file contents. // The namespace key is engine-specific: // - MySQL: schema name diff --git a/pkg/testutil/container.go b/pkg/testutil/container.go index 0f4807d9b..ffa5915c9 100644 --- a/pkg/testutil/container.go +++ b/pkg/testutil/container.go @@ -47,8 +47,8 @@ func ContainerEndpoint(ctx context.Context, c testcontainers.Container, proto st }) } -// ContainerConnectionString returns the connection string for a MySQL container, -// retrying on transient Docker failures. +// ContainerConnectionString returns the connection string for a database +// container, retrying on transient Docker failures. func ContainerConnectionString(ctx context.Context, c interface { ConnectionString(context.Context, ...string) (string, error) }, args ...string) (string, error) {