HIVE-29643: Add missing primary keys to backend tables across differe…

standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.3.0.derby.sql

@@ -536,6 +536,7 @@ INSERT INTO TXNS (TXN_ID, TXN_STATE, TXN_STARTED, TXN_LAST_HEARTBEAT, TXN_USER,
   VALUES(0, 'c', 0, 0, '', '');
 
 CREATE TABLE TXN_COMPONENTS (
+  TC_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID),
   TC_DATABASE varchar(128) NOT NULL,
   TC_TABLE varchar(256),
@@ -547,6 +548,7 @@ CREATE TABLE TXN_COMPONENTS (
 CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID);
 
 CREATE TABLE COMPLETED_TXN_COMPONENTS (
+  CTC_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   CTC_TXNID bigint NOT NULL,
   CTC_DATABASE varchar(128) NOT NULL,
   CTC_TABLE varchar(256),
@@ -559,9 +561,10 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
 CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
 
 CREATE TABLE TXN_LOCK_TBL (
+  TLT_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   TXN_LOCK bigint NOT NULL
 );
-INSERT INTO TXN_LOCK_TBL VALUES(1);
+INSERT INTO TXN_LOCK_TBL (TXN_LOCK) VALUES(1);
 
 CREATE TABLE HIVE_LOCKS (
   HL_LOCK_EXT_ID bigint NOT NULL,
@@ -587,9 +590,10 @@ CREATE TABLE HIVE_LOCKS (
 CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID);
 
 CREATE TABLE NEXT_LOCK_ID (
+  NL_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   NL_NEXT bigint NOT NULL
 );
-INSERT INTO NEXT_LOCK_ID VALUES(1);
+INSERT INTO NEXT_LOCK_ID (NL_NEXT) VALUES(1);
 
 CREATE TABLE COMPACTION_QUEUE (
   CQ_ID bigint PRIMARY KEY,
@@ -621,9 +625,10 @@ CREATE TABLE COMPACTION_QUEUE (
 );
 
 CREATE TABLE NEXT_COMPACTION_QUEUE_ID (
+  NCQI_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   NCQ_NEXT bigint NOT NULL
 );
-INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1);
+INSERT INTO NEXT_COMPACTION_QUEUE_ID (NCQ_NEXT) VALUES(1);
 
 CREATE TABLE COMPLETED_COMPACTIONS (
   CC_ID bigint PRIMARY KEY,
@@ -657,6 +662,7 @@ CREATE INDEX COMPLETED_COMPACTIONS_RES ON COMPLETED_COMPACTIONS (CC_DATABASE,CC_
 
 -- HIVE-25842
 CREATE TABLE COMPACTION_METRICS_CACHE (
+  CMC_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   CMC_DATABASE varchar(128) NOT NULL,
   CMC_TABLE varchar(256) NOT NULL,
   CMC_PARTITION varchar(767),
@@ -672,9 +678,8 @@ CREATE TABLE AUX_TABLE (
   PRIMARY KEY(MT_KEY1, MT_KEY2)
 );
 
---1st 4 cols make up a PK but since WS_PARTITION is nullable we can't declare such PK
---This is a good candidate for Index orgainzed table
 CREATE TABLE WRITE_SET (
+  WS_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   WS_DATABASE varchar(128) NOT NULL,
   WS_TABLE varchar(256) NOT NULL,
   WS_PARTITION varchar(767),
@@ -687,21 +692,21 @@ CREATE TABLE TXN_TO_WRITE_ID (
   T2W_TXNID bigint NOT NULL,
   T2W_DATABASE varchar(128) NOT NULL,
   T2W_TABLE varchar(256) NOT NULL,
-  T2W_WRITEID bigint NOT NULL
+  T2W_WRITEID bigint NOT NULL,
+  PRIMARY KEY (T2W_DATABASE, T2W_TABLE, T2W_TXNID)
 );
 
-CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID);
 CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID);
 
 CREATE TABLE NEXT_WRITE_ID (
   NWI_DATABASE varchar(128) NOT NULL,
   NWI_TABLE varchar(256) NOT NULL,
-  NWI_NEXT bigint NOT NULL
+  NWI_NEXT bigint NOT NULL,
+  PRIMARY KEY (NWI_DATABASE, NWI_TABLE)
 );
 
-CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE);
-
 CREATE TABLE MIN_HISTORY_WRITE_ID (
+  MH_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
   MH_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID),
   MH_DATABASE varchar(128) NOT NULL,
   MH_TABLE varchar(256) NOT NULL,

standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.2.0-to-4.3.0.derby.sql

@@ -1,7 +1,94 @@
 ALTER TABLE HIVE_LOCKS ADD COLUMN HL_CATALOG varchar(128) NOT NULL DEFAULT 'hive';
 ALTER TABLE MATERIALIZATION_REBUILD_LOCKS ADD COLUMN MRL_CAT_NAME varchar(128) NOT NULL DEFAULT 'hive';
 
+-- Add surrogate primary keys for HA database replication. Adding IDENTITY primary
+-- keys via ALTER TABLE rewrites large Derby tables and holds exclusive locks for
+-- the duration. Rebuild each populated table via a TMP_* copy/swap instead.
+-- Counter tables contain a single row and keep the one-step ALTER form. Plan a
+-- maintenance window for non-trivial embedded metastores; duration scales with
+-- row counts.
+
+CREATE TABLE TMP_TXN_COMPONENTS (
+  TC_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
+  TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID),
+  TC_DATABASE varchar(128) NOT NULL,
+  TC_TABLE varchar(256),
+  TC_PARTITION varchar(767),
+  TC_OPERATION_TYPE char(1) NOT NULL,
+  TC_WRITEID bigint
+);
+INSERT INTO TMP_TXN_COMPONENTS (TC_TXNID, TC_DATABASE, TC_TABLE, TC_PARTITION, TC_OPERATION_TYPE, TC_WRITEID)
+  SELECT TC_TXNID, TC_DATABASE, TC_TABLE, TC_PARTITION, TC_OPERATION_TYPE, TC_WRITEID FROM TXN_COMPONENTS;
+RENAME TABLE TXN_COMPONENTS TO TXN_COMPONENTS_OLD, TMP_TXN_COMPONENTS TO TXN_COMPONENTS;
+DROP TABLE TXN_COMPONENTS_OLD;
+CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID);
+
+CREATE TABLE TMP_COMPLETED_TXN_COMPONENTS (
+  CTC_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
+  CTC_TXNID bigint NOT NULL,
+  CTC_DATABASE varchar(128) NOT NULL,
+  CTC_TABLE varchar(256),
+  CTC_PARTITION varchar(767),
+  CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
+  CTC_WRITEID bigint,
+  CTC_UPDATE_DELETE char(1) NOT NULL
+);
+INSERT INTO TMP_COMPLETED_TXN_COMPONENTS (CTC_TXNID, CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_TIMESTAMP, CTC_WRITEID, CTC_UPDATE_DELETE)
+  SELECT CTC_TXNID, CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_TIMESTAMP, CTC_WRITEID, CTC_UPDATE_DELETE FROM COMPLETED_TXN_COMPONENTS;
+RENAME TABLE COMPLETED_TXN_COMPONENTS TO COMPLETED_TXN_COMPONENTS_OLD, TMP_COMPLETED_TXN_COMPONENTS TO COMPLETED_TXN_COMPONENTS;
+DROP TABLE COMPLETED_TXN_COMPONENTS_OLD;
+CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
+
+CREATE TABLE TMP_COMPACTION_METRICS_CACHE (
+  CMC_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
+  CMC_DATABASE varchar(128) NOT NULL,
+  CMC_TABLE varchar(256) NOT NULL,
+  CMC_PARTITION varchar(767),
+  CMC_METRIC_TYPE varchar(128) NOT NULL,
+  CMC_METRIC_VALUE integer NOT NULL,
+  CMC_VERSION integer NOT NULL
+);
+INSERT INTO TMP_COMPACTION_METRICS_CACHE (CMC_DATABASE, CMC_TABLE, CMC_PARTITION, CMC_METRIC_TYPE, CMC_METRIC_VALUE, CMC_VERSION)
+  SELECT CMC_DATABASE, CMC_TABLE, CMC_PARTITION, CMC_METRIC_TYPE, CMC_METRIC_VALUE, CMC_VERSION FROM COMPACTION_METRICS_CACHE;
+RENAME TABLE COMPACTION_METRICS_CACHE TO COMPACTION_METRICS_CACHE_OLD, TMP_COMPACTION_METRICS_CACHE TO COMPACTION_METRICS_CACHE;
+DROP TABLE COMPACTION_METRICS_CACHE_OLD;
+
+CREATE TABLE TMP_WRITE_SET (
+  WS_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
+  WS_DATABASE varchar(128) NOT NULL,
+  WS_TABLE varchar(256) NOT NULL,
+  WS_PARTITION varchar(767),
+  WS_TXNID bigint NOT NULL,
+  WS_COMMIT_ID bigint NOT NULL,
+  WS_OPERATION_TYPE char(1) NOT NULL
+);
+INSERT INTO TMP_WRITE_SET (WS_DATABASE, WS_TABLE, WS_PARTITION, WS_TXNID, WS_COMMIT_ID, WS_OPERATION_TYPE)
+  SELECT WS_DATABASE, WS_TABLE, WS_PARTITION, WS_TXNID, WS_COMMIT_ID, WS_OPERATION_TYPE FROM WRITE_SET;
+RENAME TABLE WRITE_SET TO WRITE_SET_OLD, TMP_WRITE_SET TO WRITE_SET;
+DROP TABLE WRITE_SET_OLD;
+
+DROP INDEX TBL_TO_TXN_ID_IDX;
+ALTER TABLE TXN_TO_WRITE_ID ADD PRIMARY KEY (T2W_DATABASE, T2W_TABLE, T2W_TXNID);
+
+DROP INDEX NEXT_WRITE_ID_IDX;
+ALTER TABLE NEXT_WRITE_ID ADD PRIMARY KEY (NWI_DATABASE, NWI_TABLE);
+
+CREATE TABLE TMP_MIN_HISTORY_WRITE_ID (
+  MH_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
+  MH_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID),
+  MH_DATABASE varchar(128) NOT NULL,
+  MH_TABLE varchar(256) NOT NULL,
+  MH_WRITEID bigint NOT NULL
+);
+INSERT INTO TMP_MIN_HISTORY_WRITE_ID (MH_TXNID, MH_DATABASE, MH_TABLE, MH_WRITEID)
+  SELECT MH_TXNID, MH_DATABASE, MH_TABLE, MH_WRITEID FROM MIN_HISTORY_WRITE_ID;
+RENAME TABLE MIN_HISTORY_WRITE_ID TO MIN_HISTORY_WRITE_ID_OLD, TMP_MIN_HISTORY_WRITE_ID TO MIN_HISTORY_WRITE_ID;
+DROP TABLE MIN_HISTORY_WRITE_ID_OLD;
 CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID);
 
+ALTER TABLE TXN_LOCK_TBL ADD COLUMN TLT_ID BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY;
+ALTER TABLE NEXT_LOCK_ID ADD COLUMN NL_ID BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY;
+ALTER TABLE NEXT_COMPACTION_QUEUE_ID ADD COLUMN NCQI_ID BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY;
+
 -- This needs to be the last thing done.  Insert any changes above this line.
 UPDATE "APP".VERSION SET SCHEMA_VERSION='4.3.0', VERSION_COMMENT='Hive release version 4.3.0' where VER_ID=1;

standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.3.0.mssql.sql

@@ -1033,22 +1033,26 @@ CREATE INDEX COMPLETED_COMPACTIONS_RES ON COMPLETED_COMPACTIONS (CC_DATABASE,CC_
 
 -- HIVE-25842
 CREATE TABLE COMPACTION_METRICS_CACHE (
+    CMC_ID bigint NOT NULL IDENTITY(1,1),
     CMC_DATABASE nvarchar(128) NOT NULL,
     CMC_TABLE nvarchar(256) NOT NULL,
     CMC_PARTITION nvarchar(767) NULL,
     CMC_METRIC_TYPE nvarchar(128) NOT NULL,
     CMC_METRIC_VALUE int NOT NULL,
-    CMC_VERSION int NOT NULL
+    CMC_VERSION int NOT NULL,
+    PRIMARY KEY CLUSTERED (CMC_ID ASC)
 );
 
 CREATE TABLE COMPLETED_TXN_COMPONENTS(
+	CTC_ID bigint NOT NULL IDENTITY(1,1),
 	CTC_TXNID bigint NOT NULL,
 	CTC_DATABASE nvarchar(128) NOT NULL,
 	CTC_TABLE nvarchar(256) NULL,
 	CTC_PARTITION nvarchar(767) NULL,
     CTC_TIMESTAMP datetime2 DEFAULT CURRENT_TIMESTAMP NOT NULL,
     CTC_WRITEID bigint,
-    CTC_UPDATE_DELETE char(1) NOT NULL
+    CTC_UPDATE_DELETE char(1) NOT NULL,
+    PRIMARY KEY CLUSTERED (CTC_ID ASC)
 );
 
 
@@ -1080,22 +1084,28 @@ PRIMARY KEY CLUSTERED
 );
 
 CREATE TABLE NEXT_COMPACTION_QUEUE_ID(
-	NCQ_NEXT bigint NOT NULL
+	NCQI_ID bigint NOT NULL IDENTITY(1,1),
+	NCQ_NEXT bigint NOT NULL,
+	PRIMARY KEY CLUSTERED (NCQI_ID ASC)
 );
 
-INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1);
+INSERT INTO NEXT_COMPACTION_QUEUE_ID (NCQ_NEXT) VALUES(1);
 
 CREATE TABLE NEXT_LOCK_ID(
-	NL_NEXT bigint NOT NULL
+	NL_ID bigint NOT NULL IDENTITY(1,1),
+	NL_NEXT bigint NOT NULL,
+	PRIMARY KEY CLUSTERED (NL_ID ASC)
 );
 
-INSERT INTO NEXT_LOCK_ID VALUES(1);
+INSERT INTO NEXT_LOCK_ID (NL_NEXT) VALUES(1);
 
 CREATE TABLE TXN_LOCK_TBL(
-    TXN_LOCK bigint NOT NULL
+    TLT_ID bigint NOT NULL IDENTITY(1,1),
+    TXN_LOCK bigint NOT NULL,
+    PRIMARY KEY CLUSTERED (TLT_ID ASC)
 );
 
-INSERT INTO TXN_LOCK_TBL VALUES(1);
+INSERT INTO TXN_LOCK_TBL (TXN_LOCK) VALUES(1);
 
 CREATE TABLE TXNS(
     TXN_ID bigint NOT NULL IDENTITY(1,1),
@@ -1118,12 +1128,14 @@ INSERT INTO TXNS (TXN_ID, TXN_STATE, TXN_STARTED, TXN_LAST_HEARTBEAT, TXN_USER,
   VALUES(0, 'c', 0, 0, '', '');
 
 CREATE TABLE TXN_COMPONENTS(
+	TC_ID bigint NOT NULL IDENTITY(1,1),
 	TC_TXNID bigint NOT NULL,
 	TC_DATABASE nvarchar(128) NOT NULL,
 	TC_TABLE nvarchar(256) NULL,
 	TC_PARTITION nvarchar(767) NULL,
     TC_OPERATION_TYPE char(1) NOT NULL,
-    TC_WRITEID bigint
+    TC_WRITEID bigint,
+    PRIMARY KEY CLUSTERED (TC_ID ASC)
 );
 
 ALTER TABLE TXN_COMPONENTS  WITH CHECK ADD FOREIGN KEY(TC_TXNID) REFERENCES TXNS (TXN_ID);
@@ -1165,12 +1177,14 @@ CREATE INDEX CONSTRAINTS_PARENT_TBL_ID__INDEX ON KEY_CONSTRAINTS(PARENT_TBL_ID);
 CREATE INDEX CONSTRAINTS_CONSTRAINT_TYPE_INDEX ON KEY_CONSTRAINTS(CONSTRAINT_TYPE);
 
 CREATE TABLE WRITE_SET (
+  WS_ID bigint NOT NULL IDENTITY(1,1),
   WS_DATABASE nvarchar(128) NOT NULL,
   WS_TABLE nvarchar(256) NOT NULL,
   WS_PARTITION nvarchar(767),
   WS_TXNID bigint NOT NULL,
   WS_COMMIT_ID bigint NOT NULL,
-  WS_OPERATION_TYPE char(1) NOT NULL
+  WS_OPERATION_TYPE char(1) NOT NULL,
+  PRIMARY KEY CLUSTERED (WS_ID ASC)
 );
 
 CREATE TABLE METASTORE_DB_PROPERTIES (
@@ -1186,25 +1200,26 @@ CREATE TABLE TXN_TO_WRITE_ID (
   T2W_TXNID bigint NOT NULL,
   T2W_DATABASE nvarchar(128) NOT NULL,
   T2W_TABLE nvarchar(256) NOT NULL,
-  T2W_WRITEID bigint NOT NULL
+  T2W_WRITEID bigint NOT NULL,
+  PRIMARY KEY CLUSTERED (T2W_DATABASE ASC, T2W_TABLE ASC, T2W_TXNID ASC)
 );
 
-CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID);
 CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID);
 
 CREATE TABLE NEXT_WRITE_ID (
   NWI_DATABASE nvarchar(128) NOT NULL,
   NWI_TABLE nvarchar(256) NOT NULL,
-  NWI_NEXT bigint NOT NULL
+  NWI_NEXT bigint NOT NULL,
+  PRIMARY KEY CLUSTERED (NWI_DATABASE ASC, NWI_TABLE ASC)
 );
 
-CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE);
-
 CREATE TABLE MIN_HISTORY_WRITE_ID (
+  MH_ID bigint NOT NULL IDENTITY(1,1),
   MH_TXNID bigint NOT NULL,
   MH_DATABASE nvarchar(128) NOT NULL,
   MH_TABLE nvarchar(256) NOT NULL,
-  MH_WRITEID bigint NOT NULL
+  MH_WRITEID bigint NOT NULL,
+  PRIMARY KEY CLUSTERED (MH_ID ASC)
 );
 
 CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID);