Implemented rebuild cmd for history snapshots

Author: eastorski

cmd/integration/commands/state_history.go

@@ -17,6 +17,7 @@
 package commands
 
 import (
+	"context"
 	"os"
 
 	"github.com/erigontech/erigon/db/config3"
@@ -34,7 +35,12 @@ func init() {
 	withDataDir2(printCmd)
 	withHistoryDomain(printCmd)
 
+	withDataDir2(rebuildCmd)
+	withHistoryDomain(rebuildCmd)
+
 	historyCmd.AddCommand(printCmd)
+	historyCmd.AddCommand(rebuildCmd)
+
 	rootCmd.AddCommand(historyCmd)
 }
 
@@ -98,3 +104,55 @@ var printCmd = &cobra.Command{
 		}
 	},
 }
+
+var rebuildCmd = &cobra.Command{
+	Use:   "rebuild",
+	Short: "Regenerate .ef .efi .v .vi domain history snapshots from step 0",
+	Run: func(cmd *cobra.Command, args []string) {
+		logger := debug.SetupCobra(cmd, "integration")
+
+		dirs, l, err := datadir.New(datadirCli).MustFlock()
+		if err != nil {
+			logger.Error("Opening Datadir", "error", err)
+			return
+		}
+		defer l.Unlock()
+
+		domainKV, err := kv.String2Domain(historyDomain)
+		if err != nil {
+			logger.Error("Failed to resolve domain", "error", err)
+			return
+		}
+
+		history, err := state.NewHistory(
+			statecfg.Schema.GetDomainCfg(domainKV).Hist,
+			config3.DefaultStepSize,
+			config3.DefaultStepsInFrozenFile,
+			dirs,
+			logger,
+		)
+		if err != nil {
+			logger.Error("Failed to init history", "error", err)
+			return
+		}
+		history.Scan(toStep * config3.DefaultStepSize)
+
+		roTx := history.BeginFilesRo()
+		defer roTx.Close()
+
+		for i := uint64(0); i < roTx.FirstStepNotInFiles().ToTxNum(config3.DefaultStepSize); {
+			fromTxNum := i
+			i += config3.DefaultStepSize * config3.DefaultStepsInFrozenFile
+
+			if i > roTx.FirstStepNotInFiles().ToTxNum(config3.DefaultStepSize) {
+				i = roTx.FirstStepNotInFiles().ToTxNum(config3.DefaultStepSize)
+			}
+
+			err = roTx.CompactRange(context.TODO(), fromTxNum, i)
+			if err != nil {
+				logger.Error("Failed to rebuild history", "error", err)
+				return
+			}
+		}
+	},
+}

db/state/history.go

@@ -1417,6 +1417,31 @@ func (ht *HistoryRoTx) HistoryDump(fromTxNum, toTxNum int, dumpTo io.Writer) err
 	return nil
 }
 
+// CompactRange rebuilds the history files within the specified transaction range by performing a forced self-merge.
+// If the range contains existing static files, the method collects all files belonging to that span and merges them.
+func (ht *HistoryRoTx) CompactRange(ctx context.Context, fromTxNum, toTxNum uint64) error {
+	if len(ht.iit.files) == 0 {
+		return nil
+	}
+
+	mergeRange := NewHistoryRanges(
+		*NewMergeRange("", true, fromTxNum, toTxNum),
+		*NewMergeRange("", true, fromTxNum, toTxNum),
+	)
+
+	efFiles, vFiles, err := ht.staticFilesInRange(mergeRange)
+	if err != nil {
+		return err
+	}
+
+	_, _, err = ht.mergeFiles(ctx, efFiles, vFiles, mergeRange, background.NewProgressSet())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (ht *HistoryRoTx) idxRangeOnDB(key []byte, startTxNum, endTxNum int, asc order.By, limit int, roTx kv.Tx) (stream.U64, error) {
 	if ht.h.HistoryLargeValues {
 		from := make([]byte, len(key)+8)