diff --git a/parser/v1/parser.go b/parser/v1/parser.go index 587fd80b..e0054058 100644 --- a/parser/v1/parser.go +++ b/parser/v1/parser.go @@ -70,7 +70,7 @@ func NewParser(helper *helper.Helper, logger *logger.Logger, metrics metrics.Met logger: logger2.GetSafeLogger(logger), multisigEventGenerator: multisigTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config), minerEventGenerator: minerTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config), - dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, networkName), + dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, networkName, config), metrics: parsermetrics.NewClient(metrics, "parserV1"), actorsCacheMetrics: cacheMetrics.NewClient(metrics, "actorsCache"), config: config, @@ -86,7 +86,7 @@ func NewActorsV2Parser(network string, helper *helper.Helper, logger *logger.Log logger: logger2.GetSafeLogger(logger), multisigEventGenerator: multisigTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config), minerEventGenerator: minerTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config), - dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, network), + dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, network, config), metrics: parsermetrics.NewClient(metrics, "parserV1"), actorsCacheMetrics: cacheMetrics.NewClient(metrics, "actorsCache"), config: config, diff --git a/parser/v2/parser.go b/parser/v2/parser.go index 94bb796d..41144ca5 100644 --- a/parser/v2/parser.go +++ b/parser/v2/parser.go @@ -72,7 +72,7 @@ func NewParser(helper *helper.Helper, logger *logger.Logger, metrics metrics.Met logger: logger2.GetSafeLogger(logger), multisigEventGenerator: multisigTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config), minerEventGenerator: minerTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config), - dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, networkName), + dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, networkName, config), metrics: parsermetrics.NewClient(metrics, "parserV2"), actorsCacheMetrics: cacheMetrics.NewClient(metrics, "actorsCache"), config: config, @@ -91,7 +91,7 @@ func NewActorsV2Parser(network string, helper *helper.Helper, logger *logger.Log logger: logger2.GetSafeLogger(logger), multisigEventGenerator: multisigTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config), minerEventGenerator: minerTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config), - dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, network), + dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, network, config), metrics: parsermetrics.NewClient(metrics, "parserV2"), actorsCacheMetrics: cacheMetrics.NewClient(metrics, "actorsCache"), config: config, diff --git a/tools/deals/activations.go b/tools/deals/activations.go index 25bd21d1..34662a44 100644 --- a/tools/deals/activations.go +++ b/tools/deals/activations.go @@ -88,16 +88,20 @@ func (eg *eventGenerator) parseVerifyDealsForActivation(tx *types.Transaction, p return nil, err } - dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{ - ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealIDs)), - Height: tx.Height, - TxCid: tx.TxCid, - DealIDs: dealIDs, - NonVerifiedDealSpace: nonVerifiedDealWeight, - VerifiedDealSpace: verifiedDealWeight, - ActionType: tx.TxType, - TxTimestamp: tx.TxTimestamp, - }) + for _, dealID := range dealIDs { + dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{ + ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealID)), + Height: tx.Height, + ActorAddress: tx.TxFrom, + TxCid: tx.TxCid, + DealID: dealID, + GroupDealIDs: dealIDs, + NonVerifiedDealSpace: nonVerifiedDealWeight, + VerifiedDealSpace: verifiedDealWeight, + ActionType: tx.TxType, + TxTimestamp: tx.TxTimestamp, + }) + } return dealSpaceInfo, nil } @@ -147,17 +151,21 @@ func (eg *eventGenerator) parseVerifyDealsForActivation(tx *types.Transaction, p return nil, err } - dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{ - ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealIDs[i])), - Height: tx.Height, - TxCid: tx.TxCid, - DealIDs: dealIDs, - NonVerifiedDealSpace: nonVerifiedDealWeight, - VerifiedDealSpace: verifiedDealWeight, - SpaceAsWeight: true, - ActionType: tx.TxType, - TxTimestamp: tx.TxTimestamp, - }) + for _, dealID := range dealIDs { + dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{ + ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealID)), + Height: tx.Height, + ActorAddress: tx.TxFrom, + TxCid: tx.TxCid, + DealID: dealID, + GroupDealIDs: dealIDs, + NonVerifiedDealSpace: nonVerifiedDealWeight, + VerifiedDealSpace: verifiedDealWeight, + SpaceAsWeight: true, + ActionType: tx.TxType, + TxTimestamp: tx.TxTimestamp, + }) + } } return dealSpaceInfo, nil } @@ -180,6 +188,7 @@ func (eg *eventGenerator) parseActivateDeals(tx *types.Transaction, params, ret for _, dealID := range dealIDs { dealActivations = append(dealActivations, &types.DealsActivations{ ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealID)), + ActorAddress: tx.TxFrom, Height: tx.Height, TxCid: tx.TxCid, DealID: dealID, @@ -196,16 +205,20 @@ func (eg *eventGenerator) parseActivateDeals(tx *types.Transaction, params, ret if err != nil { return err } - dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{ - ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealIDs)), - Height: tx.Height, - TxCid: tx.TxCid, - DealIDs: dealIDs, - NonVerifiedDealSpace: nonVerifiedDealSpace, - VerifiedDealSpace: verifiedDealSpace, - ActionType: tx.TxType, - TxTimestamp: tx.TxTimestamp, - }) + for _, dealID := range dealIDs { + dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{ + ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealID)), + DealID: dealID, + Height: tx.Height, + ActorAddress: tx.TxFrom, + TxCid: tx.TxCid, + GroupDealIDs: dealIDs, + NonVerifiedDealSpace: nonVerifiedDealSpace, + VerifiedDealSpace: verifiedDealSpace, + ActionType: tx.TxType, + TxTimestamp: tx.TxTimestamp, + }) + } return nil } diff --git a/tools/deals/deals.go b/tools/deals/deals.go index b4c0eeaa..9ef2fd5c 100644 --- a/tools/deals/deals.go +++ b/tools/deals/deals.go @@ -30,14 +30,16 @@ type eventGenerator struct { logger *logger.Logger metrics *dealsMetricsClient network string + config parser.Config } -func NewEventGenerator(helper *helper.Helper, logger *logger.Logger, metrics metrics.MetricsClient, network string) EventGenerator { +func NewEventGenerator(helper *helper.Helper, logger *logger.Logger, metrics metrics.MetricsClient, network string, config parser.Config) EventGenerator { return &eventGenerator{ helper: helper, logger: logger, metrics: newClient(metrics, "deals"), network: network, + config: config, } } @@ -55,13 +57,7 @@ func (eg *eventGenerator) GenerateDealsEvents(ctx context.Context, transactions continue } - actorAddress := tx.TxTo - // this is executed by(from) the miner actor - if tx.TxType == parser.MethodUpdateClaimedPower || tx.TxType == parser.TotalFeeOp { - actorAddress = tx.TxFrom - } - - addr, err := address.NewFromString(actorAddress) + addr, err := address.NewFromString(tx.TxTo) if err != nil { eg.logger.Errorf("could not parse address. Err: %s", err) } @@ -78,6 +74,8 @@ func (eg *eventGenerator) GenerateDealsEvents(ctx context.Context, transactions continue } + // the address that calls the market actor + actorAddress := tx.TxFrom dealMessage, err := eg.createDealMessage(tx, tipsetCid, actorAddress) if err != nil { eg.logger.Errorf("could not create deal message. Err: %s", err) @@ -86,7 +84,7 @@ func (eg *eventGenerator) GenerateDealsEvents(ctx context.Context, transactions events.DealsMessages = append(events.DealsMessages, dealMessage) - if eg.isPublishStorageDeals(actorName, tx.TxType) { + if eg.isPublishStorageDeals(tx.TxType) { dealsInfo, err := eg.createDealsInfo(ctx, tx) if err != nil { eg.logger.Errorf("could not create deal proposal. Err: %s", err) @@ -126,11 +124,7 @@ func (eg *eventGenerator) isDealsStateMessage(actorName, txType string) bool { return false } -func (eg *eventGenerator) isPublishStorageDeals(actorName, txType string) bool { - if !strings.Contains(actorName, manifest.MarketKey) { - return false - } - +func (eg *eventGenerator) isPublishStorageDeals(txType string) bool { switch { case strings.EqualFold(txType, parser.MethodPublishStorageDeals), strings.EqualFold(txType, parser.MethodPublishStorageDealsExported): diff --git a/tools/deals/messages.go b/tools/deals/messages.go index 3ccb958b..29aa99a8 100644 --- a/tools/deals/messages.go +++ b/tools/deals/messages.go @@ -25,7 +25,7 @@ func (eg *eventGenerator) createDealMessage(tx *types.Transaction, tipsetCid, ac Height: tx.Height, TxCid: tx.TxCid, ActionType: txType, - Value: tx.TxMetadata, + Data: tx.TxMetadata, TxTimestamp: tx.TxTimestamp, } diff --git a/tools/deals/proposal.go b/tools/deals/proposal.go index 2f7aacb0..022155ee 100644 --- a/tools/deals/proposal.go +++ b/tools/deals/proposal.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" + "github.com/filecoin-project/go-address" + "github.com/zondax/fil-parser/actors" "github.com/zondax/fil-parser/tools" "github.com/zondax/fil-parser/tools/common" "github.com/zondax/fil-parser/types" @@ -72,7 +74,8 @@ func (eg *eventGenerator) parsePublishStorageDeals(tx *types.Transaction, params // use the return to get the deal ids because the actor may drop invalid deals and return less ids than the params // From NV0 - NV13 the verified deals are in PublishStorageDealsReturn.IDs - if version.NodeVersion() < tools.V14.NodeVersion() { + // This only applies to mainnet. In calibration the verified deals are in PublishStorageDealsReturn.ValidDeals as a bitfield in all versions. + if version.NodeVersion() < tools.V14.NodeVersion() && eg.network == tools.MainnetNetwork { for i, id := range dealIDs { // #nosec G115 validDealIndexToDealID[uint64(i)] = id @@ -172,9 +175,24 @@ func (eg *eventGenerator) parsePublishStorageDeals(tx *types.Transaction, params if err != nil { return nil, fmt.Errorf("error parsing client collateral: %w", err) } + if eg.config.ConsolidateRobustAddress { + consolidatedProviderAddress, err := eg.consolidateAddress(providerAddress) + if err != nil { + eg.logger.Errorf("error consolidating provider address: %s", err.Error()) + } else { + providerAddress = consolidatedProviderAddress + } + consolidatedClientAddress, err := eg.consolidateAddress(clientAddress) + if err != nil { + eg.logger.Errorf("error consolidating client address: %s", err.Error()) + } else { + clientAddress = consolidatedClientAddress + } + } dealsInfo = append(dealsInfo, &types.DealsProposals{ ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealID)), + ActorAddress: tx.TxFrom, Height: tx.Height, DealID: dealID, TxCid: tx.TxCid, @@ -187,7 +205,7 @@ func (eg *eventGenerator) parsePublishStorageDeals(tx *types.Transaction, params Label: label, StartEpoch: startEpoch, EndEpoch: endEpoch, - PricePerEpoch: storagePricePerEpoch.Uint64(), + PricePerEpoch: storagePricePerEpoch, ProviderCollateral: providerCollateral, ClientCollateral: clientCollateral, TxTimestamp: tx.TxTimestamp, @@ -197,3 +215,15 @@ func (eg *eventGenerator) parsePublishStorageDeals(tx *types.Transaction, params return dealsInfo, nil } + +func (eg *eventGenerator) consolidateAddress(addrStr string) (string, error) { + addr, err := address.NewFromString(addrStr) + if err != nil { + return "", fmt.Errorf("error parsing address: %w", err) + } + consolidatedAddress, err := actors.ConsolidateToRobustAddress(addr, eg.helper, eg.logger, eg.config.RobustAddressBestEffort) + if err != nil { + return "", fmt.Errorf("error consolidating address: %w", err) + } + return consolidatedAddress, nil +} diff --git a/tools/miner/sector.go b/tools/miner/sector.go index 5604b7ea..1911e368 100644 --- a/tools/miner/sector.go +++ b/tools/miner/sector.go @@ -400,17 +400,17 @@ func (eg *eventGenerator) consolidatePieceActivationManifests(_ context.Context, for _, piece := range pieces { verifiedAllocationKey, err := common.GetItem[map[string]interface{}](piece, KeyVerifiedAllocationKey, true) if err != nil { - return nil, fmt.Errorf("error parsing verified allocation key: %w", err) + return nil, fmt.Errorf("error parsing verified allocation key: %s", err) } if len(verifiedAllocationKey) > 0 { - clientIDAddrStr, err := common.GetInteger[uint64](verifiedAllocationKey, KeyAddress, false) + clientIDAddrStr, err := common.GetItem[string](verifiedAllocationKey, KeyAddress, false) if err != nil { - eg.logger.Errorf("error parsing client id address: %w", err) + eg.logger.Errorf("error parsing client id address: %s", err) break } - consolidatedClientIDAddr, err := eg.consolidateIDAddress(clientIDAddrStr) + consolidatedClientIDAddr, err := eg.consolidateAddress(clientIDAddrStr) if err != nil { - eg.logger.Errorf("error consolidating client id address: %w", err) + eg.logger.Errorf("error consolidating client id address: %s", err) break } verifiedAllocationKey[KeyAddress] = consolidatedClientIDAddr @@ -427,12 +427,12 @@ func (eg *eventGenerator) consolidatePieceActivationManifests(_ context.Context, for _, notify := range dataActivationNotifications { addrStr, err := common.GetItem[string](notify, KeyAddress, false) if err != nil { - eg.logger.Errorf("error parsing notify number: %w", err) + eg.logger.Errorf("error parsing notify number: %s", err) break } consolidatedAddr, err := eg.consolidateAddress(addrStr) if err != nil { - eg.logger.Errorf("error consolidating address: %w", err) + eg.logger.Errorf("error consolidating address: %s", err) break } notify[KeyAddress] = consolidatedAddr diff --git a/types/deals.go b/types/deals.go index c903ef52..02996078 100644 --- a/types/deals.go +++ b/types/deals.go @@ -18,15 +18,16 @@ type DealsMessages struct { Height uint64 `json:"height"` TxCid string `json:"tx_cid"` ActionType string `json:"action_type"` - Value string `json:"value"` + Data string `json:"data"` TxTimestamp time.Time `json:"tx_timestamp"` } type DealsProposals struct { - ID string `json:"id"` - Height uint64 `json:"height"` - DealID uint64 `json:"deal_id"` - TxCid string `json:"tx_cid"` + ID string `json:"id"` + Height uint64 `json:"height"` + ActorAddress string `json:"actor_address"` + DealID uint64 `json:"deal_id"` + TxCid string `json:"tx_cid"` // proposal details ClientSignature string `json:"client_signature"` @@ -43,12 +44,12 @@ type DealsProposals struct { // with total amount StoragePricePerEpoch * (EndEpoch - StartEpoch). // Storage deal must appear in a sealed (proven) sector no later than StartEpoch, // otherwise it is invalid. - StartEpoch int64 `json:"start_epoch"` - EndEpoch int64 `json:"end_epoch"` - PricePerEpoch uint64 `json:"price_per_epoch"` + StartEpoch int64 `json:"start_epoch"` + EndEpoch int64 `json:"end_epoch"` + PricePerEpoch *big.Int `json:"price_per_epoch" gorm:"column:price_per_epoch;type:UInt256"` - ProviderCollateral *big.Int `json:"provider_collateral"` - ClientCollateral *big.Int `json:"client_collateral"` + ProviderCollateral *big.Int `json:"provider_collateral" gorm:"column:provider_collateral;type:UInt256"` + ClientCollateral *big.Int `json:"client_collateral" gorm:"column:client_collateral;type:UInt256"` TxTimestamp time.Time `json:"tx_timestamp"` } @@ -56,6 +57,7 @@ type DealsProposals struct { type DealsActivations struct { ID string `json:"id"` Height uint64 `json:"height"` + ActorAddress string `json:"actor_address"` TxCid string `json:"tx_cid"` DealID uint64 `json:"deal_id"` SectorExpiry int64 `json:"sector_expiry"` @@ -64,16 +66,18 @@ type DealsActivations struct { } type DealsSpaceInfo struct { - ID string `json:"id"` - Height uint64 `json:"height"` - TxCid string `json:"tx_cid"` - DealIDs []uint64 `json:"deal_ids"` + ID string `json:"id"` + Height uint64 `json:"height"` + ActorAddress string `json:"actor_address"` + TxCid string `json:"tx_cid"` + DealID uint64 `json:"deal_id"` + GroupDealIDs []uint64 `json:"group_deal_ids"` // NonVerifiedDealWeight is the sum(piece_size * deal_duration) of all the non-verified deals // VerifiedDealWeight is the sum(piece_size * deal_duration) of all the verified deals // NonVerifiedDealSpace is the sum(piece_size) of all the deals // VerifiedDealSpace is the sum(piece_size) of all the verified deals - NonVerifiedDealSpace *big.Int `json:"non_verified_deal_space"` - VerifiedDealSpace *big.Int `json:"verified_deal_space"` + NonVerifiedDealSpace *big.Int `json:"non_verified_deal_space" gorm:"column:non_verified_deal_space;type:UInt256"` + VerifiedDealSpace *big.Int `json:"verified_deal_space" gorm:"column:verified_deal_space;type:UInt256"` // SpaceAsWeight is true if the deal space is expressed as a weight. Retrieve the deal space by dividing the VerifiedDealSpace/NonVerifiedDealSpace by the DealDuration for each dealId in the DealIDs slice. SpaceAsWeight bool `json:"space_as_weight"` ActionType string `json:"action_type"`