Skip to content

New Release #474

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ziscky merged 4 commits into from
Aug 14, 2025
Merged

New Release #474

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ef50fd2
Fix deals and activation manifest addr parsing (#473)
ziscky Aug 11, 2025
5e4497d
Fix deal space info (#475)
ziscky Aug 12, 2025
1ac9d01
Fix deals message filter (#477)
ziscky Aug 13, 2025
b04b916
fix: use correct tags for uint256 (#478)
ziscky Aug 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions parser/v1/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func NewParser(helper *helper.Helper, logger *logger.Logger, metrics metrics.Met
logger: logger2.GetSafeLogger(logger),
multisigEventGenerator: multisigTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config),
minerEventGenerator: minerTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config),
dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, networkName),
dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, networkName, config),
metrics: parsermetrics.NewClient(metrics, "parserV1"),
actorsCacheMetrics: cacheMetrics.NewClient(metrics, "actorsCache"),
config: config,
Expand All @@ -86,7 +86,7 @@ func NewActorsV2Parser(network string, helper *helper.Helper, logger *logger.Log
logger: logger2.GetSafeLogger(logger),
multisigEventGenerator: multisigTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config),
minerEventGenerator: minerTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config),
dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, network),
dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, network, config),
metrics: parsermetrics.NewClient(metrics, "parserV1"),
actorsCacheMetrics: cacheMetrics.NewClient(metrics, "actorsCache"),
config: config,
Expand Down
4 changes: 2 additions & 2 deletions parser/v2/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func NewParser(helper *helper.Helper, logger *logger.Logger, metrics metrics.Met
logger: logger2.GetSafeLogger(logger),
multisigEventGenerator: multisigTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config),
minerEventGenerator: minerTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config),
dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, networkName),
dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, networkName, config),
metrics: parsermetrics.NewClient(metrics, "parserV2"),
actorsCacheMetrics: cacheMetrics.NewClient(metrics, "actorsCache"),
config: config,
Expand All @@ -91,7 +91,7 @@ func NewActorsV2Parser(network string, helper *helper.Helper, logger *logger.Log
logger: logger2.GetSafeLogger(logger),
multisigEventGenerator: multisigTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config),
minerEventGenerator: minerTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, config),
dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, network),
dealsEventGenerator: dealsTools.NewEventGenerator(helper, logger2.GetSafeLogger(logger), metrics, network, config),
metrics: parsermetrics.NewClient(metrics, "parserV2"),
actorsCacheMetrics: cacheMetrics.NewClient(metrics, "actorsCache"),
config: config,
Expand Down
75 changes: 44 additions & 31 deletions tools/deals/activations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,16 +88,20 @@ func (eg *eventGenerator) parseVerifyDealsForActivation(tx *types.Transaction, p
return nil, err
}

dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{
ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealIDs)),
Height: tx.Height,
TxCid: tx.TxCid,
DealIDs: dealIDs,
NonVerifiedDealSpace: nonVerifiedDealWeight,
VerifiedDealSpace: verifiedDealWeight,
ActionType: tx.TxType,
TxTimestamp: tx.TxTimestamp,
})
for _, dealID := range dealIDs {
dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{
ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealID)),
Height: tx.Height,
ActorAddress: tx.TxFrom,
TxCid: tx.TxCid,
DealID: dealID,
GroupDealIDs: dealIDs,
NonVerifiedDealSpace: nonVerifiedDealWeight,
VerifiedDealSpace: verifiedDealWeight,
ActionType: tx.TxType,
TxTimestamp: tx.TxTimestamp,
})
}
Comment thread
ziscky marked this conversation as resolved.

return dealSpaceInfo, nil
}
Expand Down Expand Up @@ -147,17 +151,21 @@ func (eg *eventGenerator) parseVerifyDealsForActivation(tx *types.Transaction, p
return nil, err
}

dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{
ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealIDs[i])),
Height: tx.Height,
TxCid: tx.TxCid,
DealIDs: dealIDs,
NonVerifiedDealSpace: nonVerifiedDealWeight,
VerifiedDealSpace: verifiedDealWeight,
SpaceAsWeight: true,
ActionType: tx.TxType,
TxTimestamp: tx.TxTimestamp,
})
for _, dealID := range dealIDs {
dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{
ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealID)),
Height: tx.Height,
ActorAddress: tx.TxFrom,
TxCid: tx.TxCid,
DealID: dealID,
GroupDealIDs: dealIDs,
NonVerifiedDealSpace: nonVerifiedDealWeight,
VerifiedDealSpace: verifiedDealWeight,
SpaceAsWeight: true,
ActionType: tx.TxType,
TxTimestamp: tx.TxTimestamp,
})
}
}
return dealSpaceInfo, nil
}
Expand All @@ -180,6 +188,7 @@ func (eg *eventGenerator) parseActivateDeals(tx *types.Transaction, params, ret
for _, dealID := range dealIDs {
dealActivations = append(dealActivations, &types.DealsActivations{
ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealID)),
ActorAddress: tx.TxFrom,
Height: tx.Height,
TxCid: tx.TxCid,
DealID: dealID,
Expand All @@ -196,16 +205,20 @@ func (eg *eventGenerator) parseActivateDeals(tx *types.Transaction, params, ret
if err != nil {
return err
}
dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{
ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealIDs)),
Height: tx.Height,
TxCid: tx.TxCid,
DealIDs: dealIDs,
NonVerifiedDealSpace: nonVerifiedDealSpace,
VerifiedDealSpace: verifiedDealSpace,
ActionType: tx.TxType,
TxTimestamp: tx.TxTimestamp,
})
for _, dealID := range dealIDs {
dealSpaceInfo = append(dealSpaceInfo, &types.DealsSpaceInfo{
ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealID)),
DealID: dealID,
Height: tx.Height,
ActorAddress: tx.TxFrom,
TxCid: tx.TxCid,
GroupDealIDs: dealIDs,
NonVerifiedDealSpace: nonVerifiedDealSpace,
VerifiedDealSpace: verifiedDealSpace,
ActionType: tx.TxType,
TxTimestamp: tx.TxTimestamp,
})
}
return nil
}

Expand Down
22 changes: 8 additions & 14 deletions tools/deals/deals.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,16 @@ type eventGenerator struct {
logger *logger.Logger
metrics *dealsMetricsClient
network string
config parser.Config
}

func NewEventGenerator(helper *helper.Helper, logger *logger.Logger, metrics metrics.MetricsClient, network string) EventGenerator {
func NewEventGenerator(helper *helper.Helper, logger *logger.Logger, metrics metrics.MetricsClient, network string, config parser.Config) EventGenerator {
return &eventGenerator{
helper: helper,
logger: logger,
metrics: newClient(metrics, "deals"),
network: network,
config: config,
}
}

Expand All @@ -55,13 +57,7 @@ func (eg *eventGenerator) GenerateDealsEvents(ctx context.Context, transactions
continue
}

actorAddress := tx.TxTo
// this is executed by(from) the miner actor
if tx.TxType == parser.MethodUpdateClaimedPower || tx.TxType == parser.TotalFeeOp {
actorAddress = tx.TxFrom
}

addr, err := address.NewFromString(actorAddress)
addr, err := address.NewFromString(tx.TxTo)
if err != nil {
eg.logger.Errorf("could not parse address. Err: %s", err)
}
Expand All @@ -78,6 +74,8 @@ func (eg *eventGenerator) GenerateDealsEvents(ctx context.Context, transactions
continue
}

// the address that calls the market actor
actorAddress := tx.TxFrom
dealMessage, err := eg.createDealMessage(tx, tipsetCid, actorAddress)
if err != nil {
eg.logger.Errorf("could not create deal message. Err: %s", err)
Expand All @@ -86,7 +84,7 @@ func (eg *eventGenerator) GenerateDealsEvents(ctx context.Context, transactions

events.DealsMessages = append(events.DealsMessages, dealMessage)

if eg.isPublishStorageDeals(actorName, tx.TxType) {
if eg.isPublishStorageDeals(tx.TxType) {
dealsInfo, err := eg.createDealsInfo(ctx, tx)
if err != nil {
eg.logger.Errorf("could not create deal proposal. Err: %s", err)
Expand Down Expand Up @@ -126,11 +124,7 @@ func (eg *eventGenerator) isDealsStateMessage(actorName, txType string) bool {
return false
}

func (eg *eventGenerator) isPublishStorageDeals(actorName, txType string) bool {
if !strings.Contains(actorName, manifest.MarketKey) {
return false
}

func (eg *eventGenerator) isPublishStorageDeals(txType string) bool {
switch {
case strings.EqualFold(txType, parser.MethodPublishStorageDeals),
strings.EqualFold(txType, parser.MethodPublishStorageDealsExported):
Expand Down
2 changes: 1 addition & 1 deletion tools/deals/messages.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func (eg *eventGenerator) createDealMessage(tx *types.Transaction, tipsetCid, ac
Height: tx.Height,
TxCid: tx.TxCid,
ActionType: txType,
Value: tx.TxMetadata,
Data: tx.TxMetadata,
TxTimestamp: tx.TxTimestamp,
}

Expand Down
34 changes: 32 additions & 2 deletions tools/deals/proposal.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"encoding/json"
"fmt"

"github.com/filecoin-project/go-address"
"github.com/zondax/fil-parser/actors"
"github.com/zondax/fil-parser/tools"
"github.com/zondax/fil-parser/tools/common"
"github.com/zondax/fil-parser/types"
Expand Down Expand Up @@ -72,7 +74,8 @@ func (eg *eventGenerator) parsePublishStorageDeals(tx *types.Transaction, params

// use the return to get the deal ids because the actor may drop invalid deals and return less ids than the params
// From NV0 - NV13 the verified deals are in PublishStorageDealsReturn.IDs
if version.NodeVersion() < tools.V14.NodeVersion() {
// This only applies to mainnet. In calibration the verified deals are in PublishStorageDealsReturn.ValidDeals as a bitfield in all versions.
if version.NodeVersion() < tools.V14.NodeVersion() && eg.network == tools.MainnetNetwork {
for i, id := range dealIDs {
// #nosec G115
validDealIndexToDealID[uint64(i)] = id
Expand Down Expand Up @@ -172,9 +175,24 @@ func (eg *eventGenerator) parsePublishStorageDeals(tx *types.Transaction, params
if err != nil {
return nil, fmt.Errorf("error parsing client collateral: %w", err)
}
if eg.config.ConsolidateRobustAddress {
consolidatedProviderAddress, err := eg.consolidateAddress(providerAddress)
if err != nil {
eg.logger.Errorf("error consolidating provider address: %s", err.Error())
} else {
providerAddress = consolidatedProviderAddress
}
consolidatedClientAddress, err := eg.consolidateAddress(clientAddress)
if err != nil {
eg.logger.Errorf("error consolidating client address: %s", err.Error())
} else {
clientAddress = consolidatedClientAddress
}
}
Comment thread
ziscky marked this conversation as resolved.

dealsInfo = append(dealsInfo, &types.DealsProposals{
ID: tools.BuildId(tx.TxCid, tx.TxFrom, tx.TxTo, fmt.Sprint(tx.Height), tx.TxType, fmt.Sprint(dealID)),
ActorAddress: tx.TxFrom,
Height: tx.Height,
DealID: dealID,
TxCid: tx.TxCid,
Expand All @@ -187,7 +205,7 @@ func (eg *eventGenerator) parsePublishStorageDeals(tx *types.Transaction, params
Label: label,
StartEpoch: startEpoch,
EndEpoch: endEpoch,
PricePerEpoch: storagePricePerEpoch.Uint64(),
PricePerEpoch: storagePricePerEpoch,
ProviderCollateral: providerCollateral,
ClientCollateral: clientCollateral,
TxTimestamp: tx.TxTimestamp,
Expand All @@ -197,3 +215,15 @@ func (eg *eventGenerator) parsePublishStorageDeals(tx *types.Transaction, params

return dealsInfo, nil
}

func (eg *eventGenerator) consolidateAddress(addrStr string) (string, error) {
addr, err := address.NewFromString(addrStr)
if err != nil {
return "", fmt.Errorf("error parsing address: %w", err)
}
consolidatedAddress, err := actors.ConsolidateToRobustAddress(addr, eg.helper, eg.logger, eg.config.RobustAddressBestEffort)
if err != nil {
return "", fmt.Errorf("error consolidating address: %w", err)
}
return consolidatedAddress, nil
}
Comment thread
ziscky marked this conversation as resolved.
14 changes: 7 additions & 7 deletions tools/miner/sector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,17 +400,17 @@ func (eg *eventGenerator) consolidatePieceActivationManifests(_ context.Context,
for _, piece := range pieces {
verifiedAllocationKey, err := common.GetItem[map[string]interface{}](piece, KeyVerifiedAllocationKey, true)
if err != nil {
return nil, fmt.Errorf("error parsing verified allocation key: %w", err)
return nil, fmt.Errorf("error parsing verified allocation key: %s", err)
}
Comment thread
ziscky marked this conversation as resolved.
if len(verifiedAllocationKey) > 0 {
clientIDAddrStr, err := common.GetInteger[uint64](verifiedAllocationKey, KeyAddress, false)
clientIDAddrStr, err := common.GetItem[string](verifiedAllocationKey, KeyAddress, false)
if err != nil {
eg.logger.Errorf("error parsing client id address: %w", err)
eg.logger.Errorf("error parsing client id address: %s", err)
break
}
consolidatedClientIDAddr, err := eg.consolidateIDAddress(clientIDAddrStr)
consolidatedClientIDAddr, err := eg.consolidateAddress(clientIDAddrStr)
if err != nil {
eg.logger.Errorf("error consolidating client id address: %w", err)
eg.logger.Errorf("error consolidating client id address: %s", err)
break
}
verifiedAllocationKey[KeyAddress] = consolidatedClientIDAddr
Expand All @@ -427,12 +427,12 @@ func (eg *eventGenerator) consolidatePieceActivationManifests(_ context.Context,
for _, notify := range dataActivationNotifications {
addrStr, err := common.GetItem[string](notify, KeyAddress, false)
if err != nil {
eg.logger.Errorf("error parsing notify number: %w", err)
eg.logger.Errorf("error parsing notify number: %s", err)
break
}
consolidatedAddr, err := eg.consolidateAddress(addrStr)
if err != nil {
eg.logger.Errorf("error consolidating address: %w", err)
eg.logger.Errorf("error consolidating address: %s", err)
break
}
notify[KeyAddress] = consolidatedAddr
Expand Down
36 changes: 20 additions & 16 deletions types/deals.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,16 @@ type DealsMessages struct {
Height uint64 `json:"height"`
TxCid string `json:"tx_cid"`
ActionType string `json:"action_type"`
Value string `json:"value"`
Data string `json:"data"`
Comment thread
ziscky marked this conversation as resolved.
TxTimestamp time.Time `json:"tx_timestamp"`
}

type DealsProposals struct {
ID string `json:"id"`
Height uint64 `json:"height"`
DealID uint64 `json:"deal_id"`
TxCid string `json:"tx_cid"`
ID string `json:"id"`
Height uint64 `json:"height"`
ActorAddress string `json:"actor_address"`
DealID uint64 `json:"deal_id"`
TxCid string `json:"tx_cid"`

// proposal details
ClientSignature string `json:"client_signature"`
Expand All @@ -43,19 +44,20 @@ type DealsProposals struct {
// with total amount StoragePricePerEpoch * (EndEpoch - StartEpoch).
// Storage deal must appear in a sealed (proven) sector no later than StartEpoch,
// otherwise it is invalid.
StartEpoch int64 `json:"start_epoch"`
EndEpoch int64 `json:"end_epoch"`
PricePerEpoch uint64 `json:"price_per_epoch"`
StartEpoch int64 `json:"start_epoch"`
EndEpoch int64 `json:"end_epoch"`
PricePerEpoch *big.Int `json:"price_per_epoch" gorm:"column:price_per_epoch;type:UInt256"`

ProviderCollateral *big.Int `json:"provider_collateral"`
ClientCollateral *big.Int `json:"client_collateral"`
ProviderCollateral *big.Int `json:"provider_collateral" gorm:"column:provider_collateral;type:UInt256"`
ClientCollateral *big.Int `json:"client_collateral" gorm:"column:client_collateral;type:UInt256"`

TxTimestamp time.Time `json:"tx_timestamp"`
}

type DealsActivations struct {
ID string `json:"id"`
Height uint64 `json:"height"`
ActorAddress string `json:"actor_address"`
TxCid string `json:"tx_cid"`
DealID uint64 `json:"deal_id"`
SectorExpiry int64 `json:"sector_expiry"`
Expand All @@ -64,16 +66,18 @@ type DealsActivations struct {
}

type DealsSpaceInfo struct {
ID string `json:"id"`
Height uint64 `json:"height"`
TxCid string `json:"tx_cid"`
DealIDs []uint64 `json:"deal_ids"`
ID string `json:"id"`
Height uint64 `json:"height"`
ActorAddress string `json:"actor_address"`
TxCid string `json:"tx_cid"`
DealID uint64 `json:"deal_id"`
GroupDealIDs []uint64 `json:"group_deal_ids"`
Comment thread
ziscky marked this conversation as resolved.
// NonVerifiedDealWeight is the sum(piece_size * deal_duration) of all the non-verified deals
// VerifiedDealWeight is the sum(piece_size * deal_duration) of all the verified deals
// NonVerifiedDealSpace is the sum(piece_size) of all the deals
// VerifiedDealSpace is the sum(piece_size) of all the verified deals
NonVerifiedDealSpace *big.Int `json:"non_verified_deal_space"`
VerifiedDealSpace *big.Int `json:"verified_deal_space"`
NonVerifiedDealSpace *big.Int `json:"non_verified_deal_space" gorm:"column:non_verified_deal_space;type:UInt256"`
VerifiedDealSpace *big.Int `json:"verified_deal_space" gorm:"column:verified_deal_space;type:UInt256"`
// SpaceAsWeight is true if the deal space is expressed as a weight. Retrieve the deal space by dividing the VerifiedDealSpace/NonVerifiedDealSpace by the DealDuration for each dealId in the DealIDs slice.
SpaceAsWeight bool `json:"space_as_weight"`
ActionType string `json:"action_type"`
Expand Down
Loading