Add feature to stagger sector prove commit submission

api/api_full.go

@@ -955,7 +955,7 @@ type MsgLookup struct {
 
 type MsgGasCost struct {
 	Message            cid.Cid // Can be different than requested, in case it was replaced, but only gas values changed
-	GasUsed            abi.TokenAmount
+	GasUsed            uint64
 	BaseFeeBurn        abi.TokenAmount
 	OverEstimationBurn abi.TokenAmount
 	MinerPenalty       abi.TokenAmount

chain/stmgr/utils.go

@@ -204,7 +204,7 @@ func CheckTotalFIL(ctx context.Context, cs *store.ChainStore, ts *types.TipSet)
 func MakeMsgGasCost(msg *types.Message, ret *vm.ApplyRet) api.MsgGasCost {
 	return api.MsgGasCost{
 		Message:            msg.Cid(),
-		GasUsed:            big.NewInt(ret.GasUsed),
+		GasUsed:            uint64(ret.GasUsed),
 		BaseFeeBurn:        ret.GasCosts.BaseFeeBurn,
 		OverEstimationBurn: ret.GasCosts.OverEstimationBurn,
 		MinerPenalty:       ret.GasCosts.MinerPenalty,

documentation/en/api-v0-methods.md

@@ -4856,7 +4856,7 @@ Response:
     "Message": {
       "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
     },
-    "GasUsed": "0",
+    "GasUsed": 42,
     "BaseFeeBurn": "0",
     "OverEstimationBurn": "0",
     "MinerPenalty": "0",
@@ -5086,7 +5086,7 @@ Response:
         "Message": {
           "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
         },
-        "GasUsed": "0",
+        "GasUsed": 42,
         "BaseFeeBurn": "0",
         "OverEstimationBurn": "0",
         "MinerPenalty": "0",
@@ -6472,7 +6472,7 @@ Response:
     "Message": {
       "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
     },
-    "GasUsed": "0",
+    "GasUsed": 42,
     "BaseFeeBurn": "0",
     "OverEstimationBurn": "0",
     "MinerPenalty": "0",

documentation/en/api-v1-unstable-methods.md

@@ -6281,7 +6281,7 @@ Response:
     "Message": {
       "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
     },
-    "GasUsed": "0",
+    "GasUsed": 42,
     "BaseFeeBurn": "0",
     "OverEstimationBurn": "0",
     "MinerPenalty": "0",
@@ -6511,7 +6511,7 @@ Response:
         "Message": {
           "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
         },
-        "GasUsed": "0",
+        "GasUsed": 42,
         "BaseFeeBurn": "0",
         "OverEstimationBurn": "0",
         "MinerPenalty": "0",
@@ -7980,7 +7980,7 @@ Response:
     "Message": {
       "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
     },
-    "GasUsed": "0",
+    "GasUsed": 42,
     "BaseFeeBurn": "0",
     "OverEstimationBurn": "0",
     "MinerPenalty": "0",

documentation/en/default-lotus-miner-config.toml

@@ -657,6 +657,16 @@
   # env var: LOTUS_SEALING_AGGREGATEABOVEBASEFEE
   #AggregateAboveBaseFee = "0.00000000032 FIL"
 
+  # When submitting several sector prove commit messages simultaneously, this option allows you to
+  # stagger the number of prove commits submitted per epoch
+  # This is done because gas estimates for ProveCommits are non deterministic and increasing as a large
+  # number of sectors get committed within the same epoch resulting in occasionally failed msgs.
+  # Submitting a smaller number of prove commits per epoch would reduce the possibility of failed msgs
+  #
+  # type: uint64
+  # env var: LOTUS_SEALING_MAXSECTORPROVECOMMITSSUBMITTEDPEREPOCH
+  #MaxSectorProveCommitsSubmittedPerEpoch = 0
+
   # type: uint64
   # env var: LOTUS_SEALING_TERMINATEBATCHMAX
   #TerminateBatchMax = 100

node/config/types.go

@@ -422,6 +422,13 @@ type SealingConfig struct {
 	// submitting proofs to the chain individually
 	AggregateAboveBaseFee types.FIL
 
+	// When submitting several sector prove commit messages simultaneously, this option allows you to
+	// stagger the number of prove commits submitted per epoch
+	// This is done because gas estimates for ProveCommits are non deterministic and increasing as a large
+	// number of sectors get committed within the same epoch resulting in occasionally failed msgs.
+	// Submitting a smaller number of prove commits per epoch would reduce the possibility of failed msgs
+	MaxSectorProveCommitsSubmittedPerEpoch uint64
+
 	TerminateBatchMax  uint64
 	TerminateBatchMin  uint64
 	TerminateBatchWait Duration

node/modules/storageminer.go

@@ -1014,9 +1014,10 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error
 				AggregateAboveBaseFee:      types.FIL(cfg.AggregateAboveBaseFee),
 				BatchPreCommitAboveBaseFee: types.FIL(cfg.BatchPreCommitAboveBaseFee),
 
-				TerminateBatchMax:  cfg.TerminateBatchMax,
-				TerminateBatchMin:  cfg.TerminateBatchMin,
-				TerminateBatchWait: config.Duration(cfg.TerminateBatchWait),
+				TerminateBatchMax:                      cfg.TerminateBatchMax,
+				TerminateBatchMin:                      cfg.TerminateBatchMin,
+				TerminateBatchWait:                     config.Duration(cfg.TerminateBatchWait),
+				MaxSectorProveCommitsSubmittedPerEpoch: cfg.MaxSectorProveCommitsSubmittedPerEpoch,
 			}
 			c.SetSealingConfig(newCfg)
 		})
@@ -1051,13 +1052,14 @@ func ToSealingConfig(dealmakingCfg config.DealmakingConfig, sealingCfg config.Se
 		PreCommitBatchWait:  time.Duration(sealingCfg.PreCommitBatchWait),
 		PreCommitBatchSlack: time.Duration(sealingCfg.PreCommitBatchSlack),
 
-		AggregateCommits:           sealingCfg.AggregateCommits,
-		MinCommitBatch:             sealingCfg.MinCommitBatch,
-		MaxCommitBatch:             sealingCfg.MaxCommitBatch,
-		CommitBatchWait:            time.Duration(sealingCfg.CommitBatchWait),
-		CommitBatchSlack:           time.Duration(sealingCfg.CommitBatchSlack),
-		AggregateAboveBaseFee:      types.BigInt(sealingCfg.AggregateAboveBaseFee),
-		BatchPreCommitAboveBaseFee: types.BigInt(sealingCfg.BatchPreCommitAboveBaseFee),
+		AggregateCommits:                       sealingCfg.AggregateCommits,
+		MinCommitBatch:                         sealingCfg.MinCommitBatch,
+		MaxCommitBatch:                         sealingCfg.MaxCommitBatch,
+		CommitBatchWait:                        time.Duration(sealingCfg.CommitBatchWait),
+		CommitBatchSlack:                       time.Duration(sealingCfg.CommitBatchSlack),
+		AggregateAboveBaseFee:                  types.BigInt(sealingCfg.AggregateAboveBaseFee),
+		BatchPreCommitAboveBaseFee:             types.BigInt(sealingCfg.BatchPreCommitAboveBaseFee),
+		MaxSectorProveCommitsSubmittedPerEpoch: sealingCfg.MaxSectorProveCommitsSubmittedPerEpoch,
 
 		TerminateBatchMax:  sealingCfg.TerminateBatchMax,
 		TerminateBatchMin:  sealingCfg.TerminateBatchMin,

storage/pipeline/commit_batch.go

@@ -388,6 +388,7 @@ func (b *CommitBatcher) processBatch(cfg sealiface.Config) ([]sealiface.CommitBa
 }
 
 func (b *CommitBatcher) processIndividually(cfg sealiface.Config) ([]sealiface.CommitBatchRes, error) {
+
 	mi, err := b.api.StateMinerInfo(b.mctx, b.maddr, types.EmptyTSK)
 	if err != nil {
 		return nil, xerrors.Errorf("couldn't get miner info: %w", err)
@@ -414,12 +415,30 @@ func (b *CommitBatcher) processIndividually(cfg sealiface.Config) ([]sealiface.C
 
 	var res []sealiface.CommitBatchRes
 
+	sectorsProcessed := 0
+
 	for sn, info := range b.todo {
 		r := sealiface.CommitBatchRes{
 			Sectors:       []abi.SectorNumber{sn},
 			FailedSectors: map[abi.SectorNumber]string{},
 		}
 
+		if cfg.MaxSectorProveCommitsSubmittedPerEpoch > 0 &&
+			uint64(sectorsProcessed) >= cfg.MaxSectorProveCommitsSubmittedPerEpoch {
+
+			tmp := ts
+			for tmp.Height() <= ts.Height() {
+				tmp, err = b.api.ChainHead(b.mctx)
+				if err != nil {
+					log.Errorf("getting chain head: %+v", err)
+					return nil, err
+				}
+				time.Sleep(3 * time.Second)
+			}
+
+			sectorsProcessed = 0
+		}
+
 		mcid, err := b.processSingle(cfg, mi, &avail, sn, info, ts.Key())
 		if err != nil {
 			log.Errorf("process single error: %+v", err) // todo: return to user
@@ -429,6 +448,8 @@ func (b *CommitBatcher) processIndividually(cfg sealiface.Config) ([]sealiface.C
 		}
 
 		res = append(res, r)
+
+		sectorsProcessed++
 	}
 
 	return res, nil

storage/pipeline/sealiface/config.go

@@ -58,6 +58,8 @@ type Config struct {
 	AggregateAboveBaseFee      abi.TokenAmount
 	BatchPreCommitAboveBaseFee abi.TokenAmount
 
+	MaxSectorProveCommitsSubmittedPerEpoch uint64
+
 	TerminateBatchMax  uint64
 	TerminateBatchMin  uint64
 	TerminateBatchWait time.Duration