Skip to content
This repository has been archived by the owner on Dec 19, 2022. It is now read-only.

Commit

Permalink
Release/v1.6.0 (#206)
Browse files Browse the repository at this point in the history 
* fix missing file type

* Less verbose sector manager logging

* avoid panic

* PRU insufficient collateral

* Include worker name in sealing errors

* Don't log CIDs as binary

* Output primary sector status correctly

* Estimate deal weight in sector list when upgrading

* Remove sector copies from workers after snapdeals

* Fix error loop on bad event

* FinalizeSector doesn't need sealed replica access

* always do cooldown in handleSubmitReplicaUpdateFailed

* Handle inputLk correctly

* remove endpoint from cli

* Sector upgrade queue

* Rename error message

* More SnapDeals config knobs

* Move MakeNewSectorForDeals config into the Sealing sectoin

* check fd limit on startup

* proving check --faulty, faults in storage list sectors

* Change Propose-worker msg

* API/CLI to compute window-post

* Thread safe piecereader

* Fix PR1 worker selection

* generate api doc

* release worker tracker lock when call cb func

* Release unsealed sector files after snapdeals abort

* Improve worker assigning logic

* Don't panic in ReleaseUnsealed with no ranges

* Fail to add expired precommits to a batch

* Make FSOverhead numbers more accurate

* update ffi

* upgrade venus & venus-market

* fix builtin actors bundles

* fix: worker: Download proofs if PRU2 is enabled

* fix builtin actor bundle

* Use new go-state-types state accessors

* opt: sector fsm & storage

* chore: upgrade go mod

* opt: for sector cmd

* cmd: not check network name

* feat: Snap Deals full unseal

* chore: upgrade venus

* cmd: Expand repo path

* chore: upgrade go version & mod ref

* chore: upgrade version to v1.6.0-rc1

* chore: update specs-storage to v0.4.0

* build: bump version to v1.6.0-rc2

* feat(sealing): DataCid on workers

* feat: cli command for compactPartitions

* opt(worker): do not log normal storage stat calls

* feat(wdpost): Ignore faults when compute window-post

* chore: remove unuse file

* fix: BlockDelaySecs for butterfly

* fix: Trigger wdpost multiple times per cycle

* feat: upgrade to 8.0.0-rc.1 bundle & manifest

* feat: Add new proofs version type

* feat: check index out of bounds against correct param length not return length

* fix: imported and not used

* chore: upgrade venus & venus-market version

* chore: bump version to 1.6.0-rc3

* chore: bump version to 1.6.0-rc4

Co-authored-by: simlecode <[email protected]>
Co-authored-by: 一页素书 <[email protected]>
  • Loading branch information
@diwufeiwen @simlecode @ta0li
3 people authored Jun 16, 2022
1 parent 9cc11ff commit 655dbd3
Show file tree
Hide file tree
Showing 113 changed files with 3,863 additions and 2,587 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ lotus-fix
.idea
.vscode

builtin-actors/v7/*
builtin-actors/v8/*

*.coverprofile
*.out
*.so
Expand Down
1 change: 0 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ deps:
api-docs-gen:
go run ./tool/api-docs-gen/cmd ./api/storage_struct.go StorageMiner api ./api/ > ./docs/api-documents.md


lotus-fix:
rm -f lotus-fix
go build -o lotus-fix ./tool/convert-with-lotus/to-lotus
Expand Down
2 changes: 1 addition & 1 deletion api/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func GetFullNodeAPIV2(cctx *cli.Context) (FullNode, jsonrpc.ClientCloser, error)
apiInfo, err := GetFullNodeAPIFromConfig(cctx)
if err != nil {
if !cctx.IsSet("node-url") || !cctx.IsSet("node-token") {
return nil, nil, xerrors.New("must set url or token")
return nil, nil, xerrors.Errorf("must set url or token %w", err)
}

apiInfo = apiinfo.APIInfo{
Expand Down
22 changes: 12 additions & 10 deletions api/fullnode.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,16 @@ package api

import (
"context"

"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/builtin/v8/miner"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/dline"
stnetwork "github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/venus/venus-shared/actors/builtin/miner"
lminer "github.com/filecoin-project/venus/venus-shared/actors/builtin/miner"
"github.com/filecoin-project/venus/venus-shared/types"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
Expand Down Expand Up @@ -287,7 +289,7 @@ type FullNode interface {
// StateMinerPower returns the power of the indicated miner
StateMinerPower(context.Context, address.Address, types.TipSetKey) (*types.MinerPower, error)
// StateMinerInfo returns info about the indicated miner
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error)
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (types.MinerInfo, error)
// StateMinerDeadlines returns all the proving deadlines for the given miner
StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]types.Deadline, error)
// StateMinerPartitions returns all partitions in the specified deadline
Expand All @@ -313,9 +315,9 @@ type FullNode interface {
// expiration epoch
StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)
// StateSectorExpiration returns epoch at which given sector will expire
StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error)
StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*lminer.SectorExpiration, error)
// StateSectorPartition finds deadline/partition with the specified sector
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error)
StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*lminer.SectorLocation, error)
// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*types.MsgLookup, error)
// StateSearchMsgLimited looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
Expand Down Expand Up @@ -577,7 +579,7 @@ type FullNodeStruct struct {
StateMinerActiveSectors func(context.Context, address.Address, types.TipSetKey) ([]*miner.SectorOnChainInfo, error) `perm:"read"`
StateMinerProvingDeadline func(context.Context, address.Address, types.TipSetKey) (*dline.Info, error) `perm:"read"`
StateMinerPower func(context.Context, address.Address, types.TipSetKey) (*types.MinerPower, error) `perm:"read"`
StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) `perm:"read"`
StateMinerInfo func(context.Context, address.Address, types.TipSetKey) (types.MinerInfo, error) `perm:"read"`
StateMinerDeadlines func(context.Context, address.Address, types.TipSetKey) ([]types.Deadline, error) `perm:"read"`
StateMinerPartitions func(ctx context.Context, m address.Address, dlIdx uint64, tsk types.TipSetKey) ([]types.Partition, error) `perm:"read"`
StateMinerFaults func(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error) `perm:"read"`
Expand All @@ -589,8 +591,8 @@ type FullNodeStruct struct {
StateMinerSectorAllocated func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (bool, error) `perm:"read"`
StateSectorPreCommitInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) `perm:"read"`
StateSectorGetInfo func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error) `perm:"read"`
StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) `perm:"read"`
StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorLocation, error) `perm:"read"`
StateSectorExpiration func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*lminer.SectorExpiration, error) `perm:"read"`
StateSectorPartition func(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*lminer.SectorLocation, error) `perm:"read"`
StateCall func(context.Context, *types.Message, types.TipSetKey) (*types.InvocResult, error) `perm:"read"`
StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*types.InvocResult, error) `perm:"read"`
StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"`
Expand Down Expand Up @@ -1023,7 +1025,7 @@ func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address,
return c.Internal.StateMinerPower(ctx, a, tsk)
}

func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) {
func (c *FullNodeStruct) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (types.MinerInfo, error) {
return c.Internal.StateMinerInfo(ctx, actor, tsk)
}

Expand Down Expand Up @@ -1071,11 +1073,11 @@ func (c *FullNodeStruct) StateSectorGetInfo(ctx context.Context, maddr address.A
return c.Internal.StateSectorGetInfo(ctx, maddr, n, tsk)
}

func (c *FullNodeStruct) StateSectorExpiration(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorExpiration, error) {
func (c *FullNodeStruct) StateSectorExpiration(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*lminer.SectorExpiration, error) {
return c.Internal.StateSectorExpiration(ctx, maddr, n, tsk)
}

func (c *FullNodeStruct) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) {
func (c *FullNodeStruct) StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*lminer.SectorLocation, error) {
return c.Internal.StateSectorPartition(ctx, maddr, sectorNumber, tok)
}

Expand Down
35 changes: 29 additions & 6 deletions api/impl/strageminer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/filecoin-project/venus/venus-shared/api/market"
mtypes "github.com/filecoin-project/venus/venus-shared/types/market"
"github.com/filecoin-project/venus/venus-shared/types/messager"
"net/http"
"strconv"
"time"

"github.com/filecoin-project/venus/venus-shared/api/market"
mtypes "github.com/filecoin-project/venus/venus-shared/types/market"
"github.com/filecoin-project/venus/venus-shared/types/messager"

"github.com/google/uuid"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log/v2"
Expand All @@ -26,6 +27,7 @@ import (
sto "github.com/filecoin-project/specs-storage/storage"
multi "github.com/hashicorp/go-multierror"

"github.com/filecoin-project/go-state-types/builtin/v8/miner"
proof7 "github.com/filecoin-project/specs-actors/v7/actors/runtime/proof"

"github.com/filecoin-project/venus-sealer/api"
Expand Down Expand Up @@ -59,6 +61,8 @@ type StorageMinerAPI struct {

AddrSel *storage.AddressSelector

WdPoSt *storage.WindowPoStScheduler

Stor *stores.Remote

MarketClient market.IMarket
Expand Down Expand Up @@ -92,7 +96,7 @@ func (sm *StorageMinerAPI) UpdateDealStatus(ctx context.Context, dealId abi.Deal
return errEmptyMarketClient
}
addr := sm.Miner.Address()
return sm.MarketClient.UpdateDealStatus(ctx, addr, dealId, status)
return sm.MarketClient.UpdateDealStatus(ctx, addr, dealId, mtypes.PieceStatus(status))
}

func (sm *StorageMinerAPI) IsUnsealed(ctx context.Context, sector sto.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) {
Expand Down Expand Up @@ -137,6 +141,25 @@ func (sm *StorageMinerAPI) ActorSectorSize(ctx context.Context, addr address.Add
return mi.SectorSize, nil
}

func (sm *StorageMinerAPI) ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) {
var ts *types.TipSet
var err error
if tsk == types.EmptyTSK {
ts, err = sm.Full.ChainHead(ctx)
} else {
ts, err = sm.Full.ChainGetTipSet(ctx, tsk)
}
if err != nil {
return nil, err
}

return sm.WdPoSt.ComputePoSt(ctx, dlIdx, ts)
}

func (sm *StorageMinerAPI) ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData sto.Data) (abi.PieceInfo, error) {
return sm.StorageMgr.DataCid(ctx, pieceSize, pieceData)
}

func (sm *StorageMinerAPI) PledgeSector(ctx context.Context) (abi.SectorID, error) {
sr, err := sm.Miner.PledgeSector(ctx)
if err != nil {
Expand Down Expand Up @@ -221,7 +244,7 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb
PreCommitMsg: info.PreCommitMessage,
CommitMsg: info.CommitMessage,
Retries: info.InvalidProofs,
ToUpgrade: sm.Miner.IsMarkedForUpgrade(sid),
ToUpgrade: false,

CCUpdate: info.CCUpdate,
UpdateSealed: info.UpdateSealed,
Expand Down Expand Up @@ -373,7 +396,7 @@ func (sm *StorageMinerAPI) SectorsInfoListInStates(ctx context.Context, states [
PreCommitMsg: sector.PreCommitMessage,
CommitMsg: sector.CommitMessage,
Retries: sector.InvalidProofs,
ToUpgrade: sm.Miner.IsMarkedForUpgrade(sector.SectorNumber),
ToUpgrade: false,

LastErr: sector.LastErr,
Log: log,
Expand Down
25 changes: 24 additions & 1 deletion api/storage_struct.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ package api

import (
"context"
"time"

"github.com/filecoin-project/go-state-types/builtin/v8/miner"
mtypes "github.com/filecoin-project/venus/venus-shared/types/market"
"github.com/filecoin-project/venus/venus-shared/types/messager"
"golang.org/x/xerrors"
"time"

"github.com/google/uuid"
"github.com/ipfs/go-cid"
Expand Down Expand Up @@ -52,6 +54,10 @@ type StorageMiner interface {
ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error)
ActorAddressConfig(ctx context.Context) (AddressConfig, error)

ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types2.TipSetKey) ([]miner.SubmitWindowedPoStParams, error)

ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error)

// Temp api for testing
PledgeSector(context.Context) (abi.SectorID, error)

Expand Down Expand Up @@ -195,6 +201,10 @@ type StorageMinerStruct struct {
ActorAddressConfig func(ctx context.Context) (AddressConfig, error) `perm:"read"`
NetParamsConfig func(ctx context.Context) (*config.NetParamsConfig, error) `perm:"read"`

ComputeWindowPoSt func(ctx context.Context, dlIdx uint64, tsk types2.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) `perm:"admin"`

ComputeDataCid func(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) `perm:"admin"`

PledgeSector func(context.Context) (abi.SectorID, error) `perm:"write"`

CurrentSectorID func(ctx context.Context) (abi.SectorNumber, error) `perm:"read"`
Expand Down Expand Up @@ -229,6 +239,7 @@ type StorageMinerStruct struct {
WorkerStats func(context.Context) (map[uuid.UUID]storiface.WorkerStats, error) `perm:"admin"`
WorkerJobs func(context.Context) (map[uuid.UUID][]storiface.WorkerJob, error) `perm:"admin"`

ReturnDataCid func(ctx context.Context, callID types.CallID, pi abi.PieceInfo, err *storiface.CallError) error `perm:"admin" retry:"true"`
ReturnAddPiece func(ctx context.Context, callID types.CallID, pi abi.PieceInfo, err *storiface.CallError) error `perm:"admin" retry:"true"`
ReturnSealPreCommit1 func(ctx context.Context, callID types.CallID, p1o storage.PreCommit1Out, err *storiface.CallError) error `perm:"admin" retry:"true"`
ReturnSealPreCommit2 func(ctx context.Context, callID types.CallID, sealed storage.SectorCids, err *storiface.CallError) error `perm:"admin" retry:"true"`
Expand Down Expand Up @@ -333,6 +344,14 @@ func (c *StorageMinerStruct) ActorAddressConfig(ctx context.Context) (AddressCon
return c.Internal.ActorAddressConfig(ctx)
}

func (c *StorageMinerStruct) ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types2.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) {
return c.Internal.ComputeWindowPoSt(ctx, dlIdx, tsk)
}

func (c *StorageMinerStruct) ComputeDataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) {
return c.Internal.ComputeDataCid(ctx, pieceSize, pieceData)
}

func (c *StorageMinerStruct) PledgeSector(ctx context.Context) (abi.SectorID, error) {
return c.Internal.PledgeSector(ctx)
}
Expand Down Expand Up @@ -453,6 +472,10 @@ func (c *StorageMinerStruct) WorkerJobs(ctx context.Context) (map[uuid.UUID][]st
return c.Internal.WorkerJobs(ctx)
}

func (c *StorageMinerStruct) ReturnDataCid(ctx context.Context, callID types.CallID, pi abi.PieceInfo, err *storiface.CallError) error {
return c.Internal.ReturnDataCid(ctx, callID, pi, err)
}

func (c *StorageMinerStruct) ReturnAddPiece(ctx context.Context, callID types.CallID, pi abi.PieceInfo, err *storiface.CallError) error {
return c.Internal.ReturnAddPiece(ctx, callID, pi, err)
}
Expand Down
5 changes: 5 additions & 0 deletions api/worker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type WorkerStruct struct {
Paths func(context.Context) ([]stores.StoragePath, error) `perm:"admin"`
Info func(context.Context) (storiface.WorkerInfo, error) `perm:"admin"`

DataCid func(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (types.CallID, error) `perm:"admin"`
AddPiece func(ctx context.Context, sector storage.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (types.CallID, error) `perm:"admin"`
SealPreCommit1 func(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (types.CallID, error) `perm:"admin"`
SealPreCommit2 func(ctx context.Context, sector storage.SectorRef, pc1o storage.PreCommit1Out) (types.CallID, error) `perm:"admin"`
Expand Down Expand Up @@ -79,6 +80,10 @@ func (w *WorkerStruct) Info(ctx context.Context) (storiface.WorkerInfo, error) {
return w.Internal.Info(ctx)
}

func (w *WorkerStruct) DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (types.CallID, error) {
return w.Internal.DataCid(ctx, pieceSize, pieceData)
}

func (w *WorkerStruct) AddPiece(ctx context.Context, sector storage.SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (types.CallID, error) {
return w.Internal.AddPiece(ctx, sector, pieceSizes, newPieceSize, pieceData)
}
Expand Down
Loading

0 comments on commit 655dbd3

Please sign in to comment.