From 839e185d791f92057484992321db9bfc74a68da0 Mon Sep 17 00:00:00 2001
From: LexLuthr <88259624+LexLuthr@users.noreply.github.com>
Date: Wed, 18 Sep 2024 01:36:32 +0400
Subject: [PATCH] feat: index provider (#182)
* basic mk12 scafolding
* fix gen test
* poller redesign
* remove indexing table, add defaults
* fix ingest, market tasks
* incomplete basic UI code
* finish libp2p init, fix tests
* fix test-all
* ipni provider
* fix storeiface, publisher multiaddrs
* close rows
* apply suggestions
* fix error
* fix contextID type, move ipni in market
* gen, remove old market sql
---
.circleci/config.yml | 6 +
Makefile | 11 +-
cmd/curio/tasks/tasks.go | 3 +-
deps/config/doc_gen.go | 61 ++
deps/config/types.go | 47 ++
.../default-curio-configuration.md | 41 ++
go.mod | 63 +-
go.sum | 150 +++--
harmony/harmonydb/sql/20240228-piece-park.sql | 3 -
.../sql/20240731-market-migration.sql | 20 +-
harmony/harmonydb/sql/20240823-ipni.sql | 165 +++++
itests/alertnow_test.go | 1 -
lib/dealdata/urlpiecereader.go | 30 +-
lib/paths/local_test.go | 6 +-
lib/paths/remote_test.go | 6 +-
lib/urltomultiaddr/urltomultiaddr.go | 53 ++
market/ipni/chunker/chunker.go | 156 +++++
market/ipni/ipni-provider/ipni-provider.go | 604 ++++++++++++++++++
market/ipni/ipniculib/ipniculib.go | 11 +
market/mk12/mk12.go | 12 +-
market/mk12/types.go | 2 +
market/storageingest/deal_ingest_snap.go | 6 +-
tasks/gc/pipeline_meta_gc.go | 3 +
tasks/indexing/task_indexing.go | 30 +-
tasks/indexing/task_ipni.go | 461 +++++++++++++
tasks/piece/task_park_piece.go | 50 +-
tasks/seal/task_movestorage.go | 2 +-
tasks/snap/task_movestorage.go | 2 +-
tasks/storage-market/task_commp.go | 2 +-
tasks/storage-market/task_find_deal.go | 2 +-
tasks/storage-market/task_psd.go | 2 +-
web/api/webrpc/sync_state.go | 2 +
web/static/ux/curio-ux.mjs | 1 -
33 files changed, 1854 insertions(+), 160 deletions(-)
create mode 100644 harmony/harmonydb/sql/20240823-ipni.sql
create mode 100644 lib/urltomultiaddr/urltomultiaddr.go
create mode 100644 market/ipni/chunker/chunker.go
create mode 100644 market/ipni/ipni-provider/ipni-provider.go
create mode 100644 market/ipni/ipniculib/ipniculib.go
create mode 100644 tasks/indexing/task_ipni.go
diff --git a/.circleci/config.yml b/.circleci/config.yml
index b4dadc809..4bb43029c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -283,3 +283,9 @@ workflows:
suite: test-all
get-params: true
resource_class: 2xlarge
+ - test:
+ name: test-idxStore
+ requires:
+ - build
+ suite: test-all
+ get-params: true
diff --git a/Makefile b/Makefile
index e84088a1f..1b8b4a4c1 100644
--- a/Makefile
+++ b/Makefile
@@ -66,10 +66,11 @@ build/.update-modules:
# end git modules
## CUDA Library Path
-CUDA_PATH := $(shell dirname $$(dirname $$(which nvcc)))
-CUDA_LIB_PATH := $(CUDA_PATH)/lib64
-LIBRARY_PATH ?= $(CUDA_LIB_PATH)
-export LIBRARY_PATH
+setup_cuda:
+ $(eval CUDA_PATH := $(shell dirname $$(dirname $$(which nvcc))))
+ $(eval CUDA_LIB_PATH := $(CUDA_PATH)/lib64)
+ export LIBRARY_PATH=$(CUDA_LIB_PATH)
+.PHONY: setup_cuda
## MAIN BINARIES
@@ -97,7 +98,7 @@ BINS+=sptool
ifeq ($(shell uname),Linux)
-batchdep: build/.supraseal-install
+batchdep: setup_cuda build/.supraseal-install
batchdep: $(BUILD_DEPS)
.PHONY: batchdep
diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go
index 2157f26ad..25a8e9f6f 100644
--- a/cmd/curio/tasks/tasks.go
+++ b/cmd/curio/tasks/tasks.go
@@ -241,7 +241,8 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
}
indexingTask := indexing.NewIndexingTask(db, sc, iStore, pp, cfg)
- activeTasks = append(activeTasks, indexingTask)
+ ipniTask := indexing.NewIPNITask(db, sc, iStore, pp, cfg)
+ activeTasks = append(activeTasks, indexingTask, ipniTask)
}
diff --git a/deps/config/doc_gen.go b/deps/config/doc_gen.go
index 904de0000..7dc32400d 100644
--- a/deps/config/doc_gen.go
+++ b/deps/config/doc_gen.go
@@ -757,6 +757,55 @@ on a sinle node. Enabling on multiple nodes will cause issues with libp2p deals.
Comment: ``,
},
},
+ "HTTPConfig": {
+ {
+ Name: "ListenAddress",
+ Type: "string",
+
+ Comment: `ListenAddress is where HTTP server will be listening on. Default is "0.0.0.0:12400"`,
+ },
+ {
+ Name: "AnnounceAddresses",
+ Type: "[]string",
+
+ Comment: `AnnounceAddresses is a list of addresses clients can use to reach to the HTTP market node.
+Curio allows running more than one node for HTTP server and thus all addressed can be announced
+simultaneously to the client. Example: ["https://mycurio.com", "http://myNewCurio:433/XYZ", "http://1.2.3.4:433"]`,
+ },
+ },
+ "IPNIConfig": {
+ {
+ Name: "Disable",
+ Type: "bool",
+
+ Comment: `Disable set whether to disable indexing announcement to the network and expose endpoints that
+allow indexer nodes to process announcements. Default: False`,
+ },
+ {
+ Name: "EntriesCacheCapacity",
+ Type: "int",
+
+ Comment: `EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement
+entries. Defaults to 4096 if not specified. The cache is evicted using LRU policy. The
+maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize(16384) and
+the length of multihashes being advertised. For example, advertising 128-bit long multihashes
+with the default EntriesCacheCapacity, and EntriesChunkSize(16384) means the cache size can grow to
+1GiB when full.`,
+ },
+ {
+ Name: "WebHost",
+ Type: "string",
+
+ Comment: `The network indexer host that the web UI should link to for published announcements
+TODO: should we use this for checking published heas before publishing? Later commit`,
+ },
+ {
+ Name: "DirectAnnounceURLs",
+ Type: "[]string",
+
+ Comment: `The list of URLs of indexing nodes to announce to.`,
+ },
+ },
"IndexingConfig": {
{
Name: "InsertBatchSize",
@@ -858,6 +907,12 @@ sector will need to be sent again`,
Comment: `StorageMarketConfig houses all the deal related market configuration`,
},
+ {
+ Name: "HTTP",
+ Type: "HTTPConfig",
+
+ Comment: `HTTP configuration for market HTTP server`,
+ },
},
"PagerDutyConfig": {
{
@@ -942,6 +997,12 @@ The server must have 2 endpoints
Comment: `Indexing configuration for deal indexing`,
},
+ {
+ Name: "IPNI",
+ Type: "IPNIConfig",
+
+ Comment: `IPNI configuration for ipni-provider`,
+ },
{
Name: "MK12",
Type: "MK12Config",
diff --git a/deps/config/types.go b/deps/config/types.go
index 80276ae5f..b3a686d4e 100644
--- a/deps/config/types.go
+++ b/deps/config/types.go
@@ -71,6 +71,10 @@ func DefaultCurioConfig() *CurioConfig {
},
},
Market: MarketConfig{
+ HTTP: HTTPConfig{
+ ListenAddress: "0.0.0.0:12400",
+ AnnounceAddresses: []string{},
+ },
StorageMarketConfig: StorageMarketConfig{
PieceLocator: []PieceLocatorConfig{},
Indexing: IndexingConfig{
@@ -90,6 +94,11 @@ func DefaultCurioConfig() *CurioConfig {
ExpectedPoRepSealDuration: Duration(8 * time.Hour),
ExpectedSnapSealDuration: Duration(2 * time.Hour),
},
+ IPNI: IPNIConfig{
+ EntriesCacheCapacity: 4096,
+ WebHost: "https://cid.contact",
+ DirectAnnounceURLs: []string{"https://cid.contact/ingest/announce"},
+ },
},
},
}
@@ -576,6 +585,9 @@ type ApisConfig struct {
type MarketConfig struct {
// StorageMarketConfig houses all the deal related market configuration
StorageMarketConfig StorageMarketConfig
+
+ // HTTP configuration for market HTTP server
+ HTTP HTTPConfig
}
type StorageMarketConfig struct {
@@ -589,6 +601,9 @@ type StorageMarketConfig struct {
// Indexing configuration for deal indexing
Indexing IndexingConfig
+ // IPNI configuration for ipni-provider
+ IPNI IPNIConfig
+
// MK12 encompasses all configuration related to deal protocol mk1.2.0 and mk1.2.1 (i.e. Boost deals)
MK12 MK12Config
}
@@ -638,6 +653,7 @@ type IndexingConfig struct {
type Libp2pConfig struct {
// Miners ID for which MK12 deals (boosts) should be disabled
DisabledMiners []string
+
// Binding address for the libp2p host - 0 means random port.
// Format: multiaddress; see https://multiformats.io/multiaddr/
ListenAddresses []string
@@ -649,3 +665,34 @@ type Libp2pConfig struct {
// Format: multiaddress
NoAnnounceAddresses []string
}
+
+type IPNIConfig struct {
+ // Disable set whether to disable indexing announcement to the network and expose endpoints that
+ // allow indexer nodes to process announcements. Default: False
+ Disable bool
+
+ // EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement
+ // entries. Defaults to 4096 if not specified. The cache is evicted using LRU policy. The
+ // maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize(16384) and
+ // the length of multihashes being advertised. For example, advertising 128-bit long multihashes
+ // with the default EntriesCacheCapacity, and EntriesChunkSize(16384) means the cache size can grow to
+ // 1GiB when full.
+ EntriesCacheCapacity int
+
+ // The network indexer host that the web UI should link to for published announcements
+ // TODO: should we use this for checking published heas before publishing? Later commit
+ WebHost string
+
+ // The list of URLs of indexing nodes to announce to.
+ DirectAnnounceURLs []string
+}
+
+type HTTPConfig struct {
+ // ListenAddress is where HTTP server will be listening on. Default is "0.0.0.0:12400"
+ ListenAddress string
+
+ // AnnounceAddresses is a list of addresses clients can use to reach to the HTTP market node.
+ // Curio allows running more than one node for HTTP server and thus all addressed can be announced
+ // simultaneously to the client. Example: ["https://mycurio.com", "http://myNewCurio:433/XYZ", "http://1.2.3.4:433"]
+ AnnounceAddresses []string
+}
diff --git a/documentation/en/configuration/default-curio-configuration.md b/documentation/en/configuration/default-curio-configuration.md
index 503b74f37..056b02def 100644
--- a/documentation/en/configuration/default-curio-configuration.md
+++ b/documentation/en/configuration/default-curio-configuration.md
@@ -431,6 +431,34 @@ description: The default curio configuration
# type: int
#InsertConcurrency = 8
+ [Market.StorageMarketConfig.IPNI]
+ # Disable set whether to disable indexing announcement to the network and expose endpoints that
+ # allow indexer nodes to process announcements. Default: False
+ #
+ # type: bool
+ #Disable = false
+
+ # EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement
+ # entries. Defaults to 4096 if not specified. The cache is evicted using LRU policy. The
+ # maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize(16384) and
+ # the length of multihashes being advertised. For example, advertising 128-bit long multihashes
+ # with the default EntriesCacheCapacity, and EntriesChunkSize(16384) means the cache size can grow to
+ # 1GiB when full.
+ #
+ # type: int
+ #EntriesCacheCapacity = 4096
+
+ # The network indexer host that the web UI should link to for published announcements
+ # TODO: should we use this for checking published heas before publishing? Later commit
+ #
+ # type: string
+ #WebHost = "https://cid.contact"
+
+ # The list of URLs of indexing nodes to announce to.
+ #
+ # type: []string
+ #DirectAnnounceURLs = ["https://cid.contact/ingest/announce"]
+
[Market.StorageMarketConfig.MK12]
# When a deal is ready to publish, the amount of time to wait for more
# deals to be ready to publish before publishing them all as a batch
@@ -493,6 +521,19 @@ description: The default curio configuration
# type: []string
#NoAnnounceAddresses = []
+ [Market.HTTP]
+ # ListenAddress is where HTTP server will be listening on. Default is "0.0.0.0:12400"
+ #
+ # type: string
+ #ListenAddress = "0.0.0.0:12400"
+
+ # AnnounceAddresses is a list of addresses clients can use to reach to the HTTP market node.
+ # Curio allows running more than one node for HTTP server and thus all addressed can be announced
+ # simultaneously to the client. Example: ["https://mycurio.com", "http://myNewCurio:433/XYZ", "http://1.2.3.4:433"]
+ #
+ # type: []string
+ #AnnounceAddresses = []
+
[Ingest]
# Maximum number of sectors that can be queued waiting for deals to start processing.
diff --git a/go.mod b/go.mod
index 4434ba23e..1dfc00659 100644
--- a/go.mod
+++ b/go.mod
@@ -56,20 +56,21 @@ require (
github.com/ipfs/go-ipld-format v0.6.0
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipld/go-car/v2 v2.13.1
- github.com/ipni/go-libipni v0.0.8
+ github.com/ipld/go-ipld-prime v0.21.0
+ github.com/ipni/go-libipni v0.6.11
github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438
github.com/kelseyhightower/envconfig v1.4.0
github.com/libp2p/go-buffer-pool v0.1.0
- github.com/libp2p/go-libp2p v0.35.4
+ github.com/libp2p/go-libp2p v0.36.2
github.com/manifoldco/promptui v0.9.0
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
github.com/minio/sha256-simd v1.0.1
github.com/mitchellh/go-homedir v1.1.0
- github.com/multiformats/go-multiaddr v0.12.4
+ github.com/multiformats/go-multiaddr v0.13.0
github.com/multiformats/go-multihash v0.2.3
github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333
github.com/pkg/errors v0.9.1
- github.com/prometheus/client_golang v1.19.1
+ github.com/prometheus/client_golang v1.20.0
github.com/puzpuzpuz/xsync/v2 v2.4.0
github.com/raulk/clock v1.1.0
github.com/samber/lo v1.39.0
@@ -85,12 +86,12 @@ require (
go.opencensus.io v0.24.0
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.0
- golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
- golang.org/x/net v0.26.0
- golang.org/x/sync v0.7.0
- golang.org/x/sys v0.23.0
- golang.org/x/text v0.16.0
- golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d
+ golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa
+ golang.org/x/net v0.28.0
+ golang.org/x/sync v0.8.0
+ golang.org/x/sys v0.24.0
+ golang.org/x/text v0.17.0
+ golang.org/x/tools v0.24.0
golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9
)
@@ -134,7 +135,7 @@ require (
github.com/drand/kyber-bls12381 v0.3.1 // indirect
github.com/elastic/go-elasticsearch/v7 v7.14.0 // indirect
github.com/elastic/go-windows v1.0.0 // indirect
- github.com/elastic/gosigar v0.14.2 // indirect
+ github.com/elastic/gosigar v0.14.3 // indirect
github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect
github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect
github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 // indirect
@@ -174,7 +175,7 @@ require (
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gopacket v1.1.19 // indirect
- github.com/google/pprof v0.0.0-20240509144519-723abb6459b7 // indirect
+ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 // indirect
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
@@ -202,7 +203,6 @@ require (
github.com/ipfs/go-verifcid v0.0.3 // indirect
github.com/ipld/go-car v0.6.2 // indirect
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
- github.com/ipld/go-ipld-prime v0.21.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
@@ -222,7 +222,7 @@ require (
github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.25.2 // indirect
github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect
- github.com/libp2p/go-libp2p-pubsub v0.11.0 // indirect
+ github.com/libp2p/go-libp2p-pubsub v0.12.0 // indirect
github.com/libp2p/go-libp2p-record v0.2.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.7.3 // indirect
github.com/libp2p/go-maddr-filter v0.1.0 // indirect
@@ -239,7 +239,7 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mattn/go-sqlite3 v1.14.16 // indirect
- github.com/miekg/dns v1.1.59 // indirect
+ github.com/miekg/dns v1.1.62 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
@@ -256,27 +256,27 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nikkolasg/hexjson v0.1.0 // indirect
github.com/nkovacs/streamquote v1.0.0 // indirect
- github.com/onsi/ginkgo/v2 v2.17.3 // indirect
+ github.com/onsi/ginkgo/v2 v2.20.0 // indirect
github.com/opencontainers/runtime-spec v1.2.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
- github.com/pion/datachannel v1.5.6 // indirect
- github.com/pion/dtls/v2 v2.2.11 // indirect
- github.com/pion/ice/v2 v2.3.25 // indirect
- github.com/pion/interceptor v0.1.29 // indirect
+ github.com/pion/datachannel v1.5.8 // indirect
+ github.com/pion/dtls/v2 v2.2.12 // indirect
+ github.com/pion/ice/v2 v2.3.34 // indirect
+ github.com/pion/interceptor v0.1.30 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/mdns v0.0.12 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pion/rtcp v1.2.14 // indirect
- github.com/pion/rtp v1.8.6 // indirect
- github.com/pion/sctp v1.8.16 // indirect
+ github.com/pion/rtp v1.8.9 // indirect
+ github.com/pion/sctp v1.8.33 // indirect
github.com/pion/sdp/v3 v3.0.9 // indirect
- github.com/pion/srtp/v2 v2.0.18 // indirect
+ github.com/pion/srtp/v2 v2.0.20 // indirect
github.com/pion/stun v0.6.1 // indirect
- github.com/pion/transport/v2 v2.2.5 // indirect
+ github.com/pion/transport/v2 v2.2.10 // indirect
github.com/pion/turn/v2 v2.1.6 // indirect
- github.com/pion/webrtc/v3 v3.2.40 // indirect
+ github.com/pion/webrtc/v3 v3.3.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
@@ -284,7 +284,7 @@ require (
github.com/prometheus/procfs v0.15.1 // indirect
github.com/prometheus/statsd_exporter v0.22.7 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
- github.com/quic-go/quic-go v0.44.0 // indirect
+ github.com/quic-go/quic-go v0.46.0 // indirect
github.com/quic-go/webtransport-go v0.8.0 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
@@ -299,6 +299,7 @@ require (
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
+ github.com/wlynxg/anet v0.0.4 // indirect
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
github.com/zondax/hid v0.9.2 // indirect
github.com/zondax/ledger-filecoin-go v0.11.1 // indirect
@@ -315,13 +316,13 @@ require (
go.opentelemetry.io/otel/sdk/metric v1.28.0 // indirect
go.opentelemetry.io/otel/trace v1.28.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
- go.uber.org/dig v1.17.1 // indirect
- go.uber.org/fx v1.22.1 // indirect
+ go.uber.org/dig v1.18.0 // indirect
+ go.uber.org/fx v1.22.2 // indirect
go.uber.org/mock v0.4.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
- golang.org/x/crypto v0.25.0 // indirect
- golang.org/x/mod v0.17.0 // indirect
- golang.org/x/term v0.22.0 // indirect
+ golang.org/x/crypto v0.26.0 // indirect
+ golang.org/x/mod v0.20.0 // indirect
+ golang.org/x/term v0.23.0 // indirect
golang.org/x/time v0.5.0 // indirect
gonum.org/v1/gonum v0.15.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect
diff --git a/go.sum b/go.sum
index 96454a26c..15df3dfc0 100644
--- a/go.sum
+++ b/go.sum
@@ -107,10 +107,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
-github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
-github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
@@ -241,8 +241,8 @@ github.com/elastic/go-sysinfo v1.7.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6
github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY=
github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
-github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4=
-github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
+github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
+github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -496,8 +496,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20240509144519-723abb6459b7 h1:velgFPYr1X9TDwLIfkV7fWqsFlf7TeP11M/7kPd/dVI=
-github.com/google/pprof v0.0.0-20240509144519-723abb6459b7/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
+github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k=
+github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -688,6 +688,8 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j
github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U=
github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg=
github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
+github.com/ipfs/go-test v0.0.4 h1:DKT66T6GBB6PsDFLoO56QZPrOmzJkqU1FZH5C9ySkew=
+github.com/ipfs/go-test v0.0.4/go.mod h1:qhIM1EluEfElKKM6fnWxGn822/z9knUGM1+I/OAQNKI=
github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
github.com/ipfs/go-unixfsnode v1.9.0 h1:ubEhQhr22sPAKO2DNsyVBW7YB/zA8Zkif25aBvz8rc8=
github.com/ipfs/go-unixfsnode v1.9.0/go.mod h1:HxRu9HYHOjK6HUqFBAi++7DVoWAHn0o4v/nZ/VA+0g8=
@@ -707,8 +709,8 @@ github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOan
github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU=
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw=
-github.com/ipni/go-libipni v0.0.8 h1:0wLfZRSBG84swmZwmaLKul/iB/FlBkkl9ZcR1ub+Z+w=
-github.com/ipni/go-libipni v0.0.8/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk=
+github.com/ipni/go-libipni v0.6.11 h1:i+a+OCVgtKd0FMg8L9PrNpJgq//MYTxsl7YlyCHKAJY=
+github.com/ipni/go-libipni v0.6.11/go.mod h1:hHkfaG5zP8M8RQX8C84gTUre5KODHGPxEbI8E2SgCNw=
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0=
@@ -793,6 +795,8 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
@@ -810,8 +814,8 @@ github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFG
github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM=
github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8=
-github.com/libp2p/go-libp2p v0.35.4 h1:FDiBUYLkueFwsuNJUZaxKRdpKvBOWU64qQPL768bSeg=
-github.com/libp2p/go-libp2p v0.35.4/go.mod h1:RKCDNt30IkFipGL0tl8wQW/3zVWEGFUZo8g2gAKxwjU=
+github.com/libp2p/go-libp2p v0.36.2 h1:BbqRkDaGC3/5xfaJakLV/BrpjlAuYqSB0lRvtzL3B/U=
+github.com/libp2p/go-libp2p v0.36.2/go.mod h1:XO3joasRE4Eup8yCTTP/+kX+g92mOgRaadk46LmPhHY=
github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8=
@@ -833,8 +837,8 @@ github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCv
github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU=
github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY=
github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY=
-github.com/libp2p/go-libp2p-pubsub v0.11.0 h1:+JvS8Kty0OiyUiN0i8H5JbaCgjnJTRnTHe4rU88dLFc=
-github.com/libp2p/go-libp2p-pubsub v0.11.0/go.mod h1:QEb+hEV9WL9wCiUAnpY29FZR6W3zK8qYlaml8R4q6gQ=
+github.com/libp2p/go-libp2p-pubsub v0.12.0 h1:PENNZjSfk8KYxANRlpipdS7+BfLmOl3L2E/6vSNjbdI=
+github.com/libp2p/go-libp2p-pubsub v0.12.0/go.mod h1:Oi0zw9aw8/Y5GC99zt+Ef2gYAl+0nZlwdJonDyOz/sE=
github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
@@ -922,8 +926,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
-github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
-github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
+github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
+github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8=
github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms=
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc=
@@ -969,8 +973,8 @@ github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lg
github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y=
-github.com/multiformats/go-multiaddr v0.12.4 h1:rrKqpY9h+n80EwhhC/kkcunCZZ7URIF8yN1WEUt2Hvc=
-github.com/multiformats/go-multiaddr v0.12.4/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII=
+github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ=
+github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII=
github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A=
@@ -1022,14 +1026,14 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.17.3 h1:oJcvKpIb7/8uLpDDtnQuf18xVnwKp8DTD7DQ6gTd/MU=
-github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc=
+github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw=
+github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.33.0 h1:snPCflnZrpMsy94p4lXVEkHo12lmPnc3vY5XBbreexE=
-github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxjX3wY=
+github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
+github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 h1:CznVS40zms0Dj5he4ERo+fRPtO0qxUk8lA8Xu3ddet0=
github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI=
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
@@ -1049,15 +1053,15 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
-github.com/pion/datachannel v1.5.6 h1:1IxKJntfSlYkpUj8LlYRSWpYiTTC02nUrOE8T3DqGeg=
-github.com/pion/datachannel v1.5.6/go.mod h1:1eKT6Q85pRnr2mHiWHxJwO50SfZRtWHTsNIVb/NfGW4=
+github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo=
+github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI=
github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
-github.com/pion/dtls/v2 v2.2.11 h1:9U/dpCYl1ySttROPWJgqWKEylUdT0fXp/xst6JwY5Ks=
-github.com/pion/dtls/v2 v2.2.11/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
-github.com/pion/ice/v2 v2.3.25 h1:M5rJA07dqhi3nobJIg+uPtcVjFECTrhcR3n0ns8kDZs=
-github.com/pion/ice/v2 v2.3.25/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw=
-github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M=
-github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4=
+github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk=
+github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
+github.com/pion/ice/v2 v2.3.34 h1:Ic1ppYCj4tUOcPAp76U6F3fVrlSw8A9JtRXLqw6BbUM=
+github.com/pion/ice/v2 v2.3.34/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ=
+github.com/pion/interceptor v0.1.30 h1:au5rlVHsgmxNi+v/mjOPazbW1SHzfx7/hYOEYQnUcxA=
+github.com/pion/interceptor v0.1.30/go.mod h1:RQuKT5HTdkP2Fi0cuOS5G5WNymTjzXaGF75J4k7z2nc=
github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8=
@@ -1068,31 +1072,29 @@ github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9
github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE=
github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4=
github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU=
-github.com/pion/rtp v1.8.6 h1:MTmn/b0aWWsAzux2AmP8WGllusBVw4NPYPVFFd7jUPw=
-github.com/pion/rtp v1.8.6/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU=
-github.com/pion/sctp v1.8.13/go.mod h1:YKSgO/bO/6aOMP9LCie1DuD7m+GamiK2yIiPM6vH+GA=
-github.com/pion/sctp v1.8.16 h1:PKrMs+o9EMLRvFfXq59WFsC+V8mN1wnKzqrv+3D/gYY=
-github.com/pion/sctp v1.8.16/go.mod h1:P6PbDVA++OJMrVNg2AL3XtYHV4uD6dvfyOovCgMs0PE=
+github.com/pion/rtp v1.8.9 h1:E2HX740TZKaqdcPmf4pw6ZZuG8u5RlMMt+l3dxeu6Wk=
+github.com/pion/rtp v1.8.9/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU=
+github.com/pion/sctp v1.8.33 h1:dSE4wX6uTJBcNm8+YlMg7lw1wqyKHggsP5uKbdj+NZw=
+github.com/pion/sctp v1.8.33/go.mod h1:beTnqSzewI53KWoG3nqB282oDMGrhNxBdb+JZnkCwRM=
github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY=
github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M=
-github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo=
-github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA=
+github.com/pion/srtp/v2 v2.0.20 h1:HNNny4s+OUmG280ETrCdgFndp4ufx3/uy85EawYEhTk=
+github.com/pion/srtp/v2 v2.0.20/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA=
github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4=
github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8=
github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g=
-github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++NjXLOUorc=
github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
-github.com/pion/transport/v2 v2.2.5 h1:iyi25i/21gQck4hfRhomF6SktmUQjRsRW4WJdhfc3Kc=
-github.com/pion/transport/v2 v2.2.5/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
+github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q=
+github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0=
-github.com/pion/transport/v3 v3.0.2 h1:r+40RJR25S9w3jbA6/5uEPTzcdn7ncyU44RWCbHkLg4=
-github.com/pion/transport/v3 v3.0.2/go.mod h1:nIToODoOlb5If2jF9y2Igfx3PFYWfuXi37m0IlWa/D0=
+github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0=
+github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
-github.com/pion/webrtc/v3 v3.2.40 h1:Wtfi6AZMQg+624cvCXUuSmrKWepSB7zfgYDOYqsSOVU=
-github.com/pion/webrtc/v3 v3.2.40/go.mod h1:M1RAe3TNTD1tzyvqHrbVODfwdPGSXOUo/OgpoGGJqFY=
+github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I=
+github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -1113,8 +1115,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
-github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
-github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
+github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI=
+github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -1146,8 +1148,8 @@ github.com/puzpuzpuz/xsync/v2 v2.4.0 h1:5sXAMHrtx1bg9nbRZTOn8T4MkWe5V+o8yKRH02Ez
github.com/puzpuzpuz/xsync/v2 v2.4.0/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU=
github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
-github.com/quic-go/quic-go v0.44.0 h1:So5wOr7jyO4vzL2sd8/pD9Kesciv91zSk8BoFngItQ0=
-github.com/quic-go/quic-go v0.44.0/go.mod h1:z4cx/9Ny9UtGITIPzmPTXh1ULfOyWh4qGQlpnPcWmek=
+github.com/quic-go/quic-go v0.46.0 h1:uuwLClEEyk1DNvchH8uCByQVjo3yKL9opKulExNDs7Y=
+github.com/quic-go/quic-go v0.46.0/go.mod h1:1dLehS7TIR64+vxGR70GDcatWTOtMX2PUtnKsjbTurI=
github.com/quic-go/webtransport-go v0.8.0 h1:HxSrwun11U+LlmwpgM1kEqIqH90IT4N8auv/cD7QFJg=
github.com/quic-go/webtransport-go v0.8.0/go.mod h1:N99tjprW432Ut5ONql/aUhSLT0YVSlwHohQsuac9WaM=
github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y=
@@ -1317,6 +1319,9 @@ github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
+github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
+github.com/wlynxg/anet v0.0.4 h1:0de1OFQxnNqAu+x2FAKKCVIrnfGKQbs7FQz++tB0+Uw=
+github.com/wlynxg/anet v0.0.4/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ=
github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZDpfMfWg7xk29yEOZiXmo/wZl+utTI8=
github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ=
@@ -1382,10 +1387,10 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc=
-go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
-go.uber.org/fx v1.22.1 h1:nvvln7mwyT5s1q201YE29V/BFrGor6vMiDNpU/78Mys=
-go.uber.org/fx v1.22.1/go.mod h1:HT2M7d7RHo+ebKGh9NRcrsrHHfpZ60nW3QRubMRfv48=
+go.uber.org/dig v1.18.0 h1:imUL1UiY0Mg4bqbFfsRQO5G4CGRBec/ZujWTvSVp3pw=
+go.uber.org/dig v1.18.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
+go.uber.org/fx v1.22.2 h1:iPW+OPxv0G8w75OemJ1RAnTUrF55zOJlXlo1TbJ0Buw=
+go.uber.org/fx v1.22.2/go.mod h1:o/D9n+2mLP6v1EG+qsdT1O8wKopYAsqZasju97SDFCU=
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
@@ -1433,10 +1438,8 @@ golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1447,8 +1450,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
-golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
+golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI=
+golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1473,8 +1476,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1528,13 +1531,10 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1559,8 +1559,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1645,10 +1645,8 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
-golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1658,10 +1656,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
-golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
-golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
+golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
+golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1674,8 +1670,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
+golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1736,8 +1732,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/harmony/harmonydb/sql/20240228-piece-park.sql b/harmony/harmonydb/sql/20240228-piece-park.sql
index efd529da7..add0a4093 100644
--- a/harmony/harmonydb/sql/20240228-piece-park.sql
+++ b/harmony/harmonydb/sql/20240228-piece-park.sql
@@ -33,8 +33,5 @@ create table parked_piece_refs (
data_url text,
data_headers jsonb not null default '{}',
- -- host Added in 202240730-market-migrations.sql
- -- host text,
-
foreign key (piece_id) references parked_pieces(id) on delete cascade
);
diff --git a/harmony/harmonydb/sql/20240731-market-migration.sql b/harmony/harmonydb/sql/20240731-market-migration.sql
index 7eaef853b..64c283689 100644
--- a/harmony/harmonydb/sql/20240731-market-migration.sql
+++ b/harmony/harmonydb/sql/20240731-market-migration.sql
@@ -178,6 +178,8 @@ CREATE TABLE market_mk12_deal_pipeline (
indexing_task_id BIGINT DEFAULT NULL,
indexed BOOLEAN DEFAULT FALSE,
+ announce BOOLEAN DEFAULT FALSE,
+
complete BOOLEAN NOT NULL DEFAULT FALSE,
constraint market_mk12_deal_pipeline_identity_key unique (uuid)
@@ -213,21 +215,21 @@ BEGIN
market_mk12_deal_pipeline dp ON ssp.sp_id = dp.sp_id AND ssp.sector_num = dp.sector
WHERE
ssp.task_id_move_storage = $1', sealing_table);
-ELSE
+ ELSE
RAISE EXCEPTION 'Invalid sealing_table name: %', sealing_table;
-END IF;
+ END IF;
-- Execute the dynamic SQL query with the task_id parameter
FOR pms IN EXECUTE query USING task_id
LOOP
-- Update the market_mk12_deal_pipeline table with the reg_seal_proof and indexing_created_at values
-UPDATE market_mk12_deal_pipeline
-SET
- reg_seal_proof = pms.reg_seal_proof,
- indexing_created_at = NOW() AT TIME ZONE 'UTC'
-WHERE
- uuid = pms.uuid;
-END LOOP;
+ UPDATE market_mk12_deal_pipeline
+ SET
+ reg_seal_proof = pms.reg_seal_proof,
+ indexing_created_at = NOW() AT TIME ZONE 'UTC'
+ WHERE
+ uuid = pms.uuid;
+ END LOOP;
-- If everything is successful, simply exit
RETURN;
diff --git a/harmony/harmonydb/sql/20240823-ipni.sql b/harmony/harmonydb/sql/20240823-ipni.sql
new file mode 100644
index 000000000..04dbb93bf
--- /dev/null
+++ b/harmony/harmonydb/sql/20240823-ipni.sql
@@ -0,0 +1,165 @@
+-- Table for storing IPNI ads
+CREATE TABLE ipni (
+ order_number BIGSERIAL PRIMARY KEY, -- Unique increasing order number
+ ad_cid TEXT NOT NULL,
+ context_id BYTEA NOT NULL, -- abi.PieceInfo in Curio
+ -- metadata column in not required as Curio only supports one type of metadata(HTTP)
+ is_rm BOOLEAN NOT NULL,
+
+ previous TEXT, -- previous ad will only be null for first ad in chain
+
+ provider TEXT NOT NULL, -- peerID from libp2p, this is main identifier on IPNI side
+ addresses TEXT NOT NULL, -- HTTP retrieval server addresses
+
+ signature BYTEA NOT NULL,
+ entries TEXT NOT NULL, -- CID of first link in entry chain
+
+ unique (ad_cid)
+);
+
+-- This index will help speed up the lookup of all ads for a specific provider and ensure fast ordering by order_number
+CREATE INDEX ipni_provider_order_number ON ipni(provider, order_number);
+
+-- This index will speed up lookups based on the ad_cid, which is frequently used to identify specific ads
+CREATE UNIQUE INDEX ipni_ad_cid ON ipni(ad_cid);
+
+-- This index will speed up lookups based on the ad_cid, which is frequently used to identify specific ads
+CREATE UNIQUE INDEX ipni_context_id ON ipni(context_id, ad_cid, is_rm);
+
+-- Since the get_ad_chain function relies on both provider and ad_cid to find the order_number, this index will optimize that query:
+CREATE INDEX ipni_provider_ad_cid ON ipni(provider, ad_cid);
+
+
+CREATE TABLE ipni_head (
+ provider TEXT NOT NULL PRIMARY KEY, -- PeerID from libp2p, this is the main identifier
+ head TEXT NOT NULL, -- ad_cid from the ipni table, representing the head of the ad chain
+
+ FOREIGN KEY (head) REFERENCES ipni(ad_cid) ON DELETE RESTRICT -- Prevents deletion if it's referenced
+);
+
+CREATE OR REPLACE FUNCTION insert_ad_and_update_head(
+ _ad_cid TEXT,
+ _context_id BYTEA,
+ _is_rm BOOLEAN,
+ _provider TEXT,
+ _addresses TEXT,
+ _signature BYTEA,
+ _entries TEXT
+) RETURNS VOID AS $$
+DECLARE
+_previous TEXT;
+BEGIN
+ -- Determine the previous ad_cid in the chain for this provider
+ SELECT head INTO _previous
+ FROM ipni_head
+ WHERE provider = _provider;
+
+ -- Insert the new ad into the ipni table with an automatically assigned order_number
+ INSERT INTO ipni (ad_cid, context_id, is_rm, previous, provider, addresses, signature, entries)
+ VALUES (_ad_cid, _context_id, _is_rm, _previous, _provider, _addresses, _signature, _entries);
+
+ -- Update the ipni_head table to set the new ad as the head of the chain
+ INSERT INTO ipni_head (provider, head)
+ VALUES (_provider, _ad_cid)
+ ON CONFLICT (provider) DO UPDATE SET head = EXCLUDED.head;
+END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION get_ad_chain(
+ _provider TEXT,
+ _ad_cid TEXT
+) RETURNS TABLE (
+ ad_cid TEXT,
+ context_id BYTEA,
+ is_rm BOOLEAN,
+ previous TEXT,
+ provider TEXT,
+ addresses TEXT,
+ signature BYTEA,
+ entries TEXT,
+ order_number BIGINT
+) AS $$
+DECLARE
+_order_number BIGINT;
+BEGIN
+ -- Get the order_number for the specified ad_cid and provider
+ SELECT order_number INTO _order_number
+ FROM ipni
+ WHERE ad_cid = _ad_cid AND provider = _provider;
+
+ -- Return all ads from the head to the specified order_number
+ RETURN QUERY
+ SELECT ad_cid, context_id, is_rm, previous, provider, addresses, signature, entries, order_number
+ FROM ipni
+ WHERE provider = _provider AND order_number <= _order_number
+ ORDER BY order_number ASC;
+END;
+$$ LANGUAGE plpgsql;
+
+-- IPNI pipeline is kept separate from rest for robustness
+-- and reuse. This allows for removing, recreating ads using CLI.
+CREATE TABLE ipni_task (
+ sp_id BIGINT NOT NULL,
+ sector BIGINT NOT NULL,
+ reg_seal_proof INT NOT NULL,
+ sector_offset BIGINT,
+
+ context_id BYTEA NOT NULL,
+ is_rm BOOLEAN NOT NULL,
+
+ provider TEXT NOT NULL,
+
+ created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()),
+ task_id BIGINT DEFAULT NULL,
+ complete BOOLEAN DEFAULT FALSE,
+
+ PRIMARY KEY (provider, context_id, is_rm)
+);
+
+-- Function to create ipni tasks
+CREATE OR REPLACE FUNCTION insert_ipni_task(
+ _sp_id BIGINT,
+ _sector BIGINT,
+ _reg_seal_proof INT,
+ _sector_offset BIGINT,
+ _context_id BYTEA,
+ _is_rm BOOLEAN,
+ _provider TEXT,
+ _task_id BIGINT DEFAULT NULL
+) RETURNS VOID AS $$
+DECLARE
+ _existing_is_rm BOOLEAN;
+ _latest_is_rm BOOLEAN;
+BEGIN
+ -- Check if ipni_task has the same context_id and provider with a different is_rm value
+ SELECT is_rm INTO _existing_is_rm
+ FROM ipni_task
+ WHERE provider = _provider AND context_id = _context_id AND is_rm != _is_rm
+ LIMIT 1;
+
+ -- If a different is_rm exists for the same context_id and provider, insert the new task
+ IF FOUND THEN
+ INSERT INTO ipni_task (sp_id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete)
+ VALUES (_sp_id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE);
+ RETURN;
+ END IF;
+
+ -- If no conflicting entry is found in ipni_task, check the latest ad in ipni table
+ SELECT is_rm INTO _latest_is_rm
+ FROM ipni
+ WHERE provider = _provider AND context_id = _context_id
+ ORDER BY order_number DESC
+ LIMIT 1;
+
+ -- If the latest ad has the same is_rm value, raise an exception
+ IF FOUND AND _latest_is_rm = _is_rm THEN
+ RAISE EXCEPTION 'already published';
+ END IF;
+
+ -- If all conditions are met, insert the new task into ipni_task
+ INSERT INTO ipni_task (sp_id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete)
+ VALUES (_sp_id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE);
+END;
+$$ LANGUAGE plpgsql;
+
diff --git a/itests/alertnow_test.go b/itests/alertnow_test.go
index 53c947a25..0eef39730 100644
--- a/itests/alertnow_test.go
+++ b/itests/alertnow_test.go
@@ -22,7 +22,6 @@ func TestAlertNow(t *testing.T) {
// Create dependencies
sharedITestID := harmonydb.ITestNewID()
db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID)
-
require.NoError(t, err)
an := alertmanager.NewAlertNow(db, "alertNowMachine")
diff --git a/lib/dealdata/urlpiecereader.go b/lib/dealdata/urlpiecereader.go
index a7c5d683d..938c59caa 100644
--- a/lib/dealdata/urlpiecereader.go
+++ b/lib/dealdata/urlpiecereader.go
@@ -4,6 +4,7 @@ import (
"io"
"net/http"
"net/url"
+ "os"
"golang.org/x/xerrors"
)
@@ -62,8 +63,33 @@ func (u *UrlPieceReader) Read(p []byte) (n int, err error) {
return 0, xerrors.Errorf("a non 200 response code: %s", resp.Status)
}
- // Set 'active' to the response body
- u.active = resp.Body
+ if goUrl.Scheme == "file" {
+ fileUrl := goUrl.Path
+ file, err := os.Open(fileUrl)
+ if err != nil {
+ return 0, xerrors.Errorf("error opening file: %w", err)
+ }
+ u.active = file
+ } else {
+ req, err := http.NewRequest(http.MethodGet, u.Url, nil)
+ if err != nil {
+ return 0, xerrors.Errorf("error creating request: %w", err)
+ }
+ // Add custom headers for security and authentication
+ req.Header = u.Headers
+ // Create a client and make the request
+ client := &http.Client{}
+ resp, err := client.Do(req)
+ if err != nil {
+ return 0, xerrors.Errorf("error making GET request: %w", err)
+ }
+ if resp.StatusCode != 200 {
+ return 0, xerrors.Errorf("a non 200 response code: %s", resp.Status)
+ }
+
+ // Set 'active' to the response body
+ u.active = resp.Body
+ }
}
// Calculate the maximum number of bytes we can read without exceeding RawSize
diff --git a/lib/paths/local_test.go b/lib/paths/local_test.go
index 92c9b5074..4edcef6a9 100644
--- a/lib/paths/local_test.go
+++ b/lib/paths/local_test.go
@@ -11,7 +11,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/filecoin-project/curio/harmony/harmonydb"
- storiface "github.com/filecoin-project/curio/lib/storiface"
+ "github.com/filecoin-project/curio/lib/storiface"
"github.com/filecoin-project/lotus/storage/sealer/fsutil"
)
@@ -82,7 +82,9 @@ func TestLocalStorage(t *testing.T) {
root: root,
}
- db, err := harmonydb.NewFromConfigWithITestID(t, "testlocalstorage")
+ sharedITestID := harmonydb.ITestNewID()
+
+ db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID)
require.NoError(t, err)
index := NewDBIndex(nil, db)
diff --git a/lib/paths/remote_test.go b/lib/paths/remote_test.go
index 7602eb116..af397d00f 100644
--- a/lib/paths/remote_test.go
+++ b/lib/paths/remote_test.go
@@ -25,7 +25,7 @@ import (
"github.com/filecoin-project/curio/lib/partialfile"
"github.com/filecoin-project/curio/lib/paths"
"github.com/filecoin-project/curio/lib/paths/mocks"
- storiface "github.com/filecoin-project/curio/lib/storiface"
+ "github.com/filecoin-project/curio/lib/storiface"
)
const metaFile = "sectorstore.json"
@@ -59,7 +59,9 @@ func createTestStorage(t *testing.T, p string, seal bool, att ...*paths.Local) s
func TestMoveShared(t *testing.T) {
logging.SetAllLoggers(logging.LevelDebug)
- db, err := harmonydb.NewFromConfigWithITestID(t, "testlocalstorage")
+ sharedITestID := harmonydb.ITestNewID()
+
+ db, err := harmonydb.NewFromConfigWithITestID(t, sharedITestID)
require.NoError(t, err)
index := paths.NewDBIndex(nil, db)
diff --git a/lib/urltomultiaddr/urltomultiaddr.go b/lib/urltomultiaddr/urltomultiaddr.go
new file mode 100644
index 000000000..dfed65ea0
--- /dev/null
+++ b/lib/urltomultiaddr/urltomultiaddr.go
@@ -0,0 +1,53 @@
+package urltomultiaddr
+
+import (
+ "errors"
+ "fmt"
+ "net"
+ "net/url"
+
+ "github.com/multiformats/go-multiaddr"
+)
+
+const protocol = "tcp"
+
+func UrlToMultiaddr(urlStr string) (multiaddr.Multiaddr, error) {
+ // Parse the URL
+ parsedURL, err := url.Parse(urlStr)
+ if err != nil {
+ return nil, err
+ }
+
+ // Extract the host and port
+ host := parsedURL.Hostname()
+ port := parsedURL.Port()
+ if port == "" {
+ // Default port based on the scheme
+ switch parsedURL.Scheme {
+ case "https":
+ port = "443"
+ case "http":
+ port = "80"
+ default:
+ return nil, errors.New("invalid URL scheme")
+ }
+ }
+
+ // Determine if the host is an IP address or a DNS name
+ var addrStr string
+ if ip := net.ParseIP(host); ip != nil {
+ // It's an IP address, check if it's IPv4 or IPv6
+ if ip.To4() != nil {
+ // IPv4
+ addrStr = fmt.Sprintf("/ip4/%s/%s/%s/%s", ip.String(), protocol, port, parsedURL.Scheme)
+ } else {
+ // IPv6
+ addrStr = fmt.Sprintf("/ip6/%s/%s/%s/%s", ip.String(), protocol, port, parsedURL.Scheme)
+ }
+ } else {
+ // It's a DNS name
+ addrStr = fmt.Sprintf("/dns/%s/%s/%s/%s", host, protocol, port, parsedURL.Scheme)
+ }
+
+ return multiaddr.NewMultiaddr(addrStr)
+}
diff --git a/market/ipni/chunker/chunker.go b/market/ipni/chunker/chunker.go
new file mode 100644
index 000000000..21ff1010e
--- /dev/null
+++ b/market/ipni/chunker/chunker.go
@@ -0,0 +1,156 @@
+package chunker
+
+import (
+ "cmp"
+ "errors"
+ "fmt"
+ "io"
+ "slices"
+
+ lru "github.com/hashicorp/golang-lru/v2"
+ logging "github.com/ipfs/go-log/v2"
+ "github.com/ipld/go-car/v2/index"
+ "github.com/ipld/go-ipld-prime"
+ "github.com/ipld/go-ipld-prime/datamodel"
+ "github.com/ipni/go-libipni/ingest/schema"
+ "github.com/multiformats/go-multihash"
+
+ "github.com/filecoin-project/curio/market/ipni/ipniculib"
+)
+
+var log = logging.Logger("chunker")
+
+const entriesChunkSize = 16384
+
+// Chunker chunks advertisement entries as a chained series of schema.EntryChunk nodes.
+// See: NewChunker
+type Chunker struct {
+ chunkSize int
+ cache *lru.Cache[ipld.Link, datamodel.Node]
+}
+
+// NewChunker instantiates a new chain chunker that given a provider.MultihashIterator it drains
+// all its mulithashes and stores them in the given link system represented as a chain of
+// schema.EntryChunk nodes where each chunk contains no more than chunkSize number of multihashes.
+//
+// See: schema.EntryChunk.
+func NewChunker(cache *lru.Cache[ipld.Link, datamodel.Node]) *Chunker {
+ return &Chunker{
+ chunkSize: entriesChunkSize,
+ cache: cache,
+ }
+}
+
+// Chunk chunks all the mulithashes returned by the given iterator into a chain of schema.EntryChunk
+// nodes where each chunk contains no more than chunkSize number of multihashes and returns the link
+// the root chunk node.
+//
+// See: schema.EntryChunk.
+func (ls *Chunker) Chunk(mhi SliceMhIterator) (ipld.Link, error) {
+ mhs := make([]multihash.Multihash, 0, ls.chunkSize)
+ var next ipld.Link
+ var mhCount, chunkCount int
+ for {
+ mh, err := mhi.Next()
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ return nil, err
+ }
+ mhs = append(mhs, mh)
+ if len(mhs) >= ls.chunkSize {
+ cNode, err := newEntriesChunkNode(mhs, next)
+ if err != nil {
+ return nil, err
+ }
+ next, err = ipniculib.NodeToLink(cNode, schema.Linkproto)
+ if err != nil {
+ return nil, err
+ }
+ if ls.cache != nil {
+ ls.cache.Add(next, cNode)
+ }
+ chunkCount++
+ mhCount += len(mhs)
+ // NewLinkedListOfMhs makes it own copy, so safe to reuse mhs
+ mhs = mhs[:0]
+ }
+ }
+ if len(mhs) != 0 {
+ cNode, err := newEntriesChunkNode(mhs, next)
+ if err != nil {
+ return nil, err
+ }
+ next, err = ipniculib.NodeToLink(cNode, schema.Linkproto)
+ if err != nil {
+ return nil, err
+ }
+ chunkCount++
+ mhCount += len(mhs)
+ }
+
+ log.Infow("Generated linked chunks of multihashes", "totalMhCount", mhCount, "chunkCount", chunkCount)
+ return next, nil
+}
+
+func newEntriesChunkNode(mhs []multihash.Multihash, next ipld.Link) (datamodel.Node, error) {
+ chunk := schema.EntryChunk{
+ Entries: mhs,
+ }
+ if next != nil {
+ chunk.Next = next
+ }
+ return chunk.ToNode()
+}
+
+// SliceMhIterator is a simple MultihashIterator implementation that
+// iterates a slice of multihash.Multihash.
+type SliceMhIterator struct {
+ mhs []multihash.Multihash
+ pos int
+}
+
+type iteratorStep struct {
+ mh multihash.Multihash
+ offset uint64
+}
+
+// CarMultihashIterator constructs a new MultihashIterator from a CAR index.
+//
+// This iterator supplies multihashes in deterministic order of their
+// corresponding CAR offset. The order is maintained consistently regardless of
+// the underlying IterableIndex implementation. Returns error if duplicate
+// offsets detected.
+func CarMultihashIterator(idx index.IterableIndex) (*SliceMhIterator, error) {
+ var steps []iteratorStep
+ if err := idx.ForEach(func(mh multihash.Multihash, offset uint64) error {
+ steps = append(steps, iteratorStep{mh, offset})
+ return nil
+ }); err != nil {
+ return nil, err
+ }
+ slices.SortFunc(steps, func(a, b iteratorStep) int {
+ return cmp.Compare(a.offset, b.offset)
+ })
+
+ var lastOffset uint64
+ mhs := make([]multihash.Multihash, len(steps))
+ for i := range steps {
+ if steps[i].offset == lastOffset {
+ return nil, fmt.Errorf("car multihash iterator has duplicate offset %d", steps[i].offset)
+ }
+ mhs[i] = steps[i].mh
+ }
+ return &SliceMhIterator{mhs: mhs}, nil
+}
+
+// Next implements the MultihashIterator interface.
+func (it *SliceMhIterator) Next() (multihash.Multihash, error) {
+ if it.pos >= len(it.mhs) {
+ return nil, io.EOF
+ }
+ mh := it.mhs[it.pos]
+ it.pos++
+ return mh, nil
+}
diff --git a/market/ipni/ipni-provider/ipni-provider.go b/market/ipni/ipni-provider/ipni-provider.go
new file mode 100644
index 000000000..0e745cfbf
--- /dev/null
+++ b/market/ipni/ipni-provider/ipni-provider.go
@@ -0,0 +1,604 @@
+package ipni_provider
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+
+ "github.com/gorilla/mux"
+ lru "github.com/hashicorp/golang-lru/v2"
+ "github.com/ipfs/go-cid"
+ logging "github.com/ipfs/go-log/v2"
+ carv2 "github.com/ipld/go-car/v2"
+ "github.com/ipld/go-car/v2/index"
+ "github.com/ipld/go-ipld-prime"
+ "github.com/ipld/go-ipld-prime/codec/dagjson"
+ "github.com/ipld/go-ipld-prime/datamodel"
+ cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+ "github.com/ipni/go-libipni/announce"
+ "github.com/ipni/go-libipni/announce/httpsender"
+ "github.com/ipni/go-libipni/dagsync/ipnisync/head"
+ "github.com/ipni/go-libipni/ingest/schema"
+ "github.com/ipni/go-libipni/metadata"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multiaddr"
+ "golang.org/x/xerrors"
+
+ "github.com/filecoin-project/go-state-types/abi"
+
+ "github.com/filecoin-project/curio/deps"
+ "github.com/filecoin-project/curio/harmony/harmonydb"
+ "github.com/filecoin-project/curio/lib/pieceprovider"
+ "github.com/filecoin-project/curio/lib/storiface"
+ "github.com/filecoin-project/curio/lib/urltomultiaddr"
+ "github.com/filecoin-project/curio/market/ipni/chunker"
+ "github.com/filecoin-project/curio/market/ipni/ipniculib"
+
+ "github.com/filecoin-project/lotus/node/modules/dtypes"
+)
+
+const IPNIRoutePath = "/ipni-provider"
+const IPNIPath = "/ipni/v1/ad/"
+const ProviderPath = "/ipni-provider"
+const publishInterval = 10 * time.Minute
+
+var (
+ log = logging.Logger("ipni-provider")
+ ErrNotFound = errors.New("not found")
+)
+
+type ipniAPI interface {
+ StateNetworkName(context.Context) (dtypes.NetworkName, error)
+}
+
+type peerInfo struct {
+ ID peer.ID
+ Key crypto.PrivKey
+}
+
+type Provider struct {
+ api ipniAPI
+ db *harmonydb.DB
+ pieceProvider *pieceprovider.PieceProvider
+ entriesChunker *chunker.Chunker
+ cache *lru.Cache[ipld.Link, datamodel.Node]
+ httpPrefix string
+ keys map[string]*peerInfo // map[peerID String]Private_Key
+ // announceURLs enables sending direct announcements via HTTP. This is
+ // the list of indexer URLs to send direct HTTP announce messages to.
+ announceURLs []*url.URL
+ httpServerAddresses []multiaddr.Multiaddr
+}
+
+func NewProvider(api ipniAPI, deps *deps.Deps) (*Provider, error) {
+ c, err := lru.New[ipld.Link, datamodel.Node](deps.Cfg.Market.StorageMarketConfig.IPNI.EntriesCacheCapacity)
+ if err != nil {
+ return nil, xerrors.Errorf("creating new cache: %w", err)
+ }
+
+ ctx := context.Background()
+
+ keyMap := make(map[string]*peerInfo)
+
+ rows, err := deps.DB.Query(ctx, `SELECT priv_key FROM libp2p`)
+ if err != nil {
+ return nil, xerrors.Errorf("failed to get private libp2p keys from DB: %w", err)
+ }
+
+ defer rows.Close()
+
+ for rows.Next() && rows.Err() == nil {
+ var priv []byte
+ err := rows.Scan(&priv)
+ if err != nil {
+ return nil, xerrors.Errorf("failed to scan the row: %w", err)
+ }
+
+ pkey, err := crypto.UnmarshalPrivateKey(priv)
+ if err != nil {
+ return nil, xerrors.Errorf("unmarshaling private key: %w", err)
+ }
+
+ id, err := peer.IDFromPublicKey(pkey.GetPublic())
+ if err != nil {
+ return nil, xerrors.Errorf("generating peer ID from private key: %w", err)
+ }
+
+ keyMap[id.String()] = &peerInfo{
+ Key: pkey,
+ ID: id,
+ }
+ }
+
+ if rows.Err() != nil {
+ return nil, err
+ }
+
+ announceURLs := make([]*url.URL, 0, len(deps.Cfg.Market.StorageMarketConfig.IPNI.DirectAnnounceURLs))
+
+ for i, us := range deps.Cfg.Market.StorageMarketConfig.IPNI.DirectAnnounceURLs {
+ u, err := url.Parse(us)
+ if err != nil {
+ return nil, err
+ }
+ announceURLs[i] = u
+ }
+
+ httpServerAddresses := make([]multiaddr.Multiaddr, 0, len(deps.Cfg.Market.HTTP.AnnounceAddresses))
+
+ for i, a := range deps.Cfg.Market.HTTP.AnnounceAddresses {
+ addr, err := urltomultiaddr.UrlToMultiaddr(a)
+ if err != nil {
+ return nil, err
+ }
+ addr, err = multiaddr.NewMultiaddr(addr.String() + IPNIRoutePath)
+ if err != nil {
+ return nil, err
+ }
+ httpServerAddresses[i] = addr
+ }
+
+ return &Provider{
+ api: api,
+ db: deps.DB,
+ pieceProvider: deps.PieceProvider,
+ cache: c,
+ keys: keyMap,
+ announceURLs: announceURLs,
+ httpServerAddresses: httpServerAddresses,
+ }, nil
+}
+
+func (p *Provider) getAd(ctx context.Context, ad cid.Cid, provider string) (schema.Advertisement, error) {
+ var ads []struct {
+ PreviousID string
+ Provider string
+ Addresses string
+ Signature []byte
+ Entries string
+ ContextID []byte
+ IsRm bool
+ }
+
+ err := p.db.Select(ctx, &ads, `SELECT
+ context_id,
+ is_rm,
+ previous,
+ provider,
+ addresses,
+ signature,
+ entries
+ FROM ipni
+ WHERE ad_cid = $1
+ AND provider = $2`, ad.String(), provider)
+
+ if err != nil {
+ return schema.Advertisement{}, xerrors.Errorf("getting ad from DB: %w", err)
+ }
+
+ if len(ads) == 0 {
+ return schema.Advertisement{}, ErrNotFound
+ }
+
+ if len(ads) > 1 {
+ return schema.Advertisement{}, xerrors.Errorf("expected 1 ad but got %d", len(ads))
+ }
+
+ a := ads[0]
+
+ prev, err := cid.Parse(a.PreviousID)
+ if err != nil {
+ return schema.Advertisement{}, xerrors.Errorf("parsing previous CID: %w", err)
+ }
+
+ e, err := cid.Parse(a.Entries)
+ if err != nil {
+ return schema.Advertisement{}, xerrors.Errorf("parsing entry CID: %w", err)
+ }
+
+ mds := metadata.IpfsGatewayHttp{}
+ md, err := mds.MarshalBinary()
+ if err != nil {
+ return schema.Advertisement{}, xerrors.Errorf("marshalling metadata: %w", err)
+ }
+
+ return schema.Advertisement{
+ PreviousID: cidlink.Link{Cid: prev},
+ Provider: a.Provider,
+ Addresses: strings.Split(a.Addresses, ","),
+ Signature: a.Signature,
+ Entries: cidlink.Link{Cid: e},
+ ContextID: a.ContextID,
+ IsRm: a.IsRm,
+ Metadata: md,
+ }, nil
+}
+
+func (p *Provider) getHead(ctx context.Context, provider string) ([]byte, error) {
+ var headStr string
+ err := p.db.QueryRow(ctx, `SELECT head FROM ipni_head WHERE provider = $1`, provider).Scan(&headStr)
+ if err != nil {
+ return nil, xerrors.Errorf("querying previous head: %w", err)
+ }
+
+ if headStr == "" {
+ return nil, ErrNotFound
+ }
+
+ ad, err := cid.Parse(headStr)
+ if err != nil {
+ return nil, err
+ }
+
+ h, err := p.getAd(ctx, ad, provider)
+ if err != nil {
+ return nil, err
+ }
+
+ hn, err := h.ToNode()
+ if err != nil {
+ return nil, err
+ }
+
+ lnk, err := ipniculib.NodeToLink(hn, schema.Linkproto)
+ if err != nil {
+ return nil, err
+ }
+
+ signedHead, err := head.NewSignedHead(lnk.(cidlink.Link).Cid, "", p.keys[provider].Key)
+ if err != nil {
+ return nil, xerrors.Errorf("failed to generate signed head for peer %s: %w", provider, err)
+ }
+
+ return signedHead.Encode()
+}
+
+func (p *Provider) GetEntry(block cid.Cid, provider string) ([]byte, error) {
+ // We should use background context to avoid early exit
+ // while chunking as first attempt will always fail
+ ctx := context.Background()
+
+ // Check if cache has it
+ cacheKey := cidlink.Link{Cid: block}
+ if value, ok := p.cache.Get(cacheKey); ok {
+ b := new(bytes.Buffer)
+ err := dagjson.Encode(value, b)
+ if err != nil {
+ return nil, err
+ }
+ return b.Bytes(), nil
+ }
+
+ // If cache does not have it then this must be Entry Head
+ // We should find the relevant ContextID(abi.PieceInfo) and chunk the piece again
+ // to generate all the required links
+
+ var pis []abi.PieceInfo
+
+ rows, err := p.db.Query(ctx, `SELECT context_id FROM ipni WHERE entries = $1 AND provider = $2`, block.String(), provider)
+ if err != nil {
+ return nil, xerrors.Errorf("querying ads with entry link %s: %w", block, err)
+ }
+
+ defer rows.Close()
+
+ for rows.Next() && rows.Err() == nil {
+ var contextID []byte
+ err := rows.Scan(&contextID)
+ if err != nil {
+ return nil, xerrors.Errorf("failed to scan the row: %w", err)
+ }
+
+ var pi abi.PieceInfo
+ err = pi.UnmarshalCBOR(bytes.NewReader(contextID))
+ if err != nil {
+ return nil, xerrors.Errorf("unmarshaling piece info: %w", err)
+ }
+
+ pis = append(pis, pi)
+ }
+
+ if rows.Err() != nil {
+ return nil, err
+ }
+
+ type info struct {
+ SPID int64 `db:"sp_id"`
+ Sector abi.SectorNumber `db:"sector_num"`
+ Offset int64 `db:"piece_offset"`
+ Length int64 `db:"piece_length"`
+ RawSize int64 `db:"raw_size"`
+ Proof abi.RegisteredSealProof `db:"reg_seal_proof"`
+ }
+
+ pieceMap := make(map[abi.PieceInfo][]info)
+
+ for _, pi := range pis {
+ if _, ok := pieceMap[pi]; ok {
+ continue
+ }
+
+ var infos []info
+
+ err := p.db.Select(ctx, &infos, `SELECT
+ mpd.sp_id,
+ mpd.sector_num,
+ mpd.piece_offset,
+ mpd.piece_length,
+ mpd.raw_size,
+ sm.reg_seal_proof
+ FROM
+ market_piece_deal mpd
+ INNER JOIN
+ sectors_meta sm
+ ON
+ mpd.sp_id = sm.sp_id
+ AND mpd.sector_num = sm.sector_num
+ WHERE piece_cid = $1`, pi.PieceCID.String())
+
+ if err != nil {
+ return nil, xerrors.Errorf("getting deal info from database: %w", err)
+ }
+ pieceMap[pi] = infos
+ }
+
+ // Chunk the piece and generate links
+ for pi, infos := range pieceMap {
+ for _, i := range infos {
+ unsealed, err := p.pieceProvider.IsUnsealed(ctx, storiface.SectorRef{
+ ID: abi.SectorID{
+ Miner: abi.ActorID(i.SPID),
+ Number: i.Sector,
+ },
+ ProofType: i.Proof,
+ }, storiface.UnpaddedByteIndex(i.Offset), pi.Size.Unpadded())
+ if err != nil {
+ return nil, xerrors.Errorf("checking if sector is unsealed :%w", err)
+ }
+
+ if !unsealed {
+ continue
+ }
+
+ reader, err := p.pieceProvider.ReadPiece(ctx, storiface.SectorRef{
+ ID: abi.SectorID{
+ Miner: abi.ActorID(i.SPID),
+ Number: i.Sector,
+ },
+ ProofType: i.Proof,
+ }, storiface.UnpaddedByteIndex(i.Offset), abi.UnpaddedPieceSize(pi.Size), pi.PieceCID)
+ if err != nil {
+ return nil, xerrors.Errorf("getting piece reader: %w", err)
+ }
+
+ var recs []index.Record
+ opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)}
+ blockReader, err := carv2.NewBlockReader(reader, opts...)
+ if err != nil {
+ return nil, fmt.Errorf("getting block reader over piece: %w", err)
+ }
+
+ blockMetadata, err := blockReader.SkipNext()
+ for err == nil {
+ recs = append(recs, index.Record{
+ Cid: blockMetadata.Cid,
+ Offset: blockMetadata.Offset,
+ })
+ blockMetadata, err = blockReader.SkipNext()
+ }
+ if !errors.Is(err, io.EOF) {
+ return nil, fmt.Errorf("generating index for piece: %w", err)
+ }
+
+ mis := make(index.MultihashIndexSorted)
+ err = mis.Load(recs)
+ if err != nil {
+ return nil, xerrors.Errorf("failed to load indexed in multihash sorter: %w", err)
+ }
+
+ // To avoid - Cannot assert pinter to interface
+ idxF := func(sorted *index.MultihashIndexSorted) index.Index {
+ return sorted
+ }
+
+ idx := idxF(&mis)
+ iterableIndex := idx.(index.IterableIndex)
+
+ mhi, err := chunker.CarMultihashIterator(iterableIndex)
+ if err != nil {
+ return nil, xerrors.Errorf("getting CAR multihash iterator: %w", err)
+ }
+
+ _, err = p.entriesChunker.Chunk(*mhi)
+ if err != nil {
+ return nil, xerrors.Errorf("chunking CAR multihash iterator: %w", err)
+ }
+
+ if value, ok := p.cache.Get(cacheKey); ok {
+ b := new(bytes.Buffer)
+ err := dagjson.Encode(value, b)
+ if err != nil {
+ return nil, err
+ }
+ return b.Bytes(), nil
+ }
+ }
+ }
+
+ return nil, ErrNotFound
+}
+
+func (p *Provider) handleGet(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case http.MethodGet:
+ default:
+ http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+
+ pp := strings.TrimPrefix(r.URL.RawPath, p.httpPrefix+ProviderPath)
+ pps := strings.Split(pp, "/")
+ providerID := pps[0]
+
+ req := strings.TrimPrefix(pp, IPNIPath)
+ switch req {
+ case "head":
+ sh, err := p.getHead(r.Context(), providerID)
+ if err != nil {
+ if errors.Is(err, ErrNotFound) {
+ http.Error(w, "", http.StatusNoContent)
+ return
+ }
+ log.Errorf("failed to get signed head for peer %s: %w", providerID, err)
+ http.Error(w, "", http.StatusInternalServerError)
+ }
+ w.WriteHeader(http.StatusOK)
+ _, err = w.Write(sh)
+ if err != nil {
+ log.Errorw("failed to write HTTP response", "err", err)
+ }
+ default:
+ b, err := cid.Parse(req)
+ if err != nil {
+ log.Debugw("invalid CID as path parameter while getting content", "request", req, "err", err)
+ http.Error(w, "invalid CID: "+req, http.StatusBadRequest)
+ return
+ }
+ ad, err := p.getAd(r.Context(), b, providerID)
+ if err != nil {
+ if errors.Is(err, ErrNotFound) {
+ // Check if this is an entry CID
+ entry, err := p.GetEntry(b, providerID)
+ if err != nil {
+ if errors.Is(err, ErrNotFound) {
+ log.Debugw("No Content Found", "CID", b.String())
+ http.Error(w, "", http.StatusNoContent)
+ return
+ }
+ log.Errorf("failed to get entry %s for peer %s: %w", b.String(), providerID, err)
+ http.Error(w, "", http.StatusInternalServerError)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ _, err = w.Write(entry)
+ if err != nil {
+ log.Errorw("failed to write HTTP response", "err", err)
+ }
+ return
+ }
+ log.Errorf("failed to get ad %s for peer %s: %w", b.String(), providerID, err)
+ http.Error(w, "", http.StatusInternalServerError)
+ return
+ }
+ adn, err := ad.ToNode()
+ if err != nil {
+ log.Errorf("failed to convert ad %s for peer %s to IPLD node: %w", b.String(), providerID, err)
+ http.Error(w, "", http.StatusInternalServerError)
+ return
+ }
+ // Use local buffer for better error handing
+ resp := new(bytes.Buffer)
+ err = dagjson.Encode(adn, resp)
+ if err != nil {
+ log.Errorf("failed to encode ad %s for peer %s: %w", b.String(), providerID, err)
+ http.Error(w, "", http.StatusInternalServerError)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ _, err = w.Write(resp.Bytes())
+ if err != nil {
+ log.Errorw("failed to write HTTP response", "err", err)
+ }
+ return
+ }
+}
+
+func contentRouter(r *mux.Router, p *Provider) {
+ r.Methods("GET").Path("/*").HandlerFunc(p.handleGet)
+}
+
+func Routes(r *mux.Router, p *Provider) {
+ contentRouter(r.PathPrefix(IPNIRoutePath).Subrouter(), p)
+}
+
+func (p *Provider) StartPublishing(ctx context.Context) {
+ // A poller which publishes head for each provider
+ // every 10 minutes
+ ticker := time.NewTicker(publishInterval)
+ go func() {
+ for {
+ select {
+ case <-ticker.C:
+ // Call the function to publish head for each provider
+ p.publishHead(ctx)
+ case <-ctx.Done():
+ ticker.Stop()
+ return
+ }
+ }
+ }()
+}
+
+func (p *Provider) getHeadCID(ctx context.Context, provider string) (cid.Cid, error) {
+ var headStr string
+ err := p.db.QueryRow(ctx, `SELECT head FROM ipni_head WHERE provider = $1`, provider).Scan(&headStr)
+ if err != nil {
+ return cid.Undef, xerrors.Errorf("querying previous head: %w", err)
+ }
+
+ if headStr == "" {
+ return cid.Undef, ErrNotFound
+ }
+
+ return cid.Parse(headStr)
+}
+
+func (p *Provider) publishHead(ctx context.Context) {
+ for provider := range p.keys {
+ c, err := p.getHeadCID(ctx, provider)
+ if err != nil {
+ log.Errorw("failed to get head CID", "provider", provider, "error", err)
+ continue
+ }
+ err = p.publishhttp(ctx, c, provider)
+ if err != nil {
+ log.Errorw("failed to publish head for provide", "provider", provider, "error", err)
+ }
+ }
+}
+
+func (p *Provider) publishhttp(ctx context.Context, adCid cid.Cid, peer string) error {
+ // Create the http announce sender.
+ httpSender, err := httpsender.New(p.announceURLs, p.keys[peer].ID)
+ if err != nil {
+ return fmt.Errorf("cannot create http announce sender: %w", err)
+ }
+
+ addrs, err := p.getHTTPAddressForPeer(peer)
+ if err != nil {
+ return fmt.Errorf("cannot create provider http addresses: %w", err)
+ }
+
+ log.Infow("Announcing advertisements over HTTP", "urls", p.announceURLs)
+ return announce.Send(ctx, adCid, addrs, httpSender)
+}
+
+func (p *Provider) getHTTPAddressForPeer(peer string) ([]multiaddr.Multiaddr, error) {
+ var ret []multiaddr.Multiaddr
+ for _, addr := range p.httpServerAddresses {
+ a, err := multiaddr.NewMultiaddr(addr.String() + "/" + peer)
+ if err != nil {
+ return nil, err
+ }
+ ret = append(ret, a)
+ }
+
+ return ret, nil
+}
diff --git a/market/ipni/ipniculib/ipniculib.go b/market/ipni/ipniculib/ipniculib.go
new file mode 100644
index 000000000..714416238
--- /dev/null
+++ b/market/ipni/ipniculib/ipniculib.go
@@ -0,0 +1,11 @@
+package ipniculib
+
+import (
+ "github.com/ipld/go-ipld-prime/datamodel"
+ cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+)
+
+func NodeToLink(node datamodel.Node, lp datamodel.LinkPrototype) (datamodel.Link, error) {
+ linkSystem := cidlink.DefaultLinkSystem()
+ return linkSystem.ComputeLink(lp, node)
+}
diff --git a/market/mk12/mk12.go b/market/mk12/mk12.go
index 65cd50b71..b21b56edd 100644
--- a/market/mk12/mk12.go
+++ b/market/mk12/mk12.go
@@ -425,19 +425,19 @@ func (m *MK12) processDeal(ctx context.Context, deal *ProviderDealState) (*Provi
Opaque: fmt.Sprintf("%d", refID),
}
- _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline (uuid, sp_id, piece_cid, piece_size, offline, url, raw_size, should_index)
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (uuid) DO NOTHING`,
+ _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline (uuid, sp_id, piece_cid, piece_size, offline, url, raw_size, should_index, announce)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (uuid) DO NOTHING`,
deal.DealUuid.String(), mid, prop.PieceCID.String(), prop.PieceSize, deal.IsOffline, pieceIDUrl, deal.Transfer.Size,
- deal.FastRetrieval)
+ deal.FastRetrieval, deal.AnnounceToIPNI)
if err != nil {
return false, xerrors.Errorf("inserting deal into deal pipeline: %w", err)
}
} else {
// Insert the offline deal into the deal pipeline
- _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline (uuid, sp_id, piece_cid, piece_size, offline, should_index)
- VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (uuid) DO NOTHING`,
- deal.DealUuid.String(), mid, prop.PieceCID.String(), prop.PieceSize, deal.IsOffline, deal.FastRetrieval)
+ _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline (uuid, sp_id, piece_cid, piece_size, offline, should_index, announce)
+ VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (uuid) DO NOTHING`,
+ deal.DealUuid.String(), mid, prop.PieceCID.String(), prop.PieceSize, deal.IsOffline, deal.FastRetrieval, deal.AnnounceToIPNI)
if err != nil {
return false, xerrors.Errorf("inserting deal into deal pipeline: %w", err)
}
diff --git a/market/mk12/types.go b/market/mk12/types.go
index 3070b75f3..57245f703 100644
--- a/market/mk12/types.go
+++ b/market/mk12/types.go
@@ -266,6 +266,8 @@ type ProviderDealState struct {
SectorID abi.SectorNumber
Offset abi.PaddedPieceSize
+ Length abi.PaddedPieceSize
+
// set if there's an error
Err string
diff --git a/market/storageingest/deal_ingest_snap.go b/market/storageingest/deal_ingest_snap.go
index 445f6d7aa..09d91a3ef 100644
--- a/market/storageingest/deal_ingest_snap.go
+++ b/market/storageingest/deal_ingest_snap.go
@@ -499,12 +499,12 @@ func (p *PieceIngesterSnap) allocateToExisting(ctx context.Context, tx *harmonyd
var allocated bool
var rerr error
- head, err := p.api.ChainHead(ctx)
+ openSectors, err := p.getOpenSectors(tx, p.addToID[maddr])
if err != nil {
- return false, api.SectorOffset{}, xerrors.Errorf("getting chain head: %w", err)
+ return false, api.SectorOffset{}, err
}
- openSectors, err := p.getOpenSectors(tx, p.addToID[maddr])
+ head, err := p.api.ChainHead(ctx)
if err != nil {
return false, api.SectorOffset{}, err
}
diff --git a/tasks/gc/pipeline_meta_gc.go b/tasks/gc/pipeline_meta_gc.go
index 42fa80540..157b27a7f 100644
--- a/tasks/gc/pipeline_meta_gc.go
+++ b/tasks/gc/pipeline_meta_gc.go
@@ -31,6 +31,9 @@ func (s *PipelineGC) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done
if err := s.cleanupUpgrade(); err != nil {
return false, xerrors.Errorf("cleanupUpgrade: %w", err)
}
+ if err := s.cleanupMK12DealPipeline(); err != nil {
+ return false, xerrors.Errorf("cleanupMK12DealPipeline: %w", err)
+ }
if err := s.cleanupMK12DealPipeline(); err != nil {
return false, xerrors.Errorf("cleanupMK12DealPipeline: %w", err)
diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go
index 872cec8ea..a53d9ff51 100644
--- a/tasks/indexing/task_indexing.go
+++ b/tasks/indexing/task_indexing.go
@@ -62,6 +62,7 @@ type itask struct {
ChainID abi.DealID `db:"chain_deal_id"`
RawSize int64 `db:"raw_size"`
ShouldIndex bool `db:"should_index"`
+ Announce bool `db:"announce"`
}
func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
@@ -80,7 +81,8 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do
reg_seal_proof,
chain_deal_id,
raw_size,
- should_index
+ should_index,
+ announce
FROM
market_mk12_deal_pipeline
WHERE
@@ -212,13 +214,25 @@ func (i *IndexingTask) recordCompletion(ctx context.Context, task itask, taskID
return xerrors.Errorf("failed to update piece metadata and piece deal for deal %s: %w", task.UUID, err)
}
- n, err := i.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET indexed = TRUE, complete = TRUE WHERE uuid = $1`, task.UUID)
- if err != nil {
- return xerrors.Errorf("store indexing success: updating pipeline: %w", err)
- }
- if n != 1 {
- return xerrors.Errorf("store indexing success: updated %d rows", n)
+ // If IPNI is disabled then mark deal as complete otherwise just mark as indexed
+ if i.cfg.Market.StorageMarketConfig.IPNI.Disable {
+ n, err := i.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET indexed = TRUE, complete = TRUE WHERE uuid = $1`, task.UUID)
+ if err != nil {
+ return xerrors.Errorf("store indexing success: updating pipeline: %w", err)
+ }
+ if n != 1 {
+ return xerrors.Errorf("store indexing success: updated %d rows", n)
+ }
+ } else {
+ n, err := i.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET indexed = TRUE WHERE uuid = $1`, task.UUID)
+ if err != nil {
+ return xerrors.Errorf("store indexing success: updating pipeline: %w", err)
+ }
+ if n != 1 {
+ return xerrors.Errorf("store indexing success: updated %d rows", n)
+ }
}
+
return nil
}
@@ -277,6 +291,8 @@ func (i *IndexingTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.T
}
func (i *IndexingTask) TypeDetails() harmonytask.TaskTypeDetails {
+ //dealCfg := i.cfg.Market.StorageMarketConfig
+ //chanSize := dealCfg.Indexing.InsertConcurrency * dealCfg.Indexing.InsertBatchSize * 56 // (56 = size of each index.Record)
return harmonytask.TaskTypeDetails{
Name: "Indexing",
diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go
new file mode 100644
index 000000000..b6502b5ac
--- /dev/null
+++ b/tasks/indexing/task_ipni.go
@@ -0,0 +1,461 @@
+package indexing
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "strings"
+ "time"
+
+ "github.com/ipfs/go-cid"
+ logging "github.com/ipfs/go-log/v2"
+ carv2 "github.com/ipld/go-car/v2"
+ "github.com/ipld/go-car/v2/index"
+ cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+ "github.com/ipni/go-libipni/ingest/schema"
+ "github.com/ipni/go-libipni/metadata"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "golang.org/x/xerrors"
+
+ "github.com/filecoin-project/go-state-types/abi"
+
+ "github.com/filecoin-project/curio/deps/config"
+ "github.com/filecoin-project/curio/harmony/harmonydb"
+ "github.com/filecoin-project/curio/harmony/harmonytask"
+ "github.com/filecoin-project/curio/harmony/resources"
+ "github.com/filecoin-project/curio/lib/ffi"
+ "github.com/filecoin-project/curio/lib/passcall"
+ "github.com/filecoin-project/curio/lib/pieceprovider"
+ "github.com/filecoin-project/curio/lib/storiface"
+ "github.com/filecoin-project/curio/market/indexstore"
+ "github.com/filecoin-project/curio/market/ipni/chunker"
+ "github.com/filecoin-project/curio/market/ipni/ipniculib"
+)
+
+var ilog = logging.Logger("ipni")
+
+type IPNITask struct {
+ db *harmonydb.DB
+ indexStore *indexstore.IndexStore
+ pieceProvider *pieceprovider.PieceProvider
+ sc *ffi.SealCalls
+ cfg *config.CurioConfig
+}
+
+func NewIPNITask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, pieceProvider *pieceprovider.PieceProvider, cfg *config.CurioConfig) *IPNITask {
+
+ return &IPNITask{
+ db: db,
+ indexStore: indexStore,
+ pieceProvider: pieceProvider,
+ sc: sc,
+ cfg: cfg,
+ }
+}
+
+func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
+ ctx := context.Background()
+
+ var tasks []struct {
+ SPID int64 `db:"sp_id"`
+ Sector abi.SectorNumber `db:"sector"`
+ Proof abi.RegisteredSealProof `db:"reg_seal_proof"`
+ Offset int64 `db:"sector_offset"`
+ CtxID []byte `db:"context_id"`
+ Rm bool `db:"is_rm"`
+ Prov string `db:"provider"`
+ }
+
+ err = I.db.Select(ctx, &tasks, `SELECT
+ sp_id,
+ sector,
+ reg_seal_proof,
+ sector_offset,
+ context_id,
+ is_rm,
+ provider
+ FROM
+ ipni_task
+ WHERE
+ task_id = $1;`, taskID)
+ if err != nil {
+ return false, xerrors.Errorf("getting ipni task params: %w", err)
+ }
+
+ if len(tasks) != 1 {
+ return false, xerrors.Errorf("expected 1 ipni task params, got %d", len(tasks))
+ }
+
+ task := tasks[0]
+
+ var pi abi.PieceInfo
+ err = pi.UnmarshalCBOR(bytes.NewReader(task.CtxID))
+ if err != nil {
+ return false, xerrors.Errorf("unmarshaling piece info: %w", err)
+ }
+
+ unsealed, err := I.pieceProvider.IsUnsealed(ctx, storiface.SectorRef{
+ ID: abi.SectorID{
+ Miner: abi.ActorID(task.SPID),
+ Number: task.Sector,
+ },
+ ProofType: task.Proof,
+ }, storiface.UnpaddedByteIndex(task.Offset), pi.Size.Unpadded())
+ if err != nil {
+ return false, xerrors.Errorf("checking if sector is unsealed :%w", err)
+ }
+
+ if !unsealed {
+ return false, xerrors.Errorf("sector %d for miner %d is not unsealed", task.Sector, task.SPID)
+ }
+
+ reader, err := I.pieceProvider.ReadPiece(ctx, storiface.SectorRef{
+ ID: abi.SectorID{
+ Miner: abi.ActorID(task.SPID),
+ Number: task.Sector,
+ },
+ ProofType: task.Proof,
+ }, storiface.UnpaddedByteIndex(task.Offset), abi.UnpaddedPieceSize(pi.Size), pi.PieceCID)
+ if err != nil {
+ return false, xerrors.Errorf("getting piece reader: %w", err)
+ }
+
+ var recs []index.Record
+ opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)}
+ blockReader, err := carv2.NewBlockReader(reader, opts...)
+ if err != nil {
+ return false, fmt.Errorf("getting block reader over piece: %w", err)
+ }
+
+ blockMetadata, err := blockReader.SkipNext()
+ for err == nil {
+ recs = append(recs, index.Record{
+ Cid: blockMetadata.Cid,
+ Offset: blockMetadata.Offset,
+ })
+ blockMetadata, err = blockReader.SkipNext()
+ }
+ if !errors.Is(err, io.EOF) {
+ return false, fmt.Errorf("generating index for piece: %w", err)
+ }
+
+ mis := make(index.MultihashIndexSorted)
+ err = mis.Load(recs)
+ if err != nil {
+ return false, xerrors.Errorf("failed to load indexed in multihash sorter: %w", err)
+ }
+
+ // To avoid - Cannot assert pinter to interface
+ idxF := func(sorted *index.MultihashIndexSorted) index.Index {
+ return sorted
+ }
+
+ idx := idxF(&mis)
+ iterableIndex := idx.(index.IterableIndex)
+
+ mhi, err := chunker.CarMultihashIterator(iterableIndex)
+ if err != nil {
+ return false, xerrors.Errorf("getting CAR multihash iterator: %w", err)
+ }
+
+ lnk, err := chunker.NewChunker(nil).Chunk(*mhi)
+ if err != nil {
+ return false, xerrors.Errorf("chunking CAR multihash iterator: %w", err)
+ }
+
+ _, err = I.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
+ var prev string
+ err = tx.QueryRow(`SELECT head FROM ipni_head WHERE provider = $1`, task.Prov).Scan(&prev)
+ if err != nil {
+ return false, xerrors.Errorf("querying previous head: %w", err)
+ }
+
+ prevCID, err := cid.Parse(prev)
+ if err != nil {
+ return false, xerrors.Errorf("parsing previous CID: %w", err)
+ }
+
+ mds := metadata.IpfsGatewayHttp{}
+ md, err := mds.MarshalBinary()
+ if err != nil {
+ return false, xerrors.Errorf("marshaling metadata: %w", err)
+ }
+
+ var privKey []byte
+ err = tx.QueryRow(`SELECT priv_key FROM libp2p WHERE sp_id = $1`, task.SPID).Scan(&privKey)
+ if err != nil {
+ return false, xerrors.Errorf("failed to get private libp2p key for miner %d: %w", task.SPID, err)
+ }
+
+ pkey, err := crypto.UnmarshalPrivateKey(privKey)
+ if err != nil {
+ return false, xerrors.Errorf("unmarshaling private key: %w", err)
+ }
+
+ adv := schema.Advertisement{
+ PreviousID: cidlink.Link{Cid: prevCID},
+ Provider: task.Prov,
+ Addresses: make([]string, 0),
+ Entries: lnk,
+ ContextID: task.CtxID,
+ Metadata: md,
+ IsRm: task.Rm,
+ }
+
+ err = adv.Sign(pkey)
+ if err != nil {
+ return false, xerrors.Errorf("signing the advertisement: %w", err)
+ }
+
+ err = adv.Validate()
+ if err != nil {
+ return false, xerrors.Errorf("validating the advertisement: %w", err)
+ }
+
+ adNode, err := adv.ToNode()
+ if err != nil {
+ return false, xerrors.Errorf("converting advertisement to node: %w", err)
+ }
+
+ ad, err := ipniculib.NodeToLink(adNode, schema.Linkproto)
+ if err != nil {
+ return false, xerrors.Errorf("converting advertisement to link: %w", err)
+ }
+
+ n, err := I.db.Exec(ctx, `SELECT insert_ad_and_update_head($1, $2, $3, $4, $5, $6, $7)`,
+ ad.(cidlink.Link).Cid.String(), adv.ContextID, adv.IsRm, adv.Provider, adv.Addresses,
+ adv.Signature, adv.Entries.String())
+
+ if err != nil {
+ return false, xerrors.Errorf("adding advertisement to the database: %w", err)
+ }
+
+ if n != 1 {
+ return false, xerrors.Errorf("updated %d rows", n)
+ }
+
+ return true, nil
+
+ })
+ if err != nil {
+ return false, xerrors.Errorf("store IPNI success: %w", err)
+ }
+
+ return true, nil
+}
+
+func (I *IPNITask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
+ var tasks []struct {
+ TaskID harmonytask.TaskID `db:"task_id"`
+ SpID int64 `db:"sp_id"`
+ SectorNumber int64 `db:"sector_number"`
+ StorageID string `db:"storage_id"`
+ }
+
+ if storiface.FTUnsealed != 1 {
+ panic("storiface.FTUnsealed != 1")
+ }
+
+ ctx := context.Background()
+
+ indIDs := make([]int64, len(ids))
+ for i, id := range ids {
+ indIDs[i] = int64(id)
+ }
+
+ err := I.db.Select(ctx, &tasks, `
+ SELECT dp.indexing_task_id, dp.sp_id, dp.sector_number, l.storage_id FROM market_mk12_deal_pipeline dp
+ INNER JOIN sector_location l ON dp.sp_id = l.miner_id AND dp.sector_number = l.sector_num
+ WHERE dp.indexing_task_id = ANY ($1) AND l.sector_filetype = 1
+`, indIDs)
+ if err != nil {
+ return nil, xerrors.Errorf("getting tasks: %w", err)
+ }
+
+ ls, err := I.sc.LocalStorage(ctx)
+ if err != nil {
+ return nil, xerrors.Errorf("getting local storage: %w", err)
+ }
+
+ acceptables := map[harmonytask.TaskID]bool{}
+
+ for _, t := range ids {
+ acceptables[t] = true
+ }
+
+ for _, t := range tasks {
+ if _, ok := acceptables[t.TaskID]; !ok {
+ continue
+ }
+
+ for _, l := range ls {
+ if string(l.ID) == t.StorageID {
+ return &t.TaskID, nil
+ }
+ }
+ }
+
+ return nil, nil
+}
+
+func (I *IPNITask) TypeDetails() harmonytask.TaskTypeDetails {
+ return harmonytask.TaskTypeDetails{
+ Name: "IPNI",
+ Cost: resources.Resources{
+ Cpu: 1,
+ Ram: 8 << 30, // 8 GiB for the most dense cars
+ },
+ MaxFailures: 3,
+ IAmBored: passcall.Every(5*time.Minute, func(taskFunc harmonytask.AddTaskFunc) error {
+ return I.schedule(context.Background(), taskFunc)
+ }),
+ }
+}
+
+func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error {
+ // If IPNI is disabled then don't schedule any tasks
+ // Deals should already be marked as complete by task_indexing
+ // This check is to cover any edge case
+ if I.cfg.Market.StorageMarketConfig.IPNI.Disable {
+ return nil
+ }
+
+ // schedule submits
+ var stop bool
+ for !stop {
+ taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) {
+ stop = true // assume we're done until we find a task to schedule
+
+ var pendings []itask
+
+ err := I.db.Select(ctx, &pendings, `SELECT
+ uuid,
+ sp_id,
+ sector_number,
+ piece_cid,
+ piece_size,
+ piece_offset,
+ reg_seal_proof,
+ chain_deal_id,
+ raw_size,
+ should_index,
+ announce
+ FROM market_mk12_deal_pipeline
+ WHERE sealed = TRUE
+ AND indexed = TRUE
+ AND complete = FALSE
+ ORDER BY indexing_created_at ASC;`)
+ if err != nil {
+ return false, xerrors.Errorf("getting pending IPNI announcing tasks: %w", err)
+ }
+
+ if len(pendings) == 0 {
+ return false, nil
+ }
+
+ p := pendings[0]
+
+ if !p.Announce {
+ n, err := tx.Exec(`UPDATE market_mk12_deal_pipeline SET complete = TRUE WHERE uuid = $1`, p.UUID)
+ if err != nil {
+ return false, xerrors.Errorf("store IPNI success: updating pipeline: %w", err)
+ }
+ if n != 1 {
+ return false, xerrors.Errorf("store IPNI success: updated %d rows", n)
+ }
+ stop = false // we found a task to schedule, keep going
+ return true, nil
+ }
+
+ var privKey []byte
+ err = tx.QueryRow(`SELECT priv_key FROM libp2p WHERE sp_id = $1`, p.SpID).Scan(&privKey)
+ if err != nil {
+ return false, xerrors.Errorf("failed to get private libp2p key for miner %d: %w", p.SpID, err)
+ }
+
+ pkey, err := crypto.UnmarshalPrivateKey(privKey)
+ if err != nil {
+ return false, xerrors.Errorf("unmarshaling private key: %w", err)
+ }
+
+ pubK := pkey.GetPublic()
+ pid, err := peer.IDFromPublicKey(pubK)
+ if err != nil {
+ return false, fmt.Errorf("getting peer ID: %w", err)
+ }
+
+ pcid, err := cid.Parse(p.PieceCid)
+ if err != nil {
+ return false, xerrors.Errorf("parsing piece CID: %w", err)
+ }
+
+ pi := abi.PieceInfo{
+ PieceCID: pcid,
+ Size: p.Size,
+ }
+
+ b := new(bytes.Buffer)
+ err = pi.MarshalCBOR(b)
+ if err != nil {
+ return false, xerrors.Errorf("marshaling piece info: %w", err)
+ }
+
+ _, err = tx.Exec(`SELECT insert_ipni_task($1, $2, $3, $4, $5, $6, $7, $8)`, p.SpID,
+ p.Sector, p.Proof, p.Offset, b.Bytes(), false, pid.String(), id)
+ if err != nil {
+ if harmonydb.IsErrUniqueContraint(err) {
+ ilog.Infof("Another IPNI announce task already present for piece %s in deal %s", p.PieceCid, p.UUID)
+ // SET "complete" status to true for this deal, so it is not considered next time
+ n, err := tx.Exec(`UPDATE market_mk12_deal_pipeline SET complete = TRUE WHERE uuid = $1`, p.UUID)
+ if err != nil {
+ return false, xerrors.Errorf("store IPNI success: updating pipeline: %w", err)
+ }
+ if n != 1 {
+ return false, xerrors.Errorf("store IPNI success: updated %d rows", n)
+ }
+ // We should commit this transaction to set the value
+ stop = false // we found a task to schedule, keep going
+ return true, nil
+ }
+ if strings.Contains(err.Error(), "already published") {
+ ilog.Infof("Piece %s in deal %s is already published", p.PieceCid, p.UUID)
+ // SET "complete" status to true for this deal, so it is not considered next time
+ n, err := tx.Exec(`UPDATE market_mk12_deal_pipeline SET complete = TRUE WHERE uuid = $1`, p.UUID)
+ if err != nil {
+ return false, xerrors.Errorf("store IPNI success: updating pipeline: %w", err)
+ }
+ if n != 1 {
+ return false, xerrors.Errorf("store IPNI success: updated %d rows", n)
+ }
+ // We should commit this transaction to set the value
+ stop = false // we found a task to schedule, keep going
+ return true, nil
+ }
+ return false, xerrors.Errorf("updating IPNI announcing task id: %w", err)
+ }
+
+ stop = false // we found a task to schedule, keep going
+ return true, nil
+ })
+ }
+
+ return nil
+}
+
+func (I *IPNITask) Adder(taskFunc harmonytask.AddTaskFunc) {}
+
+func (I *IPNITask) GetSpid(db *harmonydb.DB, taskID int64) string {
+ var spid string
+ err := db.QueryRow(context.Background(), `SELECT sp_id FROM ipni_task WHERE task_id = $1`, taskID).Scan(&spid)
+ if err != nil {
+ log.Errorf("getting spid: %s", err)
+ return ""
+ }
+ return spid
+}
+
+var _ = harmonytask.Reg(&IPNITask{})
+var _ harmonytask.TaskInterface = &IPNITask{}
diff --git a/tasks/piece/task_park_piece.go b/tasks/piece/task_park_piece.go
index 4fb9caf82..1a89ffcd5 100644
--- a/tasks/piece/task_park_piece.go
+++ b/tasks/piece/task_park_piece.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"net/http"
+ "net/url"
"strconv"
"time"
@@ -46,12 +47,20 @@ func NewParkPieceTask(db *harmonydb.DB, sc *ffi2.SealCalls, max int) (*ParkPiece
ctx := context.Background()
- // We should delete all incomplete pieces before we start
+ // We should delete all incomplete pieces that do not have a file URL scheme before we start
// as we would have lost reader for these. The RPC caller will get an error
// when Curio shuts down before parking a piece. They can always retry.
// Leaving these pieces we utilise unnecessary resources in the form of ParkPieceTask
- _, err := db.Exec(ctx, `DELETE FROM parked_pieces WHERE complete = FALSE AND task_id IS NULL`)
+ _, err := db.Exec(ctx, `DELETE FROM parked_pieces pp
+ WHERE complete = FALSE
+ AND task_id IS NULL
+ AND NOT EXISTS (
+ SELECT 1
+ FROM parked_piece_refs ppr
+ WHERE ppr.piece_id = pp.id
+ AND ppr.data_url LIKE 'file:///%'
+ );`)
if err != nil {
return nil, xerrors.Errorf("failed to delete incomplete parked pieces: %w", err)
}
@@ -141,7 +150,8 @@ func (p *ParkPieceTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d
err = p.db.Select(ctx, &refData, `
SELECT data_url, data_headers
FROM parked_piece_refs
- WHERE piece_id = $1 AND data_url IS NOT NULL`, pieceData.PieceID)
+ WHERE piece_id = $1 AND data_url IS NOT NULL
+ ORDER BY CASE WHEN data_url LIKE 'file:///%' THEN 0 ELSE 1 END`, pieceData.PieceID)
if err != nil {
return false, xerrors.Errorf("fetching reference data: %w", err)
}
@@ -193,8 +203,38 @@ func (p *ParkPieceTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d
}
func (p *ParkPieceTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
- id := ids[0]
- return &id, nil
+ var pieces []struct {
+ Tid int64 `db:"task_id"`
+ URL string `db:"data_url"`
+ Host string `db:"host"`
+ }
+ err := p.db.Select(context.Background(), &pieces, `SELECT pp.task_id, ppr.host, ppr.data_url
+ FROM parked_pieces pp
+ JOIN parked_piece_refs ppr ON pp.id = ppr.piece_id
+ WHERE pp.task_id = ANY($1)
+ ORDER BY pp.task_id,
+ CASE WHEN ppr.data_url LIKE 'file:///%' THEN 0 ELSE 1 END;`, ids)
+ if err != nil {
+ return nil, xerrors.Errorf("failed to get pending task details from DB: %w", err)
+ }
+
+ for _, p := range pieces {
+ ret := harmonytask.TaskID(int(p.Tid))
+ goUrl, err := url.Parse(p.URL)
+ if err != nil {
+ return nil, xerrors.Errorf("failed to parse the URL: %w", err)
+ }
+ if goUrl.Scheme == "file" {
+ if p.Host == engine.Host() {
+
+ return &ret, nil
+ }
+ continue
+ }
+ return &ret, nil
+ }
+
+ return nil, xerrors.Errorf("Host %s is not sutiable for pending PiecePark tasks", engine.Host())
}
func (p *ParkPieceTask) TypeDetails() harmonytask.TaskTypeDetails {
diff --git a/tasks/seal/task_movestorage.go b/tasks/seal/task_movestorage.go
index d64da0e04..f8e2f32b7 100644
--- a/tasks/seal/task_movestorage.go
+++ b/tasks/seal/task_movestorage.go
@@ -14,7 +14,7 @@ import (
"github.com/filecoin-project/curio/harmony/taskhelp"
ffi2 "github.com/filecoin-project/curio/lib/ffi"
"github.com/filecoin-project/curio/lib/paths"
- storiface "github.com/filecoin-project/curio/lib/storiface"
+ "github.com/filecoin-project/curio/lib/storiface"
)
type MoveStorageTask struct {
diff --git a/tasks/snap/task_movestorage.go b/tasks/snap/task_movestorage.go
index a4f941a97..d85171559 100644
--- a/tasks/snap/task_movestorage.go
+++ b/tasks/snap/task_movestorage.go
@@ -16,7 +16,7 @@ import (
"github.com/filecoin-project/curio/lib/ffi"
"github.com/filecoin-project/curio/lib/passcall"
"github.com/filecoin-project/curio/lib/paths"
- storiface "github.com/filecoin-project/curio/lib/storiface"
+ "github.com/filecoin-project/curio/lib/storiface"
"github.com/filecoin-project/curio/tasks/seal"
)
diff --git a/tasks/storage-market/task_commp.go b/tasks/storage-market/task_commp.go
index bfa34c810..a4eea3151 100644
--- a/tasks/storage-market/task_commp.go
+++ b/tasks/storage-market/task_commp.go
@@ -9,7 +9,6 @@ import (
"net/url"
"strconv"
- "github.com/filecoin-project/curio/harmony/taskhelp"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
@@ -22,6 +21,7 @@ import (
"github.com/filecoin-project/curio/harmony/harmonydb"
"github.com/filecoin-project/curio/harmony/harmonytask"
"github.com/filecoin-project/curio/harmony/resources"
+ "github.com/filecoin-project/curio/harmony/taskhelp"
"github.com/filecoin-project/curio/lib/ffi"
"github.com/filecoin-project/curio/lib/storiface"
diff --git a/tasks/storage-market/task_find_deal.go b/tasks/storage-market/task_find_deal.go
index 5a681a89b..e71e601f6 100644
--- a/tasks/storage-market/task_find_deal.go
+++ b/tasks/storage-market/task_find_deal.go
@@ -6,7 +6,6 @@ import (
"encoding/json"
"fmt"
- "github.com/filecoin-project/curio/harmony/taskhelp"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
@@ -20,6 +19,7 @@ import (
"github.com/filecoin-project/curio/harmony/harmonydb"
"github.com/filecoin-project/curio/harmony/harmonytask"
"github.com/filecoin-project/curio/harmony/resources"
+ "github.com/filecoin-project/curio/harmony/taskhelp"
"github.com/filecoin-project/curio/lib/promise"
"github.com/filecoin-project/lotus/api"
diff --git a/tasks/storage-market/task_psd.go b/tasks/storage-market/task_psd.go
index 21cfa95fc..0ad56a177 100644
--- a/tasks/storage-market/task_psd.go
+++ b/tasks/storage-market/task_psd.go
@@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
- "github.com/filecoin-project/curio/harmony/taskhelp"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
@@ -20,6 +19,7 @@ import (
"github.com/filecoin-project/curio/harmony/harmonydb"
"github.com/filecoin-project/curio/harmony/harmonytask"
"github.com/filecoin-project/curio/harmony/resources"
+ "github.com/filecoin-project/curio/harmony/taskhelp"
"github.com/filecoin-project/curio/lib/multictladdr"
"github.com/filecoin-project/curio/lib/promise"
"github.com/filecoin-project/curio/tasks/message"
diff --git a/web/api/webrpc/sync_state.go b/web/api/webrpc/sync_state.go
index 81a381173..7bfc1cfa1 100644
--- a/web/api/webrpc/sync_state.go
+++ b/web/api/webrpc/sync_state.go
@@ -43,6 +43,8 @@ func (a *WebRPC) loadConfigs(ctx context.Context) (map[string]string, error) {
return nil, xerrors.Errorf("getting db configs: %w", err)
}
+ defer rows.Close()
+
configs := make(map[string]string)
for rows.Next() {
var title, config string
diff --git a/web/static/ux/curio-ux.mjs b/web/static/ux/curio-ux.mjs
index 673545428..fb7a4d82f 100644
--- a/web/static/ux/curio-ux.mjs
+++ b/web/static/ux/curio-ux.mjs
@@ -164,7 +164,6 @@ class CurioUX extends LitElement {
Curio
-<<<<<<< HEAD