feat: index provider (#182)

* basic mk12 scafolding

* fix gen test

* poller redesign

* remove indexing table, add defaults

* fix ingest, market tasks

* incomplete basic UI code

* finish libp2p init, fix tests

* fix test-all

* ipni provider

* fix storeiface, publisher multiaddrs

* close rows

* apply suggestions

* fix error

* fix contextID type, move ipni in market

* gen, remove old market sql

.circleci/config.yml

@@ -283,3 +283,9 @@ workflows:
           suite: test-all
           get-params: true
           resource_class: 2xlarge
+      - test:
+          name: test-idxStore
+          requires:
+            - build
+          suite: test-all
+          get-params: true

Makefile

@@ -66,10 +66,11 @@ build/.update-modules:
 # end git modules
 
 ## CUDA Library Path
-CUDA_PATH := $(shell dirname $$(dirname $$(which nvcc)))
-CUDA_LIB_PATH := $(CUDA_PATH)/lib64
-LIBRARY_PATH ?= $(CUDA_LIB_PATH)
-export LIBRARY_PATH
+setup_cuda:
+	$(eval CUDA_PATH := $(shell dirname $$(dirname $$(which nvcc))))
+	$(eval CUDA_LIB_PATH := $(CUDA_PATH)/lib64)
+	export LIBRARY_PATH=$(CUDA_LIB_PATH)
+.PHONY: setup_cuda
 
 ## MAIN BINARIES
 
@@ -97,7 +98,7 @@ BINS+=sptool
 
 ifeq ($(shell uname),Linux)
 
-batchdep: build/.supraseal-install
+batchdep: setup_cuda build/.supraseal-install
 batchdep: $(BUILD_DEPS)
 .PHONY: batchdep
 

cmd/curio/tasks/tasks.go

@@ -241,7 +241,8 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
 		}
 
 		indexingTask := indexing.NewIndexingTask(db, sc, iStore, pp, cfg)
-		activeTasks = append(activeTasks, indexingTask)
+		ipniTask := indexing.NewIPNITask(db, sc, iStore, pp, cfg)
+		activeTasks = append(activeTasks, indexingTask, ipniTask)
 
 	}
 

deps/config/types.go

@@ -71,6 +71,10 @@ func DefaultCurioConfig() *CurioConfig {
 			},
 		},
 		Market: MarketConfig{
+			HTTP: HTTPConfig{
+				ListenAddress:     "0.0.0.0:12400",
+				AnnounceAddresses: []string{},
+			},
 			StorageMarketConfig: StorageMarketConfig{
 				PieceLocator: []PieceLocatorConfig{},
 				Indexing: IndexingConfig{
@@ -90,6 +94,11 @@ func DefaultCurioConfig() *CurioConfig {
 					ExpectedPoRepSealDuration: Duration(8 * time.Hour),
 					ExpectedSnapSealDuration:  Duration(2 * time.Hour),
 				},
+				IPNI: IPNIConfig{
+					EntriesCacheCapacity: 4096,
+					WebHost:              "https://cid.contact",
+					DirectAnnounceURLs:   []string{"https://cid.contact/ingest/announce"},
+				},
 			},
 		},
 	}
@@ -576,6 +585,9 @@ type ApisConfig struct {
 type MarketConfig struct {
 	// StorageMarketConfig houses all the deal related market configuration
 	StorageMarketConfig StorageMarketConfig
+
+	// HTTP configuration for market HTTP server
+	HTTP HTTPConfig
 }
 
 type StorageMarketConfig struct {
@@ -589,6 +601,9 @@ type StorageMarketConfig struct {
 	// Indexing configuration for deal indexing
 	Indexing IndexingConfig
 
+	// IPNI configuration for ipni-provider
+	IPNI IPNIConfig
+
 	// MK12 encompasses all configuration related to deal protocol mk1.2.0 and mk1.2.1 (i.e. Boost deals)
 	MK12 MK12Config
 }
@@ -638,6 +653,7 @@ type IndexingConfig struct {
 type Libp2pConfig struct {
 	// Miners ID for which MK12 deals (boosts) should be disabled
 	DisabledMiners []string
+
 	// Binding address for the libp2p host - 0 means random port.
 	// Format: multiaddress; see https://multiformats.io/multiaddr/
 	ListenAddresses []string
@@ -649,3 +665,34 @@ type Libp2pConfig struct {
 	// Format: multiaddress
 	NoAnnounceAddresses []string
 }
+
+type IPNIConfig struct {
+	// Disable set whether to disable indexing announcement to the network and expose endpoints that
+	// allow indexer nodes to process announcements. Default: False
+	Disable bool
+
+	// EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement
+	// entries. Defaults to 4096 if not specified. The cache is evicted using LRU policy. The
+	// maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize(16384) and
+	// the length of multihashes being advertised. For example, advertising 128-bit long multihashes
+	// with the default EntriesCacheCapacity, and EntriesChunkSize(16384) means the cache size can grow to
+	// 1GiB when full.
+	EntriesCacheCapacity int
+
+	// The network indexer host that the web UI should link to for published announcements
+	// TODO: should we use this for checking published heas before publishing? Later commit
+	WebHost string
+
+	// The list of URLs of indexing nodes to announce to.
+	DirectAnnounceURLs []string
+}
+
+type HTTPConfig struct {
+	// ListenAddress is where HTTP server will be listening on. Default is "0.0.0.0:12400"
+	ListenAddress string
+
+	// AnnounceAddresses is a list of addresses clients can use to reach to the HTTP market node.
+	// Curio allows running more than one node for HTTP server and thus all addressed can be announced
+	// simultaneously to the client. Example: ["https://mycurio.com", "http://myNewCurio:433/XYZ", "http://1.2.3.4:433"]
+	AnnounceAddresses []string
+}

documentation/en/configuration/default-curio-configuration.md

@@ -431,6 +431,34 @@ description: The default curio configuration
       # type: int
       #InsertConcurrency = 8
 
+    [Market.StorageMarketConfig.IPNI]
+      # Disable set whether to disable indexing announcement to the network and expose endpoints that
+      # allow indexer nodes to process announcements. Default: False
+      #
+      # type: bool
+      #Disable = false
+
+      # EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement
+      # entries. Defaults to 4096 if not specified. The cache is evicted using LRU policy. The
+      # maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize(16384) and
+      # the length of multihashes being advertised. For example, advertising 128-bit long multihashes
+      # with the default EntriesCacheCapacity, and EntriesChunkSize(16384) means the cache size can grow to
+      # 1GiB when full.
+      #
+      # type: int
+      #EntriesCacheCapacity = 4096
+
+      # The network indexer host that the web UI should link to for published announcements
+      # TODO: should we use this for checking published heas before publishing? Later commit
+      #
+      # type: string
+      #WebHost = "https://cid.contact"
+
+      # The list of URLs of indexing nodes to announce to.
+      #
+      # type: []string
+      #DirectAnnounceURLs = ["https://cid.contact/ingest/announce"]
+
     [Market.StorageMarketConfig.MK12]
       # When a deal is ready to publish, the amount of time to wait for more
       # deals to be ready to publish before publishing them all as a batch
@@ -493,6 +521,19 @@ description: The default curio configuration
         # type: []string
         #NoAnnounceAddresses = []
 
+  [Market.HTTP]
+    # ListenAddress is where HTTP server will be listening on. Default is "0.0.0.0:12400"
+    #
+    # type: string
+    #ListenAddress = "0.0.0.0:12400"
+
+    # AnnounceAddresses is a list of addresses clients can use to reach to the HTTP market node.
+    # Curio allows running more than one node for HTTP server and thus all addressed can be announced
+    # simultaneously to the client. Example: ["https://mycurio.com", "http://myNewCurio:433/XYZ", "http://1.2.3.4:433"]
+    #
+    # type: []string
+    #AnnounceAddresses = []
+
 
 [Ingest]
   # Maximum number of sectors that can be queued waiting for deals to start processing.