feat: retrievals (#208)

* enables retrievals

* fix itest

* correct test path

* apply changes from review

.circleci/config.yml

@@ -279,13 +279,15 @@ workflows:
           name: test-all
           requires:
             - build
-          target: "`go list ./... | grep -v curio/itests`"
+          target: "`go list ./... | grep -v curio/itests | grep -v market/indexstore`"
           suite: test-all
           get-params: true
           resource_class: 2xlarge
       - test:
           name: test-idxStore
           requires:
             - build
-          suite: test-all
+          suite: test-idxStore
+          target: "./market/indexstore"
           get-params: true
+          resource_class: 2xlarge

cmd/curio/tasks/tasks.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/filecoin-project/curio/alertmanager"
 	"github.com/filecoin-project/curio/api"
+	"github.com/filecoin-project/curio/cuhttp"
 	"github.com/filecoin-project/curio/deps"
 	"github.com/filecoin-project/curio/deps/config"
 	"github.com/filecoin-project/curio/harmony/harmonydb"
@@ -244,6 +245,12 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
 		ipniTask := indexing.NewIPNITask(db, sc, iStore, pp, cfg)
 		activeTasks = append(activeTasks, indexingTask, ipniTask)
 
+		if cfg.HTTP.Enable {
+			err = cuhttp.StartHTTPServer(ctx, dependencies)
+			if err != nil {
+				return nil, xerrors.Errorf("failed to start the HTTP server: %w", err)
+			}
+		}
 	}
 
 	amTask := alertmanager.NewAlertTask(full, db, cfg.Alerting, dependencies.Al)

cuhttp/server.go

@@ -19,6 +19,8 @@ import (
 	"github.com/filecoin-project/curio/deps"
 	"github.com/filecoin-project/curio/deps/config"
 	"github.com/filecoin-project/curio/harmony/harmonydb"
+	ipni_provider "github.com/filecoin-project/curio/market/ipni/ipni-provider"
+	"github.com/filecoin-project/curio/market/retrieval"
 )
 
 var log = logging.Logger("cu-http")
@@ -66,14 +68,15 @@ func compressionMiddleware(config *config.CompressionConfig) (func(http.Handler)
 	return adapter, nil
 }
 
-func NewHTTPServer(ctx context.Context, deps *deps.Deps, config *config.HTTPConfig) error {
-	ch := cache{db: deps.DB}
+func StartHTTPServer(ctx context.Context, d *deps.Deps) error {
+	ch := cache{db: d.DB}
+	cfg := d.Cfg.HTTP
 
 	// Set up the autocert manager for Let's Encrypt
 	certManager := autocert.Manager{
 		Cache:      ch,
 		Prompt:     autocert.AcceptTOS, // Automatically accept the Terms of Service
-		HostPolicy: autocert.HostWhitelist(config.DomainName),
+		HostPolicy: autocert.HostWhitelist(cfg.DomainName),
 	}
 
 	// Setup the Chi router for more complex routing (if needed in the future)
@@ -86,12 +89,12 @@ func NewHTTPServer(ctx context.Context, deps *deps.Deps, config *config.HTTPConf
 	chiRouter.Use(handlers.ProxyHeaders) // Handle reverse proxy headers like X-Forwarded-For
 	chiRouter.Use(secureHeaders)
 
-	if config.EnableCORS {
-		chiRouter.Use(handlers.CORS(handlers.AllowedOrigins([]string{"https://" + config.DomainName})))
+	if cfg.EnableCORS {
+		chiRouter.Use(handlers.CORS(handlers.AllowedOrigins([]string{"https://" + cfg.DomainName})))
 	}
 
 	// Set up the compression middleware with custom compression levels
-	compressionMw, err := compressionMiddleware(&config.CompressionLevels)
+	compressionMw, err := compressionMiddleware(&cfg.CompressionLevels)
 	if err != nil {
 		log.Fatalf("Failed to initialize compression middleware: %s", err)
 	}
@@ -114,16 +117,19 @@ func NewHTTPServer(ctx context.Context, deps *deps.Deps, config *config.HTTPConf
 		fmt.Fprintf(w, "Service is up and running")
 	})
 
-	// TODO: Attach other subrouters here
+	chiRouter, err = attachRouters(ctx, chiRouter, d)
+	if err != nil {
+		return xerrors.Errorf("failed to attach routers: %w", err)
+	}
 
 	// Set up the HTTP server with proper timeouts
 	server := &http.Server{
-		Addr:              config.ListenAddress,
+		Addr:              cfg.ListenAddress,
 		Handler:           loggingMiddleware(compressionMw(chiRouter)), // Attach middlewares
-		ReadTimeout:       config.ReadTimeout,
-		WriteTimeout:      config.WriteTimeout,
-		IdleTimeout:       config.IdleTimeout,
-		ReadHeaderTimeout: config.ReadHeaderTimeout,
+		ReadTimeout:       cfg.ReadTimeout,
+		WriteTimeout:      cfg.WriteTimeout,
+		IdleTimeout:       cfg.IdleTimeout,
+		ReadHeaderTimeout: cfg.ReadHeaderTimeout,
 		TLSConfig: &tls.Config{
 			GetCertificate: certManager.GetCertificate,
 		},
@@ -134,7 +140,7 @@ func NewHTTPServer(ctx context.Context, deps *deps.Deps, config *config.HTTPConf
 	// Start the server with TLS
 	eg := errgroup.Group{}
 	eg.Go(func() error {
-		log.Infof("Starting HTTPS server on https://%s", config.DomainName)
+		log.Infof("Starting HTTPS server for https://%s on %s", cfg.DomainName, cfg.ListenAddress)
 		serr := server.ListenAndServeTLS("", "")
 		if serr != nil {
 			return xerrors.Errorf("failed to start listening: %w", serr)
@@ -185,3 +191,18 @@ func (c cache) Delete(ctx context.Context, key string) error {
 }
 
 var _ autocert.Cache = cache{}
+
+func attachRouters(ctx context.Context, r *chi.Mux, d *deps.Deps) (*chi.Mux, error) {
+	// Attach retrievals
+	rp := retrieval.NewRetrievalProvider(ctx, d.DB, d.IndexStore, d.PieceProvider)
+	retrieval.Router(r, rp)
+
+	// Attach IPNI
+	ipp, err := ipni_provider.NewProvider(d)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to create new ipni provider: %w", err)
+	}
+	ipni_provider.Routes(r, ipp)
+
+	return r, nil
+}

deps/config/types.go

@@ -97,19 +97,19 @@ func DefaultCurioConfig() *CurioConfig {
 					AnnounceAddresses:    []string{},
 				},
 			},
-			HTTP: HTTPConfig{
-				DomainName:        "",
-				ListenAddress:     "0.0.0.0:12310",
-				ReadTimeout:       time.Second * 10,
-				WriteTimeout:      time.Second * 10,
-				IdleTimeout:       time.Minute * 2,
-				ReadHeaderTimeout: time.Second * 5,
-				EnableCORS:        true,
-				CompressionLevels: CompressionConfig{
-					GzipLevel:    6,
-					BrotliLevel:  4,
-					DeflateLevel: 6,
-				},
+		},
+		HTTP: HTTPConfig{
+			DomainName:        "",
+			ListenAddress:     "0.0.0.0:12310",
+			ReadTimeout:       time.Second * 10,
+			WriteTimeout:      time.Second * 10,
+			IdleTimeout:       time.Minute * 2,
+			ReadHeaderTimeout: time.Second * 5,
+			EnableCORS:        true,
+			CompressionLevels: CompressionConfig{
+				GzipLevel:    6,
+				BrotliLevel:  4,
+				DeflateLevel: 6,
 			},
 		},
 	}
@@ -597,9 +597,6 @@ type ApisConfig struct {
 type MarketConfig struct {
 	// StorageMarketConfig houses all the deal related market configuration
 	StorageMarketConfig StorageMarketConfig
-
-	// HTTP configuration for market HTTP server
-	HTTP HTTPConfig
 }
 
 type StorageMarketConfig struct {
@@ -706,7 +703,11 @@ type IPNIConfig struct {
 
 // HTTPConfig represents the configuration for an HTTP server.
 type HTTPConfig struct {
-	// DomainName specifies the domain name that the server uses to serve HTTP requests.
+	// Enable the HTTP server on the node
+	Enable bool
+
+	// DomainName specifies the domain name that the server uses to serve HTTP requests. DomainName cannot be empty and cannot be
+	// an IP address
 	DomainName string
 
 	// CertCacheDir path to the cache directory for storing SSL certificates needed for HTTPS.

documentation/en/configuration/default-curio-configuration.md

@@ -528,57 +528,6 @@ description: The default curio configuration
         # type: []string
         #NoAnnounceAddresses = []
 
-  [Market.HTTP]
-    # DomainName specifies the domain name that the server uses to serve HTTP requests.
-    #
-    # type: string
-    #DomainName = ""
-
-    # CertCacheDir path to the cache directory for storing SSL certificates needed for HTTPS.
-    #
-    # type: string
-    #CertCacheDir = ""
-
-    # ListenAddress is the address that the server listens for HTTP requests.
-    #
-    # type: string
-    #ListenAddress = "0.0.0.0:12310"
-
-    # ReadTimeout is the maximum duration for reading the entire or next request, including body, from the client.
-    #
-    # type: time.Duration
-    #ReadTimeout = "10s"
-
-    # WriteTimeout is the maximum duration before timing out writes of the response to the client.
-    #
-    # type: time.Duration
-    #WriteTimeout = "10s"
-
-    # IdleTimeout is the maximum duration of an idle session. If set, idle connections are closed after this duration.
-    #
-    # type: time.Duration
-    #IdleTimeout = "2m0s"
-
-    # ReadHeaderTimeout is amount of time allowed to read request headers
-    #
-    # type: time.Duration
-    #ReadHeaderTimeout = "5s"
-
-    # EnableCORS indicates whether Cross-Origin Resource Sharing (CORS) is enabled or not.
-    #
-    # type: bool
-    #EnableCORS = true
-
-    [Market.HTTP.CompressionLevels]
-      # type: int
-      #GzipLevel = 6
-
-      # type: int
-      #BrotliLevel = 4
-
-      # type: int
-      #DeflateLevel = 6
-
 
 [Ingest]
   # Maximum number of sectors that can be queued waiting for deals to start processing.
@@ -734,7 +683,13 @@ description: The default curio configuration
 
 
 [HTTP]
-  # DomainName specifies the domain name that the server uses to serve HTTP requests.
+  # Enable the HTTP server on the node
+  #
+  # type: bool
+  #Enable = false
+
+  # DomainName specifies the domain name that the server uses to serve HTTP requests. DomainName cannot be empty and cannot be
+  # an IP address
   #
   # type: string
   #DomainName = ""
@@ -747,41 +702,41 @@ description: The default curio configuration
   # ListenAddress is the address that the server listens for HTTP requests.
   #
   # type: string
-  #ListenAddress = ""
+  #ListenAddress = "0.0.0.0:12310"
 
   # ReadTimeout is the maximum duration for reading the entire or next request, including body, from the client.
   #
   # type: time.Duration
-  #ReadTimeout = "0s"
+  #ReadTimeout = "10s"
 
   # WriteTimeout is the maximum duration before timing out writes of the response to the client.
   #
   # type: time.Duration
-  #WriteTimeout = "0s"
+  #WriteTimeout = "10s"
 
   # IdleTimeout is the maximum duration of an idle session. If set, idle connections are closed after this duration.
   #
   # type: time.Duration
-  #IdleTimeout = "0s"
+  #IdleTimeout = "2m0s"
 
   # ReadHeaderTimeout is amount of time allowed to read request headers
   #
   # type: time.Duration
-  #ReadHeaderTimeout = "0s"
+  #ReadHeaderTimeout = "5s"
 
   # EnableCORS indicates whether Cross-Origin Resource Sharing (CORS) is enabled or not.
   #
   # type: bool
-  #EnableCORS = false
+  #EnableCORS = true
 
   [HTTP.CompressionLevels]
     # type: int
-    #GzipLevel = 0
+    #GzipLevel = 6
 
     # type: int
-    #BrotliLevel = 0
+    #BrotliLevel = 4
 
     # type: int
-    #DeflateLevel = 0
+    #DeflateLevel = 6
 
 ```