feat!: gateway: fix rate limiting, better stateful handling

CHANGELOG.md

@@ -19,6 +19,10 @@
 - https://github.com/filecoin-project/lotus/pull/12332: fix: ETH RPC: receipts: use correct txtype in receipts
 - https://github.com/filecoin-project/lotus/pull/12335: fix: lotus-shed: store processed tipset after backfilling events
 
+## ☢️ Upgrade Warnings ☢️
+
+- lotus-gateway behaviour, CLI arguments and APIs have received minor changes. See the improvements section below.
+
 ## New features
 
 - feat: Add trace filter API supporting RPC method `trace_filter` ([filecoin-project/lotus#12123](https://github.com/filecoin-project/lotus/pull/12123)). Configuring `EthTraceFilterMaxResults` sets a limit on how many results are returned in any individual `trace_filter` RPC API call.
@@ -30,6 +34,18 @@
 
 - fix: add datacap balance to circ supply internal accounting as unCirc #12348
 
+## Improvements
+
+- feat!: gateway: fix rate limiting, better stateful handling ([filecoin-project/lotus#12315](https://github.com/filecoin-project/lotus/pull/12315)).
+  - CLI usage documentation has been improved for `lotus-gateway`
+  - `--per-conn-rate-limit` now works as advertised.
+  - `--eth-max-filters-per-conn` is new and allows you to set the maximum number of filters and subscription per connection, it defaults to 16.
+    - Previously, this limit was set to `16` and applied separately to filters and subscriptions. This limit is now unified and applies to both filters and subscriptions.
+  - Stateful Ethereum APIs (those involving filters and subscriptions) are now disabled for plain HTTP connections. A client must be using websockets to access these APIs.
+    - These APIs are also now automatically removed from the node by the gateway when a client disconnects.
+  - Some APIs have changed which may impact users consuming Lotus Gateway code as a library.
+  - The default value for the `Events.FilterTTL` config option has been reduced from 24h to 1h. This means that filters will expire on a Lotus node after 1 hour of not being accessed by the client.
+
 # v1.28.1 / 2024-07-24
 
 This is the MANDATORY Lotus v1.28.1 release, which will deliver the Filecoin network version 23, codenamed Waffle 🧇. v1.28.1 is also the minimal version that supports nv23.

cmd/lotus-gateway/main.go

@@ -124,33 +124,44 @@ var runCmd = &cli.Command{
 		&cli.DurationFlag{
 			Name:  "api-max-lookback",
 			Usage: "maximum duration allowable for tipset lookbacks",
-			Value: gateway.DefaultLookbackCap,
+			Value: gateway.DefaultMaxLookbackDuration,
 		},
 		&cli.Int64Flag{
 			Name:  "api-wait-lookback-limit",
 			Usage: "maximum number of blocks to search back through for message inclusion",
-			Value: int64(gateway.DefaultStateWaitLookbackLimit),
+			Value: int64(gateway.DefaultMaxMessageLookbackEpochs),
 		},
 		&cli.Int64Flag{
-			Name:  "rate-limit",
-			Usage: "rate-limit API calls. Use 0 to disable",
+			Name: "rate-limit",
+			Usage: fmt.Sprintf(
+				"Global API call throttling rate limit (per second), weighted by relative expense of the call, with the most expensive calls counting for %d. Use 0 to disable",
+				gateway.MaxRateLimitTokens,
+			),
 			Value: 0,
 		},
 		&cli.Int64Flag{
-			Name:  "per-conn-rate-limit",
-			Usage: "rate-limit API calls per each connection. Use 0 to disable",
+			Name: "per-conn-rate-limit",
+			Usage: fmt.Sprintf(
+				"API call throttling rate limit (per second) per WebSocket connection, weighted by relative expense of the call, with the most expensive calls counting for %d. Use 0 to disable",
+				gateway.MaxRateLimitTokens,
+			),
 			Value: 0,
 		},
 		&cli.DurationFlag{
 			Name:  "rate-limit-timeout",
-			Usage: "the maximum time to wait for the rate limiter before returning an error to clients",
+			Usage: "The maximum time to wait for the API call throttling rate limiter before returning an error to clients",
 			Value: gateway.DefaultRateLimitTimeout,
 		},
 		&cli.Int64Flag{
 			Name:  "conn-per-minute",
-			Usage: "The number of incomming connections to accept from a single IP per minute.  Use 0 to disable",
+			Usage: "A hard limit on the number of incoming connections (requests) to accept per remote host per minute. Use 0 to disable",
 			Value: 0,
 		},
+		&cli.IntFlag{
+			Name:  "eth-max-filters-per-conn",
+			Usage: "The maximum number of filters plus subscriptions that a single websocket connection can maintain",
+			Value: gateway.DefaultEthMaxFiltersPerConn,
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		log.Info("Starting lotus gateway")
@@ -171,13 +182,14 @@ var runCmd = &cli.Command{
 		defer closer()
 
 		var (
-			lookbackCap      = cctx.Duration("api-max-lookback")
-			address          = cctx.String("listen")
-			waitLookback     = abi.ChainEpoch(cctx.Int64("api-wait-lookback-limit"))
-			rateLimit        = cctx.Int64("rate-limit")
-			perConnRateLimit = cctx.Int64("per-conn-rate-limit")
-			rateLimitTimeout = cctx.Duration("rate-limit-timeout")
-			connPerMinute    = cctx.Int64("conn-per-minute")
+			lookbackCap                 = cctx.Duration("api-max-lookback")
+			address                     = cctx.String("listen")
+			waitLookback                = abi.ChainEpoch(cctx.Int64("api-wait-lookback-limit"))
+			globalRateLimit             = cctx.Int("rate-limit")
+			perConnectionRateLimit      = cctx.Int("per-conn-rate-limit")
+			rateLimitTimeout            = cctx.Duration("rate-limit-timeout")
+			perHostConnectionsPerMinute = cctx.Int("conn-per-minute")
+			maxFiltersPerConn           = cctx.Int("eth-max-filters-per-conn")
 		)
 
 		serverOptions := make([]jsonrpc.ServerOption, 0)
@@ -197,21 +209,36 @@ var runCmd = &cli.Command{
 			return xerrors.Errorf("failed to convert endpoint address to multiaddr: %w", err)
 		}
 
-		gwapi := gateway.NewNode(api, subHnd, lookbackCap, waitLookback, rateLimit, rateLimitTimeout)
-		h, err := gateway.Handler(gwapi, api, perConnRateLimit, connPerMinute, serverOptions...)
+		gwapi := gateway.NewNode(
+			api,
+			gateway.WithEthSubHandler(subHnd),
+			gateway.WithMaxLookbackDuration(lookbackCap),
+			gateway.WithMaxMessageLookbackEpochs(waitLookback),
+			gateway.WithRateLimit(globalRateLimit),
+			gateway.WithRateLimitTimeout(rateLimitTimeout),
+			gateway.WithEthMaxFiltersPerConn(maxFiltersPerConn),
+		)
+		handler, err := gateway.Handler(
+			gwapi,
+			api,
+			gateway.WithPerConnectionAPIRateLimit(perConnectionRateLimit),
+			gateway.WithPerHostConnectionsPerMinute(perHostConnectionsPerMinute),
+			gateway.WithJsonrpcServerOptions(serverOptions...),
+		)
 		if err != nil {
 			return xerrors.Errorf("failed to set up gateway HTTP handler")
 		}
 
-		stopFunc, err := node.ServeRPC(h, "lotus-gateway", maddr)
+		stopFunc, err := node.ServeRPC(handler, "lotus-gateway", maddr)
 		if err != nil {
 			return xerrors.Errorf("failed to serve rpc endpoint: %w", err)
 		}
 
-		<-node.MonitorShutdown(nil, node.ShutdownHandler{
-			Component: "rpc",
-			StopFunc:  stopFunc,
-		})
+		<-node.MonitorShutdown(
+			nil,
+			node.ShutdownHandler{Component: "rpc", StopFunc: stopFunc},
+			node.ShutdownHandler{Component: "rpc-handler", StopFunc: handler.Shutdown},
+		)
 		return nil
 	},
 }

documentation/en/default-lotus-config.toml

@@ -268,13 +268,17 @@
   #EnableActorEventsAPI = false
 
   # FilterTTL specifies the time to live for actor event filters. Filters that haven't been accessed longer than
-  # this time become eligible for automatic deletion.
+  # this time become eligible for automatic deletion. Filters consume resources, so if they are unused they
+  # should not be retained.
   #
   # type: Duration
   # env var: LOTUS_EVENTS_FILTERTTL
-  #FilterTTL = "24h0m0s"
+  #FilterTTL = "1h0m0s"
 
   # MaxFilters specifies the maximum number of filters that may exist at any one time.
+  # Multi-tenant environments may want to increase this value to serve a larger number of clients. If using
+  # lotus-gateway, this global limit can be coupled with --eth-max-filters-per-conn which limits the number
+  # of filters per connection.
   #
   # type: int
   # env var: LOTUS_EVENTS_MAXFILTERS