From 1ad48dfe438c5601ab5da40cac5e55e1f65e7436 Mon Sep 17 00:00:00 2001 From: Steve Loeppky Date: Tue, 24 Jan 2023 16:20:04 -0800 Subject: [PATCH 1/6] Showing how want to adjust the default resource manager limits --- core/node/libp2p/rcmgr_defaults.go | 31 +++++++++--------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/core/node/libp2p/rcmgr_defaults.go b/core/node/libp2p/rcmgr_defaults.go index 4d578a9b650..ff1a4d2caa6 100644 --- a/core/node/libp2p/rcmgr_defaults.go +++ b/core/node/libp2p/rcmgr_defaults.go @@ -72,33 +72,19 @@ Run 'ipfs swarm limit all' to see the resulting limits. Memory: int64(maxMemory), FD: int(numFD), - // By default, we just limit connections on the inbound side. + // We limit the amount of memory and FD - everything else is infinite. Conns: bigEnough, - ConnsInbound: rcmgr.DefaultLimits.SystemBaseLimit.ConnsInbound, // same as libp2p default + ConnsInbound: bigEnough, ConnsOutbound: bigEnough, - - // We limit streams since they not only take up memory and CPU. - // The Memory limit protects us on the memory side, - // but a StreamsInbound limit helps protect against unbound CPU consumption from stream processing. Streams: bigEnough, - StreamsInbound: rcmgr.DefaultLimits.SystemBaseLimit.StreamsInbound, + StreamsInbound: bigEnough, StreamsOutbound: bigEnough, }, - // Most limits don't see an increase because they're already infinite/bigEnough or at their max value. - // The values that should scale based on the amount of memory allocated to libp2p need to increase accordingly. - SystemLimitIncrease: rcmgr.BaseLimitIncrease{ - Memory: 0, - FDFraction: 0, - - Conns: 0, - ConnsInbound: rcmgr.DefaultLimits.SystemLimitIncrease.ConnsInbound, - ConnsOutbound: 0, - - Streams: 0, - StreamsInbound: rcmgr.DefaultLimits.SystemLimitIncrease.StreamsInbound, - StreamsOutbound: 0, - }, + SystemLimitIncrease: noLimitIncrease, + // Transient connections won't cause any memory to accounted for by the resource manager. + // Only established connections do. + // As a result, we can't rely on System.Memory to protect us from a bunch of transient connection being opened. TransientBaseLimit: rcmgr.BaseLimit{ Memory: rcmgr.DefaultLimits.TransientBaseLimit.Memory, FD: rcmgr.DefaultLimits.TransientBaseLimit.FD, @@ -111,7 +97,6 @@ Run 'ipfs swarm limit all' to see the resulting limits. StreamsInbound: rcmgr.DefaultLimits.TransientBaseLimit.StreamsInbound, StreamsOutbound: bigEnough, }, - TransientLimitIncrease: rcmgr.BaseLimitIncrease{ Memory: rcmgr.DefaultLimits.TransientLimitIncrease.Memory, FDFraction: rcmgr.DefaultLimits.TransientLimitIncrease.FDFraction, @@ -152,6 +137,7 @@ Run 'ipfs swarm limit all' to see the resulting limits. StreamBaseLimit: infiniteBaseLimit, StreamLimitIncrease: noLimitIncrease, + // Steve 2023-01-24: I don't see any reason to change this. // Limit the resources consumed by a peer. // This doesn't protect us against intentional DoS attacks since an attacker can easily spin up multiple peers. // We specify this limit against unintentional DoS attacks (e.g., a peer has a bug and is sending too much traffic intentionally). @@ -181,6 +167,7 @@ Run 'ipfs swarm limit all' to see the resulting limits. }, } + // Steve 2023-01-24: I don't see any reasons to change this. // Whatever limits libp2p has specifically tuned for its protocols/services we'll apply. libp2p.SetDefaultServiceLimits(&scalingLimitConfig) From af03df33017372ffa50b513e550914f013759cfb Mon Sep 17 00:00:00 2001 From: Steve Loeppky Date: Wed, 25 Jan 2023 16:29:05 -0800 Subject: [PATCH 2/6] Add limit on System.ConnsInbound This is since can't rely on go-libp2p currently. --- core/node/libp2p/rcmgr_defaults.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/core/node/libp2p/rcmgr_defaults.go b/core/node/libp2p/rcmgr_defaults.go index ff1a4d2caa6..72477047a79 100644 --- a/core/node/libp2p/rcmgr_defaults.go +++ b/core/node/libp2p/rcmgr_defaults.go @@ -44,7 +44,7 @@ var noLimitIncrease = rcmgr.BaseLimitIncrease{ // This file defines implicit limit defaults used when Swarm.ResourceMgr.Enabled // createDefaultLimitConfig creates LimitConfig to pass to libp2p's resource manager. -// The defaults follow the documentation in docs/config.md. +// The defaults follow the documentation in docs/libp2p-resource-management.md. // Any changes in the logic here should be reflected there. func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error) { maxMemoryDefaultString := humanize.Bytes(uint64(memory.TotalMemory()) / 4) @@ -72,9 +72,18 @@ Run 'ipfs swarm limit all' to see the resulting limits. Memory: int64(maxMemory), FD: int(numFD), - // We limit the amount of memory and FD - everything else is infinite. + // At least as of 2023-01-25, it's possible to open a connection that + // doesn't for any memory usage with the libp2p Resource Manager/Accountant + // (see https://github.com/libp2p/go-libp2p/issues/2010#issuecomment-1404280736). + // As a result, we can't curretly rely on Memory limits to full protect us. + // Until https://github.com/libp2p/go-libp2p/issues/2010 is addressed, + // we take a proxy now of restricting to 1 inbound connection per MB. + // Note: this is more generous than go-libp2p's default autoscaled limits which do + // 64 connections per 1GB + // (see https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L357 ). + ConnsInbound: maxMemory >> 20, // Steve: I assume this isn't valid Go code. + Conns: bigEnough, - ConnsInbound: bigEnough, ConnsOutbound: bigEnough, Streams: bigEnough, StreamsInbound: bigEnough, From d9156b61e38f078a44d98dbee730470119f90206 Mon Sep 17 00:00:00 2001 From: Steve Loeppky Date: Wed, 25 Jan 2023 16:32:56 -0800 Subject: [PATCH 3/6] Update libp2p-resource-management.md --- docs/libp2p-resource-management.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/libp2p-resource-management.md b/docs/libp2p-resource-management.md index d6b782da1f9..7924849d891 100644 --- a/docs/libp2p-resource-management.md +++ b/docs/libp2p-resource-management.md @@ -70,8 +70,7 @@ The reason these scopes are chosen is because: Within these scopes, limits are just set on [memory](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#memory), -[file descriptors (FD)](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#file-descriptors), [*inbound* connections](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connections), -and [*inbound* streams](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#streams). +[file descriptors (FD)](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#file-descriptors), and [*inbound* connections](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connections). Limits are set based on the `Swarm.ResourceMgr.MaxMemory` and `Swarm.ResourceMgr.MaxFileDescriptors` inputs above. There are also some special cases where minimum values are enforced. @@ -89,7 +88,6 @@ These become the [active limits](#how-does-one-see-the-active-limits). While `Swarm.ResourceMgr.Limits` can be edited directly, it is also possible to use `ipfs swarm limit` command to inspect and tweak specific limits at runtime. - To see all resources that are close to hitting their respective limit: ```console From 2b4bc9aeb6854910d6e0f9cf726206c53c86d764 Mon Sep 17 00:00:00 2001 From: Steve Loeppky Date: Wed, 25 Jan 2023 17:05:10 -0800 Subject: [PATCH 4/6] Create v0.18.1 --- docs/changelogs/v0.18.1 | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs/changelogs/v0.18.1 diff --git a/docs/changelogs/v0.18.1 b/docs/changelogs/v0.18.1 new file mode 100644 index 00000000000..343b3ba5f88 --- /dev/null +++ b/docs/changelogs/v0.18.1 @@ -0,0 +1,36 @@ +# Kubo changelog v0.18 + +## v0.18.1 + +### Overview + +Below is an outline of all that is in this release, so you get a sense of all that's included. + + + +- [Overview](#overview) +- [๐Ÿ”ฆ Highlights](#-highlights) + - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration) +- [๐Ÿ“ Changelog](#-changelog) +- [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) + + + +### ๐Ÿ”ฆ Highlights + +#### Improving libp2p resource management integration +This builds on the default protection nodes get against DoS (resource exhaustion) and eclipse attacks +with the [go-libp2p Network Resource Manager/Accountant](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md) +that was fine-tuned in [Kubo 0.18](https://github.com/ipfs/kubo/blob/biglep/resource-manager-example-of-what-want/docs/changelogs/v0.18.md#improving-libp2p-resource-management-integration). + +Adding default hard-limits from the Resource Manager/Accountant after the fact is tricky, +and some additional improvements have been made to improve the [computed defaults](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#computed-default-limits). +As much as possible, the aim is for a user to only think about how much memory they want to bound libp2p to, +and not need to think about translating that to hard numbers for connections, streams, etc. +More updates are likely in future Kubo releases, but with this release: +1.``System.StreamsInbound`` is no longer bounded directly +2. ``System.ConnsInbound`` has higher default computed values. + +### ๐Ÿ“ Changelog + +### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors From d3a48e98ae03186f4db29cef75a10a7768d931f8 Mon Sep 17 00:00:00 2001 From: Steve Loeppky Date: Thu, 26 Jan 2023 08:50:28 -0800 Subject: [PATCH 5/6] Updating based on peer feedback --- core/node/libp2p/rcmgr_defaults.go | 58 +++++++++++++----------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/core/node/libp2p/rcmgr_defaults.go b/core/node/libp2p/rcmgr_defaults.go index 72477047a79..74b739afd7f 100644 --- a/core/node/libp2p/rcmgr_defaults.go +++ b/core/node/libp2p/rcmgr_defaults.go @@ -53,8 +53,10 @@ func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error) if err != nil { return rcmgr.LimitConfig{}, err } + maxMemory = int64(maxMemory) + maxMemoryMB = maxMemory / (1024 * 1024) - numFD := cfg.ResourceMgr.MaxFileDescriptors.WithDefault(int64(fd.GetNumFDs()) / 2) + maxFD := int(cfg.ResourceMgr.MaxFileDescriptors.WithDefault(int64(fd.GetNumFDs()) / 2)) // We want to see this message on startup, that's why we are using fmt instead of log. fmt.Printf(` @@ -67,21 +69,23 @@ Run 'ipfs swarm limit all' to see the resulting limits. `, maxMemoryString, numFD) + // At least as of 2023-01-25, it's possible to open a connection that + // doesn't for any memory usage with the libp2p Resource Manager/Accountant + // (see https://github.com/libp2p/go-libp2p/issues/2010#issuecomment-1404280736). + // As a result, we can't curretly rely on Memory limits to full protect us. + // Until https://github.com/libp2p/go-libp2p/issues/2010 is addressed, + // we take a proxy now of restricting to 1 inbound connection per MB. + // Note: this is more generous than go-libp2p's default autoscaled limits which do + // 64 connections per 1GB + // (see https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L357 ). + systemConnsInbound = 1 * maxMemoryMB + scalingLimitConfig := rcmgr.ScalingLimitConfig{ SystemBaseLimit: rcmgr.BaseLimit{ - Memory: int64(maxMemory), - FD: int(numFD), - - // At least as of 2023-01-25, it's possible to open a connection that - // doesn't for any memory usage with the libp2p Resource Manager/Accountant - // (see https://github.com/libp2p/go-libp2p/issues/2010#issuecomment-1404280736). - // As a result, we can't curretly rely on Memory limits to full protect us. - // Until https://github.com/libp2p/go-libp2p/issues/2010 is addressed, - // we take a proxy now of restricting to 1 inbound connection per MB. - // Note: this is more generous than go-libp2p's default autoscaled limits which do - // 64 connections per 1GB - // (see https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L357 ). - ConnsInbound: maxMemory >> 20, // Steve: I assume this isn't valid Go code. + Memory: maxMemory, + FD: maxFD, + + ConnsInbound: systemConnsInbound, Conns: bigEnough, ConnsOutbound: bigEnough, @@ -94,30 +98,20 @@ Run 'ipfs swarm limit all' to see the resulting limits. // Transient connections won't cause any memory to accounted for by the resource manager. // Only established connections do. // As a result, we can't rely on System.Memory to protect us from a bunch of transient connection being opened. + // We limit the same values as the System scope, but only allow the Transient scope to take 25% of what is allowed for the System scope. TransientBaseLimit: rcmgr.BaseLimit{ - Memory: rcmgr.DefaultLimits.TransientBaseLimit.Memory, - FD: rcmgr.DefaultLimits.TransientBaseLimit.FD, + Memory: maxMemory / 4, + FD: maxFD / 4, + ConnsInbound: systemConnsInbound / 4, + Conns: bigEnough, - ConnsInbound: rcmgr.DefaultLimits.TransientBaseLimit.ConnsInbound, ConnsOutbound: bigEnough, - Streams: bigEnough, - StreamsInbound: rcmgr.DefaultLimits.TransientBaseLimit.StreamsInbound, + StreamsInbound: bigEnough, StreamsOutbound: bigEnough, }, - TransientLimitIncrease: rcmgr.BaseLimitIncrease{ - Memory: rcmgr.DefaultLimits.TransientLimitIncrease.Memory, - FDFraction: rcmgr.DefaultLimits.TransientLimitIncrease.FDFraction, - - Conns: 0, - ConnsInbound: rcmgr.DefaultLimits.TransientLimitIncrease.ConnsInbound, - ConnsOutbound: 0, - - Streams: 0, - StreamsInbound: rcmgr.DefaultLimits.TransientLimitIncrease.StreamsInbound, - StreamsOutbound: 0, - }, + TransientLimitIncrease: noLimitIncrease, // Lets get out of the way of the allow list functionality. // If someone specified "Swarm.ResourceMgr.Allowlist" we should let it go through. @@ -146,7 +140,6 @@ Run 'ipfs swarm limit all' to see the resulting limits. StreamBaseLimit: infiniteBaseLimit, StreamLimitIncrease: noLimitIncrease, - // Steve 2023-01-24: I don't see any reason to change this. // Limit the resources consumed by a peer. // This doesn't protect us against intentional DoS attacks since an attacker can easily spin up multiple peers. // We specify this limit against unintentional DoS attacks (e.g., a peer has a bug and is sending too much traffic intentionally). @@ -176,7 +169,6 @@ Run 'ipfs swarm limit all' to see the resulting limits. }, } - // Steve 2023-01-24: I don't see any reasons to change this. // Whatever limits libp2p has specifically tuned for its protocols/services we'll apply. libp2p.SetDefaultServiceLimits(&scalingLimitConfig) From 92f6823db9988c159f824c972c8c292a122a4c3a Mon Sep 17 00:00:00 2001 From: Steve Loeppky Date: Thu, 26 Jan 2023 08:53:25 -0800 Subject: [PATCH 6/6] Update v0.18.1 --- docs/changelogs/v0.18.1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelogs/v0.18.1 b/docs/changelogs/v0.18.1 index 343b3ba5f88..f7d6ba5d73d 100644 --- a/docs/changelogs/v0.18.1 +++ b/docs/changelogs/v0.18.1 @@ -28,8 +28,8 @@ and some additional improvements have been made to improve the [computed default As much as possible, the aim is for a user to only think about how much memory they want to bound libp2p to, and not need to think about translating that to hard numbers for connections, streams, etc. More updates are likely in future Kubo releases, but with this release: -1.``System.StreamsInbound`` is no longer bounded directly -2. ``System.ConnsInbound`` has higher default computed values. +1. ``System.StreamsInbound`` is no longer bounded directly +2. ``System.ConnsInbound``, ``Transient.Memory``, ``Transiet.ConnsInbound`` have higher default computed values. ### ๐Ÿ“ Changelog