Skip to content

Commit

Permalink
feat: built-in periodic gc (#87)
Browse files Browse the repository at this point in the history
  • Loading branch information
hacdias authored Feb 19, 2024
1 parent 2cecd47 commit d7b8111
Show file tree
Hide file tree
Showing 8 changed files with 182 additions and 7 deletions.
17 changes: 17 additions & 0 deletions docs/environment-variables.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
- [`RAINBOW_GATEWAY_DOMAINS`](#rainbow_gateway_domains)
- [`RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS`](#rainbow_subdomain_gateway_domains)
- [`RAINBOW_TRUSTLESS_GATEWAY_DOMAINS`](#rainbow_trustless_gateway_domains)
- [`RAINBOW_GC_INTERVAL`](#rainbow_gc_interval)
- [`RAINBOW_GC_THRESHOLD`](#rainbow_gc_threshold)
- [`KUBO_RPC_URL`](#kubo_rpc_url)
- [Logging](#logging)
- [`GOLOG_LOG_LEVEL`](#golog_log_level)
Expand Down Expand Up @@ -61,6 +63,21 @@ when request comes with the `Host` header set to `trustless-gateway.link`.

Default: none (`Host` is ignored and gateway at `127.0.0.1` supports both deserialized and verifiable response types)

## `RAINBOW_GC_INTERVAL`

The interval at which the garbage collector will be called. This is given as a string that corresponds to the duration of the interval. Set 0 to disable.


Default: `60m`

## `RAINBOW_GC_THRESHOLD`

The threshold of how much free space one wants to always have available on disk. This is used with the periodic garbage collector.

When the periodic GC runs, it checks for the total and available space on disk. If the available space is larger than the threshold, the GC is not called. Otherwise, the GC is asked to remove how many bytes necessary such that the threshold of available space on disk is met.

Default: `0.3` (always keep 30% of the disk available)

### `KUBO_RPC_URL`

Single URL or a comma separated list of RPC endpoints that provide legacy `/api/v0` from Kubo.
Expand Down
26 changes: 26 additions & 0 deletions gc.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ package main

import (
"context"

badger4 "github.com/ipfs/go-ds-badger4"
"github.com/shirou/gopsutil/v3/disk"
)

// GC is a really stupid simple algorithm where we just delete things until
Expand Down Expand Up @@ -45,3 +47,27 @@ deleteBlocks:

return nil
}

func (nd *Node) periodicGC(ctx context.Context, threshold float64) error {
stat, err := disk.Usage(nd.dataDir)
if err != nil {
return err
}

// Calculate % of the total space
minFreeBytes := uint64((float64(stat.Total) * threshold))

goLog.Infow("disk data collected", "total_bytes", stat.Total, "available_bytes", stat.Free, "min_free_bytes", minFreeBytes)

// If there's enough free space, do nothing.
if minFreeBytes < stat.Free {
return nil
}

bytesToFree := (minFreeBytes - stat.Free)
if bytesToFree <= 0 {
return nil
}

return nd.GC(ctx, int64(bytesToFree))
}
46 changes: 46 additions & 0 deletions gc_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package main

import (
"context"
"testing"

"github.com/ipfs/go-cid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestPeriodicGC(t *testing.T) {
t.Parallel()

gnd := mustTestNode(t, Config{})

cids := []cid.Cid{
mustAddFile(t, gnd, []byte("a")),
mustAddFile(t, gnd, []byte("b")),
mustAddFile(t, gnd, []byte("c")),
mustAddFile(t, gnd, []byte("d")),
mustAddFile(t, gnd, []byte("e")),
mustAddFile(t, gnd, []byte("f")),
}

for i, cid := range cids {
has, err := gnd.blockstore.Has(context.Background(), cid)
assert.NoError(t, err, i)
assert.True(t, has, i)
}

// NOTE: ideally, we'd be able to spawn an isolated Rainbow instance with the
// periodic GC settings configured (interval, threshold). The way it is now,
// we can only test if the periodicGC function and the GC function work, but
// not if the timer is being correctly set-up.
//
// Tracked in https://github.com/ipfs/rainbow/issues/89
err := gnd.periodicGC(context.Background(), 1)
require.NoError(t, err)

for i, cid := range cids {
has, err := gnd.blockstore.Has(context.Background(), cid)
assert.NoError(t, err, i)
assert.False(t, has, i)
}
}
6 changes: 5 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/felixge/httpsnoop v1.0.4
github.com/ipfs-shipyard/nopfs v0.0.12
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a
github.com/ipfs/boxo v0.17.1-0.20240126101119-fdfcfcc0708a
github.com/ipfs/boxo v0.17.1-0.20240219101629-39f4588f1548
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-datastore v0.6.0
github.com/ipfs/go-ds-badger4 v0.1.5
Expand All @@ -31,6 +31,7 @@ require (
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
github.com/prometheus/client_golang v1.18.0
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
github.com/shirou/gopsutil/v3 v3.24.1
github.com/stretchr/testify v1.8.4
github.com/urfave/cli/v2 v2.27.1
go.opencensus.io v0.24.0
Expand Down Expand Up @@ -67,6 +68,7 @@ require (
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down Expand Up @@ -138,6 +140,7 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
Expand All @@ -156,6 +159,7 @@ require (
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.opentelemetry.io/contrib/propagators/aws v1.23.0 // indirect
go.opentelemetry.io/contrib/propagators/b3 v1.23.0 // indirect
go.opentelemetry.io/contrib/propagators/jaeger v1.23.0 // indirect
Expand Down
24 changes: 22 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
Expand Down Expand Up @@ -172,6 +174,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
Expand Down Expand Up @@ -223,8 +227,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a h1:MKG
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.17.1-0.20240126101119-fdfcfcc0708a h1:BMxa0aXrjyGh5gAkzxVsjDN71YhAWGfjbOoNvZt4/jg=
github.com/ipfs/boxo v0.17.1-0.20240126101119-fdfcfcc0708a/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
github.com/ipfs/boxo v0.17.1-0.20240219101629-39f4588f1548 h1:E6eJPd7mIIWVtZoXgLLje1OvEcXWDZp7fS9jgosXpxU=
github.com/ipfs/boxo v0.17.1-0.20240219101629-39f4588f1548/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
Expand Down Expand Up @@ -391,6 +395,7 @@ github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQsc
github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU=
github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ=
github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
Expand Down Expand Up @@ -501,6 +506,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4=
github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
Expand Down Expand Up @@ -539,6 +546,10 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI=
github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU=
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
Expand Down Expand Up @@ -600,6 +611,8 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ=
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
Expand Down Expand Up @@ -632,6 +645,8 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
Expand Down Expand Up @@ -776,11 +791,13 @@ golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand All @@ -789,6 +806,9 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down
11 changes: 7 additions & 4 deletions handler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func TestRPCNotImplemented(t *testing.T) {
}
}

func mustTestServer(t *testing.T, cfg Config) (*httptest.Server, *Node) {
func mustTestNode(t *testing.T, cfg Config) *Node {
cfg.DataDir = t.TempDir()
cfg.BlockstoreType = "flatfs"

Expand All @@ -114,9 +114,12 @@ func mustTestServer(t *testing.T, cfg Config) (*httptest.Server, *Node) {
})

gnd, err := Setup(ctx, cfg, sk, cdns)
if err != nil {
require.NoError(t, err)
}
require.NoError(t, err)
return gnd
}

func mustTestServer(t *testing.T, cfg Config) (*httptest.Server, *Node) {
gnd := mustTestNode(t, cfg)

handler, err := setupGatewayHandler(cfg, gnd)
if err != nil {
Expand Down
47 changes: 47 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,18 @@ Generate an identity seed and launch a gateway:
EnvVars: []string{"RAINBOW_CTL_LISTEN_ADDRESS"},
Usage: "Listen address for the management api and metrics",
},
&cli.DurationFlag{
Name: "gc-interval",
Value: time.Minute * 60,
EnvVars: []string{"RAINBOW_GC_INTERVAL"},
Usage: "The interval between automatic GC runs. Set 0 to disable.",
},
&cli.Float64Flag{
Name: "gc-threshold",
Value: 0.3,
EnvVars: []string{"RAINBOW_GC_THRESHOLD"},
Usage: "Percentage of how much of the disk free space must be available.",
},
&cli.IntFlag{
Name: "connmgr-low",
Value: 100,
Expand Down Expand Up @@ -286,6 +298,8 @@ share the same seed as long as the indexes are different.
DHTSharedHost: cctx.Bool("dht-shared-host"),
DenylistSubs: getCommaSeparatedList(cctx.String("denylists")),
Peering: peeringAddrs,
GCInterval: cctx.Duration("gc-interval"),
GCThreshold: cctx.Float64("gc-threshold"),
}

goLog.Debugf("Rainbow config: %+v", cfg)
Expand Down Expand Up @@ -360,6 +374,8 @@ share the same seed as long as the indexes are different.
}
}()

_ = gnd.periodicGC(cctx.Context, cfg.GCThreshold)

go func() {
defer wg.Done()

Expand All @@ -370,6 +386,31 @@ share the same seed as long as the indexes are different.
}
}()

var gcTicker *time.Ticker
var gcTickerDone chan bool

if cfg.GCInterval > 0 {
gcTicker = time.NewTicker(cfg.GCInterval)
gcTickerDone = make(chan bool)
wg.Add(1)

go func() {
defer wg.Done()

for {
select {
case <-gcTickerDone:
return
case <-gcTicker.C:
err = gnd.periodicGC(cctx.Context, cfg.GCThreshold)
if err != nil {
goLog.Errorf("error when running periodic gc: %w", err)
}
}
}
}()
}

sddaemon.SdNotify(false, sddaemon.SdNotifyReady)
signal.Notify(
quit,
Expand All @@ -382,6 +423,12 @@ share the same seed as long as the indexes are different.
goLog.Info("Closing servers...")
go gatewaySrv.Close()
go apiSrv.Close()

if gcTicker != nil {
gcTicker.Stop()
gcTickerDone <- true
}

for _, sub := range gnd.denylistSubs {
sub.Stop()
}
Expand Down
12 changes: 12 additions & 0 deletions setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ type Node struct {
vs routing.ValueStore
host host.Host

dataDir string
datastore datastore.Batching
blockstore blockstore.Blockstore
bsClient *bsclient.Client
Expand Down Expand Up @@ -103,9 +104,19 @@ type Config struct {

DenylistSubs []string
Peering []peer.AddrInfo

GCInterval time.Duration
GCThreshold float64
}

func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cachedDNS) (*Node, error) {
var err error

cfg.DataDir, err = filepath.Abs(cfg.DataDir)
if err != nil {
return nil, err
}

ds, err := setupDatastore(cfg)
if err != nil {
return nil, err
Expand Down Expand Up @@ -350,6 +361,7 @@ func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cached
return &Node{
host: h,
blockstore: blkst,
dataDir: cfg.DataDir,
datastore: ds,
bsClient: bswap,
ns: ns,
Expand Down

0 comments on commit d7b8111

Please sign in to comment.