Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(options): allow special handling and checking when creating options from superflag #1688

Merged
merged 2 commits into from
Apr 5, 2021
Merged

feat(options): allow special handling and checking when creating options from superflag #1688

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
49e68bb
fix super flags
NamanJain8 Apr 1, 2021
82aa9f5
badger superflag works
NamanJain8 Apr 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 162 additions & 16 deletions options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,15 @@
package badger

import (
"fmt"
"os"
"reflect"
"strconv"
"strings"
"time"

"github.com/dgraph-io/ristretto/z"
"github.com/pkg/errors"

"github.com/dgraph-io/badger/v3/options"
"github.com/dgraph-io/badger/v3/table"
Expand Down Expand Up @@ -224,6 +227,100 @@ func LSMOnlyOptions(path string) Options {
return DefaultOptions(path).WithValueThreshold(maxValueThreshold /* 1 MB */)
}

// parseCompression returns badger.compressionType and compression level given compression string
// of format compression-type:compression-level
func parseCompression(cStr string) (options.CompressionType, int, error) {
cStrSplit := strings.Split(cStr, ":")
cType := cStrSplit[0]
level := 3

var err error
if len(cStrSplit) == 2 {
level, err = strconv.Atoi(cStrSplit[1])
y.Check(err)
if level <= 0 {
return 0, 0,
errors.Errorf("ERROR: compression level(%v) must be greater than zero", level)
}
} else if len(cStrSplit) > 2 {
return 0, 0, errors.Errorf("ERROR: Invalid badger.compression argument")
}
switch cType {
case "zstd":
return options.ZSTD, level, nil
case "snappy":
return options.Snappy, 0, nil
case "none":
return options.None, 0, nil
}
return 0, 0, errors.Errorf("ERROR: compression type (%s) invalid", cType)
}

// getCachePercentages returns the slice of cache percentages given the "," (comma) separated
// cache percentages(integers) string and expected number of caches.
func getCachePercentages(cpString string, numExpected int) ([]int64, error) {
cp := strings.Split(cpString, ",")
// Sanity checks
if len(cp) != numExpected {
return nil, errors.Errorf("ERROR: expected %d cache percentages, got %d",
numExpected, len(cp))
}

var cachePercent []int64
percentSum := 0
for _, percent := range cp {
x, err := strconv.Atoi(percent)
if err != nil {
return nil, errors.Errorf("ERROR: unable to parse cache percentage(%s)", percent)
}
if x < 0 {
return nil, errors.Errorf("ERROR: cache percentage(%s) cannot be negative", percent)
}
cachePercent = append(cachePercent, int64(x))
percentSum += x
}

if percentSum != 100 {
return nil, errors.Errorf("ERROR: cache percentages (%s) does not sum up to 100",
strings.Join(cp, "+"))
}

return cachePercent, nil
}

// generateSuperFlag generates an identical SuperFlag string from the provided Options.
func generateSuperFlag(options Options) string {
superflag := ""
v := reflect.ValueOf(&options).Elem()
optionsStruct := v.Type()
for i := 0; i < v.NumField(); i++ {
if field := v.Field(i); field.CanInterface() {
name := strings.ToLower(optionsStruct.Field(i).Name)
kind := v.Field(i).Kind()
switch kind {
case reflect.Bool:
superflag += name + "="
superflag += fmt.Sprintf("%v; ", field.Bool())
case reflect.Int, reflect.Int64:
superflag += name + "="
superflag += fmt.Sprintf("%v; ", field.Int())
case reflect.Uint32, reflect.Uint64:
superflag += name + "="
superflag += fmt.Sprintf("%v; ", field.Uint())
case reflect.Float64:
superflag += name + "="
superflag += fmt.Sprintf("%v; ", field.Float())
case reflect.String:
superflag += name + "="
superflag += fmt.Sprintf("%v; ", field.String())
default:
continue
}
}
}
return superflag
}

// FromSuperFlag fills Options fields for each flag within the superflag. For
// example, replacing the default Options.NumGoroutines:
//
Expand All @@ -233,9 +330,29 @@ func LSMOnlyOptions(path string) Options {
// will not fill it with default values. FromSuperFlag only writes to the fields
// present within the superflag string (case insensitive).
//
// Special Handling: compression, cache-mb, cache-percentage sub-flags.
// Valid values for special flags:
// compression: valid options are {none,snappy,zstd:<level>}
// cache-mb: uint
// cache-percentage: 2 comma-separated values summing upto 100. Format: blockCache,indexCache
// goroutines: alias for numgoroutines.
// Example: compression=zstd:3; cache-mb=2048; cache-percentage=70,30; goroutines=8;
// Unsupported: Options.Logger, Options.EncryptionKey
func (opt Options) FromSuperFlag(superflag string) Options {
flags := z.NewSuperFlag(superflag)
var specialFlags = map[string]struct{}{
"compression": {},
"goroutines": {},
"cache-mb": {},
"cache-percentage": {},
}

// currentOptions act as a default value for the options superflag.
currentOptions := generateSuperFlag(opt)
for flag := range specialFlags {
currentOptions += fmt.Sprintf("%s=; ", flag)
}

flags := z.NewSuperFlag(superflag).MergeAndCheckDefault(currentOptions)
v := reflect.ValueOf(&opt).Elem()
optionsStruct := v.Type()
for i := 0; i < v.NumField(); i++ {
Expand All @@ -244,25 +361,54 @@ func (opt Options) FromSuperFlag(superflag string) Options {
// z.SuperFlag stores keys as lowercase, keep everything case
// insensitive
name := strings.ToLower(optionsStruct.Field(i).Name)
if _, ok := specialFlags[name]; ok {
// We will specially handle these flags later. Skip them here.
continue
}
kind := v.Field(i).Kind()
// make sure the option exists in the SuperFlag first, otherwise
// we'd overwrite the defaults with 0 values
if flags.Has(name) {
switch kind {
case reflect.Bool:
field.SetBool(flags.GetBool(name))
case reflect.Int, reflect.Int64:
field.SetInt(flags.GetInt64(name))
case reflect.Uint32:
field.SetUint(uint64(flags.GetUint32(name)))
case reflect.Float64:
field.SetFloat(flags.GetFloat64(name))
case reflect.String:
field.SetString(flags.GetString(name))
}
switch kind {
case reflect.Bool:
field.SetBool(flags.GetBool(name))
case reflect.Int, reflect.Int64:
field.SetInt(flags.GetInt64(name))
case reflect.Uint32, reflect.Uint64:
field.SetUint(uint64(flags.GetUint64(name)))
case reflect.Float64:
field.SetFloat(flags.GetFloat64(name))
case reflect.String:
field.SetString(flags.GetString(name))
}
}
}

// Only update the options for special flags that were present in the input superflag.
inputFlag := z.NewSuperFlag(superflag)
if inputFlag.Has("compression") {
ctype, clevel, err := parseCompression(flags.GetString("compression"))
switch err {
case nil:
opt.Compression = ctype
opt.ZSTDCompressionLevel = clevel
default:
ctype = options.CompressionType(flags.GetUint32("compression"))
y.AssertTruef(ctype <= 2, "ERROR: Invalid format or compression type. Got: %s",
flags.GetString("compression"))
opt.Compression = ctype
}
}
if inputFlag.Has("cache-mb") {
totalCache := flags.GetInt64("cache-mb")
y.AssertTruef(totalCache >= 0, "ERROR: Cache size must be non-negative")
cachePercentage := flags.GetString("cache-percentage")
cachePercent, err := getCachePercentages(cachePercentage, 2)
y.Check(err)
opt.BlockCacheSize = (cachePercent[0] * (totalCache << 20)) / 100
opt.IndexCacheSize = (cachePercent[1] * (totalCache << 20)) / 100
}
if inputFlag.Has("goroutines") {
opt.NumGoroutines = int(flags.GetUint32("goroutines"))
}

return opt
}

Expand Down
82 changes: 35 additions & 47 deletions options_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,60 +17,48 @@
package badger

import (
"fmt"
"reflect"
"strings"
"testing"

"github.com/dgraph-io/badger/v3/options"
)

func TestOptions(t *testing.T) {
// copy all the default options over to a big SuperFlag string
defaultSuperFlag := generateSuperFlag(DefaultOptions(""))
// fill an empty Options with values from the SuperFlag
generated := Options{}.FromSuperFlag(defaultSuperFlag)
// make sure they're equal
if !optionsEqual(DefaultOptions(""), generated) {
t.Fatal("generated default SuperFlag != default Options")
}
// check that values are overwritten properly
overwritten := DefaultOptions("").FromSuperFlag("numgoroutines=1234")
if overwritten.NumGoroutines != 1234 {
t.Fatal("Option value not overwritten by SuperFlag value")
}
}
t.Run("default options", func(t *testing.T) {
// copy all the default options over to a big SuperFlag string
defaultSuperFlag := generateSuperFlag(DefaultOptions(""))
// fill an empty Options with values from the SuperFlag
generated := Options{}.FromSuperFlag(defaultSuperFlag)
// make sure they're equal
if !optionsEqual(DefaultOptions(""), generated) {
t.Fatal("generated default SuperFlag != default Options")
}
// check that values are overwritten properly
overwritten := DefaultOptions("").FromSuperFlag("numgoroutines=1234")
if overwritten.NumGoroutines != 1234 {
t.Fatal("Option value not overwritten by SuperFlag value")
}
})

// generateSuperFlag generates an identical SuperFlag string from the provided
// Options--useful for testing.
func generateSuperFlag(options Options) string {
superflag := ""
v := reflect.ValueOf(&options).Elem()
optionsStruct := v.Type()
for i := 0; i < v.NumField(); i++ {
if field := v.Field(i); field.CanInterface() {
name := strings.ToLower(optionsStruct.Field(i).Name)
kind := v.Field(i).Kind()
switch kind {
case reflect.Bool:
superflag += name + "="
superflag += fmt.Sprintf("%v; ", field.Bool())
case reflect.Int, reflect.Int64:
superflag += name + "="
superflag += fmt.Sprintf("%v; ", field.Int())
case reflect.Uint32, reflect.Uint64:
superflag += name + "="
superflag += fmt.Sprintf("%v; ", field.Uint())
case reflect.Float64:
superflag += name + "="
superflag += fmt.Sprintf("%v; ", field.Float())
case reflect.String:
superflag += name + "="
superflag += fmt.Sprintf("%v; ", field.String())
default:
continue
}
t.Run("special flags", func(t *testing.T) {
o1 := DefaultOptions("")
o1.NamespaceOffset = 10
o1.Compression = options.ZSTD
o1.ZSTDCompressionLevel = 2
o1.BlockCacheSize = 70 << 20
o1.IndexCacheSize = 30 << 20
o1.NumGoroutines = 20

o2 := DefaultOptions("")
o2.NamespaceOffset = 10
o2 = o2.FromSuperFlag("compression=zstd:2; cache-mb=100; cache-percentage=70,30; " +
"goroutines=20;")

// make sure they're equal
if !optionsEqual(o1, o2) {
t.Fatal("generated superFlag != expected options")
}
}
return superflag
})
}

// optionsEqual just compares the values of two Options structs
Expand Down