Skip to content
This repository has been archived by the owner on Feb 21, 2024. It is now read-only.

Set & retrieve field values. #702

Merged
merged 1 commit into from
Jun 30, 2017
Merged

Set & retrieve field values. #702

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions fragment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,69 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
return changed, nil
}

func (f *Fragment) bit(rowID, columnID uint64) (bool, error) {
pos, err := f.pos(rowID, columnID)
if err != nil {
return false, err
}
return f.storage.Contains(pos), nil
}

// FieldValue uses a column of bits to read a multi-bit value.
func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()

// If existance bit is unset then ignore remaining bits.
if v, err := f.bit(uint64(bitDepth), columnID); err != nil {
return 0, false, err
} else if !v {
return 0, false, nil
}

// Compute other bits into a value.
for i := uint(0); i < bitDepth; i++ {
if v, err := f.bit(uint64(i), columnID); err != nil {
return 0, false, err
} else if !v {
value |= (1 << i)
}
}

return value, true, nil
}

// SetFieldValue uses a column of bits to set a multi-bit value.
func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()

for i := uint(0); i < bitDepth; i++ {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For base-2 range-encoding, we only have to store the 0th bit vector, which ends up being the "flip" of the actual value. So if value = 20, base-2 = 10100 and base-2 range-encoded = 01011. All that to say I think the set and clear here need to be reversed. That would change the logic in FieldValue() as well.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, I fixed the flip in 800e384. I don't quite understand the need for the flip though. Why not store in "unflipped" format?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It makes the range comparisons easier. For example, if you want to find all B greater than F, starting with the most significant bit of a range-encoded bitmap (bits flipped) you can do a NOR operation:

(range-encoded case where B is definitely greater than F):
B 0 0 1 1
F 0 1 0 1
---------
  T - - -

If you store it as the actual value

(equality encoded case where B is definitely greater than F):
B 0 0 1 1
F 0 1 0 1
---------
  - - T -

it's not a straightforward bitwise operation.

if value&(1<<i) != 0 {
if c, err := f.clearBit(uint64(i), columnID); err != nil {
return changed, err
} else if c {
changed = true
}
} else {
if c, err := f.setBit(uint64(i), columnID); err != nil {
return changed, err
} else if c {
changed = true
}
}
}

// Mark value as set.
if c, err := f.setBit(uint64(bitDepth), columnID); err != nil {
return changed, err
} else if c {
changed = true
}

return changed, nil
}

// pos translates the row ID and column ID into a position in the storage bitmap.
func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) {
// Return an error if the column ID is out of the range of the fragment's slice.
Expand Down
123 changes: 123 additions & 0 deletions fragment_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"math"
"reflect"
"testing"
"testing/quick"

"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa"
Expand Down Expand Up @@ -92,6 +93,128 @@ func TestFragment_ClearBit(t *testing.T) {
}
}

// Ensure a fragment can set & read a field value.
func TestFragment_SetFieldValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()

// Set value.
if changed, err := f.SetFieldValue(100, 16, 3829); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}

// Read value.
if value, exists, err := f.FieldValue(100, 16); err != nil {
t.Fatal(err)
} else if value != 3829 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected to exist")
}

// Setting value should return no change.
if changed, err := f.SetFieldValue(100, 16, 3829); err != nil {
t.Fatal(err)
} else if changed {
t.Fatal("expected no change")
}
})

t.Run("Overwrite", func(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()

// Set value.
if changed, err := f.SetFieldValue(100, 16, 3829); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}

// Overwriting value should overwrite all bits.
if changed, err := f.SetFieldValue(100, 16, 2028); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}

// Read value.
if value, exists, err := f.FieldValue(100, 16); err != nil {
t.Fatal(err)
} else if value != 2028 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected to exist")
}
})

t.Run("NotExists", func(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()

// Set value.
if changed, err := f.SetFieldValue(100, 10, 20); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}

// Non-existant value.
if value, exists, err := f.FieldValue(100, 11); err != nil {
t.Fatal(err)
} else if value != 0 {
t.Fatalf("unexpected value: %d", value)
} else if exists {
t.Fatal("expected to not exist")
}
})

t.Run("QuickCheck", func(t *testing.T) {
if err := quick.Check(func(bitDepth uint, columnN uint64, values []uint64) bool {
// Limit bit depth & maximum values.
bitDepth = (bitDepth % 62) + 1
columnN = (columnN % 100)
for i := range values {
values[i] = values[i] % (1 << bitDepth)
}

f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()

// Set values.
m := make(map[uint64]int64)
for _, value := range values {
columnID := value % columnN

m[columnID] = int64(value)

if _, err := f.SetFieldValue(columnID, bitDepth, value); err != nil {
t.Fatal(err)
}
}

// Ensure values are set.
for columnID, value := range m {
v, exists, err := f.FieldValue(columnID, bitDepth)
if err != nil {
t.Fatal(err)
} else if value != int64(v) {
t.Fatalf("value mismatch: column=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v)
} else if !exists {
t.Fatalf("value should exist: column=%d", columnID)
}
}

return true
}, nil); err != nil {
t.Fatal(err)
}
})
}

// Ensure a fragment can snapshot correctly.
func TestFragment_Snapshot(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
Expand Down
74 changes: 70 additions & 4 deletions frame.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,16 @@ func (f *Frame) Schema() *FrameSchema {
return f.schema
}

// Field returns a field from the schema by name.
func (f *Frame) Field(name string) *Field {
for _, field := range f.Schema().Fields {
if field.Name == name {
return field
}
}
return nil
}

// TimeQuantum returns the time quantum for the frame.
func (f *Frame) TimeQuantum() TimeQuantum {
f.mu.Lock()
Expand Down Expand Up @@ -576,6 +586,52 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change
return changed, nil
}

// FieldValue reads a field value for a column.
func (f *Frame) FieldValue(columnID uint64, name string) (value int64, exists bool, err error) {
field := f.Field(name)
if field == nil {
return 0, false, ErrFieldNotFound
}

// Fetch target view.
view := f.View(ViewFieldPrefix + name)
if view == nil {
return 0, false, nil
}

v, exists, err := view.FieldValue(columnID, field.BitDepth())
if err != nil {
return 0, false, err
} else if !exists {
return 0, false, nil
}
return int64(v) + field.Min, true, nil
}

// SetFieldValue sets a field value for a column.
func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (changed bool, err error) {
// Fetch field and validate value.
field := f.Field(name)
if field == nil {
return false, ErrFieldNotFound
} else if value < field.Min {
return false, ErrFieldValueTooLow
} else if value > field.Max {
return false, ErrFieldValueTooHigh
}

// Fetch target view.
view, err := f.CreateViewIfNotExists(ViewFieldPrefix + name)
if err != nil {
return false, err
}

// Determine base value to store.
baseValue := uint64(value - field.Min)

return view.SetFieldValue(columnID, field.BitDepth(), baseValue)
}

// Import bulk imports data.
func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error {
// Determine quantum if timestamps are set.
Expand Down Expand Up @@ -761,8 +817,18 @@ func IsValidFieldType(v string) bool {
type Field struct {
Name string `json:"name,omitempty"`
Type string `json:"type,omitempty"`
Min int `json:"min,omitempty"`
Max int `json:"max,omitempty"`
Min int64 `json:"min,omitempty"`
Max int64 `json:"max,omitempty"`
}

// BitDepth returns the number of bits required to store a value between min & max.
func (f *Field) BitDepth() uint {
for i := uint(0); i < 63; i++ {
if f.Max-f.Min < (1 << i) {
return i
}
}
return 63
}

func ValidateField(f *Field) error {
Expand Down Expand Up @@ -817,8 +883,8 @@ func decodeField(f *internal.Field) *Field {
return &Field{
Name: f.Name,
Type: f.Type,
Min: int(f.Min),
Max: int(f.Max),
Min: f.Min,
Max: f.Max,
}
}

Expand Down
Loading