[SDKS-8921] Proxy Large Segments imp

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
 
       - name: SonarQube Scan (Push)
         if: ${{ github.event_name == 'push' }}
-        uses: SonarSource/sonarcloud-github-action@v2.3.0
+        uses: SonarSource/sonarcloud-github-action@v3.0.0
         env:
           SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -46,7 +46,7 @@ jobs:
 
       - name: SonarQube Scan (Pull Request)
         if: ${{ github.event_name == 'pull_request' }}
-        uses: SonarSource/sonarcloud-github-action@v2.3.0
+        uses: SonarSource/sonarcloud-github-action@v3.0.0
         env:
           SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/update-license-year.yml

@@ -38,7 +38,7 @@ jobs:
           git commit -m "Updated License Year" -a
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           title: Update License Year

CHANGES

@@ -1,6 +1,12 @@
+6.0.2 (Dec 2, 2024)
+- Adding HTTPLargeSegmentsFetcher implementation.
+- Adding MembershipsResponseDTO.
+- Adding LargeSegmentDTO.
+- Bump dependencies for vulnerability fixes.
+
 6.0.1 (Sep 12, 2024)
 - Updated split version filter to support 1.2.
-- Fixed healcheck monitor for cases with no segments.
+- Fixed healthcheck monitor for cases with no segments.
 
 6.0.0 (May 14, 2024)
 - BREAKING CHANGE:

conf/conf.go

@@ -87,4 +87,15 @@ type AdvancedConfig struct {
 	FlagSetsFilter         []string
 	AuthSpecVersion        string
 	FlagsSpecVersion       string
+	LargeSegment           *LargeSegmentConfig
+}
+
+type LargeSegmentConfig struct {
+	Enable          bool
+	Version         string
+	LazyLoad        bool
+	Workers         int
+	UpdateQueueSize int64
+	QueueSize       int
+	RefreshRate     int
 }

conf/defaults.go

@@ -1,21 +1,26 @@
 package conf
 
 const (
-	defaultHTTPTimeout            = 30
-	defaultSegmentQueueSize       = 500
-	defaultSegmentWorkers         = 10
-	defaultEventsBulkSize         = 5000
-	defaultEventsQueueSize        = 10000
-	defaultImpressionsQueueSize   = 10000
-	defaultImpressionsBulkSize    = 5000
-	defaultStreamingEnabled       = true
-	defaultSplitUpdateQueueSize   = 5000
-	defaultSegmentUpdateQueueSize = 5000
-	defaultAuthServiceURL         = "https://auth.split.io"
-	defaultEventsURL              = "https://events.split.io/api"
-	defaultSdkURL                 = "https://sdk.split.io/api"
-	defaultStreamingServiceURL    = "https://streaming.split.io/sse"
-	defaultTelemetryServiceURL    = "https://telemetry.split.io/api/v1"
+	defaultHTTPTimeout                 = 30
+	defaultSegmentQueueSize            = 500
+	defaultSegmentWorkers              = 10
+	defaultEventsBulkSize              = 5000
+	defaultEventsQueueSize             = 10000
+	defaultImpressionsQueueSize        = 10000
+	defaultImpressionsBulkSize         = 5000
+	defaultStreamingEnabled            = true
+	defaultSplitUpdateQueueSize        = 5000
+	defaultSegmentUpdateQueueSize      = 5000
+	defaultLargeSegmentUpdateQueueSize = 5000
+	defaultLargeSegmentQueueSize       = 5000
+	defaultLargeSegmentWorkers         = 5
+	defaultLargeSegmentLazyLoad        = false
+	defaultLargeSegmentEnabled         = false
+	defaultAuthServiceURL              = "https://auth.split.io"
+	defaultEventsURL                   = "https://events.split.io/api"
+	defaultSdkURL                      = "https://sdk.split.io/api"
+	defaultStreamingServiceURL         = "https://streaming.split.io/sse"
+	defaultTelemetryServiceURL         = "https://telemetry.split.io/api/v1"
 )
 
 const (
@@ -52,5 +57,12 @@ func GetDefaultAdvancedConfig() AdvancedConfig {
 		SdkURL:                 defaultSdkURL,
 		StreamingServiceURL:    defaultStreamingServiceURL,
 		TelemetryServiceURL:    defaultTelemetryServiceURL,
+		LargeSegment: &LargeSegmentConfig{
+			Enable:          defaultLargeSegmentEnabled,
+			UpdateQueueSize: defaultLargeSegmentUpdateQueueSize,
+			LazyLoad:        defaultLargeSegmentLazyLoad,
+			Workers:         defaultLargeSegmentWorkers,
+			QueueSize:       defaultLargeSegmentQueueSize,
+		},
 	}
 }

dtos/largesegment.go

@@ -0,0 +1,42 @@
+package dtos
+
+import "net/http"
+
+// Params
+type Params struct {
+	Method  string      `json:"m"`           // method
+	URL     string      `json:"u"`           // url
+	Headers http.Header `json:"h"`           // headers
+	Body    []byte      `json:"b,omitempty"` // body
+}
+
+// Data
+type Data struct {
+	Interval  *int64 `json:"i,omitempty"` // interval
+	Format    int    `json:"f"`           // 0 unknown | 1 csv | otro // format
+	TotalKeys int64  `json:"k"`           // totalKeys
+	FileSize  int64  `json:"s"`           // fileSize
+	ExpiresAt int64  `json:"e"`           // expiration time url
+}
+
+// RFD struct
+type RFD struct {
+	Data   Data   `json:"d"`
+	Params Params `json:"p"`
+}
+
+// LargeSegmentRFDResponseDTO
+type LargeSegmentRFDResponseDTO struct {
+	Name             string `json:"n"`
+	NotificationType string `json:"t"`
+	RFD              *RFD   `json:"rfd,omitempty"`
+	SpecVersion      string `json:"v"`
+	ChangeNumber     int64  `json:"cn"`
+}
+
+// LargeSegment
+type LargeSegment struct {
+	Name         string
+	Keys         []string
+	ChangeNumber int64
+}

dtos/membership.go

@@ -0,0 +1,18 @@
+package dtos
+
+// Segment struct
+type Segment struct {
+	Name string `json:"n"`
+}
+
+// Memberships struct mapping segments data for memberships endpoint
+type Memberships struct {
+	Segments     []Segment `json:"k"`
+	ChangeNumber *int64    `json:"cn,omitempty"`
+}
+
+// MembershipsResponseDTO struct mapping memberships data for memberships endpoint
+type MembershipsResponseDTO struct {
+	MySegments      Memberships `json:"ms"`
+	MyLargeSegments Memberships `json:"ls"`
+}

dtos/notification.go

@@ -14,10 +14,11 @@ const (
 
 // Update type constants
 const (
-	UpdateTypeSplitChange   = "SPLIT_UPDATE"
-	UpdateTypeSplitKill     = "SPLIT_KILL"
-	UpdateTypeSegmentChange = "SEGMENT_UPDATE"
-	UpdateTypeContol        = "CONTROL"
+	UpdateTypeSplitChange        = "SPLIT_UPDATE"
+	UpdateTypeSplitKill          = "SPLIT_KILL"
+	UpdateTypeSegmentChange      = "SEGMENT_UPDATE"
+	UpdateTypeContol             = "CONTROL"
+	UpdateTypeLargeSegmentChange = "LS_DEFINITION_UPDATE"
 )
 
 // Control type constants
@@ -310,13 +311,36 @@ func (u *ControlUpdate) String() string {
 		u.Channel(), u.controlType, u.Timestamp())
 }
 
+type LargeSegmentChangeUpdate struct {
+	BaseUpdate
+	LargeSegments []LargeSegmentRFDResponseDTO `json:"ls"`
+}
+
+func NewLargeSegmentChangeUpdate(baseUpdate BaseUpdate, largeSegments []LargeSegmentRFDResponseDTO) *LargeSegmentChangeUpdate {
+	return &LargeSegmentChangeUpdate{
+		BaseUpdate:    baseUpdate,
+		LargeSegments: largeSegments,
+	}
+}
+
+// UpdateType is always UpdateTypeLargeSegmentChange for Large Segmet Updates
+func (u *LargeSegmentChangeUpdate) UpdateType() string { return UpdateTypeLargeSegmentChange }
+
+// String returns the string representation of a segment update notification
+func (u *LargeSegmentChangeUpdate) String() string {
+	return fmt.Sprintf("LargeSegmentChange(channel=%s,changeNumber=%d,count=%d,timestamp=%d)",
+		u.Channel(), u.ChangeNumber(), len(u.LargeSegments), u.Timestamp())
+}
+
 // Compile-type assertions of interface requirements
 var _ Event = &AblyError{}
 var _ Message = &OccupancyMessage{}
 var _ Message = &SplitChangeUpdate{}
 var _ Message = &SplitKillUpdate{}
 var _ Message = &SegmentChangeUpdate{}
 var _ Message = &ControlUpdate{}
+var _ Message = &LargeSegmentChangeUpdate{}
 var _ Update = &SplitChangeUpdate{}
 var _ Update = &SplitKillUpdate{}
 var _ Update = &SegmentChangeUpdate{}
+var _ Update = &LargeSegmentChangeUpdate{}

dtos/notification_test.go

@@ -225,3 +225,41 @@ func TestControlUpdate(t *testing.T) {
 		t.Error("Unexpected String")
 	}
 }
+
+func TestLargeSegmentChangeUpdate(t *testing.T) {
+	ls := []LargeSegmentRFDResponseDTO{
+		{
+			Name:             "ls1",
+			NotificationType: UpdateTypeLargeSegmentChange,
+			SpecVersion:      "1.0",
+			RFD:              &RFD{},
+			ChangeNumber:     123123,
+		},
+	}
+
+	lsUpdate := NewLargeSegmentChangeUpdate(NewBaseUpdate(NewBaseMessage(123456789, "ls_channel"), 123456), ls)
+	if lsUpdate.EventType() != SSEEventTypeMessage {
+		t.Error("Unexpected EventType")
+	}
+	if lsUpdate.Timestamp() != 123456789 {
+		t.Error("Unexpected Timestamp")
+	}
+	if lsUpdate.Channel() != "ls_channel" {
+		t.Error("Unexpected Channel")
+	}
+	if lsUpdate.MessageType() != MessageTypeUpdate {
+		t.Error("Unexpected MessageType")
+	}
+	if lsUpdate.ChangeNumber() != 123456 {
+		t.Error("Unexpected ChangeNumber")
+	}
+	if lsUpdate.UpdateType() != UpdateTypeLargeSegmentChange {
+		t.Error("Unexpected UpdateType")
+	}
+	if lsUpdate.String() != "LargeSegmentChange(channel=ls_channel,changeNumber=123456,count=1,timestamp=123456789)" {
+		t.Error("Unexpected String", lsUpdate.String())
+	}
+	if len(lsUpdate.LargeSegments) != 1 {
+		t.Error("LargeSegments len should be 1. Actual: ", len(lsUpdate.LargeSegments))
+	}
+}

dtos/split.go

@@ -53,17 +53,23 @@ type MatcherGroupDTO struct {
 
 // MatcherDTO structure to map a Matcher definition fetched from JSON message.
 type MatcherDTO struct {
-	KeySelector        *KeySelectorDTO                   `json:"keySelector"`
-	MatcherType        string                            `json:"matcherType"`
-	Negate             bool                              `json:"negate"`
-	UserDefinedSegment *UserDefinedSegmentMatcherDataDTO `json:"userDefinedSegmentMatcherData"`
-	Whitelist          *WhitelistMatcherDataDTO          `json:"whitelistMatcherData"`
-	UnaryNumeric       *UnaryNumericMatcherDataDTO       `json:"unaryNumericMatcherData"`
-	Between            *BetweenMatcherDataDTO            `json:"betweenMatcherData"`
-	BetweenString      *BetweenStringMatcherDataDTO      `json:"betweenStringMatcherData"`
-	Dependency         *DependencyMatcherDataDTO         `json:"dependencyMatcherData"`
-	Boolean            *bool                             `json:"booleanMatcherData"`
-	String             *string                           `json:"stringMatcherData"`
+	KeySelector             *KeySelectorDTO                        `json:"keySelector"`
+	MatcherType             string                                 `json:"matcherType"`
+	Negate                  bool                                   `json:"negate"`
+	UserDefinedSegment      *UserDefinedSegmentMatcherDataDTO      `json:"userDefinedSegmentMatcherData"`
+	Whitelist               *WhitelistMatcherDataDTO               `json:"whitelistMatcherData"`
+	UnaryNumeric            *UnaryNumericMatcherDataDTO            `json:"unaryNumericMatcherData"`
+	Between                 *BetweenMatcherDataDTO                 `json:"betweenMatcherData"`
+	BetweenString           *BetweenStringMatcherDataDTO           `json:"betweenStringMatcherData"`
+	Dependency              *DependencyMatcherDataDTO              `json:"dependencyMatcherData"`
+	Boolean                 *bool                                  `json:"booleanMatcherData"`
+	String                  *string                                `json:"stringMatcherData"`
+	UserDefinedLargeSegment *UserDefinedLargeSegmentMatcherDataDTO `json:"userDefinedLargeSegmentMatcherData"`
+}
+
+// UserDefinedLargeSegmentMatcherDataDTO structure to map a Matcher definition fetched from JSON message.
+type UserDefinedLargeSegmentMatcherDataDTO struct {
+	LargeSegmentName string `json:"largeSegmentName"`
 }
 
 // UserDefinedSegmentMatcherDataDTO structure to map a Matcher definition fetched from JSON message.

dtos/split_test.go

@@ -4,12 +4,12 @@ package dtos
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"testing"
 )
 
-var splitsMock, _ = ioutil.ReadFile("../testdata/splits_mock.json")
-var splitMock, _ = ioutil.ReadFile("../testdata/split_mock.json")
+var splitsMock, _ = os.ReadFile("../testdata/splits_mock.json")
+var splitMock, _ = os.ReadFile("../testdata/split_mock.json")
 
 func TestSplitDTO(t *testing.T) {
 	mockedData := fmt.Sprintf(string(splitsMock), splitMock)
@@ -63,5 +63,19 @@ func TestSplitDTO(t *testing.T) {
 		if len(splitChangesDtoFromMarshal.Splits[0].Sets) != 0 {
 			t.Error("Marshal struct mal formed [Sets]")
 		}
+
+		condition := splitChangesDtoFromMarshal.Splits[0].Conditions[0]
+		if condition.ConditionType != "WHITELIST" {
+			t.Error("Marshal struct mal formed [ConditionType]. Actual: ", condition.ConditionType)
+		}
+
+		matcher := condition.MatcherGroup.Matchers[0]
+		if matcher.MatcherType != "IN_LARGE_SEGMENT" {
+			t.Error("Marshal struct mal formed [MatcherType]")
+		}
+
+		if matcher.UserDefinedLargeSegment.LargeSegmentName != "mauro_sanz_ls" {
+			t.Error("Marshal struct mal formed [UserDefinedLargeSegment]")
+		}
 	}
 }

engine/evaluator/evaluator_test.go

@@ -219,6 +219,7 @@ func (s *mockStorage) FetchMany(
 }
 func (s *mockStorage) All() []dtos.SplitDTO                      { return make([]dtos.SplitDTO, 0) }
 func (s *mockStorage) SegmentNames() *set.ThreadUnsafeSet        { return nil }
+func (s *mockStorage) LargeSegmentNames() *set.ThreadUnsafeSet   { return nil }
 func (s *mockStorage) SplitNames() []string                      { return make([]string, 0) }
 func (s *mockStorage) TrafficTypeExists(trafficType string) bool { return true }
 func (s *mockStorage) ChangeNumber() (int64, error)              { return 0, nil }

engine/grammar/matchers/inlargesegment.go

@@ -0,0 +1,39 @@
+package matchers
+
+import (
+	"fmt"
+
+	"github.com/splitio/go-split-commons/v6/storage"
+)
+
+// InLargeSegmentMatcher matches if the key passed is in the large segment which the matcher was constructed with
+type InLargeSegmentMatcher struct {
+	Matcher
+	name string
+}
+
+// Match returns true if the key is in the matcher's segment
+func (m *InLargeSegmentMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool {
+	storage, ok := m.Context.Dependency("largeSegmentStorage").(storage.LargeSegmentStorageConsumer)
+	if !ok {
+		m.logger.Error("InLargeSegmentMatcher: Unable to retrieve large segment storage!")
+		return false
+	}
+
+	isInLargeSegment, err := storage.IsInLargeSegment(m.name, key)
+	if err != nil {
+		m.logger.Error(fmt.Printf("InLargeSegmentMatcher: Large Segment %s not found", m.name))
+	}
+	return isInLargeSegment
+}
+
+// NewInLargeSegmentMatcher instantiates a new InLargeSegmentMatcher
+func NewInLargeSegmentMatcher(negate bool, name string, attributeName *string) *InLargeSegmentMatcher {
+	return &InLargeSegmentMatcher{
+		Matcher: Matcher{
+			negate:        negate,
+			attributeName: attributeName,
+		},
+		name: name,
+	}
+}