Merge pull request #1080 from Permify/feature/deduplicate-tuples-attr…

…ibutes feat: deduplicate tuples and attributes during data writing
Permify · Feb 23, 2024 · 67f0694 · 67f0694
2 parents c865365 + 0f63678
commit 67f0694
Show file tree

Hide file tree

Showing 3 changed files with 87 additions and 9 deletions.
diff --git a/internal/servers/dataServer.go b/internal/servers/dataServer.go
@@ -3,13 +3,16 @@ package servers
 import (
 	"log/slog"
 
+	otelCodes "go.opentelemetry.io/otel/codes"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc/status"
+
 	"github.com/Permify/permify/internal/storage"
 	"github.com/Permify/permify/internal/validation"
+	"github.com/Permify/permify/pkg/attribute"
 	"github.com/Permify/permify/pkg/database"
 	v1 "github.com/Permify/permify/pkg/pb/base/v1"
-	otelCodes "go.opentelemetry.io/otel/codes"
-	"golang.org/x/net/context"
-	"google.golang.org/grpc/status"
+	"github.com/Permify/permify/pkg/tuple"
 )
 
 // DataServer - Structure for Data Server
@@ -144,7 +147,18 @@ func (r *DataServer) Write(ctx context.Context, request *v1.DataWriteRequest) (*
 
 	relationships := make([]*v1.Tuple, 0, len(request.GetTuples()))
 
+	relationshipsMap := map[string]struct{}{}
+
 	for _, tup := range request.GetTuples() {
+
+		key := tuple.ToString(tup)
+
+		if _, ok := relationshipsMap[key]; ok {
+			continue
+		}
+
+		relationshipsMap[key] = struct{}{}
+
 		definition, _, err := r.sr.ReadEntityDefinition(ctx, request.GetTenantId(), tup.GetEntity().GetType(), version)
 		if err != nil {
 			span.RecordError(err)
@@ -162,27 +176,38 @@ func (r *DataServer) Write(ctx context.Context, request *v1.DataWriteRequest) (*
 		relationships = append(relationships, tup)
 	}
 
-	attributes := make([]*v1.Attribute, 0, len(request.GetAttributes()))
+	attrs := make([]*v1.Attribute, 0, len(request.GetAttributes()))
 
-	for _, attribute := range request.GetAttributes() {
-		definition, _, err := r.sr.ReadEntityDefinition(ctx, request.GetTenantId(), attribute.GetEntity().GetType(), version)
+	attributesMap := map[string]struct{}{}
+
+	for _, attr := range request.GetAttributes() {
+
+		key := attribute.EntityAndAttributeToString(attr.GetEntity(), attr.GetAttribute())
+
+		if _, ok := attributesMap[key]; ok {
+			continue
+		}
+
+		attributesMap[key] = struct{}{}
+
+		definition, _, err := r.sr.ReadEntityDefinition(ctx, request.GetTenantId(), attr.GetEntity().GetType(), version)
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(otelCodes.Error, err.Error())
 			return nil, status.Error(GetStatus(err), err.Error())
 		}
 
-		err = validation.ValidateAttribute(definition, attribute)
+		err = validation.ValidateAttribute(definition, attr)
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(otelCodes.Error, err.Error())
 			return nil, status.Error(GetStatus(err), err.Error())
 		}
 
-		attributes = append(attributes, attribute)
+		attrs = append(attrs, attr)
 	}
 
-	snap, err := r.dw.Write(ctx, request.GetTenantId(), database.NewTupleCollection(relationships...), database.NewAttributeCollection(attributes...))
+	snap, err := r.dw.Write(ctx, request.GetTenantId(), database.NewTupleCollection(relationships...), database.NewAttributeCollection(attrs...))
 	if err != nil {
 		span.RecordError(err)
 		span.SetStatus(otelCodes.Error, err.Error())
@@ -218,7 +243,18 @@ func (r *DataServer) WriteRelationships(ctx context.Context, request *v1.Relatio
 
 	relationships := make([]*v1.Tuple, 0, len(request.GetTuples()))
 
+	relationshipsMap := map[string]struct{}{}
+
 	for _, tup := range request.GetTuples() {
+
+		key := tuple.ToString(tup)
+
+		if _, ok := relationshipsMap[key]; ok {
+			continue
+		}
+
+		relationshipsMap[key] = struct{}{}
+
 		definition, _, err := r.sr.ReadEntityDefinition(ctx, request.GetTenantId(), tup.GetEntity().GetType(), version)
 		if err != nil {
 			span.RecordError(err)

diff --git a/pkg/attribute/attribute.go b/pkg/attribute/attribute.go
@@ -149,6 +149,18 @@ func ToString(attribute *base.Attribute) string {
 	return result
 }
 
+// EntityAndAttributeToString converts an entity and attribute to a single string.
+func EntityAndAttributeToString(entity *base.Entity, attr string) string {
+	// Convert the entity to string format
+	strEntity := EntityToString(entity)
+
+	// Combine the entity string with the attribute using a dollar sign as the separator
+	result := fmt.Sprintf("%s$%s", strEntity, attr)
+
+	// Return the combined string
+	return result
+}
+
 // EntityToString function takes an Entity object and converts it into a string.
 func EntityToString(entity *base.Entity) string {
 	return fmt.Sprintf(ENTITY, entity.GetType(), entity.GetId())

diff --git a/pkg/attribute/attribute_test.go b/pkg/attribute/attribute_test.go
@@ -591,5 +591,35 @@ var _ = Describe("attribute", func() {
 				}
 			}
 		})
+
+		It("EntityAndAttributeToString", func() {
+			tests := []struct {
+				entity    *base.Entity
+				attribute string
+				result    string
+			}{
+				{
+					entity: &base.Entity{
+						Type: "repository",
+						Id:   "1",
+					},
+					attribute: "is_public",
+					result:    "repository:1$is_public",
+				},
+				{
+					entity: &base.Entity{
+						Type: "organization",
+						Id:   "organization-879",
+					},
+					attribute: "credit",
+					result:    "organization:organization-879$credit",
+				},
+			}
+
+			for _, tt := range tests {
+				result := EntityAndAttributeToString(tt.entity, tt.attribute)
+				Expect(result).Should(Equal(tt.result))
+			}
+		})
 	})
 })