From 086c6017e007dfbc61f06af9b22418d6b5b48279 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 01:03:51 -0400 Subject: [PATCH 01/25] added start of DataStore --- datastore/change_meta.go | 92 +++++++++++++ datastore/complete_post_object.go | 79 +++++++++++ datastore/get_meta.go | 95 +++++++++++++ datastore/get_metas_multiple_param.go | 158 +++++++++++++++++++++ datastore/prepare_get_object.go | 91 ++++++++++++ datastore/prepare_post_object.go | 104 ++++++++++++++ datastore/protocol.go | 190 ++++++++++++++++++++++++++ datastore/rate_objects.go | 92 +++++++++++++ 8 files changed, 901 insertions(+) create mode 100644 datastore/change_meta.go create mode 100644 datastore/complete_post_object.go create mode 100644 datastore/get_meta.go create mode 100644 datastore/get_metas_multiple_param.go create mode 100644 datastore/prepare_get_object.go create mode 100644 datastore/prepare_post_object.go create mode 100644 datastore/protocol.go create mode 100644 datastore/rate_objects.go diff --git a/datastore/change_meta.go b/datastore/change_meta.go new file mode 100644 index 0000000..d312e2a --- /dev/null +++ b/datastore/change_meta.go @@ -0,0 +1,92 @@ +package datastore + +import ( + nex "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func changeMeta(err error, client *nex.Client, callID uint32, param *datastore_types.DataStoreChangeMetaParam) uint32 { + if commonDataStoreProtocol.getObjectInfoByDataIDHandler == nil { + common_globals.Logger.Warning("GetObjectInfoByDataID not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { + common_globals.Logger.Warning("VerifyObjectPermission not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.updateObjectPeriodByDataIDWithPasswordHandler == nil { + common_globals.Logger.Warning("UpdateObjectPeriodByDataIDWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.updateObjectMetaBinaryByDataIDWithPasswordHandler == nil { + common_globals.Logger.Warning("UpdateObjectMetaBinaryByDataIDWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.updateObjectDataTypeByDataIDWithPasswordHandler == nil { + common_globals.Logger.Warning("UpdateObjectDataTypeByDataIDWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + metaInfo, errCode := commonDataStoreProtocol.getObjectInfoByDataIDHandler(param.DataID) + if errCode != 0 { + return errCode + } + + // TODO - Is this the right permission? + errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(metaInfo.OwnerID, client.PID(), metaInfo.DelPermission) + if errCode != 0 { + return errCode + } + + if param.ModifiesFlag&0x08 != 0 { + errCode = commonDataStoreProtocol.updateObjectPeriodByDataIDWithPasswordHandler(param.DataID, param.Period, param.UpdatePassword) + if errCode != 0 { + return errCode + } + } + + if param.ModifiesFlag&0x10 != 0 { + errCode = commonDataStoreProtocol.updateObjectMetaBinaryByDataIDWithPasswordHandler(param.DataID, param.MetaBinary, param.UpdatePassword) + if errCode != 0 { + return errCode + } + } + + if param.ModifiesFlag&0x80 != 0 { + errCode = commonDataStoreProtocol.updateObjectDataTypeByDataIDWithPasswordHandler(param.DataID, param.DataType, param.UpdatePassword) + if errCode != 0 { + return errCode + } + } + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodChangeMeta, nil) + + rmcResponseBytes := rmcResponse.Bytes() + + responsePacket, _ := nex.NewPacketV1(client, nil) + + responsePacket.SetVersion(1) + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} diff --git a/datastore/complete_post_object.go b/datastore/complete_post_object.go new file mode 100644 index 0000000..7063351 --- /dev/null +++ b/datastore/complete_post_object.go @@ -0,0 +1,79 @@ +package datastore + +import ( + "fmt" + + nex "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func completePostObject(err error, client *nex.Client, callID uint32, param *datastore_types.DataStoreCompletePostParam) uint32 { + if commonDataStoreProtocol.s3ObjectSizeHandler == nil { + common_globals.Logger.Warning("S3ObjectSize not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.getObjectSizeDataIDHandler == nil { + common_globals.Logger.Warning("GetObjectSizeDataID not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.updateObjectUploadCompletedByDataIDHandler == nil { + common_globals.Logger.Warning("UpdateObjectUploadCompletedByDataID not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + // TODO - What is param.IsSuccess? Is this correct? + if param.IsSuccess { + bucket := commonDataStoreProtocol.s3Bucket + key := fmt.Sprintf("%d.bin", param.DataID) + + objectSizeS3, err := commonDataStoreProtocol.s3ObjectSizeHandler(bucket, key) + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.NotFound + } + + objectSizeDB, errCode := commonDataStoreProtocol.getObjectSizeDataIDHandler(param.DataID) + if errCode != 0 { + return errCode + } + + if objectSizeS3 != uint64(objectSizeDB) { + // TODO - Is this a good error? + return nex.Errors.DataStore.Unknown + } + + errCode = commonDataStoreProtocol.updateObjectUploadCompletedByDataIDHandler(param.DataID, true) + if errCode != 0 { + return errCode + } + } + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodCompletePostObject, nil) + + rmcResponseBytes := rmcResponse.Bytes() + + responsePacket, _ := nex.NewPacketV1(client, nil) + + responsePacket.SetVersion(1) + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} diff --git a/datastore/get_meta.go b/datastore/get_meta.go new file mode 100644 index 0000000..785477d --- /dev/null +++ b/datastore/get_meta.go @@ -0,0 +1,95 @@ +package datastore + +import ( + nex "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func getMeta(err error, client *nex.Client, callID uint32, param *datastore_types.DataStoreGetMetaParam) uint32 { + if commonDataStoreProtocol.getObjectInfoByPersistenceTargetWithPasswordHandler == nil { + common_globals.Logger.Warning("GetObjectInfoByPersistenceTargetWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.getObjectInfoByDataIDWithPasswordHandler == nil { + common_globals.Logger.Warning("GetObjectInfoByDataIDWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { + common_globals.Logger.Warning("VerifyObjectPermission not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + var pMetaInfo *datastore_types.DataStoreMetaInfo + var errCode uint32 + + // * Real server ignores PersistenceTarget if DataID is set + if param.DataID == 0 { + pMetaInfo, errCode = commonDataStoreProtocol.getObjectInfoByPersistenceTargetWithPasswordHandler(param.PersistenceTarget, param.AccessPassword) + } else { + pMetaInfo, errCode = commonDataStoreProtocol.getObjectInfoByDataIDWithPasswordHandler(param.DataID, param.AccessPassword) + } + + if errCode != 0 { + return errCode + } + + errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(pMetaInfo.OwnerID, client.PID(), pMetaInfo.Permission) + if errCode != 0 { + return errCode + } + + // * This is kind of backwards. + // * The database pulls this data + // * by default, so it can be done + // * in a single query. So instead + // * of checking if a flag *IS* + // * set, and conditionally *ADDING* + // * the fields, we check if a flag + // * is *NOT* set and conditionally + // * *REMOVE* the field + if param.ResultOption&0x1 == 0 { + pMetaInfo.Tags = make([]string, 0) + } + + if param.ResultOption&0x2 == 0 { + pMetaInfo.Ratings = make([]*datastore_types.DataStoreRatingInfoWithSlot, 0) + } + + if param.ResultOption&0x4 == 0 { + pMetaInfo.MetaBinary = make([]byte, 0) + } + + rmcResponseStream := nex.NewStreamOut(commonDataStoreProtocol.server) + rmcResponseStream.WriteStructure(pMetaInfo) + + rmcResponseBody := rmcResponseStream.Bytes() + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodGetMeta, rmcResponseBody) + + rmcResponseBytes := rmcResponse.Bytes() + + responsePacket, _ := nex.NewPacketV1(client, nil) + + responsePacket.SetVersion(1) + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} diff --git a/datastore/get_metas_multiple_param.go b/datastore/get_metas_multiple_param.go new file mode 100644 index 0000000..a7faf9a --- /dev/null +++ b/datastore/get_metas_multiple_param.go @@ -0,0 +1,158 @@ +package datastore + +import ( + nex "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func getMetasMultipleParam(err error, client *nex.Client, callID uint32, params []*datastore_types.DataStoreGetMetaParam) uint32 { + if commonDataStoreProtocol.getObjectInfoByPersistenceTargetWithPasswordHandler == nil { + common_globals.Logger.Warning("GetObjectInfoByPersistenceTargetWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.getObjectInfoByDataIDWithPasswordHandler == nil { + common_globals.Logger.Warning("GetObjectInfoByDataIDWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { + common_globals.Logger.Warning("VerifyObjectPermission not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + pMetaInfo := make([]*datastore_types.DataStoreMetaInfo, 0, len(params)) + pResults := make([]*nex.Result, 0, len(params)) + + for _, param := range params { + var objectInfo *datastore_types.DataStoreMetaInfo + var errCode uint32 + + // * Real server ignores PersistenceTarget if DataID is set + if param.DataID == 0 { + objectInfo, errCode = commonDataStoreProtocol.getObjectInfoByPersistenceTargetWithPasswordHandler(param.PersistenceTarget, param.AccessPassword) + } else { + objectInfo, errCode = commonDataStoreProtocol.getObjectInfoByDataIDWithPasswordHandler(param.DataID, param.AccessPassword) + } + + if errCode != 0 { + // TODO - Maybe this should be broken out into a util function in globals? + objectInfo = datastore_types.NewDataStoreMetaInfo() + objectInfo.DataID = 0 + objectInfo.OwnerID = 0 + objectInfo.Size = 0 + objectInfo.Name = "" + objectInfo.DataType = 0 + objectInfo.MetaBinary = []byte{} + objectInfo.Permission = datastore_types.NewDataStorePermission() + objectInfo.Permission.Permission = 0 + objectInfo.Permission.RecipientIDs = []uint32{} + objectInfo.DelPermission = datastore_types.NewDataStorePermission() + objectInfo.DelPermission.Permission = 0 + objectInfo.DelPermission.RecipientIDs = []uint32{} + objectInfo.CreatedTime = nex.NewDateTime(0) + objectInfo.UpdatedTime = nex.NewDateTime(0) + objectInfo.Period = 0 + objectInfo.Status = 0 + objectInfo.ReferredCnt = 0 + objectInfo.ReferDataID = 0 + objectInfo.Flag = 0 + objectInfo.ReferredTime = nex.NewDateTime(0) + objectInfo.ExpireTime = nex.NewDateTime(0) + objectInfo.Tags = []string{} + objectInfo.Ratings = []*datastore_types.DataStoreRatingInfoWithSlot{} + + pResults = append(pResults, nex.NewResultError(errCode)) + } else { + errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(objectInfo.OwnerID, client.PID(), objectInfo.Permission) + if errCode != 0 { + // TODO - Maybe this should be broken out into a util function in globals? + objectInfo = datastore_types.NewDataStoreMetaInfo() + objectInfo.DataID = 0 + objectInfo.OwnerID = 0 + objectInfo.Size = 0 + objectInfo.Name = "" + objectInfo.DataType = 0 + objectInfo.MetaBinary = []byte{} + objectInfo.Permission = datastore_types.NewDataStorePermission() + objectInfo.Permission.Permission = 0 + objectInfo.Permission.RecipientIDs = []uint32{} + objectInfo.DelPermission = datastore_types.NewDataStorePermission() + objectInfo.DelPermission.Permission = 0 + objectInfo.DelPermission.RecipientIDs = []uint32{} + objectInfo.CreatedTime = nex.NewDateTime(0) + objectInfo.UpdatedTime = nex.NewDateTime(0) + objectInfo.Period = 0 + objectInfo.Status = 0 + objectInfo.ReferredCnt = 0 + objectInfo.ReferDataID = 0 + objectInfo.Flag = 0 + objectInfo.ReferredTime = nex.NewDateTime(0) + objectInfo.ExpireTime = nex.NewDateTime(0) + objectInfo.Tags = []string{} + objectInfo.Ratings = []*datastore_types.DataStoreRatingInfoWithSlot{} + + pResults = append(pResults, nex.NewResultError(errCode)) + } else { + pResults = append(pResults, nex.NewResultSuccess(nex.Errors.DataStore.Unknown)) + } + + // * This is kind of backwards. + // * The database pulls this data + // * by default, so it can be done + // * in a single query. So instead + // * of checking if a flag *IS* + // * set, and conditionally *ADDING* + // * the fields, we check if a flag + // * is *NOT* set and conditionally + // * *REMOVE* the field + if param.ResultOption&0x1 == 0 { + objectInfo.Tags = make([]string, 0) + } + + if param.ResultOption&0x2 == 0 { + objectInfo.Ratings = make([]*datastore_types.DataStoreRatingInfoWithSlot, 0) + } + + if param.ResultOption&0x4 == 0 { + objectInfo.MetaBinary = make([]byte, 0) + } + } + + pMetaInfo = append(pMetaInfo, objectInfo) + } + + rmcResponseStream := nex.NewStreamOut(commonDataStoreProtocol.server) + + rmcResponseStream.WriteListStructure(pMetaInfo) + rmcResponseStream.WriteListResult(pResults) + + rmcResponseBody := rmcResponseStream.Bytes() + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodGetMetasMultipleParam, rmcResponseBody) + + rmcResponseBytes := rmcResponse.Bytes() + + responsePacket, _ := nex.NewPacketV1(client, nil) + + responsePacket.SetVersion(1) + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} diff --git a/datastore/prepare_get_object.go b/datastore/prepare_get_object.go new file mode 100644 index 0000000..b0d6e71 --- /dev/null +++ b/datastore/prepare_get_object.go @@ -0,0 +1,91 @@ +package datastore + +import ( + "fmt" + "time" + + nex "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func prepareGetObject(err error, client *nex.Client, callID uint32, param *datastore_types.DataStorePrepareGetParam) uint32 { + if commonDataStoreProtocol.getObjectInfoByDataIDHandler == nil { + common_globals.Logger.Warning("GetObjectInfoByDataID not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { + common_globals.Logger.Warning("VerifyObjectPermission not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.presignGetObjectHandler == nil { + common_globals.Logger.Warning("PresignGetObject not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.Core.Unknown + } + + bucket := commonDataStoreProtocol.s3Bucket + key := fmt.Sprintf("%d.bin", param.DataID) + + objectInfo, errCode := commonDataStoreProtocol.getObjectInfoByDataIDHandler(param.DataID) + if errCode != 0 { + return errCode + } + + errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(objectInfo.OwnerID, client.PID(), objectInfo.Permission) + if errCode != 0 { + return errCode + } + + URL, err := commonDataStoreProtocol.presignGetObjectHandler(bucket, key, time.Minute*15) + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.OperationNotAllowed + } + + requestHeaders, errCode := commonDataStoreProtocol.s3GetRequestHeadersHandler() + if errCode != 0 { + return errCode + } + + pReqGetInfo := datastore_types.NewDataStoreReqGetInfo() + + pReqGetInfo.URL = URL.String() + pReqGetInfo.RequestHeaders = requestHeaders + pReqGetInfo.Size = objectInfo.Size + pReqGetInfo.RootCACert = commonDataStoreProtocol.rootCACert + pReqGetInfo.DataID = param.DataID + + rmcResponseStream := nex.NewStreamOut(commonDataStoreProtocol.server) + + rmcResponseStream.WriteStructure(pReqGetInfo) + + rmcResponseBody := rmcResponseStream.Bytes() + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodPrepareGetObject, rmcResponseBody) + + rmcResponseBytes := rmcResponse.Bytes() + + responsePacket, _ := nex.NewPacketV1(client, nil) + + responsePacket.SetVersion(1) + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} diff --git a/datastore/prepare_post_object.go b/datastore/prepare_post_object.go new file mode 100644 index 0000000..208fb87 --- /dev/null +++ b/datastore/prepare_post_object.go @@ -0,0 +1,104 @@ +package datastore + +import ( + "fmt" + "time" + + nex "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func preparePostObject(err error, client *nex.Client, callID uint32, param *datastore_types.DataStorePreparePostParam) uint32 { + if commonDataStoreProtocol.initializeObjectByPreparePostParamHandler == nil { + common_globals.Logger.Warning("InitializeObjectByPreparePostParam not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.initializeObjectRatingWithSlotHandler == nil { + common_globals.Logger.Warning("InitializeObjectRatingWithSlot not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.presignPostObjectHandler == nil { + common_globals.Logger.Warning("PresignPostObject not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + dataID, errCode := commonDataStoreProtocol.initializeObjectByPreparePostParamHandler(client.PID(), param) + if errCode != 0 { + common_globals.Logger.Errorf("Error code %d on object init", errCode) + return errCode + } + + // TODO - Should this be moved to InitializeObjectByPreparePostParam? + for _, ratingInitParamWithSlot := range param.RatingInitParams { + errCode = commonDataStoreProtocol.initializeObjectRatingWithSlotHandler(dataID, ratingInitParamWithSlot) + if errCode != 0 { + common_globals.Logger.Errorf("Error code %d on rating init", errCode) + return errCode + } + } + + bucket := commonDataStoreProtocol.s3Bucket + key := fmt.Sprintf("%d.bin", dataID) + + URL, formData, err := commonDataStoreProtocol.presignPostObjectHandler(bucket, key, time.Minute*15) + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.OperationNotAllowed + } + + requestHeaders, errCode := commonDataStoreProtocol.s3PostRequestHeadersHandler() + if errCode != 0 { + return errCode + } + + pReqPostInfo := datastore_types.NewDataStoreReqPostInfo() + + pReqPostInfo.DataID = dataID + pReqPostInfo.URL = URL.String() + pReqPostInfo.RequestHeaders = requestHeaders + pReqPostInfo.FormFields = make([]*datastore_types.DataStoreKeyValue, 0, len(formData)) + pReqPostInfo.RootCACert = commonDataStoreProtocol.rootCACert + + for key, value := range formData { + field := datastore_types.NewDataStoreKeyValue() + field.Key = key + field.Value = value + + pReqPostInfo.FormFields = append(pReqPostInfo.FormFields, field) + } + + rmcResponseStream := nex.NewStreamOut(commonDataStoreProtocol.server) + + rmcResponseStream.WriteStructure(pReqPostInfo) + + rmcResponseBody := rmcResponseStream.Bytes() + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodPreparePostObject, rmcResponseBody) + + rmcResponseBytes := rmcResponse.Bytes() + + responsePacket, _ := nex.NewPacketV1(client, nil) + + responsePacket.SetVersion(1) + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} diff --git a/datastore/protocol.go b/datastore/protocol.go new file mode 100644 index 0000000..18a96d4 --- /dev/null +++ b/datastore/protocol.go @@ -0,0 +1,190 @@ +package datastore + +import ( + "net/url" + "strings" + "time" + + nex "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_super_mario_maker "github.com/PretendoNetwork/nex-protocols-go/datastore/super-mario-maker" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +var commonDataStoreProtocol *CommonDataStoreProtocol + +type CommonDataStoreProtocol struct { + server *nex.Server + DefaultProtocol *datastore.Protocol + SuperMarioMakerProtocol *datastore_super_mario_maker.Protocol + + s3Bucket string + rootCACert []byte + getObjectInfoByDataIDHandler func(dataID uint64) (*datastore_types.DataStoreMetaInfo, uint32) + verifyObjectPermissionHandler func(ownerPID uint32, accessorPID uint32, permission *datastore_types.DataStorePermission) uint32 + updateObjectPeriodByDataIDWithPasswordHandler func(dataID uint64, dataType uint16, password uint64) uint32 + updateObjectMetaBinaryByDataIDWithPasswordHandler func(dataID uint64, metaBinary []byte, password uint64) uint32 + updateObjectDataTypeByDataIDWithPasswordHandler func(dataID uint64, period uint16, password uint64) uint32 + s3ObjectSizeHandler func(bucket string, key string) (uint64, error) + getObjectSizeDataIDHandler func(dataID uint64) (uint32, uint32) + updateObjectUploadCompletedByDataIDHandler func(dataID uint64, uploadCompleted bool) uint32 + getObjectInfoByPersistenceTargetWithPasswordHandler func(persistenceTarget *datastore_types.DataStorePersistenceTarget, password uint64) (*datastore_types.DataStoreMetaInfo, uint32) + getObjectInfoByDataIDWithPasswordHandler func(dataID uint64, password uint64) (*datastore_types.DataStoreMetaInfo, uint32) + presignGetObjectHandler func(bucket string, key string, lifetime time.Duration) (*url.URL, error) + presignPostObjectHandler func(bucket string, key string, lifetime time.Duration) (*url.URL, map[string]string, error) + s3GetRequestHeadersHandler func() ([]*datastore_types.DataStoreKeyValue, uint32) + s3PostRequestHeadersHandler func() ([]*datastore_types.DataStoreKeyValue, uint32) + initializeObjectByPreparePostParamHandler func(ownerPID uint32, param *datastore_types.DataStorePreparePostParam) (uint64, uint32) + initializeObjectRatingWithSlotHandler func(dataID uint64, param *datastore_types.DataStoreRatingInitParamWithSlot) uint32 + rateObjectWithPasswordHandler func(dataID uint64, slot uint8, ratingValue int32, accessPassword uint64) (*datastore_types.DataStoreRatingInfo, uint32) +} + +// SetS3Bucket sets the S3 bucket +func (c *CommonDataStoreProtocol) SetS3Bucket(bucket string) { + c.s3Bucket = bucket +} + +// SetRootCACert sets the S3 root CA +func (c *CommonDataStoreProtocol) SetRootCACert(rootCACert []byte) { + c.rootCACert = rootCACert +} + +// GetObjectInfoByDataID sets the GetObjectInfoByDataID handler function +func (c *CommonDataStoreProtocol) GetObjectInfoByDataID(handler func(dataID uint64) (*datastore_types.DataStoreMetaInfo, uint32)) { + c.getObjectInfoByDataIDHandler = handler +} + +// VerifyObjectPermission sets the VerifyObjectPermission handler function +func (c *CommonDataStoreProtocol) VerifyObjectPermission(handler func(ownerPID uint32, accessorPID uint32, permission *datastore_types.DataStorePermission) uint32) { + c.verifyObjectPermissionHandler = handler +} + +// UpdateObjectPeriodByDataIDWithPassword sets the UpdateObjectPeriodByDataIDWithPassword handler function +func (c *CommonDataStoreProtocol) UpdateObjectPeriodByDataIDWithPassword(handler func(dataID uint64, dataType uint16, password uint64) uint32) { + c.updateObjectPeriodByDataIDWithPasswordHandler = handler +} + +// UpdateObjectMetaBinaryByDataIDWithPassword sets the UpdateObjectMetaBinaryByDataIDWithPassword handler function +func (c *CommonDataStoreProtocol) UpdateObjectMetaBinaryByDataIDWithPassword(handler func(dataID uint64, metaBinary []byte, password uint64) uint32) { + c.updateObjectMetaBinaryByDataIDWithPasswordHandler = handler +} + +// UpdateObjectDataTypeByDataIDWithPassword sets the UpdateObjectDataTypeByDataIDWithPassword handler function +func (c *CommonDataStoreProtocol) UpdateObjectDataTypeByDataIDWithPassword(handler func(dataID uint64, period uint16, password uint64) uint32) { + c.updateObjectDataTypeByDataIDWithPasswordHandler = handler +} + +// S3ObjectSize sets the S3ObjectSize handler function +func (c *CommonDataStoreProtocol) S3ObjectSize(handler func(bucket string, key string) (uint64, error)) { + c.s3ObjectSizeHandler = handler +} + +// GetObjectSizeDataID sets the GetObjectSizeDataID handler function +func (c *CommonDataStoreProtocol) GetObjectSizeDataID(handler func(dataID uint64) (uint32, uint32)) { + c.getObjectSizeDataIDHandler = handler +} + +// UpdateObjectUploadCompletedByDataID sets the UpdateObjectUploadCompletedByDataID handler function +func (c *CommonDataStoreProtocol) UpdateObjectUploadCompletedByDataID(handler func(dataID uint64, uploadCompleted bool) uint32) { + c.updateObjectUploadCompletedByDataIDHandler = handler +} + +// GetObjectInfoByPersistenceTargetWithPassword sets the GetObjectInfoByPersistenceTargetWithPassword handler function +func (c *CommonDataStoreProtocol) GetObjectInfoByPersistenceTargetWithPassword(handler func(persistenceTarget *datastore_types.DataStorePersistenceTarget, password uint64) (*datastore_types.DataStoreMetaInfo, uint32)) { + c.getObjectInfoByPersistenceTargetWithPasswordHandler = handler +} + +// GetObjectInfoByDataIDWithPassword sets the GetObjectInfoByDataIDWithPassword handler function +func (c *CommonDataStoreProtocol) GetObjectInfoByDataIDWithPassword(handler func(dataID uint64, password uint64) (*datastore_types.DataStoreMetaInfo, uint32)) { + c.getObjectInfoByDataIDWithPasswordHandler = handler +} + +// PresignGetObject sets the PresignGetObject handler function +func (c *CommonDataStoreProtocol) PresignGetObject(handler func(bucket string, key string, lifetime time.Duration) (*url.URL, error)) { + c.presignGetObjectHandler = handler +} + +// PresignPostObject sets the PresignPostObject handler function +func (c *CommonDataStoreProtocol) PresignPostObject(handler func(bucket string, key string, lifetime time.Duration) (*url.URL, map[string]string, error)) { + c.presignPostObjectHandler = handler +} + +// S3GetRequestHeaders sets the S3GetRequestHeaders handler function +func (c *CommonDataStoreProtocol) S3GetRequestHeaders(handler func() ([]*datastore_types.DataStoreKeyValue, uint32)) { + c.s3GetRequestHeadersHandler = handler +} + +// S3PostRequestHeaders sets the S3PostRequestHeaders handler function +func (c *CommonDataStoreProtocol) S3PostRequestHeaders(handler func() ([]*datastore_types.DataStoreKeyValue, uint32)) { + c.s3PostRequestHeadersHandler = handler +} + +// InitializeObjectByPreparePostParam sets the InitializeObjectByPreparePostParam handler function +func (c *CommonDataStoreProtocol) InitializeObjectByPreparePostParam(handler func(ownerPID uint32, param *datastore_types.DataStorePreparePostParam) (uint64, uint32)) { + c.initializeObjectByPreparePostParamHandler = handler +} + +// InitializeObjectRatingWithSlot sets the InitializeObjectRatingWithSlot handler function +func (c *CommonDataStoreProtocol) InitializeObjectRatingWithSlot(handler func(dataID uint64, param *datastore_types.DataStoreRatingInitParamWithSlot) uint32) { + c.initializeObjectRatingWithSlotHandler = handler +} + +// RateObjectWithPassword sets the RateObjectWithPassword handler function +func (c *CommonDataStoreProtocol) RateObjectWithPassword(handler func(dataID uint64, slot uint8, ratingValue int32, accessPassword uint64) (*datastore_types.DataStoreRatingInfo, uint32)) { + c.rateObjectWithPasswordHandler = handler +} + +func initDefault(c *CommonDataStoreProtocol) { + // TODO - Organize by method ID + c.DefaultProtocol = datastore.NewProtocol(c.server) + c.DefaultProtocol.GetMeta(getMeta) + c.DefaultProtocol.PreparePostObject(preparePostObject) + c.DefaultProtocol.PrepareGetObject(prepareGetObject) + c.DefaultProtocol.CompletePostObject(completePostObject) + c.DefaultProtocol.GetMetasMultipleParam(getMetasMultipleParam) + c.DefaultProtocol.ChangeMeta(changeMeta) + c.DefaultProtocol.RateObjects(rateObjects) +} + +func initSuperMarioMaker(c *CommonDataStoreProtocol) { + // TODO - Organize by method ID + c.SuperMarioMakerProtocol = datastore_super_mario_maker.NewProtocol(c.server) + c.SuperMarioMakerProtocol.GetMeta(getMeta) + c.SuperMarioMakerProtocol.PreparePostObject(preparePostObject) + c.SuperMarioMakerProtocol.PrepareGetObject(prepareGetObject) + c.SuperMarioMakerProtocol.CompletePostObject(completePostObject) + c.SuperMarioMakerProtocol.GetMetasMultipleParam(getMetasMultipleParam) + c.SuperMarioMakerProtocol.ChangeMeta(changeMeta) + c.SuperMarioMakerProtocol.RateObjects(rateObjects) +} + +// NewCommonDataStoreProtocol returns a new CommonDataStoreProtocol +func NewCommonDataStoreProtocol(server *nex.Server) *CommonDataStoreProtocol { + commonDataStoreProtocol = &CommonDataStoreProtocol{ + server: server, + rootCACert: []byte{}, + s3GetRequestHeadersHandler: func() ([]*datastore_types.DataStoreKeyValue, uint32) { + return []*datastore_types.DataStoreKeyValue{}, 0 + }, + s3PostRequestHeadersHandler: func() ([]*datastore_types.DataStoreKeyValue, uint32) { + return []*datastore_types.DataStoreKeyValue{}, 0 + }, + } + + patch := server.DataStoreProtocolVersion().GameSpecificPatch + + if strings.EqualFold(patch, "AMAJ") { + common_globals.Logger.Info("Using Super Mario Maker DataStore protocol") + initSuperMarioMaker(commonDataStoreProtocol) + } else { + if patch != "" { + common_globals.Logger.Infof("DataStore version patch %q not recognized", patch) + } + + common_globals.Logger.Info("Using default DataStore protocol") + initDefault(commonDataStoreProtocol) + } + + return commonDataStoreProtocol +} diff --git a/datastore/rate_objects.go b/datastore/rate_objects.go new file mode 100644 index 0000000..9415079 --- /dev/null +++ b/datastore/rate_objects.go @@ -0,0 +1,92 @@ +package datastore + +import ( + nex "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func rateObjects(err error, client *nex.Client, callID uint32, targets []*datastore_types.DataStoreRatingTarget, params []*datastore_types.DataStoreRateObjectParam, transactional bool, fetchRatings bool) uint32 { + if commonDataStoreProtocol.getObjectInfoByDataIDWithPasswordHandler == nil { + common_globals.Logger.Warning("GetObjectInfoByDataIDWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { + common_globals.Logger.Warning("VerifyObjectPermission not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.rateObjectWithPasswordHandler == nil { + common_globals.Logger.Warning("RateObjectWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + pRatings := make([]*datastore_types.DataStoreRatingInfo, 0) + pResults := make([]*nex.Result, 0) + + // * Real DataStore does not actually check this. + // * I just didn't feel like working out the + // * logic for differing sized lists. So force + // * them to always be the same + if len(targets) != len(params) { + return nex.Errors.DataStore.InvalidArgument + } + + for i := 0; i < len(targets); i++ { + target := targets[i] + param := params[i] + + objectInfo, errCode := commonDataStoreProtocol.getObjectInfoByDataIDWithPasswordHandler(target.DataID, param.AccessPassword) + if errCode != 0 { + return errCode + } + + errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(objectInfo.OwnerID, client.PID(), objectInfo.Permission) + if errCode != 0 { + return errCode + } + + rating, errCode := commonDataStoreProtocol.rateObjectWithPasswordHandler(target.DataID, target.Slot, param.RatingValue, param.AccessPassword) + if errCode != 0 { + return errCode + } + + if fetchRatings { + pRatings = append(pRatings, rating) + } + } + + rmcResponseStream := nex.NewStreamOut(commonDataStoreProtocol.server) + + rmcResponseStream.WriteListStructure(pRatings) + rmcResponseStream.WriteListResult(pResults) // * pResults is ALWAYS empty in SMM? + + rmcResponseBody := rmcResponseStream.Bytes() + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodRateObjects, rmcResponseBody) + + rmcResponseBytes := rmcResponse.Bytes() + + responsePacket, _ := nex.NewPacketV1(client, nil) + + responsePacket.SetVersion(1) + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} From adec20f92c6a2dd8fe15d99dd5710cb09b20b91e Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 01:08:11 -0400 Subject: [PATCH 02/25] added server PRUDP version checks for response packets --- datastore/change_meta.go | 11 +++++++++-- datastore/complete_post_object.go | 11 +++++++++-- datastore/get_meta.go | 11 +++++++++-- datastore/get_metas_multiple_param.go | 11 +++++++++-- datastore/prepare_get_object.go | 11 +++++++++-- datastore/prepare_post_object.go | 11 +++++++++-- datastore/rate_objects.go | 11 +++++++++-- 7 files changed, 63 insertions(+), 14 deletions(-) diff --git a/datastore/change_meta.go b/datastore/change_meta.go index d312e2a..07f15be 100644 --- a/datastore/change_meta.go +++ b/datastore/change_meta.go @@ -75,9 +75,16 @@ func changeMeta(err error, client *nex.Client, callID uint32, param *datastore_t rmcResponseBytes := rmcResponse.Bytes() - responsePacket, _ := nex.NewPacketV1(client, nil) + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } - responsePacket.SetVersion(1) responsePacket.SetSource(0xA1) responsePacket.SetDestination(0xAF) responsePacket.SetType(nex.DataPacket) diff --git a/datastore/complete_post_object.go b/datastore/complete_post_object.go index 7063351..7fab569 100644 --- a/datastore/complete_post_object.go +++ b/datastore/complete_post_object.go @@ -62,9 +62,16 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat rmcResponseBytes := rmcResponse.Bytes() - responsePacket, _ := nex.NewPacketV1(client, nil) + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } - responsePacket.SetVersion(1) responsePacket.SetSource(0xA1) responsePacket.SetDestination(0xAF) responsePacket.SetType(nex.DataPacket) diff --git a/datastore/get_meta.go b/datastore/get_meta.go index 785477d..7856642 100644 --- a/datastore/get_meta.go +++ b/datastore/get_meta.go @@ -78,9 +78,16 @@ func getMeta(err error, client *nex.Client, callID uint32, param *datastore_type rmcResponseBytes := rmcResponse.Bytes() - responsePacket, _ := nex.NewPacketV1(client, nil) + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } - responsePacket.SetVersion(1) responsePacket.SetSource(0xA1) responsePacket.SetDestination(0xAF) responsePacket.SetType(nex.DataPacket) diff --git a/datastore/get_metas_multiple_param.go b/datastore/get_metas_multiple_param.go index a7faf9a..81d0293 100644 --- a/datastore/get_metas_multiple_param.go +++ b/datastore/get_metas_multiple_param.go @@ -141,9 +141,16 @@ func getMetasMultipleParam(err error, client *nex.Client, callID uint32, params rmcResponseBytes := rmcResponse.Bytes() - responsePacket, _ := nex.NewPacketV1(client, nil) + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } - responsePacket.SetVersion(1) responsePacket.SetSource(0xA1) responsePacket.SetDestination(0xAF) responsePacket.SetType(nex.DataPacket) diff --git a/datastore/prepare_get_object.go b/datastore/prepare_get_object.go index b0d6e71..479080e 100644 --- a/datastore/prepare_get_object.go +++ b/datastore/prepare_get_object.go @@ -74,9 +74,16 @@ func prepareGetObject(err error, client *nex.Client, callID uint32, param *datas rmcResponseBytes := rmcResponse.Bytes() - responsePacket, _ := nex.NewPacketV1(client, nil) + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } - responsePacket.SetVersion(1) responsePacket.SetSource(0xA1) responsePacket.SetDestination(0xAF) responsePacket.SetType(nex.DataPacket) diff --git a/datastore/prepare_post_object.go b/datastore/prepare_post_object.go index 208fb87..6a2319e 100644 --- a/datastore/prepare_post_object.go +++ b/datastore/prepare_post_object.go @@ -87,9 +87,16 @@ func preparePostObject(err error, client *nex.Client, callID uint32, param *data rmcResponseBytes := rmcResponse.Bytes() - responsePacket, _ := nex.NewPacketV1(client, nil) + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } - responsePacket.SetVersion(1) responsePacket.SetSource(0xA1) responsePacket.SetDestination(0xAF) responsePacket.SetType(nex.DataPacket) diff --git a/datastore/rate_objects.go b/datastore/rate_objects.go index 9415079..23e3503 100644 --- a/datastore/rate_objects.go +++ b/datastore/rate_objects.go @@ -75,9 +75,16 @@ func rateObjects(err error, client *nex.Client, callID uint32, targets []*datast rmcResponseBytes := rmcResponse.Bytes() - responsePacket, _ := nex.NewPacketV1(client, nil) + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } - responsePacket.SetVersion(1) responsePacket.SetSource(0xA1) responsePacket.SetDestination(0xAF) responsePacket.SetType(nex.DataPacket) From 9953bced1bc36afe0dc6b5a53a6c137c79de185e Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 10:11:39 -0400 Subject: [PATCH 03/25] removed method order todo comment. already in ID order --- datastore/protocol.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/datastore/protocol.go b/datastore/protocol.go index 18a96d4..056f9ac 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -136,7 +136,6 @@ func (c *CommonDataStoreProtocol) RateObjectWithPassword(handler func(dataID uin } func initDefault(c *CommonDataStoreProtocol) { - // TODO - Organize by method ID c.DefaultProtocol = datastore.NewProtocol(c.server) c.DefaultProtocol.GetMeta(getMeta) c.DefaultProtocol.PreparePostObject(preparePostObject) @@ -148,7 +147,6 @@ func initDefault(c *CommonDataStoreProtocol) { } func initSuperMarioMaker(c *CommonDataStoreProtocol) { - // TODO - Organize by method ID c.SuperMarioMakerProtocol = datastore_super_mario_maker.NewProtocol(c.server) c.SuperMarioMakerProtocol.GetMeta(getMeta) c.SuperMarioMakerProtocol.PreparePostObject(preparePostObject) From 554491872cf15c6d69053e746b3c331487b89a4a Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 10:13:00 -0400 Subject: [PATCH 04/25] added CompletePostObjects --- datastore/complete_post_objects.go | 84 ++++++++++++++++++++++++++++++ datastore/protocol.go | 2 + 2 files changed, 86 insertions(+) create mode 100644 datastore/complete_post_objects.go diff --git a/datastore/complete_post_objects.go b/datastore/complete_post_objects.go new file mode 100644 index 0000000..bd0f25d --- /dev/null +++ b/datastore/complete_post_objects.go @@ -0,0 +1,84 @@ +package datastore + +import ( + "fmt" + + "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" +) + +func completePostObjects(err error, client *nex.Client, callID uint32, dataIDs []uint64) uint32 { + if commonDataStoreProtocol.s3ObjectSizeHandler == nil { + common_globals.Logger.Warning("S3ObjectSize not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.getObjectSizeDataIDHandler == nil { + common_globals.Logger.Warning("GetObjectSizeDataID not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.updateObjectUploadCompletedByDataIDHandler == nil { + common_globals.Logger.Warning("UpdateObjectUploadCompletedByDataID not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + for _, dataID := range dataIDs { + bucket := commonDataStoreProtocol.s3Bucket + key := fmt.Sprintf("%d.bin", dataID) + + objectSizeS3, err := commonDataStoreProtocol.s3ObjectSizeHandler(bucket, key) + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.NotFound + } + + objectSizeDB, errCode := commonDataStoreProtocol.getObjectSizeDataIDHandler(dataID) + if errCode != 0 { + return errCode + } + + if objectSizeS3 != uint64(objectSizeDB) { + // TODO - Is this a good error? + return nex.Errors.DataStore.Unknown + } + + errCode = commonDataStoreProtocol.updateObjectUploadCompletedByDataIDHandler(dataID, true) + if errCode != 0 { + return errCode + } + } + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodCompletePostObjects, nil) + + rmcResponseBytes := rmcResponse.Bytes() + + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } + + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} diff --git a/datastore/protocol.go b/datastore/protocol.go index 056f9ac..2bdc0d9 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -142,6 +142,7 @@ func initDefault(c *CommonDataStoreProtocol) { c.DefaultProtocol.PrepareGetObject(prepareGetObject) c.DefaultProtocol.CompletePostObject(completePostObject) c.DefaultProtocol.GetMetasMultipleParam(getMetasMultipleParam) + c.DefaultProtocol.CompletePostObjects(completePostObjects) c.DefaultProtocol.ChangeMeta(changeMeta) c.DefaultProtocol.RateObjects(rateObjects) } @@ -153,6 +154,7 @@ func initSuperMarioMaker(c *CommonDataStoreProtocol) { c.SuperMarioMakerProtocol.PrepareGetObject(prepareGetObject) c.SuperMarioMakerProtocol.CompletePostObject(completePostObject) c.SuperMarioMakerProtocol.GetMetasMultipleParam(getMetasMultipleParam) + c.SuperMarioMakerProtocol.CompletePostObjects(completePostObjects) c.SuperMarioMakerProtocol.ChangeMeta(changeMeta) c.SuperMarioMakerProtocol.RateObjects(rateObjects) } From cb8f1fb5c01b9962185f9899d1df90aa257ee3c9 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 10:25:14 -0400 Subject: [PATCH 05/25] added DeleteObject --- datastore/delete_object.go | 65 ++++++++++++++++++++++++++++++++++++++ datastore/protocol.go | 8 +++++ 2 files changed, 73 insertions(+) create mode 100644 datastore/delete_object.go diff --git a/datastore/delete_object.go b/datastore/delete_object.go new file mode 100644 index 0000000..63faf2a --- /dev/null +++ b/datastore/delete_object.go @@ -0,0 +1,65 @@ +package datastore + +import ( + "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func deleteObject(err error, client *nex.Client, callID uint32, param *datastore_types.DataStoreDeleteParam) uint32 { + if commonDataStoreProtocol.getObjectInfoByDataIDHandler == nil { + common_globals.Logger.Warning("GetObjectInfoByDataID not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { + common_globals.Logger.Warning("VerifyObjectPermission not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.deleteObjectByDataIDWithPasswordHandler == nil { + common_globals.Logger.Warning("DeleteObjectByDataIDWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + metaInfo, errCode := commonDataStoreProtocol.getObjectInfoByDataIDHandler(param.DataID) + if errCode != 0 { + return errCode + } + + errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(metaInfo.OwnerID, client.PID(), metaInfo.DelPermission) + if errCode != 0 { + return errCode + } + + errCode = commonDataStoreProtocol.deleteObjectByDataIDWithPasswordHandler(param.DataID, param.UpdatePassword) + if errCode != 0 { + return errCode + } + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodDeleteObject, nil) + + rmcResponseBytes := rmcResponse.Bytes() + + responsePacket, _ := nex.NewPacketV1(client, nil) + + responsePacket.SetVersion(1) + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} diff --git a/datastore/protocol.go b/datastore/protocol.go index 2bdc0d9..e59494b 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -38,6 +38,7 @@ type CommonDataStoreProtocol struct { initializeObjectByPreparePostParamHandler func(ownerPID uint32, param *datastore_types.DataStorePreparePostParam) (uint64, uint32) initializeObjectRatingWithSlotHandler func(dataID uint64, param *datastore_types.DataStoreRatingInitParamWithSlot) uint32 rateObjectWithPasswordHandler func(dataID uint64, slot uint8, ratingValue int32, accessPassword uint64) (*datastore_types.DataStoreRatingInfo, uint32) + deleteObjectByDataIDWithPasswordHandler func(dataID uint64, password uint64) uint32 } // SetS3Bucket sets the S3 bucket @@ -135,8 +136,14 @@ func (c *CommonDataStoreProtocol) RateObjectWithPassword(handler func(dataID uin c.rateObjectWithPasswordHandler = handler } +// DeleteObjectByDataIDWithPassword sets the DeleteObjectByDataIDWithPassword handler function +func (c *CommonDataStoreProtocol) DeleteObjectByDataIDWithPassword(handler func(dataID uint64, password uint64) uint32) { + c.deleteObjectByDataIDWithPasswordHandler = handler +} + func initDefault(c *CommonDataStoreProtocol) { c.DefaultProtocol = datastore.NewProtocol(c.server) + c.DefaultProtocol.DeleteObject(deleteObject) c.DefaultProtocol.GetMeta(getMeta) c.DefaultProtocol.PreparePostObject(preparePostObject) c.DefaultProtocol.PrepareGetObject(prepareGetObject) @@ -149,6 +156,7 @@ func initDefault(c *CommonDataStoreProtocol) { func initSuperMarioMaker(c *CommonDataStoreProtocol) { c.SuperMarioMakerProtocol = datastore_super_mario_maker.NewProtocol(c.server) + c.SuperMarioMakerProtocol.DeleteObject(deleteObject) c.SuperMarioMakerProtocol.GetMeta(getMeta) c.SuperMarioMakerProtocol.PreparePostObject(preparePostObject) c.SuperMarioMakerProtocol.PrepareGetObject(prepareGetObject) From 85ac1b29c1d26610522c33f25fd7eaa2bf474257 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 11:03:57 -0400 Subject: [PATCH 06/25] added GetMetas --- datastore/get_metas.go | 159 +++++++++++++++++++++++++++++++++++++++++ datastore/protocol.go | 2 + 2 files changed, 161 insertions(+) create mode 100644 datastore/get_metas.go diff --git a/datastore/get_metas.go b/datastore/get_metas.go new file mode 100644 index 0000000..0d27388 --- /dev/null +++ b/datastore/get_metas.go @@ -0,0 +1,159 @@ +package datastore + +import ( + "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func getMetas(err error, client *nex.Client, callID uint32, dataIDs []uint64, param *datastore_types.DataStoreGetMetaParam) uint32 { + if commonDataStoreProtocol.getObjectInfoByDataIDHandler == nil { + common_globals.Logger.Warning("GetObjectInfoByDataID not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { + common_globals.Logger.Warning("VerifyObjectPermission not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + // TODO - Verify if param.PersistenceTarget is respected? It wouldn't make sense here but who knows + + pMetaInfo := make([]*datastore_types.DataStoreMetaInfo, 0, len(dataIDs)) + pResults := make([]*nex.Result, 0, len(dataIDs)) + + // * param has an AccessPassword, but it goes unchecked here. + // * The password would need to be the same for every object + // * in the input array, which doesn't make any sense. Assuming + // * it's unused until proven otherwise + + for i := 0; i < len(dataIDs); i++ { + objectInfo, errCode := commonDataStoreProtocol.getObjectInfoByDataIDHandler(dataIDs[i]) + + if errCode != 0 { + // TODO - Maybe this should be broken out into a util function in globals? + objectInfo = datastore_types.NewDataStoreMetaInfo() + objectInfo.DataID = 0 + objectInfo.OwnerID = 0 + objectInfo.Size = 0 + objectInfo.Name = "" + objectInfo.DataType = 0 + objectInfo.MetaBinary = []byte{} + objectInfo.Permission = datastore_types.NewDataStorePermission() + objectInfo.Permission.Permission = 0 + objectInfo.Permission.RecipientIDs = []uint32{} + objectInfo.DelPermission = datastore_types.NewDataStorePermission() + objectInfo.DelPermission.Permission = 0 + objectInfo.DelPermission.RecipientIDs = []uint32{} + objectInfo.CreatedTime = nex.NewDateTime(0) + objectInfo.UpdatedTime = nex.NewDateTime(0) + objectInfo.Period = 0 + objectInfo.Status = 0 + objectInfo.ReferredCnt = 0 + objectInfo.ReferDataID = 0 + objectInfo.Flag = 0 + objectInfo.ReferredTime = nex.NewDateTime(0) + objectInfo.ExpireTime = nex.NewDateTime(0) + objectInfo.Tags = []string{} + objectInfo.Ratings = []*datastore_types.DataStoreRatingInfoWithSlot{} + + pResults = append(pResults, nex.NewResultError(errCode)) + } else { + errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(objectInfo.OwnerID, client.PID(), objectInfo.Permission) + if errCode != 0 { + // TODO - Maybe this should be broken out into a util function in globals? + objectInfo = datastore_types.NewDataStoreMetaInfo() + objectInfo.DataID = 0 + objectInfo.OwnerID = 0 + objectInfo.Size = 0 + objectInfo.Name = "" + objectInfo.DataType = 0 + objectInfo.MetaBinary = []byte{} + objectInfo.Permission = datastore_types.NewDataStorePermission() + objectInfo.Permission.Permission = 0 + objectInfo.Permission.RecipientIDs = []uint32{} + objectInfo.DelPermission = datastore_types.NewDataStorePermission() + objectInfo.DelPermission.Permission = 0 + objectInfo.DelPermission.RecipientIDs = []uint32{} + objectInfo.CreatedTime = nex.NewDateTime(0) + objectInfo.UpdatedTime = nex.NewDateTime(0) + objectInfo.Period = 0 + objectInfo.Status = 0 + objectInfo.ReferredCnt = 0 + objectInfo.ReferDataID = 0 + objectInfo.Flag = 0 + objectInfo.ReferredTime = nex.NewDateTime(0) + objectInfo.ExpireTime = nex.NewDateTime(0) + objectInfo.Tags = []string{} + objectInfo.Ratings = []*datastore_types.DataStoreRatingInfoWithSlot{} + + pResults = append(pResults, nex.NewResultError(errCode)) + } else { + pResults = append(pResults, nex.NewResultSuccess(nex.Errors.DataStore.Unknown)) + } + + // * This is kind of backwards. + // * The database pulls this data + // * by default, so it can be done + // * in a single query. So instead + // * of checking if a flag *IS* + // * set, and conditionally *ADDING* + // * the fields, we check if a flag + // * is *NOT* set and conditionally + // * *REMOVE* the field + if param.ResultOption&0x1 == 0 { + objectInfo.Tags = make([]string, 0) + } + + if param.ResultOption&0x2 == 0 { + objectInfo.Ratings = make([]*datastore_types.DataStoreRatingInfoWithSlot, 0) + } + + if param.ResultOption&0x4 == 0 { + objectInfo.MetaBinary = make([]byte, 0) + } + } + + pMetaInfo = append(pMetaInfo, objectInfo) + } + + rmcResponseStream := nex.NewStreamOut(commonDataStoreProtocol.server) + + rmcResponseStream.WriteListStructure(pMetaInfo) + rmcResponseStream.WriteListResult(pResults) + + rmcResponseBody := rmcResponseStream.Bytes() + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodGetMetas, rmcResponseBody) + + rmcResponseBytes := rmcResponse.Bytes() + + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } + + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} diff --git a/datastore/protocol.go b/datastore/protocol.go index e59494b..053de83 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -145,6 +145,7 @@ func initDefault(c *CommonDataStoreProtocol) { c.DefaultProtocol = datastore.NewProtocol(c.server) c.DefaultProtocol.DeleteObject(deleteObject) c.DefaultProtocol.GetMeta(getMeta) + c.DefaultProtocol.GetMetas(getMetas) c.DefaultProtocol.PreparePostObject(preparePostObject) c.DefaultProtocol.PrepareGetObject(prepareGetObject) c.DefaultProtocol.CompletePostObject(completePostObject) @@ -158,6 +159,7 @@ func initSuperMarioMaker(c *CommonDataStoreProtocol) { c.SuperMarioMakerProtocol = datastore_super_mario_maker.NewProtocol(c.server) c.SuperMarioMakerProtocol.DeleteObject(deleteObject) c.SuperMarioMakerProtocol.GetMeta(getMeta) + c.SuperMarioMakerProtocol.GetMetas(getMetas) c.SuperMarioMakerProtocol.PreparePostObject(preparePostObject) c.SuperMarioMakerProtocol.PrepareGetObject(prepareGetObject) c.SuperMarioMakerProtocol.CompletePostObject(completePostObject) From 823830b36c7981e9e52f235d9d6625aa54e8f7f4 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 11:04:32 -0400 Subject: [PATCH 07/25] DeleteObject was always using PRUDPv1 --- datastore/delete_object.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/datastore/delete_object.go b/datastore/delete_object.go index 63faf2a..6322c71 100644 --- a/datastore/delete_object.go +++ b/datastore/delete_object.go @@ -48,9 +48,16 @@ func deleteObject(err error, client *nex.Client, callID uint32, param *datastore rmcResponseBytes := rmcResponse.Bytes() - responsePacket, _ := nex.NewPacketV1(client, nil) + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } - responsePacket.SetVersion(1) responsePacket.SetSource(0xA1) responsePacket.SetDestination(0xAF) responsePacket.SetType(nex.DataPacket) From b643ac5b31ab352e5b10623a9bb219ff73e21b49 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 11:08:48 -0400 Subject: [PATCH 08/25] added TODO comment to PreparePostObject --- datastore/prepare_post_object.go | 1 + 1 file changed, 1 insertion(+) diff --git a/datastore/prepare_post_object.go b/datastore/prepare_post_object.go index 6a2319e..c0ea5ce 100644 --- a/datastore/prepare_post_object.go +++ b/datastore/prepare_post_object.go @@ -31,6 +31,7 @@ func preparePostObject(err error, client *nex.Client, callID uint32, param *data return nex.Errors.DataStore.Unknown } + // TODO - Need to verify what param.PersistenceInitParam.DeleteLastObject really means. It's often set to true even when it wouldn't make sense dataID, errCode := commonDataStoreProtocol.initializeObjectByPreparePostParamHandler(client.PID(), param) if errCode != 0 { common_globals.Logger.Errorf("Error code %d on object init", errCode) From a2c1d4f70421c2a77c3085f6547638e975ddd2fb Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 11:15:38 -0400 Subject: [PATCH 09/25] added PostMetaBinary --- datastore/post_meta_binary.go | 81 +++++++++++++++++++++++++++++++++++ datastore/protocol.go | 2 + 2 files changed, 83 insertions(+) create mode 100644 datastore/post_meta_binary.go diff --git a/datastore/post_meta_binary.go b/datastore/post_meta_binary.go new file mode 100644 index 0000000..88fa835 --- /dev/null +++ b/datastore/post_meta_binary.go @@ -0,0 +1,81 @@ +package datastore + +import ( + "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func postMetaBinary(err error, client *nex.Client, callID uint32, param *datastore_types.DataStorePreparePostParam) uint32 { + // * This method looks to function identically to DataStore::PreparePostObject, + // * except the only difference being it doesn't return an S3 upload URL. This + // * needs to be verified though, as there are other methods in the family such + // * as DataStore::PostMetaBinaryWithDataID which make less sense in this context, + // * unless those are just used to *update* a meta binary? Or maybe the DataID in + // * those methods is a pre-allocated DataID from the server? Needs more testing + + if commonDataStoreProtocol.initializeObjectByPreparePostParamHandler == nil { + common_globals.Logger.Warning("InitializeObjectByPreparePostParam not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.initializeObjectRatingWithSlotHandler == nil { + common_globals.Logger.Warning("InitializeObjectRatingWithSlot not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + // TODO - Need to verify what param.PersistenceInitParam.DeleteLastObject really means. It's often set to true even when it wouldn't make sense + dataID, errCode := commonDataStoreProtocol.initializeObjectByPreparePostParamHandler(client.PID(), param) + if errCode != 0 { + common_globals.Logger.Errorf("Error code %d on object init", errCode) + return errCode + } + + // TODO - Should this be moved to InitializeObjectByPreparePostParam? + for _, ratingInitParamWithSlot := range param.RatingInitParams { + errCode = commonDataStoreProtocol.initializeObjectRatingWithSlotHandler(dataID, ratingInitParamWithSlot) + if errCode != 0 { + common_globals.Logger.Errorf("Error code %d on rating init", errCode) + return errCode + } + } + + rmcResponseStream := nex.NewStreamOut(commonDataStoreProtocol.server) + + rmcResponseStream.WriteUInt64LE(uint64(dataID)) + + rmcResponseBody := rmcResponseStream.Bytes() + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodPostMetaBinary, rmcResponseBody) + + rmcResponseBytes := rmcResponse.Bytes() + + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } + + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} diff --git a/datastore/protocol.go b/datastore/protocol.go index 053de83..4e83d46 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -146,6 +146,7 @@ func initDefault(c *CommonDataStoreProtocol) { c.DefaultProtocol.DeleteObject(deleteObject) c.DefaultProtocol.GetMeta(getMeta) c.DefaultProtocol.GetMetas(getMetas) + c.DefaultProtocol.PostMetaBinary(postMetaBinary) c.DefaultProtocol.PreparePostObject(preparePostObject) c.DefaultProtocol.PrepareGetObject(prepareGetObject) c.DefaultProtocol.CompletePostObject(completePostObject) @@ -160,6 +161,7 @@ func initSuperMarioMaker(c *CommonDataStoreProtocol) { c.SuperMarioMakerProtocol.DeleteObject(deleteObject) c.SuperMarioMakerProtocol.GetMeta(getMeta) c.SuperMarioMakerProtocol.GetMetas(getMetas) + c.SuperMarioMakerProtocol.PostMetaBinary(postMetaBinary) c.SuperMarioMakerProtocol.PreparePostObject(preparePostObject) c.SuperMarioMakerProtocol.PrepareGetObject(prepareGetObject) c.SuperMarioMakerProtocol.CompletePostObject(completePostObject) From c3d36833d79b30bb7f6d0e9c7f5609b95dc3d3b7 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 11:23:14 -0400 Subject: [PATCH 10/25] added RateObject --- datastore/protocol.go | 2 + datastore/rate_object.go | 86 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 datastore/rate_object.go diff --git a/datastore/protocol.go b/datastore/protocol.go index 4e83d46..8a8ebb5 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -146,6 +146,7 @@ func initDefault(c *CommonDataStoreProtocol) { c.DefaultProtocol.DeleteObject(deleteObject) c.DefaultProtocol.GetMeta(getMeta) c.DefaultProtocol.GetMetas(getMetas) + c.DefaultProtocol.RateObject(rateObject) c.DefaultProtocol.PostMetaBinary(postMetaBinary) c.DefaultProtocol.PreparePostObject(preparePostObject) c.DefaultProtocol.PrepareGetObject(prepareGetObject) @@ -161,6 +162,7 @@ func initSuperMarioMaker(c *CommonDataStoreProtocol) { c.SuperMarioMakerProtocol.DeleteObject(deleteObject) c.SuperMarioMakerProtocol.GetMeta(getMeta) c.SuperMarioMakerProtocol.GetMetas(getMetas) + c.SuperMarioMakerProtocol.RateObject(rateObject) c.SuperMarioMakerProtocol.PostMetaBinary(postMetaBinary) c.SuperMarioMakerProtocol.PreparePostObject(preparePostObject) c.SuperMarioMakerProtocol.PrepareGetObject(prepareGetObject) diff --git a/datastore/rate_object.go b/datastore/rate_object.go new file mode 100644 index 0000000..b897cce --- /dev/null +++ b/datastore/rate_object.go @@ -0,0 +1,86 @@ +package datastore + +import ( + "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func rateObject(err error, client *nex.Client, callID uint32, target *datastore_types.DataStoreRatingTarget, param *datastore_types.DataStoreRateObjectParam, fetchRatings bool) uint32 { + if commonDataStoreProtocol.getObjectInfoByDataIDWithPasswordHandler == nil { + common_globals.Logger.Warning("GetObjectInfoByDataIDWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { + common_globals.Logger.Warning("VerifyObjectPermission not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.rateObjectWithPasswordHandler == nil { + common_globals.Logger.Warning("RateObjectWithPassword not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + objectInfo, errCode := commonDataStoreProtocol.getObjectInfoByDataIDWithPasswordHandler(target.DataID, param.AccessPassword) + if errCode != 0 { + return errCode + } + + errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(objectInfo.OwnerID, client.PID(), objectInfo.Permission) + if errCode != 0 { + return errCode + } + + pRating, errCode := commonDataStoreProtocol.rateObjectWithPasswordHandler(target.DataID, target.Slot, param.RatingValue, param.AccessPassword) + if errCode != 0 { + return errCode + } + + // * This is kinda backwards. Server returns + // * the rating by default, so we check if + // * the client DOESN'T want it and then just + // * zero it out + if !fetchRatings { + pRating = datastore_types.NewDataStoreRatingInfo() + } + + rmcResponseStream := nex.NewStreamOut(commonDataStoreProtocol.server) + + rmcResponseStream.WriteStructure(pRating) + + rmcResponseBody := rmcResponseStream.Bytes() + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodRateObject, rmcResponseBody) + + rmcResponseBytes := rmcResponse.Bytes() + + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } + + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} From 3bbd382549f175925a6c5640c0c5267923392822 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 11:42:34 -0400 Subject: [PATCH 11/25] added SearchObject --- datastore/protocol.go | 8 +++ datastore/search_object.go | 106 +++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 datastore/search_object.go diff --git a/datastore/protocol.go b/datastore/protocol.go index 8a8ebb5..6feca95 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -39,6 +39,7 @@ type CommonDataStoreProtocol struct { initializeObjectRatingWithSlotHandler func(dataID uint64, param *datastore_types.DataStoreRatingInitParamWithSlot) uint32 rateObjectWithPasswordHandler func(dataID uint64, slot uint8, ratingValue int32, accessPassword uint64) (*datastore_types.DataStoreRatingInfo, uint32) deleteObjectByDataIDWithPasswordHandler func(dataID uint64, password uint64) uint32 + getObjectInfosByDataStoreSearchParamHandler func(param *datastore_types.DataStoreSearchParam) ([]*datastore_types.DataStoreMetaInfo, uint32, uint32) } // SetS3Bucket sets the S3 bucket @@ -141,11 +142,17 @@ func (c *CommonDataStoreProtocol) DeleteObjectByDataIDWithPassword(handler func( c.deleteObjectByDataIDWithPasswordHandler = handler } +// GetObjectInfosByDataStoreSearchParam sets the GetObjectInfosByDataStoreSearchParam handler function +func (c *CommonDataStoreProtocol) GetObjectInfosByDataStoreSearchParam(handler func(param *datastore_types.DataStoreSearchParam) ([]*datastore_types.DataStoreMetaInfo, uint32, uint32)) { + c.getObjectInfosByDataStoreSearchParamHandler = handler +} + func initDefault(c *CommonDataStoreProtocol) { c.DefaultProtocol = datastore.NewProtocol(c.server) c.DefaultProtocol.DeleteObject(deleteObject) c.DefaultProtocol.GetMeta(getMeta) c.DefaultProtocol.GetMetas(getMetas) + c.DefaultProtocol.SearchObject(searchObject) c.DefaultProtocol.RateObject(rateObject) c.DefaultProtocol.PostMetaBinary(postMetaBinary) c.DefaultProtocol.PreparePostObject(preparePostObject) @@ -162,6 +169,7 @@ func initSuperMarioMaker(c *CommonDataStoreProtocol) { c.SuperMarioMakerProtocol.DeleteObject(deleteObject) c.SuperMarioMakerProtocol.GetMeta(getMeta) c.SuperMarioMakerProtocol.GetMetas(getMetas) + c.SuperMarioMakerProtocol.SearchObject(searchObject) c.SuperMarioMakerProtocol.RateObject(rateObject) c.SuperMarioMakerProtocol.PostMetaBinary(postMetaBinary) c.SuperMarioMakerProtocol.PreparePostObject(preparePostObject) diff --git a/datastore/search_object.go b/datastore/search_object.go new file mode 100644 index 0000000..faec2a0 --- /dev/null +++ b/datastore/search_object.go @@ -0,0 +1,106 @@ +package datastore + +import ( + "github.com/PretendoNetwork/nex-go" + common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" + datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" + datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" +) + +func searchObject(err error, client *nex.Client, callID uint32, param *datastore_types.DataStoreSearchParam) uint32 { + if commonDataStoreProtocol.getObjectInfosByDataStoreSearchParamHandler == nil { + common_globals.Logger.Warning("GetObjectInfosByDataStoreSearchParam not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { + common_globals.Logger.Warning("VerifyObjectPermission not defined") + return nex.Errors.Core.NotImplemented + } + + if err != nil { + common_globals.Logger.Error(err.Error()) + return nex.Errors.DataStore.Unknown + } + + // * This is likely game-specific. Also developer note: + // * Please keep in mind that no results is allowed. errCode + // * should NEVER be DataStore::NotFound! + // * + // * DataStoreSearchParam contains a ResultRange to limit the + // * returned results. TotalCount is the total matching objects + // * in the database, whereas objects is the limited results + objects, totalCount, errCode := commonDataStoreProtocol.getObjectInfosByDataStoreSearchParamHandler(param) + if errCode != 0 { + return errCode + } + + pSearchResult := datastore_types.NewDataStoreSearchResult() + + pSearchResult.TotalCount = totalCount + pSearchResult.Result = make([]*datastore_types.DataStoreMetaInfo, 0, len(objects)) + pSearchResult.TotalCountType = uint8(param.DataType) // TODO - Is this right? + + for _, object := range objects { + errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(object.OwnerID, client.PID(), object.Permission) + if errCode != 0 { + continue + } + + // * This is kind of backwards. + // * The database pulls this data + // * by default, so it can be done + // * in a single query. So instead + // * of checking if a flag *IS* + // * set, and conditionally *ADDING* + // * the fields, we check if a flag + // * is *NOT* set and conditionally + // * *REMOVE* the field + if param.ResultOption&0x1 == 0 { + object.Tags = make([]string, 0) + } + + if param.ResultOption&0x2 == 0 { + object.Ratings = make([]*datastore_types.DataStoreRatingInfoWithSlot, 0) + } + + if param.ResultOption&0x4 == 0 { + object.MetaBinary = make([]byte, 0) + } + + pSearchResult.Result = append(pSearchResult.Result, object) + } + + rmcResponseStream := nex.NewStreamOut(commonDataStoreProtocol.server) + + rmcResponseStream.WriteStructure(pSearchResult) + + rmcResponseBody := rmcResponseStream.Bytes() + + rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) + rmcResponse.SetSuccess(datastore.MethodSearchObject, rmcResponseBody) + + rmcResponseBytes := rmcResponse.Bytes() + + var responsePacket nex.PacketInterface + + if commonDataStoreProtocol.server.PRUDPVersion() == 0 { + responsePacket, _ = nex.NewPacketV0(client, nil) + responsePacket.SetVersion(0) + } else { + responsePacket, _ = nex.NewPacketV1(client, nil) + responsePacket.SetVersion(1) + } + + responsePacket.SetSource(0xA1) + responsePacket.SetDestination(0xAF) + responsePacket.SetType(nex.DataPacket) + responsePacket.SetPayload(rmcResponseBytes) + + responsePacket.AddFlag(nex.FlagNeedsAck) + responsePacket.AddFlag(nex.FlagReliable) + + commonDataStoreProtocol.server.Send(responsePacket) + + return 0 +} From cdfa4a68c3bc82ab435ba15f2817d6db584eac3f Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 14:08:01 -0400 Subject: [PATCH 12/25] remove TODO and delete unsuccessful objects in CompletePostObject --- datastore/complete_post_object.go | 11 ++++++++++- datastore/protocol.go | 6 ++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/datastore/complete_post_object.go b/datastore/complete_post_object.go index 7fab569..23ddc61 100644 --- a/datastore/complete_post_object.go +++ b/datastore/complete_post_object.go @@ -25,12 +25,16 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat return nex.Errors.Core.NotImplemented } + if commonDataStoreProtocol.deleteObjectByDataIDHandler == nil { + common_globals.Logger.Warning("DeleteObjectByDataIDHandler not defined") + return nex.Errors.Core.NotImplemented + } + if err != nil { common_globals.Logger.Error(err.Error()) return nex.Errors.DataStore.Unknown } - // TODO - What is param.IsSuccess? Is this correct? if param.IsSuccess { bucket := commonDataStoreProtocol.s3Bucket key := fmt.Sprintf("%d.bin", param.DataID) @@ -55,6 +59,11 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat if errCode != 0 { return errCode } + } else { + errCode := commonDataStoreProtocol.deleteObjectByDataIDHandler(param.DataID) + if errCode != 0 { + return errCode + } } rmcResponse := nex.NewRMCResponse(datastore.ProtocolID, callID) diff --git a/datastore/protocol.go b/datastore/protocol.go index 6feca95..faf15a6 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -39,6 +39,7 @@ type CommonDataStoreProtocol struct { initializeObjectRatingWithSlotHandler func(dataID uint64, param *datastore_types.DataStoreRatingInitParamWithSlot) uint32 rateObjectWithPasswordHandler func(dataID uint64, slot uint8, ratingValue int32, accessPassword uint64) (*datastore_types.DataStoreRatingInfo, uint32) deleteObjectByDataIDWithPasswordHandler func(dataID uint64, password uint64) uint32 + deleteObjectByDataIDHandler func(dataID uint64) uint32 getObjectInfosByDataStoreSearchParamHandler func(param *datastore_types.DataStoreSearchParam) ([]*datastore_types.DataStoreMetaInfo, uint32, uint32) } @@ -142,6 +143,11 @@ func (c *CommonDataStoreProtocol) DeleteObjectByDataIDWithPassword(handler func( c.deleteObjectByDataIDWithPasswordHandler = handler } +// DeleteObjectByDataID sets the DeleteObjectByDataID handler function +func (c *CommonDataStoreProtocol) DeleteObjectByDataID(handler func(dataID uint64) uint32) { + c.deleteObjectByDataIDHandler = handler +} + // GetObjectInfosByDataStoreSearchParam sets the GetObjectInfosByDataStoreSearchParam handler function func (c *CommonDataStoreProtocol) GetObjectInfosByDataStoreSearchParam(handler func(param *datastore_types.DataStoreSearchParam) ([]*datastore_types.DataStoreMetaInfo, uint32, uint32)) { c.getObjectInfosByDataStoreSearchParamHandler = handler From fc7039b169a7f6bb5c9b697a97b19f0688317938 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 14:10:26 -0400 Subject: [PATCH 13/25] log an error when upload sizes dont match --- datastore/complete_post_object.go | 1 + datastore/complete_post_objects.go | 1 + 2 files changed, 2 insertions(+) diff --git a/datastore/complete_post_object.go b/datastore/complete_post_object.go index 23ddc61..03ce7e1 100644 --- a/datastore/complete_post_object.go +++ b/datastore/complete_post_object.go @@ -51,6 +51,7 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat } if objectSizeS3 != uint64(objectSizeDB) { + common_globals.Logger.Errorf("Object with DataID %d did not upload correctly! Mismatched sizes", param.DataID) // TODO - Is this a good error? return nex.Errors.DataStore.Unknown } diff --git a/datastore/complete_post_objects.go b/datastore/complete_post_objects.go index bd0f25d..310f6c3 100644 --- a/datastore/complete_post_objects.go +++ b/datastore/complete_post_objects.go @@ -45,6 +45,7 @@ func completePostObjects(err error, client *nex.Client, callID uint32, dataIDs [ } if objectSizeS3 != uint64(objectSizeDB) { + common_globals.Logger.Errorf("Object with DataID %d did not upload correctly! Mismatched sizes", dataID) // TODO - Is this a good error? return nex.Errors.DataStore.Unknown } From 014ad34eeaf3d36cd95eda8420b066f8881530d4 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 14:11:22 -0400 Subject: [PATCH 14/25] URL to url in PrepareGetObject --- datastore/prepare_get_object.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore/prepare_get_object.go b/datastore/prepare_get_object.go index 479080e..615c996 100644 --- a/datastore/prepare_get_object.go +++ b/datastore/prepare_get_object.go @@ -44,7 +44,7 @@ func prepareGetObject(err error, client *nex.Client, callID uint32, param *datas return errCode } - URL, err := commonDataStoreProtocol.presignGetObjectHandler(bucket, key, time.Minute*15) + url, err := commonDataStoreProtocol.presignGetObjectHandler(bucket, key, time.Minute*15) if err != nil { common_globals.Logger.Error(err.Error()) return nex.Errors.DataStore.OperationNotAllowed @@ -57,7 +57,7 @@ func prepareGetObject(err error, client *nex.Client, callID uint32, param *datas pReqGetInfo := datastore_types.NewDataStoreReqGetInfo() - pReqGetInfo.URL = URL.String() + pReqGetInfo.URL = url.String() pReqGetInfo.RequestHeaders = requestHeaders pReqGetInfo.Size = objectInfo.Size pReqGetInfo.RootCACert = commonDataStoreProtocol.rootCACert From e143a8cc2443f134053d6f54c6685cafa5bedc4d Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 14:21:47 -0400 Subject: [PATCH 15/25] properly handle TotalCount and TotalCountType in SearchObject --- datastore/search_object.go | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/datastore/search_object.go b/datastore/search_object.go index faec2a0..d0120fd 100644 --- a/datastore/search_object.go +++ b/datastore/search_object.go @@ -37,13 +37,14 @@ func searchObject(err error, client *nex.Client, callID uint32, param *datastore pSearchResult := datastore_types.NewDataStoreSearchResult() - pSearchResult.TotalCount = totalCount pSearchResult.Result = make([]*datastore_types.DataStoreMetaInfo, 0, len(objects)) - pSearchResult.TotalCountType = uint8(param.DataType) // TODO - Is this right? for _, object := range objects { errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(object.OwnerID, client.PID(), object.Permission) if errCode != 0 { + // * Since we don't error here, should we also + // * "hide" these results by also decrementing + // * totalCount? continue } @@ -71,6 +72,32 @@ func searchObject(err error, client *nex.Client, callID uint32, param *datastore pSearchResult.Result = append(pSearchResult.Result, object) } + var totalCountType uint8 + + // * Doing this here since the object + // * the permissions checks in the + // * previous loop will mutate the data + // * returned from the database + if totalCount == uint32(len(pSearchResult.Result)) { + totalCountType = 0 // * Has no more data. All possible results were returned + } else { + totalCountType = 1 // * Has more data. Not all possible results were returned + } + + // * Disables the TotalCount + // * + // * Only seen in struct revision 3 or + // * NEX 4.0+ + if param.StructureVersion() >= 3 || commonDataStoreProtocol.server.DataStoreProtocolVersion().GreaterOrEqual("4.0.0") { + if !param.TotalCountEnabled { + totalCount = 0 + totalCountType = 3 + } + } + + pSearchResult.TotalCount = totalCount + pSearchResult.TotalCountType = totalCountType + rmcResponseStream := nex.NewStreamOut(commonDataStoreProtocol.server) rmcResponseStream.WriteStructure(pSearchResult) From 4f97f681615c6260ac1ff35e7463fdc5a5969e79 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 14:30:09 -0400 Subject: [PATCH 16/25] added SetDataKeyBase and SetNotifyKeyBase to separate data and notify objects --- datastore/complete_post_object.go | 2 +- datastore/complete_post_objects.go | 2 +- datastore/prepare_get_object.go | 2 +- datastore/prepare_post_object.go | 2 +- datastore/protocol.go | 18 ++++++++++++++++++ 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/datastore/complete_post_object.go b/datastore/complete_post_object.go index 03ce7e1..dd95dbb 100644 --- a/datastore/complete_post_object.go +++ b/datastore/complete_post_object.go @@ -37,7 +37,7 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat if param.IsSuccess { bucket := commonDataStoreProtocol.s3Bucket - key := fmt.Sprintf("%d.bin", param.DataID) + key := fmt.Sprintf("%s/%d.bin", commonDataStoreProtocol.s3DataKeyBase, param.DataID) objectSizeS3, err := commonDataStoreProtocol.s3ObjectSizeHandler(bucket, key) if err != nil { diff --git a/datastore/complete_post_objects.go b/datastore/complete_post_objects.go index 310f6c3..c67f9a4 100644 --- a/datastore/complete_post_objects.go +++ b/datastore/complete_post_objects.go @@ -31,7 +31,7 @@ func completePostObjects(err error, client *nex.Client, callID uint32, dataIDs [ for _, dataID := range dataIDs { bucket := commonDataStoreProtocol.s3Bucket - key := fmt.Sprintf("%d.bin", dataID) + key := fmt.Sprintf("%s/%d.bin", commonDataStoreProtocol.s3DataKeyBase, dataID) objectSizeS3, err := commonDataStoreProtocol.s3ObjectSizeHandler(bucket, key) if err != nil { diff --git a/datastore/prepare_get_object.go b/datastore/prepare_get_object.go index 615c996..9b42921 100644 --- a/datastore/prepare_get_object.go +++ b/datastore/prepare_get_object.go @@ -32,7 +32,7 @@ func prepareGetObject(err error, client *nex.Client, callID uint32, param *datas } bucket := commonDataStoreProtocol.s3Bucket - key := fmt.Sprintf("%d.bin", param.DataID) + key := fmt.Sprintf("%s/%d.bin", commonDataStoreProtocol.s3DataKeyBase, param.DataID) objectInfo, errCode := commonDataStoreProtocol.getObjectInfoByDataIDHandler(param.DataID) if errCode != 0 { diff --git a/datastore/prepare_post_object.go b/datastore/prepare_post_object.go index c0ea5ce..a517633 100644 --- a/datastore/prepare_post_object.go +++ b/datastore/prepare_post_object.go @@ -48,7 +48,7 @@ func preparePostObject(err error, client *nex.Client, callID uint32, param *data } bucket := commonDataStoreProtocol.s3Bucket - key := fmt.Sprintf("%d.bin", dataID) + key := fmt.Sprintf("%s/%d.bin", commonDataStoreProtocol.s3DataKeyBase, dataID) URL, formData, err := commonDataStoreProtocol.presignPostObjectHandler(bucket, key, time.Minute*15) if err != nil { diff --git a/datastore/protocol.go b/datastore/protocol.go index faf15a6..8d1d779 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -20,6 +20,8 @@ type CommonDataStoreProtocol struct { SuperMarioMakerProtocol *datastore_super_mario_maker.Protocol s3Bucket string + s3DataKeyBase string + s3NotifyKeyBase string rootCACert []byte getObjectInfoByDataIDHandler func(dataID uint64) (*datastore_types.DataStoreMetaInfo, uint32) verifyObjectPermissionHandler func(ownerPID uint32, accessorPID uint32, permission *datastore_types.DataStorePermission) uint32 @@ -48,6 +50,22 @@ func (c *CommonDataStoreProtocol) SetS3Bucket(bucket string) { c.s3Bucket = bucket } +// SetDataKeyBase sets the base for the key to be used when uploading standard DataStore objects +func (c *CommonDataStoreProtocol) SetDataKeyBase(base string) { + // * Just in case someone passes a badly formatted key + base = strings.TrimPrefix(base, "/") + base = strings.TrimSuffix(base, "/") + c.s3DataKeyBase = base +} + +// SetNotifyKeyBase sets the base for the key to be used when uploading DataStore notification data +func (c *CommonDataStoreProtocol) SetNotifyKeyBase(base string) { + // * Just in case someone passes a badly formatted key + base = strings.TrimPrefix(base, "/") + base = strings.TrimSuffix(base, "/") + c.s3NotifyKeyBase = base +} + // SetRootCACert sets the S3 root CA func (c *CommonDataStoreProtocol) SetRootCACert(rootCACert []byte) { c.rootCACert = rootCACert From 719cd89f700014afeb935b3d35292219cb76127d Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 17:20:02 -0400 Subject: [PATCH 17/25] moved minio s3 client to common --- datastore/complete_post_object.go | 6 ++-- datastore/complete_post_objects.go | 6 ++-- datastore/prepare_get_object.go | 6 ++-- datastore/prepare_post_object.go | 6 ++-- datastore/protocol.go | 48 ++++++++++++++++------------ datastore/s3_presigner.go | 51 ++++++++++++++++++++++++++++++ go.mod | 16 ++++++++++ go.sum | 49 ++++++++++++++++++++++++++++ 8 files changed, 156 insertions(+), 32 deletions(-) create mode 100644 datastore/s3_presigner.go diff --git a/datastore/complete_post_object.go b/datastore/complete_post_object.go index dd95dbb..08a38c8 100644 --- a/datastore/complete_post_object.go +++ b/datastore/complete_post_object.go @@ -10,8 +10,8 @@ import ( ) func completePostObject(err error, client *nex.Client, callID uint32, param *datastore_types.DataStoreCompletePostParam) uint32 { - if commonDataStoreProtocol.s3ObjectSizeHandler == nil { - common_globals.Logger.Warning("S3ObjectSize not defined") + if commonDataStoreProtocol.MinIOClient == nil { + common_globals.Logger.Warning("MinIOClient not defined") return nex.Errors.Core.NotImplemented } @@ -39,7 +39,7 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat bucket := commonDataStoreProtocol.s3Bucket key := fmt.Sprintf("%s/%d.bin", commonDataStoreProtocol.s3DataKeyBase, param.DataID) - objectSizeS3, err := commonDataStoreProtocol.s3ObjectSizeHandler(bucket, key) + objectSizeS3, err := commonDataStoreProtocol.S3ObjectSize(bucket, key) if err != nil { common_globals.Logger.Error(err.Error()) return nex.Errors.DataStore.NotFound diff --git a/datastore/complete_post_objects.go b/datastore/complete_post_objects.go index c67f9a4..32576d0 100644 --- a/datastore/complete_post_objects.go +++ b/datastore/complete_post_objects.go @@ -9,8 +9,8 @@ import ( ) func completePostObjects(err error, client *nex.Client, callID uint32, dataIDs []uint64) uint32 { - if commonDataStoreProtocol.s3ObjectSizeHandler == nil { - common_globals.Logger.Warning("S3ObjectSize not defined") + if commonDataStoreProtocol.MinIOClient == nil { + common_globals.Logger.Warning("MinIOClient not defined") return nex.Errors.Core.NotImplemented } @@ -33,7 +33,7 @@ func completePostObjects(err error, client *nex.Client, callID uint32, dataIDs [ bucket := commonDataStoreProtocol.s3Bucket key := fmt.Sprintf("%s/%d.bin", commonDataStoreProtocol.s3DataKeyBase, dataID) - objectSizeS3, err := commonDataStoreProtocol.s3ObjectSizeHandler(bucket, key) + objectSizeS3, err := commonDataStoreProtocol.S3ObjectSize(bucket, key) if err != nil { common_globals.Logger.Error(err.Error()) return nex.Errors.DataStore.NotFound diff --git a/datastore/prepare_get_object.go b/datastore/prepare_get_object.go index 9b42921..1881649 100644 --- a/datastore/prepare_get_object.go +++ b/datastore/prepare_get_object.go @@ -21,8 +21,8 @@ func prepareGetObject(err error, client *nex.Client, callID uint32, param *datas return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.presignGetObjectHandler == nil { - common_globals.Logger.Warning("PresignGetObject not defined") + if commonDataStoreProtocol.S3Presigner == nil { + common_globals.Logger.Warning("S3Presigner not defined") return nex.Errors.Core.NotImplemented } @@ -44,7 +44,7 @@ func prepareGetObject(err error, client *nex.Client, callID uint32, param *datas return errCode } - url, err := commonDataStoreProtocol.presignGetObjectHandler(bucket, key, time.Minute*15) + url, err := commonDataStoreProtocol.S3Presigner.GetObject(bucket, key, time.Minute*15) if err != nil { common_globals.Logger.Error(err.Error()) return nex.Errors.DataStore.OperationNotAllowed diff --git a/datastore/prepare_post_object.go b/datastore/prepare_post_object.go index a517633..a377546 100644 --- a/datastore/prepare_post_object.go +++ b/datastore/prepare_post_object.go @@ -21,8 +21,8 @@ func preparePostObject(err error, client *nex.Client, callID uint32, param *data return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.presignPostObjectHandler == nil { - common_globals.Logger.Warning("PresignPostObject not defined") + if commonDataStoreProtocol.S3Presigner == nil { + common_globals.Logger.Warning("S3Presigner not defined") return nex.Errors.Core.NotImplemented } @@ -50,7 +50,7 @@ func preparePostObject(err error, client *nex.Client, callID uint32, param *data bucket := commonDataStoreProtocol.s3Bucket key := fmt.Sprintf("%s/%d.bin", commonDataStoreProtocol.s3DataKeyBase, dataID) - URL, formData, err := commonDataStoreProtocol.presignPostObjectHandler(bucket, key, time.Minute*15) + URL, formData, err := commonDataStoreProtocol.S3Presigner.PostObject(bucket, key, time.Minute*15) if err != nil { common_globals.Logger.Error(err.Error()) return nex.Errors.DataStore.OperationNotAllowed diff --git a/datastore/protocol.go b/datastore/protocol.go index 8d1d779..4c1a44e 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -1,15 +1,15 @@ package datastore import ( - "net/url" + "context" "strings" - "time" nex "github.com/PretendoNetwork/nex-go" common_globals "github.com/PretendoNetwork/nex-protocols-common-go/globals" datastore "github.com/PretendoNetwork/nex-protocols-go/datastore" datastore_super_mario_maker "github.com/PretendoNetwork/nex-protocols-go/datastore/super-mario-maker" datastore_types "github.com/PretendoNetwork/nex-protocols-go/datastore/types" + "github.com/minio/minio-go/v7" ) var commonDataStoreProtocol *CommonDataStoreProtocol @@ -23,18 +23,17 @@ type CommonDataStoreProtocol struct { s3DataKeyBase string s3NotifyKeyBase string rootCACert []byte + MinIOClient *minio.Client + S3Presigner S3PresignerInterface getObjectInfoByDataIDHandler func(dataID uint64) (*datastore_types.DataStoreMetaInfo, uint32) verifyObjectPermissionHandler func(ownerPID uint32, accessorPID uint32, permission *datastore_types.DataStorePermission) uint32 updateObjectPeriodByDataIDWithPasswordHandler func(dataID uint64, dataType uint16, password uint64) uint32 updateObjectMetaBinaryByDataIDWithPasswordHandler func(dataID uint64, metaBinary []byte, password uint64) uint32 updateObjectDataTypeByDataIDWithPasswordHandler func(dataID uint64, period uint16, password uint64) uint32 - s3ObjectSizeHandler func(bucket string, key string) (uint64, error) getObjectSizeDataIDHandler func(dataID uint64) (uint32, uint32) updateObjectUploadCompletedByDataIDHandler func(dataID uint64, uploadCompleted bool) uint32 getObjectInfoByPersistenceTargetWithPasswordHandler func(persistenceTarget *datastore_types.DataStorePersistenceTarget, password uint64) (*datastore_types.DataStoreMetaInfo, uint32) getObjectInfoByDataIDWithPasswordHandler func(dataID uint64, password uint64) (*datastore_types.DataStoreMetaInfo, uint32) - presignGetObjectHandler func(bucket string, key string, lifetime time.Duration) (*url.URL, error) - presignPostObjectHandler func(bucket string, key string, lifetime time.Duration) (*url.URL, map[string]string, error) s3GetRequestHeadersHandler func() ([]*datastore_types.DataStoreKeyValue, uint32) s3PostRequestHeadersHandler func() ([]*datastore_types.DataStoreKeyValue, uint32) initializeObjectByPreparePostParamHandler func(ownerPID uint32, param *datastore_types.DataStorePreparePostParam) (uint64, uint32) @@ -45,6 +44,19 @@ type CommonDataStoreProtocol struct { getObjectInfosByDataStoreSearchParamHandler func(param *datastore_types.DataStoreSearchParam) ([]*datastore_types.DataStoreMetaInfo, uint32, uint32) } +func (c *CommonDataStoreProtocol) S3StatObject(bucket, key string) (minio.ObjectInfo, error) { + return c.MinIOClient.StatObject(context.TODO(), bucket, key, minio.StatObjectOptions{}) +} + +func (c *CommonDataStoreProtocol) S3ObjectSize(bucket, key string) (uint64, error) { + info, err := c.S3StatObject(bucket, key) + if err != nil { + return 0, err + } + + return uint64(info.Size), nil +} + // SetS3Bucket sets the S3 bucket func (c *CommonDataStoreProtocol) SetS3Bucket(bucket string) { c.s3Bucket = bucket @@ -71,6 +83,17 @@ func (c *CommonDataStoreProtocol) SetRootCACert(rootCACert []byte) { c.rootCACert = rootCACert } +// SetMinIOClient sets the MinIO S3 client +func (c *CommonDataStoreProtocol) SetMinIOClient(client *minio.Client) { + c.MinIOClient = client + c.SetS3Presigner(NewS3Presigner(c.MinIOClient)) +} + +// SetS3Presigner sets the struct which creates presigned S3 URLs +func (c *CommonDataStoreProtocol) SetS3Presigner(presigner S3PresignerInterface) { + c.S3Presigner = presigner +} + // GetObjectInfoByDataID sets the GetObjectInfoByDataID handler function func (c *CommonDataStoreProtocol) GetObjectInfoByDataID(handler func(dataID uint64) (*datastore_types.DataStoreMetaInfo, uint32)) { c.getObjectInfoByDataIDHandler = handler @@ -96,11 +119,6 @@ func (c *CommonDataStoreProtocol) UpdateObjectDataTypeByDataIDWithPassword(handl c.updateObjectDataTypeByDataIDWithPasswordHandler = handler } -// S3ObjectSize sets the S3ObjectSize handler function -func (c *CommonDataStoreProtocol) S3ObjectSize(handler func(bucket string, key string) (uint64, error)) { - c.s3ObjectSizeHandler = handler -} - // GetObjectSizeDataID sets the GetObjectSizeDataID handler function func (c *CommonDataStoreProtocol) GetObjectSizeDataID(handler func(dataID uint64) (uint32, uint32)) { c.getObjectSizeDataIDHandler = handler @@ -121,16 +139,6 @@ func (c *CommonDataStoreProtocol) GetObjectInfoByDataIDWithPassword(handler func c.getObjectInfoByDataIDWithPasswordHandler = handler } -// PresignGetObject sets the PresignGetObject handler function -func (c *CommonDataStoreProtocol) PresignGetObject(handler func(bucket string, key string, lifetime time.Duration) (*url.URL, error)) { - c.presignGetObjectHandler = handler -} - -// PresignPostObject sets the PresignPostObject handler function -func (c *CommonDataStoreProtocol) PresignPostObject(handler func(bucket string, key string, lifetime time.Duration) (*url.URL, map[string]string, error)) { - c.presignPostObjectHandler = handler -} - // S3GetRequestHeaders sets the S3GetRequestHeaders handler function func (c *CommonDataStoreProtocol) S3GetRequestHeaders(handler func() ([]*datastore_types.DataStoreKeyValue, uint32)) { c.s3GetRequestHeadersHandler = handler diff --git a/datastore/s3_presigner.go b/datastore/s3_presigner.go new file mode 100644 index 0000000..e2e84f6 --- /dev/null +++ b/datastore/s3_presigner.go @@ -0,0 +1,51 @@ +package datastore + +import ( + "context" + "net/url" + "time" + + "github.com/minio/minio-go/v7" +) + +type S3PresignerInterface interface { + GetObject(bucket, key string, lifetime time.Duration) (*url.URL, error) + PostObject(bucket, key string, lifetime time.Duration) (*url.URL, map[string]string, error) +} + +type S3Presigner struct { + minio *minio.Client +} + +func (p *S3Presigner) GetObject(bucket, key string, lifetime time.Duration) (*url.URL, error) { + reqParams := make(url.Values) + + return p.minio.PresignedGetObject(context.Background(), bucket, key, lifetime, reqParams) +} + +func (p *S3Presigner) PostObject(bucket, key string, lifetime time.Duration) (*url.URL, map[string]string, error) { + policy := minio.NewPostPolicy() + + err := policy.SetBucket(bucket) + if err != nil { + return nil, nil, err + } + + err = policy.SetKey(key) + if err != nil { + return nil, nil, err + } + + err = policy.SetExpires(time.Now().UTC().Add(lifetime).UTC()) + if err != nil { + return nil, nil, err + } + + return p.minio.PresignedPostPolicy(context.Background(), policy) +} + +func NewS3Presigner(minioClient *minio.Client) *S3Presigner { + return &S3Presigner{ + minio: minioClient, + } +} diff --git a/go.mod b/go.mod index 76ade1d..9fae24d 100644 --- a/go.mod +++ b/go.mod @@ -6,16 +6,32 @@ require ( github.com/PretendoNetwork/nex-go v1.0.41 github.com/PretendoNetwork/nex-protocols-go v1.0.52 github.com/PretendoNetwork/plogger-go v1.0.4 + github.com/minio/minio-go/v7 v7.0.63 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.15.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/jwalton/go-supportscolor v1.2.0 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/rs/xid v1.5.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/superwhiskers/crunch/v3 v3.5.7 // indirect + golang.org/x/crypto v0.12.0 // indirect golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/term v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 40cafe2..563ee3d 100644 --- a/go.sum +++ b/go.sum @@ -4,28 +4,77 @@ github.com/PretendoNetwork/nex-protocols-go v1.0.52 h1:romSI1brKF+IGcK6+v2p8Ajgb github.com/PretendoNetwork/nex-protocols-go v1.0.52/go.mod h1:9r4KbNELVZj01TY8p+FGYDb9+e4mHLiWKo5NL1fPqD8= github.com/PretendoNetwork/plogger-go v1.0.4 h1:PF7xHw9eDRHH+RsAP9tmAE7fG0N0p6H4iPwHKnsoXwc= github.com/PretendoNetwork/plogger-go v1.0.4/go.mod h1:7kD6M4vPq1JL4LTuPg6kuB1OvUBOwQOtAvTaUwMbwvU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jwalton/go-supportscolor v1.2.0 h1:g6Ha4u7Vm3LIsQ5wmeBpS4gazu0UP1DRDE8y6bre4H8= github.com/jwalton/go-supportscolor v1.2.0/go.mod h1:hFVUAZV2cWg+WFFC4v8pT2X/S2qUUBYMioBD9AINXGs= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= +github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/superwhiskers/crunch/v3 v3.5.7 h1:N9RLxaR65C36i26BUIpzPXGy2f6pQ7wisu2bawbKNqg= github.com/superwhiskers/crunch/v3 v3.5.7/go.mod h1:4ub2EKgF1MAhTjoOCTU4b9uLMsAweHEa89aRrfAypXA= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 890b9dc32a01b464604679856f8cd35b0f4e77de Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 17:30:02 -0400 Subject: [PATCH 18/25] moved VerifyObjectPermission to common --- datastore/change_meta.go | 7 +--- datastore/delete_object.go | 7 +--- datastore/get_meta.go | 7 +--- datastore/get_metas.go | 7 +--- datastore/get_metas_multiple_param.go | 7 +--- datastore/prepare_get_object.go | 7 +--- datastore/protocol.go | 49 +++++++++++++++++++++++---- datastore/rate_object.go | 7 +--- datastore/rate_objects.go | 7 +--- datastore/search_object.go | 7 +--- 10 files changed, 52 insertions(+), 60 deletions(-) diff --git a/datastore/change_meta.go b/datastore/change_meta.go index 07f15be..163f0a5 100644 --- a/datastore/change_meta.go +++ b/datastore/change_meta.go @@ -13,11 +13,6 @@ func changeMeta(err error, client *nex.Client, callID uint32, param *datastore_t return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { - common_globals.Logger.Warning("VerifyObjectPermission not defined") - return nex.Errors.Core.NotImplemented - } - if commonDataStoreProtocol.updateObjectPeriodByDataIDWithPasswordHandler == nil { common_globals.Logger.Warning("UpdateObjectPeriodByDataIDWithPassword not defined") return nex.Errors.Core.NotImplemented @@ -44,7 +39,7 @@ func changeMeta(err error, client *nex.Client, callID uint32, param *datastore_t } // TODO - Is this the right permission? - errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(metaInfo.OwnerID, client.PID(), metaInfo.DelPermission) + errCode = commonDataStoreProtocol.VerifyObjectPermission(metaInfo.OwnerID, client.PID(), metaInfo.DelPermission) if errCode != 0 { return errCode } diff --git a/datastore/delete_object.go b/datastore/delete_object.go index 6322c71..4b24c3f 100644 --- a/datastore/delete_object.go +++ b/datastore/delete_object.go @@ -13,11 +13,6 @@ func deleteObject(err error, client *nex.Client, callID uint32, param *datastore return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { - common_globals.Logger.Warning("VerifyObjectPermission not defined") - return nex.Errors.Core.NotImplemented - } - if commonDataStoreProtocol.deleteObjectByDataIDWithPasswordHandler == nil { common_globals.Logger.Warning("DeleteObjectByDataIDWithPassword not defined") return nex.Errors.Core.NotImplemented @@ -33,7 +28,7 @@ func deleteObject(err error, client *nex.Client, callID uint32, param *datastore return errCode } - errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(metaInfo.OwnerID, client.PID(), metaInfo.DelPermission) + errCode = commonDataStoreProtocol.VerifyObjectPermission(metaInfo.OwnerID, client.PID(), metaInfo.DelPermission) if errCode != 0 { return errCode } diff --git a/datastore/get_meta.go b/datastore/get_meta.go index 7856642..8f80ad5 100644 --- a/datastore/get_meta.go +++ b/datastore/get_meta.go @@ -18,11 +18,6 @@ func getMeta(err error, client *nex.Client, callID uint32, param *datastore_type return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { - common_globals.Logger.Warning("VerifyObjectPermission not defined") - return nex.Errors.Core.NotImplemented - } - if err != nil { common_globals.Logger.Error(err.Error()) return nex.Errors.DataStore.Unknown @@ -42,7 +37,7 @@ func getMeta(err error, client *nex.Client, callID uint32, param *datastore_type return errCode } - errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(pMetaInfo.OwnerID, client.PID(), pMetaInfo.Permission) + errCode = commonDataStoreProtocol.VerifyObjectPermission(pMetaInfo.OwnerID, client.PID(), pMetaInfo.Permission) if errCode != 0 { return errCode } diff --git a/datastore/get_metas.go b/datastore/get_metas.go index 0d27388..bea0ff1 100644 --- a/datastore/get_metas.go +++ b/datastore/get_metas.go @@ -13,11 +13,6 @@ func getMetas(err error, client *nex.Client, callID uint32, dataIDs []uint64, pa return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { - common_globals.Logger.Warning("VerifyObjectPermission not defined") - return nex.Errors.Core.NotImplemented - } - if err != nil { common_globals.Logger.Error(err.Error()) return nex.Errors.DataStore.Unknown @@ -65,7 +60,7 @@ func getMetas(err error, client *nex.Client, callID uint32, dataIDs []uint64, pa pResults = append(pResults, nex.NewResultError(errCode)) } else { - errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(objectInfo.OwnerID, client.PID(), objectInfo.Permission) + errCode = commonDataStoreProtocol.VerifyObjectPermission(objectInfo.OwnerID, client.PID(), objectInfo.Permission) if errCode != 0 { // TODO - Maybe this should be broken out into a util function in globals? objectInfo = datastore_types.NewDataStoreMetaInfo() diff --git a/datastore/get_metas_multiple_param.go b/datastore/get_metas_multiple_param.go index 81d0293..117c03a 100644 --- a/datastore/get_metas_multiple_param.go +++ b/datastore/get_metas_multiple_param.go @@ -18,11 +18,6 @@ func getMetasMultipleParam(err error, client *nex.Client, callID uint32, params return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { - common_globals.Logger.Warning("VerifyObjectPermission not defined") - return nex.Errors.Core.NotImplemented - } - if err != nil { common_globals.Logger.Error(err.Error()) return nex.Errors.DataStore.Unknown @@ -71,7 +66,7 @@ func getMetasMultipleParam(err error, client *nex.Client, callID uint32, params pResults = append(pResults, nex.NewResultError(errCode)) } else { - errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(objectInfo.OwnerID, client.PID(), objectInfo.Permission) + errCode = commonDataStoreProtocol.VerifyObjectPermission(objectInfo.OwnerID, client.PID(), objectInfo.Permission) if errCode != 0 { // TODO - Maybe this should be broken out into a util function in globals? objectInfo = datastore_types.NewDataStoreMetaInfo() diff --git a/datastore/prepare_get_object.go b/datastore/prepare_get_object.go index 1881649..325b12d 100644 --- a/datastore/prepare_get_object.go +++ b/datastore/prepare_get_object.go @@ -16,11 +16,6 @@ func prepareGetObject(err error, client *nex.Client, callID uint32, param *datas return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { - common_globals.Logger.Warning("VerifyObjectPermission not defined") - return nex.Errors.Core.NotImplemented - } - if commonDataStoreProtocol.S3Presigner == nil { common_globals.Logger.Warning("S3Presigner not defined") return nex.Errors.Core.NotImplemented @@ -39,7 +34,7 @@ func prepareGetObject(err error, client *nex.Client, callID uint32, param *datas return errCode } - errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(objectInfo.OwnerID, client.PID(), objectInfo.Permission) + errCode = commonDataStoreProtocol.VerifyObjectPermission(objectInfo.OwnerID, client.PID(), objectInfo.Permission) if errCode != 0 { return errCode } diff --git a/datastore/protocol.go b/datastore/protocol.go index 4c1a44e..96fb158 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -2,6 +2,7 @@ package datastore import ( "context" + "slices" "strings" nex "github.com/PretendoNetwork/nex-go" @@ -25,8 +26,8 @@ type CommonDataStoreProtocol struct { rootCACert []byte MinIOClient *minio.Client S3Presigner S3PresignerInterface + getUserFriendPIDsHandler func(pid uint32) []uint32 getObjectInfoByDataIDHandler func(dataID uint64) (*datastore_types.DataStoreMetaInfo, uint32) - verifyObjectPermissionHandler func(ownerPID uint32, accessorPID uint32, permission *datastore_types.DataStorePermission) uint32 updateObjectPeriodByDataIDWithPasswordHandler func(dataID uint64, dataType uint16, password uint64) uint32 updateObjectMetaBinaryByDataIDWithPasswordHandler func(dataID uint64, metaBinary []byte, password uint64) uint32 updateObjectDataTypeByDataIDWithPasswordHandler func(dataID uint64, period uint16, password uint64) uint32 @@ -57,6 +58,42 @@ func (c *CommonDataStoreProtocol) S3ObjectSize(bucket, key string) (uint64, erro return uint64(info.Size), nil } +func (c *CommonDataStoreProtocol) VerifyObjectPermission(ownerPID, accessorPID uint32, permission *datastore_types.DataStorePermission) uint32 { + if permission.Permission > 3 { + return nex.Errors.DataStore.InvalidArgument + } + + // * Allow anyone + if permission.Permission == 0 { + return 0 + } + + // * Allow friends + if permission.Permission == 1 { + friendsList := c.getUserFriendPIDsHandler(ownerPID) + + if !slices.Contains(friendsList, accessorPID) { + return nex.Errors.DataStore.PermissionDenied + } + } + + // * Allow people in permission.RecipientIDs + if permission.Permission == 2 { + if !slices.Contains(permission.RecipientIDs, accessorPID) { + return nex.Errors.DataStore.PermissionDenied + } + } + + // * Allow only the owner + if permission.Permission == 3 { + if ownerPID != accessorPID { + return nex.Errors.DataStore.PermissionDenied + } + } + + return 0 +} + // SetS3Bucket sets the S3 bucket func (c *CommonDataStoreProtocol) SetS3Bucket(bucket string) { c.s3Bucket = bucket @@ -94,16 +131,16 @@ func (c *CommonDataStoreProtocol) SetS3Presigner(presigner S3PresignerInterface) c.S3Presigner = presigner } +// SetGetUserFriendPIDs sets the handler for a function which gets a list of a users friend PIDs +func (c *CommonDataStoreProtocol) SetGetUserFriendPIDs(handler func(pid uint32) []uint32) { + c.getUserFriendPIDsHandler = handler +} + // GetObjectInfoByDataID sets the GetObjectInfoByDataID handler function func (c *CommonDataStoreProtocol) GetObjectInfoByDataID(handler func(dataID uint64) (*datastore_types.DataStoreMetaInfo, uint32)) { c.getObjectInfoByDataIDHandler = handler } -// VerifyObjectPermission sets the VerifyObjectPermission handler function -func (c *CommonDataStoreProtocol) VerifyObjectPermission(handler func(ownerPID uint32, accessorPID uint32, permission *datastore_types.DataStorePermission) uint32) { - c.verifyObjectPermissionHandler = handler -} - // UpdateObjectPeriodByDataIDWithPassword sets the UpdateObjectPeriodByDataIDWithPassword handler function func (c *CommonDataStoreProtocol) UpdateObjectPeriodByDataIDWithPassword(handler func(dataID uint64, dataType uint16, password uint64) uint32) { c.updateObjectPeriodByDataIDWithPasswordHandler = handler diff --git a/datastore/rate_object.go b/datastore/rate_object.go index b897cce..30cb819 100644 --- a/datastore/rate_object.go +++ b/datastore/rate_object.go @@ -13,11 +13,6 @@ func rateObject(err error, client *nex.Client, callID uint32, target *datastore_ return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { - common_globals.Logger.Warning("VerifyObjectPermission not defined") - return nex.Errors.Core.NotImplemented - } - if commonDataStoreProtocol.rateObjectWithPasswordHandler == nil { common_globals.Logger.Warning("RateObjectWithPassword not defined") return nex.Errors.Core.NotImplemented @@ -33,7 +28,7 @@ func rateObject(err error, client *nex.Client, callID uint32, target *datastore_ return errCode } - errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(objectInfo.OwnerID, client.PID(), objectInfo.Permission) + errCode = commonDataStoreProtocol.VerifyObjectPermission(objectInfo.OwnerID, client.PID(), objectInfo.Permission) if errCode != 0 { return errCode } diff --git a/datastore/rate_objects.go b/datastore/rate_objects.go index 23e3503..220aee1 100644 --- a/datastore/rate_objects.go +++ b/datastore/rate_objects.go @@ -13,11 +13,6 @@ func rateObjects(err error, client *nex.Client, callID uint32, targets []*datast return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { - common_globals.Logger.Warning("VerifyObjectPermission not defined") - return nex.Errors.Core.NotImplemented - } - if commonDataStoreProtocol.rateObjectWithPasswordHandler == nil { common_globals.Logger.Warning("RateObjectWithPassword not defined") return nex.Errors.Core.NotImplemented @@ -48,7 +43,7 @@ func rateObjects(err error, client *nex.Client, callID uint32, targets []*datast return errCode } - errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(objectInfo.OwnerID, client.PID(), objectInfo.Permission) + errCode = commonDataStoreProtocol.VerifyObjectPermission(objectInfo.OwnerID, client.PID(), objectInfo.Permission) if errCode != 0 { return errCode } diff --git a/datastore/search_object.go b/datastore/search_object.go index d0120fd..d5cb7de 100644 --- a/datastore/search_object.go +++ b/datastore/search_object.go @@ -13,11 +13,6 @@ func searchObject(err error, client *nex.Client, callID uint32, param *datastore return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.verifyObjectPermissionHandler == nil { - common_globals.Logger.Warning("VerifyObjectPermission not defined") - return nex.Errors.Core.NotImplemented - } - if err != nil { common_globals.Logger.Error(err.Error()) return nex.Errors.DataStore.Unknown @@ -40,7 +35,7 @@ func searchObject(err error, client *nex.Client, callID uint32, param *datastore pSearchResult.Result = make([]*datastore_types.DataStoreMetaInfo, 0, len(objects)) for _, object := range objects { - errCode = commonDataStoreProtocol.verifyObjectPermissionHandler(object.OwnerID, client.PID(), object.Permission) + errCode = commonDataStoreProtocol.VerifyObjectPermission(object.OwnerID, client.PID(), object.Permission) if errCode != 0 { // * Since we don't error here, should we also // * "hide" these results by also decrementing From 49bca9e60d1c2f2a08b57ccc4e263aa5f1afc2f4 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sun, 29 Oct 2023 23:28:44 -0400 Subject: [PATCH 19/25] removed creating zeroed structs manually --- datastore/get_metas.go | 48 --------------------------- datastore/get_metas_multiple_param.go | 48 --------------------------- go.mod | 22 ++++++------ go.sum | 44 ++++++++++++------------ 4 files changed, 33 insertions(+), 129 deletions(-) diff --git a/datastore/get_metas.go b/datastore/get_metas.go index bea0ff1..4b2c0a6 100644 --- a/datastore/get_metas.go +++ b/datastore/get_metas.go @@ -32,61 +32,13 @@ func getMetas(err error, client *nex.Client, callID uint32, dataIDs []uint64, pa objectInfo, errCode := commonDataStoreProtocol.getObjectInfoByDataIDHandler(dataIDs[i]) if errCode != 0 { - // TODO - Maybe this should be broken out into a util function in globals? objectInfo = datastore_types.NewDataStoreMetaInfo() - objectInfo.DataID = 0 - objectInfo.OwnerID = 0 - objectInfo.Size = 0 - objectInfo.Name = "" - objectInfo.DataType = 0 - objectInfo.MetaBinary = []byte{} - objectInfo.Permission = datastore_types.NewDataStorePermission() - objectInfo.Permission.Permission = 0 - objectInfo.Permission.RecipientIDs = []uint32{} - objectInfo.DelPermission = datastore_types.NewDataStorePermission() - objectInfo.DelPermission.Permission = 0 - objectInfo.DelPermission.RecipientIDs = []uint32{} - objectInfo.CreatedTime = nex.NewDateTime(0) - objectInfo.UpdatedTime = nex.NewDateTime(0) - objectInfo.Period = 0 - objectInfo.Status = 0 - objectInfo.ReferredCnt = 0 - objectInfo.ReferDataID = 0 - objectInfo.Flag = 0 - objectInfo.ReferredTime = nex.NewDateTime(0) - objectInfo.ExpireTime = nex.NewDateTime(0) - objectInfo.Tags = []string{} - objectInfo.Ratings = []*datastore_types.DataStoreRatingInfoWithSlot{} pResults = append(pResults, nex.NewResultError(errCode)) } else { errCode = commonDataStoreProtocol.VerifyObjectPermission(objectInfo.OwnerID, client.PID(), objectInfo.Permission) if errCode != 0 { - // TODO - Maybe this should be broken out into a util function in globals? objectInfo = datastore_types.NewDataStoreMetaInfo() - objectInfo.DataID = 0 - objectInfo.OwnerID = 0 - objectInfo.Size = 0 - objectInfo.Name = "" - objectInfo.DataType = 0 - objectInfo.MetaBinary = []byte{} - objectInfo.Permission = datastore_types.NewDataStorePermission() - objectInfo.Permission.Permission = 0 - objectInfo.Permission.RecipientIDs = []uint32{} - objectInfo.DelPermission = datastore_types.NewDataStorePermission() - objectInfo.DelPermission.Permission = 0 - objectInfo.DelPermission.RecipientIDs = []uint32{} - objectInfo.CreatedTime = nex.NewDateTime(0) - objectInfo.UpdatedTime = nex.NewDateTime(0) - objectInfo.Period = 0 - objectInfo.Status = 0 - objectInfo.ReferredCnt = 0 - objectInfo.ReferDataID = 0 - objectInfo.Flag = 0 - objectInfo.ReferredTime = nex.NewDateTime(0) - objectInfo.ExpireTime = nex.NewDateTime(0) - objectInfo.Tags = []string{} - objectInfo.Ratings = []*datastore_types.DataStoreRatingInfoWithSlot{} pResults = append(pResults, nex.NewResultError(errCode)) } else { diff --git a/datastore/get_metas_multiple_param.go b/datastore/get_metas_multiple_param.go index 117c03a..3b8082d 100644 --- a/datastore/get_metas_multiple_param.go +++ b/datastore/get_metas_multiple_param.go @@ -38,61 +38,13 @@ func getMetasMultipleParam(err error, client *nex.Client, callID uint32, params } if errCode != 0 { - // TODO - Maybe this should be broken out into a util function in globals? objectInfo = datastore_types.NewDataStoreMetaInfo() - objectInfo.DataID = 0 - objectInfo.OwnerID = 0 - objectInfo.Size = 0 - objectInfo.Name = "" - objectInfo.DataType = 0 - objectInfo.MetaBinary = []byte{} - objectInfo.Permission = datastore_types.NewDataStorePermission() - objectInfo.Permission.Permission = 0 - objectInfo.Permission.RecipientIDs = []uint32{} - objectInfo.DelPermission = datastore_types.NewDataStorePermission() - objectInfo.DelPermission.Permission = 0 - objectInfo.DelPermission.RecipientIDs = []uint32{} - objectInfo.CreatedTime = nex.NewDateTime(0) - objectInfo.UpdatedTime = nex.NewDateTime(0) - objectInfo.Period = 0 - objectInfo.Status = 0 - objectInfo.ReferredCnt = 0 - objectInfo.ReferDataID = 0 - objectInfo.Flag = 0 - objectInfo.ReferredTime = nex.NewDateTime(0) - objectInfo.ExpireTime = nex.NewDateTime(0) - objectInfo.Tags = []string{} - objectInfo.Ratings = []*datastore_types.DataStoreRatingInfoWithSlot{} pResults = append(pResults, nex.NewResultError(errCode)) } else { errCode = commonDataStoreProtocol.VerifyObjectPermission(objectInfo.OwnerID, client.PID(), objectInfo.Permission) if errCode != 0 { - // TODO - Maybe this should be broken out into a util function in globals? objectInfo = datastore_types.NewDataStoreMetaInfo() - objectInfo.DataID = 0 - objectInfo.OwnerID = 0 - objectInfo.Size = 0 - objectInfo.Name = "" - objectInfo.DataType = 0 - objectInfo.MetaBinary = []byte{} - objectInfo.Permission = datastore_types.NewDataStorePermission() - objectInfo.Permission.Permission = 0 - objectInfo.Permission.RecipientIDs = []uint32{} - objectInfo.DelPermission = datastore_types.NewDataStorePermission() - objectInfo.DelPermission.Permission = 0 - objectInfo.DelPermission.RecipientIDs = []uint32{} - objectInfo.CreatedTime = nex.NewDateTime(0) - objectInfo.UpdatedTime = nex.NewDateTime(0) - objectInfo.Period = 0 - objectInfo.Status = 0 - objectInfo.ReferredCnt = 0 - objectInfo.ReferDataID = 0 - objectInfo.Flag = 0 - objectInfo.ReferredTime = nex.NewDateTime(0) - objectInfo.ExpireTime = nex.NewDateTime(0) - objectInfo.Tags = []string{} - objectInfo.Ratings = []*datastore_types.DataStoreRatingInfoWithSlot{} pResults = append(pResults, nex.NewResultError(errCode)) } else { diff --git a/go.mod b/go.mod index 9fae24d..4f4bdba 100644 --- a/go.mod +++ b/go.mod @@ -4,22 +4,22 @@ go 1.19 require ( github.com/PretendoNetwork/nex-go v1.0.41 - github.com/PretendoNetwork/nex-protocols-go v1.0.52 + github.com/PretendoNetwork/nex-protocols-go v1.0.56 github.com/PretendoNetwork/plogger-go v1.0.4 github.com/minio/minio-go/v7 v7.0.63 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d ) require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.15.0 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/jwalton/go-supportscolor v1.2.0 // indirect - github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/compress v1.17.2 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -27,11 +27,11 @@ require ( github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/superwhiskers/crunch/v3 v3.5.7 // indirect - golang.org/x/crypto v0.12.0 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 563ee3d..679dfdf 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/PretendoNetwork/nex-go v1.0.41 h1:TcBb1Bpe2KAB+AXaGX1K9NVQBRtZJIoy4yCvRde2xbI= github.com/PretendoNetwork/nex-go v1.0.41/go.mod h1:QwHEa165DeVd0xIuthrgc3j6NWXT8lyPSG6t5kC/Shk= -github.com/PretendoNetwork/nex-protocols-go v1.0.52 h1:romSI1brKF+IGcK6+v2p8AjgbztVnTOVENT2U+dcWq0= -github.com/PretendoNetwork/nex-protocols-go v1.0.52/go.mod h1:9r4KbNELVZj01TY8p+FGYDb9+e4mHLiWKo5NL1fPqD8= +github.com/PretendoNetwork/nex-protocols-go v1.0.56 h1:cFoyANBXv2McRhBnVDsOWIXxoRv066qO1tkkknFL7WE= +github.com/PretendoNetwork/nex-protocols-go v1.0.56/go.mod h1:136762CMIcAsTZDrt4W7gDE4ppiBuc9zN2QFOHESjS8= github.com/PretendoNetwork/plogger-go v1.0.4 h1:PF7xHw9eDRHH+RsAP9tmAE7fG0N0p6H4iPwHKnsoXwc= github.com/PretendoNetwork/plogger-go v1.0.4/go.mod h1:7kD6M4vPq1JL4LTuPg6kuB1OvUBOwQOtAvTaUwMbwvU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -13,22 +13,22 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jwalton/go-supportscolor v1.2.0 h1:g6Ha4u7Vm3LIsQ5wmeBpS4gazu0UP1DRDE8y6bre4H8= github.com/jwalton/go-supportscolor v1.2.0/go.mod h1:hFVUAZV2cWg+WFFC4v8pT2X/S2qUUBYMioBD9AINXGs= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= @@ -52,27 +52,27 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/superwhiskers/crunch/v3 v3.5.7 h1:N9RLxaR65C36i26BUIpzPXGy2f6pQ7wisu2bawbKNqg= github.com/superwhiskers/crunch/v3 v3.5.7/go.mod h1:4ub2EKgF1MAhTjoOCTU4b9uLMsAweHEa89aRrfAypXA= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= From 9ed9b132e80035a380af12cefdc8c051c00abe50 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Mon, 30 Oct 2023 00:08:37 -0400 Subject: [PATCH 20/25] use FilterPropertiesByResultOption when requesting object data --- datastore/get_meta.go | 21 +-------------------- datastore/get_metas.go | 21 +-------------------- datastore/get_metas_multiple_param.go | 21 +-------------------- datastore/search_object.go | 21 +-------------------- go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 7 insertions(+), 83 deletions(-) diff --git a/datastore/get_meta.go b/datastore/get_meta.go index 8f80ad5..97d626b 100644 --- a/datastore/get_meta.go +++ b/datastore/get_meta.go @@ -42,26 +42,7 @@ func getMeta(err error, client *nex.Client, callID uint32, param *datastore_type return errCode } - // * This is kind of backwards. - // * The database pulls this data - // * by default, so it can be done - // * in a single query. So instead - // * of checking if a flag *IS* - // * set, and conditionally *ADDING* - // * the fields, we check if a flag - // * is *NOT* set and conditionally - // * *REMOVE* the field - if param.ResultOption&0x1 == 0 { - pMetaInfo.Tags = make([]string, 0) - } - - if param.ResultOption&0x2 == 0 { - pMetaInfo.Ratings = make([]*datastore_types.DataStoreRatingInfoWithSlot, 0) - } - - if param.ResultOption&0x4 == 0 { - pMetaInfo.MetaBinary = make([]byte, 0) - } + pMetaInfo.FilterPropertiesByResultOption(param.ResultOption) rmcResponseStream := nex.NewStreamOut(commonDataStoreProtocol.server) rmcResponseStream.WriteStructure(pMetaInfo) diff --git a/datastore/get_metas.go b/datastore/get_metas.go index 4b2c0a6..31ee337 100644 --- a/datastore/get_metas.go +++ b/datastore/get_metas.go @@ -45,26 +45,7 @@ func getMetas(err error, client *nex.Client, callID uint32, dataIDs []uint64, pa pResults = append(pResults, nex.NewResultSuccess(nex.Errors.DataStore.Unknown)) } - // * This is kind of backwards. - // * The database pulls this data - // * by default, so it can be done - // * in a single query. So instead - // * of checking if a flag *IS* - // * set, and conditionally *ADDING* - // * the fields, we check if a flag - // * is *NOT* set and conditionally - // * *REMOVE* the field - if param.ResultOption&0x1 == 0 { - objectInfo.Tags = make([]string, 0) - } - - if param.ResultOption&0x2 == 0 { - objectInfo.Ratings = make([]*datastore_types.DataStoreRatingInfoWithSlot, 0) - } - - if param.ResultOption&0x4 == 0 { - objectInfo.MetaBinary = make([]byte, 0) - } + objectInfo.FilterPropertiesByResultOption(param.ResultOption) } pMetaInfo = append(pMetaInfo, objectInfo) diff --git a/datastore/get_metas_multiple_param.go b/datastore/get_metas_multiple_param.go index 3b8082d..64ef360 100644 --- a/datastore/get_metas_multiple_param.go +++ b/datastore/get_metas_multiple_param.go @@ -51,26 +51,7 @@ func getMetasMultipleParam(err error, client *nex.Client, callID uint32, params pResults = append(pResults, nex.NewResultSuccess(nex.Errors.DataStore.Unknown)) } - // * This is kind of backwards. - // * The database pulls this data - // * by default, so it can be done - // * in a single query. So instead - // * of checking if a flag *IS* - // * set, and conditionally *ADDING* - // * the fields, we check if a flag - // * is *NOT* set and conditionally - // * *REMOVE* the field - if param.ResultOption&0x1 == 0 { - objectInfo.Tags = make([]string, 0) - } - - if param.ResultOption&0x2 == 0 { - objectInfo.Ratings = make([]*datastore_types.DataStoreRatingInfoWithSlot, 0) - } - - if param.ResultOption&0x4 == 0 { - objectInfo.MetaBinary = make([]byte, 0) - } + objectInfo.FilterPropertiesByResultOption(param.ResultOption) } pMetaInfo = append(pMetaInfo, objectInfo) diff --git a/datastore/search_object.go b/datastore/search_object.go index d5cb7de..7475813 100644 --- a/datastore/search_object.go +++ b/datastore/search_object.go @@ -43,26 +43,7 @@ func searchObject(err error, client *nex.Client, callID uint32, param *datastore continue } - // * This is kind of backwards. - // * The database pulls this data - // * by default, so it can be done - // * in a single query. So instead - // * of checking if a flag *IS* - // * set, and conditionally *ADDING* - // * the fields, we check if a flag - // * is *NOT* set and conditionally - // * *REMOVE* the field - if param.ResultOption&0x1 == 0 { - object.Tags = make([]string, 0) - } - - if param.ResultOption&0x2 == 0 { - object.Ratings = make([]*datastore_types.DataStoreRatingInfoWithSlot, 0) - } - - if param.ResultOption&0x4 == 0 { - object.MetaBinary = make([]byte, 0) - } + object.FilterPropertiesByResultOption(param.ResultOption) pSearchResult.Result = append(pSearchResult.Result, object) } diff --git a/go.mod b/go.mod index 4f4bdba..7350458 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/PretendoNetwork/nex-go v1.0.41 - github.com/PretendoNetwork/nex-protocols-go v1.0.56 + github.com/PretendoNetwork/nex-protocols-go v1.0.57 github.com/PretendoNetwork/plogger-go v1.0.4 github.com/minio/minio-go/v7 v7.0.63 golang.org/x/exp v0.0.0-20231006140011-7918f672742d diff --git a/go.sum b/go.sum index 679dfdf..a374d79 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/PretendoNetwork/nex-go v1.0.41 h1:TcBb1Bpe2KAB+AXaGX1K9NVQBRtZJIoy4yCvRde2xbI= github.com/PretendoNetwork/nex-go v1.0.41/go.mod h1:QwHEa165DeVd0xIuthrgc3j6NWXT8lyPSG6t5kC/Shk= -github.com/PretendoNetwork/nex-protocols-go v1.0.56 h1:cFoyANBXv2McRhBnVDsOWIXxoRv066qO1tkkknFL7WE= -github.com/PretendoNetwork/nex-protocols-go v1.0.56/go.mod h1:136762CMIcAsTZDrt4W7gDE4ppiBuc9zN2QFOHESjS8= +github.com/PretendoNetwork/nex-protocols-go v1.0.57 h1:7VosWcDuYbciGgWfgvRd2zDW7uTfcmFzlPbmaPkI7L4= +github.com/PretendoNetwork/nex-protocols-go v1.0.57/go.mod h1:136762CMIcAsTZDrt4W7gDE4ppiBuc9zN2QFOHESjS8= github.com/PretendoNetwork/plogger-go v1.0.4 h1:PF7xHw9eDRHH+RsAP9tmAE7fG0N0p6H4iPwHKnsoXwc= github.com/PretendoNetwork/plogger-go v1.0.4/go.mod h1:7kD6M4vPq1JL4LTuPg6kuB1OvUBOwQOtAvTaUwMbwvU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 271cc3fab2f1e68e05e6ac5281bf25ba2b3df160 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Mon, 30 Oct 2023 00:16:55 -0400 Subject: [PATCH 21/25] misspelled GetObjectSizeByDataID --- datastore/complete_post_object.go | 6 +++--- datastore/complete_post_objects.go | 6 +++--- datastore/protocol.go | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/datastore/complete_post_object.go b/datastore/complete_post_object.go index 08a38c8..f42358e 100644 --- a/datastore/complete_post_object.go +++ b/datastore/complete_post_object.go @@ -15,8 +15,8 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.getObjectSizeDataIDHandler == nil { - common_globals.Logger.Warning("GetObjectSizeDataID not defined") + if commonDataStoreProtocol.getObjectSizeByDataIDHandler == nil { + common_globals.Logger.Warning("GetObjectSizeByDataID not defined") return nex.Errors.Core.NotImplemented } @@ -45,7 +45,7 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat return nex.Errors.DataStore.NotFound } - objectSizeDB, errCode := commonDataStoreProtocol.getObjectSizeDataIDHandler(param.DataID) + objectSizeDB, errCode := commonDataStoreProtocol.getObjectSizeByDataIDHandler(param.DataID) if errCode != 0 { return errCode } diff --git a/datastore/complete_post_objects.go b/datastore/complete_post_objects.go index 32576d0..43cd548 100644 --- a/datastore/complete_post_objects.go +++ b/datastore/complete_post_objects.go @@ -14,8 +14,8 @@ func completePostObjects(err error, client *nex.Client, callID uint32, dataIDs [ return nex.Errors.Core.NotImplemented } - if commonDataStoreProtocol.getObjectSizeDataIDHandler == nil { - common_globals.Logger.Warning("GetObjectSizeDataID not defined") + if commonDataStoreProtocol.getObjectSizeByDataIDHandler == nil { + common_globals.Logger.Warning("GetObjectSizeByDataID not defined") return nex.Errors.Core.NotImplemented } @@ -39,7 +39,7 @@ func completePostObjects(err error, client *nex.Client, callID uint32, dataIDs [ return nex.Errors.DataStore.NotFound } - objectSizeDB, errCode := commonDataStoreProtocol.getObjectSizeDataIDHandler(dataID) + objectSizeDB, errCode := commonDataStoreProtocol.getObjectSizeByDataIDHandler(dataID) if errCode != 0 { return errCode } diff --git a/datastore/protocol.go b/datastore/protocol.go index 96fb158..a94187e 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -31,7 +31,7 @@ type CommonDataStoreProtocol struct { updateObjectPeriodByDataIDWithPasswordHandler func(dataID uint64, dataType uint16, password uint64) uint32 updateObjectMetaBinaryByDataIDWithPasswordHandler func(dataID uint64, metaBinary []byte, password uint64) uint32 updateObjectDataTypeByDataIDWithPasswordHandler func(dataID uint64, period uint16, password uint64) uint32 - getObjectSizeDataIDHandler func(dataID uint64) (uint32, uint32) + getObjectSizeByDataIDHandler func(dataID uint64) (uint32, uint32) updateObjectUploadCompletedByDataIDHandler func(dataID uint64, uploadCompleted bool) uint32 getObjectInfoByPersistenceTargetWithPasswordHandler func(persistenceTarget *datastore_types.DataStorePersistenceTarget, password uint64) (*datastore_types.DataStoreMetaInfo, uint32) getObjectInfoByDataIDWithPasswordHandler func(dataID uint64, password uint64) (*datastore_types.DataStoreMetaInfo, uint32) @@ -158,7 +158,7 @@ func (c *CommonDataStoreProtocol) UpdateObjectDataTypeByDataIDWithPassword(handl // GetObjectSizeDataID sets the GetObjectSizeDataID handler function func (c *CommonDataStoreProtocol) GetObjectSizeDataID(handler func(dataID uint64) (uint32, uint32)) { - c.getObjectSizeDataIDHandler = handler + c.getObjectSizeByDataIDHandler = handler } // UpdateObjectUploadCompletedByDataID sets the UpdateObjectUploadCompletedByDataID handler function From 444944d57b52dae073f63d2c6f43c109730d7a0e Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Mon, 30 Oct 2023 19:34:36 -0400 Subject: [PATCH 22/25] update VerifyObjectPermission to always allow an owner to access their objects --- datastore/protocol.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/datastore/protocol.go b/datastore/protocol.go index a94187e..07f7723 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -63,12 +63,17 @@ func (c *CommonDataStoreProtocol) VerifyObjectPermission(ownerPID, accessorPID u return nex.Errors.DataStore.InvalidArgument } + // * Owner can always access their own objects + if ownerPID == accessorPID { + return 0 + } + // * Allow anyone if permission.Permission == 0 { return 0 } - // * Allow friends + // * Allow only friends of the owner if permission.Permission == 1 { friendsList := c.getUserFriendPIDsHandler(ownerPID) @@ -77,7 +82,7 @@ func (c *CommonDataStoreProtocol) VerifyObjectPermission(ownerPID, accessorPID u } } - // * Allow people in permission.RecipientIDs + // * Allow only users whose PIDs are defined in permission.RecipientIDs if permission.Permission == 2 { if !slices.Contains(permission.RecipientIDs, accessorPID) { return nex.Errors.DataStore.PermissionDenied From 8095267d16e01e4b9aef6896cbca28fbffe21844 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 31 Oct 2023 16:14:39 -0400 Subject: [PATCH 23/25] unexport DataStore MinIO client --- datastore/complete_post_object.go | 18 +++++++++++------- datastore/complete_post_objects.go | 2 +- datastore/protocol.go | 8 ++++---- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/datastore/complete_post_object.go b/datastore/complete_post_object.go index f42358e..b399aa3 100644 --- a/datastore/complete_post_object.go +++ b/datastore/complete_post_object.go @@ -10,7 +10,7 @@ import ( ) func completePostObject(err error, client *nex.Client, callID uint32, param *datastore_types.DataStoreCompletePostParam) uint32 { - if commonDataStoreProtocol.MinIOClient == nil { + if commonDataStoreProtocol.minIOClient == nil { common_globals.Logger.Warning("MinIOClient not defined") return nex.Errors.Core.NotImplemented } @@ -35,10 +35,10 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat return nex.Errors.DataStore.Unknown } - if param.IsSuccess { - bucket := commonDataStoreProtocol.s3Bucket - key := fmt.Sprintf("%s/%d.bin", commonDataStoreProtocol.s3DataKeyBase, param.DataID) + bucket := commonDataStoreProtocol.s3Bucket + key := fmt.Sprintf("%s/%d.bin", commonDataStoreProtocol.s3DataKeyBase, param.DataID) + if param.IsSuccess { objectSizeS3, err := commonDataStoreProtocol.S3ObjectSize(bucket, key) if err != nil { common_globals.Logger.Error(err.Error()) @@ -61,9 +61,13 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat return errCode } } else { - errCode := commonDataStoreProtocol.deleteObjectByDataIDHandler(param.DataID) - if errCode != 0 { - return errCode + // * Query to see if the + _, err := commonDataStoreProtocol.S3StatObject(bucket, key) + if err != nil { + errCode := commonDataStoreProtocol.deleteObjectByDataIDHandler(param.DataID) + if errCode != 0 { + return errCode + } } } diff --git a/datastore/complete_post_objects.go b/datastore/complete_post_objects.go index 43cd548..e51ae9b 100644 --- a/datastore/complete_post_objects.go +++ b/datastore/complete_post_objects.go @@ -9,7 +9,7 @@ import ( ) func completePostObjects(err error, client *nex.Client, callID uint32, dataIDs []uint64) uint32 { - if commonDataStoreProtocol.MinIOClient == nil { + if commonDataStoreProtocol.minIOClient == nil { common_globals.Logger.Warning("MinIOClient not defined") return nex.Errors.Core.NotImplemented } diff --git a/datastore/protocol.go b/datastore/protocol.go index 07f7723..2a73982 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -24,7 +24,7 @@ type CommonDataStoreProtocol struct { s3DataKeyBase string s3NotifyKeyBase string rootCACert []byte - MinIOClient *minio.Client + minIOClient *minio.Client S3Presigner S3PresignerInterface getUserFriendPIDsHandler func(pid uint32) []uint32 getObjectInfoByDataIDHandler func(dataID uint64) (*datastore_types.DataStoreMetaInfo, uint32) @@ -46,7 +46,7 @@ type CommonDataStoreProtocol struct { } func (c *CommonDataStoreProtocol) S3StatObject(bucket, key string) (minio.ObjectInfo, error) { - return c.MinIOClient.StatObject(context.TODO(), bucket, key, minio.StatObjectOptions{}) + return c.minIOClient.StatObject(context.TODO(), bucket, key, minio.StatObjectOptions{}) } func (c *CommonDataStoreProtocol) S3ObjectSize(bucket, key string) (uint64, error) { @@ -127,8 +127,8 @@ func (c *CommonDataStoreProtocol) SetRootCACert(rootCACert []byte) { // SetMinIOClient sets the MinIO S3 client func (c *CommonDataStoreProtocol) SetMinIOClient(client *minio.Client) { - c.MinIOClient = client - c.SetS3Presigner(NewS3Presigner(c.MinIOClient)) + c.minIOClient = client + c.SetS3Presigner(NewS3Presigner(c.minIOClient)) } // SetS3Presigner sets the struct which creates presigned S3 URLs From 63149ad6f65db6ccffce31a8bde48a8b59d61569 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 31 Oct 2023 17:23:09 -0400 Subject: [PATCH 24/25] undo incomplete changes to CompletePostObject. Waiting for conversation to finish --- datastore/complete_post_object.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/datastore/complete_post_object.go b/datastore/complete_post_object.go index b399aa3..ee5ec9d 100644 --- a/datastore/complete_post_object.go +++ b/datastore/complete_post_object.go @@ -61,13 +61,9 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat return errCode } } else { - // * Query to see if the - _, err := commonDataStoreProtocol.S3StatObject(bucket, key) - if err != nil { - errCode := commonDataStoreProtocol.deleteObjectByDataIDHandler(param.DataID) - if errCode != 0 { - return errCode - } + errCode := commonDataStoreProtocol.deleteObjectByDataIDHandler(param.DataID) + if errCode != 0 { + return errCode } } From 0df41b9a214975816ed01803f57c362970119f4a Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Tue, 31 Oct 2023 18:10:24 -0400 Subject: [PATCH 25/25] check owner and upload status in CompletePostObject --- datastore/complete_post_object.go | 30 +++++++++++++++++++++++++++++- datastore/protocol.go | 6 ++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/datastore/complete_post_object.go b/datastore/complete_post_object.go index ee5ec9d..a97f5f9 100644 --- a/datastore/complete_post_object.go +++ b/datastore/complete_post_object.go @@ -15,6 +15,16 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat return nex.Errors.Core.NotImplemented } + if commonDataStoreProtocol.getObjectInfoByDataIDHandler == nil { + common_globals.Logger.Warning("GetObjectInfoByDataID not defined") + return nex.Errors.Core.NotImplemented + } + + if commonDataStoreProtocol.getObjectOwnerByDataIDHandler == nil { + common_globals.Logger.Warning("GetObjectOwnerByDataID not defined") + return nex.Errors.Core.NotImplemented + } + if commonDataStoreProtocol.getObjectSizeByDataIDHandler == nil { common_globals.Logger.Warning("GetObjectSizeByDataID not defined") return nex.Errors.Core.NotImplemented @@ -26,7 +36,7 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat } if commonDataStoreProtocol.deleteObjectByDataIDHandler == nil { - common_globals.Logger.Warning("DeleteObjectByDataIDHandler not defined") + common_globals.Logger.Warning("DeleteObjectByDataID not defined") return nex.Errors.Core.NotImplemented } @@ -35,6 +45,24 @@ func completePostObject(err error, client *nex.Client, callID uint32, param *dat return nex.Errors.DataStore.Unknown } + // * If GetObjectInfoByDataID returns data then that means + // * the object has already been marked as uploaded. So do + // * nothing + objectInfo, _ := commonDataStoreProtocol.getObjectInfoByDataIDHandler(param.DataID) + if objectInfo != nil { + return nex.Errors.DataStore.PermissionDenied + } + + // * Only allow an objects owner to make this request + ownerPID, errCode := commonDataStoreProtocol.getObjectOwnerByDataIDHandler(param.DataID) + if errCode != 0 { + return errCode + } + + if ownerPID != client.PID() { + return nex.Errors.DataStore.PermissionDenied + } + bucket := commonDataStoreProtocol.s3Bucket key := fmt.Sprintf("%s/%d.bin", commonDataStoreProtocol.s3DataKeyBase, param.DataID) diff --git a/datastore/protocol.go b/datastore/protocol.go index 2a73982..b1bfadf 100644 --- a/datastore/protocol.go +++ b/datastore/protocol.go @@ -43,6 +43,7 @@ type CommonDataStoreProtocol struct { deleteObjectByDataIDWithPasswordHandler func(dataID uint64, password uint64) uint32 deleteObjectByDataIDHandler func(dataID uint64) uint32 getObjectInfosByDataStoreSearchParamHandler func(param *datastore_types.DataStoreSearchParam) ([]*datastore_types.DataStoreMetaInfo, uint32, uint32) + getObjectOwnerByDataIDHandler func(dataID uint64) (uint32, uint32) } func (c *CommonDataStoreProtocol) S3StatObject(bucket, key string) (minio.ObjectInfo, error) { @@ -221,6 +222,11 @@ func (c *CommonDataStoreProtocol) GetObjectInfosByDataStoreSearchParam(handler f c.getObjectInfosByDataStoreSearchParamHandler = handler } +// GetObjectOwnerByDataID sets the GetObjectOwnerByDataID handler function +func (c *CommonDataStoreProtocol) GetObjectOwnerByDataID(handler func(dataID uint64) (uint32, uint32)) { + c.getObjectOwnerByDataIDHandler = handler +} + func initDefault(c *CommonDataStoreProtocol) { c.DefaultProtocol = datastore.NewProtocol(c.server) c.DefaultProtocol.DeleteObject(deleteObject)