VAULT-22642: Include secret syncs in activity log responses

changelog/24710.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core/activity: Include secret_syncs in activity log responses
+```

vault/activity/query.go

@@ -32,6 +32,10 @@ type CountsRecord struct {
 	SecretSyncs      int `json:"secret_syncs"`
 }
 
+func (c *CountsRecord) HasCounts() bool {
+	return c.EntityClients+c.NonEntityClients+c.SecretSyncs != 0
+}
+
 type NewClientRecord struct {
 	Counts     *CountsRecord             `json:"counts"`
 	Namespaces []*MonthlyNamespaceRecord `json:"namespaces"`

vault/activity_log.go

@@ -1578,6 +1578,19 @@ type ResponseCounts struct {
 	NonEntityTokens  int `json:"non_entity_tokens" mapstructure:"non_entity_tokens"`
 	NonEntityClients int `json:"non_entity_clients" mapstructure:"non_entity_clients"`
 	Clients          int `json:"clients"`
+	SecretSyncs      int `json:"secret_syncs" mapstructure:"secret_syncs"`
+}
+
+func (existingRecord *ResponseCounts) Add(newRecord *ResponseCounts) {
+	if newRecord == nil {
+		return
+	}
+	existingRecord.EntityClients += newRecord.EntityClients
+	existingRecord.Clients += newRecord.Clients
+	existingRecord.DistinctEntities += newRecord.DistinctEntities
+	existingRecord.NonEntityClients += newRecord.NonEntityClients
+	existingRecord.NonEntityTokens += newRecord.NonEntityTokens
+	existingRecord.SecretSyncs += newRecord.SecretSyncs
 }
 
 type ResponseNamespace struct {
@@ -1587,6 +1600,27 @@ type ResponseNamespace struct {
 	Mounts        []*ResponseMount `json:"mounts"`
 }
 
+func (existingRecord *ResponseNamespace) Add(newRecord *ResponseNamespace) {
+	// Create a map of the existing mounts, so we don't duplicate them
+	mountMap := make(map[string]*ResponseCounts)
+	for _, erm := range existingRecord.Mounts {
+		mountMap[erm.MountPath] = erm.Counts
+	}
+
+	existingRecord.Counts.Add(&newRecord.Counts)
+
+	// Check the current month mounts against the existing mounts and if there are matches, update counts
+	// accordingly. If there is no match, append the new mount to the existing mounts, so it will be counted
+	// later.
+	for _, newRecordMount := range newRecord.Mounts {
+		if existingRecordMountCounts, ook := mountMap[newRecordMount.MountPath]; ook {
+			existingRecordMountCounts.Add(&newRecord.Counts)
+		} else {
+			existingRecord.Mounts = append(existingRecord.Mounts, newRecordMount)
+		}
+	}
+}
+
 type ResponseMonth struct {
 	Timestamp  string               `json:"timestamp"`
 	Counts     *ResponseCounts      `json:"counts"`
@@ -1681,7 +1715,7 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
 
 	// Calculate the namespace response breakdowns and totals for entities and tokens from the initial
 	// namespace data.
-	totalEntities, totalTokens, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, pq.Namespaces)
+	totalCounts, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, pq.Namespaces)
 	if err != nil {
 		return nil, err
 	}
@@ -1690,9 +1724,8 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
 	// breakdown for the current month as well.
 	var partialByMonth map[int64]*processMonth
 	var partialByNamespace map[string]*processByNamespace
-	var totalEntitiesCurrent int
-	var totalTokensCurrent int
 	var byNamespaceResponseCurrent []*ResponseNamespace
+	var totalCurrentCounts *ResponseCounts
 	if computePartial {
 		// Traverse through current month's activitylog data and group clients
 		// into months and namespaces
@@ -1705,9 +1738,9 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
 		// endpoints.
 		byNamespaceComputation := a.transformALNamespaceBreakdowns(partialByNamespace)
 
-		// Calculate the namespace response breakdowns and totals for entities and tokens from the initial
-		// namespace data.
-		totalEntitiesCurrent, totalTokensCurrent, byNamespaceResponseCurrent, err = a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
+		// Calculate the namespace response breakdowns and totals for entities
+		// and tokens from current month namespace data.
+		totalCurrentCounts, byNamespaceResponseCurrent, err = a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
 		if err != nil {
 			return nil, err
 		}
@@ -1723,34 +1756,7 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
 		// month counts, and append or update as necessary. We also want to account for mounts and their counts.
 		for _, nrc := range byNamespaceResponseCurrent {
 			if ndx, ok := nsrMap[nrc.NamespaceID]; ok {
-				existingRecord := byNamespaceResponse[ndx]
-
-				// Create a map of the existing mounts, so we don't duplicate them
-				mountMap := make(map[string]*ResponseCounts)
-				for _, erm := range existingRecord.Mounts {
-					mountMap[erm.MountPath] = erm.Counts
-				}
-
-				existingRecord.Counts.EntityClients += nrc.Counts.EntityClients
-				existingRecord.Counts.Clients += nrc.Counts.Clients
-				existingRecord.Counts.DistinctEntities += nrc.Counts.DistinctEntities
-				existingRecord.Counts.NonEntityClients += nrc.Counts.NonEntityClients
-				existingRecord.Counts.NonEntityTokens += nrc.Counts.NonEntityTokens
-
-				// Check the current month mounts against the existing mounts and if there are matches, update counts
-				// accordingly. If there is no match, append the new mount to the existing mounts, so it will be counted
-				// later.
-				for _, nrcMount := range nrc.Mounts {
-					if existingRecordMountCounts, ook := mountMap[nrcMount.MountPath]; ook {
-						existingRecordMountCounts.EntityClients += nrcMount.Counts.EntityClients
-						existingRecordMountCounts.Clients += nrcMount.Counts.Clients
-						existingRecordMountCounts.DistinctEntities += nrcMount.Counts.DistinctEntities
-						existingRecordMountCounts.NonEntityClients += nrcMount.Counts.NonEntityClients
-						existingRecordMountCounts.NonEntityTokens += nrcMount.Counts.NonEntityTokens
-					} else {
-						existingRecord.Mounts = append(existingRecord.Mounts, nrcMount)
-					}
-				}
+				byNamespaceResponse[ndx].Add(nrc)
 			} else {
 				byNamespaceResponse = append(byNamespaceResponse, nrc)
 			}
@@ -1761,10 +1767,10 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
 	a.sortALResponseNamespaces(byNamespaceResponse)
 
 	if limitNamespaces > 0 {
-		totalEntities, totalTokens, byNamespaceResponse = a.limitNamespacesInALResponse(byNamespaceResponse, limitNamespaces)
+		totalCounts, byNamespaceResponse = a.limitNamespacesInALResponse(byNamespaceResponse, limitNamespaces)
 	}
 
-	distinctEntitiesResponse := totalEntities
+	distinctEntitiesResponse := totalCounts.EntityClients
 	if computePartial {
 		currentMonth, err := a.computeCurrentMonthForBillingPeriod(ctx, partialByMonth, startTime, endTime)
 		if err != nil {
@@ -1808,13 +1814,9 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
 	}
 
 	responseData["by_namespace"] = byNamespaceResponse
-	responseData["total"] = &ResponseCounts{
-		DistinctEntities: distinctEntitiesResponse,
-		EntityClients:    totalEntities + totalEntitiesCurrent,
-		NonEntityTokens:  totalTokens + totalTokensCurrent,
-		NonEntityClients: totalTokens + totalTokensCurrent,
-		Clients:          totalEntities + totalEntitiesCurrent + totalTokens + totalTokensCurrent,
-	}
+	totalCounts.Add(totalCurrentCounts)
+	totalCounts.DistinctEntities = distinctEntitiesResponse
+	responseData["total"] = totalCounts
 
 	// Create and populate the month response structs based on the monthly breakdown.
 	months, err := a.prepareMonthsResponseForQuery(ctx, pq.Months)
@@ -2610,32 +2612,25 @@ func (a *ActivityLog) transformMonthBreakdowns(byMonth map[int64]*processMonth)
 	return monthly
 }
 
-func (a *ActivityLog) calculateByNamespaceResponseForQuery(ctx context.Context, byNamespace []*activity.NamespaceRecord) (int, int, []*ResponseNamespace, error) {
+func (a *ActivityLog) calculateByNamespaceResponseForQuery(ctx context.Context, byNamespace []*activity.NamespaceRecord) (*ResponseCounts, []*ResponseNamespace, error) {
 	queryNS, err := namespace.FromContext(ctx)
 	if err != nil {
-		return 0, 0, nil, err
+		return nil, nil, err
 	}
 	byNamespaceResponse := make([]*ResponseNamespace, 0)
-	totalEntities := 0
-	totalTokens := 0
+	totalCounts := &ResponseCounts{}
 
 	for _, nsRecord := range byNamespace {
 		ns, err := NamespaceByID(ctx, nsRecord.NamespaceID, a.core)
 		if err != nil {
-			return 0, 0, nil, err
+			return nil, nil, err
 		}
 		if a.includeInResponse(queryNS, ns) {
 			mountResponse := make([]*ResponseMount, 0, len(nsRecord.Mounts))
 			for _, mountRecord := range nsRecord.Mounts {
 				mountResponse = append(mountResponse, &ResponseMount{
 					MountPath: mountRecord.MountPath,
-					Counts: &ResponseCounts{
-						DistinctEntities: int(mountRecord.Counts.EntityClients),
-						EntityClients:    int(mountRecord.Counts.EntityClients),
-						NonEntityClients: int(mountRecord.Counts.NonEntityClients),
-						NonEntityTokens:  int(mountRecord.Counts.NonEntityClients),
-						Clients:          int(mountRecord.Counts.EntityClients + mountRecord.Counts.NonEntityClients),
-					},
+					Counts:    a.countsRecordToCountsResponse(mountRecord.Counts, true),
 				})
 			}
 			// Sort the mounts in descending order of usage
@@ -2649,55 +2644,41 @@ func (a *ActivityLog) calculateByNamespaceResponseForQuery(ctx context.Context,
 			} else {
 				displayPath = ns.Path
 			}
+			nsCounts := a.namespaceRecordToCountsResponse(nsRecord)
 			byNamespaceResponse = append(byNamespaceResponse, &ResponseNamespace{
 				NamespaceID:   nsRecord.NamespaceID,
 				NamespacePath: displayPath,
-				Counts: ResponseCounts{
-					DistinctEntities: int(nsRecord.Entities),
-					EntityClients:    int(nsRecord.Entities),
-					NonEntityTokens:  int(nsRecord.NonEntityTokens),
-					NonEntityClients: int(nsRecord.NonEntityTokens),
-					Clients:          int(nsRecord.Entities + nsRecord.NonEntityTokens),
-				},
-				Mounts: mountResponse,
+				Counts:        *nsCounts,
+				Mounts:        mountResponse,
 			})
-			totalEntities += int(nsRecord.Entities)
-			totalTokens += int(nsRecord.NonEntityTokens)
+			totalCounts.Add(nsCounts)
 		}
 	}
-	return totalEntities, totalTokens, byNamespaceResponse, nil
+	return totalCounts, byNamespaceResponse, nil
 }
 
 func (a *ActivityLog) prepareMonthsResponseForQuery(ctx context.Context, byMonth []*activity.MonthRecord) ([]*ResponseMonth, error) {
 	months := make([]*ResponseMonth, 0, len(byMonth))
 	for _, monthsRecord := range byMonth {
 		newClientsResponse := &ResponseNewClients{}
-		if int(monthsRecord.NewClients.Counts.EntityClients+monthsRecord.NewClients.Counts.NonEntityClients) != 0 {
+		if monthsRecord.NewClients.Counts.HasCounts() {
 			newClientsNSResponse, err := a.prepareNamespaceResponse(ctx, monthsRecord.NewClients.Namespaces)
 			if err != nil {
 				return nil, err
 			}
-			newClientsResponse.Counts = &ResponseCounts{
-				EntityClients:    int(monthsRecord.NewClients.Counts.EntityClients),
-				NonEntityClients: int(monthsRecord.NewClients.Counts.NonEntityClients),
-				Clients:          int(monthsRecord.NewClients.Counts.EntityClients + monthsRecord.NewClients.Counts.NonEntityClients),
-			}
+			newClientsResponse.Counts = a.countsRecordToCountsResponse(monthsRecord.NewClients.Counts, false)
 			newClientsResponse.Namespaces = newClientsNSResponse
 		}
 
 		monthResponse := &ResponseMonth{
 			Timestamp: time.Unix(monthsRecord.Timestamp, 0).UTC().Format(time.RFC3339),
 		}
-		if int(monthsRecord.Counts.EntityClients+monthsRecord.Counts.NonEntityClients) != 0 {
+		if monthsRecord.Counts.HasCounts() {
 			nsResponse, err := a.prepareNamespaceResponse(ctx, monthsRecord.Namespaces)
 			if err != nil {
 				return nil, err
 			}
-			monthResponse.Counts = &ResponseCounts{
-				EntityClients:    int(monthsRecord.Counts.EntityClients),
-				NonEntityClients: int(monthsRecord.Counts.NonEntityClients),
-				Clients:          int(monthsRecord.Counts.EntityClients + monthsRecord.Counts.NonEntityClients),
-			}
+			monthResponse.Counts = a.countsRecordToCountsResponse(monthsRecord.Counts, false)
 			monthResponse.Namespaces = nsResponse
 			monthResponse.NewClients = newClientsResponse
 			months = append(months, monthResponse)
@@ -2715,7 +2696,7 @@ func (a *ActivityLog) prepareNamespaceResponse(ctx context.Context, nsRecords []
 	}
 	nsResponse := make([]*ResponseNamespace, 0, len(nsRecords))
 	for _, nsRecord := range nsRecords {
-		if int(nsRecord.Counts.EntityClients) == 0 && int(nsRecord.Counts.NonEntityClients) == 0 {
+		if !nsRecord.Counts.HasCounts() {
 			continue
 		}
 
@@ -2726,17 +2707,13 @@ func (a *ActivityLog) prepareNamespaceResponse(ctx context.Context, nsRecords []
 		if a.includeInResponse(queryNS, ns) {
 			mountResponse := make([]*ResponseMount, 0, len(nsRecord.Mounts))
 			for _, mountRecord := range nsRecord.Mounts {
-				if int(mountRecord.Counts.EntityClients) == 0 && int(mountRecord.Counts.NonEntityClients) == 0 {
+				if !mountRecord.Counts.HasCounts() {
 					continue
 				}
 
 				mountResponse = append(mountResponse, &ResponseMount{
 					MountPath: mountRecord.MountPath,
-					Counts: &ResponseCounts{
-						EntityClients:    int(mountRecord.Counts.EntityClients),
-						NonEntityClients: int(mountRecord.Counts.NonEntityClients),
-						Clients:          int(mountRecord.Counts.EntityClients + mountRecord.Counts.NonEntityClients),
-					},
+					Counts:    a.countsRecordToCountsResponse(mountRecord.Counts, false),
 				})
 			}
 
@@ -2749,12 +2726,8 @@ func (a *ActivityLog) prepareNamespaceResponse(ctx context.Context, nsRecords []
 			nsResponse = append(nsResponse, &ResponseNamespace{
 				NamespaceID:   nsRecord.NamespaceID,
 				NamespacePath: displayPath,
-				Counts: ResponseCounts{
-					EntityClients:    int(nsRecord.Counts.EntityClients),
-					NonEntityClients: int(nsRecord.Counts.NonEntityClients),
-					Clients:          int(nsRecord.Counts.EntityClients + nsRecord.Counts.NonEntityClients),
-				},
-				Mounts: mountResponse,
+				Counts:        *a.countsRecordToCountsResponse(nsRecord.Counts, false),
+				Mounts:        mountResponse,
 			})
 		}
 	}
@@ -2783,7 +2756,7 @@ func (a *ActivityLog) partialMonthClientCount(ctx context.Context) (map[string]i
 
 	// Calculate the namespace response breakdowns and totals for entities and tokens from the initial
 	// namespace data.
-	totalEntities, totalTokens, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
+	totalCounts, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
 	if err != nil {
 		return nil, err
 	}
@@ -2794,11 +2767,12 @@ func (a *ActivityLog) partialMonthClientCount(ctx context.Context) (map[string]i
 	// Now populate the response based on breakdowns.
 	responseData := make(map[string]interface{})
 	responseData["by_namespace"] = byNamespaceResponse
-	responseData["distinct_entities"] = totalEntities
-	responseData["entity_clients"] = totalEntities
-	responseData["non_entity_tokens"] = totalTokens
-	responseData["non_entity_clients"] = totalTokens
-	responseData["clients"] = totalEntities + totalTokens
+	responseData["distinct_entities"] = totalCounts.EntityClients
+	responseData["entity_clients"] = totalCounts.EntityClients
+	responseData["non_entity_tokens"] = totalCounts.NonEntityClients
+	responseData["non_entity_clients"] = totalCounts.NonEntityClients
+	responseData["clients"] = totalCounts.Clients
+	responseData["secret_syncs"] = totalCounts.SecretSyncs
 
 	// The partialMonthClientCount should not have more than one month worth of data.
 	// If it does, something has gone wrong and we should warn that the activity log data

vault/activity_log_util_common.go

@@ -197,19 +197,17 @@ func (a *ActivityLog) transformALNamespaceBreakdowns(nsData map[string]*processB
 
 // limitNamespacesInALResponse will truncate the number of namespaces shown in the activity
 // endpoints to the number specified in limitNamespaces (the API filtering parameter)
-func (a *ActivityLog) limitNamespacesInALResponse(byNamespaceResponse []*ResponseNamespace, limitNamespaces int) (int, int, []*ResponseNamespace) {
+func (a *ActivityLog) limitNamespacesInALResponse(byNamespaceResponse []*ResponseNamespace, limitNamespaces int) (*ResponseCounts, []*ResponseNamespace) {
 	if limitNamespaces > len(byNamespaceResponse) {
 		limitNamespaces = len(byNamespaceResponse)
 	}
 	byNamespaceResponse = byNamespaceResponse[:limitNamespaces]
 	// recalculate total entities and tokens
-	totalEntities := 0
-	totalTokens := 0
+	totalCounts := &ResponseCounts{}
 	for _, namespaceData := range byNamespaceResponse {
-		totalEntities += namespaceData.Counts.DistinctEntities
-		totalTokens += namespaceData.Counts.NonEntityTokens
+		totalCounts.Add(&namespaceData.Counts)
 	}
-	return totalEntities, totalTokens, byNamespaceResponse
+	return totalCounts, byNamespaceResponse
 }
 
 // transformActivityLogMounts is a helper used to reformat data for transformMonthlyNamespaceBreakdowns.
@@ -383,3 +381,28 @@ func (e *segmentReader) ReadEntity(ctx context.Context) (*activity.EntityActivit
 	}
 	return out, nil
 }
+
+func (a *ActivityLog) countsRecordToCountsResponse(record *activity.CountsRecord, includeDeprecated bool) *ResponseCounts {
+	response := &ResponseCounts{
+		EntityClients:    record.EntityClients,
+		NonEntityClients: record.NonEntityClients,
+		Clients:          record.EntityClients + record.NonEntityClients,
+		SecretSyncs:      record.SecretSyncs,
+	}
+	if includeDeprecated {
+		response.NonEntityTokens = response.NonEntityClients
+		response.DistinctEntities = response.EntityClients
+	}
+	return response
+}
+
+func (a *ActivityLog) namespaceRecordToCountsResponse(record *activity.NamespaceRecord) *ResponseCounts {
+	return &ResponseCounts{
+		DistinctEntities: int(record.Entities),
+		EntityClients:    int(record.Entities),
+		NonEntityTokens:  int(record.NonEntityTokens),
+		NonEntityClients: int(record.NonEntityTokens),
+		Clients:          int(record.Entities + record.NonEntityTokens),
+		SecretSyncs:      int(record.SecretSyncs),
+	}
+}