Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

statistics: reduce the copy when to merge global status #47372

Merged
merged 3 commits into from
Oct 7, 2023
Merged

statistics: reduce the copy when to merge global status #47372

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e52a9d9
statistics: reduce the copy when to merge global status
hawkingrei Sep 28, 2023
d9866d1
u
hawkingrei Oct 7, 2023
72c1fea
use new skip partition function
hawkingrei Oct 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions statistics/cmsketch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -904,6 +904,9 @@ func CheckEmptyTopNs(topNs []*TopN) bool {
count := uint64(0)
for _, topN := range topNs {
count += topN.TotalCount()
if count != 0 {
return false
}
}
return count == 0
}
Expand Down
5 changes: 1 addition & 4 deletions statistics/handle/globalstats/global_stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ func MergePartitionStats2GlobalStats(
for i := 0; i < globalStats.Num; i++ {
// GetStatsInfo will return the copy of the statsInfo, so we don't need to worry about the data race.
// partitionStats will be released after the for loop.
hg, cms, topN, fms, analyzed := partitionStats.GetStatsInfo(histIDs[i], isIndex)
hg, cms, topN, fms, analyzed := partitionStats.GetStatsInfo(histIDs[i], isIndex, externalCache)
skipPartition := false
if !analyzed {
var missingPart string
Expand Down Expand Up @@ -201,9 +201,6 @@ func MergePartitionStats2GlobalStats(
allFms[i] = append(allFms[i], fms)
}
}
if !externalCache {
partitionStats.ReleaseAndPutToPool()
}
}

// After collect all the statistics from the partition-level stats,
Expand Down
20 changes: 15 additions & 5 deletions statistics/table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,18 +377,28 @@ func (t *Table) ColumnByName(colName string) *Column {
}

// GetStatsInfo returns their statistics according to the ID of the column or index, including histogram, CMSketch, TopN and FMSketch.
func (t *Table) GetStatsInfo(id int64, isIndex bool) (*Histogram, *CMSketch, *TopN, *FMSketch, bool) {
//
// needCopy: In order to protect the item in the cache from being damaged, we need to copy the item.
func (t *Table) GetStatsInfo(id int64, isIndex bool, needCopy bool) (*Histogram, *CMSketch, *TopN, *FMSketch, bool) {
winoros marked this conversation as resolved.
Show resolved Hide resolved
if isIndex {
if idxStatsInfo, ok := t.Indices[id]; ok {
return idxStatsInfo.Histogram.Copy(),
idxStatsInfo.CMSketch.Copy(), idxStatsInfo.TopN.Copy(), idxStatsInfo.FMSketch.Copy(), true
if needCopy {
return idxStatsInfo.Histogram.Copy(),
idxStatsInfo.CMSketch.Copy(), idxStatsInfo.TopN.Copy(), idxStatsInfo.FMSketch.Copy(), true
}
return &idxStatsInfo.Histogram,
idxStatsInfo.CMSketch, idxStatsInfo.TopN, idxStatsInfo.FMSketch, true
}
// newly added index which is not analyzed yet
return nil, nil, nil, nil, false
}
if colStatsInfo, ok := t.Columns[id]; ok {
return colStatsInfo.Histogram.Copy(), colStatsInfo.CMSketch.Copy(),
colStatsInfo.TopN.Copy(), colStatsInfo.FMSketch.Copy(), true
if needCopy {
return colStatsInfo.Histogram.Copy(), colStatsInfo.CMSketch.Copy(),
colStatsInfo.TopN.Copy(), colStatsInfo.FMSketch.Copy(), true
}
return &colStatsInfo.Histogram, colStatsInfo.CMSketch,
colStatsInfo.TopN, colStatsInfo.FMSketch, true
}
// newly added column which is not analyzed yet
return nil, nil, nil, nil, false
Expand Down