Return -1 to neredis if report olap table row count for new table is …

…not done for all tablets.
Jibing-Li · Sep 6, 2024 · ccaeccb · ccaeccb
1 parent 5d41b05
commit ccaeccb
Show file tree

Hide file tree

Showing 16 changed files with 181 additions and 55 deletions.
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
@@ -1065,6 +1065,7 @@ void TabletManager::build_all_report_tablets_info(std::map<TTabletId, TTablet>*
         t_tablet_stat.__set_row_count(tablet_info.row_count);
         t_tablet_stat.__set_total_version_count(tablet_info.total_version_count);
         t_tablet_stat.__set_visible_version_count(tablet_info.visible_version_count);
+        t_tablet_stat.__set_visible_version(tablet_info.version);
     };
     for_each_tablet(handler, filter_all_tablets);
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
@@ -75,8 +75,10 @@ public class ShowTableStatsStmt extends ShowStmt implements NotFallbackInParser
             new ImmutableList.Builder<String>()
             .add("table_name")
             .add("index_name")
-            .add("row_count")
+            .add("analyze_row_count")
             .add("update_time")
+            .add("report_row_count")
+            .add("report_row_count_for_nereids")
             .build();
 
     private static final ImmutableList<String> COLUMN_PARTITION_TITLE_NAMES =
@@ -200,47 +202,39 @@ public long getTableId() {
 
     public ShowResultSet constructResultSet(TableStatsMeta tableStatistic, TableIf table) {
         if (indexName != null) {
-            return constructIndexResultSet(tableStatistic);
+            return constructIndexResultSet(tableStatistic, table);
         }
         if (partitionNames == null) {
             return constructTableResultSet(tableStatistic, table);
         }
         if (columnNames == null) {
-            return constructPartitionResultSet(tableStatistic);
+            return constructPartitionResultSet(tableStatistic, table);
         } else {
-            return constructColumnPartitionResultSet(tableStatistic);
+            return constructColumnPartitionResultSet(tableStatistic, table);
         }
     }
 
     public ShowResultSet constructEmptyResultSet() {
         return new ShowResultSet(getMetaData(), new ArrayList<>());
     }
 
-    public ShowResultSet constructResultSet(TableIf table) {
-        List<List<String>> result = Lists.newArrayList();
-        if (partitionNames != null) {
-            // For partition, return empty result if table stats not exist.
-            return new ShowResultSet(getMetaData(), result);
-        }
-        List<String> row = Lists.newArrayList();
-        row.add("");
-        row.add("");
-        row.add(String.valueOf(table.getCachedRowCount()));
-        row.add("");
-        row.add("");
-        row.add("");
-        row.add("");
-        row.add("");
-        row.add(String.valueOf(table.autoAnalyzeEnabled()));
-        result.add(row);
-        return new ShowResultSet(getMetaData(), result);
-    }
-
     public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, TableIf table) {
-        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
         if (tableStatistic == null) {
-            return new ShowResultSet(getMetaData(), new ArrayList<>());
+            List<List<String>> result = Lists.newArrayList();
+            List<String> row = Lists.newArrayList();
+            row.add("");
+            row.add("");
+            row.add(String.valueOf(table.getCachedRowCount()));
+            row.add("");
+            row.add("");
+            row.add("");
+            row.add("");
+            row.add("");
+            row.add(String.valueOf(table.autoAnalyzeEnabled()));
+            result.add(row);
+            return new ShowResultSet(getMetaData(), result);
         }
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
         List<List<String>> result = Lists.newArrayList();
         List<String> row = Lists.newArrayList();
         row.add(String.valueOf(tableStatistic.updatedRows));
@@ -260,9 +254,9 @@ public ShowResultSet constructTableResultSet(TableStatsMeta tableStatistic, Tabl
         return new ShowResultSet(getMetaData(), result);
     }
 
-    public ShowResultSet constructPartitionResultSet(TableStatsMeta tableStatistic) {
+    public ShowResultSet constructPartitionResultSet(TableStatsMeta tableStatistic, TableIf table) {
         List<List<String>> result = Lists.newArrayList();
-        if (!(table instanceof OlapTable)) {
+        if (!(table instanceof OlapTable) || tableStatistic == null) {
             return new ShowResultSet(getMetaData(), result);
         }
         Collection<String> partitions = partitionNames.isStar()
@@ -286,7 +280,7 @@ public ShowResultSet constructPartitionResultSet(TableStatsMeta tableStatistic)
         return new ShowResultSet(getMetaData(), result);
     }
 
-    public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic) {
+    public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic, TableIf table) {
         List<List<String>> result = Lists.newArrayList();
         if (!(table instanceof OlapTable)) {
             return new ShowResultSet(getMetaData(), result);
@@ -296,23 +290,22 @@ public ShowResultSet constructIndexResultSet(TableStatsMeta tableStatistic) {
         if (indexId == null) {
             throw new RuntimeException(String.format("Index %s not exist.", indexName));
         }
-        long rowCount = tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
-        if (rowCount == -1) {
-            return new ShowResultSet(getMetaData(), result);
-        }
-        long updateTime = tableStatistic.getRowCountUpdateTime(olapTable.getIndexIdByName(indexName));
+        long rowCount = tableStatistic == null ? -1 : tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
+        long updateTime = tableStatistic == null ? -1 : tableStatistic.getRowCountUpdateTime(indexId);
         List<String> row = Lists.newArrayList();
         row.add(table.getName());
         row.add(indexName);
         row.add(String.valueOf(rowCount));
         row.add(String.valueOf(updateTime));
+        row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, false)));
+        row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, true)));
         result.add(row);
         return new ShowResultSet(getMetaData(), result);
     }
 
-    public ShowResultSet constructColumnPartitionResultSet(TableStatsMeta tableStatistic) {
+    public ShowResultSet constructColumnPartitionResultSet(TableStatsMeta tableStatistic, TableIf table) {
         List<List<String>> result = Lists.newArrayList();
-        if (!(table instanceof OlapTable)) {
+        if (!(table instanceof OlapTable) || tableStatistic == null) {
             return new ShowResultSet(getMetaData(), result);
         }
         OlapTable olapTable = (OlapTable) table;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
@@ -73,6 +73,8 @@ public enum IndexExtState {
     @SerializedName(value = "rollupFinishedVersion")
     private long rollupFinishedVersion;
 
+    private boolean rowCountReported = false;
+
     public MaterializedIndex() {
         this.state = IndexState.NORMAL;
         this.idToTablets = new HashMap<>();
@@ -206,6 +208,14 @@ public int getTabletOrderIdx(long tabletId) {
         return -1;
     }
 
+    public void setRowCountReported(boolean reported) {
+        this.rowCountReported = reported;
+    }
+
+    public boolean getRowCountReported() {
+        return this.rowCountReported;
+    }
+
     @Deprecated
     public void readFields(DataInput in) throws IOException {
         super.readFields(in);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1564,13 +1564,16 @@ public Set<Pair<String, String>> getColumnIndexPairs(Set<String> columns) {
 
     @Override
     public long fetchRowCount() {
-        return getRowCountForIndex(baseIndexId);
+        return getRowCountForIndex(baseIndexId, false);
     }
 
-    public long getRowCountForIndex(long indexId) {
+    public long getRowCountForIndex(long indexId, boolean strict) {
         long rowCount = 0;
         for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
             MaterializedIndex index = entry.getValue().getIndex(indexId);
+            if (strict && !index.getRowCountReported()) {
+                return -1;
+            }
             rowCount += (index == null || index.getRowCount() == -1) ? 0 : index.getRowCount();
         }
         return rowCount;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
@@ -177,6 +177,8 @@ public static class ReplicaContext {
 
     private long userDropTime = -1;
 
+    private long lastReportVersion = 0;
+
     public Replica() {
     }
 
@@ -839,4 +841,12 @@ public boolean isScheduleAvailable() {
         return Env.getCurrentSystemInfo().checkBackendScheduleAvailable(backendId)
             && !isUserDrop();
     }
+
+    public void setLastReportVersion(long version) {
+        this.lastReportVersion = version;
+    }
+
+    public long getLastReportVersion() {
+        return lastReportVersion;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -128,16 +128,34 @@ protected void runAfterCatalogReady() {
                         long version = partition.getVisibleVersion();
                         for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
                             long indexRowCount = 0L;
+                            boolean indexReported = true;
                             for (Tablet tablet : index.getTablets()) {
 
                                 Long tabletDataSize = 0L;
                                 Long tabletRemoteDataSize = 0L;
 
                                 Long tabletRowCount = 0L;
 
+                                boolean tabletReported = false;
                                 for (Replica replica : tablet.getReplicas()) {
+                                    LOG.debug("Table {} replica {} current version {}, report version {}",
+                                            olapTable.getName(), replica.getId(),
+                                            replica.getVersion(), replica.getLastReportVersion());
                                     if (replica.checkVersionCatchUp(version, false)
-                                            && replica.getRowCount() > tabletRowCount) {
+                                            && replica.getRowCount() >= tabletRowCount) {
+                                        // 1. If replica version and reported replica version are all equal to
+                                        // PARTITION_INIT_VERSION, set tabletReported to true, which indicates this
+                                        // tablet is empty for sure when previous report.
+                                        // 2. If last report version is larger than PARTITION_INIT_VERSION, set
+                                        // tabletReported to true as well. That is, we only guarantee all replicas of
+                                        // the tablet are reported for the init version.
+                                        // e.g. When replica version is 2, but last reported version is 1,
+                                        // tabletReported would be false.
+                                        if (replica.getVersion() == Partition.PARTITION_INIT_VERSION
+                                                && replica.getLastReportVersion() == Partition.PARTITION_INIT_VERSION
+                                                || replica.getLastReportVersion() > Partition.PARTITION_INIT_VERSION) {
+                                            tabletReported = true;
+                                        }
                                         tabletRowCount = replica.getRowCount();
                                     }
 
@@ -157,8 +175,14 @@ protected void runAfterCatalogReady() {
 
                                 tableRowCount += tabletRowCount;
                                 indexRowCount += tabletRowCount;
+                                // Only when all tablets of this index are reported, we set indexReported to true.
+                                indexReported = indexReported && tabletReported;
                             } // end for tablets
+                            index.setRowCountReported(indexReported);
                             index.setRowCount(indexRowCount);
+                            LOG.debug("Table {} index {} all tablets reported[{}], row count {}",
+                                    olapTable.getName(), olapTable.getIndexNameById(index.getId()),
+                                    indexReported, tableRowCount);
                         } // end for indices
                     } // end for partitions
 
@@ -193,6 +217,9 @@ private void updateTabletStat(Long beId, TTabletStatResult result) {
                         replica.setTotalVersionCount(stat.getTotalVersionCount());
                         replica.setVisibleVersionCount(stat.isSetVisibleVersionCount() ? stat.getVisibleVersionCount()
                                 : stat.getTotalVersionCount());
+                        // Older version BE doesn't set visible version. Set it to max for compatibility.
+                        replica.setLastReportVersion(stat.isSetVisibleVersion() ? stat.getVisibleVersion()
+                                : Long.MAX_VALUE);
                     }
                 }
             }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -366,7 +366,7 @@ private void checkIfUnknownStatsUsedAsKey(StatisticsBuilder builder) {
 
     private Statistics computeOlapScan(OlapScan olapScan) {
         OlapTable olapTable = olapScan.getTable();
-        double tableRowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId());
+        double tableRowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true);
         if (tableRowCount <= 0) {
             AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
             TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId());

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -2725,14 +2725,7 @@ private void handleShowTableStats() {
             return;
         }
         TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId());
-        /*
-           tableStats == null means it's not analyzed, in this case show the estimated row count.
-         */
-        if (tableStats == null) {
-            resultSet = showTableStatsStmt.constructResultSet(tableIf);
-        } else {
-            resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf);
-        }
+        resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf);
     }
 
     private void handleShowColumnStats() throws AnalysisException {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -3319,6 +3319,9 @@ public TStatus invalidateStatsCache(TInvalidateFollowerStatsCacheRequest request
             partitionNames = new PartitionNames(false, new ArrayList<>(target.partitions));
         }
         if (target.isTruncate) {
+            if (partitionNames == null || partitionNames.isStar() || partitionNames.getPartitionNames() == null) {
+                tableStats.clearIndexesRowCountAndTime();
+            }
             analysisManager.submitAsyncDropStatsTask(target.catalogId, target.dbId,
                     target.tableId, tableStats, partitionNames);
         } else {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -682,6 +682,9 @@ public void dropStats(TableIf table, PartitionNames partitionNames) throws DdlEx
         long catalogId = table.getDatabase().getCatalog().getId();
         long dbId = table.getDatabase().getId();
         long tableId = table.getId();
+        if (partitionNames == null || partitionNames.isStar() || partitionNames.getPartitionNames() == null) {
+            tableStats.clearIndexesRowCountAndTime();
+        }
         submitAsyncDropStatsTask(catalogId, dbId, tableId, tableStats, partitionNames);
         // Drop stats ddl is master only operation.
         Set<String> partitions = null;
@@ -810,6 +813,8 @@ public void invalidateLocalStats(long catalogId, long dbId, long tableId, Set<St
         }
         if (allColumn && allPartition) {
             tableStats.removeAllColumn();
+            tableStats.clearIndexesRowCountAndTime();
+            removeTableStats(tableId);
         }
         tableStats.updatedTime = 0;
         tableStats.userInjected = false;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -101,7 +101,7 @@ protected void doSample() {
         List<Long> tabletIds = pair.first;
         long totalRowCount = info.indexId == -1
                 ? tbl.getRowCount()
-                : ((OlapTable) tbl).getRowCountForIndex(info.indexId);
+                : ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
         double scaleFactor = (double) totalRowCount / (double) pair.second;
         // might happen if row count in fe metadata hasn't been updated yet
         if (Double.isInfinite(scaleFactor) || Double.isNaN(scaleFactor)) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -101,10 +101,10 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
     public ConcurrentMap<Long, Long> partitionUpdateRows = new ConcurrentHashMap<>();
 
     @SerializedName("irc")
-    public ConcurrentMap<Long, Long> indexesRowCount = new ConcurrentHashMap<>();
+    private ConcurrentMap<Long, Long> indexesRowCount = new ConcurrentHashMap<>();
 
     @SerializedName("ircut")
-    public ConcurrentMap<Long, Long> indexesRowCountUpdateTime = new ConcurrentHashMap<>();
+    private ConcurrentMap<Long, Long> indexesRowCountUpdateTime = new ConcurrentHashMap<>();
 
     @VisibleForTesting
     public TableStatsMeta() {
@@ -240,6 +240,11 @@ public long getRowCountUpdateTime(long indexId) {
         return indexesRowCountUpdateTime.getOrDefault(indexId, 0L);
     }
 
+    public void clearIndexesRowCountAndTime() {
+        indexesRowCount.clear();
+        indexesRowCountUpdateTime.clear();
+    }
+
     private void clearStaleIndexRowCountAndTime(OlapTable table) {
         Iterator<Long> iterator = indexesRowCount.keySet().iterator();
         List<Long> indexIds = table.getIndexIds();
@@ -259,8 +264,8 @@ private void clearStaleIndexRowCountAndTime(OlapTable table) {
     }
 
     public long getBaseIndexDeltaRowCount(OlapTable table) {
-        if (colToColStatsMeta == null) {
-            return -1;
+        if (colToColStatsMeta == null || colToColStatsMeta.isEmpty() || userInjected) {
+            return 0;
         }
         long maxUpdateRows = 0;
         String baseIndexName = table.getIndexNameById(table.getBaseIndexId());

diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -2294,6 +2294,9 @@ private boolean updateCatalogAfterVisible(TransactionState transactionState, Dat
                                         replica.getId(), newVersion, lastFailedVersion, lastSuccessVersion);
                             }
                             replica.updateVersionWithFailed(newVersion, lastFailedVersion, lastSuccessVersion);
+                            if (newVersion == Partition.PARTITION_INIT_VERSION + 1) {
+                                index.setRowCountReported(false);
+                            }
                             Set<Long> partitionIds = backendPartitions.get(replica.getBackendId());
                             if (partitionIds == null) {
                                 partitionIds = Sets.newHashSet();

diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift
@@ -38,6 +38,7 @@ struct TTabletStat {
     4: optional i64 total_version_count
     5: optional i64 remote_data_size
     6: optional i64 visible_version_count
+    7: optional i64 visible_version
 }
 
 struct TTabletStatResult {

diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy
@@ -2743,7 +2743,7 @@ PARTITION `p599` VALUES IN (599)
     assertEquals("true", alter_result[0][7])
     sql """drop stats alter_test"""
     alter_result = sql """show table stats alter_test"""
-    assertEquals("false", alter_result[0][7])
+    assertEquals("", alter_result[0][7])
     sql """alter table alter_test modify column id set stats ('row_count'='100', 'ndv'='0', 'num_nulls'='0.0', 'data_size'='2.69975443E8', 'min_value'='1', 'max_value'='2');"""
     alter_result = sql """show column stats alter_test(id)"""
     logger.info("show column alter_test(id) stats: " + alter_result)