Merge branch 'master' into persist-partition-learner

src/common/base/SanitizerOptions.cpp

@@ -21,7 +21,9 @@ const char* __asan_default_options() {
            "fast_unwind_on_malloc=0 \n"
            "detect_stack_use_after_return=1 \n"
            "alloc_dealloc_mismatch=1 \n"
-           "new_delete_type_mismatch=1 \n"
+           // todo(doodle): Reopen when https://github.com/vesoft-inc/nebula/issues/3690
+           // addressed throughly
+           "new_delete_type_mismatch=0 \n"
            "strict_init_order=1 \n"
            "intercept_tls_get_addr=1 \n"
            "symbolize_inline_frames=1 \n"

src/common/expression/PropertyExpression.h

@@ -155,6 +155,10 @@ class LabelTagPropertyExpression final : public PropertyExpression {
     return label_;
   }
 
+  void setLabel(Expression* label) {
+    label_ = label;
+  }
+
  private:
   LabelTagPropertyExpression(ObjectPool* pool,
                              Expression* label = nullptr,

src/graph/optimizer/CMakeLists.txt

@@ -47,6 +47,7 @@ nebula_add_library(
     rule/PushLimitDownEdgeIndexPrefixScanRule.cpp
     rule/PushLimitDownEdgeIndexRangeScanRule.cpp
     rule/PushLimitDownProjectRule.cpp
+    rule/EliminateRowCollectRule.cpp
     rule/PushLimitDownScanAppendVerticesRule.cpp
     rule/GetEdgesTransformRule.cpp
     rule/PushLimitDownScanEdgesAppendVerticesRule.cpp

src/graph/optimizer/OptGroup.cpp

@@ -45,6 +45,7 @@ void OptGroup::addGroupNode(OptGroupNode *groupNode) {
   DCHECK(groupNode != nullptr);
   DCHECK(groupNode->group() == this);
   groupNodes_.emplace_back(groupNode);
+  groupNode->node()->updateSymbols();
 }
 
 OptGroupNode *OptGroup::makeGroupNode(PlanNode *node) {

src/graph/optimizer/rule/EliminateRowCollectRule.cpp

@@ -0,0 +1,73 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License.
+ */
+
+#include "graph/optimizer/rule/EliminateRowCollectRule.h"
+
+#include "graph/optimizer/OptContext.h"
+#include "graph/optimizer/OptGroup.h"
+#include "graph/planner/plan/PlanNode.h"
+#include "graph/planner/plan/Query.h"
+
+using nebula::graph::DataCollect;
+using nebula::graph::PlanNode;
+using nebula::graph::Project;
+using nebula::graph::QueryContext;
+
+namespace nebula {
+namespace opt {
+
+std::unique_ptr<OptRule> EliminateRowCollectRule::kInstance =
+    std::unique_ptr<EliminateRowCollectRule>(new EliminateRowCollectRule());
+
+EliminateRowCollectRule::EliminateRowCollectRule() {
+  RuleSet::QueryRules().addRule(this);
+}
+
+// TODO match DataCollect->(Any Node with real result)
+const Pattern& EliminateRowCollectRule::pattern() const {
+  static Pattern pattern = Pattern::create(graph::PlanNode::Kind::kDataCollect,
+                                           {Pattern::create(graph::PlanNode::Kind::kProject)});
+  return pattern;
+}
+
+bool EliminateRowCollectRule::match(OptContext* octx, const MatchedResult& matched) const {
+  if (!OptRule::match(octx, matched)) {
+    return false;
+  }
+  const auto* collectNode = static_cast<const DataCollect*>(matched.node->node());
+  if (collectNode->kind() != DataCollect::DCKind::kRowBasedMove) {
+    return false;
+  }
+  return true;
+}
+
+StatusOr<OptRule::TransformResult> EliminateRowCollectRule::transform(
+    OptContext* octx, const MatchedResult& matched) const {
+  auto dataCollectGroupNode = matched.node;
+  auto projGroupNode = matched.dependencies.front().node;
+
+  const auto dataCollect = static_cast<const DataCollect*>(dataCollectGroupNode->node());
+  const auto proj = static_cast<const Project*>(projGroupNode->node());
+
+  auto newProj = static_cast<Project*>(proj->clone());
+  newProj->setOutputVar(dataCollect->outputVar());
+  auto newProjGroupNode = OptGroupNode::create(octx, newProj, dataCollectGroupNode->group());
+
+  for (auto dep : projGroupNode->dependencies()) {
+    newProjGroupNode->dependsOn(dep);
+  }
+
+  TransformResult result;
+  result.eraseAll = true;
+  result.newGroupNodes.emplace_back(newProjGroupNode);
+  return result;
+}
+
+std::string EliminateRowCollectRule::toString() const {
+  return "EliminateRowCollectRule";
+}
+
+}  // namespace opt
+}  // namespace nebula

src/graph/optimizer/rule/EliminateRowCollectRule.h

@@ -0,0 +1,30 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License.
+ */
+
+#pragma once
+
+#include "graph/optimizer/OptRule.h"
+
+namespace nebula {
+namespace opt {
+
+class EliminateRowCollectRule final : public OptRule {
+ public:
+  const Pattern &pattern() const override;
+
+  bool match(OptContext *ctx, const MatchedResult &matched) const override;
+
+  StatusOr<TransformResult> transform(OptContext *ctx, const MatchedResult &matched) const override;
+
+  std::string toString() const override;
+
+ private:
+  EliminateRowCollectRule();
+
+  static std::unique_ptr<OptRule> kInstance;
+};
+
+}  // namespace opt
+}  // namespace nebula

src/graph/planner/plan/PlanNode.cpp

@@ -370,6 +370,15 @@ void PlanNode::releaseSymbols() {
   }
 }
 
+void PlanNode::updateSymbols() {
+  auto symTbl = qctx_->symTable();
+  for (auto out : outputVars_) {
+    if (out != nullptr) {
+      symTbl->updateWrittenBy(out->name, out->name, this);
+    }
+  }
+}
+
 std::ostream& operator<<(std::ostream& os, PlanNode::Kind kind) {
   os << PlanNode::toString(kind);
   return os;

src/graph/planner/plan/PlanNode.h

@@ -269,6 +269,8 @@ class PlanNode {
 
   void releaseSymbols();
 
+  void updateSymbols();
+
   static const char* toString(Kind kind);
   std::string toString() const;
 

src/graph/util/ExpressionUtils.cpp

@@ -128,18 +128,24 @@ bool ExpressionUtils::isEvaluableExpr(const Expression *expr, const QueryContext
 }
 
 // rewrite Attribute to LabelTagProp
-Expression *ExpressionUtils::rewriteAttr2LabelTagProp(const Expression *expr) {
+Expression *ExpressionUtils::rewriteAttr2LabelTagProp(
+    const Expression *expr, const std::unordered_map<std::string, AliasType> &aliasTypeMap) {
   ObjectPool *pool = expr->getObjPool();
 
-  auto matcher = [](const Expression *e) -> bool {
-    if (e->kind() == Expression::Kind::kAttribute) {
-      auto attrExpr = static_cast<const AttributeExpression *>(e);
-      if (attrExpr->left()->kind() == Expression::Kind::kLabelAttribute &&
-          attrExpr->right()->kind() == Expression::Kind::kConstant) {
-        return true;
-      }
+  auto matcher = [&aliasTypeMap](const Expression *e) -> bool {
+    if (e->kind() != Expression::Kind::kAttribute) {
+      return false;
     }
-    return false;
+    auto attrExpr = static_cast<const AttributeExpression *>(e);
+    if (attrExpr->left()->kind() != Expression::Kind::kLabelAttribute) {
+      return false;
+    }
+    auto label = static_cast<const LabelAttributeExpression *>(attrExpr->left())->left()->name();
+    auto iter = aliasTypeMap.find(label);
+    if (iter == aliasTypeMap.end() || iter->second != AliasType::kNode) {
+      return false;
+    }
+    return true;
   };
 
   auto rewriter = [pool](const Expression *e) -> Expression * {

src/graph/util/ExpressionUtils.h

@@ -15,6 +15,7 @@
 #include "common/expression/PropertyExpression.h"
 #include "common/expression/TypeCastingExpression.h"
 #include "common/expression/UnaryExpression.h"
+#include "graph/context/ast/CypherAstContext.h"
 #include "graph/visitor/EvaluableExprVisitor.h"
 #include "graph/visitor/FindVisitor.h"
 #include "graph/visitor/RewriteVisitor.h"
@@ -24,7 +25,6 @@ DECLARE_int32(max_expression_depth);
 namespace nebula {
 class ObjectPool;
 namespace graph {
-
 class ExpressionUtils {
  public:
   explicit ExpressionUtils(...) = delete;
@@ -57,7 +57,8 @@ class ExpressionUtils {
 
   static bool isEvaluableExpr(const Expression* expr, const QueryContext* qctx = nullptr);
 
-  static Expression* rewriteAttr2LabelTagProp(const Expression* expr);
+  static Expression* rewriteAttr2LabelTagProp(
+      const Expression* expr, const std::unordered_map<std::string, AliasType>& aliasTypeMap);
 
   static Expression* rewriteLabelAttr2TagProp(const Expression* expr);
 

src/graph/validator/MatchValidator.cpp

@@ -273,7 +273,8 @@ Status MatchValidator::validateFilter(const Expression *filter,
   auto transformRes = ExpressionUtils::filterTransform(filter);
   NG_RETURN_IF_ERROR(transformRes);
   // rewrite Attribute to LabelTagProperty
-  whereClauseCtx.filter = ExpressionUtils::rewriteAttr2LabelTagProp(transformRes.value());
+  whereClauseCtx.filter = ExpressionUtils::rewriteAttr2LabelTagProp(
+      transformRes.value(), whereClauseCtx.aliasesAvailable);
 
   auto typeStatus = deduceExprType(whereClauseCtx.filter);
   NG_RETURN_IF_ERROR(typeStatus);
@@ -383,7 +384,8 @@ Status MatchValidator::validateReturn(MatchReturn *ret,
           ExpressionUtils::hasAny(column->expr(), {Expression::Kind::kAggregate})) {
         retClauseCtx.yield->hasAgg_ = true;
       }
-      column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(column->expr()));
+      column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(
+          column->expr(), retClauseCtx.yield->aliasesAvailable));
       exprs.push_back(column->expr());
       columns->addColumn(column->clone().release());
     }
@@ -459,7 +461,8 @@ Status MatchValidator::validateWith(const WithClause *with,
   }
   if (with->returnItems()->columns()) {
     for (auto *column : with->returnItems()->columns()->columns()) {
-      column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(column->expr()));
+      column->setExpr(ExpressionUtils::rewriteAttr2LabelTagProp(
+          column->expr(), withClauseCtx.yield->aliasesAvailable));
       columns->addColumn(column->clone().release());
     }
   }
@@ -819,7 +822,7 @@ Status MatchValidator::checkAlias(
       auto name = static_cast<const VariablePropertyExpression *>(labelExpr)->prop();
       auto res = getAliasType(aliasesAvailable, name);
       NG_RETURN_IF_ERROR(res);
-      if (res.value() != AliasType::kNode) {
+      if (res.value() == AliasType::kEdge || res.value() == AliasType::kPath) {
         return Status::SemanticError("The type of `%s' should be tag", name.c_str());
       }
       return Status::OK();

src/graph/visitor/RewriteVisitor.cpp

@@ -237,6 +237,18 @@ void RewriteVisitor::visit(PathBuildExpression *expr) {
   }
 }
 
+void RewriteVisitor::visit(LabelTagPropertyExpression *expr) {
+  if (!care(expr->kind())) {
+    return;
+  }
+  auto label = expr->label();
+  if (matcher_(label)) {
+    expr->setLabel(rewriter_(label));
+  } else {
+    label->accept(this);
+  }
+}
+
 void RewriteVisitor::visit(AttributeExpression *expr) {
   if (!care(expr->kind())) {
     return;

src/graph/visitor/RewriteVisitor.h

@@ -67,12 +67,12 @@ class RewriteVisitor final : public ExprVisitorImpl {
   void visit(RelationalExpression*) override;
   void visit(SubscriptExpression*) override;
   void visit(PathBuildExpression*) override;
+  void visit(LabelTagPropertyExpression*) override;
   void visit(SubscriptRangeExpression*) override;
   void visit(ConstantExpression*) override {}
   void visit(LabelExpression*) override {}
   void visit(UUIDExpression*) override {}
   void visit(LabelAttributeExpression*) override {}
-  void visit(LabelTagPropertyExpression*) override {}
   void visit(VariableExpression*) override {}
   void visit(VersionedVariableExpression*) override {}
   void visit(TagPropertyExpression*) override {}

src/kvstore/KVEngine.h

@@ -49,6 +49,18 @@ class KVEngine {
                                                    bool sync,
                                                    bool wait) = 0;
 
+  /**
+   * @brief Get the Snapshot from kv engine.
+   *
+   * @return const void* snapshot pointer.
+   */
+  virtual const void* GetSnapshot() = 0;
+  /**
+   * @brief Release snapshot from kv engine.
+   *
+   * @param snapshot
+   */
+  virtual void ReleaseSnapshot(const void* snapshot) = 0;
   // Read a single key
   virtual nebula::cpp2::ErrorCode get(const std::string& key, std::string* value) = 0;
 
@@ -62,9 +74,17 @@ class KVEngine {
                                         const std::string& end,
                                         std::unique_ptr<KVIterator>* iter) = 0;
 
-  // Get all results with 'prefix' str as prefix.
+  /**
+   * @brief Get all results with 'prefix' str as prefix.
+   *
+   * @param prefix Prefix string.
+   * @param snapshot Snapshot from kv engine. nullptr means no snapshot.
+   * @param iter Iterator for this prefix range.
+   * @return nebula::cpp2::ErrorCode
+   */
   virtual nebula::cpp2::ErrorCode prefix(const std::string& prefix,
-                                         std::unique_ptr<KVIterator>* iter) = 0;
+                                         std::unique_ptr<KVIterator>* iter,
+                                         const void* snapshot = nullptr) = 0;
 
   // Get all results with 'prefix' str as prefix starting form 'start'
   virtual nebula::cpp2::ErrorCode rangeWithPrefix(const std::string& start,