Skip to content

Commit

Permalink
Folder: expression
Browse files Browse the repository at this point in the history 
relative pr:

Allow decimal in casting string to int oap-project#215
Add mapping from named_struct to row_constructor oap-project#214
Fix semantic issues in cast function oap-project#280
  • Loading branch information
@zhejiangxiaomai
zhejiangxiaomai committed Jul 11, 2023
1 parent 4f3ed22 commit 00d07a3
Show file tree
Hide file tree
Showing 4 changed files with 208 additions and 34 deletions.
156 changes: 129 additions & 27 deletions velox/expression/CastExpr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "velox/expression/StringWriter.h"
#include "velox/external/date/tz.h"
#include "velox/functions/lib/RowsTranslationUtil.h"
#include "velox/type/DecimalUtilOp.h"
#include "velox/vector/ComplexVector.h"
#include "velox/vector/FunctionVector.h"
#include "velox/vector/SelectivityVector.h"
Expand All @@ -41,26 +42,35 @@ namespace {
/// @param row The index of the current row
/// @param input The input vector (of type FromKind)
/// @param result The output vector (of type ToKind)
/// @return False if the result is null
template <TypeKind ToKind, TypeKind FromKind, bool Truncate>
template <TypeKind ToKind, TypeKind FromKind, bool Truncate, bool AllowDecimal>
void applyCastKernel(
vector_size_t row,
const SimpleVector<typename TypeTraits<FromKind>::NativeType>* input,
FlatVector<typename TypeTraits<ToKind>::NativeType>* result,
bool& nullOutput) {
auto output = util::Converter<ToKind, void, Truncate>::cast(
input->valueAt(row), nullOutput);
if (nullOutput) {
return;
}

FlatVector<typename TypeTraits<ToKind>::NativeType>* result) {
// Special handling for string target type
if constexpr (ToKind == TypeKind::VARCHAR || ToKind == TypeKind::VARBINARY) {
std::string output;
if (input->type()->isDecimal()) {
output = util::Converter<ToKind, void, Truncate, AllowDecimal>::cast(
input->valueAt(row), input->type());
} else {
output = util::Converter<ToKind, void, Truncate, AllowDecimal>::cast(
input->valueAt(row));
}
// Write the result output to the output vector
auto writer = exec::StringWriter<>(result, row);
writer.copy_from(output);
writer.finalize();
} else {
result->set(row, output);
if (input->type()->isDecimal()) {
auto output = util::Converter<ToKind, void, Truncate, AllowDecimal>::cast(
input->valueAt(row), input->type());
result->set(row, output);
} else {
auto output = util::Converter<ToKind, void, Truncate, AllowDecimal>::cast(
input->valueAt(row));
result->set(row, output);
}
}
}

Expand Down Expand Up @@ -127,6 +137,78 @@ void applyIntToDecimalCastKernel(
});
}

template <typename TInput, typename TOutput>
void applyDateToDecimalCastKernel(
const SelectivityVector& rows,
const BaseVector& input,
exec::EvalCtx& context,
const TypePtr& toType,
VectorPtr castResult) {
auto sourceVector = input.as<SimpleVector<Date>>();
auto castResultRawBuffer =
castResult->asUnchecked<FlatVector<TOutput>>()->mutableRawValues();
const auto& toPrecisionScale = getDecimalPrecisionScale(*toType);
context.applyToSelectedNoThrow(rows, [&](vector_size_t row) {
auto rescaledValue = DecimalUtil::rescaleInt<TInput, TOutput>(
sourceVector->valueAt(row).days(),
toPrecisionScale.first,
toPrecisionScale.second);
if (rescaledValue.has_value()) {
castResultRawBuffer[row] = rescaledValue.value();
} else {
castResult->setNull(row, true);
}
});
}

template <typename TInput, typename TOutput>
void applyDoubleToDecimalCastKernel(
const SelectivityVector& rows,
const BaseVector& input,
exec::EvalCtx& context,
const TypePtr& toType,
VectorPtr castResult) {
auto sourceVector = input.as<SimpleVector<TInput>>();
auto castResultRawBuffer =
castResult->asUnchecked<FlatVector<TOutput>>()->mutableRawValues();
const auto& toPrecisionScale = getDecimalPrecisionScale(*toType);
context.applyToSelectedNoThrow(rows, [&](vector_size_t row) {
auto rescaledValue = DecimalUtilOp::rescaleDouble<TInput, TOutput>(
sourceVector->valueAt(row),
toPrecisionScale.first,
toPrecisionScale.second);
if (rescaledValue.has_value()) {
castResultRawBuffer[row] = rescaledValue.value();
} else {
castResult->setNull(row, true);
}
});
}

template <typename TOutput>
void applyVarCharToDecimalCastKernel(
const SelectivityVector& rows,
const BaseVector& input,
exec::EvalCtx& context,
const TypePtr& toType,
VectorPtr castResult) {
auto sourceVector = input.as<SimpleVector<StringView>>();
auto castResultRawBuffer =
castResult->asUnchecked<FlatVector<TOutput>>()->mutableRawValues();
const auto& toPrecisionScale = getDecimalPrecisionScale(*toType);
context.applyToSelectedNoThrow(rows, [&](vector_size_t row) {
auto rescaledValue = DecimalUtilOp::rescaleVarchar<TOutput>(
sourceVector->valueAt(row),
toPrecisionScale.first,
toPrecisionScale.second);
if (rescaledValue.has_value()) {
castResultRawBuffer[row] = rescaledValue.value();
} else {
castResult->setNull(row, true);
}
});
}

template <TypeKind ToKind, TypeKind FromKind>
void applyCastPrimitives(
const SelectivityVector& rows,
Expand All @@ -136,17 +218,21 @@ void applyCastPrimitives(
using To = typename TypeTraits<ToKind>::NativeType;
using From = typename TypeTraits<FromKind>::NativeType;
auto* resultFlatVector = result->as<FlatVector<To>>();
auto* inputSimpleVector = input.as<SimpleVector<From>>();

const auto& queryConfig = context.execCtx()->queryCtx()->queryConfig();
const bool isCastIntAllowDecimal = queryConfig.isCastIntAllowDecimal();
auto* inputSimpleVector = input.as<SimpleVector<From>>();

if (!queryConfig.isCastToIntByTruncate()) {
context.applyToSelectedNoThrow(rows, [&](int row) {
bool nullOutput = false;
try {
// Passing a false truncate flag
applyCastKernel<ToKind, FromKind, false>(
row, inputSimpleVector, resultFlatVector, nullOutput);
if (isCastIntAllowDecimal) {
applyCastKernel<ToKind, FromKind, false, true>(
row, inputSimpleVector, resultFlatVector);
} else {
applyCastKernel<ToKind, FromKind, false, false>(
row, inputSimpleVector, resultFlatVector);
}
} catch (const VeloxRuntimeError& re) {
VELOX_FAIL(
makeErrorMessage(input, row, resultFlatVector->type()) + " " +
Expand All @@ -160,18 +246,18 @@ void applyCastPrimitives(
makeErrorMessage(input, row, resultFlatVector->type()) + " " +
e.what());
}

if (nullOutput) {
VELOX_USER_FAIL(makeErrorMessage(input, row, resultFlatVector->type()));
}
});
} else {
context.applyToSelectedNoThrow(rows, [&](int row) {
bool nullOutput = false;
try {
// Passing a true truncate flag
applyCastKernel<ToKind, FromKind, true>(
row, inputSimpleVector, resultFlatVector, nullOutput);
if (isCastIntAllowDecimal) {
applyCastKernel<ToKind, FromKind, true, true>(
row, inputSimpleVector, resultFlatVector);
} else {
applyCastKernel<ToKind, FromKind, true, false>(
row, inputSimpleVector, resultFlatVector);
}
} catch (const VeloxRuntimeError& re) {
VELOX_FAIL(
makeErrorMessage(input, row, resultFlatVector->type()) + " " +
Expand All @@ -185,10 +271,6 @@ void applyCastPrimitives(
makeErrorMessage(input, row, resultFlatVector->type()) + " " +
e.what());
}

if (nullOutput) {
VELOX_USER_FAIL(makeErrorMessage(input, row, resultFlatVector->type()));
}
});
}

Expand Down Expand Up @@ -468,6 +550,10 @@ VectorPtr CastExpr::applyDecimal(
(*castResult).clearNulls(rows);
// toType is a decimal
switch (fromType->kind()) {
case TypeKind::BOOLEAN:
applyIntToDecimalCastKernel<bool, toDecimalType>(
rows, input, context, toType, castResult);
break;
case TypeKind::TINYINT:
applyIntToDecimalCastKernel<int8_t, toDecimalType>(
rows, input, context, toType, castResult);
Expand Down Expand Up @@ -497,6 +583,22 @@ VectorPtr CastExpr::applyDecimal(
break;
}
}
case TypeKind::DATE:
applyDateToDecimalCastKernel<int32_t, toDecimalType>(
rows, input, context, toType, castResult);
break;
case TypeKind::REAL:
applyDoubleToDecimalCastKernel<float, toDecimalType>(
rows, input, context, toType, castResult);
break;
case TypeKind::DOUBLE:
applyDoubleToDecimalCastKernel<double, toDecimalType>(
rows, input, context, toType, castResult);
break;
case TypeKind::VARCHAR:
applyVarCharToDecimalCastKernel<toDecimalType>(
rows, input, context, toType, castResult);
break;
default:
VELOX_UNSUPPORTED(
"Cast from {} to {} is not supported",
Expand Down
25 changes: 25 additions & 0 deletions velox/expression/ExprCompiler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ using core::TypedExprPtr;
const char* const kAnd = "and";
const char* const kOr = "or";
const char* const kRowConstructor = "row_constructor";
const char* const kRowConstructorWithNull = "row_constructor_with_null";

struct ITypedExprHasher {
size_t operator()(const ITypedExpr* expr) const {
Expand Down Expand Up @@ -212,6 +213,25 @@ ExprPtr getRowConstructorExpr(
trackCpuUsage);
}

ExprPtr getRowConstructorWithNullExpr(
const TypePtr& type,
std::vector<ExprPtr>&& compiledChildren,
bool trackCpuUsage) {
static auto rowConstructorVectorFunction =
vectorFunctionFactories().withRLock([](auto& functionMap) {
auto functionIterator = functionMap.find(exec::kRowConstructorWithNull);
return functionIterator->second.factory(
exec::kRowConstructorWithNull, {});
});

return std::make_shared<Expr>(
type,
std::move(compiledChildren),
rowConstructorVectorFunction,
"row_constructor_with_null",
trackCpuUsage);
}

ExprPtr getSpecialForm(
const std::string& name,
const TypePtr& type,
Expand All @@ -222,6 +242,11 @@ ExprPtr getSpecialForm(
type, std::move(compiledChildren), trackCpuUsage);
}

if (name == kRowConstructorWithNull) {
return getRowConstructorWithNullExpr(
type, std::move(compiledChildren), trackCpuUsage);
}

// If we just check the output of constructSpecialForm we'll have moved
// compiledChildren, and if the function isn't a special form we'll still need
// compiledChildren. Splitting the check in two avoids this use after move.
Expand Down
14 changes: 7 additions & 7 deletions velox/expression/ExprToSubfieldFilter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,44 +439,44 @@ std::unique_ptr<common::Filter> leafCallToSubfieldFilter(
common::Subfield& subfield,
core::ExpressionEvaluator* evaluator,
bool negated) {
if (call.name() == "eq") {
if (call.name() == "eq" || call.name() == "equalto") {
if (auto field = asField(&call, 0)) {
if (toSubfield(field, subfield)) {
return negated ? makeNotEqualFilter(call.inputs()[1], evaluator)
: makeEqualFilter(call.inputs()[1], evaluator);
}
}
} else if (call.name() == "neq") {
} else if (call.name() == "neq" || call.name() == "notequalto") {
if (auto field = asField(&call, 0)) {
if (toSubfield(field, subfield)) {
return negated ? makeEqualFilter(call.inputs()[1], evaluator)
: makeNotEqualFilter(call.inputs()[1], evaluator);
}
}
} else if (call.name() == "lte") {
} else if (call.name() == "lte" || call.name() == "lessthanorequal") {
if (auto field = asField(&call, 0)) {
if (toSubfield(field, subfield)) {
return negated ? makeGreaterThanFilter(call.inputs()[1], evaluator)
: makeLessThanOrEqualFilter(call.inputs()[1], evaluator);
}
}
} else if (call.name() == "lt") {
} else if (call.name() == "lt" || call.name() == "lessthan") {
if (auto field = asField(&call, 0)) {
if (toSubfield(field, subfield)) {
return negated
? makeGreaterThanOrEqualFilter(call.inputs()[1], evaluator)
: makeLessThanFilter(call.inputs()[1], evaluator);
}
}
} else if (call.name() == "gte") {
} else if (call.name() == "gte" || call.name() == "greaterthanorequal") {
if (auto field = asField(&call, 0)) {
if (toSubfield(field, subfield)) {
return negated
? makeLessThanFilter(call.inputs()[1], evaluator)
: makeGreaterThanOrEqualFilter(call.inputs()[1], evaluator);
}
}
} else if (call.name() == "gt") {
} else if (call.name() == "gt" || call.name() == "greaterthan") {
if (auto field = asField(&call, 0)) {
if (toSubfield(field, subfield)) {
return negated ? makeLessThanOrEqualFilter(call.inputs()[1], evaluator)
Expand All @@ -496,7 +496,7 @@ std::unique_ptr<common::Filter> leafCallToSubfieldFilter(
return makeInFilter(call.inputs()[1], evaluator, negated);
}
}
} else if (call.name() == "is_null") {
} else if (call.name() == "is_null" || call.name() == "isnull") {
if (auto field = asField(&call, 0)) {
if (toSubfield(field, subfield)) {
if (negated) {
Expand Down
Loading

0 comments on commit 00d07a3

Please sign in to comment.