Skip to content

Commit

Permalink
Merge branch 'master' into zimuxia/add-col-default
Browse files Browse the repository at this point in the history
  • Loading branch information
@ti-chi-bot
ti-chi-bot[bot] authored Nov 25, 2024
2 parents 7d7d97d + b335225 commit 0c22cb5
Show file tree
Hide file tree
Showing 51 changed files with 3,365 additions and 632 deletions.
40 changes: 17 additions & 23 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,13 @@ sudo ./llvm.sh 17 all
sudo apt install -y cmake ninja-build zlib1g-dev libcurl4-openssl-dev ccache
```

Then, expose the LLVM17 toolchain as the default one in order to use it later in compile:

```shell
export CC="/usr/bin/clang-17"
export CXX="/usr/bin/clang++-17"
```

**Note for Ubuntu 18.04 and Ubuntu 20.04:**

The default installed cmake may be not recent enough. You can install a newer cmake from the [Kitware APT Repository](https://apt.kitware.com):
Expand Down Expand Up @@ -125,9 +132,14 @@ xcode-select --install
brew install ninja cmake [email protected] ccache

brew install llvm@17
```

Then, expose the LLVM17 toolchain as the default one in order to use it later in compile:

# check llvm version
clang --version # should be 17.0.0 or higher
```shell
export PATH="$(brew --prefix)/opt/llvm@17/bin:$PATH"
export CC="$(brew --prefix)/opt/llvm@17/bin/clang"
export CXX="$(brew --prefix)/opt/llvm@17/bin/clang++"
```

</details>
Expand All @@ -148,34 +160,16 @@ To build TiFlash for development:
cmake --workflow --preset dev
```

Note: In Linux, usually you need to explicitly specify to use LLVM.

```shell
export CC="/usr/bin/clang-17"
export CXX="/usr/bin/clang++-17"
```

In MacOS, if you install llvm clang, you need to explicitly specify to use llvm clang.
Or if you don't like the preset:

Add the following lines to your shell environment, e.g. `~/.bash_profile`.
```shell
export PATH="$(brew --prefix)/opt/llvm/bin:$PATH"
export CC="$(brew --prefix)/opt/llvm/bin/clang"
export CXX="$(brew --prefix)/opt/llvm/bin/clang++"
```

Or use `CMAKE_C_COMPILER` and `CMAKE_CXX_COMPILER` to specify the compiler, like this:
```shell
# In the TiFlash repository root:
mkdir cmake-build-debug
cd cmake-build-debug

cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG -DCMAKE_C_COMPILER="$(brew --prefix)/opt/llvm/bin/clang" -DCMAKE_CXX_COMPILER="$(brew --prefix)/opt/llvm/bin/clang++"

cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG
ninja tiflash
```

After building, you can get TiFlash binary in `dbms/src/Server/tiflash` in the `cmake-build-debug` directory.

### Build Options

TiFlash has several CMake build options to tweak for development purposes. These options SHOULD NOT be changed for production usage, as they may introduce unexpected build errors and unpredictable runtime behaviors.
Expand Down
4 changes: 4 additions & 0 deletions dbms/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,10 @@ check_then_add_sources_compile_flag (
src/Columns/ColumnString.cpp
src/Columns/ColumnsCommon.cpp
src/Columns/ColumnVector.cpp
src/Columns/ColumnDecimal.cpp
src/Columns/ColumnArray.cpp
src/Columns/ColumnNullable.cpp
src/Columns/ColumnFixedString.cpp
src/DataTypes/DataTypeString.cpp
src/Interpreters/Join.cpp
src/IO/Compression/EncodingUtil.cpp
Expand Down
47 changes: 47 additions & 0 deletions dbms/src/Columns/ColumnAggregateFunction.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,53 @@ class ColumnAggregateFunction final : public COWPtrHelper<IColumn, ColumnAggrega

const char * deserializeAndInsertFromArena(const char * src_arena, const TiDB::TiDBCollatorPtr &) override;

void countSerializeByteSize(PaddedPODArray<size_t> & /* byte_size */) const override
{
throw Exception("Method countSerializeByteSize is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void countSerializeByteSizeForColumnArray(
PaddedPODArray<size_t> & /* byte_size */,
const IColumn::Offsets & /* offsets */) const override
{
throw Exception(
"Method countSerializeByteSizeForColumnArray is not supported for " + getName(),
ErrorCodes::NOT_IMPLEMENTED);
}

void serializeToPos(
PaddedPODArray<char *> & /* pos */,
size_t /* start */,
size_t /* length */,
bool /* has_null */) const override
{
throw Exception("Method serializeToPos is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void serializeToPosForColumnArray(
PaddedPODArray<char *> & /* pos */,
size_t /* start */,
size_t /* length */,
bool /* has_null */,
const IColumn::Offsets & /* offsets */) const override
{
throw Exception(
"Method serializeToPosForColumnArray is not supported for " + getName(),
ErrorCodes::NOT_IMPLEMENTED);
}

void deserializeAndInsertFromPos(PaddedPODArray<char *> & /* pos */, ColumnsAlignBufferAVX2 & /* align_buffer */)
override
{
throw Exception(
"Method deserializeAndInsertFromPos is not supported for " + getName(),
ErrorCodes::NOT_IMPLEMENTED);
}
void deserializeAndInsertFromPosForColumnArray(PaddedPODArray<char *> &, const Offsets &) override
{
throw Exception(
"Method deserializeAndInsertFromPosForColumnArray is not supported for " + getName(),
ErrorCodes::NOT_IMPLEMENTED);
}

void updateHashWithValue(size_t n, SipHash & hash, const TiDB::TiDBCollatorPtr &, String &) const override;

void updateHashWithValues(IColumn::HashValues & hash_values, const TiDB::TiDBCollatorPtr &, String &)
Expand Down
79 changes: 77 additions & 2 deletions dbms/src/Columns/ColumnArray.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,75 @@ const char * ColumnArray::deserializeAndInsertFromArena(const char * pos, const
return pos;
}

void ColumnArray::countSerializeByteSize(PaddedPODArray<size_t> & byte_size) const
{
RUNTIME_CHECK_MSG(byte_size.size() == size(), "size of byte_size({}) != column size({})", byte_size.size(), size());

if unlikely (!getOffsets().empty() && getOffsets().back() > UINT32_MAX)
{
size_t sz = size();
for (size_t i = 0; i < sz; ++i)
RUNTIME_CHECK_MSG(
sizeAt(i) <= UINT32_MAX,
"size of ({}) is ({}), which is greater than UINT32_MAX",
i,
sizeAt(i));
}

size_t size = byte_size.size();
for (size_t i = 0; i < size; ++i)
byte_size[i] += sizeof(UInt32);

getData().countSerializeByteSizeForColumnArray(byte_size, getOffsets());
}

void ColumnArray::serializeToPos(PaddedPODArray<char *> & pos, size_t start, size_t length, bool has_null) const
{
if (has_null)
serializeToPosImpl<true>(pos, start, length);
else
serializeToPosImpl<false>(pos, start, length);
}

template <bool has_null>
void ColumnArray::serializeToPosImpl(PaddedPODArray<char *> & pos, size_t start, size_t length) const
{
RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size());
RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size());

/// countSerializeByteSize has already checked that the size of one element is not greater than UINT32_MAX
for (size_t i = 0; i < length; ++i)
{
if constexpr (has_null)
{
if (pos[i] == nullptr)
continue;
}
UInt32 len = sizeAt(start + i);
tiflash_compiler_builtin_memcpy(pos[i], &len, sizeof(UInt32));
pos[i] += sizeof(UInt32);
}

getData().serializeToPosForColumnArray(pos, start, length, has_null, getOffsets());
}

void ColumnArray::deserializeAndInsertFromPos(PaddedPODArray<char *> & pos, ColumnsAlignBufferAVX2 & /* align_buffer */)
{
auto & offsets = getOffsets();
size_t prev_size = offsets.size();
size_t size = pos.size();

offsets.resize(prev_size + size);
for (size_t i = 0; i < size; ++i)
{
UInt32 len;
tiflash_compiler_builtin_memcpy(&len, pos[i], sizeof(UInt32));
offsets[prev_size + i] = len + offsets[prev_size + i - 1];
pos[i] += sizeof(UInt32);
}

getData().deserializeAndInsertFromPosForColumnArray(pos, offsets);
}

void ColumnArray::updateHashWithValue(
size_t n,
Expand Down Expand Up @@ -414,10 +483,16 @@ struct Less
void ColumnArray::reserve(size_t n)
{
getOffsets().reserve(n);
getData().reserve(
n); /// The average size of arrays is not taken into account here. Or it is considered to be no more than 1.
/// The average size of arrays is not taken into account here. Or it is considered to be no more than 1.
getData().reserve(n);
}

void ColumnArray::reserveAlign(size_t n, size_t alignment)
{
getOffsets().reserve(n, alignment);
/// The average size of arrays is not taken into account here. Or it is considered to be no more than 1.
getData().reserveAlign(n, alignment);
}

size_t ColumnArray::byteSize() const
{
Expand Down
51 changes: 43 additions & 8 deletions dbms/src/Columns/ColumnArray.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,43 @@ class ColumnArray final : public COWPtrHelper<IColumn, ColumnArray>
const TiDB::TiDBCollatorPtr &,
String &) const override;
const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override;

void countSerializeByteSize(PaddedPODArray<size_t> & byte_size) const override;
void countSerializeByteSizeForColumnArray(
PaddedPODArray<size_t> & /* byte_size */,
const IColumn::Offsets & /* array_offsets */) const override
{
throw Exception(
"Method countSerializeByteSizeForColumnArray is not supported for " + getName(),
ErrorCodes::NOT_IMPLEMENTED);
}

void serializeToPos(PaddedPODArray<char *> & pos, size_t start, size_t length, bool has_null) const override;
template <bool has_null>
void serializeToPosImpl(PaddedPODArray<char *> & pos, size_t start, size_t length) const;

void serializeToPosForColumnArray(
PaddedPODArray<char *> & /* pos */,
size_t /* start */,
size_t /* length */,
bool /* has_null */,
const IColumn::Offsets & /* array_offsets */) const override
{
throw Exception(
"Method serializeToPosForColumnArray is not supported for " + getName(),
ErrorCodes::NOT_IMPLEMENTED);
}

void deserializeAndInsertFromPos(PaddedPODArray<char *> & pos, ColumnsAlignBufferAVX2 & align_buffer) override;
void deserializeAndInsertFromPosForColumnArray(
PaddedPODArray<char *> & /* pos */,
const IColumn::Offsets & /* array_offsets */) override
{
throw Exception(
"Method deserializeAndInsertFromPosForColumnArray is not supported for " + getName(),
ErrorCodes::NOT_IMPLEMENTED);
}

void updateHashWithValue(size_t n, SipHash & hash, const TiDB::TiDBCollatorPtr &, String &) const override;
void updateHashWithValues(IColumn::HashValues & hash_values, const TiDB::TiDBCollatorPtr &, String &)
const override;
Expand Down Expand Up @@ -111,6 +148,7 @@ class ColumnArray final : public COWPtrHelper<IColumn, ColumnArray>
int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
void reserve(size_t n) override;
void reserveAlign(size_t n, size_t alignment) override;
size_t byteSize() const override;
size_t byteSize(size_t offset, size_t limit) const override;
size_t allocatedBytes() const override;
Expand All @@ -125,12 +163,12 @@ class ColumnArray final : public COWPtrHelper<IColumn, ColumnArray>
IColumn & getData() { return data->assumeMutableRef(); }
const IColumn & getData() const { return *data; }

IColumn & getOffsetsColumn() { return offsets->assumeMutableRef(); }
const IColumn & getOffsetsColumn() const { return *offsets; }
ColumnOffsets & getOffsetsColumn() { return static_cast<ColumnOffsets &>(offsets->assumeMutableRef()); }
const ColumnOffsets & getOffsetsColumn() const { return static_cast<const ColumnOffsets &>(*offsets); }

Offsets & ALWAYS_INLINE getOffsets() { return static_cast<ColumnOffsets &>(offsets->assumeMutableRef()).getData(); }
Offsets & ALWAYS_INLINE getOffsets() { return getOffsetsColumn().getData(); }

const Offsets & ALWAYS_INLINE getOffsets() const { return static_cast<const ColumnOffsets &>(*offsets).getData(); }
const Offsets & ALWAYS_INLINE getOffsets() const { return getOffsetsColumn().getData(); }

const ColumnPtr & getDataPtr() const { return data; }
ColumnPtr & getDataPtr() { return data; }
Expand Down Expand Up @@ -172,10 +210,7 @@ class ColumnArray final : public COWPtrHelper<IColumn, ColumnArray>

std::pair<UInt32, StringRef> getElementRef(size_t element_idx) const;

size_t ALWAYS_INLINE sizeAt(size_t i) const
{
return i == 0 ? getOffsets()[0] : (getOffsets()[i] - getOffsets()[i - 1]);
}
size_t ALWAYS_INLINE sizeAt(ssize_t i) const { return (getOffsets()[i] - getOffsets()[i - 1]); }

private:
ColumnPtr data;
Expand Down
49 changes: 49 additions & 0 deletions dbms/src/Columns/ColumnConst.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,55 @@ class ColumnConst final : public COWPtrHelper<IColumn, ColumnConst>
return res;
}

void countSerializeByteSize(PaddedPODArray<size_t> & /* byte_size */) const override
{
throw Exception("Method countSerializeByteSize is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void countSerializeByteSizeForColumnArray(
PaddedPODArray<size_t> & /* byte_size */,
const IColumn::Offsets & /* array_offsets */) const override
{
throw Exception(
"Method countSerializeByteSizeForColumnArray is not supported for " + getName(),
ErrorCodes::NOT_IMPLEMENTED);
}

void serializeToPos(
PaddedPODArray<char *> & /* pos */,
size_t /* start */,
size_t /* length */,
bool /* has_null */) const override
{
throw Exception("Method serializeToPos is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void serializeToPosForColumnArray(
PaddedPODArray<char *> & /* pos */,
size_t /* start */,
size_t /* length */,
bool /* has_null */,
const IColumn::Offsets & /* array_offsets */) const override
{
throw Exception(
"Method serializeToPosForColumnArray is not supported for " + getName(),
ErrorCodes::NOT_IMPLEMENTED);
}

void deserializeAndInsertFromPos(PaddedPODArray<char *> & /* pos */, ColumnsAlignBufferAVX2 & /* align_buffer */)
override
{
throw Exception(
"Method deserializeAndInsertFromPos is not supported for " + getName(),
ErrorCodes::NOT_IMPLEMENTED);
}
void deserializeAndInsertFromPosForColumnArray(
PaddedPODArray<char *> & /* pos */,
const IColumn::Offsets & /* array_offsets */) override
{
throw Exception(
"Method deserializeAndInsertFromPosForColumnArray is not supported for " + getName(),
ErrorCodes::NOT_IMPLEMENTED);
}

void updateHashWithValue(
size_t,
SipHash & hash,
Expand Down
Loading

0 comments on commit 0c22cb5

Please sign in to comment.