Segmentation fault in coprocessor

[leiysky]

Table schema:

create table if not exists tmp.t_product_dtl_ord_srvpkg(
`SRVPKG_ORDER_ID` bigint NULL,
`CUST_ORDER_ID` bigint NULL,
`OFFER_ORDER_ID` bigint NULL,
`SRVPKG_ID` bigint NULL,
`SRVPKG_INST_ID` bigint NULL,
`VALID_TYPE` tinyint NULL,
`STATE` tinyint NULL,
`ORDER_STATE` tinyint NULL,
`PROD_ORDER_ID` bigint NULL,
`DONE_CODE` bigint NULL,
`CREATE_DATE` datetime NULL,
`DONE_DATE` datetime NULL,
`VALID_DATE` datetime NULL,
`EXPIRE_DATE` datetime NULL,
`OP_ID` bigint NULL,
`ORG_ID` bigint NULL,
`SRC_SYSTEM_ID` bigint NULL,
`REGION_ID` varchar(6) NULL,
`SRC_SYSTEM_EXT_CODE` varchar(200) NULL,
`PROD_SERVICE_ID` bigint NULL,
`OS_STATUS` varchar(30) NULL,
`DATA_SOURCE_CD` varchar(200) NULL,
`IS_HISTORY_DATA` int NULL,
`ETL_CYCLE` int NULL,
`ETL_DATE` datetime NULL
);

Query:

select t1.prod_inst_id ## distinct t1.prod_inst_id\r\n
from dw.dwa_fact_ins_prod_20201025 t1
join dw.dwa_fact_ins_srvpkg_20201025 t2
on t1.prod_inst_id = t2.prod_inst_id
and t2.prod_service_id=1002
and substr(t2.os_status,-1,1) in('1','2')
## and truncate(TIMESTAMPDIFF(month,date_format(t2.done_date,'%Y%m%d'),'20201025'),0)>= 3
and t2.done_date <= date_add('20201025',interval -3 month)
group by t1.prod_inst_id

Stack trace:

2020.10.27 09:37:26.315976 [ 223 ] <Error> BaseDaemon: ########################################
2020.10.27 09:37:26.315986 [ 223 ] <Error> BaseDaemon: (from thread 222) Received signal Segmentation fault (11).
2020.10.27 09:37:26.315993 [ 223 ] <Error> BaseDaemon: Address: 0xfffffffffffffff8
2020.10.27 09:37:26.316001 [ 223 ] <Error> BaseDaemon: Access: read.
2020.10.27 09:37:26.316008 [ 223 ] <Error> BaseDaemon: Address not mapped to object.
2020.10.27 09:37:26.344387 [ 223 ] <Error> BaseDaemon: 0. bin/tiflash/tiflash(DB::FunctionSubstringUTF8::executeImpl(DB::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long)+0x77c) [0x40669cc]
2020.10.27 09:37:26.344403 [ 223 ] <Error> BaseDaemon: 1. bin/tiflash/tiflash(DB::PreparedFunctionImpl::defaultImplementationForNulls(DB::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long)+0x132) [0x6a672a2]
2020.10.27 09:37:26.344413 [ 223 ] <Error> BaseDaemon: 2. bin/tiflash/tiflash(DB::PreparedFunctionImpl::execute(DB::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long)+0x41) [0x6a67121]
2020.10.27 09:37:26.344423 [ 223 ] <Error> BaseDaemon: 3. bin/tiflash/tiflash(DB::IFunctionBase::execute(DB::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long)+0x3c) [0x34fd24c]
2020.10.27 09:37:26.344433 [ 223 ] <Error> BaseDaemon: 4. bin/tiflash/tiflash(DB::ExpressionAction::execute(DB::Block&) const+0x51a) [0x6be872a]
2020.10.27 09:37:26.344442 [ 223 ] <Error> BaseDaemon: 5. bin/tiflash/tiflash(DB::ExpressionActions::execute(DB::Block&) const+0x6a) [0x6becf7a]
2020.10.27 09:37:26.344451 [ 223 ] <Error> BaseDaemon: 6. bin/tiflash/tiflash(DB::FilterBlockInputStream::readImpl()+0x1e3) [0x6ac9233]
2020.10.27 09:37:26.344462 [ 223 ] <Error> BaseDaemon: 7. bin/tiflash/tiflash(DB::IProfilingBlockInputStream::read(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool)+0x3bd) [0x5ef262d]
2020.10.27 09:37:26.344471 [ 223 ] <Error> BaseDaemon: 8. bin/tiflash/tiflash(DB::IProfilingBlockInputStream::read()+0x17) [0x5ef2817]
2020.10.27 09:37:26.344480 [ 223 ] <Error> BaseDaemon: 9. bin/tiflash/tiflash(DB::ExpressionBlockInputStream::readImpl()+0x1b) [0x6ac4dab]
2020.10.27 09:37:26.344490 [ 223 ] <Error> BaseDaemon: 10. bin/tiflash/tiflash(DB::IProfilingBlockInputStream::read(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool)+0x3bd) [0x5ef262d]
2020.10.27 09:37:26.344499 [ 223 ] <Error> BaseDaemon: 11. bin/tiflash/tiflash(DB::IProfilingBlockInputStream::read()+0x17) [0x5ef2817]
2020.10.27 09:37:26.344508 [ 223 ] <Error> BaseDaemon: 12. bin/tiflash/tiflash(DB::ExpressionBlockInputStream::readImpl()+0x1b) [0x6ac4dab]
2020.10.27 09:37:26.344517 [ 223 ] <Error> BaseDaemon: 13. bin/tiflash/tiflash(DB::IProfilingBlockInputStream::read(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool)+0x3bd) [0x5ef262d]
2020.10.27 09:37:26.344526 [ 223 ] <Error> BaseDaemon: 14. bin/tiflash/tiflash(DB::IProfilingBlockInputStream::read()+0x17) [0x5ef2817]
2020.10.27 09:37:26.344536 [ 223 ] <Error> BaseDaemon: 15. bin/tiflash/tiflash(DB::copyData(DB::IBlockInputStream&, DB::IBlockOutputStream&, std::atomic<bool>*)+0x5d) [0x6b7be3d]
2020.10.27 09:37:26.344545 [ 223 ] <Error> BaseDaemon: 16. bin/tiflash/tiflash(DB::DAGDriver<false>::execute()+0x3bd) [0x6e9bcfd]
2020.10.27 09:37:26.344554 [ 223 ] <Error> BaseDaemon: 17. bin/tiflash/tiflash(DB::CoprocessorHandler::execute()+0x640) [0x6ecbf70]
2020.10.27 09:37:26.344564 [ 223 ] <Error> BaseDaemon: 18. bin/tiflash/tiflash(DB::FlashService::Coprocessor(grpc_impl::ServerContext*, coprocessor::Request const*, coprocessor::Response*)+0x63c) [0x6e81a8c]
2020.10.27 09:37:26.344590 [ 223 ] <Error> BaseDaemon: 19. bin/tiflash/tiflash(std::_Function_handler<grpc::Status (tikvpb::Tikv::Service*, grpc_impl::ServerContext*, coprocessor::Request const*, coprocessor::Response*), std::_Mem_fn<grpc::Status (tikvpb::Tikv::Service::*)(grpc_impl::ServerContext*, coprocessor::Request const*, coprocessor::Response*)> >::_M_invoke(std::_Any_data const&, tikvpb::Tikv::Service*&&, grpc_impl::ServerContext*&&, coprocessor::Request const*&&, coprocessor::Response*&&)+0x39) [0x7621ad9]
2020.10.27 09:37:26.344605 [ 223 ] <Error> BaseDaemon: 20. bin/tiflash/tiflash(grpc::Status grpc::internal::CatchingFunctionHandler<grpc::internal::RpcMethodHandler<tikvpb::Tikv::Service, coprocessor::Request, coprocessor::Response>::RunHandler(grpc::internal::MethodHandler::HandlerParameter const&)::{lambda()#1}>(grpc::internal::RpcMethodHandler<tikvpb::Tikv::Service, coprocessor::Request, coprocessor::Response>::RunHandler(grpc::internal::MethodHandler::HandlerParameter const&)::{lambda()#1}&&)+0x54) [0x7678104]
2020.10.27 09:37:26.344617 [ 223 ] <Error> BaseDaemon: 21. bin/tiflash/tiflash(grpc::internal::RpcMethodHandler<tikvpb::Tikv::Service, coprocessor::Request, coprocessor::Response>::RunHandler(grpc::internal::MethodHandler::HandlerParameter const&)+0x1fe) [0x76af74e]
2020.10.27 09:37:26.344627 [ 223 ] <Error> BaseDaemon: 22. bin/tiflash/tiflash(grpc_impl::Server::SyncRequest::CallData::ContinueRunAfterInterception()+0x161) [0x76feca1]
2020.10.27 09:37:26.344636 [ 223 ] <Error> BaseDaemon: 23. bin/tiflash/tiflash(grpc_impl::Server::SyncRequestThreadManager::DoWork(void*, bool, bool)+0x430) [0x7700260]
2020.10.27 09:37:26.344646 [ 223 ] <Error> BaseDaemon: 24. bin/tiflash/tiflash(grpc::ThreadManager::MainWorkLoop()+0x9b) [0x7705e9b]
2020.10.27 09:37:26.344655 [ 223 ] <Error> BaseDaemon: 25. bin/tiflash/tiflash(grpc::ThreadManager::WorkerThread::Run()+0xc) [0x7705f7c]
2020.10.27 09:37:26.344663 [ 223 ] <Error> BaseDaemon: 26. bin/tiflash/tiflash() [0x7a7cc33]
2020.10.27 09:37:26.344671 [ 223 ] <Error> BaseDaemon: 27. /lib64/libpthread.so.0(+0x7ea5) [0x7f4f8a3b3ea5]

[leiysky]

It seems caused by substr.

[leiysky]

Cannot reproduce. Seems fixed by #1296, I think we can close this.

[windtalker]

reopen it as customer hit this issue again in v4.0.10, which already have the fix of #1296.