diff --git a/.travis-scripts/linux/build.sh b/.travis-scripts/linux/build.sh index 8e9286f80b..a31d4226ea 100755 --- a/.travis-scripts/linux/build.sh +++ b/.travis-scripts/linux/build.sh @@ -17,8 +17,16 @@ # limitations under the License. # set -e -export CC="gcc-4.8" -export CXX="g++-4.8" + +arch="$(uname -i)" + +if [[ "$arch" == "s390x" ]] || [[ "$arch" == "ppc64le" ]]; then + export CC="gcc-7" + export CXX="g++-7" +else + export CC="gcc-4.8" + export CXX="g++-4.8" +fi wget https://s3.amazonaws.com/download.draios.com/dependencies/cmake-3.3.2.tar.gz tar -xzf cmake-3.3.2.tar.gz cd cmake-3.3.2 diff --git a/.travis-scripts/linux/install.sh b/.travis-scripts/linux/install.sh index 3f9ad6e1f1..02dcf70933 100755 --- a/.travis-scripts/linux/install.sh +++ b/.travis-scripts/linux/install.sh @@ -16,6 +16,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # -sudo apt-get --force-yes install g++-4.8 +arch="$(uname -i)" + +if [[ "$arch" != "s390x" ]] && [[ "$arch" != "ppc64le" ]]; then + sudo apt-get --force-yes install g++-4.8 +fi sudo apt-get install rpm linux-headers-$(uname -r) libelf-dev sudo apt-get purge cmake diff --git a/.travis.yml b/.travis.yml index 9e9fee5c28..3a039cf279 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,12 +16,34 @@ # limitations under the License. # language: c -os: - - linux - - osx -env: - - BUILD_TYPE=Debug - - BUILD_TYPE=Release + +matrix: + include: + - env: BUILD_TYPE=Debug + os: linux + - env: BUILD_TYPE=Release + os: linux + - env: BUILD_TYPE=Debug + os: osx + - env: BUILD_TYPE=Release + os: osx + - env: BUILD_TYPE=Debug + os: linux + arch: ppc64le + dist: bionic + - env: BUILD_TYPE=Release + os: linux + arch: ppc64le + dist: bionic + - env: BUILD_TYPE=Debug + os: linux + arch: s390x + dist: bionic + - env: BUILD_TYPE=Release + os: linux + arch: s390x + dist: bionic + sudo: required services: - docker diff --git a/CMakeLists.txt b/CMakeLists.txt index ab89b93f58..46ab92bd43 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,6 +160,27 @@ else() set(LUAJIT_INCLUDE "${LUAJIT_SRC}") if(NOT WIN32) set(LUAJIT_LIB "${LUAJIT_SRC}/libluajit.a") + if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "ppc64le") + ExternalProject_Add(luajit + GIT_REPOSITORY "https://github.com/moonjit/moonjit" + GIT_TAG "2.1.2" + PATCH_COMMAND sed -i "s/luaL_reg/luaL_Reg/g" ${PROJECT_SOURCE_DIR}/userspace/libsinsp/chisel.cpp && sed -i "s/luaL_reg/luaL_Reg/g" ${PROJECT_SOURCE_DIR}/userspace/libsinsp/lua_parser.cpp && sed -i "s/luaL_getn/lua_objlen /g" ${PROJECT_SOURCE_DIR}/userspace/libsinsp/lua_parser_api.cpp + CONFIGURE_COMMAND "" + BUILD_COMMAND ${CMD_MAKE} + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${LUAJIT_LIB} + INSTALL_COMMAND "") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "s390x") + ExternalProject_Add(luajit + GIT_REPOSITORY "https://github.com/linux-on-ibm-z/LuaJIT.git" + GIT_TAG "v2.1" + PATCH_COMMAND sed -i "s/luaL_reg/luaL_Reg/g" ${PROJECT_SOURCE_DIR}/userspace/libsinsp/chisel.cpp && sed -i "s/luaL_reg/luaL_Reg/g" ${PROJECT_SOURCE_DIR}/userspace/libsinsp/lua_parser.cpp && sed -i "s/luaL_getn/lua_objlen /g" ${PROJECT_SOURCE_DIR}/userspace/libsinsp/lua_parser_api.cpp + CONFIGURE_COMMAND "" + BUILD_COMMAND ${CMD_MAKE} + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${LUAJIT_LIB} + INSTALL_COMMAND "") + else() ExternalProject_Add(luajit URL "http://download.draios.com/dependencies/LuaJIT-2.0.3.tar.gz" URL_MD5 "f14e9104be513913810cd59c8c658dc0" @@ -168,6 +189,7 @@ else() BUILD_IN_SOURCE 1 BUILD_BYPRODUCTS ${LUAJIT_LIB} INSTALL_COMMAND "") + endif() else() set(LUAJIT_LIB "${LUAJIT_SRC}/lua51.lib") ExternalProject_Add(luajit @@ -464,17 +486,31 @@ if(NOT WIN32 AND NOT APPLE) set(PROTOC "${PROTOBUF_SRC}/target/bin/protoc") set(PROTOBUF_INCLUDE "${PROTOBUF_SRC}/target/include") set(PROTOBUF_LIB "${PROTOBUF_SRC}/target/lib/libprotobuf.a") - ExternalProject_Add(protobuf - DEPENDS openssl zlib - URL "http://download.sysdig.com/dependencies/protobuf-cpp-3.5.0.tar.gz" - URL_MD5 "e4ba8284a407712168593e79e6555eb2" - # TODO what if using system zlib? - CONFIGURE_COMMAND /usr/bin/env CPPFLAGS=-I${ZLIB_INCLUDE} LDFLAGS=-L${ZLIB_SRC} ./configure --with-zlib --prefix=${PROTOBUF_SRC}/target - BUILD_COMMAND ${CMD_MAKE} - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS ${PROTOC} ${PROTOBUF_INCLUDE} ${PROTOBUF_LIB} - # TODO s390x support - INSTALL_COMMAND make install) + if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "s390x") + ExternalProject_Add(protobuf + DEPENDS openssl zlib + URL "http://download.sysdig.com/dependencies/protobuf-cpp-3.5.0.tar.gz" + URL_MD5 "e4ba8284a407712168593e79e6555eb2" + PATCH_COMMAND wget http://download.sysdig.com/dependencies/protobuf-3.5.0-s390x.patch && patch -p1 -i protobuf-3.5.0-s390x.patch + # TODO what if using system zlib? + CONFIGURE_COMMAND /usr/bin/env CPPFLAGS=-I${ZLIB_INCLUDE} LDFLAGS=-L${ZLIB_SRC} ./configure --with-zlib --prefix=${PROTOBUF_SRC}/target + COMMAND aclocal && automake + BUILD_COMMAND ${CMD_MAKE} + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${PROTOC} ${PROTOBUF_INCLUDE} ${PROTOBUF_LIB} + INSTALL_COMMAND make install) + else() + ExternalProject_Add(protobuf + DEPENDS openssl zlib + URL "http://download.sysdig.com/dependencies/protobuf-cpp-3.5.0.tar.gz" + URL_MD5 "e4ba8284a407712168593e79e6555eb2" + # TODO what if using system zlib? + CONFIGURE_COMMAND /usr/bin/env CPPFLAGS=-I${ZLIB_INCLUDE} LDFLAGS=-L${ZLIB_SRC} ./configure --with-zlib --prefix=${PROTOBUF_SRC}/target + BUILD_COMMAND ${CMD_MAKE} + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${PROTOC} ${PROTOBUF_INCLUDE} ${PROTOBUF_LIB} + INSTALL_COMMAND make install) + endif() endif() option(USE_BUNDLED_GRPC "Enable building of the bundled grpc" ${USE_BUNDLED_DEPS}) @@ -581,7 +617,7 @@ if(CREATE_TEST_TARGETS AND NOT WIN32) # This is preferred vs using ctest's add_test because it will build # the code and output to stdout. add_custom_target(run-unit-tests - COMMAND $(MAKE) run-unit-test-libsinsp + COMMAND ${CMAKE_MAKE_PROGRAM} run-unit-test-libsinsp ) endif() @@ -598,12 +634,12 @@ set(CPACK_GENERATOR DEB RPM TGZ) set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Sysdig ") set(CPACK_DEBIAN_PACKAGE_SECTION "utils") -set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "http://www.sysdig.org") +set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "http://www.sysdig.com") set(CPACK_DEBIAN_PACKAGE_DEPENDS "dkms (>= 2.1.0.0)") set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_BINARY_DIR}/scripts/debian/postinst;${CMAKE_BINARY_DIR}/scripts/debian/prerm") set(CPACK_RPM_PACKAGE_LICENSE "Apache v2.0") -set(CPACK_RPM_PACKAGE_URL "http://www.sysdig.org") +set(CPACK_RPM_PACKAGE_URL "http://www.sysdig.com") set(CPACK_RPM_PACKAGE_REQUIRES "dkms, gcc, make, kernel-devel, perl") set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${PROJECT_SOURCE_DIR}/scripts/rpm/postinstall") set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${PROJECT_SOURCE_DIR}/scripts/rpm/preuninstall") diff --git a/README.md b/README.md index fd4ee68174..f084ac17c6 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ sysdig Where to start? --- -If this is your first time hearing about sysdig, we recommend you [start with the website](http://www.sysdig.org). +If this is your first time hearing about sysdig, we recommend you [start with the website](http://www.sysdig.com). What does sysdig do and why should I use it? --- diff --git a/driver/bpf/filler_helpers.h b/driver/bpf/filler_helpers.h index 3cd3924e4b..0b236af038 100644 --- a/driver/bpf/filler_helpers.h +++ b/driver/bpf/filler_helpers.h @@ -931,6 +931,9 @@ static __always_inline int bpf_val_to_ring_type(struct filler_data *data, static __always_inline bool bpf_in_ia32_syscall() { +#ifdef __ppc64__ + return 0; +#else struct task_struct *task; u32 status; @@ -947,6 +950,7 @@ static __always_inline bool bpf_in_ia32_syscall() #endif return status & TS_COMPAT; +#endif // #ifdef __ppc64__ } #endif diff --git a/driver/ppm.h b/driver/ppm.h index 86c372bc78..acd5fa01f3 100644 --- a/driver/ppm.h +++ b/driver/ppm.h @@ -39,6 +39,7 @@ or GPL2.txt for full copies of the license. #define PPM_PORT_MYSQL 3306 #define PPM_PORT_POSTGRES 5432 #define PPM_PORT_STATSD 8125 +#define PPM_PORT_MONGODB 27017 /* * The ring descriptor. diff --git a/driver/ppm_events.c b/driver/ppm_events.c index 6c321f689c..69db0ac4af 100644 --- a/driver/ppm_events.c +++ b/driver/ppm_events.c @@ -188,6 +188,28 @@ inline int sock_getname(struct socket* sock, struct sockaddr* sock_address, int #endif } +/** + * Compute the snaplen for the arguments. + * + * The snaplen is the amount of argument data returned along with the event. + * Normally, the driver performs a dynamic calculation to figure out snaplen + * per-event. However, if this calculation is disabled + * (i.e. args->consumer->do_dynamic_snaplen == false), the snaplen will always + * be args->consumer->snaplen. + * + * If dynamic snaplen is enabled, here's how the calculation works: + * + * 1. If the event is a write to /dev/null, it gets a special snaplen because + * writes to /dev/null is a backdoor method for inserting special events + * into the event stream. + * 2. If the event is NOT a socket operation, return args->consumer->snaplen. + * 3. If the sending port OR destination port falls within the fullcapture port + * range specified by the user, return 16000. + * 4. Protocol detection. A number of applications are detected heuristically + * and given a longer snaplen (2000). These applications are MYSQL, Postgres, + * HTTP, mongodb, and statsd. + * 5. If none of the above apply, return args->consumer->snaplen. + */ inline u32 compute_snaplen(struct event_filler_arguments *args, char *buf, u32 lookahead_size) { u32 res = args->consumer->snaplen; @@ -197,7 +219,18 @@ inline u32 compute_snaplen(struct event_filler_arguments *args, char *buf, u32 l struct sockaddr_storage sock_address; struct sockaddr_storage peer_address; u16 sport, dport; + u16 min_port = 0, max_port = 0; + u32 dynamic_snaplen = 2000; + + if (args->consumer->snaplen > dynamic_snaplen) { + /* + * If the user requested a default snaplen greater than the custom + * snaplen given to certain applications, just use the greater value. + */ + dynamic_snaplen = args->consumer->snaplen; + } + /* Increase snaplen on writes to /dev/null */ if (g_tracers_enabled && args->event_type == PPME_SYSCALL_WRITE_X) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) struct fd f = fdget(args->fd); @@ -242,195 +275,216 @@ inline u32 compute_snaplen(struct event_filler_arguments *args, char *buf, u32 l sock = sockfd_lookup(args->fd, &err); - if (sock) { + if (!sock) { + return res; + } - if (sock->sk) { - err = sock_getname(sock, (struct sockaddr *)&sock_address, 0); + if (!sock->sk) { + goto done; + } - if (err == 0) { - if(args->event_type == PPME_SOCKET_SENDTO_X) - { - unsigned long syscall_args[6] = {}; - unsigned long val; - struct sockaddr __user * usrsockaddr; - /* - * Get the address - */ - if (!args->is_socketcall) { - ppm_syscall_get_arguments(current, args->regs, syscall_args); - val = syscall_args[4]; - } else - val = args->socketcall_args[4]; + err = sock_getname(sock, (struct sockaddr *)&sock_address, 0); - usrsockaddr = (struct sockaddr __user *)val; + if (err != 0) { + goto done; + } - if(usrsockaddr == NULL) { - /* - * Suppose is a connected socket, fall back to fd - */ - err = sock_getname(sock, (struct sockaddr *)&peer_address, 1); - } else { - /* - * Get the address len - */ - if (!args->is_socketcall) { - ppm_syscall_get_arguments(current, args->regs, syscall_args); - val = syscall_args[5]; - } else - val = args->socketcall_args[5]; - - if (val != 0) { - /* - * Copy the address - */ - err = addr_to_kernel(usrsockaddr, val, (struct sockaddr *)&peer_address); - } else { - /* - * This case should be very rare, fallback again to sock - */ - err = sock_getname(sock, (struct sockaddr *)&peer_address, 1); - } - } - } else if (args->event_type == PPME_SOCKET_SENDMSG_X) { - unsigned long syscall_args[6] = {}; - unsigned long val; - struct sockaddr __user * usrsockaddr; - int addrlen; + /* Try to get the source and destination port */ + if (args->event_type == PPME_SOCKET_SENDTO_X) { + unsigned long syscall_args[6] = {}; + unsigned long val; + struct sockaddr __user * usrsockaddr; + /* + * Get the address + */ + if (!args->is_socketcall) { + ppm_syscall_get_arguments(current, args->regs, syscall_args); + val = syscall_args[4]; + } else { + val = args->socketcall_args[4]; + } + + usrsockaddr = (struct sockaddr __user *)val; + + if(usrsockaddr == NULL) { + /* + * Suppose is a connected socket, fall back to fd + */ + err = sock_getname(sock, (struct sockaddr *)&peer_address, 1); + } else { + /* + * Get the address len + */ + if (!args->is_socketcall) { + ppm_syscall_get_arguments(current, args->regs, syscall_args); + val = syscall_args[5]; + } else { + val = args->socketcall_args[5]; + } + + if (val != 0) { + /* + * Copy the address + */ + err = addr_to_kernel(usrsockaddr, val, (struct sockaddr *)&peer_address); + } else { + /* + * This case should be very rare, fallback again to sock + */ + err = sock_getname(sock, (struct sockaddr *)&peer_address, 1); + } + } + } else if (args->event_type == PPME_SOCKET_SENDMSG_X) { + unsigned long syscall_args[6] = {}; + unsigned long val; + struct sockaddr __user * usrsockaddr; + int addrlen; #ifdef CONFIG_COMPAT - struct compat_msghdr compat_mh; + struct compat_msghdr compat_mh; #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) - struct user_msghdr mh; + struct user_msghdr mh; #else - struct msghdr mh; + struct msghdr mh; #endif - if (!args->is_socketcall) { - ppm_syscall_get_arguments(current, args->regs, syscall_args); - val = syscall_args[1]; - } else - val = args->socketcall_args[1]; + if (!args->is_socketcall) { + ppm_syscall_get_arguments(current, args->regs, syscall_args); + val = syscall_args[1]; + } else { + val = args->socketcall_args[1]; + } #ifdef CONFIG_COMPAT - if (!args->compat) { + if (!args->compat) { #endif - if (unlikely(ppm_copy_from_user(&mh, (const void __user *)val, sizeof(mh)))) { - usrsockaddr = NULL; - addrlen = 0; - } else { - usrsockaddr = (struct sockaddr __user *)mh.msg_name; - addrlen = mh.msg_namelen; - } + if (unlikely(ppm_copy_from_user(&mh, (const void __user *)val, sizeof(mh)))) { + usrsockaddr = NULL; + addrlen = 0; + } else { + usrsockaddr = (struct sockaddr __user *)mh.msg_name; + addrlen = mh.msg_namelen; + } #ifdef CONFIG_COMPAT - } else { - if (unlikely(ppm_copy_from_user(&compat_mh, (const void __user *)compat_ptr(val), sizeof(compat_mh)))) { - usrsockaddr = NULL; - addrlen = 0; - } else { - usrsockaddr = (struct sockaddr __user *)compat_ptr(compat_mh.msg_name); - addrlen = compat_mh.msg_namelen; - } - } + } else { + if (unlikely(ppm_copy_from_user(&compat_mh, (const void __user *)compat_ptr(val), sizeof(compat_mh)))) { + usrsockaddr = NULL; + addrlen = 0; + } else { + usrsockaddr = (struct sockaddr __user *)compat_ptr(compat_mh.msg_name); + addrlen = compat_mh.msg_namelen; + } + } #endif - if (usrsockaddr != NULL && addrlen != 0) { - /* - * Copy the address - */ - err = addr_to_kernel(usrsockaddr, addrlen, (struct sockaddr *)&peer_address); - } else - /* - * Suppose it is a connected socket, fall back to fd - */ - err = sock_getname(sock, (struct sockaddr *)&peer_address, 1); - } else - err = sock_getname(sock, (struct sockaddr *)&peer_address, 1); - - if (err == 0) { - uint16_t min_port = args->consumer->fullcapture_port_range_start; - uint16_t max_port = args->consumer->fullcapture_port_range_end; - family = sock->sk->sk_family; - - if (family == AF_INET) { - sport = ntohs(((struct sockaddr_in *) &sock_address)->sin_port); - dport = ntohs(((struct sockaddr_in *) &peer_address)->sin_port); - } else if (family == AF_INET6) { - sport = ntohs(((struct sockaddr_in6 *) &sock_address)->sin6_port); - dport = ntohs(((struct sockaddr_in6 *) &peer_address)->sin6_port); - } else { - sport = 0; - dport = 0; - } - - if (max_port > 0 && - (in_port_range(sport, min_port, max_port) || - in_port_range(dport, min_port, max_port))) { - /* - * Before checking the well-known ports, see if the user has requested - * an increased snaplen for the port in question. - */ - sockfd_put(sock); - return RW_MAX_FULLCAPTURE_PORT_SNAPLEN; - } else if (sport == PPM_PORT_MYSQL || dport == PPM_PORT_MYSQL) { - if (lookahead_size >= 5) { - if (buf[0] == 3 || buf[1] == 3 || buf[2] == 3 || buf[3] == 3 || buf[4] == 3) { - sockfd_put(sock); - return 2000; - } else if (buf[2] == 0 && buf[3] == 0) { - sockfd_put(sock); - return 2000; - } - } - } else if (sport == PPM_PORT_POSTGRES || dport == PPM_PORT_POSTGRES) { - if (lookahead_size >= 2) { - if ((buf[0] == 'Q' && buf[1] == 0) || /* SimpleQuery command */ - (buf[0] == 'P' && buf[1] == 0) || /* Prepare statement command */ - (buf[4] == 0 && buf[5] == 3 && buf[6] == 0) || /* startup command */ - (buf[0] == 'E' && buf[1] == 0) /* error or execute command */ - ) { - sockfd_put(sock); - return 2000; - } - } - } else if ((lookahead_size >= 4 && buf[1] == 0 && buf[2] == 0 && buf[2] == 0) || /* matches command */ - (lookahead_size >= 16 && (*(int32_t *)(buf+12) == 1 || /* matches header */ - *(int32_t *)(buf+12) == 2001 || - *(int32_t *)(buf+12) == 2002 || - *(int32_t *)(buf+12) == 2003 || - *(int32_t *)(buf+12) == 2004 || - *(int32_t *)(buf+12) == 2005 || - *(int32_t *)(buf+12) == 2006 || - *(int32_t *)(buf+12) == 2007) - ) - ) { - sockfd_put(sock); - return 2000; - } else if (dport == args->consumer->statsd_port) { - sockfd_put(sock); - return 2000; - } else { - if (lookahead_size >= 5) { - if (*(u32 *)buf == g_http_get_intval || - *(u32 *)buf == g_http_post_intval || - *(u32 *)buf == g_http_put_intval || - *(u32 *)buf == g_http_delete_intval || - *(u32 *)buf == g_http_trace_intval || - *(u32 *)buf == g_http_connect_intval || - *(u32 *)buf == g_http_options_intval || - ((*(u32 *)buf == g_http_resp_intval) && (buf[4] == '/')) - ) { - sockfd_put(sock); - return 2000; - } - } - } - } - } + if (usrsockaddr != NULL && addrlen != 0) { + /* + * Copy the address + */ + err = addr_to_kernel(usrsockaddr, addrlen, (struct sockaddr *)&peer_address); + } else { + /* + * Suppose it is a connected socket, fall back to fd + */ + err = sock_getname(sock, (struct sockaddr *)&peer_address, 1); } + } else { + err = sock_getname(sock, (struct sockaddr *)&peer_address, 1); + } + + if (err != 0) { + goto done; + } + + /* + * If there's a valid source / dest port, use it to run heuristics + * for determining snaplen. + */ + min_port = args->consumer->fullcapture_port_range_start; + max_port = args->consumer->fullcapture_port_range_end; + family = sock->sk->sk_family; + + if (family == AF_INET) { + sport = ntohs(((struct sockaddr_in *) &sock_address)->sin_port); + dport = ntohs(((struct sockaddr_in *) &peer_address)->sin_port); + } else if (family == AF_INET6) { + sport = ntohs(((struct sockaddr_in6 *) &sock_address)->sin6_port); + dport = ntohs(((struct sockaddr_in6 *) &peer_address)->sin6_port); + } else { + sport = 0; + dport = 0; + } + if (max_port > 0 && + (in_port_range(sport, min_port, max_port) || + in_port_range(dport, min_port, max_port))) { + /* + * Before checking the well-known ports, see if the user has requested + * an increased snaplen for the port in question. + */ sockfd_put(sock); + return RW_MAX_FULLCAPTURE_PORT_SNAPLEN; + } else if (sport == PPM_PORT_MYSQL || dport == PPM_PORT_MYSQL) { + if (lookahead_size >= 5) { + if (buf[0] == 3 || buf[1] == 3 || buf[2] == 3 || buf[3] == 3 || buf[4] == 3) { + res = dynamic_snaplen; + goto done; + } else if (buf[2] == 0 && buf[3] == 0) { + res = dynamic_snaplen; + goto done; + } + } + } else if (sport == PPM_PORT_POSTGRES || dport == PPM_PORT_POSTGRES) { + if (lookahead_size >= 2) { + if ((buf[0] == 'Q' && buf[1] == 0) || /* SimpleQuery command */ + (buf[0] == 'P' && buf[1] == 0) || /* Prepare statement command */ + (buf[0] == 'E' && buf[1] == 0) /* error or execute command */ + ) { + res = dynamic_snaplen; + goto done; + } + } + if (lookahead_size >= 7 && + (buf[4] == 0 && buf[5] == 3 && buf[6] == 0)) { /* startup command */ + res = dynamic_snaplen; + goto done; + } + } else if ((sport == PPM_PORT_MONGODB || dport == PPM_PORT_MONGODB) || + (lookahead_size >= 16 && + (*(int32_t *)(buf+12) == 1 || /* matches header */ + *(int32_t *)(buf+12) == 2001 || + *(int32_t *)(buf+12) == 2002 || + *(int32_t *)(buf+12) == 2003 || + *(int32_t *)(buf+12) == 2004 || + *(int32_t *)(buf+12) == 2005 || + *(int32_t *)(buf+12) == 2006 || + *(int32_t *)(buf+12) == 2007) + ) + ) { + res = dynamic_snaplen; + goto done; + } else if (dport == args->consumer->statsd_port) { + res = dynamic_snaplen; + goto done; + } else { + if (lookahead_size >= 5) { + if (*(u32 *)buf == g_http_get_intval || + *(u32 *)buf == g_http_post_intval || + *(u32 *)buf == g_http_put_intval || + *(u32 *)buf == g_http_delete_intval || + *(u32 *)buf == g_http_trace_intval || + *(u32 *)buf == g_http_connect_intval || + *(u32 *)buf == g_http_options_intval || + ((*(u32 *)buf == g_http_resp_intval) && (buf[4] == '/')) + ) { + res = dynamic_snaplen; + goto done; + } + } } +done: + sockfd_put(sock); return res; } diff --git a/userspace/libscap/scap.c b/userspace/libscap/scap.c index 59b04e0a21..e9faea5184 100644 --- a/userspace/libscap/scap.c +++ b/userspace/libscap/scap.c @@ -977,30 +977,35 @@ int32_t scap_readbuf(scap_t* handle, uint32_t cpuid, OUT char** buf, OUT uint32_ return SCAP_SUCCESS; } -static bool are_buffers_empty(scap_t* handle) +static uint64_t buf_size_used(scap_t* handle, uint32_t cpu) { - uint32_t j; + uint64_t read_size; - for(j = 0; j < handle->m_ndevs; j++) + if (handle->m_bpf) { - uint64_t read_size; + uint64_t thead; + uint64_t ttail; - if(handle->m_bpf) - { - uint64_t thead; - uint64_t ttail; + scap_bpf_get_buf_pointers(handle->m_devs[cpu].m_buffer, &thead, &ttail, &read_size); + } + else + { + uint32_t thead; + uint32_t ttail; - scap_bpf_get_buf_pointers(handle->m_devs[j].m_buffer, &thead, &ttail, &read_size); - } - else - { - uint32_t thead; - uint32_t ttail; + get_buf_pointers(handle->m_devs[cpu].m_bufinfo, &thead, &ttail, &read_size); + } - get_buf_pointers(handle->m_devs[j].m_bufinfo, &thead, &ttail, &read_size); - } + return read_size; +} + +static bool are_buffers_empty(scap_t* handle) +{ + uint32_t j; - if(read_size > BUFFER_EMPTY_THRESHOLD_B) + for(j = 0; j < handle->m_ndevs; j++) + { + if(buf_size_used(handle, j) > BUFFER_EMPTY_THRESHOLD_B) { return false; } @@ -1181,6 +1186,24 @@ static int32_t scap_next_nodriver(scap_t* handle, OUT scap_evt** pevent, OUT uin } #endif // _WIN32 +uint64_t scap_max_buf_used(scap_t* handle) +{ +#if defined(HAS_CAPTURE) && !defined(CYGWING_AGENT) + uint64_t i; + uint64_t max = 0; + + for(i = 0; i < handle->m_ndevs; i++) + { + uint64_t size = buf_size_used(handle, i); + max = size > max ? size : max; + } + + return max; +#else + return 0; +#endif +} + int32_t scap_next(scap_t* handle, OUT scap_evt** pevent, OUT uint16_t* pcpuid) { int32_t res = SCAP_FAILURE; diff --git a/userspace/libscap/scap.def b/userspace/libscap/scap.def index 88dfd5ea3c..d11b73da96 100644 --- a/userspace/libscap/scap.def +++ b/userspace/libscap/scap.def @@ -9,6 +9,7 @@ EXPORTS scap_get_os_platform scap_get_ndevs scap_getlasterr + scap_max_buf_used scap_next scap_event_getlen scap_event_get_ts diff --git a/userspace/libscap/scap.h b/userspace/libscap/scap.h index ecc045551b..e0cd418007 100644 --- a/userspace/libscap/scap.h +++ b/userspace/libscap/scap.h @@ -605,6 +605,11 @@ scap_os_platform scap_get_os_platform(scap_t* handle); */ const char* scap_getlasterr(scap_t* handle); +/*! + * \brief returns the maximum amount of memory used by any driver queue + */ +uint64_t scap_max_buf_used(scap_t* handle); + /*! \brief Get the next event from the from the given capture instance diff --git a/userspace/libsinsp/container_info.cpp b/userspace/libsinsp/container_info.cpp index 1df12cd918..6b63d16bed 100644 --- a/userspace/libsinsp/container_info.cpp +++ b/userspace/libsinsp/container_info.cpp @@ -148,7 +148,7 @@ const sinsp_container_info::container_mount_info *sinsp_container_info::mount_by std::shared_ptr sinsp_container_info::get_tinfo(sinsp* inspector) const { - auto tinfo = make_shared(inspector); + std::shared_ptr tinfo(inspector->build_threadinfo()); tinfo->m_tid = -1; tinfo->m_pid = -1; tinfo->m_vtid = -2; diff --git a/userspace/libsinsp/grpc_channel_registry.cpp b/userspace/libsinsp/grpc_channel_registry.cpp index 2ff8fe7f7d..6fc7ebfdf5 100644 --- a/userspace/libsinsp/grpc_channel_registry.cpp +++ b/userspace/libsinsp/grpc_channel_registry.cpp @@ -22,24 +22,29 @@ limitations under the License. std::map> libsinsp::grpc_channel_registry::s_channels; -std::shared_ptr libsinsp::grpc_channel_registry::get_channel(const std::string &url) +std::shared_ptr libsinsp::grpc_channel_registry::get_channel(const std::string &url, const grpc::ChannelArguments *args) { + std::shared_ptr chan; auto it = s_channels.find(url); - if(it == s_channels.end()) + if(it != s_channels.end()) { - auto chan = grpc::CreateChannel(url, grpc::InsecureChannelCredentials()); - s_channels[url] = chan; - - return chan; + chan = it->second.lock(); + if (chan != nullptr) + { + return chan; + } + } + if (args) + { + chan = grpc::CreateCustomChannel(url, + grpc::InsecureChannelCredentials(), *args); } else { - auto chan = it->second.lock(); - if(chan == nullptr) - { - chan = grpc::CreateChannel(url, grpc::InsecureChannelCredentials()); - s_channels[url] = chan; - } - return chan; + chan = grpc::CreateChannel(url, + grpc::InsecureChannelCredentials()); } + s_channels[url] = chan; + + return chan; } diff --git a/userspace/libsinsp/grpc_channel_registry.h b/userspace/libsinsp/grpc_channel_registry.h index bffee59e13..cd16c08817 100644 --- a/userspace/libsinsp/grpc_channel_registry.h +++ b/userspace/libsinsp/grpc_channel_registry.h @@ -32,7 +32,8 @@ class grpc_channel_registry { public: // Return a (shared) grpc::Channel for the provided url. - static std::shared_ptr get_channel(const std::string &url); + static std::shared_ptr get_channel(const std::string &url, + const grpc::ChannelArguments *args = nullptr); private: static std::map> s_channels; diff --git a/userspace/libsinsp/include/sinsp_external_processor.h b/userspace/libsinsp/include/sinsp_external_processor.h index 6e08277b72..9be656f5fc 100644 --- a/userspace/libsinsp/include/sinsp_external_processor.h +++ b/userspace/libsinsp/include/sinsp_external_processor.h @@ -8,9 +8,11 @@ // Required until the chisel_api dependency on the analyzer can be removed class statsd_metric; +class sinsp; +class threadinfo; -namespace libsinsp { - +namespace libsinsp +{ enum event_return { EVENT_RETURN_TIMEOUT, @@ -40,6 +42,15 @@ class event_processor * Required until the chisel_api dependency on the analyzer can be removed */ virtual void add_chisel_metric(statsd_metric* metric) = 0; + + /** + * Some event processors allocate different thread types with extra data. + * + * This allows the processor to override the thread builder. Note that + * If this is overriden by the event processor, the processor MUST be registered + * before the sinsp object is init-ed + */ + virtual sinsp_threadinfo* build_threadinfo(sinsp* inspector); }; -} +} // namespace libsinsp diff --git a/userspace/libsinsp/parsers.cpp b/userspace/libsinsp/parsers.cpp index ef50562f72..ad37a0833f 100644 --- a/userspace/libsinsp/parsers.cpp +++ b/userspace/libsinsp/parsers.cpp @@ -1159,7 +1159,7 @@ void sinsp_parser::parse_clone_exit(sinsp_evt *evt) // XXX this should absolutely not do a malloc, but get the item from a // preallocated list // - sinsp_threadinfo* tinfo = new sinsp_threadinfo(m_inspector); + sinsp_threadinfo* tinfo = m_inspector->build_threadinfo(); // // Set the tid and parent tid diff --git a/userspace/libsinsp/sinsp.cpp b/userspace/libsinsp/sinsp.cpp index 7d1e0f1113..5525a22376 100644 --- a/userspace/libsinsp/sinsp.cpp +++ b/userspace/libsinsp/sinsp.cpp @@ -828,7 +828,7 @@ void sinsp::on_new_entry_from_proc(void* context, if(fdinfo == NULL) { bool thread_added = false; - sinsp_threadinfo* newti = new sinsp_threadinfo(this); + sinsp_threadinfo* newti = build_threadinfo(); newti->init(tinfo); if(is_nodriver()) { @@ -852,7 +852,7 @@ void sinsp::on_new_entry_from_proc(void* context, if(!sinsp_tinfo) { - sinsp_threadinfo* newti = new sinsp_threadinfo(this); + sinsp_threadinfo* newti = build_threadinfo(); newti->init(tinfo); if (!m_thread_manager->add_thread(newti, true)) { @@ -895,7 +895,7 @@ void sinsp::import_thread_table() // HASH_ITER(hh, table, pi, tpi) { - sinsp_threadinfo* newti = new sinsp_threadinfo(this); + sinsp_threadinfo* newti = build_threadinfo(); newti->init(pi); m_thread_manager->add_thread(newti, true); } @@ -1027,6 +1027,18 @@ void sinsp::restart_capture_at_filepos(uint64_t filepos) } } +uint64_t sinsp::max_buf_used() +{ + if(m_h) + { + return scap_max_buf_used(m_h); + } + else + { + return 0; + } +} + int32_t sinsp::next(OUT sinsp_evt **puevt) { sinsp_evt* evt; @@ -1413,7 +1425,7 @@ threadinfo_map_t::ptr_t sinsp::get_thread_ref(int64_t tid, bool query_os_if_not_ } scap_threadinfo* scap_proc = NULL; - sinsp_threadinfo* newti = new sinsp_threadinfo(this); + sinsp_threadinfo* newti = build_threadinfo(); m_n_proc_lookups++; @@ -2566,3 +2578,9 @@ std::shared_ptr sinsp::lookup_cgroup_dir(const string& subsys) } } #endif + +sinsp_threadinfo* +libsinsp::event_processor::build_threadinfo(sinsp* inspector) +{ + return new sinsp_threadinfo(inspector); +} diff --git a/userspace/libsinsp/sinsp.h b/userspace/libsinsp/sinsp.h index 499d1d07ee..b1e1ecfae9 100644 --- a/userspace/libsinsp/sinsp.h +++ b/userspace/libsinsp/sinsp.h @@ -253,6 +253,11 @@ class SINSP_PUBLIC sinsp : public capture_stats_source, public wmi_handle_source */ virtual int32_t next(OUT sinsp_evt **evt); + /*! + \brief Get the maximum number of bytes currently in use by any CPU buffer + */ + uint64_t max_buf_used(); + /*! \brief Get the number of events that have been captured and processed since the call to \ref open() @@ -514,6 +519,12 @@ class SINSP_PUBLIC sinsp : public capture_stats_source, public wmi_handle_source libsinsp::event_processor* m_external_event_processor; + sinsp_threadinfo* build_threadinfo() + { + return m_external_event_processor ? m_external_event_processor->build_threadinfo(this) + : new sinsp_threadinfo(this); + } + /*! \brief registers external event processor. After this, callbacks on libsinsp::event_processor will happen at diff --git a/userspace/libsinsp/threadinfo.cpp b/userspace/libsinsp/threadinfo.cpp index e54fc12bf3..3788972dd0 100644 --- a/userspace/libsinsp/threadinfo.cpp +++ b/userspace/libsinsp/threadinfo.cpp @@ -40,19 +40,11 @@ static void copy_ipv6_address(uint32_t* dest, uint32_t* src) /////////////////////////////////////////////////////////////////////////////// // sinsp_threadinfo implementation /////////////////////////////////////////////////////////////////////////////// -sinsp_threadinfo::sinsp_threadinfo() : - m_fdtable(NULL) -{ - m_inspector = NULL; - m_tracer_parser = NULL; - init(); -} - -sinsp_threadinfo::sinsp_threadinfo(sinsp *inspector) : +sinsp_threadinfo::sinsp_threadinfo(sinsp* inspector) : + m_tracer_parser(NULL), + m_inspector(inspector), m_fdtable(inspector) { - m_inspector = inspector; - m_tracer_parser = NULL; init(); } @@ -883,7 +875,7 @@ double sinsp_threadinfo::get_fd_usage_pct_d() } } -uint64_t sinsp_threadinfo::get_fd_opencount() +uint64_t sinsp_threadinfo::get_fd_opencount() const { return get_main_thread()->m_fdtable.size(); } @@ -977,7 +969,7 @@ bool sinsp_threadinfo::is_health_probe() m_category == sinsp_threadinfo::CAT_READINESS_PROBE); } -shared_ptr sinsp_threadinfo::lookup_thread() +shared_ptr sinsp_threadinfo::lookup_thread() const { return m_inspector->get_thread_ref(m_pid, true, true, true); } diff --git a/userspace/libsinsp/threadinfo.h b/userspace/libsinsp/threadinfo.h index 51d26b4fe4..f228135913 100644 --- a/userspace/libsinsp/threadinfo.h +++ b/userspace/libsinsp/threadinfo.h @@ -71,9 +71,8 @@ typedef struct erase_fd_params class SINSP_PUBLIC sinsp_threadinfo { public: - sinsp_threadinfo(); - sinsp_threadinfo(sinsp *inspector); - ~sinsp_threadinfo(); + sinsp_threadinfo(sinsp *inspector = nullptr); + virtual ~sinsp_threadinfo(); /*! \brief Return the name of the process containing this thread, e.g. "top". @@ -119,7 +118,7 @@ class SINSP_PUBLIC sinsp_threadinfo \brief Get the main thread of the process containing this thread. */ #ifndef _WIN32 - inline sinsp_threadinfo* get_main_thread() + inline sinsp_threadinfo* get_main_thread() const { auto main_thread = m_main_thread.lock(); if(!main_thread) @@ -136,7 +135,7 @@ class SINSP_PUBLIC sinsp_threadinfo // invoked for a threadinfo that is in the stack. Caching the this pointer // would cause future mess. // - return this; + return const_cast(this); } else { @@ -218,7 +217,7 @@ class SINSP_PUBLIC sinsp_threadinfo /*! \brief Return the number of open FDs for this thread. */ - uint64_t get_fd_opencount(); + uint64_t get_fd_opencount() const; /*! \brief Return the maximum number of FDs this thread can open. @@ -352,33 +351,42 @@ class SINSP_PUBLIC sinsp_threadinfo } }; -VISIBILITY_PRIVATE - void init(); - // return true if, based on the current inspector filter, this thread should be kept - void init(scap_threadinfo* pi); - void fix_sockets_coming_from_proc(); - sinsp_fdinfo_t* add_fd(int64_t fd, sinsp_fdinfo_t *fdinfo); - void add_fd_from_scap(scap_fdinfo *fdinfo, OUT sinsp_fdinfo_t *res); - void remove_fd(int64_t fd); +protected: inline sinsp_fdtable* get_fd_table() { - sinsp_threadinfo* root; - if(!(m_flags & PPM_CL_CLONE_FILES)) { - root = this; + return &m_fdtable;; } else { - root = get_main_thread(); - if(NULL == root) - { - return NULL; - } + sinsp_threadinfo* root = get_main_thread(); + return (root == nullptr) ? nullptr : &(root->m_fdtable); } + } - return &(root->m_fdtable); + inline const sinsp_fdtable* get_fd_table() const + { + if(!(m_flags & PPM_CL_CLONE_FILES)) + { + return &m_fdtable;; + } + else + { + sinsp_threadinfo* root = get_main_thread(); + return (root == nullptr) ? nullptr : &(root->m_fdtable); + } } + +public: +VISIBILITY_PRIVATE + void init(); + // return true if, based on the current inspector filter, this thread should be kept + void init(scap_threadinfo* pi); + void fix_sockets_coming_from_proc(); + sinsp_fdinfo_t* add_fd(int64_t fd, sinsp_fdinfo_t *fdinfo); + void add_fd_from_scap(scap_fdinfo *fdinfo, OUT sinsp_fdinfo_t *res); + void remove_fd(int64_t fd); void set_cwd(const char *cwd, uint32_t cwdlen); sinsp_threadinfo* get_cwd_root(); void set_args(const char* args, size_t len); @@ -399,7 +407,7 @@ VISIBILITY_PRIVATE } void allocate_private_state(); void compute_program_hash(); - std::shared_ptr lookup_thread(); + std::shared_ptr lookup_thread() const; size_t strvec_len(const std::vector &strs) const; void strvec_to_iovec(const std::vector &strs, @@ -424,7 +432,7 @@ VISIBILITY_PRIVATE // sinsp_fdtable m_fdtable; // The fd table of this thread std::string m_cwd; // current working directory - std::weak_ptr m_main_thread; + mutable std::weak_ptr m_main_thread; uint8_t* m_lastevent_data; // Used by some event parsers to store the last enter event std::vector m_private_state; diff --git a/userspace/libsinsp/tracers.cpp b/userspace/libsinsp/tracers.cpp index e5f4766005..e4e4c1c249 100644 --- a/userspace/libsinsp/tracers.cpp +++ b/userspace/libsinsp/tracers.cpp @@ -1304,17 +1304,12 @@ inline void sinsp_tracerparser::init_partial_tracer(sinsp_partial_tracer* pae) void sinsp_tracerparser::test() { -// char doc[] = "[\">\\\"\", 12435, [\"mysql\", \"query\", \"init\"], [{\"argname1\":\"argval1\"}, {\"argname2\":\"argval2\"}, {\"argname3\":\"argval3\"}]]"; -// char doc1[] = "[\"\", 12345, [\"mysql\", \"query\", \"init\"], [{\"argname1\":\"argval1\"}, {\"argname2\":\"argval2\"}, {\"argname3\":\"argval3\"}]]"; -// char doc1[] = ">:1111:u\\:\\=a.u\\:\\>.aaa.33.aa\\::a=b\\:\\=,c=d\\:\\=a:"; - sinsp_threadinfo tinfo; - - m_tinfo = &tinfo; - tinfo.m_ptid = 11; - tinfo.m_pid = 22; - tinfo.m_tid = 33; + m_tinfo = new sinsp_threadinfo(nullptr); + m_tinfo->m_ptid = 11; + m_tinfo->m_pid = 22; + m_tinfo->m_tid = 33; printf("1\n"); @@ -1339,4 +1334,7 @@ void sinsp_tracerparser::test() cpu_time = ((float)clock()/ CLOCKS_PER_SEC) - cpu_time; printf ("time: %5.2f\n", cpu_time); + + delete m_tinfo; + m_tinfo = nullptr; }