Async container info

CMakeLists.txt

@@ -241,7 +241,7 @@ endif()
 #
 # Intel tbb
 #
-if(NOT WIN32 AND NOT APPLE)
+if(NOT WIN32)
 	option(USE_BUNDLED_TBB "Enable building of the bundled tbb" ${USE_BUNDLED_DEPS})
 	if(NOT USE_BUNDLED_TBB)
 		find_path(TBB_INCLUDE_DIR tbb.h PATH_SUFFIXES tbb)
@@ -411,7 +411,7 @@ if(NOT WIN32)
 			DEPENDS openssl
 			URL "http://download.draios.com/dependencies/curl-7.61.0.tar.bz2"
 			URL_MD5 "31d0a9f48dc796a7db351898a1e5058a"
-			CONFIGURE_COMMAND ./configure ${CURL_SSL_OPTION} --disable-threaded-resolver --disable-shared --enable-optimize --disable-curldebug --disable-rt --enable-http --disable-ftp --disable-file --disable-ldap --disable-ldaps --disable-rtsp --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-sspi --disable-ntlm-wb --disable-tls-srp --without-winssl --without-darwinssl --without-polarssl --without-cyassl --without-nss --without-axtls --without-ca-path --without-ca-bundle --without-libmetalink --without-librtmp --without-winidn --without-libidn --without-nghttp2 --without-libssh2
+			CONFIGURE_COMMAND ./configure ${CURL_SSL_OPTION} --disable-threaded-resolver --disable-shared --enable-optimize --disable-curldebug --disable-rt --enable-http --disable-ftp --disable-file --disable-ldap --disable-ldaps --disable-rtsp --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-sspi --disable-ntlm-wb --disable-tls-srp --without-winssl --without-darwinssl --without-polarssl --without-cyassl --without-nss --without-axtls --without-ca-path --without-ca-bundle --without-libmetalink --without-librtmp --without-winidn --without-libidn --without-libidn2 --without-nghttp2 --without-libssh2
 			BUILD_COMMAND ${CMD_MAKE}
 			BUILD_IN_SOURCE 1
 			BUILD_BYPRODUCTS ${CURL_LIBRARIES}

driver/event_table.c

@@ -284,8 +284,8 @@ const struct ppm_event_info g_event_info[PPM_EVENT_MAX] = {
 	/* PPME_TRACER_X */{ "tracer", EC_OTHER, EF_NONE, 3, { { "id", PT_INT64, PF_DEC }, { "tags", PT_CHARBUFARRAY, PF_NA }, { "args", PT_CHARBUF_PAIR_ARRAY, PF_NA } } },
 	/* PPME_MESOS_E */{"mesos", EC_INTERNAL, EF_SKIPPARSERESET | EF_MODIFIES_STATE, 1, {{"json", PT_CHARBUF, PF_NA} } },
 	/* PPME_MESOS_X */{"NA4", EC_SYSTEM, EF_UNUSED, 0},
-	/* PPME_CONTAINER_JSON_E */{"container", EC_INTERNAL, EF_SKIPPARSERESET | EF_MODIFIES_STATE, 1, {{"json", PT_CHARBUF, PF_NA} } },
-	/* PPME_CONTAINER_JSON_X */{"container", EC_INTERNAL, EF_UNUSED, 0},
+	/* PPME_CONTAINER_JSON_E */{"container", EC_PROCESS, EF_MODIFIES_STATE, 1, {{"json", PT_CHARBUF, PF_NA} } },
+	/* PPME_CONTAINER_JSON_X */{"container", EC_PROCESS, EF_UNUSED, 0},
 	/* PPME_SYSCALL_SETSID_E */{"setsid", EC_PROCESS, EF_MODIFIES_STATE, 0},
 	/* PPME_SYSCALL_SETSID_X */{"setsid", EC_PROCESS, EF_MODIFIES_STATE, 1, {{"res", PT_PID, PF_DEC} } },
 	/* PPME_SYSCALL_MKDIR_2_E */{"mkdir", EC_FILE, EF_NONE, 1, {{"mode", PT_UINT32, PF_HEX} } },

userspace/libsinsp/CMakeLists.txt

@@ -18,21 +18,25 @@
 include_directories(./)
 include_directories(../../common)
 include_directories(../libscap)
+include_directories(../async)
 include_directories("${JSONCPP_INCLUDE}")
 include_directories("${LUAJIT_INCLUDE}")
+include_directories("${TBB_INCLUDE_DIR}")
 
 if(NOT WIN32 AND NOT APPLE)
 	include_directories("${B64_INCLUDE}")
-	include_directories("${CURL_INCLUDE_DIR}")
 	include_directories("${CURSES_INCLUDE_DIR}")
 	include_directories("${GRPC_INCLUDE}")
 	include_directories("${JQ_INCLUDE}")
 	include_directories("${OPENSSL_INCLUDE_DIR}")
 	include_directories("${PROTOBUF_INCLUDE}")
-	include_directories("${TBB_INCLUDE_DIR}")
 	include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 endif()
 
+if(NOT WIN32)
+	include_directories("${CURL_INCLUDE_DIR}")
+endif()
+
 set(SINSP_SOURCES
 	chisel.cpp
 	chisel_api.cpp
@@ -149,9 +153,13 @@ if(NOT WIN32)
 	endif()
 	if(USE_BUNDLED_CURL)
 		add_dependencies(sinsp curl)
+		target_link_libraries(sinsp
+			"${CURL_LIBRARIES}")
 	endif()
 	if(USE_BUNDLED_TBB)
 		add_dependencies(sinsp tbb)
+		target_link_libraries(sinsp
+			"${TBB_LIB}")
 	endif()
 
 	if(NOT APPLE)
@@ -173,8 +181,6 @@ if(NOT WIN32)
 			"${CARES_LIB}"
 			"${JQ_LIB}"
 			"${B64_LIB}"
-			"${CURL_LIBRARIES}"
-			"${TBB_LIB}"
 			rt
 			anl)
 	endif()

userspace/libsinsp/container.cpp

@@ -147,6 +147,7 @@ string sinsp_container_manager::container_to_json(const sinsp_container_info& co
 	container["imagetag"] = container_info.m_imagetag;
 	container["imagedigest"] = container_info.m_imagedigest;
 	container["privileged"] = container_info.m_privileged;
+	container["is_pod_sandbox"] = container_info.m_is_pod_sandbox;
 
 	Json::Value mounts = Json::arrayValue;
 
@@ -175,14 +176,51 @@ string sinsp_container_manager::container_to_json(const sinsp_container_info& co
 	inet_ntop(AF_INET, &iph, addrbuff, sizeof(addrbuff));
 	container["ip"] = addrbuff;
 
+	Json::Value port_mappings = Json::arrayValue;
+
+	for(auto &mapping : container_info.m_port_mappings)
+	{
+		Json::Value jmap;
+		jmap["HostIp"] = mapping.m_host_ip;
+		jmap["HostPort"] = mapping.m_host_port;
+		jmap["ContainerPort"] = mapping.m_container_port;
+
+		port_mappings.append(jmap);
+	}
+
+	container["port_mappings"] = port_mappings;
+
+	Json::Value labels;
+	for (auto &pair : container_info.m_labels)
+	{
+		labels[pair.first] = pair.second;
+	}
+	container["labels"] = labels;
+
+	Json::Value env_vars = Json::arrayValue;
+
+	for (auto &var : container_info.m_env)
+	{
+		env_vars.append(var);
+	}
+	container["env"] = env_vars;
+
+	container["memory_limit"] = (Json::Value::Int64) container_info.m_memory_limit;
+	container["swap_limit"] = (Json::Value::Int64) container_info.m_swap_limit;
+	container["cpu_shares"] = (Json::Value::Int64) container_info.m_cpu_shares;
+	container["cpu_quota"] = (Json::Value::Int64) container_info.m_cpu_quota;
+	container["cpu_period"] = (Json::Value::Int64) container_info.m_cpu_period;
+
 	if(!container_info.m_mesos_task_id.empty())
 	{
 		container["mesos_task_id"] = container_info.m_mesos_task_id;
 	}
+
+	container["metadata_deadline"] = (Json::Value::UInt64) container_info.m_metadata_deadline;
 	return Json::FastWriter().write(obj);
 }
 
-bool sinsp_container_manager::container_to_sinsp_event(const string& json, sinsp_evt* evt)
+bool sinsp_container_manager::container_to_sinsp_event(const string& json, sinsp_evt* evt, int64_t tid)
 {
 	// TODO: variable event length
 	size_t evt_len = SP_EVT_BUF_SIZE;
@@ -201,7 +239,7 @@ bool sinsp_container_manager::container_to_sinsp_event(const string& json, sinsp
 	scap_evt* scapevt = evt->m_pevt;
 
 	scapevt->ts = m_inspector->m_lastevent_ts;
-	scapevt->tid = 0;
+	scapevt->tid = tid;
 	scapevt->len = (uint32_t)totlen;
 	scapevt->type = PPME_CONTAINER_JSON_E;
 	scapevt->nparams = 1;
@@ -231,11 +269,22 @@ void sinsp_container_manager::add_container(const sinsp_container_info& containe
 	}
 }
 
-void sinsp_container_manager::notify_new_container(const sinsp_container_info& container_info)
+void sinsp_container_manager::notify_new_container(const sinsp_container_info& container_info, int64_t tid)
 {
-	if(container_to_sinsp_event(container_to_json(container_info), &m_inspector->m_meta_evt))
+	sinsp_evt *evt = new sinsp_evt();
+	evt->m_pevt_storage = new char[SP_EVT_BUF_SIZE];
+	evt->m_pevt = (scap_evt *) evt->m_pevt_storage;
+
+	if(container_to_sinsp_event(container_to_json(container_info), evt, tid))
 	{
-		m_inspector->m_meta_evt_pending = true;
+		std::shared_ptr<sinsp_evt> cevt(evt);
+
+		// Enqueue it onto the queue of pending container events for the inspector
+		m_inspector->m_pending_container_evts.push(cevt);
+	}
+	else
+	{
+		delete evt;
 	}
 }
 
@@ -363,7 +412,7 @@ void sinsp_container_manager::cleanup()
 
 void sinsp_container_manager::set_query_docker_image_info(bool query_image_info)
 {
-	libsinsp::container_engine::docker::set_query_image_info(query_image_info);
+	libsinsp::container_engine::docker_async_source::set_query_image_info(query_image_info);
 }
 
 void sinsp_container_manager::set_cri_extra_queries(bool extra_queries)
@@ -385,4 +434,4 @@ void sinsp_container_manager::set_cri_timeout(int64_t timeout_ms)
 #if defined(HAS_CAPTURE)
 	libsinsp::container_engine::cri::set_cri_timeout(timeout_ms);
 #endif
-}
+}

userspace/libsinsp/container.h

@@ -39,7 +39,7 @@ class sinsp_container_manager
 	bool remove_inactive_containers();
 	void add_container(const sinsp_container_info& container_info, sinsp_threadinfo *thread);
 	sinsp_container_info * get_container(const string &id);
-	void notify_new_container(const sinsp_container_info& container_info);
+	void notify_new_container(const sinsp_container_info& container_info, int64_t tid);
 	template<typename E> bool resolve_container_impl(sinsp_threadinfo* tinfo, bool query_os_for_missing_info);
 	template<typename E1, typename E2, typename... Args> bool resolve_container_impl(sinsp_threadinfo* tinfo, bool query_os_for_missing_info);
 	bool resolve_container(sinsp_threadinfo* tinfo, bool query_os_for_missing_info);
@@ -71,7 +71,7 @@ class sinsp_container_manager
 	sinsp* get_inspector() { return m_inspector; }
 private:
 	string container_to_json(const sinsp_container_info& container_info);
-	bool container_to_sinsp_event(const string& json, sinsp_evt* evt);
+	bool container_to_sinsp_event(const string& json, sinsp_evt* evt, int64_t tid=0);
 	string get_docker_env(const Json::Value &env_vars, const string &mti);
 
 	sinsp* m_inspector;

userspace/libsinsp/container_engine/cri.cpp

@@ -197,17 +197,18 @@ void cri::set_extra_queries(bool extra_queries) {
 bool cri::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, bool query_os_for_missing_info)
 {
 	sinsp_container_info container_info;
+	sinsp_container_info *existing_container_info;
 
-	if(matches_runc_cgroups(tinfo, CRI_CGROUP_LAYOUT, container_info.m_id))
-	{
-		container_info.m_type = s_cri_runtime_type;
-	}
-	else
+	if(!matches_runc_cgroups(tinfo, CRI_CGROUP_LAYOUT, container_info.m_id))
 	{
 		return false;
 	}
 	tinfo->m_container_id = container_info.m_id;
-	if (!manager->container_exists(container_info.m_id))
+
+	existing_container_info = manager->get_container(container_info.m_id);
+
+	if (!existing_container_info ||
+	    existing_container_info->m_metadata_complete == false)
 	{
 		if (query_os_for_missing_info)
 		{
@@ -217,6 +218,11 @@ bool cri::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, boo
 						container_info.m_id.c_str());
 				return false;
 			}
+
+			// If here, parse_cri succeeded so we can
+			// assign an actual type.
+			container_info.m_type = s_cri_runtime_type;
+
 		}
 		if (mesos::set_mesos_task_id(&container_info, tinfo))
 		{
@@ -225,7 +231,7 @@ bool cri::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, boo
 					container_info.m_id.c_str(), container_info.m_mesos_task_id.c_str());
 		}
 		manager->add_container(container_info, tinfo);
-		manager->notify_new_container(container_info);
+		manager->notify_new_container(container_info, tinfo->m_tid);
 	}
 	return true;
 }

userspace/libsinsp/container_engine/docker.h

@@ -22,18 +22,37 @@ limitations under the License.
 #include <memory>
 #include <string>
 #include <vector>
+#include <atomic>
+
+#if !defined(_WIN32)
+#include <curl/curl.h>
+#include <curl/easy.h>
+#include <curl/multi.h>
+#endif
+
+#include "tbb/concurrent_hash_map.h"
 
 #include "json/json.h"
 
+#include "async_key_value_source.h"
+
 #include "container_info.h"
 
+class sinsp;
 class sinsp_container_manager;
 class sinsp_container_info;
 class sinsp_threadinfo;
 
 namespace libsinsp {
 namespace container_engine {
-class docker
+
+struct container_lookup_result
+{
+	bool m_successful;
+	sinsp_container_info m_container_info;
+};
+
+class docker_async_source : public sysdig::async_key_value_source<std::string, container_lookup_result>
 {
 	enum docker_response
 	{
@@ -42,22 +61,83 @@ class docker
 		RESP_ERROR = 2
 	};
 
+public:
+	docker_async_source(uint64_t max_wait_ms, uint64_t ttl_ms, sinsp *inspector);
+	virtual ~docker_async_source();
+
+	static void set_query_image_info(bool query_image_info);
+
+	// Note that this tid is the current top tid for this container
+	void set_top_tid(const std::string &container_id, sinsp_threadinfo *tinfo);
+
+	// Update the mapping from container id to top running tid for
+	// that container.
+	void update_top_tid(std::string &container_id, sinsp_threadinfo *tinfo);
+
+	// Get the thread id of the top thread running in this container.
+	int64_t get_top_tid(const std::string &container_id);
+
+        bool pending_lookup(std::string &container_id);
+
+protected:
+	void run_impl();
+
+private:
+	// These 4 methods are OS-dependent and defined in docker_{linux,win}.cpp
+	void init_docker_conn();
+	void free_docker_conn();
+	std::string build_request(const std::string& url);
+	docker_response get_docker(const std::string& url, std::string &json);
+
+	bool parse_docker(std::string &container_id, sinsp_container_info *container);
+
+	sinsp *m_inspector;
+
+	std::string m_docker_unix_socket_path;
+	std::string m_api_version;
+
+#ifndef _WIN32
+	CURLM *m_curlm;
+	CURL *m_curl;
+#endif
+
+	static bool m_query_image_info;
+
+	// Maps from container id to the "top" threadinfo in the
+	// process heirarchy having that container id that
+	// exists. These associations are only maintained while an
+	// async lookup of container information is in progress. We
+	// use this to ensure that the tid of the CONTAINER_JSON event
+	// we eventually emit is a valid thread, and the top running
+	// thread, in the container.
+	//
+        // We want the container event to have a valid thread because
+        // all the container.* filterchecks first look up the
+        // threadinfo for the event and then use the threadinfo's
+        // m_container_id to look up to the container. If the container
+        // event didn't have a valid threadinfo, that lookup would
+        // fail and the container.* filterchecks would not return anything.
+	typedef tbb::concurrent_hash_map<std::string, int64_t> top_tid_table;
+	top_tid_table m_top_tids;
+};
+
+class docker
+{
 public:
 	docker();
 
 	bool resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, bool query_os_for_missing_info);
 	static void cleanup();
-	static void set_query_image_info(bool query_image_info);
 	static void parse_json_mounts(const Json::Value &mnt_obj, std::vector<sinsp_container_info::container_mount_info> &mounts);
 
+	// Container name only set for windows. For linux name must be fetched via lookup
+	static bool detect_docker(const sinsp_threadinfo* tinfo, std::string& container_id, std::string &container_name);
 protected:
-	docker_response get_docker(sinsp_container_manager* manager, const std::string& url, std::string &json);
-	std::string build_request(const std::string& url);
-	bool parse_docker(sinsp_container_manager* manager, sinsp_container_info *container, sinsp_threadinfo* tinfo);
+	void parse_docker_async(sinsp *inspector, std::string &container_id, sinsp_container_manager *manager);
 
-	static std::string m_api_version;
-	static bool m_query_image_info;
-	static bool m_enabled;
+	static std::unique_ptr<docker_async_source> g_docker_info_source;
+
+	static std::string s_incomplete_info_name;
 };
 }
 }