From 16fe92f040b38bf0ca14ba10d162a95c11ade019 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Wed, 6 Mar 2019 19:33:49 -0800
Subject: [PATCH 01/15] Read from a queue of pending events in sinsp::next

Add a non-blocking queue to the inspector and in sinsp::next() try to
read events from that queue, similar to how we look at m_metaevt.

If any events are found, they are returned from sinsp::next() instead of
reading from the libscap.

In order to support truly portable sinsp_event objects, we
need (optional) storage for the matching libscap event in m_pevt. This
is in m_pevt_storage, NULL by default but if non-NULL will be
freed. This will be used to pass container events to the inspector via
the queue.
---
 userspace/libsinsp/event.cpp |  6 ++++++
 userspace/libsinsp/event.h   |  1 +
 userspace/libsinsp/sinsp.cpp |  5 +++++
 userspace/libsinsp/sinsp.h   | 10 ++++++++++
 4 files changed, 22 insertions(+)

diff --git a/userspace/libsinsp/event.cpp b/userspace/libsinsp/event.cpp
index 146b6c0d39..b29539bbd0 100644
--- a/userspace/libsinsp/event.cpp
+++ b/userspace/libsinsp/event.cpp
@@ -62,6 +62,7 @@ extern sinsp_evttables g_infotables;
 // sinsp_evt implementation
 ///////////////////////////////////////////////////////////////////////////////
 sinsp_evt::sinsp_evt() :
+	m_pevt_storage(NULL),
 	m_paramstr_storage(256), m_resolved_paramstr_storage(1024)
 {
 	m_flags = EF_NONE;
@@ -73,6 +74,7 @@ sinsp_evt::sinsp_evt() :
 }
 
 sinsp_evt::sinsp_evt(sinsp *inspector) :
+	m_pevt_storage(NULL),
 	m_paramstr_storage(1024), m_resolved_paramstr_storage(1024)
 {
 	m_inspector = inspector;
@@ -86,6 +88,10 @@ sinsp_evt::sinsp_evt(sinsp *inspector) :
 
 sinsp_evt::~sinsp_evt()
 {
+	if(m_pevt_storage)
+	{
+		delete[] m_pevt_storage;
+	}
 }
 
 uint32_t sinsp_evt::get_dump_flags()
diff --git a/userspace/libsinsp/event.h b/userspace/libsinsp/event.h
index 596b90570a..d693b7d29c 100644
--- a/userspace/libsinsp/event.h
+++ b/userspace/libsinsp/event.h
@@ -435,6 +435,7 @@ VISIBILITY_PRIVATE
 	sinsp* m_inspector;
 	scap_evt* m_pevt;
 	scap_evt* m_poriginal_evt;	// This is used when the original event is replaced by a different one (e.g. in the case of user events)
+	char *m_pevt_storage;           // In some cases an alternate buffer is used to hold m_pevt. This points to that storage.
 	uint16_t m_cpuid;
 	uint64_t m_evtnum;
 	uint32_t m_flags;
diff --git a/userspace/libsinsp/sinsp.cpp b/userspace/libsinsp/sinsp.cpp
index bb32831e1c..26c0f39ea0 100644
--- a/userspace/libsinsp/sinsp.cpp
+++ b/userspace/libsinsp/sinsp.cpp
@@ -1046,6 +1046,11 @@ int32_t sinsp::next(OUT sinsp_evt **puevt)
 		evt = m_meta_skipped_evt;
 		m_meta_evt_pending = false;
 	}
+	else if (m_pending_container_evts.try_pop(m_container_evt))
+	{
+		res = SCAP_SUCCESS;
+		evt = m_container_evt.get();
+	}
 	else
 	{
 		evt = &m_evt;
diff --git a/userspace/libsinsp/sinsp.h b/userspace/libsinsp/sinsp.h
index 5214260600..be8248d1ec 100644
--- a/userspace/libsinsp/sinsp.h
+++ b/userspace/libsinsp/sinsp.h
@@ -49,6 +49,8 @@ limitations under the License.
 #pragma warning(disable: 4251 4200 4221 4190)
 #endif
 
+#include "tbb/concurrent_queue.h"
+
 #include "sinsp_inet.h"
 #include "sinsp_public.h"
 
@@ -1164,6 +1166,14 @@ VISIBILITY_PRIVATE
 	meta_event_callback m_meta_event_callback;
 	void* m_meta_event_callback_data;
 
+	// A queue of pending container events. Written from async
+	// callbacks that occur after looking up container
+	// information, read from sinsp::next().
+	tbb::concurrent_queue<shared_ptr<sinsp_evt>> m_pending_container_evts;
+
+	// Holds an event dequeued from the above queue
+	std::shared_ptr<sinsp_evt> m_container_evt;
+
 	//
 	// End of second housekeeping
 	//

From ff599a381f376d2c837ba3a09cac22c33afcf567 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Wed, 6 Mar 2019 19:38:49 -0800
Subject: [PATCH 02/15] Add tid for container events, pass to inspector

Make sure container_json events have a real thread id by passing it as
an argument to container_to_sinsp_event.

Modify notify_new_container to use the non-blocking queue to pass events
to the inspector instead of m_meta_evt.
---
 userspace/libsinsp/container.cpp              | 21 ++++++++++++++-----
 userspace/libsinsp/container.h                |  4 ++--
 userspace/libsinsp/container_engine/cri.cpp   |  2 +-
 .../libsinsp/container_engine/libvirt_lxc.cpp |  2 +-
 userspace/libsinsp/container_engine/lxc.cpp   |  2 +-
 userspace/libsinsp/container_engine/mesos.cpp |  2 +-
 userspace/libsinsp/container_engine/rkt.cpp   |  2 +-
 7 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/userspace/libsinsp/container.cpp b/userspace/libsinsp/container.cpp
index 869a6f6654..077ed8e01f 100644
--- a/userspace/libsinsp/container.cpp
+++ b/userspace/libsinsp/container.cpp
@@ -182,7 +182,7 @@ string sinsp_container_manager::container_to_json(const sinsp_container_info& co
 	return Json::FastWriter().write(obj);
 }
 
-bool sinsp_container_manager::container_to_sinsp_event(const string& json, sinsp_evt* evt)
+bool sinsp_container_manager::container_to_sinsp_event(const string& json, sinsp_evt* evt, int64_t tid)
 {
 	// TODO: variable event length
 	size_t evt_len = SP_EVT_BUF_SIZE;
@@ -201,7 +201,7 @@ bool sinsp_container_manager::container_to_sinsp_event(const string& json, sinsp
 	scap_evt* scapevt = evt->m_pevt;
 
 	scapevt->ts = m_inspector->m_lastevent_ts;
-	scapevt->tid = 0;
+	scapevt->tid = tid;
 	scapevt->len = (uint32_t)totlen;
 	scapevt->type = PPME_CONTAINER_JSON_E;
 	scapevt->nparams = 1;
@@ -231,11 +231,22 @@ void sinsp_container_manager::add_container(const sinsp_container_info& containe
 	}
 }
 
-void sinsp_container_manager::notify_new_container(const sinsp_container_info& container_info)
+void sinsp_container_manager::notify_new_container(const sinsp_container_info& container_info, int64_t tid)
 {
-	if(container_to_sinsp_event(container_to_json(container_info), &m_inspector->m_meta_evt))
+	sinsp_evt *evt = new sinsp_evt();
+	evt->m_pevt_storage = new char[SP_EVT_BUF_SIZE];
+	evt->m_pevt = (scap_evt *) evt->m_pevt_storage;
+
+	if(container_to_sinsp_event(container_to_json(container_info), evt, tid))
+	{
+		std::shared_ptr<sinsp_evt> cevt(evt);
+
+		// Enqueue it onto the queue of pending container events for the inspector
+		m_inspector->m_pending_container_evts.push(cevt);
+	}
+	else
 	{
-		m_inspector->m_meta_evt_pending = true;
+		delete evt;
 	}
 }
 
diff --git a/userspace/libsinsp/container.h b/userspace/libsinsp/container.h
index 6aeafa0f79..79f14d8801 100644
--- a/userspace/libsinsp/container.h
+++ b/userspace/libsinsp/container.h
@@ -39,7 +39,7 @@ class sinsp_container_manager
 	bool remove_inactive_containers();
 	void add_container(const sinsp_container_info& container_info, sinsp_threadinfo *thread);
 	sinsp_container_info * get_container(const string &id);
-	void notify_new_container(const sinsp_container_info& container_info);
+	void notify_new_container(const sinsp_container_info& container_info, int64_t tid);
 	template<typename E> bool resolve_container_impl(sinsp_threadinfo* tinfo, bool query_os_for_missing_info);
 	template<typename E1, typename E2, typename... Args> bool resolve_container_impl(sinsp_threadinfo* tinfo, bool query_os_for_missing_info);
 	bool resolve_container(sinsp_threadinfo* tinfo, bool query_os_for_missing_info);
@@ -71,7 +71,7 @@ class sinsp_container_manager
 	sinsp* get_inspector() { return m_inspector; }
 private:
 	string container_to_json(const sinsp_container_info& container_info);
-	bool container_to_sinsp_event(const string& json, sinsp_evt* evt);
+	bool container_to_sinsp_event(const string& json, sinsp_evt* evt, int64_t tid=0);
 	string get_docker_env(const Json::Value &env_vars, const string &mti);
 
 	sinsp* m_inspector;
diff --git a/userspace/libsinsp/container_engine/cri.cpp b/userspace/libsinsp/container_engine/cri.cpp
index 06f2748ba7..444f454b2c 100644
--- a/userspace/libsinsp/container_engine/cri.cpp
+++ b/userspace/libsinsp/container_engine/cri.cpp
@@ -225,7 +225,7 @@ bool cri::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, boo
 					container_info.m_id.c_str(), container_info.m_mesos_task_id.c_str());
 		}
 		manager->add_container(container_info, tinfo);
-		manager->notify_new_container(container_info);
+		manager->notify_new_container(container_info, tinfo->m_tid);
 	}
 	return true;
 }
diff --git a/userspace/libsinsp/container_engine/libvirt_lxc.cpp b/userspace/libsinsp/container_engine/libvirt_lxc.cpp
index 61eda09e17..5b01870445 100644
--- a/userspace/libsinsp/container_engine/libvirt_lxc.cpp
+++ b/userspace/libsinsp/container_engine/libvirt_lxc.cpp
@@ -87,7 +87,7 @@ bool libvirt_lxc::resolve(sinsp_container_manager* manager, sinsp_threadinfo* ti
 	{
 		container_info.m_name = container_info.m_id;
 		manager->add_container(container_info, tinfo);
-		manager->notify_new_container(container_info);
+		manager->notify_new_container(container_info, tinfo->m_tid);
 	}
 	return true;
 }
diff --git a/userspace/libsinsp/container_engine/lxc.cpp b/userspace/libsinsp/container_engine/lxc.cpp
index ec16b02c9a..74e7ceadb7 100644
--- a/userspace/libsinsp/container_engine/lxc.cpp
+++ b/userspace/libsinsp/container_engine/lxc.cpp
@@ -66,7 +66,7 @@ bool lxc::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, boo
 	{
 		container_info.m_name = container_info.m_id;
 		manager->add_container(container_info, tinfo);
-		manager->notify_new_container(container_info);
+		manager->notify_new_container(container_info, tinfo->m_tid);
 	}
 	return true;
 }
diff --git a/userspace/libsinsp/container_engine/mesos.cpp b/userspace/libsinsp/container_engine/mesos.cpp
index 7be539857c..3feace3c60 100644
--- a/userspace/libsinsp/container_engine/mesos.cpp
+++ b/userspace/libsinsp/container_engine/mesos.cpp
@@ -64,7 +64,7 @@ bool libsinsp::container_engine::mesos::resolve(sinsp_container_manager* manager
 	{
 		container_info.m_name = container_info.m_id;
 		manager->add_container(container_info, tinfo);
-		manager->notify_new_container(container_info);
+		manager->notify_new_container(container_info, tinfo->m_tid);
 	}
 	return true;
 }
diff --git a/userspace/libsinsp/container_engine/rkt.cpp b/userspace/libsinsp/container_engine/rkt.cpp
index f39ddc604b..720c22ec88 100644
--- a/userspace/libsinsp/container_engine/rkt.cpp
+++ b/userspace/libsinsp/container_engine/rkt.cpp
@@ -190,7 +190,7 @@ bool rkt::rkt::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo
 	if (have_rkt)
 	{
 		manager->add_container(container_info, tinfo);
-		manager->notify_new_container(container_info);
+		manager->notify_new_container(container_info, tinfo->m_tid);
 		return true;
 	}
 	else

From 8546f3e470bdb7ce7069fd057c76228ac35b376d Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Wed, 6 Mar 2019 19:44:09 -0800
Subject: [PATCH 03/15] Add all info to parsing/dumping of json events

Previously, CONTAINER_JSON events had some, but not all, of the info
that was in a sinsp_container_info object. Fix this so all important
info is both dumped to json in container_to_json and parsed in
parse_container_json_evt.
---
 driver/event_table.c             |  4 +-
 userspace/libsinsp/container.cpp | 39 ++++++++++++++
 userspace/libsinsp/parsers.cpp   | 90 +++++++++++++++++++++++++++++++-
 3 files changed, 130 insertions(+), 3 deletions(-)

diff --git a/driver/event_table.c b/driver/event_table.c
index bf81e85c60..4907d2113e 100644
--- a/driver/event_table.c
+++ b/driver/event_table.c
@@ -284,8 +284,8 @@ const struct ppm_event_info g_event_info[PPM_EVENT_MAX] = {
 	/* PPME_TRACER_X */{ "tracer", EC_OTHER, EF_NONE, 3, { { "id", PT_INT64, PF_DEC }, { "tags", PT_CHARBUFARRAY, PF_NA }, { "args", PT_CHARBUF_PAIR_ARRAY, PF_NA } } },
 	/* PPME_MESOS_E */{"mesos", EC_INTERNAL, EF_SKIPPARSERESET | EF_MODIFIES_STATE, 1, {{"json", PT_CHARBUF, PF_NA} } },
 	/* PPME_MESOS_X */{"NA4", EC_SYSTEM, EF_UNUSED, 0},
-	/* PPME_CONTAINER_JSON_E */{"container", EC_INTERNAL, EF_SKIPPARSERESET | EF_MODIFIES_STATE, 1, {{"json", PT_CHARBUF, PF_NA} } },
-	/* PPME_CONTAINER_JSON_X */{"container", EC_INTERNAL, EF_UNUSED, 0},
+	/* PPME_CONTAINER_JSON_E */{"container", EC_PROCESS, EF_MODIFIES_STATE, 1, {{"json", PT_CHARBUF, PF_NA} } },
+	/* PPME_CONTAINER_JSON_X */{"container", EC_PROCESS, EF_UNUSED, 0},
 	/* PPME_SYSCALL_SETSID_E */{"setsid", EC_PROCESS, EF_MODIFIES_STATE, 0},
 	/* PPME_SYSCALL_SETSID_X */{"setsid", EC_PROCESS, EF_MODIFIES_STATE, 1, {{"res", PT_PID, PF_DEC} } },
 	/* PPME_SYSCALL_MKDIR_2_E */{"mkdir", EC_FILE, EF_NONE, 1, {{"mode", PT_UINT32, PF_HEX} } },
diff --git a/userspace/libsinsp/container.cpp b/userspace/libsinsp/container.cpp
index 077ed8e01f..3ef8c96249 100644
--- a/userspace/libsinsp/container.cpp
+++ b/userspace/libsinsp/container.cpp
@@ -175,10 +175,49 @@ string sinsp_container_manager::container_to_json(const sinsp_container_info& co
 	inet_ntop(AF_INET, &iph, addrbuff, sizeof(addrbuff));
 	container["ip"] = addrbuff;
 
+	Json::Value port_mappings = Json::arrayValue;
+
+	for(auto &mapping : container_info.m_port_mappings)
+	{
+		Json::Value jmap;
+		jmap["HostIp"] = mapping.m_host_ip;
+		jmap["HostPort"] = mapping.m_host_port;
+		jmap["ContainerPort"] = mapping.m_container_port;
+
+		port_mappings.append(jmap);
+	}
+
+	container["port_mappings"] = port_mappings;
+
+	Json::Value labels;
+	for (auto &pair : container_info.m_labels)
+	{
+		labels[pair.first] = pair.second;
+	}
+	container["labels"] = labels;
+
+	Json::Value env_vars = Json::arrayValue;
+
+	for (auto &var : container_info.m_env)
+	{
+		env_vars.append(var);
+	}
+	container["env"] = env_vars;
+
+	container["memory_limit"] = (Json::Value::Int64) container_info.m_memory_limit;
+	container["swap_limit"] = (Json::Value::Int64) container_info.m_swap_limit;
+	container["cpu_shares"] = (Json::Value::Int64) container_info.m_cpu_shares;
+	container["cpu_quota"] = (Json::Value::Int64) container_info.m_cpu_quota;
+	container["cpu_period"] = (Json::Value::Int64) container_info.m_cpu_period;
+
 	if(!container_info.m_mesos_task_id.empty())
 	{
 		container["mesos_task_id"] = container_info.m_mesos_task_id;
 	}
+
+#ifdef HAS_ANALYZER
+	container["metadata_deadline"] = (Json::Value::UInt64) container_info.m_metadata_deadline;
+#endif
 	return Json::FastWriter().write(obj);
 }
 
diff --git a/userspace/libsinsp/parsers.cpp b/userspace/libsinsp/parsers.cpp
index b9017c4c2c..3103bb4f20 100644
--- a/userspace/libsinsp/parsers.cpp
+++ b/userspace/libsinsp/parsers.cpp
@@ -33,6 +33,7 @@ limitations under the License.
 #include <fcntl.h>
 #include <limits>
 
+#include "container_engine/mesos.h"
 #include "sinsp.h"
 #include "sinsp_int.h"
 #include "../../driver/ppm_ringbuffer.h"
@@ -4478,7 +4479,7 @@ void sinsp_parser::parse_container_json_evt(sinsp_evt *evt)
 	ASSERT(parinfo);
 	ASSERT(parinfo->m_len > 0);
 	std::string json(parinfo->m_val, parinfo->m_len);
-	g_logger.log(json, sinsp_logger::SEV_DEBUG);
+	g_logger.format(sinsp_logger::SEV_DEBUG, "Parsing Container JSON=%s", json.c_str());
 	ASSERT(m_inspector);
 	Json::Value root;
 	if(Json::Reader().parse(json, root))
@@ -4499,6 +4500,10 @@ void sinsp_parser::parse_container_json_evt(sinsp_evt *evt)
 		if(!name.isNull() && name.isConvertibleTo(Json::stringValue))
 		{
 			container_info.m_name = name.asString();
+			if(container_info.m_type == CT_DOCKER && container_info.m_name.find("k8s_POD") == 0)
+			{
+				container_info.m_is_pod_sandbox = true;
+			}
 		}
 		const Json::Value& image = container["image"];
 		if(!image.isNull() && image.isConvertibleTo(Json::stringValue))
@@ -4546,11 +4551,84 @@ void sinsp_parser::parse_container_json_evt(sinsp_evt *evt)
 
 			container_info.m_container_ip = ntohl(ip);
 		}
+
+		const Json::Value &port_mappings = container["port_mappings"];
+
+		if(!port_mappings.isNull() && port_mappings.isConvertibleTo(Json::arrayValue))
+		{
+			for (Json::Value::ArrayIndex i = 0; i != port_mappings.size(); i++)
+			{
+				sinsp_container_info::container_port_mapping map;
+				map.m_host_ip = port_mappings[i]["HostIp"].asInt();
+				map.m_host_port = (uint16_t) port_mappings[i]["HostPort"].asInt();
+				map.m_container_port = (uint16_t) port_mappings[i]["ContainerPort"].asInt();
+
+				container_info.m_port_mappings.push_back(map);
+			}
+		}
+
+		vector<string> labels = container["labels"].getMemberNames();
+		for(vector<string>::const_iterator it = labels.begin(); it != labels.end(); ++it)
+		{
+			string val = container["labels"][*it].asString();
+			container_info.m_labels[*it] = val;
+		}
+
+		const Json::Value& env_vars = container["env"];
+
+		for(const auto& env_var : env_vars)
+		{
+			if(env_var.isString())
+			{
+				container_info.m_env.emplace_back(env_var.asString());
+			}
+		}
+
+		const Json::Value& memory_limit = container["memory_limit"];
+		if(!memory_limit.isNull() && memory_limit.isConvertibleTo(Json::uintValue))
+		{
+			container_info.m_memory_limit = memory_limit.asUInt();
+		}
+
+		const Json::Value& swap_limit = container["swap_limit"];
+		if(!swap_limit.isNull() && swap_limit.isConvertibleTo(Json::uintValue))
+		{
+			container_info.m_swap_limit = swap_limit.asUInt();
+		}
+
+		const Json::Value& cpu_shares = container["cpu_shares"];
+		if(!cpu_shares.isNull() && cpu_shares.isConvertibleTo(Json::uintValue))
+		{
+			container_info.m_cpu_shares = cpu_shares.asUInt();
+		}
+
+		const Json::Value& cpu_quota = container["cpu_quota"];
+		if(!cpu_quota.isNull() && cpu_quota.isConvertibleTo(Json::uintValue))
+		{
+			container_info.m_cpu_quota = cpu_quota.asUInt();
+		}
+
+		const Json::Value& cpu_period = container["cpu_period"];
+		if(!cpu_period.isNull() && cpu_period.isConvertibleTo(Json::uintValue))
+		{
+			container_info.m_cpu_period = cpu_period.asUInt();
+		}
+
 		const Json::Value& mesos_task_id = container["mesos_task_id"];
 		if(!mesos_task_id.isNull() && mesos_task_id.isConvertibleTo(Json::stringValue))
 		{
 			container_info.m_mesos_task_id = mesos_task_id.asString();
 		}
+
+#ifdef HAS_ANALYZER
+		const Json::Value& metadata_deadline = container["metadata_deadline"];
+		// isConvertibleTo doesn't seem to work on large 64 bit numbers
+		if(!metadata_deadline.isNull() && metadata_deadline.isUInt64())
+		{
+			container_info.m_metadata_deadline = metadata_deadline.asUInt64();
+		}
+#endif
+
 		m_inspector->m_container_manager.add_container(container_info, evt->get_thread_info(true));
 		/*
 		g_logger.log("Container\n-------\nID:" + container_info.m_id +
@@ -4559,6 +4637,16 @@ void sinsp_parser::parse_container_json_evt(sinsp_evt *evt)
 					 "\nImage: " + container_info.m_image +
 					 "\nMesos Task ID: " + container_info.m_mesos_task_id, sinsp_logger::SEV_DEBUG);
 		*/
+
+		if(evt->m_tinfo)
+		{
+			if (libsinsp::container_engine::mesos::set_mesos_task_id(&container_info, evt->m_tinfo))
+			{
+				g_logger.format(sinsp_logger::SEV_DEBUG,
+						"Mesos Docker container: [%s], Mesos task ID: [%s]",
+						container_info.m_id.c_str(), container_info.m_mesos_task_id.c_str());
+			}
+		}
 	}
 	else
 	{

From 4f2ae8e210ded5a5d7c67dfff31a68f1d3ec8527 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Wed, 6 Mar 2019 19:46:36 -0800
Subject: [PATCH 04/15] Refactor docker info fetches to be async

Refactor docker metadata fetches to be asynchronous, using the framework
in sysdig::async_key_value_source.

docker_async_source (a global static so it can be long-lived) is now
responsible for looking up docker metadata. Some methods that used to be
in docker:: like parse_docker(), get_docker(), etc move to
docker_async_source. It dequeues requests, calling parse_docker() to
look up the information as needed, and calls store_value() to pass the
metadata back to the caller.

docker::resolve now uses parse_docker_async to schedule the lookup with
docker_async_source. Before scheduling the lookup it creates a stub
container with type UNKNOWN (UNKNOWN because you can't tell the
difference between cri and docker containers only from the cgroup), with
the id set, and with a name/image set to "incomplete". This ensures that
threads have some associated container info object with it. In the
callback once the full metadata is available, it calls
notify_new_container, which creates the CONTAINER_JSON event and pushes
it to the inspector.

There's a fair amount of bookeeping to make sure that the container
metadata has a valid tid. The very first tid that creates the container
often exits after forking of the real container entrypoint, so you need
to keep track of the "top tid" in the container for every call to
docker::resolve() and replace it if you find it's exited.

Previously, on the first error fetching container metadata, a flag
m_enabled would be set to false, and all subsequent attempts to fetch
container metadata would be skipped. Now that lookups are done in the
background, I think it makes sense to always try a lookup for every
container, even after failures. So remove m_enabled entirely from the
docker engine.

Also, as a part of this change, reorganize the way the async lookups are
done to better support the choices of different osen (linux, windows,
and HAS_CAPTURE). Instead of compiling out the curl handles when
HAS_CAPTURE is false, always compile the code for them but don't even
attempt any lookups in docker::resolve. Note that
docker_async_source::get_docker no longer changes behavior based on
HAS_CAPTURE.
---
 userspace/libsinsp/CMakeLists.txt             |   1 +
 userspace/libsinsp/container.cpp              |   2 +-
 userspace/libsinsp/container_engine/docker.h  |  88 ++++++-
 .../container_engine/docker_common.cpp        | 247 ++++++++++++++++--
 .../container_engine/docker_linux.cpp         | 137 ++++------
 .../libsinsp/container_engine/docker_win.cpp  |  35 ++-
 userspace/libsinsp/container_info.h           |  11 +-
 7 files changed, 385 insertions(+), 136 deletions(-)

diff --git a/userspace/libsinsp/CMakeLists.txt b/userspace/libsinsp/CMakeLists.txt
index 46046c16e6..422c704feb 100644
--- a/userspace/libsinsp/CMakeLists.txt
+++ b/userspace/libsinsp/CMakeLists.txt
@@ -18,6 +18,7 @@
 include_directories(./)
 include_directories(../../common)
 include_directories(../libscap)
+include_directories(../async)
 include_directories("${JSONCPP_INCLUDE}")
 include_directories("${LUAJIT_INCLUDE}")
 
diff --git a/userspace/libsinsp/container.cpp b/userspace/libsinsp/container.cpp
index 3ef8c96249..7ed0c2ea71 100644
--- a/userspace/libsinsp/container.cpp
+++ b/userspace/libsinsp/container.cpp
@@ -413,7 +413,7 @@ void sinsp_container_manager::cleanup()
 
 void sinsp_container_manager::set_query_docker_image_info(bool query_image_info)
 {
-	libsinsp::container_engine::docker::set_query_image_info(query_image_info);
+	libsinsp::container_engine::docker_async_source::set_query_image_info(query_image_info);
 }
 
 void sinsp_container_manager::set_cri_extra_queries(bool extra_queries)
diff --git a/userspace/libsinsp/container_engine/docker.h b/userspace/libsinsp/container_engine/docker.h
index d69f0429f5..92ed888f43 100644
--- a/userspace/libsinsp/container_engine/docker.h
+++ b/userspace/libsinsp/container_engine/docker.h
@@ -22,9 +22,22 @@ limitations under the License.
 #include <memory>
 #include <string>
 #include <vector>
+#include <atomic>
+
+#if !defined(_WIN32)
+#include <curl/curl.h>
+#include <curl/easy.h>
+#include <curl/multi.h>
+#endif
+
+#include "tbb/concurrent_hash_map.h"
 
 #include "json/json.h"
 
+#include "async_key_value_source.h"
+
+#include "sinsp.h"
+
 #include "container_info.h"
 
 class sinsp_container_manager;
@@ -33,7 +46,14 @@ class sinsp_threadinfo;
 
 namespace libsinsp {
 namespace container_engine {
-class docker
+
+struct container_lookup_result
+{
+	bool m_successful;
+	sinsp_container_info m_container_info;
+};
+
+class docker_async_source : public sysdig::async_key_value_source<std::string, container_lookup_result>
 {
 	enum docker_response
 	{
@@ -42,22 +62,74 @@ class docker
 		RESP_ERROR = 2
 	};
 
+public:
+	docker_async_source(uint64_t max_wait_ms, uint64_t ttl_ms, sinsp *inspector);
+	virtual ~docker_async_source();
+
+	static void set_query_image_info(bool query_image_info);
+
+	// Note that this tid is the current top tid for this container
+	void set_top_tid(const std::string &container_id, sinsp_threadinfo *tinfo);
+
+	// Update the mapping from container id to top running tid for
+	// that container.
+	void update_top_tid(std::string &container_id, sinsp_threadinfo *tinfo);
+
+	// Get the thread id of the top thread running in this container.
+	int64_t get_top_tid(const std::string &container_id);
+
+        bool pending_lookup(std::string &container_id);
+
+protected:
+	void run_impl();
+
+private:
+	// These 4 methods are OS-dependent and defined in docker_{linux,win}.cpp
+	void init_docker_conn();
+	void free_docker_conn();
+	std::string build_request(const std::string& url);
+	docker_response get_docker(const std::string& url, std::string &json);
+
+	bool parse_docker(std::string &container_id, sinsp_container_info *container);
+
+	sinsp *m_inspector;
+
+	std::string m_docker_unix_socket_path;
+	std::string m_api_version;
+
+#ifndef _WIN32
+	CURLM *m_curlm;
+	CURL *m_curl;
+#endif
+
+	static bool m_query_image_info;
+
+	// Maps from container id to the "top" threadinfo in the
+	// process heirarchy having that container id that
+	// exists. These associations are only maintained while an
+	// async lookup of container information is in progress. We
+	// use this to ensure that the tid of the CONTAINER_JSON event
+	// we eventually emit is the top running thread in the
+	// container.
+	typedef tbb::concurrent_hash_map<std::string, int64_t> top_tid_table;
+	top_tid_table m_top_tids;
+};
+
+class docker
+{
 public:
 	docker();
 
 	bool resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, bool query_os_for_missing_info);
 	static void cleanup();
-	static void set_query_image_info(bool query_image_info);
 	static void parse_json_mounts(const Json::Value &mnt_obj, std::vector<sinsp_container_info::container_mount_info> &mounts);
 
+	// Container name only set for windows. For linux name must be fetched via lookup
+	static bool detect_docker(const sinsp_threadinfo* tinfo, std::string& container_id, std::string &container_name);
 protected:
-	docker_response get_docker(sinsp_container_manager* manager, const std::string& url, std::string &json);
-	std::string build_request(const std::string& url);
-	bool parse_docker(sinsp_container_manager* manager, sinsp_container_info *container, sinsp_threadinfo* tinfo);
+	void parse_docker_async(sinsp *inspector, std::string &container_id, sinsp_container_manager *manager);
 
-	static std::string m_api_version;
-	static bool m_query_image_info;
-	static bool m_enabled;
+	static std::unique_ptr<docker_async_source> g_docker_info_source;
 };
 }
 }
diff --git a/userspace/libsinsp/container_engine/docker_common.cpp b/userspace/libsinsp/container_engine/docker_common.cpp
index 75a554ddb9..690a757b39 100644
--- a/userspace/libsinsp/container_engine/docker_common.cpp
+++ b/userspace/libsinsp/container_engine/docker_common.cpp
@@ -25,7 +25,127 @@ limitations under the License.
 
 using namespace libsinsp::container_engine;
 
-bool docker::m_query_image_info = true;
+docker_async_source::docker_async_source(uint64_t max_wait_ms, uint64_t ttl_ms, sinsp *inspector)
+	: async_key_value_source(max_wait_ms, ttl_ms),
+	  m_inspector(inspector),
+	  m_docker_unix_socket_path("/var/run/docker.sock"),
+#ifdef _WIN32
+	  m_api_version("/v1.30"),
+#else
+	  m_api_version("/v1.24"),
+	  m_curlm(NULL),
+	  m_curl(NULL)
+#endif
+{
+	init_docker_conn();
+}
+
+docker_async_source::~docker_async_source()
+{
+	free_docker_conn();
+}
+
+void docker_async_source::run_impl()
+{
+	std::string container_id;
+
+	while (dequeue_next_key(container_id))
+	{
+		container_lookup_result res;
+
+		res.m_successful = true;
+		res.m_container_info.m_type = CT_DOCKER;
+		res.m_container_info.m_id = container_id;
+
+		if(!parse_docker(container_id, &res.m_container_info))
+		{
+			g_logger.format(sinsp_logger::SEV_ERROR, "Failed to get Docker metadata for container %s",
+					container_id.c_str());
+			res.m_successful = false;
+		}
+
+		// Return a result object either way, to ensure any
+		// new container callbacks are called.
+		store_value(container_id, res);
+	}
+}
+
+void docker_async_source::set_top_tid(const std::string &container_id, sinsp_threadinfo *tinfo)
+{
+	top_tid_table::accessor acc;
+
+	m_top_tids.insert(acc, container_id);
+	acc->second = tinfo->m_tid;
+}
+
+int64_t docker_async_source::get_top_tid(const std::string &container_id)
+{
+	top_tid_table::const_accessor cacc;
+
+	if(m_top_tids.find(cacc, container_id))
+	{
+		return cacc->second;
+	}
+
+	g_logger.log("Did not find container id->tid mapping for in-progress container metadata lookup?", sinsp_logger::SEV_ERROR);
+	return 0;
+}
+
+bool docker_async_source::pending_lookup(std::string &container_id)
+{
+	top_tid_table::const_accessor cacc;
+
+	return m_top_tids.find(cacc, container_id);
+}
+
+void docker_async_source::update_top_tid(std::string &container_id, sinsp_threadinfo *tinfo)
+{
+	// See if the current top tid still exists.
+	top_tid_table::const_accessor cacc;
+	sinsp_threadinfo *top_tinfo;
+
+	int64_t top_tid = get_top_tid(container_id);
+
+	if(((top_tinfo = m_inspector->get_thread(top_tid)) == NULL) ||
+	   top_tinfo->m_flags & PPM_CL_CLOSED)
+	{
+		// The top thread no longer exists. Starting with tinfo,
+		// walk the parent heirarchy to find the top thread in
+		// the same container.
+
+		sinsp_threadinfo::visitor_func_t visitor =
+		[&] (sinsp_threadinfo *ptinfo)
+		{
+			// Stop at the first thread not in this container.
+			if(ptinfo->m_container_id != container_id)
+			{
+				return false;
+			}
+
+			// Ignore exited threads but keep traversing.
+			if(ptinfo->m_flags & PPM_CL_CLOSED)
+			{
+				return true;
+			}
+
+			if(ptinfo->m_container_id == container_id)
+			{
+				top_tinfo = ptinfo;
+			}
+
+			return true;
+		};
+
+		top_tinfo = tinfo;
+		tinfo->traverse_parent_state(visitor);
+
+		// At this point, top_tinfo should point to the top
+		// thread of those running in the container.
+		set_top_tid(container_id, top_tinfo);
+	}
+}
+
+bool docker_async_source::m_query_image_info = true;
 
 void docker::parse_json_mounts(const Json::Value &mnt_obj, vector<sinsp_container_info::container_mount_info> &mounts)
 {
@@ -41,20 +161,106 @@ void docker::parse_json_mounts(const Json::Value &mnt_obj, vector<sinsp_containe
 	}
 }
 
-void docker::set_query_image_info(bool query_image_info)
+void docker_async_source::set_query_image_info(bool query_image_info)
 {
 	m_query_image_info = query_image_info;
 }
 
-bool docker::parse_docker(sinsp_container_manager* manager, sinsp_container_info *container, sinsp_threadinfo* tinfo)
+bool docker::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, bool query_os_for_missing_info)
+{
+	std::string container_id, container_name;
+	sinsp_container_info *existing_container_info;
+
+	if(!g_docker_info_source)
+	{
+		uint64_t max_wait_ms = 10000;
+		docker_async_source *src = new docker_async_source(docker_async_source::NO_WAIT_LOOKUP, max_wait_ms, manager->get_inspector());
+		g_docker_info_source.reset(src);
+	}
+
+	if(!detect_docker(tinfo, container_id, container_name))
+	{
+		return false;
+	}
+
+	tinfo->m_container_id = container_id;
+
+	existing_container_info = manager->get_container(container_id);
+
+	if(!existing_container_info)
+	{
+		// Add a minimal container_info object where only the
+		// container id, (possibly) name, and a container
+		// image=incomplete is filled in. This may be
+		// overidden later once parse_docker_async completes.
+		sinsp_container_info container_info;
+
+		container_info.m_type = CT_UNKNOWN;
+		container_info.m_id = container_id;
+		container_info.m_name = container_name;
+		container_info.m_image="incomplete";
+		container_info.m_metadata_complete = false;
+
+		manager->add_container(container_info, tinfo);
+
+		existing_container_info = manager->get_container(container_id);
+	}
+
+#ifdef HAS_CAPTURE
+	// Possibly start a lookup for this container info
+	if(!existing_container_info->m_metadata_complete &&
+	    query_os_for_missing_info)
+	{
+		if(!g_docker_info_source->pending_lookup(container_id))
+		{
+			// give CRI a chance to return metadata for this container
+			parse_docker_async(manager->get_inspector(), container_id, manager);
+
+			g_docker_info_source->set_top_tid(container_id, tinfo);
+		}
+		else
+		{
+			g_docker_info_source->update_top_tid(container_id, tinfo);
+		}
+	}
+#endif
+
+	// Returning true will prevent other container engines from
+	// trying to resolve the container, so only return true if we
+	// have complete metadata.
+	return existing_container_info->m_metadata_complete;
+}
+
+void docker::parse_docker_async(sinsp *inspector, std::string &container_id, sinsp_container_manager *manager)
+{
+	auto cb = [manager](const std::string &container_id, const container_lookup_result &res)
+        {
+		if(res.m_successful)
+		{
+			int64_t top_tid = docker::g_docker_info_source->get_top_tid(container_id);
+			manager->notify_new_container(res.m_container_info, top_tid);
+		}
+	};
+
+        container_lookup_result dummy;
+
+	if (g_docker_info_source->lookup(container_id, dummy, cb))
+	{
+		// This should *never* happen, as ttl is 0 (never wait)
+		g_logger.log("Unexpected immediate return from docker_info_source.lookup()", sinsp_logger::SEV_ERROR);
+	}
+}
+
+bool docker_async_source::parse_docker(std::string &container_id, sinsp_container_info *container)
 {
 	string json;
-	docker_response resp = get_docker(manager, build_request("/containers/" + container->m_id + "/json"), json);
+
+	docker_response resp = get_docker(build_request("/containers/" + container_id + "/json"), json);
 	switch(resp) {
 		case docker_response::RESP_BAD_REQUEST:
 			m_api_version = "";
 			json = "";
-			resp = get_docker(manager, build_request("/containers/" + container->m_id + "/json"), json);
+			resp = get_docker(build_request("/containers/" + container_id + "/json"), json);
 			if (resp == docker_response::RESP_OK)
 			{
 				break;
@@ -117,7 +323,7 @@ bool docker::parse_docker(sinsp_container_manager* manager, sinsp_container_info
 	   (no_name || container->m_imagedigest.empty() || (!container->m_imagedigest.empty() && container->m_imagetag.empty())))
 	{
 		string img_json;
-		if(get_docker(manager, build_request("/images/" + container->m_imageid + "/json?digests=1"), img_json) == docker_response::RESP_OK)
+		if(get_docker(build_request("/images/" + container->m_imageid + "/json?digests=1"), img_json) == docker_response::RESP_OK)
 		{
 			Json::Value img_root;
 			if(reader.parse(img_json, img_root))
@@ -176,27 +382,24 @@ bool docker::parse_docker(sinsp_container_manager* manager, sinsp_container_info
 	const Json::Value& net_obj = root["NetworkSettings"];
 
 	string ip = net_obj["IPAddress"].asString();
+
 	if(ip.empty())
 	{
-		const Json::Value& hconfig_obj = root["HostConfig"];
+ 		const Json::Value& hconfig_obj = root["HostConfig"];
 		string net_mode = hconfig_obj["NetworkMode"].asString();
+
 		if(strncmp(net_mode.c_str(), "container:", strlen("container:")) == 0)
 		{
 			std::string container_id = net_mode.substr(net_mode.find(":") + 1);
-			uint32_t container_ip;
-			const sinsp_container_info *container_info = manager->get_container(container_id);
-			if(container_info)
-			{
-				container_ip = container_info->m_container_ip;
-			}
-			else
-			{
-				sinsp_container_info pcnt;
-				pcnt.m_id = container_id;
-				parse_docker(manager, &pcnt, tinfo);
-				container_ip = pcnt.m_container_ip;
-			}
-			container->m_container_ip = container_ip;
+
+			sinsp_container_info pcnt;
+			pcnt.m_id = container_id;
+
+			// This is a *blocking* fetch of the
+			// secondary container, but we're in a
+			// separate thread so this is ok.
+			parse_docker(container_id, &pcnt);
+			container->m_container_ip = pcnt.m_container_ip;
 		}
 	}
 	else
@@ -280,7 +483,7 @@ bool docker::parse_docker(sinsp_container_manager* manager, sinsp_container_info
 		container->m_privileged = privileged.asBool();
 	}
 
-	parse_json_mounts(root["Mounts"], container->m_mounts);
+	docker::parse_json_mounts(root["Mounts"], container->m_mounts);
 
 #ifdef HAS_ANALYZER
 	sinsp_utils::find_env(container->m_sysdig_agent_conf, container->get_env(), "SYSDIG_AGENT_CONF");
diff --git a/userspace/libsinsp/container_engine/docker_linux.cpp b/userspace/libsinsp/container_engine/docker_linux.cpp
index 9e13207dd0..0de85329e6 100644
--- a/userspace/libsinsp/container_engine/docker_linux.cpp
+++ b/userspace/libsinsp/container_engine/docker_linux.cpp
@@ -28,10 +28,6 @@ using namespace libsinsp::container_engine;
 using namespace libsinsp::runc;
 
 namespace {
-std::string s_docker_unix_socket_path = "/var/run/docker.sock";
-#if defined(HAS_CAPTURE)
-thread_local CURLM *s_curlm = NULL;
-thread_local CURL *s_curl = NULL;
 
 size_t docker_curl_write_callback(const char* ptr, size_t size, size_t nmemb, string* json)
 {
@@ -39,7 +35,6 @@ size_t docker_curl_write_callback(const char* ptr, size_t size, size_t nmemb, st
 	json->append(ptr, total);
 	return total;
 }
-#endif
 
 constexpr const cgroup_layout DOCKER_CGROUP_LAYOUT[] = {
 	{"/", ""}, // non-systemd docker
@@ -48,108 +43,74 @@ constexpr const cgroup_layout DOCKER_CGROUP_LAYOUT[] = {
 };
 }
 
-std::string docker::m_api_version = "/v1.24";
-bool libsinsp::container_engine::docker::m_enabled = true;
+std::unique_ptr<docker_async_source> docker::g_docker_info_source;
 
 docker::docker()
 {
-#if defined(HAS_CAPTURE)
-	if(!s_curlm)
+}
+
+void docker::cleanup()
+{
+	g_docker_info_source.reset(NULL);
+}
+
+void docker_async_source::init_docker_conn()
+{
+	if(!m_curlm)
 	{
-		s_curl = curl_easy_init();
-		s_curlm = curl_multi_init();
+		m_curl = curl_easy_init();
+		m_curlm = curl_multi_init();
 
-		if(s_curlm)
+		if(m_curlm)
 		{
-			curl_multi_setopt(s_curlm, CURLMOPT_PIPELINING, CURLPIPE_HTTP1|CURLPIPE_MULTIPLEX);
+			curl_multi_setopt(m_curlm, CURLMOPT_PIPELINING, CURLPIPE_HTTP1|CURLPIPE_MULTIPLEX);
 		}
 
-		if(s_curl)
+		if(m_curl)
 		{
-			auto docker_path = scap_get_host_root() + s_docker_unix_socket_path;
-			curl_easy_setopt(s_curl, CURLOPT_UNIX_SOCKET_PATH, docker_path.c_str());
-			curl_easy_setopt(s_curl, CURLOPT_HTTPGET, 1);
-			curl_easy_setopt(s_curl, CURLOPT_FOLLOWLOCATION, 1);
-			curl_easy_setopt(s_curl, CURLOPT_WRITEFUNCTION, docker_curl_write_callback);
+			auto docker_path = scap_get_host_root() + m_docker_unix_socket_path;
+			curl_easy_setopt(m_curl, CURLOPT_UNIX_SOCKET_PATH, docker_path.c_str());
+			curl_easy_setopt(m_curl, CURLOPT_HTTPGET, 1);
+			curl_easy_setopt(m_curl, CURLOPT_FOLLOWLOCATION, 1);
+			curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, docker_curl_write_callback);
 		}
 	}
-#endif
 }
 
-void docker::cleanup()
+void docker_async_source::free_docker_conn()
 {
-#if defined(HAS_CAPTURE)
-	curl_easy_cleanup(s_curl);
-	s_curl = NULL;
-	curl_multi_cleanup(s_curlm);
-	s_curlm = NULL;
-
-	m_enabled = true;
-#endif
-}
-
-std::string docker::build_request(const std::string &url)
-{
-	return "http://localhost" + m_api_version + url;
-}
-
-bool docker::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, bool query_os_for_missing_info)
-{
-	sinsp_container_info container_info;
-
-	if (!m_enabled)
+	if(m_curl)
 	{
-		return false;
+		curl_easy_cleanup(m_curl);
+		m_curl = NULL;
 	}
 
-	if(matches_runc_cgroups(tinfo, DOCKER_CGROUP_LAYOUT, container_info.m_id))
-	{
-		container_info.m_type = CT_DOCKER;
-		tinfo->m_container_id = container_info.m_id;
-	}
-	else
+	if(m_curlm)
 	{
-		return false;
+		curl_multi_cleanup(m_curlm);
+		m_curlm = NULL;
 	}
-	if (!manager->container_exists(container_info.m_id))
-	{
-		if (query_os_for_missing_info)
-		{
-			if (!parse_docker(manager, &container_info, tinfo))
-			{
-				// give CRI a chance to return metadata for this container
-				g_logger.format(sinsp_logger::SEV_DEBUG, "Failed to get Docker metadata for container %s",
-					container_info.m_id.c_str());
-				return false;
-			}
-		}
-		if (mesos::set_mesos_task_id(&container_info, tinfo))
-		{
-			g_logger.format(sinsp_logger::SEV_DEBUG,
-					"Mesos Docker container: [%s], Mesos task ID: [%s]",
-					container_info.m_id.c_str(), container_info.m_mesos_task_id.c_str());
-		}
-		manager->add_container(container_info, tinfo);
-		manager->notify_new_container(container_info);
-	}
-	return true;
 }
 
-docker::docker_response libsinsp::container_engine::docker::get_docker(sinsp_container_manager* manager, const std::string& url, std::string &json)
+std::string docker_async_source::build_request(const std::string &url)
 {
-#ifdef HAS_CAPTURE
-	if(curl_easy_setopt(s_curl, CURLOPT_URL, url.c_str()) != CURLE_OK)
+	return "http://localhost" + m_api_version + url;
+}
+
+docker_async_source::docker_response docker_async_source::get_docker(const std::string& url, std::string &json)
+{
+	if(curl_easy_setopt(m_curl, CURLOPT_URL, url.c_str()) != CURLE_OK)
 	{
 		ASSERT(false);
 		return docker_response::RESP_ERROR;
 	}
-	if(curl_easy_setopt(s_curl, CURLOPT_WRITEDATA, &json) != CURLE_OK)
+	if(curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, &json) != CURLE_OK)
 	{
 		ASSERT(false);
 		return docker_response::RESP_ERROR;
 	}
 
-	if(curl_multi_add_handle(s_curlm, s_curl) != CURLM_OK)
+	if(curl_multi_add_handle(m_curlm, m_curl) != CURLM_OK)
 	{
 		ASSERT(false);
 		return docker_response::RESP_ERROR;
@@ -158,7 +119,7 @@ docker::docker_response libsinsp::container_engine::docker::get_docker(sinsp_con
 	while(true)
 	{
 		int still_running;
-		CURLMcode res = curl_multi_perform(s_curlm, &still_running);
+		CURLMcode res = curl_multi_perform(m_curlm, &still_running);
 		if(res != CURLM_OK)
 		{
 			ASSERT(false);
@@ -171,7 +132,7 @@ docker::docker_response libsinsp::container_engine::docker::get_docker(sinsp_con
 		}
 
 		int numfds;
-		res = curl_multi_wait(s_curlm, NULL, 0, -1, &numfds);
+		res = curl_multi_wait(m_curlm, NULL, 0, -1, &numfds);
 		if(res != CURLM_OK)
 		{
 			ASSERT(false);
@@ -179,14 +140,14 @@ docker::docker_response libsinsp::container_engine::docker::get_docker(sinsp_con
 		}
 	}
 
-	if(curl_multi_remove_handle(s_curlm, s_curl) != CURLM_OK)
+	if(curl_multi_remove_handle(m_curlm, m_curl) != CURLM_OK)
 	{
 		ASSERT(false);
 		return docker_response::RESP_ERROR;
 	}
 
 	long http_code = 0;
-	if(curl_easy_getinfo(s_curl, CURLINFO_RESPONSE_CODE, &http_code) != CURLE_OK)
+	if(curl_easy_getinfo(m_curl, CURLINFO_RESPONSE_CODE, &http_code) != CURLE_OK)
 	{
 		ASSERT(false);
 		return docker_response::RESP_ERROR;
@@ -195,7 +156,6 @@ docker::docker_response libsinsp::container_engine::docker::get_docker(sinsp_con
 	{
 		case 0: /* connection failed, apparently */
 			g_logger.format(sinsp_logger::SEV_NOTICE, "Docker connection failed, disabling Docker container engine");
-			m_enabled = false;
 			return docker_response::RESP_ERROR;
 		case 200:
 			return docker_response::RESP_OK;
@@ -204,8 +164,17 @@ docker::docker_response libsinsp::container_engine::docker::get_docker(sinsp_con
 	}
 
 	return docker_response::RESP_OK;
-#else
-	return docker_response::RESP_ERROR;
-#endif
 }
 
+bool docker::detect_docker(const sinsp_threadinfo *tinfo, std::string &container_id, std::string &container_name)
+{
+	if(matches_runc_cgroups(tinfo, DOCKER_CGROUP_LAYOUT, container_id))
+	{
+		// The container name is only available in windows
+		container_name = "incomplete";
+
+		return true;
+	}
+
+	return false;
+}
diff --git a/userspace/libsinsp/container_engine/docker_win.cpp b/userspace/libsinsp/container_engine/docker_win.cpp
index 24ccaca3aa..9a3e0aabcb 100644
--- a/userspace/libsinsp/container_engine/docker_win.cpp
+++ b/userspace/libsinsp/container_engine/docker_win.cpp
@@ -24,22 +24,29 @@ limitations under the License.
 
 using namespace libsinsp::container_engine;
 
-std::string docker::m_api_version = "/v1.30";
-
 docker::docker()
 {
 }
 
 void docker::cleanup()
 {
+	g_docker_info_source.reset(NULL);
 }
 
-std::string docker::build_request(const std::string &url)
+void docker_async_source::init_docker_conn()
+{
+}
+
+void docker_async_source::free_docker_conn()
+{
+}
+
+std::string docker_async_source::build_request(const std::string &url)
 {
 	return "GET " + m_api_version + url + " HTTP/1.1\r\nHost: docker\r\n\r\n";
 }
 
-bool docker::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, bool query_os_for_missing_info)
+bool docker::detect_docker(sinsp_threadinfo *tinfo, std::string &container_id, std::string &container_name)
 {
 	wh_docker_container_info wcinfo = wh_docker_resolve_pid(manager->get_inspector()->get_wmi_handle(), tinfo->m_pid);
 	if(!wcinfo.m_res)
@@ -47,28 +54,16 @@ bool docker::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo,
 		return false;
 	}
 
-	sinsp_container_info container_info;
-	container_info.m_type = CT_DOCKER;
-	container_info.m_id = wcinfo.m_container_id;
-	container_info.m_name = wcinfo.m_container_name;
+	container_id = wcinfo.m_container_id;
+	container_name = wcinfo.m_container_name;
 
-	tinfo->m_container_id = container_info.m_id;
-	if (!manager->container_exists(container_info.m_id))
-	{
-		if (query_os_for_missing_info)
-		{
-			parse_docker(manager, &container_info, tinfo);
-		}
-		manager->add_container(container_info, tinfo);
-		manager->notify_new_container(container_info);
-	}
 	return true;
 }
 
-docker::docker_response libsinsp::container_engine::docker::get_docker(sinsp_container_manager* manager, const string& url, string &json)
+docker_async_source::docker_response docker_async_source::get_docker(const std::string& url, std::string &json)
 {
 	const char* response = NULL;
-	bool qdres = wh_query_docker(manager->get_inspector()->get_wmi_handle(),
+	bool qdres = wh_query_docker(m_inspector->get_wmi_handle(),
 				     (char*)url.c_str(),
 				     &response);
 	if(qdres == false)
diff --git a/userspace/libsinsp/container_info.h b/userspace/libsinsp/container_info.h
index 27de99a5be..80871f4c0d 100644
--- a/userspace/libsinsp/container_info.h
+++ b/userspace/libsinsp/container_info.h
@@ -37,6 +37,9 @@ enum sinsp_container_type
 	CT_CRI = 6,
 	CT_CONTAINERD = 7,
 	CT_CRIO = 8,
+	// This is used when we are still fetching information about a
+	// container and don't exactly know its type yet.
+	CT_UNKNOWN = 9,
 };
 
 // Docker and CRI-compatible runtimes are very similar
@@ -134,7 +137,8 @@ class sinsp_container_info
 		m_cpu_period(100000),
 		m_has_healthcheck(false),
 		m_healthcheck_exe(""),
-		m_is_pod_sandbox(false)
+		m_is_pod_sandbox(false),
+		m_metadata_complete(true)
 #ifdef HAS_ANALYZER
 		,m_metadata_deadline(0)
 #endif
@@ -179,6 +183,11 @@ class sinsp_container_info
 	std::string m_healthcheck_exe;
 	std::vector<std::string> m_healthcheck_args;
 	bool m_is_pod_sandbox;
+
+	// If false, this represents incomplete information about the
+	// container that will be filled in later as a result of an
+	// async fetch of container info.
+	bool m_metadata_complete;
 #ifdef HAS_ANALYZER
 	std::string m_sysdig_agent_conf;
 	uint64_t m_metadata_deadline;

From 037b7570065f004dd6c7d37236ef73df292b3e9d Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Wed, 6 Mar 2019 20:31:27 -0800
Subject: [PATCH 05/15] Fix cri::resolve to work w UNKNOWN container infos

cri::resolve might be called after docker::resolve, so there could be a
container_info object with type == UNKNOWN.

Update it to handle this i.e. do the lookup anyway.
---
 userspace/libsinsp/container_engine/cri.cpp | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/userspace/libsinsp/container_engine/cri.cpp b/userspace/libsinsp/container_engine/cri.cpp
index 444f454b2c..a66c26f646 100644
--- a/userspace/libsinsp/container_engine/cri.cpp
+++ b/userspace/libsinsp/container_engine/cri.cpp
@@ -197,17 +197,24 @@ void cri::set_extra_queries(bool extra_queries) {
 bool cri::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, bool query_os_for_missing_info)
 {
 	sinsp_container_info container_info;
+	sinsp_container_info *existing_container_info;
 
 	if(matches_runc_cgroups(tinfo, CRI_CGROUP_LAYOUT, container_info.m_id))
 	{
-		container_info.m_type = s_cri_runtime_type;
+		// It might also be a docker container, so set the type
+		// to UNKNOWN for now.
+		container_info.m_type = CT_UNKNOWN;
 	}
 	else
 	{
 		return false;
 	}
 	tinfo->m_container_id = container_info.m_id;
-	if (!manager->container_exists(container_info.m_id))
+
+	existing_container_info = manager->get_container(container_info.m_id);
+
+	if (!existing_container_info ||
+	    existing_container_info->m_type == CT_UNKNOWN)
 	{
 		if (query_os_for_missing_info)
 		{
@@ -217,6 +224,11 @@ bool cri::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, boo
 						container_info.m_id.c_str());
 				return false;
 			}
+
+			// If here, parse_cri succeeded so we can
+			// assign an actual type.
+			container_info.m_type = s_cri_runtime_type;
+
 		}
 		if (mesos::set_mesos_task_id(&container_info, tinfo))
 		{

From 74fb5c1cc750cac1429f4bb9b7f07c08bcbfa887 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Thu, 7 Mar 2019 20:19:35 -0800
Subject: [PATCH 06/15] Use tbb, libcurl on osx, disable libidn2

We need it now that tbb is a core part of the inspector and that the
curl part of the docker container engine isn't #ifdefd with HAS_CAPTURE.

Both are already downloaded/built as dependencies of sysdig so it's just
a matter of CMakeLists.txt config.

Also, disable libidn2 explicitly when building libcurl. On OSX builds
this gets picked up from an autodetected library, while it does not get
picked up on linux. We already disable libidn1, so also disable libidn2.
---
 CMakeLists.txt                    |  4 ++--
 userspace/libsinsp/CMakeLists.txt | 13 +++++++++----
 userspace/sysdig/CMakeLists.txt   |  4 +---
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5b135816ef..d40b65fce5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -241,7 +241,7 @@ endif()
 #
 # Intel tbb
 #
-if(NOT WIN32 AND NOT APPLE)
+if(NOT WIN32)
 	option(USE_BUNDLED_TBB "Enable building of the bundled tbb" ${USE_BUNDLED_DEPS})
 	if(NOT USE_BUNDLED_TBB)
 		find_path(TBB_INCLUDE_DIR tbb.h PATH_SUFFIXES tbb)
@@ -411,7 +411,7 @@ if(NOT WIN32)
 			DEPENDS openssl
 			URL "http://download.draios.com/dependencies/curl-7.61.0.tar.bz2"
 			URL_MD5 "31d0a9f48dc796a7db351898a1e5058a"
-			CONFIGURE_COMMAND ./configure ${CURL_SSL_OPTION} --disable-threaded-resolver --disable-shared --enable-optimize --disable-curldebug --disable-rt --enable-http --disable-ftp --disable-file --disable-ldap --disable-ldaps --disable-rtsp --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-sspi --disable-ntlm-wb --disable-tls-srp --without-winssl --without-darwinssl --without-polarssl --without-cyassl --without-nss --without-axtls --without-ca-path --without-ca-bundle --without-libmetalink --without-librtmp --without-winidn --without-libidn --without-nghttp2 --without-libssh2
+			CONFIGURE_COMMAND ./configure ${CURL_SSL_OPTION} --disable-threaded-resolver --disable-shared --enable-optimize --disable-curldebug --disable-rt --enable-http --disable-ftp --disable-file --disable-ldap --disable-ldaps --disable-rtsp --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-sspi --disable-ntlm-wb --disable-tls-srp --without-winssl --without-darwinssl --without-polarssl --without-cyassl --without-nss --without-axtls --without-ca-path --without-ca-bundle --without-libmetalink --without-librtmp --without-winidn --without-libidn --without-libidn2 --without-nghttp2 --without-libssh2
 			BUILD_COMMAND ${CMD_MAKE}
 			BUILD_IN_SOURCE 1
 			BUILD_BYPRODUCTS ${CURL_LIBRARIES}
diff --git a/userspace/libsinsp/CMakeLists.txt b/userspace/libsinsp/CMakeLists.txt
index 422c704feb..6429a4196f 100644
--- a/userspace/libsinsp/CMakeLists.txt
+++ b/userspace/libsinsp/CMakeLists.txt
@@ -21,19 +21,22 @@ include_directories(../libscap)
 include_directories(../async)
 include_directories("${JSONCPP_INCLUDE}")
 include_directories("${LUAJIT_INCLUDE}")
+include_directories("${TBB_INCLUDE_DIR}")
 
 if(NOT WIN32 AND NOT APPLE)
 	include_directories("${B64_INCLUDE}")
-	include_directories("${CURL_INCLUDE_DIR}")
 	include_directories("${CURSES_INCLUDE_DIR}")
 	include_directories("${GRPC_INCLUDE}")
 	include_directories("${JQ_INCLUDE}")
 	include_directories("${OPENSSL_INCLUDE_DIR}")
 	include_directories("${PROTOBUF_INCLUDE}")
-	include_directories("${TBB_INCLUDE_DIR}")
 	include_directories("${CMAKE_CURRENT_BINARY_DIR}")
 endif()
 
+if(NOT WIN32)
+	include_directories("${CURL_INCLUDE_DIR}")
+endif()
+
 set(SINSP_SOURCES
 	chisel.cpp
 	chisel_api.cpp
@@ -150,9 +153,13 @@ if(NOT WIN32)
 	endif()
 	if(USE_BUNDLED_CURL)
 		add_dependencies(sinsp curl)
+		target_link_libraries(sinsp
+			"${CURL_LIBRARIES}")
 	endif()
 	if(USE_BUNDLED_TBB)
 		add_dependencies(sinsp tbb)
+		target_link_libraries(sinsp
+			"${TBB_LIB}")
 	endif()
 
 	if(NOT APPLE)
@@ -174,8 +181,6 @@ if(NOT WIN32)
 			"${CARES_LIB}"
 			"${JQ_LIB}"
 			"${B64_LIB}"
-			"${CURL_LIBRARIES}"
-			"${TBB_LIB}"
 			rt
 			anl)
 	endif()
diff --git a/userspace/sysdig/CMakeLists.txt b/userspace/sysdig/CMakeLists.txt
index a71435215f..648ff3cbd8 100644
--- a/userspace/sysdig/CMakeLists.txt
+++ b/userspace/sysdig/CMakeLists.txt
@@ -16,12 +16,10 @@
 # limitations under the License.
 #
 include_directories("${JSONCPP_INCLUDE}")
+include_directories("${TBB_INCLUDE_DIR}")
 
 if(NOT WIN32)
-    if(NOT APPLE)
 	include_directories("${CURL_INCLUDE_DIR}")
-	include_directories("${TBB_INCLUDE_DIR}")
-    endif()
 	include_directories("${CURSES_INCLUDE_DIR}")
 endif()
 

From 98fd835300e6d1edff1461dfb75ce28350c5c779 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Mon, 25 Mar 2019 12:57:09 -0700
Subject: [PATCH 07/15] Serialize/unserialize is_pod_sandbox

Instead of inferring is_pod_sandbox from the container name only for
docker, save/load it to json directly from the container info. This
ensures it works for other container engines than docker.
---
 userspace/libsinsp/container.cpp |  1 +
 userspace/libsinsp/parsers.cpp   | 11 +++++++----
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/userspace/libsinsp/container.cpp b/userspace/libsinsp/container.cpp
index 7ed0c2ea71..2d0fe93bca 100644
--- a/userspace/libsinsp/container.cpp
+++ b/userspace/libsinsp/container.cpp
@@ -147,6 +147,7 @@ string sinsp_container_manager::container_to_json(const sinsp_container_info& co
 	container["imagetag"] = container_info.m_imagetag;
 	container["imagedigest"] = container_info.m_imagedigest;
 	container["privileged"] = container_info.m_privileged;
+	container["is_pod_sandbox"] = container_info.m_is_pod_sandbox;
 
 	Json::Value mounts = Json::arrayValue;
 
diff --git a/userspace/libsinsp/parsers.cpp b/userspace/libsinsp/parsers.cpp
index 3103bb4f20..2583feca7c 100644
--- a/userspace/libsinsp/parsers.cpp
+++ b/userspace/libsinsp/parsers.cpp
@@ -4500,11 +4500,14 @@ void sinsp_parser::parse_container_json_evt(sinsp_evt *evt)
 		if(!name.isNull() && name.isConvertibleTo(Json::stringValue))
 		{
 			container_info.m_name = name.asString();
-			if(container_info.m_type == CT_DOCKER && container_info.m_name.find("k8s_POD") == 0)
-			{
-				container_info.m_is_pod_sandbox = true;
-			}
 		}
+
+		const Json::Value& is_pod_sandbox = container["isPodSandbox"];
+		if(!is_pod_sandbox.isNull() && is_pod_sandbox.isConvertibleTo(Json::booleanValue))
+		{
+			container_info.m_is_pod_sandbox = is_pod_sandbox.asBool();
+		}
+
 		const Json::Value& image = container["image"];
 		if(!image.isNull() && image.isConvertibleTo(Json::stringValue))
 		{

From 3e43e0844c7295fbbe29797597d295534d308b84 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Thu, 28 Mar 2019 11:51:14 -0700
Subject: [PATCH 08/15] Get rid of CT_UNKNOWN

Instead of having a CT_UNKNOWN type, iniitally set the type to CT_DOCKER
when starting the async lookup. Cri runs next and if the grpc lookup
completes successfully this will be replaced with CT_CRIO. If both grpc
and docker metadata lookups fail the type will remain at CT_DOCKER.
---
 userspace/libsinsp/container_engine/cri.cpp           | 10 ++--------
 userspace/libsinsp/container_engine/docker_common.cpp |  2 +-
 userspace/libsinsp/container_info.h                   |  3 ---
 3 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/userspace/libsinsp/container_engine/cri.cpp b/userspace/libsinsp/container_engine/cri.cpp
index a66c26f646..153d61b175 100644
--- a/userspace/libsinsp/container_engine/cri.cpp
+++ b/userspace/libsinsp/container_engine/cri.cpp
@@ -199,13 +199,7 @@ bool cri::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, boo
 	sinsp_container_info container_info;
 	sinsp_container_info *existing_container_info;
 
-	if(matches_runc_cgroups(tinfo, CRI_CGROUP_LAYOUT, container_info.m_id))
-	{
-		// It might also be a docker container, so set the type
-		// to UNKNOWN for now.
-		container_info.m_type = CT_UNKNOWN;
-	}
-	else
+	if(!matches_runc_cgroups(tinfo, CRI_CGROUP_LAYOUT, container_info.m_id))
 	{
 		return false;
 	}
@@ -214,7 +208,7 @@ bool cri::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, boo
 	existing_container_info = manager->get_container(container_info.m_id);
 
 	if (!existing_container_info ||
-	    existing_container_info->m_type == CT_UNKNOWN)
+	    existing_container_info->m_metadata_complete == false)
 	{
 		if (query_os_for_missing_info)
 		{
diff --git a/userspace/libsinsp/container_engine/docker_common.cpp b/userspace/libsinsp/container_engine/docker_common.cpp
index 690a757b39..807355e516 100644
--- a/userspace/libsinsp/container_engine/docker_common.cpp
+++ b/userspace/libsinsp/container_engine/docker_common.cpp
@@ -195,7 +195,7 @@ bool docker::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo,
 		// overidden later once parse_docker_async completes.
 		sinsp_container_info container_info;
 
-		container_info.m_type = CT_UNKNOWN;
+		container_info.m_type = CT_DOCKER;
 		container_info.m_id = container_id;
 		container_info.m_name = container_name;
 		container_info.m_image="incomplete";
diff --git a/userspace/libsinsp/container_info.h b/userspace/libsinsp/container_info.h
index 80871f4c0d..0b8df1925b 100644
--- a/userspace/libsinsp/container_info.h
+++ b/userspace/libsinsp/container_info.h
@@ -37,9 +37,6 @@ enum sinsp_container_type
 	CT_CRI = 6,
 	CT_CONTAINERD = 7,
 	CT_CRIO = 8,
-	// This is used when we are still fetching information about a
-	// container and don't exactly know its type yet.
-	CT_UNKNOWN = 9,
 };
 
 // Docker and CRI-compatible runtimes are very similar

From 4594484b83de2c7026684f591a19cba8a24361ed Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Thu, 28 Mar 2019 11:51:42 -0700
Subject: [PATCH 09/15] Just include sinsp class

It's a bit simpler than including sinsp.h, which has many many dependent
header files.
---
 userspace/libsinsp/container_engine/docker.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/userspace/libsinsp/container_engine/docker.h b/userspace/libsinsp/container_engine/docker.h
index 92ed888f43..15fda982b1 100644
--- a/userspace/libsinsp/container_engine/docker.h
+++ b/userspace/libsinsp/container_engine/docker.h
@@ -36,10 +36,9 @@ limitations under the License.
 
 #include "async_key_value_source.h"
 
-#include "sinsp.h"
-
 #include "container_info.h"
 
+class sinsp;
 class sinsp_container_manager;
 class sinsp_container_info;
 class sinsp_threadinfo;

From 9ab808dfdaedac96346dd845dfef51565857b9dd Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Thu, 28 Mar 2019 12:03:40 -0700
Subject: [PATCH 10/15] Add note on why the container evt needs a tid

It's required for the container.* filterchecks to work.
---
 userspace/libsinsp/container_engine/docker.h | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/userspace/libsinsp/container_engine/docker.h b/userspace/libsinsp/container_engine/docker.h
index 15fda982b1..a65759043f 100644
--- a/userspace/libsinsp/container_engine/docker.h
+++ b/userspace/libsinsp/container_engine/docker.h
@@ -108,8 +108,15 @@ class docker_async_source : public sysdig::async_key_value_source<std::string, c
 	// exists. These associations are only maintained while an
 	// async lookup of container information is in progress. We
 	// use this to ensure that the tid of the CONTAINER_JSON event
-	// we eventually emit is the top running thread in the
-	// container.
+	// we eventually emit is a valid thread, and the top running
+	// thread, in the container.
+	//
+        // We want the container event to have a valid thread because
+        // all the container.* filterchecks first look up the
+        // threadinfo for the event and then use the threadinfo's
+        // m_container_id to look up to the container. If the container
+        // event didn't have a valid threadinfo, that lookup would
+        // fail and the container.* filterchecks would not return anything.
 	typedef tbb::concurrent_hash_map<std::string, int64_t> top_tid_table;
 	top_tid_table m_top_tids;
 };

From 71b08a6bcfd64b61e5b1adb8d1975861aea31f75 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Thu, 28 Mar 2019 12:32:35 -0700
Subject: [PATCH 11/15] Only start async lookup thread if docker found

Call detect_docker, which identifies a docker container, before starting
the async lookup thread. If no docker containers are ever used, the
async thread won't be started.
---
 userspace/libsinsp/container_engine/docker_common.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/userspace/libsinsp/container_engine/docker_common.cpp b/userspace/libsinsp/container_engine/docker_common.cpp
index 807355e516..24b5b05c06 100644
--- a/userspace/libsinsp/container_engine/docker_common.cpp
+++ b/userspace/libsinsp/container_engine/docker_common.cpp
@@ -171,6 +171,11 @@ bool docker::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo,
 	std::string container_id, container_name;
 	sinsp_container_info *existing_container_info;
 
+	if(!detect_docker(tinfo, container_id, container_name))
+	{
+		return false;
+	}
+
 	if(!g_docker_info_source)
 	{
 		uint64_t max_wait_ms = 10000;
@@ -178,11 +183,6 @@ bool docker::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo,
 		g_docker_info_source.reset(src);
 	}
 
-	if(!detect_docker(tinfo, container_id, container_name))
-	{
-		return false;
-	}
-
 	tinfo->m_container_id = container_id;
 
 	existing_container_info = manager->get_container(container_id);

From 454756315b1a832edb72c73f0c67919c5a0dad57 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Thu, 28 Mar 2019 12:44:06 -0700
Subject: [PATCH 12/15] Use a static for the string "incomplete"

---
 userspace/libsinsp/container_engine/docker.h          | 2 ++
 userspace/libsinsp/container_engine/docker_common.cpp | 6 ++++--
 userspace/libsinsp/container_engine/docker_linux.cpp  | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/userspace/libsinsp/container_engine/docker.h b/userspace/libsinsp/container_engine/docker.h
index a65759043f..8fd656798d 100644
--- a/userspace/libsinsp/container_engine/docker.h
+++ b/userspace/libsinsp/container_engine/docker.h
@@ -136,6 +136,8 @@ class docker
 	void parse_docker_async(sinsp *inspector, std::string &container_id, sinsp_container_manager *manager);
 
 	static std::unique_ptr<docker_async_source> g_docker_info_source;
+
+	static std::string s_incomplete_info_name;
 };
 }
 }
diff --git a/userspace/libsinsp/container_engine/docker_common.cpp b/userspace/libsinsp/container_engine/docker_common.cpp
index 24b5b05c06..a1363b4141 100644
--- a/userspace/libsinsp/container_engine/docker_common.cpp
+++ b/userspace/libsinsp/container_engine/docker_common.cpp
@@ -166,6 +166,8 @@ void docker_async_source::set_query_image_info(bool query_image_info)
 	m_query_image_info = query_image_info;
 }
 
+std::string docker::s_incomplete_info_name = "incomplete";
+
 bool docker::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo, bool query_os_for_missing_info)
 {
 	std::string container_id, container_name;
@@ -191,14 +193,14 @@ bool docker::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo,
 	{
 		// Add a minimal container_info object where only the
 		// container id, (possibly) name, and a container
-		// image=incomplete is filled in. This may be
+		// image = incomplete is filled in. This may be
 		// overidden later once parse_docker_async completes.
 		sinsp_container_info container_info;
 
 		container_info.m_type = CT_DOCKER;
 		container_info.m_id = container_id;
 		container_info.m_name = container_name;
-		container_info.m_image="incomplete";
+		container_info.m_image = s_incomplete_info_name;
 		container_info.m_metadata_complete = false;
 
 		manager->add_container(container_info, tinfo);
diff --git a/userspace/libsinsp/container_engine/docker_linux.cpp b/userspace/libsinsp/container_engine/docker_linux.cpp
index 0de85329e6..acb5f5cafb 100644
--- a/userspace/libsinsp/container_engine/docker_linux.cpp
+++ b/userspace/libsinsp/container_engine/docker_linux.cpp
@@ -171,7 +171,7 @@ bool docker::detect_docker(const sinsp_threadinfo *tinfo, std::string &container
 	if(matches_runc_cgroups(tinfo, DOCKER_CGROUP_LAYOUT, container_id))
 	{
 		// The container name is only available in windows
-		container_name = "incomplete";
+		container_name = s_incomplete_info_name;
 
 		return true;
 	}

From 84a6558473e435107f86758e1604eaf754bb3335 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Thu, 28 Mar 2019 12:46:47 -0700
Subject: [PATCH 13/15] Fix typo CRI -> docker

---
 userspace/libsinsp/container_engine/docker_common.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/userspace/libsinsp/container_engine/docker_common.cpp b/userspace/libsinsp/container_engine/docker_common.cpp
index a1363b4141..a78e8eddb2 100644
--- a/userspace/libsinsp/container_engine/docker_common.cpp
+++ b/userspace/libsinsp/container_engine/docker_common.cpp
@@ -215,7 +215,7 @@ bool docker::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo,
 	{
 		if(!g_docker_info_source->pending_lookup(container_id))
 		{
-			// give CRI a chance to return metadata for this container
+			// give docker a chance to return metadata for this container
 			parse_docker_async(manager->get_inspector(), container_id, manager);
 
 			g_docker_info_source->set_top_tid(container_id, tinfo);

From 8d65ac8c52bb51192012795d5bf81038f93ad772 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Tue, 2 Apr 2019 10:45:37 -0700
Subject: [PATCH 14/15] Set all container metadata fields to incomplete

Instead of just setting the image to incomplete, set all container
metadata fields to incomplete.
---
 userspace/libsinsp/container_engine/docker_common.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/userspace/libsinsp/container_engine/docker_common.cpp b/userspace/libsinsp/container_engine/docker_common.cpp
index a78e8eddb2..23bf088fc3 100644
--- a/userspace/libsinsp/container_engine/docker_common.cpp
+++ b/userspace/libsinsp/container_engine/docker_common.cpp
@@ -201,6 +201,10 @@ bool docker::resolve(sinsp_container_manager* manager, sinsp_threadinfo* tinfo,
 		container_info.m_id = container_id;
 		container_info.m_name = container_name;
 		container_info.m_image = s_incomplete_info_name;
+		container_info.m_imageid = s_incomplete_info_name;
+		container_info.m_imagerepo = s_incomplete_info_name;
+		container_info.m_imagetag = s_incomplete_info_name;
+		container_info.m_imagedigest = s_incomplete_info_name;
 		container_info.m_metadata_complete = false;
 
 		manager->add_container(container_info, tinfo);

From 9c7f3dfd6a77ce7c0e3c5aa78b3712fb52687c69 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Thu, 18 Apr 2019 09:39:40 -0700
Subject: [PATCH 15/15] Don't protect m_metadata_deadline w HAS_ANALYZER

It's filled in when using the analyzer, but we can define it in both
cases and just not use it when there is no analyzer.
---
 userspace/libsinsp/container.cpp    | 4 +---
 userspace/libsinsp/container_info.h | 8 +++-----
 userspace/libsinsp/parsers.cpp      | 4 +---
 3 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/userspace/libsinsp/container.cpp b/userspace/libsinsp/container.cpp
index 2d0fe93bca..58fb650e29 100644
--- a/userspace/libsinsp/container.cpp
+++ b/userspace/libsinsp/container.cpp
@@ -216,9 +216,7 @@ string sinsp_container_manager::container_to_json(const sinsp_container_info& co
 		container["mesos_task_id"] = container_info.m_mesos_task_id;
 	}
 
-#ifdef HAS_ANALYZER
 	container["metadata_deadline"] = (Json::Value::UInt64) container_info.m_metadata_deadline;
-#endif
 	return Json::FastWriter().write(obj);
 }
 
@@ -436,4 +434,4 @@ void sinsp_container_manager::set_cri_timeout(int64_t timeout_ms)
 #if defined(HAS_CAPTURE)
 	libsinsp::container_engine::cri::set_cri_timeout(timeout_ms);
 #endif
-}
\ No newline at end of file
+}
diff --git a/userspace/libsinsp/container_info.h b/userspace/libsinsp/container_info.h
index 0b8df1925b..e5fdf5bf00 100644
--- a/userspace/libsinsp/container_info.h
+++ b/userspace/libsinsp/container_info.h
@@ -135,10 +135,8 @@ class sinsp_container_info
 		m_has_healthcheck(false),
 		m_healthcheck_exe(""),
 		m_is_pod_sandbox(false),
-		m_metadata_complete(true)
-#ifdef HAS_ANALYZER
-		,m_metadata_deadline(0)
-#endif
+		m_metadata_complete(true),
+		m_metadata_deadline(0)
 	{
 	}
 
@@ -187,6 +185,6 @@ class sinsp_container_info
 	bool m_metadata_complete;
 #ifdef HAS_ANALYZER
 	std::string m_sysdig_agent_conf;
-	uint64_t m_metadata_deadline;
 #endif
+	uint64_t m_metadata_deadline;
 };
diff --git a/userspace/libsinsp/parsers.cpp b/userspace/libsinsp/parsers.cpp
index 2583feca7c..44596b1aae 100644
--- a/userspace/libsinsp/parsers.cpp
+++ b/userspace/libsinsp/parsers.cpp
@@ -2262,7 +2262,7 @@ inline void sinsp_parser::add_socket(sinsp_evt *evt, int64_t fd, uint32_t domain
  *
  * Preconditions: evt->m_fdinfo == nullptr and
  *                evt->m_tinfo != nullptr
- * 
+ *
  */
 inline void sinsp_parser::infer_sendto_fdinfo(sinsp_evt* const evt)
 {
@@ -4623,14 +4623,12 @@ void sinsp_parser::parse_container_json_evt(sinsp_evt *evt)
 			container_info.m_mesos_task_id = mesos_task_id.asString();
 		}
 
-#ifdef HAS_ANALYZER
 		const Json::Value& metadata_deadline = container["metadata_deadline"];
 		// isConvertibleTo doesn't seem to work on large 64 bit numbers
 		if(!metadata_deadline.isNull() && metadata_deadline.isUInt64())
 		{
 			container_info.m_metadata_deadline = metadata_deadline.asUInt64();
 		}
-#endif
 
 		m_inspector->m_container_manager.add_container(container_info, evt->get_thread_info(true));
 		/*