From 8baef60d600c5254662633d8275f321a6dafb82c Mon Sep 17 00:00:00 2001
From: Victor Julien <victor@inliniac.net>
Date: Mon, 1 Feb 2021 22:23:47 +0100
Subject: [PATCH 01/19] app-layer: fix transaction cleanup

Fix a 'skipped' transaction early in the list leading to all further
transactions getting skipped, even if they were fully processed and
ready to be cleaned up.
---
 src/app-layer-parser.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c
index f94c81ddebb1..a80ac9484e9f 100644
--- a/src/app-layer-parser.c
+++ b/src/app-layer-parser.c
@@ -922,6 +922,7 @@ void AppLayerParserTransactionsCleanup(Flow *f)
         if (ires.tx_ptr == NULL)
             break;
 
+        bool tx_skipped = false;
         void *tx = ires.tx_ptr;
         i = ires.tx_id; // actual tx id for the tx the IterFunc returned
 
@@ -950,7 +951,7 @@ void AppLayerParserTransactionsCleanup(Flow *f)
                 if (!(detect_flags_ts & APP_LAYER_TX_INSPECTED_FLAG)) {
                     SCLogDebug("%p/%"PRIu64" skipping: TS inspect not done: ts:%"PRIx64,
                             tx, i, detect_flags_ts);
-                    skipped = true;
+                    tx_skipped = skipped = true;
                 } else {
                     inspected = true;
                 }
@@ -960,7 +961,7 @@ void AppLayerParserTransactionsCleanup(Flow *f)
                 if (!(detect_flags_tc & APP_LAYER_TX_INSPECTED_FLAG)) {
                     SCLogDebug("%p/%"PRIu64" skipping: TC inspect not done: tc:%"PRIx64,
                             tx, i, detect_flags_tc);
-                    skipped = true;
+                    tx_skipped = skipped = true;
                 } else {
                     inspected = true;
                 }
@@ -969,7 +970,8 @@ void AppLayerParserTransactionsCleanup(Flow *f)
 
         // If not a unidirectional transaction both sides are required to have
         // been inspected.
-        if (!is_unidir && skipped) {
+        if (!is_unidir && tx_skipped) {
+            SCLogDebug("%p/%" PRIu64 " !is_unidir && tx_skipped", tx, i);
             goto next;
         }
 
@@ -977,7 +979,8 @@ void AppLayerParserTransactionsCleanup(Flow *f)
         // inspected, which the inspected flag tells us. This is also guarded
         // with skip to limit this check to transactions that actually had the
         // tx inspected flag checked.
-        if (is_unidir && skipped && !inspected) {
+        if (is_unidir && tx_skipped && !inspected) {
+            SCLogDebug("%p/%" PRIu64 " is_unidir && tx_skipped && !inspected", tx, i);
             goto next;
         }
 

From 17a38f1823adeb9eb059f666686e35509f3a13d2 Mon Sep 17 00:00:00 2001
From: Victor Julien <victor@inliniac.net>
Date: Wed, 3 Feb 2021 12:00:51 +0100
Subject: [PATCH 02/19] flow/manager: (u)sleep slightly longer

Sleep 250 microseconds instead of 100 as running in KVM cause the
old value to use 100% CPU for these threads.

Perf testing suggests no measurable impact for the non-KVM case.

Ticket: #4096
---
 src/flow-manager.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/flow-manager.c b/src/flow-manager.c
index cb430ec74da4..d58a49637d6a 100644
--- a/src/flow-manager.c
+++ b/src/flow-manager.c
@@ -1011,7 +1011,7 @@ static TmEcode FlowManager(ThreadVars *th_v, void *thread_data)
         memset(&sleep_startts, 0, sizeof(sleep_startts));
         gettimeofday(&sleep_startts, NULL);
 #endif
-        usleep(100);
+        usleep(250);
 
 #ifdef FM_PROFILE
         struct timeval sleep_endts;
@@ -1207,7 +1207,7 @@ static TmEcode FlowRecycler(ThreadVars *th_v, void *thread_data)
         memset(&sleep_startts, 0, sizeof(sleep_startts));
         gettimeofday(&sleep_startts, NULL);
 #endif
-        usleep(100);
+        usleep(250);
 #ifdef FM_PROFILE
         struct timeval sleep_endts;
         memset(&sleep_endts, 0, sizeof(sleep_endts));

From 97ca2e01ffaf53b05d808e457bf836d9f69a6087 Mon Sep 17 00:00:00 2001
From: Victor Julien <victor@inliniac.net>
Date: Thu, 4 Feb 2021 14:48:11 +0100
Subject: [PATCH 03/19] detect: fix heap overflow issue with buffer setup

In some cases, the InspectionBufferGet function would be followed by
a failure to set the buffer up, for example due to a HTTP body limit
not yet being reached. Yet each call to InspectionBufferGet would lead
to the matching list_id to be added to the
DetectEngineThreadCtx::inspect.to_clear_queue. This array is sized to
add each list only once, but in this case the same id could be added
multiple times, potentially overflowing the array.
---
 src/detect-dce-stub-data.c                 |  4 ++--
 src/detect-dnp3.c                          |  2 +-
 src/detect-dns-query.c                     |  2 +-
 src/detect-engine.c                        | 15 +++++++++------
 src/detect-engine.h                        |  2 +-
 src/detect-file-data.c                     |  4 ++--
 src/detect-filemagic.c                     |  2 +-
 src/detect-filename.c                      |  2 +-
 src/detect-http-client-body.c              |  2 +-
 src/detect-http-cookie.c                   |  4 ++--
 src/detect-http-header-names.c             |  6 +++---
 src/detect-http-header.c                   |  4 ++--
 src/detect-http-headers-stub.h             |  4 ++--
 src/detect-http-host.c                     |  4 ++--
 src/detect-http-method.c                   |  2 +-
 src/detect-http-protocol.c                 |  2 +-
 src/detect-http-raw-header.c               |  2 +-
 src/detect-http-request-line.c             |  2 +-
 src/detect-http-response-line.c            |  2 +-
 src/detect-http-start.c                    |  6 +++---
 src/detect-http-stat-code.c                |  2 +-
 src/detect-http-stat-msg.c                 |  2 +-
 src/detect-http-ua.c                       |  2 +-
 src/detect-http-uri.c                      |  4 ++--
 src/detect-http2.c                         |  4 ++--
 src/detect-icmpv4hdr.c                     |  2 +-
 src/detect-icmpv6hdr.c                     |  2 +-
 src/detect-ipv4hdr.c                       |  2 +-
 src/detect-ipv6hdr.c                       |  2 +-
 src/detect-krb5-cname.c                    |  2 +-
 src/detect-krb5-sname.c                    |  2 +-
 src/detect-mqtt-connect-clientid.c         |  2 +-
 src/detect-mqtt-connect-password.c         |  2 +-
 src/detect-mqtt-connect-username.c         |  2 +-
 src/detect-mqtt-connect-willmessage.c      |  2 +-
 src/detect-mqtt-connect-willtopic.c        |  2 +-
 src/detect-mqtt-publish-message.c          |  2 +-
 src/detect-mqtt-publish-topic.c            |  2 +-
 src/detect-mqtt-subscribe-topic.c          |  2 +-
 src/detect-mqtt-unsubscribe-topic.c        |  2 +-
 src/detect-rfb-name.c                      |  2 +-
 src/detect-sip-method.c                    |  2 +-
 src/detect-sip-protocol.c                  |  2 +-
 src/detect-sip-request-line.c              |  2 +-
 src/detect-sip-response-line.c             |  2 +-
 src/detect-sip-stat-code.c                 |  2 +-
 src/detect-sip-stat-msg.c                  |  2 +-
 src/detect-sip-uri.c                       |  2 +-
 src/detect-smb-share.c                     |  4 ++--
 src/detect-snmp-community.c                |  2 +-
 src/detect-ssh-hassh-server-string.c       |  2 +-
 src/detect-ssh-hassh-server.c              |  2 +-
 src/detect-ssh-hassh-string.c              |  2 +-
 src/detect-ssh-hassh.c                     |  2 +-
 src/detect-ssh-proto.c                     |  2 +-
 src/detect-ssh-software.c                  |  2 +-
 src/detect-tcphdr.c                        |  2 +-
 src/detect-template-buffer.c               |  2 +-
 src/detect-tls-cert-fingerprint.c          |  2 +-
 src/detect-tls-cert-issuer.c               |  2 +-
 src/detect-tls-cert-serial.c               |  2 +-
 src/detect-tls-cert-subject.c              |  2 +-
 src/detect-tls-certs.c                     |  2 +-
 src/detect-tls-ja3-hash.c                  |  2 +-
 src/detect-tls-ja3-string.c                |  2 +-
 src/detect-tls-ja3s-hash.c                 |  2 +-
 src/detect-tls-ja3s-string.c               |  2 +-
 src/detect-tls-sni.c                       |  2 +-
 src/detect-transform-compress-whitespace.c |  8 ++++----
 src/detect-transform-dotprefix.c           |  6 +++---
 src/detect-transform-md5.c                 |  4 ++--
 src/detect-transform-sha1.c                |  2 +-
 src/detect-transform-sha256.c              |  4 ++--
 src/detect-transform-strip-whitespace.c    |  6 +++---
 src/detect-transform-urldecode.c           |  2 +-
 src/detect-udphdr.c                        |  2 +-
 76 files changed, 106 insertions(+), 103 deletions(-)

diff --git a/src/detect-dce-stub-data.c b/src/detect-dce-stub-data.c
index 416109fe0f57..69a9f1a84974 100644
--- a/src/detect-dce-stub-data.c
+++ b/src/detect-dce-stub-data.c
@@ -79,7 +79,7 @@ static InspectionBuffer *GetSMBData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         SCLogDebug("have data!");
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
@@ -105,7 +105,7 @@ static InspectionBuffer *GetDCEData(DetectEngineThreadCtx *det_ctx,
         } else {
             buffer->flags |= DETECT_CI_FLAGS_DCE_BE;
         }
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-dnp3.c b/src/detect-dnp3.c
index d0f78e7ac754..232dd28cddce 100644
--- a/src/detect-dnp3.c
+++ b/src/detect-dnp3.c
@@ -170,7 +170,7 @@ static InspectionBuffer *GetDNP3Data(DetectEngineThreadCtx *det_ctx,
             return NULL;
 
         SCLogDebug("tx %p data %p data_len %u", tx, data, data_len);
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-dns-query.c b/src/detect-dns-query.c
index 5e4b5f10517b..a029f8bac2a9 100644
--- a/src/detect-dns-query.c
+++ b/src/detect-dns-query.c
@@ -91,7 +91,7 @@ static InspectionBuffer *DnsQueryGetData(DetectEngineThreadCtx *det_ctx,
                 &data, &data_len) == 0) {
         return NULL;
     }
-    InspectionBufferSetup(buffer, data, data_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
     InspectionBufferApplyTransforms(buffer, transforms);
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-engine.c b/src/detect-engine.c
index cbb681963ba7..1f8d945bce3d 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -996,11 +996,7 @@ void InspectionBufferClean(DetectEngineThreadCtx *det_ctx)
 
 InspectionBuffer *InspectionBufferGet(DetectEngineThreadCtx *det_ctx, const int list_id)
 {
-    InspectionBuffer *buffer = &det_ctx->inspect.buffers[list_id];
-    if (buffer->inspect == NULL) {
-        det_ctx->inspect.to_clear_queue[det_ctx->inspect.to_clear_idx++] = list_id;
-    }
-    return buffer;
+    return &det_ctx->inspect.buffers[list_id];
 }
 
 /** \brief for a InspectionBufferMultipleForList get a InspectionBuffer
@@ -1053,8 +1049,15 @@ void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size)
 }
 
 /** \brief setup the buffer with our initial data */
-void InspectionBufferSetup(InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len)
+void InspectionBufferSetup(DetectEngineThreadCtx *det_ctx,
+        const int list_id, InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len)
 {
+    if (buffer->inspect == NULL) {
+#ifdef UNITTESTS
+        if (det_ctx && list_id != -1)
+#endif
+            det_ctx->inspect.to_clear_queue[det_ctx->inspect.to_clear_idx++] = list_id;
+    }
     buffer->inspect = buffer->orig = data;
     buffer->inspect_len = buffer->orig_len = data_len;
     buffer->len = 0;
diff --git a/src/detect-engine.h b/src/detect-engine.h
index dc72b2f67095..bbfde4cc4f76 100644
--- a/src/detect-engine.h
+++ b/src/detect-engine.h
@@ -29,7 +29,7 @@
 #include "flow-private.h"
 
 void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size);
-void InspectionBufferSetup(InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len);
+void InspectionBufferSetup(DetectEngineThreadCtx *det_ctx, const int list_id, InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len);
 void InspectionBufferFree(InspectionBuffer *buffer);
 void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size);
 void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len);
diff --git a/src/detect-file-data.c b/src/detect-file-data.c
index cc39cb91817e..78e6bcb9f4d0 100644
--- a/src/detect-file-data.c
+++ b/src/detect-file-data.c
@@ -405,7 +405,7 @@ static InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *de
 
     StreamingBufferGetDataAtOffset(body->sb,
             &data, &data_len, offset);
-    InspectionBufferSetup(buffer, data, data_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
     buffer->inspect_offset = offset;
 
     /* built-in 'transformation' */
@@ -483,7 +483,7 @@ static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
     StreamingBufferGetDataAtOffset(cur_file->sb,
             &data, &data_len,
             cur_file->content_inspected);
-    InspectionBufferSetup(buffer, data, data_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
     SCLogDebug("[list %d] [before] buffer offset %" PRIu64 "; buffer len %" PRIu32
                "; data_len %" PRIu32 "; file_size %" PRIu64,
             list_id, buffer->inspect_offset, buffer->inspect_len, data_len, file_size);
diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c
index 536e88803f14..2f3c23a3b431 100644
--- a/src/detect-filemagic.c
+++ b/src/detect-filemagic.c
@@ -461,7 +461,7 @@ static InspectionBuffer *FilemagicGetDataCallback(DetectEngineThreadCtx *det_ctx
     const uint8_t *data = (const uint8_t *)cur_file->magic;
     uint32_t data_len = (uint32_t)strlen(cur_file->magic);
 
-    InspectionBufferSetup(buffer, data, data_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
     InspectionBufferApplyTransforms(buffer, transforms);
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-filename.c b/src/detect-filename.c
index 2c82fede9af1..301f51ea9401 100644
--- a/src/detect-filename.c
+++ b/src/detect-filename.c
@@ -367,7 +367,7 @@ static InspectionBuffer *FilenameGetDataCallback(DetectEngineThreadCtx *det_ctx,
     const uint8_t *data = cur_file->name;
     uint32_t data_len = cur_file->name_len;
 
-    InspectionBufferSetup(buffer, data, data_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
     InspectionBufferApplyTransforms(buffer, transforms);
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c
index 97d2927829a2..cacc3ed6a9ce 100644
--- a/src/detect-http-client-body.c
+++ b/src/detect-http-client-body.c
@@ -257,7 +257,7 @@ static InspectionBuffer *HttpClientBodyGetDataCallback(DetectEngineThreadCtx *de
 
     StreamingBufferGetDataAtOffset(body->sb,
             &data, &data_len, offset);
-    InspectionBufferSetup(buffer, data, data_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
     InspectionBufferApplyTransforms(buffer, transforms);
     buffer->inspect_offset = offset;
 
diff --git a/src/detect-http-cookie.c b/src/detect-http-cookie.c
index 74fc9f98a44f..a1068da0f069 100644
--- a/src/detect-http-cookie.c
+++ b/src/detect-http-cookie.c
@@ -182,7 +182,7 @@ static InspectionBuffer *GetRequestData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(h->value);
         const uint8_t *data = bstr_ptr(h->value);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
@@ -210,7 +210,7 @@ static InspectionBuffer *GetResponseData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(h->value);
         const uint8_t *data = bstr_ptr(h->value);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http-header-names.c b/src/detect-http-header-names.c
index dde59f959755..989a24c3e517 100644
--- a/src/detect-http-header-names.c
+++ b/src/detect-http-header-names.c
@@ -189,7 +189,7 @@ static void PrefilterTxHttpRequestHeaderNames(DetectEngineThreadCtx *det_ctx,
             return;
 
         /* setup buffer and apply transforms */
-        InspectionBufferSetup(buffer, rawdata, rawdata_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, rawdata, rawdata_len);
         InspectionBufferApplyTransforms(buffer, ctx->transforms);
     }
 
@@ -267,7 +267,7 @@ static void PrefilterTxHttpResponseHeaderNames(DetectEngineThreadCtx *det_ctx,
             return;
 
         /* setup buffer and apply transforms */
-        InspectionBufferSetup(buffer, rawdata, rawdata_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, rawdata, rawdata_len);
         InspectionBufferApplyTransforms(buffer, ctx->transforms);
     }
 
@@ -332,7 +332,7 @@ static int InspectEngineHttpHeaderNames(
             goto end;
         }
         /* setup buffer and apply transforms */
-        InspectionBufferSetup(buffer, rawdata, rawdata_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, rawdata, rawdata_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http-header.c b/src/detect-http-header.c
index e2b1904ede9c..8eabfebecf90 100644
--- a/src/detect-http-header.c
+++ b/src/detect-http-header.c
@@ -176,7 +176,7 @@ static int DetectEngineInspectBufferHttpHeader(
             goto end;
         }
         /* setup buffer and apply transforms */
-        InspectionBufferSetup(buffer, rawdata, rawdata_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, rawdata, rawdata_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
@@ -243,7 +243,7 @@ static void PrefilterMpmHttpHeader(DetectEngineThreadCtx *det_ctx,
             return;
 
         /* setup buffer and apply transforms */
-        InspectionBufferSetup(buffer, rawdata, rawdata_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, rawdata, rawdata_len);
         InspectionBufferApplyTransforms(buffer, ctx->transforms);
     }
 
diff --git a/src/detect-http-headers-stub.h b/src/detect-http-headers-stub.h
index 07e18e7aae6b..33fea365331e 100644
--- a/src/detect-http-headers-stub.h
+++ b/src/detect-http-headers-stub.h
@@ -74,7 +74,7 @@ static InspectionBuffer *GetRequestData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(h->value);
         const uint8_t *data = bstr_ptr(h->value);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
@@ -107,7 +107,7 @@ static InspectionBuffer *GetResponseData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(h->value);
         const uint8_t *data = bstr_ptr(h->value);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http-host.c b/src/detect-http-host.c
index fdf47641feb0..0870fd920f60 100644
--- a/src/detect-http-host.c
+++ b/src/detect-http-host.c
@@ -233,7 +233,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(tx->request_hostname);
         const uint8_t *data = bstr_ptr(tx->request_hostname);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
@@ -306,7 +306,7 @@ static InspectionBuffer *GetRawData(DetectEngineThreadCtx *det_ctx,
             data_len = bstr_len(tx->parsed_uri->hostname);
         }
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http-method.c b/src/detect-http-method.c
index 9d185cedc0a8..4674ea6467b7 100644
--- a/src/detect-http-method.c
+++ b/src/detect-http-method.c
@@ -201,7 +201,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(tx->request_method);
         const uint8_t *data = bstr_ptr(tx->request_method);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http-protocol.c b/src/detect-http-protocol.c
index 1cae37e2d79f..6a35ac7e9be0 100644
--- a/src/detect-http-protocol.c
+++ b/src/detect-http-protocol.c
@@ -109,7 +109,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         }
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http-raw-header.c b/src/detect-http-raw-header.c
index 57d1b648f2cc..e75dbb96eea4 100644
--- a/src/detect-http-raw-header.c
+++ b/src/detect-http-raw-header.c
@@ -189,7 +189,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = ts ?
             tx_ud->request_headers_raw_len : tx_ud->response_headers_raw_len;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http-request-line.c b/src/detect-http-request-line.c
index 963ca3d08fc9..8f6ae56267a2 100644
--- a/src/detect-http-request-line.c
+++ b/src/detect-http-request-line.c
@@ -139,7 +139,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(tx->request_line);
         const uint8_t *data = bstr_ptr(tx->request_line);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-http-response-line.c b/src/detect-http-response-line.c
index 42d7c9a28325..64d915c675ac 100644
--- a/src/detect-http-response-line.c
+++ b/src/detect-http-response-line.c
@@ -138,7 +138,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(tx->response_line);
         const uint8_t *data = bstr_ptr(tx->response_line);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-http-start.c b/src/detect-http-start.c
index 4e6decc03ff8..1a42e088b13c 100644
--- a/src/detect-http-start.c
+++ b/src/detect-http-start.c
@@ -190,7 +190,7 @@ static void PrefilterTxHttpRequestStart(DetectEngineThreadCtx *det_ctx,
             return;
 
         /* setup buffer and apply transforms */
-        InspectionBufferSetup(buffer, rawdata, rawdata_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, rawdata, rawdata_len);
         InspectionBufferApplyTransforms(buffer, ctx->transforms);
     }
 
@@ -264,7 +264,7 @@ static void PrefilterTxHttpResponseStart(DetectEngineThreadCtx *det_ctx,
             return;
 
         /* setup buffer and apply transforms */
-        InspectionBufferSetup(buffer, rawdata, rawdata_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, rawdata, rawdata_len);
         InspectionBufferApplyTransforms(buffer, ctx->transforms);
     }
 
@@ -329,7 +329,7 @@ static int InspectEngineHttpStart(
             goto end;
         }
         /* setup buffer and apply transforms */
-        InspectionBufferSetup(buffer, rawdata, rawdata_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, rawdata, rawdata_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http-stat-code.c b/src/detect-http-stat-code.c
index 22433a6b8e47..bd0405508e63 100644
--- a/src/detect-http-stat-code.c
+++ b/src/detect-http-stat-code.c
@@ -163,7 +163,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(tx->response_status);
         const uint8_t *data = bstr_ptr(tx->response_status);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http-stat-msg.c b/src/detect-http-stat-msg.c
index c59f9c267182..6eeb5685cca1 100644
--- a/src/detect-http-stat-msg.c
+++ b/src/detect-http-stat-msg.c
@@ -163,7 +163,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(tx->response_message);
         const uint8_t *data = bstr_ptr(tx->response_message);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http-ua.c b/src/detect-http-ua.c
index a4d9f39ccff7..f7be34d05501 100644
--- a/src/detect-http-ua.c
+++ b/src/detect-http-ua.c
@@ -169,7 +169,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(h->value);
         const uint8_t *data = bstr_ptr(h->value);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http-uri.c b/src/detect-http-uri.c
index dafbae723361..a05c203e83f3 100644
--- a/src/detect-http-uri.c
+++ b/src/detect-http-uri.c
@@ -228,7 +228,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(tx_ud->request_uri_normalized);
         const uint8_t *data = bstr_ptr(tx_ud->request_uri_normalized);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
@@ -298,7 +298,7 @@ static InspectionBuffer *GetRawData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = bstr_len(tx->request_uri);
         const uint8_t *data = bstr_ptr(tx->request_uri);
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-http2.c b/src/detect-http2.c
index af51cd1eb0f7..4e89214d673c 100644
--- a/src/detect-http2.c
+++ b/src/detect-http2.c
@@ -706,7 +706,7 @@ static InspectionBuffer *GetHttp2HNameData(DetectEngineThreadCtx *det_ctx,
     if (b == NULL || b_len == 0)
         return NULL;
 
-    InspectionBufferSetup(buffer, b, b_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
     InspectionBufferApplyTransforms(buffer, transforms);
 
     SCReturnPtr(buffer, "InspectionBuffer");
@@ -840,7 +840,7 @@ static InspectionBuffer *GetHttp2HeaderData(DetectEngineThreadCtx *det_ctx,
     if (b == NULL || b_len == 0)
         return NULL;
 
-    InspectionBufferSetup(buffer, b, b_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
     InspectionBufferApplyTransforms(buffer, transforms);
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-icmpv4hdr.c b/src/detect-icmpv4hdr.c
index 6e626c45b3cf..42a6b4922278 100644
--- a/src/detect-icmpv4hdr.c
+++ b/src/detect-icmpv4hdr.c
@@ -112,7 +112,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = hlen;
         const uint8_t *data = (const uint8_t *)p->icmpv4h;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-icmpv6hdr.c b/src/detect-icmpv6hdr.c
index ffe9595aa98d..f892f5921b5f 100644
--- a/src/detect-icmpv6hdr.c
+++ b/src/detect-icmpv6hdr.c
@@ -118,7 +118,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = hlen;
         const uint8_t *data = (const uint8_t *)p->icmpv6h;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-ipv4hdr.c b/src/detect-ipv4hdr.c
index 87b29c5aefd0..e39dddea0568 100644
--- a/src/detect-ipv4hdr.c
+++ b/src/detect-ipv4hdr.c
@@ -117,7 +117,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = hlen;
         const uint8_t *data = (const uint8_t *)p->ip4h;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-ipv6hdr.c b/src/detect-ipv6hdr.c
index e5592234205a..e7221a9f9ea2 100644
--- a/src/detect-ipv6hdr.c
+++ b/src/detect-ipv6hdr.c
@@ -118,7 +118,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = hlen;
         const uint8_t *data = (const uint8_t *)p->ip6h;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-krb5-cname.c b/src/detect-krb5-cname.c
index 251baa0d0196..e7ca0fe84a89 100644
--- a/src/detect-krb5-cname.c
+++ b/src/detect-krb5-cname.c
@@ -76,7 +76,7 @@ static InspectionBuffer *GetKrb5CNameData(DetectEngineThreadCtx *det_ctx,
     if (b == NULL || b_len == 0)
         return NULL;
 
-    InspectionBufferSetup(buffer, b, b_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
     InspectionBufferApplyTransforms(buffer, transforms);
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-krb5-sname.c b/src/detect-krb5-sname.c
index e4d86346004a..fd33bb49bfbb 100644
--- a/src/detect-krb5-sname.c
+++ b/src/detect-krb5-sname.c
@@ -76,7 +76,7 @@ static InspectionBuffer *GetKrb5SNameData(DetectEngineThreadCtx *det_ctx,
     if (b == NULL || b_len == 0)
         return NULL;
 
-    InspectionBufferSetup(buffer, b, b_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
     InspectionBufferApplyTransforms(buffer, transforms);
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-mqtt-connect-clientid.c b/src/detect-mqtt-connect-clientid.c
index 2fca0c4c8851..421b486d1e87 100644
--- a/src/detect-mqtt-connect-clientid.c
+++ b/src/detect-mqtt-connect-clientid.c
@@ -63,7 +63,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-mqtt-connect-password.c b/src/detect-mqtt-connect-password.c
index 506b874e58d6..9c964418cd45 100644
--- a/src/detect-mqtt-connect-password.c
+++ b/src/detect-mqtt-connect-password.c
@@ -63,7 +63,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-mqtt-connect-username.c b/src/detect-mqtt-connect-username.c
index 57e4a2f631ee..a4f5a723d56d 100644
--- a/src/detect-mqtt-connect-username.c
+++ b/src/detect-mqtt-connect-username.c
@@ -63,7 +63,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-mqtt-connect-willmessage.c b/src/detect-mqtt-connect-willmessage.c
index 4d82e68d25f3..4fdcab6250b3 100644
--- a/src/detect-mqtt-connect-willmessage.c
+++ b/src/detect-mqtt-connect-willmessage.c
@@ -63,7 +63,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-mqtt-connect-willtopic.c b/src/detect-mqtt-connect-willtopic.c
index 3bc9db630165..2794a7d48c7f 100644
--- a/src/detect-mqtt-connect-willtopic.c
+++ b/src/detect-mqtt-connect-willtopic.c
@@ -63,7 +63,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-mqtt-publish-message.c b/src/detect-mqtt-publish-message.c
index ba43aa99565a..32671bc32c22 100644
--- a/src/detect-mqtt-publish-message.c
+++ b/src/detect-mqtt-publish-message.c
@@ -63,7 +63,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-mqtt-publish-topic.c b/src/detect-mqtt-publish-topic.c
index d7fb7ba22ed1..3a33acf732c3 100644
--- a/src/detect-mqtt-publish-topic.c
+++ b/src/detect-mqtt-publish-topic.c
@@ -63,7 +63,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-mqtt-subscribe-topic.c b/src/detect-mqtt-subscribe-topic.c
index 30b4e48326bb..57ab2c4c03d6 100644
--- a/src/detect-mqtt-subscribe-topic.c
+++ b/src/detect-mqtt-subscribe-topic.c
@@ -83,7 +83,7 @@ static InspectionBuffer *MQTTSubscribeTopicGetData(DetectEngineThreadCtx *det_ct
         return NULL;
     }
 
-    InspectionBufferSetup(buffer, data, data_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
     InspectionBufferApplyTransforms(buffer, transforms);
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-mqtt-unsubscribe-topic.c b/src/detect-mqtt-unsubscribe-topic.c
index 3eb7490e010b..307f843ea41e 100644
--- a/src/detect-mqtt-unsubscribe-topic.c
+++ b/src/detect-mqtt-unsubscribe-topic.c
@@ -83,7 +83,7 @@ static InspectionBuffer *MQTTUnsubscribeTopicGetData(DetectEngineThreadCtx *det_
         return NULL;
     }
 
-    InspectionBufferSetup(buffer, data, data_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
     InspectionBufferApplyTransforms(buffer, transforms);
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-rfb-name.c b/src/detect-rfb-name.c
index d46819fd32a0..556c3e0c687e 100644
--- a/src/detect-rfb-name.c
+++ b/src/detect-rfb-name.c
@@ -82,7 +82,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-sip-method.c b/src/detect-sip-method.c
index 477f96138761..05417755c645 100644
--- a/src/detect-sip-method.c
+++ b/src/detect-sip-method.c
@@ -115,7 +115,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-sip-protocol.c b/src/detect-sip-protocol.c
index e8e4de72f0ef..ed8e780fbd36 100644
--- a/src/detect-sip-protocol.c
+++ b/src/detect-sip-protocol.c
@@ -85,7 +85,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         if (b == NULL || b_len == 0)
             return NULL;
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-sip-request-line.c b/src/detect-sip-request-line.c
index 9341f2b60469..2e91a01b504b 100644
--- a/src/detect-sip-request-line.c
+++ b/src/detect-sip-request-line.c
@@ -86,7 +86,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-sip-response-line.c b/src/detect-sip-response-line.c
index a6aaeb3795b4..879d5bc69357 100644
--- a/src/detect-sip-response-line.c
+++ b/src/detect-sip-response-line.c
@@ -86,7 +86,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-sip-stat-code.c b/src/detect-sip-stat-code.c
index b98e520217f8..8c15b050eabd 100644
--- a/src/detect-sip-stat-code.c
+++ b/src/detect-sip-stat-code.c
@@ -88,7 +88,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-sip-stat-msg.c b/src/detect-sip-stat-msg.c
index c29851b9e0f2..f8aac9a5c4b9 100644
--- a/src/detect-sip-stat-msg.c
+++ b/src/detect-sip-stat-msg.c
@@ -88,7 +88,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-sip-uri.c b/src/detect-sip-uri.c
index 074e8e43e8d8..53d3ed954c20 100644
--- a/src/detect-sip-uri.c
+++ b/src/detect-sip-uri.c
@@ -98,7 +98,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-smb-share.c b/src/detect-smb-share.c
index f770929d4533..c54ffa3779cc 100644
--- a/src/detect-smb-share.c
+++ b/src/detect-smb-share.c
@@ -69,7 +69,7 @@ static InspectionBuffer *GetNamedPipeData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
@@ -132,7 +132,7 @@ static InspectionBuffer *GetShareData(DetectEngineThreadCtx *det_ctx,
         if (b == NULL || b_len == 0)
             return NULL;
 
-        InspectionBufferSetup(buffer, b, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, b, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
diff --git a/src/detect-snmp-community.c b/src/detect-snmp-community.c
index 6769d7d4b4a6..f1882c269e38 100644
--- a/src/detect-snmp-community.c
+++ b/src/detect-snmp-community.c
@@ -104,7 +104,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         }
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-ssh-hassh-server-string.c b/src/detect-ssh-hassh-server-string.c
index 6c6b0b10eee5..d6a76f5f1207 100644
--- a/src/detect-ssh-hassh-server-string.c
+++ b/src/detect-ssh-hassh-server-string.c
@@ -77,7 +77,7 @@ static InspectionBuffer *GetSshData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         }
 
-        InspectionBufferSetup(buffer, hassh, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, hassh, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-ssh-hassh-server.c b/src/detect-ssh-hassh-server.c
index 3978dc3a0f7e..3f703c051b06 100644
--- a/src/detect-ssh-hassh-server.c
+++ b/src/detect-ssh-hassh-server.c
@@ -77,7 +77,7 @@ static InspectionBuffer *GetSshData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         }
 
-        InspectionBufferSetup(buffer, hasshServer, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, hasshServer, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-ssh-hassh-string.c b/src/detect-ssh-hassh-string.c
index 5c3437169331..a6ad4223b42b 100644
--- a/src/detect-ssh-hassh-string.c
+++ b/src/detect-ssh-hassh-string.c
@@ -77,7 +77,7 @@ static InspectionBuffer *GetSshData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         }
 
-        InspectionBufferSetup(buffer, hassh, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, hassh, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-ssh-hassh.c b/src/detect-ssh-hassh.c
index 8cd4b5749ddd..dbc0dd832f3d 100644
--- a/src/detect-ssh-hassh.c
+++ b/src/detect-ssh-hassh.c
@@ -77,7 +77,7 @@ static InspectionBuffer *GetSshData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         }
 
-        InspectionBufferSetup(buffer, hassh, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, hassh, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-ssh-proto.c b/src/detect-ssh-proto.c
index d8eec71d4e56..4f4874a9161b 100644
--- a/src/detect-ssh-proto.c
+++ b/src/detect-ssh-proto.c
@@ -74,7 +74,7 @@ static InspectionBuffer *GetSshData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         }
 
-        InspectionBufferSetup(buffer, protocol, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, protocol, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-ssh-software.c b/src/detect-ssh-software.c
index 9765e3844e3f..ac7b1973cedf 100644
--- a/src/detect-ssh-software.c
+++ b/src/detect-ssh-software.c
@@ -74,7 +74,7 @@ static InspectionBuffer *GetSshData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         }
 
-        InspectionBufferSetup(buffer, software, b_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, software, b_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-tcphdr.c b/src/detect-tcphdr.c
index 3b0cde3f318a..4b6a65986d27 100644
--- a/src/detect-tcphdr.c
+++ b/src/detect-tcphdr.c
@@ -119,7 +119,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = hlen;
         const uint8_t *data = (const uint8_t *)p->tcph;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-template-buffer.c b/src/detect-template-buffer.c
index 6264f81b589a..e988255b9ebf 100644
--- a/src/detect-template-buffer.c
+++ b/src/detect-template-buffer.c
@@ -133,7 +133,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
             return NULL; /* no buffer */
         }
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-tls-cert-fingerprint.c b/src/detect-tls-cert-fingerprint.c
index 7bb827bc7553..130daadf3ce3 100644
--- a/src/detect-tls-cert-fingerprint.c
+++ b/src/detect-tls-cert-fingerprint.c
@@ -141,7 +141,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = strlen(ssl_state->server_connp.cert0_fingerprint);
         const uint8_t *data = (uint8_t *)ssl_state->server_connp.cert0_fingerprint;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-tls-cert-issuer.c b/src/detect-tls-cert-issuer.c
index 18591661dd58..bf89e6550a8e 100644
--- a/src/detect-tls-cert-issuer.c
+++ b/src/detect-tls-cert-issuer.c
@@ -131,7 +131,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = strlen(ssl_state->server_connp.cert0_issuerdn);
         const uint8_t *data = (uint8_t *)ssl_state->server_connp.cert0_issuerdn;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-tls-cert-serial.c b/src/detect-tls-cert-serial.c
index ce0950b32104..4d5bfdbf82a8 100644
--- a/src/detect-tls-cert-serial.c
+++ b/src/detect-tls-cert-serial.c
@@ -140,7 +140,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = strlen(ssl_state->server_connp.cert0_serial);
         const uint8_t *data = (uint8_t *)ssl_state->server_connp.cert0_serial;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-tls-cert-subject.c b/src/detect-tls-cert-subject.c
index d22a686cbe94..0e43a45a1acd 100644
--- a/src/detect-tls-cert-subject.c
+++ b/src/detect-tls-cert-subject.c
@@ -130,7 +130,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = strlen(ssl_state->server_connp.cert0_subject);
         const uint8_t *data = (uint8_t *)ssl_state->server_connp.cert0_subject;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-tls-certs.c b/src/detect-tls-certs.c
index 01244d1657b5..9c3144dfa8b0 100644
--- a/src/detect-tls-certs.c
+++ b/src/detect-tls-certs.c
@@ -158,7 +158,7 @@ static InspectionBuffer *TlsCertsGetData(DetectEngineThreadCtx *det_ctx,
         return NULL;
     }
 
-    InspectionBufferSetup(buffer, cbdata->cert->cert_data,
+    InspectionBufferSetup(det_ctx, list_id, buffer, cbdata->cert->cert_data,
 		          cbdata->cert->cert_len);
     InspectionBufferApplyTransforms(buffer, transforms);
 
diff --git a/src/detect-tls-ja3-hash.c b/src/detect-tls-ja3-hash.c
index 03787ce2cc09..9787ecd3f1eb 100644
--- a/src/detect-tls-ja3-hash.c
+++ b/src/detect-tls-ja3-hash.c
@@ -153,7 +153,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = strlen(ssl_state->client_connp.ja3_hash);
         const uint8_t *data = (uint8_t *)ssl_state->client_connp.ja3_hash;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-tls-ja3-string.c b/src/detect-tls-ja3-string.c
index 6869ea960458..76c44c8c0d56 100644
--- a/src/detect-tls-ja3-string.c
+++ b/src/detect-tls-ja3-string.c
@@ -143,7 +143,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = strlen(ssl_state->client_connp.ja3_str->data);
         const uint8_t *data = (uint8_t *)ssl_state->client_connp.ja3_str->data;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-tls-ja3s-hash.c b/src/detect-tls-ja3s-hash.c
index d8acf0344c44..7dd7077da9de 100644
--- a/src/detect-tls-ja3s-hash.c
+++ b/src/detect-tls-ja3s-hash.c
@@ -151,7 +151,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = strlen(ssl_state->server_connp.ja3_hash);
         const uint8_t *data = (uint8_t *)ssl_state->server_connp.ja3_hash;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-tls-ja3s-string.c b/src/detect-tls-ja3s-string.c
index ea4c9aa5e48b..7b22fce6c29e 100644
--- a/src/detect-tls-ja3s-string.c
+++ b/src/detect-tls-ja3s-string.c
@@ -142,7 +142,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = strlen(ssl_state->server_connp.ja3_str->data);
         const uint8_t *data = (uint8_t *)ssl_state->server_connp.ja3_str->data;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-tls-sni.c b/src/detect-tls-sni.c
index 0fe36c020871..5bcb4172a728 100644
--- a/src/detect-tls-sni.c
+++ b/src/detect-tls-sni.c
@@ -129,7 +129,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = strlen(ssl_state->client_connp.sni);
         const uint8_t *data = (uint8_t *)ssl_state->client_connp.sni;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 
diff --git a/src/detect-transform-compress-whitespace.c b/src/detect-transform-compress-whitespace.c
index 0c720fb0f667..13b5f4d01e1c 100644
--- a/src/detect-transform-compress-whitespace.c
+++ b/src/detect-transform-compress-whitespace.c
@@ -161,7 +161,7 @@ static int DetectTransformCompressWhitespaceTest01(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, 9);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     TransformCompressWhitespace(&buffer, NULL);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
@@ -176,7 +176,7 @@ static int DetectTransformCompressWhitespaceTest02(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, 9);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     TransformDoubleWhitespace(&buffer);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
@@ -195,7 +195,7 @@ static int DetectTransformCompressWhitespaceTest03(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, 10);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     FAIL_IF(TransformCompressWhitespaceValidate(buffer.inspect, buffer.inspect_len, NULL));
     PASS;
@@ -208,7 +208,7 @@ static int DetectTransformCompressWhitespaceTest04(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, 9);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     TransformDoubleWhitespace(&buffer);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     FAIL_IF(TransformCompressWhitespaceValidate(buffer.inspect, buffer.inspect_len, NULL));
diff --git a/src/detect-transform-dotprefix.c b/src/detect-transform-dotprefix.c
index 22fe0ff5ee82..061b0f1925b2 100644
--- a/src/detect-transform-dotprefix.c
+++ b/src/detect-transform-dotprefix.c
@@ -128,7 +128,7 @@ static int DetectTransformDotPrefixTest01(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, input_len);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     TransformDotPrefix(&buffer, NULL);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
@@ -148,7 +148,7 @@ static int DetectTransformDotPrefixTest02(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, input_len);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     TransformDotPrefix(&buffer, NULL);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
@@ -182,4 +182,4 @@ static void DetectTransformDotPrefixRegisterTests(void)
     UtRegisterTest("DetectTransformDotPrefixTest02", DetectTransformDotPrefixTest02);
     UtRegisterTest("DetectTransformDotPrefixTest03", DetectTransformDotPrefixTest03);
 }
-#endif
\ No newline at end of file
+#endif
diff --git a/src/detect-transform-md5.c b/src/detect-transform-md5.c
index 7d891d8619a8..2e6d8d4ea6f7 100644
--- a/src/detect-transform-md5.c
+++ b/src/detect-transform-md5.c
@@ -99,7 +99,7 @@ static int DetectTransformToMd5Test01(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, 8);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     TransformToMd5(&buffer, NULL);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
@@ -112,4 +112,4 @@ static void DetectTransformToMd5RegisterTests(void)
     UtRegisterTest("DetectTransformToMd5Test01",
             DetectTransformToMd5Test01);
 }
-#endif
\ No newline at end of file
+#endif
diff --git a/src/detect-transform-sha1.c b/src/detect-transform-sha1.c
index 3f6e85ca5688..6ff84b5ccbef 100644
--- a/src/detect-transform-sha1.c
+++ b/src/detect-transform-sha1.c
@@ -100,7 +100,7 @@ static int DetectTransformToSha1Test01(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, 8);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     TransformToSha1(&buffer, NULL);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
diff --git a/src/detect-transform-sha256.c b/src/detect-transform-sha256.c
index c01cc9220b15..af07454d8a6d 100644
--- a/src/detect-transform-sha256.c
+++ b/src/detect-transform-sha256.c
@@ -100,7 +100,7 @@ static int DetectTransformToSha256Test01(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, 8);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     TransformToSha256(&buffer, NULL);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
@@ -113,4 +113,4 @@ static void DetectTransformToSha256RegisterTests(void)
     UtRegisterTest("DetectTransformToSha256Test01",
             DetectTransformToSha256Test01);
 }
-#endif
\ No newline at end of file
+#endif
diff --git a/src/detect-transform-strip-whitespace.c b/src/detect-transform-strip-whitespace.c
index 6757941f8c15..055a7e00d061 100644
--- a/src/detect-transform-strip-whitespace.c
+++ b/src/detect-transform-strip-whitespace.c
@@ -148,7 +148,7 @@ static int DetectTransformStripWhitespaceTest01(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, 8);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     TransformStripWhitespace(&buffer, NULL);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
@@ -163,7 +163,7 @@ static int DetectTransformStripWhitespaceTest02(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, 8);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     TransformDoubleWhitespace(&buffer);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
@@ -202,4 +202,4 @@ static void DetectTransformStripWhitespaceRegisterTests(void)
     UtRegisterTest("DetectTransformStripWhitespaceTest03",
             DetectTransformStripWhitespaceTest03);
 }
-#endif
\ No newline at end of file
+#endif
diff --git a/src/detect-transform-urldecode.c b/src/detect-transform-urldecode.c
index 50781c8f72fb..526561f2f992 100644
--- a/src/detect-transform-urldecode.c
+++ b/src/detect-transform-urldecode.c
@@ -136,7 +136,7 @@ static int DetectTransformUrlDecodeTest01(void)
 
     InspectionBuffer buffer;
     InspectionBufferInit(&buffer, 8);
-    InspectionBufferSetup(&buffer, input, input_len);
+    InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
     TransformUrlDecode(&buffer, NULL);
     PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
diff --git a/src/detect-udphdr.c b/src/detect-udphdr.c
index a8900048bea4..6c58a8883c59 100644
--- a/src/detect-udphdr.c
+++ b/src/detect-udphdr.c
@@ -114,7 +114,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const uint32_t data_len = UDP_HEADER_LEN;
         const uint8_t *data = (const uint8_t *)p->udph;
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
 

From da66db148f1c7df227eb42fcaa52ca34301de11d Mon Sep 17 00:00:00 2001
From: Victor Julien <victor@inliniac.net>
Date: Fri, 5 Feb 2021 08:41:22 +0100
Subject: [PATCH 04/19] detect/state: optimize state keeping

---
 src/detect-engine-state.c | 24 +++++++++---------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c
index ac83a4bc335f..865801ed428c 100644
--- a/src/detect-engine-state.c
+++ b/src/detect-engine-state.c
@@ -123,27 +123,22 @@ static int DeStateSearchState(DetectEngineState *state, uint8_t direction, SigIn
 static void DeStateSignatureAppend(DetectEngineState *state,
         const Signature *s, uint32_t inspect_flags, uint8_t direction)
 {
-    int jump = 0;
-    int i = 0;
+    SCEnter();
+
     DetectEngineStateDirection *dir_state = &state->dir_state[direction & STREAM_TOSERVER ? 0 : 1];
 
 #ifdef DEBUG_VALIDATION
     BUG_ON(DeStateSearchState(state, direction, s->num));
 #endif
-    DeStateStore *store = dir_state->head;
+    DeStateStore *store = dir_state->tail;
 
     if (store == NULL) {
         store = DeStateStoreAlloc();
-        if (store != NULL) {
-            dir_state->head = store;
-            dir_state->tail = store;
-        }
+        dir_state->head = store;
+        dir_state->tail = store;
     } else {
-        jump = dir_state->cnt / DE_STATE_CHUNK_SIZE;
-        for (i = 0; i < jump; i++) {
-            store = store->next;
-        }
-        if (store == NULL) {
+        SCLogDebug("dir_state->cnt %u mod chunksize %u", dir_state->cnt, dir_state->cnt % DE_STATE_CHUNK_SIZE);
+        if (dir_state->cnt && dir_state->cnt % DE_STATE_CHUNK_SIZE == 0) {
             store = DeStateStoreAlloc();
             if (store != NULL) {
                 dir_state->tail->next = store;
@@ -151,15 +146,14 @@ static void DeStateSignatureAppend(DetectEngineState *state,
             }
         }
     }
-
     if (store == NULL)
-        return;
+        SCReturn;
 
     SigIntId idx = dir_state->cnt++ % DE_STATE_CHUNK_SIZE;
     store->store[idx].sid = s->num;
     store->store[idx].flags = inspect_flags;
 
-    return;
+    SCReturn;
 }
 
 DetectEngineState *DetectEngineStateAlloc(void)

From adcce44472eed7da9ab2393efd2f0165bf9a8995 Mon Sep 17 00:00:00 2001
From: Philippe Antoine <contact@catenacyber.fr>
Date: Fri, 5 Feb 2021 15:15:27 +0100
Subject: [PATCH 05/19] template: use response_gap in rust parser

---
 rust/src/applayertemplate/template.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/rust/src/applayertemplate/template.rs b/rust/src/applayertemplate/template.rs
index 99d9343eb951..feabfc65686b 100644
--- a/rust/src/applayertemplate/template.rs
+++ b/rust/src/applayertemplate/template.rs
@@ -177,6 +177,17 @@ impl TemplateState {
             return AppLayerResult::ok();
         }
 
+        if self.response_gap {
+            if probe(input).is_err() {
+                // The parser now needs to decide what to do as we are not in sync.
+                // For this template, we'll just try again next time.
+                return AppLayerResult::ok();
+            }
+
+            // It looks like we're in sync with a message header, clear gap
+            // state and keep parsing.
+            self.response_gap = false;
+        }
         let mut start = input;
         while start.len() > 0 {
             match parser::parse_message(start) {

From b893e0f6b06923e5ac068d6f9b730cbf8c896b57 Mon Sep 17 00:00:00 2001
From: Jeff Lucovsky <jeff@lucovsky.org>
Date: Mon, 8 Feb 2021 08:05:41 -0500
Subject: [PATCH 06/19] detct/pcre: Correct capture group count check

This commit corrects the validation check between the number of
variables used and the number of specified capture groups.
---
 src/detect-pcre.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/detect-pcre.c b/src/detect-pcre.c
index c6588ba6ab17..2d4a4dc7668c 100644
--- a/src/detect-pcre.c
+++ b/src/detect-pcre.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2020 Open Information Security Foundation
+/* Copyright (C) 2007-2021 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -723,7 +723,7 @@ static int DetectPcreParseCapture(const char *regexstr, DetectEngineCtx *de_ctx,
     {
         char *ptr = NULL;
         while ((name_array[name_idx] = strtok_r(name_idx == 0 ? capture_names : NULL, " ,", &ptr))){
-            if (name_idx > capture_cnt) {
+            if (name_idx > (capture_cnt - 1)) {
                 SCLogError(SC_ERR_VAR_LIMIT, "more pkt/flow "
                         "var capture names than capturing substrings");
                 return -1;

From 6284f665618a157d2ecfad2aa0d2e4d0b7415d08 Mon Sep 17 00:00:00 2001
From: Jeff Lucovsky <jeff@lucovsky.org>
Date: Mon, 8 Feb 2021 08:06:53 -0500
Subject: [PATCH 07/19] detect/pcre: Test capture group/var mismatch

---
 src/detect-pcre.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/detect-pcre.c b/src/detect-pcre.c
index 2d4a4dc7668c..9bff6d044d4f 100644
--- a/src/detect-pcre.c
+++ b/src/detect-pcre.c
@@ -3552,6 +3552,11 @@ static int DetectPcreParseCaptureTest(void)
     s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any "
             "(content:\"Server: \"; http_header; pcre:\"/([a-z]+)([0-9]+)\\r\\n/HR, flow:somecapture, pkt:anothercap\"; content:\"xyz\"; http_header; sid:3;)");
     FAIL_IF(s == NULL);
+    s = DetectEngineAppendSig(de_ctx,
+            "alert http any any -> any any "
+            "(content:\"Server: \"; http_header; pcre:\"/([a-z]+)\\r\\n/HR, flow:somecapture, "
+            "pkt:anothercap\"; content:\"xyz\"; http_header; sid:3;)");
+    FAIL_IF_NOT_NULL(s);
 
     SigGroupBuild(de_ctx);
 

From 0cccb160c88c1b2858709995b4dfee54318babf2 Mon Sep 17 00:00:00 2001
From: Eric Leblond <eric@regit.org>
Date: Wed, 27 Jan 2021 11:38:34 +0100
Subject: [PATCH 08/19] eve: only output ja3 and ja3s if present

This will prevent JSON entries like the following that occur
with the dedault configuration (ja3 deactivated and extended
tls ouput activated):

  "tls": {
    "subject": "C=GB, ST=London, L=London, O=Global Security, OU=IT Department, CN=example.com",
    "issuerdn": "C=GB, ST=London, L=London, O=Global Security, OU=IT Department, CN=example.com",
    "serial": "00:9C:FC:DA:1D:A4:70:87:5D",
    "fingerprint": "b8:18:2d:cb:c9:f8:1a:66:75:13:18:31:24:e0:92:35:42:ab:96:89",
    "version": "TLSv1",
    "notbefore": "2020-05-03T11:07:28",
    "notafter": "2021-05-03T11:07:28",
    "ja3": {},
    "ja3s": {}
  }
---
 src/output-json-tls.c | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/output-json-tls.c b/src/output-json-tls.c
index 3bf30a805898..412c18e0fff2 100644
--- a/src/output-json-tls.c
+++ b/src/output-json-tls.c
@@ -215,12 +215,16 @@ static void JsonTlsLogJa3String(JsonBuilder *js, SSLState *ssl_state)
 
 static void JsonTlsLogJa3(JsonBuilder *js, SSLState *ssl_state)
 {
-    jb_open_object(js, "ja3");
+    if ((ssl_state->client_connp.ja3_hash != NULL) ||
+            ((ssl_state->client_connp.ja3_str != NULL) &&
+                    ssl_state->client_connp.ja3_str->data != NULL)) {
+        jb_open_object(js, "ja3");
 
-    JsonTlsLogJa3Hash(js, ssl_state);
-    JsonTlsLogJa3String(js, ssl_state);
+        JsonTlsLogJa3Hash(js, ssl_state);
+        JsonTlsLogJa3String(js, ssl_state);
 
-    jb_close(js);
+        jb_close(js);
+    }
 }
 
 static void JsonTlsLogJa3SHash(JsonBuilder *js, SSLState *ssl_state)
@@ -242,12 +246,16 @@ static void JsonTlsLogJa3SString(JsonBuilder *js, SSLState *ssl_state)
 
 static void JsonTlsLogJa3S(JsonBuilder *js, SSLState *ssl_state)
 {
-    jb_open_object(js, "ja3s");
+    if ((ssl_state->server_connp.ja3_hash != NULL) ||
+            ((ssl_state->server_connp.ja3_str != NULL) &&
+                    ssl_state->server_connp.ja3_str->data != NULL)) {
+        jb_open_object(js, "ja3s");
 
-    JsonTlsLogJa3SHash(js, ssl_state);
-    JsonTlsLogJa3SString(js, ssl_state);
+        JsonTlsLogJa3SHash(js, ssl_state);
+        JsonTlsLogJa3SString(js, ssl_state);
 
-    jb_close(js);
+        jb_close(js);
+    }
 }
 
 static void JsonTlsLogCertificate(JsonBuilder *js, SSLState *ssl_state)

From 1ea9fb7e10ebe5c8a913df84119aa6c918a48b75 Mon Sep 17 00:00:00 2001
From: Eric Leblond <eric@regit.org>
Date: Fri, 29 Jan 2021 23:59:11 +0100
Subject: [PATCH 09/19] suricata: unix-socket mode and -l are compatible

Commit 93642a0d1dd29c96e98824935ef963f6b1ef40eb did prevent to
specify the logging directory on command line and use the unix
socket.

It looks like the implementation has evolved and the arbitrary
limitation can be removed allowing a user to start unix socket
without editing the configuration file.
---
 src/suricata.c | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/suricata.c b/src/suricata.c
index 4f054aeb1dbc..4fc13d3b180d 100644
--- a/src/suricata.c
+++ b/src/suricata.c
@@ -1840,12 +1840,6 @@ static TmEcode ParseCommandLine(int argc, char** argv, SCInstance *suri)
         return TM_ECODE_FAILED;
     }
 
-    if ((suri->run_mode == RUNMODE_UNIX_SOCKET) && suri->set_logdir) {
-        SCLogError(SC_ERR_INITIALIZATION,
-                "can't use -l and unix socket runmode at the same time");
-        return TM_ECODE_FAILED;
-    }
-
     /* save the runmode from the commandline (if any) */
     suri->aux_run_mode = suri->run_mode;
 

From c6bbc5d61a392c157ea3fc55e25c12aa8ace186b Mon Sep 17 00:00:00 2001
From: Eric Leblond <eric@regit.org>
Date: Wed, 10 Feb 2021 16:23:25 +0100
Subject: [PATCH 10/19] log/pcap: exit on invalid filename

If the filename has to % sign and if pcap logging is using multi
mode, then the pcap capture will fail. So let's exit if ever this
is the case.
---
 src/log-pcap.c | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/log-pcap.c b/src/log-pcap.c
index afee53c0927a..07ac49452c5b 100644
--- a/src/log-pcap.c
+++ b/src/log-pcap.c
@@ -1125,6 +1125,13 @@ static int ParseFilename(PcapLogData *pl, const char *filename)
                 }
             }
         }
+
+        if ((tok == 0) && (pl->mode == LOGMODE_MULTI)) {
+            SCLogError(SC_ERR_INVALID_ARGUMENT,
+                    "Invalid filename for multimode. Need at list one %%-sign option");
+            goto error;
+        }
+
         if (s) {
             if (tok >= MAX_TOKS) {
                 SCLogError(SC_ERR_INVALID_ARGUMENT,
@@ -1222,11 +1229,6 @@ static OutputInitResult PcapLogInitCtx(ConfNode *conf)
 
     pl->suffix = "";
 
-    if (filename) {
-        if (ParseFilename(pl, filename) != 0)
-            exit(EXIT_FAILURE);
-    }
-
     pl->size_limit = DEFAULT_LIMIT;
     if (conf != NULL) {
         const char *s_limit = NULL;
@@ -1418,6 +1420,11 @@ static OutputInitResult PcapLogInitCtx(ConfNode *conf)
                 compression_str ? compression_str : "none");
     }
 
+    if (filename) {
+        if (ParseFilename(pl, filename) != 0)
+            exit(EXIT_FAILURE);
+    }
+
     SCLogInfo("using %s logging", pl->mode == LOGMODE_SGUIL ?
               "Sguil compatible" : (pl->mode == LOGMODE_MULTI ? "multi" : "normal"));
 

From b325b4150210ac161bada7fec128cd0c29467169 Mon Sep 17 00:00:00 2001
From: Eric Leblond <eric@regit.org>
Date: Wed, 10 Feb 2021 17:26:47 +0100
Subject: [PATCH 11/19] util/running-modes: don't exit in running mode

---
 src/util-running-modes.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/util-running-modes.c b/src/util-running-modes.c
index b4f52ea6da23..8748c0b15d80 100644
--- a/src/util-running-modes.c
+++ b/src/util-running-modes.c
@@ -37,7 +37,7 @@ int ListKeywords(const char *keyword_info)
     AppLayerSetup();
     SigTableSetup(); /* load the rule keywords */
     SigTableList(keyword_info);
-    exit(EXIT_SUCCESS);
+    return TM_ECODE_DONE;
 }
 
 int ListAppLayerProtocols(const char *conf_filename)
@@ -49,6 +49,6 @@ int ListAppLayerProtocols(const char *conf_filename)
     AppLayerSetup();
     AppLayerListSupportedProtocols();
 
-    exit(EXIT_SUCCESS);
+    return TM_ECODE_DONE;
 }
 

From 334049edd54e44d60d58a0b0bb0f8327792af869 Mon Sep 17 00:00:00 2001
From: Eric Leblond <eric@regit.org>
Date: Wed, 10 Feb 2021 17:28:18 +0100
Subject: [PATCH 12/19] suricata: return error value of custom run modes

---
 src/suricata.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/suricata.c b/src/suricata.c
index 4fc13d3b180d..5b80e974a0c6 100644
--- a/src/suricata.c
+++ b/src/suricata.c
@@ -2085,15 +2085,13 @@ static int StartInternalRunMode(SCInstance *suri, int argc, char **argv)
     /* Treat internal running mode */
     switch(suri->run_mode) {
         case RUNMODE_LIST_KEYWORDS:
-            ListKeywords(suri->keyword_info);
-            return TM_ECODE_DONE;
+            return ListKeywords(suri->keyword_info);
         case RUNMODE_LIST_APP_LAYERS:
             if (suri->conf_filename != NULL) {
-                ListAppLayerProtocols(suri->conf_filename);
+                return ListAppLayerProtocols(suri->conf_filename);
             } else {
-                ListAppLayerProtocols(DEFAULT_CONF_FILE);
+                return ListAppLayerProtocols(DEFAULT_CONF_FILE);
             }
-            return TM_ECODE_DONE;
         case RUNMODE_PRINT_VERSION:
             PrintVersion();
             return TM_ECODE_DONE;

From fc215fe0f3059c18e5857e1a8fd50e3e02d396d3 Mon Sep 17 00:00:00 2001
From: Eric Leblond <eric@regit.org>
Date: Wed, 10 Feb 2021 17:45:16 +0100
Subject: [PATCH 13/19] suricata: improve list keywords

Exit with error if a keyword is not supported or not existing
and display a message.
---
 src/detect-engine-register.c | 10 ++++++----
 src/detect-engine-register.h |  2 +-
 src/util-running-modes.c     |  3 +--
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c
index 3f849125a5a0..1ee5ef91f697 100644
--- a/src/detect-engine-register.c
+++ b/src/detect-engine-register.c
@@ -351,7 +351,7 @@ static void SigMultilinePrint(int i, const char *prefix)
     printf("\n");
 }
 
-void SigTableList(const char *keyword)
+int SigTableList(const char *keyword)
 {
     size_t size = sizeof(sigmatch_table) / sizeof(SigTableElmt);
     size_t i;
@@ -414,14 +414,16 @@ void SigTableList(const char *keyword)
                 printf("= %s =\n", sigmatch_table[i].name);
                 if (sigmatch_table[i].flags & SIGMATCH_NOT_BUILT) {
                     printf("Not built-in\n");
-                    return;
+                    return TM_ECODE_FAILED;
                 }
                 SigMultilinePrint(i, "");
-                return;
+                return TM_ECODE_DONE;
             }
         }
+        printf("Non existing keyword\n");
+        return TM_ECODE_FAILED;
     }
-    return;
+    return TM_ECODE_DONE;
 }
 
 void SigTableSetup(void)
diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h
index 5e3fc5e75b4f..aea54e28f0d5 100644
--- a/src/detect-engine-register.h
+++ b/src/detect-engine-register.h
@@ -303,7 +303,7 @@ enum DetectKeywordId {
     DETECT_TBLSIZE,
 };
 
-void SigTableList(const char *keyword);
+int SigTableList(const char *keyword);
 void SigTableSetup(void);
 void SigTableRegisterTests(void);
 
diff --git a/src/util-running-modes.c b/src/util-running-modes.c
index 8748c0b15d80..22b933c7afc8 100644
--- a/src/util-running-modes.c
+++ b/src/util-running-modes.c
@@ -36,8 +36,7 @@ int ListKeywords(const char *keyword_info)
     SpmTableSetup();
     AppLayerSetup();
     SigTableSetup(); /* load the rule keywords */
-    SigTableList(keyword_info);
-    return TM_ECODE_DONE;
+    return SigTableList(keyword_info);
 }
 
 int ListAppLayerProtocols(const char *conf_filename)

From 1ae5fd536cc2443aaeea6b938f23cd6f7937d200 Mon Sep 17 00:00:00 2001
From: Philippe Antoine <contact@catenacyber.fr>
Date: Mon, 16 Nov 2020 10:14:46 +0100
Subject: [PATCH 14/19] fuzz: driver running directories as well as single
 files

---
 src/tests/fuzz/onefile.c | 60 +++++++++++++++++++++++++++++++---------
 1 file changed, 47 insertions(+), 13 deletions(-)

diff --git a/src/tests/fuzz/onefile.c b/src/tests/fuzz/onefile.c
index ef86d921af89..f59a12c3260d 100644
--- a/src/tests/fuzz/onefile.c
+++ b/src/tests/fuzz/onefile.c
@@ -1,25 +1,19 @@
+#define _DEFAULT_SOURCE 1 // for DT_REG
 #include <stdint.h>
 #include <stdlib.h>
 #include <stdio.h>
+#include <dirent.h>
+#include <unistd.h>
 #include "autoconf.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
 
-int main(int argc, char** argv)
+static int runOneFile(const char *fname)
 {
-    FILE * fp;
+    // opens the file, get its size, and reads it into a buffer
     uint8_t *data;
     size_t size;
-
-    if (argc != 2) {
-        return 1;
-    }
-#ifdef AFLFUZZ_PERSISTANT_MODE
-    while (__AFL_LOOP(1000)) {
-#endif /* AFLFUZZ_PERSISTANT_MODE */
-
-    //opens the file, get its size, and reads it into a buffer
-    fp = fopen(argv[1], "rb");
+    FILE *fp = fopen(fname, "rb");
     if (fp == NULL) {
         return 2;
     }
@@ -51,10 +45,50 @@ int main(int argc, char** argv)
     LLVMFuzzerTestOneInput(data, size);
     free(data);
     fclose(fp);
+    return 0;
+}
+
+int main(int argc, char **argv)
+{
+    DIR *d;
+    struct dirent *dir;
+    int r;
+
+    if (argc != 2) {
+        return 1;
+    }
+#ifdef AFLFUZZ_PERSISTANT_MODE
+    while (__AFL_LOOP(1000)) {
+#endif /* AFLFUZZ_PERSISTANT_MODE */
+
+        d = opendir(argv[1]);
+        if (d == NULL) {
+            // run one file
+            r = runOneFile(argv[1]);
+            if (r != 0) {
+                return r;
+            }
+        } else {
+            // run every file in one directory
+            if (chdir(argv[1]) != 0) {
+                closedir(d);
+                printf("Invalid directory\n");
+                return 2;
+            }
+            while ((dir = readdir(d)) != NULL) {
+                if (dir->d_type != DT_REG) {
+                    continue;
+                }
+                r = runOneFile(dir->d_name);
+                if (r != 0) {
+                    return r;
+                }
+            }
+            closedir(d);
+        }
 #ifdef AFLFUZZ_PERSISTANT_MODE
     }
 #endif /* AFLFUZZ_PERSISTANT_MODE */
 
     return 0;
 }
-

From aebf635dd065da84272bcf6d18cd97ab2de0918f Mon Sep 17 00:00:00 2001
From: Philippe Antoine <contact@catenacyber.fr>
Date: Thu, 28 Jan 2021 17:02:19 +0100
Subject: [PATCH 15/19] detect: initializes memory in bytemath parsing

---
 src/detect-bytemath.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/detect-bytemath.c b/src/detect-bytemath.c
index 8994c13e92c3..9a7005d81b8d 100644
--- a/src/detect-bytemath.c
+++ b/src/detect-bytemath.c
@@ -262,7 +262,7 @@ static DetectByteMathData *DetectByteMathParse(DetectEngineCtx *de_ctx, const ch
 #undef MAX_SUBSTRINGS
 #define MAX_SUBSTRINGS 100
     int ov[MAX_SUBSTRINGS];
-    char tmp_str[128];
+    char tmp_str[128] = "";
 
     ret = DetectParsePcreExec(&parse_regex, arg, 0, 0, ov, MAX_SUBSTRINGS);
     if (ret < MIN_GROUP || ret > MAX_GROUP) {

From 0f9d4bbb05e662b848c97e18e181cc72d467225f Mon Sep 17 00:00:00 2001
From: Victor Julien <victor@inliniac.net>
Date: Thu, 21 Jan 2021 13:28:04 +0100
Subject: [PATCH 16/19] fuzz: run OSS-Fuzz corpus and track coverage

---
 .github/codecov.yml          |  2 +-
 .github/workflows/builds.yml | 76 ++++++++++++++++++++++++++++++++++++
 qa/run-ossfuzz-corpus.sh     | 13 ++++++
 src/tests/fuzz/onefile.c     |  8 +---
 4 files changed, 91 insertions(+), 8 deletions(-)
 create mode 100755 qa/run-ossfuzz-corpus.sh

diff --git a/.github/codecov.yml b/.github/codecov.yml
index d32a46f1a708..e7b195e24b9e 100644
--- a/.github/codecov.yml
+++ b/.github/codecov.yml
@@ -1,7 +1,7 @@
 codecov:
   require_ci_to_pass: yes
   notify:
-    after_n_builds: 2
+    after_n_builds: 3
 
 coverage:
   precision: 2
diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index f8afe83da95e..a861f75b6ce5 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -666,6 +666,82 @@ jobs:
         with:
           flags: unittests
 
+  ubuntu-20-04-cov-fuzz:
+    name: Ubuntu 20.04 (fuzz corpus coverage)
+    runs-on: ubuntu-latest
+    container: ubuntu:20.04
+    needs: [prepare-deps, prepare-cbindgen]
+    steps:
+      - name: Install dependencies
+        run: |
+          apt update
+          apt -y install \
+                libpcre3 \
+                libpcre3-dev \
+                build-essential \
+                autoconf \
+                automake \
+                llvm-10 \
+                clang-10 \
+                git \
+                jq \
+                libc++-dev \
+                libc++abi-dev \
+                libtool \
+                libpcap-dev \
+                libnet1-dev \
+                libyaml-0-2 \
+                libyaml-dev \
+                libcap-ng-dev \
+                libcap-ng0 \
+                libmagic-dev \
+                libnetfilter-queue-dev \
+                libnetfilter-queue1 \
+                libnfnetlink-dev \
+                libnfnetlink0 \
+                libhiredis-dev \
+                liblua5.1-dev \
+                libjansson-dev \
+                libevent-dev \
+                libevent-pthreads-2.1-7 \
+                libjansson-dev \
+                libpython2.7 \
+                make \
+                parallel \
+                python3-yaml \
+                rustc \
+                software-properties-common \
+                zlib1g \
+                zlib1g-dev \
+                exuberant-ctags \
+                unzip \
+                curl \
+                wget
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: prep
+          path: prep
+      - run: tar xf prep/libhtp.tar.gz
+      - name: Setup cbindgen
+        run: |
+          mkdir -p $HOME/.cargo/bin
+          cp prep/cbindgen $HOME/.cargo/bin
+          chmod 755 $HOME/.cargo/bin/cbindgen
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+      - run: ./autogen.sh
+      - run: LIB_FUZZING_ENGINE="fail_to_onefile_driver" CC=clang-10 CXX=clang++-10 CFLAGS="-fprofile-arcs -ftest-coverage -g -fno-strict-aliasing -fsanitize=address -fno-omit-frame-pointer -fPIC -Wno-unused-parameter -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" CXXFLAGS="-fprofile-arcs -ftest-coverage -g -fno-strict-aliasing -fsanitize=address -fno-omit-frame-pointer -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1 -stdlib=libc++" ac_cv_func_malloc_0_nonnull=yes ac_cv_func_realloc_0_nonnull=yes ./configure --with-gnu-ld --enable-fuzztargets --disable-shared --enable-gccprotect
+      - run: make -j2
+      - run: ./qa/run-ossfuzz-corpus.sh
+      - name: Gcov
+        run: |
+          cd src
+          llvm-cov-10 gcov -p *.c
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          flags: fuzzcorpus
+
   ubuntu-20-04-ndebug:
     name: Ubuntu 20.04 (-DNDEBUG)
     runs-on: ubuntu-latest
diff --git a/qa/run-ossfuzz-corpus.sh b/qa/run-ossfuzz-corpus.sh
new file mode 100755
index 000000000000..c2ac05b3b695
--- /dev/null
+++ b/qa/run-ossfuzz-corpus.sh
@@ -0,0 +1,13 @@
+#/bin/sh
+ls src/fuzz_* | while read ftarget
+do
+    target=$(basename $ftarget)
+    echo "target $target"
+    #download public corpus
+    rm -f public.zip
+    wget --quiet "https://storage.googleapis.com/suricata-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/suricata_$target/public.zip"
+    rm -rf corpus_$target
+    unzip -q public.zip -d corpus_$target
+    #run target on corpus.
+    ./src/$target corpus_$target
+done
diff --git a/src/tests/fuzz/onefile.c b/src/tests/fuzz/onefile.c
index f59a12c3260d..f2dd471b8ea2 100644
--- a/src/tests/fuzz/onefile.c
+++ b/src/tests/fuzz/onefile.c
@@ -1,10 +1,4 @@
-#define _DEFAULT_SOURCE 1 // for DT_REG
-#include <stdint.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <dirent.h>
-#include <unistd.h>
-#include "autoconf.h"
+#include "suricata-common.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
 

From dd4a66aedffdbe3c61776d06c1c71fcf46091dce Mon Sep 17 00:00:00 2001
From: Jason Ish <jason.ish@oisf.net>
Date: Thu, 26 Nov 2020 16:20:36 -0600
Subject: [PATCH 17/19] .gitignore: ignore .a files

Ignore .a library files as we now have one created in src/
as part of adding a Suricata library.
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 4bdfb76e2f60..ca98ed119e66 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 *.o
 *.lo
+*.a
 *.in
 *.[ch]e
 *.log

From 9e83ac729d962d2d2e449cdff151e8e3898dd0f4 Mon Sep 17 00:00:00 2001
From: Jason Ish <jason.ish@oisf.net>
Date: Thu, 26 Nov 2020 16:21:46 -0600
Subject: [PATCH 18/19] rust/ffi: provide AppLayerRegisterParser in context

AppLayerRegisterParser was creating a link error when attempting
to use a convenience library for the Suricata C code, then linking
the library of C code with the library of Rust code into a final
Suricata executable, or use with fuzz targets.

By moving AppLayerRegisterParser to the context structure and
calling it like a callback the circular reference is removed
allowing the convenience libraries to work again.

This is also a stepping block to proving a Suricata library
as a single .a or .so file.
---
 rust/src/applayer.rs | 7 ++++++-
 rust/src/core.rs     | 2 ++
 src/rust-context.h   | 4 ++++
 src/suricata.c       | 3 +++
 4 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs
index 476ea1cb361f..684d0f27596a 100644
--- a/rust/src/applayer.rs
+++ b/rust/src/applayer.rs
@@ -22,6 +22,7 @@ use crate::core::{DetectEngineState,Flow,AppLayerEventType,AppLayerDecoderEvents
 use crate::filecontainer::FileContainer;
 use crate::applayer;
 use std::os::raw::{c_void,c_char,c_int};
+use crate::core::SC;
 
 #[repr(C)]
 #[derive(Debug,PartialEq)]
@@ -289,7 +290,11 @@ pub type TruncateFn = unsafe extern "C" fn (*mut c_void, u8);
 // Defined in app-layer-register.h
 extern {
     pub fn AppLayerRegisterProtocolDetection(parser: *const RustParser, enable_default: c_int) -> AppProto;
-    pub fn AppLayerRegisterParser(parser: *const RustParser, alproto: AppProto) -> c_int;
+}
+
+#[allow(non_snake_case)]
+pub unsafe fn AppLayerRegisterParser(parser: *const RustParser, alproto: AppProto) -> c_int {
+    (SC.unwrap().AppLayerRegisterParser)(parser, alproto)
 }
 
 // Defined in app-layer-detect-proto.h
diff --git a/rust/src/core.rs b/rust/src/core.rs
index db929303217a..69ed658a2347 100644
--- a/rust/src/core.rs
+++ b/rust/src/core.rs
@@ -141,6 +141,8 @@ pub struct SuricataContext {
     pub FileContainerRecycle: SCFileContainerRecycle,
     pub FilePrune: SCFilePrune,
     pub FileSetTx: SCFileSetTx,
+
+    pub AppLayerRegisterParser: extern fn(parser: *const crate::applayer::RustParser, alproto: AppProto) -> std::os::raw::c_int,
 }
 
 #[allow(non_snake_case)]
diff --git a/src/rust-context.h b/src/rust-context.h
index 8efd2678a3d4..f0005314f7cf 100644
--- a/src/rust-context.h
+++ b/src/rust-context.h
@@ -25,6 +25,8 @@
 #include "app-layer-snmp.h" //SNMPState, SNMPTransaction
 #include "app-layer-tftp.h" //TFTPState, TFTPTransaction
 
+struct AppLayerParser;
+
 typedef struct SuricataContext_ {
     SCError (*SCLogMessage)(const SCLogLevel, const char *, const unsigned int,
             const char *, const SCError, const char *message);
@@ -46,6 +48,8 @@ typedef struct SuricataContext_ {
     void (*FilePrune)(FileContainer *ffc);
     void (*FileSetTx)(FileContainer *, uint64_t);
 
+    int (*AppLayerRegisterParser)(const struct AppLayerParser *p, AppProto alproto);
+
 } SuricataContext;
 
 extern SuricataContext suricata_context;
diff --git a/src/suricata.c b/src/suricata.c
index 5b80e974a0c6..f386dc4159c8 100644
--- a/src/suricata.c
+++ b/src/suricata.c
@@ -113,6 +113,7 @@
 
 #include "app-layer.h"
 #include "app-layer-parser.h"
+#include "app-layer-register.h"
 #include "app-layer-htp.h"
 #include "app-layer-ssl.h"
 #include "app-layer-ssh.h"
@@ -2661,6 +2662,8 @@ int InitGlobal(void) {
     suricata_context.FilePrune = FilePrune;
     suricata_context.FileSetTx = FileContainerSetTx;
 
+    suricata_context.AppLayerRegisterParser = AppLayerRegisterParser;
+
     rs_init(&suricata_context);
 
     SC_ATOMIC_INIT(engine_stage);

From e75dee8b4f99d76322cd40f2d06bd514e9fb69ce Mon Sep 17 00:00:00 2001
From: Jason Ish <jason.ish@oisf.net>
Date: Wed, 10 Feb 2021 09:01:49 -0600
Subject: [PATCH 19/19] github-ci: run suricata-verify on centos-7 build

---
 .github/workflows/builds.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index a861f75b6ce5..0bd6457faa70 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -250,7 +250,7 @@ jobs:
     name: CentOS 7
     runs-on: ubuntu-latest
     container: centos:7
-    needs: centos-8
+    needs: [prepare-deps, centos-8]
     steps:
       - name: Install system dependencies
         run: |
@@ -280,6 +280,7 @@ jobs:
                 nss-devel \
                 pcre-devel \
                 pkgconfig \
+                python36-PyYAML \
                 rust \
                 sudo \
                 which \
@@ -302,6 +303,12 @@ jobs:
       - run: make distcheck
       - run: make clean
       - run: make -j2
+      - uses: actions/download-artifact@v2
+        with:
+          name: prep
+          path: prep
+      - run: tar xf prep/suricata-verify.tar.gz
+      - run: python3 ./suricata-verify/run.py
 
   fedora-33:
     name: Fedora 33 (debug, clang, asan, wshadow, rust-strict)