Next/20210212/v6

.github/codecov.yml

@@ -1,7 +1,7 @@
 codecov:
   require_ci_to_pass: yes
   notify:
-    after_n_builds: 2
+    after_n_builds: 3
 
 coverage:
   precision: 2

.github/workflows/builds.yml

@@ -250,7 +250,7 @@ jobs:
     name: CentOS 7
     runs-on: ubuntu-latest
     container: centos:7
-    needs: centos-8
+    needs: [prepare-deps, centos-8]
     steps:
       - name: Install system dependencies
         run: |
@@ -280,6 +280,7 @@ jobs:
                 nss-devel \
                 pcre-devel \
                 pkgconfig \
+                python36-PyYAML \
                 rust \
                 sudo \
                 which \
@@ -302,6 +303,12 @@ jobs:
       - run: make distcheck
       - run: make clean
       - run: make -j2
+      - uses: actions/download-artifact@v2
+        with:
+          name: prep
+          path: prep
+      - run: tar xf prep/suricata-verify.tar.gz
+      - run: python3 ./suricata-verify/run.py
 
   fedora-33:
     name: Fedora 33 (debug, clang, asan, wshadow, rust-strict)
@@ -666,6 +673,82 @@ jobs:
         with:
           flags: unittests
 
+  ubuntu-20-04-cov-fuzz:
+    name: Ubuntu 20.04 (fuzz corpus coverage)
+    runs-on: ubuntu-latest
+    container: ubuntu:20.04
+    needs: [prepare-deps, prepare-cbindgen]
+    steps:
+      - name: Install dependencies
+        run: |
+          apt update
+          apt -y install \
+                libpcre3 \
+                libpcre3-dev \
+                build-essential \
+                autoconf \
+                automake \
+                llvm-10 \
+                clang-10 \
+                git \
+                jq \
+                libc++-dev \
+                libc++abi-dev \
+                libtool \
+                libpcap-dev \
+                libnet1-dev \
+                libyaml-0-2 \
+                libyaml-dev \
+                libcap-ng-dev \
+                libcap-ng0 \
+                libmagic-dev \
+                libnetfilter-queue-dev \
+                libnetfilter-queue1 \
+                libnfnetlink-dev \
+                libnfnetlink0 \
+                libhiredis-dev \
+                liblua5.1-dev \
+                libjansson-dev \
+                libevent-dev \
+                libevent-pthreads-2.1-7 \
+                libjansson-dev \
+                libpython2.7 \
+                make \
+                parallel \
+                python3-yaml \
+                rustc \
+                software-properties-common \
+                zlib1g \
+                zlib1g-dev \
+                exuberant-ctags \
+                unzip \
+                curl \
+                wget
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: prep
+          path: prep
+      - run: tar xf prep/libhtp.tar.gz
+      - name: Setup cbindgen
+        run: |
+          mkdir -p $HOME/.cargo/bin
+          cp prep/cbindgen $HOME/.cargo/bin
+          chmod 755 $HOME/.cargo/bin/cbindgen
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+      - run: ./autogen.sh
+      - run: LIB_FUZZING_ENGINE="fail_to_onefile_driver" CC=clang-10 CXX=clang++-10 CFLAGS="-fprofile-arcs -ftest-coverage -g -fno-strict-aliasing -fsanitize=address -fno-omit-frame-pointer -fPIC -Wno-unused-parameter -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" CXXFLAGS="-fprofile-arcs -ftest-coverage -g -fno-strict-aliasing -fsanitize=address -fno-omit-frame-pointer -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1 -stdlib=libc++" ac_cv_func_malloc_0_nonnull=yes ac_cv_func_realloc_0_nonnull=yes ./configure --with-gnu-ld --enable-fuzztargets --disable-shared --enable-gccprotect
+      - run: make -j2
+      - run: ./qa/run-ossfuzz-corpus.sh
+      - name: Gcov
+        run: |
+          cd src
+          llvm-cov-10 gcov -p *.c
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          flags: fuzzcorpus
+
   ubuntu-20-04-ndebug:
     name: Ubuntu 20.04 (-DNDEBUG)
     runs-on: ubuntu-latest

.gitignore

@@ -1,5 +1,6 @@
 *.o
 *.lo
+*.a
 *.in
 *.[ch]e
 *.log

qa/run-ossfuzz-corpus.sh

@@ -0,0 +1,13 @@
+#/bin/sh
+ls src/fuzz_* | while read ftarget
+do
+    target=$(basename $ftarget)
+    echo "target $target"
+    #download public corpus
+    rm -f public.zip
+    wget --quiet "https://storage.googleapis.com/suricata-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/suricata_$target/public.zip"
+    rm -rf corpus_$target
+    unzip -q public.zip -d corpus_$target
+    #run target on corpus.
+    ./src/$target corpus_$target
+done

rust/src/applayer.rs

@@ -22,6 +22,7 @@ use crate::core::{DetectEngineState,Flow,AppLayerEventType,AppLayerDecoderEvents
 use crate::filecontainer::FileContainer;
 use crate::applayer;
 use std::os::raw::{c_void,c_char,c_int};
+use crate::core::SC;
 
 #[repr(C)]
 #[derive(Debug,PartialEq)]
@@ -289,7 +290,11 @@ pub type TruncateFn = unsafe extern "C" fn (*mut c_void, u8);
 // Defined in app-layer-register.h
 extern {
     pub fn AppLayerRegisterProtocolDetection(parser: *const RustParser, enable_default: c_int) -> AppProto;
-    pub fn AppLayerRegisterParser(parser: *const RustParser, alproto: AppProto) -> c_int;
+}
+
+#[allow(non_snake_case)]
+pub unsafe fn AppLayerRegisterParser(parser: *const RustParser, alproto: AppProto) -> c_int {
+    (SC.unwrap().AppLayerRegisterParser)(parser, alproto)
 }
 
 // Defined in app-layer-detect-proto.h

rust/src/applayertemplate/template.rs

@@ -177,6 +177,17 @@ impl TemplateState {
             return AppLayerResult::ok();
         }
 
+        if self.response_gap {
+            if probe(input).is_err() {
+                // The parser now needs to decide what to do as we are not in sync.
+                // For this template, we'll just try again next time.
+                return AppLayerResult::ok();
+            }
+
+            // It looks like we're in sync with a message header, clear gap
+            // state and keep parsing.
+            self.response_gap = false;
+        }
         let mut start = input;
         while start.len() > 0 {
             match parser::parse_message(start) {

rust/src/core.rs

@@ -141,6 +141,8 @@ pub struct SuricataContext {
     pub FileContainerRecycle: SCFileContainerRecycle,
     pub FilePrune: SCFilePrune,
     pub FileSetTx: SCFileSetTx,
+
+    pub AppLayerRegisterParser: extern fn(parser: *const crate::applayer::RustParser, alproto: AppProto) -> std::os::raw::c_int,
 }
 
 #[allow(non_snake_case)]

src/app-layer-parser.c

@@ -922,6 +922,7 @@ void AppLayerParserTransactionsCleanup(Flow *f)
         if (ires.tx_ptr == NULL)
             break;
 
+        bool tx_skipped = false;
         void *tx = ires.tx_ptr;
         i = ires.tx_id; // actual tx id for the tx the IterFunc returned
 
@@ -950,7 +951,7 @@ void AppLayerParserTransactionsCleanup(Flow *f)
                 if (!(detect_flags_ts & APP_LAYER_TX_INSPECTED_FLAG)) {
                     SCLogDebug("%p/%"PRIu64" skipping: TS inspect not done: ts:%"PRIx64,
                             tx, i, detect_flags_ts);
-                    skipped = true;
+                    tx_skipped = skipped = true;
                 } else {
                     inspected = true;
                 }
@@ -960,7 +961,7 @@ void AppLayerParserTransactionsCleanup(Flow *f)
                 if (!(detect_flags_tc & APP_LAYER_TX_INSPECTED_FLAG)) {
                     SCLogDebug("%p/%"PRIu64" skipping: TC inspect not done: tc:%"PRIx64,
                             tx, i, detect_flags_tc);
-                    skipped = true;
+                    tx_skipped = skipped = true;
                 } else {
                     inspected = true;
                 }
@@ -969,15 +970,17 @@ void AppLayerParserTransactionsCleanup(Flow *f)
 
         // If not a unidirectional transaction both sides are required to have
         // been inspected.
-        if (!is_unidir && skipped) {
+        if (!is_unidir && tx_skipped) {
+            SCLogDebug("%p/%" PRIu64 " !is_unidir && tx_skipped", tx, i);
             goto next;
         }
 
         // If this is a unidirectional transaction require only one side to be
         // inspected, which the inspected flag tells us. This is also guarded
         // with skip to limit this check to transactions that actually had the
         // tx inspected flag checked.
-        if (is_unidir && skipped && !inspected) {
+        if (is_unidir && tx_skipped && !inspected) {
+            SCLogDebug("%p/%" PRIu64 " is_unidir && tx_skipped && !inspected", tx, i);
             goto next;
         }
 

src/detect-bytemath.c

@@ -262,7 +262,7 @@ static DetectByteMathData *DetectByteMathParse(DetectEngineCtx *de_ctx, const ch
 #undef MAX_SUBSTRINGS
 #define MAX_SUBSTRINGS 100
     int ov[MAX_SUBSTRINGS];
-    char tmp_str[128];
+    char tmp_str[128] = "";
 
     ret = DetectParsePcreExec(&parse_regex, arg, 0, 0, ov, MAX_SUBSTRINGS);
     if (ret < MIN_GROUP || ret > MAX_GROUP) {

src/detect-dce-stub-data.c

@@ -79,7 +79,7 @@ static InspectionBuffer *GetSMBData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         SCLogDebug("have data!");
 
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;
@@ -105,7 +105,7 @@ static InspectionBuffer *GetDCEData(DetectEngineThreadCtx *det_ctx,
         } else {
             buffer->flags |= DETECT_CI_FLAGS_DCE_BE;
         }
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;

src/detect-dnp3.c

@@ -170,7 +170,7 @@ static InspectionBuffer *GetDNP3Data(DetectEngineThreadCtx *det_ctx,
             return NULL;
 
         SCLogDebug("tx %p data %p data_len %u", tx, data, data_len);
-        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
         InspectionBufferApplyTransforms(buffer, transforms);
     }
     return buffer;

src/detect-dns-query.c

@@ -91,7 +91,7 @@ static InspectionBuffer *DnsQueryGetData(DetectEngineThreadCtx *det_ctx,
                 &data, &data_len) == 0) {
         return NULL;
     }
-    InspectionBufferSetup(buffer, data, data_len);
+    InspectionBufferSetup(det_ctx, list_id, buffer, data, data_len);
     InspectionBufferApplyTransforms(buffer, transforms);
 
     SCReturnPtr(buffer, "InspectionBuffer");

src/detect-engine-register.c

@@ -351,7 +351,7 @@ static void SigMultilinePrint(int i, const char *prefix)
     printf("\n");
 }
 
-void SigTableList(const char *keyword)
+int SigTableList(const char *keyword)
 {
     size_t size = sizeof(sigmatch_table) / sizeof(SigTableElmt);
     size_t i;
@@ -414,14 +414,16 @@ void SigTableList(const char *keyword)
                 printf("= %s =\n", sigmatch_table[i].name);
                 if (sigmatch_table[i].flags & SIGMATCH_NOT_BUILT) {
                     printf("Not built-in\n");
-                    return;
+                    return TM_ECODE_FAILED;
                 }
                 SigMultilinePrint(i, "");
-                return;
+                return TM_ECODE_DONE;
             }
         }
+        printf("Non existing keyword\n");
+        return TM_ECODE_FAILED;
     }
-    return;
+    return TM_ECODE_DONE;
 }
 
 void SigTableSetup(void)

src/detect-engine-register.h

@@ -303,7 +303,7 @@ enum DetectKeywordId {
     DETECT_TBLSIZE,
 };
 
-void SigTableList(const char *keyword);
+int SigTableList(const char *keyword);
 void SigTableSetup(void);
 void SigTableRegisterTests(void);
 

src/detect-engine-state.c

@@ -123,43 +123,37 @@ static int DeStateSearchState(DetectEngineState *state, uint8_t direction, SigIn
 static void DeStateSignatureAppend(DetectEngineState *state,
         const Signature *s, uint32_t inspect_flags, uint8_t direction)
 {
-    int jump = 0;
-    int i = 0;
+    SCEnter();
+
     DetectEngineStateDirection *dir_state = &state->dir_state[direction & STREAM_TOSERVER ? 0 : 1];
 
 #ifdef DEBUG_VALIDATION
     BUG_ON(DeStateSearchState(state, direction, s->num));
 #endif
-    DeStateStore *store = dir_state->head;
+    DeStateStore *store = dir_state->tail;
 
     if (store == NULL) {
         store = DeStateStoreAlloc();
-        if (store != NULL) {
-            dir_state->head = store;
-            dir_state->tail = store;
-        }
+        dir_state->head = store;
+        dir_state->tail = store;
     } else {
-        jump = dir_state->cnt / DE_STATE_CHUNK_SIZE;
-        for (i = 0; i < jump; i++) {
-            store = store->next;
-        }
-        if (store == NULL) {
+        SCLogDebug("dir_state->cnt %u mod chunksize %u", dir_state->cnt, dir_state->cnt % DE_STATE_CHUNK_SIZE);
+        if (dir_state->cnt && dir_state->cnt % DE_STATE_CHUNK_SIZE == 0) {
             store = DeStateStoreAlloc();
             if (store != NULL) {
                 dir_state->tail->next = store;
                 dir_state->tail = store;
             }
         }
     }
-
     if (store == NULL)
-        return;
+        SCReturn;
 
     SigIntId idx = dir_state->cnt++ % DE_STATE_CHUNK_SIZE;
     store->store[idx].sid = s->num;
     store->store[idx].flags = inspect_flags;
 
-    return;
+    SCReturn;
 }
 
 DetectEngineState *DetectEngineStateAlloc(void)