diff --git a/src/detect-byte-extract.c b/src/detect-byte-extract.c index e9241488158f..d590183f9c77 100644 --- a/src/detect-byte-extract.c +++ b/src/detect-byte-extract.c @@ -1945,9 +1945,8 @@ static int DetectByteExtractTest43(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "three", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_OFFSET_VAR | - DETECT_CONTENT_OFFSET) || - cd->offset != bed->local_id) { + cd->flags != (DETECT_CONTENT_OFFSET_VAR | DETECT_CONTENT_OFFSET | DETECT_CONTENT_MPM) || + cd->offset != bed->local_id) { printf("three failed\n"); result = 0; goto end; @@ -2049,9 +2048,8 @@ static int DetectByteExtractTest44(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "four", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_OFFSET_VAR | - DETECT_CONTENT_OFFSET) || - cd->offset != bed1->local_id) { + cd->flags != (DETECT_CONTENT_OFFSET_VAR | DETECT_CONTENT_OFFSET | DETECT_CONTENT_MPM) || + cd->offset != bed1->local_id) { printf("four failed\n"); result = 0; goto end; @@ -2158,10 +2156,8 @@ static int DetectByteExtractTest45(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "three", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_DEPTH_VAR | - DETECT_CONTENT_DEPTH) || - cd->depth != bed->local_id || - cd->offset != 0) { + cd->flags != (DETECT_CONTENT_DEPTH_VAR | DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) || + cd->depth != bed->local_id || cd->offset != 0) { printf("three failed\n"); result = 0; goto end; @@ -2263,9 +2259,8 @@ static int DetectByteExtractTest46(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "four", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_DEPTH_VAR | - DETECT_CONTENT_DEPTH) || - cd->depth != bed1->local_id) { + cd->flags != (DETECT_CONTENT_DEPTH_VAR | DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) || + cd->depth != bed1->local_id) { printf("four failed\n"); result = 0; goto end; @@ -2372,11 +2367,9 @@ static int DetectByteExtractTest47(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "three", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_DISTANCE_VAR | - DETECT_CONTENT_DISTANCE) || - cd->distance != bed->local_id || - cd->offset != 0 || - cd->depth != 0) { + cd->flags != + (DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM) || + cd->distance != bed->local_id || cd->offset != 0 || cd->depth != 0) { printf("three failed\n"); result = 0; goto end; @@ -2478,12 +2471,9 @@ static int DetectByteExtractTest48(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "four", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_DISTANCE_VAR | - DETECT_CONTENT_DISTANCE | - DETECT_CONTENT_DISTANCE_NEXT) || - cd->distance != bed1->local_id || - cd->depth != 0 || - cd->offset != 0) { + cd->flags != (DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_DISTANCE | + DETECT_CONTENT_DISTANCE_NEXT | DETECT_CONTENT_MPM) || + cd->distance != bed1->local_id || cd->depth != 0 || cd->offset != 0) { printf("four failed\n"); result = 0; goto end; @@ -2592,12 +2582,8 @@ static int DetectByteExtractTest49(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "three", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_WITHIN_VAR | - DETECT_CONTENT_WITHIN) || - cd->within != bed->local_id || - cd->offset != 0 || - cd->depth != 0 || - cd->distance != 0) { + cd->flags != (DETECT_CONTENT_WITHIN_VAR | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) || + cd->within != bed->local_id || cd->offset != 0 || cd->depth != 0 || cd->distance != 0) { printf("three failed\n"); result = 0; goto end; @@ -2699,13 +2685,10 @@ static int DetectByteExtractTest50(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "four", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_WITHIN_VAR | - DETECT_CONTENT_WITHIN| - DETECT_CONTENT_WITHIN_NEXT) || - cd->within != bed1->local_id || - cd->depth != 0 || - cd->offset != 0 || - cd->distance != 0) { + cd->flags != (DETECT_CONTENT_WITHIN_VAR | DETECT_CONTENT_WITHIN | + DETECT_CONTENT_WITHIN_NEXT | DETECT_CONTENT_MPM) || + cd->within != bed1->local_id || cd->depth != 0 || cd->offset != 0 || + cd->distance != 0) { printf("four failed\n"); result = 0; goto end; @@ -2970,7 +2953,7 @@ static int DetectByteExtractTest53(void) SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_PMATCH]; FAIL_IF(sm->type != DETECT_CONTENT); DetectContentData *cd = (DetectContentData *)sm->ctx; - FAIL_IF(cd->flags != 0); + FAIL_IF(cd->flags != (DETECT_CONTENT_MPM | DETECT_CONTENT_NO_DOUBLE_INSPECTION_REQUIRED)); sm = sm->next; FAIL_IF_NULL(sm); @@ -3092,11 +3075,8 @@ static int DetectByteExtractTest54(void) goto end; } bjd = (DetectBytejumpData *)sm->ctx; - if (bjd->flags != DETECT_BYTEJUMP_OFFSET_VAR || bjd->offset != 1) { - printf("four failed\n"); - result = 0; - goto end; - } + FAIL_IF(bjd->flags != DETECT_BYTEJUMP_OFFSET_VAR); + FAIL_IF(bjd->offset != 1); if (sm->next != NULL) goto end; @@ -3197,12 +3177,10 @@ static int DetectByteExtractTest55(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "four", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_DISTANCE_VAR | - DETECT_CONTENT_WITHIN_VAR | - DETECT_CONTENT_DISTANCE | - DETECT_CONTENT_WITHIN) || - cd->within != bed1->local_id || - cd->distance != bed2->local_id) { + cd->flags != + (DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_WITHIN_VAR | + DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) || + cd->within != bed1->local_id || cd->distance != bed2->local_id) { printf("four failed: "); goto end; } @@ -3829,8 +3807,8 @@ static int DetectByteExtractTest60(void) goto end; } cd = (DetectContentData *)sm->ctx; - if (cd->flags != DETECT_CONTENT_RELATIVE_NEXT || - strncmp((char *)cd->content, "three", cd->content_len) != 0) { + if (cd->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM) || + strncmp((char *)cd->content, "three", cd->content_len) != 0) { printf("one failed\n"); result = 0; goto end; @@ -3950,8 +3928,8 @@ static int DetectByteExtractTest61(void) goto end; } cd = (DetectContentData *)sm->ctx; - if (cd->flags != DETECT_CONTENT_RELATIVE_NEXT || - strncmp((char *)cd->content, "three", cd->content_len) != 0) { + if (cd->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM) || + strncmp((char *)cd->content, "three", cd->content_len) != 0) { printf("one failed\n"); result = 0; goto end; diff --git a/src/detect-content.c b/src/detect-content.c index 9625e7426d45..849094ca223e 100644 --- a/src/detect-content.c +++ b/src/detect-content.c @@ -1523,7 +1523,7 @@ static int DetectContentParseTest19(void) FAIL_IF_NOT(s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL); DetectContentData *data = (DetectContentData *)sm->ctx; - FAIL_IF_NOT(data->flags == DETECT_CONTENT_DISTANCE); + FAIL_IF_NOT(data->flags == (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM)); s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index c85f53d5051d..037980e5e825 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -668,7 +668,7 @@ static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigG any5_cnt++; } - prefilter_cnt += (s->init_data->prefilter_sm != 0); + prefilter_cnt += (s->init_data->prefilter_sm != NULL); if (s->init_data->mpm_sm == NULL) { nonmpm_cnt++; @@ -1703,7 +1703,6 @@ void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s) } } -extern int g_skip_prefilter; /** * \brief Preprocess signature, classify ip-only, etc, build sig array * @@ -1787,39 +1786,6 @@ int SigPrepareStage1(DetectEngineCtx *de_ctx) RuleSetScore(s); - /* if keyword engines are enabled in the config, handle them here */ - if (!g_skip_prefilter && de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO && - !(s->flags & SIG_FLAG_PREFILTER)) { - int prefilter_list = DETECT_TBLSIZE; - - // TODO buffers? - - /* get the keyword supporting prefilter with the lowest type */ - for (int i = 0; i < DETECT_SM_LIST_MAX; i++) { - for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) { - if (sigmatch_table[sm->type].SupportsPrefilter != NULL) { - if (sigmatch_table[sm->type].SupportsPrefilter(s)) { - prefilter_list = MIN(prefilter_list, sm->type); - } - } - } - } - - /* apply that keyword as prefilter */ - if (prefilter_list != DETECT_TBLSIZE) { - for (int i = 0; i < DETECT_SM_LIST_MAX; i++) { - for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) { - if (sm->type == prefilter_list) { - s->init_data->prefilter_sm = sm; - s->flags |= SIG_FLAG_PREFILTER; - SCLogConfig("sid %u: prefilter is on \"%s\"", s->id, sigmatch_table[sm->type].name); - break; - } - } - } - } - } - /* run buffer type callbacks if any */ for (int x = 0; x < DETECT_SM_LIST_MAX; x++) { if (s->init_data->smlists[x]) diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c index 950812a187c9..30e931165cbc 100644 --- a/src/detect-engine-loader.c +++ b/src/detect-engine-loader.c @@ -167,7 +167,6 @@ static int DetectLoadSigFile( sig = DetectEngineAppendSig(de_ctx, line); if (sig != NULL) { if (rule_engine_analysis_set || fp_engine_analysis_set) { - RetrieveFPForSig(de_ctx, sig); if (fp_engine_analysis_set) { EngineAnalysisFP(de_ctx, sig, line); } diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 5e8687e34686..7b12786662c6 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -2415,12 +2415,7 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx) { uint32_t cnt = 0; for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) { - if (s->flags & SIG_FLAG_PREFILTER) - continue; - - RetrieveFPForSig(de_ctx, s); if (s->init_data->mpm_sm != NULL) { - s->flags |= SIG_FLAG_PREFILTER; cnt++; } } diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c index feff1251e4e2..9d41f00404a4 100644 --- a/src/detect-engine-prefilter.c +++ b/src/detect-engine-prefilter.c @@ -144,6 +144,35 @@ void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, } } +/** \brief invoke post-rule match "prefilter" engines + * + * Invoke prefilter engines that depend on a rule match to run. + * e.g. the flowbits:set prefilter that adds sids that depend on + * a flowbit "set" to the match array. + */ +void PrefilterPostRuleMatch( + DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, Flow *f) +{ + SCLogDebug("post-rule-match engines %p", sgh->post_rule_match_engines); + if (sgh->post_rule_match_engines) { + PrefilterEngine *engine = sgh->post_rule_match_engines; + do { + SCLogDebug("running post-rule-match engine"); + PREFILTER_PROFILING_START(det_ctx); + engine->cb.PrefilterPostRule(det_ctx, engine->pectx, p, f); + PREFILTER_PROFILING_END(det_ctx, engine->gid); + + if (engine->is_last) + break; + engine++; + } while (1); + + if (det_ctx->pmq.rule_id_array_cnt > 1) { + QuickSortSigIntId(det_ctx->pmq.rule_id_array, det_ctx->pmq.rule_id_array_cnt); + } + } +} + void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const uint8_t flags, const SignatureMask mask) { @@ -342,6 +371,39 @@ int PrefilterAppendFrameEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, return 0; } +int PrefilterAppendPostRuleEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, + void (*PrefilterPostRuleFunc)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f), + void *pectx, void (*FreeFunc)(void *pectx), const char *name) +{ + if (sgh == NULL || PrefilterPostRuleFunc == NULL || pectx == NULL) + return -1; + + PrefilterEngineList *e = SCMallocAligned(sizeof(*e), CLS); + if (e == NULL) + return -1; + memset(e, 0x00, sizeof(*e)); + e->PrefilterPostRule = PrefilterPostRuleFunc; + e->pectx = pectx; + e->Free = FreeFunc; + + if (sgh->init->post_rule_match_engines == NULL) { + sgh->init->post_rule_match_engines = e; + } else { + PrefilterEngineList *t = sgh->init->post_rule_match_engines; + while (t->next != NULL) { + t = t->next; + } + + t->next = e; + e->id = t->id + 1; + } + + e->name = name; + e->gid = PrefilterStoreGetId(de_ctx, e->name, e->Free); + return 0; +} + static void PrefilterFreeEngineList(PrefilterEngineList *e) { if (e->Free && e->pectx) { @@ -396,6 +458,10 @@ void PrefilterCleanupRuleGroup(const DetectEngineCtx *de_ctx, SigGroupHead *sgh) PrefilterFreeEngines(de_ctx, sgh->frame_engines); sgh->frame_engines = NULL; } + if (sgh->post_rule_match_engines) { + PrefilterFreeEngines(de_ctx, sgh->post_rule_match_engines); + sgh->post_rule_match_engines = NULL; + } } static int PrefilterSetupRuleGroupSortHelper(const void *a, const void *b) @@ -589,6 +655,30 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) e++; } } + if (sgh->init->post_rule_match_engines != NULL) { + uint32_t cnt = 0; + for (el = sgh->init->post_rule_match_engines; el != NULL; el = el->next) { + cnt++; + } + sgh->post_rule_match_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS); + if (sgh->post_rule_match_engines == NULL) { + return; + } + memset(sgh->post_rule_match_engines, 0x00, (cnt * sizeof(PrefilterEngine))); + + uint16_t local_id = 0; + PrefilterEngine *e = sgh->post_rule_match_engines; + for (el = sgh->init->post_rule_match_engines; el != NULL; el = el->next) { + e->local_id = local_id++; + e->cb.PrefilterPostRule = el->PrefilterPostRule; + e->pectx = el->pectx; + el->pectx = NULL; // e now owns the ctx + e->gid = el->gid; + e->is_last = (el->next == NULL); + e++; + } + SCLogDebug("sgh %p max local_id %u", sgh, local_id); + } } /* hash table for assigning a unique id to each engine type. */ @@ -889,3 +979,31 @@ int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, M } return r; } + +// TODO move to common file +#define QUEUE_STEP 16 + +void PostRuleMatchWorkQueueAppend( + DetectEngineThreadCtx *det_ctx, const Signature *s, const int type, const uint32_t value) +{ + if (det_ctx->post_rule_work_queue.q == NULL) { + det_ctx->post_rule_work_queue.q = + SCCalloc(1, sizeof(PostRuleMatchWorkQueueItem) * QUEUE_STEP); + BUG_ON(det_ctx->post_rule_work_queue.q == NULL); + det_ctx->post_rule_work_queue.size = QUEUE_STEP; + } else if (det_ctx->post_rule_work_queue.len == det_ctx->post_rule_work_queue.size) { + void *ptr = SCRealloc( + det_ctx->post_rule_work_queue.q, (det_ctx->post_rule_work_queue.size + QUEUE_STEP) * + sizeof(PostRuleMatchWorkQueueItem)); + BUG_ON(ptr == NULL); + det_ctx->post_rule_work_queue.q = ptr; + det_ctx->post_rule_work_queue.size += QUEUE_STEP; + } + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].sm_type = type; + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].value = value; +#ifdef DEBUG + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].id = s->num; +#endif + det_ctx->post_rule_work_queue.len++; + SCLogDebug("det_ctx->post_rule_work_queue.len %u", det_ctx->post_rule_work_queue.len); +} diff --git a/src/detect-engine-prefilter.h b/src/detect-engine-prefilter.h index ec58594b9b00..0b27c913826a 100644 --- a/src/detect-engine-prefilter.h +++ b/src/detect-engine-prefilter.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2016 Open Information Security Foundation +/* Copyright (C) 2016-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -52,6 +52,10 @@ void Prefilter(DetectEngineThreadCtx *, const SigGroupHead *, Packet *p, const u int PrefilterAppendEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, PrefilterPktFn PrefilterFunc, SignatureMask mask, void *pectx, void (*FreeFunc)(void *pectx), const char *name); + +void PrefilterPostRuleMatch( + DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, Flow *f); + int PrefilterAppendPayloadEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, PrefilterPktFn PrefilterFunc, void *pectx, void (*FreeFunc)(void *pectx), const char *name); int PrefilterAppendTxEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, @@ -60,6 +64,10 @@ int PrefilterAppendTxEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, int PrefilterAppendFrameEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, PrefilterFrameFn PrefilterFrameFunc, AppProto alproto, uint8_t frame_type, void *pectx, void (*FreeFunc)(void *pectx), const char *name); +int PrefilterAppendPostRuleEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, + void (*PrefilterPostRuleFunc)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f), + void *pectx, void (*FreeFunc)(void *pectx), const char *name); void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, @@ -91,4 +99,7 @@ int PrefilterMultiGenericMpmRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, const DetectBufferMpmRegistry *mpm_reg, int list_id); +void PostRuleMatchWorkQueueAppend( + DetectEngineThreadCtx *det_ctx, const Signature *s, const int type, const uint32_t value); + #endif diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 49982f028b28..29bc3b24011b 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -81,6 +81,7 @@ void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid) PrefilterFreeEnginesList(sghid->pkt_engines); PrefilterFreeEnginesList(sghid->payload_engines); PrefilterFreeEnginesList(sghid->frame_engines); + PrefilterFreeEnginesList(sghid->post_rule_match_engines); SCFree(sghid); } diff --git a/src/detect-engine.c b/src/detect-engine.c index 77c25a1cf3a9..b72b91e6fe51 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2523,6 +2523,9 @@ static DetectEngineCtx *DetectEngineCtxInitReal( goto error; } + // TODO test enable flowbits prefilter + // de_ctx->sm_types_prefilter[DETECT_FLOWBITS] = true; + SigGroupHeadHashInit(de_ctx); MpmStoreInit(de_ctx); DetectParseDupSigHashInit(de_ctx); @@ -3522,6 +3525,9 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) AlertQueueFree(det_ctx); + if (det_ctx->post_rule_work_queue.q) + SCFree(det_ctx->post_rule_work_queue.q); + if (det_ctx->byte_values != NULL) SCFree(det_ctx->byte_values); diff --git a/src/detect-flow.c b/src/detect-flow.c index c268dedb155f..683d53fb70cb 100644 --- a/src/detect-flow.c +++ b/src/detect-flow.c @@ -445,14 +445,18 @@ void DetectFlowFree(DetectEngineCtx *de_ctx, void *ptr) static void PrefilterPacketFlowMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) { + SCEnter(); + const PrefilterPacketHeaderCtx *ctx = pectx; if (!PrefilterPacketHeaderExtraMatch(ctx, p)) return; if (FlowMatch(p->flags, p->flowflags, ctx->v1.u16[0], ctx->v1.u16[1])) { + SCLogDebug("match: adding sids"); PrefilterAddSids(&det_ctx->pmq, ctx->sigs_array, ctx->sigs_cnt); } + SCReturn; } static void diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 40f04d75f305..78e4e4d8d85a 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -42,6 +42,9 @@ #include "detect-engine-mpm.h" #include "detect-engine-state.h" #include "detect-engine-build.h" +#include "detect-engine-prefilter.h" + +#include "tree.h" #include "util-var-name.h" #include "util-unittest.h" @@ -61,6 +64,8 @@ void DetectFlowbitFree (DetectEngineCtx *, void *); #ifdef UNITTESTS void FlowBitsRegisterTests(void); #endif +static bool PrefilterFlowbitIsPrefilterable(const Signature *s); +static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh); void DetectFlowbitsRegister (void) { @@ -76,6 +81,8 @@ void DetectFlowbitsRegister (void) /* this is compatible to ip-only signatures */ sigmatch_table[DETECT_FLOWBITS].flags |= SIGMATCH_IPONLY_COMPAT; + sigmatch_table[DETECT_FLOWBITS].SupportsPrefilter = PrefilterFlowbitIsPrefilterable; + sigmatch_table[DETECT_FLOWBITS].SetupPrefilter = PrefilterSetupFlowbits; DetectSetupParseRegexes(PARSE_REGEX, &parse_regex); } @@ -132,11 +139,10 @@ static int FlowbitOrAddData(DetectEngineCtx *de_ctx, DetectFlowbitsData *cd, cha static int DetectFlowbitMatchToggle (Packet *p, const DetectFlowbitsData *fd) { if (p->flow == NULL) - return 0; - - FlowBitToggle(p->flow,fd->idx); + return -1; - return 1; + bool set = FlowBitToggle(p->flow, fd->idx); + return set == true; } static int DetectFlowbitMatchUnset (Packet *p, const DetectFlowbitsData *fd) @@ -152,11 +158,11 @@ static int DetectFlowbitMatchUnset (Packet *p, const DetectFlowbitsData *fd) static int DetectFlowbitMatchSet (Packet *p, const DetectFlowbitsData *fd) { if (p->flow == NULL) - return 0; - - FlowBitSet(p->flow,fd->idx); + return -1; - return 1; + int r = FlowBitSet(p->flow, fd->idx); + SCLogDebug("set %u", fd->idx); + return r; } static int DetectFlowbitMatchIsset (Packet *p, const DetectFlowbitsData *fd) @@ -206,12 +212,25 @@ int DetectFlowbitMatch (DetectEngineThreadCtx *det_ctx, Packet *p, return DetectFlowbitMatchIsset(p,fd); case DETECT_FLOWBITS_CMD_ISNOTSET: return DetectFlowbitMatchIsnotset(p,fd); - case DETECT_FLOWBITS_CMD_SET: - return DetectFlowbitMatchSet(p,fd); + case DETECT_FLOWBITS_CMD_SET: { + int r = DetectFlowbitMatchSet(p, fd); + /* only on a new "set" invoke the prefilter */ + if (r == 1 && fd->post_rule_match_prefilter) { + SCLogDebug("flowbit set, appending to work queue"); + PostRuleMatchWorkQueueAppend(det_ctx, s, DETECT_FLOWBITS, fd->idx); + } + return (r != -1); + } case DETECT_FLOWBITS_CMD_UNSET: return DetectFlowbitMatchUnset(p,fd); - case DETECT_FLOWBITS_CMD_TOGGLE: - return DetectFlowbitMatchToggle(p,fd); + case DETECT_FLOWBITS_CMD_TOGGLE: { + int r = DetectFlowbitMatchToggle(p, fd); + if (r == 1 && fd->post_rule_match_prefilter) { + SCLogDebug("flowbit set (by toggle), appending to work queue"); + PostRuleMatchWorkQueueAppend(det_ctx, s, DETECT_FLOWBITS, fd->idx); + } + return (r != -1); + } default: SCLogError("unknown cmd %" PRIu32 "", fd->cmd); return 0; @@ -388,6 +407,11 @@ void DetectFlowbitFree (DetectEngineCtx *de_ctx, void *ptr) SCFree(fd); } +struct FBAnalyzer { + struct FBAnalyze *array; + uint32_t array_size; +}; + struct FBAnalyze { uint16_t cnts[DETECT_FLOWBITS_CMD_MAX]; uint16_t state_cnts[DETECT_FLOWBITS_CMD_MAX]; @@ -417,20 +441,143 @@ extern bool rule_engine_analysis_set; static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx, struct FBAnalyze *array, uint32_t elements); +static void FBAnalyzerArrayFree(struct FBAnalyze *array, const uint32_t array_size) +{ + if (array) { + for (uint32_t i = 0; i < array_size; i++) { + SCFree(array[i].set_sids); + SCFree(array[i].unset_sids); + SCFree(array[i].isset_sids); + SCFree(array[i].isnotset_sids); + SCFree(array[i].toggle_sids); + } + SCFree(array); + } +} + +static void FBAnalyzerFree(struct FBAnalyzer *fba) +{ + if (fba && fba->array) { + FBAnalyzerArrayFree(fba->array, fba->array_size); + fba->array = NULL; + fba->array_size = 0; + } +} + +#define MAX_SIDS 8 +static bool CheckExpand(const uint32_t sids_idx, uint32_t **sids, uint32_t *sids_size) +{ + if (sids_idx >= *sids_size) { + const uint32_t old_size = *sids_size; + const uint32_t new_size = MAX(2 * old_size, MAX_SIDS); + + void *ptr = SCRealloc(*sids, new_size * sizeof(uint32_t)); + if (ptr == NULL) + return false; + *sids_size = new_size; + *sids = ptr; + } + return true; +} + +static int DetectFlowbitsAnalyzeSignature(const Signature *s, struct FBAnalyzer *fba) +{ + struct FBAnalyze *array = fba->array; + if (array == NULL) + return -1; + + /* see if the signature uses stateful matching TODO is there not a flag? */ + bool has_state = (s->init_data->buffer_index != 0); + + for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; + sm = sm->next) { + if (sm->type != DETECT_FLOWBITS) + continue; + /* figure out the flowbit action */ + const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + // Handle flowbit array in case of ORed flowbits + for (uint8_t k = 0; k < fb->or_list_size; k++) { + struct FBAnalyze *fa = &array[fb->or_list[k]]; + fa->cnts[fb->cmd]++; + fa->state_cnts[fb->cmd] += has_state; + + if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { + if (!CheckExpand(fa->isset_sids_idx, &fa->isset_sids, &fa->isset_sids_size)) + return -1; + fa->isset_sids[fa->isset_sids_idx] = s->num; + fa->isset_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { + if (!CheckExpand( + fa->isnotset_sids_idx, &fa->isnotset_sids, &fa->isnotset_sids_size)) + return -1; + fa->isnotset_sids[fa->isnotset_sids_idx] = s->num; + fa->isnotset_sids_idx++; + } + } + if (fb->or_list_size == 0) { + struct FBAnalyze *fa = &array[fb->idx]; + fa->cnts[fb->cmd]++; + fa->state_cnts[fb->cmd] += has_state; + + if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { + if (!CheckExpand(fa->isset_sids_idx, &fa->isset_sids, &fa->isset_sids_size)) + return -1; + fa->isset_sids[fa->isset_sids_idx] = s->num; + fa->isset_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { + if (!CheckExpand( + fa->isnotset_sids_idx, &fa->isnotset_sids, &fa->isnotset_sids_size)) + return -1; + fa->isnotset_sids[fa->isnotset_sids_idx] = s->num; + fa->isnotset_sids_idx++; + } + } + } + for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH]; sm != NULL; + sm = sm->next) { + if (sm->type != DETECT_FLOWBITS) + continue; + /* figure out what flowbit action */ + const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + struct FBAnalyze *fa = &array[fb->idx]; + fa->cnts[fb->cmd]++; + fa->state_cnts[fb->cmd] += has_state; + + if (fb->cmd == DETECT_FLOWBITS_CMD_SET) { + if (!CheckExpand(fa->set_sids_idx, &fa->set_sids, &fa->set_sids_size)) + return -1; + fa->set_sids[fa->set_sids_idx] = s->num; + fa->set_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_UNSET) { + if (!CheckExpand(fa->unset_sids_idx, &fa->unset_sids, &fa->unset_sids_size)) + return -1; + fa->unset_sids[fa->unset_sids_idx] = s->num; + fa->unset_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) { + if (!CheckExpand(fa->toggle_sids_idx, &fa->toggle_sids, &fa->toggle_sids_size)) + return -1; + fa->toggle_sids[fa->toggle_sids_idx] = s->num; + fa->toggle_sids_idx++; + } + } + return 0; +} + int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) { const uint32_t max_fb_id = de_ctx->max_fb_id; if (max_fb_id == 0) return 0; -#define MAX_SIDS 8 - uint32_t array_size = max_fb_id + 1; + struct FBAnalyzer fba = { .array = NULL, .array_size = 0 }; + const uint32_t array_size = max_fb_id + 1; struct FBAnalyze *array = SCCalloc(array_size, sizeof(struct FBAnalyze)); - if (array == NULL) { SCLogError("Unable to allocate flowbit analyze array"); return -1; } + fba.array = array; + fba.array_size = array_size; SCLogDebug("fb analyzer array size: %"PRIu64, (uint64_t)(array_size * sizeof(struct FBAnalyze))); @@ -439,143 +586,10 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) for (uint32_t i = 0; i < de_ctx->sig_array_len; i++) { const Signature *s = de_ctx->sig_array[i]; - /* see if the signature uses stateful matching */ - bool has_state = (s->init_data->buffer_index != 0); - - for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) { - switch (sm->type) { - case DETECT_FLOWBITS: - { - /* figure out the flowbit action */ - const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; - // Handle flowbit array in case of ORed flowbits - for (uint8_t k = 0; k < fb->or_list_size; k++) { - array[fb->or_list[k]].cnts[fb->cmd]++; - if (has_state) - array[fb->or_list[k]].state_cnts[fb->cmd]++; - if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { - if (array[fb->or_list[k]].isset_sids_idx >= array[fb->or_list[k]].isset_sids_size) { - uint32_t old_size = array[fb->or_list[k]].isset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->or_list[k]].isset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->or_list[k]].isset_sids_size = new_size; - array[fb->or_list[k]].isset_sids = ptr; - } - - array[fb->or_list[k]].isset_sids[array[fb->or_list[k]].isset_sids_idx] = s->num; - array[fb->or_list[k]].isset_sids_idx++; - } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { - if (array[fb->or_list[k]].isnotset_sids_idx >= array[fb->or_list[k]].isnotset_sids_size) { - uint32_t old_size = array[fb->or_list[k]].isnotset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->or_list[k]].isnotset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->or_list[k]].isnotset_sids_size = new_size; - array[fb->or_list[k]].isnotset_sids = ptr; - } - - array[fb->or_list[k]].isnotset_sids[array[fb->or_list[k]].isnotset_sids_idx] = s->num; - array[fb->or_list[k]].isnotset_sids_idx++; - } - } - if (fb->or_list_size == 0) { - array[fb->idx].cnts[fb->cmd]++; - if (has_state) - array[fb->idx].state_cnts[fb->cmd]++; - if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { - if (array[fb->idx].isset_sids_idx >= array[fb->idx].isset_sids_size) { - uint32_t old_size = array[fb->idx].isset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].isset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].isset_sids_size = new_size; - array[fb->idx].isset_sids = ptr; - } - - array[fb->idx].isset_sids[array[fb->idx].isset_sids_idx] = s->num; - array[fb->idx].isset_sids_idx++; - } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { - if (array[fb->idx].isnotset_sids_idx >= array[fb->idx].isnotset_sids_size) { - uint32_t old_size = array[fb->idx].isnotset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].isnotset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].isnotset_sids_size = new_size; - array[fb->idx].isnotset_sids = ptr; - } - - array[fb->idx].isnotset_sids[array[fb->idx].isnotset_sids_idx] = s->num; - array[fb->idx].isnotset_sids_idx++; - } - } - } - } - } - for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH] ; sm != NULL; sm = sm->next) { - switch (sm->type) { - case DETECT_FLOWBITS: - { - /* figure out what flowbit action */ - const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; - array[fb->idx].cnts[fb->cmd]++; - if (has_state) - array[fb->idx].state_cnts[fb->cmd]++; - if (fb->cmd == DETECT_FLOWBITS_CMD_SET) { - if (array[fb->idx].set_sids_idx >= array[fb->idx].set_sids_size) { - uint32_t old_size = array[fb->idx].set_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].set_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].set_sids_size = new_size; - array[fb->idx].set_sids = ptr; - } - - array[fb->idx].set_sids[array[fb->idx].set_sids_idx] = s->num; - array[fb->idx].set_sids_idx++; - } - else if (fb->cmd == DETECT_FLOWBITS_CMD_UNSET) { - if (array[fb->idx].unset_sids_idx >= array[fb->idx].unset_sids_size) { - uint32_t old_size = array[fb->idx].unset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].unset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].unset_sids_size = new_size; - array[fb->idx].unset_sids = ptr; - } - - array[fb->idx].unset_sids[array[fb->idx].unset_sids_idx] = s->num; - array[fb->idx].unset_sids_idx++; - } - else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) { - if (array[fb->idx].toggle_sids_idx >= array[fb->idx].toggle_sids_size) { - uint32_t old_size = array[fb->idx].toggle_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].toggle_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].toggle_sids_size = new_size; - array[fb->idx].toggle_sids = ptr; - } - - array[fb->idx].toggle_sids[array[fb->idx].toggle_sids_idx] = s->num; - array[fb->idx].toggle_sids_idx++; - } - } - } + int r = DetectFlowbitsAnalyzeSignature(s, &fba); + if (r < 0) { + FBAnalyzerFree(&fba); + return -1; } } @@ -640,17 +654,93 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) DetectFlowbitsAnalyzeDump(de_ctx, array, array_size); } -end: + FBAnalyzerFree(&fba); + return 0; +} + +// TODO misses IPOnly rules. IPOnly flowbit rules are set only though. +static struct FBAnalyzer DetectFlowbitsAnalyzeForGroup( + const DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + struct FBAnalyzer fba = { .array = NULL, .array_size = 0 }; + + const uint32_t max_fb_id = de_ctx->max_fb_id; + if (max_fb_id == 0) + return fba; + + uint32_t array_size = max_fb_id + 1; + struct FBAnalyze *array = SCCalloc(array_size, sizeof(struct FBAnalyze)); + if (array == NULL) { + SCLogError("Unable to allocate flowbit analyze array"); + return fba; + } + SCLogDebug( + "fb analyzer array size: %" PRIu64, (uint64_t)(array_size * sizeof(struct FBAnalyze))); + fba.array = array; + fba.array_size = array_size; + + /* fill flowbit array, updating counters per sig */ + for (uint32_t i = 0; i < sgh->init->sig_cnt; i++) { + const Signature *s = sgh->init->match_array[i]; + SCLogDebug("sgh %p: s->id %u", sgh, s->id); + + int r = DetectFlowbitsAnalyzeSignature(s, &fba); + if (r < 0) { + FBAnalyzerFree(&fba); + return fba; + } + } + + /* walk array to see if all bits make sense */ for (uint32_t i = 0; i < array_size; i++) { - SCFree(array[i].set_sids); - SCFree(array[i].unset_sids); - SCFree(array[i].isset_sids); - SCFree(array[i].isnotset_sids); - SCFree(array[i].toggle_sids); + const char *varname = VarNameStoreSetupLookup(i, VAR_TYPE_FLOW_BIT); + if (varname == NULL) + continue; + + bool to_state = false; + if (array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] && + array[i].state_cnts[DETECT_FLOWBITS_CMD_SET] == 0) { + SCLogDebug("flowbit %s/%u: isset in state, set not in state", varname, i); + } + + /* if signature depends on 'stateful' flowbits, then turn the + * sig into a stateful sig itself */ + if (array[i].cnts[DETECT_FLOWBITS_CMD_ISSET] > 0 && + array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] == 0 && + array[i].state_cnts[DETECT_FLOWBITS_CMD_SET]) { + SCLogDebug("flowbit %s/%u: isset not in state, set in state", varname, i); + to_state = true; + } + + SCLogDebug("ALL flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u", + varname, i, array[i].cnts[DETECT_FLOWBITS_CMD_SET], + array[i].cnts[DETECT_FLOWBITS_CMD_TOGGLE], array[i].cnts[DETECT_FLOWBITS_CMD_UNSET], + array[i].cnts[DETECT_FLOWBITS_CMD_ISNOTSET], + array[i].cnts[DETECT_FLOWBITS_CMD_ISSET]); + SCLogDebug("STATE flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u", + varname, i, array[i].state_cnts[DETECT_FLOWBITS_CMD_SET], + array[i].state_cnts[DETECT_FLOWBITS_CMD_TOGGLE], + array[i].state_cnts[DETECT_FLOWBITS_CMD_UNSET], + array[i].state_cnts[DETECT_FLOWBITS_CMD_ISNOTSET], + array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET]); + for (uint32_t x = 0; x < array[i].set_sids_idx; x++) { + SCLogDebug("SET flowbit %s/%u: SID %u", varname, i, + de_ctx->sig_array[array[i].set_sids[x]]->id); + } + for (uint32_t x = 0; x < array[i].isset_sids_idx; x++) { + Signature *s = de_ctx->sig_array[array[i].isset_sids[x]]; + SCLogDebug("GET flowbit %s/%u: SID %u", varname, i, s->id); + + if (to_state) { + s->init_data->init_flags |= SIG_FLAG_INIT_STATE_MATCH; + SCLogDebug("made SID %u stateful because it depends on " + "stateful rules that set flowbit %s", + s->id, varname); + } + } } - SCFree(array); - return 0; + return fba; } SCMutex g_flowbits_dump_write_m = SCMUTEX_INITIALIZER; @@ -745,6 +835,463 @@ static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx, jb_free(js); } +static bool PrefilterFlowbitIsPrefilterable(const Signature *s) +{ + SCLogDebug("sid:%u: checking", s->id); + + for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; + sm = sm->next) { + switch (sm->type) { + case DETECT_FLOWBITS: { + const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { + SCLogDebug("sid:%u: FLOWBITS ISSET can prefilter", s->id); + return true; + } + } break; + } + } + SCLogDebug("sid:%u: no flowbit prefilter", s->id); + return false; +} + +/** core flowbit data structure: map a flowbit id to the signatures that need inspecting after it is + * found. Part of a rb-tree. */ +typedef struct PrefilterFlowbit { + uint32_t id; /**< flowbit id */ + uint32_t rule_id_size; /**< size in elements of `rule_id` */ + uint32_t rule_id_cnt; /**< usage in elements of `rule_id` */ + uint32_t *rule_id; /**< array of signature iid that are part of this prefilter */ + RB_ENTRY(PrefilterFlowbit) __attribute__((__packed__)) rb; +} __attribute__((__packed__)) PrefilterFlowbit; + +static int PrefilterFlowbitCompare(const PrefilterFlowbit *a, const PrefilterFlowbit *b) +{ + if (a->id > b->id) + return 1; + else if (a->id < b->id) + return -1; + else + return 0; +} + +/** red-black tree prototype for PFB (Prefilter Flow Bits) */ +RB_HEAD(PFB, PrefilterFlowbit); +RB_PROTOTYPE(PFB, PrefilterFlowbit, rb, PrefilterFlowbitCompare); +RB_GENERATE(PFB, PrefilterFlowbit, rb, PrefilterFlowbitCompare); + +struct PrefilterEngineFlowbits { + struct PFB fb_tree; +}; + +static void PrefilterFlowbitFree(void *vctx) +{ + struct PrefilterEngineFlowbits *ctx = vctx; + struct PrefilterFlowbit *rec, *safe = NULL; + RB_FOREACH_SAFE (rec, PFB, &ctx->fb_tree, safe) { + PFB_RB_REMOVE(&ctx->fb_tree, rec); + SCFree(rec->rule_id); + SCFree(rec); + } + + SCFree(ctx); +} + +static void PrefilterFlowbitMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) +{ + struct PrefilterEngineFlowbits *ctx = (struct PrefilterEngineFlowbits *)pectx; + SCLogDebug("%" PRIu64 ": ctx %p", p->pcap_cnt, ctx); + + if (p->flow == NULL) { + SCReturn; + } + + for (GenericVar *gv = p->flow->flowvar; gv != NULL; gv = gv->next) { + if (gv->type != DETECT_FLOWBITS) + continue; + + PrefilterFlowbit lookup; + memset(&lookup, 0, sizeof(lookup)); + lookup.id = gv->idx; + SCLogDebug("flowbit %u", gv->idx); + + PrefilterFlowbit *b = PFB_RB_FIND(&ctx->fb_tree, &lookup); + if (b == NULL) { + SCLogDebug("flowbit %u not in the tree", lookup.id); + } else { + SCLogDebug("flowbit %u found in the tree: %u", lookup.id, b->id); + + PrefilterAddSids(&det_ctx->pmq, b->rule_id, b->rule_id_cnt); +#ifdef DEBUG + for (uint32_t x = 0; x < b->rule_id_cnt; x++) { + const Signature *s = det_ctx->de_ctx->sig_array[b->rule_id[x]]; + SCLogDebug("flowbit %u -> sig %u", gv->idx, s->id); + } +#endif + } + } +} + +static void PrefilterFlowbitPostRuleMatch( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f) +{ + struct PrefilterEngineFlowbits *ctx = (struct PrefilterEngineFlowbits *)pectx; + SCLogDebug("%" PRIu64 ": ctx %p", p->pcap_cnt, ctx); + + if (p->flow == NULL) { + SCReturn; + } + + for (uint32_t i = 0; i < det_ctx->post_rule_work_queue.len; i++) { + const PostRuleMatchWorkQueueItem *w = &det_ctx->post_rule_work_queue.q[i]; + if (w->sm_type != DETECT_FLOWBITS) + continue; + + PrefilterFlowbit lookup; + memset(&lookup, 0, sizeof(lookup)); + lookup.id = w->value; + + PrefilterFlowbit *b = PFB_RB_FIND(&ctx->fb_tree, &lookup); + if (b == NULL) { + SCLogDebug("flowbit %u not in the tree", lookup.id); + } else { + SCLogDebug("flowbit %u found in the tree: %u. Adding %u sids", lookup.id, b->id, + b->rule_id_cnt); + PrefilterAddSids(&det_ctx->pmq, b->rule_id, b->rule_id_cnt); +#ifdef DEBUG + // SCLogDebug("b %u", b->rule_id_cnt); + for (uint32_t x = 0; x < b->rule_id_cnt; x++) { + Signature *s = det_ctx->de_ctx->sig_array[b->rule_id[x]]; + SCLogDebug("flowbit %u -> sig %u (triggered by %u)", w->value, s->id, + det_ctx->de_ctx->sig_array[w->id]->id); + } +#endif + } + } +} + +#define BLOCK_SIZE 8 + +static int AddBitAndSid( + struct PrefilterEngineFlowbits *ctx, const Signature *s, const uint32_t flowbit_id) +{ + PrefilterFlowbit x; + memset(&x, 0, sizeof(x)); + x.id = flowbit_id; + + PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x); + if (pfb == NULL) { + PrefilterFlowbit *add = SCCalloc(1, sizeof(*add)); + if (add == NULL) + return -1; + + add->id = flowbit_id; + add->rule_id = SCCalloc(1, BLOCK_SIZE * sizeof(uint32_t)); + if (add->rule_id == NULL) { + SCFree(add); + return -1; + } + add->rule_id_size = BLOCK_SIZE; + add->rule_id_cnt = 1; + add->rule_id[0] = s->num; + + PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add); + SCLogDebug("not found, so added (res %p)", res); + if (res != NULL) { + // duplicate, shouldn't be possible after the FIND above + BUG_ON(1); + return -1; + } + } else { + SCLogDebug("found! pfb %p id %u", pfb, pfb->id); + + if (pfb->rule_id_cnt < pfb->rule_id_size) { + pfb->rule_id[pfb->rule_id_cnt++] = s->num; + } else { + uint32_t *ptr = + SCRealloc(pfb->rule_id, (pfb->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t)); + if (ptr == NULL) { + // memory stays in the tree + return -1; + } + pfb->rule_id = ptr; + pfb->rule_id_size += BLOCK_SIZE; + pfb->rule_id[pfb->rule_id_cnt++] = s->num; + } + } + return 0; +} + +static int AddBitsAndSid(const DetectEngineCtx *de_ctx, struct PrefilterEngineFlowbits *ctx, + const DetectFlowbitsData *fb, const Signature *s) +{ + if (fb->or_list_size == 0) { + if (AddBitAndSid(ctx, s, fb->idx) < 0) { + return -1; + } + } else { + for (uint8_t i = 0; i < fb->or_list_size; i++) { + SCLogDebug("flowbit OR: bit %u", fb->or_list[i]); + if (AddBitAndSid(ctx, s, fb->or_list[i]) < 0) { + return -1; + } + } + } + return 0; +} + +static uint32_t NextMultiple(const uint32_t v, const uint32_t m) +{ + return v + (m - v % m); +} + +/** \internal + * \brief adds sids for 'isset' prefilter flowbits + * \retval int 1 if we added sid(s), 0 if we didn't, -1 on error */ +// TODO skip sids that aren't set by this sgh +// TODO skip sids that doesn't have a isset in the same direction +static int AddSids(const DetectEngineCtx *de_ctx, const struct FBAnalyzer *fba, + const DetectFlowbitsData *fb, PrefilterFlowbit *add) +{ + int added = 0; + for (uint32_t i = 0; i < fba->array[fb->idx].isset_sids_idx; i++) { + const uint32_t sig_iid = fba->array[fb->idx].isset_sids[i]; + const Signature *s = de_ctx->sig_array[sig_iid]; + SCLogDebug("flowbit: %u => considering sid %u (iid:%u)", fb->idx, s->id, s->num); + + /* Skip sids that aren't prefilter. These would just run all the time. */ + if (s->init_data->prefilter_sm == NULL || + s->init_data->prefilter_sm->type != DETECT_FLOWBITS) { +#ifdef DEBUG + const char *name = s->init_data->prefilter_sm + ? sigmatch_table[s->init_data->prefilter_sm->type].name + : "none"; + SCLogDebug("flowbit: %u => rejected sid %u (iid:%u). No prefilter or prefilter not " + "flowbits (%p, %s, %d)", + fb->idx, s->id, sig_iid, s->init_data->prefilter_sm, name, + s->init_data->prefilter_sm ? s->init_data->prefilter_sm->type : -1); +#endif + continue; + } + + /* only add sids that match our bit */ + const DetectFlowbitsData *fs_fb = + (const DetectFlowbitsData *)s->init_data->prefilter_sm->ctx; + if (fs_fb->idx != fb->idx) { + SCLogDebug( + "flowbit: %u => rejected sid %u (iid:%u). Sig prefilters on different bit %u", + fb->idx, s->id, sig_iid, fs_fb->idx); + continue; + } + + bool dup = false; + for (uint32_t x = 0; x < add->rule_id_cnt; x++) { + if (add->rule_id[x] == sig_iid) { + dup = true; + } + } + + if (!dup) { + if (add->rule_id_cnt < add->rule_id_size) { + add->rule_id[add->rule_id_cnt++] = sig_iid; + } else { + uint32_t *ptr = SCRealloc( + add->rule_id, (add->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t)); + if (ptr == NULL) { + return -1; + } + add->rule_id = ptr; + add->rule_id_size += BLOCK_SIZE; + add->rule_id[add->rule_id_cnt++] = sig_iid; + } + added = 1; + SCLogDebug("flowbit: %u => accepted sid %u (iid:%u)", fb->idx, s->id, sig_iid); + } + } + return added; +} + +/* TODO shouldn't add sids for which Signature::num is < our num. Is this possible after sorting? */ +static int AddBitSetToggle(const DetectEngineCtx *de_ctx, struct FBAnalyzer *fba, + struct PrefilterEngineFlowbits *ctx, const DetectFlowbitsData *fb, const Signature *s) +{ + PrefilterFlowbit x; + memset(&x, 0, sizeof(x)); + x.id = fb->idx; + PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x); + if (pfb == NULL) { + PrefilterFlowbit *add = SCCalloc(1, sizeof(*add)); + if (add == NULL) + return -1; + + add->id = fb->idx; + add->rule_id_size = NextMultiple(fba->array[fb->idx].isset_sids_idx, BLOCK_SIZE); + add->rule_id = SCCalloc(1, add->rule_id_size * sizeof(uint32_t)); + if (add->rule_id == NULL) { + SCFree(add); + return -1; + } + + if (!AddSids(de_ctx, fba, fb, add)) { + SCLogDebug("no sids added"); + SCFree(add->rule_id); + SCFree(add); + return 0; + } + PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add); + SCLogDebug("not found, so added (res %p)", res); + BUG_ON(res != NULL); // TODO if res != NULL we have a duplicate which should be impossible + } else { + SCLogDebug("found! pfb %p id %u", pfb, pfb->id); + + int r = AddSids(de_ctx, fba, fb, pfb); + if (r < 0) { + return -1; + } else if (r == 0) { + SCLogDebug("no sids added"); + return 0; + } + } + return 1; +} + +/** \brief build flowbit prefilter state(s) + * + * Build "set" and "isset" states. + * + * For each flowbit "isset" in the sgh, we need to check: + * 1. is it supported + * 2. is prefilter enabled + * 3. does it match in the same dir or only opposing dir + */ +static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + if (sgh == NULL) + return 0; + + SCLogDebug("sgh %p: setting up prefilter", sgh); + struct PrefilterEngineFlowbits *isset_ctx = NULL; + struct PrefilterEngineFlowbits *set_ctx = NULL; + + struct FBAnalyzer fb_analysis = DetectFlowbitsAnalyzeForGroup(de_ctx, sgh); + if (fb_analysis.array == NULL) + goto error; + + for (uint32_t i = 0; i < sgh->init->sig_cnt; i++) { + Signature *s = sgh->init->match_array[i]; + if (s == NULL) + continue; + + SCLogDebug("checking sid %u", s->id); + + /* first build the 'set' state */ + for (SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH]; sm != NULL; + sm = sm->next) { + if (sm->type != DETECT_FLOWBITS) { + SCLogDebug("skip non flowbits sm"); + continue; + } + + DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + if (fb->cmd == DETECT_FLOWBITS_CMD_SET) { + SCLogDebug( + "DETECT_SM_LIST_POSTMATCH: sid %u DETECT_FLOWBITS set %u", s->id, fb->idx); + } else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) { + SCLogDebug("DETECT_SM_LIST_POSTMATCH: sid %u DETECT_FLOWBITS toggle %u", s->id, + fb->idx); + } else { + SCLogDebug("unsupported flowbits setting"); + continue; + } + + if (fb_analysis.array[fb->idx].isnotset_sids_idx || + fb_analysis.array[fb->idx].unset_sids_idx) { + SCLogDebug("flowbit %u not supported: unset in use", fb->idx); + continue; + } + + if (set_ctx == NULL) { + set_ctx = SCCalloc(1, sizeof(*set_ctx)); + if (set_ctx == NULL) + goto error; + } + + SCLogDebug("setting up sets/toggles for sid %u", s->id); + if (AddBitSetToggle(de_ctx, &fb_analysis, set_ctx, fb, s) == 1) { + // flag the set/toggle to trigger the post-rule match logic + SCLogDebug("set up sets/toggles for sid %u", s->id); + fb->post_rule_match_prefilter = true; + } + + // TODO don't add for sigs that don't have isset in this sgh. Reasoning: + // prefilter post match logic only makes sense in the same dir as otherwise + // the regular 'isset' logic can simply run with the regular prefilters + // before the rule loop + } + + /* next, build the 'isset' state */ + if (s->init_data->prefilter_sm == NULL || + s->init_data->prefilter_sm->type != DETECT_FLOWBITS) { + SCLogDebug("no prefilter or prefilter not flowbits"); + continue; + } + + const DetectFlowbitsData *fb = (DetectFlowbitsData *)s->init_data->prefilter_sm->ctx; + if (fb_analysis.array[fb->idx].isnotset_sids_idx || + fb_analysis.array[fb->idx].unset_sids_idx) { + SCLogDebug("flowbit %u not supported: toggle or unset in use", fb->idx); + s->init_data->prefilter_sm = NULL; + s->flags &= ~SIG_FLAG_PREFILTER; + continue; + } + + SCLogDebug("isset: adding sid %u, flowbit %u", s->id, fb->idx); + + if (isset_ctx == NULL) { + isset_ctx = SCCalloc(1, sizeof(*isset_ctx)); + if (isset_ctx == NULL) + goto error; + } + if (AddBitsAndSid(de_ctx, isset_ctx, fb, s) < 0) { + goto error; + } + } + + /* finally, register the states with their engines */ + static const char *g_prefilter_flowbits_isset = "flowbits:isset"; + if (isset_ctx != NULL) { + PrefilterAppendEngine(de_ctx, sgh, PrefilterFlowbitMatch, SIG_MASK_REQUIRE_FLOW, isset_ctx, + PrefilterFlowbitFree, g_prefilter_flowbits_isset); + SCLogDebug("isset: added prefilter engine"); + + if (set_ctx != NULL && !RB_EMPTY(&set_ctx->fb_tree)) { + static const char *g_prefilter_flowbits_set = "flowbits:set"; + PrefilterAppendPostRuleEngine(de_ctx, sgh, PrefilterFlowbitPostRuleMatch, set_ctx, + PrefilterFlowbitFree, g_prefilter_flowbits_set); + SCLogDebug("set/toggle: added prefilter engine"); + } else { + if (set_ctx) { + PrefilterFlowbitFree(set_ctx); + } + SCLogDebug("set/toggle: NO prefilter engine added"); + } + } else if (set_ctx != NULL) { + PrefilterFlowbitFree(set_ctx); + } + FBAnalyzerFree(&fb_analysis); + return 0; + +error: + if (set_ctx) { + PrefilterFlowbitFree(set_ctx); + } + if (isset_ctx) { + PrefilterFlowbitFree(isset_ctx); + } + FBAnalyzerFree(&fb_analysis); + return -1; +} + #ifdef UNITTESTS static int FlowBitsTestParse01(void) diff --git a/src/detect-flowbits.h b/src/detect-flowbits.h index 0a5823ef60b3..d04d6846bf46 100644 --- a/src/detect-flowbits.h +++ b/src/detect-flowbits.h @@ -36,6 +36,9 @@ typedef struct DetectFlowbitsData_ { uint32_t idx; uint8_t cmd; uint8_t or_list_size; + /** Flag to trigger post rule match prefilter following a 'set' match. */ + bool post_rule_match_prefilter; /**< set/toggle command should trigger post-rule-match + "prefilter" */ uint32_t *or_list; } DetectFlowbitsData; diff --git a/src/detect-parse.c b/src/detect-parse.c index 984501c1dd8a..7a69f238e1ef 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -913,6 +913,9 @@ static int SigParseOptions(DetectEngineCtx *de_ctx, Signature *s, char *optstr, } s->init_data->negated = false; + const enum DetectKeywordId idx = SigTableGetIndex(st); + s->init_data->has_possible_prefilter |= de_ctx->sm_types_prefilter[idx]; + if (st->flags & SIGMATCH_INFO_DEPRECATED) { #define URL "https://suricata.io/our-story/deprecation-policy/" if (st->alternative == 0) @@ -1020,7 +1023,6 @@ static int SigParseOptions(DetectEngineCtx *de_ctx, Signature *s, char *optstr, /* handle 'silent' error case */ if (setup_ret == -2) { - enum DetectKeywordId idx = SigTableGetIndex(st); if (de_ctx->sm_types_silent_error[idx] == false) { de_ctx->sm_types_silent_error[idx] = true; return -1; @@ -1876,6 +1878,57 @@ SigMatchData* SigMatchList2DataArray(SigMatch *head) return out; } +extern int g_skip_prefilter; + +static void SigSetupPrefilter(DetectEngineCtx *de_ctx, Signature *s) +{ + SCEnter(); + if (s->init_data->prefilter_sm == NULL && s->init_data->mpm_sm == NULL) { + SCLogDebug("s %u: set up prefilter/mpm", s->id); + + RetrieveFPForSig(de_ctx, s); + if (s->init_data->mpm_sm != NULL) { + s->flags |= SIG_FLAG_PREFILTER; + } else { + SCLogDebug("s %u: no mpm; prefilter? de_ctx->prefilter_setting %u " + "s->init_data->has_possible_prefilter %s", + s->id, de_ctx->prefilter_setting, + BOOL2STR(s->init_data->has_possible_prefilter)); + + if (!g_skip_prefilter && de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO && + !(s->flags & SIG_FLAG_PREFILTER)) { + int prefilter_list = DETECT_TBLSIZE; + /* get the keyword supporting prefilter with the lowest type */ + for (int i = 0; i < DETECT_SM_LIST_MAX; i++) { + for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) { + if (sigmatch_table[sm->type].SupportsPrefilter != NULL) { + if (sigmatch_table[sm->type].SupportsPrefilter(s)) { + prefilter_list = MIN(prefilter_list, sm->type); + } + } + } + } + + /* apply that keyword as prefilter */ + if (prefilter_list != DETECT_TBLSIZE) { + for (int i = 0; i < DETECT_SM_LIST_MAX; i++) { + for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) { + if (sm->type == prefilter_list) { + s->init_data->prefilter_sm = sm; + s->flags |= SIG_FLAG_PREFILTER; + SCLogConfig("sid %u: prefilter is on \"%s\"", s->id, + sigmatch_table[sm->type].name); + break; + } + } + } + } + } + } + } + SCReturn; +} + /** * \internal * \brief validate a just parsed signature for internal inconsistencies @@ -2215,6 +2268,8 @@ static Signature *SigInitHelper(DetectEngineCtx *de_ctx, const char *sigstr, DetectEngineBufferRunSetupCallback(de_ctx, sig->init_data->buffers[x].id, sig); } + SigSetupPrefilter(de_ctx, sig); + /* validate signature, SigValidate will report the error reason */ if (SigValidate(de_ctx, sig) == 0) { goto error; diff --git a/src/detect-prefilter.c b/src/detect-prefilter.c index f38b56bf8b9c..bd5a6445a4f1 100644 --- a/src/detect-prefilter.c +++ b/src/detect-prefilter.c @@ -99,6 +99,8 @@ static int DetectPrefilterSetup (DetectEngineCtx *de_ctx, Signature *s, const ch } s->init_data->prefilter_sm = sm; + SCLogDebug( + "sid %u: prefilter is on \"%s\" (%u)", s->id, sigmatch_table[sm->type].name, sm->type); SCReturnInt(0); } diff --git a/src/detect.c b/src/detect.c index 03fa8437068d..188cf136f27c 100644 --- a/src/detect.c +++ b/src/detect.c @@ -725,6 +725,16 @@ static inline void DetectRunPrefilterPkt( #endif } +static int SortHelper(const void *a, const void *b) +{ + const Signature *sa = *(const Signature **)a; + const Signature *sb = *(const Signature **)b; + if (sa->num == sb->num) + return 0; + else + return sa->num > sb->num ? 1 : -1; +} + static inline void DetectRulePacketRules( ThreadVars * const tv, DetectEngineCtx * const de_ctx, @@ -772,7 +782,7 @@ static inline void DetectRulePacketRules( } const uint8_t s_proto_flags = s->proto.flags; - SCLogDebug("inspecting signature id %"PRIu32"", s->id); + SCLogDebug("packet %" PRIu64 ": inspecting signature id %" PRIu32 "", p->pcap_cnt, s->id); if (s->app_inspect != NULL) { goto next; // handle sig in DetectRunTx @@ -834,6 +844,51 @@ static inline void DetectRulePacketRules( } } AlertQueueAppend(det_ctx, s, p, txid, alert_flags); + + if (det_ctx->post_rule_work_queue.len > 0) { + /* run post match prefilter engines on work queue */ + PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, pflow); + + if (det_ctx->pmq.rule_id_array_cnt > 0) { + /* undo "prefetch" */ + if (next_s) + match_array--; + /* create temporary rule pointer array starting + * at where we are in the current match array */ + const Signature *replace[de_ctx->sig_array_len]; // TODO heap? + SCLogDebug("sig_array_len %u det_ctx->pmq.rule_id_array_cnt %u", + de_ctx->sig_array_len, det_ctx->pmq.rule_id_array_cnt); + const Signature **r = replace; + for (uint32_t x = 0; x < match_cnt; x++) { + *r++ = match_array[x]; + SCLogDebug("appended %u", match_array[x]->id); + } + /* append the prefilter results, then sort it */ + for (uint32_t x = 0; x < det_ctx->pmq.rule_id_array_cnt; x++) { + // TODO what happens if a tx engine is added? + SCLogDebug("adding iid %u", det_ctx->pmq.rule_id_array[x]); + Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[x]]; + SCLogDebug("adding id %u", ts->id); + if (ts->app_inspect == NULL) { + *r++ = ts; + match_cnt++; + } + } + // TODO if sid 12 is added twice, how do we dedup + qsort(replace, match_cnt, sizeof(Signature *), SortHelper); + /* rewrite match_array to include the new additions */ + Signature **m = match_array; + for (uint32_t x = 0; x < match_cnt; x++) { + *m++ = (Signature *)replace[x]; + } + /* prefetch next */ + next_s = *match_array++; + next_sflags = next_s->flags; + SCLogDebug("%u rules added", det_ctx->pmq.rule_id_array_cnt); + det_ctx->post_rule_work_queue.len = 0; + PMQ_RESET(&det_ctx->pmq); + } + } next: DetectVarProcessList(det_ctx, pflow, p); DetectReplaceFree(det_ctx); @@ -1609,6 +1664,38 @@ static void DetectRunTx(ThreadVars *tv, } DetectVarProcessList(det_ctx, p->flow, p); RULE_PROFILING_END(det_ctx, s, r, p); + + if (det_ctx->post_rule_work_queue.len > 0) { + SCLogDebug("%p/%" PRIu64 " post_rule_work_queue len %u", tx.tx_ptr, tx.tx_id, + det_ctx->post_rule_work_queue.len); + /* run post match prefilter engines on work queue */ + PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, f); + + uint32_t prev_array_idx = array_idx; + for (uint32_t j = 0; j < det_ctx->pmq.rule_id_array_cnt; j++) { + const Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[j]]; + if (ts->app_inspect != NULL) { + const SigIntId id = ts->num; + det_ctx->tx_candidates[array_idx].s = ts; + det_ctx->tx_candidates[array_idx].id = id; + det_ctx->tx_candidates[array_idx].flags = NULL; + det_ctx->tx_candidates[array_idx].stream_reset = 0; + array_idx++; + + SCLogDebug("%p/%" PRIu64 " rule %u (%u) added from 'post match' prefilter", + tx.tx_ptr, tx.tx_id, ts->id, id); + } + } + SCLogDebug("%p/%" PRIu64 " rules added from 'post match' prefilter: %u", tx.tx_ptr, + tx.tx_id, array_idx - prev_array_idx); + if (prev_array_idx != array_idx) { + /* sort, but only part of array we're still going to process */ + qsort(det_ctx->tx_candidates + i, array_idx - i, sizeof(RuleMatchCandidateTx), + DetectRunTxSortHelper); + } + det_ctx->post_rule_work_queue.len = 0; + PMQ_RESET(&det_ctx->pmq); + } } det_ctx->tx_id = 0; diff --git a/src/detect.h b/src/detect.h index fe755b7f0d14..35bbf278a91c 100644 --- a/src/detect.h +++ b/src/detect.h @@ -549,6 +549,9 @@ typedef struct SignatureInitData_ { bool src_contains_negation; bool dst_contains_negation; + /** see if any of the sigmatches supports an enabled prefilter */ + bool has_possible_prefilter; + /* used to hold flags that are used during init */ uint32_t init_flags; /* coccinelle: SignatureInitData:init_flags:SIG_FLAG_INIT_ */ @@ -1087,6 +1090,25 @@ typedef struct RuleMatchCandidateTx { const Signature *s; /**< ptr to sig */ } RuleMatchCandidateTx; +/** Stores a single u32 for a rule match of the type `sm_type`. Used by + * flowbits prefilter to register DETECT_FLOWBITS, for post + * match handling. */ +typedef struct PostRuleMatchWorkQueueItem { + int sm_type; /**< sigmatch type e.g. DETECT_FLOWBITS */ + uint32_t value; /**< value to be interpreted by the sm_type + * implementation. E.g. flowbit id. */ +#ifdef DEBUG + SigIntId id; +#endif +} PostRuleMatchWorkQueueItem; + +/** Array of PostRuleMatchWorkQueueItem's. */ +typedef struct PostRuleMatchWorkQueue { + PostRuleMatchWorkQueueItem *q; /**< array pointer */ + uint32_t len; /**< number of array elements in use. */ + uint32_t size; /**< allocation size in number of elements. */ +} PostRuleMatchWorkQueue; + /** * Detection engine thread data. */ @@ -1197,6 +1219,9 @@ typedef struct DetectEngineThreadCtx_ { uint32_t non_pf_store_cnt; MpmThreadCtx mtc; /**< thread ctx for the mpm */ + /* work queue for post-rule matching affecting prefilter */ + PostRuleMatchWorkQueue post_rule_work_queue; + PrefilterRuleStore pmq; /* string to replace */ @@ -1388,6 +1413,8 @@ typedef struct PrefilterEngineList_ { PrefilterPktFn Prefilter; PrefilterTxFn PrefilterTx; PrefilterFrameFn PrefilterFrame; + void (*PrefilterPostRule)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f); struct PrefilterEngineList_ *next; @@ -1421,6 +1448,8 @@ typedef struct PrefilterEngine_ { PrefilterPktFn Prefilter; PrefilterTxFn PrefilterTx; PrefilterFrameFn PrefilterFrame; + void (*PrefilterPostRule)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f); } cb; /* global id for this prefilter */ @@ -1448,6 +1477,7 @@ typedef struct SigGroupHeadInitData_ { PrefilterEngineList *payload_engines; PrefilterEngineList *tx_engines; PrefilterEngineList *frame_engines; + PrefilterEngineList *post_rule_match_engines; /** number of sigs in this group */ SigIntId sig_cnt; @@ -1478,6 +1508,7 @@ typedef struct SigGroupHead_ { PrefilterEngine *payload_engines; PrefilterEngine *tx_engines; PrefilterEngine *frame_engines; + PrefilterEngine *post_rule_match_engines; /**< engines to run after rules modified a state */ /* ptr to our init data we only use at... init :) */ SigGroupHeadInitData *init; diff --git a/src/flow-bit.c b/src/flow-bit.c index f59e7eae943e..f481220e64b6 100644 --- a/src/flow-bit.c +++ b/src/flow-bit.c @@ -55,19 +55,25 @@ static FlowBit *FlowBitGet(Flow *f, uint32_t idx) return NULL; } -/* add a flowbit to the flow */ -static void FlowBitAdd(Flow *f, uint32_t idx) +/** \brief add a flowbit to the flow + * \retval -1 error + * \retval 0 not added, already set before + * \retval 1 added */ +static int FlowBitAdd(Flow *f, uint32_t idx) { FlowBit *fb = FlowBitGet(f, idx); if (fb == NULL) { fb = SCMalloc(sizeof(FlowBit)); if (unlikely(fb == NULL)) - return; + return -1; fb->type = DETECT_FLOWBITS; fb->idx = idx; fb->next = NULL; GenericVarAppend(&f->flowvar, (GenericVar *)fb); + return 1; + } else { + return 0; } } @@ -81,9 +87,13 @@ static void FlowBitRemove(Flow *f, uint32_t idx) FlowBitFree(fb); } -void FlowBitSet(Flow *f, uint32_t idx) +/** \brief add a flowbit to the flow + * \retval -1 error + * \retval 0 not added, already set before + * \retval 1 added */ +int FlowBitSet(Flow *f, uint32_t idx) { - FlowBitAdd(f, idx); + return FlowBitAdd(f, idx); } void FlowBitUnset(Flow *f, uint32_t idx) @@ -91,13 +101,18 @@ void FlowBitUnset(Flow *f, uint32_t idx) FlowBitRemove(f, idx); } -void FlowBitToggle(Flow *f, uint32_t idx) +/** + * \retval bool true if bit is set, false is unset + */ +bool FlowBitToggle(Flow *f, uint32_t idx) { FlowBit *fb = FlowBitGet(f, idx); if (fb != NULL) { FlowBitRemove(f, idx); + return false; } else { FlowBitAdd(f, idx); + return true; } } diff --git a/src/flow-bit.h b/src/flow-bit.h index e10c58dafde0..2290e94de65b 100644 --- a/src/flow-bit.h +++ b/src/flow-bit.h @@ -39,9 +39,9 @@ typedef struct FlowBit_ { void FlowBitFree(FlowBit *); void FlowBitRegisterTests(void); -void FlowBitSet(Flow *, uint32_t); +int FlowBitSet(Flow *, uint32_t); void FlowBitUnset(Flow *, uint32_t); -void FlowBitToggle(Flow *, uint32_t); +bool FlowBitToggle(Flow *, uint32_t); int FlowBitIsset(Flow *, uint32_t); int FlowBitIsnotset(Flow *, uint32_t); #endif /* SURICATA_FLOW_BIT_H */ diff --git a/src/tests/detect-http-client-body.c b/src/tests/detect-http-client-body.c index bbeb4d33bde8..707c7492cb1d 100644 --- a/src/tests/detect-http-client-body.c +++ b/src/tests/detect-http-client-body.c @@ -2231,7 +2231,7 @@ static int DetectHttpClientBodyTest22(void) FAIL_IF(memcmp(cd2->content, "four", cd2->content_len) != 0); FAIL_IF(hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT); FAIL_IF(memcmp(hcbd1->content, "two", hcbd1->content_len) != 0); - FAIL_IF(hcbd2->flags != DETECT_CONTENT_DISTANCE); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM)); FAIL_IF(memcmp(hcbd2->content, "three", hcbd1->content_len) != 0); FAIL_IF(!DETECT_CONTENT_IS_SINGLE(cd1)); @@ -2273,7 +2273,7 @@ static int DetectHttpClientBodyTest23(void) FAIL_IF(memcmp(cd2->content, "four", cd2->content_len) != 0); FAIL_IF(hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT); FAIL_IF(memcmp(hcbd1->content, "one", hcbd1->content_len) != 0); - FAIL_IF(hcbd2->flags != DETECT_CONTENT_DISTANCE); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM)); FAIL_IF(memcmp(hcbd2->content, "three", hcbd1->content_len) != 0); FAIL_IF(!DETECT_CONTENT_IS_SINGLE(cd2)); FAIL_IF(DETECT_CONTENT_IS_SINGLE(hcbd1)); @@ -2285,31 +2285,17 @@ static int DetectHttpClientBodyTest23(void) static int DetectHttpClientBodyTest24(void) { - DetectEngineCtx *de_ctx = NULL; - int result = 0; - - if ( (de_ctx = DetectEngineCtxInit()) == NULL) - goto end; - + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; Signature *s = DetectEngineAppendSig(de_ctx, "alert icmp any any -> any any " "(content:\"one\"; http_client_body; pcre:/two/; " "content:\"three\"; distance:10; within:15; " "http_client_body; content:\"four\"; sid:1;)"); - if (de_ctx->sig_list == NULL) { - printf("de_ctx->sig_list == NULL\n"); - goto end; - } - - if (de_ctx->sig_list->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) { - printf("de_ctx->sig_list->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL\n"); - goto end; - } + FAIL_IF_NULL(s); - if (DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id) == NULL) { - printf("DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id) == NULL\n"); - goto end; - } + FAIL_IF_NULL(de_ctx->sig_list->init_data->smlists[DETECT_SM_LIST_PMATCH]); + FAIL_IF_NULL(DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id)); DetectPcreData *pd1 = (DetectPcreData *)de_ctx->sig_list->init_data->smlists_tail[DETECT_SM_LIST_PMATCH] @@ -2322,56 +2308,35 @@ static int DetectHttpClientBodyTest24(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != 0 || - cd2->flags != 0 || memcmp(cd2->content, "four", cd2->content_len) != 0 || - hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN) || - memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { - goto end; - } - - if (!DETECT_CONTENT_IS_SINGLE(cd2) || - DETECT_CONTENT_IS_SINGLE(hcbd1) || - DETECT_CONTENT_IS_SINGLE(hcbd2)) { - goto end; - } + FAIL_IF(pd1->flags != 0); + FAIL_IF(cd2->flags != 0); + FAIL_IF(memcmp(cd2->content, "four", cd2->content_len) != 0); + FAIL_IF(hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT); + FAIL_IF(memcmp(hcbd1->content, "one", hcbd1->content_len) != 0); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd2->content, "three", hcbd1->content_len) != 0); - result = 1; + FAIL_IF(!DETECT_CONTENT_IS_SINGLE(cd2)); + FAIL_IF(DETECT_CONTENT_IS_SINGLE(hcbd1)); + FAIL_IF(DETECT_CONTENT_IS_SINGLE(hcbd2)); - end: DetectEngineCtxFree(de_ctx); - return result; + PASS; } static int DetectHttpClientBodyTest25(void) { - DetectEngineCtx *de_ctx = NULL; - int result = 0; - - if ( (de_ctx = DetectEngineCtxInit()) == NULL) - goto end; - + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; Signature *s = DetectEngineAppendSig(de_ctx, "alert icmp any any -> any any " "(content:\"one\"; http_client_body; pcre:/two/; " "content:\"three\"; distance:10; http_client_body; " "content:\"four\"; distance:10; sid:1;)"); - if (de_ctx->sig_list == NULL) { - printf("de_ctx->sig_list == NULL\n"); - goto end; - } - - if (de_ctx->sig_list->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) { - printf("de_ctx->sig_list->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL\n"); - goto end; - } - - if (DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id) == NULL) { - printf("DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id) == NULL\n"); - goto end; - } + FAIL_IF_NULL(s); + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_PMATCH]); + FAIL_IF_NULL(DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id)); DetectPcreData *pd1 = (DetectPcreData *)de_ctx->sig_list->init_data->smlists_tail[DETECT_SM_LIST_PMATCH] @@ -2384,27 +2349,20 @@ static int DetectHttpClientBodyTest25(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != DETECT_PCRE_RELATIVE_NEXT || - cd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(cd2->content, "four", cd2->content_len) != 0 || - hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { - goto end; - } - - if (DETECT_CONTENT_IS_SINGLE(cd2) || - DETECT_CONTENT_IS_SINGLE(hcbd1) || - DETECT_CONTENT_IS_SINGLE(hcbd2)) { - goto end; - } + FAIL_IF(pd1->flags != DETECT_PCRE_RELATIVE_NEXT); + FAIL_IF(cd2->flags != DETECT_CONTENT_DISTANCE); + FAIL_IF(memcmp(cd2->content, "four", cd2->content_len) != 0); + FAIL_IF(hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT); + FAIL_IF(memcmp(hcbd1->content, "one", hcbd1->content_len) != 0); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd2->content, "three", hcbd1->content_len) != 0); - result = 1; + FAIL_IF(DETECT_CONTENT_IS_SINGLE(cd2)); + FAIL_IF(DETECT_CONTENT_IS_SINGLE(hcbd1)); + FAIL_IF(DETECT_CONTENT_IS_SINGLE(hcbd2)); - end: DetectEngineCtxFree(de_ctx); - return result; + PASS; } static int DetectHttpClientBodyTest26(void) @@ -2447,13 +2405,13 @@ static int DetectHttpClientBodyTest26(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || - cd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(cd2->content, "four", cd2->content_len) != 0 || - hcbd1->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_OFFSET) || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN) || - memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { + if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || cd2->flags != DETECT_CONTENT_DISTANCE || + memcmp(cd2->content, "four", cd2->content_len) != 0 || + hcbd1->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_OFFSET) || + memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || + hcbd2->flags != + (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) || + memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { printf ("failed: http_client_body incorrect flags"); goto end; } @@ -2533,13 +2491,11 @@ static int DetectHttpClientBodyTest28(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || - cd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(cd2->content, "four", cd2->content_len) != 0 || - hcbd1->flags != 0 || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != DETECT_CONTENT_DEPTH || - memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { + if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || cd2->flags != DETECT_CONTENT_DISTANCE || + memcmp(cd2->content, "four", cd2->content_len) != 0 || hcbd1->flags != 0 || + memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || + hcbd2->flags != (DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) || + memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { goto end; } @@ -2589,12 +2545,10 @@ static int DetectHttpClientBodyTest29(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(hcbd2->content, "two", hcbd1->content_len) != 0) { - goto end; - } + FAIL_IF(hcbd1->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd1->content, "one", hcbd1->content_len) != 0); + FAIL_IF(hcbd2->flags != DETECT_CONTENT_DISTANCE); + FAIL_IF(memcmp(hcbd2->content, "two", hcbd1->content_len) != 0); result = 1; @@ -2636,12 +2590,10 @@ static int DetectHttpClientBodyTest30(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != DETECT_CONTENT_WITHIN || - memcmp(hcbd2->content, "two", hcbd1->content_len) != 0) { - goto end; - } + FAIL_IF(hcbd1->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd1->content, "one", hcbd1->content_len) != 0); + FAIL_IF(hcbd2->flags != DETECT_CONTENT_WITHIN); + FAIL_IF(memcmp(hcbd2->content, "two", hcbd1->content_len) != 0); result = 1; @@ -2754,11 +2706,9 @@ static int DetectHttpClientBodyTest34(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || - hcbd2->flags != DETECT_CONTENT_WITHIN || - memcmp(hcbd2->content, "two", hcbd2->content_len) != 0) { - goto end; - } + FAIL_IF(pd1->flags != (DETECT_PCRE_RELATIVE_NEXT)); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd2->content, "two", hcbd2->content_len) != 0); result = 1; @@ -2808,11 +2758,9 @@ static int DetectHttpClientBodyTest35(void) ->prev->ctx; DetectPcreData *pd2 = (DetectPcreData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd2->flags != (DETECT_PCRE_RELATIVE) || - hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT || - memcmp(hcbd1->content, "two", hcbd1->content_len) != 0) { - goto end; - } + FAIL_IF(pd2->flags != (DETECT_PCRE_RELATIVE)); + FAIL_IF(hcbd1->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd1->content, "two", hcbd1->content_len) != 0); result = 1; @@ -2863,11 +2811,9 @@ static int DetectHttpClientBodyTest36(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || - hcbd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(hcbd2->content, "two", hcbd2->content_len) != 0) { - goto end; - } + FAIL_IF(pd1->flags != (DETECT_PCRE_RELATIVE_NEXT)); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd2->content, "two", hcbd2->content_len) != 0); result = 1;