From 72b7778ac19bbc0d253ef950c871a9d541f38dc1 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 10 Jun 2022 21:07:15 +0200 Subject: [PATCH 01/17] flowbits: unset returns state of the bit --- src/flow-bit.c | 7 ++++++- src/flow-bit.h | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/flow-bit.c b/src/flow-bit.c index f59e7eae943e..304e6bd41569 100644 --- a/src/flow-bit.c +++ b/src/flow-bit.c @@ -91,13 +91,18 @@ void FlowBitUnset(Flow *f, uint32_t idx) FlowBitRemove(f, idx); } -void FlowBitToggle(Flow *f, uint32_t idx) +/** + * \retval bool true if bit is set, false is unset + */ +bool FlowBitToggle(Flow *f, uint32_t idx) { FlowBit *fb = FlowBitGet(f, idx); if (fb != NULL) { FlowBitRemove(f, idx); + return false; } else { FlowBitAdd(f, idx); + return true; } } diff --git a/src/flow-bit.h b/src/flow-bit.h index e10c58dafde0..adf9944a3ea6 100644 --- a/src/flow-bit.h +++ b/src/flow-bit.h @@ -41,7 +41,7 @@ void FlowBitRegisterTests(void); void FlowBitSet(Flow *, uint32_t); void FlowBitUnset(Flow *, uint32_t); -void FlowBitToggle(Flow *, uint32_t); +bool FlowBitToggle(Flow *, uint32_t); int FlowBitIsset(Flow *, uint32_t); int FlowBitIsnotset(Flow *, uint32_t); #endif /* SURICATA_FLOW_BIT_H */ From a81984d4771443de5a3c8112e35ad6b2f1012b06 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 10 Oct 2022 11:42:02 +0200 Subject: [PATCH 02/17] flowbits: allow setter to know if set is new --- src/flow-bit.c | 20 +++++++++++++++----- src/flow-bit.h | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/flow-bit.c b/src/flow-bit.c index 304e6bd41569..f481220e64b6 100644 --- a/src/flow-bit.c +++ b/src/flow-bit.c @@ -55,19 +55,25 @@ static FlowBit *FlowBitGet(Flow *f, uint32_t idx) return NULL; } -/* add a flowbit to the flow */ -static void FlowBitAdd(Flow *f, uint32_t idx) +/** \brief add a flowbit to the flow + * \retval -1 error + * \retval 0 not added, already set before + * \retval 1 added */ +static int FlowBitAdd(Flow *f, uint32_t idx) { FlowBit *fb = FlowBitGet(f, idx); if (fb == NULL) { fb = SCMalloc(sizeof(FlowBit)); if (unlikely(fb == NULL)) - return; + return -1; fb->type = DETECT_FLOWBITS; fb->idx = idx; fb->next = NULL; GenericVarAppend(&f->flowvar, (GenericVar *)fb); + return 1; + } else { + return 0; } } @@ -81,9 +87,13 @@ static void FlowBitRemove(Flow *f, uint32_t idx) FlowBitFree(fb); } -void FlowBitSet(Flow *f, uint32_t idx) +/** \brief add a flowbit to the flow + * \retval -1 error + * \retval 0 not added, already set before + * \retval 1 added */ +int FlowBitSet(Flow *f, uint32_t idx) { - FlowBitAdd(f, idx); + return FlowBitAdd(f, idx); } void FlowBitUnset(Flow *f, uint32_t idx) diff --git a/src/flow-bit.h b/src/flow-bit.h index adf9944a3ea6..2290e94de65b 100644 --- a/src/flow-bit.h +++ b/src/flow-bit.h @@ -39,7 +39,7 @@ typedef struct FlowBit_ { void FlowBitFree(FlowBit *); void FlowBitRegisterTests(void); -void FlowBitSet(Flow *, uint32_t); +int FlowBitSet(Flow *, uint32_t); void FlowBitUnset(Flow *, uint32_t); bool FlowBitToggle(Flow *, uint32_t); int FlowBitIsset(Flow *, uint32_t); From 3fee5905bf1c9fc0e27cbd009868799288072b76 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 5 Oct 2022 08:40:15 +0200 Subject: [PATCH 03/17] detect: set mpm/prefilter during signature parsing In preparation of flowbit prefilter work that needs this info earlier. Track potential prefilter sm's to avoid unnecessary looping during setup. --- src/detect-engine-build.c | 34 ----------------------- src/detect-engine-loader.c | 1 - src/detect-engine-mpm.c | 5 ---- src/detect-flow.c | 4 +++ src/detect-parse.c | 57 +++++++++++++++++++++++++++++++++++++- src/detect.h | 3 ++ 6 files changed, 63 insertions(+), 41 deletions(-) diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index c85f53d5051d..38ea4df86f24 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -1703,7 +1703,6 @@ void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s) } } -extern int g_skip_prefilter; /** * \brief Preprocess signature, classify ip-only, etc, build sig array * @@ -1787,39 +1786,6 @@ int SigPrepareStage1(DetectEngineCtx *de_ctx) RuleSetScore(s); - /* if keyword engines are enabled in the config, handle them here */ - if (!g_skip_prefilter && de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO && - !(s->flags & SIG_FLAG_PREFILTER)) { - int prefilter_list = DETECT_TBLSIZE; - - // TODO buffers? - - /* get the keyword supporting prefilter with the lowest type */ - for (int i = 0; i < DETECT_SM_LIST_MAX; i++) { - for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) { - if (sigmatch_table[sm->type].SupportsPrefilter != NULL) { - if (sigmatch_table[sm->type].SupportsPrefilter(s)) { - prefilter_list = MIN(prefilter_list, sm->type); - } - } - } - } - - /* apply that keyword as prefilter */ - if (prefilter_list != DETECT_TBLSIZE) { - for (int i = 0; i < DETECT_SM_LIST_MAX; i++) { - for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) { - if (sm->type == prefilter_list) { - s->init_data->prefilter_sm = sm; - s->flags |= SIG_FLAG_PREFILTER; - SCLogConfig("sid %u: prefilter is on \"%s\"", s->id, sigmatch_table[sm->type].name); - break; - } - } - } - } - } - /* run buffer type callbacks if any */ for (int x = 0; x < DETECT_SM_LIST_MAX; x++) { if (s->init_data->smlists[x]) diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c index 950812a187c9..30e931165cbc 100644 --- a/src/detect-engine-loader.c +++ b/src/detect-engine-loader.c @@ -167,7 +167,6 @@ static int DetectLoadSigFile( sig = DetectEngineAppendSig(de_ctx, line); if (sig != NULL) { if (rule_engine_analysis_set || fp_engine_analysis_set) { - RetrieveFPForSig(de_ctx, sig); if (fp_engine_analysis_set) { EngineAnalysisFP(de_ctx, sig, line); } diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 5e8687e34686..7b12786662c6 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -2415,12 +2415,7 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx) { uint32_t cnt = 0; for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) { - if (s->flags & SIG_FLAG_PREFILTER) - continue; - - RetrieveFPForSig(de_ctx, s); if (s->init_data->mpm_sm != NULL) { - s->flags |= SIG_FLAG_PREFILTER; cnt++; } } diff --git a/src/detect-flow.c b/src/detect-flow.c index c268dedb155f..683d53fb70cb 100644 --- a/src/detect-flow.c +++ b/src/detect-flow.c @@ -445,14 +445,18 @@ void DetectFlowFree(DetectEngineCtx *de_ctx, void *ptr) static void PrefilterPacketFlowMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) { + SCEnter(); + const PrefilterPacketHeaderCtx *ctx = pectx; if (!PrefilterPacketHeaderExtraMatch(ctx, p)) return; if (FlowMatch(p->flags, p->flowflags, ctx->v1.u16[0], ctx->v1.u16[1])) { + SCLogDebug("match: adding sids"); PrefilterAddSids(&det_ctx->pmq, ctx->sigs_array, ctx->sigs_cnt); } + SCReturn; } static void diff --git a/src/detect-parse.c b/src/detect-parse.c index 984501c1dd8a..7a69f238e1ef 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -913,6 +913,9 @@ static int SigParseOptions(DetectEngineCtx *de_ctx, Signature *s, char *optstr, } s->init_data->negated = false; + const enum DetectKeywordId idx = SigTableGetIndex(st); + s->init_data->has_possible_prefilter |= de_ctx->sm_types_prefilter[idx]; + if (st->flags & SIGMATCH_INFO_DEPRECATED) { #define URL "https://suricata.io/our-story/deprecation-policy/" if (st->alternative == 0) @@ -1020,7 +1023,6 @@ static int SigParseOptions(DetectEngineCtx *de_ctx, Signature *s, char *optstr, /* handle 'silent' error case */ if (setup_ret == -2) { - enum DetectKeywordId idx = SigTableGetIndex(st); if (de_ctx->sm_types_silent_error[idx] == false) { de_ctx->sm_types_silent_error[idx] = true; return -1; @@ -1876,6 +1878,57 @@ SigMatchData* SigMatchList2DataArray(SigMatch *head) return out; } +extern int g_skip_prefilter; + +static void SigSetupPrefilter(DetectEngineCtx *de_ctx, Signature *s) +{ + SCEnter(); + if (s->init_data->prefilter_sm == NULL && s->init_data->mpm_sm == NULL) { + SCLogDebug("s %u: set up prefilter/mpm", s->id); + + RetrieveFPForSig(de_ctx, s); + if (s->init_data->mpm_sm != NULL) { + s->flags |= SIG_FLAG_PREFILTER; + } else { + SCLogDebug("s %u: no mpm; prefilter? de_ctx->prefilter_setting %u " + "s->init_data->has_possible_prefilter %s", + s->id, de_ctx->prefilter_setting, + BOOL2STR(s->init_data->has_possible_prefilter)); + + if (!g_skip_prefilter && de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO && + !(s->flags & SIG_FLAG_PREFILTER)) { + int prefilter_list = DETECT_TBLSIZE; + /* get the keyword supporting prefilter with the lowest type */ + for (int i = 0; i < DETECT_SM_LIST_MAX; i++) { + for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) { + if (sigmatch_table[sm->type].SupportsPrefilter != NULL) { + if (sigmatch_table[sm->type].SupportsPrefilter(s)) { + prefilter_list = MIN(prefilter_list, sm->type); + } + } + } + } + + /* apply that keyword as prefilter */ + if (prefilter_list != DETECT_TBLSIZE) { + for (int i = 0; i < DETECT_SM_LIST_MAX; i++) { + for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) { + if (sm->type == prefilter_list) { + s->init_data->prefilter_sm = sm; + s->flags |= SIG_FLAG_PREFILTER; + SCLogConfig("sid %u: prefilter is on \"%s\"", s->id, + sigmatch_table[sm->type].name); + break; + } + } + } + } + } + } + } + SCReturn; +} + /** * \internal * \brief validate a just parsed signature for internal inconsistencies @@ -2215,6 +2268,8 @@ static Signature *SigInitHelper(DetectEngineCtx *de_ctx, const char *sigstr, DetectEngineBufferRunSetupCallback(de_ctx, sig->init_data->buffers[x].id, sig); } + SigSetupPrefilter(de_ctx, sig); + /* validate signature, SigValidate will report the error reason */ if (SigValidate(de_ctx, sig) == 0) { goto error; diff --git a/src/detect.h b/src/detect.h index fe755b7f0d14..3970764f8ea9 100644 --- a/src/detect.h +++ b/src/detect.h @@ -549,6 +549,9 @@ typedef struct SignatureInitData_ { bool src_contains_negation; bool dst_contains_negation; + /** see if any of the sigmatches supports an enabled prefilter */ + bool has_possible_prefilter; + /* used to hold flags that are used during init */ uint32_t init_flags; /* coccinelle: SignatureInitData:init_flags:SIG_FLAG_INIT_ */ From f5c45522bd98fe38325854074b0a46241acef052 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 13 Nov 2020 16:09:00 +0100 Subject: [PATCH 04/17] detect/flowbits: implement prefilter support Allow for more efficient rules that 'prefilter' on flowbits with 'isset' logic. This prefilter is enabled by default, which means that if no mpm is present or no explicit prefilter is used, the flowbits prefilter will be set up for a rule. flowbits 'isset' prefilter For rules that have a 'flowbits:isset,' statement, a "regular" prefilter facility is created. It means that the rules are removed from the normal match list(s) and added to a prefilter engine that runs prior to the individual rule inspection stage. Implementation: the prefilter is implemented as an RB_TREE of flowbits, with the rule id's they "enable" stored per tree node. The matching logic is walking the list of bits set in the flow and looking each of them up in the RB_TREE, adding the rule ids of each of the matching bits to the list of rule candidates. The 'isset' prefilter has one important corner case, which is that bits can in fact be set during the rule evaluation stage. This is different from all other prefilter engines, that evaluate an immutable state (for the lifetime of the packets inspection). flowbits 'set' post-match prefilter For flowbits 'set' action, special post-match 'prefilter' facilities deal with this corner case. The high level logic is that these track which 'isset' sigs depend on them, and add these dependencies to the candidates list when a 'set' action occurs. This is implemented in a few steps: 1. flowbits 'set' is flagged 2. when 'set' action occurs the flowbit is added to a "post rule match work queue" 3. when the rule evaluation ends, the post-match "prefilter" engine is run on each of the flowbits in the "post rule match work queue" 4. these engines ammend the candidates list with the rule id dependencies for the flowbit 5. the candidates list is sorted to make sure within the execution for that packet the inspection order is maintained Ticket: #2486. --- src/detect-engine-build.c | 2 +- src/detect-engine-prefilter.c | 90 ++++ src/detect-engine-prefilter.h | 10 +- src/detect-engine-siggroup.c | 1 + src/detect-engine.c | 6 + src/detect-flowbits.c | 911 ++++++++++++++++++++++++++++------ src/detect-flowbits.h | 1 + src/detect-prefilter.c | 2 + src/detect.c | 89 +++- src/detect.h | 23 + 10 files changed, 972 insertions(+), 163 deletions(-) diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index 38ea4df86f24..037980e5e825 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -668,7 +668,7 @@ static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigG any5_cnt++; } - prefilter_cnt += (s->init_data->prefilter_sm != 0); + prefilter_cnt += (s->init_data->prefilter_sm != NULL); if (s->init_data->mpm_sm == NULL) { nonmpm_cnt++; diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c index feff1251e4e2..768549e4ea18 100644 --- a/src/detect-engine-prefilter.c +++ b/src/detect-engine-prefilter.c @@ -144,6 +144,35 @@ void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, } } +/** \brief invoke post-rule match "prefilter" engines + * + * Invoke prefilter engines that depend on a rule match to run. + * e.g. the flowbits:set prefilter that adds sids that depend on + * a flowbit "set" to the match array. + */ +void PrefilterPostRuleMatch( + DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, Flow *f) +{ + SCLogDebug("post-rule-match engines %p", sgh->post_rule_match_engines); + if (sgh->post_rule_match_engines) { + PrefilterEngine *engine = sgh->post_rule_match_engines; + do { + SCLogDebug("running post-rule-match engine"); + PREFILTER_PROFILING_START(det_ctx); + engine->cb.PrefilterPostRule(det_ctx, engine->pectx, p, f); + PREFILTER_PROFILING_END(det_ctx, engine->gid); + + if (engine->is_last) + break; + engine++; + } while (1); + + if (det_ctx->pmq.rule_id_array_cnt > 1) { + QuickSortSigIntId(det_ctx->pmq.rule_id_array, det_ctx->pmq.rule_id_array_cnt); + } + } +} + void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const uint8_t flags, const SignatureMask mask) { @@ -342,6 +371,39 @@ int PrefilterAppendFrameEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, return 0; } +int PrefilterAppendPostRuleEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, + void (*PrefilterPostRuleFunc)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f), + void *pectx, void (*FreeFunc)(void *pectx), const char *name) +{ + if (sgh == NULL || PrefilterPostRuleFunc == NULL || pectx == NULL) + return -1; + + PrefilterEngineList *e = SCMallocAligned(sizeof(*e), CLS); + if (e == NULL) + return -1; + memset(e, 0x00, sizeof(*e)); + e->PrefilterPostRule = PrefilterPostRuleFunc; + e->pectx = pectx; + e->Free = FreeFunc; + + if (sgh->init->post_rule_match_engines == NULL) { + sgh->init->post_rule_match_engines = e; + } else { + PrefilterEngineList *t = sgh->init->post_rule_match_engines; + while (t->next != NULL) { + t = t->next; + } + + t->next = e; + e->id = t->id + 1; + } + + e->name = name; + e->gid = PrefilterStoreGetId(de_ctx, e->name, e->Free); + return 0; +} + static void PrefilterFreeEngineList(PrefilterEngineList *e) { if (e->Free && e->pectx) { @@ -396,6 +458,10 @@ void PrefilterCleanupRuleGroup(const DetectEngineCtx *de_ctx, SigGroupHead *sgh) PrefilterFreeEngines(de_ctx, sgh->frame_engines); sgh->frame_engines = NULL; } + if (sgh->post_rule_match_engines) { + PrefilterFreeEngines(de_ctx, sgh->post_rule_match_engines); + sgh->post_rule_match_engines = NULL; + } } static int PrefilterSetupRuleGroupSortHelper(const void *a, const void *b) @@ -589,6 +655,30 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) e++; } } + if (sgh->init->post_rule_match_engines != NULL) { + uint32_t cnt = 0; + for (el = sgh->init->post_rule_match_engines; el != NULL; el = el->next) { + cnt++; + } + sgh->post_rule_match_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS); + if (sgh->post_rule_match_engines == NULL) { + return; + } + memset(sgh->post_rule_match_engines, 0x00, (cnt * sizeof(PrefilterEngine))); + + uint32_t local_id = 0; + PrefilterEngine *e = sgh->post_rule_match_engines; + for (el = sgh->init->post_rule_match_engines; el != NULL; el = el->next) { + e->local_id = local_id++; + e->cb.PrefilterPostRule = el->PrefilterPostRule; + e->pectx = el->pectx; + el->pectx = NULL; // e now owns the ctx + e->gid = el->gid; + e->is_last = (el->next == NULL); + e++; + } + SCLogDebug("sgh %p max local_id %u", sgh, local_id); + } } /* hash table for assigning a unique id to each engine type. */ diff --git a/src/detect-engine-prefilter.h b/src/detect-engine-prefilter.h index ec58594b9b00..8f37f2b8a756 100644 --- a/src/detect-engine-prefilter.h +++ b/src/detect-engine-prefilter.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2016 Open Information Security Foundation +/* Copyright (C) 2016-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -52,6 +52,10 @@ void Prefilter(DetectEngineThreadCtx *, const SigGroupHead *, Packet *p, const u int PrefilterAppendEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, PrefilterPktFn PrefilterFunc, SignatureMask mask, void *pectx, void (*FreeFunc)(void *pectx), const char *name); + +void PrefilterPostRuleMatch( + DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, Flow *f); + int PrefilterAppendPayloadEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, PrefilterPktFn PrefilterFunc, void *pectx, void (*FreeFunc)(void *pectx), const char *name); int PrefilterAppendTxEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, @@ -60,6 +64,10 @@ int PrefilterAppendTxEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, int PrefilterAppendFrameEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, PrefilterFrameFn PrefilterFrameFunc, AppProto alproto, uint8_t frame_type, void *pectx, void (*FreeFunc)(void *pectx), const char *name); +int PrefilterAppendPostRuleEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, + void (*PrefilterPostRuleFunc)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f), + void *pectx, void (*FreeFunc)(void *pectx), const char *name); void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 49982f028b28..29bc3b24011b 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -81,6 +81,7 @@ void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid) PrefilterFreeEnginesList(sghid->pkt_engines); PrefilterFreeEnginesList(sghid->payload_engines); PrefilterFreeEnginesList(sghid->frame_engines); + PrefilterFreeEnginesList(sghid->post_rule_match_engines); SCFree(sghid); } diff --git a/src/detect-engine.c b/src/detect-engine.c index 77c25a1cf3a9..b72b91e6fe51 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2523,6 +2523,9 @@ static DetectEngineCtx *DetectEngineCtxInitReal( goto error; } + // TODO test enable flowbits prefilter + // de_ctx->sm_types_prefilter[DETECT_FLOWBITS] = true; + SigGroupHeadHashInit(de_ctx); MpmStoreInit(de_ctx); DetectParseDupSigHashInit(de_ctx); @@ -3522,6 +3525,9 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) AlertQueueFree(det_ctx); + if (det_ctx->post_rule_work_queue.q) + SCFree(det_ctx->post_rule_work_queue.q); + if (det_ctx->byte_values != NULL) SCFree(det_ctx->byte_values); diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 40f04d75f305..a3d038f77b5a 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -61,6 +61,8 @@ void DetectFlowbitFree (DetectEngineCtx *, void *); #ifdef UNITTESTS void FlowBitsRegisterTests(void); #endif +static bool PrefilterFlowbitIsPrefilterable(const Signature *s); +static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh); void DetectFlowbitsRegister (void) { @@ -76,6 +78,8 @@ void DetectFlowbitsRegister (void) /* this is compatible to ip-only signatures */ sigmatch_table[DETECT_FLOWBITS].flags |= SIGMATCH_IPONLY_COMPAT; + sigmatch_table[DETECT_FLOWBITS].SupportsPrefilter = PrefilterFlowbitIsPrefilterable; + sigmatch_table[DETECT_FLOWBITS].SetupPrefilter = PrefilterSetupFlowbits; DetectSetupParseRegexes(PARSE_REGEX, &parse_regex); } @@ -132,11 +136,10 @@ static int FlowbitOrAddData(DetectEngineCtx *de_ctx, DetectFlowbitsData *cd, cha static int DetectFlowbitMatchToggle (Packet *p, const DetectFlowbitsData *fd) { if (p->flow == NULL) - return 0; - - FlowBitToggle(p->flow,fd->idx); + return -1; - return 1; + bool set = FlowBitToggle(p->flow, fd->idx); + return set == true; } static int DetectFlowbitMatchUnset (Packet *p, const DetectFlowbitsData *fd) @@ -152,11 +155,11 @@ static int DetectFlowbitMatchUnset (Packet *p, const DetectFlowbitsData *fd) static int DetectFlowbitMatchSet (Packet *p, const DetectFlowbitsData *fd) { if (p->flow == NULL) - return 0; - - FlowBitSet(p->flow,fd->idx); + return -1; - return 1; + int r = FlowBitSet(p->flow, fd->idx); + SCLogDebug("set %u", fd->idx); + return r; } static int DetectFlowbitMatchIsset (Packet *p, const DetectFlowbitsData *fd) @@ -188,6 +191,34 @@ static int DetectFlowbitMatchIsnotset (Packet *p, const DetectFlowbitsData *fd) return FlowBitIsnotset(p->flow,fd->idx); } +// TODO move to common file +#define QUEUE_STEP 16 + +static void PostRuleMatchWorkQueueAppend( + DetectEngineThreadCtx *det_ctx, const Signature *s, const int type, const uint32_t value) +{ + if (det_ctx->post_rule_work_queue.q == NULL) { + det_ctx->post_rule_work_queue.q = + SCCalloc(1, sizeof(PostRuleMatchWorkQueueItem) * QUEUE_STEP); + BUG_ON(det_ctx->post_rule_work_queue.q == NULL); + det_ctx->post_rule_work_queue.size = QUEUE_STEP; + } else if (det_ctx->post_rule_work_queue.len == det_ctx->post_rule_work_queue.size) { + void *ptr = SCRealloc( + det_ctx->post_rule_work_queue.q, (det_ctx->post_rule_work_queue.size + QUEUE_STEP) * + sizeof(PostRuleMatchWorkQueueItem)); + BUG_ON(ptr == NULL); + det_ctx->post_rule_work_queue.q = ptr; + det_ctx->post_rule_work_queue.size += QUEUE_STEP; + } + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].sm_type = type; + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].value = value; +#ifdef DEBUG + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].id = s->num; +#endif + det_ctx->post_rule_work_queue.len++; + SCLogDebug("det_ctx->post_rule_work_queue.len %u", det_ctx->post_rule_work_queue.len); +} + /* * returns 0: no match * 1: match @@ -206,12 +237,25 @@ int DetectFlowbitMatch (DetectEngineThreadCtx *det_ctx, Packet *p, return DetectFlowbitMatchIsset(p,fd); case DETECT_FLOWBITS_CMD_ISNOTSET: return DetectFlowbitMatchIsnotset(p,fd); - case DETECT_FLOWBITS_CMD_SET: - return DetectFlowbitMatchSet(p,fd); + case DETECT_FLOWBITS_CMD_SET: { + int r = DetectFlowbitMatchSet(p, fd); + /* only on a new "set" invoke the prefilter */ + if (r == 1 && fd->prefilter) { + SCLogDebug("flowbit set, appending to work queue"); + PostRuleMatchWorkQueueAppend(det_ctx, s, DETECT_FLOWBITS, fd->idx); + } + return (r != -1); + } case DETECT_FLOWBITS_CMD_UNSET: return DetectFlowbitMatchUnset(p,fd); - case DETECT_FLOWBITS_CMD_TOGGLE: - return DetectFlowbitMatchToggle(p,fd); + case DETECT_FLOWBITS_CMD_TOGGLE: { + int r = DetectFlowbitMatchToggle(p, fd); + if (r == 1 && fd->prefilter) { + SCLogDebug("flowbit set (by toggle), appending to work queue"); + PostRuleMatchWorkQueueAppend(det_ctx, s, DETECT_FLOWBITS, fd->idx); + } + return (r != -1); + } default: SCLogError("unknown cmd %" PRIu32 "", fd->cmd); return 0; @@ -388,6 +432,11 @@ void DetectFlowbitFree (DetectEngineCtx *de_ctx, void *ptr) SCFree(fd); } +struct FBAnalyzer { + struct FBAnalyze *array; + uint32_t array_size; +}; + struct FBAnalyze { uint16_t cnts[DETECT_FLOWBITS_CMD_MAX]; uint16_t state_cnts[DETECT_FLOWBITS_CMD_MAX]; @@ -417,20 +466,140 @@ extern bool rule_engine_analysis_set; static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx, struct FBAnalyze *array, uint32_t elements); +static void FBAnalyzerArrayFree(struct FBAnalyze *array, const uint32_t array_size) +{ + if (array) { + for (uint32_t i = 0; i < array_size; i++) { + SCFree(array[i].set_sids); + SCFree(array[i].unset_sids); + SCFree(array[i].isset_sids); + SCFree(array[i].isnotset_sids); + SCFree(array[i].toggle_sids); + } + SCFree(array); + } +} + +static void FBAnalyzerFree(struct FBAnalyzer *fba) +{ + if (fba && fba->array) { + FBAnalyzerArrayFree(fba->array, fba->array_size); + } +} + +#define MAX_SIDS 8 +static bool CheckExpand(const uint32_t sids_idx, uint32_t **sids, uint32_t *sids_size) +{ + if (sids_idx >= *sids_size) { + const uint32_t old_size = *sids_size; + const uint32_t new_size = MAX(2 * old_size, MAX_SIDS); + + void *ptr = SCRealloc(*sids, new_size * sizeof(uint32_t)); + if (ptr == NULL) + return false; + *sids_size = new_size; + *sids = ptr; + } + return true; +} + +static int DetectFlowbitsAnalyzeSignature(const Signature *s, struct FBAnalyzer *fba) +{ + // const uint32_t array_size = fba->array_size; + struct FBAnalyze *array = fba->array; + + /* see if the signature uses stateful matching TODO is there not a flag? */ + bool has_state = (s->init_data->buffer_index != 0); + + for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; + sm = sm->next) { + if (sm->type != DETECT_FLOWBITS) + continue; + /* figure out the flowbit action */ + const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + // Handle flowbit array in case of ORed flowbits + for (uint8_t k = 0; k < fb->or_list_size; k++) { + struct FBAnalyze *fa = &array[fb->or_list[k]]; + fa->cnts[fb->cmd]++; + fa->state_cnts[fb->cmd] += has_state; + + if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { + if (!CheckExpand(fa->isset_sids_idx, &fa->isset_sids, &fa->isset_sids_size)) + return -1; + fa->isset_sids[fa->isset_sids_idx] = s->num; + fa->isset_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { + if (!CheckExpand( + fa->isnotset_sids_idx, &fa->isnotset_sids, &fa->isnotset_sids_size)) + return -1; + fa->isnotset_sids[fa->isnotset_sids_idx] = s->num; + fa->isnotset_sids_idx++; + } + } + if (fb->or_list_size == 0) { + struct FBAnalyze *fa = &array[fb->idx]; + fa->cnts[fb->cmd]++; + fa->state_cnts[fb->cmd] += has_state; + + if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { + if (!CheckExpand(fa->isset_sids_idx, &fa->isset_sids, &fa->isset_sids_size)) + return -1; + fa->isset_sids[fa->isset_sids_idx] = s->num; + fa->isset_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { + if (!CheckExpand( + fa->isnotset_sids_idx, &fa->isnotset_sids, &fa->isnotset_sids_size)) + return -1; + fa->isnotset_sids[fa->isnotset_sids_idx] = s->num; + fa->isnotset_sids_idx++; + } + } + } + for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH]; sm != NULL; + sm = sm->next) { + if (sm->type != DETECT_FLOWBITS) + continue; + /* figure out what flowbit action */ + const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + struct FBAnalyze *fa = &array[fb->idx]; + fa->cnts[fb->cmd]++; + fa->state_cnts[fb->cmd] += has_state; + + if (fb->cmd == DETECT_FLOWBITS_CMD_SET) { + if (!CheckExpand(fa->set_sids_idx, &fa->set_sids, &fa->set_sids_size)) + return -1; + fa->set_sids[fa->set_sids_idx] = s->num; + fa->set_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_UNSET) { + if (!CheckExpand(fa->unset_sids_idx, &fa->unset_sids, &fa->unset_sids_size)) + return -1; + fa->unset_sids[fa->unset_sids_idx] = s->num; + fa->unset_sids_idx++; + } else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) { + if (!CheckExpand(fa->toggle_sids_idx, &fa->toggle_sids, &fa->toggle_sids_size)) + return -1; + fa->toggle_sids[fa->toggle_sids_idx] = s->num; + fa->toggle_sids_idx++; + } + } + return 0; +} + int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) { const uint32_t max_fb_id = de_ctx->max_fb_id; if (max_fb_id == 0) return 0; -#define MAX_SIDS 8 - uint32_t array_size = max_fb_id + 1; + struct FBAnalyzer fba = { .array = NULL, .array_size = 0 }; + const uint32_t array_size = max_fb_id + 1; struct FBAnalyze *array = SCCalloc(array_size, sizeof(struct FBAnalyze)); - if (array == NULL) { SCLogError("Unable to allocate flowbit analyze array"); return -1; } + fba.array = array; + fba.array_size = array_size; SCLogDebug("fb analyzer array size: %"PRIu64, (uint64_t)(array_size * sizeof(struct FBAnalyze))); @@ -439,143 +608,10 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) for (uint32_t i = 0; i < de_ctx->sig_array_len; i++) { const Signature *s = de_ctx->sig_array[i]; - /* see if the signature uses stateful matching */ - bool has_state = (s->init_data->buffer_index != 0); - - for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) { - switch (sm->type) { - case DETECT_FLOWBITS: - { - /* figure out the flowbit action */ - const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; - // Handle flowbit array in case of ORed flowbits - for (uint8_t k = 0; k < fb->or_list_size; k++) { - array[fb->or_list[k]].cnts[fb->cmd]++; - if (has_state) - array[fb->or_list[k]].state_cnts[fb->cmd]++; - if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { - if (array[fb->or_list[k]].isset_sids_idx >= array[fb->or_list[k]].isset_sids_size) { - uint32_t old_size = array[fb->or_list[k]].isset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->or_list[k]].isset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->or_list[k]].isset_sids_size = new_size; - array[fb->or_list[k]].isset_sids = ptr; - } - - array[fb->or_list[k]].isset_sids[array[fb->or_list[k]].isset_sids_idx] = s->num; - array[fb->or_list[k]].isset_sids_idx++; - } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { - if (array[fb->or_list[k]].isnotset_sids_idx >= array[fb->or_list[k]].isnotset_sids_size) { - uint32_t old_size = array[fb->or_list[k]].isnotset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->or_list[k]].isnotset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->or_list[k]].isnotset_sids_size = new_size; - array[fb->or_list[k]].isnotset_sids = ptr; - } - - array[fb->or_list[k]].isnotset_sids[array[fb->or_list[k]].isnotset_sids_idx] = s->num; - array[fb->or_list[k]].isnotset_sids_idx++; - } - } - if (fb->or_list_size == 0) { - array[fb->idx].cnts[fb->cmd]++; - if (has_state) - array[fb->idx].state_cnts[fb->cmd]++; - if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { - if (array[fb->idx].isset_sids_idx >= array[fb->idx].isset_sids_size) { - uint32_t old_size = array[fb->idx].isset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].isset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].isset_sids_size = new_size; - array[fb->idx].isset_sids = ptr; - } - - array[fb->idx].isset_sids[array[fb->idx].isset_sids_idx] = s->num; - array[fb->idx].isset_sids_idx++; - } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) { - if (array[fb->idx].isnotset_sids_idx >= array[fb->idx].isnotset_sids_size) { - uint32_t old_size = array[fb->idx].isnotset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].isnotset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].isnotset_sids_size = new_size; - array[fb->idx].isnotset_sids = ptr; - } - - array[fb->idx].isnotset_sids[array[fb->idx].isnotset_sids_idx] = s->num; - array[fb->idx].isnotset_sids_idx++; - } - } - } - } - } - for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH] ; sm != NULL; sm = sm->next) { - switch (sm->type) { - case DETECT_FLOWBITS: - { - /* figure out what flowbit action */ - const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; - array[fb->idx].cnts[fb->cmd]++; - if (has_state) - array[fb->idx].state_cnts[fb->cmd]++; - if (fb->cmd == DETECT_FLOWBITS_CMD_SET) { - if (array[fb->idx].set_sids_idx >= array[fb->idx].set_sids_size) { - uint32_t old_size = array[fb->idx].set_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].set_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].set_sids_size = new_size; - array[fb->idx].set_sids = ptr; - } - - array[fb->idx].set_sids[array[fb->idx].set_sids_idx] = s->num; - array[fb->idx].set_sids_idx++; - } - else if (fb->cmd == DETECT_FLOWBITS_CMD_UNSET) { - if (array[fb->idx].unset_sids_idx >= array[fb->idx].unset_sids_size) { - uint32_t old_size = array[fb->idx].unset_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].unset_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].unset_sids_size = new_size; - array[fb->idx].unset_sids = ptr; - } - - array[fb->idx].unset_sids[array[fb->idx].unset_sids_idx] = s->num; - array[fb->idx].unset_sids_idx++; - } - else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) { - if (array[fb->idx].toggle_sids_idx >= array[fb->idx].toggle_sids_size) { - uint32_t old_size = array[fb->idx].toggle_sids_size; - uint32_t new_size = MAX(2 * old_size, MAX_SIDS); - - void *ptr = SCRealloc(array[fb->idx].toggle_sids, new_size * sizeof(uint32_t)); - if (ptr == NULL) - goto end; - array[fb->idx].toggle_sids_size = new_size; - array[fb->idx].toggle_sids = ptr; - } - - array[fb->idx].toggle_sids[array[fb->idx].toggle_sids_idx] = s->num; - array[fb->idx].toggle_sids_idx++; - } - } - } + int r = DetectFlowbitsAnalyzeSignature(s, &fba); + if (r < 0) { + FBAnalyzerFree(&fba); + return -1; } } @@ -640,17 +676,96 @@ int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx) DetectFlowbitsAnalyzeDump(de_ctx, array, array_size); } -end: + FBAnalyzerFree(&fba); + return 0; +} + +// TODO misses IPOnly rules. IPOnly flowbit rules are set only though. +static struct FBAnalyzer DetectFlowbitsAnalyzeForGroup( + const DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + struct FBAnalyzer fba = { .array = NULL, .array_size = 0 }; + + const uint32_t max_fb_id = de_ctx->max_fb_id; + if (max_fb_id == 0) + return fba; + +#define MAX_SIDS 8 + uint32_t array_size = max_fb_id + 1; + struct FBAnalyze *array = SCCalloc(array_size, sizeof(struct FBAnalyze)); + if (array == NULL) { + SCLogError("Unable to allocate flowbit analyze array"); + return fba; + } + SCLogDebug( + "fb analyzer array size: %" PRIu64, (uint64_t)(array_size * sizeof(struct FBAnalyze))); + fba.array = array; + fba.array_size = array_size; + + /* fill flowbit array, updating counters per sig */ + for (uint32_t i = 0; i < sgh->init->sig_cnt; i++) { + const Signature *s = sgh->init->match_array[i]; + SCLogDebug("sgh %p: s->id %u", sgh, s->id); + + int r = DetectFlowbitsAnalyzeSignature(s, &fba); + if (r < 0) { + SCFree(fba.array); + array = fba.array = NULL; + array_size = fba.array_size = 0; + return fba; + } + } + + /* walk array to see if all bits make sense */ for (uint32_t i = 0; i < array_size; i++) { - SCFree(array[i].set_sids); - SCFree(array[i].unset_sids); - SCFree(array[i].isset_sids); - SCFree(array[i].isnotset_sids); - SCFree(array[i].toggle_sids); + const char *varname = VarNameStoreSetupLookup(i, VAR_TYPE_FLOW_BIT); + if (varname == NULL) + continue; + + bool to_state = false; + if (array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] && + array[i].state_cnts[DETECT_FLOWBITS_CMD_SET] == 0) { + SCLogDebug("flowbit %s/%u: isset in state, set not in state", varname, i); + } + + /* if signature depends on 'stateful' flowbits, then turn the + * sig into a stateful sig itself */ + if (array[i].cnts[DETECT_FLOWBITS_CMD_ISSET] > 0 && + array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] == 0 && + array[i].state_cnts[DETECT_FLOWBITS_CMD_SET]) { + SCLogDebug("flowbit %s/%u: isset not in state, set in state", varname, i); + to_state = true; + } + + SCLogDebug("ALL flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u", + varname, i, array[i].cnts[DETECT_FLOWBITS_CMD_SET], + array[i].cnts[DETECT_FLOWBITS_CMD_TOGGLE], array[i].cnts[DETECT_FLOWBITS_CMD_UNSET], + array[i].cnts[DETECT_FLOWBITS_CMD_ISNOTSET], + array[i].cnts[DETECT_FLOWBITS_CMD_ISSET]); + SCLogDebug("STATE flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u", + varname, i, array[i].state_cnts[DETECT_FLOWBITS_CMD_SET], + array[i].state_cnts[DETECT_FLOWBITS_CMD_TOGGLE], + array[i].state_cnts[DETECT_FLOWBITS_CMD_UNSET], + array[i].state_cnts[DETECT_FLOWBITS_CMD_ISNOTSET], + array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET]); + for (uint32_t x = 0; x < array[i].set_sids_idx; x++) { + SCLogDebug("SET flowbit %s/%u: SID %u", varname, i, + de_ctx->sig_array[array[i].set_sids[x]]->id); + } + for (uint32_t x = 0; x < array[i].isset_sids_idx; x++) { + Signature *s = de_ctx->sig_array[array[i].isset_sids[x]]; + SCLogDebug("GET flowbit %s/%u: SID %u", varname, i, s->id); + + if (to_state) { + s->init_data->init_flags |= SIG_FLAG_INIT_STATE_MATCH; + SCLogDebug("made SID %u stateful because it depends on " + "stateful rules that set flowbit %s", + s->id, varname); + } + } } - SCFree(array); - return 0; + return fba; } SCMutex g_flowbits_dump_write_m = SCMUTEX_INITIALIZER; @@ -745,6 +860,482 @@ static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx, jb_free(js); } +static bool PrefilterFlowbitIsPrefilterable(const Signature *s) +{ + SCLogDebug("sid:%u: checking", s->id); + + for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; + sm = sm->next) { + switch (sm->type) { + case DETECT_FLOWBITS: { + const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { + SCLogDebug("sid:%u: FLOWBITS ISSET can prefilter", s->id); + return true; + } + } break; + } + } + SCLogDebug("sid:%u: no flowbit prefilter", s->id); + return false; +} + +#include "detect-engine-prefilter.h" +#include "tree.h" + +typedef struct PrefilterFlowbit { + uint32_t id; + uint16_t rule_id_size; + uint16_t rule_id_cnt; + uint32_t *rule_id; + RB_ENTRY(PrefilterFlowbit) __attribute__((__packed__)) rb; +} __attribute__((__packed__)) PrefilterFlowbit; + +static int PrefilterFlowbitCompare(const PrefilterFlowbit *a, const PrefilterFlowbit *b) +{ + if (a->id > b->id) + return 1; + else if (a->id < b->id) + return -1; + else + return 0; +} + +/* red-black tree prototype for PFB (Prefilter Flow Bits) */ +RB_HEAD(PFB, PrefilterFlowbit); +RB_PROTOTYPE(PFB, PrefilterFlowbit, rb, PrefilterFlowbitCompare); +RB_GENERATE(PFB, PrefilterFlowbit, rb, PrefilterFlowbitCompare); + +struct PrefilterEngineFlowbits { + struct PFB fb_tree; +}; + +static void PrefilterFlowbitFree(void *vctx) +{ + struct PrefilterEngineFlowbits *ctx = vctx; + struct PrefilterFlowbit *rec, *safe = NULL; + RB_FOREACH_SAFE (rec, PFB, &ctx->fb_tree, safe) { + PFB_RB_REMOVE(&ctx->fb_tree, rec); + SCFree(rec->rule_id); + SCFree(rec); + } + + SCFree(ctx); +} + +static void PrefilterFlowbitMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx) +{ + struct PrefilterEngineFlowbits *ctx = (struct PrefilterEngineFlowbits *)pectx; + SCLogDebug("%" PRIu64 ": ctx %p", p->pcap_cnt, ctx); + + if (p->flow == NULL) { + SCReturn; + } + + for (GenericVar *gv = p->flow->flowvar; gv != NULL; gv = gv->next) { + if (gv->type != DETECT_FLOWBITS) + continue; + + PrefilterFlowbit lookup; + memset(&lookup, 0, sizeof(lookup)); + lookup.id = gv->idx; + SCLogDebug("flowbit %u", gv->idx); + + PrefilterFlowbit *b = PFB_RB_FIND(&ctx->fb_tree, &lookup); + if (b == NULL) { + SCLogDebug("flowbit %u not in the tree", lookup.id); + } else { + SCLogDebug("flowbit %u found in the tree: %u", lookup.id, b->id); + + // TODO check/log which are in the tree here + // TODO rule id's wrong? + + PrefilterAddSids(&det_ctx->pmq, b->rule_id, b->rule_id_cnt); +#ifdef DEBUG + for (uint32_t x = 0; x < b->rule_id_cnt; x++) { + Signature *s = det_ctx->de_ctx->sig_array[b->rule_id[x]]; + SCLogDebug("flowbit %u -> sig %u", gv->idx, s->id); + } +#endif + } + } +} + +static void PrefilterFlowbitPostRuleMatch( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f) +{ + struct PrefilterEngineFlowbits *ctx = (struct PrefilterEngineFlowbits *)pectx; + SCLogDebug("%" PRIu64 ": ctx %p", p->pcap_cnt, ctx); + + if (p->flow == NULL) { + SCReturn; + } + + for (uint32_t i = 0; i < det_ctx->post_rule_work_queue.len; i++) { + const PostRuleMatchWorkQueueItem *w = &det_ctx->post_rule_work_queue.q[i]; + if (w->sm_type != DETECT_FLOWBITS) + continue; + + PrefilterFlowbit lookup; + memset(&lookup, 0, sizeof(lookup)); + lookup.id = w->value; + + PrefilterFlowbit *b = PFB_RB_FIND(&ctx->fb_tree, &lookup); + if (b == NULL) { + SCLogDebug("flowbit %u not in the tree", lookup.id); + } else { + SCLogDebug("flowbit %u found in the tree: %u. Adding %u sids", lookup.id, b->id, + b->rule_id_cnt); + PrefilterAddSids(&det_ctx->pmq, b->rule_id, b->rule_id_cnt); +#ifdef DEBUG + // SCLogDebug("b %u", b->rule_id_cnt); + for (uint32_t x = 0; x < b->rule_id_cnt; x++) { + Signature *s = det_ctx->de_ctx->sig_array[b->rule_id[x]]; + SCLogDebug("flowbit %u -> sig %u (triggered by %u)", w->value, s->id, + det_ctx->de_ctx->sig_array[w->id]->id); + } +#endif + } + } +} + +#define BLOCK_SIZE 8 + +static int AddBitIsset(const DetectEngineCtx *de_ctx, struct PrefilterEngineFlowbits *ctx, + const DetectFlowbitsData *fb, const Signature *s) +{ + if (fb->or_list_size == 0) { + PrefilterFlowbit x; + memset(&x, 0, sizeof(x)); + x.id = fb->idx; + + PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x); + if (pfb == NULL) { + PrefilterFlowbit *add = SCCalloc(1, sizeof(*add)); + BUG_ON(add == NULL); + + add->id = fb->idx; + add->rule_id = SCCalloc(1, BLOCK_SIZE * sizeof(uint32_t)); + BUG_ON(add->rule_id == NULL); + add->rule_id_size = BLOCK_SIZE; + add->rule_id_cnt = 1; + add->rule_id[0] = s->num; + + PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add); + SCLogDebug("not found, so added (res %p)", res); + if (res != NULL) { + // duplicate, shouldn't be possible after the FIND above + BUG_ON(1); + return -1; + } + } else { + SCLogDebug("found! pfb %p id %u", pfb, pfb->id); + + if (pfb->rule_id_cnt < pfb->rule_id_size) { + pfb->rule_id[pfb->rule_id_cnt++] = s->num; + } else { + uint32_t *ptr = SCRealloc( + pfb->rule_id, (pfb->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t)); + BUG_ON(ptr == NULL); + pfb->rule_id = ptr; + pfb->rule_id_size += BLOCK_SIZE; + pfb->rule_id[pfb->rule_id_cnt++] = s->num; + } + } + } else { + for (uint8_t i = 0; i < fb->or_list_size; i++) { + PrefilterFlowbit x; + memset(&x, 0, sizeof(x)); + x.id = fb->or_list[i]; + SCLogDebug("flowbit OR: bit %u", fb->or_list[i]); + + PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x); + if (pfb == NULL) { + PrefilterFlowbit *add = SCCalloc(1, sizeof(*add)); + BUG_ON(add == NULL); + + add->id = x.id; + add->rule_id = SCCalloc(1, BLOCK_SIZE * sizeof(uint32_t)); + BUG_ON(add->rule_id == NULL); + add->rule_id_size = BLOCK_SIZE; + add->rule_id_cnt = 1; + add->rule_id[0] = s->num; + + PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add); + SCLogDebug("not found, so added (res %p)", res); + if (res != NULL) { + // duplicate, shouldn't be possible after the FIND above + BUG_ON(1); + return -1; + } + } else { + SCLogDebug("found! pfb %p id %u", pfb, pfb->id); + + if (pfb->rule_id_cnt < pfb->rule_id_size) { + pfb->rule_id[pfb->rule_id_cnt++] = s->num; + } else { + uint32_t *ptr = SCRealloc( + pfb->rule_id, (pfb->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t)); + BUG_ON(ptr == NULL); + pfb->rule_id = ptr; + pfb->rule_id_size += BLOCK_SIZE; + pfb->rule_id[pfb->rule_id_cnt++] = s->num; + } + } + } + } + return 0; +} + +static uint32_t NextMultiple(const uint32_t v, const uint32_t m) +{ + return v + (m - v % m); +} + +/** \internal + * \brief adds sids for 'isset' prefilter flowbits + * \retval int 1 if we added sid(s), 0 if we didn't, -1 on error */ +// TODO skip sids that aren't set by this sgh +// TODO skip sids that doesn't have a isset in the same direction +static int AddSids(const DetectEngineCtx *de_ctx, const struct FBAnalyzer *fba, + const DetectFlowbitsData *fb, PrefilterFlowbit *add) +{ + int added = 0; + for (uint32_t i = 0; i < fba->array[fb->idx].isset_sids_idx; i++) { + const uint32_t sig_iid = fba->array[fb->idx].isset_sids[i]; + const Signature *s = de_ctx->sig_array[sig_iid]; + SCLogDebug("flowbit: %u => considering sid %u (iid:%u)", fb->idx, s->id, s->num); + + /* Skip sids that aren't prefilter. These would just run all the time. */ + if (s->init_data->prefilter_sm == NULL || + s->init_data->prefilter_sm->type != DETECT_FLOWBITS) { +#ifdef DEBUG + const char *name = s->init_data->prefilter_sm + ? sigmatch_table[s->init_data->prefilter_sm->type].name + : "none"; + SCLogDebug("flowbit: %u => rejected sid %u (iid:%u). No prefilter or prefilter not " + "flowbits (%p, %s, %d)", + fb->idx, s->id, sig_iid, s->init_data->prefilter_sm, name, + s->init_data->prefilter_sm ? s->init_data->prefilter_sm->type : -1); +#endif + continue; + } + + /* only add sids that match our bit */ + const DetectFlowbitsData *fs_fb = + (const DetectFlowbitsData *)s->init_data->prefilter_sm->ctx; + if (fs_fb->idx != fb->idx) { + SCLogDebug( + "flowbit: %u => rejected sid %u (iid:%u). Sig prefilters on different bit %u", + fb->idx, s->id, sig_iid, fs_fb->idx); + continue; + } + + bool dup = false; + for (uint32_t x = 0; x < add->rule_id_cnt; x++) { + if (add->rule_id[x] == sig_iid) { + dup = true; + } + } + + if (!dup) { + if (add->rule_id_cnt < add->rule_id_size) { + add->rule_id[add->rule_id_cnt++] = sig_iid; + } else { + uint32_t *ptr = SCRealloc( + add->rule_id, (add->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t)); + if (ptr == NULL) { + return -1; + } + add->rule_id = ptr; + add->rule_id_size += BLOCK_SIZE; + add->rule_id[add->rule_id_cnt++] = sig_iid; + } + added = 1; + SCLogDebug("flowbit: %u => accepted sid %u (iid:%u)", fb->idx, s->id, sig_iid); + } + } + return added; +} + +/* TODO shouldn't add sids for which Signature::num is < our num. Is this possible after sorting? */ +static int AddBitSetToggle(const DetectEngineCtx *de_ctx, struct FBAnalyzer *fba, + struct PrefilterEngineFlowbits *ctx, const DetectFlowbitsData *fb, const Signature *s) +{ + PrefilterFlowbit x; + memset(&x, 0, sizeof(x)); + x.id = fb->idx; + PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x); + if (pfb == NULL) { + PrefilterFlowbit *add = SCCalloc(1, sizeof(*add)); + if (add == NULL) + return -1; + + add->id = fb->idx; + add->rule_id_size = NextMultiple(fba->array[fb->idx].isset_sids_idx, BLOCK_SIZE); + add->rule_id = SCCalloc(1, add->rule_id_size * sizeof(uint32_t)); + if (add->rule_id == NULL) { + SCFree(add); + return -1; + } + + if (!AddSids(de_ctx, fba, fb, add)) { + SCLogDebug("no sids added"); + SCFree(add->rule_id); + SCFree(add); + return 0; + } + PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add); + SCLogDebug("not found, so added (res %p)", res); + BUG_ON(res != NULL); // TODO if res != NULL we have a duplicate which should be impossible + } else { + SCLogDebug("found! pfb %p id %u", pfb, pfb->id); + + int r = AddSids(de_ctx, fba, fb, pfb); + if (r < 0) { + return -1; + } else if (r == 0) { + SCLogDebug("no sids added"); + return 0; + } + } + return 1; +} + +/** \brief build flowbit prefilter state(s) + * + * Build "set" and "isset" states. + * + * For each flowbit "isset" in the sgh, we need to check: + * 1. is it supported + * 2. is prefilter enabled + * 3. does it match in the same dir or only opposing dir + */ +static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + if (sgh == NULL) + return 0; + + SCLogDebug("sgh %p: setting up prefilter", sgh); + struct FBAnalyzer fb_analysis = DetectFlowbitsAnalyzeForGroup(de_ctx, sgh); + struct PrefilterEngineFlowbits *isset_ctx = NULL; + struct PrefilterEngineFlowbits *set_ctx = NULL; + + for (uint32_t i = 0; i < sgh->init->sig_cnt; i++) { + Signature *s = sgh->init->match_array[i]; + if (s == NULL) + continue; + + SCLogDebug("checking sid %u", s->id); + + /* first build the 'set' state */ + for (SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH]; sm != NULL; + sm = sm->next) { + if (sm->type != DETECT_FLOWBITS) { + SCLogDebug("skip non flowbits sm"); + continue; + } + + DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + if (fb->cmd == DETECT_FLOWBITS_CMD_SET) { + SCLogDebug( + "DETECT_SM_LIST_POSTMATCH: sid %u DETECT_FLOWBITS set %u", s->id, fb->idx); + // else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) { + // SCLogDebug("DETECT_SM_LIST_POSTMATCH: sid %u DETECT_FLOWBITS toggle %u", s->id, + // fb->idx); + } else { + SCLogDebug("unsupported flowbits setting"); + continue; + } + + if (fb_analysis.array[fb->idx].toggle_sids_idx || + fb_analysis.array[fb->idx].isnotset_sids_idx || + fb_analysis.array[fb->idx].unset_sids_idx) { + SCLogDebug("flowbit %u not supported: toggle or unset in use", fb->idx); + continue; + } + + if (set_ctx == NULL) { + set_ctx = SCCalloc(1, sizeof(*set_ctx)); + if (set_ctx == NULL) + goto error; + } + + SCLogDebug("setting up sets/toggles for sid %u", s->id); + if (AddBitSetToggle(de_ctx, &fb_analysis, set_ctx, fb, s) == 1) { + fb->prefilter = true; // flag the set/toggle to trigger the post-rule match logic + } + + // TODO don't add for sigs that don't have isset in this sgh. Reasoning: + // prefilter post match logic only makes sense in the same dir as otherwise + // the regular 'isset' logic can simply run with the regular prefilters + // before the rule loop + } + + /* next, build the 'isset' state */ + if (s->init_data->prefilter_sm == NULL || + s->init_data->prefilter_sm->type != DETECT_FLOWBITS) { + SCLogDebug("no prefilter or prefilter not flowbits"); + continue; + } + + const DetectFlowbitsData *fb = (DetectFlowbitsData *)s->init_data->prefilter_sm->ctx; + if (fb_analysis.array[fb->idx].toggle_sids_idx || + fb_analysis.array[fb->idx].isnotset_sids_idx || + fb_analysis.array[fb->idx].unset_sids_idx) { + SCLogDebug("flowbit %u not supported: toggle or unset in use", fb->idx); + s->init_data->prefilter_sm = NULL; + s->flags &= ~SIG_FLAG_PREFILTER; + continue; + } + + SCLogDebug("isset: adding sid %u, flowbit %u", s->id, fb->idx); + + if (isset_ctx == NULL) { + isset_ctx = SCCalloc(1, sizeof(*isset_ctx)); + if (isset_ctx == NULL) + goto error; + } + if (AddBitIsset(de_ctx, isset_ctx, fb, s) < 0) { + goto error; + } + } + + /* finally, register the states with their engines */ + static const char *g_prefilter_flowbits_isset = "flowbits:isset"; + if (isset_ctx != NULL) { + PrefilterAppendEngine(de_ctx, sgh, PrefilterFlowbitMatch, SIG_MASK_REQUIRE_FLOW, isset_ctx, + PrefilterFlowbitFree, g_prefilter_flowbits_isset); + SCLogDebug("isset: added prefilter engine"); + + if (set_ctx != NULL && !RB_EMPTY(&set_ctx->fb_tree)) { + static const char *g_prefilter_flowbits_set = "flowbits:set"; + PrefilterAppendPostRuleEngine(de_ctx, sgh, PrefilterFlowbitPostRuleMatch, set_ctx, + PrefilterFlowbitFree, g_prefilter_flowbits_set); + SCLogDebug("set/toggle: added prefilter engine"); + } else { + if (set_ctx) { + PrefilterFlowbitFree(set_ctx); + } + SCLogDebug("set/toggle: NO prefilter engine added"); + } + } else if (set_ctx != NULL) { + PrefilterFlowbitFree(set_ctx); + } + FBAnalyzerFree(&fb_analysis); + return 0; + +error: + if (set_ctx) { + PrefilterFlowbitFree(set_ctx); + } + if (isset_ctx) { + PrefilterFlowbitFree(isset_ctx); + } + FBAnalyzerFree(&fb_analysis); + return -1; +} + #ifdef UNITTESTS static int FlowBitsTestParse01(void) diff --git a/src/detect-flowbits.h b/src/detect-flowbits.h index 0a5823ef60b3..fc68dbdca779 100644 --- a/src/detect-flowbits.h +++ b/src/detect-flowbits.h @@ -36,6 +36,7 @@ typedef struct DetectFlowbitsData_ { uint32_t idx; uint8_t cmd; uint8_t or_list_size; + bool prefilter; uint32_t *or_list; } DetectFlowbitsData; diff --git a/src/detect-prefilter.c b/src/detect-prefilter.c index f38b56bf8b9c..bd5a6445a4f1 100644 --- a/src/detect-prefilter.c +++ b/src/detect-prefilter.c @@ -99,6 +99,8 @@ static int DetectPrefilterSetup (DetectEngineCtx *de_ctx, Signature *s, const ch } s->init_data->prefilter_sm = sm; + SCLogDebug( + "sid %u: prefilter is on \"%s\" (%u)", s->id, sigmatch_table[sm->type].name, sm->type); SCReturnInt(0); } diff --git a/src/detect.c b/src/detect.c index 03fa8437068d..188cf136f27c 100644 --- a/src/detect.c +++ b/src/detect.c @@ -725,6 +725,16 @@ static inline void DetectRunPrefilterPkt( #endif } +static int SortHelper(const void *a, const void *b) +{ + const Signature *sa = *(const Signature **)a; + const Signature *sb = *(const Signature **)b; + if (sa->num == sb->num) + return 0; + else + return sa->num > sb->num ? 1 : -1; +} + static inline void DetectRulePacketRules( ThreadVars * const tv, DetectEngineCtx * const de_ctx, @@ -772,7 +782,7 @@ static inline void DetectRulePacketRules( } const uint8_t s_proto_flags = s->proto.flags; - SCLogDebug("inspecting signature id %"PRIu32"", s->id); + SCLogDebug("packet %" PRIu64 ": inspecting signature id %" PRIu32 "", p->pcap_cnt, s->id); if (s->app_inspect != NULL) { goto next; // handle sig in DetectRunTx @@ -834,6 +844,51 @@ static inline void DetectRulePacketRules( } } AlertQueueAppend(det_ctx, s, p, txid, alert_flags); + + if (det_ctx->post_rule_work_queue.len > 0) { + /* run post match prefilter engines on work queue */ + PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, pflow); + + if (det_ctx->pmq.rule_id_array_cnt > 0) { + /* undo "prefetch" */ + if (next_s) + match_array--; + /* create temporary rule pointer array starting + * at where we are in the current match array */ + const Signature *replace[de_ctx->sig_array_len]; // TODO heap? + SCLogDebug("sig_array_len %u det_ctx->pmq.rule_id_array_cnt %u", + de_ctx->sig_array_len, det_ctx->pmq.rule_id_array_cnt); + const Signature **r = replace; + for (uint32_t x = 0; x < match_cnt; x++) { + *r++ = match_array[x]; + SCLogDebug("appended %u", match_array[x]->id); + } + /* append the prefilter results, then sort it */ + for (uint32_t x = 0; x < det_ctx->pmq.rule_id_array_cnt; x++) { + // TODO what happens if a tx engine is added? + SCLogDebug("adding iid %u", det_ctx->pmq.rule_id_array[x]); + Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[x]]; + SCLogDebug("adding id %u", ts->id); + if (ts->app_inspect == NULL) { + *r++ = ts; + match_cnt++; + } + } + // TODO if sid 12 is added twice, how do we dedup + qsort(replace, match_cnt, sizeof(Signature *), SortHelper); + /* rewrite match_array to include the new additions */ + Signature **m = match_array; + for (uint32_t x = 0; x < match_cnt; x++) { + *m++ = (Signature *)replace[x]; + } + /* prefetch next */ + next_s = *match_array++; + next_sflags = next_s->flags; + SCLogDebug("%u rules added", det_ctx->pmq.rule_id_array_cnt); + det_ctx->post_rule_work_queue.len = 0; + PMQ_RESET(&det_ctx->pmq); + } + } next: DetectVarProcessList(det_ctx, pflow, p); DetectReplaceFree(det_ctx); @@ -1609,6 +1664,38 @@ static void DetectRunTx(ThreadVars *tv, } DetectVarProcessList(det_ctx, p->flow, p); RULE_PROFILING_END(det_ctx, s, r, p); + + if (det_ctx->post_rule_work_queue.len > 0) { + SCLogDebug("%p/%" PRIu64 " post_rule_work_queue len %u", tx.tx_ptr, tx.tx_id, + det_ctx->post_rule_work_queue.len); + /* run post match prefilter engines on work queue */ + PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, f); + + uint32_t prev_array_idx = array_idx; + for (uint32_t j = 0; j < det_ctx->pmq.rule_id_array_cnt; j++) { + const Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[j]]; + if (ts->app_inspect != NULL) { + const SigIntId id = ts->num; + det_ctx->tx_candidates[array_idx].s = ts; + det_ctx->tx_candidates[array_idx].id = id; + det_ctx->tx_candidates[array_idx].flags = NULL; + det_ctx->tx_candidates[array_idx].stream_reset = 0; + array_idx++; + + SCLogDebug("%p/%" PRIu64 " rule %u (%u) added from 'post match' prefilter", + tx.tx_ptr, tx.tx_id, ts->id, id); + } + } + SCLogDebug("%p/%" PRIu64 " rules added from 'post match' prefilter: %u", tx.tx_ptr, + tx.tx_id, array_idx - prev_array_idx); + if (prev_array_idx != array_idx) { + /* sort, but only part of array we're still going to process */ + qsort(det_ctx->tx_candidates + i, array_idx - i, sizeof(RuleMatchCandidateTx), + DetectRunTxSortHelper); + } + det_ctx->post_rule_work_queue.len = 0; + PMQ_RESET(&det_ctx->pmq); + } } det_ctx->tx_id = 0; diff --git a/src/detect.h b/src/detect.h index 3970764f8ea9..04336b4490a3 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1090,6 +1090,20 @@ typedef struct RuleMatchCandidateTx { const Signature *s; /**< ptr to sig */ } RuleMatchCandidateTx; +typedef struct PostRuleMatchWorkQueueItem { + int sm_type; + uint32_t value; +#ifdef DEBUG + SigIntId id; +#endif +} PostRuleMatchWorkQueueItem; + +typedef struct PostRuleMatchWorkQueue { + PostRuleMatchWorkQueueItem *q; + uint32_t len; + uint32_t size; +} PostRuleMatchWorkQueue; + /** * Detection engine thread data. */ @@ -1200,6 +1214,9 @@ typedef struct DetectEngineThreadCtx_ { uint32_t non_pf_store_cnt; MpmThreadCtx mtc; /**< thread ctx for the mpm */ + /* work queue for post-rule matching affecting prefilter */ + PostRuleMatchWorkQueue post_rule_work_queue; + PrefilterRuleStore pmq; /* string to replace */ @@ -1391,6 +1408,8 @@ typedef struct PrefilterEngineList_ { PrefilterPktFn Prefilter; PrefilterTxFn PrefilterTx; PrefilterFrameFn PrefilterFrame; + void (*PrefilterPostRule)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f); struct PrefilterEngineList_ *next; @@ -1424,6 +1443,8 @@ typedef struct PrefilterEngine_ { PrefilterPktFn Prefilter; PrefilterTxFn PrefilterTx; PrefilterFrameFn PrefilterFrame; + void (*PrefilterPostRule)( + DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f); } cb; /* global id for this prefilter */ @@ -1451,6 +1472,7 @@ typedef struct SigGroupHeadInitData_ { PrefilterEngineList *payload_engines; PrefilterEngineList *tx_engines; PrefilterEngineList *frame_engines; + PrefilterEngineList *post_rule_match_engines; /** number of sigs in this group */ SigIntId sig_cnt; @@ -1481,6 +1503,7 @@ typedef struct SigGroupHead_ { PrefilterEngine *payload_engines; PrefilterEngine *tx_engines; PrefilterEngine *frame_engines; + PrefilterEngine *post_rule_match_engines; /**< engines to run after rules modified a state */ /* ptr to our init data we only use at... init :) */ SigGroupHeadInitData *init; From 5c71c86fa5042230093a2d9079637f6948014d11 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 19 Nov 2024 16:41:45 +0100 Subject: [PATCH 05/17] SQUASH scan-build issues --- src/detect-flowbits.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index a3d038f77b5a..1bb4e7784e1b 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -690,7 +690,6 @@ static struct FBAnalyzer DetectFlowbitsAnalyzeForGroup( if (max_fb_id == 0) return fba; -#define MAX_SIDS 8 uint32_t array_size = max_fb_id + 1; struct FBAnalyze *array = SCCalloc(array_size, sizeof(struct FBAnalyze)); if (array == NULL) { @@ -709,9 +708,7 @@ static struct FBAnalyzer DetectFlowbitsAnalyzeForGroup( int r = DetectFlowbitsAnalyzeSignature(s, &fba); if (r < 0) { - SCFree(fba.array); - array = fba.array = NULL; - array_size = fba.array_size = 0; + FBAnalyzerFree(&fba); return fba; } } @@ -1217,9 +1214,11 @@ static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh) return 0; SCLogDebug("sgh %p: setting up prefilter", sgh); - struct FBAnalyzer fb_analysis = DetectFlowbitsAnalyzeForGroup(de_ctx, sgh); struct PrefilterEngineFlowbits *isset_ctx = NULL; struct PrefilterEngineFlowbits *set_ctx = NULL; + struct FBAnalyzer fb_analysis = DetectFlowbitsAnalyzeForGroup(de_ctx, sgh); + if (fb_analysis.array == NULL) + goto error; for (uint32_t i = 0; i < sgh->init->sig_cnt; i++) { Signature *s = sgh->init->match_array[i]; From dc397372a2e43632555979bbec3a259d2e606653 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 19 Nov 2024 16:49:09 +0100 Subject: [PATCH 06/17] WIP PostRuleMatchWorkQueueItem doc --- src/detect.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/detect.h b/src/detect.h index 04336b4490a3..35bbf278a91c 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1090,18 +1090,23 @@ typedef struct RuleMatchCandidateTx { const Signature *s; /**< ptr to sig */ } RuleMatchCandidateTx; +/** Stores a single u32 for a rule match of the type `sm_type`. Used by + * flowbits prefilter to register DETECT_FLOWBITS, for post + * match handling. */ typedef struct PostRuleMatchWorkQueueItem { - int sm_type; - uint32_t value; + int sm_type; /**< sigmatch type e.g. DETECT_FLOWBITS */ + uint32_t value; /**< value to be interpreted by the sm_type + * implementation. E.g. flowbit id. */ #ifdef DEBUG SigIntId id; #endif } PostRuleMatchWorkQueueItem; +/** Array of PostRuleMatchWorkQueueItem's. */ typedef struct PostRuleMatchWorkQueue { - PostRuleMatchWorkQueueItem *q; - uint32_t len; - uint32_t size; + PostRuleMatchWorkQueueItem *q; /**< array pointer */ + uint32_t len; /**< number of array elements in use. */ + uint32_t size; /**< allocation size in number of elements. */ } PostRuleMatchWorkQueue; /** From ef2360eced135d1d68ce0596335b9847445514a8 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 19 Nov 2024 16:57:54 +0100 Subject: [PATCH 07/17] SQUASH docs --- src/detect-flowbits.c | 12 +++++++----- src/detect-flowbits.h | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 1bb4e7784e1b..1cac76d84f9a 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -880,11 +880,13 @@ static bool PrefilterFlowbitIsPrefilterable(const Signature *s) #include "detect-engine-prefilter.h" #include "tree.h" +/** core flowbit data structure: map a flowbit id to the signatures that need inspecting after it is + * found. Part of a rb-tree. */ typedef struct PrefilterFlowbit { - uint32_t id; - uint16_t rule_id_size; - uint16_t rule_id_cnt; - uint32_t *rule_id; + uint32_t id; /**< flowbit id */ + uint16_t rule_id_size; /**< size in elements of `rule_id` */ + uint16_t rule_id_cnt; /**< usage in elements of `rule_id` */ + uint32_t *rule_id; /**< array of signature iid that are part of this prefilter */ RB_ENTRY(PrefilterFlowbit) __attribute__((__packed__)) rb; } __attribute__((__packed__)) PrefilterFlowbit; @@ -898,7 +900,7 @@ static int PrefilterFlowbitCompare(const PrefilterFlowbit *a, const PrefilterFlo return 0; } -/* red-black tree prototype for PFB (Prefilter Flow Bits) */ +/** red-black tree prototype for PFB (Prefilter Flow Bits) */ RB_HEAD(PFB, PrefilterFlowbit); RB_PROTOTYPE(PFB, PrefilterFlowbit, rb, PrefilterFlowbitCompare); RB_GENERATE(PFB, PrefilterFlowbit, rb, PrefilterFlowbitCompare); diff --git a/src/detect-flowbits.h b/src/detect-flowbits.h index fc68dbdca779..3d34bb824fd7 100644 --- a/src/detect-flowbits.h +++ b/src/detect-flowbits.h @@ -36,7 +36,7 @@ typedef struct DetectFlowbitsData_ { uint32_t idx; uint8_t cmd; uint8_t or_list_size; - bool prefilter; + bool prefilter; /**< set/toggle command should trigger post-rule-match "prefilter" */ uint32_t *or_list; } DetectFlowbitsData; From 3c68795ee0049542eb92e4e9660fd8f80baa2b2a Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 19 Nov 2024 17:02:59 +0100 Subject: [PATCH 08/17] SQUASH flowbits todo removal --- src/detect-flowbits.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 1cac76d84f9a..171a2396fede 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -946,13 +946,10 @@ static void PrefilterFlowbitMatch(DetectEngineThreadCtx *det_ctx, Packet *p, con } else { SCLogDebug("flowbit %u found in the tree: %u", lookup.id, b->id); - // TODO check/log which are in the tree here - // TODO rule id's wrong? - PrefilterAddSids(&det_ctx->pmq, b->rule_id, b->rule_id_cnt); #ifdef DEBUG for (uint32_t x = 0; x < b->rule_id_cnt; x++) { - Signature *s = det_ctx->de_ctx->sig_array[b->rule_id[x]]; + const Signature *s = det_ctx->de_ctx->sig_array[b->rule_id[x]]; SCLogDebug("flowbit %u -> sig %u", gv->idx, s->id); } #endif From acf22c5f67364d7ae4f5b27854449ba77a55c741 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 19 Nov 2024 17:03:50 +0100 Subject: [PATCH 09/17] SQUASH flowbits cleanup --- src/detect-flowbits.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 171a2396fede..57f04b5ad150 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -505,7 +505,6 @@ static bool CheckExpand(const uint32_t sids_idx, uint32_t **sids, uint32_t *sids static int DetectFlowbitsAnalyzeSignature(const Signature *s, struct FBAnalyzer *fba) { - // const uint32_t array_size = fba->array_size; struct FBAnalyze *array = fba->array; /* see if the signature uses stateful matching TODO is there not a flag? */ From 7e796ab05009bcb1d22c241fdc5d0a98f9d08ae9 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 19 Nov 2024 20:35:55 +0100 Subject: [PATCH 10/17] SQUASH move common code to prefilter --- src/detect-engine-prefilter.c | 28 ++++++++++++++++++++++++++++ src/detect-engine-prefilter.h | 3 +++ src/detect-flowbits.c | 34 +++------------------------------- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c index 768549e4ea18..6675c7fe5e40 100644 --- a/src/detect-engine-prefilter.c +++ b/src/detect-engine-prefilter.c @@ -979,3 +979,31 @@ int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, M } return r; } + +// TODO move to common file +#define QUEUE_STEP 16 + +void PostRuleMatchWorkQueueAppend( + DetectEngineThreadCtx *det_ctx, const Signature *s, const int type, const uint32_t value) +{ + if (det_ctx->post_rule_work_queue.q == NULL) { + det_ctx->post_rule_work_queue.q = + SCCalloc(1, sizeof(PostRuleMatchWorkQueueItem) * QUEUE_STEP); + BUG_ON(det_ctx->post_rule_work_queue.q == NULL); + det_ctx->post_rule_work_queue.size = QUEUE_STEP; + } else if (det_ctx->post_rule_work_queue.len == det_ctx->post_rule_work_queue.size) { + void *ptr = SCRealloc( + det_ctx->post_rule_work_queue.q, (det_ctx->post_rule_work_queue.size + QUEUE_STEP) * + sizeof(PostRuleMatchWorkQueueItem)); + BUG_ON(ptr == NULL); + det_ctx->post_rule_work_queue.q = ptr; + det_ctx->post_rule_work_queue.size += QUEUE_STEP; + } + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].sm_type = type; + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].value = value; +#ifdef DEBUG + det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].id = s->num; +#endif + det_ctx->post_rule_work_queue.len++; + SCLogDebug("det_ctx->post_rule_work_queue.len %u", det_ctx->post_rule_work_queue.len); +} diff --git a/src/detect-engine-prefilter.h b/src/detect-engine-prefilter.h index 8f37f2b8a756..0b27c913826a 100644 --- a/src/detect-engine-prefilter.h +++ b/src/detect-engine-prefilter.h @@ -99,4 +99,7 @@ int PrefilterMultiGenericMpmRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, const DetectBufferMpmRegistry *mpm_reg, int list_id); +void PostRuleMatchWorkQueueAppend( + DetectEngineThreadCtx *det_ctx, const Signature *s, const int type, const uint32_t value); + #endif diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 57f04b5ad150..8feb7e9054fb 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -42,6 +42,9 @@ #include "detect-engine-mpm.h" #include "detect-engine-state.h" #include "detect-engine-build.h" +#include "detect-engine-prefilter.h" + +#include "tree.h" #include "util-var-name.h" #include "util-unittest.h" @@ -191,34 +194,6 @@ static int DetectFlowbitMatchIsnotset (Packet *p, const DetectFlowbitsData *fd) return FlowBitIsnotset(p->flow,fd->idx); } -// TODO move to common file -#define QUEUE_STEP 16 - -static void PostRuleMatchWorkQueueAppend( - DetectEngineThreadCtx *det_ctx, const Signature *s, const int type, const uint32_t value) -{ - if (det_ctx->post_rule_work_queue.q == NULL) { - det_ctx->post_rule_work_queue.q = - SCCalloc(1, sizeof(PostRuleMatchWorkQueueItem) * QUEUE_STEP); - BUG_ON(det_ctx->post_rule_work_queue.q == NULL); - det_ctx->post_rule_work_queue.size = QUEUE_STEP; - } else if (det_ctx->post_rule_work_queue.len == det_ctx->post_rule_work_queue.size) { - void *ptr = SCRealloc( - det_ctx->post_rule_work_queue.q, (det_ctx->post_rule_work_queue.size + QUEUE_STEP) * - sizeof(PostRuleMatchWorkQueueItem)); - BUG_ON(ptr == NULL); - det_ctx->post_rule_work_queue.q = ptr; - det_ctx->post_rule_work_queue.size += QUEUE_STEP; - } - det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].sm_type = type; - det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].value = value; -#ifdef DEBUG - det_ctx->post_rule_work_queue.q[det_ctx->post_rule_work_queue.len].id = s->num; -#endif - det_ctx->post_rule_work_queue.len++; - SCLogDebug("det_ctx->post_rule_work_queue.len %u", det_ctx->post_rule_work_queue.len); -} - /* * returns 0: no match * 1: match @@ -876,9 +851,6 @@ static bool PrefilterFlowbitIsPrefilterable(const Signature *s) return false; } -#include "detect-engine-prefilter.h" -#include "tree.h" - /** core flowbit data structure: map a flowbit id to the signatures that need inspecting after it is * found. Part of a rb-tree. */ typedef struct PrefilterFlowbit { From 11fb4c5cad1a0ae7b6f36309702267052fa7e3cd Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 19 Nov 2024 20:37:56 +0100 Subject: [PATCH 11/17] SQUASH error check --- src/detect-flowbits.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 8feb7e9054fb..0bdeb3128fc4 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -481,6 +481,8 @@ static bool CheckExpand(const uint32_t sids_idx, uint32_t **sids, uint32_t *sids static int DetectFlowbitsAnalyzeSignature(const Signature *s, struct FBAnalyzer *fba) { struct FBAnalyze *array = fba->array; + if (array == NULL) + return -1; /* see if the signature uses stateful matching TODO is there not a flag? */ bool has_state = (s->init_data->buffer_index != 0); From a29557c6ae4cc86659d69d30d914e17ee877f0c4 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 19 Nov 2024 21:09:27 +0100 Subject: [PATCH 12/17] SQUASH error check --- src/detect-flowbits.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 0bdeb3128fc4..cc6d75e51aba 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -459,6 +459,8 @@ static void FBAnalyzerFree(struct FBAnalyzer *fba) { if (fba && fba->array) { FBAnalyzerArrayFree(fba->array, fba->array_size); + fba->array = NULL; + fba->array_size = 0; } } @@ -1188,6 +1190,7 @@ static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh) SCLogDebug("sgh %p: setting up prefilter", sgh); struct PrefilterEngineFlowbits *isset_ctx = NULL; struct PrefilterEngineFlowbits *set_ctx = NULL; + struct FBAnalyzer fb_analysis = DetectFlowbitsAnalyzeForGroup(de_ctx, sgh); if (fb_analysis.array == NULL) goto error; From b991723943edcce74d524ee4f32d8decf567a42b Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 20 Nov 2024 13:45:43 +0100 Subject: [PATCH 13/17] SQUASH unittests after mpm changes --- src/detect-byte-extract.c | 84 +++++-------- src/detect-content.c | 2 +- src/tests/detect-http-client-body.c | 180 ++++++++++------------------ 3 files changed, 95 insertions(+), 171 deletions(-) diff --git a/src/detect-byte-extract.c b/src/detect-byte-extract.c index e9241488158f..d590183f9c77 100644 --- a/src/detect-byte-extract.c +++ b/src/detect-byte-extract.c @@ -1945,9 +1945,8 @@ static int DetectByteExtractTest43(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "three", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_OFFSET_VAR | - DETECT_CONTENT_OFFSET) || - cd->offset != bed->local_id) { + cd->flags != (DETECT_CONTENT_OFFSET_VAR | DETECT_CONTENT_OFFSET | DETECT_CONTENT_MPM) || + cd->offset != bed->local_id) { printf("three failed\n"); result = 0; goto end; @@ -2049,9 +2048,8 @@ static int DetectByteExtractTest44(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "four", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_OFFSET_VAR | - DETECT_CONTENT_OFFSET) || - cd->offset != bed1->local_id) { + cd->flags != (DETECT_CONTENT_OFFSET_VAR | DETECT_CONTENT_OFFSET | DETECT_CONTENT_MPM) || + cd->offset != bed1->local_id) { printf("four failed\n"); result = 0; goto end; @@ -2158,10 +2156,8 @@ static int DetectByteExtractTest45(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "three", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_DEPTH_VAR | - DETECT_CONTENT_DEPTH) || - cd->depth != bed->local_id || - cd->offset != 0) { + cd->flags != (DETECT_CONTENT_DEPTH_VAR | DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) || + cd->depth != bed->local_id || cd->offset != 0) { printf("three failed\n"); result = 0; goto end; @@ -2263,9 +2259,8 @@ static int DetectByteExtractTest46(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "four", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_DEPTH_VAR | - DETECT_CONTENT_DEPTH) || - cd->depth != bed1->local_id) { + cd->flags != (DETECT_CONTENT_DEPTH_VAR | DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) || + cd->depth != bed1->local_id) { printf("four failed\n"); result = 0; goto end; @@ -2372,11 +2367,9 @@ static int DetectByteExtractTest47(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "three", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_DISTANCE_VAR | - DETECT_CONTENT_DISTANCE) || - cd->distance != bed->local_id || - cd->offset != 0 || - cd->depth != 0) { + cd->flags != + (DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM) || + cd->distance != bed->local_id || cd->offset != 0 || cd->depth != 0) { printf("three failed\n"); result = 0; goto end; @@ -2478,12 +2471,9 @@ static int DetectByteExtractTest48(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "four", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_DISTANCE_VAR | - DETECT_CONTENT_DISTANCE | - DETECT_CONTENT_DISTANCE_NEXT) || - cd->distance != bed1->local_id || - cd->depth != 0 || - cd->offset != 0) { + cd->flags != (DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_DISTANCE | + DETECT_CONTENT_DISTANCE_NEXT | DETECT_CONTENT_MPM) || + cd->distance != bed1->local_id || cd->depth != 0 || cd->offset != 0) { printf("four failed\n"); result = 0; goto end; @@ -2592,12 +2582,8 @@ static int DetectByteExtractTest49(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "three", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_WITHIN_VAR | - DETECT_CONTENT_WITHIN) || - cd->within != bed->local_id || - cd->offset != 0 || - cd->depth != 0 || - cd->distance != 0) { + cd->flags != (DETECT_CONTENT_WITHIN_VAR | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) || + cd->within != bed->local_id || cd->offset != 0 || cd->depth != 0 || cd->distance != 0) { printf("three failed\n"); result = 0; goto end; @@ -2699,13 +2685,10 @@ static int DetectByteExtractTest50(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "four", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_WITHIN_VAR | - DETECT_CONTENT_WITHIN| - DETECT_CONTENT_WITHIN_NEXT) || - cd->within != bed1->local_id || - cd->depth != 0 || - cd->offset != 0 || - cd->distance != 0) { + cd->flags != (DETECT_CONTENT_WITHIN_VAR | DETECT_CONTENT_WITHIN | + DETECT_CONTENT_WITHIN_NEXT | DETECT_CONTENT_MPM) || + cd->within != bed1->local_id || cd->depth != 0 || cd->offset != 0 || + cd->distance != 0) { printf("four failed\n"); result = 0; goto end; @@ -2970,7 +2953,7 @@ static int DetectByteExtractTest53(void) SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_PMATCH]; FAIL_IF(sm->type != DETECT_CONTENT); DetectContentData *cd = (DetectContentData *)sm->ctx; - FAIL_IF(cd->flags != 0); + FAIL_IF(cd->flags != (DETECT_CONTENT_MPM | DETECT_CONTENT_NO_DOUBLE_INSPECTION_REQUIRED)); sm = sm->next; FAIL_IF_NULL(sm); @@ -3092,11 +3075,8 @@ static int DetectByteExtractTest54(void) goto end; } bjd = (DetectBytejumpData *)sm->ctx; - if (bjd->flags != DETECT_BYTEJUMP_OFFSET_VAR || bjd->offset != 1) { - printf("four failed\n"); - result = 0; - goto end; - } + FAIL_IF(bjd->flags != DETECT_BYTEJUMP_OFFSET_VAR); + FAIL_IF(bjd->offset != 1); if (sm->next != NULL) goto end; @@ -3197,12 +3177,10 @@ static int DetectByteExtractTest55(void) } cd = (DetectContentData *)sm->ctx; if (strncmp((char *)cd->content, "four", cd->content_len) != 0 || - cd->flags != (DETECT_CONTENT_DISTANCE_VAR | - DETECT_CONTENT_WITHIN_VAR | - DETECT_CONTENT_DISTANCE | - DETECT_CONTENT_WITHIN) || - cd->within != bed1->local_id || - cd->distance != bed2->local_id) { + cd->flags != + (DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_WITHIN_VAR | + DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) || + cd->within != bed1->local_id || cd->distance != bed2->local_id) { printf("four failed: "); goto end; } @@ -3829,8 +3807,8 @@ static int DetectByteExtractTest60(void) goto end; } cd = (DetectContentData *)sm->ctx; - if (cd->flags != DETECT_CONTENT_RELATIVE_NEXT || - strncmp((char *)cd->content, "three", cd->content_len) != 0) { + if (cd->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM) || + strncmp((char *)cd->content, "three", cd->content_len) != 0) { printf("one failed\n"); result = 0; goto end; @@ -3950,8 +3928,8 @@ static int DetectByteExtractTest61(void) goto end; } cd = (DetectContentData *)sm->ctx; - if (cd->flags != DETECT_CONTENT_RELATIVE_NEXT || - strncmp((char *)cd->content, "three", cd->content_len) != 0) { + if (cd->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM) || + strncmp((char *)cd->content, "three", cd->content_len) != 0) { printf("one failed\n"); result = 0; goto end; diff --git a/src/detect-content.c b/src/detect-content.c index 9625e7426d45..849094ca223e 100644 --- a/src/detect-content.c +++ b/src/detect-content.c @@ -1523,7 +1523,7 @@ static int DetectContentParseTest19(void) FAIL_IF_NOT(s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL); DetectContentData *data = (DetectContentData *)sm->ctx; - FAIL_IF_NOT(data->flags == DETECT_CONTENT_DISTANCE); + FAIL_IF_NOT(data->flags == (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM)); s = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " diff --git a/src/tests/detect-http-client-body.c b/src/tests/detect-http-client-body.c index bbeb4d33bde8..707c7492cb1d 100644 --- a/src/tests/detect-http-client-body.c +++ b/src/tests/detect-http-client-body.c @@ -2231,7 +2231,7 @@ static int DetectHttpClientBodyTest22(void) FAIL_IF(memcmp(cd2->content, "four", cd2->content_len) != 0); FAIL_IF(hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT); FAIL_IF(memcmp(hcbd1->content, "two", hcbd1->content_len) != 0); - FAIL_IF(hcbd2->flags != DETECT_CONTENT_DISTANCE); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM)); FAIL_IF(memcmp(hcbd2->content, "three", hcbd1->content_len) != 0); FAIL_IF(!DETECT_CONTENT_IS_SINGLE(cd1)); @@ -2273,7 +2273,7 @@ static int DetectHttpClientBodyTest23(void) FAIL_IF(memcmp(cd2->content, "four", cd2->content_len) != 0); FAIL_IF(hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT); FAIL_IF(memcmp(hcbd1->content, "one", hcbd1->content_len) != 0); - FAIL_IF(hcbd2->flags != DETECT_CONTENT_DISTANCE); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM)); FAIL_IF(memcmp(hcbd2->content, "three", hcbd1->content_len) != 0); FAIL_IF(!DETECT_CONTENT_IS_SINGLE(cd2)); FAIL_IF(DETECT_CONTENT_IS_SINGLE(hcbd1)); @@ -2285,31 +2285,17 @@ static int DetectHttpClientBodyTest23(void) static int DetectHttpClientBodyTest24(void) { - DetectEngineCtx *de_ctx = NULL; - int result = 0; - - if ( (de_ctx = DetectEngineCtxInit()) == NULL) - goto end; - + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; Signature *s = DetectEngineAppendSig(de_ctx, "alert icmp any any -> any any " "(content:\"one\"; http_client_body; pcre:/two/; " "content:\"three\"; distance:10; within:15; " "http_client_body; content:\"four\"; sid:1;)"); - if (de_ctx->sig_list == NULL) { - printf("de_ctx->sig_list == NULL\n"); - goto end; - } - - if (de_ctx->sig_list->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) { - printf("de_ctx->sig_list->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL\n"); - goto end; - } + FAIL_IF_NULL(s); - if (DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id) == NULL) { - printf("DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id) == NULL\n"); - goto end; - } + FAIL_IF_NULL(de_ctx->sig_list->init_data->smlists[DETECT_SM_LIST_PMATCH]); + FAIL_IF_NULL(DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id)); DetectPcreData *pd1 = (DetectPcreData *)de_ctx->sig_list->init_data->smlists_tail[DETECT_SM_LIST_PMATCH] @@ -2322,56 +2308,35 @@ static int DetectHttpClientBodyTest24(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != 0 || - cd2->flags != 0 || memcmp(cd2->content, "four", cd2->content_len) != 0 || - hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN) || - memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { - goto end; - } - - if (!DETECT_CONTENT_IS_SINGLE(cd2) || - DETECT_CONTENT_IS_SINGLE(hcbd1) || - DETECT_CONTENT_IS_SINGLE(hcbd2)) { - goto end; - } + FAIL_IF(pd1->flags != 0); + FAIL_IF(cd2->flags != 0); + FAIL_IF(memcmp(cd2->content, "four", cd2->content_len) != 0); + FAIL_IF(hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT); + FAIL_IF(memcmp(hcbd1->content, "one", hcbd1->content_len) != 0); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd2->content, "three", hcbd1->content_len) != 0); - result = 1; + FAIL_IF(!DETECT_CONTENT_IS_SINGLE(cd2)); + FAIL_IF(DETECT_CONTENT_IS_SINGLE(hcbd1)); + FAIL_IF(DETECT_CONTENT_IS_SINGLE(hcbd2)); - end: DetectEngineCtxFree(de_ctx); - return result; + PASS; } static int DetectHttpClientBodyTest25(void) { - DetectEngineCtx *de_ctx = NULL; - int result = 0; - - if ( (de_ctx = DetectEngineCtxInit()) == NULL) - goto end; - + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; Signature *s = DetectEngineAppendSig(de_ctx, "alert icmp any any -> any any " "(content:\"one\"; http_client_body; pcre:/two/; " "content:\"three\"; distance:10; http_client_body; " "content:\"four\"; distance:10; sid:1;)"); - if (de_ctx->sig_list == NULL) { - printf("de_ctx->sig_list == NULL\n"); - goto end; - } - - if (de_ctx->sig_list->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) { - printf("de_ctx->sig_list->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL\n"); - goto end; - } - - if (DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id) == NULL) { - printf("DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id) == NULL\n"); - goto end; - } + FAIL_IF_NULL(s); + FAIL_IF_NULL(s->init_data->smlists[DETECT_SM_LIST_PMATCH]); + FAIL_IF_NULL(DetectBufferGetFirstSigMatch(s, g_http_client_body_buffer_id)); DetectPcreData *pd1 = (DetectPcreData *)de_ctx->sig_list->init_data->smlists_tail[DETECT_SM_LIST_PMATCH] @@ -2384,27 +2349,20 @@ static int DetectHttpClientBodyTest25(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != DETECT_PCRE_RELATIVE_NEXT || - cd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(cd2->content, "four", cd2->content_len) != 0 || - hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { - goto end; - } - - if (DETECT_CONTENT_IS_SINGLE(cd2) || - DETECT_CONTENT_IS_SINGLE(hcbd1) || - DETECT_CONTENT_IS_SINGLE(hcbd2)) { - goto end; - } + FAIL_IF(pd1->flags != DETECT_PCRE_RELATIVE_NEXT); + FAIL_IF(cd2->flags != DETECT_CONTENT_DISTANCE); + FAIL_IF(memcmp(cd2->content, "four", cd2->content_len) != 0); + FAIL_IF(hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT); + FAIL_IF(memcmp(hcbd1->content, "one", hcbd1->content_len) != 0); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd2->content, "three", hcbd1->content_len) != 0); - result = 1; + FAIL_IF(DETECT_CONTENT_IS_SINGLE(cd2)); + FAIL_IF(DETECT_CONTENT_IS_SINGLE(hcbd1)); + FAIL_IF(DETECT_CONTENT_IS_SINGLE(hcbd2)); - end: DetectEngineCtxFree(de_ctx); - return result; + PASS; } static int DetectHttpClientBodyTest26(void) @@ -2447,13 +2405,13 @@ static int DetectHttpClientBodyTest26(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || - cd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(cd2->content, "four", cd2->content_len) != 0 || - hcbd1->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_OFFSET) || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN) || - memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { + if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || cd2->flags != DETECT_CONTENT_DISTANCE || + memcmp(cd2->content, "four", cd2->content_len) != 0 || + hcbd1->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_OFFSET) || + memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || + hcbd2->flags != + (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) || + memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { printf ("failed: http_client_body incorrect flags"); goto end; } @@ -2533,13 +2491,11 @@ static int DetectHttpClientBodyTest28(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || - cd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(cd2->content, "four", cd2->content_len) != 0 || - hcbd1->flags != 0 || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != DETECT_CONTENT_DEPTH || - memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { + if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || cd2->flags != DETECT_CONTENT_DISTANCE || + memcmp(cd2->content, "four", cd2->content_len) != 0 || hcbd1->flags != 0 || + memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || + hcbd2->flags != (DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) || + memcmp(hcbd2->content, "three", hcbd1->content_len) != 0) { goto end; } @@ -2589,12 +2545,10 @@ static int DetectHttpClientBodyTest29(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(hcbd2->content, "two", hcbd1->content_len) != 0) { - goto end; - } + FAIL_IF(hcbd1->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd1->content, "one", hcbd1->content_len) != 0); + FAIL_IF(hcbd2->flags != DETECT_CONTENT_DISTANCE); + FAIL_IF(memcmp(hcbd2->content, "two", hcbd1->content_len) != 0); result = 1; @@ -2636,12 +2590,10 @@ static int DetectHttpClientBodyTest30(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT || - memcmp(hcbd1->content, "one", hcbd1->content_len) != 0 || - hcbd2->flags != DETECT_CONTENT_WITHIN || - memcmp(hcbd2->content, "two", hcbd1->content_len) != 0) { - goto end; - } + FAIL_IF(hcbd1->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd1->content, "one", hcbd1->content_len) != 0); + FAIL_IF(hcbd2->flags != DETECT_CONTENT_WITHIN); + FAIL_IF(memcmp(hcbd2->content, "two", hcbd1->content_len) != 0); result = 1; @@ -2754,11 +2706,9 @@ static int DetectHttpClientBodyTest34(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || - hcbd2->flags != DETECT_CONTENT_WITHIN || - memcmp(hcbd2->content, "two", hcbd2->content_len) != 0) { - goto end; - } + FAIL_IF(pd1->flags != (DETECT_PCRE_RELATIVE_NEXT)); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd2->content, "two", hcbd2->content_len) != 0); result = 1; @@ -2808,11 +2758,9 @@ static int DetectHttpClientBodyTest35(void) ->prev->ctx; DetectPcreData *pd2 = (DetectPcreData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd2->flags != (DETECT_PCRE_RELATIVE) || - hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT || - memcmp(hcbd1->content, "two", hcbd1->content_len) != 0) { - goto end; - } + FAIL_IF(pd2->flags != (DETECT_PCRE_RELATIVE)); + FAIL_IF(hcbd1->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd1->content, "two", hcbd1->content_len) != 0); result = 1; @@ -2863,11 +2811,9 @@ static int DetectHttpClientBodyTest36(void) ->prev->ctx; DetectContentData *hcbd2 = (DetectContentData *)DetectBufferGetLastSigMatch(s, g_http_client_body_buffer_id)->ctx; - if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT) || - hcbd2->flags != DETECT_CONTENT_DISTANCE || - memcmp(hcbd2->content, "two", hcbd2->content_len) != 0) { - goto end; - } + FAIL_IF(pd1->flags != (DETECT_PCRE_RELATIVE_NEXT)); + FAIL_IF(hcbd2->flags != (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM)); + FAIL_IF(memcmp(hcbd2->content, "two", hcbd2->content_len) != 0); result = 1; From 03fc6874dec2561fea04d187b8a936db6b1ed567 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 20 Nov 2024 18:39:43 +0100 Subject: [PATCH 14/17] SQUASH flowbit prefilter: int warning --- src/detect-engine-prefilter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c index 6675c7fe5e40..9d41f00404a4 100644 --- a/src/detect-engine-prefilter.c +++ b/src/detect-engine-prefilter.c @@ -666,7 +666,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh) } memset(sgh->post_rule_match_engines, 0x00, (cnt * sizeof(PrefilterEngine))); - uint32_t local_id = 0; + uint16_t local_id = 0; PrefilterEngine *e = sgh->post_rule_match_engines; for (el = sgh->init->post_rule_match_engines; el != NULL; el = el->next) { e->local_id = local_id++; From 3a0f362e61748ebea08323d0a89c6ea09d489ed6 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 21 Nov 2024 11:42:32 +0100 Subject: [PATCH 15/17] SQUASH flowbit prefilter --- src/detect-flowbits.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index cc6d75e51aba..0dac81ff444e 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -859,8 +859,8 @@ static bool PrefilterFlowbitIsPrefilterable(const Signature *s) * found. Part of a rb-tree. */ typedef struct PrefilterFlowbit { uint32_t id; /**< flowbit id */ - uint16_t rule_id_size; /**< size in elements of `rule_id` */ - uint16_t rule_id_cnt; /**< usage in elements of `rule_id` */ + uint32_t rule_id_size; /**< size in elements of `rule_id` */ + uint32_t rule_id_cnt; /**< usage in elements of `rule_id` */ uint32_t *rule_id; /**< array of signature iid that are part of this prefilter */ RB_ENTRY(PrefilterFlowbit) __attribute__((__packed__)) rb; } __attribute__((__packed__)) PrefilterFlowbit; From 4280d8234f66c60f3e2ad51b47e7abda39b7a8f0 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 21 Nov 2024 12:43:01 +0100 Subject: [PATCH 16/17] SQUASH flowbit --- src/detect-flowbits.c | 135 ++++++++++++++++++------------------------ src/detect-flowbits.h | 4 +- 2 files changed, 62 insertions(+), 77 deletions(-) diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 0dac81ff444e..5e1616a72b05 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -215,7 +215,7 @@ int DetectFlowbitMatch (DetectEngineThreadCtx *det_ctx, Packet *p, case DETECT_FLOWBITS_CMD_SET: { int r = DetectFlowbitMatchSet(p, fd); /* only on a new "set" invoke the prefilter */ - if (r == 1 && fd->prefilter) { + if (r == 1 && fd->post_rule_match_prefilter) { SCLogDebug("flowbit set, appending to work queue"); PostRuleMatchWorkQueueAppend(det_ctx, s, DETECT_FLOWBITS, fd->idx); } @@ -225,7 +225,7 @@ int DetectFlowbitMatch (DetectEngineThreadCtx *det_ctx, Packet *p, return DetectFlowbitMatchUnset(p,fd); case DETECT_FLOWBITS_CMD_TOGGLE: { int r = DetectFlowbitMatchToggle(p, fd); - if (r == 1 && fd->prefilter) { + if (r == 1 && fd->post_rule_match_prefilter) { SCLogDebug("flowbit set (by toggle), appending to work queue"); PostRuleMatchWorkQueueAppend(det_ctx, s, DETECT_FLOWBITS, fd->idx); } @@ -972,86 +972,68 @@ static void PrefilterFlowbitPostRuleMatch( #define BLOCK_SIZE 8 -static int AddBitIsset(const DetectEngineCtx *de_ctx, struct PrefilterEngineFlowbits *ctx, - const DetectFlowbitsData *fb, const Signature *s) +static int AddBitAndSid( + struct PrefilterEngineFlowbits *ctx, const Signature *s, const uint32_t flowbit_id) { - if (fb->or_list_size == 0) { - PrefilterFlowbit x; - memset(&x, 0, sizeof(x)); - x.id = fb->idx; - - PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x); - if (pfb == NULL) { - PrefilterFlowbit *add = SCCalloc(1, sizeof(*add)); - BUG_ON(add == NULL); - - add->id = fb->idx; - add->rule_id = SCCalloc(1, BLOCK_SIZE * sizeof(uint32_t)); - BUG_ON(add->rule_id == NULL); - add->rule_id_size = BLOCK_SIZE; - add->rule_id_cnt = 1; - add->rule_id[0] = s->num; - - PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add); - SCLogDebug("not found, so added (res %p)", res); - if (res != NULL) { - // duplicate, shouldn't be possible after the FIND above - BUG_ON(1); + PrefilterFlowbit x; + memset(&x, 0, sizeof(x)); + x.id = flowbit_id; + + PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x); + if (pfb == NULL) { + PrefilterFlowbit *add = SCCalloc(1, sizeof(*add)); + if (add == NULL) + return -1; + + add->id = flowbit_id; + add->rule_id = SCCalloc(1, BLOCK_SIZE * sizeof(uint32_t)); + if (add->rule_id == NULL) { + SCFree(add); + return -1; + } + add->rule_id_size = BLOCK_SIZE; + add->rule_id_cnt = 1; + add->rule_id[0] = s->num; + + PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add); + SCLogDebug("not found, so added (res %p)", res); + if (res != NULL) { + // duplicate, shouldn't be possible after the FIND above + BUG_ON(1); + return -1; + } + } else { + SCLogDebug("found! pfb %p id %u", pfb, pfb->id); + + if (pfb->rule_id_cnt < pfb->rule_id_size) { + pfb->rule_id[pfb->rule_id_cnt++] = s->num; + } else { + uint32_t *ptr = + SCRealloc(pfb->rule_id, (pfb->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t)); + if (ptr == NULL) { + // memory stays in the tree return -1; } - } else { - SCLogDebug("found! pfb %p id %u", pfb, pfb->id); + pfb->rule_id = ptr; + pfb->rule_id_size += BLOCK_SIZE; + pfb->rule_id[pfb->rule_id_cnt++] = s->num; + } + } + return 0; +} - if (pfb->rule_id_cnt < pfb->rule_id_size) { - pfb->rule_id[pfb->rule_id_cnt++] = s->num; - } else { - uint32_t *ptr = SCRealloc( - pfb->rule_id, (pfb->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t)); - BUG_ON(ptr == NULL); - pfb->rule_id = ptr; - pfb->rule_id_size += BLOCK_SIZE; - pfb->rule_id[pfb->rule_id_cnt++] = s->num; - } +static int AddBitsAndSid(const DetectEngineCtx *de_ctx, struct PrefilterEngineFlowbits *ctx, + const DetectFlowbitsData *fb, const Signature *s) +{ + if (fb->or_list_size == 0) { + if (AddBitAndSid(ctx, s, fb->idx) < 0) { + return -1; } } else { for (uint8_t i = 0; i < fb->or_list_size; i++) { - PrefilterFlowbit x; - memset(&x, 0, sizeof(x)); - x.id = fb->or_list[i]; SCLogDebug("flowbit OR: bit %u", fb->or_list[i]); - - PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x); - if (pfb == NULL) { - PrefilterFlowbit *add = SCCalloc(1, sizeof(*add)); - BUG_ON(add == NULL); - - add->id = x.id; - add->rule_id = SCCalloc(1, BLOCK_SIZE * sizeof(uint32_t)); - BUG_ON(add->rule_id == NULL); - add->rule_id_size = BLOCK_SIZE; - add->rule_id_cnt = 1; - add->rule_id[0] = s->num; - - PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add); - SCLogDebug("not found, so added (res %p)", res); - if (res != NULL) { - // duplicate, shouldn't be possible after the FIND above - BUG_ON(1); - return -1; - } - } else { - SCLogDebug("found! pfb %p id %u", pfb, pfb->id); - - if (pfb->rule_id_cnt < pfb->rule_id_size) { - pfb->rule_id[pfb->rule_id_cnt++] = s->num; - } else { - uint32_t *ptr = SCRealloc( - pfb->rule_id, (pfb->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t)); - BUG_ON(ptr == NULL); - pfb->rule_id = ptr; - pfb->rule_id_size += BLOCK_SIZE; - pfb->rule_id[pfb->rule_id_cnt++] = s->num; - } + if (AddBitAndSid(ctx, s, fb->or_list[i]) < 0) { + return -1; } } } @@ -1237,7 +1219,8 @@ static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh) SCLogDebug("setting up sets/toggles for sid %u", s->id); if (AddBitSetToggle(de_ctx, &fb_analysis, set_ctx, fb, s) == 1) { - fb->prefilter = true; // flag the set/toggle to trigger the post-rule match logic + // flag the set/toggle to trigger the post-rule match logic + fb->post_rule_match_prefilter = true; } // TODO don't add for sigs that don't have isset in this sgh. Reasoning: @@ -1270,7 +1253,7 @@ static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh) if (isset_ctx == NULL) goto error; } - if (AddBitIsset(de_ctx, isset_ctx, fb, s) < 0) { + if (AddBitsAndSid(de_ctx, isset_ctx, fb, s) < 0) { goto error; } } diff --git a/src/detect-flowbits.h b/src/detect-flowbits.h index 3d34bb824fd7..d04d6846bf46 100644 --- a/src/detect-flowbits.h +++ b/src/detect-flowbits.h @@ -36,7 +36,9 @@ typedef struct DetectFlowbitsData_ { uint32_t idx; uint8_t cmd; uint8_t or_list_size; - bool prefilter; /**< set/toggle command should trigger post-rule-match "prefilter" */ + /** Flag to trigger post rule match prefilter following a 'set' match. */ + bool post_rule_match_prefilter; /**< set/toggle command should trigger post-rule-match + "prefilter" */ uint32_t *or_list; } DetectFlowbitsData; From c03660065c0a59dae6cde2c326be894bff5e2c6e Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 21 Nov 2024 13:26:02 +0100 Subject: [PATCH 17/17] SQUASH flowbits prefilter toggle --- src/detect-flowbits.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/detect-flowbits.c b/src/detect-flowbits.c index 5e1616a72b05..78e4e4d8d85a 100644 --- a/src/detect-flowbits.c +++ b/src/detect-flowbits.c @@ -1196,18 +1196,17 @@ static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh) if (fb->cmd == DETECT_FLOWBITS_CMD_SET) { SCLogDebug( "DETECT_SM_LIST_POSTMATCH: sid %u DETECT_FLOWBITS set %u", s->id, fb->idx); - // else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) { - // SCLogDebug("DETECT_SM_LIST_POSTMATCH: sid %u DETECT_FLOWBITS toggle %u", s->id, - // fb->idx); + } else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) { + SCLogDebug("DETECT_SM_LIST_POSTMATCH: sid %u DETECT_FLOWBITS toggle %u", s->id, + fb->idx); } else { SCLogDebug("unsupported flowbits setting"); continue; } - if (fb_analysis.array[fb->idx].toggle_sids_idx || - fb_analysis.array[fb->idx].isnotset_sids_idx || + if (fb_analysis.array[fb->idx].isnotset_sids_idx || fb_analysis.array[fb->idx].unset_sids_idx) { - SCLogDebug("flowbit %u not supported: toggle or unset in use", fb->idx); + SCLogDebug("flowbit %u not supported: unset in use", fb->idx); continue; } @@ -1220,6 +1219,7 @@ static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh) SCLogDebug("setting up sets/toggles for sid %u", s->id); if (AddBitSetToggle(de_ctx, &fb_analysis, set_ctx, fb, s) == 1) { // flag the set/toggle to trigger the post-rule match logic + SCLogDebug("set up sets/toggles for sid %u", s->id); fb->post_rule_match_prefilter = true; } @@ -1237,8 +1237,7 @@ static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh) } const DetectFlowbitsData *fb = (DetectFlowbitsData *)s->init_data->prefilter_sm->ctx; - if (fb_analysis.array[fb->idx].toggle_sids_idx || - fb_analysis.array[fb->idx].isnotset_sids_idx || + if (fb_analysis.array[fb->idx].isnotset_sids_idx || fb_analysis.array[fb->idx].unset_sids_idx) { SCLogDebug("flowbit %u not supported: toggle or unset in use", fb->idx); s->init_data->prefilter_sm = NULL;