Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/flowbit prefilter/v24 #12132

Closed
wants to merge 17 commits into from
Closed

Feat/flowbit prefilter/v24 #12132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
72b7778
flowbits: unset returns state of the bit
victorjulien Jun 10, 2022
a81984d
flowbits: allow setter to know if set is new
victorjulien Oct 10, 2022
3fee590
detect: set mpm/prefilter during signature parsing
victorjulien Oct 5, 2022
f5c4552
detect/flowbits: implement prefilter support
victorjulien Nov 13, 2020
5c71c86
SQUASH scan-build issues
victorjulien Nov 19, 2024
dc39737
WIP PostRuleMatchWorkQueueItem doc
victorjulien Nov 19, 2024
ef2360e
SQUASH docs
victorjulien Nov 19, 2024
3c68795
SQUASH flowbits todo removal
victorjulien Nov 19, 2024
acf22c5
SQUASH flowbits cleanup
victorjulien Nov 19, 2024
7e796ab
SQUASH move common code to prefilter
victorjulien Nov 19, 2024
11fb4c5
SQUASH error check
victorjulien Nov 19, 2024
a29557c
SQUASH error check
victorjulien Nov 19, 2024
b991723
SQUASH unittests after mpm changes
victorjulien Nov 20, 2024
03fc687
SQUASH flowbit prefilter: int warning
victorjulien Nov 20, 2024
3a0f362
SQUASH flowbit prefilter
victorjulien Nov 21, 2024
4280d82
SQUASH flowbit
victorjulien Nov 21, 2024
c036600
SQUASH flowbits prefilter toggle
victorjulien Nov 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 31 additions & 53 deletions src/detect-byte-extract.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1945,9 +1945,8 @@ static int DetectByteExtractTest43(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_OFFSET_VAR |
DETECT_CONTENT_OFFSET) ||
cd->offset != bed->local_id) {
cd->flags != (DETECT_CONTENT_OFFSET_VAR | DETECT_CONTENT_OFFSET | DETECT_CONTENT_MPM) ||
cd->offset != bed->local_id) {
printf("three failed\n");
result = 0;
goto end;
Expand Down Expand Up @@ -2049,9 +2048,8 @@ static int DetectByteExtractTest44(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_OFFSET_VAR |
DETECT_CONTENT_OFFSET) ||
cd->offset != bed1->local_id) {
cd->flags != (DETECT_CONTENT_OFFSET_VAR | DETECT_CONTENT_OFFSET | DETECT_CONTENT_MPM) ||
cd->offset != bed1->local_id) {
printf("four failed\n");
result = 0;
goto end;
Expand Down Expand Up @@ -2158,10 +2156,8 @@ static int DetectByteExtractTest45(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_DEPTH_VAR |
DETECT_CONTENT_DEPTH) ||
cd->depth != bed->local_id ||
cd->offset != 0) {
cd->flags != (DETECT_CONTENT_DEPTH_VAR | DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) ||
cd->depth != bed->local_id || cd->offset != 0) {
printf("three failed\n");
result = 0;
goto end;
Expand Down Expand Up @@ -2263,9 +2259,8 @@ static int DetectByteExtractTest46(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_DEPTH_VAR |
DETECT_CONTENT_DEPTH) ||
cd->depth != bed1->local_id) {
cd->flags != (DETECT_CONTENT_DEPTH_VAR | DETECT_CONTENT_DEPTH | DETECT_CONTENT_MPM) ||
cd->depth != bed1->local_id) {
printf("four failed\n");
result = 0;
goto end;
Expand Down Expand Up @@ -2372,11 +2367,9 @@ static int DetectByteExtractTest47(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_DISTANCE_VAR |
DETECT_CONTENT_DISTANCE) ||
cd->distance != bed->local_id ||
cd->offset != 0 ||
cd->depth != 0) {
cd->flags !=
(DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM) ||
cd->distance != bed->local_id || cd->offset != 0 || cd->depth != 0) {
printf("three failed\n");
result = 0;
goto end;
Expand Down Expand Up @@ -2478,12 +2471,9 @@ static int DetectByteExtractTest48(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_DISTANCE_VAR |
DETECT_CONTENT_DISTANCE |
DETECT_CONTENT_DISTANCE_NEXT) ||
cd->distance != bed1->local_id ||
cd->depth != 0 ||
cd->offset != 0) {
cd->flags != (DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_DISTANCE |
DETECT_CONTENT_DISTANCE_NEXT | DETECT_CONTENT_MPM) ||
cd->distance != bed1->local_id || cd->depth != 0 || cd->offset != 0) {
printf("four failed\n");
result = 0;
goto end;
Expand Down Expand Up @@ -2592,12 +2582,8 @@ static int DetectByteExtractTest49(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "three", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_WITHIN_VAR |
DETECT_CONTENT_WITHIN) ||
cd->within != bed->local_id ||
cd->offset != 0 ||
cd->depth != 0 ||
cd->distance != 0) {
cd->flags != (DETECT_CONTENT_WITHIN_VAR | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) ||
cd->within != bed->local_id || cd->offset != 0 || cd->depth != 0 || cd->distance != 0) {
printf("three failed\n");
result = 0;
goto end;
Expand Down Expand Up @@ -2699,13 +2685,10 @@ static int DetectByteExtractTest50(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_WITHIN_VAR |
DETECT_CONTENT_WITHIN|
DETECT_CONTENT_WITHIN_NEXT) ||
cd->within != bed1->local_id ||
cd->depth != 0 ||
cd->offset != 0 ||
cd->distance != 0) {
cd->flags != (DETECT_CONTENT_WITHIN_VAR | DETECT_CONTENT_WITHIN |
DETECT_CONTENT_WITHIN_NEXT | DETECT_CONTENT_MPM) ||
cd->within != bed1->local_id || cd->depth != 0 || cd->offset != 0 ||
cd->distance != 0) {
printf("four failed\n");
result = 0;
goto end;
Expand Down Expand Up @@ -2970,7 +2953,7 @@ static int DetectByteExtractTest53(void)
SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_PMATCH];
FAIL_IF(sm->type != DETECT_CONTENT);
DetectContentData *cd = (DetectContentData *)sm->ctx;
FAIL_IF(cd->flags != 0);
FAIL_IF(cd->flags != (DETECT_CONTENT_MPM | DETECT_CONTENT_NO_DOUBLE_INSPECTION_REQUIRED));

sm = sm->next;
FAIL_IF_NULL(sm);
Expand Down Expand Up @@ -3092,11 +3075,8 @@ static int DetectByteExtractTest54(void)
goto end;
}
bjd = (DetectBytejumpData *)sm->ctx;
if (bjd->flags != DETECT_BYTEJUMP_OFFSET_VAR || bjd->offset != 1) {
printf("four failed\n");
result = 0;
goto end;
}
FAIL_IF(bjd->flags != DETECT_BYTEJUMP_OFFSET_VAR);
FAIL_IF(bjd->offset != 1);

if (sm->next != NULL)
goto end;
Expand Down Expand Up @@ -3197,12 +3177,10 @@ static int DetectByteExtractTest55(void)
}
cd = (DetectContentData *)sm->ctx;
if (strncmp((char *)cd->content, "four", cd->content_len) != 0 ||
cd->flags != (DETECT_CONTENT_DISTANCE_VAR |
DETECT_CONTENT_WITHIN_VAR |
DETECT_CONTENT_DISTANCE |
DETECT_CONTENT_WITHIN) ||
cd->within != bed1->local_id ||
cd->distance != bed2->local_id) {
cd->flags !=
(DETECT_CONTENT_DISTANCE_VAR | DETECT_CONTENT_WITHIN_VAR |
DETECT_CONTENT_DISTANCE | DETECT_CONTENT_WITHIN | DETECT_CONTENT_MPM) ||
cd->within != bed1->local_id || cd->distance != bed2->local_id) {
printf("four failed: ");
goto end;
}
Expand Down Expand Up @@ -3829,8 +3807,8 @@ static int DetectByteExtractTest60(void)
goto end;
}
cd = (DetectContentData *)sm->ctx;
if (cd->flags != DETECT_CONTENT_RELATIVE_NEXT ||
strncmp((char *)cd->content, "three", cd->content_len) != 0) {
if (cd->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM) ||
strncmp((char *)cd->content, "three", cd->content_len) != 0) {
printf("one failed\n");
result = 0;
goto end;
Expand Down Expand Up @@ -3950,8 +3928,8 @@ static int DetectByteExtractTest61(void)
goto end;
}
cd = (DetectContentData *)sm->ctx;
if (cd->flags != DETECT_CONTENT_RELATIVE_NEXT ||
strncmp((char *)cd->content, "three", cd->content_len) != 0) {
if (cd->flags != (DETECT_CONTENT_RELATIVE_NEXT | DETECT_CONTENT_MPM) ||
strncmp((char *)cd->content, "three", cd->content_len) != 0) {
printf("one failed\n");
result = 0;
goto end;
Expand Down
2 changes: 1 addition & 1 deletion src/detect-content.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1523,7 +1523,7 @@ static int DetectContentParseTest19(void)
FAIL_IF_NOT(s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL);

DetectContentData *data = (DetectContentData *)sm->ctx;
FAIL_IF_NOT(data->flags == DETECT_CONTENT_DISTANCE);
FAIL_IF_NOT(data->flags == (DETECT_CONTENT_DISTANCE | DETECT_CONTENT_MPM));

s = DetectEngineAppendSig(de_ctx,
"alert tcp any any -> any any "
Expand Down
36 changes: 1 addition & 35 deletions src/detect-engine-build.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -668,7 +668,7 @@ static json_t *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx, const SigG
any5_cnt++;
}

prefilter_cnt += (s->init_data->prefilter_sm != 0);
prefilter_cnt += (s->init_data->prefilter_sm != NULL);
if (s->init_data->mpm_sm == NULL) {
nonmpm_cnt++;

Expand Down Expand Up @@ -1703,7 +1703,6 @@ void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s)
}
}

extern int g_skip_prefilter;
/**
* \brief Preprocess signature, classify ip-only, etc, build sig array
*
Expand Down Expand Up @@ -1787,39 +1786,6 @@ int SigPrepareStage1(DetectEngineCtx *de_ctx)

RuleSetScore(s);

/* if keyword engines are enabled in the config, handle them here */
if (!g_skip_prefilter && de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO &&
!(s->flags & SIG_FLAG_PREFILTER)) {
int prefilter_list = DETECT_TBLSIZE;

// TODO buffers?

/* get the keyword supporting prefilter with the lowest type */
for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) {
if (sigmatch_table[sm->type].SupportsPrefilter != NULL) {
if (sigmatch_table[sm->type].SupportsPrefilter(s)) {
prefilter_list = MIN(prefilter_list, sm->type);
}
}
}
}

/* apply that keyword as prefilter */
if (prefilter_list != DETECT_TBLSIZE) {
for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
for (SigMatch *sm = s->init_data->smlists[i]; sm != NULL; sm = sm->next) {
if (sm->type == prefilter_list) {
s->init_data->prefilter_sm = sm;
s->flags |= SIG_FLAG_PREFILTER;
SCLogConfig("sid %u: prefilter is on \"%s\"", s->id, sigmatch_table[sm->type].name);
break;
}
}
}
}
}

/* run buffer type callbacks if any */
for (int x = 0; x < DETECT_SM_LIST_MAX; x++) {
if (s->init_data->smlists[x])
Expand Down
1 change: 0 additions & 1 deletion src/detect-engine-loader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,6 @@ static int DetectLoadSigFile(
sig = DetectEngineAppendSig(de_ctx, line);
if (sig != NULL) {
if (rule_engine_analysis_set || fp_engine_analysis_set) {
RetrieveFPForSig(de_ctx, sig);
if (fp_engine_analysis_set) {
EngineAnalysisFP(de_ctx, sig, line);
}
Expand Down
5 changes: 0 additions & 5 deletions src/detect-engine-mpm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2415,12 +2415,7 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
{
uint32_t cnt = 0;
for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
if (s->flags & SIG_FLAG_PREFILTER)
continue;

RetrieveFPForSig(de_ctx, s);
if (s->init_data->mpm_sm != NULL) {
s->flags |= SIG_FLAG_PREFILTER;
cnt++;
}
}
Expand Down
Loading
Loading