Skip to content

Commit

Permalink
applayer: add stats counters for exception errors
Browse files Browse the repository at this point in the history 
Add stats counters for exception policy are applied for app-layer errors

Part of
Task OISF#5816

(cherry picked from commit a71ace8)
  • Loading branch information
@jufajardini @jlucovsky
jufajardini authored and jlucovsky committed Oct 22, 2024
1 parent ea880cb commit 1a2e6bb
Show file tree
Hide file tree
Showing 4 changed files with 145 additions and 2 deletions.
10 changes: 10 additions & 0 deletions etc/schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3750,6 +3750,11 @@
"error": {
"type": "object",
"properties": {
"exception_policy": {
"description":
"Consolidated stats on how many times app-layer error exception policy was applied, and which one",
"$ref": "#/$defs/exceptionPolicy"
},
"bittorrent-dht": {
"$ref": "#/$defs/stats_applayer_error"
},
Expand Down Expand Up @@ -5549,6 +5554,11 @@
},
"internal": {
"type": "integer"
},
"exception_policy": {
"description":
"How many times app-layer error exception policy was applied, and which one",
"$ref": "#/$defs/exceptionPolicy"
}
},
"additionalProperties": false
Expand Down
121 changes: 120 additions & 1 deletion src/app-layer.c
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* Copyright (C) 2007-2023 Open Information Security Foundation
/* Copyright (C) 2007-2024 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
Expand Down Expand Up @@ -49,6 +49,7 @@
#include "app-layer-htp-mem.h"
#include "util-exception-policy.h"

extern bool g_stats_eps_per_app_proto_errors;
/**
* \brief This is for the app layer in general and it contains per thread
* context relevant to both the alpd and alp.
Expand Down Expand Up @@ -80,6 +81,7 @@ typedef struct AppLayerCounterNames_ {
char parser_error[MAX_COUNTER_SIZE];
char internal_error[MAX_COUNTER_SIZE];
char alloc_error[MAX_COUNTER_SIZE];
char eps_name[EXCEPTION_POLICY_MAX][MAX_COUNTER_SIZE];
} AppLayerCounterNames;

typedef struct AppLayerCounters_ {
Expand All @@ -89,12 +91,41 @@ typedef struct AppLayerCounters_ {
uint16_t parser_error_id;
uint16_t internal_error_id;
uint16_t alloc_error_id;
ExceptionPolicyCounters eps_error;
} AppLayerCounters;

/* counter names. Only used at init. */
AppLayerCounterNames applayer_counter_names[FLOW_PROTO_APPLAYER_MAX][ALPROTO_MAX];
/* counter id's. Used that runtime. */
AppLayerCounters applayer_counters[FLOW_PROTO_APPLAYER_MAX][ALPROTO_MAX];
/* Exception policy global counters ids */
ExceptionPolicyCounters eps_error_summary;

/* Settings order as in the enum */
// clang-format off
ExceptionPolicyStatsSetts app_layer_error_eps_stats = {
.valid_settings_ids = {
/* EXCEPTION_POLICY_NOT_SET */ false,
/* EXCEPTION_POLICY_AUTO */ false,
/* EXCEPTION_POLICY_PASS_PACKET */ true,
/* EXCEPTION_POLICY_PASS_FLOW */ true,
/* EXCEPTION_POLICY_BYPASS_FLOW */ true,
/* EXCEPTION_POLICY_DROP_PACKET */ false,
/* EXCEPTION_POLICY_DROP_FLOW */ false,
/* EXCEPTION_POLICY_REJECT */ true,
},
.valid_settings_ips = {
/* EXCEPTION_POLICY_NOT_SET */ false,
/* EXCEPTION_POLICY_AUTO */ false,
/* EXCEPTION_POLICY_PASS_PACKET */ true,
/* EXCEPTION_POLICY_PASS_FLOW */ true,
/* EXCEPTION_POLICY_BYPASS_FLOW */ true,
/* EXCEPTION_POLICY_DROP_PACKET */ true,
/* EXCEPTION_POLICY_DROP_FLOW */ true,
/* EXCEPTION_POLICY_REJECT */ true,
},
};
// clang-format on

void AppLayerSetupCounters(void);
void AppLayerDeSetupCounters(void);
Expand Down Expand Up @@ -159,6 +190,25 @@ void AppLayerIncInternalErrorCounter(ThreadVars *tv, Flow *f)
}
}

static void AppLayerIncrErrorExcPolicyCounter(ThreadVars *tv, Flow *f, enum ExceptionPolicy policy)
{
#ifdef UNITTESTS
if (tv == NULL) {
return;
}
#endif
uint16_t id = applayer_counters[f->protomap][f->alproto].eps_error.eps_id[policy];
/* for the summary values */
uint16_t g_id = eps_error_summary.eps_id[policy];

if (likely(id > 0)) {
StatsIncr(tv, id);
}
if (likely(g_id > 0)) {
StatsIncr(tv, g_id);
}
}

/* in IDS mode protocol detection is done in reverse order:
* when TCP data is ack'd. We want to flag the correct packet,
* so in this case we set a flag in the flow so that the first
Expand Down Expand Up @@ -640,6 +690,7 @@ static int TCPProtoDetect(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
SCReturnInt(0);
parser_error:
ExceptionPolicyApply(p, g_applayerparser_error_policy, PKT_DROP_REASON_APPLAYER_ERROR);
AppLayerIncrErrorExcPolicyCounter(tv, f, g_applayerparser_error_policy);
SCReturnInt(-1);
detect_error:
DisableAppLayer(tv, f, p);
Expand Down Expand Up @@ -707,6 +758,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet
StreamTcpUpdateAppLayerProgress(ssn, direction, data_len);
if (r < 0) {
ExceptionPolicyApply(p, g_applayerparser_error_policy, PKT_DROP_REASON_APPLAYER_ERROR);
AppLayerIncrErrorExcPolicyCounter(tv, f, g_applayerparser_error_policy);
SCReturnInt(-1);
}
goto end;
Expand Down Expand Up @@ -793,6 +845,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet
if (r < 0) {
ExceptionPolicyApply(
p, g_applayerparser_error_policy, PKT_DROP_REASON_APPLAYER_ERROR);
AppLayerIncrErrorExcPolicyCounter(tv, f, g_applayerparser_error_policy);
SCReturnInt(-1);
}
}
Expand Down Expand Up @@ -933,6 +986,7 @@ int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *tctx, Packet *p, Flow *
}
if (r < 0) {
ExceptionPolicyApply(p, g_applayerparser_error_policy, PKT_DROP_REASON_APPLAYER_ERROR);
AppLayerIncrErrorExcPolicyCounter(tv, f, g_applayerparser_error_policy);
SCReturnInt(-1);
}

Expand Down Expand Up @@ -1063,6 +1117,30 @@ void AppLayerRegisterGlobalCounters(void)
StatsRegisterGlobalCounter("app_layer.expectations", ExpectationGetCounter);
}

static bool IsAppLayerErrorExceptionPolicyStatsValid(enum ExceptionPolicy policy)
{
if (EngineModeIsIPS()) {
return app_layer_error_eps_stats.valid_settings_ips[policy];
}
return app_layer_error_eps_stats.valid_settings_ids[policy];
}

static void AppLayerSetupExceptionPolicyPerProtoCounters(
uint8_t ipproto_map, AppProto alproto, const char *alproto_str, const char *ipproto_suffix)
{
if (g_stats_eps_per_app_proto_errors &&
g_applayerparser_error_policy != EXCEPTION_POLICY_NOT_SET) {
for (enum ExceptionPolicy i = EXCEPTION_POLICY_NOT_SET + 1; i < EXCEPTION_POLICY_MAX; i++) {
if (IsAppLayerErrorExceptionPolicyStatsValid(i)) {
snprintf(applayer_counter_names[ipproto_map][alproto].eps_name[i],
sizeof(applayer_counter_names[ipproto_map][alproto].eps_name[i]),
"app_layer.error.%s%s.exception_policy.%s", alproto_str, ipproto_suffix,
ExceptionPolicyEnumToString(i, true));
}
}
}
}

#define IPPROTOS_MAX 2
void AppLayerSetupCounters(void)
{
Expand All @@ -1071,6 +1149,19 @@ void AppLayerSetupCounters(void)
const char *str = "app_layer.flow.";
const char *estr = "app_layer.error.";

/* We don't log stats counters if exception policy is `ignore`/`not set` */
if (g_applayerparser_error_policy != EXCEPTION_POLICY_NOT_SET) {
/* Register global counters for app layer error exception policy summary */
const char *eps_default_str = "app_layer.error.exception_policy.";
for (enum ExceptionPolicy i = EXCEPTION_POLICY_NOT_SET + 1; i < EXCEPTION_POLICY_MAX; i++) {
if (IsAppLayerErrorExceptionPolicyStatsValid(i)) {
snprintf(app_layer_error_eps_stats.eps_name[i],
sizeof(app_layer_error_eps_stats.eps_name[i]), "%s%s", eps_default_str,
ExceptionPolicyEnumToString(i, true));
}
}
}

AppLayerProtoDetectSupportedAppProtocols(alprotos);

for (uint8_t p = 0; p < IPPROTOS_MAX; p++) {
Expand Down Expand Up @@ -1107,6 +1198,9 @@ void AppLayerSetupCounters(void)
snprintf(applayer_counter_names[ipproto_map][alproto].internal_error,
sizeof(applayer_counter_names[ipproto_map][alproto].internal_error),
"%s%s%s.internal", estr, alproto_str, ipproto_suffix);

AppLayerSetupExceptionPolicyPerProtoCounters(
ipproto_map, alproto, alproto_str, ipproto_suffix);
} else {
snprintf(applayer_counter_names[ipproto_map][alproto].name,
sizeof(applayer_counter_names[ipproto_map][alproto].name),
Expand All @@ -1129,6 +1223,8 @@ void AppLayerSetupCounters(void)
snprintf(applayer_counter_names[ipproto_map][alproto].internal_error,
sizeof(applayer_counter_names[ipproto_map][alproto].internal_error),
"%s%s.internal", estr, alproto_str);
AppLayerSetupExceptionPolicyPerProtoCounters(
ipproto_map, alproto, alproto_str, "");
}
} else if (alproto == ALPROTO_FAILED) {
snprintf(applayer_counter_names[ipproto_map][alproto].name,
Expand All @@ -1150,6 +1246,17 @@ void AppLayerRegisterThreadCounters(ThreadVars *tv)
AppProto alprotos[ALPROTO_MAX];
AppLayerProtoDetectSupportedAppProtocols(alprotos);

/* We don't log stats counters if exception policy is `ignore`/`not set` */
if (g_applayerparser_error_policy != EXCEPTION_POLICY_NOT_SET) {
/* Register global counters for app layer error exception policy summary */
for (enum ExceptionPolicy i = EXCEPTION_POLICY_NOT_SET + 1; i < EXCEPTION_POLICY_MAX; i++) {
if (IsAppLayerErrorExceptionPolicyStatsValid(i)) {
eps_error_summary.eps_id[i] =
StatsRegisterCounter(app_layer_error_eps_stats.eps_name[i], tv);
}
}
}

for (uint8_t p = 0; p < IPPROTOS_MAX; p++) {
const uint8_t ipproto = ipprotos[p];
const uint8_t ipproto_map = FlowGetProtoMapping(ipproto);
Expand All @@ -1172,6 +1279,18 @@ void AppLayerRegisterThreadCounters(ThreadVars *tv)
applayer_counter_names[ipproto_map][alproto].parser_error, tv);
applayer_counters[ipproto_map][alproto].internal_error_id = StatsRegisterCounter(
applayer_counter_names[ipproto_map][alproto].internal_error, tv);
/* We don't log stats counters if exception policy is `ignore`/`not set` */
if (g_stats_eps_per_app_proto_errors &&
g_applayerparser_error_policy != EXCEPTION_POLICY_NOT_SET) {
for (enum ExceptionPolicy i = EXCEPTION_POLICY_NOT_SET + 1;
i < EXCEPTION_POLICY_MAX; i++) {
if (IsAppLayerErrorExceptionPolicyStatsValid(i)) {
applayer_counters[ipproto_map][alproto]
.eps_error.eps_id[i] = StatsRegisterCounter(
applayer_counter_names[ipproto_map][alproto].eps_name[i], tv);
}
}
}
} else if (alproto == ALPROTO_FAILED) {
applayer_counters[ipproto_map][alproto].counter_id =
StatsRegisterCounter(applayer_counter_names[ipproto_map][alproto].name, tv);
Expand Down
13 changes: 12 additions & 1 deletion src/suricata.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,12 @@ uint16_t g_livedev_mask = 0xffff;
* support */
bool g_disable_hashing = false;

/* snapshot of the system's hugepages before system intitialization. */
SystemHugepageSnapshot *prerun_snap = NULL;

/** add per-proto app-layer error counters for exception policies stats? disabled by default */
bool g_stats_eps_per_app_proto_errors = false;

/** Suricata instance */
SCInstance suricata;

Expand Down Expand Up @@ -2700,6 +2706,12 @@ int PostConfLoadedSetup(SCInstance *suri)
/* Must occur prior to output mod registration
and app layer setup. */
FeatureTrackingRegister();
ConfNode *eps = ConfGetNode("stats.exception-policy");
if (eps != NULL) {
if (ConfNodeChildValueIsTrue(eps, "per-app-proto-errors")) {
g_stats_eps_per_app_proto_errors = true;
}
}

AppLayerSetup();

Expand Down Expand Up @@ -2988,7 +3000,6 @@ int SuricataMain(int argc, char **argv)
goto out;
}

SystemHugepageSnapshot *prerun_snap = NULL;
if (run_mode == RUNMODE_DPDK)
prerun_snap = SystemHugepageSnapshotCreate();

Expand Down
3 changes: 3 additions & 0 deletions suricata.yaml.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ stats:
#decoder-events-prefix: "decoder.event"
# Add stream events as stats.
#stream-events: false
exception-policy:
#per-app-proto-errors: false # default: false. True will log errors for
# each app-proto. Warning: VERY verbose

# Plugins -- Experimental -- specify the filename for each plugin shared object
plugins:
Expand Down

0 comments on commit 1a2e6bb

Please sign in to comment.