diff --git a/lib/Makefile.am b/lib/Makefile.am index 6f94b1ab0886b..f7eb421c67e95 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -80,6 +80,7 @@ pkginclude_HEADERS += \ lib/tls-support.h \ lib/tlscontext.h \ lib/tlstransport.h \ + lib/type-hinting.h \ lib/utils.h \ lib/uuid.h \ lib/value-pairs.h \ @@ -150,6 +151,7 @@ lib_libsyslog_ng_la_SOURCES = \ lib/syslog-names.c \ lib/tags.c \ lib/timeutils.c \ + lib/type-hinting.c \ lib/utils.c \ lib/value-pairs.c \ lib/vptransform.c \ diff --git a/lib/cfg-grammar.y b/lib/cfg-grammar.y index e78b03227574f..5b215f3c2a044 100644 --- a/lib/cfg-grammar.y +++ b/lib/cfg-grammar.y @@ -32,6 +32,7 @@ /* YYSTYPE and YYLTYPE is defined by the lexer */ #include "cfg-lexer.h" #include "afinter.h" +#include "type-hinting.h" #include "filter/filter-expr-parser.h" #include "filter/filter-pipe.h" #include "parser/parser-expr-parser.h" @@ -745,6 +746,9 @@ template_content_inner CHECK_ERROR(log_template_compile(last_template, $3, &error), @3, "Error compiling template (%s)", error->message); free($3); + + CHECK_ERROR(log_template_set_type_hint(last_template, $1, &error), @1, "Error setting the template type-hint (%s)", error->message); + free($1); } ; diff --git a/lib/cfg.c b/lib/cfg.c index 4f465138d94c1..4d30163c39768 100644 --- a/lib/cfg.c +++ b/lib/cfg.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary * Copyright (c) 1998-2012 Balázs Scheidler * * This library is free software; you can redistribute it and/or @@ -322,6 +322,8 @@ cfg_new(gint version) self->recv_time_zone = NULL; self->keep_timestamp = TRUE; + self->type_cast_strictness = TYPE_CAST_DROP_MESSAGE; + cfg_tree_init_instance(&self->tree, self); return self; } diff --git a/lib/cfg.h b/lib/cfg.h index 3776aa0d1d40f..cc5c1bf91aa15 100644 --- a/lib/cfg.h +++ b/lib/cfg.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary * Copyright (c) 1998-2012 Balázs Scheidler * * This library is free software; you can redistribute it and/or @@ -31,6 +31,7 @@ #include "cfg-parser.h" #include "persist-state.h" #include "template/templates.h" +#include "type-hinting.h" #include #include @@ -84,6 +85,7 @@ struct _GlobalConfig gint time_reopen; gint time_reap; gint suppress; + gint type_cast_strictness; gint log_fifo_size; gint log_msg_size; diff --git a/lib/scratch-buffers.c b/lib/scratch-buffers.c index f6636dee050ed..b2e507c60aae1 100644 --- a/lib/scratch-buffers.c +++ b/lib/scratch-buffers.c @@ -29,6 +29,7 @@ TLS_BLOCK_START { GTrashStack *sb_gstrings; + GTrashStack *sb_th_gstrings; GList *sb_registry; } TLS_BLOCK_END; @@ -80,6 +81,52 @@ ScratchBufferStack SBGStringStack = { .free_stack = sb_gstring_free_stack }; +/* Type-hinted GStrings */ + +#define local_sb_th_gstrings __tls_deref(sb_th_gstrings) + +GTrashStack * +sb_th_gstring_acquire_buffer (void) +{ + SBTHGString *sb; + + sb = g_trash_stack_pop(&local_sb_th_gstrings); + if (!sb) + { + sb = g_new(SBTHGString, 1); + g_string_steal(sb_th_gstring_string(sb)); + sb->type_hint = TYPE_HINT_STRING; + } + else + g_string_set_size(sb_th_gstring_string(sb), 0); + + return (GTrashStack *)sb; +} + +void +sb_th_gstring_release_buffer(GTrashStack *s) +{ + g_trash_stack_push(&local_sb_th_gstrings, s); +} + +void +sb_th_gstring_free_stack(void) +{ + SBGString *sb; + + while ((sb = g_trash_stack_pop(&local_sb_th_gstrings)) != NULL) + { + g_free(sb_gstring_string(sb)->str); + g_free(sb); + } +} + +ScratchBufferStack SBTHGStringStack = { + .acquire_buffer = sb_th_gstring_acquire_buffer, + .release_buffer = sb_th_gstring_release_buffer, + .free_stack = sb_th_gstring_free_stack +}; + /* Global API */ #define local_sb_registry __tls_deref(sb_registry) @@ -95,6 +142,7 @@ scratch_buffers_init(void) { local_sb_registry = NULL; scratch_buffers_register(&SBGStringStack); + scratch_buffers_register(&SBTHGStringStack); } static void diff --git a/lib/scratch-buffers.h b/lib/scratch-buffers.h index 6aeda6fb63252..fdd4c6108bab5 100644 --- a/lib/scratch-buffers.h +++ b/lib/scratch-buffers.h @@ -26,6 +26,7 @@ #define SCRATCH_BUFFERS_H_INCLUDED 1 #include +#include "type-hinting.h" /* Global API */ @@ -67,4 +68,20 @@ extern ScratchBufferStack SBGStringStack; #define sb_gstring_string(buffer) (&buffer->s) +/* Type-hinted GStrings */ + +typedef struct +{ + GTrashStack stackp; + GString s; + TypeHint type_hint; +} SBTHGString; + +extern ScratchBufferStack SBTHGStringStack; + +#define sb_th_gstring_acquire() ((SBTHGString *)scratch_buffer_acquire(&SBTHGStringStack)) +#define sb_th_gstring_release(b) (scratch_buffer_release(&SBTHGStringStack, (GTrashStack *)b)) + +#define sb_th_gstring_string(buffer) (&buffer->s) + #endif diff --git a/lib/template/templates.c b/lib/template/templates.c index 1454cd2a16036..91b52722c1185 100644 --- a/lib/template/templates.c +++ b/lib/template/templates.c @@ -1,6 +1,6 @@ /* - * Copyright (c) 2002-2012 BalaBit IT Ltd, Budapest, Hungary - * Copyright (c) 1998-2012 Balázs Scheidler + * Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 1998-2013 Balázs Scheidler * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -756,6 +756,14 @@ log_template_set_escape(LogTemplate *self, gboolean enable) self->escape = enable; } +gboolean +log_template_set_type_hint(LogTemplate *self, const gchar *type_hint, GError **error) +{ + g_return_val_if_fail(error == NULL || *error == NULL, FALSE); + + return type_hint_parse(type_hint, &self->type_hint, error); +} + static void log_template_reset_compiled(LogTemplate *self) { @@ -1275,6 +1283,7 @@ log_template_new(GlobalConfig *cfg, gchar *name) LogTemplate *self = g_new0(LogTemplate, 1); self->name = g_strdup(name); + self->type_cast_strictness = TYPE_CAST_DROP_MESSAGE; self->ref_cnt = 1; self->cfg = cfg; g_static_mutex_init(&self->arg_lock); diff --git a/lib/template/templates.h b/lib/template/templates.h index 258780c02b2f6..4dd401887f71b 100644 --- a/lib/template/templates.h +++ b/lib/template/templates.h @@ -1,6 +1,6 @@ /* - * Copyright (c) 2002-2012 BalaBit IT Ltd, Budapest, Hungary - * Copyright (c) 1998-2012 Balázs Scheidler + * Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 1998-2013 Balázs Scheidler * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -27,6 +27,7 @@ #include "syslog-ng.h" #include "timeutils.h" +#include "type-hinting.h" #define LTZ_LOCAL 0 #define LTZ_SEND 1 @@ -54,6 +55,8 @@ typedef struct _LogTemplate GlobalConfig *cfg; GStaticMutex arg_lock; GPtrArray *arg_bufs; + TypeHint type_hint; + gint type_cast_strictness; } LogTemplate; /* template expansion options that can be influenced by the user and @@ -188,6 +191,7 @@ void tf_simple_func_free_state(gpointer state); /* appends the formatted output into result */ void log_template_set_escape(LogTemplate *self, gboolean enable); +gboolean log_template_set_type_hint(LogTemplate *self, const gchar *hint, GError **error); gboolean log_template_compile(LogTemplate *self, const gchar *template, GError **error); void log_template_format(LogTemplate *self, LogMessage *lm, LogTemplateOptions *opts, gint tz, gint32 seq_num, const gchar *context_id, GString *result); void log_template_append_format(LogTemplate *self, LogMessage *lm, LogTemplateOptions *opts, gint tz, gint32 seq_num, const gchar *context_id, GString *result); diff --git a/lib/tests/Makefile.am b/lib/tests/Makefile.am index 7117983dd9546..6bead1d3f5871 100644 --- a/lib/tests/Makefile.am +++ b/lib/tests/Makefile.am @@ -1,4 +1,8 @@ -lib_tests_TESTS = lib/tests/test_log_message lib/tests/test_cfg_lexer_subst +lib_tests_TESTS = \ + lib/tests/test_log_message \ + lib/tests/test_cfg_lexer_subst \ + lib/tests/test_type_hints + check_PROGRAMS += ${lib_tests_TESTS} lib_tests_test_log_message_CFLAGS = \ @@ -10,3 +14,8 @@ lib_tests_test_cfg_lexer_subst_CFLAGS = \ $(TEST_CFLAGS) lib_tests_test_cfg_lexer_subst_LDADD = \ $(TEST_LDADD) + +lib_tests_test_type_hints_CFLAGS = \ + $(TEST_CFLAGS) +lib_tests_test_type_hints_LDADD = \ + $(TEST_LDADD) diff --git a/lib/tests/test_type_hints.c b/lib/tests/test_type_hints.c new file mode 100644 index 0000000000000..d453792e68b17 --- /dev/null +++ b/lib/tests/test_type_hints.c @@ -0,0 +1,191 @@ +#include "testutils.h" +#include "type-hinting.h" +#include "apphook.h" + +#include +#include + +#define assert_type_hint(hint,expected) \ + do \ + { \ + TypeHint t; \ + GError *e = NULL; \ + \ + assert_true(type_hint_parse(hint, &t, &e), \ + "Parsing '%s' as type hint", hint); \ + \ + assert_gint(t, expected, \ + "Parsing '%s' as type hint results in correct type", hint); \ + } while(0) \ + +static void +assert_error(GError *error, gint code, const gchar *expected_message) +{ + assert_not_null(error, "GError expected to be non-NULL"); + + assert_gint(error->code, code, "GError error code is as expected"); + if (expected_message) + assert_string(error->message, expected_message, "GError error message is as expected"); +} + +static void +test_type_hint_parse(void) +{ + TypeHint t; + GError *e = NULL; + + testcase_begin("Testing type hint parsing"); + + assert_type_hint(NULL, TYPE_HINT_STRING); + assert_type_hint("string", TYPE_HINT_STRING); + assert_type_hint("literal", TYPE_HINT_LITERAL); + assert_type_hint("boolean", TYPE_HINT_BOOLEAN); + assert_type_hint("int", TYPE_HINT_INT32); + assert_type_hint("int32", TYPE_HINT_INT32); + assert_type_hint("int64", TYPE_HINT_INT64); + assert_type_hint("datetime", TYPE_HINT_DATETIME); + assert_type_hint("default", TYPE_HINT_DEFAULT); + + assert_false(type_hint_parse("invalid-hint", &t, &e), + "Parsing an invalid hint results in an error."); + + assert_error(e, TYPE_HINTING_INVALID_TYPE, "invalid-hint"); + + testcase_end(); +} + +#define assert_type_cast(target,value,out) \ + do \ + { \ + assert_true(type_cast_to_##target(value, out, &error), \ + "Casting '%s' to %s works", value, #target); \ + assert_no_error(error, "Successful casting returns no error"); \ + } while(0) + +#define assert_type_cast_fail(target,value,out) \ + do \ + { \ + assert_false(type_cast_to_##target(value, out, &error), \ + "Casting '%s' to %s fails", value, #target); \ + assert_error(error, TYPE_HINTING_INVALID_CAST, NULL); \ + error = NULL; \ + } while(0) + +#define assert_bool_cast(value,expected) \ + do \ + { \ + gboolean ob; \ + assert_type_cast(boolean, value, &ob); \ + assert_gboolean(ob, expected, "'%s' casted to boolean is %s", \ + value, #expected); \ + } while(0) + +#define assert_int_cast(value,width,expected) \ + do \ + { \ + gint##width i; \ + assert_type_cast(int##width, value, &i); \ + assert_gint##width(i, expected, "'%s' casted to int%s is %u", \ + value, expected); \ + } while(0) \ + +static void +test_type_cast(void) +{ + GError *error = NULL; + gboolean ob; + gint32 i32; + gint64 i64; + guint64 dt; + + testcase_begin("Testing type casting"); + + /* Boolean */ + + assert_bool_cast("True", TRUE); + assert_bool_cast("true", TRUE); + assert_bool_cast("1", TRUE); + assert_bool_cast("totally true", TRUE); + assert_bool_cast("False", FALSE); + assert_bool_cast("false", FALSE); + assert_bool_cast("0", FALSE); + assert_bool_cast("fatally false", FALSE); + + assert_type_cast_fail(boolean, "booyah", &ob); + + /* int32 */ + assert_int_cast("12345", 32, 12345); + assert_type_cast_fail(int32, "12345a", &i32); + + /* int64 */ + assert_int_cast("12345", 64, 12345); + assert_type_cast_fail(int64, "12345a", &i64); + + /* datetime */ + assert_type_cast(datetime_int, "12345", &dt); + assert_guint64(dt, 12345000, "Casting '12345' to datetime works"); + assert_type_cast(datetime_int, "12345.5", &dt); + assert_guint64(dt, 12345500, "Casting '12345.5' to datetime works"); + assert_type_cast(datetime_int, "12345.54", &dt); + assert_guint64(dt, 12345540, "Casting '12345.54' to datetime works"); + assert_type_cast(datetime_int, "12345.543", &dt); + assert_guint64(dt, 12345543, "Casting '12345.543' to datetime works"); + assert_type_cast(datetime_int, "12345.54321", &dt); + assert_guint64(dt, 12345543, "Casting '12345.54321' to datetime works"); + + assert_type_cast_fail(datetime_int, "invalid", &dt); + + testcase_end(); +} + +#define assert_type_cast_parse_strictness(strictness,expected) \ + do \ + { \ + gint r; \ + \ + assert_true(type_cast_strictness_parse(strictness, &r, &error), \ + "Parsing '%s' works", strictness); \ + assert_no_error(error, "Successful strictness parsing returns no error"); \ + assert_gint32(r, expected, "'%s' parses down to '%s'", strictness, #expected); \ + } while(0) + +static void +test_type_cast_strictness(void) +{ + GError *error = NULL; + gint r; + + testcase_begin("Testing type cast strictness"); + + assert_type_cast_parse_strictness("drop-message", TYPE_CAST_DROP_MESSAGE); + assert_type_cast_parse_strictness("silently-drop-message", + TYPE_CAST_DROP_MESSAGE | TYPE_CAST_SILENTLY); + + assert_type_cast_parse_strictness("drop-property", TYPE_CAST_DROP_PROPERTY); + assert_type_cast_parse_strictness("silently-drop-property", + TYPE_CAST_DROP_PROPERTY | TYPE_CAST_SILENTLY); + + assert_type_cast_parse_strictness("fallback-to-string", TYPE_CAST_FALLBACK_TO_STRING); + assert_type_cast_parse_strictness("silently-fallback-to-string", + TYPE_CAST_FALLBACK_TO_STRING | TYPE_CAST_SILENTLY); + + assert_false(type_cast_strictness_parse("do-what-i-mean", &r, &error), + "The 'do-what-i-mean' strictness is not recognised"); + assert_error(error, TYPE_CAST_INVALID_STRICTNESS, NULL); + + testcase_end(); +} + +int +main (void) +{ + app_startup(); + + test_type_hint_parse(); + test_type_cast(); + test_type_cast_strictness(); + + app_shutdown(); + + return 0; +} diff --git a/lib/type-hinting.c b/lib/type-hinting.c new file mode 100644 index 0000000000000..84c305b4ecf2b --- /dev/null +++ b/lib/type-hinting.c @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2012-2013 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2012-2013 Gergely Nagy + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include "messages.h" +#include "type-hinting.h" +#include "template/templates.h" + +#include +#include + +GQuark +type_hinting_error_quark() +{ + return g_quark_from_static_string("type-hinting-error-quark"); +} + +gboolean +type_hint_parse(const gchar *hint, TypeHint *out_type, GError **error) +{ + if (hint == NULL) + { + *out_type = TYPE_HINT_STRING; + return TRUE; + } + + if (strcmp(hint, "string") == 0) + *out_type = TYPE_HINT_STRING; + else if (strcmp(hint, "literal") == 0) + *out_type = TYPE_HINT_LITERAL; + else if (strcmp(hint, "int32") == 0 || strcmp(hint, "int") == 0) + *out_type = TYPE_HINT_INT32; + else if (strcmp(hint, "int64") == 0) + *out_type = TYPE_HINT_INT64; + else if (strcmp(hint, "datetime") == 0) + *out_type = TYPE_HINT_DATETIME; + else if (strcmp(hint, "boolean") == 0) + *out_type = TYPE_HINT_BOOLEAN; + else if (strcmp(hint, "default") == 0) + *out_type = TYPE_HINT_DEFAULT; + else + { + g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_TYPE, + "%s", hint); + return FALSE; + } + + return TRUE; +} + +gboolean +type_cast_strictness_parse(const gchar *strictness, gint *out, GError **error) +{ + const gchar *p = strictness; + gboolean silently = FALSE; + + if (!strictness) + { + *out = TYPE_CAST_DROP_MESSAGE; + return TRUE; + } + + if (strncmp(strictness, "silently-", strlen("silently-")) == 0) + { + silently = TRUE; + p = strictness + strlen("silently-"); + } + + if (strcmp(p, "drop-message") == 0) + *out = TYPE_CAST_DROP_MESSAGE; + else if (strcmp(p, "drop-property") == 0) + *out = TYPE_CAST_DROP_PROPERTY; + else if (strcmp(p, "fallback-to-string") == 0) + *out = TYPE_CAST_FALLBACK_TO_STRING; + else + { + g_set_error(error, TYPE_HINTING_ERROR, TYPE_CAST_INVALID_STRICTNESS, + "%s",strictness); + return FALSE; + } + + if (silently) + *out |= TYPE_CAST_SILENTLY; + + return TRUE; +} + +gboolean +type_cast_drop_helper(gint drop_flags, const gchar *value, + const gchar *type_hint) +{ + if (!(drop_flags & TYPE_CAST_SILENTLY)) + { + msg_error("Casting error", + evt_tag_str("value", value), + evt_tag_str("type-hint", type_hint), + NULL); + } + return drop_flags & TYPE_CAST_DROP_MESSAGE; +} + +gboolean +type_cast_to_boolean(const gchar *value, gboolean *out, GError **error) +{ + if (value[0] == 'T' || value[0] == 't' || value[0] == '1') + *out = TRUE; + else if (value[0] == 'F' || value[0] == 'f' || value[0] == '0') + *out = FALSE; + else + { + if (error) + g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST, + "boolean(%s)", value); + return FALSE; + } + + return TRUE; +} + +gboolean +type_cast_to_int32(const gchar *value, gint32 *out, GError **error) +{ + gchar *endptr; + + *out = (gint32)strtol(value, &endptr, 10); + + if (endptr[0] != '\0') + { + if (error) + g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST, + "int32(%s)", value); + return FALSE; + } + return TRUE; +} + +gboolean +type_cast_to_int64(const gchar *value, gint64 *out, GError **error) +{ + gchar *endptr; + + *out = (gint64)strtoll(value, &endptr, 10); + + if (endptr[0] != '\0') + { + if (error) + g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST, + "int64(%s)", value); + return FALSE; + } + return TRUE; +} + +gboolean +type_cast_to_datetime_int(const gchar *value, guint64 *out, GError **error) +{ + gchar *endptr; + + *out = (gint64)strtoll(value, &endptr, 10) * 1000; + + if (endptr[0] == '.') + { + gsize len = strlen(endptr) - 1, p; + gchar *e, tmp[4]; + glong i; + + if (len > 3) + len = 3; + + memcpy(tmp, endptr + 1, len); + tmp[len] = '\0'; + + i = strtoll(tmp, &e, 10); + + if (e[0] != '\0') + { + if (error) + g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST, + "datetime(%s)", value); + return FALSE; + } + + for (p = 3 - len; p > 0; p--) + i *= 10; + + *out += i; + } + else if (endptr[0] != '\0') + { + if (error) + g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST, + "datetime(%s)", value); + return FALSE; + } + return TRUE; +} + +gboolean +type_cast_to_datetime_str(const gchar *value, const char *format, + gchar **out, GError **error) +{ + if (error) + g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST, + "datetime_str is not supported yet"); + return FALSE; +} diff --git a/lib/type-hinting.h b/lib/type-hinting.h new file mode 100644 index 0000000000000..94ec75756b0e8 --- /dev/null +++ b/lib/type-hinting.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2012-2013 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2012-2013 Gergely Nagy + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef TYPE_HINTING_H +#define TYPE_HINTING_H + +#include "syslog-ng.h" + +#define TYPE_HINTING_ERROR type_hinting_error_quark() + +GQuark type_hinting_error_quark(void); + +enum TypeHintingError +{ + TYPE_HINTING_INVALID_TYPE, + TYPE_HINTING_INVALID_CAST, + TYPE_CAST_INVALID_STRICTNESS, +}; + +typedef enum +{ + TYPE_HINT_STRING, + TYPE_HINT_LITERAL, + TYPE_HINT_BOOLEAN, + TYPE_HINT_INT32, + TYPE_HINT_INT64, + TYPE_HINT_DATETIME, + TYPE_HINT_DEFAULT, +} TypeHint; + +typedef enum +{ + TYPE_CAST_DROP_MESSAGE = 0x01, + TYPE_CAST_DROP_PROPERTY = 0x02, + TYPE_CAST_FALLBACK_TO_STRING = 0x04, + TYPE_CAST_SILENTLY = 0x08 +} TypeCastStrictness; + +gboolean type_cast_strictness_parse(const gchar *strictness, gint *out, GError **error); +gboolean type_hint_parse(const gchar *hint, TypeHint *out_hint, GError **error); + +gboolean type_cast_drop_helper(gint drop_flags, const gchar *value, + const gchar *type_hint); + +gboolean type_cast_to_boolean(const gchar *value, gboolean *out, GError **error); +gboolean type_cast_to_int32(const gchar *value, gint32 *out, GError **error); +gboolean type_cast_to_int64(const gchar *value, gint64 *out, GError **error); +gboolean type_cast_to_datetime_int(const gchar *value, guint64 *out, GError **error); +gboolean type_cast_to_datetime_str(const gchar *value, const char *format, + gchar **out, GError **error); + +#endif diff --git a/lib/value-pairs.c b/lib/value-pairs.c index 5fb3e8d3e3b98..7fc7133fd2c34 100644 --- a/lib/value-pairs.c +++ b/lib/value-pairs.c @@ -26,6 +26,7 @@ #include "vptransform.h" #include "logmsg.h" #include "template/templates.h" +#include "type-hinting.h" #include "cfg-parser.h" #include "misc.h" #include "scratch-buffers.h" @@ -192,22 +193,20 @@ vp_pairs_foreach(gpointer data, gpointer user_data) LogMessage *msg = ((gpointer *)user_data)[2]; gint32 seq_num = GPOINTER_TO_INT (((gpointer *)user_data)[3]); GTree *scope_set = ((gpointer *)user_data)[5]; - SBGString *sb = sb_gstring_acquire(); + SBTHGString *sb = sb_th_gstring_acquire(); VPPairConf *vpc = (VPPairConf *)data; - log_template_format((LogTemplate *)vpc->template, msg, NULL, LTZ_LOCAL, - seq_num, NULL, sb_gstring_string(sb)); + sb->type_hint = vpc->template->type_hint; + log_template_append_format((LogTemplate *)vpc->template, msg, NULL, LTZ_LOCAL, + seq_num, NULL, sb_th_gstring_string(sb)); - if (!sb_gstring_string(sb)->str[0]) + if (sb_th_gstring_string(sb)->len == 0) { - sb_gstring_release(sb); + sb_th_gstring_release(sb); return; } - g_tree_insert(scope_set, vp_transform_apply(vp, vpc->name), - sb_gstring_string(sb)->str); - g_string_steal(sb_gstring_string(sb)); - sb_gstring_release(sb); + g_tree_insert(scope_set, vp_transform_apply(vp, vpc->name), sb); } /* runs over the LogMessage nv-pairs, and inserts them unless excluded */ @@ -233,8 +232,11 @@ vp_msg_nvpairs_foreach(NVHandle handle, gchar *name, (log_msg_is_handle_sdata(handle) && (vp->scopes & VPS_SDATA))) || inc) { - /* NOTE: the key is a borrowed reference in the hash, and value is freed */ - g_tree_insert(scope_set, vp_transform_apply(vp, name), g_strndup(value, value_len)); + SBTHGString *sb = sb_th_gstring_acquire(); + + g_string_append_len(sb_th_gstring_string(sb), value, value_len); + sb->type_hint = TYPE_HINT_STRING; + g_tree_insert(scope_set, vp_transform_apply(vp, name), sb); } return FALSE; @@ -245,7 +247,7 @@ static void vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set, GTree *dest) { gint i; - SBGString *sb = sb_gstring_acquire(); + SBTHGString *sb; for (i = 0; set[i].name; i++) { @@ -261,10 +263,12 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set if (exclude) continue; + sb = sb_th_gstring_acquire(); + switch (set[i].type) { case VPT_MACRO: - log_macro_expand(sb_gstring_string(sb), set[i].id, FALSE, + log_macro_expand(sb_th_gstring_string(sb), set[i].id, FALSE, NULL, LTZ_LOCAL, seq_num, NULL, msg); break; case VPT_NVPAIR: @@ -273,34 +277,55 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set gssize len; nv = log_msg_get_value(msg, (NVHandle) set[i].id, &len); - g_string_append_len(sb_gstring_string(sb), nv, len); + g_string_append_len(sb_th_gstring_string(sb), nv, len); break; } default: g_assert_not_reached(); } - if (!sb_gstring_string(sb)->str[0]) - continue; + if (sb_th_gstring_string(sb)->len == 0) + { + sb_th_gstring_release(sb); + continue; + } - g_tree_insert(dest, vp_transform_apply(vp, set[i].name), - sb_gstring_string(sb)->str); - g_string_steal(sb_gstring_string(sb)); + g_tree_insert(dest, vp_transform_apply(vp, set[i].name), sb); } - sb_gstring_release(sb); } -void +static gboolean +vp_foreach_helper (const gchar *name, const SBTHGString *hinted_value, + gpointer data) +{ + VPForeachFunc func = ((gpointer *)data)[0]; + gpointer user_data = ((gpointer *)data)[1]; + gboolean *r = ((gpointer *)data)[2]; + + *r &= !func(name, hinted_value->type_hint, + sb_th_gstring_string(hinted_value)->str, user_data); + return !*r; +} + +static void +vp_data_free (SBTHGString *s) +{ + sb_th_gstring_release (s); +} + +gboolean value_pairs_foreach_sorted (ValuePairs *vp, VPForeachFunc func, GCompareDataFunc compare_func, LogMessage *msg, gint32 seq_num, gpointer user_data) { gpointer args[] = { vp, func, msg, GINT_TO_POINTER (seq_num), user_data, NULL }; + gboolean result = TRUE; + gpointer helper_args[] = { func, user_data, &result }; GTree *scope_set; scope_set = g_tree_new_full((GCompareDataFunc)compare_func, NULL, (GDestroyNotify)g_free, - (GDestroyNotify)g_free); + (GDestroyNotify)vp_data_free); args[5] = scope_set; /* @@ -327,18 +352,20 @@ value_pairs_foreach_sorted (ValuePairs *vp, VPForeachFunc func, g_ptr_array_foreach(vp->vpairs, (GFunc)vp_pairs_foreach, args); /* Aaand we run it through the callback! */ - g_tree_foreach(scope_set, (GTraverseFunc)func, user_data); + g_tree_foreach(scope_set, (GTraverseFunc)vp_foreach_helper, helper_args); g_tree_destroy(scope_set); + + return result; } -void +gboolean value_pairs_foreach(ValuePairs *vp, VPForeachFunc func, LogMessage *msg, gint32 seq_num, gpointer user_data) { - value_pairs_foreach_sorted(vp, func, (GCompareDataFunc) strcmp, - msg, seq_num, user_data); + return value_pairs_foreach_sorted(vp, func, (GCompareDataFunc) strcmp, + msg, seq_num, user_data); } typedef struct @@ -482,7 +509,7 @@ vp_walker_name_split(vp_walk_stack_t **stack, vp_walk_state_t *state, } static gboolean -value_pairs_walker(const gchar *name, const gchar *value, +value_pairs_walker(const gchar *name, TypeHint type, const gchar *value, gpointer user_data) { vp_walk_state_t *state = (vp_walk_state_t *)user_data; @@ -495,10 +522,14 @@ value_pairs_walker(const gchar *name, const gchar *value, key = vp_walker_name_split (&st, state, name); if (st) - result = state->process_value(key, st->prefix, value, &st->data, + result = state->process_value(key, st->prefix, + type, value, + &st->data, state->user_data); else - result = state->process_value(key, NULL, value, NULL, + result = state->process_value(key, NULL, + type, value, + NULL, state->user_data); g_free(key); @@ -515,7 +546,7 @@ vp_walk_cmp(const gchar *s1, const gchar *s2) return strcmp(s2, s1); } -void +gboolean value_pairs_walk(ValuePairs *vp, VPWalkCallbackFunc obj_start_func, VPWalkValueCallbackFunc process_value_func, @@ -524,6 +555,7 @@ value_pairs_walk(ValuePairs *vp, gpointer user_data) { vp_walk_state_t state; + gboolean result; state.user_data = user_data; state.obj_start = obj_start_func; @@ -532,10 +564,12 @@ value_pairs_walk(ValuePairs *vp, state.stack = NULL; state.obj_start(NULL, NULL, NULL, NULL, NULL, user_data); - value_pairs_foreach_sorted(vp, value_pairs_walker, - (GCompareDataFunc)vp_walk_cmp, msg, seq_num, &state); + result = value_pairs_foreach_sorted(vp, value_pairs_walker, + (GCompareDataFunc)vp_walk_cmp, msg, seq_num, &state); vp_walker_stack_unwind_all(&state.stack, &state); state.obj_end(NULL, NULL, NULL, NULL, NULL, user_data); + + return result; } static void @@ -730,6 +764,32 @@ vp_cmdline_parse_rekey(const gchar *option_name, const gchar *value, return TRUE; } +static void +value_pairs_parse_type(gchar *spec, gchar **value, gchar **type) +{ + char *sp, *ep; + + *type = NULL; + + sp = strchr(spec, '('); + if (sp == NULL) + { + *value = spec; + return; + } + ep = strchr(sp, ')'); + if (ep == NULL || ep[1] != '\0') + { + *value = spec; + return; + } + + *value = sp + 1; + *type = spec; + sp[0] = '\0'; + ep[0] = '\0'; +} + static gboolean vp_cmdline_parse_pair (const gchar *option_name, const gchar *value, gpointer data, GError **error) @@ -737,7 +797,7 @@ vp_cmdline_parse_pair (const gchar *option_name, const gchar *value, gpointer *args = (gpointer *) data; ValuePairs *vp = (ValuePairs *) args[1]; GlobalConfig *cfg = (GlobalConfig *) args[0]; - gchar **kv; + gchar **kv, *v, *t; gboolean res = FALSE; LogTemplate *template; @@ -750,9 +810,12 @@ vp_cmdline_parse_pair (const gchar *option_name, const gchar *value, } kv = g_strsplit(value, "=", 2); + value_pairs_parse_type(kv[1], &v, &t); template = log_template_new(cfg, NULL); - if (!log_template_compile(template, kv[1], error)) + if (!log_template_compile(template, v, error)) + goto error; + if (!log_template_set_type_hint(template, t, error)) goto error; value_pairs_add_pair(vp, kv[0], template); diff --git a/lib/value-pairs.h b/lib/value-pairs.h index b060d5425506c..74b6474992347 100644 --- a/lib/value-pairs.h +++ b/lib/value-pairs.h @@ -27,13 +27,15 @@ #include "syslog-ng.h" #include "nvtable.h" +#include "type-hinting.h" #include "template/templates.h" typedef struct _ValuePairs ValuePairs; -typedef gboolean (*VPForeachFunc)(const gchar *name, const gchar *value, gpointer user_data); +typedef gboolean (*VPForeachFunc)(const gchar *name, TypeHint type, + const gchar *value, gpointer user_data); typedef gboolean (*VPWalkValueCallbackFunc)(const gchar *name, const gchar *prefix, - const gchar *value, + TypeHint type, const gchar *value, gpointer *prefix_data, gpointer user_data); typedef gboolean (*VPWalkCallbackFunc)(const gchar *name, const gchar *prefix, gpointer *prefix_data, @@ -46,20 +48,20 @@ gboolean value_pairs_add_pair(ValuePairs *vp, const gchar *key, LogTemplate *val void value_pairs_add_transforms(ValuePairs *vp, gpointer vpts); -void value_pairs_foreach_sorted(ValuePairs *vp, VPForeachFunc func, - GCompareDataFunc compare_func, - LogMessage *msg, gint32 seq_num, - gpointer user_data); -void value_pairs_foreach(ValuePairs *vp, VPForeachFunc func, - LogMessage *msg, gint32 seq_num, - gpointer user_data); +gboolean value_pairs_foreach_sorted(ValuePairs *vp, VPForeachFunc func, + GCompareDataFunc compare_func, + LogMessage *msg, gint32 seq_num, + gpointer user_data); +gboolean value_pairs_foreach(ValuePairs *vp, VPForeachFunc func, + LogMessage *msg, gint32 seq_num, + gpointer user_data); -void value_pairs_walk(ValuePairs *vp, - VPWalkCallbackFunc obj_start_func, - VPWalkValueCallbackFunc process_value_func, - VPWalkCallbackFunc obj_end_func, - LogMessage *msg, gint32 seq_num, - gpointer user_data); +gboolean value_pairs_walk(ValuePairs *vp, + VPWalkCallbackFunc obj_start_func, + VPWalkValueCallbackFunc process_value_func, + VPWalkCallbackFunc obj_end_func, + LogMessage *msg, gint32 seq_num, + gpointer user_data); ValuePairs *value_pairs_new(void); void value_pairs_free(ValuePairs *vp); diff --git a/modules/afamqp/afamqp.c b/modules/afamqp/afamqp.c index 9a6c07dd656f0..a37e396205ab7 100644 --- a/modules/afamqp/afamqp.c +++ b/modules/afamqp/afamqp.c @@ -375,7 +375,8 @@ afamqp_dd_connect(AMQPDestDriver *self, gboolean reconnect) */ static gboolean -afamqp_vp_foreach(const gchar *name, const gchar *value, +afamqp_vp_foreach(const gchar *name, + TypeHint type, const gchar *value, gpointer user_data) { amqp_table_entry_t **entries = (amqp_table_entry_t **) ((gpointer *)user_data)[0]; diff --git a/modules/afmongodb/afmongodb.c b/modules/afmongodb/afmongodb.c index deafc8f8e243c..44ccd274894f3 100644 --- a/modules/afmongodb/afmongodb.c +++ b/modules/afmongodb/afmongodb.c @@ -320,7 +320,7 @@ afmongodb_vp_obj_end(const gchar *name, static gboolean afmongodb_vp_process_value(const gchar *name, const gchar *prefix, - const gchar *value, + TypeHint type, const gchar *value, gpointer *prefix_data, gpointer user_data) { bson *o; diff --git a/modules/json/format-json.c b/modules/json/format-json.c index 9bfe80ad9d3d9..1c64e12bda10c 100644 --- a/modules/json/format-json.c +++ b/modules/json/format-json.c @@ -163,7 +163,8 @@ tf_json_obj_end(const gchar *name, } static gboolean -tf_json_value(const gchar *name, const gchar *prefix, const gchar *value, +tf_json_value(const gchar *name, const gchar *prefix, + TypeHint type, const gchar *value, gpointer *prefix_data, gpointer user_data) { json_state_t *state = (json_state_t *)user_data; diff --git a/tests/unit/test_value_pairs.c b/tests/unit/test_value_pairs.c index 84e4a71fb0d4a..404285d9ccb54 100644 --- a/tests/unit/test_value_pairs.c +++ b/tests/unit/test_value_pairs.c @@ -10,7 +10,7 @@ gboolean success = TRUE; gboolean -vp_keys_foreach(const gchar *name, const gchar *value, gpointer user_data) +vp_keys_foreach(const gchar *name, TypeHint type, const gchar *value, gpointer user_data) { gpointer *args = (gpointer *) user_data; GList **keys = (GList **) args[0]; @@ -46,6 +46,17 @@ create_message(void) return msg; } +static LogTemplate * +create_template(const gchar *type_hint_string, const gchar *template_string) +{ + LogTemplate *template; + + template = log_template_new(configuration, NULL); + log_template_compile(template, template_string, NULL); + log_template_set_type_hint(template, type_hint_string, NULL); + return template; +} + void testcase(const gchar *scope, const gchar *exclude, const gchar *expected, GPtrArray *transformers) { @@ -62,7 +73,7 @@ testcase(const gchar *scope, const gchar *exclude, const gchar *expected, GPtrAr value_pairs_add_scope(vp, scope); if (exclude) value_pairs_add_glob_pattern(vp, exclude, FALSE); - value_pairs_add_pair(vp, configuration, "test.key", "$MESSAGE"); + value_pairs_add_pair(vp, "test.key", create_template("string", "$MESSAGE")); if (transformers) {