value-pairs: Type hinting support

This implements very basic support for type hinting template strings, where those hints will be available to value-pairs too, allowing any user of either LogTemplate or value-pairs to handle different types of values than plain strings. How they handle it, is up to the callers, this patch just lays the groundwork that makes it possible to pass these hints around. This fixes elastic#4 and elastic#5. Signed-off-by: Balazs Scheidler <[email protected]> Signed-off-by: Gergely Nagy <[email protected]>
ccin2p3 · Sep 6, 2013 · a62140d · a62140d
1 parent 05443fb
commit a62140d
Show file tree

Hide file tree

Showing 18 changed files with 726 additions and 61 deletions.
diff --git a/lib/Makefile.am b/lib/Makefile.am
@@ -80,6 +80,7 @@ pkginclude_HEADERS			+= \
 	lib/tls-support.h		\
 	lib/tlscontext.h  		\
 	lib/tlstransport.h		\
+	lib/type-hinting.h		\
 	lib/utils.h			\
 	lib/uuid.h			\
 	lib/value-pairs.h		\
@@ -150,6 +151,7 @@ lib_libsyslog_ng_la_SOURCES		= \
 	lib/syslog-names.c		\
 	lib/tags.c			\
 	lib/timeutils.c			\
+	lib/type-hinting.c		\
 	lib/utils.c			\
 	lib/value-pairs.c		\
 	lib/vptransform.c		\

diff --git a/lib/cfg-grammar.y b/lib/cfg-grammar.y
@@ -32,6 +32,7 @@
 /* YYSTYPE and YYLTYPE is defined by the lexer */
 #include "cfg-lexer.h"
 #include "afinter.h"
+#include "type-hinting.h"
 #include "filter/filter-expr-parser.h"
 #include "filter/filter-pipe.h"
 #include "parser/parser-expr-parser.h"
@@ -745,6 +746,9 @@ template_content_inner
 
           CHECK_ERROR(log_template_compile(last_template, $3, &error), @3, "Error compiling template (%s)", error->message);
           free($3);
+
+          CHECK_ERROR(log_template_set_type_hint(last_template, $1, &error), @1, "Error setting the template type-hint (%s)", error->message);
+          free($1);
         }
         ;
 

diff --git a/lib/cfg.c b/lib/cfg.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2012 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary
  * Copyright (c) 1998-2012 Balázs Scheidler
  *
  * This library is free software; you can redistribute it and/or
@@ -322,6 +322,8 @@ cfg_new(gint version)
   self->recv_time_zone = NULL;
   self->keep_timestamp = TRUE;
 
+  self->type_cast_strictness = TYPE_CAST_DROP_MESSAGE;
+
   cfg_tree_init_instance(&self->tree, self);
   return self;
 }

diff --git a/lib/cfg.h b/lib/cfg.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002-2012 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary
  * Copyright (c) 1998-2012 Balázs Scheidler
  *
  * This library is free software; you can redistribute it and/or
@@ -31,6 +31,7 @@
 #include "cfg-parser.h"
 #include "persist-state.h"
 #include "template/templates.h"
+#include "type-hinting.h"
 
 #include <sys/types.h>
 #include <regex.h>
@@ -84,6 +85,7 @@ struct _GlobalConfig
   gint time_reopen;
   gint time_reap;
   gint suppress;
+  gint type_cast_strictness;
 
   gint log_fifo_size;
   gint log_msg_size;

diff --git a/lib/scratch-buffers.c b/lib/scratch-buffers.c
@@ -29,6 +29,7 @@
 TLS_BLOCK_START
 {
   GTrashStack *sb_gstrings;
+  GTrashStack *sb_th_gstrings;
   GList *sb_registry;
 }
 TLS_BLOCK_END;
@@ -80,6 +81,52 @@ ScratchBufferStack SBGStringStack = {
   .free_stack = sb_gstring_free_stack
 };
 
+/* Type-hinted GStrings */
+
+#define local_sb_th_gstrings        __tls_deref(sb_th_gstrings)
+
+GTrashStack *
+sb_th_gstring_acquire_buffer (void)
+{
+  SBTHGString *sb;
+
+  sb = g_trash_stack_pop(&local_sb_th_gstrings);
+  if (!sb)
+    {
+      sb = g_new(SBTHGString, 1);
+      g_string_steal(sb_th_gstring_string(sb));
+      sb->type_hint = TYPE_HINT_STRING;
+    }
+  else
+    g_string_set_size(sb_th_gstring_string(sb), 0);
+
+  return (GTrashStack *)sb;
+}
+
+void
+sb_th_gstring_release_buffer(GTrashStack *s)
+{
+  g_trash_stack_push(&local_sb_th_gstrings, s);
+}
+
+void
+sb_th_gstring_free_stack(void)
+{
+  SBGString *sb;
+
+  while ((sb = g_trash_stack_pop(&local_sb_th_gstrings)) != NULL)
+    {
+      g_free(sb_gstring_string(sb)->str);
+      g_free(sb);
+    }
+}
+
+ScratchBufferStack SBTHGStringStack = {
+  .acquire_buffer = sb_th_gstring_acquire_buffer,
+  .release_buffer = sb_th_gstring_release_buffer,
+  .free_stack = sb_th_gstring_free_stack
+};
+
 /* Global API */
 
 #define local_sb_registry  __tls_deref(sb_registry)
@@ -95,6 +142,7 @@ scratch_buffers_init(void)
 {
   local_sb_registry = NULL;
   scratch_buffers_register(&SBGStringStack);
+  scratch_buffers_register(&SBTHGStringStack);
 }
 
 static void

diff --git a/lib/scratch-buffers.h b/lib/scratch-buffers.h
@@ -26,6 +26,7 @@
 #define SCRATCH_BUFFERS_H_INCLUDED 1
 
 #include <glib.h>
+#include "type-hinting.h"
 
 /* Global API */
 
@@ -67,4 +68,20 @@ extern ScratchBufferStack SBGStringStack;
 
 #define sb_gstring_string(buffer) (&buffer->s)
 
+/* Type-hinted GStrings */
+
+typedef struct
+{
+  GTrashStack stackp;
+  GString s;
+  TypeHint type_hint;
+} SBTHGString;
+
+extern ScratchBufferStack SBTHGStringStack;
+
+#define sb_th_gstring_acquire() ((SBTHGString *)scratch_buffer_acquire(&SBTHGStringStack))
+#define sb_th_gstring_release(b) (scratch_buffer_release(&SBTHGStringStack, (GTrashStack *)b))
+
+#define sb_th_gstring_string(buffer) (&buffer->s)
+
 #endif
diff --git a/lib/template/templates.c b/lib/template/templates.c
@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2002-2012 BalaBit IT Ltd, Budapest, Hungary
- * Copyright (c) 1998-2012 Balázs Scheidler
+ * Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 1998-2013 Balázs Scheidler
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -756,6 +756,14 @@ log_template_set_escape(LogTemplate *self, gboolean enable)
   self->escape = enable;
 }
 
+gboolean
+log_template_set_type_hint(LogTemplate *self, const gchar *type_hint, GError **error)
+{
+  g_return_val_if_fail(error == NULL || *error == NULL, FALSE);
+
+  return type_hint_parse(type_hint, &self->type_hint, error);
+}
+
 static void
 log_template_reset_compiled(LogTemplate *self)
 {
@@ -1275,6 +1283,7 @@ log_template_new(GlobalConfig *cfg, gchar *name)
   LogTemplate *self = g_new0(LogTemplate, 1);
 
   self->name = g_strdup(name);
+  self->type_cast_strictness = TYPE_CAST_DROP_MESSAGE;
   self->ref_cnt = 1;
   self->cfg = cfg;
   g_static_mutex_init(&self->arg_lock);

diff --git a/lib/template/templates.h b/lib/template/templates.h
@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2002-2012 BalaBit IT Ltd, Budapest, Hungary
- * Copyright (c) 1998-2012 Balázs Scheidler
+ * Copyright (c) 2002-2013 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 1998-2013 Balázs Scheidler
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -27,6 +27,7 @@
 
 #include "syslog-ng.h"
 #include "timeutils.h"
+#include "type-hinting.h"
 
 #define LTZ_LOCAL 0
 #define LTZ_SEND  1
@@ -54,6 +55,8 @@ typedef struct _LogTemplate
   GlobalConfig *cfg;
   GStaticMutex arg_lock;
   GPtrArray *arg_bufs;
+  TypeHint type_hint;
+  gint type_cast_strictness;
 } LogTemplate;
 
 /* template expansion options that can be influenced by the user and
@@ -188,6 +191,7 @@ void tf_simple_func_free_state(gpointer state);
 /* appends the formatted output into result */
 
 void log_template_set_escape(LogTemplate *self, gboolean enable);
+gboolean log_template_set_type_hint(LogTemplate *self, const gchar *hint, GError **error);
 gboolean log_template_compile(LogTemplate *self, const gchar *template, GError **error);
 void log_template_format(LogTemplate *self, LogMessage *lm, LogTemplateOptions *opts, gint tz, gint32 seq_num, const gchar *context_id, GString *result);
 void log_template_append_format(LogTemplate *self, LogMessage *lm, LogTemplateOptions *opts, gint tz, gint32 seq_num, const gchar *context_id, GString *result);

diff --git a/lib/tests/Makefile.am b/lib/tests/Makefile.am
@@ -1,4 +1,8 @@
-lib_tests_TESTS		= lib/tests/test_log_message lib/tests/test_cfg_lexer_subst
+lib_tests_TESTS		= \
+	lib/tests/test_log_message	\
+	lib/tests/test_cfg_lexer_subst	\
+	lib/tests/test_type_hints
+
 check_PROGRAMS		+= ${lib_tests_TESTS}
 
 lib_tests_test_log_message_CFLAGS	=	\
@@ -10,3 +14,8 @@ lib_tests_test_cfg_lexer_subst_CFLAGS	=	\
 	$(TEST_CFLAGS)
 lib_tests_test_cfg_lexer_subst_LDADD	=	\
 	$(TEST_LDADD)
+
+lib_tests_test_type_hints_CFLAGS	=	\
+	$(TEST_CFLAGS)
+lib_tests_test_type_hints_LDADD	=	\
+	$(TEST_LDADD)