add arithmetics and indexed access to code generator

run all rule-based parser tests with and without code generation
refactor test functions to work around square/javapoet#526

plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AdditionExpression.java

@@ -72,6 +72,10 @@ public Object evaluateUnsafe(EvaluationContext context) {
         }
     }
 
+    public boolean isPlus() {
+        return isPlus;
+    }
+
     @Override
     public Class getType() {
         return type;

plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MultiplicationExpression.java

@@ -84,6 +84,10 @@ public Object evaluateUnsafe(EvaluationContext context) {
         }
     }
 
+    public char getOperator() {
+        return operator;
+    }
+
     @Override
     public Class getType() {
         return type;

plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/SignedExpression.java

@@ -67,4 +67,7 @@ public String toString() {
         return (isPlus ? " + " : " - ") + right.toString();
     }
 
+    public boolean isPlus() {
+        return isPlus;
+    }
 }

plugin/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/CodeGenerator.java

@@ -3,10 +3,12 @@
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.common.primitives.Ints;
 import com.google.common.primitives.Primitives;
 
 import com.squareup.javapoet.AnnotationSpec;
@@ -27,20 +29,26 @@
 import org.graylog.plugins.pipelineprocessor.ast.Rule;
 import org.graylog.plugins.pipelineprocessor.ast.RuleAstBaseListener;
 import org.graylog.plugins.pipelineprocessor.ast.RuleAstWalker;
+import org.graylog.plugins.pipelineprocessor.ast.expressions.AdditionExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanValuedFunctionWrapper;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.ComparisonExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.ConstantExpression;
+import org.graylog.plugins.pipelineprocessor.ast.expressions.DoubleExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.EqualityExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldAccessExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldRefExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression;
+import org.graylog.plugins.pipelineprocessor.ast.expressions.IndexedAccessExpression;
+import org.graylog.plugins.pipelineprocessor.ast.expressions.LongExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.MapLiteralExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.MessageRefExpression;
+import org.graylog.plugins.pipelineprocessor.ast.expressions.MultiplicationExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.NotExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.OrExpression;
+import org.graylog.plugins.pipelineprocessor.ast.expressions.SignedExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.StringExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.VarRefExpression;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
@@ -71,13 +79,6 @@
 
 public class CodeGenerator {
     private static final Logger log = LoggerFactory.getLogger(CodeGenerator.class);
-    private static AtomicLong ruleNameCounter = new AtomicLong();
-    private final FunctionRegistry functionRegistry;
-
-    @Inject
-    public CodeGenerator(FunctionRegistry functionRegistry) {
-        this.functionRegistry = functionRegistry;
-    }
 
     public static String sourceCodeForRule(Rule rule) {
         final JavaPoetListener javaPoetListener = new JavaPoetListener();
@@ -94,9 +95,11 @@ public Class<? extends GeneratedRule> generateCompiledRule(Rule rule) {
         ClassLoader ruleClassloader = new ClassLoader() {
         };
         try {
+            log.info("Sourcecode:\n{}", sourceCode);
             return (Class<GeneratedRule>) CompilerUtils.CACHED_COMPILER.loadFromJava(ruleClassloader, "org.graylog.plugins.pipelineprocessor.$dynamic.rules.rule$" + rule.id() , sourceCode);
         } catch (ClassNotFoundException e) {
-            return null;
+            log.error("Unable to compile code\n{}", sourceCode);
+            throw new RuntimeException(e);
         }
 
     }
@@ -458,6 +461,18 @@ public void exitString(StringExpression expr) {
             codeSnippet.putIfAbsent(expr, CodeBlock.of("$S", expr.evaluateUnsafe()));
         }
 
+        @Override
+        public void exitLong(LongExpression expr) {
+            // long needs a suffix
+            codeSnippet.putIfAbsent(expr, CodeBlock.of("$LL", expr.evaluateUnsafe()));
+        }
+
+        @Override
+        public void exitDouble(DoubleExpression expr) {
+            // double should have a suffix
+            codeSnippet.putIfAbsent(expr, CodeBlock.of("$Ld", expr.evaluateUnsafe()));
+        }
+
         @Override
         public void exitMessageRef(MessageRefExpression expr) {
             final Object field = blockOrMissing(codeSnippet.get(expr.getFieldExpr()), expr.getFieldExpr());
@@ -470,26 +485,13 @@ public void exitVariableAssignStatement(VarAssignStatement assign) {
             final Object value = blockOrMissing(codeSnippet.get(assign.getValueExpression()), assign.getValueExpression());
             final Class type = assign.getValueExpression().getType();
 
-            // potentially annotate the value with the type width, if we assign a constant
-            String suffix = "";
-            final boolean constantValue = assign.getValueExpression().isConstant();
-            if (constantValue) {
-                if (type.equals(Long.class)) {
-                    suffix = "L";
-                } else if (type.equals(Double.class)) {
-                    suffix = "D";
-                }
-            }
             // always hoist declaration
             hoistedExpressionMembers.add(FieldSpec.builder(type, "var$" + assign.getName(), Modifier.PRIVATE).build());
             if (assign.getValueExpression().isConstant()) {
                 // also hoist the assignment
-                hoistedConstantExpressions.addStatement("var$$$L = $L" + suffix,
-                        assign.getName(), value);
+                hoistedConstantExpressions.addStatement("var$$$L = $L", assign.getName(), value);
             } else {
-                currentMethod.addStatement("var$$$L = $L" + suffix,
-                        assign.getName(),
-                        value);
+                currentMethod.addStatement("var$$$L = $L", assign.getName(), value);
             }
         }
 
@@ -554,6 +556,64 @@ public void exitArrayLiteral(ArrayLiteralExpression expr) {
             codeSnippet.putIfAbsent(expr, CodeBlock.of("$L", listName));
         }
 
+        @Override
+        public void exitAddition(AdditionExpression expr) {
+            final Object leftBlock = blockOrMissing(codeSnippet.get(expr.left()), expr.left());
+            final Object rightBlock = blockOrMissing(codeSnippet.get(expr.right()), expr.right());
+
+            codeSnippet.putIfAbsent(expr,
+                    CodeBlock.of("$L " + (expr.isPlus() ? "+" : "-") + " $L", leftBlock, rightBlock));
+        }
+
+        @Override
+        public void exitMultiplication(MultiplicationExpression expr) {
+            final Object leftBlock = blockOrMissing(codeSnippet.get(expr.left()), expr.left());
+            final Object rightBlock = blockOrMissing(codeSnippet.get(expr.right()), expr.right());
+
+            codeSnippet.putIfAbsent(expr,
+                    CodeBlock.of("$L " + expr.getOperator() + " $L", leftBlock, rightBlock));
+        }
+
+        @Override
+        public void exitSigned(SignedExpression expr) {
+            final Object rightBlock = blockOrMissing(codeSnippet.get(expr.right()), expr.right());
+            codeSnippet.putIfAbsent(expr, CodeBlock.of((expr.isPlus() ? "+" : "-") + "$L", rightBlock));
+        }
+
+        @Override
+        public void exitIndexedAccess(IndexedAccessExpression expr) {
+            final Expression indexableObject = expr.getIndexableObject();
+            final Expression index = expr.getIndex();
+
+            final Object objectBlock = blockOrMissing(codeSnippet.get(indexableObject), indexableObject);
+            final Object indexBlock = blockOrMissing(codeSnippet.get(index), index);
+
+            final Class indexType = index.getType();
+            final Class indexableObjectType = indexableObject.getType();
+            CodeBlock block;
+            if (Long.class.equals(indexType)) {
+                // array indexing
+                if (indexableObjectType.isArray()) {
+                    block = CodeBlock.of("Arrays.get($L, $L)", objectBlock, indexBlock);
+                } else if (List.class.isAssignableFrom(indexableObjectType)) {
+                    block = CodeBlock.of("$L.get($T.saturatedCast($L))", objectBlock, ClassName.get(Ints.class), indexBlock);
+                } else if (Iterable.class.isAssignableFrom(indexableObjectType)) {
+                    block = CodeBlock.of("$T.get($L, $L)", ClassName.get(Iterables.class), objectBlock, indexBlock);
+                } else {
+                    log.error("Unhandled indexable object type: {}", indexableObject);
+                    block = null;
+                }
+            } else if (String.class.equals(indexType) && Map.class.isAssignableFrom(indexableObjectType)) {
+                // map indexing
+                block = CodeBlock.of("$L.get($L)", objectBlock, indexBlock);
+            } else {
+                // illegal
+                log.error("Invalid index type: {}", index);
+                block = null;
+            }
+            codeSnippet.putIfAbsent(expr, block);
+        }
+
         @Nonnull
         private String subExpressionName() {
             return "im$" + counter++;

plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java

@@ -97,6 +97,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.Stack;
+import java.util.concurrent.atomic.AtomicLong;
 
 import javax.inject.Inject;
 
@@ -110,6 +111,7 @@ public class PipelineRuleParser {
     private final CodeGenerator codeGenerator;
 
     private static boolean allowCodeGeneration = false;
+    private static AtomicLong uniqueId = new AtomicLong(0);
 
     @Inject
     public PipelineRuleParser(FunctionRegistry functionRegistry, CodeGenerator codeGenerator) {
@@ -130,7 +132,7 @@ public static void setAllowCodeGeneration(boolean allowCodeGeneration) {
 
 
     public Rule parseRule(String rule, boolean silent) throws ParseException {
-        return parseRule(null, rule, silent);
+        return parseRule("dummy" + uniqueId.getAndIncrement(), rule, silent);
     }
 
     public Rule parseRule(String id, String rule, boolean silent) throws ParseException {

plugin/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java

@@ -26,6 +26,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.statements.Statement;
 import org.graylog.plugins.pipelineprocessor.codegen.CodeGenerator;
+import org.graylog.plugins.pipelineprocessor.codegen.GeneratedRule;
 import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry;
 import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser;
 import org.graylog2.plugin.Message;
@@ -77,30 +78,45 @@ public FunctionDescriptor<Void> descriptor() {
 
     @Before
     public void setup() {
-        parser = new PipelineRuleParser(functionRegistry, new CodeGenerator(functionRegistry));
+        parser = new PipelineRuleParser(functionRegistry, new CodeGenerator());
         // initialize before every test!
         actionsTriggered.set(false);
     }
 
     protected EvaluationContext contextForRuleEval(Rule rule, Message message) {
         final EvaluationContext context = new EvaluationContext(message);
-        if (rule.when().evaluateBool(context)) {
-
-            for (Statement statement : rule.then()) {
-                statement.evaluate(context);
+        final GeneratedRule generatedRule = rule.generatedRule();
+        if (generatedRule != null) {
+            if (generatedRule.when(context)) {
+                generatedRule.then(context);
+            }
+        } else {
+            if (rule.when().evaluateBool(context)) {
+                for (Statement statement : rule.then()) {
+                    statement.evaluate(context);
+                }
             }
         }
         return context;
     }
 
     protected Message evaluateRule(Rule rule, Message message) {
         final EvaluationContext context = new EvaluationContext(message);
+        final GeneratedRule generatedRule = rule.generatedRule();
+        if (generatedRule != null) {
+            if (generatedRule.when(context)) {
+                generatedRule.then(context);
+                return context.currentMessage();
+            } else {
+                return null;
+            }
+        }
         if (rule.when().evaluateBool(context)) {
 
             for (Statement statement : rule.then()) {
                 statement.evaluate(context);
             }
-            return message;
+            return context.currentMessage();
         } else {
             return null;
         }

plugin/src/test/java/org/graylog/plugins/pipelineprocessor/codegen/Codegen.java

@@ -20,6 +20,7 @@
 import org.graylog2.plugin.Message;
 import org.graylog2.plugin.Tools;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.reflections.ReflectionUtils;
 import org.slf4j.Logger;
@@ -36,6 +37,7 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
+@Ignore("Replaced by CodegenPipelineRuleParserTest")
 public class Codegen extends BaseParserTest {
     private static final Logger log = LoggerFactory.getLogger(Codegen.class);
     private static final Path PARENT = Paths.get("/Users/kroepke/projects/graylog/graylog-project-repos/graylog-plugin-pipeline-processor/plugin/");

plugin/src/test/java/org/graylog/plugins/pipelineprocessor/parser/CodegenPipelineRuleParserTest.java

@@ -0,0 +1,25 @@
+/**
+ * This file is part of Graylog Pipeline Processor.
+ *
+ * Graylog Pipeline Processor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog Pipeline Processor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog Pipeline Processor.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog.plugins.pipelineprocessor.parser;
+
+public class CodegenPipelineRuleParserTest extends PipelineRuleParserTest {
+
+    // runs the same tests as in PipelineRuleParserTest but with dynamic code generation turned on.
+    public CodegenPipelineRuleParserTest() {
+        PipelineRuleParser.setAllowCodeGeneration(true);
+    }
+}