EQL: Add field resolution and verification

x-pack/plugin/eql/build.gradle

@@ -27,6 +27,7 @@ dependencies {
   testCompile project(':test:framework')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
   testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
+  testCompile project(path: xpackModule('ql'), configuration: 'testArtifacts')
   testCompile project(path: ':modules:reindex', configuration: 'runtime')
   testCompile project(path: ':modules:parent-join', configuration: 'runtime')
   testCompile project(path: ':modules:analysis-common', configuration: 'runtime')

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalysisUtils.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.analysis;
+
+import org.elasticsearch.xpack.ql.expression.Attribute;
+import org.elasticsearch.xpack.ql.expression.FieldAttribute;
+import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+import org.elasticsearch.xpack.ql.type.InvalidMappedField;
+import org.elasticsearch.xpack.ql.type.UnsupportedEsField;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+
+import static java.util.stream.Collectors.toList;
+
+public final class AnalysisUtils {
+
+    private AnalysisUtils() {}
+
+    //
+    // Shared methods around the analyzer rules
+    //
+    static Attribute resolveAgainstList(UnresolvedAttribute u, Collection<Attribute> attrList) {
+        return resolveAgainstList(u, attrList, false);
+    }
+
+    static Attribute resolveAgainstList(UnresolvedAttribute u, Collection<Attribute> attrList, boolean allowCompound) {
+        List<Attribute> matches = new ArrayList<>();
+
+        // first take into account the qualified version
+        boolean qualified = u.qualifier() != null;
+
+        for (Attribute attribute : attrList) {
+            if (!attribute.synthetic()) {
+                boolean match = qualified ? Objects.equals(u.qualifiedName(), attribute.qualifiedName()) :
+                // if the field is unqualified
+                // first check the names directly
+                        (Objects.equals(u.name(), attribute.name())
+                                // but also if the qualifier might not be quoted and if there's any ambiguity with nested fields
+                                || Objects.equals(u.name(), attribute.qualifiedName()));
+                if (match) {
+                    matches.add(attribute.withLocation(u.source()));
+                }
+            }
+        }
+
+        // none found
+        if (matches.isEmpty()) {
+            return null;
+        }
+
+        if (matches.size() == 1) {
+            return handleSpecialFields(u, matches.get(0), allowCompound);
+        }
+
+        return u.withUnresolvedMessage(
+                "Reference [" + u.qualifiedName() + "] is ambiguous (to disambiguate use quotes or qualifiers); matches any of "
+                        + matches.stream().map(a -> "\"" + a.qualifier() + "\".\"" + a.name() + "\"").sorted().collect(toList()));
+    }
+
+    private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute named, boolean allowCompound) {
+        // if it's a object/compound type, keep it unresolved with a nice error message
+        if (named instanceof FieldAttribute) {
+            FieldAttribute fa = (FieldAttribute) named;
+
+            // incompatible mappings
+            if (fa.field() instanceof InvalidMappedField) {
+                named = u.withUnresolvedMessage("Cannot use field [" + fa.name() + "] due to ambiguities being "
+                        + ((InvalidMappedField) fa.field()).errorMessage());
+            }
+            // unsupported types
+            else if (DataTypes.isUnsupported(fa.dataType())) {
+                UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
+                if (unsupportedField.hasInherited()) {
+                    named = u.withUnresolvedMessage("Cannot use field [" + fa.name() + "] with unsupported type ["
+                            + unsupportedField.getOriginalType() + "] " + "in hierarchy (field [" + unsupportedField.getInherited() + "])");
+                } else {
+                    named = u.withUnresolvedMessage(
+                            "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]");
+                }
+            }
+            // compound fields
+            else if (allowCompound == false && DataTypes.isPrimitive(fa.dataType()) == false) {
+                named = u.withUnresolvedMessage(
+                        "Cannot use field [" + fa.name() + "] type [" + fa.dataType().typeName() + "] only its subfields");
+            }
+        }
+        return named;
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java

@@ -6,13 +6,20 @@
 
 package org.elasticsearch.xpack.eql.analysis;
 
+import org.elasticsearch.xpack.ql.expression.Attribute;
+import org.elasticsearch.xpack.ql.expression.NamedExpression;
+import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.ql.rule.Rule;
 import org.elasticsearch.xpack.ql.rule.RuleExecutor;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 import static java.util.Arrays.asList;
+import static org.elasticsearch.xpack.eql.analysis.AnalysisUtils.resolveAgainstList;
 
 public class Analyzer extends RuleExecutor<LogicalPlan> {
 
@@ -26,7 +33,8 @@ public Analyzer(FunctionRegistry functionRegistry, Verifier verifier) {
 
     @Override
     protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
-        Batch resolution = new Batch("Resolution");
+        Batch resolution = new Batch("Resolution",
+                new ResolveRefs());
 
         return asList(resolution);
     }
@@ -42,4 +50,56 @@ private LogicalPlan verify(LogicalPlan plan) {
         }
         return plan;
     }
-}
+
+    private static class ResolveRefs extends AnalyzeRule<LogicalPlan> {
+
+        @Override
+        protected LogicalPlan rule(LogicalPlan plan) {
+            // if the children are not resolved, there's no way the node can be resolved
+            if (!plan.childrenResolved()) {
+                return plan;
+            }
+
+            // okay, there's a chance so let's get started
+            if (log.isTraceEnabled()) {
+                log.trace("Attempting to resolve {}", plan.nodeString());
+            }
+
+            return plan.transformExpressionsUp(e -> {
+                if (e instanceof UnresolvedAttribute) {
+                    UnresolvedAttribute u = (UnresolvedAttribute) e;
+                    List<Attribute> childrenOutput = new ArrayList<>();
+                    for (LogicalPlan child : plan.children()) {
+                        childrenOutput.addAll(child.output());
+                    }
+                    NamedExpression named = resolveAgainstList(u, childrenOutput);
+                    // if resolved, return it; otherwise keep it in place to be resolved later
+                    if (named != null) {
+                        if (log.isTraceEnabled()) {
+                            log.trace("Resolved {} to {}", u, named);
+                        }
+                        return named;
+                    }
+                }
+                return e;
+            });
+        }
+    }
+
+    abstract static class AnalyzeRule<SubPlan extends LogicalPlan> extends Rule<SubPlan, LogicalPlan> {
+
+        // transformUp (post-order) - that is first children and then the node
+        // but with a twist; only if the tree is not resolved or analyzed
+        @Override
+        public final LogicalPlan apply(LogicalPlan plan) {
+            return plan.transformUp(t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t), typeToken());
+        }
+
+        @Override
+        protected abstract LogicalPlan rule(SubPlan plan);
+
+        protected boolean skipResolved() {
+            return true;
+        }
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerRule.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.analysis;
+
+import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.ql.rule.Rule;
+
+public abstract class AnalyzerRule<SubPlan extends LogicalPlan> extends Rule<SubPlan, LogicalPlan> {
+
+    // transformUp (post-order) - that is first children and then the node
+    // but with a twist; only if the tree is not resolved or analyzed
+    @Override
+    public final LogicalPlan apply(LogicalPlan plan) {
+        return plan.transformUp(t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t), typeToken());
+    }
+
+    @Override
+    protected abstract LogicalPlan rule(SubPlan plan);
+
+    protected boolean skipResolved() {
+        return true;
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Verifier.java

@@ -99,6 +99,8 @@ Collection<Failure> verify(LogicalPlan plan) {
                     });
                 });
             }
+
+            failures.addAll(localFailures);
         });
 
         return failures;

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function;
+
+import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
+
+public class EqlFunctionRegistry extends FunctionRegistry {
+
+    public EqlFunctionRegistry() {
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java

@@ -11,20 +11,16 @@
 import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
-import org.elasticsearch.xpack.ql.index.EsIndex;
-import org.elasticsearch.xpack.ql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.ql.plan.logical.Filter;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 
-import static java.util.Collections.emptyMap;
-
 public abstract class LogicalPlanBuilder extends ExpressionBuilder {
 
     // TODO: these need to be made configurable
-    private static final String EVENT_TYPE = "event.category";
-    private static final EsIndex esIndex = new EsIndex("<not-specified>", emptyMap());
+    private static final String EVENT_TYPE = "event_type";
 
     @Override
     public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) {
@@ -43,6 +39,6 @@ public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) {
 
         }
 
-        return new Filter(source(ctx), new EsRelation(Source.EMPTY, esIndex, false), condition);
+        return new Filter(source(ctx), new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition);
     }
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Configuration.java

@@ -23,8 +23,7 @@ public class Configuration extends org.elasticsearch.xpack.ql.session.Configurat
     private QueryBuilder filter;
 
     public Configuration(String[] indices, ZoneId zi, String username, String clusterName, QueryBuilder filter,
-            TimeValue requestTimeout,
-            boolean includeFrozen, String clientId) {
+            TimeValue requestTimeout, boolean includeFrozen, String clientId) {
 
         super(zi, username, clusterName);
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
 import org.elasticsearch.xpack.eql.execution.PlanExecutor;
 import org.elasticsearch.xpack.eql.optimizer.Optimizer;
+import org.elasticsearch.xpack.eql.parser.EqlParser;
 import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.eql.planner.Planner;
 import org.elasticsearch.xpack.ql.index.IndexResolver;
@@ -97,7 +98,6 @@ private <T> void preAnalyze(LogicalPlan parsed, ActionListener<LogicalPlan> list
 
     private LogicalPlan doParse(String eql, List<Object> params) {
         Check.isTrue(params.isEmpty(), "Parameters were given despite being ignored - server bug");
-        //LogicalPlan plan = new EqlParser().createStatement(eql);
-        throw new UnsupportedOperationException();
+        return new EqlParser().createStatement(eql);
     }
 }

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.xpack.eql.session.Configuration;
+
+import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
+import static org.elasticsearch.test.ESTestCase.randomBoolean;
+import static org.elasticsearch.test.ESTestCase.randomNonNegativeLong;
+import static org.elasticsearch.test.ESTestCase.randomZone;
+
+public final class EqlTestUtils {
+
+    private EqlTestUtils() {}
+
+    public static final Configuration TEST_CFG = new Configuration(new String[] { "none" }, org.elasticsearch.xpack.ql.util.DateUtils.UTC,
+            "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, "");
+
+    public static Configuration randomConfiguration() {
+        return new Configuration(new String[] {randomAlphaOfLength(16)},
+                randomZone(),
+                randomAlphaOfLength(16),
+                randomAlphaOfLength(16),
+                null,
+                new TimeValue(randomNonNegativeLong()),
+                randomBoolean(),
+                randomAlphaOfLength(16));
+    }
+}