Skip to content

Commit

Permalink
Update/fix more tests.
Browse files Browse the repository at this point in the history 
Signed-off-by: currantw <[email protected]>
  • Loading branch information
@currantw
currantw committed Jan 28, 2025
1 parent 2a2bcb7 commit 8cac79c
Show file tree
Hide file tree
Showing 8 changed files with 75 additions and 109 deletions.
3 changes: 2 additions & 1 deletion core/src/main/java/org/opensearch/sql/analysis/Analyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -467,14 +467,15 @@ public LogicalPlan visitFlatten(Flatten node, AnalysisContext context) {
LogicalPlan child = node.getChild().getFirst().accept(this, context);
TypeEnvironment env = context.peek();

// Verify that the field type is valid.
// Verify that the field name is valid.
ReferenceExpression fieldExpr;
try {
fieldExpr = (ReferenceExpression) expressionAnalyzer.analyze(node.getField(), context);
} catch (SemanticCheckException e) {
throw new IllegalArgumentException("Invalid field name for flatten command", e);
}

// Verify that the field type is valid.
ExprType fieldType = fieldExpr.type();
if (fieldType != STRUCT) {
throw new IllegalArgumentException(
Expand Down
24 changes: 10 additions & 14 deletions core/src/test/java/org/opensearch/sql/planner/physical/FlattenOperatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,19 +34,14 @@ class FlattenOperatorTest extends PhysicalPlanTestBase {
@Mock private PhysicalPlan inputPlan;

@Test
void testFlattenStruct() {
void testFlattenStructBasic() {
Map<String, Object> structMap =
ImmutableMap.of(
"string_field",
"string_value",
"integer_field",
1,
"long_field",
1L,
"boolean_field",
true,
"list_field",
List.of("a", "b"));
ImmutableMap.ofEntries(
Map.entry("string_field", "string_value"),
Map.entry("integer_field", 1),
Map.entry("long_field", 1L),
Map.entry("boolean_field", true),
Map.entry("list_field", List.of("a", "b")));

Map<String, Object> rowMap = ImmutableMap.of("struct_field", structMap);
ExprValue rowValue = ExprValueUtils.tupleValue(rowMap);
Expand Down Expand Up @@ -77,9 +72,10 @@ void testFlattenStructEmpty() {

@Test
void testFlattenStructNested() {
Map<String, Object> nestedStructMap =
ImmutableMap.ofEntries(Map.entry("nested_string_field", "string_value"));
Map<String, Object> structMap =
ImmutableMap.of(
"nested_struct_field", ImmutableMap.of("nested_string_field", "string_value"));
ImmutableMap.ofEntries(Map.entry("nested_struct_field", nestedStructMap));
Map<String, Object> rowMap = ImmutableMap.of("struct_field", structMap);
ExprValue rowValue = ExprValueUtils.tupleValue(rowMap);

Expand Down
8 changes: 7 additions & 1 deletion integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import static org.opensearch.sql.legacy.TestUtils.getAccountIndexMapping;
import static org.opensearch.sql.legacy.TestUtils.getBankIndexMapping;
import static org.opensearch.sql.legacy.TestUtils.getBankWithNullValuesIndexMapping;
import static org.opensearch.sql.legacy.TestUtils.getCitiesIndexMapping;
import static org.opensearch.sql.legacy.TestUtils.getDataTypeNonnumericIndexMapping;
import static org.opensearch.sql.legacy.TestUtils.getDataTypeNumericIndexMapping;
import static org.opensearch.sql.legacy.TestUtils.getDateIndexMapping;
Expand Down Expand Up @@ -751,7 +752,12 @@ public enum Index {
TestsConstants.TEST_INDEX_JSON_TEST,
"json",
getJsonTestIndexMapping(),
"src/test/resources/json_test.json");
"src/test/resources/json_test.json"),
CITIES(
TestsConstants.TEST_INDEX_CITIES,
"cities",
getCitiesIndexMapping(),
"src/test/resources/cities.json");

private final String name;
private final String type;
Expand Down
5 changes: 5 additions & 0 deletions integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,11 @@ public static String getJsonTestIndexMapping() {
return getMappingFile(mappingFile);
}

public static String getCitiesIndexMapping() {
String mappingFile = "cities.mapping.json";
return getMappingFile(mappingFile);
}

public static void loadBulk(Client client, String jsonPath, String defaultIndex)
throws Exception {
System.out.println(String.format("Loading file %s into opensearch cluster", jsonPath));
Expand Down
1 change: 1 addition & 0 deletions integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ public class TestsConstants {
public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls";
public static final String TEST_INDEX_GEOPOINT = TEST_INDEX + "_geopoint";
public static final String TEST_INDEX_JSON_TEST = TEST_INDEX + "_json_test";
public static final String TEST_INDEX_CITIES = TEST_INDEX + "_cities";
public static final String DATASOURCES = ".ql-datasources";

public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
Expand Down
105 changes: 12 additions & 93 deletions integ-test/src/test/java/org/opensearch/sql/ppl/FlattenCommandIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,7 @@

package org.opensearch.sql.ppl;

import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_MULTI_NESTED_TYPE;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_WITH_NULLS;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CITIES;
import static org.opensearch.sql.util.MatcherUtils.rows;
import static org.opensearch.sql.util.MatcherUtils.schema;
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
Expand All @@ -21,106 +19,27 @@ public class FlattenCommandIT extends PPLIntegTestCase {

@Override
public void init() throws IOException {
loadIndex(Index.NESTED_WITHOUT_ARRAYS);
loadIndex(Index.NESTED_WITH_NULLS);
loadIndex(Index.MULTI_NESTED);
loadIndex(Index.CITIES);
}

@Test
public void testFlattenStructBasic() throws IOException {
public void testFlattenStruct() throws IOException {
String query =
String.format(
"source=%s | flatten message | fields info, author, dayOfWeek",
TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS);
"source=%s | flatten location | fields state, province, country, latitude, longitude",
TEST_INDEX_CITIES);
JSONObject result = executeQuery(query);

verifySchema(
result,
schema("info", "string"),
schema("author", "string"),
schema("dayOfWeek", "integer"));
schema("state", "string"),
schema("province", "string"),
schema("country", "string"),
schema("latitude", "float"),
schema("longitude", "float"));
verifyDataRows(
result,
rows("a", "e", 1),
rows("b", "f", 2),
rows("c", "g", 1),
rows("c", "h", 4),
rows("zz", "zz", 6));
}

@Test
public void testFlattenStructMultiple() throws IOException {
String query =
String.format(
"source=%s | flatten message | flatten comment "
+ "| fields info, author, dayOfWeek, data, likes",
TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS);
JSONObject result = executeQuery(query);

verifySchema(
result,
schema("info", "string"),
schema("author", "string"),
schema("dayOfWeek", "integer"),
schema("data", "string"),
schema("likes", "integer"));
verifyDataRows(
result,
rows("a", "e", 1, "ab", 1),
rows("b", "f", 2, "aa", 2),
rows("c", "g", 1, "aa", 3),
rows("c", "h", 4, "ab", 1),
rows("zz", "zz", 6, "bb", 10));
}

@Test
public void testFlattenStructNull() throws IOException {
String query =
String.format(
"source=%s | flatten message | fields info, author, dayOfWeek",
TEST_INDEX_NESTED_WITH_NULLS);
JSONObject result = executeQuery(query);

verifySchema(
result,
schema("info", "string"),
schema("author", "string"),
schema("dayOfWeek", "integer"));
verifyDataRows(
result,
rows("e", null, 5),
rows("c", "g", 1),
rows("c", "h", 4),
rows("zz", "zz", 6),
rows("zz", "z\"z", 6),
rows(null, "e", 7),
rows("a", "e", 1),
rows("rr", "this \"value\" contains quotes", 3),
rows(null, null, null),
rows(null, null, null));
}

@Test
public void testFlattenStructNested() throws IOException {
String query =
String.format(
"source=%s | flatten message | fields info, name, street, number, dayOfWeek",
TEST_INDEX_MULTI_NESTED_TYPE);
JSONObject result = executeQuery(query);

verifySchema(
result,
schema("info", "string"),
schema("name", "string"),
schema("street", "string"),
schema("number", "integer"),
schema("dayOfWeek", "integer"));
verifyDataRows(
result,
rows("a", "e", "bc", 1, 1),
rows("b", "f", "ab", 2, 2),
rows("c", "g", "sk", 3, 1),
rows("d", "h", "mb", 4, 4),
rows("zz", "yy", "qc", 6, 6));
rows("Washington", null, "United States", 47.6061, -122.3328),
rows(null, "British Columbia", "Canada", 49.2827, -123.1207));
}
}
4 changes: 4 additions & 0 deletions integ-test/src/test/resources/cities.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{"index":{"_id":"1"}}
{"name": "Seattle", "location": { "state": "Washington", "country": "United States", "coordinates": {"latitude": 47.6061, "longitude": -122.3328}}}
{"index":{"_id":"2"}}
{"name": "Vancouver", "location": { "province": "British Columbia", "country": "Canada", "coordinates": {"latitude": 49.2827, "longitude": -123.1207}}}
34 changes: 34 additions & 0 deletions integ-test/src/test/resources/indexDefinitions/cities_mapping.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{
"mappings": {
"properties": {
"name": {
"type": "keyword"
},
"location": {
"type": "object",
"properties": {
"state": {
"type": "keyword"
},
"province": {
"type": "keyword"
},
"country": {
"type": "keyword"
},
"coordinates": {
"type": "object",
"properties": {
"latitude": {
"type": "double"
},
"longitude": {
"type": "double"
}
}
}
}
}
}
}
}

0 comments on commit 8cac79c

Please sign in to comment.