Skip to content

Commit

Permalink
Merge branch 'feature/acarbo_json_cast_ppl' into feature/json-extract
Browse files Browse the repository at this point in the history 
Signed-off-by: 14yapkc1 <[email protected]>
Signed-off-by: Kenrick Yap <[email protected]>
  • Loading branch information
@kenrickyap
kenrickyap committed Jan 28, 2025
2 parents 2b08007 + a5652ea commit 6bd2f40
Show file tree
Hide file tree
Showing 7 changed files with 283 additions and 97 deletions.
4 changes: 4 additions & 0 deletions core/src/main/java/org/opensearch/sql/expression/DSL.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -691,6 +691,10 @@ public static FunctionExpression jsonExtract(Expression... expressions) {
return compile(FunctionProperties.None, BuiltinFunctionName.JSON_EXTRACT, expressions);
}

public static FunctionExpression stringToJson(Expression value) {
return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value);
}

public static Aggregator avg(Expression... expressions) {
return aggregate(BuiltinFunctionName.AVG, expressions);
}
Expand Down
18 changes: 12 additions & 6 deletions core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,8 @@ private static DefaultFunctionResolver castToShort() {
impl(
nullMissingHandling((v) -> new ExprShortValue(v.booleanValue() ? 1 : 0)),
SHORT,
BOOLEAN));
BOOLEAN),
impl(nullMissingHandling((v) -> v), SHORT, UNDEFINED));
}

private static DefaultFunctionResolver castToInt() {
Expand All @@ -122,7 +123,8 @@ private static DefaultFunctionResolver castToInt() {
impl(
nullMissingHandling((v) -> new ExprIntegerValue(v.booleanValue() ? 1 : 0)),
INTEGER,
BOOLEAN));
BOOLEAN),
impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED));
}

private static DefaultFunctionResolver castToLong() {
Expand All @@ -136,7 +138,8 @@ private static DefaultFunctionResolver castToLong() {
impl(
nullMissingHandling((v) -> new ExprLongValue(v.booleanValue() ? 1L : 0L)),
LONG,
BOOLEAN));
BOOLEAN),
impl(nullMissingHandling((v) -> v), LONG, UNDEFINED));
}

private static DefaultFunctionResolver castToFloat() {
Expand All @@ -150,7 +153,8 @@ private static DefaultFunctionResolver castToFloat() {
impl(
nullMissingHandling((v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)),
FLOAT,
BOOLEAN));
BOOLEAN),
impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED));
}

private static DefaultFunctionResolver castToDouble() {
Expand All @@ -164,7 +168,8 @@ private static DefaultFunctionResolver castToDouble() {
impl(
nullMissingHandling((v) -> new ExprDoubleValue(v.booleanValue() ? 1D : 0D)),
DOUBLE,
BOOLEAN));
BOOLEAN),
impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED));
}

private static DefaultFunctionResolver castToBoolean() {
Expand All @@ -176,7 +181,8 @@ private static DefaultFunctionResolver castToBoolean() {
STRING),
impl(
nullMissingHandling((v) -> ExprBooleanValue.of(v.doubleValue() != 0)), BOOLEAN, DOUBLE),
impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN));
impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN),
impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED));
}

private static DefaultFunctionResolver castToIp() {
Expand Down
103 changes: 54 additions & 49 deletions core/src/main/java/org/opensearch/sql/utils/JsonUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,11 @@
import java.util.List;
import java.util.Map;
import lombok.experimental.UtilityClass;
import org.opensearch.sql.data.model.ExprBooleanValue;
import org.opensearch.sql.data.model.ExprCollectionValue;
import org.opensearch.sql.data.model.ExprDoubleValue;
import org.opensearch.sql.data.model.ExprIntegerValue;
import org.opensearch.sql.data.model.ExprNullValue;
import org.opensearch.sql.data.model.ExprStringValue;
import org.opensearch.sql.data.model.ExprTupleValue;
import org.opensearch.sql.data.model.ExprValue;
Expand All @@ -31,7 +33,7 @@ public class JsonUtils {
* Checks if given JSON string can be parsed as valid JSON.
*
* @param jsonExprValue JSON string (e.g. "{\"hello\": \"world\"}").
* @return true if the string can be parsed as valid JSON, else false.
* @return true if the string can be parsed as valid JSON, else false (including null or missing).
*/
public static ExprValue isValidJson(ExprValue jsonExprValue) {
ObjectMapper objectMapper = new ObjectMapper();
Expand All @@ -48,9 +50,31 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) {
}
}

/** Converts a JSON encoded string to an Expression object. */
/**
* Converts a JSON encoded string to a {@link ExprValue}. Expression type will be UNDEFINED.
*
* @param json JSON string (e.g. "{\"hello\": \"world\"}").
* @return ExprValue returns an expression that best represents the provided JSON-encoded string.
* <ol>
* <li>{@link ExprTupleValue} if the JSON is an object
* <li>{@link ExprCollectionValue} if the JSON is an array
* <li>{@link ExprDoubleValue} if the JSON is a floating-point number scalar
* <li>{@link ExprIntegerValue} if the JSON is an integral number scalar
* <li>{@link ExprStringValue} if the JSON is a string scalar
* <li>{@link ExprBooleanValue} if the JSON is a boolean scalar
* <li>{@link ExprNullValue} if the JSON is null, empty, or invalid
* </ol>
*/
public static ExprValue castJson(ExprValue json) {
JsonNode jsonNode = jsonToNode(json);
ObjectMapper objectMapper = new ObjectMapper();
JsonNode jsonNode;
try {
jsonNode = objectMapper.readTree(json.stringValue());
} catch (JsonProcessingException e) {
final String errorFormat = "JSON string '%s' is not valid. Error details: %s";
throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e);
}

return processJsonNode(jsonNode);
}

Expand Down Expand Up @@ -83,53 +107,34 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) {
}
}

private static JsonNode jsonToNode(ExprValue json) {
return jsonStringToNode(json.stringValue());
}

private static JsonNode jsonStringToNode(String jsonString) {
ObjectMapper objectMapper = new ObjectMapper();
JsonNode jsonNode;
try {
jsonNode = objectMapper.readTree(jsonString);
} catch (JsonProcessingException e) {
final String errorFormat = "JSON string '%s' is not valid. Error details: %s";
throw new SemanticCheckException(String.format(errorFormat, jsonString, e.getMessage()), e);
}
return jsonNode;
}

private static ExprValue processJsonNode(JsonNode jsonNode) {
if (jsonNode.isFloatingPointNumber()) {
return new ExprDoubleValue(jsonNode.asDouble());
}
if (jsonNode.isIntegralNumber()) {
return new ExprIntegerValue(jsonNode.asLong());
}
if (jsonNode.isBoolean()) {
return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE;
}
if (jsonNode.isTextual()) {
return new ExprStringValue(jsonNode.asText());
switch (jsonNode.getNodeType()) {
case ARRAY:
List<ExprValue> elements = new LinkedList<>();
for (var iter = jsonNode.iterator(); iter.hasNext(); ) {
jsonNode = iter.next();
elements.add(processJsonNode(jsonNode));
}
return new ExprCollectionValue(elements);
case OBJECT:
Map<String, ExprValue> values = new LinkedHashMap<>();
for (var iter = jsonNode.fields(); iter.hasNext(); ) {
Map.Entry<String, JsonNode> entry = iter.next();
values.put(entry.getKey(), processJsonNode(entry.getValue()));
}
return ExprTupleValue.fromExprValueMap(values);
case STRING:
return new ExprStringValue(jsonNode.asText());
case NUMBER:
if (jsonNode.isFloatingPointNumber()) {
return new ExprDoubleValue(jsonNode.asDouble());
}
return new ExprIntegerValue(jsonNode.asLong());
case BOOLEAN:
return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE;
default:
// in all other cases, return null
return LITERAL_NULL;
}
if (jsonNode.isArray()) {
List<ExprValue> elements = new LinkedList<>();
for (var iter = jsonNode.iterator(); iter.hasNext(); ) {
jsonNode = iter.next();
elements.add(processJsonNode(jsonNode));
}
return new ExprCollectionValue(elements);
}
if (jsonNode.isObject()) {
Map<String, ExprValue> values = new LinkedHashMap<>();
for (var iter = jsonNode.fields(); iter.hasNext(); ) {
Map.Entry<String, JsonNode> entry = iter.next();
values.put(entry.getKey(), processJsonNode(entry.getValue()));
}
return ExprTupleValue.fromExprValueMap(values);
}

// in all other cases, return null
return LITERAL_NULL;
}
}
93 changes: 59 additions & 34 deletions core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,43 +38,57 @@

@ExtendWith(MockitoExtension.class)
public class JsonFunctionsTest {
private static final ExprValue JsonNestedObject =
ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}");
private static final ExprValue JsonObject =
ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}");
private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]");
private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\"");
private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue("");
private static final ExprValue JsonInvalidObject =
ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}");
private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc");

@Test
public void json_valid_returns_false() {
assertEquals(LITERAL_FALSE, execute(JsonInvalidObject));
assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar));
assertEquals(LITERAL_FALSE, execute(LITERAL_NULL));
assertEquals(LITERAL_FALSE, execute(LITERAL_MISSING));
assertEquals(
LITERAL_FALSE,
DSL.jsonValid(DSL.literal(ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}")))
.valueOf());
assertEquals(
LITERAL_FALSE, DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue("abc")))).valueOf());
assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_NULL))).valueOf());
assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_MISSING))).valueOf());
}

@Test
public void json_valid_throws_ExpressionEvaluationException() {
assertThrows(
ExpressionEvaluationException.class, () -> execute(ExprValueUtils.booleanValue(true)));
ExpressionEvaluationException.class,
() -> DSL.jsonValid(DSL.literal((ExprValueUtils.booleanValue(true)))).valueOf());
}

@Test
public void json_valid_returns_true() {
assertEquals(LITERAL_TRUE, execute(JsonNestedObject));
assertEquals(LITERAL_TRUE, execute(JsonObject));
assertEquals(LITERAL_TRUE, execute(JsonArray));
assertEquals(LITERAL_TRUE, execute(JsonScalarString));
assertEquals(LITERAL_TRUE, execute(JsonEmptyString));
}

private ExprValue execute(ExprValue jsonString) {
FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString));
return exp.valueOf();
List<String> validJsonStrings =
List.of(
// test json objects are valid
"{\"a\":\"1\",\"b\":\"2\"}",
"{\"a\":1,\"b\":{\"c\":2,\"d\":3}}",
"{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}",

// test json arrays are valid
"[1, 2, 3, 4]",
"[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]",

// test json scalars are valid
"\"abc\"",
"1234",
"12.34",
"true",
"false",
"null",

// test empty string is valid
"");

validJsonStrings.stream()
.forEach(
str ->
assertEquals(
LITERAL_TRUE,
DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue(str)))).valueOf(),
String.format("String %s must be valid json", str)));
}

@Test
Expand All @@ -101,12 +115,16 @@ void json_returnsJsonObject() {
ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap);

// exercise
exp = DSL.json_function(DSL.literal(objectJson));
exp = DSL.stringToJson(DSL.literal(objectJson));

// Verify
var value = exp.valueOf();
assertTrue(value instanceof ExprTupleValue);
assertEquals(expectedTupleExpr, value);

// also test the empty object case
assertEquals(
ExprTupleValue.fromExprValueMap(Map.of()), DSL.stringToJson(DSL.literal("{}")).valueOf());
}

@Test
Expand All @@ -127,29 +145,36 @@ void json_returnsJsonArray() {
LITERAL_NULL));

// exercise
exp = DSL.json_function(DSL.literal(arrayJson));
exp = DSL.stringToJson(DSL.literal(arrayJson));

// Verify
var value = exp.valueOf();
assertTrue(value instanceof ExprCollectionValue);
assertEquals(expectedArrayExpr, value);

// also test the empty-array case
assertEquals(new ExprCollectionValue(List.of()), DSL.stringToJson(DSL.literal("[]")).valueOf());
}

@Test
void json_returnsScalar() {
assertEquals(
new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf());
new ExprStringValue("foobar"), DSL.stringToJson(DSL.literal("\"foobar\"")).valueOf());

assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf());
assertEquals(new ExprIntegerValue(1234), DSL.stringToJson(DSL.literal("1234")).valueOf());

assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf());
assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf());

assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf());
assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf());
assertEquals(LITERAL_FALSE, DSL.stringToJson(DSL.literal("false")).valueOf());

assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf());
assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf());

assertEquals(
ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf());
assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal(LITERAL_NULL)).valueOf());

assertEquals(LITERAL_MISSING, DSL.stringToJson(DSL.literal(LITERAL_MISSING)).valueOf());

assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("")).valueOf());
}

@Test
Expand Down
Loading

0 comments on commit 6bd2f40

Please sign in to comment.