Merge pull request #43 from nipunayf/fix-6141

Add the integration test suite for the readString API

ballerina/Ballerina.toml

@@ -1,7 +1,7 @@
 [package]
 org = "ballerina"
 name = "yaml"
-version = "0.5.2"
+version = "0.5.3"
 authors = ["Ballerina"]
 keywords = ["yaml"]
 repository = "https://github.com/ballerina-platform/module-ballerina-yaml"

ballerina/Dependencies.toml

@@ -147,7 +147,7 @@ dependencies = [
 [[package]]
 org = "ballerina"
 name = "yaml"
-version = "0.5.2"
+version = "0.5.3"
 dependencies = [
 	{org = "ballerina", name = "file"},
 	{org = "ballerina", name = "io"},

ballerina/modules/composer/state.bal

@@ -36,10 +36,10 @@ public class ComposerState {
     # Flag is set if same map keys are allowed in a mapping
     readonly & boolean allowMapEntryRedefinition;
 
-    public isolated function init(string[]|string yamlInput, map<schema:YAMLTypeConstructor> tagSchema,
+    public isolated function init(string[] lines, map<schema:YAMLTypeConstructor> tagSchema,
         boolean allowAnchorRedefinition, boolean allowMapEntryRedefinition) returns parser:ParsingError? {
 
-        self.parserState = check new (yamlInput);
+        self.parserState = check new (lines);
         self.tagSchema = tagSchema;
         self.allowAnchorRedefinition = allowAnchorRedefinition;
         self.allowMapEntryRedefinition = allowMapEntryRedefinition;

ballerina/modules/lexer/lexer.bal

@@ -62,12 +62,6 @@ public isolated function scan(LexerState state) returns LexerState|LexicalError
         return state.index == 0 ? state.tokenize(EMPTY_LINE) : state.tokenize(EOL);
     }
 
-    // Check for line breaks when reading from string
-    if state.peek() == "\n" && state.context != LEXER_DOUBLE_QUOTE {
-        state.isNewLine = true;
-        return state.tokenize(EOL);
-    }
-
     match state.context {
         LEXER_START => {
             return contextStart(state);

ballerina/modules/lexer/scan.bal

@@ -158,7 +158,7 @@ isolated function scanPlanarChar(LexerState state) returns boolean|LexicalError
     }
 
     // Step back from the white spaces if EOL or ':' is reached 
-    if state.peek() == () || state.peek() == "\n" {
+    if state.peek() == () || state.isNewLine() {
         state.forward(-numWhitespace);
         return true;
     }

ballerina/modules/lexer/state.bal

@@ -70,8 +70,6 @@ public class LexerState {
 
     public boolean firstLine = true;
 
-    public boolean isNewLine = false;
-
     int mappingKeyColumn = -1;
 
     # Output YAML token
@@ -150,7 +148,6 @@ public class LexerState {
         self.indentStartIndex = -1;
         self.tokensForMappingValue = [];
         self.tabInWhitespace = -1;
-        self.isNewLine = false;
         self.keyDefinedForLine = false;
     }
 
@@ -167,6 +164,10 @@ public class LexerState {
     }
 
     public isolated function isFlowCollection() returns boolean => self.numOpenedFlowCollections > 0;
-
-    public isolated function isEndOfStream() returns boolean => self.index >= self.line.length();
+
+    # Check if the current character is a new line. 
+    # This should be replaced by the os module once it supports an API: #4931.
+    # 
+    # + return - True if the current character is a new line
+    public isolated function isNewLine() returns boolean => self.peek() == "\n" || self.peek() == "\r\n";
 }

ballerina/modules/parser/parser.bal

@@ -250,8 +250,8 @@ public isolated function parse(ParserState state, ParserOption option = DEFAULT,
 # + return - True if the string is a valid planar scalar. Else, false.
 public isolated function isValidPlanarScalar(string value) returns boolean {
     string? planarScalarResult = ();
-    do {   
-        ParserState parserState = check new (value);
+    do {
+        ParserState parserState = check new ([value]);
         planarScalarResult = check planarScalar(parserState, false);
     } on fail {
         return false;

ballerina/modules/parser/state.bal

@@ -11,13 +11,12 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 import yaml.lexer;
 import yaml.common;
 
 public class ParserState {
     # Properties for the YAML lines
-    string[]|string yamlInput;
+    string[] lines;
     int numLines;
     int lineIndex = -1;
 
@@ -62,12 +61,9 @@ public class ParserState {
 
     common:Event[] eventBuffer = [];
 
-    boolean isStringInput;
-
-    public isolated function init(string[]|string yamlInput) returns ParsingError? {
-        self.yamlInput = yamlInput;
-        self.isStringInput = yamlInput is string;
-        self.numLines = self.isStringInput ? 1 : (<string[]>yamlInput).length();
+    public isolated function init(string[] lines) returns ParsingError? {
+        self.lines = lines;
+        self.numLines = lines.length();
         ParsingError? err = self.initLexer();
         if err is ParsingError {
             self.eventBuffer.push({endType: common:STREAM});
@@ -90,36 +86,16 @@ public class ParserState {
         self.lineIndex += 1;
         string line;
 
-        if self.isStringInput {
-            string currentLine = self.lexerState.line;
-            if self.lexerState.isNewLine {
-                line = currentLine.substring(self.lexerState.index);
-            } else {
-                if self.lineIndex == 0 {
-                    line = <string>self.yamlInput;
-                } else {
-                    int? index = currentLine.indexOf("\n");
-                    if index is int {
-                        line = currentLine.substring(index + 1);
-                    } else {
-                        return generateGrammarError(self, message);
-                    }
-                }
-            }
-        } else {
-            if self.lineIndex >= self.numLines {
-                return generateGrammarError(self, message);
-            }
-            string[] lines = <string[]>self.yamlInput;
-            line = lines[self.lineIndex];
+        if self.lineIndex >= self.numLines {
+            return generateGrammarError(self, message);
         }
+        line = self.lines[self.lineIndex];
 
         self.explicitDoc = false;
         self.expectBlockSequenceValue = false;
         self.tagPropertiesInLine = false;
         self.lexerState.setLine(line, self.lineIndex);
     }
 
-    isolated function isEndOfFile() returns boolean =>
-        self.isStringInput ? self.lexerState.isEndOfStream() : self.lineIndex >= self.numLines - 1;
+    isolated function isEndOfFile() returns boolean => self.lineIndex >= self.numLines - 1;
 }

ballerina/tests/api_test.bal

@@ -17,8 +17,7 @@ import ballerina/io;
 import ballerina/test;
 
 @test:Config {
-    groups: ["api"],
-    enable: false
+    groups: ["api"]
 }
 function testReadYamlString() returns error? {
     string input = string `
@@ -76,8 +75,7 @@ function testWriteYamlFile() returns error? {
 
 @test:Config {
     dataProvider: yamlSchemaDataGen,
-    groups: ["api"],
-    enable: false
+    groups: ["api"]
 }
 function testReadYAMLSchema(YAMLSchema schema, json expectedOutput) returns error? {
     string input = string `

ballerina/tests/it.bal

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import ballerina/io;
 import ballerina/test;
 
 YamlType[] customYamlTypes = [];
@@ -35,7 +36,7 @@ map<FailSafeSchema> customTags = {
 };
 
 @test:BeforeSuite
-function initYamlCustomeTypes() {
+function initYamlCustomTypes() {
     customTags.entries().forEach(function([string, FailSafeSchema] entry) {
         customYamlTypes.push({
             tag: entry[0],
@@ -50,11 +51,27 @@ function initYamlCustomeTypes() {
 @test:Config {
     dataProvider: yamlDataGen
 }
-function testYAMLIntegrationTest(string filePath, json expectedOutput, boolean isStream, boolean isError) returns error? {
+function testYAMLIntegrationTestForReadFile(string filePath, json expectedOutput, boolean isStream, boolean isError) returns error? {
     json|Error output = readFile(filePath, yamlTypes = customYamlTypes, isStream = isStream);
+    assertOutput(output, expectedOutput, isError);
+}
+
+@test:Config {
+    dataProvider: yamlDataGen
+}
+function testYAMLIntegrationTestForReadString(string filePath, json expectedOutput, boolean isStream, boolean isError) returns error? {
+    io:ReadableByteChannel byteChannel = check io:openReadableFile(filePath);
+    (byte[] & readonly) readBytes = check byteChannel.readAll();
+    string yamlContent = check string:fromBytes(readBytes);
+
+    json|Error output = readString(yamlContent, yamlTypes = customYamlTypes, isStream = isStream);
+    assertOutput(output, expectedOutput, isError);
+}
+
+function assertOutput(json|Error actualOut, json expectedOut, boolean isError) {
     if isError {
-        test:assertTrue(output is Error);
+        test:assertTrue(actualOut is Error);
     } else {
-        test:assertEquals(output, expectedOutput);
+        test:assertEquals(actualOut, expectedOut);
     }
 }

ballerina/utils.bal

@@ -11,8 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 import yaml.schema;
+import yaml.composer;
+
 import ballerina/file;
 
 # Checks if the file exists. If not, creates a new file.
@@ -61,3 +62,15 @@ isolated function generateTagHandlesMap(YamlType[] yamlTypes, YAMLSchema yamlSch
 
     return tagHandles;
 }
+
+# Parses the given lines of YAML and returns the JSON representation.
+#
+# + lines - Lines of the YAML document
+# + config - Configuration for the YAML parser
+# + return - JSON representation of the YAML document
+isolated function readLines(string[] lines, ReadConfig config) returns json|Error {
+    composer:ComposerState composerState = check new (lines, generateTagHandlesMap(config.yamlTypes, config.schema),
+        config.allowAnchorRedefinition, config.allowMapEntryRedefinition
+    );
+    return config.isStream ? composer:composeStream(composerState) : composer:composeDocument(composerState);
+}

ballerina/yaml.bal

@@ -15,18 +15,17 @@
 import ballerina/io;
 import yaml.emitter;
 import yaml.serializer;
-import yaml.composer;
 
 # Parses a Ballerina string of YAML content into a Ballerina map object.
 #
 # + yamlString - YAML content
 # + config - Configuration for reading a YAML file
 # + return - YAML map object on success. Else, returns an error
 public isolated function readString(string yamlString, *ReadConfig config) returns json|Error {
-    composer:ComposerState composerState = check new (yamlString,
-        generateTagHandlesMap(config.yamlTypes, config.schema), config.allowAnchorRedefinition,
-        config.allowMapEntryRedefinition);
-    return composer:composeDocument(composerState);
+    io:ReadableByteChannel byteChannel = check io:createReadableChannel(yamlString.toBytes());
+    io:ReadableCharacterChannel charChannel = new (byteChannel, io:DEFAULT_ENCODING);
+    string[] lines = check charChannel.readAllLines();
+    return readLines(lines, config);
 }
 
 # Parses a YAML file into a Ballerina json object.
@@ -36,9 +35,7 @@ public isolated function readString(string yamlString, *ReadConfig config) retur
 # + return - YAML map object on success. Else, returns an error
 public isolated function readFile(string filePath, *ReadConfig config) returns json|Error {
     string[] lines = check io:fileReadLines(filePath);
-    composer:ComposerState composerState = check new (lines, generateTagHandlesMap(config.yamlTypes, config.schema),
-        config.allowAnchorRedefinition, config.allowMapEntryRedefinition);
-    return config.isStream ? composer:composeStream(composerState) : composer:composeDocument(composerState);
+    return readLines(lines, config);
 }
 
 # Converts the YAML structure to an array of strings.