[SYSTEMDS-3240] Fix IOGEN test path if the iogen directory doesn't exist

* Also, bad format data generated due to generateRandomData() method is fixed which lead to failures in `org.apache.sysds.test.functions.iogen.FrameGenerateReaderCSVTest` Closes #1475.
apache · Dec 7, 2021 · 9a295c0 · 9a295c0
1 parent 0a3a726
commit 9a295c0
Show file tree

Hide file tree

Showing 7 changed files with 23 additions and 11 deletions.
diff --git a/src/test/java/org/apache/sysds/test/functions/iogen/FrameGenerateReaderCSVTest.java b/src/test/java/org/apache/sysds/test/functions/iogen/FrameGenerateReaderCSVTest.java
@@ -86,7 +86,7 @@ public void test4() {
 	public void test5() {
 		String[] naStrings = {"NULL", "inf", "NaN"};
 		String separator = ",,,,";
-		generateRandomData(10, 10, -10, 10, 0.2, naStrings);
+		generateRandomData(10, 10, -10, 10, 0.5, naStrings);
 		extractSampleRawCSV(separator);
 		runGenerateReaderTest();
 	}
@@ -95,7 +95,7 @@ public void test5() {
 	public void test6() {
 		String[] naStrings = {"NULL", "inf", "NaN"};
 		String separator = "**";
-		generateRandomData(1000, 100, -10, 10, 0.2, naStrings);
+		generateRandomData(1000, 100, -10, 10, 0.4, naStrings);
 		extractSampleRawCSV(separator);
 		runGenerateReaderTest();
 	}
@@ -113,7 +113,7 @@ public void test7() {
 	public void test8() {
 		String[] naStrings = {"NULL", "inf", "NaN"};
 		String separator = "**";
-		generateRandomData(100000, 100, -10, 10, 0.5, naStrings);
+		generateRandomData(10000, 100, -10, 10, 0.5, naStrings);
 		extractSampleRawCSV(separator);
 		runGenerateReaderTest();
 	}

diff --git a/src/test/java/org/apache/sysds/test/functions/iogen/FrameGenerateReaderLibSVMTest.java b/src/test/java/org/apache/sysds/test/functions/iogen/FrameGenerateReaderLibSVMTest.java
@@ -85,7 +85,7 @@ public void test1() {
 		String[] naStrings = {"NULL", "inf", "NaN"};
 		String separator = ",";
 		String indexSeparator = ":";
-		generateRandomData(5, 10, -100, 100, 1, naStrings);
+		generateRandomData(10, 10, -100, 100, 1, naStrings);
 		extractSampleRawLibSVM(0,separator, indexSeparator);
 		runGenerateReaderTest();
 	}
@@ -115,7 +115,7 @@ public void test4() {
 		String[] naStrings = {"NULL", "inf", "NaN"};
 		String separator = ",,,,,,";
 		String indexSeparator = ":";
-		generateRandomData(10, 10, -10, 10, 0.5, naStrings);
+		generateRandomData(20, 20, -100, 100, 0.6, naStrings);
 		extractSampleRawLibSVM(0,separator, indexSeparator);
 		runGenerateReaderTest();
 	}
@@ -125,7 +125,7 @@ public void test5() {
 		String[] naStrings = {"NULL", "inf", "NaN"};
 		String separator = ",,,,,";
 		String indexSeparator = ":";
-		generateRandomData(100, 10, -100, 100, 0.7, naStrings);
+		generateRandomData(100, 50, -100, 100, 0.5, naStrings);
 		extractSampleRawLibSVM(0,separator, indexSeparator);
 		runGenerateReaderTest();
 	}
@@ -135,7 +135,7 @@ public void test6() {
 		String[] naStrings = {"NULL", "inf", "NaN"};
 		String separator = ",,,,,";
 		String indexSeparator = ":";
-		generateRandomData(10, 1000, -100, 100, 0.5, naStrings);
+		generateRandomData(10, 1000, -100, 100, 0.7, naStrings);
 		extractSampleRawLibSVM(1,separator, indexSeparator);
 		runGenerateReaderTest();
 	}

diff --git a/src/test/java/org/apache/sysds/test/functions/iogen/GenerateReaderFrameTest.java b/src/test/java/org/apache/sysds/test/functions/iogen/GenerateReaderFrameTest.java
@@ -31,6 +31,7 @@
 import org.apache.sysds.test.TestUtils;
 
 import java.io.BufferedWriter;
+import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.Random;
@@ -141,6 +142,8 @@ protected void generateRandomData(int nrows, int ncols, double min, double max,
 
 			Random rn = new Random();
 			int rnt = rn.nextInt(types.length);
+			if(i == 0|| i==ncols-1)
+				rnt = 3;
 			schema[i] = types[rnt];
 
 			if(types[rnt] == Types.ValueType.STRING)
@@ -167,6 +170,10 @@ protected void runGenerateReaderTest() {
 			FrameBlock sampleFrame = new FrameBlock(schema, names, data);
 
 			String HOME = SCRIPT_DIR + TEST_DIR;
+			File directory = new File(HOME);
+			if (! directory.exists()){
+				directory.mkdir();
+			}
 			String dataPath = HOME + "frame_data.raw";
 			int clen = data[0].length;
 			writeRawString(sampleRaw, dataPath);

diff --git a/src/test/java/org/apache/sysds/test/functions/iogen/GenerateReaderMatrixTest.java b/src/test/java/org/apache/sysds/test/functions/iogen/GenerateReaderMatrixTest.java
@@ -31,6 +31,7 @@
 import org.apache.sysds.test.TestUtils;
 
 import java.io.BufferedWriter;
+import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 
@@ -81,6 +82,10 @@ protected void runGenerateReaderTest() {
 			MatrixBlock sampleMB = DataConverter.convertToMatrixBlock(sampleMatrix);
 
 			String HOME = SCRIPT_DIR + TEST_DIR;
+			File directory = new File(HOME);
+			if (! directory.exists()){
+				directory.mkdir();
+			}
 			String dataPath = HOME + "matrix_data.raw";
 			int clen = sampleMatrix[0].length;
 			writeRawString(sampleRaw, dataPath);

diff --git a/src/test/java/org/apache/sysds/test/functions/iogen/MatrixGenerateReaderCSVTest.java b/src/test/java/org/apache/sysds/test/functions/iogen/MatrixGenerateReaderCSVTest.java
@@ -145,7 +145,7 @@ public void test12() {
 	@Test
 	public void test13() {
 		String[] naString = {"Nan", "NAN", "", "inf", "null", "NULL"};
-		generateRandomCSV(1000, 2000, -10, 10, 0.5, ",,", naString);
+		generateRandomCSV(1000, 500, -10, 10, 0.5, ",,", naString);
 		runGenerateReaderTest();
 	}
 }
diff --git a/src/test/java/org/apache/sysds/test/functions/iogen/MatrixGenerateReaderLibSVMTest.java b/src/test/java/org/apache/sysds/test/functions/iogen/MatrixGenerateReaderLibSVMTest.java
@@ -116,7 +116,7 @@ public void test0_5() {
 
 	@Test
 	public void test0_6() {
-		sampleRaw = "+1 2.0:3.0 4:5. 6.:7\n" + "-1 8.0:9.0E0 10.0:11e0";
+		sampleRaw = "+1 2:3.0 4:5. 6:7\n" + "-1 8:9.0E0 10:11e0";
 		sampleMatrix = new double[][] {{0, 0, 3, 0, 5, 0, 7, 0, 0, 0, 0, +1}, {0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 11, -1}};
 		runGenerateReaderTest();
 	}

diff --git a/...test/java/org/apache/sysds/test/functions/iogen/MatrixGenerateReaderMatrixMarketTest.java b/...test/java/org/apache/sysds/test/functions/iogen/MatrixGenerateReaderMatrixMarketTest.java
@@ -198,13 +198,13 @@ public void SymmetricTest1_2() {
 
 	@Test
 	public void SymmetricTest1_3() {
-		generateRandomSymmetricMM(1, 100, -5, 5, 1, ",", false, false);
+		generateRandomSymmetricMM(1, 50, -5, 5, 1, ",", false, false);
 		runGenerateReaderTest();
 	}
 
 	@Test
 	public void SymmetricTest1_4() {
-		generateRandomSymmetricMM(1, 200, -100, 100, 1, "  ", false, false);
+		generateRandomSymmetricMM(1, 70, -100, 100, 1, "  ", false, false);
 		runGenerateReaderTest();
 	}