diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml index d68db3eb3..d5ed9d684 100644 --- a/.github/workflows/load-test.yml +++ b/.github/workflows/load-test.yml @@ -35,7 +35,7 @@ jobs: run: mvn clean compile package -DskipTests=true - name: execute py script run: | - python perfTest/perfTestRunner.py + python perf_test/perfTestRunner.py git config user.name nitish git config user.email nitish1814@github.com git add . diff --git a/perf_test/perfTestInput.py b/perf_test/perfTestInput.py index 9dd167394..c18210e02 100644 --- a/perf_test/perfTestInput.py +++ b/perf_test/perfTestInput.py @@ -5,13 +5,13 @@ INCREMENTAL = "runIncrement" #load file config to test on -config_65 = "../examples/febrl120k/config.json" -config_120k = "../examples/febrl120k/config120k.json" -config_5m = "../examples/ncVoters5M/config.json" +config_65 = "./examples/febrl120k/config.json" +config_120k = "./examples/febrl120k/config120k.json" +config_5m = "./examples/ncVoters5M/config.json" #bash script location -ZINGG = "../scripts/zingg.sh" +ZINGG = "./scripts/zingg.sh" #add all the load to test diff --git a/perf_test/perfTestRunner.py b/perf_test/perfTestRunner.py index 88c761245..5b08084f7 100644 --- a/perf_test/perfTestRunner.py +++ b/perf_test/perfTestRunner.py @@ -3,6 +3,10 @@ import time from datetime import date, datetime from subprocess import PIPE +import os + +#set working directory +os.chdir(os.path.dirname("../")) ZINGG = ZINGG #phases to run: ftd, match @@ -16,20 +20,23 @@ start_time = time.time() +reportFile = "./perf_test/perf_test_report/loadTestReport" + def perf_test_small_all(): return "small_test_running_all" +propertyFile = "./config/zingg.conf" def run_phase(phase, conf): print("Running phase - " + phase) - return subprocess.call(ZINGG + " %s %s %s %s" % ("--phase", phase, "--conf", conf), shell=True) + return subprocess.call(ZINGG + " %s %s %s %s %s %s" % ("--phase", phase, "--conf", conf, "--properties-file", propertyFile), shell=True) def perf_test_small(phase): return "small_test_running" def write_on_start(): - f = open("./perfTestReport/loadTestReport_" + str(date.today()), "w+") + f = open(reportFile, "w+") f.write("******************************** perf test report, " + str(date.today()) + ", " + current_time + " ********************************\n\n"); f.write("------------ Test bed details ------------\n") f.write("Load samples: ") @@ -44,23 +51,23 @@ def write_on_start(): f.close() def write_on_complete(): - f = open("./perfTestReport/loadTestReport_" + str(date.today()), "a+") + f = open(reportFile, "a+") f.write("********************************************************************************************************\n\n\n\n\n\n") def write_success_stats(phase_time, load): - f = open("./perfTestReport/loadTestReport_" + str(date.today()), "a+") + f = open(reportFile, "a+") f.write("{:>50}".format("capturing for " + load) + "\n") f.write("PHASE {:>65}".format("TIME_TAKEN_IN_MINUTES") + "\n") for phase, time in phase_time.items(): - f.write(success_message(phase, time/60) + "\n") + f.write(success_message(phase, round(time/60, 1)) + "\n") f.write("\n") f.close() def write_failure_stats(phase_error): - f = open("./perfTestReport/loadTestReport_" + str(date.today()), "a+") + f = open(reportFile, "a+") for phase, error in phase_error.items(): f.write(error_message(phase, error) + "\n\n") f.close() diff --git a/perf_test/perf_test_report/loadTestReport b/perf_test/perf_test_report/loadTestReport new file mode 100644 index 000000000..5151bb7d0 --- /dev/null +++ b/perf_test/perf_test_report/loadTestReport @@ -0,0 +1,18 @@ +******************************** perf test report, 2024-11-14, 18:09:04 ******************************** + +------------ Test bed details ------------ +Load samples: 65_samples +Phases: findTrainingData match +------------------------------------------ + + capturing for 65_samples +PHASE TIME_TAKEN_IN_MINUTES +findTrainingData 3.0 +match 4.5 + +******************************************************************************************************** + + + + + diff --git a/scripts/zingg.sh b/scripts/zingg.sh index 34632556f..125690e77 100755 --- a/scripts/zingg.sh +++ b/scripts/zingg.sh @@ -1,5 +1,4 @@ #!/bin/bash -#ZINGG_HOME=./assembly/target ZINGG_JARS=$ZINGG_HOME/zingg-0.4.1-SNAPSHOT.jar EMAIL=zingg@zingg.ai LICENSE=zinggLicense.txt @@ -63,5 +62,5 @@ if [[ $RUN_PYTHON_DB_CONNECT_PHASE -eq 1 ]]; then python $EXECUTABLE else # All the additional options must be added here - $SPARK_HOME/bin/spark-submit --master $SPARK_MASTER $PROPERTIES --files "./log4j2.properties" --conf spark.driver.memory=10g --conf spark.executor.extraJavaOptions="$log4j_setting -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+HeapDumpOnOutOfMemoryError -Xloggc:/tmp/memLog.txt -XX:+UseCompressedOops" --conf spark.driver.extraJavaOptions="$log4j_setting" $LOGGING --driver-class-path $ZINGG_JARS $EXECUTABLE $@ --email $EMAIL --license $LICENSE + $SPARK_HOME/bin/spark-submit --master $SPARK_MASTER $PROPERTIES --files "./log4j2.properties" --conf spark.executor.extraJavaOptions="$log4j_setting -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+HeapDumpOnOutOfMemoryError -Xloggc:/tmp/memLog.txt -XX:+UseCompressedOops" --conf spark.driver.extraJavaOptions="$log4j_setting" $LOGGING --driver-class-path $ZINGG_JARS $EXECUTABLE $@ --email $EMAIL --license $LICENSE fi