Autoscheduler test reorg, part 1

.github/workflows/presubmit.yml

@@ -50,10 +50,10 @@ jobs:
       - name: Run test sources check
         run: |
           shopt -s nullglob
-          (cd test/auto_schedule       && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
-          (cd test/correctness         && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
-          (cd test/error               && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
-          (cd test/generator           && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
-          (cd test/failing_with_issue  && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
-          (cd test/performance         && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
-          (cd test/warning             && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
+          (cd test/autoschedulers/mullapudi2016 && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
+          (cd test/correctness                  && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
+          (cd test/error                        && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
+          (cd test/generator                    && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
+          (cd test/failing_with_issue           && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
+          (cd test/performance                  && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])
+          (cd test/warning                      && comm -23 <(ls *.{c,cpp} | sort) <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) | tee missing_files && [ ! -s missing_files ])

Makefile

@@ -1175,7 +1175,7 @@ RUNTIME_TESTS = $(shell ls $(ROOT_DIR)/test/runtime/*.cpp)
 GENERATOR_EXTERNAL_TESTS := $(shell ls $(ROOT_DIR)/test/generator/*test.cpp)
 GENERATOR_EXTERNAL_TEST_GENERATOR := $(shell ls $(ROOT_DIR)/test/generator/*_generator.cpp)
 TUTORIALS = $(filter-out %_generate.cpp, $(shell ls $(ROOT_DIR)/tutorial/*.cpp))
-AUTO_SCHEDULE_TESTS = $(shell ls $(ROOT_DIR)/test/auto_schedule/*.cpp)
+MULLAPUDI2016_TESTS = $(shell ls $(ROOT_DIR)/test/autoschedulers/mullapudi2016/*.cpp)
 
 test_correctness: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=quiet_correctness_%) $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.c=quiet_correctness_%)
 test_performance: $(PERFORMANCE_TESTS:$(ROOT_DIR)/test/performance/%.cpp=performance_%)
@@ -1185,7 +1185,8 @@ test_runtime: $(RUNTIME_TESTS:$(ROOT_DIR)/test/runtime/%.cpp=runtime_%)
 test_tutorial: $(TUTORIALS:$(ROOT_DIR)/tutorial/%.cpp=tutorial_%)
 test_valgrind: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=valgrind_%)
 test_avx512: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=avx512_%)
-test_auto_schedule: test_mullapudi2016 test_li2018 test_adams2019
+test_autoschedulers: test_mullapudi2016 test_li2018 test_adams2019
+test_auto_schedule: test_autoschedulers
 
 .PHONY: test_correctness_multi_gpu
 test_correctness_multi_gpu: correctness_gpu_multi_device
@@ -1282,7 +1283,7 @@ init_time_compilation_%:
 TIME_COMPILATION ?= /usr/bin/time -a -f "$@,%U,%S,%E" -o
 
 run_tests: $(ALL_TESTS)
-	make -f $(THIS_MAKEFILE) test_performance test_auto_schedule
+	make -f $(THIS_MAKEFILE) test_performance test_autoschedulers
 
 .PHONY: build_tests
 build_tests: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=$(BIN_DIR)/correctness_%) \
@@ -1292,7 +1293,7 @@ build_tests: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=$(BIN_DIR)/c
 	$(RUNTIME_TESTS:$(ROOT_DIR)/test/runtime/%.cpp=$(BIN_DIR)/runtime_%) \
 	$(GENERATOR_EXTERNAL_TESTS:$(ROOT_DIR)/test/generator/%_aottest.cpp=$(BIN_DIR)/$(TARGET)/generator_aot_%) \
 	$(GENERATOR_EXTERNAL_TESTS:$(ROOT_DIR)/test/generator/%_jittest.cpp=$(BIN_DIR)/generator_jit_%) \
-	$(AUTO_SCHEDULE_TESTS:$(ROOT_DIR)/test/auto_schedule/%.cpp=$(BIN_DIR)/auto_schedule_%)
+	$(MULLAPUDI2016_TESTS:$(ROOT_DIR)/test/autoschedulers/mullapudi2016/%.cpp=$(BIN_DIR)/mullapudi2016_%)
 
 clean_generator:
 	rm -rf $(BIN_DIR)/*.generator
@@ -1368,7 +1369,7 @@ $(BIN_DIR)/runtime_%: $(ROOT_DIR)/test/runtime/%.cpp $(ROOT_DIR)/test/runtime/co
 	$(CXX) $(TEST_CXX_FLAGS) $(RUNTIME_TESTS_CXXFLAGS) -I$(ROOT_DIR)/test/runtime -I$(ROOT_DIR)/src/runtime $(OPTIMIZE_FOR_BUILD_TIME) $< $(COMMON_LD_FLAGS) -o $@
 
 # Auto schedule tests that link against libHalide
-$(BIN_DIR)/auto_schedule_%: $(ROOT_DIR)/test/auto_schedule/%.cpp $(BIN_DIR)/libHalide.$(SHARED_EXT) $(INCLUDE_DIR)/Halide.h
+$(BIN_DIR)/mullapudi2016_%: $(ROOT_DIR)/test/autoschedulers/mullapudi2016/%.cpp $(BIN_DIR)/libHalide.$(SHARED_EXT) $(INCLUDE_DIR)/Halide.h
 	$(CXX) $(TEST_CXX_FLAGS) $(OPTIMIZE_FOR_BUILD_TIME) $< -I$(INCLUDE_DIR) $(TEST_LD_FLAGS) -o $@
 
 # TODO(srj): this doesn't auto-delete, why not?
@@ -1998,11 +1999,11 @@ tutorial_%: $(BIN_DIR)/tutorial_% $(TMP_DIR)/images/rgb.png $(TMP_DIR)/images/gr
 	cd $(TMP_DIR) ; $(CURDIR)/$<
 	@-echo
 
-test_mullapudi2016: $(AUTO_SCHEDULE_TESTS:$(ROOT_DIR)/test/auto_schedule/%.cpp=auto_schedule_%)
+test_mullapudi2016: $(MULLAPUDI2016_TESTS:$(ROOT_DIR)/test/autoschedulers/mullapudi2016/%.cpp=mullapudi2016_%)
 
 # These tests were written for the Mullapudi2016 autoscheduler.
 # TODO: either make them work with all autoschedulers or move them under src/autoschedulers/mullapudi2016
-auto_schedule_%: $(BIN_DIR)/auto_schedule_% $(BIN_MULLAPUDI2016)
+mullapudi2016_%: $(BIN_DIR)/mullapudi2016_% $(BIN_MULLAPUDI2016)
 	@-mkdir -p $(TMP_DIR)
 	cd $(TMP_DIR) ; $(CURDIR)/$< $(realpath $(BIN_MULLAPUDI2016))
 	@-echo

test/CMakeLists.txt

@@ -9,9 +9,9 @@ target_precompile_headers(_test_internal PRIVATE <Halide.h>)
 
 add_halide_test(_test_internal GROUPS internal)
 
-option(WITH_TEST_AUTO_SCHEDULE "Build auto_schedule tests" ON)
+option(WITH_TEST_AUTO_SCHEDULE "Build autoscheduler tests" ON)
 if (WITH_TEST_AUTO_SCHEDULE)
-    add_subdirectory(auto_schedule)
+    add_subdirectory(autoschedulers)
 endif ()
 
 option(WITH_TEST_CORRECTNESS "Build correctness tests" ON)

test/autoschedulers/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(mullapudi2016)

test/autoschedulers/mullapudi2016/CMakeLists.txt

@@ -0,0 +1,28 @@
+if (NOT TARGET Halide::Mullapudi2016)
+    message(STATUS "Disabling mullapudi2016 tests for static Halide")
+    return()
+endif()
+
+tests(GROUPS mullapudi2016 autoschedulers auto_schedule
+      SOURCES
+      extern.cpp
+      param.cpp
+      ARGS $<TARGET_FILE:Halide::Mullapudi2016>)
+
+tests(GROUPS mullapudi2016 autoschedulers auto_schedule multithreaded
+      SOURCES
+      cost_function.cpp
+      data_dependent.cpp
+      fibonacci.cpp
+      histogram.cpp
+      large_window.cpp
+      mat_mul.cpp
+      max_filter.cpp
+      multi_output.cpp
+      overlap.cpp
+      reorder.cpp
+      small_pure_update.cpp
+      tile_vs_inline.cpp
+      unused_func.cpp
+      vectorize_var_in_update.cpp
+      ARGS $<TARGET_FILE:Halide::Mullapudi2016>)

test/auto_schedule/cost_function.cpp → test/autoschedulers/mullapudi2016/cost_function.cpp

@@ -54,13 +54,14 @@ int main(int argc, char **argv) {
     AutoSchedulerResults results = p.apply_autoscheduler(target, {"Mullapudi2016"});
 #endif
 
-    std::cout << "\n\n******************************************\nSCHEDULE:\n"
-              << "******************************************\n"
-              << results.schedule_source
-              << "\n******************************************\n\n";
-
-    // Inspect the schedule
-    stencils[num_stencils - 1].print_loop_nest();
+    // Don't dump to stdout (this is only for debugging)
+    // std::cout << "\n\n******************************************\nSCHEDULE:\n"
+    //           << "******************************************\n"
+    //           << results.schedule_source
+    //           << "\n******************************************\n\n";
+
+    // Inspect the schedule (only for debugging))
+    // stencils[num_stencils - 1].print_loop_nest();
 
     // Run the schedule
     p.realize({6204, 4604});

test/auto_schedule/data_dependent.cpp → test/autoschedulers/mullapudi2016/data_dependent.cpp

@@ -46,8 +46,8 @@ int main(int argc, char **argv) {
     p.apply_autoscheduler(target, {"Mullapudi2016"});
 #endif
 
-    // Inspect the schedule
-    g.print_loop_nest();
+    // Inspect the schedule (only for debugging))
+    // g.print_loop_nest();
 
     // Run the schedule
     Buffer<uint16_t> out = p.realize({input.width() - 1, input.height()});

test/auto_schedule/extern.cpp → test/autoschedulers/mullapudi2016/extern.cpp

@@ -58,8 +58,8 @@ void test_case_1() {
     p.apply_autoscheduler(target, {"Mullapudi2016"});
 #endif
 
-    // Inspect the schedule
-    g.print_loop_nest();
+    // Inspect the schedule (only for debugging))
+    // g.print_loop_nest();
 }
 
 // Test with an extern Func which consumes a trivial Func; autoscheduler
@@ -92,8 +92,8 @@ void test_case_2() {
     p.apply_autoscheduler(target, {"Mullapudi2016"});
 #endif
 
-    // Inspect the schedule
-    g.print_loop_nest();
+    // Inspect the schedule (only for debugging))
+    // g.print_loop_nest();
 }
 
 // Test with an extern Func that consumes a non-pure Func.
@@ -128,8 +128,8 @@ void test_case_3() {
     p.apply_autoscheduler(target, {"Mullapudi2016"});
 #endif
 
-    // Inspect the schedule
-    g.print_loop_nest();
+    // Inspect the schedule (only for debugging))
+    // g.print_loop_nest();
 }
 
 int main(int argc, char **argv) {

test/auto_schedule/fibonacci.cpp → test/autoschedulers/mullapudi2016/fibonacci.cpp

@@ -29,8 +29,8 @@ double run_test(bool auto_schedule) {
 #endif
     }
 
-    // Inspect the schedule
-    g.print_loop_nest();
+    // Inspect the schedule (only for debugging))
+    // g.print_loop_nest();
 
     // Benchmark the schedule
     Buffer<int> out(100);
@@ -57,10 +57,15 @@ int main(int argc, char **argv) {
     double manual_time = run_test(false);
     double auto_time = run_test(true);
 
-    std::cout << "======================\n"
-              << "Manual time: " << manual_time << "ms\n"
-              << "Auto time: " << auto_time << "ms\n"
-              << "======================\n";
+    const double slowdown_factor = 10.0;
+    if (!get_jit_target_from_environment().has_gpu_feature() && auto_time > manual_time * slowdown_factor) {
+        std::cerr << "Autoscheduler time is slower than expected:\n"
+                  << "======================\n"
+                  << "Manual time: " << manual_time << "ms\n"
+                  << "Auto time: " << auto_time << "ms\n"
+                  << "======================\n";
+        exit(1);
+    }
 
     printf("Success!\n");
     return 0;

test/auto_schedule/histogram.cpp → test/autoschedulers/mullapudi2016/histogram.cpp

@@ -112,7 +112,8 @@ double run_test(bool auto_schedule) {
     }
 
     p.compile_to_lowered_stmt("histogram.html", {in}, HTML, target);
-    color.print_loop_nest();
+    // Inspect the schedule (only for debugging))
+    // color.print_loop_nest();
 
     Buffer<uint8_t> out(in.width(), in.height(), in.channels());
     double t = benchmark(3, 10, [&]() {
@@ -138,13 +139,14 @@ int main(int argc, char **argv) {
     double manual_time = run_test(false);
     double auto_time = run_test(true);
 
-    std::cout << "======================" << std::endl;
-    std::cout << "Manual time: " << manual_time << "ms" << std::endl;
-    std::cout << "Auto time: " << auto_time << "ms" << std::endl;
-    std::cout << "======================" << std::endl;
-
-    if (auto_time > manual_time * 3) {
-        fprintf(stderr, "Warning: Auto-scheduler is much much slower than it should be.\n");
+    const double slowdown_factor = 5.0;
+    if (!get_jit_target_from_environment().has_gpu_feature() && auto_time > manual_time * slowdown_factor) {
+        std::cerr << "Autoscheduler time is slower than expected:\n"
+                  << "======================\n"
+                  << "Manual time: " << manual_time << "ms\n"
+                  << "Auto time: " << auto_time << "ms\n"
+                  << "======================\n";
+        exit(1);
     }
 
     printf("Success!\n");

test/auto_schedule/large_window.cpp → test/autoschedulers/mullapudi2016/large_window.cpp

@@ -52,8 +52,8 @@ int main(int argc, char **argv) {
     p.apply_autoscheduler(target, {"Mullapudi2016"});
 #endif
 
-    // Inspect the schedule
-    g.print_loop_nest();
+    // Inspect the schedule (only for debugging))
+    // g.print_loop_nest();
 
     // Run the schedule
     Buffer<uint16_t> out = p.realize({input.width(), input.height()});

test/auto_schedule/mat_mul.cpp → test/autoschedulers/mullapudi2016/mat_mul.cpp

@@ -110,11 +110,12 @@ double run_test(bool auto_schedule) {
         prod.compute_at(out, x).vectorize(x);
         prod.update().reorder(x, y, r).vectorize(x).unroll(y);
         out.tile(x, y, xi, yi, 16, 4).vectorize(xi).unroll(yi).parallel(y);
-        out.print_loop_nest();
+        // Inspect the schedule (only for debugging))
+        // out.print_loop_nest();
     }
 
-    // Inspect the schedule
-    out.print_loop_nest();
+    // Inspect the schedule (only for debugging))
+    // out.print_loop_nest();
 
     // Benchmark the schedule
     Buffer<float> result(size, size);
@@ -141,14 +142,14 @@ int main(int argc, char **argv) {
     double manual_time = run_test(false);
     double auto_time = run_test(true);
 
-    std::cout << "======================\n"
-              << "Manual time: " << manual_time << "ms\n"
-              << "Auto time: " << auto_time << "ms\n"
-              << "======================\n";
-
-    if (!get_jit_target_from_environment().has_gpu_feature() &&
-        (auto_time > manual_time * 5.0)) {
-        fprintf(stderr, "Warning: Auto-scheduler is much much slower than it should be.\n");
+    const double slowdown_factor = 8.0;
+    if (!get_jit_target_from_environment().has_gpu_feature() && auto_time > manual_time * slowdown_factor) {
+        std::cerr << "Autoscheduler time is slower than expected:\n"
+                  << "======================\n"
+                  << "Manual time: " << manual_time << "ms\n"
+                  << "Auto time: " << auto_time << "ms\n"
+                  << "======================\n";
+        exit(1);
     }
 
     printf("Success!\n");

test/auto_schedule/max_filter.cpp → test/autoschedulers/mullapudi2016/max_filter.cpp

@@ -113,8 +113,8 @@ double run_test(bool auto_schedule) {
     }
 
     p.compile_to_lowered_stmt("max_filter.html", {in}, HTML, target);
-    // Inspect the schedule
-    final.print_loop_nest();
+    // Inspect the schedule (only for debugging))
+    // final.print_loop_nest();
 
     // Run the schedule
     Buffer<float> out(in.width(), in.height(), in.channels());
@@ -141,13 +141,14 @@ int main(int argc, char **argv) {
     double manual_time = run_test(false);
     double auto_time = run_test(true);
 
-    std::cout << "======================\n"
-              << "Manual time: " << manual_time << "ms\n"
-              << "Auto time: " << auto_time << "ms\n"
-              << "======================\n";
-
-    if (auto_time > manual_time * 2) {
-        fprintf(stderr, "Warning: Auto-scheduler is much much slower than it should be.\n");
+    const double slowdown_factor = 4.0;
+    if (!get_jit_target_from_environment().has_gpu_feature() && auto_time > manual_time * slowdown_factor) {
+        std::cerr << "Autoscheduler time is slower than expected:\n"
+                  << "======================\n"
+                  << "Manual time: " << manual_time << "ms\n"
+                  << "Auto time: " << auto_time << "ms\n"
+                  << "======================\n";
+        exit(1);
     }
 
     printf("Success!\n");

test/auto_schedule/multi_output.cpp → test/autoschedulers/mullapudi2016/multi_output.cpp

@@ -53,9 +53,9 @@ int main(int argc, char **argv) {
     p.apply_autoscheduler(target, {"Mullapudi2016"});
 #endif
 
-    // Inspect the schedule
-    h.print_loop_nest();
-    g.print_loop_nest();
+    // Inspect the schedule (only for debugging))
+    // h.print_loop_nest();
+    // g.print_loop_nest();
 
     Buffer<uint16_t> out_1(999, 999), out_2(999, 999);
 

test/auto_schedule/overlap.cpp → test/autoschedulers/mullapudi2016/overlap.cpp

@@ -56,8 +56,8 @@ int main(int argc, char **argv) {
     p.apply_autoscheduler(target, {"Mullapudi2016"});
 #endif
 
-    // Inspect the schedule
-    up[num_levels - 1].print_loop_nest();
+    // Inspect the schedule (only for debugging))
+    // up[num_levels - 1].print_loop_nest();
 
     // Run the schedule
     Buffer<float> out = p.realize({1500, 1500});