Add test artifacts check to CI (aws#52)

* Add test artifacts check to CI

* Fix bash syntax

* Change /my to /tmp/my

* Use TemporaryDirectory()

* Add some grep filters

* Patch so passes CI

.gitignore

@@ -112,23 +112,10 @@ log
 
 .DS_Store
 .idea/
-*.json
-cm.json
-cm.ts
-collections.json
-worker*.json
 
 # Wheel and tar files from running `python bin/build_containers.py`
 *.whl
 bin/sagemaker-containers/**/*.tar.gz
 
-# outdirs
-data/
-default/
-newlogsRunTest*/
-MNIST-data/
-logs/
-savedParams/
-ts_outputs*/
-test_output*/
+# TODO: Figure out what is generating this and move it to /tmp/MNIST-data
 MNIST-data/

config/tests.sh

@@ -5,7 +5,7 @@ set -ex
 check_logs() {
   if grep -e "AssertionError" $1;
    then
-    echo "Integration tests:FAILED."
+    echo "Integration tests: FAILED."
     exit 1
   else
         echo "Integration tests: SUCCESS."
@@ -43,3 +43,9 @@ if [ "$run_pytest_pytorch" = "enable" ] ; then
 fi
 
 check_logs $REPORT_DIR/*
+
+# Only look at newly added files
+if [ -n "$(git status --porcelain | grep ^?? | grep -v tornasolecodebuildtest | grep -v upload)" ]; then
+  echo "ERROR: Test artifacts were created. Please place these in /tmp."
+  exit 1
+fi

tests/core/test_paths.py

@@ -1,6 +1,7 @@
 # Standard Library
 import os
 import uuid
+from tempfile import TemporaryDirectory
 
 # First Party
 import smdebug.pytorch as smd
@@ -97,18 +98,19 @@ def test_s3_path_that_exists_without_end_of_job():
 
 
 def test_outdir_sagemaker(monkeypatch):
-    json_file_contents = """
-            {
-                "S3OutputPath": "s3://sagemaker-test",
-                "LocalPath": "/my/own/path/tensors",
-                "HookParameters" : {
-                    "save_interval": "2",
-                    "include_workers": "all"
-                }
-            }
-            """
-    from smdebug.tensorflow import get_hook
-
-    with SagemakerSimulator(json_file_contents=json_file_contents) as sim:
-        hook = get_hook("keras", create_if_not_exists=True)
-        assert hook.out_dir == "/my/own/path/tensors"
+    with TemporaryDirectory() as dir_name:
+        json_file_contents = f"""
+                {{
+                    "S3OutputPath": "s3://sagemaker-test",
+                    "LocalPath": "{dir_name}",
+                    "HookParameters" : {{
+                        "save_interval": "2",
+                        "include_workers": "all"
+                    }}
+                }}
+                """
+        from smdebug.tensorflow import get_hook
+
+        with SagemakerSimulator(json_file_contents=json_file_contents) as sim:
+            hook = get_hook("keras", create_if_not_exists=True)
+            assert hook.out_dir == dir_name

tests/mxnet/test_hook_all_zero.py

@@ -22,6 +22,7 @@ def test_hook_all_zero(hook=None, out_dir=None):
         run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
         out_dir = "/tmp/newlogsRunTest/" + run_id
         print("Registering the hook with out_dir {0}".format(out_dir))
+        shutil.rmtree(out_dir, ignore_errors=True)
         hook = t_hook(
             out_dir=out_dir,
             save_config=save_config,