diff --git a/tools/nni_cmd/config_utils.py b/tools/nni_cmd/config_utils.py
index e6472ee3de..ad3c1fe5cb 100644
--- a/tools/nni_cmd/config_utils.py
+++ b/tools/nni_cmd/config_utils.py
@@ -3,7 +3,9 @@
 
 import os
 import json
+import shutil
 from .constants import NNICTL_HOME_DIR
+from .command_utils import print_error
 
 class Config:
     '''a util class to load and save config'''
@@ -77,7 +79,13 @@ def update_experiment(self, expId, key, value):
     def remove_experiment(self, expId):
         '''remove an experiment by id'''
         if expId in self.experiments:
-            self.experiments.pop(expId)
+            fileName = self.experiments.pop(expId).get('fileName')
+            if fileName:
+                logPath = os.path.join(NNICTL_HOME_DIR, fileName)
+                try:
+                    shutil.rmtree(logPath)
+                except FileNotFoundError:
+                    print_error('{0} does not exist.'.format(logPath))
         self.write_file()
 
     def get_all_experiments(self):