fix get best cfg in sft and lora mode bug

PaddlePaddle · Sep 18, 2023 · b1bc3eb · b1bc3eb
1 parent fda5145
commit b1bc3eb
Showing 1 changed file with 12 additions and 2 deletions.
diff --git a/python/paddle/distributed/auto_tuner/recorder.py b/python/paddle/distributed/auto_tuner/recorder.py
@@ -54,18 +54,28 @@ def get_best(self, metric, direction, mode=None) -> Tuple[dict, bool]:
             return (self.history[0], True)
         if mode == "SFT" or mode == "LoRA":
             best_cfg = self.history[0]
+            if (
+                isinstance(best_cfg["max_mem_usage"], str)
+                or best_cfg["time"] == -1
+            ):
+                return (best_cfg, True)
             first_few = 1
             for cfg in self.history:
                 if (
-                    cfg["max_mem_usage"] < best_cfg["max_mem_usage"]
+                    not isinstance(cfg["max_mem_usage"], str)
+                    and cfg["max_mem_usage"] < best_cfg["max_mem_usage"]
                     and cfg["time"] != -1
                 ):
                     best_cfg = cfg
                 first_few += 1
                 if first_few >= 5:
                     break
             return (best_cfg, False)
-
+        if (
+            isinstance(self.history[0]["max_mem_usage"], str)
+            or self.history[0]["time"] == -1
+        ):
+            return (self.history[0], False)
         return (self.history[0], False)
 
     def store_history(self, path="./history.csv"):