Seperate code and query

코드부분과 쿼리문을 분리했습니다.
State-of-The-MLOps · Sep 18, 2021 · 36a465f · 36a465f
1 parent cd0a7bd
commit 36a465f
Show file tree

Hide file tree

Showing 3 changed files with 79 additions and 71 deletions.
diff --git a/app/api/router/train.py b/app/api/router/train.py
@@ -29,7 +29,7 @@ def train_insurance(
 
     Args:
         experiment_name (str): 실험이름. 기본 값: exp1
-        experimeter (str): 실험자의 이름. 기본 값: DongUk
+        experimenter (str): 실험자의 이름. 기본 값: DongUk
         model_name (str): 모델의 이름. 기본 값: insurance_fee_model
         version (float): 실험의 버전. 기본 값: 0.1
 
@@ -60,8 +60,6 @@ def train_insurance(
         if sucs_msg in nni_create_result:
             p = re.compile(r"The experiment id is ([a-zA-Z0-9]+)\n")
             expr_id = p.findall(nni_create_result)[0]
-            # expr id 랑 expr name 주고 instance 만들어서
-            # a.excute를 target으로 넘겨주자.
             nni_watcher = NniWatcher(expr_id, experiment_name)
             m_process = multiprocessing.Process(
                 target=nni_watcher.excute

diff --git a/app/query.py b/app/query.py
@@ -0,0 +1,58 @@
+UPDATE_TEMP_MODEL_DATA = """
+            DELETE FROM temp_model_data
+            WHERE id NOT IN (
+                SELECT id
+                FROM temp_model_data
+                WHERE experiment_name = '{}'
+                ORDER BY {}
+                LIMIT {}
+            )
+        """
+
+
+SELECT_TEMP_MODEL_BY_EXPR_NAME = """
+            SELECT * 
+            FROM temp_model_data
+            WHERE experiment_name = '{}'
+            ORDER BY {};
+        """
+
+
+SELECT_MODEL_METADATA_BY_EXPR_NAME = """
+            SELECT *
+            FROM model_metadata
+            WHERE experiment_name = '{}'
+        """
+
+INSERT_MODEL_CORE = """
+                INSERT INTO model_core (
+                    model_name,
+                    model_file
+                ) VALUES(
+                    '{}',
+                    '{}'
+                )
+            """
+
+UPDATE_MODEL_CORE = """
+            UPDATE model_core
+            SET
+                model_file = '{}'
+            WHERE
+                model_name = '{}'
+        """
+
+UPDATE_MODEL_METADATA = """
+            UPDATE model_metadata
+            SET 
+                train_mae = {},
+                val_mae = {},
+                train_mse = {},
+                val_mse = {}
+            WHERE experiment_name = '{}'
+        """
+
+DELETE_ALL_EXPERIMENTS_BY_EXPR_NAME = """
+    DELETE FROM temp_model_data
+    WHERE experiment_name = '{}'
+"""
diff --git a/app/utils.py b/app/utils.py
@@ -17,6 +17,8 @@
 
 from app.database import engine
 from logger import L
+from app.query import *
+
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
 
 base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -148,7 +150,7 @@ def write_yml(
             'experimentName': f'{experiment_name}',
             'trialConcurrency': 1,
             'maxExecDuration': '1h',
-            'maxTrialNum': 1,
+            'maxTrialNum': 10,
             'trainingServicePlatform': 'local',
             'searchSpacePath': 'search_space.json',
             'useAnnotation': False,
@@ -172,15 +174,6 @@ def write_yml(
     return
 
 
-# class NniExperiments:
-#     def __init__(self, expriment_name, author, model_name, version, waiting=None):
-#         self.experiment_name = expriment_name
-#         self.author = author
-#         self.model_name = model_name
-#         self.version = version
-#         self.waiting = waiting
-
-
 class NniWatcher:
     def __init__(
         self,
@@ -201,16 +194,6 @@ def __init__(
         self._wait_minute = minute * 60
         self._experiment_list = None
         self._running_experiment = None
-        self._update_model_query = """
-            DELETE FROM temp_model_data
-            WHERE id NOT IN (
-                SELECT id
-                FROM temp_model_data
-                WHERE experiment_name = '{}'
-                ORDER BY {}
-                LIMIT {}
-            )
-        """
 
     def excute(self):
         self.watch_process()
@@ -245,58 +228,28 @@ def watch_process(self):
 
     def model_keep_update(self):
         engine.execute(
-            self._update_model_query.format(
+            UPDATE_TEMP_MODEL_DATA.format(
                 self.experiment_name,
                 self.evaluation_criteria,
                 self.top_cnt)
         )
 
     def model_final_update(self):
-        final_model = """
-            SELECT * 
-            FROM temp_model_data
-            WHERE experiment_name = '{}'
-            ORDER BY {};
-        """.format(self.experiment_name, self.evaluation_criteria)
-        final_result = engine.execute(final_model).fetchone()
-
-        saved_model = """
-            SELECT *
-            FROM model_metadata
-            WHERE experiment_name = '{}'
-        """.format(self.experiment_name)
-        saved_result = engine.execute(saved_model).fetchone()
-        INSERT_MODEL_CORE = """
-                INSERT INTO model_core (
-                    model_name,
-                    model_file
-                ) VALUES(
-                    '{}',
-                    '{}'
-                )
-            """
-
-        UPDATE_MODEL_CORE = """
-            UPDATE model_core
-            SET
-                model_file = '{}'
-            WHERE
-                model_name = '{}'
-        """
-        UPDATE_MODEL_METADATA = """
-            UPDATE model_metadata
-            SET 
-                train_mae = {},
-                val_mae = {},
-                train_mse = {},
-                val_mse = {}
-            WHERE experiment_name = '{}'
-        """
-        print(saved_result)
+        final_result = engine.execute(
+            SELECT_TEMP_MODEL_BY_EXPR_NAME.format(
+                self.experiment_name,
+                self.evaluation_criteria)
+        ).fetchone()
+
+        saved_result = engine.execute(
+            SELECT_MODEL_METADATA_BY_EXPR_NAME.format(
+                self.experiment_name)
+        ).fetchone()
+
         if saved_result is None:
             engine.execute(INSERT_MODEL_CORE.format(
                 final_result.model_name, final_result.model_file))
-        elif float(saved_result[self.evaluation_criteria]) > float(final_result[self.evaluation_criteria]):
+        elif saved_result[self.evaluation_criteria] > final_result[self.evaluation_criteria]:
             engine.execute(UPDATE_MODEL_CORE.format(
                 final_result.model_file, final_result.model_name))
             engine.execute(UPDATE_MODEL_METADATA.format(
@@ -307,11 +260,10 @@ def model_final_update(self):
                 self.experiment_name)
             )
 
-        DELETE_EXPERIMENTS = """
-            DELETE FROM temp_model_data
-            WHERE experiment_name = '{}'
-        """
-        engine.execute(DELETE_EXPERIMENTS.format(self.experiment_name))
+        engine.execute(
+            DELETE_ALL_EXPERIMENTS_BY_EXPR_NAME.format(
+                self.experiment_name)
+        )
 
 
 def zip_model(model_path):