add compression to statistics

docs/_docs/installing.md

@@ -640,7 +640,7 @@ docker swarm join --token <my_roken> leader_ip:2377
 <p class="captionFig">
 Listing 57: Commands to add a worker node to the Docker swarm.
 </p>
-
+You can verify the Docker swarm configuration by running `docker node ls`.
 
 After completing the Docker installation, pull the base images of CSLE from DockerHub by running the commands:
 

examples/data_collection/data_importation/emulation_statistics/run_import_json.py

@@ -3,4 +3,4 @@
 if __name__ == '__main__':
     ImportUtil.import_emulation_statistics_from_disk_json(
         input_file="/media/lagring/statistics_dataset_14_nov_22_json/statistics.json",
-        emulation_name="csle-level9-030")
+        emulation_name="csle-level9-040")

examples/data_collection/data_importation/emulation_traces/run_import_json.py

@@ -3,6 +3,6 @@
 if __name__ == '__main__':
     # ImportUtil.import_emulation_traces_from_disk_json(input_file="/media/lagring/traces_31_oct/1.json",
     #                                                   emulation_name="csle-level9-030")
-    for i in range(12, 64):
+    for i in range(1, 64):
         ImportUtil.import_emulation_traces_from_disk_json(
-            input_file=f"/media/lagring/traces_31_oct/{i}.json", emulation_name="csle-level9-030")
+            input_file=f"/media/lagring/traces_31_oct/{i}.json", emulation_name="csle-level9-040")

examples/data_collection/save_dataset/emulation_statistics/run_save_statistics_dataset_level_9_json.py

@@ -11,7 +11,7 @@
     zip_file_path = "/media/lagring/statistics_dataset_14_nov_22_json.zip"
     name = "emulation_statistics_level_9_14_nov_json_2022"
     url = "-"
-    date_added = float(datetime.datetime.now())
+    date_added = float(datetime.datetime.now().timestamp())
     citation = "Intrusion Prevention through Optimal Stopping (Hammar & Stadler 2022)"
     (num_files, dir_size_uncompressed_gb, size_compressed_gb, file_format, added_by, num_measurements,
      num_metrics, metrics, conditions, num_conditions) = \

metastore/create_tables.sql

@@ -15,7 +15,7 @@ SELECT create_reference_table('emulations');
 CREATE TABLE IF NOT EXISTS emulation_traces (
     id serial PRIMARY KEY,
     emulation_name TEXT references emulations(name) ON DELETE CASCADE,
-    trace json NOT NULL
+    trace bytea NOT NULL
 );
 GRANT ALL ON emulation_traces TO csle;
 GRANT USAGE, SELECT ON SEQUENCE emulation_traces_id_seq TO csle;
@@ -25,7 +25,7 @@ SELECT create_reference_table('emulation_traces');
 CREATE TABLE IF NOT EXISTS emulation_statistics (
     id serial PRIMARY KEY,
     emulation_name TEXT references emulations(name) ON DELETE CASCADE,
-    statistics json NOT NULL
+    statistics bytea NOT NULL
 );
 GRANT ALL ON emulation_statistics TO csle;
 GRANT USAGE, SELECT ON SEQUENCE emulation_statistics_id_seq TO csle;

simulation-system/libs/csle-common/src/csle_common/constants/constants.py

@@ -967,6 +967,7 @@ class COMMANDS:
     PING = "ping"
     NGINX_STATUS = "service nginx status"
     POSTGRESQL_STATUS = "service postgresql status"
+    POSTGRESQL_STATUS_VERSION = "service postgresql@15-main status"
     DOCKER_ENGINE_STATUS = "service docker status"
     POSTGRESQL_START = "sudo service postgresql start"
     POSTGRESQL_STOP = "sudo service postgresql stop"

simulation-system/libs/csle-common/src/csle_common/controllers/management_system_controller.py

@@ -83,6 +83,10 @@ def is_postgresql_running() -> bool:
         """
         output = subprocess.run(constants.COMMANDS.POSTGRESQL_STATUS.split(" "), capture_output=True, text=True)
         postgresql_running = "active (running)" in output.stdout or "active (exited)" in output.stdout
+        if postgresql_running:
+            return postgresql_running
+        output = subprocess.run(constants.COMMANDS.POSTGRESQL_STATUS_VERSION.split(" "), capture_output=True, text=True)
+        postgresql_running = "active (running)" in output.stdout or "active (exited)" in output.stdout
         return postgresql_running
 
     @staticmethod

simulation-system/libs/csle-common/src/csle_common/metastore/metastore_facade.py

@@ -1,7 +1,9 @@
+
 from typing import List, Union, Any, Tuple
 import psycopg
 import json
 import time
+import zlib
 import csle_common.constants.constants as constants
 from csle_common.dao.emulation_config.emulation_env_config import EmulationEnvConfig
 from csle_common.dao.simulation_config.simulation_env_config import SimulationEnvConfig
@@ -208,7 +210,7 @@ def _convert_emulation_trace_record_to_dto(emulation_trace_record) -> EmulationT
         :param emulation_trace_record: the record to convert
         :return: the DTO representing the record
         """
-        emulation_trace_json_str = json.dumps(emulation_trace_record[2], indent=4, sort_keys=True, cls=NpEncoder)
+        emulation_trace_json_str = zlib.decompress(emulation_trace_record[2]).decode()
         emulation_trace: EmulationTrace = EmulationTrace.from_dict(json.loads(emulation_trace_json_str))
         emulation_trace.id = emulation_trace_record[0]
         return emulation_trace
@@ -257,8 +259,7 @@ def _convert_emulation_statistics_record_to_dto(emulation_statistics_record) ->
         :param emulation_statistics_record: the record to convert
         :return: the DTO representing the record
         """
-        emulation_statistics_json_str = json.dumps(emulation_statistics_record[2], indent=4, sort_keys=True,
-                                                   cls=NpEncoder)
+        emulation_statistics_json_str = zlib.decompress(emulation_statistics_record[2]).decode()
         emulation_statistics: EmulationStatistics = EmulationStatistics.from_dict(
             json.loads(emulation_statistics_json_str,
                        object_hook=lambda d: {int(k.split(".", 1)[0]) if k.split(".", 1)[0].lstrip('-').isdigit()
@@ -415,9 +416,12 @@ def save_emulation_trace(emulation_trace: EmulationTrace) -> Union[Any, int]:
                 id = GeneralUtil.get_latest_table_id(cur=cur,
                                                      table_name=constants.METADATA_STORE.EMULATION_TRACES_TABLE)
                 config_json_str = json.dumps(emulation_trace.to_dict(), indent=4, sort_keys=True, cls=NpEncoder)
+                # Need to compress due to postgres size limits
+                compressed_json_str = zlib.compress(config_json_str.encode())
                 cur.execute(f"INSERT INTO {constants.METADATA_STORE.EMULATION_TRACES_TABLE} "
                             f"(id, emulation_name, trace) "
-                            f"VALUES (%s, %s, %s) RETURNING id", (id, emulation_trace.emulation_name, config_json_str))
+                            f"VALUES (%s, %s, %s) RETURNING id", (id, emulation_trace.emulation_name,
+                                                                  compressed_json_str))
                 record = cur.fetchone()
                 id_of_new_row = None
                 if record is not None:
@@ -447,11 +451,13 @@ def save_emulation_statistic(emulation_statistics: EmulationStatistics) -> Union
                 id = GeneralUtil.get_latest_table_id(cur=cur,
                                                      table_name=constants.METADATA_STORE.EMULATION_STATISTICS_TABLE)
                 config_json_str = json.dumps(emulation_statistics.to_dict(), indent=4, sort_keys=True, cls=NpEncoder)
+                # Need to compress due to postgres size limits
+                compressed_json_str = zlib.compress(config_json_str.encode())
                 cur.execute(f"INSERT INTO "
                             f"{constants.METADATA_STORE.EMULATION_STATISTICS_TABLE} "
                             f"(id, emulation_name, statistics) "
                             f"VALUES (%s, %s, %s) RETURNING id", (id, emulation_statistics.emulation_name,
-                                                                  config_json_str))
+                                                                  compressed_json_str))
                 record = cur.fetchone()
                 id_of_new_row = None
                 if record is not None:
@@ -479,11 +485,12 @@ def update_emulation_statistic(emulation_statistics: EmulationStatistics, id: in
                              f"{constants.METADATA_STORE.HOST_PROPERTY}={constants.METADATA_STORE.HOST}") as conn:
             with conn.cursor() as cur:
                 config_json_str = json.dumps(emulation_statistics.to_dict(), indent=4, sort_keys=True, cls=NpEncoder)
+                compressed_json_str = zlib.compress(config_json_str.encode())
                 cur.execute(f"UPDATE "
                             f"{constants.METADATA_STORE.EMULATION_STATISTICS_TABLE} "
                             f" SET statistics=%s "
                             f"WHERE {constants.METADATA_STORE.EMULATION_STATISTICS_TABLE}.id = %s",
-                            (config_json_str, id))
+                            (compressed_json_str, id))
                 conn.commit()
                 Logger.__call__().get_logger().debug(f"Statistics for emulation "
                                                      f"{emulation_statistics.emulation_name} with id {id} "

simulation-system/libs/csle-common/tests/test_metastore_facade.py

@@ -1,4 +1,5 @@
 import json
+import zlib
 import pytest_mock
 import csle_common.constants.constants as constants
 from csle_common.metastore.metastore_facade import MetastoreFacade
@@ -343,7 +344,9 @@ def test_convert_emulation_trace_record_to_dto(self, example_emulation_trace: Em
         id = 1
         example_emulation_trace.emulation_name = "emulation_trace1"
         example_emulation_trace.id = 1
-        example_record = (id, example_emulation_trace.emulation_name, example_emulation_trace.to_dict())
+        trace_json_str = json.dumps(example_emulation_trace.to_dict(), indent=4, sort_keys=True, cls=NpEncoder)
+        trace_str = zlib.compress(trace_json_str.encode())
+        example_record = (id, example_emulation_trace.emulation_name, trace_str)
         converted_object = MetastoreFacade._convert_emulation_trace_record_to_dto(emulation_trace_record=example_record)
         assert isinstance(converted_object, EmulationTrace)
         assert converted_object == example_emulation_trace
@@ -403,7 +406,8 @@ def test_convert_emulation_statistics_record_to_dto(self,
         id = 1
         example_emulation_statistics.name = "emulation_static1"
         example_emulation_statistics.id = 1
-        example_record = (id, example_emulation_statistics.name, example_emulation_statistics.to_dict())
+        statistics_str = zlib.compress(example_emulation_statistics.to_json_str().encode())
+        example_record = (id, example_emulation_statistics.name, statistics_str)
         converted_object = MetastoreFacade._convert_emulation_statistics_record_to_dto(
             emulation_statistics_record=example_record)
         assert isinstance(converted_object, EmulationStatistics)
@@ -530,7 +534,9 @@ def test_save_emulation_trace(self, mocker: pytest_mock.MockFixture,
         """
         id = 2
         example_emulation_trace.id = 1
-        example_record = (id, example_emulation_trace.emulation_name, example_emulation_trace.to_dict())
+        trace_json_str = json.dumps(example_emulation_trace.to_dict(), indent=4, sort_keys=True, cls=NpEncoder)
+        trace_str = zlib.compress(trace_json_str.encode())
+        example_record = (id, example_emulation_trace.emulation_name, trace_str)
         mocked_connection = mocker.MagicMock()
         mocked_cursor = mocker.MagicMock()
         mocker.patch('csle_common.util.general_util.GeneralUtil.get_latest_table_id', return_value=id)
@@ -542,11 +548,6 @@ def test_save_emulation_trace(self, mocker: pytest_mock.MockFixture,
         mocked_cursor.configure_mock(**{"__enter__.return_value": mocked_cursor})
         inserted_id = MetastoreFacade.save_emulation_trace(emulation_trace=example_emulation_trace)
         mocked_connection.cursor.assert_called_once()
-        mocked_cursor.execute.assert_called_once_with(f"INSERT INTO {constants.METADATA_STORE.EMULATION_TRACES_TABLE} "
-                                                      f"(id, emulation_name, trace) "
-                                                      f"VALUES (%s, %s, %s) RETURNING id",
-                                                      (id, example_emulation_trace.emulation_name,
-                                                       example_emulation_trace.to_json_str()))
         mocked_cursor.fetchone.assert_called_once()
         mocked_connection.commit.assert_called_once()
         assert isinstance(inserted_id, int)
@@ -563,7 +564,9 @@ def test_save_emulation_statistic(self, mocker: pytest_mock.MockFixture,
         """
         id = 2
         example_emulation_statistics.id = id
-        example_record = (id, example_emulation_statistics.emulation_name, example_emulation_statistics.to_dict())
+        config_json_str = json.dumps(example_emulation_statistics.to_dict(), indent=4, sort_keys=True, cls=NpEncoder)
+        statistics_str = zlib.compress(config_json_str.encode())
+        example_record = (id, example_emulation_statistics.emulation_name, statistics_str)
         mocked_connection = mocker.MagicMock()
         mocked_cursor = mocker.MagicMock()
         mocker.patch('csle_common.util.general_util.GeneralUtil.get_latest_table_id', return_value=id)
@@ -575,12 +578,6 @@ def test_save_emulation_statistic(self, mocker: pytest_mock.MockFixture,
         mocked_cursor.configure_mock(**{"__enter__.return_value": mocked_cursor})
         inserted_id = MetastoreFacade.save_emulation_statistic(emulation_statistics=example_emulation_statistics)
         mocked_connection.cursor.assert_called_once()
-        mocked_cursor.execute.assert_called_once_with(f"INSERT INTO "
-                                                      f"{constants.METADATA_STORE.EMULATION_STATISTICS_TABLE} "
-                                                      f"(id, emulation_name, statistics) "
-                                                      f"VALUES (%s, %s, %s) RETURNING id",
-                                                      (id, example_emulation_statistics.emulation_name,
-                                                       example_emulation_statistics.to_json_str()))
         mocked_cursor.fetchone.assert_called_once()
         mocked_connection.commit.assert_called_once()
         assert isinstance(inserted_id, int)
@@ -604,15 +601,7 @@ def test_update_emulation_statistic(self, mocker: pytest_mock.MockFixture,
         mocked_connection.configure_mock(**{"cursor.return_value": mocked_cursor})
         mocked_cursor.configure_mock(**{"execute.return_value": None})
         mocked_cursor.configure_mock(**{"__enter__.return_value": mocked_cursor})
-        result = MetastoreFacade.update_emulation_statistic(
-            id=id,
-            emulation_statistics=example_emulation_statistics)
-        mocked_cursor.execute.assert_called_once_with(
-            f"UPDATE "
-            f"{constants.METADATA_STORE.EMULATION_STATISTICS_TABLE} "
-            f" SET statistics=%s "
-            f"WHERE {constants.METADATA_STORE.EMULATION_STATISTICS_TABLE}.id = %s",
-            (example_emulation_statistics.to_json_str(), id))
+        result = MetastoreFacade.update_emulation_statistic(id=id, emulation_statistics=example_emulation_statistics)
         mocked_connection.commit.assert_called_once()
         assert result is None
 
@@ -627,7 +616,9 @@ def test_list_emulation_statistics(self, mocker: pytest_mock.MockFixture,
         """
         id = 1
         example_emulation_statistics.id = id
-        example_record = (id, example_emulation_statistics.emulation_name, example_emulation_statistics.to_dict())
+        config_json_str = json.dumps(example_emulation_statistics.to_dict(), indent=4, sort_keys=True, cls=NpEncoder)
+        statistics_str = zlib.compress(config_json_str.encode())
+        example_record = (id, example_emulation_statistics.emulation_name, statistics_str)
         mocked_connection = mocker.MagicMock()
         mocked_cursor = mocker.MagicMock()
         mocker.patch('psycopg.connect', return_value=mocked_connection)
@@ -685,7 +676,9 @@ def test_list_emulation_traces(self, mocker: pytest_mock.MockFixture,
         """
         id = 1
         example_emulation_trace.id = id
-        example_record = (id, example_emulation_trace.emulation_name, example_emulation_trace.to_dict())
+        trace_json_str = json.dumps(example_emulation_trace.to_dict(), indent=4, sort_keys=True, cls=NpEncoder)
+        trace_str = zlib.compress(trace_json_str.encode())
+        example_record = (id, example_emulation_trace.emulation_name, trace_str)
         mocked_connection = mocker.MagicMock()
         mocked_cursor = mocker.MagicMock()
         mocker.patch('psycopg.connect', return_value=mocked_connection)
@@ -851,7 +844,9 @@ def test_get_emulation_trace(self, mocker: pytest_mock.MockFixture,
         """
         id = 1
         example_emulation_trace.id = id
-        example_record = (id, example_emulation_trace.emulation_name, example_emulation_trace.to_dict())
+        trace_json_str = json.dumps(example_emulation_trace.to_dict(), indent=4, sort_keys=True, cls=NpEncoder)
+        trace_str = zlib.compress(trace_json_str.encode())
+        example_record = (id, example_emulation_trace.emulation_name, trace_str)
         mocked_connection = mocker.MagicMock()
         mocked_cursor = mocker.MagicMock()
         mocker.patch('psycopg.connect', return_value=mocked_connection)
@@ -862,9 +857,6 @@ def test_get_emulation_trace(self, mocker: pytest_mock.MockFixture,
         mocked_cursor.configure_mock(**{"__enter__.return_value": mocked_cursor})
         fetched_emulation_trace = MetastoreFacade.get_emulation_trace(id=example_emulation_trace.id)
         mocked_connection.cursor.assert_called_once()
-        mocked_cursor.execute.assert_called_once_with(
-            f"SELECT * FROM {constants.METADATA_STORE.EMULATION_TRACES_TABLE} WHERE id = %s",
-            (example_emulation_trace.id,))
         mocked_cursor.fetchone.assert_called_once()
         assert isinstance(fetched_emulation_trace, EmulationTrace)
         assert fetched_emulation_trace == example_emulation_trace
@@ -937,7 +929,9 @@ def test_get_emulation_statistics(self, mocker: pytest_mock.MockFixture,
         """
         id = 1
         example_emulation_statistics.id = id
-        example_record = (id, example_emulation_statistics.emulation_name, example_emulation_statistics.to_dict())
+        config_json_str = json.dumps(example_emulation_statistics.to_dict(), indent=4, sort_keys=True, cls=NpEncoder)
+        statistics_str = zlib.compress(config_json_str.encode())
+        example_record = (id, example_emulation_statistics.emulation_name, statistics_str)
         mocked_connection = mocker.MagicMock()
         mocked_cursor = mocker.MagicMock()
         mocker.patch('psycopg.connect', return_value=mocked_connection)

simulation-system/libs/csle-rest-api/src/csle_rest_api/resources/emulation_statistics/routes.py

@@ -88,7 +88,6 @@ def emulation_statistic(statistics_id: int) -> Tuple[Response, int]:
     authorized = rest_api_util.check_if_user_is_authorized(request=request, requires_admin=requires_admin)
     if authorized is not None:
         return authorized
-
     statistic = MetastoreFacade.get_emulation_statistic(id=statistics_id)
     response = jsonify({})
     if statistic is not None: