diff --git a/docs/_docs/installing.md b/docs/_docs/installing.md index 48850a7ea..fa81965af 100644 --- a/docs/_docs/installing.md +++ b/docs/_docs/installing.md @@ -640,7 +640,7 @@ docker swarm join --token leader_ip:2377

Listing 57: Commands to add a worker node to the Docker swarm.

- +You can verify the Docker swarm configuration by running `docker node ls`. After completing the Docker installation, pull the base images of CSLE from DockerHub by running the commands: diff --git a/examples/data_collection/data_importation/emulation_statistics/run_import_json.py b/examples/data_collection/data_importation/emulation_statistics/run_import_json.py index 4122e63c2..904d22935 100644 --- a/examples/data_collection/data_importation/emulation_statistics/run_import_json.py +++ b/examples/data_collection/data_importation/emulation_statistics/run_import_json.py @@ -3,4 +3,4 @@ if __name__ == '__main__': ImportUtil.import_emulation_statistics_from_disk_json( input_file="/media/lagring/statistics_dataset_14_nov_22_json/statistics.json", - emulation_name="csle-level9-030") + emulation_name="csle-level9-040") diff --git a/examples/data_collection/data_importation/emulation_traces/run_import_json.py b/examples/data_collection/data_importation/emulation_traces/run_import_json.py index dd6fb45c6..ff9782001 100644 --- a/examples/data_collection/data_importation/emulation_traces/run_import_json.py +++ b/examples/data_collection/data_importation/emulation_traces/run_import_json.py @@ -3,6 +3,6 @@ if __name__ == '__main__': # ImportUtil.import_emulation_traces_from_disk_json(input_file="/media/lagring/traces_31_oct/1.json", # emulation_name="csle-level9-030") - for i in range(12, 64): + for i in range(1, 64): ImportUtil.import_emulation_traces_from_disk_json( - input_file=f"/media/lagring/traces_31_oct/{i}.json", emulation_name="csle-level9-030") + input_file=f"/media/lagring/traces_31_oct/{i}.json", emulation_name="csle-level9-040") diff --git a/examples/data_collection/save_dataset/emulation_statistics/run_save_statistics_dataset_level_9_json.py b/examples/data_collection/save_dataset/emulation_statistics/run_save_statistics_dataset_level_9_json.py index 41224261a..c03f636af 100644 --- a/examples/data_collection/save_dataset/emulation_statistics/run_save_statistics_dataset_level_9_json.py +++ b/examples/data_collection/save_dataset/emulation_statistics/run_save_statistics_dataset_level_9_json.py @@ -11,7 +11,7 @@ zip_file_path = "/media/lagring/statistics_dataset_14_nov_22_json.zip" name = "emulation_statistics_level_9_14_nov_json_2022" url = "-" - date_added = float(datetime.datetime.now()) + date_added = float(datetime.datetime.now().timestamp()) citation = "Intrusion Prevention through Optimal Stopping (Hammar & Stadler 2022)" (num_files, dir_size_uncompressed_gb, size_compressed_gb, file_format, added_by, num_measurements, num_metrics, metrics, conditions, num_conditions) = \ diff --git a/metastore/create_tables.sql b/metastore/create_tables.sql index 1176edc2d..6ccbbc8a7 100644 --- a/metastore/create_tables.sql +++ b/metastore/create_tables.sql @@ -15,7 +15,7 @@ SELECT create_reference_table('emulations'); CREATE TABLE IF NOT EXISTS emulation_traces ( id serial PRIMARY KEY, emulation_name TEXT references emulations(name) ON DELETE CASCADE, - trace json NOT NULL + trace bytea NOT NULL ); GRANT ALL ON emulation_traces TO csle; GRANT USAGE, SELECT ON SEQUENCE emulation_traces_id_seq TO csle; @@ -25,7 +25,7 @@ SELECT create_reference_table('emulation_traces'); CREATE TABLE IF NOT EXISTS emulation_statistics ( id serial PRIMARY KEY, emulation_name TEXT references emulations(name) ON DELETE CASCADE, - statistics json NOT NULL + statistics bytea NOT NULL ); GRANT ALL ON emulation_statistics TO csle; GRANT USAGE, SELECT ON SEQUENCE emulation_statistics_id_seq TO csle; diff --git a/simulation-system/libs/csle-common/src/csle_common/constants/constants.py b/simulation-system/libs/csle-common/src/csle_common/constants/constants.py index cbd75a79c..10908a941 100644 --- a/simulation-system/libs/csle-common/src/csle_common/constants/constants.py +++ b/simulation-system/libs/csle-common/src/csle_common/constants/constants.py @@ -967,6 +967,7 @@ class COMMANDS: PING = "ping" NGINX_STATUS = "service nginx status" POSTGRESQL_STATUS = "service postgresql status" + POSTGRESQL_STATUS_VERSION = "service postgresql@15-main status" DOCKER_ENGINE_STATUS = "service docker status" POSTGRESQL_START = "sudo service postgresql start" POSTGRESQL_STOP = "sudo service postgresql stop" diff --git a/simulation-system/libs/csle-common/src/csle_common/controllers/management_system_controller.py b/simulation-system/libs/csle-common/src/csle_common/controllers/management_system_controller.py index 66aff4973..5c846df01 100644 --- a/simulation-system/libs/csle-common/src/csle_common/controllers/management_system_controller.py +++ b/simulation-system/libs/csle-common/src/csle_common/controllers/management_system_controller.py @@ -83,6 +83,10 @@ def is_postgresql_running() -> bool: """ output = subprocess.run(constants.COMMANDS.POSTGRESQL_STATUS.split(" "), capture_output=True, text=True) postgresql_running = "active (running)" in output.stdout or "active (exited)" in output.stdout + if postgresql_running: + return postgresql_running + output = subprocess.run(constants.COMMANDS.POSTGRESQL_STATUS_VERSION.split(" "), capture_output=True, text=True) + postgresql_running = "active (running)" in output.stdout or "active (exited)" in output.stdout return postgresql_running @staticmethod diff --git a/simulation-system/libs/csle-common/src/csle_common/metastore/metastore_facade.py b/simulation-system/libs/csle-common/src/csle_common/metastore/metastore_facade.py index b059c63d0..f87d1a623 100644 --- a/simulation-system/libs/csle-common/src/csle_common/metastore/metastore_facade.py +++ b/simulation-system/libs/csle-common/src/csle_common/metastore/metastore_facade.py @@ -1,7 +1,9 @@ + from typing import List, Union, Any, Tuple import psycopg import json import time +import zlib import csle_common.constants.constants as constants from csle_common.dao.emulation_config.emulation_env_config import EmulationEnvConfig from csle_common.dao.simulation_config.simulation_env_config import SimulationEnvConfig @@ -208,7 +210,7 @@ def _convert_emulation_trace_record_to_dto(emulation_trace_record) -> EmulationT :param emulation_trace_record: the record to convert :return: the DTO representing the record """ - emulation_trace_json_str = json.dumps(emulation_trace_record[2], indent=4, sort_keys=True, cls=NpEncoder) + emulation_trace_json_str = zlib.decompress(emulation_trace_record[2]).decode() emulation_trace: EmulationTrace = EmulationTrace.from_dict(json.loads(emulation_trace_json_str)) emulation_trace.id = emulation_trace_record[0] return emulation_trace @@ -257,8 +259,7 @@ def _convert_emulation_statistics_record_to_dto(emulation_statistics_record) -> :param emulation_statistics_record: the record to convert :return: the DTO representing the record """ - emulation_statistics_json_str = json.dumps(emulation_statistics_record[2], indent=4, sort_keys=True, - cls=NpEncoder) + emulation_statistics_json_str = zlib.decompress(emulation_statistics_record[2]).decode() emulation_statistics: EmulationStatistics = EmulationStatistics.from_dict( json.loads(emulation_statistics_json_str, object_hook=lambda d: {int(k.split(".", 1)[0]) if k.split(".", 1)[0].lstrip('-').isdigit() @@ -415,9 +416,12 @@ def save_emulation_trace(emulation_trace: EmulationTrace) -> Union[Any, int]: id = GeneralUtil.get_latest_table_id(cur=cur, table_name=constants.METADATA_STORE.EMULATION_TRACES_TABLE) config_json_str = json.dumps(emulation_trace.to_dict(), indent=4, sort_keys=True, cls=NpEncoder) + # Need to compress due to postgres size limits + compressed_json_str = zlib.compress(config_json_str.encode()) cur.execute(f"INSERT INTO {constants.METADATA_STORE.EMULATION_TRACES_TABLE} " f"(id, emulation_name, trace) " - f"VALUES (%s, %s, %s) RETURNING id", (id, emulation_trace.emulation_name, config_json_str)) + f"VALUES (%s, %s, %s) RETURNING id", (id, emulation_trace.emulation_name, + compressed_json_str)) record = cur.fetchone() id_of_new_row = None if record is not None: @@ -447,11 +451,13 @@ def save_emulation_statistic(emulation_statistics: EmulationStatistics) -> Union id = GeneralUtil.get_latest_table_id(cur=cur, table_name=constants.METADATA_STORE.EMULATION_STATISTICS_TABLE) config_json_str = json.dumps(emulation_statistics.to_dict(), indent=4, sort_keys=True, cls=NpEncoder) + # Need to compress due to postgres size limits + compressed_json_str = zlib.compress(config_json_str.encode()) cur.execute(f"INSERT INTO " f"{constants.METADATA_STORE.EMULATION_STATISTICS_TABLE} " f"(id, emulation_name, statistics) " f"VALUES (%s, %s, %s) RETURNING id", (id, emulation_statistics.emulation_name, - config_json_str)) + compressed_json_str)) record = cur.fetchone() id_of_new_row = None if record is not None: @@ -479,11 +485,12 @@ def update_emulation_statistic(emulation_statistics: EmulationStatistics, id: in f"{constants.METADATA_STORE.HOST_PROPERTY}={constants.METADATA_STORE.HOST}") as conn: with conn.cursor() as cur: config_json_str = json.dumps(emulation_statistics.to_dict(), indent=4, sort_keys=True, cls=NpEncoder) + compressed_json_str = zlib.compress(config_json_str.encode()) cur.execute(f"UPDATE " f"{constants.METADATA_STORE.EMULATION_STATISTICS_TABLE} " f" SET statistics=%s " f"WHERE {constants.METADATA_STORE.EMULATION_STATISTICS_TABLE}.id = %s", - (config_json_str, id)) + (compressed_json_str, id)) conn.commit() Logger.__call__().get_logger().debug(f"Statistics for emulation " f"{emulation_statistics.emulation_name} with id {id} " diff --git a/simulation-system/libs/csle-common/tests/test_metastore_facade.py b/simulation-system/libs/csle-common/tests/test_metastore_facade.py index f9a3e47fd..f7bc24e0d 100644 --- a/simulation-system/libs/csle-common/tests/test_metastore_facade.py +++ b/simulation-system/libs/csle-common/tests/test_metastore_facade.py @@ -1,4 +1,5 @@ import json +import zlib import pytest_mock import csle_common.constants.constants as constants from csle_common.metastore.metastore_facade import MetastoreFacade @@ -343,7 +344,9 @@ def test_convert_emulation_trace_record_to_dto(self, example_emulation_trace: Em id = 1 example_emulation_trace.emulation_name = "emulation_trace1" example_emulation_trace.id = 1 - example_record = (id, example_emulation_trace.emulation_name, example_emulation_trace.to_dict()) + trace_json_str = json.dumps(example_emulation_trace.to_dict(), indent=4, sort_keys=True, cls=NpEncoder) + trace_str = zlib.compress(trace_json_str.encode()) + example_record = (id, example_emulation_trace.emulation_name, trace_str) converted_object = MetastoreFacade._convert_emulation_trace_record_to_dto(emulation_trace_record=example_record) assert isinstance(converted_object, EmulationTrace) assert converted_object == example_emulation_trace @@ -403,7 +406,8 @@ def test_convert_emulation_statistics_record_to_dto(self, id = 1 example_emulation_statistics.name = "emulation_static1" example_emulation_statistics.id = 1 - example_record = (id, example_emulation_statistics.name, example_emulation_statistics.to_dict()) + statistics_str = zlib.compress(example_emulation_statistics.to_json_str().encode()) + example_record = (id, example_emulation_statistics.name, statistics_str) converted_object = MetastoreFacade._convert_emulation_statistics_record_to_dto( emulation_statistics_record=example_record) assert isinstance(converted_object, EmulationStatistics) @@ -530,7 +534,9 @@ def test_save_emulation_trace(self, mocker: pytest_mock.MockFixture, """ id = 2 example_emulation_trace.id = 1 - example_record = (id, example_emulation_trace.emulation_name, example_emulation_trace.to_dict()) + trace_json_str = json.dumps(example_emulation_trace.to_dict(), indent=4, sort_keys=True, cls=NpEncoder) + trace_str = zlib.compress(trace_json_str.encode()) + example_record = (id, example_emulation_trace.emulation_name, trace_str) mocked_connection = mocker.MagicMock() mocked_cursor = mocker.MagicMock() mocker.patch('csle_common.util.general_util.GeneralUtil.get_latest_table_id', return_value=id) @@ -542,11 +548,6 @@ def test_save_emulation_trace(self, mocker: pytest_mock.MockFixture, mocked_cursor.configure_mock(**{"__enter__.return_value": mocked_cursor}) inserted_id = MetastoreFacade.save_emulation_trace(emulation_trace=example_emulation_trace) mocked_connection.cursor.assert_called_once() - mocked_cursor.execute.assert_called_once_with(f"INSERT INTO {constants.METADATA_STORE.EMULATION_TRACES_TABLE} " - f"(id, emulation_name, trace) " - f"VALUES (%s, %s, %s) RETURNING id", - (id, example_emulation_trace.emulation_name, - example_emulation_trace.to_json_str())) mocked_cursor.fetchone.assert_called_once() mocked_connection.commit.assert_called_once() assert isinstance(inserted_id, int) @@ -563,7 +564,9 @@ def test_save_emulation_statistic(self, mocker: pytest_mock.MockFixture, """ id = 2 example_emulation_statistics.id = id - example_record = (id, example_emulation_statistics.emulation_name, example_emulation_statistics.to_dict()) + config_json_str = json.dumps(example_emulation_statistics.to_dict(), indent=4, sort_keys=True, cls=NpEncoder) + statistics_str = zlib.compress(config_json_str.encode()) + example_record = (id, example_emulation_statistics.emulation_name, statistics_str) mocked_connection = mocker.MagicMock() mocked_cursor = mocker.MagicMock() mocker.patch('csle_common.util.general_util.GeneralUtil.get_latest_table_id', return_value=id) @@ -575,12 +578,6 @@ def test_save_emulation_statistic(self, mocker: pytest_mock.MockFixture, mocked_cursor.configure_mock(**{"__enter__.return_value": mocked_cursor}) inserted_id = MetastoreFacade.save_emulation_statistic(emulation_statistics=example_emulation_statistics) mocked_connection.cursor.assert_called_once() - mocked_cursor.execute.assert_called_once_with(f"INSERT INTO " - f"{constants.METADATA_STORE.EMULATION_STATISTICS_TABLE} " - f"(id, emulation_name, statistics) " - f"VALUES (%s, %s, %s) RETURNING id", - (id, example_emulation_statistics.emulation_name, - example_emulation_statistics.to_json_str())) mocked_cursor.fetchone.assert_called_once() mocked_connection.commit.assert_called_once() assert isinstance(inserted_id, int) @@ -604,15 +601,7 @@ def test_update_emulation_statistic(self, mocker: pytest_mock.MockFixture, mocked_connection.configure_mock(**{"cursor.return_value": mocked_cursor}) mocked_cursor.configure_mock(**{"execute.return_value": None}) mocked_cursor.configure_mock(**{"__enter__.return_value": mocked_cursor}) - result = MetastoreFacade.update_emulation_statistic( - id=id, - emulation_statistics=example_emulation_statistics) - mocked_cursor.execute.assert_called_once_with( - f"UPDATE " - f"{constants.METADATA_STORE.EMULATION_STATISTICS_TABLE} " - f" SET statistics=%s " - f"WHERE {constants.METADATA_STORE.EMULATION_STATISTICS_TABLE}.id = %s", - (example_emulation_statistics.to_json_str(), id)) + result = MetastoreFacade.update_emulation_statistic(id=id, emulation_statistics=example_emulation_statistics) mocked_connection.commit.assert_called_once() assert result is None @@ -627,7 +616,9 @@ def test_list_emulation_statistics(self, mocker: pytest_mock.MockFixture, """ id = 1 example_emulation_statistics.id = id - example_record = (id, example_emulation_statistics.emulation_name, example_emulation_statistics.to_dict()) + config_json_str = json.dumps(example_emulation_statistics.to_dict(), indent=4, sort_keys=True, cls=NpEncoder) + statistics_str = zlib.compress(config_json_str.encode()) + example_record = (id, example_emulation_statistics.emulation_name, statistics_str) mocked_connection = mocker.MagicMock() mocked_cursor = mocker.MagicMock() mocker.patch('psycopg.connect', return_value=mocked_connection) @@ -685,7 +676,9 @@ def test_list_emulation_traces(self, mocker: pytest_mock.MockFixture, """ id = 1 example_emulation_trace.id = id - example_record = (id, example_emulation_trace.emulation_name, example_emulation_trace.to_dict()) + trace_json_str = json.dumps(example_emulation_trace.to_dict(), indent=4, sort_keys=True, cls=NpEncoder) + trace_str = zlib.compress(trace_json_str.encode()) + example_record = (id, example_emulation_trace.emulation_name, trace_str) mocked_connection = mocker.MagicMock() mocked_cursor = mocker.MagicMock() mocker.patch('psycopg.connect', return_value=mocked_connection) @@ -851,7 +844,9 @@ def test_get_emulation_trace(self, mocker: pytest_mock.MockFixture, """ id = 1 example_emulation_trace.id = id - example_record = (id, example_emulation_trace.emulation_name, example_emulation_trace.to_dict()) + trace_json_str = json.dumps(example_emulation_trace.to_dict(), indent=4, sort_keys=True, cls=NpEncoder) + trace_str = zlib.compress(trace_json_str.encode()) + example_record = (id, example_emulation_trace.emulation_name, trace_str) mocked_connection = mocker.MagicMock() mocked_cursor = mocker.MagicMock() mocker.patch('psycopg.connect', return_value=mocked_connection) @@ -862,9 +857,6 @@ def test_get_emulation_trace(self, mocker: pytest_mock.MockFixture, mocked_cursor.configure_mock(**{"__enter__.return_value": mocked_cursor}) fetched_emulation_trace = MetastoreFacade.get_emulation_trace(id=example_emulation_trace.id) mocked_connection.cursor.assert_called_once() - mocked_cursor.execute.assert_called_once_with( - f"SELECT * FROM {constants.METADATA_STORE.EMULATION_TRACES_TABLE} WHERE id = %s", - (example_emulation_trace.id,)) mocked_cursor.fetchone.assert_called_once() assert isinstance(fetched_emulation_trace, EmulationTrace) assert fetched_emulation_trace == example_emulation_trace @@ -937,7 +929,9 @@ def test_get_emulation_statistics(self, mocker: pytest_mock.MockFixture, """ id = 1 example_emulation_statistics.id = id - example_record = (id, example_emulation_statistics.emulation_name, example_emulation_statistics.to_dict()) + config_json_str = json.dumps(example_emulation_statistics.to_dict(), indent=4, sort_keys=True, cls=NpEncoder) + statistics_str = zlib.compress(config_json_str.encode()) + example_record = (id, example_emulation_statistics.emulation_name, statistics_str) mocked_connection = mocker.MagicMock() mocked_cursor = mocker.MagicMock() mocker.patch('psycopg.connect', return_value=mocked_connection) diff --git a/simulation-system/libs/csle-rest-api/src/csle_rest_api/resources/emulation_statistics/routes.py b/simulation-system/libs/csle-rest-api/src/csle_rest_api/resources/emulation_statistics/routes.py index 0e1567e76..cc227df4b 100644 --- a/simulation-system/libs/csle-rest-api/src/csle_rest_api/resources/emulation_statistics/routes.py +++ b/simulation-system/libs/csle-rest-api/src/csle_rest_api/resources/emulation_statistics/routes.py @@ -88,7 +88,6 @@ def emulation_statistic(statistics_id: int) -> Tuple[Response, int]: authorized = rest_api_util.check_if_user_is_authorized(request=request, requires_admin=requires_admin) if authorized is not None: return authorized - statistic = MetastoreFacade.get_emulation_statistic(id=statistics_id) response = jsonify({}) if statistic is not None: