From a5402ae526fdcc0d025530172fd93ac5f203e0ea Mon Sep 17 00:00:00 2001 From: felix-20 <39854388+felix-20@users.noreply.github.com> Date: Wed, 20 Jul 2022 17:48:19 +0200 Subject: [PATCH] [B / Docker] Logging for api (#517) * merge multiple experiments commit 8652a678520884a4c7325c87965cc4cfe42ffb48 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 18:17:20 2022 +0200 missed `handle_requests` on last commit commit a354f890afa46be9f678ffa88dfe9d3e7815a838 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 18:16:15 2022 +0200 implement #356 commit fd54365178e6b65c607c0fbb45a3ad8dca023e12 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 17:56:35 2022 +0200 docstrings commit 9f43c9dbfacc50764c5e7d1062b60a3d94f64057 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 15:07:47 2022 +0200 fix docker test commit b3e6e80628f9743f8de3d2cb99a6e8839ab0f74e Merge: 6f64f9e 091eee6 Author: felix-20 <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 14:31:39 2022 +0200 Merge branch 'development' into 379-multiple-experiments commit 6f64f9ec9a5e56798594762ed03310fb4bff39da Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 14:20:47 2022 +0200 clean up docker port commit 8c35e8a6634c6516af71a6eeb959f880c01cd8ba Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 11:33:30 2022 +0200 merge development commit 091eee66ed4e2870d860059ed0a1ec9ca515ee6d Author: Nikkel Mollenhauer <57323886+NikkelM@users.noreply.github.com> Date: Mon Apr 11 10:43:00 2022 +0200 Tests for `config_validation.py` (#404) * Don't get default data (shouldn't be necessary) * Unpack default data * Refactored utils functions to return dicts instead of strings * Adapted to new mock format * Adapted to new mock format * Some first tests * More tests * Fixed testcase-names * Moved file-endings to initial function call * Fixed tests * More asserts * More tests * More tests * More tests * `validate_sub_keys`-tests commit f8bc1622e763f3502d103b77174517383c8fa8a2 Author: jannikgro <52510222+jannikgro@users.noreply.github.com> Date: Fri Apr 8 13:20:42 2022 +0200 [D] Stable baselines integration (#384) * refactored reinforcement learning agent to accept marketplace * adapted test_exampleprinter.py to marketplace initialization * add market option to accept continuos actions * fixed action space check * initial stable baselines integration * Agent init by env (#390) * introduced self.network in actorcritic_agent * added network_architecture in QLearningAgent * changed actorcritic_agent to network_architecture * set back training_scenario * am_configuration initialize rl-agent via marketplace * added final analyse to stable baselines training * added more stable baselines algorithms * added ppo algorithm * introduced stable_baselines_folder * renamed training to callback * satisfied linter * fixed loading problem * try to make tqdm run in stable_baselines * make tqdm running * reduced pmonitoring episodes in sb training * save model only if significantly better * fixed too long test time bug * moved back to 250 episodes testing * set timeout to 15 minutes * added first batch of fixes to @NikkelM feedback * added type annotations and asserts in stable_baselines_model * added sbtraining to training_scenario * applied comments in am_configuration * solved .dat problem and fixed crashing asserts * reintroduced _end_of_training * removed deprecated if * Moved '.dat' to function call instead of appending within function * Fixed assert * fixed model file ending bug * Add short explanation docstring Co-authored-by: Johann Schulze Tast <35633229+blackjack2693@users.noreply.github.com> * fixed wrong docstring * Fixed tests Co-authored-by: NikkelM Co-authored-by: Johann Schulze Tast <35633229+blackjack2693@users.noreply.github.com> commit 018387606ffe99bdeaf65c91720dce75149f59e1 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon Apr 11 11:31:15 2022 +0200 name from `names` for container commit ad70b7ca95d876889965eb5d9c5125e91b860434 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Sun Apr 10 16:32:47 2022 +0200 fix #380 commit 8e1314ca0f8cd631dce0959dacefaed9b51dd292 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Sun Apr 10 16:05:28 2022 +0200 multiple experiments are supported on the webserver commit 53a40007f7c6ddcf2dc4be0930298234269400d3 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Fri Apr 8 14:36:44 2022 +0200 support for starting multiple container on docker side commit 3c94f4325123f82d8bb2fe25554004560ebd0c3e Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Fri Apr 8 11:46:07 2022 +0200 ability to add `DockerInfo` for multiple container * first attempt for websocket * websocket on docker site working * just send all of it if things have changed * started on webserver site * webserver sends push notification to user about stopped container * commit * websocket to ssl * fix * try * started on database manager * more db * fix? * debug statements * more types * does table exist fix * colorful terminal output * commit * debug * extra health checker * debug for force stop * some logging * fix tests * telegram notifications * better logging * better logging * fix test? * different logging level * small fixes * started on system monitoring * system monitoring * csv possibility for system data * two buttons in webserver * more fail prove * fix configuration form * silent_starter * logging statements for silent starter * debug * debug * more debugging * fix error * more debugging * more local monitoring * adjusted logging in silent starter * gpu * os.system * subprocess * Squashed commit of the following: commit af12e13b04a741fddf530451ee3c7c9e1ad821be Merge: bc9d3a3 ab704ca Author: Nikkel Mollenhauer Date: Tue Jun 7 18:20:56 2022 +0200 Merge branch '484-configuration-remove-combined-config' of https://github.com/hpi-epic/BP2021 into 484-configuration-remove-combined-config commit bc9d3a387fdc79676d83ac3f75dd60ab049465e7 Author: Nikkel Mollenhauer Date: Tue Jun 7 18:20:50 2022 +0200 Removed dead methods commit ab704ca883df5626227a9c6dbf1936b54adf79c5 Author: felix-20 <39854388+felix-20@users.noreply.github.com> Date: Tue Jun 7 16:55:34 2022 +0200 improved prefill (#496) * improved prefill with consideration of the current formdata * remove debug statements * remove print debug * Added lxml dependency * Removed debug comment Co-authored-by: Nikkel Mollenhauer commit 8839f9017a5c5ea658a9da693f4d470ec49f38fe Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Tue Jun 7 12:01:43 2022 +0200 remove `config_is_final` commit 53d79f9c110d2d4802bf741058ecbd28083f3b7a Author: NikkelM Date: Tue Jun 7 10:41:51 2022 +0200 Review feedback by @SinNeax commit 99ab68189c7493d071581289a217f1fa5cb4cbfb Author: NikkelM Date: Sat Jun 4 15:02:34 2022 +0200 Fixed config validation commit 6a09267883dd6731656148fb064f1c134bf1a35a Merge: 179e0da e623413 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Fri Jun 3 14:50:15 2022 +0200 Merge branch '484-configuration-remove-combined-config' of https://github.com/hpi-epic/BP2021 into 484-configuration-remove-combined-config commit 179e0da87b19b2f75dc2f18391df43fe5193348a Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Fri Jun 3 14:49:49 2022 +0200 fix javascript error commit e62341322c9b35d95a246b914c2e9032e6d84674 Author: NikkelM Date: Fri Jun 3 14:38:20 2022 +0200 Added `config_type` field to default modelfiles commit bd151084fd88f9d500535052b50c2fda1a12ab89 Merge: dac92d3 507cbc1 Author: NikkelM Date: Fri Jun 3 09:45:11 2022 +0200 Merge branch '484-configuration-remove-combined-config' of https://github.com/hpi-epic/BP2021 into 484-configuration-remove-combined-config commit dac92d3e81a53bf7e88e9a4ec3575e4b13b71c2a Author: NikkelM Date: Fri Jun 3 09:44:49 2022 +0200 Fixed tests commit 507cbc19b3e9d98b72196242cf3ae16f8fd20597 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Fri Jun 3 07:52:30 2022 +0200 dynamic table for sim_market possible commit bce03bdf0e581bb35e0dbfa8ca5620a5332800de Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Thu Jun 2 20:15:17 2022 +0200 fix webserver tests commit 57dbb332d1c6073a044a4bb51a90fe547ab3270c Author: Nikkel Mollenhauer Date: Thu Jun 2 14:29:40 2022 +0200 Webserver format commit b3780cd2e28b675972e70ef08e6759f2ddff2589 Author: Nikkel Mollenhauer Date: Thu Jun 2 14:15:01 2022 +0200 Fixed validation commit c9657a90c32bb81e2e83df91a1b8b7caf1c54981 Author: Nikkel Mollenhauer Date: Thu Jun 2 13:50:20 2022 +0200 Added back needed functionality of validating "complete" configs commit dca24f332dfccca5c54fbf891db26bf82338b4d7 Author: Nikkel Mollenhauer Date: Thu Jun 2 13:34:21 2022 +0200 Added some tests commit fff9b47597ed5999fefc9444631489bc648656cb Author: Nikkel Mollenhauer Date: Thu Jun 2 13:12:41 2022 +0200 Renamed `market` to `sim_market` commit caf5c019f313fc888825b54759f36094ff25ebcf Merge: b8c43b7 f50fe65 Author: Nikkel Mollenhauer Date: Thu Jun 2 11:30:18 2022 +0200 Merge branch '484-configuration-remove-combined-config' of https://github.com/hpi-epic/BP2021 into 484-configuration-remove-combined-config commit b8c43b7997647b0b98a4548526496185be591cde Author: Nikkel Mollenhauer Date: Thu Jun 2 11:29:34 2022 +0200 New config format commit f50fe65fccbed1db759c42ffb6e3afd0e277fe4a Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Thu Jun 2 11:21:10 2022 +0200 implement new config validation for webserver commit 135e8f300a5da0f853f07ddc4faec7ae3a13d0e9 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Wed Jun 1 20:52:12 2022 +0200 fix webserver tests commit d3262458eae5d33c96e81bfbb4872d0f99ca9273 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Wed Jun 1 17:26:50 2022 +0200 fix prefill commit 70492c86bc08f2daa7277466cf8f0b05965823d0 Author: Nikkel Mollenhauer Date: Wed Jun 1 17:08:34 2022 +0200 Fixed remaining tests(?) commit 59136b14a51a4db3513b369e21042dc1a9dc07b3 Author: Nikkel Mollenhauer Date: Wed Jun 1 16:35:00 2022 +0200 Fixed most tests commit c91230934ea9f0c648d1c479bfe5300803e27147 Author: Nikkel Mollenhauer Date: Wed Jun 1 16:11:20 2022 +0200 Removed references to `class`-field commit 2ccee7898d5ac00b7cd3f950858296b5b63603c2 Author: Nikkel Mollenhauer Date: Wed Jun 1 15:12:56 2022 +0200 Removed `class` keywords from config files commit c51dab7f0a4523b42fd76b3315b5dc90ecaffe98 Author: Nikkel Mollenhauer Date: Wed Jun 1 15:09:46 2022 +0200 renamed rl_config to q_learning_config commit f9aa012d86d3f0699d81ace7a979c3f0b43a3a40 Author: NikkelM Date: Tue May 31 19:49:14 2022 +0200 Fixed Agent_monitoring with stable_baselines agents commit 16df8c4a5facceac365e78255075be03a59ca437 Merge: 7610c6e 26cb54b Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Tue May 31 13:50:24 2022 +0200 Merge branch '484-configuration-remove-combined-config' of https://github.com/hpi-epic/BP2021 into 484-configuration-remove-combined-config commit 7610c6e448d2e320526f20a0ac92fc904dcf2760 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Tue May 31 13:50:14 2022 +0200 script for creating rl model commit 26cb54b00dec8586c2d433b1a8d92362d7ecbb13 Author: NikkelM Date: Tue May 31 11:21:46 2022 +0200 Simplified and extended key validation commit 0fa7eb5bab808494692bd8c039f8fcfd85aeb1b3 Author: Judith <39854388+felix-20@users.noreply.github.com> Date: Mon May 30 14:17:22 2022 +0200 rl config works in view commit da0547478cf0d38702c5096f28de36e6c2accf26 Author: NikkelM Date: Fri May 27 13:37:39 2022 +0200 New feather website commit dd4010438e9a211ca5cf13b155a6f49f3767f5a1 Author: NikkelM Date: Fri May 27 12:56:28 2022 +0200 Removed debug comment, fixed policyanalyzer commit 1d025c911c8764c339cd5944459b9751ae8e6c6e Author: NikkelM Date: Fri May 27 12:39:17 2022 +0200 Reintroduced `main` for testing purposes commit c90a8d6d77f6840b6997bcfd3f71c30587d66263 Author: NikkelM Date: Fri May 27 12:26:10 2022 +0200 Fixed config validation for webserver commit 4107aaf77ac1755f5b327569ba9fccf1c0ee3c84 Author: NikkelM Date: Fri May 27 10:57:04 2022 +0200 Added some debugging to docker commit 2588f6da51e7342774ef1c90cc95d6b297cb8e2a Author: Jan Niklas Groeneveld Date: Thu May 26 22:22:39 2022 +0200 preconditions for actor critic parameters and stable baselines parameter have been created commit 4e174e6a0fae69541320bc7c75ea82776a281e3e Author: Jan Niklas Groeneveld Date: Thu May 26 19:42:23 2022 +0200 restored changes commit 14f181df344a8032aeee76a06b0d05c41c5b4e26 Author: Jan Niklas Groeneveld Date: Thu May 26 19:37:23 2022 +0200 restored last changes commit 3d3ac9dd39c9790cff265da2296ceabbc9957934 Author: Jan Niklas Groeneveld Date: Thu May 26 19:34:11 2022 +0200 more information commit 31a212f2fa6d92b17fdecfbd040c4f5bb3b582ed Author: Jan Niklas Groeneveld Date: Thu May 26 19:31:31 2022 +0200 more information commit 1b139e08c55af2baf940b171c4bfd2435c486a33 Author: Jan Niklas Groeneveld Date: Thu May 26 19:28:02 2022 +0200 test commit 1e80e8cb2792d9f6822111ac0d812c63a42f1fb9 Author: Jan Niklas Groeneveld Date: Thu May 26 19:25:16 2022 +0200 next try commit 09a433e05717aa542a29b4563cfa5307f6abb01d Author: Jan Niklas Groeneveld Date: Thu May 26 19:22:45 2022 +0200 another printf try commit de9f5eb8ed50b8bd93fb4908bd7c0e1301329c8f Author: Jan Niklas Groeneveld Date: Thu May 26 19:19:07 2022 +0200 more printf debug commit 511d38ae0caf7837a517b70f03ce2b9e1c3f8a7c Author: Jan Niklas Groeneveld Date: Thu May 26 19:16:11 2022 +0200 printf debug commit 1d1a3c4b37810e1881692db252eed8b5838e6309 Author: Jan Niklas Groeneveld Date: Thu May 26 19:12:06 2022 +0200 restored last changes commit 61c0d8eeea0f7c3a03413466213020f8a0f91bdc Author: Jan Niklas Groeneveld Date: Thu May 26 19:09:22 2022 +0200 tried without reading check commit 068abaaa1502c6a3d232d45270329270e88ee504 Author: Jan Niklas Groeneveld Date: Thu May 26 19:03:54 2022 +0200 tried with dirty fix commit f1cdf451552278d3abfce15d0ef844e57710f528 Author: Jan Niklas Groeneveld Date: Thu May 26 19:02:03 2022 +0200 debug print commit 75a674e32add34911288e25e07e6299954c02443 Author: Jan Niklas Groeneveld Date: Thu May 26 18:56:08 2022 +0200 restored docker_manager commit 39bc23876eddd54243afaba77371799ad04b7ad5 Author: Jan Niklas Groeneveld Date: Thu May 26 18:51:31 2022 +0200 tried with assert False commit 698651350684d9d9c98297b2990a8f59e08291fd Author: Jan Niklas Groeneveld Date: Thu May 26 18:21:59 2022 +0200 small change in docker_manager commit 561fc0090c14976770c32c318b13454523f05de2 Author: Jan Niklas Groeneveld Date: Thu May 26 18:13:18 2022 +0200 tried to fix config_validation commit 3fded29c003ac3f8c59058e9390fe00ab7b499f8 Author: Jan Niklas Groeneveld Date: Thu May 26 17:25:10 2022 +0200 reintroduced test_hyperparameter_config_market commit b6a49450f5ce017746b0405c37b431bf6e7f55e8 Author: Jan Niklas Groeneveld Date: Thu May 26 15:52:19 2022 +0200 reintroduced test_hyperparameter_config_rl commit 7cef3e754ad5aa466561af4367ea8a951837da54 Author: Jan Niklas Groeneveld Date: Thu May 26 15:19:29 2022 +0200 added rules and verifications commit 473ddf761fea2d205356ec5010e9ee71ba382b27 Author: Jan Niklas Groeneveld Date: Thu May 26 12:43:12 2022 +0200 introduced JSONConfigurable class and demanded class entry in config commit 36024c8d4c6d6a084b94ed453055dfbf55a3fbc7 Author: Jan Niklas Groeneveld Date: Thu May 26 11:50:48 2022 +0200 solved #484 without check * automatically write template files * try fix pre-commit * try fix pre-commit * try except * more debug * more debug * debugging :) * hotfix for qlearning file * adopting to the given agent works again * restore docker manager and app.py * try restore again * webserver tests are running again merge from dev was very very strange * new rl_config * precommit? * precommit! * assert print Co-Authored-By: Nikkel Mollenhauer <57323886+NikkelM@users.noreply.github.com> * remove print Co-Authored-By: Nikkel Mollenhauer <57323886+NikkelM@users.noreply.github.com> * new regex for matching ce agents Co-Authored-By: Nikkel Mollenhauer <57323886+NikkelM@users.noreply.github.com> * fixed fixed price agent Co-Authored-By: Nikkel Mollenhauer <57323886+NikkelM@users.noreply.github.com> * comment out assert Co-Authored-By: Nikkel Mollenhauer <57323886+NikkelM@users.noreply.github.com> Co-Authored-By: jannikgro <52510222+jannikgro@users.noreply.github.com> * delete some files * bcolors back * readme websocket * debug websocket * fixed websocket * small websocket fix * logging for websocket * more logging? * logging * logging relevant content * reset? * reset webserver * relevant gitignore stuff * some resets in docker manager * init files again? * separate_markets again * remove unnecessary code * fixed docker tests, introduce Mocked logger * logger into DockerManager * Fixed typo bug * log to file when executing `docker_manager` Co-authored-by: Nikkel Mollenhauer <57323886+NikkelM@users.noreply.github.com> Co-authored-by: jannikgro <52510222+jannikgro@users.noreply.github.com> Co-authored-by: Johann Schulze Tast --- .gitignore | 8 +- docker/app.py | 16 +- docker/docker_manager.py | 81 +++++---- docker/log_api.ini | 21 +++ docker/test_docker_manager.py | 26 ++- tests/test_config_validation.py | 162 +++++++++--------- ...010_alter_environmentconfig_marketplace.py | 36 ++-- ...011_alter_agentconfig_argument_and_more.py | 46 ++--- .../migrations/0014_merge_20220609_1023.py | 28 +-- 9 files changed, 240 insertions(+), 184 deletions(-) create mode 100644 docker/log_api.ini diff --git a/.gitignore b/.gitignore index 3246aa90..4b613f5e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,12 +8,10 @@ data # results from the tests tests/test_results -# all webserver related directories -webserver/configurations -webserver/data -webserver/package-lock.json -webserver/node_modules +# all webserver and docker related directories webserver/.env.txt +docker_api/.env.txt +docker_api/log_files/* # file where the users data path is saved recommerce/configuration/user_path.txt diff --git a/docker/app.py b/docker/app.py index ad83c7ee..6cc6f377 100644 --- a/docker/app.py +++ b/docker/app.py @@ -1,5 +1,5 @@ -# app.py import hashlib +import logging import os import time @@ -21,8 +21,12 @@ # If using a remote machine use # uvicorn --host 0.0.0.0 app:app --reload # instead to expose it to the local network -manager = DockerManager() +path_to_log_files = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'log_files') +if not os.path.isdir(path_to_log_files): + os.makedirs(path_to_log_files) +logger = logging.getLogger('uvicorn.error') +manager = DockerManager(logger) app = FastAPI() @@ -47,17 +51,15 @@ def verify_token(request: Request) -> bool: """ verifies for a given request that the header contains the right AUTHORIZATION_TOKEN. Warning: This cannot be considered 100% secure, without https, any network sniffer can read the token - Args: request (Request): The request to the API - Returns: bool: if the given authorization token matches our authorization token. """ try: token = request.headers['Authorization'] except KeyError: - print('The request did not set an Authorization header') + logger.error('The request did not set an Authorization header') return False master_secret_as_int = sum(ord(c) for c in os.environ['AUTHORIZATION_TOKEN']) current_time = int(time.time() / 3600) # unix time in hours @@ -84,6 +86,7 @@ async def start_container(num_experiments: int, config: Request, authorized: boo if not authorized: return JSONResponse(status_code=401, content=vars(DockerInfo('', 'Not authorized'))) all_container_infos = manager.start(config=await config.json(), count=num_experiments) + # check if all prerequisites were met if type(all_container_infos) == DockerInfo: return JSONResponse(status_code=404, content=vars(all_container_infos)) @@ -93,7 +96,7 @@ async def start_container(num_experiments: int, config: Request, authorized: boo if (is_invalid_status(all_container_infos[index].status) or all_container_infos[index].data is False): return JSONResponse(status_code=404, content=vars(all_container_infos[index])) return_dict[index] = vars(all_container_infos[index]) - print(f'successfully started {num_experiments} container') + logger.info(f'successfully started {num_experiments} container') return JSONResponse(return_dict, status_code=200) @@ -283,5 +286,6 @@ async def check_if_api_is_available(authorized: bool = Depends(verify_token)) -> uvicorn.run('app:app', host='0.0.0.0', port=8000, + log_config='./log_api.ini', ssl_keyfile='/etc/sslzertifikat/api_cert.key', ssl_certfile='/etc/sslzertifikat/api_cert.crt') diff --git a/docker/docker_manager.py b/docker/docker_manager.py index be971c00..6e1bbe4e 100644 --- a/docker/docker_manager.py +++ b/docker/docker_manager.py @@ -59,21 +59,24 @@ class DockerManager(): _allowed_commands = ['training', 'exampleprinter', 'agent_monitoring'] # dictionary of container_id:host-port pairs _port_mapping = {} + _logger = None - def __new__(cls): + def __new__(cls, logger): """ This function makes sure that the `DockerManager` is a singleton. Returns: DockerManager: The DockerManager instance. """ + cls._logger = logger if cls._instance is None: - print('A new instance of DockerManager is being initialized') + cls._logger.info('A new instance of DockerManager is being initialized') cls._instance = super(DockerManager, cls).__new__(cls) cls._client = cls._get_client() if cls._client is not None: cls._update_port_mapping() + return cls._instance def start(self, config: dict, count: int) -> DockerInfo or list: @@ -96,7 +99,7 @@ def start(self, config: dict, count: int) -> DockerInfo or list: command_id = config['environment']['task'] if command_id not in self._allowed_commands: - print(f'Command with ID {command_id} not allowed') + self._logger.warning(f'Command with ID {command_id} not allowed') return DockerInfo(id='No container was started', status=f'Command not allowed: {command_id}') if not self._confirm_image_exists(): @@ -105,7 +108,6 @@ def start(self, config: dict, count: int) -> DockerInfo or list: all_container_infos = [] for _ in range(count): # start a container for the image of the requested command - print('cuda?', is_available()) container_info: DockerInfo = self._create_container(command_id, config, use_gpu=is_available()) if 'Image not found' in container_info.status or container_info.data is False: # something is wrong with our container @@ -125,7 +127,7 @@ def health(self, container_id: str) -> DockerInfo: Returns: DockerInfo: A JSON serializable object containing the id and the status of the new container. """ - print(f'Checking health status for: {container_id}') + self._logger.info(f'Checking health status for: {container_id}') container: Container = self._get_container(container_id) if not container: return DockerInfo(container_id, status='Container not found') @@ -206,7 +208,7 @@ def start_tensorboard(self, container_id: str) -> DockerInfo: if container.status != 'running': return DockerInfo(container_id, status='Container is not running. Download the data and start a tensorboard locally.') - print(f'Starting tensorboard for: {container_id}') + self._logger.info(f'Starting tensorboard for: {container_id}') container.exec_run(cmd='tensorboard serve --host 0.0.0.0 --logdir ./results/runs', detach=True) port = self._port_mapping[container.id] return DockerInfo(container_id, status=container.status, data=str(port)) @@ -229,7 +231,7 @@ def get_container_logs(self, container_id: str, timestamps: bool, stream: bool, if not container: return DockerInfo(container_id, status='Container not found') - print(f'Getting logs for {container_id}...') + self._logger.info(f'Getting logs for {container_id}...') logs = container.logs(stream=stream, timestamps=timestamps, tail=tail, stderr=docker.APIClient().inspect_container(container.id)['State']['ExitCode'] != 0) @@ -276,10 +278,10 @@ def remove_container(self, container_id: str) -> DockerInfo: container_info = self._stop_container(container_id) if container_info.status != 'exited': - print(f'Container not stopped successfully. Status: {container_info.status}') + self._logger.warning(f'Container not stopped successfully. Status: {container_info.status}') return DockerInfo(id=container_id, status=f'Container not stopped successfully. Status: {container_info.status}') - print(f'Removing container: {container_id}') + self._logger.info(f'Removing container: {container_id}') try: exit_code = container.wait()['StatusCode'] container.remove() @@ -297,12 +299,12 @@ def ping(self) -> bool: Returns: bool: If the server is running or not. """ - print('Pinging docker server...') + self._logger.info('Pinging docker server...') try: return self._get_client().ping() except Exception: - print('Docker server is not responding!') - print(f'Client is: {self._get_client()}') + self._logger.warning('Docker server is not responding!') + self._logger.info(f'Client is: {self._get_client()}') return False # PRIVATE METHODS @@ -344,19 +346,19 @@ def _confirm_image_exists(self, update: bool = False) -> str: tagged_images = [image.tags[0].rsplit(':')[0] for image in all_images if len(image.tags)] if len(all_images) != len(tagged_images): - print('You have untagged images and may want to remove them:') + self._logger.info('You have untagged images and may want to remove them:') for image in all_images: if len(image.tags) == 0: - print(image.id) + self._logger.info(image.id) if update: - print(f'{IMAGE_NAME} image will be created/updated.') + self._logger.info(f'{IMAGE_NAME} image will be created/updated.') return self._build_image() if IMAGE_NAME not in tagged_images: - print(f'{IMAGE_NAME} image does not exist and will be created') + self._logger.info(f'{IMAGE_NAME} image does not exist and will be created') return self._build_image() - print(f'{IMAGE_NAME} image already exists') + self._logger.info(f'{IMAGE_NAME} image already exists') return self._get_client().images.get(IMAGE_NAME).id[7:] def _build_image(self) -> str: @@ -367,7 +369,7 @@ def _build_image(self) -> str: str: The id of the image or None if the build failed. """ # https://docker-py.readthedocs.io/en/stable/images.html - print(f'Building {IMAGE_NAME} image') + self._logger.info(f'Building {IMAGE_NAME} image') # Find out if an image with the name already exists to remove it afterwards try: @@ -381,13 +383,13 @@ def _build_image(self) -> str: for output in logs: if 'stream' in output: output_str = output['stream'].strip('\r\n').strip('\n') - print(output_str) + self._logger.info(output_str) img = self._get_client().images.get(IMAGE_NAME) except docker.errors.BuildError or docker.errors.APIError as error: - print(f'An error occurred while building the {IMAGE_NAME} image\n{error}') + self._logger.error(f'An error occurred while building the {IMAGE_NAME} image\n{error}') return None if old_img is not None and old_img.id != img.id: - print(f'\nA {IMAGE_NAME} image already exists, it will be overwritten') + self._logger.warning(f'\nA {IMAGE_NAME} image already exists, it will be overwritten') self._get_client().images.remove(old_img.id[7:]) # return id without the 'sha256:'-prefix return img.id[7:] @@ -405,7 +407,7 @@ def _create_container(self, command_id: str, config: dict, use_gpu: bool = True) DockerInfo: A DockerInfo object with id and status set. """ # https://docker-py.readthedocs.io/en/stable/containers.html - print(f'Creating container for command: {command_id}') + self._logger.info(f'Creating container for command: {command_id}') # first update the port mapping in case containers were added/removed without our knowledge self._update_port_mapping() @@ -438,7 +440,7 @@ def _create_container(self, command_id: str, config: dict, use_gpu: bool = True) upload_info = self._upload_config(container.id, command_id, config) if not upload_info.data: - print('Failed to upload configuration file!') + self._logger.warning('Failed to upload configuration file!') return upload_info def _start_container(self, container_id: str) -> DockerInfo: @@ -456,10 +458,10 @@ def _start_container(self, container_id: str) -> DockerInfo: return DockerInfo(id=container_id, status='Container not found.') if container.status == 'running': - print(f'Container is already running: {container_id}') + self._logger.info(f'Container is already running: {container_id}') return DockerInfo(id=container_id, status='running') - print(f'Starting container: {container_id}') + self._logger.info(f'Starting container: {container_id}') try: container.start() # Reload the attributes to get the correct status @@ -484,6 +486,13 @@ def _get_container(self, container_id: str) -> Container: except docker.errors.NotFound: return None + def _get_container_exit_code(self, container: Container) -> str: + try: + exit_code = container.wait()['StatusCode'] + except docker.errors.APIError as error: + exit_code = f'could not get, {error}' + return exit_code + def _stop_container(self, container_id: str) -> DockerInfo: """ Stop a running container. @@ -500,7 +509,7 @@ def _stop_container(self, container_id: str) -> DockerInfo: if not container: return DockerInfo(container_id, status='Container not found.') - print(f'Stopping container: {container_id}') + self._logger.info(f'Stopping container: {container_id}') try: container.stop(timeout=10) # Reload the attributes to get the correct status @@ -525,7 +534,7 @@ def _upload_config(self, container_id: str, command_id: str, config_dict: dict) if not container: return DockerInfo(id=container_id, status='Container not found.') - print('Copying config files into container...') + self._logger.info('Copying config files into container...') # create a directory to store the files safely os.makedirs('config_tmp', exist_ok=True) os.chdir('config_tmp') @@ -557,7 +566,7 @@ def _upload_config(self, container_id: str, command_id: str, config_dict: dict) if upload_ok: os.chdir('..') shutil.rmtree('config_tmp') - print('Copying config files complete.') + self._logger.info('Copying config files complete') return DockerInfo(id=container_id, status=container.status, data=upload_ok) @classmethod @@ -572,10 +581,22 @@ def _update_port_mapping(cls): running_recommerce_containers = list(cls._get_client().containers.list(filters={'label': IMAGE_NAME})) # Get the port mapped to '6006/tcp' within the container occupied_ports = [int(container.ports['6006/tcp'][0]['HostPort']) for container in running_recommerce_containers] - # Create a dictionary of container_id: mapped port cls._port_mapping = dict(zip([container.id for container in running_recommerce_containers], occupied_ports)) + # Create a dictionary of container_id: mapped port if __name__ == '__main__': # pragma: no cover - manager = DockerManager() + import logging + path_to_log_files = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'log_files') + if not os.path.isdir(path_to_log_files): + os.makedirs(path_to_log_files) + + logging.basicConfig(filename=os.path.join(path_to_log_files, 'docker_manager.log'), + filemode='a', + format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', + datefmt='%H:%M:%S', + level=logging.DEBUG) + + docker_manager_logger = logging.getLogger('docker-manager') + manager = DockerManager(docker_manager_logger) print(manager._confirm_image_exists(update=True), '\n') diff --git a/docker/log_api.ini b/docker/log_api.ini new file mode 100644 index 00000000..951e38bf --- /dev/null +++ b/docker/log_api.ini @@ -0,0 +1,21 @@ +[loggers] +keys=root + +[handlers] +keys=logfile + +[formatters] +keys=logfileformatter + +[logger_root] +level=INFO +handlers=logfile + +[formatter_logfileformatter] +format=[%(asctime)s.%(msecs)03d] %(levelname)s [%(thread)d] - %(message)s + +[handler_logfile] +class=handlers.RotatingFileHandler +level=DEBUG +args=('log_files/api.log','a') +formatter=logfileformatter diff --git a/docker/test_docker_manager.py b/docker/test_docker_manager.py index 44659dba..a4fbbd51 100644 --- a/docker/test_docker_manager.py +++ b/docker/test_docker_manager.py @@ -3,6 +3,18 @@ import docker_manager import pytest + +class MockedLogger: + def info(self, logging_text): + return + + def warning(self, logging_text): + return + + def error(self, logging_text): + return + + mock_port_mapping = { '2d680af1e272e0573d44f0adccccf03361a1c4e8db98c540e03ac84f9d9c4e3c': 6006, 'c818251ed4088168b51b5677082c1bdf87200fbdb87bab25a7d2faca30ffac6e': 6008, @@ -10,7 +22,7 @@ } with patch('docker_manager.docker'), \ patch('docker_manager.DockerManager._update_port_mapping'): - manager = docker_manager.DockerManager() + manager = docker_manager.DockerManager(MockedLogger()) # Remember to ALWAYS do: # patch('docker_manager.docker') \ @@ -24,7 +36,7 @@ def setup_function(function): global manager # reset the instance for our tests, as we always want a fresh one docker_manager.DockerManager._instance = None - manager = docker_manager.DockerManager() + manager = docker_manager.DockerManager(MockedLogger()) manager._port_mapping = mock_port_mapping @@ -79,7 +91,7 @@ def test_incorrect_docker_info_initialization(id, status, data, stream, expected def test_docker_manager_is_singleton(): with patch('docker_manager.docker'), \ patch('docker_manager.DockerManager._update_port_mapping'): - manager2 = docker_manager.DockerManager() + manager2 = docker_manager.DockerManager(MockedLogger()) assert manager is manager2 @@ -108,7 +120,7 @@ def test_allowed_commands_is_up_to_date(): def test_start_with_invalid_command(test_config, expected_docker_info_params): expected_docker_info = docker_manager.DockerInfo(**expected_docker_info_params) - actual_docker_info = docker_manager.DockerManager().start(test_config, 2) + actual_docker_info = docker_manager.DockerManager(MockedLogger()).start(test_config, 2) assert expected_docker_info == actual_docker_info @@ -119,7 +131,7 @@ def test_start_but_no_image(): with patch('docker_manager.DockerManager._confirm_image_exists') as image_exists_mock: image_exists_mock.return_value = None - actual_docker_info = docker_manager.DockerManager().start(test_config, 2) + actual_docker_info = docker_manager.DockerManager(MockedLogger()).start(test_config, 2) assert expected_docker_info == actual_docker_info image_exists_mock.assert_called_once() @@ -141,7 +153,7 @@ def test_start_create_container_failed(docker_info_mock_parameter): image_exists_mock.return_value = '12345' create_container_mock.return_value = docker_manager.DockerInfo(**docker_info_mock_parameter) - actual_docker_info = docker_manager.DockerManager().start(test_config, 2) + actual_docker_info = docker_manager.DockerManager(MockedLogger()).start(test_config, 2) assert expected_docker_info == actual_docker_info image_exists_mock.assert_called_once() @@ -158,7 +170,7 @@ def test_start_container_works(): patch('docker_manager.DockerManager._start_container') as start_container_mock: image_exists_mock.return_value = '12345' create_container_mock.return_value = docker_info - actual_docker_infos = docker_manager.DockerManager().start(test_config, 2) + actual_docker_infos = docker_manager.DockerManager(MockedLogger()).start(test_config, 2) assert 2 == len(actual_docker_infos) image_exists_mock.assert_called_once() diff --git a/tests/test_config_validation.py b/tests/test_config_validation.py index c4645556..95777943 100644 --- a/tests/test_config_validation.py +++ b/tests/test_config_validation.py @@ -1,81 +1,81 @@ -import os - -import pytest -import utils_tests as ut_t - -import recommerce.configuration.config_validation as config_validation -from recommerce.configuration.path_manager import PathManager - -env_config_file = os.path.join(PathManager.user_path, 'configuration_files', 'environment_config_training.json') -market_config_file = os.path.join(PathManager.user_path, 'configuration_files', 'market_config.json') -rl_config_file = os.path.join(PathManager.user_path, 'configuration_files', 'q_learning_config.json') - - -config_environment = ut_t.load_json(env_config_file) -config_market = ut_t.load_json(market_config_file) -config_rl = ut_t.load_json(rl_config_file) - -config_environment['config_type'] = 'environment' -config_market['config_type'] = 'sim_market' -config_rl['config_type'] = 'rl' - - -def setup_function(function): - print('***SETUP***') - global config_environment - global config_market - global config_rl - - config_environment['config_type'] = 'environment' - config_market['config_type'] = 'sim_market' - config_rl['config_type'] = 'rl' - - -test_valid_config_validation_complete_testcases = [ - config_environment, - config_market, - config_rl -] - - -@pytest.mark.parametrize('config', test_valid_config_validation_complete_testcases) -def test_valid_config_validation_complete(config): - config_type = config['config_type'] - success, result = config_validation.validate_config(config) - assert success, result - assert result == ({config_type: config}, None, None) - - -test_valid_config_validation_incomplete_testcases = [ - (config_environment, 'agents'), - (config_market, 'max_price'), - (config_rl, 'learning_rate') -] - - -@pytest.mark.parametrize('config, removed_key', test_valid_config_validation_incomplete_testcases) -def test_valid_config_validation_incomplete(config, removed_key): - # Hacky, thx pytest! - tested_config = config.copy() - tested_config = ut_t.remove_key(removed_key, tested_config) - config_type = tested_config['config_type'] - success, result = config_validation.validate_config(tested_config) - assert success - assert result == ({config_type: tested_config}, None, None) - - -def test_validation_strips_redundant_keys(): - expected_config = { - 'hyperparameter': { - 'sim_market': config_market.copy(), - 'rl': config_rl.copy()}, - 'environment': config_environment.copy() - } - test_config = expected_config.copy() - test_config['hyperparameter']['rl']['test_key'] = 123 - status, new_config = config_validation.validate_config(test_config) - assert status, 'This is not valid' - - assert expected_config['hyperparameter']['rl'] == new_config[0]['rl'] - assert expected_config['hyperparameter']['sim_market'] == new_config[1]['sim_market'] - assert expected_config['environment'] == new_config[2]['environment'] +import os + +import pytest +import utils_tests as ut_t + +import recommerce.configuration.config_validation as config_validation +from recommerce.configuration.path_manager import PathManager + +env_config_file = os.path.join(PathManager.user_path, 'configuration_files', 'environment_config_training.json') +market_config_file = os.path.join(PathManager.user_path, 'configuration_files', 'market_config.json') +rl_config_file = os.path.join(PathManager.user_path, 'configuration_files', 'q_learning_config.json') + + +config_environment = ut_t.load_json(env_config_file) +config_market = ut_t.load_json(market_config_file) +config_rl = ut_t.load_json(rl_config_file) + +config_environment['config_type'] = 'environment' +config_market['config_type'] = 'sim_market' +config_rl['config_type'] = 'rl' + + +def setup_function(function): + print('***SETUP***') + global config_environment + global config_market + global config_rl + + config_environment['config_type'] = 'environment' + config_market['config_type'] = 'sim_market' + config_rl['config_type'] = 'rl' + + +test_valid_config_validation_complete_testcases = [ + config_environment, + config_market, + config_rl +] + + +@pytest.mark.parametrize('config', test_valid_config_validation_complete_testcases) +def test_valid_config_validation_complete(config): + config_type = config['config_type'] + success, result = config_validation.validate_config(config) + assert success, result + assert result == ({config_type: config}, None, None) + + +test_valid_config_validation_incomplete_testcases = [ + (config_environment, 'agents'), + (config_market, 'max_price'), + (config_rl, 'learning_rate') +] + + +@pytest.mark.parametrize('config, removed_key', test_valid_config_validation_incomplete_testcases) +def test_valid_config_validation_incomplete(config, removed_key): + # Hacky, thx pytest! + tested_config = config.copy() + tested_config = ut_t.remove_key(removed_key, tested_config) + config_type = tested_config['config_type'] + success, result = config_validation.validate_config(tested_config) + assert success + assert result == ({config_type: tested_config}, None, None) + + +def test_validation_strips_redundant_keys(): + expected_config = { + 'hyperparameter': { + 'sim_market': config_market.copy(), + 'rl': config_rl.copy()}, + 'environment': config_environment.copy() + } + test_config = expected_config.copy() + test_config['hyperparameter']['rl']['test_key'] = 123 + status, new_config = config_validation.validate_config(test_config) + assert status, 'This is not valid' + + assert expected_config['hyperparameter']['rl'] == new_config[0]['rl'] + assert expected_config['hyperparameter']['sim_market'] == new_config[1]['sim_market'] + assert expected_config['environment'] == new_config[2]['environment'] diff --git a/webserver/alpha_business_app/migrations/0010_alter_environmentconfig_marketplace.py b/webserver/alpha_business_app/migrations/0010_alter_environmentconfig_marketplace.py index 6ca3f3b2..21e644dc 100644 --- a/webserver/alpha_business_app/migrations/0010_alter_environmentconfig_marketplace.py +++ b/webserver/alpha_business_app/migrations/0010_alter_environmentconfig_marketplace.py @@ -1,18 +1,18 @@ -# Generated by Django 4.0.1 on 2022-04-01 08:18 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('alpha_business_app', '0009_rename_config_file_container_config_and_more'), - ] - - operations = [ - migrations.AlterField( - model_name='environmentconfig', - name='marketplace', - field=models.CharField(max_length=150, null=True), - ), - ] +# Generated by Django 4.0.1 on 2022-04-01 08:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('alpha_business_app', '0009_rename_config_file_container_config_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='environmentconfig', + name='marketplace', + field=models.CharField(max_length=150, null=True), + ), + ] diff --git a/webserver/alpha_business_app/migrations/0011_alter_agentconfig_argument_and_more.py b/webserver/alpha_business_app/migrations/0011_alter_agentconfig_argument_and_more.py index 258d75d8..b23e140c 100644 --- a/webserver/alpha_business_app/migrations/0011_alter_agentconfig_argument_and_more.py +++ b/webserver/alpha_business_app/migrations/0011_alter_agentconfig_argument_and_more.py @@ -1,23 +1,23 @@ -# Generated by Django 4.0.1 on 2022-04-06 08:12 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('alpha_business_app', '0010_alter_environmentconfig_marketplace'), - ] - - operations = [ - migrations.AlterField( - model_name='agentconfig', - name='argument', - field=models.CharField(default='', max_length=200), - ), - migrations.AlterField( - model_name='environmentconfig', - name='task', - field=models.CharField(choices=[(1, 'training'), (2, 'agent_monitoring'), (3, 'exampleprinter')], max_length=14, null=True), - ), - ] +# Generated by Django 4.0.1 on 2022-04-06 08:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('alpha_business_app', '0010_alter_environmentconfig_marketplace'), + ] + + operations = [ + migrations.AlterField( + model_name='agentconfig', + name='argument', + field=models.CharField(default='', max_length=200), + ), + migrations.AlterField( + model_name='environmentconfig', + name='task', + field=models.CharField(choices=[(1, 'training'), (2, 'agent_monitoring'), (3, 'exampleprinter')], max_length=14, null=True), + ), + ] diff --git a/webserver/alpha_business_app/migrations/0014_merge_20220609_1023.py b/webserver/alpha_business_app/migrations/0014_merge_20220609_1023.py index a969d3e7..4c8ab68d 100644 --- a/webserver/alpha_business_app/migrations/0014_merge_20220609_1023.py +++ b/webserver/alpha_business_app/migrations/0014_merge_20220609_1023.py @@ -1,14 +1,14 @@ -# Generated by Django 4.0.1 on 2022-06-09 08:23 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('alpha_business_app', '0013_rename_enable_live_draw_environmentconfig_separate_markets'), - ('alpha_business_app', '0013_rlconfig_stable_baseline_test_rlconfig_testvalue2_and_more'), - ] - - operations = [ - ] +# Generated by Django 4.0.1 on 2022-06-09 08:23 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('alpha_business_app', '0013_rename_enable_live_draw_environmentconfig_separate_markets'), + ('alpha_business_app', '0013_rlconfig_stable_baseline_test_rlconfig_testvalue2_and_more'), + ] + + operations = [ + ]