Merge branch '484-configuration-remove-combined-config' of https://gi…

…thub.com/hpi-epic/BP2021 into 484-configuration-remove-combined-config

webserver/alpha_business_app/buttons.py

@@ -310,7 +310,7 @@ def _start(self) -> HttpResponse:
 
 		config_dict = ConfigFlatDictParser().flat_dict_to_hierarchical_config_dict(post_request)
 
-		validate_status, validate_data = validate_config(config=config_dict, config_is_final=True)
+		validate_status, validate_data = validate_config(config=config_dict.copy(), config_is_final=True)
 		if not validate_status:
 			self.message = ['error', validate_data]
 			return self._decide_rendering()

webserver/alpha_business_app/config_merger.py

@@ -5,7 +5,7 @@ class ConfigMerger():
 	def __init__(self) -> None:
 		self.error_dict = Config.get_empty_structure_dict()
 
-	def merge_config_objects(self, config_object_ids: list) -> tuple:
+	def merge_config_objects(self, config_object_ids: list, base_config: dict) -> tuple:
 		"""
 		merge a list of config objects given by their id.
 
@@ -17,10 +17,8 @@ def merge_config_objects(self, config_object_ids: list) -> tuple:
 		"""
 		configuration_objects = [Config.objects.get(id=config_id) for config_id in config_object_ids]
 		configuration_dicts = [config.as_dict() for config in configuration_objects]
-		# for c in configuration_dicts:
-		# 	print(c)
-		# print('--------------------------------------')
-		# get initial empty dict to merge into
+
+		# initial empty dict to merge into
 		final_config = Config.get_empty_structure_dict()
 		for config in configuration_dicts:
 			final_config = self._merge_config_into_base_config(final_config, config)

webserver/alpha_business_app/handle_files.py

@@ -1,4 +1,5 @@
 import json
+import re
 import tarfile
 import zipfile
 from io import BytesIO
@@ -60,30 +61,24 @@ def handle_uploaded_file(request, uploaded_config) -> HttpResponse:
 	validate_status, validate_data = validate_config(content_as_dict, False)
 	if not validate_status:
 		return render(request, 'upload.html', {'error': validate_data})
-	config = validate_data
-	assert len(config.keys()) == 1, f'This config ({config} as multiple keys, should only be one ("environment", "rl" or "sim_market"))'
 
-	# recommerce returns either {'environment': {}}, {'rl': {}} or {'sim_markte': {}}}
-	# for parsing we need to know what the toplevel key is
-	top_level = config.keys()[0]
-	if top_level != 'environment':
-		top_level = 'hyperparameter'
+	# configs and their corresponding top level keys as list
+	config_objects = _get_top_level_and_configs(validate_data)
 
 	# parse config model to datastructure
 	parser = ConfigModelParser()
-	try:
-		resulting_config_part = parser.parse_config_dict_to_datastructure(top_level, config)
-	except ValueError as e:
-		return render(request, 'upload.html', {'error': f'Your config is wrong {e}'})
+	resulting_config_parts = []
+	for top_level, config in config_objects:
+		try:
+			resulting_config_parts += [(top_level, parser.parse_config_dict_to_datastructure(top_level, config))]
+		except ValueError:
+			return render(request, 'upload.html', {'error': 'Your config is wrong'})
+		except TypeError as error:
+			invalid_keyword_search = re.search('.*keyword argument (.*)', str(error))
+			return render(request, 'upload.html', {'error': f'Your config contains an invalid key: {invalid_keyword_search.group(1)}'})
 
 	# Make it a real config object
-	environment_config = None
-	hyperparameter_config = None
-	if top_level == 'environment':
-		environment_config = resulting_config_part
-	else:
-		hyperparameter_config = resulting_config_part
-
+	environment_config, hyperparameter_config = _get_config_parts(resulting_config_parts)
 	given_name = request.POST['config_name']
 	config_name = given_name if given_name else uploaded_config.name
 	Config.objects.create(environment=environment_config, hyperparameter=hyperparameter_config, name=config_name, user=request.user)
@@ -176,3 +171,62 @@ def _convert_tar_file_to_zip(fake_tar_archive: BytesIO) -> BytesIO:
 	tar_archive.close()
 
 	return file_like_zip
+
+
+def _get_top_level_and_configs(validate_data: tuple) -> list:
+	"""
+	Prepares data returned by the recommerce validation function for parsing.
+	Should only be used when the validated config was correct
+
+	Args:
+		validate_data (tuple): return of recommerce validation function, when config was correct
+
+	Returns:
+		list: of tuples, the first tuple value indecates the top level ('hyperparameter' or 'environment')
+			and the second value is the corresponding config.
+			Length will be between 1 and 2.
+	"""
+	assert tuple == type(validate_data), \
+		f'Data returned by "vaidate_config" for correct config should be tuple, but was {validate_data}'
+	result = []
+	for config in validate_data:
+		if not config:
+			continue
+		if 'environment' in config:
+			result += [('environment', config['environment'])]
+		elif 'hyperparameter' in config:
+			result += [('hyperparameter', config['hyperparameter'])]
+		elif 'rl' in config or 'sim_market' in config:
+			# we need to add those two to the same hyperparameter name
+			existing_hyperparameter = [item for item in result if 'hyperparameter' in item]
+			if existing_hyperparameter:
+				new_hyperparameter = ('hyperparameter', {**existing_hyperparameter[0][1], **config})
+				result.remove(existing_hyperparameter[0])
+				result += [new_hyperparameter]
+			else:
+				result += [('hyperparameter', config)]
+	return result
+
+
+def _get_config_parts(config_objects: list) -> tuple:
+	"""
+	Takes a list of tuple with the parsed objects from the config and their top level key
+	and returns an 'environment_config' and a 'hyperparameter_config' object to be inserted into the Config object
+
+	Args:
+		config_objects (list): list of tuples, first tuple value indecating top-level key ('hyperparameter' / 'environment')
+			second value, the actual parsed config object
+
+	Returns:
+		tuple: (instance of EnvironmentConfig, instance of HyperparameterConfig)
+	"""
+	assert len(config_objects) <= 2 and len(config_objects) >= 1, \
+		'At least one, at max two config parts should have been parsed'
+	environment_config = None
+	hyperparameter_config = None
+	for top_level, config_part in config_objects:
+		if top_level == 'environment':
+			environment_config = config_part
+		elif top_level == 'hyperparameter':
+			hyperparameter_config = config_part
+	return environment_config, hyperparameter_config

webserver/alpha_business_app/migrations/0013_rlconfig_stable_baseline_test_rlconfig_testvalue2_and_more.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.0.1 on 2022-06-01 18:36
+# Generated by Django 4.0.1 on 2022-06-02 19:39
 
 from django.db import migrations, models
 
@@ -65,4 +65,39 @@ class Migration(migrations.Migration):
             name='sync_target_frames',
             field=models.IntegerField(default=None, null=True),
         ),
+        migrations.AlterField(
+            model_name='simmarketconfig',
+            name='episode_length',
+            field=models.IntegerField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='simmarketconfig',
+            name='max_price',
+            field=models.IntegerField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='simmarketconfig',
+            name='max_quality',
+            field=models.IntegerField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='simmarketconfig',
+            name='max_storage',
+            field=models.IntegerField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='simmarketconfig',
+            name='number_of_customers',
+            field=models.IntegerField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='simmarketconfig',
+            name='production_price',
+            field=models.IntegerField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='simmarketconfig',
+            name='storage_cost_per_product',
+            field=models.FloatField(default=None, null=True),
+        ),
     ]

webserver/alpha_business_app/models/rl_config.py

@@ -4,14 +4,14 @@
 
 
 class RlConfig(AbstractConfig, models.Model):
-	stable_baseline_test = models.FloatField(null=True, default=None)
 	epsilon_decay_last_frame = models.IntegerField(null=True, default=None)
 	epsilon_start = models.FloatField(null=True, default=None)
 	sync_target_frames = models.IntegerField(null=True, default=None)
+	replay_size = models.IntegerField(null=True, default=None)
 	replay_start_size = models.IntegerField(null=True, default=None)
-	testvalue2 = models.FloatField(null=True, default=None)
 	gamma = models.FloatField(null=True, default=None)
-	epsilon_final = models.FloatField(null=True, default=None)
-	replay_size = models.IntegerField(null=True, default=None)
 	batch_size = models.IntegerField(null=True, default=None)
+	epsilon_final = models.FloatField(null=True, default=None)
+	stable_baseline_test = models.FloatField(null=True, default=None)
 	learning_rate = models.FloatField(null=True, default=None)
+	testvalue2 = models.FloatField(null=True, default=None)

webserver/alpha_business_app/models/sim_market_config.py

@@ -4,10 +4,10 @@
 
 
 class SimMarketConfig(AbstractConfig, models.Model):
-	max_storage = models.IntegerField(null=True)
-	episode_length = models.IntegerField(null=True)
-	max_price = models.IntegerField(null=True)
-	max_quality = models.IntegerField(null=True)
-	number_of_customers = models.IntegerField(null=True)
-	production_price = models.IntegerField(null=True)
-	storage_cost_per_product = models.FloatField(null=True)
+	max_storage = models.IntegerField(null=True, default=None)
+	storage_cost_per_product = models.FloatField(null=True, default=None)
+	episode_length = models.IntegerField(null=True, default=None)
+	max_price = models.IntegerField(null=True, default=None)
+	number_of_customers = models.IntegerField(null=True, default=None)
+	max_quality = models.IntegerField(null=True, default=None)
+	production_price = models.IntegerField(null=True, default=None)

webserver/alpha_business_app/on_recommerce_change.py

@@ -13,23 +13,40 @@ def __init__(self, top_level: str, second_level: str = None) -> None:
 		self.whitespace = '\t'
 		self.top_level = top_level
 		self.second_level = second_level
-		# self.structure_dict = get_structure_of(top_level, second_level)
 		self.name = second_level if second_level else top_level
+		self.class_name = to_config_class_name(self.name)
 
 	def write_file(self) -> None:
+		print(f'{self._warning()}WARNING: This action will override the {self.class_name} file.{self._end()}')
+		print('Press enter to continue')
+		input()
 		# imports
 		lines = ['from django.db import models', '', 'from .abstract_config import AbstractConfig', '']
 		# class definition
-		lines += [f'class {to_config_class_name(self.name)}(AbstractConfig, models.Model):']
+		lines += [f'class {self.class_name}(AbstractConfig, models.Model):']
 		# fields
 		attributes = get_structure_with_types_of(self.top_level, self.second_level)
 		for attr in attributes:
-			django_class = str(attr[1]).rsplit('.')[-1][:-1]
-			lines += [f'{self.whitespace}{attr[0]} = models.{django_class}(null=True, default=None)']
+			django_class = str(attr[1]).rsplit('.')[-1][:-2]
+			additional_attributes = self._get_additional_attributes(django_class)
+			lines += [f'{self.whitespace}{attr[0]} = models.{django_class}(null=True, default=None{additional_attributes})']
 		path_to_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'models', f'{self.name}_config.py')
 		# write to file
+		print(f'Writing class definition of {self.class_name} to file.')
 		with open(path_to_file, 'w') as config_file:
 			config_file.write('\n'.join(lines))
 
+	def _get_additional_attributes(self, django_class: str) -> str:
+		if 'CharField' in django_class:
+			return ', max_length=100'
+		return ''
+
+	def _warning(self) -> str:
+		return '\033[93m'
+
+	def _end(self) -> str:
+		return '\033[0m'
+
 
 ConfigModelWriter(top_level='hyperparameter', second_level='rl').write_file()
+ConfigModelWriter(top_level='hyperparameter', second_level='sim_market').write_file()

webserver/alpha_business_app/static/js/custom.js

@@ -11,6 +11,10 @@ $(document).ready(function() {
 		});
 	};
 	addEventToAddMoreButton()
+
+	function getFormData () {
+		return $("form.config-form");
+	};
 
 	function updateAPIHealth() {
 		// replaces the element by the element returned by ajax (html) and adds this click event to it
@@ -65,8 +69,7 @@ $(document).ready(function() {
 		$("select.agent-agent-class").change(function () {
 			// will be called when agent dropdown has changed, we need to change rl hyperparameter for that
 			var self = $(this);
-			var form = $("form.config-form");
-			var formdata = form.serializeArray();
+			var formdata = getFormData()
 			const csrftoken = getCookie("csrftoken");
 			$.ajax({
 				type: "POST",
@@ -104,8 +107,7 @@ $(document).ready(function() {
 		$("table.config-status-display").remove();
 
 		var self = $(this);
-		var form = $("form.config-form");
-		var formdata = form.serializeArray();
+		var formdata = getFormData();
 
 		const csrftoken = getCookie('csrftoken');
 		$.ajax({

webserver/alpha_business_app/tests/test_data/test_config_complete.json

@@ -0,0 +1,34 @@
+{
+	"environment": {
+		"task": "training", 
+		"marketplace": "recommerce.market.circular.circular_sim_market.CircularEconomyRebuyPriceMonopoly", 
+		"agents": [
+			{
+				"name": "QLearning Agent", 
+				"agent_class": "recommerce.rl.q_learning.q_learning_agent.QLearningAgent", 
+				"argument": ""
+			}
+		]
+	}, "hyperparameter": {
+		"rl": {
+			"gamma": 0.99, 
+			"batch_size": 32, 
+			"replay_size": 100000, 
+			"learning_rate": 1e-06, 
+			"sync_target_frames": 1000, 
+			"replay_start_size": 10000, 
+			"epsilon_decay_last_frame": 75000, 
+			"epsilon_start": 1.0, 
+			"epsilon_final": 0.1
+		}, 
+		"sim_market": {
+			"max_storage": 100, 
+			"episode_length": 50, 
+			"max_price": 10, 
+			"max_quality": 50, 
+			"number_of_customers": 20, 
+			"production_price": 3, 
+			"storage_cost_per_product": 0.1
+		}
+	}
+}

webserver/alpha_business_app/tests/test_data/test_sim_market_config.json

@@ -1,11 +1,10 @@
 {
-	"class": "recommerce.market.circular.circular_sim_market.CircularEconomyRebuyPriceMonopoly",
 	"max_storage": 100,
 	"episode_length": 50,
 	"max_price": 10,
 	"max_quality": 50,
 	"number_of_customers": 20,
 	"production_price": 3,
 	"storage_cost_per_product": 0.1,
-	"config_type": "market"
+	"config_type": "sim_market"
 }