#451 serialization robustness tweaks

MetaCell · Mar 4, 2022 · 508ea8d · 508ea8d
1 parent 443086a
commit 508ea8d
Show file tree

Hide file tree

Showing 11 changed files with 748 additions and 237 deletions.
diff --git a/applications/samples/backend/samples/openapi/openapi.yaml b/applications/samples/backend/samples/openapi/openapi.yaml
diff --git a/libraries/cloudharness-common/cloudharness/__init__.py b/libraries/cloudharness-common/cloudharness/__init__.py
@@ -1,8 +1,6 @@
 import logging
 import sys
 
-from cloudharness_model.models.base_model_ import Model
-
 log = logging
 
 from cloudharness_model.encoder import CloudHarnessJSONEncoder
@@ -20,9 +18,13 @@ def set_debug():
 json_dumps = js.dumps
 
 def dumps(o, *args, **kwargs):
-    if isinstance(o, Model):
-       return json_dumps(o.to_dict(), *args, **kwargs)
-    return json_dumps(o, *args, **kwargs)
+    try:
+        if  "cls" not in kwargs:
+            return json_dumps(o, cls=CloudHarnessJSONEncoder, *args, **kwargs)
+        return json_dumps(o, *args, **kwargs)
+    except:
+        logging.error(repr(o))
+        raise
 
 json = js
 json.dumps = dumps

diff --git a/libraries/cloudharness-common/cloudharness/applications.py b/libraries/cloudharness-common/cloudharness/applications.py
@@ -10,9 +10,6 @@ class ConfigurationCallException(Exception):
 
 class ApplicationConfiguration(ApplicationConfig):
 
-    @property
-    def name(self):
-        return self.harness.name
 
     def is_auto_service(self) -> bool:
         return self.harness.service.auto

diff --git a/libraries/cloudharness-common/cloudharness/events/client.py b/libraries/cloudharness-common/cloudharness/events/client.py
@@ -6,7 +6,7 @@
 import logging
 
 from time import sleep
-from cloudharness import json
+from cloudharness import json, dumps
 
 from cloudharness_model.util import DeserializationException
 from keycloak.exceptions import KeycloakGetError
@@ -86,7 +86,7 @@ def produce(self, message: dict):
                 True if the message was published correctly, False otherwise.
         '''
         producer = KafkaProducer(bootstrap_servers=self._get_bootstrap_servers(),
-                                 value_serializer=lambda x: json.dumps(x).encode('utf-8'))
+                                 value_serializer=lambda x: dumps(x).encode('utf-8'))
         try:
             return producer.send(self.topic_id, value=message)
         except KafkaTimeoutError as e:
@@ -157,15 +157,15 @@ def send_event(message_type, operation, obj, uid="id", func_name=None, func_args
             for kwa, kwa_val in func_kwargs.items():
                 try:
                     fkwargs.append({
-                        kwa: json.loads(json.dumps(kwa_val))
+                        kwa: json.loads(dumps(kwa_val))
                     })
                 except Exception as e:
                     # keyword argument can't be serialized
                     pass
 
             # send the message
             ec.produce(
-                {
+                CDCEvent.from_dict({
                     "meta": {
                         "app_name": CURRENT_APP_NAME,
                         "user": user,
@@ -178,7 +178,7 @@ def send_event(message_type, operation, obj, uid="id", func_name=None, func_args
                     "operation": operation,
                     "uid": resource_id,
                     "resource": resource
-                }
+                })
             )
             log.info(f"sent cdc event {message_type} - {operation} - {resource_id}")
         except Exception as e:

diff --git a/libraries/cloudharness-common/tests/test_applications.py b/libraries/cloudharness-common/tests/test_applications.py
@@ -53,6 +53,9 @@ def test_application_conf():
     assert uut.is_auto_deployment()
     assert uut.is_sentry_enabled()
 
+    d2 = {'admin': {'pass': 'metacell', 'role': 'administrator', 'user': 'admin'}, 'client': {'id': 'rest-client', 'secret': '5678eb6e-9e2c-4ee5-bd54-34e7411339e8'}, 'enabled': True, 'harness': {'aliases': [], 'database': {'auto': True, 'mongo': {'image': 'mongo:5', 'ports': [{'name': 'http', 'port': 27017}]}, 'name': 'keycloak-postgres', 'neo4j': {'dbms_security_auth_enabled': 'false', 'image': 'neo4j:4.1.9', 'memory': {'heap': {'initial': '64M', 'max': '128M'}, 'pagecache': {'size': '64M'}, 'size': '256M'}, 'ports': [{'name': 'http', 'port': 7474}, {'name': 'bolt', 'port': 7687}]}, 'pass': 'password', 'postgres': {'image': 'postgres:10.4', 'initialdb': 'auth_db', 'ports': [{'name': 'http', 'port': 5432}]}, 'resources': {'limits': {'cpu': '1000m', 'memory': '2Gi'}, 'requests': {'cpu': '100m', 'memory': '512Mi'}}, 'size': '2Gi', 'type': 'postgres', 'user': 'user'}, 'dependencies': {'build': [], 'hard': [], 'soft': []}, 'deployment': {'auto': True, 'image': 'osb/accounts:3e02a15477b4696ed554e08cedf4109c67908cbe6b03331072b5b73e83b4fc2b', 'name': 'accounts', 'port': 8080, 'replicas': 1, 'resources': {'limits': {'cpu': '500m', 'memory': '1024Mi'}, 'requests': {'cpu': '10m', 'memory': '512Mi'}}}, 'domain': None, 'env': [{'name': 'KEYCLOAK_IMPORT', 'value': '/tmp/realm.json'}, {'name': 'KEYCLOAK_USER', 'value': 'admin'}, {'name': 'KEYCLOAK_PASSWORD', 'value': 'metacell'}, {'name': 'PROXY_ADDRESS_FORWARDING', 'value': 'true'}, {'name': 'DB_VENDOR', 'value': 'POSTGRES'}, {'name': 'DB_ADDR', 'value': 'keycloak-postgres'}, {'name': 'DB_DATABASE', 'value': 'auth_db'}, {'name': 'DB_USER', 'value': 'user'}, {'name': 'DB_PASSWORD', 'value': 'password'}, {'name': 'JAVA_OPTS', 'value': '-server -Xms64m -Xmx896m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true  --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED --add-exports=jdk.unsupported/sun.reflect=ALL-UNNAMED'}], 'name': 'accounts', 'readinessProbe': {'path': '/auth/realms/master'}, 'resources': [{'dst': '/tmp/realm.json', 'name': 'realm-config', 'src': 'realm.json'}], 'secrets': '', 'secured': False, 'service': {'auto': True, 'name': 'accounts', 'port': 8080}, 'subdomain': 'accounts', 'uri_role_mapping': [{'roles': ['administrator'], 'uri': '/*'}], 'use_services': []}, 'harvest': True, 'image': 'osb/accounts:latest', 'name': 'accounts', 'port': 8080, 'resources': {'limits': {'cpu': '500m', 'memory': '1024Mi'}, 'requests': {'cpu': '10m', 'memory': '512Mi'}}, 'task-images': {}, 'webclient': {'id': 'web-client', 'secret': '452952ae-922c-4766-b912-7b106271e34b'}}
+    ApplicationConfiguration.from_dict(d2)
+
 def test_get_configuration():
     CloudharnessConfig.apps = {
         'a': conf_1,

diff --git a/libraries/models/.openapi-generator-ignore b/libraries/models/.openapi-generator-ignore
@@ -34,4 +34,5 @@ encoder.py
 *.sh
 */util.py
 */models/base_model_.py
-requirements.txt
+requirements.txt
+*/encoder.py
diff --git a/libraries/models/cloudharness_model/encoder.py b/libraries/models/cloudharness_model/encoder.py
@@ -8,6 +8,6 @@ class CloudHarnessJSONEncoder(JSONEncoder):
     include_nulls = False
 
     def default(self, o):
-        if isinstance(o, Model):
-            JSONEncoder.default(self, o.to_dict())
-        return JSONEncoder.default(self, o)
+        if hasattr(o, "to_dict"):
+            return o.to_dict()
+        return JSONEncoder.default(self, o)
diff --git a/libraries/models/cloudharness_model/util.py b/libraries/models/cloudharness_model/util.py
@@ -25,8 +25,12 @@ def _deserialize(data, klass):
         return deserialize_datetime(data)
     elif typing_utils.is_generic(klass):
         if typing_utils.is_list(klass):
+            if isinstance(klass.__args__[0], typing.TypeVar):
+                return data
             return _deserialize_list(data, klass.__args__[0])
         if typing_utils.is_dict(klass):
+            if isinstance(klass.__args__[1], typing.TypeVar):
+                return data
             return _deserialize_dict(data, klass.__args__[1])
     else:
         return deserialize_model(data, klass)
@@ -90,7 +94,9 @@ def deserialize_datetime(string):
         return string
 
 
-class DeserializationException(Exception): pass
+class DeserializationException(Exception):
+    pass
+
 
 def deserialize_model(data, klass):
     """Deserializes list or dict to model.
@@ -99,10 +105,16 @@ def deserialize_model(data, klass):
     :param klass: class literal.
     :return: model object.
     """
-    instance = klass()
+
+    if isinstance(data, klass) or not hasattr(klass, "__call__"):
+        return data
+    try:
+        instance = klass()
+    except:
+        raise
     instance._raw_dict = data
 
-    if not instance.openapi_types:
+    if not instance.openapi_types or isinstance(data, klass):
         return data
     if data is None:
         return instance
@@ -114,20 +126,29 @@ def deserialize_model(data, klass):
                     setattr(instance, attr, _deserialize(value, attr_type))
 
         elif isinstance(data, dict):
-            
+
             for attr in data:
                 value = data[attr]
-                try:
-                    if attr in instance.attribute_map:
+
+                if attr in instance.attribute_map:
+                    try:
                         setattr(instance, attr, _deserialize(value, instance.openapi_types[attr]))
-                    else:
+                    except:
+                        logging.warning(
+                            "Deserialization error: could not set attribute `%s` to value `%s` in class `%s`.", attr, value, klass.__name__)
                         setattr(instance, attr, value)
-                except AttributeError as e:
-                    logging.warning("Deserialization error: could not set attribute `%s` to value `%s` in class `%s`.", attr, value, klass.__name__, exc_info=True)
-                    logging.debug("Instance is %s", instance)
+                        logging.debug("Instance is %s", instance, exc_info=True)
+                else:
+                    try:
+                        setattr(instance, attr, value)
+                    except:
+                        logging.warning(
+                            "Deserialization error: could not set attribute `%s` to value `%s` in class `%s`.", attr, value, klass.__name__)
+                        logging.debug("Instance is %s", instance, exc_info=True)
     except Exception as e:
-        raise DeserializationException(f"Cannot convert data to class {klass.__name__}. Data is\n{repr(data)}") from e
-
+        logging.error("Deserialize error", exc_info=True)
+        raise DeserializationException(
+            f"Cannot convert data to class {klass.__name__}. Data is\n{repr(data)}") from e
 
     return instance
 
@@ -153,4 +174,4 @@ def _deserialize_dict(data, boxed_type):
     :rtype: dict
     """
     return {k: _deserialize(v, boxed_type)
-            for k, v in six.iteritems(data)}
+            for k, v in six.iteritems(data)}
diff --git a/libraries/models/test/test_deserialize.py b/libraries/models/test/test_deserialize.py
@@ -0,0 +1,61 @@
+import pytest
+from os.path import join, dirname as dn, realpath
+import yaml
+
+from cloudharness_model import HarnessMainConfig, ApplicationConfig, User, ApplicationHarnessConfig, CDCEvent
+
+HERE = dn(realpath(__file__))
+
+
+def test_helm_values_deserialize():
+    with open(join(HERE, "resources/values.yaml")) as f:
+        values = yaml.load(f)
+    v = HarnessMainConfig.from_dict(values)
+
+    assert v.domain
+    assert v.apps["accounts"].harness.deployment.name == "accounts"
+
+    app = ApplicationConfig.from_dict(values["apps"]["accounts"])
+    assert app.harness.deployment.name == "accounts"
+
+    assert v.apps["accounts"].webclient.get('id')
+
+    u = User(last_name="a")
+    assert u.last_name == "a"
+    assert u["last_name"] == "a"
+
+
+def test_robustness():
+    d = {'aliases': [], 'database': {'auto': True, 'mongo': {'image': 'mongo:5', 'ports': [{'name': 'http', 'port': 27017}]}, 'name': 'keycloak-postgres', 'neo4j': {'dbms_security_auth_enabled': 'false', 'image': 'neo4j:4.1.9', 'memory': {'heap': {'initial': '64M', 'max': '128M'}, 'pagecache': {'size': '64M'}, 'size': '256M'}, 'ports': [{'name': 'http', 'port': 7474}, {'name': 'bolt', 'port': 7687}]}, 'pass': 'password', 'postgres': {'image': 'postgres:10.4', 'initialdb': 'auth_db', 'ports': [{'name': 'http', 'port': 5432}]}, 'resources': {'limits': {'cpu': '1000m', 'memory': '2Gi'}, 'requests': {'cpu': '100m', 'memory': '512Mi'}}, 'size': '2Gi', 'type': 'postgres', 'user': 'user'}, 'dependencies': {'build': [], 'hard': [], 'soft': []}, 'deployment': {'auto': True, 'image': 'osb/accounts:3e02a15477b4696ed554e08cedf4109c67908cbe6b03331072b5b73e83b4fc2b', 'name': 'accounts', 'port': 8080, 'replicas': 1, 'resources': {'limits': {'cpu': '500m', 'memory': '1024Mi'}, 'requests': {'cpu': '10m', 'memory': '512Mi'}}}, 'domain': None, 'env': [{'name': 'KEYCLOAK_IMPORT', 'value': '/tmp/realm.json'},
+            {'name': 'KEYCLOAK_USER', 'value': 'admin'}, {'name': 'KEYCLOAK_PASSWORD', 'value': 'metacell'}, {'name': 'PROXY_ADDRESS_FORWARDING', 'value': 'true'}, {'name': 'DB_VENDOR', 'value': 'POSTGRES'}, {'name': 'DB_ADDR', 'value': 'keycloak-postgres'}, {'name': 'DB_DATABASE', 'value': 'auth_db'}, {'name': 'DB_USER', 'value': 'user'}, {'name': 'DB_PASSWORD', 'value': 'password'}, {'name': 'JAVA_OPTS', 'value': '-server -Xms64m -Xmx896m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true  --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED --add-exports=jdk.unsupported/sun.reflect=ALL-UNNAMED'}], 'name': 'accounts', 'readinessProbe': {'path': '/auth/realms/master'}, 'resources': [{'dst': '/tmp/realm.json', 'name': 'realm-config', 'src': 'realm.json'}], 'secrets': '', 'secured': False, 'service': {'auto': True, 'name': 'accounts', 'port': 8080}, 'subdomain': 'accounts', 'uri_role_mapping': [{'roles': ['administrator'], 'uri': '/*'}], 'use_services': []}
+
+
+    app = ApplicationHarnessConfig.from_dict(d)
+    assert app.database.auto == True
+
+    cdc = {'meta': {'app_name': 'workspaces', 'user': {'access': {'impersonate': True,          'manage': True,          'manageGroupMembership': True,          'mapRoles': True,          'view': True},
+ 'additional_properties': None,
+ 'attributes': None,
+ 'client_roles': None,
+ 'created_timestamp': None,
+ 'credentials': None,
+ 'disableable_credential_types': None,
+ 'email': '[email protected]',
+ 'email_verified': None,
+ 'enabled': True,
+ 'federation_link': None,
+ 'first_name': None,
+ 'groups': None,
+ 'id': '61a2cc58-d98c-4318-8828-fbdae85041ac',
+ 'last_name': None,
+ 'realm_roles': None,
+ 'required_actions': None,
+ 'service_account_client_id': None,
+ 'username': 'a'}, 'func': '<function WorkspaceService.post at 0x7fecdda2ee60>', 'args': [{'name': 'weerewerw', 'description': 'weerewerw', 'resources': [{'name': 'notebook', 'status': 'p', 'resource_type': 'g', 'workspace_id': -1, 'origin': '{"path": "http://www.osb.local/workspace-data/notebook.ipynb"}'}], 'user_id': '61a2cc58-d98c-4318-8828-fbdae85041ac'}], 'kwargs': [], 'description': 'workspace - 10'}, 'message_type': 'workspace', 'operation': 'create', 'uid': 10, 'resource': {'id': 10, 'name': 'weerewerw', 'description': 'weerewerw', 'timestamp_created': '2022-03-04T17:51:36.267012', 'timestamp_updated': '2022-03-04T17:51:36.267012', 'last_opened_resource_id': None, 'thumbnail': None, 'gallery': [], 'user_id': '61a2cc58-d98c-4318-8828-fbdae85041ac', 'publicable': False, 'featured': False, 'license': None, 'collaborators': [], 'storage': None, 'tags': [], 'resources': [{'id': 11, 'name': 'notebook', 'folder': 'notebook', 'status': 'p', 'timestamp_created': None, 'timestamp_updated': None, 'timestamp_last_opened': None, 'resource_type': 'g', 'origin': '{"path": "http://www.osb.local/workspace-data/notebook.ipynb"}', 'workspace_id': 10}]}}
+
+    e = CDCEvent.from_dict(cdc)
+
+
+    app = {'admin': {'pass': 'metacell', 'role': 'administrator', 'user': 'admin'}, 'client': {'id': 'rest-client', 'secret': '5678eb6e-9e2c-4ee5-bd54-34e7411339e8'}, 'enabled': True, 'harness': {'aliases': [], 'database': {'auto': True, 'mongo': {'image': 'mongo:5', 'ports': [{'name': 'http', 'port': 27017}]}, 'name': 'keycloak-postgres', 'neo4j': {'dbms_security_auth_enabled': 'false', 'image': 'neo4j:4.1.9', 'memory': {'heap': {'initial': '64M', 'max': '128M'}, 'pagecache': {'size': '64M'}, 'size': '256M'}, 'ports': [{'name': 'http', 'port': 7474}, {'name': 'bolt', 'port': 7687}]}, 'pass': 'password', 'postgres': {'image': 'postgres:10.4', 'initialdb': 'auth_db', 'ports': [{'name': 'http', 'port': 5432}]}, 'resources': {'limits': {'cpu': '1000m', 'memory': '2Gi'}, 'requests': {'cpu': '100m', 'memory': '512Mi'}}, 'size': '2Gi', 'type': 'postgres', 'user': 'user'}, 'dependencies': {'build': [], 'hard': [], 'soft': []}, 'deployment': {'auto': True, 'image': 'osb/accounts:3e02a15477b4696ed554e08cedf4109c67908cbe6b03331072b5b73e83b4fc2b', 'name': 'accounts', 'port': 8080, 'replicas': 1, 'resources': {'limits': {'cpu': '500m', 'memory': '1024Mi'}, 'requests': {'cpu': '10m', 'memory': '512Mi'}}}, 'domain': None, 'env': [{'name': 'KEYCLOAK_IMPORT', 'value': '/tmp/realm.json'}, {'name': 'KEYCLOAK_USER', 'value': 'admin'}, {'name': 'KEYCLOAK_PASSWORD', 'value': 'metacell'}, {'name': 'PROXY_ADDRESS_FORWARDING', 'value': 'true'}, {'name': 'DB_VENDOR', 'value': 'POSTGRES'}, {'name': 'DB_ADDR', 'value': 'keycloak-postgres'}, {'name': 'DB_DATABASE', 'value': 'auth_db'}, {'name': 'DB_USER', 'value': 'user'}, {'name': 'DB_PASSWORD', 'value': 'password'}, {'name': 'JAVA_OPTS', 'value': '-server -Xms64m -Xmx896m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true  --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED --add-exports=jdk.unsupported/sun.reflect=ALL-UNNAMED'}], 'name': 'accounts', 'readinessProbe': {'path': '/auth/realms/master'}, 'resources': [{'dst': '/tmp/realm.json', 'name': 'realm-config', 'src': 'realm.json'}], 'secrets': '', 'secured': False, 'service': {'auto': True, 'name': 'accounts', 'port': 8080}, 'subdomain': 'accounts', 'uri_role_mapping': [{'roles': ['administrator'], 'uri': '/*'}], 'use_services': []}, 'harvest': True, 'image': 'osb/accounts:latest', 'name': 'accounts', 'port': 8080, 'resources': {'limits': {'cpu': '500m', 'memory': '1024Mi'}, 'requests': {'cpu': '10m', 'memory': '512Mi'}}, 'task-images': {}, 'webclient': {'id': 'web-client', 'secret': '452952ae-922c-4766-b912-7b106271e34b'}}
+
+    ApplicationConfig.from_dict(app)