new: [internal] Add support for orjson

orjson is much faster library for decoding and encoding JSON formats

.github/workflows/pytest.yml

@@ -36,5 +36,11 @@ jobs:
         poetry run pytest --cov=pymisp tests/test_*.py
         poetry run mypy tests/testlive_comprehensive.py tests/test_mispevent.py tests/testlive_sync.py pymisp
 
+    - name: Test with nosetests with orjson
+      run: |
+        pip3 install orjson
+        poetry run pytest --cov=pymisp tests/test_*.py
+        poetry run mypy tests/testlive_comprehensive.py tests/test_mispevent.py tests/testlive_sync.py pymisp
+
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3

pymisp/abstract.py

@@ -1,53 +1,44 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
+import logging
 from datetime import date, datetime
-
 from deprecated import deprecated  # type: ignore
 from json import JSONEncoder
 from uuid import UUID
 from abc import ABCMeta
-
-try:
-    from rapidjson import load  # type: ignore
-    from rapidjson import loads  # type: ignore
-    from rapidjson import dumps  # type: ignore
-    HAS_RAPIDJSON = True
-except ImportError:
-    from json import load
-    from json import loads
-    from json import dumps
-    HAS_RAPIDJSON = False
-
-import logging
 from enum import Enum
 from typing import Union, Optional, Any, Dict, List, Set, Mapping
-
-from .exceptions import PyMISPInvalidFormat, PyMISPError
-
-
 from collections.abc import MutableMapping
 from functools import lru_cache
 from pathlib import Path
 
+try:
+    from orjson import loads, dumps  # type: ignore
+    HAS_ORJSON = True
+except ImportError:
+    from json import loads, dumps
+    HAS_ORJSON = False
+
+from .exceptions import PyMISPInvalidFormat, PyMISPError
+
 logger = logging.getLogger('pymisp')
 
+
 resources_path = Path(__file__).parent / 'data'
 misp_objects_path = resources_path / 'misp-objects' / 'objects'
-with (resources_path / 'describeTypes.json').open('r') as f:
-    describe_types = load(f)['result']
+with (resources_path / 'describeTypes.json').open('rb') as f:
+    describe_types = loads(f.read())['result']
 
 
 class MISPFileCache(object):
     # cache up to 150 JSON structures in class attribute
 
     @staticmethod
     @lru_cache(maxsize=150)
-    def _load_json(path: Path) -> Union[dict, None]:
+    def _load_json(path: Path) -> Optional[dict]:
         if not path.exists():
             return None
-        with path.open('r', encoding='utf-8') as f:
-            data = load(f)
+        with path.open('rb') as f:
+            data = loads(f.read())
         return data
 
 
@@ -249,7 +240,10 @@ def _to_feed(self) -> Dict:
 
     def to_json(self, sort_keys: bool = False, indent: Optional[int] = None) -> str:
         """Dump recursively any class of type MISPAbstract to a json string"""
-        return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent)
+        json_string = dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent)
+        if HAS_ORJSON:
+            json_string = json_string.decode("utf-8")
+        return json_string
 
     def __getitem__(self, key):
         try:
@@ -406,16 +400,14 @@ def __repr__(self) -> str:
         return '<{self.__class__.__name__}(NotInitialized)>'.format(self=self)
 
 
-if HAS_RAPIDJSON:
+if HAS_ORJSON:
+    # UUID, datetime, date and Enum is serialized by ORJSON by default
+    # datetime and date are serialized by ORJSON by default
     def pymisp_json_default(obj: Union[AbstractMISP, datetime, date, Enum, UUID]) -> Union[Dict, str]:
         if isinstance(obj, AbstractMISP):
             return obj.jsonable()
-        elif isinstance(obj, (datetime, date)):
-            return obj.isoformat()
-        elif isinstance(obj, Enum):
-            return obj.value
-        elif isinstance(obj, UUID):
-            return str(obj)
+        raise TypeError
+
 else:
     def pymisp_json_default(obj: Union[AbstractMISP, datetime, date, Enum, UUID]) -> Union[Dict, str]:
         if isinstance(obj, AbstractMISP):

pymisp/api.py

@@ -7,7 +7,6 @@
 from pathlib import Path
 import logging
 from urllib.parse import urljoin
-import json
 import requests
 from requests.auth import AuthBase
 import re
@@ -18,6 +17,13 @@
 import urllib3  # type: ignore
 from io import BytesIO, StringIO
 
+try:
+    from orjson import loads, dumps  # type: ignore
+    HAS_ORJSON = True
+except ImportError:
+    from json import loads, dumps
+    HAS_ORJSON = False
+
 from . import __version__, everything_broken
 from .exceptions import MISPServerError, PyMISPUnexpectedResponse, PyMISPError, NoURL, NoKey
 from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObject, \
@@ -297,7 +303,7 @@ def misp_instance_version_master(self) -> Dict:
         """Get the most recent version from github"""
         r = requests.get('https://raw.githubusercontent.com/MISP/MISP/2.4/VERSION.json')
         if r.status_code == 200:
-            master_version = json.loads(r.text)
+            master_version = loads(r.content)
             return {'version': '{}.{}.{}'.format(master_version['major'], master_version['minor'], master_version['hotfix'])}
         return {'error': 'Impossible to retrieve the version of the master branch.'}
 
@@ -3345,7 +3351,7 @@ def set_user_setting(self, user_setting: str, value: Union[str, dict], user: Opt
         """
         query: Dict[str, Any] = {'setting': user_setting}
         if isinstance(value, dict):
-            value = json.dumps(value)
+            value = dumps(value).decode("utf-8") if HAS_ORJSON else dumps(value)
         query['value'] = value
         if user:
             query['user_id'] = get_uuid_or_id_from_abstract_misp(user)
@@ -3682,7 +3688,7 @@ def _check_response(self, response: requests.Response, lenient_response_type: bo
         if 400 <= response.status_code < 500:
             # The server returns a json message with the error details
             try:
-                error_message = response.json()
+                error_message = loads(response.content)
             except Exception:
                 raise MISPServerError(f'Error code {response.status_code}:\n{response.text}')
 
@@ -3692,7 +3698,7 @@ def _check_response(self, response: requests.Response, lenient_response_type: bo
         # At this point, we had no error.
 
         try:
-            response_json = response.json()
+            response_json = loads(response.content)
             logger.debug(response_json)
             if isinstance(response_json, dict) and response_json.get('response') is not None:
                 # Cleanup.
@@ -3721,7 +3727,7 @@ def _prepare_request(self, request_type: str, url: str, data: Optional[Union[Ite
         if url[0] == '/':
             # strip it: it will fail if MISP is in a sub directory
             url = url[1:]
-        # Cake PHP being an idiot, it doesn't accepts %20 (space) in the URL path,
+        # Cake PHP being an idiot, it doesn't accept %20 (space) in the URL path,
         # so we need to make it a + instead and hope for the best
         url = url.replace(' ', '+')
         url = urljoin(self.root_url, url)
@@ -3733,7 +3739,7 @@ def _prepare_request(self, request_type: str, url: str, data: Optional[Union[Ite
                 if isinstance(data, dict):
                     # Remove None values.
                     data = {k: v for k, v in data.items() if v is not None}
-                d = json.dumps(data, default=pymisp_json_default)
+                d = dumps(data, default=pymisp_json_default)
 
         logger.debug(f'{request_type} - {url}')
         if d is not None: