diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 08ff3e7a..febc0481 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -36,5 +36,11 @@ jobs: poetry run pytest --cov=pymisp tests/test_*.py poetry run mypy tests/testlive_comprehensive.py tests/test_mispevent.py tests/testlive_sync.py pymisp + - name: Test with nosetests with orjson + run: | + pip3 install orjson + poetry run pytest --cov=pymisp tests/test_*.py + poetry run mypy tests/testlive_comprehensive.py tests/test_mispevent.py tests/testlive_sync.py pymisp + - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 diff --git a/README.md b/README.md index 5c1509fe..89da00f1 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,6 @@ logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode='w', for # From poetry pytest --cov=pymisp tests/test_*.py tests/testlive_comprehensive.py:TestComprehensive.[test_name] - ``` ## Documentation @@ -180,9 +179,9 @@ poetry build mv dist/*.whl offline/packages/ ``` -2. Copy the content of `offline/packages/` to the machine with no internet access. +3. Copy the content of `offline/packages/` to the machine with no internet access. -3. Install the packages: +4. Install the packages: ```bash python -m pip install --no-index --no-deps packages/*.whl diff --git a/poetry.lock b/poetry.lock index 7cf0ed10..f4695754 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1183,6 +1183,7 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, + {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, ] [[package]] @@ -1515,6 +1516,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1574,13 +1585,13 @@ files = [ [[package]] name = "msoffcrypto-tool" -version = "5.1.1" +version = "5.2.0" description = "Python tool and library for decrypting MS Office files with passwords or other keys" optional = true python-versions = ">=3.8,<4.0" files = [ - {file = "msoffcrypto_tool-5.1.1-py3-none-any.whl", hash = "sha256:27475aaf8a70485471ad86426c0be10ee4e24c6fad70335e4a8f88d2da323ca1"}, - {file = "msoffcrypto_tool-5.1.1.tar.gz", hash = "sha256:5585a303fa3ee49eec0253f912be17b82cf83f13f0f7489b4ea10f4ecb285278"}, + {file = "msoffcrypto_tool-5.2.0-py3-none-any.whl", hash = "sha256:2c6c1040df7a4f8e08256f9562f7898d49ec99ae7a6525f9725681b799b8af70"}, + {file = "msoffcrypto_tool-5.2.0.tar.gz", hash = "sha256:2f334cecd19eaa7426f26fc6f661a30268d4817e48d4c006708b4b29dcab6d7f"}, ] [package.dependencies] @@ -2122,6 +2133,8 @@ files = [ {file = "pydeep2-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2283893e25826b547dd1e5c71a010e86ddfd7270e2f2b8c90973c1d7984c7eb7"}, {file = "pydeep2-0.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f248e3161deb53d46a9368a7c164e36d83004faf2f11625d47a5cf23a6bdd2cb"}, {file = "pydeep2-0.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a13fca9be89a9fa8d92a4f49d7b9191eef94555f8ddf030fb2be4c8c15ad618c"}, + {file = "pydeep2-0.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1cb4757db97ac15ddf034c21cd6bab984f841586b6d53984e63c9a7803b2cd4"}, + {file = "pydeep2-0.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7809a1d6640bdbee68f075d53229d05229e11b4711f232728dd540f68e6483a4"}, {file = "pydeep2-0.5.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fedc1c9660cb5d0b73ad0b5f1dbffe16990e6721cbfc6454571a4b9882d0ea4"}, {file = "pydeep2-0.5.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ca68f7d63e2ef510d410d20b223e8e97df41707fb50c4c526b6dd1d8698d9e6"}, {file = "pydeep2-0.5.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:199d05d8b4b7544509a2ba4802ead4b41dfe7859e0ecea9d9be9e41939f11660"}, @@ -2307,6 +2320,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2314,8 +2328,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2332,6 +2353,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2339,6 +2361,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2477,13 +2500,13 @@ files = [ [[package]] name = "referencing" -version = "0.32.0" +version = "0.32.1" description = "JSON Referencing + Python" optional = false python-versions = ">=3.8" files = [ - {file = "referencing-0.32.0-py3-none-any.whl", hash = "sha256:bdcd3efb936f82ff86f993093f6da7435c7de69a3b3a5a06678a6050184bee99"}, - {file = "referencing-0.32.0.tar.gz", hash = "sha256:689e64fe121843dcfd57b71933318ef1f91188ffb45367332700a86ac8fd6161"}, + {file = "referencing-0.32.1-py3-none-any.whl", hash = "sha256:7e4dc12271d8e15612bfe35792f5ea1c40970dadf8624602e33db2758f7ee554"}, + {file = "referencing-0.32.1.tar.gz", hash = "sha256:3c57da0513e9563eb7e203ebe9bb3a1b509b042016433bd1e45a2853466c3dd3"}, ] [package.dependencies] diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 15a123ce..8a4be014 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -1,41 +1,33 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- - +import logging from datetime import date, datetime - from deprecated import deprecated # type: ignore from json import JSONEncoder from uuid import UUID from abc import ABCMeta - -try: - from rapidjson import load # type: ignore - from rapidjson import loads # type: ignore - from rapidjson import dumps # type: ignore - HAS_RAPIDJSON = True -except ImportError: - from json import load - from json import loads - from json import dumps - HAS_RAPIDJSON = False - -import logging from enum import Enum from typing import Union, Optional, Any, Dict, List, Set, Mapping - -from .exceptions import PyMISPInvalidFormat, PyMISPError - - from collections.abc import MutableMapping from functools import lru_cache from pathlib import Path +try: + import orjson # type: ignore + from orjson import loads, dumps # type: ignore + HAS_ORJSON = True +except ImportError: + from json import loads, dumps + HAS_ORJSON = False + +from .exceptions import PyMISPInvalidFormat, PyMISPError + logger = logging.getLogger('pymisp') + resources_path = Path(__file__).parent / 'data' misp_objects_path = resources_path / 'misp-objects' / 'objects' -with (resources_path / 'describeTypes.json').open('r') as f: - describe_types = load(f)['result'] +with (resources_path / 'describeTypes.json').open('rb') as f: + describe_types = loads(f.read())['result'] class MISPFileCache(object): @@ -43,11 +35,11 @@ class MISPFileCache(object): @staticmethod @lru_cache(maxsize=150) - def _load_json(path: Path) -> Union[dict, None]: + def _load_json(path: Path) -> Optional[dict]: if not path.exists(): return None - with path.open('r', encoding='utf-8') as f: - data = load(f) + with path.open('rb') as f: + data = loads(f.read()) return data @@ -249,6 +241,15 @@ def _to_feed(self) -> Dict: def to_json(self, sort_keys: bool = False, indent: Optional[int] = None) -> str: """Dump recursively any class of type MISPAbstract to a json string""" + if HAS_ORJSON: + option = 0 + if sort_keys: + option |= orjson.OPT_SORT_KEYS + if indent: + option |= orjson.OPT_INDENT_2 + + return dumps(self, default=pymisp_json_default, option=option).decode("utf-8") + return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent) def __getitem__(self, key): @@ -406,23 +407,13 @@ def __repr__(self) -> str: return '<{self.__class__.__name__}(NotInitialized)>'.format(self=self) -if HAS_RAPIDJSON: - def pymisp_json_default(obj: Union[AbstractMISP, datetime, date, Enum, UUID]) -> Union[Dict, str]: - if isinstance(obj, AbstractMISP): - return obj.jsonable() - elif isinstance(obj, (datetime, date)): - return obj.isoformat() - elif isinstance(obj, Enum): - return obj.value - elif isinstance(obj, UUID): - return str(obj) -else: - def pymisp_json_default(obj: Union[AbstractMISP, datetime, date, Enum, UUID]) -> Union[Dict, str]: - if isinstance(obj, AbstractMISP): - return obj.jsonable() - elif isinstance(obj, (datetime, date)): - return obj.isoformat() - elif isinstance(obj, Enum): - return obj.value - elif isinstance(obj, UUID): - return str(obj) +# UUID, datetime, date and Enum is serialized by ORJSON by default +def pymisp_json_default(obj: Union[AbstractMISP, datetime, date, Enum, UUID]) -> Union[Dict, str]: + if isinstance(obj, AbstractMISP): + return obj.jsonable() + elif isinstance(obj, (datetime, date)): + return obj.isoformat() + elif isinstance(obj, Enum): + return obj.value + elif isinstance(obj, UUID): + return str(obj) diff --git a/pymisp/api.py b/pymisp/api.py index baceea7e..85f6b008 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -1,13 +1,9 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - from typing import TypeVar, Optional, Tuple, List, Dict, Union, Any, Mapping, Iterable, MutableMapping from datetime import date, datetime import csv from pathlib import Path import logging from urllib.parse import urljoin -import json import requests from requests.auth import AuthBase import re @@ -18,6 +14,14 @@ import urllib3 # type: ignore from io import BytesIO, StringIO +try: + # orjson is optional dependency that speedups parsing and encoding JSON + from orjson import loads, dumps # type: ignore + HAS_ORJSON = True +except ImportError: + from json import loads, dumps + HAS_ORJSON = False + from . import __version__, everything_broken from .exceptions import MISPServerError, PyMISPUnexpectedResponse, PyMISPError, NoURL, NoKey from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog, MISPObject, \ @@ -297,7 +301,7 @@ def misp_instance_version_master(self) -> Dict: """Get the most recent version from github""" r = requests.get('https://raw.githubusercontent.com/MISP/MISP/2.4/VERSION.json') if r.status_code == 200: - master_version = json.loads(r.text) + master_version = loads(r.content) return {'version': '{}.{}.{}'.format(master_version['major'], master_version['minor'], master_version['hotfix'])} return {'error': 'Impossible to retrieve the version of the master branch.'} @@ -3345,7 +3349,7 @@ def set_user_setting(self, user_setting: str, value: Union[str, dict], user: Opt """ query: Dict[str, Any] = {'setting': user_setting} if isinstance(value, dict): - value = json.dumps(value) + value = dumps(value).decode("utf-8") if HAS_ORJSON else dumps(value) query['value'] = value if user: query['user_id'] = get_uuid_or_id_from_abstract_misp(user) @@ -3682,7 +3686,7 @@ def _check_response(self, response: requests.Response, lenient_response_type: bo if 400 <= response.status_code < 500: # The server returns a json message with the error details try: - error_message = response.json() + error_message = loads(response.content) except Exception: raise MISPServerError(f'Error code {response.status_code}:\n{response.text}') @@ -3692,7 +3696,7 @@ def _check_response(self, response: requests.Response, lenient_response_type: bo # At this point, we had no error. try: - response_json = response.json() + response_json = loads(response.content) logger.debug(response_json) if isinstance(response_json, dict) and response_json.get('response') is not None: # Cleanup. @@ -3721,7 +3725,7 @@ def _prepare_request(self, request_type: str, url: str, data: Optional[Union[Ite if url[0] == '/': # strip it: it will fail if MISP is in a sub directory url = url[1:] - # Cake PHP being an idiot, it doesn't accepts %20 (space) in the URL path, + # Cake PHP being an idiot, it doesn't accept %20 (space) in the URL path, # so we need to make it a + instead and hope for the best url = url.replace(' ', '+') url = urljoin(self.root_url, url) @@ -3733,7 +3737,7 @@ def _prepare_request(self, request_type: str, url: str, data: Optional[Union[Ite if isinstance(data, dict): # Remove None values. data = {k: v for k, v in data.items() if v is not None} - d = json.dumps(data, default=pymisp_json_default) + d = dumps(data, default=pymisp_json_default) logger.debug(f'{request_type} - {url}') if d is not None: diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 05758513..f0e69492 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -1,8 +1,5 @@ -# -*- coding: utf-8 -*- - from datetime import timezone, datetime, date import copy -import json import os import base64 import sys @@ -17,6 +14,12 @@ from typing import List, Optional, Union, IO, Dict, Any import warnings +try: + # orjson is optional dependency that speedups parsing and encoding JSON + import orjson as json # type: ignore +except ImportError: + import json + from .abstract import AbstractMISP, MISPTag from .exceptions import (UnknownMISPObjectTemplate, InvalidMISPGalaxy, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError, NewEventReportError, @@ -1090,7 +1093,7 @@ def to_dict(self, json_format: bool = False, strict: bool = False) -> Dict: self._validate() return super(MISPObject, self).to_dict(json_format) - def to_json(self, sort_keys: bool = False, indent: Optional[int] = None, strict: bool = False): + def to_json(self, sort_keys: bool = False, indent: Optional[int] = None, strict: bool = False) -> str: if strict or self._strict and self._known_template: self._validate() return super(MISPObject, self).to_json(sort_keys=sort_keys, indent=indent) @@ -1760,7 +1763,7 @@ def load(self, json_event: Union[IO, str, bytes, dict], validate: bool = False, event.pop('Object', None) self.from_dict(**event) if validate: - warnings.warn('''The validate parameter is deprecated because PyMISP is more flexible at loading event than the schema''') + warnings.warn('The validate parameter is deprecated because PyMISP is more flexible at loading event than the schema') def __setattr__(self, name, value): if name in ['date']: