SQS: Support JSON format (#7042)

.github/workflows/build.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ]
+        python-version: [ 3.7, 3.8, 3.9, "3.10" ]
 
     steps:
       - uses: actions/checkout@v2
@@ -77,7 +77,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, "3.10"]
+        python-version: [3.7, 3.8, 3.9, "3.10"]
 
     steps:
     - uses: actions/checkout@v2
@@ -122,7 +122,7 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, "3.10"]
+        python-version: [3.7, 3.8, 3.9, "3.10"]
 
     steps:
     - uses: actions/checkout@v2

moto/apigateway/models.py

@@ -6,13 +6,26 @@
 from collections import defaultdict
 from copy import copy
 
-from openapi_spec_validator import validate_spec
+try:
+    # Recommended as of 0.7.x
+    from openapi_spec_validator import validate  # type: ignore
+except ImportError:
+    # Only used in < 0.7.x
+    # (Also exists in 0.7.0, but throws a warning)
+    from openapi_spec_validator import validate_spec as validate  # type: ignore
 import time
 
 from urllib.parse import urlparse
 import responses
 
-from openapi_spec_validator.exceptions import OpenAPIValidationError
+try:
+    from openapi_spec_validator.exceptions import (  # noqa  # pylint: disable=unused-import
+        OpenAPIValidationError,
+    )
+except ImportError:
+    from openapi_spec_validator.validation.exceptions import (  # noqa  # pylint: disable=unused-import
+        OpenAPIValidationError,
+    )
 from moto.core import get_account_id, BaseBackend, BaseModel, CloudFormationModel
 from .utils import create_id, to_path
 from moto.core.utils import path_url, BackendDict
@@ -1285,9 +1298,12 @@ def import_rest_api(self, api_doc, fail_on_warnings):
         """
         if fail_on_warnings:
             try:
-                validate_spec(api_doc)
+                validate(api_doc)
             except OpenAPIValidationError as e:
                 raise InvalidOpenAPIDocumentException(e)
+            except AttributeError:
+                # Call can fail on Python3.7 due to `typing_extensions 4.6.0` throwing an error
+                pass
         name = api_doc["info"]["title"]
         description = api_doc["info"]["description"]
         api = self.create_rest_api(name=name, description=description)
@@ -1314,9 +1330,12 @@ def put_rest_api(self, function_id, api_doc, mode="merge", fail_on_warnings=Fals
 
         if fail_on_warnings:
             try:
-                validate_spec(api_doc)
+                validate(api_doc)
             except OpenAPIValidationError as e:
                 raise InvalidOpenAPIDocumentException(e)
+            except AttributeError:
+                # Call can fail on Python3.7 due to `typing_extensions 4.6.0` throwing an error
+                pass
 
         if mode == "overwrite":
             api = self.get_rest_api(function_id)

moto/backend_index.py

@@ -1,4 +1,4 @@
-# autogenerated by scripts/update_backend_index.py
+# autogenerated by /home/bblommers/Software/Code/getmoto/moto/scripts/update_backend_index.py
 import re
 
 backend_url_patterns = [
@@ -80,6 +80,8 @@
     ("iot-data", re.compile("https?://data\\.iot\\.(.+)\\.amazonaws.com")),
     ("iot-data", re.compile("https?://data-ats\\.iot\\.(.+)\\.amazonaws.com")),
     ("kinesis", re.compile("https?://kinesis\\.(.+)\\.amazonaws\\.com")),
+    ("kinesis", re.compile("https?://(.+)\\.control-kinesis\\.(.+)\\.amazonaws\\.com")),
+    ("kinesis", re.compile("https?://(.+)\\.data-kinesis\\.(.+)\\.amazonaws\\.com")),
     ("kinesisvideo", re.compile("https?://kinesisvideo\\.(.+)\\.amazonaws.com")),
     (
         "kinesis-video-archived-media",

moto/core/exceptions.py

@@ -73,6 +73,11 @@ def get_headers(self, *args, **kwargs):  # pylint: disable=unused-argument
     def get_body(self, *args, **kwargs):  # pylint: disable=unused-argument
         return self.description
 
+    def to_json(self) -> "JsonRESTError":
+        err = JsonRESTError(error_type=self.error_type, message=self.message)
+        err.code = self.code
+        return err
+
 
 class DryRunClientError(RESTError):
     code = 412

moto/core/responses.py

@@ -11,7 +11,7 @@
 from moto.core.exceptions import DryRunClientError
 
 from jinja2 import Environment, DictLoader, TemplateNotFound
-
+from gzip import decompress
 from urllib.parse import parse_qs, parse_qsl, urlparse
 
 import xmltodict
@@ -20,7 +20,7 @@
 import boto3
 from collections import OrderedDict
 from moto.core.utils import camelcase_to_underscores, method_names_from_class
-from moto.utilities.utils import load_resource
+from moto.utilities.utils import load_resource, load_resource_as_bytes
 from moto import settings
 
 log = logging.getLogger(__name__)
@@ -840,7 +840,12 @@ class AWSServiceSpec(object):
     """
 
     def __init__(self, path):
-        spec = load_resource("botocore", path)
+        try:
+            spec = load_resource("botocore", path)
+        except FileNotFoundError:
+            # botocore >= 1.32.1 sends compressed files
+            compressed = load_resource_as_bytes("botocore", f"{path}.gz")
+            spec = json.loads(decompress(compressed).decode("utf-8"))
 
         self.metadata = spec["metadata"]
         self.operations = spec["operations"]

moto/kinesis/urls.py

@@ -3,6 +3,11 @@
 url_bases = [
     # Need to avoid conflicting with kinesisvideo
     r"https?://kinesis\.(.+)\.amazonaws\.com",
+    # Somewhere around boto3-1.26.31 botocore-1.29.31, AWS started using a new endpoint:
+    # 	111122223333.control-kinesis.us-east-1.amazonaws.com
+    r"https?://(.+)\.control-kinesis\.(.+)\.amazonaws\.com",
+    # When passing in the StreamARN to get_shard_iterator/get_records, this endpoint is called:
+    r"https?://(.+)\.data-kinesis\.(.+)\.amazonaws\.com",
 ]
 
 url_paths = {"{0}/$": KinesisResponse.dispatch}

moto/sqs/models.py

@@ -153,6 +153,10 @@ def utf8(string):
     def body(self):
         return escape(self._body).replace('"', """).replace("\r", "
")
 
+    @property
+    def original_body(self) -> str:
+        return self._body
+
     def mark_sent(self, delay_seconds=None):
         self.sent_timestamp = int(unix_time_millis())
         if delay_seconds: