S3: list_object_versions() - Default value for MaxKeys can now be con…

…figured (#7721)

docs/docs/configuration/environment_variables.rst

@@ -32,6 +32,8 @@ The following is a non-exhaustive list of the environment variables that can be
 +-------------------------------+----------+-----------+-------------------------------------------------------------------------------------------------+
 | MOTO_S3_CUSTOM_ENDPOINTS      | str      |           | See :ref:`s3`.                                                                                  |
 +-------------------------------+----------+-----------+-------------------------------------------------------------------------------------------------+
+| MOTO_S3_DEFAULT_MAX_KEYS      | str      |           | See :ref:`s3`.                                                                                  |
++-------------------------------+----------+-----------+-------------------------------------------------------------------------------------------------+
 | MOTO_PRETTIFY_RESPONSES       | bool     | False     | Prettify responses from Moto, making it easier to read and debug.                               |
 +-------------------------------+----------+-----------+-------------------------------------------------------------------------------------------------+
 

docs/docs/services/s3.rst

@@ -84,8 +84,26 @@ s3
 - [ ] list_directory_buckets
 - [ ] list_multipart_uploads
 - [X] list_object_versions
+
+        The default value for the MaxKeys-argument is 100. This can be configured with an environment variable:
+
+        MOTO_S3_DEFAULT_MAX_KEYS=5
+
+
 - [X] list_objects
+
+        The default value for the MaxKeys-argument is 100. This can be configured with an environment variable:
+
+        MOTO_S3_DEFAULT_MAX_KEYS=5
+
+
 - [X] list_objects_v2
+
+        The default value for the MaxKeys-argument is 100. This can be configured with an environment variable:
+
+        MOTO_S3_DEFAULT_MAX_KEYS=5
+
+
 - [X] list_parts
 - [X] put_bucket_accelerate_configuration
 - [X] put_bucket_acl

moto/s3/models.py

@@ -1652,6 +1652,14 @@ def test_my_custom_endpoint():
 
     _-_-_-_
 
+    When listing objects, the default max-keys value is 1000, just as with AWS. Use the following environment variable to configure this this:
+
+    .. sourcecode:: bash
+
+        MOTO_S3_DEFAULT_MAX_KEYS=256
+
+    _-_-_-_
+
     CrossAccount access is allowed by default. If you want Moto to throw an AccessDenied-error when accessing a bucket in another account, use this environment variable:
 
     .. sourcecode:: bash
@@ -1881,6 +1889,11 @@ def list_object_versions(
     ) -> Tuple[
         List[FakeKey], List[str], List[FakeDeleteMarker], Optional[str], Optional[str]
     ]:
+        """
+        The default value for the MaxKeys-argument is 100. This can be configured with an environment variable:
+
+        MOTO_S3_DEFAULT_MAX_KEYS=5
+        """
         bucket = self.get_bucket(bucket_name)
 
         common_prefixes: Set[str] = set()
@@ -2571,6 +2584,11 @@ def list_objects(
         marker: Optional[str],
         max_keys: Optional[int],
     ) -> Tuple[Set[FakeKey], Set[str], bool, Optional[str]]:
+        """
+        The default value for the MaxKeys-argument is 100. This can be configured with an environment variable:
+
+        MOTO_S3_DEFAULT_MAX_KEYS=5
+        """
         key_results = set()
         folder_results = set()
         if prefix:
@@ -2624,6 +2642,11 @@ def list_objects_v2(
         start_after: Optional[str],
         max_keys: int,
     ) -> Tuple[Set[Union[FakeKey, str]], bool, Optional[str]]:
+        """
+        The default value for the MaxKeys-argument is 100. This can be configured with an environment variable:
+
+        MOTO_S3_DEFAULT_MAX_KEYS=5
+        """
         result_keys, result_folders, _, _ = self.list_objects(
             bucket, prefix, delimiter, marker=None, max_keys=None
         )

moto/s3/responses.py

@@ -619,7 +619,9 @@ def _bucket_response_get(
         elif "versions" in querystring:
             delimiter = querystring.get("delimiter", [None])[0]
             key_marker = querystring.get("key-marker", [None])[0]
-            max_keys = int(querystring.get("max-keys", [1000])[0])
+            max_keys = int(
+                querystring.get("max-keys", [settings.get_s3_default_max_keys()])[0]
+            )
             prefix = querystring.get("prefix", [""])[0]
             version_id_marker = querystring.get("version-id-marker", [None])[0]
 
@@ -693,7 +695,9 @@ def _bucket_response_get(
         if prefix and isinstance(prefix, bytes):
             prefix = prefix.decode("utf-8")
         delimiter = querystring.get("delimiter", [None])[0]
-        max_keys = int(querystring.get("max-keys", [1000])[0])
+        max_keys = int(
+            querystring.get("max-keys", [settings.get_s3_default_max_keys()])[0]
+        )
         marker = querystring.get("marker", [None])[0]
         encoding_type = querystring.get("encoding-type", [None])[0]
 
@@ -756,7 +760,9 @@ def _handle_list_objects_v2(
         delimiter = querystring.get("delimiter", [None])[0]
 
         fetch_owner = querystring.get("fetch-owner", [False])[0]
-        max_keys = int(querystring.get("max-keys", [1000])[0])
+        max_keys = int(
+            querystring.get("max-keys", [settings.get_s3_default_max_keys()])[0]
+        )
         start_after = querystring.get("start-after", [None])[0]
         encoding_type = querystring.get("encoding-type", [None])[0]
 

moto/settings.py

@@ -72,6 +72,10 @@ def get_s3_default_key_buffer_size() -> int:
     )
 
 
+def get_s3_default_max_keys() -> int:
+    return int(os.environ.get("MOTO_S3_DEFAULT_MAX_KEYS", 1000))
+
+
 def s3_allow_crossdomain_access() -> bool:
     return os.environ.get("MOTO_S3_ALLOW_CROSSACCOUNT_ACCESS", "true").lower() == "true"
 

tests/test_dynamodb/test_dynamodb_update_expressions.py

@@ -180,8 +180,8 @@ def test_update_item_add_empty_set(table_name=None):
     err = exc.value.response["Error"]
     assert err["Code"] == "ValidationException"
     assert (
-        err["Message"]
-        == "ExpressionAttributeValues contains invalid value: One or more parameter values were invalid: An string set  may not be empty"
+        "ExpressionAttributeValues contains invalid value: One or more parameter values were invalid: An string set  may not be empty"
+        in err["Message"]
     )
 
     assert dynamodb.scan(TableName=table_name)["Items"] == [

tests/test_s3/test_s3_list_object_versions.py

@@ -1,10 +1,11 @@
 from time import sleep
+from unittest import SkipTest, mock
 from uuid import uuid4
 
 import boto3
 import pytest
 
-from moto import mock_aws
+from moto import mock_aws, settings
 from moto.s3.responses import DEFAULT_REGION_NAME
 
 from . import s3_aws_verified
@@ -639,3 +640,25 @@ def enable_versioning(bucket_name, s3_client):
     while resp.get("Status") != "Enabled":
         sleep(0.1)
         resp = s3_client.get_bucket_versioning(Bucket=bucket_name)
+
+
+@mock_aws
+@mock.patch.dict("os.environ", {"MOTO_S3_DEFAULT_MAX_KEYS": "5"})
+def test_list_object_versions_with_custom_limit():
+    if not settings.TEST_DECORATOR_MODE:
+        raise SkipTest("Can only set env var in DecoratorMode")
+    s3_client = boto3.client("s3", region_name=DEFAULT_REGION_NAME)
+    s3_resource = boto3.resource("s3", region_name=DEFAULT_REGION_NAME)
+    bucket_name = "foobar"
+    bucket = s3_resource.Bucket(bucket_name)
+    bucket.create()
+    bucket.Versioning().enable()
+
+    for i in range(10):
+        s3_client.put_object(
+            Bucket=bucket_name, Key="obj", Body=b"ver" + str(i).encode("utf-8")
+        )
+
+    page1 = s3_client.list_object_versions(Bucket=bucket_name)
+    assert page1["MaxKeys"] == 5
+    assert len(page1["Versions"]) == 5

tests/test_s3/test_s3_list_objects.py

@@ -1,6 +1,10 @@
+from unittest import SkipTest, mock
+
 import boto3
 import pytest
 
+from moto import mock_aws, settings
+
 from . import s3_aws_verified
 
 
@@ -50,3 +54,36 @@ def batch_from(marker: str, *, batch_size: int):
     assert pages[1] == ["key_05", "key_06", "key_07", "key_08", "key_09"]
     assert pages[2] == ["key_10", "key_11", "key_12", "key_13", "key_14"]
     assert pages[3] == ["key_15", "key_16"]
+
+
+@mock_aws
+@mock.patch.dict("os.environ", {"MOTO_S3_DEFAULT_MAX_KEYS": "5"})
+def test_list_object_versions_with_custom_limit():
+    if not settings.TEST_DECORATOR_MODE:
+        raise SkipTest("Can only set env var in DecoratorMode")
+    s3_client = boto3.client("s3", region_name="us-east-1")
+    s3_resource = boto3.resource("s3", region_name="us-east-1")
+    bucket_name = "foobar"
+    bucket = s3_resource.Bucket(bucket_name)
+    bucket.create()
+
+    for i in range(8):
+        s3_client.put_object(Bucket=bucket_name, Key=f"obj_{i}", Body=b"data")
+
+    page1 = s3_client.list_objects(Bucket=bucket_name)
+    assert page1["MaxKeys"] == 5
+    assert len(page1["Contents"]) == 5
+
+    page2 = s3_client.list_objects(Bucket=bucket_name, Marker=page1["NextMarker"])
+    assert page2["MaxKeys"] == 5
+    assert len(page2["Contents"]) == 3
+
+    page1 = s3_client.list_objects_v2(Bucket=bucket_name)
+    assert page1["MaxKeys"] == 5
+    assert len(page1["Contents"]) == 5
+
+    page2 = s3_client.list_objects_v2(
+        Bucket=bucket_name, ContinuationToken=page1["NextContinuationToken"]
+    )
+    assert page2["MaxKeys"] == 5
+    assert len(page2["Contents"]) == 3