Refactor Class-decorator logic to reset per test (#4419)

.github/workflows/build.yml

@@ -216,50 +216,6 @@ jobs:
         path: |
           serverlogs/*
 
-  test_responses:
-    name: Test Responses versions
-    runs-on: ubuntu-latest
-    needs: lint
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: [ 3.8 ]
-        responses-version: [0.11.0, 0.12.0, 0.12.1, 0.13.0, 0.15.0, 0.17.0]
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Get pip cache dir
-        id: pip-cache
-        run: |
-          echo "::set-output name=dir::$(pip cache dir)"
-      - name: pip cache
-        uses: actions/cache@v2
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}-4
-      - name: Install project dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
-          pip install pytest-cov
-          pip install responses==${{ matrix.responses-version }}
-          pip install "coverage<=4.5.4"
-      - name: Test core-logic with responses==${{ matrix.responses-version }}
-        run: |
-          pytest -sv --cov=moto --cov-report xml ./tests/test_core ./tests/test_apigateway/test_apigateway_integration.py
-      - name: "Upload coverage to Codecov"
-        if: ${{ github.repository == 'spulec/moto'}}
-        uses: codecov/codecov-action@v1
-        with:
-          fail_ci_if_error: true
-          flags: test_responses
-
   terraform:
     name: Terraform Tests
     runs-on: ubuntu-latest
@@ -327,7 +283,7 @@ jobs:
   deploy:
     name: Deploy
     runs-on: ubuntu-latest
-    needs: [test, testserver, terraform, test_responses]
+    needs: [test, testserver, terraform ]
     if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'spulec/moto' }}
     strategy:
       matrix:

.github/workflows/dependency_test.yml

@@ -1,4 +1,4 @@
-name: DependencyTest
+name: "Service-specific Dependencies Test"
 
 on:
   workflow_dispatch:

.github/workflows/test_outdated_versions.yml

@@ -0,0 +1,38 @@
+# Run separate test cases to verify Moto works with older versions of dependencies
+#
+name: "Outdated Dependency Tests"
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.7", "3.9" ]
+        responses-version: ["0.11.0", "0.12.0", "0.12.1", "0.13.0", "0.15.0", "0.17.0" ]
+        mock-version: [ "3.0.5", "4.0.0", "4.0.3" ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Update pip
+      run: |
+        python -m pip install --upgrade pip
+
+    - name: Install project dependencies
+      run: |
+        pip install -r requirements-dev.txt
+        pip install responses==${{ matrix.responses-version }}
+        pip install mock==${{ matrix.mock-version }}
+
+    - name: Run tests
+      run: |
+        pytest -sv tests/test_core ./tests/test_apigateway/test_apigateway_integration.py

codecov.yml

@@ -1,7 +1,7 @@
 codecov:
   notify:
     # Leave a GitHub comment after all builds have passed
-    after_n_builds: 14
+    after_n_builds: 8
 coverage:
   status:
     project:

docs/docs/getting_started.rst

@@ -156,28 +156,33 @@ If you use `unittest`_ to run tests, and you want to use `moto` inside `setUp`,
             actual = object.get()['Body'].read()
             self.assertEqual(actual, content)
 
+Class Decorator
+~~~~~~~~~~~~~~~~~
 
-It is possible to use Moto as a class-decorator.
-Note that this may behave differently then you might expected - it currently creates a global state on class-level, rather than on method-level.
+It is also possible to use decorators on the class-level.
+
+The decorator is effective for every test-method inside your class. State is not shared across test-methods.
 
 .. sourcecode:: python
 
     @mock_s3
     class TestMockClassLevel(unittest.TestCase):
-        def create_my_bucket(self):
-            s3 = boto3.resource('s3')
-            bucket = s3.Bucket("mybucket")
-            bucket.create()
+        def setUp(self):
+            s3 = boto3.client("s3", region_name="us-east-1")
+            s3.create_bucket(Bucket="mybucket")
 
-        def test_1_should_create_bucket(self):
-            self.create_my_bucket()
+        def test_creating_a_bucket(self):
+            # 'mybucket', created in setUp, is accessible in this test
+            # Other clients can be created at will
 
-            client = boto3.client("s3")
-            assert len(client.list_buckets()["Buckets"]) == 1
+            s3 = boto3.client("s3", region_name="us-east-1")
+            s3.create_bucket(Bucket="bucket_inside")
 
-        def test_2_bucket_still_exists(self):
-            client = boto3.client("s3")
-            assert len(client.list_buckets()["Buckets"]) == 1
+        def test_accessing_a_bucket(self):
+            # The state has been reset before this method has started
+            # 'mybucket' is recreated as part of the setUp-method
+            # 'bucket_inside' however, created inside the other test, no longer exists
+            pass
 
 Stand-alone server mode
 ~~~~~~~~~~~~~~~~~~~~~~~

moto/core/models.py

@@ -2,6 +2,7 @@
 import boto3
 import functools
 import inspect
+import itertools
 import os
 import random
 import re
@@ -13,9 +14,11 @@
 from botocore.config import Config
 from botocore.handlers import BUILTIN_HANDLERS
 from botocore.awsrequest import AWSResponse
+from types import FunctionType
 
 from moto import settings
 import responses
+import unittest
 from unittest.mock import patch
 from .custom_responses_mock import (
     get_response_mock,
@@ -90,7 +93,7 @@ def start(self, reset=True):
             for backend in self.backends.values():
                 backend.reset()
 
-        self.enable_patching()
+        self.enable_patching(reset)
 
     def stop(self):
         self.__class__.nested_count -= 1
@@ -124,7 +127,18 @@ def wrapper(*args, **kwargs):
         return wrapper
 
     def decorate_class(self, klass):
-        for attr in dir(klass):
+        direct_methods = set(
+            x
+            for x, y in klass.__dict__.items()
+            if isinstance(y, (FunctionType, classmethod, staticmethod))
+        )
+        defined_classes = set(
+            x for x, y in klass.__dict__.items() if inspect.isclass(y)
+        )
+
+        has_setup_method = "setUp" in direct_methods
+
+        for attr in itertools.chain(direct_methods, defined_classes):
             if attr.startswith("_"):
                 continue
 
@@ -150,7 +164,18 @@ def decorate_class(self, klass):
                 continue
 
             try:
-                setattr(klass, attr, self(attr_value, reset=False))
+                # Special case for UnitTests-class
+                is_test_method = attr.startswith(unittest.TestLoader.testMethodPrefix)
+                should_reset = False
+                if attr == "setUp":
+                    should_reset = True
+                elif not has_setup_method and is_test_method:
+                    should_reset = True
+                else:
+                    # Method is unrelated to the test setup
+                    # Method is a test, but was already reset while executing the setUp-method
+                    pass
+                setattr(klass, attr, self(attr_value, reset=should_reset))
             except TypeError:
                 # Sometimes we can't set this for built-in types
                 continue
@@ -296,7 +321,7 @@ def reset(self):
         botocore_stubber.reset()
         reset_responses_mock(responses_mock)
 
-    def enable_patching(self):
+    def enable_patching(self, reset=True):
         botocore_stubber.enabled = True
         for method in BOTOCORE_HTTP_METHODS:
             for backend in self.backends_for_urls.values():
@@ -356,8 +381,8 @@ def reset(self):
 
             requests.post("http://localhost:5000/moto-api/reset")
 
-    def enable_patching(self):
-        if self.__class__.nested_count == 1:
+    def enable_patching(self, reset=True):
+        if self.__class__.nested_count == 1 and reset:
             # Just started
             self.reset()
 

tests/test_core/test_decorator_calls.py

@@ -1,9 +1,8 @@
 import boto3
-import sure  # noqa # pylint: disable=unused-import
 import os
-import unittest
-
 import pytest
+import sure  # noqa # pylint: disable=unused-import
+import unittest
 
 from botocore.exceptions import ClientError
 from moto import mock_ec2, mock_s3, settings
@@ -107,3 +106,100 @@ def static(*args):
 
     def test_no_instance_sent_to_staticmethod(self):
         self.static()
+
+
+@mock_s3
+class TestWithSetup(unittest.TestCase):
+    def setUp(self):
+        s3 = boto3.client("s3", region_name="us-east-1")
+        s3.create_bucket(Bucket="mybucket")
+
+    def test_should_find_bucket(self):
+        s3 = boto3.client("s3", region_name="us-east-1")
+        self.assertIsNotNone(s3.head_bucket(Bucket="mybucket"))
+
+    def test_should_not_find_unknown_bucket(self):
+        s3 = boto3.client("s3", region_name="us-east-1")
+        with pytest.raises(ClientError):
+            s3.head_bucket(Bucket="unknown_bucket")
+
+
+@mock_s3
+class TestWithPublicMethod(unittest.TestCase):
+    def ensure_bucket_exists(self):
+        s3 = boto3.client("s3", region_name="us-east-1")
+        s3.create_bucket(Bucket="mybucket")
+
+    def test_should_find_bucket(self):
+        self.ensure_bucket_exists()
+
+        s3 = boto3.client("s3", region_name="us-east-1")
+        s3.head_bucket(Bucket="mybucket").shouldnt.equal(None)
+
+    def test_should_not_find_bucket(self):
+        s3 = boto3.client("s3", region_name="us-east-1")
+        with pytest.raises(ClientError):
+            s3.head_bucket(Bucket="mybucket")
+
+
+@mock_s3
+class TestWithPseudoPrivateMethod(unittest.TestCase):
+    def _ensure_bucket_exists(self):
+        s3 = boto3.client("s3", region_name="us-east-1")
+        s3.create_bucket(Bucket="mybucket")
+
+    def test_should_find_bucket(self):
+        self._ensure_bucket_exists()
+        s3 = boto3.client("s3", region_name="us-east-1")
+        s3.head_bucket(Bucket="mybucket").shouldnt.equal(None)
+
+    def test_should_not_find_bucket(self):
+        s3 = boto3.client("s3", region_name="us-east-1")
+        with pytest.raises(ClientError):
+            s3.head_bucket(Bucket="mybucket")
+
+
+@mock_s3
+class TestWithNestedClasses:
+    class NestedClass(unittest.TestCase):
+        def _ensure_bucket_exists(self):
+            s3 = boto3.client("s3", region_name="us-east-1")
+            s3.create_bucket(Bucket="bucketclass1")
+
+        def test_should_find_bucket(self):
+            self._ensure_bucket_exists()
+            s3 = boto3.client("s3", region_name="us-east-1")
+            s3.head_bucket(Bucket="bucketclass1")
+
+    class NestedClass2(unittest.TestCase):
+        def _ensure_bucket_exists(self):
+            s3 = boto3.client("s3", region_name="us-east-1")
+            s3.create_bucket(Bucket="bucketclass2")
+
+        def test_should_find_bucket(self):
+            self._ensure_bucket_exists()
+            s3 = boto3.client("s3", region_name="us-east-1")
+            s3.head_bucket(Bucket="bucketclass2")
+
+        def test_should_not_find_bucket_from_different_class(self):
+            s3 = boto3.client("s3", region_name="us-east-1")
+            with pytest.raises(ClientError):
+                s3.head_bucket(Bucket="bucketclass1")
+
+    class TestWithSetup(unittest.TestCase):
+        def setUp(self):
+            s3 = boto3.client("s3", region_name="us-east-1")
+            s3.create_bucket(Bucket="mybucket")
+
+        def test_should_find_bucket(self):
+            s3 = boto3.client("s3", region_name="us-east-1")
+            s3.head_bucket(Bucket="mybucket")
+
+            s3.create_bucket(Bucket="bucketinsidetest")
+
+        def test_should_not_find_bucket_from_test_method(self):
+            s3 = boto3.client("s3", region_name="us-east-1")
+            s3.head_bucket(Bucket="mybucket")
+
+            with pytest.raises(ClientError):
+                s3.head_bucket(Bucket="bucketinsidetest")

tests/test_s3/test_s3_classdecorator.py

@@ -1,6 +1,6 @@
-import unittest
-
 import boto3
+import sure  # noqa  # pylint: disable=unused-import
+import unittest
 from moto import mock_s3