diff --git a/bin/download-wp-tests.sh b/bin/download-wp-tests.sh
index 56caa3ac87..4a245727a4 100755
--- a/bin/download-wp-tests.sh
+++ b/bin/download-wp-tests.sh
@@ -98,14 +98,16 @@ install_test_suite() {
 
 	cd $WP_TESTS_DIR_ACTUAL
 
-	if [ ! -f wp-tests-config.php ]; then
-		download https://develop.svn.wordpress.org/${WP_TESTS_TAG}/wp-tests-config-sample.php "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
-		sed $ioption "s:dirname( __FILE__ ) . '/src/':'$WP_CORE_DIR':" "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
-		sed $ioption "s/youremptytestdbnamehere/$DB_NAME/" "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
-		sed $ioption "s/yourusernamehere/$DB_USER/" "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
-		sed $ioption "s/yourpasswordhere/$DB_PASS/" "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
-		sed $ioption "s|localhost|${DB_HOST}|" "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
-	fi
+	# Always start from a clean config
+	rm -f wp-tests-config.php
+
+	# Set up the config file
+	download https://develop.svn.wordpress.org/${WP_TESTS_TAG}/wp-tests-config-sample.php "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
+	sed $ioption "s:dirname( __FILE__ ) . '/src/':'$WP_CORE_DIR':" "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
+	sed $ioption "s/youremptytestdbnamehere/$DB_NAME/" "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
+	sed $ioption "s/yourusernamehere/$DB_USER/" "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
+	sed $ioption "s/yourpasswordhere/$DB_PASS/" "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
+	sed $ioption "s|localhost|${DB_HOST}|" "$WP_TESTS_DIR_ACTUAL"/wp-tests-config.php
 
 }
 
diff --git a/bin/phpunit-docker.sh b/bin/phpunit-docker.sh
index 80b546ba4a..79eacdea01 100755
--- a/bin/phpunit-docker.sh
+++ b/bin/phpunit-docker.sh
@@ -13,26 +13,33 @@ echo "Will test with WP_VERSION=$WP_VERSION and WP_MULTISITE=$WP_MULTISITE"
 echo "--------------"
 echo
 
+MARIADB_VERSION="10.3"
+UUID=`date +%s000`
+NETWORK_NAME="tests-$UUID"
+DB_CONTAINER_NAME="db-$UUID"
 MYSQL_ROOT_PASSWORD='wordpress'
-docker network create tests
-db=$(docker run --network tests --name db -e MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD -d mariadb)
+
+docker network create $NETWORK_NAME
+
+db=$(docker run --network $NETWORK_NAME --name $DB_CONTAINER_NAME -e MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD -d mariadb/server:$MARIADB_VERSION)
 function cleanup() {
+	echo "cleanup!"
 	docker rm -f $db
-	docker network rm tests
+	docker network rm $NETWORK_NAME
 }
 trap cleanup EXIT
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-WP_VERSION=$($DIR/download-wp-tests.sh wordpress_test root "$MYSQL_ROOT_PASSWORD" "db" "$WP_VERSION")
+WP_VERSION=$($DIR/download-wp-tests.sh wordpress_test root "$MYSQL_ROOT_PASSWORD" "$DB_CONTAINER_NAME" "$WP_VERSION")
 
 ## Ensure there's a database connection for the rest of the steps
-until docker exec -it $db mysql -u root -h db --password="wordpress" -e 'CREATE DATABASE wordpress_test' > /dev/null; do
+until docker exec -it $db mysql -u root -h $DB_CONTAINER_NAME --password="wordpress" -e 'CREATE DATABASE wordpress_test' > /dev/null; do
 	echo "Waiting for database connection..."
 	sleep 5
 done
 
 docker run \
- 	--network tests \
+	--network $NETWORK_NAME \
 	-e WP_MULTISITE="$WP_MULTISITE" \
 	-v $(pwd):/app \
 	-v /tmp/wordpress-tests-lib-$WP_VERSION:/tmp/wordpress-tests-lib \