Use pytest instead of nosetests (elastic#16883)

Nosetests has been in maintenance mode for years, its last release was
released in 2015. Some functionality it uses from Python is going to be
removed in upcoming versions.

Use pytest to replace nosetests as testrunner. Features used from nose
libraries are replaced by features from the Python standard library and
from pytest.

.ci/windows.groovy

@@ -527,7 +527,8 @@ def dumpFilteredEnvironment(){
   echo "PROCESSES: ${env.PROCESSES}"
   echo "TIMEOUT: ${env.TIMEOUT}"
   echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}"
-  echo "NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS}"
+  echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}"
+  echo "PYTEST_OPTIONS: ${env.PYTEST_OPTIONS}"
   echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}"
   echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}"
   echo "STRESS_TESTS: ${env.STRESS_TESTS}"

Jenkinsfile

@@ -1091,7 +1091,8 @@ def dumpFilteredEnvironment(){
   echo "PROCESSES: ${env.PROCESSES}"
   echo "TIMEOUT: ${env.TIMEOUT}"
   echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}"
-  echo "NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS}"
+  echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}"
+  echo "PYTEST_OPTIONS: ${env.PYTEST_OPTIONS}"
   echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}"
   echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}"
   echo "STRESS_TESTS: ${env.STRESS_TESTS}"

dev-tools/mage/check.go

@@ -43,12 +43,12 @@ import (
 // if it finds any modifications. If executed in in verbose mode it will write
 // the results of 'git diff' to stdout to indicate what changes have been made.
 //
-// It checks the file permissions of nosetests test cases and YAML files.
+// It checks the file permissions of python test cases and YAML files.
 // It checks .go source files using 'go vet'.
 func Check() error {
 	fmt.Println(">> check: Checking source code for common problems")
 
-	mg.Deps(GoVet, CheckNosetestsNotExecutable, CheckYAMLNotExecutable, CheckDashboardsFormat)
+	mg.Deps(GoVet, CheckPythonTestNotExecutable, CheckYAMLNotExecutable, CheckDashboardsFormat)
 
 	changes, err := GitDiffIndex()
 	if err != nil {
@@ -124,16 +124,15 @@ func GitDiff() error {
 	return err
 }
 
-// CheckNosetestsNotExecutable checks that none of the nosetests files are
-// executable. Nosetests silently skips executable .py files and we don't want
-// this to happen.
-func CheckNosetestsNotExecutable() error {
+// CheckPythonTestNotExecutable checks that none of the python test files are
+// executable. They are silently skipped and we don't want this to happen.
+func CheckPythonTestNotExecutable() error {
 	if runtime.GOOS == "windows" {
 		// Skip windows because it doesn't have POSIX permissions.
 		return nil
 	}
 
-	tests, err := FindFiles(nosetestsTestFiles...)
+	tests, err := FindFiles(pythonTestFiles...)
 	if err != nil {
 		return err
 	}
@@ -151,7 +150,7 @@ func CheckNosetestsNotExecutable() error {
 	}
 
 	if len(executableTestFiles) > 0 {
-		return errors.Errorf("nosetests files cannot be executable because "+
+		return errors.Errorf("python test files cannot be executable because "+
 			"they will be skipped. Fix permissions of %v", executableTestFiles)
 	}
 	return nil

dev-tools/mage/pytest.go

@@ -56,7 +56,7 @@ var (
 	pythonVirtualenvLock sync.Mutex
 
 	// More globs may be needed in the future if tests are added in more places.
-	nosetestsTestFiles = []string{
+	pythonTestFiles = []string{
 		"tests/system/test_*.py",
 		"module/*/test_*.py",
 		"module/*/*/test_*.py",
@@ -77,11 +77,11 @@ func init() {
 }
 
 // PythonTestArgs are the arguments used for the "python*Test" targets and they
-// define how "nosetests" is invoked.
+// define how python tests are invoked.
 type PythonTestArgs struct {
 	TestName            string            // Test name used in logging.
 	Env                 map[string]string // Env vars to add to the current env.
-	Files               []string          // Globs used by nosetests to find tests.
+	Files               []string          // Globs used to find tests.
 	XUnitReportFile     string            // File to write the XUnit XML test report to.
 	CoverageProfileFile string            // Test coverage profile file.
 }
@@ -109,39 +109,40 @@ func DefaultPythonTestUnitArgs() PythonTestArgs { return makePythonTestArgs("Uni
 // checking for INTEGRATION_TEST=1 in the test code.
 func DefaultPythonTestIntegrationArgs() PythonTestArgs { return makePythonTestArgs("Integration") }
 
-// PythonNoseTest invokes "nosetests" via a Python virtualenv.
-func PythonNoseTest(params PythonTestArgs) error {
+// PythonTest executes python tests via a Python virtualenv.
+func PythonTest(params PythonTestArgs) error {
 	fmt.Println(">> python test:", params.TestName, "Testing")
 
 	ve, err := PythonVirtualenv()
 	if err != nil {
 		return err
 	}
 
-	nosetestsEnv := map[string]string{
+	pytestEnv := map[string]string{
 		// activate sets this. Not sure if it's ever needed.
 		"VIRTUAL_ENV": ve,
 	}
 	if IsInIntegTestEnv() {
-		nosetestsEnv["INTEGRATION_TESTS"] = "1"
+		pytestEnv["INTEGRATION_TESTS"] = "1"
 	}
 	for k, v := range params.Env {
-		nosetestsEnv[k] = v
+		pytestEnv[k] = v
 	}
 
-	nosetestsOptions := []string{
-		"--process-timeout=90",
-		"--with-timer",
-		"-v",
+	pytestOptions := []string{
+		"--timeout=90",
+		"--durations=20",
+	}
+	if mg.Verbose() {
+		pytestOptions = append(pytestOptions, "-v")
 	}
 	if params.XUnitReportFile != "" {
-		nosetestsOptions = append(nosetestsOptions,
-			"--with-xunit",
-			"--xunit-file="+createDir(params.XUnitReportFile),
+		pytestOptions = append(pytestOptions,
+			"--junit-xml="+createDir(params.XUnitReportFile),
 		)
 	}
 
-	files := nosetestsTestFiles
+	files := pythonTestFiles
 	if len(params.Files) > 0 {
 		files = params.Files
 	}
@@ -157,31 +158,31 @@ func PythonNoseTest(params PythonTestArgs) error {
 	// We check both the VE and the normal PATH because on Windows if the
 	// requirements are met by the globally installed package they are not
 	// installed to the VE.
-	nosetestsPath, err := LookVirtualenvPath(ve, "nosetests")
+	pytestPath, err := LookVirtualenvPath(ve, "pytest")
 	if err != nil {
 		return err
 	}
 
 	defer fmt.Println(">> python test:", params.TestName, "Testing Complete")
-	_, err = sh.Exec(nosetestsEnv, os.Stdout, os.Stderr, nosetestsPath, append(nosetestsOptions, testFiles...)...)
+	_, err = sh.Exec(pytestEnv, os.Stdout, os.Stderr, pytestPath, append(pytestOptions, testFiles...)...)
 	return err
 
 	// TODO: Aggregate all the individual code coverage reports and generate
 	// and HTML report.
 }
 
-// PythonNoseTestForModule executes python system tests for modules.
+// PythonTestForModule executes python system tests for modules.
 //
 // Use `MODULE=module` to run only tests for `module`.
-func PythonNoseTestForModule(params PythonTestArgs) error {
+func PythonTestForModule(params PythonTestArgs) error {
 	if module := EnvOr("MODULE", ""); module != "" {
 		params.Files = []string{
 			fmt.Sprintf("module/%s/test_*.py", module),
 			fmt.Sprintf("module/%s/*/test_*.py", module),
 		}
 		params.TestName += "-" + module
 	}
-	return PythonNoseTest(params)
+	return PythonTest(params)
 }
 
 // PythonVirtualenv constructs a virtualenv that contains the given modules as

dev-tools/mage/target/integtest/integtest.go

@@ -74,19 +74,18 @@ func GoIntegTest(ctx context.Context) error {
 
 // PythonIntegTest executes the python system tests in the integration
 // environment (Docker).
-// Use NOSE_TESTMATCH=pattern to only run tests matching the specified pattern.
-// Use any other NOSE_* environment variable to influence the behavior of
-// nosetests.
+// Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern.
+// Use any other PYTEST_* environment variable to influence the behavior of pytest.
 func PythonIntegTest(ctx context.Context) error {
 	if !devtools.IsInIntegTestEnv() {
 		mg.SerialDeps(pythonTestDeps...)
 	}
-	runner, err := devtools.NewDockerIntegrationRunner(append(whitelistedEnvVars, devtools.ListMatchingEnvVars("NOSE_")...)...)
+	runner, err := devtools.NewDockerIntegrationRunner(append(whitelistedEnvVars, devtools.ListMatchingEnvVars("PYTEST_")...)...)
 	if err != nil {
 		return err
 	}
 	return runner.Test("pythonIntegTest", func() error {
 		mg.Deps(devtools.BuildSystemTestBinary)
-		return devtools.PythonNoseTest(devtools.DefaultPythonTestIntegrationArgs())
+		return devtools.PythonTest(devtools.DefaultPythonTestIntegrationArgs())
 	})
 }

dev-tools/mage/target/unittest/unittest.go

@@ -61,5 +61,5 @@ func GoUnitTest(ctx context.Context) error {
 func PythonUnitTest() error {
 	mg.SerialDeps(pythonTestDeps...)
 	mg.Deps(devtools.BuildSystemTestBinary)
-	return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs())
+	return devtools.PythonTest(devtools.DefaultPythonTestUnitArgs())
 }

docs/devguide/modules-dev-guide.asciidoc

@@ -488,4 +488,4 @@ locally for a specific module, using the following procedure under Filebeat dire
 . Create python env: `make python-env`
 . Source python env: `./build/python-env/bin/activate`
 . Create the testing binary: `make filebeat.test`
-. Run the test, ie: `GENERATE=1 INTEGRATION_TESTS=1 BEAT_STRICT_PERMS=false TESTING_FILEBEAT_MODULES=nginx nosetests tests/system/test_modules.py`
+. Run the test, ie: `GENERATE=1 INTEGRATION_TESTS=1 BEAT_STRICT_PERMS=false TESTING_FILEBEAT_MODULES=nginx pytest tests/system/test_modules.py`

docs/devguide/python.asciidoc

@@ -52,14 +52,14 @@ of these virtual environments:
   is created by `make` or `mage` targets when needed.  
 
 Virtual environments can also be used without `make` or `mage`, this is usual
-for example when running individual system tests with `nosetests`. There are two
+for example when running individual system tests with `pytest`. There are two
 ways to run commands from the virtual environment:
 
 * "Activating" the virtual environment in your current terminal running
   `source ./build/python-env/bin/activate`. Virtual environment can be
   deactivated by running `deactivate`.
 * Directly running commands from the virtual environment path. For example
-  `nosetests` can be executed as `./build/python-env/bin/nosetests`.
+  `pytest` can be executed as `./build/python-env/bin/pytest`.
 
 To recreate a virtual environment, remove its directory. All virtual
 environments are also removed with `make clean`.

docs/devguide/testing.asciidoc

@@ -8,7 +8,8 @@ In general there are two major test suites:
 * Tests written in Go
 * Tests written in Python
 
-The tests written in Go use the https://golang.org/pkg/testing/[Go Testing package]. The tests written in Python depend on http://nose.readthedocs.io/en/latest/[nosetests] and require a compiled and executable binary from the Go code. The python test run a beat with a specific config and params and either check if the output is as expected or if the correct things show up in the logs.
+The tests written in Go use the https://golang.org/pkg/testing/[Go Testing
+package]. The tests written in Python depend on https://docs.pytest.org/en/latest/[pytest] and require a compiled and executable binary from the Go code. The python test run a beat with a specific config and params and either check if the output is as expected or if the correct things show up in the logs.
 
 For both of the above test suites so called integration tests exists. Integration tests in Beats are tests which require an external system like Elasticsearch to test if the integration with this service works as expected. Beats provides in its testsuite docker containers and docker-compose files to start these environments but a developer can run the required services also locally.
 
@@ -23,7 +24,7 @@ All Go tests are in the same package as the tested code itself and have the post
 
 ==== Running Python Tests
 
-The system tests require a testing binary to be available and the python environment to be set up. To create the testing binary run `make {beatname}.test`. This will create the test binary in the beat directory. To setup the testing environment `make python-env` can be run which will use `venv` to load the dependencies. Then `nosetests` has to be run inside `tests/system`.
+The system tests require a testing binary to be available and the python environment to be set up. To create the testing binary run `make {beatname}.test`. This will create the test binary in the beat directory. To setup the testing environment `make python-env` can be run which will use `venv` to load the dependencies. Then `pytest` has to be run inside `tests/system`.
 
 To automate all these steps into one `make system-tests` can be run. This creates the binary, the environment and runs all tests which do not require and external service.
 

filebeat/magefile.go

@@ -218,14 +218,14 @@ func PythonIntegTest(ctx context.Context) error {
 	if !devtools.IsInIntegTestEnv() {
 		mg.Deps(Fields)
 	}
-	runner, err := devtools.NewDockerIntegrationRunner(append(devtools.ListMatchingEnvVars("TESTING_FILEBEAT_", "NOSE_"), "GENERATE")...)
+	runner, err := devtools.NewDockerIntegrationRunner(append(devtools.ListMatchingEnvVars("TESTING_FILEBEAT_", "PYTEST_"), "GENERATE")...)
 	if err != nil {
 		return err
 	}
 	return runner.Test("pythonIntegTest", func() error {
 		mg.Deps(devtools.BuildSystemTestBinary)
 		args := devtools.DefaultPythonTestIntegrationArgs()
 		args.Env["MODULES_PATH"] = devtools.CWD("module")
-		return devtools.PythonNoseTest(args)
+		return devtools.PythonTest(args)
 	})
 }

filebeat/tests/system/test_crawler.py

@@ -1,13 +1,10 @@
 # -*- coding: utf-8 -*-
-
-from filebeat import BaseTest
-
 import codecs
 import os
+import shutil
 import time
 import unittest
-from nose.plugins.skip import Skip, SkipTest
-import shutil
+from filebeat import BaseTest
 
 # Additional tests to be added:
 # * Check what happens when file renamed -> no recrawling should happen
@@ -767,7 +764,7 @@ def test_file_no_permission(self):
         """
         if os.name != "nt" and os.geteuid() == 0:
             # root ignores permission flags, so we have to skip the test
-            raise SkipTest
+            raise unittest.SkipTest
 
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
@@ -789,7 +786,7 @@ def test_file_no_permission(self):
 
         if os.name == "nt":
 
-            raise SkipTest
+            raise unittest.SkipTest
             # TODO: Currently skipping this test on windows as it requires `pip install win32api`
             # which seems to have windows only dependencies.
             # To solve this problem a requirements_windows.txt could be introduced which would

filebeat/tests/system/test_load.py

@@ -1,12 +1,12 @@
-from filebeat import BaseTest
 import os
 import logging
 import logging.handlers
 import json
 import time
 import unittest
-from nose.plugins.skip import Skip, SkipTest
-from nose.plugins.attrib import attr
+import pytest
+
+from filebeat import BaseTest
 
 """
 Test filebeat under different load scenarios
@@ -25,7 +25,7 @@ def test_no_missing_events(self):
         if os.name == "nt":
             # This test is currently skipped on windows because very fast file
             # rotation cannot happen when harvester has file handler still open.
-            raise SkipTest
+            raise unittest.SkipTest
 
         log_file = self.working_dir + "/log/test.log"
         os.mkdir(self.working_dir + "/log/")
@@ -63,7 +63,7 @@ def test_no_missing_events(self):
         for i in range(total_lines):
             # Make sure each line has the same length
             line = format(i, str(line_length - 1))
-            logger.debug("%d", i)
+            logger.warning("%d", i)
 
         # wait until all lines are read
         self.wait_until(
@@ -106,7 +106,7 @@ def test_no_missing_events(self):
         assert len(entry_list) == total_lines
 
     @unittest.skipUnless(LOAD_TESTS, "load test")
-    @attr('load')
+    @pytest.mark.load
     def test_large_number_of_files(self):
         """
         Tests the number of files filebeat can open on startup
@@ -148,7 +148,7 @@ def test_large_number_of_files(self):
         assert len(data) == number_of_files
 
     @unittest.skipUnless(LOAD_TESTS, "load test")
-    @attr('load')
+    @pytest.mark.load
     def test_concurrent_harvesters(self):
         """
         Test large number of files on startup if harvester overlap happens and would create too many events

filebeat/tests/system/test_publisher.py

@@ -1,11 +1,9 @@
-from filebeat import BaseTest
-
 import os
 import platform
 import time
 import shutil
 import json
-from nose.plugins.skip import Skip, SkipTest
+from filebeat import BaseTest
 
 
 # Additional tests: to be implemented