Merge pull request #958 from DataDog/cbeauchesne/runner-in-host

Runner is now in host

.circleci/config.yml

@@ -10,16 +10,15 @@ jobs:
       - checkout
 
       - run:
-          name: Install good version of docker-compose
-          command: |
-            sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-            sudo chmod +x /usr/local/bin/docker-compose
+          name: Install python 3.9
+          command: sudo apt-get install python3.9-venv
 
       - run:
           name: versions
           command: |
             docker --version
-            docker-compose --version
+            python3.9 --version
+            pip --version
 
       - run:
           name: Build

.github/workflows/ci.yml

@@ -40,7 +40,6 @@ jobs:
       uses: actions/checkout@v3
     - name: Run lints
       uses: ./.github/actions/lint_code
-    - run: mkdir -p logs/docker/weblog/logs/
     - run: pip install -r requirements.txt
     - run: PYTHONPATH="$PWD" SYSTEMTESTS_SCENARIO=TEST_THE_TEST pytest utils/test_the_test
   test-the-tests-main:
@@ -190,7 +189,10 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v3
-    - run: mkdir logs && touch logs/.weblog.env
+    - name: Setup python 3.9
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
     - name: Pull mongo image
       run: docker pull mongo:latest
     - name: Pull cassandra image
@@ -408,7 +410,10 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v3
-    - run: mkdir logs && touch logs/.weblog.env
+    - name: Setup python 3.9
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
     - name: Pull mongo image
       run: docker pull mongo:latest
     - name: Pull cassandra image
@@ -624,29 +629,39 @@ jobs:
         DD_APP_KEY: ${{ secrets.DD_CI_APP_KEY }}
   peformances:
     runs-on: ubuntu-latest
+    if: github.event.action != 'opened' && !contains(github.event.pull_request.labels.*.name, 'run-default-scenario')
     needs:
     - lint_and_test
     env:
       DD_API_KEY: ${{ secrets.DD_API_KEY }}
     steps:
     - name: Checkout
       uses: actions/checkout@v3
+    - name: Setup python 3.9
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
     - name: Run
-      run: ./scenarios/perfs/run.sh golang
+      run: ./tests/perfs/run.sh golang
     - name: Display
-      run: python scenarios/perfs/process.py
+      run: |
+        source venv/bin/activate
+        python tests/perfs/process.py
   fuzzer:
     runs-on: ubuntu-latest
+    if: github.event.action != 'opened' && !contains(github.event.pull_request.labels.*.name, 'run-default-scenario')
     needs:
     - lint_and_test
     env:
       DD_API_KEY: ${{ secrets.DD_API_KEY }}
     steps:
     - name: Checkout
       uses: actions/checkout@v3
+    - name: Setup python 3.9
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
     - name: Build
       run: ./build.sh golang
     - name: Run
-      run: ./run.sh scenarios/fuzzer/main.py -t 60
-      env:
-        RUNNER_CMD: python
+      run: ./tests/fuzzer/run.sh

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        }
+    ]
+}

conftest.py

@@ -4,9 +4,10 @@
 import os
 import json
 
+import pytest
 from pytest_jsonreport.plugin import JSONReport
 
-from utils import context, interfaces
+from utils import context
 from utils._context._scenarios import current_scenario
 from utils.tools import logger
 from utils.scripts.junit_report import junit_modifyreport
@@ -22,6 +23,16 @@
 _coverages = {}
 _rfcs = {}
 
+
+_JSON_REPORT_FILE = f"{current_scenario.host_log_folder}/report.json"
+_XML_REPORT_FILE = f"{current_scenario.host_log_folder}/reportJunit.xml"
+
+
+def pytest_configure(config):
+    config.option.json_report_file = _JSON_REPORT_FILE
+    config.option.xmlpath = _XML_REPORT_FILE
+
+
 # Called at the very begening
 def pytest_sessionstart(session):
 
@@ -142,70 +153,47 @@ def pytest_collection_finish(session):
 
     terminal.write_line("Executing weblog warmup...")
 
-    try:
-        current_scenario.execute_warmups()
-
-        last_file = ""
-        for item in session.items:
-
-            if _item_is_skipped(item):
-                continue
+    current_scenario.execute_warmups()
 
-            if not item.instance:  # item is a method bounded to a class
-                continue
+    last_file = ""
+    for item in session.items:
 
-            # the test metohd name is like test_xxxx
-            # we replace the test_ by setup_, and call it if it exists
+        if _item_is_skipped(item):
+            continue
 
-            setup_method_name = f"setup_{item.name[5:]}"
+        if not item.instance:  # item is a method bounded to a class
+            continue
 
-            if not hasattr(item.instance, setup_method_name):
-                continue
+        # the test metohd name is like test_xxxx
+        # we replace the test_ by setup_, and call it if it exists
 
-            if last_file != item.location[0]:
-                if len(last_file) == 0:
-                    terminal.write_sep("-", "Tests setup", bold=True)
+        setup_method_name = f"setup_{item.name[5:]}"
 
-                terminal.write(f"\n{item.location[0]} ")
-                last_file = item.location[0]
+        if not hasattr(item.instance, setup_method_name):
+            continue
 
-            setup_method = getattr(item.instance, setup_method_name)
-            logger.debug(f"Call {setup_method} for {item}")
-            try:
-                setup_method()
-            except Exception:
-                logger.exception("Unexpected failure during setup method call")
-                terminal.write("x", bold=True, red=True)
-                raise
-            else:
-                terminal.write(".", bold=True, green=True)
+        if last_file != item.location[0]:
+            if len(last_file) == 0:
+                terminal.write_sep("-", "tests setup", bold=True)
 
-        terminal.write("\n\n")
-
-        if current_scenario.use_interfaces:
-            _wait_interface(interfaces.library, session, current_scenario.library_interface_timeout)
-            _wait_interface(interfaces.agent, session, current_scenario.agent_interface_timeout)
-            _wait_interface(interfaces.backend, session, current_scenario.backend_interface_timeout)
-
-            current_scenario.collect_logs()
-
-            _wait_interface(interfaces.library_stdout, session, 0)
-            _wait_interface(interfaces.library_dotnet_managed, session, 0)
-            _wait_interface(interfaces.agent_stdout, session, 0)
-
-    except:
-        current_scenario.collect_logs()
-        raise
-    finally:
-        current_scenario.close_targets()
+            terminal.write(f"\n{item.location[0]} ")
+            last_file = item.location[0]
 
+        setup_method = getattr(item.instance, setup_method_name)
+        logger.debug(f"Call {setup_method} for {item}")
+        try:
+            setup_method()
+        except Exception:
+            logger.exception("Unexpected failure during setup method call")
+            terminal.write("x", bold=True, red=True)
+            current_scenario.close_targets()
+            raise
+        else:
+            terminal.write(".", bold=True, green=True)
 
-def _wait_interface(interface, session, timeout):
-    terminal = session.config.pluginmanager.get_plugin("terminalreporter")
-    terminal.write_sep("-", f"Wait for {interface} ({timeout}s)")
-    terminal.flush()
+    terminal.write("\n\n")
 
-    interface.wait(timeout)
+    current_scenario.post_setup(session)
 
 
 def pytest_json_modifyreport(json_report):
@@ -241,7 +229,7 @@ def pytest_sessionfinish(session, exitstatus):
 
     json.dump(
         {library: sorted(versions) for library, versions in LibraryVersion.known_versions.items()},
-        open("logs/known_versions.json", "w", encoding="utf-8"),
+        open(f"{current_scenario.host_log_folder}/known_versions.json", "w", encoding="utf-8"),
         indent=2,
     )
 
@@ -253,14 +241,12 @@ def pytest_sessionfinish(session, exitstatus):
 
 
 def _pytest_junit_modifyreport():
-    json_report_path = "logs/report.json"
-    junit_report_path = "logs/reportJunit.xml"
 
-    with open(json_report_path, encoding="utf-8") as f:
+    with open(_JSON_REPORT_FILE, encoding="utf-8") as f:
         json_report = json.load(f)
         junit_modifyreport(
             json_report,
-            junit_report_path,
+            _XML_REPORT_FILE,
             _skip_reasons,
             _docs,
             _rfcs,

docs/architecture/overview.md

@@ -92,9 +92,7 @@ As system tests is blackbox testing, there will likely be very little informatio
 
 The first method of troubleshooting should be to inspect the logs folder.
 
-The folder is usually `./logs/`.
-
-If you are running a non-default scenario: the logs folder is set with the `SYSTEMTESTS_LOG_FOLDER` variable in in the `./run.sh` file.
+The folder is `./logs/` for the default scenario, or `./logs_<scenario_name>` for other scenatrios
 
 ```mermaid
 flowchart TD

docs/edit/README.md

@@ -3,7 +3,7 @@
 First of all, it'll be more confortable to set-up your dev env. The repo contains basic conf for VSCode, feel free to add conf for other IDE.
 
 ```bash
-python3 -m venv venv
+python3.9 -m venv venv
 source venv/bin/activate
 pip install -r requirements.txt
 ```

docs/edit/lint.md

@@ -16,10 +16,8 @@ There is a script `format.sh` in the root of the repository which will package t
 
 ## Using a virtualenv
 
-* [install python 3](https://www.python.org/downloads/). You may have it by default:
-  * run `python --version`. As long as the version is 3, it's ok
-  * you can try with `python3 --version`.
-* run `python -m venv venv` (use `python3` if the step above works with it).
+* [install python 3.9](https://www.python.org/downloads/). You may have it by default:
+* run `python3.9 -m venv venv`.
 * run `source venv/bin/activate`
   * Windows user, it'll be `venv\Scripts\activate.bat`
   * Fish users, i'll be `. venv/bin/activate.fish`

docs/execute/run.md

@@ -24,7 +24,7 @@
 ## Spawn componenents, but do nothing
 
 ```bash
-./run.sh scenarios/sleep.py
+./run.sh SLEEP
 
 # play with the weblog, and look inside logs/interfaces/ what's happening
 ```

docs/internals/requirements.md

@@ -1,2 +1,2 @@
-Theorically, only `docker-compose` and `git`, and very common Unix tools are required.
+Only `python3`, `docker-compose` and `git`, and very common Unix tools are required.
 

pyproject.toml

@@ -3,13 +3,12 @@ line-length = 120
 exclude = "(venv/|utils/grpc/weblog_pb2_grpc.py|utils/grpc/weblog_pb2.py|parametric/apps|parametric/protos/)"
 
 [tool.pytest.ini_options]
-addopts = "--json-report --json-report-file=logs/report.json --json-report-indent=2 --color=yes --no-header --junitxml=logs/reportJunit.xml -r Xf"
+addopts = "--json-report --json-report-indent=2 --color=yes --no-header --junitxml=reportJunit.xml -r Xf"
 # log_cli = True
 # log_cli_level = DEBUG
 log_level = "DEBUG"
 log_format = "%(asctime)s.%(msecs)03d %(levelname)-8s %(message)s"
 log_date_format = "%H:%M:%S"
-log_file = "logs/pytest.log"
 log_file_format = "%(asctime)s.%(msecs)03d %(levelname)-8s %(message)s"
 log_file_date_format = "%H:%M:%S"
 

requirements.txt

@@ -6,8 +6,14 @@ pytest-json-report==1.5.0
 black==19.10b0  # TODO : move to 22.8.0
 python-dateutil==2.8.2
 msgpack==1.0.4
-mitmproxy==9.0.1
-protobuf==3.20.2  # grpcio-tools 1.48.1 depends on protobuf<4.0dev and >=3.12.0
+watchdog==3.0.0
+
+# we'd like to update protobuf and grpcio-tools to those version, but a bug in protobuf 4.x need to be solved before
+# https://github.com/protocolbuffers/protobuf/issues/11863
+# protobuf==4.22.1
+# grpcio-tools==1.51.3
+
+protobuf==3.20.2
 grpcio-tools==1.48.1
 
 aiohttp==3.8.2