Merge everest examples and ert test-data

.github/workflows/run_ert_test_data_setups.yml

@@ -51,19 +51,19 @@ jobs:
 
     - name: Test poly example
       run: |
-        pushd test-data/poly_example
+        pushd test-data/ert/poly_example
         ert test_run poly.ert
         popd
 
     - name: Test snake_oil
       run: |
-        pushd test-data/snake_oil
+        pushd test-data/ert/snake_oil
         uv pip install oil_reservoir_synthesizer resdata
         ert test_run snake_oil.ert
         popd
 
     - name: Test snake_oil_field
       run: |
-        pushd test-data/snake_oil_field
+        pushd test-data/ert/snake_oil_field
         ert test_run snake_oil.ert
         popd

.github/workflows/test_ert.yml

@@ -44,23 +44,23 @@ jobs:
     - name: GUI Test
       if: inputs.test-type == 'gui-tests'
       run: |
-        pytest --cov=ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -v --mpl --benchmark-disable tests/ui_tests/gui
+        pytest --cov=ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -v --mpl --benchmark-disable tests/ert/ui_tests/gui
 
     - name: CLI Test
       if: inputs.test-type == 'cli-tests'
       run: |
-        pytest --cov=ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical -v --benchmark-disable  --dist loadgroup tests/ui_tests/cli
+        pytest --cov=ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical -v --benchmark-disable  --dist loadgroup tests/ert/ui_tests/cli
 
     - name: Unit Test
       if: inputs.test-type == 'unit-tests'
       run: |
-        pytest --cov=ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable --dist loadgroup tests/unit_tests
+        pytest --cov=ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable --dist loadgroup tests/ert/unit_tests
         pytest --doctest-modules --cov=ert --cov-report=xml:cov2.xml src/ --ignore src/ert/dark_storage --ignore src/ieverest
 
     - name: Performance Test
       if: inputs.test-type == 'performance-tests'
       run: |
-        pytest --cov=ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable  --dist loadgroup tests/performance_tests
+        pytest --cov=ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable  --dist loadgroup tests/ert/performance_tests
 
     - name: Test for a clean repository
       run: |

.github/workflows/test_ert_with_slurm.yml

@@ -69,13 +69,13 @@ jobs:
       run: |
         set -e
         export _ERT_TESTS_ALTERNATIVE_QUEUE=AlternativeQ
-        pytest tests/unit_tests/scheduler --slurm
+        pytest tests/ert/unit_tests/scheduler --slurm
 
     - name: Test poly-example on slurm
       run: |
         set -e
 
-        cp -a test-data/poly_example poly_slurm
+        cp -a test-data/ert/poly_example poly_slurm
         pushd poly_slurm
 
         cat << EOF >> poly.ert

.github/workflows/test_everest.yml

@@ -71,7 +71,7 @@ jobs:
       if: matrix.test_type == 'everest-models-test'
       run: |
         uv pip install git+https://github.com/equinor/everest-models.git
-        python -m pytest tests -n 4 -m everest_models_test --dist loadgroup
+        python -m pytest tests/everest -n 4 -m everest_models_test --dist loadgroup
 
     - name: Test docs entry point
       if: matrix.test_type == 'everest-docs-entry-test'

.gitmodules

@@ -1,4 +1,4 @@
-[submodule "test-data/block_storage"]
-	path = test-data/block_storage
+[submodule "test-data/ert/block_storage"]
+	path = test-data/ert/block_storage
 	url = https://github.com/equinor/ert-testdata
 	ignore = dirty

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
         exclude: tests/everest/test_data/valid_config_file/invalid_yaml_config.yml # Invalid yml needed for testing
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
-        exclude: test-data/eclipse/parse/ERROR.PRT # exact format is needed for testing
+        exclude: test-data/ert/eclipse/parse/ERROR.PRT # exact format is needed for testing
 
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.6.5

README.md

@@ -90,7 +90,7 @@ If you have not used git-lfs before, you might have to make changes to your glob
 git lfs install
 ```
 
-test-data/block_storage is a submodule and must be checked out.
+test-data/ert/block_storage is a submodule and must be checked out.
 ```sh
 git submodule update --init --recursive
 ```
@@ -131,7 +131,7 @@ As a simple test of your `ert` installation, you may try to run one of the
 examples, for instance:
 
 ```
-cd test-data/poly_example
+cd test-data/ert/poly_example
 # for non-gui trial run
 ert test_run poly.ert
 # for gui trial run
@@ -150,9 +150,9 @@ command `ulimit -a`. In order to increase maximum number of open files, run
 ## Example usage
 
 ### Basic ert test
-To test if ert itself is working, go to `test-data/poly_example` and start ert by running `poly.ert` with `ert gui`
+To test if ert itself is working, go to `test-data/ert/poly_example` and start ert by running `poly.ert` with `ert gui`
 ```
-cd test-data/poly_example
+cd test-data/ert/poly_example
 ert gui poly.ert
 ````
 This opens up the ert graphical user interface.

ci/testkomodo.sh

@@ -19,7 +19,7 @@ install_test_dependencies () {
 run_ert_with_opm () {
     pushd "${CI_TEST_ROOT}"
 
-    cp -r "${CI_SOURCE_ROOT}/test-data/flow_example" ert_with_opm
+    cp -r "${CI_SOURCE_ROOT}/test-data/ert/flow_example" ert_with_opm
     pushd ert_with_opm || exit 1
 
     ert test_run flow.ert ||

docs/ert/reference/configuration/magic_strings.rst

@@ -150,7 +150,7 @@ Experiment constants
 
    ::
 
-      YOUR_WORKFLOW_JOB /data/repos/ert/test-data/poly_example
+      YOUR_WORKFLOW_JOB /data/repos/ert/test-data/ert/poly_example
 
 .. _ms_date:
 .. topic:: DATE

tests/__init__.py

@@ -1,83 +0,0 @@
-import importlib.util
-import sys
-from copy import deepcopy
-from datetime import datetime
-from typing import Any, Dict, Optional, Sequence
-
-from pydantic import BaseModel
-
-from ert.ensemble_evaluator.snapshot import (
-    EnsembleSnapshot,
-    FMStepSnapshot,
-    RealizationSnapshot,
-    _filter_nones,
-)
-
-
-def import_from_location(name, location):
-    spec = importlib.util.spec_from_file_location(name, location)
-    if spec is None:
-        raise ImportError(f"Could not find {name}")
-    module = importlib.util.module_from_spec(spec)
-    sys.modules[name] = module
-    if spec.loader is None:
-        raise ImportError(f"No loader for {name}")
-    spec.loader.exec_module(module)
-    return module
-
-
-class SnapshotBuilder(BaseModel):
-    fm_steps: Dict[str, FMStepSnapshot] = {}
-    metadata: Dict[str, Any] = {}
-
-    def build(
-        self,
-        real_ids: Sequence[str],
-        status: Optional[str],
-        start_time: Optional[datetime] = None,
-        end_time: Optional[datetime] = None,
-    ) -> EnsembleSnapshot:
-        snapshot = EnsembleSnapshot()
-        snapshot._ensemble_state = status
-        snapshot._metadata = self.metadata
-
-        for r_id in real_ids:
-            snapshot.add_realization(
-                r_id,
-                RealizationSnapshot(
-                    active=True,
-                    fm_steps=deepcopy(self.fm_steps),
-                    start_time=start_time,
-                    end_time=end_time,
-                    status=status,
-                ),
-            )
-        return snapshot
-
-    def add_fm_step(
-        self,
-        fm_step_id: str,
-        index: str,
-        name: Optional[str],
-        status: Optional[str],
-        current_memory_usage: Optional[str] = None,
-        max_memory_usage: Optional[str] = None,
-        start_time: Optional[datetime] = None,
-        end_time: Optional[datetime] = None,
-        stdout: Optional[str] = None,
-        stderr: Optional[str] = None,
-    ) -> "SnapshotBuilder":
-        self.fm_steps[fm_step_id] = _filter_nones(
-            FMStepSnapshot(
-                status=status,
-                index=index,
-                start_time=start_time,
-                end_time=end_time,
-                name=name,
-                stdout=stdout,
-                stderr=stderr,
-                current_memory_usage=current_memory_usage,
-                max_memory_usage=max_memory_usage,
-            )
-        )
-        return self

tests/ert/__init__.py

@@ -0,0 +1,83 @@
+import importlib.util
+import sys
+from copy import deepcopy
+from datetime import datetime
+from typing import Any, Dict, Optional, Sequence
+
+from pydantic import BaseModel
+
+from ert.ensemble_evaluator.snapshot import (
+    EnsembleSnapshot,
+    FMStepSnapshot,
+    RealizationSnapshot,
+    _filter_nones,
+)
+
+
+def import_from_location(name, location):
+    spec = importlib.util.spec_from_file_location(name, location)
+    if spec is None:
+        raise ImportError(f"Could not find {name}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    if spec.loader is None:
+        raise ImportError(f"No loader for {name}")
+    spec.loader.exec_module(module)
+    return module
+
+
+class SnapshotBuilder(BaseModel):
+    fm_steps: Dict[str, FMStepSnapshot] = {}
+    metadata: Dict[str, Any] = {}
+
+    def build(
+        self,
+        real_ids: Sequence[str],
+        status: Optional[str],
+        start_time: Optional[datetime] = None,
+        end_time: Optional[datetime] = None,
+    ) -> EnsembleSnapshot:
+        snapshot = EnsembleSnapshot()
+        snapshot._ensemble_state = status
+        snapshot._metadata = self.metadata
+
+        for r_id in real_ids:
+            snapshot.add_realization(
+                r_id,
+                RealizationSnapshot(
+                    active=True,
+                    fm_steps=deepcopy(self.fm_steps),
+                    start_time=start_time,
+                    end_time=end_time,
+                    status=status,
+                ),
+            )
+        return snapshot
+
+    def add_fm_step(
+        self,
+        fm_step_id: str,
+        index: str,
+        name: Optional[str],
+        status: Optional[str],
+        current_memory_usage: Optional[str] = None,
+        max_memory_usage: Optional[str] = None,
+        start_time: Optional[datetime] = None,
+        end_time: Optional[datetime] = None,
+        stdout: Optional[str] = None,
+        stderr: Optional[str] = None,
+    ) -> "SnapshotBuilder":
+        self.fm_steps[fm_step_id] = _filter_nones(
+            FMStepSnapshot(
+                status=status,
+                index=index,
+                start_time=start_time,
+                end_time=end_time,
+                name=name,
+                stdout=stdout,
+                stderr=stderr,
+                current_memory_usage=current_memory_usage,
+                max_memory_usage=max_memory_usage,
+            )
+        )
+        return self

tests/conftest.py → tests/ert/conftest.py

@@ -91,7 +91,7 @@ def fixture_source_root():
 @pytest.fixture(scope="class")
 def class_source_root(request, source_root):
     request.cls.SOURCE_ROOT = source_root
-    request.cls.TESTDATA_ROOT = source_root / "test-data"
+    request.cls.TESTDATA_ROOT = source_root / "test-data" / "ert"
     yield
 
 
@@ -133,7 +133,7 @@ def fixture_setup_case(tmp_path_factory, source_root, monkeypatch):
     def copy_case(path, config_file):
         tmp_path = tmp_path_factory.mktemp(path.replace("/", "-"))
         shutil.copytree(
-            os.path.join(source_root, "test-data", path), tmp_path / "test_data"
+            os.path.join(source_root, "test-data/ert", path), tmp_path / "test_data"
         )
         monkeypatch.chdir(tmp_path / "test_data")
         return ErtConfig.from_file(config_file)
@@ -175,7 +175,7 @@ def fixture_copy_case(tmp_path_factory, source_root, monkeypatch):
     def _copy_case(path):
         tmp_path = tmp_path_factory.mktemp(path.replace("/", "-"))
         shutil.copytree(
-            os.path.join(source_root, "test-data", path),
+            os.path.join(source_root, "test-data/ert", path),
             tmp_path / "test_data",
             ignore=shutil.ignore_patterns("storage"),
         )
@@ -345,7 +345,9 @@ def pytest_collection_modifyitems(config, items):
 
 
 def _run_snake_oil(source_root):
-    shutil.copytree(os.path.join(source_root, "test-data", "snake_oil"), "test_data")
+    shutil.copytree(
+        os.path.join(source_root, "test-data/ert", "snake_oil"), "test_data"
+    )
     os.chdir("test_data")
     with fileinput.input("snake_oil.ert", inplace=True) as fin:
         for line in fin:
@@ -371,7 +373,7 @@ def _run_snake_oil(source_root):
 
 def _run_heat_equation(source_root):
     shutil.copytree(
-        os.path.join(source_root, "test-data", "heat_equation"), "test_data"
+        os.path.join(source_root, "test-data", "ert", "heat_equation"), "test_data"
     )
     os.chdir("test_data")
 
@@ -453,10 +455,10 @@ def snake_oil_default_storage(snake_oil_case_storage):
 
 @pytest.fixture(scope="session")
 def block_storage_path(source_root):
-    path = source_root / "test-data/block_storage/snake_oil"
+    path = source_root / "test-data/ert/block_storage/snake_oil"
     if not path.is_dir():
         pytest.skip(
-            "'test-data/block_storage' has not been checked out.\n"
+            "'test-data/ert/block_storage' has not been checked out.\n"
             "Make sure you have git-lfs installed and run: "
             "git submodule update --init --recursive"
         )

tests/performance_tests/performance_utils.py → tests/ert/performance_tests/performance_utils.py

@@ -20,7 +20,7 @@
 def source_dir() -> Path:
     current_path = Path(__file__)
     source = current_path.parent.parent
-    if not (source / "test-data" / "poly_template").exists():
+    if not (source / "test-data" / "ert" / "poly_template").exists():
         raise RuntimeError("Cannot find the source folder")
     return source
 
@@ -132,7 +132,7 @@ def make_poly_example(folder, source, **kwargs):
 
 def make_poly_template(folder, source_folder, **kwargs):
     return make_poly_example(
-        folder, source_folder / "test-data" / "poly_template", **kwargs
+        folder, source_folder / "test-data" / "ert" / "poly_template", **kwargs
     )
 
 
@@ -164,7 +164,7 @@ def dark_storage_app(monkeypatch):
 
     make_poly_example(
         folder=folder,
-        source=source_dir / "test-data/poly_template",
+        source=source_dir / "test-data/ert/poly_template",
         gen_data_count=3400,
         gen_data_entries=150,
         summary_data_entries=100,