split requirements-dev.txt into first and third party dev dependenc…

…ies (#840) Co-authored-by: Nadia Dencheva <[email protected]> Co-authored-by: Eddie Schlafly <[email protected]>
spacetelescope · Sep 21, 2023 · 9e80829 · 9e80829
1 parent 8880382
commit 9e80829
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 24 deletions.
diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml
@@ -77,7 +77,7 @@ jobs:
         - linux: check-dependencies
         - linux: build-dist
   test:
-    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
     needs: [ data ]
     with:
       setenv: |

diff --git a/JenkinsfileRT b/JenkinsfileRT
@@ -97,7 +97,8 @@ bc0.build_cmds = bc0.build_cmds + [
     "pip list"
 ]
 bc0.test_cmds = [
-    "pytest --cov-report=xml:coverage.xml --cov=./ -r sxf -n auto --bigdata --slow \
+    "pytest -r sxf -n auto --bigdata --slow \
+    --cov --cov-report=xml:coverage.xml \
     --ddtrace \
     --basetemp=${pytest_basetemp} --junit-xml=results.xml --dist=loadscope \
     --env=${artifactoryenv} ${pytest_args}",
@@ -119,5 +120,8 @@ bc1.build_cmds = bc1.build_cmds + PipInject(env.OVERRIDE_REQUIREMENTS)
 bc1.test_cmds = []
 bc1.test_configs = []
 
-utils.run([jobconfig, bc0, bc1])
+bc2 = utils.copy(bc0)
+bc2.pip_reqs_files = ['requirements-sdp.txt', 'requirements-dev-st.txt']
+
+utils.run([jobconfig, bc0, bc1, bc2])
 }  // withCredentials
diff --git a/JenkinsfileRT_dev b/JenkinsfileRT_dev
@@ -80,7 +80,7 @@ bc0.conda_packages = [
     "python=${python_version}",
     "freetds",
 ]
-bc0.pip_reqs_files = ['requirements-dev.txt']
+bc0.pip_reqs_files = ['requirements-dev-thirdparty.txt']
 bc0.build_cmds = [
     "pip install -e .[test]",
     "pip install pytest-xdist pytest-sugar",
@@ -109,5 +109,8 @@ bc1.build_cmds = bc1.build_cmds + PipInject(env.OVERRIDE_REQUIREMENTS)
 bc1.test_cmds = []
 bc1.test_configs = []
 
-utils.run([jobconfig, bc0, bc1])
+bc2 = utils.copy(bc0)
+bc2.pip_reqs_files = ['requirements-dev-thirdparty.txt', 'requirements-dev-st.txt']
+
+utils.run([jobconfig, bc0, bc1, bc2])
 }  // withCredentials
diff --git a/README.md b/README.md
@@ -102,6 +102,12 @@ tools used for linting and reproducible environments run:
 
 This will install `tox` and the `pre-commit` tools in addition to all development dependencies.
 
+Development versions of dependencies are listed in `requirements-dev-st.txt` and `requirements-dev-thirdparty.txt`, containing STScI and third-party packages respectively.
+To install these dependencies, run the following:
+
+    pip install -r requirements-dev-st.txt
+    pip install -r requirements-dev-thirdparty.txt
+
 Need other useful packages in your development environment?
 
     pip install ipython pytest-xdist

diff --git a/requirements-dev-st.txt b/requirements-dev-st.txt
@@ -0,0 +1,13 @@
+# Roman upstream packages
+git+https://github.com/spacetelescope/roman_datamodels
+git+https://github.com/spacetelescope/rad
+
+# shared upstream packages
+git+https://github.com/spacetelescope/stcal
+git+https://github.com/spacetelescope/stpipe
+
+# Other important upstream packages
+git+https://github.com/spacetelescope/crds
+git+https://github.com/spacetelescope/gwcs
+git+https://github.com/spacetelescope/metrics_logger
+git+https://github.com/spacetelescope/tweakwcs
diff --git a/requirements-dev.txt → requirements-dev-thirdparty.txt b/requirements-dev.txt → requirements-dev-thirdparty.txt
@@ -1,26 +1,16 @@
 # ASDF upstream packages
+git+https://github.com/asdf-format/asdf-standard
 git+https://github.com/asdf-format/asdf
 git+https://github.com/asdf-format/asdf-transform-schemas
+git+https://github.com/asdf-format/asdf-coordinates-schemas
 git+https://github.com/asdf-format/asdf-wcs-schemas
-git+https://github.com/astropy/asdf-astropy
 
 # Use weekly astropy dev build
+git+https://github.com/astropy/asdf-astropy
 --extra-index-url https://pypi.anaconda.org/astropy/simple astropy --pre
+git+https://github.com/astropy/photutils.git
 
 # Use Bi-weekly numpy/scipy dev builds
 --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
 numpy>=0.0.dev0
 scipy>=0.0.dev0
-
-# Roman upstream packages
-git+https://github.com/spacetelescope/roman_datamodels
-git+https://github.com/spacetelescope/rad
-
-# Other important upstream packages
-git+https://github.com/spacetelescope/gwcs
-git+https://github.com/spacetelescope/stpipe
-git+https://github.com/spacetelescope/crds
-git+https://github.com/spacetelescope/stcal
-git+https://github.com/spacetelescope/tweakwcs
-git+https://github.com/spacetelescope/metrics_logger
-git+https://github.com/astropy/photutils.git
diff --git a/tox.ini b/tox.ini
@@ -36,7 +36,8 @@ commands =
 description =
     run tests
     alldeps: with all optional dependencies
-    devdeps: with the latest developer version of key dependencies
+    stdevdeps: with the latest developer version of key upstream spacetelescope dependencies
+    devdeps: with the latest developer version of key upstream third-party dependencies
     oldestdeps: with the oldest supported version of key dependencies
     sdpdeps: with the recent STScI DMS release pinned dependencies
     pyargs: with --pyargs on installed package
@@ -54,9 +55,6 @@ pass_env =
     CODECOV_*
     DD_*
     WEBBPSF_PATH
-set_env =
-    devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simpl
-
 extras =
     test
     alldeps: all
@@ -67,7 +65,8 @@ deps =
 commands_pre =
     oldestdeps: minimum_dependencies romancal --filename requirements-min.txt
     oldestdeps: pip install -r requirements-min.txt
-    devdeps: pip install -r requirements-dev.txt -U --upgrade-strategy eager
+    stdevdeps: pip install -r requirements-dev-st.txt -U --upgrade-strategy eager
+    devdeps: pip install -r requirements-dev-thirdparty.txt -r requirements-dev-st.txt -U --upgrade-strategy eager
     sdpdeps: pip install -r requirements-sdp.txt
     pip list
 commands =