From f4000ad399b485591fb05d78baaa497707991c5d Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Mon, 21 Aug 2023 13:07:11 -0400 Subject: [PATCH 1/4] separate first and third party dev dependencies into requirements.txt files --- .github/workflows/roman_ci.yml | 2 +- JenkinsfileRT | 8 ++++++-- JenkinsfileRT_dev | 7 +++++-- requirements-dev-thirdparty.txt | 13 +++++++++++++ requirements-dev.txt | 14 -------------- tox.ini | 4 +--- 6 files changed, 26 insertions(+), 22 deletions(-) create mode 100644 requirements-dev-thirdparty.txt diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml index cd02ba3ef..bfba1bbe8 100644 --- a/.github/workflows/roman_ci.yml +++ b/.github/workflows/roman_ci.yml @@ -75,7 +75,7 @@ jobs: - linux: check-dependencies - linux: build-dist test: - uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 needs: [ data ] with: setenv: | diff --git a/JenkinsfileRT b/JenkinsfileRT index 993c3d18a..7f0dbf594 100644 --- a/JenkinsfileRT +++ b/JenkinsfileRT @@ -93,7 +93,8 @@ bc0.build_cmds = bc0.build_cmds + [ "pip list" ] bc0.test_cmds = [ - "pytest --cov-report=xml:coverage.xml --cov=./ -r sxf -n auto --bigdata --slow \ + "pytest -r sxf -n auto --bigdata --slow \ + --cov --cov-report=xml:coverage.xml \ --ddtrace \ --basetemp=${pytest_basetemp} --junit-xml=results.xml --dist=loadscope \ --env=${artifactoryenv} ${pytest_args}", @@ -115,5 +116,8 @@ bc1.build_cmds = bc1.build_cmds + PipInject(env.OVERRIDE_REQUIREMENTS) bc1.test_cmds = [] bc1.test_configs = [] -utils.run([jobconfig, bc0, bc1]) +bc2 = utils.copy(bc0) +bc2.pip_reqs_files = ['requirements-sdp.txt', 'requirements-dev.txt'] + +utils.run([jobconfig, bc0, bc1, bc2]) } // withCredentials diff --git a/JenkinsfileRT_dev b/JenkinsfileRT_dev index 1223c5462..09259856f 100644 --- a/JenkinsfileRT_dev +++ b/JenkinsfileRT_dev @@ -80,7 +80,7 @@ bc0.conda_packages = [ "python=${python_version}", "freetds", ] -bc0.pip_reqs_files = ['requirements-dev.txt'] +bc0.pip_reqs_files = ['requirements-dev-thirdparty.txt'] bc0.build_cmds = [ "pip install -e .[test]", "pip install pytest-xdist pytest-sugar", @@ -109,5 +109,8 @@ bc1.build_cmds = bc1.build_cmds + PipInject(env.OVERRIDE_REQUIREMENTS) bc1.test_cmds = [] bc1.test_configs = [] -utils.run([jobconfig, bc0, bc1]) +bc2 = utils.copy(bc0) +bc2.pip_reqs_files = ['requirements-dev-thirdparty.txt', 'requirements-dev.txt'] + +utils.run([jobconfig, bc0, bc1, bc2]) } // withCredentials diff --git a/requirements-dev-thirdparty.txt b/requirements-dev-thirdparty.txt new file mode 100644 index 000000000..997b3384a --- /dev/null +++ b/requirements-dev-thirdparty.txt @@ -0,0 +1,13 @@ +# ASDF upstream packages +git+https://github.com/asdf-format/asdf +git+https://github.com/asdf-format/asdf-transform-schemas +git+https://github.com/asdf-format/asdf-wcs-schemas +git+https://github.com/astropy/asdf-astropy + +# Use weekly astropy dev build +--extra-index-url https://pypi.anaconda.org/astropy/simple astropy --pre + +# Use Bi-weekly numpy/scipy dev builds +--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple +numpy>=0.0.dev0 +scipy>=0.0.dev0 diff --git a/requirements-dev.txt b/requirements-dev.txt index a40ea0fb0..e62f531b1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,17 +1,3 @@ -# ASDF upstream packages -git+https://github.com/asdf-format/asdf -git+https://github.com/asdf-format/asdf-transform-schemas -git+https://github.com/asdf-format/asdf-wcs-schemas -git+https://github.com/astropy/asdf-astropy - -# Use weekly astropy dev build ---extra-index-url https://pypi.anaconda.org/astropy/simple astropy --pre - -# Use Bi-weekly numpy/scipy dev builds ---extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple -numpy>=0.0.dev0 -scipy>=0.0.dev0 - # Roman upstream packages git+https://github.com/spacetelescope/roman_datamodels git+https://github.com/spacetelescope/rad diff --git a/tox.ini b/tox.ini index 935da7520..bd798e8a1 100644 --- a/tox.ini +++ b/tox.ini @@ -54,9 +54,6 @@ pass_env = CODECOV_* DD_* WEBBPSF_PATH -set_env = - devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simpl - extras = test alldeps: all @@ -68,6 +65,7 @@ commands_pre = oldestdeps: minimum_dependencies romancal --filename requirements-min.txt oldestdeps: pip install -r requirements-min.txt devdeps: pip install -r requirements-dev.txt -U --upgrade-strategy eager + devdeps: pip install -r requirements-dev-thirdparty.txt -U --upgrade-strategy eager sdpdeps: pip install -r requirements-sdp.txt pip list commands = From 5905f9282d481c5c9a45e35a1c9c798f0b9fb779 Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Thu, 24 Aug 2023 14:16:10 -0400 Subject: [PATCH 2/4] rename files --- JenkinsfileRT | 2 +- JenkinsfileRT_dev | 2 +- requirements-dev.txt => requirements-dev-st.txt | 11 ++++++----- requirements-dev-thirdparty.txt | 5 ++++- tox.ini | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) rename requirements-dev.txt => requirements-dev-st.txt (90%) diff --git a/JenkinsfileRT b/JenkinsfileRT index 7f0dbf594..95d4e2d07 100644 --- a/JenkinsfileRT +++ b/JenkinsfileRT @@ -117,7 +117,7 @@ bc1.test_cmds = [] bc1.test_configs = [] bc2 = utils.copy(bc0) -bc2.pip_reqs_files = ['requirements-sdp.txt', 'requirements-dev.txt'] +bc2.pip_reqs_files = ['requirements-sdp.txt', 'requirements-dev-st.txt'] utils.run([jobconfig, bc0, bc1, bc2]) } // withCredentials diff --git a/JenkinsfileRT_dev b/JenkinsfileRT_dev index 09259856f..86f9b22e9 100644 --- a/JenkinsfileRT_dev +++ b/JenkinsfileRT_dev @@ -110,7 +110,7 @@ bc1.test_cmds = [] bc1.test_configs = [] bc2 = utils.copy(bc0) -bc2.pip_reqs_files = ['requirements-dev-thirdparty.txt', 'requirements-dev.txt'] +bc2.pip_reqs_files = ['requirements-dev-thirdparty.txt', 'requirements-dev-st.txt'] utils.run([jobconfig, bc0, bc1, bc2]) } // withCredentials diff --git a/requirements-dev.txt b/requirements-dev-st.txt similarity index 90% rename from requirements-dev.txt rename to requirements-dev-st.txt index e62f531b1..f913d6e6e 100644 --- a/requirements-dev.txt +++ b/requirements-dev-st.txt @@ -2,11 +2,12 @@ git+https://github.com/spacetelescope/roman_datamodels git+https://github.com/spacetelescope/rad -# Other important upstream packages -git+https://github.com/spacetelescope/gwcs +# shared upstream packages +git+https://github.com/spacetelescope/stcal git+https://github.com/spacetelescope/stpipe + +# Other important upstream packages git+https://github.com/spacetelescope/crds -git+https://github.com/spacetelescope/stcal -git+https://github.com/spacetelescope/tweakwcs +git+https://github.com/spacetelescope/gwcs git+https://github.com/spacetelescope/metrics_logger -git+https://github.com/astropy/photutils.git +git+https://github.com/spacetelescope/tweakwcs diff --git a/requirements-dev-thirdparty.txt b/requirements-dev-thirdparty.txt index 997b3384a..0568dde4b 100644 --- a/requirements-dev-thirdparty.txt +++ b/requirements-dev-thirdparty.txt @@ -1,11 +1,14 @@ # ASDF upstream packages +git+https://github.com/asdf-format/asdf-standard git+https://github.com/asdf-format/asdf git+https://github.com/asdf-format/asdf-transform-schemas +git+https://github.com/asdf-format/asdf-coordinates-schemas git+https://github.com/asdf-format/asdf-wcs-schemas -git+https://github.com/astropy/asdf-astropy # Use weekly astropy dev build +git+https://github.com/astropy/asdf-astropy --extra-index-url https://pypi.anaconda.org/astropy/simple astropy --pre +git+https://github.com/astropy/photutils.git # Use Bi-weekly numpy/scipy dev builds --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple diff --git a/tox.ini b/tox.ini index bd798e8a1..1ef60d082 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,8 @@ commands = description = run tests alldeps: with all optional dependencies - devdeps: with the latest developer version of key dependencies + stdevdeps: with the latest developer version of key upstream spacetelescope dependencies + devdeps: with the latest developer version of key upstream third-party dependencies oldestdeps: with the oldest supported version of key dependencies sdpdeps: with the recent STScI DMS release pinned dependencies pyargs: with --pyargs on installed package @@ -64,7 +65,7 @@ deps = commands_pre = oldestdeps: minimum_dependencies romancal --filename requirements-min.txt oldestdeps: pip install -r requirements-min.txt - devdeps: pip install -r requirements-dev.txt -U --upgrade-strategy eager + stdevdeps: pip install -r requirements-dev-st.txt -U --upgrade-strategy eager devdeps: pip install -r requirements-dev-thirdparty.txt -U --upgrade-strategy eager sdpdeps: pip install -r requirements-sdp.txt pip list From 7b61605d8419a228c5e62034be8181d6db72c85a Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 11 Sep 2023 11:37:04 -0400 Subject: [PATCH 3/4] apply review suggestion --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1ef60d082..1d7ac3f62 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ commands_pre = oldestdeps: minimum_dependencies romancal --filename requirements-min.txt oldestdeps: pip install -r requirements-min.txt stdevdeps: pip install -r requirements-dev-st.txt -U --upgrade-strategy eager - devdeps: pip install -r requirements-dev-thirdparty.txt -U --upgrade-strategy eager + devdeps: pip install -r requirements-dev-thirdparty.txt -r requirements-dev-st.txt -U --upgrade-strategy eager sdpdeps: pip install -r requirements-sdp.txt pip list commands = From f51b8a28068cb66bdb8cf3486b40c6ddb3f0f9a2 Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Thu, 21 Sep 2023 08:37:17 -0400 Subject: [PATCH 4/4] add install instructions for development requirements --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 629b6dcc8..852037c76 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,12 @@ tools used for linting and reproducible environments run: This will install `tox` and the `pre-commit` tools in addition to all development dependencies. +Development versions of dependencies are listed in `requirements-dev-st.txt` and `requirements-dev-thirdparty.txt`, containing STScI and third-party packages respectively. +To install these dependencies, run the following: + + pip install -r requirements-dev-st.txt + pip install -r requirements-dev-thirdparty.txt + Need other useful packages in your development environment? pip install ipython pytest-xdist