Test download script in CI #28
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: download | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| tags: ['v*'] | |
| jobs: | |
| download: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: install datalad | |
| uses: awalsh128/[email protected] | |
| with: | |
| packages: datalad | |
| version: 1.0 | |
| - name: Install poetry | |
| shell: bash | |
| run: | | |
| pipx install poetry | |
| which poetry | |
| poetry --version # Check poetry installation | |
| - name: configure git email | |
| run: git config --global user.email "[email protected]" | |
| - name: configure git user | |
| run: git config --global user.name "Daniel Busch (via github actions)" | |
| - name: clone repo | |
| run: datalad -l debug clone https://github.com/primap-community/FAOSTAT_data_primap.git | |
| # - name: configure push access to gin.hemio.de | |
| # run: | | |
| # cd FAOSTAT_data_primap/ | |
| # datalad siblings add -s ginhemio --url https://https://gin.hemio.de/crdanielbusch/FAOSTAT_data_primap --pushurl gin.hemio.de:/crdanielbusch/FAOSTAT_data_primap | |
| # git config remote.ginhemio.annex-ignore false | |
| - name: setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| # - name: install requirements | |
| # run: pip install -r FAOSTAT_data_primap/requirements.txt | |
| - name: update to latest data from gin.hemio.de | |
| run: | | |
| cd FAOSTAT_data_primap/ | |
| datalad siblings add -s gin --url https://gin.hemio.de/crdanielbusch/FAOSTAT_data_primap | |
| git config remote.gin.annex-ignore false | |
| datalad update -s gin --how merge | |
| - name: Set Poetry environment | |
| shell: bash | |
| run: | | |
| cd FAOSTAT_data_primap/ | |
| poetry config virtualenvs.create true | |
| poetry config virtualenvs.in-project true | |
| poetry install --no-interaction --all-extras --only 'main,dev' | |
| poetry run python --version # Check python version just in case | |
| - name: download data | |
| run: | | |
| cd FAOSTAT_data_primap/ | |
| datalad unlock . | |
| datalad run -o downloaded_data poetry run python3 scripts/download_all_domains.py | |
| # run: datalad -C unfccc_di_data/ run -o data/annexI/*.csv.gz -o data/non-annexI/*.csv.gz -o data/all.parquet python download.py | |
| # - name: publish newly downloaded data | |
| # run: | | |
| # cd FAOSTAT_data_primap/ | |
| # datalad update -s ginhemio --how merge | |
| # datalad push --to ginhemio --data anything |