Test download script in CI #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: download | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| tags: ['v*'] | |
| jobs: | |
| download: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| # - name: set download date | |
| # run: echo "DOWNLOAD_DATE=$(date +"%Y-%m-%d")" >> $GITHUB_ENV | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/actions/setup | |
| with: | |
| os: "ubuntu-latest" | |
| python-version: "3.9" | |
| venv-id: "docs" | |
| poetry-dependency-install-flags: "--all-extras --only 'main,dev'" | |
| - name: install datalad | |
| uses: awalsh128/[email protected] | |
| with: | |
| packages: datalad | |
| version: 1.0 | |
| - name: install ssh key for gin.hemio.de access | |
| uses: shimataro/ssh-key-action@v2 | |
| with: | |
| key: ${{ secrets.GIN_HEMIO_SSH_KEY }} | |
| name: id_ginhemio | |
| known_hosts: | | |
| gin.hemio.de ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPCoq8/G7C+fdFODXWdzWxyvLfVMEUyWJh7j01XuA/OV | |
| config: | | |
| Host gin.hemio.de | |
| User git | |
| IdentityFile ~/.ssh/id_ginhemio | |
| IdentitiesOnly yes | |
| - name: configure git user | |
| run: git config --global user.email "[email protected]" | |
| - name: configure git email | |
| run: git config --global user.name "Daniel Busch (via github actions)" | |
| - name: clone repo | |
| run: datalad clone [email protected]:primap-community/FAOSTAT_data_primap.git | |
| - name: configure push access to gin.hemio.de | |
| run: | | |
| cd FAOSTAT_data_primap/ | |
| datalad siblings add -s ginhemio --url https://https://gin.hemio.de/crdanielbusch/FAOSTAT_data_primap --pushurl gin.hemio.de:/crdanielbusch/FAOSTAT_data_primap | |
| git config remote.ginhemio.annex-ignore false | |
| - name: setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| # - name: install requirements | |
| # run: pip install -r FAOSTAT_data_primap/requirements.txt | |
| - name: update to latest data from gin.hemio.de | |
| run: datalad -C FAOSTAT_data_primap/ update -s ginhemio --how merge | |
| - name: download data | |
| run: datalad run poetry run python3 scripts/download_all_domains.py -o downloaded_data | |
| # run: datalad -C unfccc_di_data/ run -o data/annexI/*.csv.gz -o data/non-annexI/*.csv.gz -o data/all.parquet python download.py | |
| # - name: publish newly downloaded data | |
| # run: | | |
| # cd FAOSTAT_data_primap/ | |
| # datalad update -s ginhemio --how merge | |
| # datalad push --to ginhemio --data anything |