Merge branch 'add-tests' [#6]

.github/actions/setup/action.yaml

@@ -0,0 +1,50 @@
+name: 'venv setup'
+description: 'configures venv, python and caches'
+
+inputs:
+  python-version:
+    description: "Python version to use"
+    required: true
+  cache-dependency-path:
+    description: "Requirement files to install. Can be a glob pattern."
+    default: '**/requirements*.txt'
+  venv-dir:
+    default: '.venv'
+
+outputs:
+  cache-hit:
+    description: "A boolean value to indicate if a cache was restored"
+    value: ${{ steps.cache-venv.outputs.cache-hit }}
+
+runs:
+  using: 'composite'
+  steps:
+    - run: cp .env.example .env
+      shell: bash
+
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c  # v5.0.0
+      id: setup-python
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - run: echo '::remove-matcher owner=python::'
+      shell: bash
+
+    - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2  # v4.0.0
+      id: cache-venv
+      with:
+        path: ${{ inputs.venv-dir }}
+        key: setup-venv-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ steps.setup-python.outputs.python-path }}-${{ hashFiles(inputs.cache-dependency-path) }}-${{ inputs.install-cmd }}
+
+    - run: python3 -m venv ${{ inputs.venv-dir }}
+      if: steps.cache-venv.outputs.cache-hit != 'true'
+      shell: bash
+
+    - run: |
+        source ${{ inputs.venv-dir }}/bin/activate
+        python -m pip install -r requirements.txt
+        python -m pip install -r requirements-dev.txt
+        echo "VIRTUAL_ENV=${VIRTUAL_ENV}" >> $GITHUB_ENV
+        echo "${VIRTUAL_ENV}/bin" >> $GITHUB_PATH
+
+      shell: bash

.github/workflows/ci.yaml

@@ -0,0 +1,29 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+    tags: ['v*']
+
+jobs:
+
+  tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ "ubuntu-latest" ]
+        python-version: [ "3.10" ]
+    runs-on: "${{ matrix.os }}"
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+      - uses: ./.github/actions/setup
+        with:
+          python-version: "${{ matrix.python-version }}"
+          venv-id: "tests-${{ runner.os }}"
+      - name: Run tests
+        run: |
+          pytest -r a -v tests
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3

.gitignore

@@ -4,6 +4,7 @@ inputs
 intermediates
 experiments
 .env
+.idea
 
 !outputs/README.md
 !inputs/README.md

omDownloadInputs.py

@@ -22,11 +22,15 @@
 This downloads the input files that rarely change and can be cached between runs.
 """
 
-from omInputs import electricityPath, fugitivesPath, landUsePath, sectoralEmissionsPath, sectoralMappingsPath, ntlPath, auShapefilePath, livestockDataPath, termitePath, wetlandPath, coalPath, oilGasPath
+from omInputs import (electricityPath,
+                      # fugitivesPath,
+                      landUsePath, sectoralEmissionsPath, sectoralMappingsPath, ntlPath, auShapefilePath, livestockDataPath, termitePath, wetlandPath, coalPath, oilGasPath)
 import requests
 import os
 from omUtils import getenv
 
+root_path = os.path.dirname(os.path.realpath(__file__))
+
 remote = getenv("PRIOR_REMOTE")
 
 electricityFile = getenv("CH4_ELECTRICITY")
@@ -41,6 +45,7 @@
 wetlandFile = getenv("WETLANDS")
 coalFile = getenv("COAL")
 oilGasFile = getenv("OILGAS")
+
 downloads = [
     [electricityFile, electricityPath],
     [coalFile, coalPath],
@@ -55,15 +60,23 @@
     [wetlandFile, wetlandPath]
 ]
 
-for filename, filepath in downloads:
-    url = f"{remote}{filename}"
+def download_input_files(root_path, downloads, remote):
+    for filename, filepath in downloads:
+        filepath = os.path.join(root_path, filepath)
+        print(filepath)
+        url = f"{remote}{filename}"
+
+        if not os.path.exists(filepath):
+            print(f"Downloading {filename} to {filepath} from {url}")
 
-    if not os.path.exists(filepath):
-        print(f"Downloading {filename} to {filepath} from {url}")
+            with requests.get(url, stream=True) as response:
+                with open(filepath, mode="wb") as file:
+                    for chunk in response.iter_content(chunk_size=10 * 1024):
+                        file.write(chunk)
+        else:
+            print(f"Skipping {filename} beacuse it already exists at {filepath}")
 
-        with requests.get(url, stream=True) as response:
-            with open(filepath, mode="wb") as file:
-                for chunk in response.iter_content(chunk_size=10 * 1024):
-                    file.write(chunk)
-    else:
-        print(f"Skipping {filename} beacuse it already exists at {filepath}")
+if __name__ == '__main__':
+    download_input_files(root_path=root_path,
+                         downloads=downloads,
+                         remote=remote)

omInputs.py

@@ -37,8 +37,13 @@
 domainFilename = getenv("DOMAIN")
 domainPath = os.path.join(inputsPath, domainFilename)
 electricityPath = os.path.join(inputsPath, getenv("CH4_ELECTRICITY"))
-oilGasPath = os.path.join( fossilPath, getenv("OILGAS"))
-coalPath = os.path.join( fossilPath, getenv("COAL"))
+# TODO: Changing this to match with the rest of the download file paths.
+# The originally specified directory does not exist and stops the download.
+# Maybe needs to be changed back later in the process.
+# oilGasPath = os.path.join( fossilPath, getenv("OILGAS"))
+oilGasPath = os.path.join( inputsPath, getenv("OILGAS"))
+# coalPath = os.path.join( fossilPath, getenv("COAL"))
+coalPath = os.path.join( inputsPath, getenv("COAL"))
 landUsePath = os.path.join(inputsPath, getenv("LAND_USE"))
 sectoralEmissionsPath = os.path.join(inputsPath, getenv("SECTORAL_EMISSIONS"))
 sectoralMappingsPath = os.path.join(inputsPath, getenv("SECTORAL_MAPPING"))

requirements-dev.txt

@@ -1,2 +1,3 @@
 licenseheaders
 ruff==0.3.7
+pytest==8.2.0

temporary_file_for_tests.py

@@ -0,0 +1,51 @@
+import os
+import dotenv
+
+dotenv.load_dotenv()
+getenv = os.environ.get
+
+remote = getenv("PRIOR_REMOTE")
+
+electricityFile = getenv("CH4_ELECTRICITY")
+fugitivesFile = getenv("CH4_FUGITIVES")
+landUseFile = getenv("LAND_USE")
+sectoralEmissionsFile = getenv("SECTORAL_EMISSIONS")
+sectoralMappingsFile = getenv("SECTORAL_MAPPING")
+ntlFile = getenv("NTL")
+auShapefileFile = getenv("AUSF")
+livestockDataFile = getenv("LIVESTOCK_DATA")
+termiteFile = getenv("TERMITES")
+wetlandFile = getenv("WETLANDS")
+coalFile = getenv("COAL")
+oilGasFile = getenv("OILGAS")
+
+inputsPath = getenv("INPUTS")
+cmaqExamplePath = getenv("CMAQ_EXAMPLE")
+climateTracePath = os.path.join( inputsPath, getenv("CLIMATETRACE"))
+fossilPath = os.path.join( climateTracePath, getenv("FOSSIL"))
+
+electricityPath = os.path.join(inputsPath, getenv("CH4_ELECTRICITY"))
+oilGasPath = os.path.join( fossilPath, getenv("OILGAS"))
+coalPath = os.path.join( fossilPath, getenv("COAL"))
+landUsePath = os.path.join(inputsPath, getenv("LAND_USE"))
+sectoralEmissionsPath = os.path.join(inputsPath, getenv("SECTORAL_EMISSIONS"))
+sectoralMappingsPath = os.path.join(inputsPath, getenv("SECTORAL_MAPPING"))
+ntlPath = os.path.join(inputsPath, getenv("NTL"))
+auShapefilePath = os.path.join(inputsPath, getenv("AUSF"))
+livestockDataPath = os.path.join(inputsPath, getenv("LIVESTOCK_DATA"))
+termitePath = os.path.join(inputsPath, getenv("TERMITES"))
+wetlandPath = os.path.join(inputsPath, getenv("WETLANDS"))
+
+downloads = [
+    [electricityFile, electricityPath],
+    [coalFile, coalPath],
+    [oilGasFile, oilGasPath],
+    [landUseFile, landUsePath],
+    [sectoralEmissionsFile, sectoralEmissionsPath],
+    [sectoralMappingsFile, sectoralMappingsPath],
+    [ntlFile, ntlPath],
+    [auShapefileFile, auShapefilePath],
+    [livestockDataFile, livestockDataPath],
+    [termiteFile, termitePath],
+    [wetlandFile, wetlandPath]
+]

tests/test_download_data.py

@@ -0,0 +1,55 @@
+# work around until folder structure is updated
+import os
+import sys
+# insert root directory into python module search path
+sys.path.insert(1, os.getcwd())
+
+from omDownloadInputs import root_path, downloads, remote, download_input_files
+
+import requests
+from pathlib import Path
+
+ROOT_DIRECTORY = Path(__file__).parent.parent
+
+def test_001_response_for_download_links() :
+    for filename, filepath in downloads :
+        url = f"{remote}{filename}"
+        with requests.get(url, stream=True) as response :
+            print(f"Response code for {url}: {response.status_code}")
+            assert response.status_code == 200
+
+
+def test_002_omDownloadInputs():
+
+    input_folder = os.path.join(ROOT_DIRECTORY, "inputs")
+
+    EXPECTED_FILES_START = ['README.md']
+    EXPECTED_FILES_END = [
+                        "ch4-electricity.csv",
+                        "coal-mining_emissions-sources.csv",
+                        "oil-and-gas-production-and-transport_emissions-sources.csv",
+                        "NLUM_ALUMV8_250m_2015_16_alb.tif",
+                        "ch4-sectoral-emissions.csv",
+                        "landuse-sector-map.csv",
+                        "nasa-nighttime-lights.tiff",
+                        "AUS_2021_AUST_SHP_GDA2020.zip",
+                        "EntericFermentation.nc",
+                        "termite_emissions_2010-2016.nc",
+                        "DLEM_totflux_CRU_diagnostic.nc",
+                        'README.md',
+                      ]
+
+    assert os.listdir(input_folder) == EXPECTED_FILES_START, f"Folder '{input_folder}' is not empty"
+
+    download_input_files(root_path=root_path,
+                         downloads=downloads,
+                         remote=remote)
+
+    downloaded_files = os.listdir(input_folder)
+
+    for file in [i for i in downloaded_files if i != 'README.md']:
+        filepath = os.path.join(input_folder, file)
+        os.remove(filepath)
+
+    assert sorted(downloaded_files) == sorted(EXPECTED_FILES_END)
+