chore: merge develop into master for v0.4.3 release

.coveragerc

@@ -1,6 +1,2 @@
 [run]
 branch = True
-omit =
-    **/test_*.py
-    **/__init__.py
-

.github/workflows/coverage.yml

@@ -0,0 +1,34 @@
+name: Code coverage
+
+on:
+  push:
+    branches:
+      - 'develop'
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+
+    name: Generate coverage report
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Start postgres container
+        run: |
+          docker-compose -f stack.yml up -d
+          while ! nc -z localhost 5432; do sleep 1; done;
+        working-directory: powersimdata/data_access
+
+      - run: python -m pip install --upgrade pip tox
+      - run: tox -e pytest-ci -- --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          name: codecov-powersimdata
+          fail_ci_if_error: true

README.md

@@ -1,6 +1,7 @@
 
 ![logo](https://raw.githubusercontent.com/Breakthrough-Energy/docs/master/source/_static/img/BE_Sciences_RGB_Horizontal_Color.svg)
 
+[![codecov](https://codecov.io/gh/Breakthrough-Energy/PowerSimData/branch/develop/graph/badge.svg?token=5A20TCV5XL)](https://codecov.io/gh/Breakthrough-Energy/PowerSimData)
 [![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg)](https://www.python.org/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 ![Tests](https://github.com/Breakthrough-Energy/PowerSimData/workflows/Pytest/badge.svg)

powersimdata/design/compare/generation.py

@@ -0,0 +1,25 @@
+from powersimdata.design.compare.helpers import _reindex_as_necessary
+from powersimdata.input.check import _check_data_frame
+
+
+def calculate_plant_difference(plant1, plant2):
+    """Calculate the capacity differences between two plant data frames. If capacity in
+    ``plant2`` is larger than capacity in ``plant1``, the return will be positive.
+
+    :param pandas.DataFrame plant1: first plant data frame.
+    :param pandas.DataFrame plant2: second plant data frame.
+    :return: (*pandas.DataFrame*) -- merged data frames with a new 'diff' column.
+    """
+    _check_data_frame(plant1, "plant1")
+    _check_data_frame(plant2, "plant2")
+    # Reindex so that we don't get NaN when calculating upgrades for new generators
+    plant1, plant2 = _reindex_as_necessary(plant1, plant2, ["bus_id", "type"])
+    plant_merge = plant1.merge(
+        plant2, how="outer", right_index=True, left_index=True, suffixes=(None, "_2")
+    )
+    plant_merge["diff"] = plant_merge.Pmax_2.fillna(0) - plant_merge.Pmax.fillna(0)
+    # Ensure that lats & lons get filled in as necessary from plant2 entries
+    for l in ["lat", "lon"]:
+        plant_merge[l].fillna(plant_merge[f"{l}_2"], inplace=True)
+
+    return plant_merge

powersimdata/design/compare/helpers.py

@@ -0,0 +1,19 @@
+def _reindex_as_necessary(df1, df2, check_columns):
+    """Check for indices with mismatched entries in specified columns. If any entries
+    don't match, reindex based on these columns such that there are no shared indices
+    with mismatched entries in these columns.
+
+    :param pandas.DataFrame df1: data frame containing ``check_columns``.
+    :param pandas.DataFrame df2: data frame containing ``check_columns``.
+    :param iterable check_columns: column
+    :return: (*tuple*) -- data frames, reindexed as necessary.
+    """
+    # Coerce to list for safety, since pandas interprets lists and tuples differently
+    check_columns_list = list(check_columns)
+    shared_indices = set(df1.index) & set(df2.index)
+    check1 = df1.loc[shared_indices, check_columns_list]
+    check2 = df2.loc[shared_indices, check_columns_list]
+    if not check1.equals(check2):
+        df1 = df1.set_index(keys=check_columns_list, drop=False, append=True)
+        df2 = df2.set_index(keys=check_columns_list, drop=False, append=True)
+    return df1, df2

powersimdata/design/compare/transmission.py

@@ -0,0 +1,61 @@
+from powersimdata.design.compare.helpers import _reindex_as_necessary
+from powersimdata.input.check import _check_data_frame, _check_grid_type
+
+
+def calculate_branch_difference(branch1, branch2):
+    """Calculate the capacity differences between two branch data frames. If capacity in
+    ``branch2`` is larger than capacity in ``branch1``, the return will be positive.
+
+    :param pandas.DataFrame branch1: first branch data frame.
+    :param pandas.DataFrame branch2: second branch data frame.
+    :param float/int difference_threshold: drop any changes less than this value from
+        the returned Series.
+    :return: (*pandas.Series*) -- capacity difference between the two branch data
+        frames.
+    """
+    _check_data_frame(branch1, "branch1")
+    _check_data_frame(branch2, "branch2")
+    if not ("rateA" in branch1.columns) and ("rateA" in branch2.columns):
+        raise ValueError("branch1 and branch2 both must have 'rateA' columns")
+    branch1, branch2 = _reindex_as_necessary(
+        branch1, branch2, ["from_bus_id", "to_bus_id"]
+    )
+    branch_merge = branch1.merge(
+        branch2, how="outer", right_index=True, left_index=True, suffixes=(None, "_2")
+    )
+    branch_merge["diff"] = branch_merge.rateA_2.fillna(0) - branch_merge.rateA.fillna(0)
+    # Ensure that lats & lons get filled in as necessary from branch2 entries
+    for l in ["from_lat", "from_lon", "to_lat", "to_lon"]:
+        branch_merge[l].fillna(branch_merge[f"{l}_2"], inplace=True)
+
+    return branch_merge
+
+
+def calculate_dcline_differences(grid1, grid2):
+    """Calculate capacity differences between dcline tables, and add to/from lat/lon.
+
+    :param powersimdata.input.grid.Grid grid1: first grid instance.
+    :param powersimdata.input.grid.Grid grid2: second grid instance.
+    :return: (*pandas.DataFrame*) -- data frame with all indices, plus new columns:
+        diff, from_lat, from_lon, to_lat, to_lon.
+    """
+    _check_grid_type(grid1)
+    _check_grid_type(grid2)
+    dcline1, dcline2 = _reindex_as_necessary(
+        grid1.dcline, grid2.dcline, ["from_bus_id", "to_bus_id"]
+    )
+    # Get latitudes and longitudes for to & from buses
+    for dcline, grid in [(dcline1, grid1), (dcline2, grid2)]:
+        dcline["from_lat"] = grid.bus.loc[dcline.from_bus_id, "lat"].to_numpy()
+        dcline["from_lon"] = grid.bus.loc[dcline.from_bus_id, "lon"].to_numpy()
+        dcline["to_lat"] = grid.bus.loc[dcline.to_bus_id, "lat"].to_numpy()
+        dcline["to_lon"] = grid.bus.loc[dcline.to_bus_id, "lon"].to_numpy()
+    dc_merge = dcline1.merge(
+        dcline2, how="outer", right_index=True, left_index=True, suffixes=(None, "_2")
+    )
+    dc_merge["diff"] = dc_merge.Pmax_2.fillna(0) - dc_merge.Pmax.fillna(0)
+    # Ensure that lats & lons get filled in as necessary from grid2.dcline entries
+    for l in ["from_lat", "from_lon", "to_lat", "to_lon"]:
+        dc_merge[l].fillna(dc_merge[f"{l}_2"], inplace=True)
+
+    return dc_merge

powersimdata/design/investment/investment_costs.py

@@ -4,12 +4,18 @@
 import numpy as np
 import pandas as pd
 
+from powersimdata.design.compare.generation import calculate_plant_difference
+from powersimdata.design.compare.transmission import (
+    calculate_branch_difference,
+    calculate_dcline_differences,
+)
 from powersimdata.design.investment import const
 from powersimdata.design.investment.create_mapping_files import (
     bus_to_neem_reg,
     bus_to_reeds_reg,
 )
 from powersimdata.design.investment.inflation import calculate_inflation
+from powersimdata.input.check import _check_grid_models_match
 from powersimdata.input.grid import Grid
 from powersimdata.utility.distance import haversine
 
@@ -40,29 +46,38 @@ def append_keep_index_name(df1, other, *args, **kwargs):
     return new_df
 
 
-def calculate_ac_inv_costs(scenario, sum_results=True, exclude_branches=None):
+def calculate_ac_inv_costs(
+    scenario,
+    sum_results=True,
+    exclude_branches=None,
+    base_grid=None,
+):
     """Calculate cost of upgrading AC lines and/or transformers in a scenario.
     NEEM regions are used to find regional multipliers.
 
     :param powersimdata.scenario.scenario.Scenario scenario: scenario instance.
     :param bool sum_results: whether to sum data frame for each branch type. Defaults to
         True.
+    :param powersimdata.input.grid.Grid base_grid: a Grid to compare against. If None,
+        the grid model and interconnect from the ``scenario`` are used to instantiate a
+        corresponding unmodified Grid.
     :return: (*dict*) -- keys are {'line_cost', 'transformer_cost'}, values are either
         float if ``sum_results``, or pandas Series indexed by branch ID.
         Whether summed or not, values are $USD, inflation-adjusted to today.
     """
-
-    base_grid = Grid(
-        scenario.info["interconnect"].split("_"), source=scenario.info["grid_model"]
-    )
     grid = scenario.state.get_grid()
+    if base_grid is None:
+        base_grid = Grid(
+            scenario.info["interconnect"].split("_"), source=scenario.info["grid_model"]
+        )
+    else:
+        _check_grid_models_match(base_grid, grid)
 
-    # find upgraded AC lines
     grid_new = cp.deepcopy(grid)
-    # Reindex so that we don't get NaN when calculating upgrades for new branches
-    base_grid.branch = base_grid.branch.reindex(grid_new.branch.index).fillna(0)
-    grid_new.branch.rateA = grid.branch.rateA - base_grid.branch.rateA
-    grid_new.branch = grid_new.branch[grid_new.branch.rateA != 0.0]
+    # find upgraded AC lines
+    capacity_difference = calculate_branch_difference(base_grid.branch, grid.branch)
+    grid_new.branch = grid.branch.assign(rateA=capacity_difference.diff)
+    grid_new.branch = grid_new.branch.query("rateA != 0.0")
     if exclude_branches is not None:
         present_exclude_branches = set(exclude_branches) & set(grid_new.branch.index)
         grid_new.branch.drop(index=present_exclude_branches, inplace=True)
@@ -257,27 +272,32 @@ def get_branch_mult(x, bus_reg, ac_reg_mult, branch_lookup_alerted=set()):
         return {"line_cost": lines.cost, "transformer_cost": transformers.cost}
 
 
-def calculate_dc_inv_costs(scenario, sum_results=True):
+def calculate_dc_inv_costs(scenario, sum_results=True, base_grid=None):
     """Calculate cost of upgrading HVDC lines in a scenario.
 
     :param powersimdata.scenario.scenario.Scenario scenario: scenario instance.
     :param bool sum_results: whether to sum series to return total cost. Defaults to
         True.
+    :param powersimdata.input.grid.Grid base_grid: a Grid to compare against. If None,
+        the grid model and interconnect from the ``scenario`` are used to instantiate a
+        corresponding unmodified Grid.
     :return: (*pandas.Series/float*) -- cost of upgrading HVDC lines, in $USD,
         inflation-adjusted to today. If ``sum_results``, a float is returned, otherwise
         a Series.
     """
-    base_grid = Grid(
-        scenario.info["interconnect"].split("_"), source=scenario.info["grid_model"]
-    )
     grid = scenario.state.get_grid()
+    if base_grid is None:
+        base_grid = Grid(
+            scenario.info["interconnect"].split("_"), source=scenario.info["grid_model"]
+        )
+    else:
+        _check_grid_models_match(base_grid, grid)
 
     grid_new = cp.deepcopy(grid)
-    # Reindex so that we don't get NaN when calculating upgrades for new DC lines
-    base_grid.dcline = base_grid.dcline.reindex(grid_new.dcline.index).fillna(0)
     # find upgraded DC lines
-    grid_new.dcline.Pmax = grid.dcline.Pmax - base_grid.dcline.Pmax
-    grid_new.dcline = grid_new.dcline[grid_new.dcline.Pmax != 0.0]
+    capacity_difference = calculate_dcline_differences(base_grid.dcline, grid.dcline)
+    grid_new.dcline = grid.dcline.assign(Pmax=capacity_difference.diff)
+    grid_new.dcline = grid_new.dcline.query("Pmax != 0.0")
 
     costs = _calculate_dc_inv_costs(grid_new, sum_results)
     return costs
@@ -329,7 +349,13 @@ def _calculate_single_line_cost(line, bus):
         return 0.0
 
 
-def calculate_gen_inv_costs(scenario, year, cost_case, sum_results=True):
+def calculate_gen_inv_costs(
+    scenario,
+    year,
+    cost_case,
+    sum_results=True,
+    base_grid=None,
+):
     """Calculate cost of upgrading generators in a scenario. ReEDS regions are used to
     find regional multipliers.
 
@@ -339,6 +365,9 @@ def calculate_gen_inv_costs(scenario, year, cost_case, sum_results=True):
         *'Conservative'*: generally higher costs, *'Advanced'*: generally lower costs
     :param bool sum_results: whether to sum data frame for plant costs. Defaults to
         True.
+    :param powersimdata.input.grid.Grid base_grid: a Grid to compare against. If None,
+        the grid model and interconnect from the ``scenario`` are used to instantiate a
+        corresponding unmodified Grid.
     :return: (*pandas.Series*) -- Overnight generation investment cost.
         If ``sum_results``, indices are technologies and values are total cost.
         Otherwise, indices are IDs of plants (including storage, which is given
@@ -349,17 +378,19 @@ def calculate_gen_inv_costs(scenario, year, cost_case, sum_results=True):
         are dropped. Wind and solar will need to be fixed based on the resource supply
         curves.
     """
-
-    base_grid = Grid(
-        scenario.info["interconnect"].split("_"), source=scenario.info["grid_model"]
-    )
     grid = scenario.state.get_grid()
+    if base_grid is None:
+        base_grid = Grid(
+            scenario.info["interconnect"].split("_"), source=scenario.info["grid_model"]
+        )
+    else:
+        _check_grid_models_match(base_grid, grid)
 
-    # Find change in generation capacity
     grid_new = cp.deepcopy(grid)
-    # Reindex so that we don't get NaN when calculating upgrades for new generators
-    base_grid.plant = base_grid.plant.reindex(grid_new.plant.index).fillna(0)
-    grid_new.plant.Pmax = grid.plant.Pmax - base_grid.plant.Pmax
+    # Find change in generation capacity
+    capacity_difference = calculate_plant_difference(base_grid.plant, grid.plant)
+    grid_new.plant = grid.plant.assign(Pmax=capacity_difference.diff)
+    grid_new.plant = grid_new.plant.query("Pmax >= 0.01")
     # Find change in storage capacity
     # Reindex so that we don't get NaN when calculating upgrades for new storage
     base_grid.storage["gen"] = base_grid.storage["gen"].reindex(
@@ -370,9 +401,6 @@ def calculate_gen_inv_costs(scenario, year, cost_case, sum_results=True):
     )
     grid_new.storage["gen"]["type"] = "storage"
 
-    # Drop small changes
-    grid_new.plant = grid_new.plant[grid_new.plant.Pmax > 0.01]
-
     costs = _calculate_gen_inv_costs(grid_new, year, cost_case, sum_results)
     return costs
 

powersimdata/design/transmission/substations.py

@@ -0,0 +1,19 @@
+def calculate_substation_capacity(grid):
+    """For each substation in a grid, calculate the total substation transmission
+    capacity (in a transport model, ignoring power flow).
+
+    :param powersimdata.input.grid.Grid grid: a grid instance.
+    :return: (*pandas.Series*) -- index is substation IDs, value are total transmission
+        capacity (MW).
+    """
+    # Get new branch data frame with 'from_sub' and 'to_sub' columns
+    branch = grid.branch.assign(
+        from_sub_id=grid.branch.from_bus_id.map(grid.bus2sub.sub_id),
+        to_sub_id=grid.branch.to_bus_id.map(grid.bus2sub.sub_id),
+    )
+    # Calculate total substation capacity for matching 'from_sub' branches
+    filtered_branch = branch.query("from_sub_id != to_sub_id")
+    from_cap = filtered_branch.groupby("from_sub_id").sum()["rateA"]
+    to_cap = filtered_branch.groupby("to_sub_id").sum()["rateA"]
+    total_capacities = from_cap.combine(to_cap, lambda x, y: x + y, fill_value=0)
+    return total_capacities

powersimdata/design/transmission/tests/test_substations.py

@@ -0,0 +1,32 @@
+import pandas as pd
+
+from powersimdata.design.transmission.substations import calculate_substation_capacity
+from powersimdata.tests.mock_grid import MockGrid
+
+
+def test_calculate_substation_capacity():
+    mock_sub = {"sub_id": [1, 2, 3, 4]}
+    mock_bus2sub = {"bus_id": [10, 20, 21, 30, 40], "sub_id": [1, 2, 2, 3, 4]}
+    mock_branch = {
+        "branch_id": [200, 400, 420, 600, 601, 1200],
+        "from_bus_id": [10, 40, 20, 20, 30, 30],
+        "to_bus_id": [20, 10, 21, 30, 20, 40],
+        "rateA": [1, 2, 4, 8, 16, 32],
+    }
+
+    mock_grid_data = {
+        "sub": mock_sub,
+        "bus2sub": mock_bus2sub,
+        "branch": mock_branch,
+    }
+    mock_grid = MockGrid(grid_attrs=mock_grid_data)
+    substation_capacity = calculate_substation_capacity(mock_grid)
+    expected_return = pd.Series(
+        {
+            1: 3,
+            2: 25,
+            3: 56,
+            4: 34,
+        }
+    )
+    assert substation_capacity.equals(expected_return)