Merge pull request #201 from e4st-dev/feat-200-adjust-yearly-col

Add year_col to adjust_yearly

src/io/data.jl

@@ -127,9 +127,6 @@ function setup_data!(config, data)
     setup_table!(config, data, :nominal_load)
     setup_table!(config, data, :gen) # needs to come after build_gen setup for newgens
     setup_table!(config, data, :af_table)
-    setup_table!(config, data, :adjust_yearly)
-    setup_table!(config, data, :adjust_hourly)
-    setup_table!(config, data, :adjust_by_age)
 end
 export setup_data!
 

src/io/util.jl

@@ -43,7 +43,7 @@ export get_year_idxs
 
     YearString(s) -> s
 
-    YearString(n::Number) -> year2string(n)
+    YearString(n::Number) -> year2str(n)
 
 This is a type that acts as a converter to ensure year columns are parsed correctly as strings.  If blank given, left blank.
 """

src/types/modifications/Adjust.jl

@@ -38,6 +38,7 @@ function summarize_table(::Val{:adjust_yearly})
         (:variable_name, AbstractString, NA, true, "The name of the variable/column to adjust"),
         (:operation, Operation, NA, true, "The operation to perform.  Could be add, scale, or set."),
         (:filter_, String, NA, true, "There can be multiple filter conditions - `filter1`, `filter2`, etc.  It denotes a comparison used for selecting the table rows to apply the adjustment to.  See `parse_comparison` for examples"),
+        (:year_col, String, NA, false, "Optional, the column for which the adjustment applies.  For example, use `year_on` to set a cost for the lifetime of a generator.  Leave blank (default) for to apply the adjustment by the simulation year value."),
         (:status, Bool, NA, false, "Whether or not to use this adjustment"),
         (:y_, Float64, Ratio, true, "Value to adjust by for each year.  Include a column for each year in the hours table.  I.e. `:y2020`, `:y2030`, etc"),
     )
@@ -113,17 +114,19 @@ export Adjust, AdjustHourly, AdjustYearly, AdjustByAge
 function modify_raw_data!(mod::Adjust{T}, config, data) where T
     file = mod.file
     name = mod.name
+    @info "Loading $T table for Modification $(mod.name) into data[:$(mod.name)]"
     table = read_table(data, file, T)
     data[name] = table
     return nothing
 end
 
 function modify_setup_data!(mod::Adjust{T}, config, data) where T
     table = get_table(data, mod.name)
+    table.num_adjusted .= 0
     for row in eachrow(table)
+        get(row, :status, true) || continue
         adjust!(mod, config, data, row)
     end
-    delete!(data, mod.name)
 end
 
 function adjust!(mod::Adjust{:adjust_yearly}, config, data, row)
@@ -164,6 +167,7 @@ function adjust_hourly!(config, data, row)
         vals = [row["h$h"] for h in 1:get_num_hours(data)]
         c = get(data, key, ByNothing(0.0))
         data[key] = operate_hourly(oper, c, vals, yr_idx, nyr)
+        row.num_adjusted = 1
         return
     end
 
@@ -173,12 +177,13 @@ function adjust_hourly!(config, data, row)
     isempty(table) && return
     hasproperty(table, variable_name) || error("Table $table_name has no column $variable_name to adjust in `adjust_hourly!`")
 
+    row.num_adjusted = nrow(table)
 
     # Perform the adjustment on each row of the table
     vals = [row["h$h"] for h in 1:get_num_hours(data)]
 
     # Make sure the appropriate column is a Vector{Container}
-    to_container!(table, variable_name)
+    to_container!(get_table(data, table_name), variable_name)
     for r in eachrow(table)
         r[variable_name] = operate_hourly(oper, r[variable_name], vals, yr_idx, nyr)
     end
@@ -188,11 +193,24 @@ end
 """
     adjust_yearly!(config, data, row)
 
-Apply an hourly adjustment given `row` from the `adjust_hourly` table.
+Apply a yearly adjustment given `row` from the `adjust_yearly` table.
 """
 function adjust_yearly!(config, data, row)
-    get(row, :status, true) || return
-
+    if haskey(row, :year_col) && !isempty(row.year_col)
+        adjust_yearly_by_year_col!(config, data, row)
+    else
+        adjust_yearly_by_sim_year!(config, data, row)
+    end
+    return nothing
+end
+
+
+"""
+    adjust_yearly_by_sim_year!(config, data, row)
+
+Adjusts values by the simulation year(s).
+"""
+function adjust_yearly_by_sim_year!(config, data, row)
     # TODO: make warning if you are trying to modify the same column of the same table hourly and yearly.
     table_name = row.table_name::AbstractString
     variable_name = row.variable_name::AbstractString
@@ -202,26 +220,73 @@ function adjust_yearly!(config, data, row)
         vals = [row[y] for y in get_years(data)]
         c = get(data, key, ByNothing(0.0))
         data[key] = operate_yearly(oper, c, vals)
+        row.num_adjusted = 1
         return
     end
 
     # Get the filtered table with which to perform the adjustment
     pairs = parse_comparisons(row)
     table = get_table(data, table_name, pairs)
     isempty(table) && return
-    hasproperty(table, variable_name) || error("Table $table_name has no column $variable_name to adjust in `adjust_hourly!`")
+    hasproperty(table, variable_name) || error("Table $table_name has no column $variable_name to adjust in `adjust_yearly!`")
 
     # Perform the adjustment on each row of the table
     vals = [row[y] for y in get_years(data)]
 
     # Make sure the appropriate column is a Vector{Container}
-    to_container!(table, variable_name) 
+    to_container!(get_table(data, table_name), variable_name)
+
+    row.num_adjusted = nrow(table)
 
     for r in eachrow(table)
         r[variable_name] = operate_yearly(oper, r[variable_name], vals)
     end
     return
 end
+export adjust_yearly_by_sim_year!
+
+"""
+    adjust_yearly_by_year_col!(config, data, row) 
+
+Adjusts by the year column specified.  Note this only works if applying an adjustment to a table.
+"""
+function adjust_yearly_by_year_col!(config, data, row)
+    table_name = row.table_name::AbstractString
+    variable_name = row.variable_name::AbstractString
+    oper = row.operation::AbstractString
+    year_col = row.year_col::String
+    nyr = get_num_years(data)
+
+    # Get the filtered table with which to perform the adjustment
+    pairs = parse_comparisons(row)
+    table = get_table(data, table_name, pairs)
+    isempty(table) && return
+    hasproperty(table, variable_name) || error("Table $table_name has no column $variable_name to adjust in `adjust_yearly!`")
+    hasproperty(table, year_col) || error("Table $table_name has no column $year_col to adjust by in `adjust_yearly!`")
+
+    to_container!(get_table(data, table_name), variable_name)
+
+    num_adjusted = 0
+    bad_years = Set{String}()
+    for r in eachrow(table)
+        year = (r[year_col] |> YearString)::String
+
+        # Skip if the year is not found.
+        if !haskey(row, year)
+            push!(bad_years, year)
+            continue
+        end
+
+        val = row[year]::Float64
+        v = fill(val, nyr)
+        r[variable_name] = operate_yearly(oper, r[variable_name], v)
+        num_adjusted += 1
+    end
+    row.num_adjusted = num_adjusted
+    isempty(bad_years) || @warn "AdjustYearly mod does not have a value for the following years, so it skipped those adjustments:\n$(sort(collect(bad_years)))"
+    return nothing
+end
+export adjust_yearly_by_year_col!
 
 """
     adjust_by_age!(config, data, row)
@@ -246,12 +311,13 @@ function adjust_by_age!(config, data, row)
     hasproperty(table, variable_name) || error("Table $table_name has no column $variable_name to adjust in `adjust_hourly!`")
 
     # Perform the adjustment on each row of the table
+    row.num_adjusted = nrow(table)
     val = row.value
     age = row.age
     age_type = row.age_type
 
     # Make sure the appropriate column is a Vector{Container}
-    to_container!(table, variable_name)
+    to_container!(get_table(data, table_name), variable_name)
 
     last_sim_year = get(config, :year_previous_sim, config[:year_gen_data])
 

test/data/3bus/adjust_yearly.csv

@@ -1,6 +1,6 @@
-description,table_name,variable_name,operation,filter1,filter2,status,y2020,y2025,y2030,y2035,y2040,y2050,y2060,y2070
-Reduce the FOM in narnia for solar generators,gen,fom,add,nation=>narnia,gentype=>solar,1,0.0,-0.1,-0.2,-0.3,-0.4,-0.5,-0.6,-0.6
-Scale the branch max power flow to match a 5% annual growth rate starting in 2020,branch,pflow_max,scale,,,1,1.0,1.276,1.629,2.079,2.653,4.322,7.040,11.467
-"Set the yearly value of damage rate of CO2, using values from Rennert et. al. (constant after 2040)",,r_dam_co2,set,,,1,150.26,166.91,183.56,198.59,213.61,213.61,213.61,213.61
-"Set the yearly value of damage rate of NOx emissions, in dollars per pound  (constant after 2040)",,r_dam_nox,set,,,1,24.18,27.11,30.05,33.00,35.60,35.60,35.60,35.60
-"Global warming potential of methane, based on SCC/SCM from Rennert et. al.  (constant after 2040)",,ch4_gwp,set,,,1,10.481081,11.812652,12.902655,14.280164,15.463878,15.463878,15.463878,15.463878
+description,table_name,variable_name,year_col,operation,filter1,filter2,status,y2020,y2025,y2030,y2035,y2040,y2050,y2060,y2070
+Reduce the FOM in narnia for solar generators,gen,fom,year_on,add,nation=>narnia,gentype=>solar,1,0.0,-0.1,-0.2,-0.3,-0.4,-0.5,-0.6,-0.6
+Scale the branch max power flow to match a 5% annual growth rate starting in 2020,branch,pflow_max,,scale,,,1,1.0,1.276,1.629,2.079,2.653,4.322,7.040,11.467
+"Set the yearly value of damage rate of CO2, using values from Rennert et. al. (constant after 2040)",,r_dam_co2,,set,,,1,150.26,166.91,183.56,198.59,213.61,213.61,213.61,213.61
+"Set the yearly value of damage rate of NOx emissions, in dollars per pound  (constant after 2040)",,r_dam_nox,,set,,,1,24.18,27.11,30.05,33.00,35.60,35.60,35.60,35.60
+"Global warming potential of methane, based on SCC/SCM from Rennert et. al.  (constant after 2040)",,ch4_gwp,,set,,,1,10.481081,11.812652,12.902655,14.280164,15.463878,15.463878,15.463878,15.463878

test/runtests.jl

@@ -30,6 +30,7 @@ rm(joinpath(@__DIR__, "out"), force=true, recursive=true)
     include("teststorage.jl")
     include("testccus.jl")
     include("testretrofits.jl")
+    include("testadjust.jl")
     include("testutil.jl")
 end
 

test/testadjust.jl

@@ -9,13 +9,20 @@
         mods[:adj_hourly] = AdjustHourly(file=joinpath(@__DIR__, "data", "3bus", "adjust_hourly.csv"), name=:adj_hourly)
         mods[:adj_by_age] = AdjustByAge(file=joinpath(@__DIR__, "data", "3bus", "adjust_by_age.csv"), name=:adj_by_age)
         data = read_data(config)
+        nyr = get_num_years(data)
         @test data isa AbstractDict
 
         @testset "Test Yearly Adjustments" begin
 
             # Test that FOM is reduced in narnia for solar generators
-            gen_idxs = get_table_row_idxs(data, :gen, "nation"=>"narnia", "genfuel"=>"solar")
-            @test all(gen_idx->(get_table_num(data, :gen, :fom, gen_idx, 3, 1) ≈ get_table_num(data0, :gen, :fom, gen_idx, 3, 1) - 0.4), gen_idxs)
+            gen_idxs = get_table_row_idxs(data, :gen, ("nation"=>"narnia", "genfuel"=>"solar", "year_on"=>"y2030"))
+            @test all(get_table_num(data, :gen, :fom, gen_idx, yr_idx, 1) ≈ get_table_num(data0, :gen, :fom, gen_idx, yr_idx, 1) - 0.2 for yr_idx in 1:nyr, gen_idx in gen_idxs)
+
+            gen_idxs = get_table_row_idxs(data, :gen, ("nation"=>"narnia", "genfuel"=>"solar", "year_on"=>"y2035"))
+            @test all(get_table_num(data, :gen, :fom, gen_idx, yr_idx, 1) ≈ get_table_num(data0, :gen, :fom, gen_idx, yr_idx, 1) - 0.3 for yr_idx in 1:nyr, gen_idx in gen_idxs)
+
+            gen_idxs = get_table_row_idxs(data, :gen, ("nation"=>"narnia", "genfuel"=>"solar", "year_on"=>"y2040"))
+            @test all(get_table_num(data, :gen, :fom, gen_idx, yr_idx, 1) ≈ get_table_num(data0, :gen, :fom, gen_idx, yr_idx, 1) - 0.4 for yr_idx in 1:nyr, gen_idx in gen_idxs)
 
             # Test that max branch power flow is greater in later years
             branch_idxs = get_table_row_idxs(data, :branch)

test/testinitializedata.jl

@@ -15,6 +15,7 @@
         for table_name in table_names
             @test has_table(data, table_name)
             table_name == :summary_table && continue
+            startswith(string(table_name), "adj") && continue
             @test summarize_table(table_name) isa DataFrame
             table = get_table(data, table_name)
             for col_name in names(table)
@@ -46,10 +47,6 @@
 
     end
 
-    #Test yearly and hourly adjustment to data
-    include("testadjust.jl")
-
-
     @testset "Test load with shaping" begin
         config = read_config(config_file)
         #config[:load_shape_file] = abspath(@__DIR__, "data", "3bus","load_shape.csv")